Logistic Regression Open or Closed- Web scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>AboutMe</th>\n",
       "      <th>Body</th>\n",
       "      <th>Comment</th>\n",
       "      <th>Downvotes</th>\n",
       "      <th>Location</th>\n",
       "      <th>PostId</th>\n",
       "      <th>ProfileImageUrl</th>\n",
       "      <th>Reputation</th>\n",
       "      <th>SOUrlCount</th>\n",
       "      <th>...</th>\n",
       "      <th>sentencesStartWithICount</th>\n",
       "      <th>sentencesStartWithYouCount</th>\n",
       "      <th>shortWordCount</th>\n",
       "      <th>smogIndex</th>\n",
       "      <th>tagCount</th>\n",
       "      <th>tagWeightSum</th>\n",
       "      <th>titleLengthInChars</th>\n",
       "      <th>urlCount</th>\n",
       "      <th>userProfileFilled</th>\n",
       "      <th>websiteUrlFilled</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;p&gt;Please someone explain complete step by ste...</td>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32203002</td>\n",
       "      <td>https://lh4.googleusercontent.com/-_GtduZ_HroM...</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>-110.462467</td>\n",
       "      <td>33</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;p&gt;I've used SFST language for my project on N...</td>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>31961877</td>\n",
       "      <td>https://lh4.googleusercontent.com/-_GtduZ_HroM...</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "      <td>-157.135019</td>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>&lt;p&gt;Currently a software developer at HiQ Finla...</td>\n",
       "      <td>&lt;p&gt;The following JSON is returned from SharePo...</td>\n",
       "      <td>100</td>\n",
       "      <td>2413</td>\n",
       "      <td>Helsinki, Finland</td>\n",
       "      <td>22080350</td>\n",
       "      <td>https://i.stack.imgur.com/w2whQ.png</td>\n",
       "      <td>33857</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "      <td>-279.862500</td>\n",
       "      <td>134</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>&lt;p&gt;Love the computer science &amp;amp; technology ...</td>\n",
       "      <td>&lt;p&gt;What's the benifit of allocating a chunk of...</td>\n",
       "      <td>100</td>\n",
       "      <td>3</td>\n",
       "      <td>中国Shanghai Shi</td>\n",
       "      <td>28019349</td>\n",
       "      <td>https://www.gravatar.com/avatar/a33ac21cad99e5...</td>\n",
       "      <td>1470</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5</td>\n",
       "      <td>-234.970000</td>\n",
       "      <td>78</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;p&gt;I'm using an S3 bucket as the origin or a C...</td>\n",
       "      <td>100</td>\n",
       "      <td>77</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32211159</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10992</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>10.7</td>\n",
       "      <td>4</td>\n",
       "      <td>-527.002535</td>\n",
       "      <td>59</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 47 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                            AboutMe  \\\n",
       "0           0                                                NaN   \n",
       "1           1                                                NaN   \n",
       "2           2  <p>Currently a software developer at HiQ Finla...   \n",
       "3           3  <p>Love the computer science &amp; technology ...   \n",
       "4           4                                                NaN   \n",
       "\n",
       "                                                Body  Comment  Downvotes  \\\n",
       "0  <p>Please someone explain complete step by ste...      100          0   \n",
       "1  <p>I've used SFST language for my project on N...      100          0   \n",
       "2  <p>The following JSON is returned from SharePo...      100       2413   \n",
       "3  <p>What's the benifit of allocating a chunk of...      100          3   \n",
       "4  <p>I'm using an S3 bucket as the origin or a C...      100         77   \n",
       "\n",
       "            Location    PostId  \\\n",
       "0                NaN  32203002   \n",
       "1                NaN  31961877   \n",
       "2  Helsinki, Finland  22080350   \n",
       "3     中国Shanghai Shi  28019349   \n",
       "4                NaN  32211159   \n",
       "\n",
       "                                     ProfileImageUrl  Reputation  SOUrlCount  \\\n",
       "0  https://lh4.googleusercontent.com/-_GtduZ_HroM...           6           0   \n",
       "1  https://lh4.googleusercontent.com/-_GtduZ_HroM...           6           0   \n",
       "2                https://i.stack.imgur.com/w2whQ.png       33857           0   \n",
       "3  https://www.gravatar.com/avatar/a33ac21cad99e5...        1470           0   \n",
       "4                                                NaN       10992           0   \n",
       "\n",
       "         ...        sentencesStartWithICount sentencesStartWithYouCount  \\\n",
       "0        ...                               0                          0   \n",
       "1        ...                               3                          0   \n",
       "2        ...                               0                          0   \n",
       "3        ...                               0                          0   \n",
       "4        ...                               2                          0   \n",
       "\n",
       "   shortWordCount  smogIndex  tagCount  tagWeightSum titleLengthInChars  \\\n",
       "0               6        0.0         2   -110.462467                 33   \n",
       "1              27        0.0         4   -157.135019                 25   \n",
       "2              25        0.0         3   -279.862500                134   \n",
       "3              10        0.0         5   -234.970000                 78   \n",
       "4              35       10.7         4   -527.002535                 59   \n",
       "\n",
       "   urlCount  userProfileFilled  websiteUrlFilled  \n",
       "0         0                  1                 0  \n",
       "1         2                  1                 0  \n",
       "2         0                  4                 1  \n",
       "3         0                  3                 0  \n",
       "4         0                  0                 0  \n",
       "\n",
       "[5 rows x 47 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#3 rows deleted:\n",
    "#20484\t,RegEx used to extract tokens from an ASCIIMath expression\"\t11\t\t1\t0\t0\thttps://i.stack.imgur.com/6VcaB.png\t\t\t\t\t\t\t\t\t0\t1\t\t0\t0\t0\t0\t0\t1\t0\t0\t0\t1\t1\t0\t0\t0\t0\t0\n",
    "#71350\t,Bootstrap modal wont trigger on mobile\"\t443\tLake Mary, FL, United States\t85\t165\t11\thttps://i.stack.imgur.com/TLzJW.png?s=128&g=1\t\t\t\t\t\t\t\t\t0\t1\t\t0\t0\t0\t0\t0\t3\t0\t0\t0\t1\t1\t0\t0\t0\t0\t0\n",
    "#60036\t<p>from group \"\"Value\"\" i can get the value of hour and it gives output as 1\t3\t2\t4\t5\t6 and so on for each generator name but since no of hour is common to all i need that i get it only once that is i should get 1\t2\t5\t6 only once</p>\t\t\t\t\t\t\t0\t1\t\t0\t0\t0\t0\t0\t1\t0\t0\t0\t1\t1\t0\t1\t0\t0\t0\n",
    "\n",
    "openCloseData=pd.read_csv(\"openCloseData9.csv\",index_col=False)\n",
    "openCloseData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Unnamed: 0                                            AboutMe  \\\n",
      "0           0                                                NaN   \n",
      "1           1                                                NaN   \n",
      "2           2  <p>Currently a software developer at HiQ Finla...   \n",
      "3           3  <p>Love the computer science &amp; technology ...   \n",
      "4           4                                                NaN   \n",
      "\n",
      "                                                Body  Comment  Downvotes  \\\n",
      "0  <p>Please someone explain complete step by ste...      100          0   \n",
      "1  <p>I've used SFST language for my project on N...      100          0   \n",
      "2  <p>The following JSON is returned from SharePo...      100       2413   \n",
      "3  <p>What's the benifit of allocating a chunk of...      100          3   \n",
      "4  <p>I'm using an S3 bucket as the origin or a C...      100         77   \n",
      "\n",
      "            Location    PostId  \\\n",
      "0                NaN  32203002   \n",
      "1                NaN  31961877   \n",
      "2  Helsinki, Finland  22080350   \n",
      "3     中国Shanghai Shi  28019349   \n",
      "4                NaN  32211159   \n",
      "\n",
      "                                     ProfileImageUrl  Reputation  SOUrlCount  \\\n",
      "0  https://lh4.googleusercontent.com/-_GtduZ_HroM...           6           0   \n",
      "1  https://lh4.googleusercontent.com/-_GtduZ_HroM...           6           0   \n",
      "2                https://i.stack.imgur.com/w2whQ.png       33857           0   \n",
      "3  https://www.gravatar.com/avatar/a33ac21cad99e5...        1470           0   \n",
      "4                                                NaN       10992           0   \n",
      "\n",
      "         ...        sentencesStartWithICount sentencesStartWithYouCount  \\\n",
      "0        ...                               0                          0   \n",
      "1        ...                               3                          0   \n",
      "2        ...                               0                          0   \n",
      "3        ...                               0                          0   \n",
      "4        ...                               2                          0   \n",
      "\n",
      "   shortWordCount  smogIndex  tagCount  tagWeightSum titleLengthInChars  \\\n",
      "0               6        0.0         2   -110.462467                 33   \n",
      "1              27        0.0         4   -157.135019                 25   \n",
      "2              25        0.0         3   -279.862500                134   \n",
      "3              10        0.0         5   -234.970000                 78   \n",
      "4              35       10.7         4   -527.002535                 59   \n",
      "\n",
      "   urlCount  userProfileFilled  websiteUrlFilled  \n",
      "0         0                  1                 0  \n",
      "1         2                  1                 0  \n",
      "2         0                  4                 1  \n",
      "3         0                  3                 0  \n",
      "4         0                  0                 0  \n",
      "\n",
      "[5 rows x 47 columns]\n"
     ]
    }
   ],
   "source": [
    "print(openCloseData.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(159996, 47)\n",
      "(159996, 46)\n"
     ]
    }
   ],
   "source": [
    "print(openCloseData.shape)\n",
    "openCloseData.drop(['Unnamed: 0'],inplace=True,axis=1)\n",
    "print(openCloseData.shape)\n",
    "#Actually 4 rows are removed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['AboutMe', 'Body', 'Comment', 'Downvotes', 'Location', 'PostId',\n",
       "       'ProfileImageUrl', 'Reputation', 'SOUrlCount', 'Tags', 'Title',\n",
       "       'Unnamed: 0.1', 'Upvotes', 'UserId', 'Views', 'WebsiteUrl',\n",
       "       'aboutMeFilled', 'bodyWordCount', 'closed', 'codeLength',\n",
       "       'colemanLiauIndex', 'countOfInterrogativeSent',\n",
       "       'daleChallReadabilityScore', 'experienceInTime', 'firstLineLength',\n",
       "       'fleschReadingEase', 'gunningFog', 'locationFilled', 'lowerUpperRatio',\n",
       "       'numAns', 'numPR', 'numPeopleReached', 'numQues',\n",
       "       'profileImageUrlFilled', 'punctuationCount', 'sentenceCount',\n",
       "       'sentencesStartWithICount', 'sentencesStartWithYouCount',\n",
       "       'shortWordCount', 'smogIndex', 'tagCount', 'tagWeightSum',\n",
       "       'titleLengthInChars', 'urlCount', 'userProfileFilled',\n",
       "       'websiteUrlFilled'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "openCloseData.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(159996, 46)\n"
     ]
    }
   ],
   "source": [
    "#https://stackoverflow.com/questions/33961028/remove-non-numeric-rows-in-one-column-with-pandas\n",
    "print(openCloseData.shape)\n",
    "#openCloseData[openCloseData.Reputation.apply(lambda x: x.isnumeric())]\n",
    "#print(openCloseData.dtypes)\n",
    "#print(openCloseData.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#openCloseData['codeTextRatio']=openCloseData['codeLength']/(openCloseData['bodyWordCount']+1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ADDING MORE FEATURES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tagWeightData=pd.read_csv(\"tagWeightData.csv\")\n",
    "# print(tagWeightData.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import re\n",
    "# idx=0\n",
    "# def calculateTagWeight(tags):\n",
    "#     forClosed=re.findall('<(.*?)>',tags)\n",
    "#     forOpen=tags.split('|')\n",
    "#     #print(forClosed)\n",
    "#     #print(forOpen)\n",
    "#     if(len(forClosed) ):  #if we pass tags for open in close it won't work, but vice versa will work hence \n",
    "#         # this particular order of if-else \n",
    "#         tagList=forClosed\n",
    "#     elif(len(forOpen) ):\n",
    "#         tagList=forOpen\n",
    "#     #print(tagList)\n",
    "#     wt=0\n",
    "# #     for t in tagList:\n",
    "# #         #print(t)\n",
    "# #         try:\n",
    "            \n",
    "# #         except:\n",
    "# #             print(t)\n",
    "# #         print(wt)\n",
    "# #     print(\"check\")\n",
    "# #     if(idx%1000==0)print(idx)\n",
    "# #     idx+=1\n",
    "#     try:\n",
    "#         for t in tagList:\n",
    "#             wt=wt+tagWeightData[tagWeightData['TagName']==t][['tagWeight']].values[0]\n",
    "#     except:\n",
    "#         #print(tagList)\n",
    "#         pass\n",
    "#     return wt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import time\n",
    "# tic=time.time()\n",
    "# openCloseData['tagWeightSum']=openCloseData['Tags'].apply(calculateTagWeight) #tagClosingWeightSum\n",
    "# toc=time.time()\n",
    "# print(str((toc-tic)/60)+\"mins\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['AboutMe', 'Body', 'Comment', 'Downvotes', 'Location', 'PostId',\n",
       "       'ProfileImageUrl', 'Reputation', 'SOUrlCount', 'Tags', 'Title',\n",
       "       'Unnamed: 0.1', 'Upvotes', 'UserId', 'Views', 'WebsiteUrl',\n",
       "       'aboutMeFilled', 'bodyWordCount', 'closed', 'codeLength',\n",
       "       'colemanLiauIndex', 'countOfInterrogativeSent',\n",
       "       'daleChallReadabilityScore', 'experienceInTime', 'firstLineLength',\n",
       "       'fleschReadingEase', 'gunningFog', 'locationFilled', 'lowerUpperRatio',\n",
       "       'numAns', 'numPR', 'numPeopleReached', 'numQues',\n",
       "       'profileImageUrlFilled', 'punctuationCount', 'sentenceCount',\n",
       "       'sentencesStartWithICount', 'sentencesStartWithYouCount',\n",
       "       'shortWordCount', 'smogIndex', 'tagCount', 'tagWeightSum',\n",
       "       'titleLengthInChars', 'urlCount', 'userProfileFilled',\n",
       "       'websiteUrlFilled'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "openCloseData.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# openCloseData['tagWeightSum'][0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# openCloseData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def getInitialEle(ar):\n",
    "#     #print(type(ar))\n",
    "#     return float(ar)\n",
    "\n",
    "# openCloseData['tagWeightSum']=openCloseData['tagWeightSum'].apply(getInitialEle)\n",
    "# print(openCloseData.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# openCloseData.to_csv(\"openCloseData[35 cols].csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(159996, 29)\n",
      "<class 'pandas.core.frame.DataFrame'>\n"
     ]
    }
   ],
   "source": [
    "X = openCloseData.loc[:,['Downvotes', \n",
    "       'Reputation', 'SOUrlCount',\n",
    "       'Upvotes', 'Views',\n",
    "       'bodyWordCount', 'closed', 'codeLength',\n",
    "       'colemanLiauIndex', 'countOfInterrogativeSent',\n",
    "       'daleChallReadabilityScore', 'experienceInTime', 'firstLineLength',\n",
    "       'fleschReadingEase', 'gunningFog', 'lowerUpperRatio',\n",
    "       'numAns', 'numQues',\n",
    "       'punctuationCount', 'sentenceCount',\n",
    "       'sentencesStartWithICount', 'sentencesStartWithYouCount',\n",
    "       'shortWordCount', 'smogIndex', 'tagCount', 'tagWeightSum',\n",
    "       'titleLengthInChars', 'urlCount', 'userProfileFilled'] ]\n",
    "\n",
    "\n",
    "X.replace(np.nan,0,inplace=True)  #replacing NaN to 0\n",
    "X.lowerUpperRatio.replace(np.inf,0,inplace=True)  #replacing infinity to 0\n",
    "print(X.shape)\n",
    "print(type(X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Downvotes</th>\n",
       "      <th>Reputation</th>\n",
       "      <th>SOUrlCount</th>\n",
       "      <th>Upvotes</th>\n",
       "      <th>Views</th>\n",
       "      <th>bodyWordCount</th>\n",
       "      <th>closed</th>\n",
       "      <th>codeLength</th>\n",
       "      <th>colemanLiauIndex</th>\n",
       "      <th>countOfInterrogativeSent</th>\n",
       "      <th>...</th>\n",
       "      <th>sentenceCount</th>\n",
       "      <th>sentencesStartWithICount</th>\n",
       "      <th>sentencesStartWithYouCount</th>\n",
       "      <th>shortWordCount</th>\n",
       "      <th>smogIndex</th>\n",
       "      <th>tagCount</th>\n",
       "      <th>tagWeightSum</th>\n",
       "      <th>titleLengthInChars</th>\n",
       "      <th>urlCount</th>\n",
       "      <th>userProfileFilled</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9.74</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>-110.462467</td>\n",
       "      <td>33</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>18.39</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "      <td>-157.135019</td>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2413</td>\n",
       "      <td>33857</td>\n",
       "      <td>0</td>\n",
       "      <td>1574</td>\n",
       "      <td>2438</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>18555</td>\n",
       "      <td>15.11</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "      <td>-279.862500</td>\n",
       "      <td>134</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1470</td>\n",
       "      <td>0</td>\n",
       "      <td>362</td>\n",
       "      <td>312</td>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13.41</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5</td>\n",
       "      <td>-234.970000</td>\n",
       "      <td>78</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>77</td>\n",
       "      <td>10992</td>\n",
       "      <td>0</td>\n",
       "      <td>269</td>\n",
       "      <td>667</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8.70</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>10.7</td>\n",
       "      <td>4</td>\n",
       "      <td>-527.002535</td>\n",
       "      <td>59</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Downvotes  Reputation  SOUrlCount  Upvotes  Views  bodyWordCount  closed  \\\n",
       "0          0           6           0        0      8             17       0   \n",
       "1          0           6           0        0      8             41       0   \n",
       "2       2413       33857           0     1574   2438             70       0   \n",
       "3          3        1470           0      362    312             25       0   \n",
       "4         77       10992           0      269    667             70       0   \n",
       "\n",
       "   codeLength  colemanLiauIndex  countOfInterrogativeSent        ...          \\\n",
       "0           0              9.74                         0        ...           \n",
       "1           0             18.39                         1        ...           \n",
       "2       18555             15.11                         1        ...           \n",
       "3           0             13.41                         3        ...           \n",
       "4           0              8.70                         1        ...           \n",
       "\n",
       "   sentenceCount  sentencesStartWithICount  sentencesStartWithYouCount  \\\n",
       "0              1                         0                           0   \n",
       "1              9                         3                           0   \n",
       "2              2                         0                           0   \n",
       "3              3                         0                           0   \n",
       "4              4                         2                           0   \n",
       "\n",
       "   shortWordCount  smogIndex  tagCount  tagWeightSum  titleLengthInChars  \\\n",
       "0               6        0.0         2   -110.462467                  33   \n",
       "1              27        0.0         4   -157.135019                  25   \n",
       "2              25        0.0         3   -279.862500                 134   \n",
       "3              10        0.0         5   -234.970000                  78   \n",
       "4              35       10.7         4   -527.002535                  59   \n",
       "\n",
       "   urlCount  userProfileFilled  \n",
       "0         0                  1  \n",
       "1         2                  1  \n",
       "2         0                  4  \n",
       "3         0                  3  \n",
       "4         0                  0  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Downvotes</th>\n",
       "      <th>Reputation</th>\n",
       "      <th>SOUrlCount</th>\n",
       "      <th>Upvotes</th>\n",
       "      <th>Views</th>\n",
       "      <th>bodyWordCount</th>\n",
       "      <th>closed</th>\n",
       "      <th>codeLength</th>\n",
       "      <th>colemanLiauIndex</th>\n",
       "      <th>countOfInterrogativeSent</th>\n",
       "      <th>...</th>\n",
       "      <th>sentenceCount</th>\n",
       "      <th>sentencesStartWithICount</th>\n",
       "      <th>sentencesStartWithYouCount</th>\n",
       "      <th>shortWordCount</th>\n",
       "      <th>smogIndex</th>\n",
       "      <th>tagCount</th>\n",
       "      <th>tagWeightSum</th>\n",
       "      <th>titleLengthInChars</th>\n",
       "      <th>urlCount</th>\n",
       "      <th>userProfileFilled</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>0 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Downvotes, Reputation, SOUrlCount, Upvotes, Views, bodyWordCount, closed, codeLength, colemanLiauIndex, countOfInterrogativeSent, daleChallReadabilityScore, experienceInTime, firstLineLength, fleschReadingEase, gunningFog, lowerUpperRatio, numAns, numQues, punctuationCount, sentenceCount, sentencesStartWithICount, sentencesStartWithYouCount, shortWordCount, smogIndex, tagCount, tagWeightSum, titleLengthInChars, urlCount, userProfileFilled]\n",
       "Index: []\n",
       "\n",
       "[0 rows x 29 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[np.isinf(X).any(axis=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(159996,)\n"
     ]
    }
   ],
   "source": [
    "y = X.closed\n",
    "X.drop(['closed'],inplace=True,axis=1)\n",
    "print(y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MEAN NORMALIZATION of X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Downvotes', 'Reputation', 'SOUrlCount', 'Upvotes', 'Views',\n",
       "       'bodyWordCount', 'codeLength', 'colemanLiauIndex',\n",
       "       'countOfInterrogativeSent', 'daleChallReadabilityScore',\n",
       "       'experienceInTime', 'firstLineLength', 'fleschReadingEase',\n",
       "       'gunningFog', 'lowerUpperRatio', 'numAns', 'numQues',\n",
       "       'punctuationCount', 'sentenceCount', 'sentencesStartWithICount',\n",
       "       'sentencesStartWithYouCount', 'shortWordCount', 'smogIndex', 'tagCount',\n",
       "       'tagWeightSum', 'titleLengthInChars', 'urlCount', 'userProfileFilled'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "   Downvotes  Reputation  SOUrlCount  Upvotes  Views  bodyWordCount  \\\n",
      "0          0           6           0        0      8             17   \n",
      "1          0           6           0        0      8             41   \n",
      "2       2413       33857           0     1574   2438             70   \n",
      "3          3        1470           0      362    312             25   \n",
      "4         77       10992           0      269    667             70   \n",
      "\n",
      "   codeLength  colemanLiauIndex  countOfInterrogativeSent  \\\n",
      "0           0              9.74                         0   \n",
      "1           0             18.39                         1   \n",
      "2       18555             15.11                         1   \n",
      "3           0             13.41                         3   \n",
      "4           0              8.70                         1   \n",
      "\n",
      "   daleChallReadabilityScore        ...          sentenceCount  \\\n",
      "0                       7.27        ...                      1   \n",
      "1                       9.66        ...                      9   \n",
      "2                      10.95        ...                      2   \n",
      "3                       8.67        ...                      3   \n",
      "4                       7.66        ...                      4   \n",
      "\n",
      "   sentencesStartWithICount  sentencesStartWithYouCount  shortWordCount  \\\n",
      "0                         0                           0               6   \n",
      "1                         3                           0              27   \n",
      "2                         0                           0              25   \n",
      "3                         0                           0              10   \n",
      "4                         2                           0              35   \n",
      "\n",
      "   smogIndex  tagCount  tagWeightSum  titleLengthInChars  urlCount  \\\n",
      "0        0.0         2   -110.462467                  33         0   \n",
      "1        0.0         4   -157.135019                  25         2   \n",
      "2        0.0         3   -279.862500                 134         0   \n",
      "3        0.0         5   -234.970000                  78         0   \n",
      "4       10.7         4   -527.002535                  59         0   \n",
      "\n",
      "   userProfileFilled  \n",
      "0                  1  \n",
      "1                  1  \n",
      "2                  4  \n",
      "3                  3  \n",
      "4                  0  \n",
      "\n",
      "[5 rows x 28 columns]\n"
     ]
    }
   ],
   "source": [
    "print(type(X))\n",
    "print(X.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "StandardScaler(copy=True, with_mean=True, with_std=True)\n",
      "[  2.18902723e+01   2.01807416e+03   1.54941374e-02   2.26216980e+02\n",
      "   3.01059583e+02   8.53872034e+01   5.68528057e+02   9.82247931e+00\n",
      "   1.19541739e+00   7.71907479e+00   5.10066908e+02   9.98289770e+01\n",
      "   5.42152991e+01   1.89378949e+01   2.99731747e+01   3.25230693e+01\n",
      "   4.77225618e+01   4.33703530e+01   5.96079902e+00   2.02549439e+00\n",
      "   8.63771594e-03   4.09886685e+01   5.76047464e+00   2.90577889e+00\n",
      "  -2.44966593e+02   5.12481187e+01   1.91942299e-01   1.54041351e+00]\n",
      "[  1.11587341e+05   1.17173655e+08   2.93294209e-02   6.03714126e+05\n",
      "   2.94724961e+06   4.70156550e+03   2.07884824e+06   3.34948687e+01\n",
      "   4.61051576e+00   3.37276468e+00   3.42615432e+05   8.75204965e+03\n",
      "   1.37349155e+03   1.23125524e+02   6.64480756e+02   3.60008314e+04\n",
      "   1.52441460e+04   2.03530533e+04   1.22872198e+02   2.81215161e+00\n",
      "   1.12006717e-02   1.21652922e+03   3.21043352e+01   1.50461647e+00\n",
      "   4.59393389e+04   4.03492388e+02   8.66455736e-01   1.23490391e+00]\n",
      "<class 'numpy.ndarray'>\n"
     ]
    }
   ],
   "source": [
    "# x=(x-u)/std\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "print(scaler.fit(X))\n",
    "print(scaler.mean_)\n",
    "print(scaler.var_)\n",
    "X=scaler.fit_transform(X)\n",
    "print(type(X))\n",
    "X = pd.DataFrame(X, columns=['Downvotes', 'Reputation', 'SOUrlCount', 'Upvotes', 'Views',\n",
    "       'bodyWordCount', 'codeLength', 'colemanLiauIndex',\n",
    "       'countOfInterrogativeSent', 'daleChallReadabilityScore',\n",
    "       'experienceInTime', 'firstLineLength', 'fleschReadingEase',\n",
    "       'gunningFog', 'lowerUpperRatio', 'numAns', 'numQues',\n",
    "       'punctuationCount', 'sentenceCount', 'sentencesStartWithICount',\n",
    "       'sentencesStartWithYouCount', 'shortWordCount', 'smogIndex', 'tagCount',\n",
    "       'tagWeightSum', 'titleLengthInChars', 'urlCount', 'userProfileFilled'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "   Downvotes  Reputation  SOUrlCount   Upvotes     Views  bodyWordCount  \\\n",
      "0  -0.065531   -0.185878   -0.090472 -0.291145 -0.170705      -0.997364   \n",
      "1  -0.065531   -0.185878   -0.090472 -0.291145 -0.170705      -0.647346   \n",
      "2   7.158006    2.941328   -0.090472  1.734620  1.244755      -0.224408   \n",
      "3  -0.056550   -0.050632   -0.090472  0.174755  0.006373      -0.880691   \n",
      "4   0.164976    0.829025   -0.090472  0.055062  0.213158      -0.224408   \n",
      "\n",
      "   codeLength  colemanLiauIndex  countOfInterrogativeSent  \\\n",
      "0   -0.394312         -0.014251                 -0.556730   \n",
      "1   -0.394312          1.480356                 -0.091010   \n",
      "2   12.474829          0.913614                 -0.091010   \n",
      "3   -0.394312          0.619877                  0.840431   \n",
      "4   -0.394312         -0.193950                 -0.091010   \n",
      "\n",
      "   daleChallReadabilityScore        ...          sentenceCount  \\\n",
      "0                  -0.244526        ...              -0.447533   \n",
      "1                   1.056856        ...               0.274178   \n",
      "2                   1.759276        ...              -0.357319   \n",
      "3                   0.517790        ...              -0.267105   \n",
      "4                  -0.032167        ...              -0.176891   \n",
      "\n",
      "   sentencesStartWithICount  sentencesStartWithYouCount  shortWordCount  \\\n",
      "0                 -1.207846                   -0.081616       -1.003151   \n",
      "1                  0.581119                   -0.081616       -0.401065   \n",
      "2                 -1.207846                   -0.081616       -0.458407   \n",
      "3                 -1.207846                   -0.081616       -0.888468   \n",
      "4                 -0.015203                   -0.081616       -0.171699   \n",
      "\n",
      "   smogIndex  tagCount  tagWeightSum  titleLengthInChars  urlCount  \\\n",
      "0  -1.016662 -0.738430      0.627543           -0.908449 -0.206204   \n",
      "1  -1.016662  0.892056      0.409787           -1.306714  1.942402   \n",
      "2  -1.016662  0.076813     -0.162810            4.119649 -0.206204   \n",
      "3  -1.016662  1.707299      0.046640            1.331793 -0.206204   \n",
      "4   0.871773  0.892056     -1.315867            0.385913 -0.206204   \n",
      "\n",
      "   userProfileFilled  \n",
      "0          -0.486306  \n",
      "1          -0.486306  \n",
      "2           2.213327  \n",
      "3           1.313449  \n",
      "4          -1.386184  \n",
      "\n",
      "[5 rows x 28 columns]\n"
     ]
    }
   ],
   "source": [
    "print(type(X))\n",
    "print(X.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Conduct Principal Component Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print('Original number of features:', X.shape[1])\n",
    "# from sklearn.decomposition import PCA\n",
    "# pca = PCA(n_components=0.99)\n",
    "# X = pca.fit_transform(X)\n",
    "# print('Reduced number of features:', X.shape[1])\n",
    "# print(type(X))\n",
    "# X = pd.DataFrame(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Downvotes</th>\n",
       "      <th>Reputation</th>\n",
       "      <th>SOUrlCount</th>\n",
       "      <th>Upvotes</th>\n",
       "      <th>Views</th>\n",
       "      <th>bodyWordCount</th>\n",
       "      <th>codeLength</th>\n",
       "      <th>colemanLiauIndex</th>\n",
       "      <th>countOfInterrogativeSent</th>\n",
       "      <th>daleChallReadabilityScore</th>\n",
       "      <th>...</th>\n",
       "      <th>sentenceCount</th>\n",
       "      <th>sentencesStartWithICount</th>\n",
       "      <th>sentencesStartWithYouCount</th>\n",
       "      <th>shortWordCount</th>\n",
       "      <th>smogIndex</th>\n",
       "      <th>tagCount</th>\n",
       "      <th>tagWeightSum</th>\n",
       "      <th>titleLengthInChars</th>\n",
       "      <th>urlCount</th>\n",
       "      <th>userProfileFilled</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.065531</td>\n",
       "      <td>-0.185878</td>\n",
       "      <td>-0.090472</td>\n",
       "      <td>-0.291145</td>\n",
       "      <td>-0.170705</td>\n",
       "      <td>-0.997364</td>\n",
       "      <td>-0.394312</td>\n",
       "      <td>-0.014251</td>\n",
       "      <td>-0.556730</td>\n",
       "      <td>-0.244526</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.447533</td>\n",
       "      <td>-1.207846</td>\n",
       "      <td>-0.081616</td>\n",
       "      <td>-1.003151</td>\n",
       "      <td>-1.016662</td>\n",
       "      <td>-0.738430</td>\n",
       "      <td>0.627543</td>\n",
       "      <td>-0.908449</td>\n",
       "      <td>-0.206204</td>\n",
       "      <td>-0.486306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.065531</td>\n",
       "      <td>-0.185878</td>\n",
       "      <td>-0.090472</td>\n",
       "      <td>-0.291145</td>\n",
       "      <td>-0.170705</td>\n",
       "      <td>-0.647346</td>\n",
       "      <td>-0.394312</td>\n",
       "      <td>1.480356</td>\n",
       "      <td>-0.091010</td>\n",
       "      <td>1.056856</td>\n",
       "      <td>...</td>\n",
       "      <td>0.274178</td>\n",
       "      <td>0.581119</td>\n",
       "      <td>-0.081616</td>\n",
       "      <td>-0.401065</td>\n",
       "      <td>-1.016662</td>\n",
       "      <td>0.892056</td>\n",
       "      <td>0.409787</td>\n",
       "      <td>-1.306714</td>\n",
       "      <td>1.942402</td>\n",
       "      <td>-0.486306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.158006</td>\n",
       "      <td>2.941328</td>\n",
       "      <td>-0.090472</td>\n",
       "      <td>1.734620</td>\n",
       "      <td>1.244755</td>\n",
       "      <td>-0.224408</td>\n",
       "      <td>12.474829</td>\n",
       "      <td>0.913614</td>\n",
       "      <td>-0.091010</td>\n",
       "      <td>1.759276</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.357319</td>\n",
       "      <td>-1.207846</td>\n",
       "      <td>-0.081616</td>\n",
       "      <td>-0.458407</td>\n",
       "      <td>-1.016662</td>\n",
       "      <td>0.076813</td>\n",
       "      <td>-0.162810</td>\n",
       "      <td>4.119649</td>\n",
       "      <td>-0.206204</td>\n",
       "      <td>2.213327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.056550</td>\n",
       "      <td>-0.050632</td>\n",
       "      <td>-0.090472</td>\n",
       "      <td>0.174755</td>\n",
       "      <td>0.006373</td>\n",
       "      <td>-0.880691</td>\n",
       "      <td>-0.394312</td>\n",
       "      <td>0.619877</td>\n",
       "      <td>0.840431</td>\n",
       "      <td>0.517790</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.267105</td>\n",
       "      <td>-1.207846</td>\n",
       "      <td>-0.081616</td>\n",
       "      <td>-0.888468</td>\n",
       "      <td>-1.016662</td>\n",
       "      <td>1.707299</td>\n",
       "      <td>0.046640</td>\n",
       "      <td>1.331793</td>\n",
       "      <td>-0.206204</td>\n",
       "      <td>1.313449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.164976</td>\n",
       "      <td>0.829025</td>\n",
       "      <td>-0.090472</td>\n",
       "      <td>0.055062</td>\n",
       "      <td>0.213158</td>\n",
       "      <td>-0.224408</td>\n",
       "      <td>-0.394312</td>\n",
       "      <td>-0.193950</td>\n",
       "      <td>-0.091010</td>\n",
       "      <td>-0.032167</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.176891</td>\n",
       "      <td>-0.015203</td>\n",
       "      <td>-0.081616</td>\n",
       "      <td>-0.171699</td>\n",
       "      <td>0.871773</td>\n",
       "      <td>0.892056</td>\n",
       "      <td>-1.315867</td>\n",
       "      <td>0.385913</td>\n",
       "      <td>-0.206204</td>\n",
       "      <td>-1.386184</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Downvotes  Reputation  SOUrlCount   Upvotes     Views  bodyWordCount  \\\n",
       "0  -0.065531   -0.185878   -0.090472 -0.291145 -0.170705      -0.997364   \n",
       "1  -0.065531   -0.185878   -0.090472 -0.291145 -0.170705      -0.647346   \n",
       "2   7.158006    2.941328   -0.090472  1.734620  1.244755      -0.224408   \n",
       "3  -0.056550   -0.050632   -0.090472  0.174755  0.006373      -0.880691   \n",
       "4   0.164976    0.829025   -0.090472  0.055062  0.213158      -0.224408   \n",
       "\n",
       "   codeLength  colemanLiauIndex  countOfInterrogativeSent  \\\n",
       "0   -0.394312         -0.014251                 -0.556730   \n",
       "1   -0.394312          1.480356                 -0.091010   \n",
       "2   12.474829          0.913614                 -0.091010   \n",
       "3   -0.394312          0.619877                  0.840431   \n",
       "4   -0.394312         -0.193950                 -0.091010   \n",
       "\n",
       "   daleChallReadabilityScore        ...          sentenceCount  \\\n",
       "0                  -0.244526        ...              -0.447533   \n",
       "1                   1.056856        ...               0.274178   \n",
       "2                   1.759276        ...              -0.357319   \n",
       "3                   0.517790        ...              -0.267105   \n",
       "4                  -0.032167        ...              -0.176891   \n",
       "\n",
       "   sentencesStartWithICount  sentencesStartWithYouCount  shortWordCount  \\\n",
       "0                 -1.207846                   -0.081616       -1.003151   \n",
       "1                  0.581119                   -0.081616       -0.401065   \n",
       "2                 -1.207846                   -0.081616       -0.458407   \n",
       "3                 -1.207846                   -0.081616       -0.888468   \n",
       "4                 -0.015203                   -0.081616       -0.171699   \n",
       "\n",
       "   smogIndex  tagCount  tagWeightSum  titleLengthInChars  urlCount  \\\n",
       "0  -1.016662 -0.738430      0.627543           -0.908449 -0.206204   \n",
       "1  -1.016662  0.892056      0.409787           -1.306714  1.942402   \n",
       "2  -1.016662  0.076813     -0.162810            4.119649 -0.206204   \n",
       "3  -1.016662  1.707299      0.046640            1.331793 -0.206204   \n",
       "4   0.871773  0.892056     -1.315867            0.385913 -0.206204   \n",
       "\n",
       "   userProfileFilled  \n",
       "0          -0.486306  \n",
       "1          -0.486306  \n",
       "2           2.213327  \n",
       "3           1.313449  \n",
       "4          -1.386184  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train/Test Split  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(111997, 28)\n",
      "(111997,)\n",
      "(47999, 28)\n",
      "(47999,)\n"
     ]
    }
   ],
   "source": [
    "X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.3,random_state=42)\n",
    "print(X_train.shape)\n",
    "print(y_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test.to_csv(\"test.csv\")\n",
    "y_test.to_csv(\"actual.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Downvotes</th>\n",
       "      <th>Reputation</th>\n",
       "      <th>SOUrlCount</th>\n",
       "      <th>Upvotes</th>\n",
       "      <th>Views</th>\n",
       "      <th>bodyWordCount</th>\n",
       "      <th>codeLength</th>\n",
       "      <th>colemanLiauIndex</th>\n",
       "      <th>countOfInterrogativeSent</th>\n",
       "      <th>daleChallReadabilityScore</th>\n",
       "      <th>...</th>\n",
       "      <th>sentenceCount</th>\n",
       "      <th>sentencesStartWithICount</th>\n",
       "      <th>sentencesStartWithYouCount</th>\n",
       "      <th>shortWordCount</th>\n",
       "      <th>smogIndex</th>\n",
       "      <th>tagCount</th>\n",
       "      <th>tagWeightSum</th>\n",
       "      <th>titleLengthInChars</th>\n",
       "      <th>urlCount</th>\n",
       "      <th>userProfileFilled</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>29416</th>\n",
       "      <td>-0.065531</td>\n",
       "      <td>-0.186063</td>\n",
       "      <td>-0.090472</td>\n",
       "      <td>-0.291145</td>\n",
       "      <td>-0.168376</td>\n",
       "      <td>-0.136904</td>\n",
       "      <td>2.121950</td>\n",
       "      <td>-0.993953</td>\n",
       "      <td>-0.091010</td>\n",
       "      <td>-0.745477</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.086677</td>\n",
       "      <td>-0.611525</td>\n",
       "      <td>-0.081616</td>\n",
       "      <td>0.143679</td>\n",
       "      <td>0.695284</td>\n",
       "      <td>1.707299</td>\n",
       "      <td>-0.265550</td>\n",
       "      <td>-1.505846</td>\n",
       "      <td>-0.206204</td>\n",
       "      <td>-0.486306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58063</th>\n",
       "      <td>-0.065531</td>\n",
       "      <td>-0.186340</td>\n",
       "      <td>-0.090472</td>\n",
       "      <td>-0.291145</td>\n",
       "      <td>-0.175365</td>\n",
       "      <td>-0.414001</td>\n",
       "      <td>-0.385296</td>\n",
       "      <td>-0.375376</td>\n",
       "      <td>-0.556730</td>\n",
       "      <td>-0.179185</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.176891</td>\n",
       "      <td>-0.015203</td>\n",
       "      <td>-0.081616</td>\n",
       "      <td>-0.515748</td>\n",
       "      <td>0.659986</td>\n",
       "      <td>0.076813</td>\n",
       "      <td>-0.479731</td>\n",
       "      <td>-0.012352</td>\n",
       "      <td>-0.206204</td>\n",
       "      <td>-0.486306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62206</th>\n",
       "      <td>-0.059543</td>\n",
       "      <td>-0.170266</td>\n",
       "      <td>-0.090472</td>\n",
       "      <td>-0.249961</td>\n",
       "      <td>-0.166046</td>\n",
       "      <td>3.567450</td>\n",
       "      <td>0.739670</td>\n",
       "      <td>1.551198</td>\n",
       "      <td>-0.556730</td>\n",
       "      <td>0.261869</td>\n",
       "      <td>...</td>\n",
       "      <td>5.687009</td>\n",
       "      <td>1.177441</td>\n",
       "      <td>-0.081616</td>\n",
       "      <td>3.154105</td>\n",
       "      <td>1.224751</td>\n",
       "      <td>0.892056</td>\n",
       "      <td>-0.017494</td>\n",
       "      <td>-1.207148</td>\n",
       "      <td>-0.206204</td>\n",
       "      <td>-0.486306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14790</th>\n",
       "      <td>-0.065531</td>\n",
       "      <td>-0.182922</td>\n",
       "      <td>-0.090472</td>\n",
       "      <td>-0.273127</td>\n",
       "      <td>-0.170123</td>\n",
       "      <td>-0.078567</td>\n",
       "      <td>0.218801</td>\n",
       "      <td>-0.342546</td>\n",
       "      <td>-0.556730</td>\n",
       "      <td>-0.429660</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.357319</td>\n",
       "      <td>-0.611525</td>\n",
       "      <td>-0.081616</td>\n",
       "      <td>-0.200370</td>\n",
       "      <td>1.277698</td>\n",
       "      <td>1.707299</td>\n",
       "      <td>-0.384663</td>\n",
       "      <td>-0.958232</td>\n",
       "      <td>-0.206204</td>\n",
       "      <td>-0.486306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152752</th>\n",
       "      <td>-0.065531</td>\n",
       "      <td>-0.182737</td>\n",
       "      <td>-0.090472</td>\n",
       "      <td>-0.284710</td>\n",
       "      <td>-0.168376</td>\n",
       "      <td>0.621468</td>\n",
       "      <td>-0.394312</td>\n",
       "      <td>0.498926</td>\n",
       "      <td>0.374711</td>\n",
       "      <td>0.888057</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003536</td>\n",
       "      <td>-0.015203</td>\n",
       "      <td>-0.081616</td>\n",
       "      <td>0.430386</td>\n",
       "      <td>1.507134</td>\n",
       "      <td>1.707299</td>\n",
       "      <td>-0.231941</td>\n",
       "      <td>-0.161702</td>\n",
       "      <td>-0.206204</td>\n",
       "      <td>1.313449</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Downvotes  Reputation  SOUrlCount   Upvotes     Views  bodyWordCount  \\\n",
       "29416   -0.065531   -0.186063   -0.090472 -0.291145 -0.168376      -0.136904   \n",
       "58063   -0.065531   -0.186340   -0.090472 -0.291145 -0.175365      -0.414001   \n",
       "62206   -0.059543   -0.170266   -0.090472 -0.249961 -0.166046       3.567450   \n",
       "14790   -0.065531   -0.182922   -0.090472 -0.273127 -0.170123      -0.078567   \n",
       "152752  -0.065531   -0.182737   -0.090472 -0.284710 -0.168376       0.621468   \n",
       "\n",
       "        codeLength  colemanLiauIndex  countOfInterrogativeSent  \\\n",
       "29416     2.121950         -0.993953                 -0.091010   \n",
       "58063    -0.385296         -0.375376                 -0.556730   \n",
       "62206     0.739670          1.551198                 -0.556730   \n",
       "14790     0.218801         -0.342546                 -0.556730   \n",
       "152752   -0.394312          0.498926                  0.374711   \n",
       "\n",
       "        daleChallReadabilityScore        ...          sentenceCount  \\\n",
       "29416                   -0.745477        ...              -0.086677   \n",
       "58063                   -0.179185        ...              -0.176891   \n",
       "62206                    0.261869        ...               5.687009   \n",
       "14790                   -0.429660        ...              -0.357319   \n",
       "152752                   0.888057        ...               0.003536   \n",
       "\n",
       "        sentencesStartWithICount  sentencesStartWithYouCount  shortWordCount  \\\n",
       "29416                  -0.611525                   -0.081616        0.143679   \n",
       "58063                  -0.015203                   -0.081616       -0.515748   \n",
       "62206                   1.177441                   -0.081616        3.154105   \n",
       "14790                  -0.611525                   -0.081616       -0.200370   \n",
       "152752                 -0.015203                   -0.081616        0.430386   \n",
       "\n",
       "        smogIndex  tagCount  tagWeightSum  titleLengthInChars  urlCount  \\\n",
       "29416    0.695284  1.707299     -0.265550           -1.505846 -0.206204   \n",
       "58063    0.659986  0.076813     -0.479731           -0.012352 -0.206204   \n",
       "62206    1.224751  0.892056     -0.017494           -1.207148 -0.206204   \n",
       "14790    1.277698  1.707299     -0.384663           -0.958232 -0.206204   \n",
       "152752   1.507134  1.707299     -0.231941           -0.161702 -0.206204   \n",
       "\n",
       "        userProfileFilled  \n",
       "29416           -0.486306  \n",
       "58063           -0.486306  \n",
       "62206           -0.486306  \n",
       "14790           -0.486306  \n",
       "152752           1.313449  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LOGISTIC REGRESSION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logreg=LogisticRegression()\n",
    "logreg.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(47999,)\n"
     ]
    }
   ],
   "source": [
    "y_pred=logreg.predict(X_test)\n",
    "print(y_pred.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.89043521740036247"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Accuracy\n",
    "np.mean(y_pred==y_test)\n",
    "#26 columns with Scaling: 0.8864976353674034"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['logreg1.joblib']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#https://scikit-learn.org/stable/modules/model_persistence.html\n",
    "#EXPORTING RANDOM FOREST\n",
    "from joblib import dump, load\n",
    "dump(logreg, 'logreg1.joblib') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.89      0.89      0.89     23916\n",
      "          1       0.89      0.89      0.89     24083\n",
      "\n",
      "avg / total       0.89      0.89      0.89     47999\n",
      "\n",
      "Precision: 0.890446383844\n",
      "Recall: 0.890425990059\n",
      "F1 Score:0.890431594644\n",
      "(0.89044638384446528, 0.89042599005861822, 0.89043159464401378, None)\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,y_pred) )\n",
    "print(\"Precision: \"+str(precision_score(y_test, y_pred, average='macro')) )\n",
    "print(\"Recall: \"+str(recall_score(y_test, y_pred, average='macro')) )\n",
    "print(\"F1 Score:\"+str(f1_score(y_test, y_pred, average='macro')) )\n",
    "print(precision_recall_fscore_support(y_test, y_pred, average='macro') )\n",
    "# Precision, recall, fscore, support in order\n",
    "\n",
    "#26 cols with Scaling:\n",
    "# Precision: 0.8873913633946493\n",
    "# Recall: 0.8873865140134614\n",
    "# F1 Score:0.8873725937122617\n",
    "\n",
    "# precision    recall  f1-score   support\n",
    "\n",
    "#           0       0.89      0.88      0.89     24085\n",
    "#           1       0.88      0.89      0.89     23914\n",
    "\n",
    "# avg / total       0.89      0.89      0.89     47999\n",
    "\n",
    "# Precision: 0.8865374593942957\n",
    "# Recall: 0.886516977164158\n",
    "# F1 Score:0.8864970500441661\n",
    "# (0.8865374593942957, 0.886516977164158, 0.8864970500441661, None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import KFold,cross_val_score\n",
    "k_fold = KFold(n_splits=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_score = cross_val_score(logreg,X,y,cv=k_fold,scoring='accuracy',n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.8213125   0.8192131   0.89177787  0.84758899  0.74483578]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd8VFX6x/HPN40AoZcooUsRpAqCCLpgA3tZdUVlV3+u2LDsWlZcl3Xdtaxt7QW3WFDQdbGjWCBWbJEmVUA6CkgNLZA8vz/uDY4xZYBMZpI879drXrlzy7nPnJnMM7ecc2RmOOecc6VJincAzjnnEp8nC+ecc2XyZOGcc65Mniycc86VyZOFc865MnmycM45VyZPFglOUn9J30jKlXRqGeveLGlMKcsXSzq6/KNMHJKelPS3cPpwSfOiWXcv95Urqe3ebl+RJJ0r6e293HaWpIHlHFLCk/SmpN/EO45E4cmigknKlrQ9/KLJLe3LLHQL8JCZZZjZyxURY1VhZh+aWcfyKCt8335bpPwMM1tUHuUX2Ve5J3Uze9bMjo1i3z9LoGZ2kJll78n+JLWWZBGf88WSbtjDsOPKzI4zs6fiHUei8GQRHyPCL5qMKL7MWgGzKiKoaEhKiXcMrlKpb2YZwBnAnyQdU9478M9kxfBkkcAkLQTaAq+Fv85qSGom6VVJ6yQtkHRRKdsPk7RE0g+S/ljGvmpKuidcf6Okj8J5hb8QL5S0FJgUrn9yeHpiQ/iru1NEWX+QtELSZknzJB0Vzu8j6UtJmyR9L+neEmKZI+nEiOcpktZKOjh8/l9J34VxfiDpoBLKGShpecTznpK+CuN6HkiPWNZA0uuS1khaH043D5fdChwOPBS+Dw+F801Su3C6nqSnw+2XSLpJUlK47PywPu8Oy/5W0nGlvR+lvE8Xhe/7uvBz0Cxi2bFhfW+U9Iik9wuPhgpjCKcl6R+SVofrzpDURdJw4Fzg+vB1vhauv/tIR1KypBslLQzrMUdSi7LiNrMvCX709IiIt5mk/4V19q2kKyOW1ZT0VFhfcyRdX+S9XBx+zmYAW8LPSGnlFfvZk5QuaYyC/5ENkr6QlBku2300KSkpfE+XhPX2tKR64bLC/5HfSFoaflZL/X+rlMzMHxX4ALKBNcBa4GNgYBnrLwaOjnj+PvAIwRddj7Cso8JlNwNjwunOQC5wBFADuBfYFVlWkf08HMaWBSQDh4XbtQYMeBqoDdQEOgBbgGOAVOB6YAGQBnQElgHNwnJbAweE01OAYeF0BnBoCbGMAp6NeH4CMDfi+f8BdcL47gOmRSx7EvhbOD0QWB5OpwFLgN+FMZ8B7IxYtxHwS6BWWPZ/gZeLvG+/LRKnAe3C6aeBV8JtWwPzgQvDZeeH+7oorNtLgZWAonnPI+YfGX5uDg5f+4PAB+GyxsAm4HQgBbgq3OdvI2L4KJweDOQA9QEBnYD9i9ZfcfEA1wEzw/dZQHegUTGxtg7rJyV8fiiwFTgtfJ4UxjAqfG/aAouAweHyOwg+6w2A5sCMwvcyIqZpQAuCz2RZ5RX72QMuBl4L3/dkoBdQt+h7TvCZWxCWmwGMB54p8lqfCGPpDuwAOsX7+6Zcv7viHUB1ewB9+fGL7jfAZsIv0xLWj/xHbQHkA3Uilt8OPBlO38yPyWIUMC5ivdpAHsV/CSUB24DuxSwr/EdoGzHvT8ALRbZfQfDl3A5YDRwNpBYp6wPgL0DjMuqoXVgvtcLnzwKjSli3fhhfvfD5kxSfLI6gyBc08AlFvhgjlvUA1kc83/3FETHPwliTwy+HzhHLLgayw+nzgQURy2qF2+5X1nteZP6/gDsjnmcQJITWwK+BKRHLRJC0i0sWRxIks0OBpCL72F1/JXwG5wGnRPE5L/zcbAg/WwbcXVj/BP8HS4tsMxL4Tzi9+4s+fP5bfp4s/q/I/1Vp5RX72SNIAp8A3Yp5Dbvfc+A94LKIZR3Duk+JeK3NI5Z/DpxdVj1VpoefhqpgZvaZmW02sx0WXDz7GDgedt91UnhB8PBiNm8GrDOzzRHzlhAcDRS37rKI/W4BfighrMYERyoLSwl9WcR0s3C/hWUXhMuzzGwBcDVB4lotaVzEqZILCY5K5oaH+ydSjLCMOcBJkmoBJwPPwe7TIHeEp0E2EXxpFL6G0jQDVlj4nxza/Rok1ZL0eHiaYRPBl0t9SclllFu478Ijl8iyI9+X7yJe39ZwMiOKsou+hsh6zyV4T7P4+fttwPKiBYTLJgEPERxNfi9ptKS6UcbQgtI/J0U1Jnid1xIk79RwfiugWXjqZ4OkDcCNQGa4/Cevp8h0cfPKKq+kz94zwERgnKSVku6UlMrP/aTuw+mUiPIh4j0mOIra0/c3oXmyiD8j+BWIBXedFF74/rCYdVcCDSXViZjXkuBXfVGrCP6xgeDLkOBUS3HWAtuBA8qIMzKOVhFlK9zXivB1PGdmA8J1DPh7OP8bMxsKNA3nvSipdgn7GwsMBU4BZocJBOCccN7RQD2CX3UQ1mEpVgFZYayFWkZMX0Pwa7GvmdUlOBKJLLe07pnXEvzKbBUxr6T3ZV8UrffaBO/pCoLX1zximSKfF2VmD5hZL+Aggi/R6woXlRHDMkr/nBS3r3wzu4fgM3ZZRDnfmln9iEcdMzs+XP6T10PEZzmy6CJxlVheSZ89M9tpZn8xs84Ep15PJDhKK+ondU/w/u4Cvt+DqqjUPFlUIEn1JQ0OL6qlSDqX4EtpYjTbm9kygkPm28MyuhH8Ynq2mNVfBE6UNEBSGsEtuMW+3+GRwb+Be8OLhMmS+kmqUUIoLwAnSDoq/BV2DcFpmE8kdZR0ZLjtdoJTEPnh6z9PUpNwfxvCsvJL2Mc44FiC8/vPRcyvE+7rB4LTObeVsH1RUwj+ua8M6/50oE+RcrcBGyQ1BP5cZPvvCc5X/4yZ5RPUya2S6khqBfweKLHNSxRSw/e48JFCUA8XSOoR1u9twGdmthh4A+gq6dRw3cuB/YorWNIhkvqG790Wgvep8H0o8XWG/gn8VVJ7BbpJKulHSFF3EFw8Tyc4TbMpvEhdM/zMdZF0SLjuC8BIBTceZAEjyii71PJK+uxJGiSpa3gEuYkg6Rf3mRwL/E5SG0kZBHX/vJntivK1V3qeLCpWKvA3frzAfQVwqpmV1dYi0lCCX9MrgZeAP5vZO0VXMrNZBF8YzxH8SltPCaclQtcSXLj8AlhH8OurpOQyDziP4ALrWuAk4CQzyyO4FnNHOP87gl9yN4abDgFmScoF7ic4p7u9hH2sIviCPwx4PmLR0wSnAFYAs4FPS3lNkeXlEVz8PZ+gLn5FcJGy0H0EFyfXhmW+VaSI+4EzFNyd80Axu7iC4It3EfARQb3/O5rYSjCBIHkVPm42s/cIrhf9j+A9PQA4O3x9a4EzgTsJEmln4EuCxFpUXYKLsesJ6vIHgusJEFwX6RyeyimuXc+9BF/kbxN8uf6LoN6i8Ua4z4vCBHsSwbWhbwnq/Z8ER4sQ/LhZHi57l+DHT3GvBdidsEsrr6TP3n5h2ZsITn2+T/FJ/t8Ep6w+CMvfTvCeVxuFF5ucc1WIgtt2lwPnmtnkeMezryRdSvAF/4t4x1Jd+ZGFc1VEeIqzfniK6kaC6y1RHXklGkn7K+jqJklSR4JTnS/FO67qzFs+Old19CM4/ZVGcIruVDPbFt+Q9loa8DjQhuAawziC9kUuTvw0lHPOuTLF9DSUpCEKuh9YoGI6EZPUStJ7CrobyFbYvUK47DcKelv9Rt7zo3POxVXMjizCW9HmE3QJsZzgLpuhZjY7Yp3/Aq+b2VOSjgQuMLNh4a2LXwK9Ce6lzgF6mdn6kvbXuHFja926dbnEvmXLFmrXLun2f1fI6yl6XlfR8XqKXnnVVU5Ozloza1LWerG8ZtGHoIuDRQCSxhE2sIpYpzNBXz0Ak4HCW/UGA++Y2bpw23cIbn0bW9LOWrduzZdfflkugWdnZzNw4MByKasq83qKntdVdLyeoldedSVpSdlrxTZZZPHT5vjLCfpviTSdoPO2+4HTgDphA5/itv1ZlxYKeskcDpCZmUl2dna5BJ6bm1tuZVVlXk/R87qKjtdT9Cq6rmKZLIrrfqHoOa9rCbp9Pp+gscsKgla20WyLmY0GRgP07t3byusXif+6iY7XU/S8rqLj9RS9iq6rWCaL5fy0P5fmBK2OdzOzlQStagmb0P/SzDYq6Ld+YJFts2MYq3POuVLE8m6oL4D2YV8qaQTdErwauYKkxmFLUwi6Ey7sHmEicGzYL0wDgj6Couo/yTnnXPmLWbIIO9gaQfAlP4dg/INZkm6RdHK42kBgnqT5BF393hpuuw74K0HC+QK4pfBit3POuYoX0xbcZjaBoEO0yHmjIqZfJOjEq7ht/82+dcTmnHOunHjfUM4558rkycI551yZPFk4F0MvT11B/zsmMXPFRvrfMYmXp5b34HnOVQzvdda5GHl56gpGjp/Jtp350AJWbNjGyPEzATi1Z3HDpjuXuPzIwrkYuWvivCBRAIs3B+1Mt+3M566JezIwonOJwZOFczGycsOPQ0n8b3FysfOdqyw8WTgXI40zauye7tqgYPd0Zt30eITj3D7xZOFcDKzbkseugoLdnZwd2/zHZLFtZz45S0rsbd+5hOTJwrlyll9gXDl2Klvy8vndMR3Iql8TgKz6Nbn22A7Uq5nKrx6fwpMff4uPVOkqC78byrlydvfb8/howVru/GU3zjqkBVce1Z7s7GyuOHcgAMP6teaaF6Zx82uzyVm6gTtO70rtGv6v6BKbH1k4V47enLmKR7MXck7flpx1SIti16lXM5XRw3pz/ZCOvDFjJac8/DELVudWcKTO7RlPFs6VkwWrN3Ptf6fTo0V9/nxS51LXTUoSlw1sx5gL+7J+Sx6nPPQRr89YWeo2zsWTJwvnysHm7Tu5+Jkc0lOTefS8g6mRklz2RsBh7RrzxpWH03G/Oox4biq3vDabnfkFZW/oXAXzZOHcPjIzrvvvDBb/sJWHzjmY/evV3KPt96uXzrjh/bigf2v+/fG3DB39Kd9t3B6jaJ3bO54snNtHj72/iLdmfcfI4w6k3wGN9qqMtJQk/nzSQTw4tCezV23ixAc/5JOFa8s5Uuf2nicL5/bBh9+s4a6Jczmx2/5cOKDNPpd3UvdmvDqiP/VqpnLePz/jkewFFBT47bUu/jxZOLeXlq/fypVjp9K+aR3uPKMbksreKArtmtbhlREDOL7r/tz51jyGP5PDxm07y6Vs5/ZWTJOFpCGS5klaIOmGYpa3lDRZ0lRJMyQdH85PlfSUpJmS5kgaGcs4ndtT23fmc8mYHHblG48N60WttPJtJ5FRI4UHh/bkzyd1Jnveak5+6CNmrdxYrvtwbk/ELFlISgYeBo4DOgNDJRW9n/AmgrG5ewJnA4+E888EaphZV6AXcLGk1rGK1bk9YWbc9PLXfL1iE//4VQ/aNK4dk/1I4oL+bXj+4kPZsbOA0x/5hP9+uSwm+3KuLLE8sugDLDCzRWaWB4wDTimyjgF1w+l6wMqI+bUlpQA1gTxgUwxjdS5qz362lBdzlnPlke04unNmzPfXq1VDXr9yAL1aNeC6F2cwcvwMtoddnztXUWKZLLKAyJ9By8N5kW4GzpO0HJgAXBHOfxHYAqwClgJ3m9m6GMbqXFS+Wrqev7w2i4Edm3DV0R0qbL+NM2rwzIV9uXzQAYz9fBlnPPYJy9ZtrbD9O6dYdWQm6UxgsJn9Nnw+DOhjZldErPP7MIZ7JPUD/gV0AfoBlwHnAw2AD4HjzGxRkX0MB4YDZGZm9ho3bly5xJ6bm0tGRka5lFWVVbd62rjD+PMn20hNgj/3q0lGWvQXtMuzrqau3sXoGTtIEgzvVoPuTapOv1LV7TO1L8qrrgYNGpRjZr3LXNHMYvIg+MKfGPF8JDCyyDqzgBYRzxcBTQmudQyLmP9v4KzS9terVy8rL5MnTy63sqqy6lRPebvy7czHPrGON02wWSs27vH25V1Xi9fm2nH3fWCt/vC63TNxru3KLyjX8uOlOn2m9lV51RXwpUXxnR7L01BfAO0ltZGURnAB+9Ui6ywFjgKQ1AlIB9aE849UoDZwKDA3hrE6V6o73pzL59+u4/bTu9K5Wd2yN4ixVo1qM/6ywzizV3MemLSA8//zOeu25MU7LFeFxSxZmNkuYAQwEZhDcNfTLEm3SDo5XO0a4CJJ04GxwPlhpnsYyAC+Jkg6/zGzGbGK1bnSvDJtBf/66FvOP6w1p/VsHu9wdktPTeauM7vz91925bNv13HiAx8ydakPquRiI6YnO81sAsGF68h5oyKmZwP9i9kul+D2Wefiau53m7jhfzM5pHUD/nhCp3iHU6xfHdKSg5rV45IxOZz1+BRGndiZ8w5tVW6NBJ0Db8HtXIk2bgt6kq2TnsLD5xxManLi/rt0yarH61cM4PD2TfjTK7P43fPT2Jq3K95huSokcT/9zsVRQYHx++ensWL9Nh4972Ca1k2Pd0hlql8rjX/+ujfXDe7Iq9NXcurDH7NwjQ+q5MqHJwvnivHgpAW8N3c1o07qTK9WDeMdTtSSksTlg9rx9P/1ZW1uHqc89DFvzlwV77BcFeDJwrkiJs39nvvem8/pB2cx7NBW8Q5nrwxo35jXrxhA+8wMLn32K259wwdVcvvGk4VzEZb8sIWrx02j0351ue20rpX6InGz+jV5fng/zj+sNU98+C3nPPEp32/yQZXc3vFk4Vxoa94uLn4mB0k8PqwX6anRDY2ayNJSkrj55IO4/+wefL1iEyc88BGfLvoh3mG5SsiThXMEPRmMHD+Ted9v5oGhPWnRsFa8QypXp/TI4pUR/albM4Vz//kZj7+/sLB3BOei4snCOeA/Hy/mlWkrueaYDvyiQ5N4hxMTHTLr8OqIAQw+KJPb35zLxc/ksGm7D6rkouPJwlV7ny36gdsmzOGYzplcNrBdvMOJqYwaQZuRP53YmUlzV3Pygx8xZ5X3/u/K5snCVWvfbdzO5c9NpWXDWtxzVneSkirvBe1oSeLCAW0YO/xQtublc9ojHzP+q+XxDsslOE8WrtrK21XApc/msDVvF48P60Xd9NR4h1ShDmkdDKrUo0V9fv/CdP740kx27PJBlVzxPFm4auuvr89m6tIN3HVGd9pn1ol3OHHRtE46Yy7syyW/OIBnP1vKmY9NYfl6H1TJ/ZwnC1ctvZiznGc+XcLwI9pyQrf94x1OXKUkJ3HDcQfy+LBefLtmCyc++BHZ81bHOyyXYDxZuGrn6xUb+eNLM+nXthHXD+4Y73ASxuCD9uO1KwawX910LnjyC/7xznzyC/z2WhfwZOGqlfVb8rj4mRwa1U7joXN6kpLAPcnGQ+vGtXnpsv6c1jOL+9/7hgue/MIHVXKAJwtXjeQXGFeOm8qazTt49LxeNMqoEe+QElLNtGTuObM7t53WlU8X/sBJD37E9GUb4h2WizNPFq7auPedeXz4zVpuOeUgureoH+9wEpokzunbkhcv7QfAmY9NYcynS7zVdzUW02QhaYikeZIWSLqhmOUtJU2WNFXSDEnHRyzrJmmKpFmSZkpK/AEFXMJ66+vveHjyQob2acHZfVrGO5xKo1vz+rx+xQD6HdCIm17+mmtemM62PL+9tjqKWbKQlEwwlvZxQGdgqKTORVa7iWBs7p7A2cAj4bYpwBjgEjM7CBgIeL8Ebq8sWJ3Ltf+dTvfm9bj55IPiHU6l06B2Gv85/xB+d3QHXpq2gtMe+Zhv126Jd1iugsXyyKIPsMDMFplZHjAOOKXIOgbUDafrASvD6WOBGWY2HcDMfjAz/znj9ljujl1cMiaHGilJPHpeL2qkVP6eZOMhKUlcdXR7nrygD99v2s7JD37EW19/F++wXAWKZbLIApZFPF8ezot0M3CepOXABOCKcH4HwCRNlPSVpOtjGKerosyM6/47nUVrcnlwaE+a1a8Z75AqvV90aMLrVx5O26YZXDImh9smzGGXD6pULaTEsOziOtkpenVsKPCkmd0jqR/wjKQuYVwDgEOArcB7knLM7L2f7EAaDgwHyMzMJDs7u1wCz83NLbeyqrJEr6cJi/J4c/5OftUxjbzlX5Mdx+6PEr2u9tSITsZYpTD6g0W8P3Mxl3avQf30ff/tWdXqKZYqvK7MLCYPoB8wMeL5SGBkkXVmAS0ini8CmhJcv3gyYv6fgOtK21+vXr2svEyePLncyqrKErmePvpmjbW54XW7bEyOFRQUxDuchK6rfTH+q2XW8aYJ1vtv79inC9fuc3lVtZ5iobzqCvjSovhOj+VpqC+A9pLaSEoLE8CrRdZZChwFIKkTkA6sASYC3STVCi92/wKYHcNYXRWyYsM2rhg7lQOaZHDnGd0q9dCoie60ns15+fL+ZNRI4Zx/fsYTHyzy22urqJglCzPbBYwg+OKfQ3DX0yxJt0g6OVztGuAiSdOBscD5YbJbD9xLkHCmAV+Z2RuxitVVHdt35nPpmBx27irgsWG9qF0jlmdaHcCB+9Xl1RH9OaZTJrdOmMOlY75isw+qVOXE9D/JzCYQXLiOnDcqYno20L+EbccQ3D7rXFTMjFGvfM2M5RsZPawXBzTJiHdI1Uad9FQePe9g/vnht9zx1lxOfuhjHjuvFx33q569+VZF3oLbVRljP1/GC18uZ8Sgdhx70H7xDqfakcRFR7Tlud/2JXfHLk59+GNenroi3mG5cuLJwlUJU5eu5+ZXZ3FEhyb87pgO8Q6nWuvbthFvXDGArln1uPr5afzp5a99UKUqwJOFq/TWbN7BpWO+IrNeDR44uwfJ1WBo1ETXtG46z17Ul+FHtOWZT5dw1uOfsmLDtniH5faBJwtXqe3KL+CKsV+xfmsej57bi/q10uIdkgulJidx4/GdeOy8g1m4OpcTH/iQ9+eviXdYbi95snCV2t/fmsuni9Zx++ld6ZJVL97huGIM6bI/r47oT9M66Zz/n8+5/91vKPBBlSodTxau0npt+kqe+PBbft2vFacf3Dze4bhStG2SwUuXH8apPbL4x7vz+b+nvmC9D6pUqXiycJXSvO82c/2LM+jVqgE3nVC0M2OXiGqlpXDvWd3526ld+GTBD5z44EfMWO6DKlUWnixcpbNx204uGZNDRnoKj5x7MGkp/jGuLCRx3qGteOGSYFClMx6dwnOfLfVW35WA/5e5SqWgwLjmhWksW7eVR849mMy6PiZWZdSjRX1eu2IAfds25MaXZnLW41Pod/t7zFyxkf53TPL2GQnIk4WrVB6evIB356zmphM6cUjrhvEOx+2DhrXTePKCPgw+KJMvFq9n1cbtrN8R9O01cvxMTxgJxpOFqzQmz1vNve/O57SeWfzmsNbxDseVg+Qk8fWKTbufP7sgGJxq28587po4L15huWKUmSwkdZD0nqSvw+fdJN0U+9Cc+9HSH7Zy1dipHLhfXW47rav3JFuFrIxorGclzHfxF82RxRMEY1HsBDCzGQTdjTtXIbbl5XPxmBwAHj+vFzXTfGjUqiRyBMM+TQqKne/iL5pkUcvMPi8yb1csgnGuKDNj5PgZzP1uE/cP7UnLRrXiHZIrZ9cN7kjN1OAHwMGNg2MLCa451vv4SiTRdFG+VtIBhEeIks4AVsU0KudCT32ymJenreT3x3RgUMem8Q7HxcCpPbMAuGviPFKTNtOgVirrt+4kyU81JpRojiwuBx4HDpS0ArgauCSmUTkHfLF4HX97Yw5Hd2rKiEHt4h2Oi6FTe2bx8Q1H0jWrHjk3HUPXrHrc+dZctu/03moTRanJQlIS0NvMjgaaAAea2QAzW1Ih0blq6/tN27ns2a9o3qAm95zVgyTvSbbaSEoSNx7fiZUbt/Pvj7+NdzguVGqyMLMCgqFRMbMtZra5QqJy1VrergIue/Yrcrfv4vFhvalXMzXeIbkK1u+ARhzdKZNHJi9kbe6OeIfjiO401DuSrpXUQlLDwkc0hUsaImmepAWSbihmeUtJkyVNlTRD0vHFLM+VdG2Ur8dVAbe+MZucJeu584xuPixnNTby+APZtjOf+9/9Jt6hOKJLFv9HcN3iAyAnfHxZ1kaSkoGHgeOAzsBQSUV7fLsJeMHMehLcjvtIkeX/AN6MIkZXRYz/ajlPTVnCbwe04aTuzeIdjoujA5pkcG7fljz3+VIWrM6NdzjVXpnJwszaFPNoG0XZfYAFZrbIzPKAccApRYsH6obT9YCVhQsknQosAmZF80Jc5ff1io2MHD+TQ9s25IbjDox3OC4BXHVUe2qlJnPHm3PiHUq1p7J6e5SUClwKHBHOygYeN7OdZWx3BjDEzH4bPh8G9DWzERHr7A+8DTQAagNHm1mOpNrAu8AxwLVArpndXcw+hgPDATIzM3uNGzeuzBccjdzcXDIyMsqlrKqsPOspN8+4eco28gvgL4fVpG6NqnVB2z9T0Smunt5YlMd/5+/kD4ek06mRN8gsVF6fqUGDBuWYWe+y1oumncWjQCo/niIaFs77bRnbFfffXjQzDQWeNLN7JPUDnpHUBfgL8A8zyy2tWwczGw2MBujdu7cNHDiwjJCik52dTXmVVZWVVz3lFxgXPPkFm/K28/zFh9KzZYN9Dy7B+GcqOsXV06H98/nknvd5Y2UqF582wO+MC1X0ZyqaZHGImXWPeD5J0vQotlsOtIh43pyI00yhC4EhAGY2RVI60BjoC5wh6U6gPlAgabuZPRTFfl0lc9+78/lg/hpuO61rlUwUbt+kpyZz/ZCOXDVuGi9PW+GjIsZJNBe488MW3ABIagtE01LmC6C9pDaS0gguYL9aZJ2lwFFhuZ2AdGCNmR1uZq3NrDVwH3CbJ4qq6e1Z3/HgpAWc1bs5Q/u0KHsDVy2d1K0Z3ZrX466J87yhXpxEkyyuAyZLypb0PjAJuKasjcxsF0EbjYnAHIK7nmZJukXSyeFq1wAXhUcqY4HzzYfMqjYWrcnlmhem0615PW45pYv3JOtKlJQk/nh8J1Zt3M6/PvKGevFQ5mkoM3tPUnugI8F1iLlmFlUrGTObAEwoMm9UxPRsoH8ZZdwczb5c5bJlxy4ufiaH1JQkHj2vF+mpfuHSla5v20Yc2zn5+0g6AAAduklEQVSTRyYv4KzeLWhSp0a8Q6pWohnP4nKgppnNMLPpQC1Jl8U+NFdVmRnXvziDhWtyeXBoT7K8K2oXpRuOO5Aduwq479358Q6l2onmNNRFZrah8ImZrQcuil1Irqp74sNFvDFzFdcPOZD+7RrHOxxXibRtksF5h7Zi3BfL+OZ7732oIkWTLJIUcTI5bJmdFruQXFX2yYK13PHmXI7rsh8XHxFN207nfurK3Q315sY7lGolmmQxEXhB0lGSjiS4EP1WbMNyVdHKDdsYMXYqbZtkcNeZ3f2CttsrDWuncfmR7Xhv7mo+WbA23uFUG9Ekiz8A7xG04r48nL4+lkG5qmf7znwuHZND3q4CHjuvFxk1omni41zxzj+sNVn1a/K3N+ZQUOA3UFaEaPqGKjCzx4BzgL8BL5mZ3+js9shfXpvF9OUbufvM7rRr6t1euH1T2FBv9qpNjJ+6It7hVAslJgtJj0k6KJyuB0wDngamShpaQfG5KmDc50sZ+/kyLht4AEO67BfvcFwVcXL3ZnRvUZ+7J85jW57/fo210o4sDjezwh5fLwDmm1lXoBd+GspFadqyDYx6ZRaHt2/MNcd2jHc4rgqRgoZ6323azr8+WhTvcKq80pJFXsT0McDLAGb2XUwjclXG2twdXDomhyZ1avDA2T1J9g7gXDnr06Yhgw/K5NHshazevD3e4VRppSWLDZJOlNSToJX1WwCSUgBvReVKtSu/gCuem8q6LXk8PqwXDWr73dYuNm44rlPYUM9H1Iul0pLFxQR9O/0HuDriiOIo4I1YB+Yqt7smzmPKoh+49bSudMmqF+9wXBXWpnHtoKHe50u9oV4MlZgszGy+mQ0xsx5m9mTE/IlmVmZHgq76emPGKh7/YBHnHdqSM3p5d9Iu9q46qj21a6Rw2wQfUS9Womln4VzU5n+/metenE7PlvUZdeJB8Q7HVRMNaqdxxZHtmDxvDR994w31YsGThSs3m7bv5JJncqiVlsKj5/YiLcU/Xq7i/Lpfa5o3qMmtE+aQ7w31yp3/N7tyUVBgXPPCdJas28rD5/Rkv3rp8Q7JVTPpqcn8YciBzFm1ifFfLY93OFVONF2U15B0jqQbJY0qfFREcBXt5akr6H/HJGau2Ej/OybxsrcMjdoj2Qt4Z/b3/PH4TvRt2yje4bhq6sRu+9OjRX3uftsb6pW3aI4sXgFOAXYBWyIeVcrLU1cwcvxMVmzYBsCKDdsYOX6mJ4wovD9/Dfe8M59TejTjgv6t4x2Oq8YkcdMJnfh+0w6e+NAb6pWnaHpza25mQ/amcElDgPuBZOCfZnZHkeUtgaeA+uE6N5jZBEnHAHcQdIWeB1xnZpP2JoZo3TVxHtvCsX0nrQxy6Lad+Yx65Ws2bd9Jekoy6WnJpKckUTMtmfTUZNJTkqmZlkSNlOSIeUmkJFefs3vL1m3lyrFT6ZhZh9tP7+o9ybq46926Icd12Y/H3l/I2X1a0LSOnxItD9Eki08kdTWzmXtScDjuxcMErb+XA19IejUcSrXQTQRjcz8qqTPBEKytgbXASWa2UlIXgm7Ss/Zk/3tqZXhEATB7/Y9feJu272LUK7OK26REqcn6MbmkJoVJJfknCSc9NZmaqeHywmWpydRMDZelJVMjJVgerBdRRsQ2qcmq8C/ol6eu4K6J8/hl1mYueieblKQkHh/Wi1pp3pOsSwx/GHIg7875nn+8M5/bT+8W73CqhGj+uwcA50v6FthBMA63mVlZ70AfYIGZLQKQNI7gdFZksjCgbjhdD1hJUPjUiHVmAemSakQ79vfeaFa/5u5TUCMOyueemUHV7F8vndeuGMD2nfnho4BtxU6XNK+A7Tvzd8/buG0nq8PpbRHr5+0q2Ku4k5O0O/mkh8mnZkTy2T2vyHSN3cmq6PIwGUUsD9ZPokZKEq9MW8nI8TPZtjOfd0liZ74hjKlLN9CqUe1yez+c2xetG9dm2KGtefKTbzn/sDZ03K9OvEOq9GRW+i1mkloVN9/MlpSx3RnAEDP7bfh8GNDXzEZErLM/8DbQAKgNHG1mOcWUc4mZHV3MPoYDwwEyMzN7jRs3rtTXUpoN23ayYv02CszIrAnfb4MkiawGNalfM3Wvy41WgRk7CyAvH/Lyg+kd+cbOfMgrCObt/psfrlcQLt+9LJj30zJgZ+S24Xp7Q0BKkpEsSBHk7hL9muZzWKaRlpzk/5ClyM3NJSPDu2YvS3nWU26ecf0HW2lXP5nf9656p6LKq64GDRqUY2a9y1qvzCMLM1siqTtweDjrQzObHkUMxZ0bKZqZhgJPmtk9kvoBz0jqYmYFAGEX6X8Hji0httHAaIDevXvbwIEDowirZIWnV85usZlxy+pw3eCOnNozpme/4sLM2LGrgB2RR0G78tmW9+OR0I/zfjwy2rEznwcmLWBnWE7/zHw+/j6ZKauDN/vbOwbG8VUltuzsbPb181kdlHc9fVdzEX97Yw7JWQdxePsm5VZuIqjoz1SZyULSVcBFwPhw1hhJo83swTI2XQ60iHjenPA0U4QLgSEAZjZFUjrQGFgtqTnwEvBrM1tY5ispB6f2zOLUnllkZ2dzxbkDK2KXcSFp9ymneuzZUdP/vlqx+3TdoU2Nj78P5jer731LusQzrF8rnpqymFvfmMMbVzb2no/3QTS37VxIcPpolJmNAg4lSB5l+QJoL6mNpDTgbODVIussJeiYEEmdgHRgjaT6BJ0VjjSzj6N7Ka4iXDe4IzVTk38yr2ZqMtcN9rEqXOKpkRI01Jv73Wb+l+MN9fZFNMlCQORZ7nyKP8X0E2a2i6DX2onAHIK7nmZJukXSyeFq1wAXSZoOjAXOt+AiygigHfAnSdPCR9OoX5WLmVN7ZnH76V3JCo8ksurX5PbTu1bJ03Wuajih6/70bBk01Nuatyve4VRa0dwN9R/gM0kvhc9PBf4VTeFmNoHgdtjIeaMipmcTjJVRdLu/EYz37RJQdTld56qGwoZ6v3x0Ck988C1XHd0+3iFVSmUeWZjZvQTDqq4D1gMXmNl9sQ7MOefKS69WDTmh6/48/sFCVm/yEfX2RonJQlLd8G9DYDEwBngGWBLOc865SuP6IR3ZmV/Ave/Mj3colVJpRxbPhX9zgC8jHoXPnXOu0mjVqDa/7teaF75cxtzvNsU7nEqntJHyTgz/tjGzthGPNmbWtuJCdM658nHFke2ok57KbRPmxjuUSieaLsrfi2aec84luvq1ghH1Ppi/hvfnr4l3OJVKadcs0sNrE40lNZDUMHy0BppVVIDOOVeeft2vNa0a1eJ2H1Fvj5R2ZHExwfWJA8O/hY9XCHqTdc65SictJWl3Q70Xc5bFO5xKo7RrFvebWRvg2ohrFW3MrLuZPVSBMTrnXLk6rst+9GrVgHvens+WHd5QLxrRtLN4UFIXSWdJ+nXhoyKCc865WJDEH0/oxOrNOxj9gY+oF41oLnD/GXgwfAwC7gROLnUj55xLcAe3bMAJ3fZn9AeL+N4b6pUpmr6hziDo7O87M7sA6A7UiGlUzjlXAf4w+EDyC4x73p4X71ASXjTJYls4vsSusFX3asDbWTjnKr2WjWrxm8Na8d+c5cxZ5Q31ShNNsvgy7DL8CYK7ob4CPo9pVM45V0FGDGpPvZqp3DZhTrxDSWjRXOC+zMw2mNljwDHAb8LTUc45V+nVq5XKlUe258Nv1pI9b3W8w0lYpTXKO7joA2gIpITTzjlXJZx3aCtaNarFbRPmsCu/IN7hJKTSxrO4J/ybDvQGphMMetQN+AwYENvQnHOuYqSlJHHDkAO59Nmv+G/Ocob2aRnvkBJOaY3yBpnZIGAJcLCZ9TazXkBPYEFFBeiccxVhSJf96O0N9UoUzQXuA81sZuETM/sa6BFN4ZKGSJonaYGkG4pZ3lLSZElTJc2QdHzEspHhdvMkDY5mf845t7cKG+qtzd3B495Q72eiSRZzJP1T0kBJv5D0BMGY2qWSlEzQh9RxQGdgqKTORVa7iWBs7p7A2cAj4badw+cHAUOAR8LynHMuZnq2bMBJ3Zsx+oOFfLfRG+pFiiZZXADMAq4CrgZmh/PK0gdYYGaLzCwPGAecUmQdA+qG0/WAleH0KcA4M9thZt8SnPbqE8U+nXNun1w/uCMFBXhDvSJKu8ANgJltB/4RPvZEFhDZpeNyoG+RdW4G3pZ0BVAbODpi20+LbJtVdAeShgPDATIzM8nOzt7DEIuXm5tbbmVVZV5P0fO6ik6i1NORLZJ5MWc5XWqspVXdxDypUdF1VWKykPSCmZ0laSbBEcBPmFm3MspWMfOKljMUeNLM7pHUD3hGUpcot8XMRgOjAXr37m0DBw4sI6ToZGdnU15lVWVeT9HzuopOotRTz747+fSuyby9ujZjTuqLVNxXUnxVdF2VdmRxVfj3xL0seznQIuJ5c348zVToQoJrEpjZFEnpQOMot3XOuZioVzOVq45qz19em032/DUM6tg03iHFXWm3zq4K/y4p7hFF2V8A7SW1kZRGcMH61SLrLCXopBBJnQjadKwJ1ztbUg1JbYD2eBcjzrkKdG7fVrRpXJvb3vCGelB6C+7NkjYV89gsqcwet8xsFzACmEhw99QLZjZL0i2SCrs4vwa4SNJ0YCxwvgVmAS8QXEx/C7jczPL37aU651z0CkfU+2Z1Li98uTze4cRdiaehzKzOvhZuZhOACUXmjYqYng30L2HbW4Fb9zUG55zbW4MPyqRP64bc+858Tu7RjIwaZd4TVGVFc+ssAJKaho3oWkrytvDOuSpPEjcWNtR7f2G8w4mraEbKO1nSN8C3wPvAYuDNGMflnHMJoUeL+pzcvRlPfLiIVRu3xTucuInmyOKvwKHAfDNrQ3BB+uOYRuWccwnkusEdKTC4e+L8eIcSN9Eki51m9gOQJCnJzCYTZd9QzjlXFbRoWIsL+rdm/NTlfL1iY7zDiYtoksUGSRnAB8Czku4HvEtG51y1ctnAdtQPR9Qz+1kb4SovmmRxCrAN+B3BbawLgZNiGZRzziWaejVTufroDnyy8AcmV8MR9UprZ/GQpMPMbIuZ5ZvZLjN7ysweCE9LOedctXJO35a0bVyb2ybMrXYN9Uo7svgGuEfSYkl/l+TXKZxz1VpqchI3HHcgC1bnMu6LZWVvUIWU1t3H/WbWD/gFsA74j6Q5kkZJ6lBhETrnXAI5pnMmfdo05L5357N5+854h1NhyrxmEfYF9fdwgKJzgNOIYvAj55yriiRx0wmdWJubx+PvV58R9aJplJcq6SRJzxI0xpsP/DLmkTnnXILq1rw+p/YIGuqt3FA9GuqVdoH7GEn/JugufDhBH08HmNmvzOzligrQOecS0bWDO2LA3dVkRL3SjixuBKYAnczsJDN71sy2VFBczjmX0Jo3qMWFA9ow/qsV1aKhXmkXuAeZ2RNmtq4iA3LOucri0oEH0LB2Gn97Y3aVb6gXda+zzjnnfqpueipXH92eTxetY9Lcqt1Qz5OFc87tg6F9WtK2SW1umzCHnVW4oZ4nC+ec2wepyUmMPK4TC9dsqdIN9WKaLCQNkTRP0gJJNxSz/B+SpoWP+ZI2RCy7U9KssCHgA5IUy1idc25vHd2pKX3bNOS+d6puQ72YJQtJycDDwHFAZ2CopM6R65jZ78ysh5n1AB4ExofbHkYw3Go3oAtwCEFLcuecSzhBQ73O/LAlj0ezq+aIerE8sugDLDCzRWaWB4wj6MG2JEOBseG0AelAGlADSAW+j2Gszjm3T7o2r8dpPbP410ffsqIKNtSL5ejjWUDkCbzlQN/iVpTUCmgDTAIwsymSJgOrAAEPmdnPuhiRNJygwSCZmZlkZ2eXS+C5ubnlVlZV5vUUPa+r6FT2ehpQt4DXCwq49un3ubhbekz3VdF1FctkUdw1hpJuRD4beNHM8gEktQM6Ac3D5e9IOsLMPvhJYWajgdEAvXv3toEDB5ZH3GRnZ1NeZVVlXk/R87qKTlWop4WayyPZC7nx9J50bV4vZvup6LqK5Wmo5UCLiOfNgZUlrHs2P56CgqCzwk/NLNfMcgn6pDo0JlE651w5unTgATSqncatE6pWQ71YJosvgPaS2khKI0gIrxZdSVJHoAFB1yKFlgK/kJQiKZXg4rb3dOucS3h10lO5+pgOfLpoHe/OqToN9WKWLMxsFzACmEjwRf+Cmc2SdIukkyNWHQqMs5+m4BcJhm+dCUwHppvZa7GK1TnnytPQQ1pwQJPa3P5m1WmoF8trFpjZBILeaiPnjSry/OZitssHLo5lbM45FyspyUnceHwnLnzqS8Z+vpRf92sd75D2mbfgds65GDjywKb0a9uI+979hk1VoKGeJwvnnIsBSfzxhE6s31o1Gup5snDOuRjpkvVjQ73l67fGO5x94snCOedi6NpjOyLg7omVe0Q9TxbOORdDzerX5KLD2/LytJVMX7ah7A0SlCcL55yLsUsGHkDjjDRunTCn0jbU82ThnHMxllEjhauP7sDn367jndmVs09UTxbOOVcBzj6kBe2aZnDHm3MrZUM9TxbOOVcBgoZ6B7Jo7Rae+2xpvMPZY54snHOuggzq2JT+7Rpx37vz2bitcjXU82ThnHMVRBI3Ht+JDdt28kj2gniHs0c8WTjnXAU6qFk9Tu/ZnP98vJhl6ypPQz1PFs45V8GuHdyBJMFdlaihnicL55yrYPvXCxrqvTp9JdMqSUM9TxbOORcHF//iABpn1OC2NypHQz1PFs45FwcZNVL4/TEd+HzxOibOSvyGep4snHMuTs7q3Zz2TTO448055O1K7IZ6MU0WkoZImidpgaQbiln+D0nTwsd8SRsilrWU9LakOZJmS2ody1idc66ipSQnceMJnVj8w1ae/WxJvMMpVcyShaRk4GHgOKAzMFRS58h1zOx3ZtbDzHoADwLjIxY/DdxlZp2APkDVGfncOedCAzs0YUC7xtz/3jcJ3VAvlkcWfYAFZrbIzPKAccAppaw/FBgLECaVFDN7B8DMcs2s8tyQ7JxzUSpsqLdx204emZy4DfVSYlh2FrAs4vlyoG9xK0pqBbQBJoWzOgAbJI0P578L3GBm+UW2Gw4MB8jMzCQ7O7tcAs/NzS23sqoyr6foeV1FpzrX04BmKfzrw0W0YxVNapX9O76i6yqWyULFzCvp/rCzgRcjkkEKcDjQE1gKPA+cD/zrJ4WZjQZGA/Tu3dsGDhy4z0EDZGdnU15lVWVeT9HzuopOda6nA3tuZ9Dd2XywsQEPHt+zzPUruq5ieRpqOdAi4nlzYGUJ655NeAoqYtup4SmsXcDLwMExidI55xLAfvXSueiItrw2fSVTl66Pdzg/E8tk8QXQXlIbSWkECeHVoitJ6gg0AKYU2baBpCbh8yOB2TGM1Tnn4u7iI9rSOKMGtyZgQ72YJYvwiGAEMBGYA7xgZrMk3SLp5IhVhwLjLKJmwtNR1wLvSZpJcErriVjF6pxziaB2jRSuObYDXy5Zz8RZ38U7nJ+I5TULzGwCMKHIvFFFnt9cwrbvAN1iFpxzziWgs3q34MmPF3PHm3M58sBM0lISo+10YkThnHMOgOQkMfL4A1n8w1bGfJo4DfU8WTjnXIIZ2LEph7dvzAOTvmHj1sRoqOfJwjnnElBhQ72HJn8T71AATxbOOZeQOu1flzN7NeepT5aw9If4d2DhycI55xLUNcd2JDlJ/H3i3HiH4snCOecSVWbddIYf0ZY3ZqwiZ0l8G+p5snDOuQR28S/a0rRODW6bEN+Gep4snHMugdVKCxrq5SxZz5tfx6+hnicL55xLcGf0asGB+9Xhjjfnxm1EPU8WzjmX4JKTgjEvlq7bytNTFsclBk8WzjlXCRzRoQlHdGjCg5MWsGFrXoXv35OFc85VEn88vhObtu9kwN8nM3PFRvrfMYmXp66okH17snDOuUpizqpNJCFyd+xiww5YsWEbI8fPrJCE4cnCOecqibsmziM/vH32w++Dr+9tO/O5a+K8mO/bk4VzzlUSKzds2z2douLnx4onC+ecqySa1a+5e/q4FgXFzo8VTxbOOVdJXDe4IzVTk38yr2ZqMtcN7hjzfcc0WUgaImmepAWSbihm+T8kTQsf8yVtKLK8rqQVkh6KZZzOOVcZnNozi9tP70pWeCSRVb8mt5/elVN7ZsV83zEbVlVSMvAwcAywHPhC0qtmNrtwHTP7XcT6VwA9ixTzV+D9WMXonHOVzak9szi1ZxbZ2dlcce7ACttvLI8s+gALzGyRmeUB44BTSll/KDC28ImkXkAm8HYMY3TOOReFmB1ZAFnAsojny4G+xa0oqRXQBpgUPk8C7gGGAUeVtANJw4HhAJmZmWRnZ5dH3OTm5pZbWVWZ11P0vK6i4/UUvYquq1gmCxUzr6T+dc8GXjSz/PD5ZcAEM1smFVdMWJjZaGA0QO/evW3gwIF7H22E7OxsyqusqszrKXpeV9HxeopeRddVLJPFcqBFxPPmwMoS1j0buDzieT/gcEmXARlAmqRcM/vZRXLnnHOxF8tk8QXQXlIbYAVBQjin6EqSOgINgCmF88zs3Ijl5wO9PVE451z8xOwCt5ntAkYAE4E5wAtmNkvSLZJOjlh1KDDO4jkElHPOuVLF8sgCM5sATCgyb1SR5zeXUcaTwJPlHJpzzrk94C24nXPOlUlV5eyPpDXAknIqrjGwtpzKqsq8nqLndRUdr6folVddtTKzJmWtVGWSRXmS9KWZ9Y53HInO6yl6XlfR8XqKXkXXlZ+Gcs45VyZPFs4558rkyaJ4o+MdQCXh9RQ9r6voeD1Fr0Lryq9ZOOecK5MfWTjnnCuTJwvnnHNl8mQRQdK/Ja2W9HW8Y0lkklpImixpjqRZkq6Kd0yJSFK6pM8lTQ/r6S/xjinRSUqWNFXS6/GOJZFJWixpZjjK6JcVsk+/ZvEjSUcAucDTZtYl3vEkKkn7A/ub2VeS6gA5wKmRoyA6UNC/fm0zy5WUCnwEXGVmn8Y5tIQl6fdAb6CumZ0Y73gSlaTFBB2sVlgDRj+yiGBmHwDr4h1HojOzVWb2VTi9maCjyNgPAlzJWCA3fJoaPvzXWQkkNQdOAP4Z71jcz3mycPtEUmuCsdM/i28kiSk8rTINWA28Y2ZeTyW7D7geKIh3IJWAAW9LyglHDI05TxZur0nKAP4HXG1mm+IdTyIys3wz60Ew+FcfSX56sxiSTgRWm1lOvGOpJPqb2cHAccDl4Sn0mPJk4fZKeA7+f8CzZjY+3vEkOjPbAGQDQ+IcSqLqD5wcnosfBxwpaUx8Q0pcZrYy/LsaeAnoE+t9erJweyy8cPsvYI6Z3RvveBKVpCaS6ofTNYGjgbnxjSoxmdlIM2tuZq0JRtWcZGbnxTmshCSpdnhjCZJqA8cCMb+D05NFBEljCYZ37ShpuaQL4x1TguoPDCP49TctfBwf76AS0P7AZEkzCIYZfsfM/JZQt68ygY8kTQc+B94ws7divVO/ddY551yZ/MjCOedcmTxZOOecK5MnC+ecc2XyZOGcc65Mniycc86VyZOFc8WQ9Mewp9gZ4a3BfctY/2ZJ14bTt0g6Opy+WlKtiojZuVhKiXcAziUaSf2AE4GDzWyHpMZAWrTbm9moiKdXA2OAreUbpXMVy5OFcz+3P7DWzHYARHYDHXZH8TwwKJx1jpktiNxY0pPA60Cz8DFZ0lozG1RkvcXAU8BJBD3SnmlmcyU1BP4NtCVIMsPNbEY5v0bn9oifhnLu594GWkiaL+kRSb8osnyTmfUBHiLoKbVYZvYAsBIYVDRRRFgbdgj3KHBtOO8vwFQz6wbcCDy9D6/FuXLhycK5IsIxKHoBw4E1wPOSzo9YZWzE3377uLvCThhzgNbh9ADgmTCWSUAjSfX2cT/O7RM/DeVcMcwsn6CX2GxJM4HfAE8WLo5cdR93tSP8m8+P/48qLqR93I9z+8SPLJwrQlJHSe0jZvUAlkQ8/1XE3yllFLcZqLOHIXwAnBvGMpDgVJWPF+Liyo8snPu5DODBsHvxXcACglNShWpI+ozgx9bQMsoaDbwpaVUp1y2Kuhn4T9hb7VaCoxok9QYuMbPfRv1KnCsn3uusc3sgvIOpd+QdUs5VB34ayjnnXJn8yMI551yZ/MjCOedcmTxZOOecK5MnC+ecc2XyZOGcc65Mniycc86V6f8BbdBz4mzrAJoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(val_score)\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.figure()\n",
    "\n",
    "plt.plot(range(1, len(val_score) + 1), val_score)\n",
    "plt.scatter(range(1, len(val_score) + 1), val_score)\n",
    "plt.title('5-fold cross validation Logistic Regression')\n",
    "plt.xlabel('Split no.')\n",
    "plt.ylabel('Validation Score')\n",
    "plt.xticks(range(1, len(val_score) + 1))\n",
    "plt.grid(True)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[21232  2684]\n",
      " [ 2575 21508]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pred = logreg.predict(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy:  0.888666660714\n"
     ]
    }
   ],
   "source": [
    "print('Training Accuracy: ', np.mean(y_train == y_train_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LASSO REGULARIZATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.01, 0.1, 1, 10, 100]\n",
      "0.01\n",
      "lambda=0.01 c=100.0Test Accuracy:0.890560220005\n",
      "Train Accuracy:  0.888568443798\n",
      "F1 Score:0.89055622495\n",
      "0.1\n",
      "lambda=0.1 c=10.0Test Accuracy:0.890560220005\n",
      "Train Accuracy:  0.888568443798\n",
      "F1 Score:0.89055622495\n",
      "1\n",
      "lambda=1 c=1.0Test Accuracy:0.890581053772\n",
      "Train Accuracy:  0.888586301419\n",
      "F1 Score:0.890577086979\n",
      "10\n",
      "lambda=10 c=0.1Test Accuracy:0.890581053772\n",
      "Train Accuracy:  0.888782735252\n",
      "F1 Score:0.890577249986\n",
      "100\n",
      "lambda=100 c=0.01Test Accuracy:0.890768557678\n",
      "Train Accuracy:  0.888443440449\n",
      "F1 Score:0.890765101274\n"
     ]
    }
   ],
   "source": [
    "#lambdas=list(range(1,50,1))\n",
    "lambdas=[0.01,0.1,1,10,100]\n",
    "print(lambdas)\n",
    "for l in lambdas:\n",
    "    print(l)\n",
    "    lassoLogReg=LogisticRegression(penalty=\"l1\",C=1/l)\n",
    "    lassoLogReg.fit(X_train,y_train)\n",
    "    y_pred=lassoLogReg.predict(X_test)\n",
    "    y_train_pred=lassoLogReg.predict(X_train)\n",
    "    #print(y_pred.shape)\n",
    "    #Accuracy\n",
    "    print(\"lambda=\"+str(l)+\" c=\"+str(1/l)+\"Test Accuracy:\"+str(np.mean(y_pred==y_test)))\n",
    "    print('Train Accuracy: ', np.mean(y_train_pred == y_train))\n",
    "    print(\"F1 Score:\"+str(f1_score(y_test, y_pred, average='macro')) )\n",
    "#for 23 features: highest accuracy obt was for lambda=650 c=0.0015384615384615385 Accuracy:0.8463926331798579\n",
    "#for 24 features: highest accuracy obt was for lambda=500 c=0.002 Accuracy:0.8551636492426925\n",
    "#if we increase lambda very much then too accuracy will reduce because of underfitting\n",
    "\n",
    "#After scaling:\n",
    "#29 features: \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(classification_report(y_test,y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RIGDE REGULARIZATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.001, 0.01, 0.1, 1, 10, 100]\n",
      "lambda=0.001 c=1000.0 Accuracy:0.89051855247\n",
      "Train Accuracy:  0.888577372608\n",
      "F1 Score:0.890514555895\n",
      "lambda=0.01 c=100.0 Accuracy:0.89051855247\n",
      "Train Accuracy:  0.888568443798\n",
      "F1 Score:0.890514555895\n",
      "lambda=0.1 c=10.0 Accuracy:0.890539386237\n",
      "Train Accuracy:  0.888559514987\n",
      "F1 Score:0.890535527026\n",
      "lambda=1 c=1.0 Accuracy:0.8904352174\n",
      "Train Accuracy:  0.888666660714\n",
      "F1 Score:0.890431594644\n",
      "lambda=10 c=0.1 Accuracy:0.889476864101\n",
      "Train Accuracy:  0.887961284677\n",
      "F1 Score:0.889474457\n",
      "lambda=100 c=0.01 Accuracy:0.883685076772\n",
      "Train Accuracy:  0.882791503344\n",
      "F1 Score:0.883684882704\n"
     ]
    }
   ],
   "source": [
    "#lambdas=list(range(1,50,2))\n",
    "lambdas=[0.001, 0.01, 0.1, 1, 10, 100]\n",
    "print(lambdas)\n",
    "for l in lambdas:\n",
    "    ridgeLogReg=LogisticRegression(penalty=\"l2\",C=1/l)\n",
    "    ridgeLogReg.fit(X_train,y_train)\n",
    "    y_pred=ridgeLogReg.predict(X_test)\n",
    "    y_train_pred=ridgeLogReg.predict(X_train)\n",
    "    #print(y_pred.shape)\n",
    "    #Accuracy\n",
    "    print(\"lambda=\"+str(l)+\" c=\"+str(1/l)+\" Accuracy:\"+str(np.mean(y_pred==y_test)))\n",
    "    print('Train Accuracy: ', np.mean(y_train_pred == y_train))\n",
    "    print(\"F1 Score:\"+str(f1_score(y_test, y_pred, average='macro')) )\n",
    "#for 24 features: highest accuracy obt was for:lambda=0.01 c=100.0 Accuracy:0.8533927790162295\n",
    "\n",
    "#After PCA: \n",
    "# 29 features to 26 features: lambda=0.001 c=1000.0 Accuracy:0.8504343840496676"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GRID SEARCH CV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.model_selection import GridSearchCV\n",
    "# param_grid={'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000],'penalty':['l1','l2'] }\n",
    "# gridSearch=GridSearchCV(LogisticRegression(), param_grid,cv=5) #5 fold cross validation\n",
    "# gridSearch.fit(X_train,y_train) \n",
    "# print(gridSearch.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #print(gridSearch.best_score_)\n",
    "# y_pred=gridSearch.predict(X_test)\n",
    "# #Accuracy on never seen before test set\n",
    "# print(np.mean(y_pred==y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Best params obtained: {'C': 0.01, 'penalty': 'l1'}\n",
    "#Accuracy obt: 0.845038438300798\n",
    "#Accuracy obt for 24 features: 0.8547469738952895"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### STOCHASTIC GRADIENT DESCENT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SGDClassifier(alpha=0.0001, average=False, class_weight=None, epsilon=0.1,\n",
       "       eta0=0.0, fit_intercept=True, l1_ratio=0.15,\n",
       "       learning_rate='optimal', loss='log', max_iter=5000, n_iter=None,\n",
       "       n_jobs=1, penalty='l2', power_t=0.5, random_state=None,\n",
       "       shuffle=True, tol=None, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.SGDClassifier.html#sklearn.linear_model.SGDClassifier\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "sgd=SGDClassifier(loss=\"log\",max_iter=5000) #logistic regression\n",
    "sgd.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(47999,)\n"
     ]
    }
   ],
   "source": [
    "y_pred=sgd.predict(X_test)\n",
    "print(y_pred.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.88939352903185487"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Accuracy\n",
    "np.mean(y_pred==y_test)\n",
    "#for 23 features: with max_iter=5000 Accuracy is:0.8080793349861456\n",
    "#for 24 features: with max_iter=5000 Accuracy is:0.8219962915894081\n",
    "#After scaling:\n",
    "#24 features: with max_iter=5000 Accuracy is: 0.8508718931644409\n",
    "#29 features: with max_iter=5000 Accuracy is: 0.8507260567928498\n",
    "\n",
    "#PCA: \n",
    "#29 features to 26 features: 0.8446634304881352\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.89      0.89      0.89     23916\n",
      "          1       0.89      0.89      0.89     24083\n",
      "\n",
      "avg / total       0.89      0.89      0.89     47999\n",
      "\n",
      "Precision: 0.889396690471\n",
      "Recall: 0.88938805838\n",
      "F1 Score:0.889391120116\n",
      "(0.88939669047090486, 0.88938805838011137, 0.88939112011587795, None)\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,y_pred) )\n",
    "print(\"Precision: \"+str(precision_score(y_test, y_pred, average='macro')) )\n",
    "print(\"Recall: \"+str(recall_score(y_test, y_pred, average='macro')) )\n",
    "print(\"F1 Score:\"+str(f1_score(y_test, y_pred, average='macro')) )\n",
    "print(precision_recall_fscore_support(y_test, y_pred, average='macro') )\n",
    "# Precision, recall, fscore, support in order\n",
    "\n",
    "#  precision    recall  f1-score   support\n",
    "\n",
    "#           0       0.88      0.81      0.84     24085\n",
    "#           1       0.82      0.89      0.86     23914\n",
    "\n",
    "# avg / total       0.85      0.85      0.85     47999\n",
    "\n",
    "# Precision: 0.8533320912052453\n",
    "# Recall: 0.8508790651450169\n",
    "# F1 Score:0.850487823093065\n",
    "# (0.8533320912052453, 0.8508790651450169, 0.850487823093065, None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy:  0.887818423708\n"
     ]
    }
   ],
   "source": [
    "y_train_pred = sgd.predict(X_train)\n",
    "print('Train Accuracy: ', np.mean(y_train == y_train_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sgd.get_params() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sgd.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "whos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}