predicting_when_companies_die.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Starting with lots of data and a good understanding of that data from *lifespan_of_companies*, we'll make a function to put in an API that will return the probability of a company going out of business in the next 2 years.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1: get the data ready for analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import statsmodels.api as sm\n",
    "import matplotlib.pyplot as plt\n",
    "import elasticsearch\n",
    "import elasticsearch.helpers\n",
    "import collections\n",
    "import re\n",
    "import copy\n",
    "import datetime\n",
    "import math\n",
    "from pandas.io.json import json_normalize\n",
    "from nltk.stem import PorterStemmer\n",
    "from collections import Counter\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(43684, 23)\n"
     ]
    }
   ],
   "source": [
    "# Data goes from Elasticsearch into Pandas:\n",
    "es = elasticsearch.Elasticsearch()\n",
    "\n",
    "data_to_load = elasticsearch.helpers.scan(\n",
    "                    es,\n",
    "                    query={\"query\": {\"match_all\": {}}},\n",
    "                    index=\"ri_sos\",\n",
    "                    doc_type=\"corporate_data\")\n",
    "\n",
    "# Initialize a double ended queue\n",
    "output_all = collections.deque()\n",
    "# Extend deque with iterator\n",
    "output_all.extend(data_to_load)\n",
    "only_data = [item[\"_source\"] for item in output_all]\n",
    "\n",
    "df = json_normalize(only_data)\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add needed variables and, using what we learned in `lifespan_of_companies`, delete bad data\n",
    "### Our dependent variable is different for a survival analysis than it was in `lifespan_of_companies`. We need to include the logged time in business for both active & inactive companies, and keep track of whether companies are active or inactive."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert date columns to the date format\n",
    "df['InactiveDate'] = pd.to_datetime(df['InactiveDate']).dt.date\n",
    "df['OrganisationDate'] = pd.to_datetime(df['OrganisationDate']).dt.date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3011     000000417\n",
      "29616    000000417\n",
      "30541    000002273\n",
      "Name: IDNumber, dtype: object\n"
     ]
    }
   ],
   "source": [
    "# Find examples where 'time_in_business' is negative (entity disolved before being formed, which is impossible):\n",
    "df.loc[df['InactiveDate'] != np.nan, 'time_in_business'] = (df['InactiveDate'] - df['OrganisationDate']).dt.days\n",
    "weird_rows = df.loc[df.time_in_business < 0]\n",
    "print(weird_rows.IDNumber)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Resulting shape: (43677, 24)\n"
     ]
    }
   ],
   "source": [
    "# If manual inspection shows you should delete them, then delete negatives and 0s:\n",
    "df.drop(df[df.time_in_business < 1].index, inplace=True)\n",
    "print(\"Resulting shape:\", df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    32455.000000\n",
       "mean      3515.187829\n",
       "std       4116.401240\n",
       "min          1.000000\n",
       "25%       1030.000000\n",
       "50%       2120.000000\n",
       "75%       4303.000000\n",
       "max      61626.000000\n",
       "Name: time_in_business, dtype: float64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.time_in_business.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original shape: (43677, 24)\n",
      "Shape after drops: (43658, 24)\n"
     ]
    }
   ],
   "source": [
    "# Some rows are missing OrganisationDate, which is a part of our dependent variable. Drop those rows.\n",
    "print(\"Original shape:\", df.shape)\n",
    "\n",
    "df.drop(df[pd.isnull(df.OrganisationDate)].index, inplace=True)\n",
    "print(\"Shape after drops:\", df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    8.133000\n",
       "1    9.402695\n",
       "2    9.809616\n",
       "3    8.293049\n",
       "4    7.459339\n",
       "Name: log_censored_tib, dtype: float64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Our dependent variable for a survival analysis is the time in business, whether the entity\n",
    "# has gone out of business or not. Assuming the data was pulled on 3/1/19:\n",
    "df.loc[pd.isnull(df.InactiveDate), 'log_censored_tib'] = np.log((datetime.date(2019, 3, 1) - df['OrganisationDate']).dt.days)\n",
    "df.loc[pd.notnull(df.InactiveDate), 'log_censored_tib'] = np.log((df['InactiveDate'] - df['OrganisationDate']).dt.days)\n",
    "\n",
    "df.log_censored_tib.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XmcHFW5//HPl0DCprIkMJAEJiA7SsAIKIgRBAJ6BRUVfrKJXq4XUFGUxRVRvFwV3K6ioBhQFlkVAVlEA7IJAdkXiRBIQkgia9i35/fHOU0qne6Zrpme6Z7p7/v1qld3nzpV/XT18nSdOnVKEYGZmVm7WarVAZiZmdXiBGVmZm3JCcrMzNqSE5SZmbUlJygzM2tLTlBmZtaWnKDMBoCkyZJmD9JzTZX07cF4rqFM0v6Srml1HNY4JyhrC5L+n6Tpkp6RNFfSnyRt2+q4zKx1nKCs5SR9Afgh8B1gdWAt4GfAbq2Mq0LS0q2OwRbn96QzOEFZS0l6E3AMcHBEnB8Rz0bEyxHxx4j4Uq6zlKQjJf1L0mOSzpa0Sp7XLSkk7SfpYUn/lvSVwvq3zHtmT0uaJ+mEwrwPSLpL0pOSpknaqDBvpqQjJN0OPCtpaUlrSjpP0gJJD0r6bKH+crmp7QlJdwNv7+V1/0jSrBzXzZLeVZh3dH6Np0lamGOcVJi/uaRb8rzfAcv28DxHS/pt4XFley2dH0+T9C1J1+b1XS5pdKH+tpKuy9tolqT9c/koSd/P23yepJ9LWi7PmyxptqTDJM3Pe8SfKKxzV0l35+ebI+mLhXn/KWmGpMclXShpzcK8kHSwpPuB+3PZhpKuyPXvk/TRQv1V8zqelnQjsG5P74m1oYjw5KllEzAFeAVYuoc6hwI3AOOAUcAvgDPzvG4ggJOB5YDNgBeBjfL864F98v0Vga3z/fWBZ4EdgWWAw4EZwMg8fyZwKzA+r3cp4Gbg68BIYB3gAWDnXP844G/AKnmZO4HZPbymvYFVgaWBw4BHgWXzvKOBF4BdgRHA/wA35HkjgYeAz+e49wBeBr5d53mOBn5beFzZXkvnx9OAf+XtsVx+fFyetxawENgrP9eqwMQ874fAhfn1vgH4I/A/ed7k/J4ek5fbFXgOWDnPnwu8K99fGdgi398e+DewRX6ffwJcXYg9gCvycy4HrADMAj6Rt+MWeflNcv2zgLNzvU2BOcA1rf7Meyrx+9DqADx19gR8HHi0lzr3ADsUHq+Rf5SXLvzgjivMvxHYM9+/GvgmMLpqnV8Dzi48Xir/gE3Oj2cCBxTmbwU8XLWOo4Bf5/sPAFMK8w6khwRV4zU+AWyW7x8N/Lkwb2Pg+Xx/O+ARQIX519G/BPXVwvyDgEsLr++CGusUKbmvWyh7B/Bgvj8ZeJ7Cnw5gPov+HDwM/Bfwxqr1/gr4buHxivl97s6PA9i+MP9jwN+q1vEL4BukxP4ysGFh3necoIbW5CY+a7XHgNG9HFNYG7ggNzM9SUpYr5KOV1U8Wrj/HOnHDeCTpL2DeyXdJOn9uXxN0p4IABHxGunf+NjCemZVxbBmJYYcx5cLMaxZVf8hepCbv+6R9FRe15uA0YUq1a9n2byN1gTmRP7FbeS5GlBv240n7V1VGwMsD9xc2BaX5vKKxyLilTrr/TBpr+ohSVdJekcur35PniF9Pnp6T7aqek8+DnTlWJamxHti7ccHGq3Vric1Z+0OnFunzizS3sy11TMkdfe08oi4H9hL0lLAh4BzJa1K2gt5S2E9Iv0gzykuXhXDgxGxXp2nmpuXvys/XqteTPl40xHADsBdEfGapCdIeya9mQuMlaRCklqL2okE0p7O8oXHXQ08R8UsYMsa5f8m7SFtEhFzaszvUUTcBOwmaRngEFIz3HjSe7J2pZ6kFUjNij29J1dFxI7VzyFpBKmZcTxwby6u+55Ye/IelLVURDxFOq7zU0m7S1pe0jKSdpH03Vzt58CxktYGkDRGUkM9/CTtLWlM3kN6Mhe/SvpRfJ+kHfIP5WGkY1fX1VnVjcDTuePEcpJGSNpUUqUzxNnAUZJWljQO+EwPYb2B9OO5AFha0teBNzbyekgJ/RXgs0odNz5E7SRScSuwnaS1lDqkHNXg8wCcDrxX0kfzc60qaWLelicDP5C0GoCksZJ27m2FkkZK+rikN0XEy8DTpPcD4AzgE5ImShpFapL7e0TMrLO6i4D1Je2TPzPLSHq7pI0i4lXgfODo/JnaGNivxGu3NuAEZS0XEScAXwC+SvrRnkX6Z/37XOVHpAPyl0taSOowsVWDq58C3CXpmbyePSPihYi4j9RR4SekPYL/AP4jIl6qE+Oruc5E4MG8zC9JTXOQjnM9lOddDvymh5guA/4E/DMv8wKLN0XVleP7ELA/6bjVx0g/xPXqXwH8Drid1MnjokaeJy/7MKkp7jDgcVKy2yzPPoLUqeQGSU8DfwY2aHDV+wAz83KfJr0PRMSVpGOD55H2FNcF9uwhvoXATrnOI6Smyv8ldbCA9BlaMZdPBX7dYHzWJrR4U7aZmVl78B6UmZm1JScoMzNrS05QZmbWlpygzMysLXX0eVCjR4+O7u7uVodhZtZRbr755n9HxJje6nV0guru7mb69OmtDsPMrKNIamhUDzfxmZlZW3KCMjOztuQEZWZmbckJyszM2pITlJmZtSUnKDMza0tOUGZm1pacoMzMrC05QZmZWVtygjIzs7bkBGVmZm3JCcrMzNqSE5SZAdDV1Y2k16euru5Wh2QdrqNHMzezRebNewiIwmO1LhgzvAdlZmZtygnKzMzakhOUmZm1JScoMzNrS05QZmbWlprSi0/SesBbgYciYnoz1mlmZp2t4T0oSR+SdImkrarKvwrcA5wN/F3Sb5sco5mZdaAyTXx7A9sBd1QKJG0KHAO8BlwLPAnsJelDzQzSzMw6T5kEtTlwW0Q8Vyjbm3Rm36ciYjvg7cDLwH82L0QzM+tEZRLUqsCcqrJ3A88AZwBExAPANcBGTYnOzMw6VpkENQp4fewTSSOBicD1EfFKod6jwOrNCc/MzDpVmQQ1F9i48Hg7UtK6tqreisDTja5U0nhJf5V0j6S7JH0ulx8taY6kW/O0a2GZoyTNkHSfpJ0L5VNy2QxJR5Z4bWZm1mbKJKirgA0lHS7prcC3SMefLq2qtykwu8R6XwEOi4iNgK2BgyVVEuEPImJini4ByPP2BDYBpgA/kzRC0gjgp8AupES6V2E9ZlbaKI9sbi1VJkEdSzre9D/AP4CtgCsj4qZKBUnrA+sAf290pRExNyJuyfcXkrqsj+1hkd2AsyLixYh4EJgBbJmnGRHxQES8BJyV65pZn7xI+g8aeaRzs8HVcIKKiH8C2wCnAn8CjmbJBLADcBtwUV+CkdRN6i1YSXCHSLpd0imSVs5lY4FZhcVm57J65dXPcaCk6ZKmL1iwoC9hmpnZICg11FFE3BkRB0TE+yPimIh4vmr+iRGxeaU5rgxJKwLnAYdGxNPAicC6pI4Yc4HjK1VrhdZDefVrOCkiJkXEpDFjxpQN08zMBklbXLBQ0jKk5HR6RJwPEBHzCvNPZtFe2WxgfGHxccAj+X69cjMzG2JKDxYr6c2Svifpmtxj7ruFeVvnJrSVSqxPwK+AeyLihEL5GoVqHwTuzPcvBPaUNErSBGA94EbgJmA9SRNyF/g9c10zMxuCSu1BSfokqafcyFwUwOhClTGkprmXgV83uNptgH2AOyTdmsu+TOqFNzE/x0zgvwAi4i5JZwN3k3oAHhwRr+b4DgEuA0YAp0TEXWVen5mZtQ9FLHGYpnZFaRtSV/NnSF3MryZ1ZpgaEQfkOssA84FpEfHBAYm4iSZNmhTTp3vwdTOA1JhR/D0oPhaN/laY9UbSzRExqbd6ZfagDid9WneJiOvzkyxWISJelnQfHurIzMz6qcwxqHcAN1aSUw9mAWv0UsfMzKxHZRLUm2hshIiRtEnvQDMzG7rKJKj5wIQG6m3AkqOem5mZlVImQV0LbCGp7oEtSTsC6wPT+hmXmZl1uDIJ6gekbj3nS9pJ0mLLStoOOIXU9fsnzQvRzMw6UZmx+P5O6sk3jjQW32OkXn27S5oH/JU09t3hEXFH3RWZmZk1oOxYfMcDuwLTgTeS9qhWIp2geyewe0T8sNlBmplZ5ynd2y4iLgUulbQqqdPECGBWRHjcO7Nha9Ri5z2uvvraPProzNaFYx2hz93BI+IxUjOfmQ17lWtDJfPm1bp4gFlzlR4s1szMbDDU3YOStG++e0FELCw8bkhEnNavyMzMrKP11MQ3lbRPfwOwsPC4UU5QZmbWZz0lqNNICempqsdmZmYDrm6Cioj9e3psZmY2kNxJwszM2pITlJmZtaXS50FJWhaYBKwJLFuvnnvxmZlZf5RKUJI+D3ydNMxRb5ygzMyszxpOUJIOAI7PD+8B7gWeHoigzGx46urqZt68h15/7CGTrCdl9qA+S+pmvk9EnDFA8ZjZMJaSk4dMssaU6SSxPnCdk5OZmQ2GMgnqOeDhgQrEzMysqEyCug7YdKACMTMzKyqToL4JbChpv4EKxszMrKJMJ4kVgBOAUyTtClxMavJ7rVbliLi6/+GZmVmnKpOgppG63wjYI0/1RMl1m5mZLaZMErmaARjNXNJ40km9XaS9sZMi4keSVgF+B3QDM4GPRsQTSted/hGwK6njxv4RcUte137AV/Oqvx0RpzY7XjMzGxwNJ6iImDxAMbwCHBYRt0h6A3CzpCuA/YErI+I4SUcCRwJHALsA6+VpK+BEYKuc0L5BGoYp8noujIgnBihusw42ivRf0Sfb2sBp+WCxETG3sgcUEQtJo1SMBXYDKntApwK75/u7AadFcgOwkqQ1gJ2BKyLi8ZyUrgCmDOJLMesgL5L+B8ZiI0OYNVPLE1SRpG5gc+DvwOoRMRdSEgNWy9XGArMKi83OZfXKq5/jQEnTJU1fsGBBs1+CmZk1SZ86MkhaAXgzadDYmmOVlO3FJ2lF4Dzg0Ih4utJ8UKtqrafrobw6rpOAkwAmTZrkKwSbmbWpsqOZv5nUQWEnet77KtWLT9IypOR0ekScn4vnSVojIubmJrz5uXw2ML6w+DjgkVw+uap8WqMxmJlZe2m4iU/SONJoErsA84AFpL2WG4B/s2gP5nrgbyXWK+BXwD0RcUJh1oVA5aTg/YA/FMr3VbI18FRuArwM2EnSypJWJiXRyxqNw8zM2kuZY1BHAqOBb0XEOOBPQETENhGxOqmTwoPAS6Tk0KhtgH2A7SXdmqddgeOAHSXdD+yYHwNcAjwAzABOBg4iBfI48C3gpjwdk8vMzGwIUkRjh2FyohgJTIiI1yT9Gtg3IkYU6qwL3EVKYscORMDNNGnSpJg+fXqrwzBrC6kxo/h7UHzc87wSvyNLrKfRZW34kHRzREzqrV6ZPahxwK0RURna6LX8RMtUKkTEv4CrgL1KrNfMzGwJZRLUC6STHyqeyberVdV7HJjQn6DMzMzKJKg5wFqFxzPy7TsqBbnDw+bAU/0Pzcwa0dXVjaTXp66u7laHZNYUZbqZ3wjsIWnZiHgBuDSX/0DSs6Ru3v9NGoLo4uaGaWb1+DLqNlyV2YO6GFgOeD9ARNxP6h4+FrgIuBX4NPAyiwZsNbNBN8p7UzYslBks9jxgmari/wbuI116YxXgXuB/IuL2pkVoZkvo6uruYQy8yjh53puyoa1f12yKiFeB4/NkZgOkdkIqdgE3G358UUGzIaD6OJOTknWCvg4WOxbYjnRuVJDGwrs6ImY3MTYzM+tgZQeLHQP8BPgwS3awCEnnA4dExPwlFjYzMyuhzIjjq5AGgV2PNIrEdaRLsUO6LPvWpM4Sm0l6h8fBM2sHi658C776rQ0tZfagjgbWB64EPp2HNXqdpAnAz4H3ki69/rkmxWhmfbaoRx+4V58NLWXOg9qddImN3auTE0BEPAh8iHTpjQ82JzwzM+tUZRLUasBVEfFsvQp53lXAmP4GZmZmna3sWHwjG6g3ktSrz8w6wiiPBWgDokyCOod0UcGuehXyvO1Jl283s45QOc6VpvojXJiVUyZBHUO6GOFfJe1SPVPSFFIHirtInSTMrIpHHjdrXJlefBcBrwIbABdJepLFu5mvlO9fD1xc7NpKujT8Dv2K1GwYKDPyeM/j7TXHYDyHWV+VSVCTC/cFrJynau+sUeZrOpuVtHgya1b38MXPi0o8pp+1pzIJ6j0DFoWZDZLFz4tyUrJ2VuZyG1cNZCBmZmZFZTpJmJmZDRonKDMza0tlRzNfGTiIdDxqTWDZOlUjItbtZ2xmZtbByoxm/mbSMEZd9H5k1b32zEpyl2+zxZXZgzoeWIN0yY0fAPcDzwxEUGadyFfNNVtc2fOgZgI7RsRLAxKN2TDjvSKzvivTSSKAG5udnCSdImm+pDsLZUdLmiPp1jztWph3lKQZku6TtHOhfEoumyHpyGbGaNZXi/aKKpOZNapMgrqVdPyp2aYCU2qU/yAiJubpEgBJGwN7ApvkZX4maYSkEcBPgV2AjYG9cl0zMxuiyiSo7wPbSqo1lFGfRcTVQKOXh98NOCsiXswXSJwBbJmnGRHxQN7DOyvXNTOzIarMSBIXSfo8aSDY/wMuA2YDr9Wp/3A/YztE0r7AdOCwiHgCGAvcUKgzO5cBzKoq36rWSiUdCBwIsNZaa/UzRLP+qjU2nplB+RN1/wHMA75M6nL+L+DBGtMD/YzrRGBdYCIwl9SDEGp3a4oeypcsjDgpIiZFxKQxY3zhX2u14rWUzKyozHlQk4FLWXRV3ccYoG7mETGv8Lwnky71AWnPaHyh6jgWXb23XrmZmQ1BZfagvkVKTt8FVomIMRExod7Un6AkrVF4+EGg0sPvQmBPSaMkTQDWA24EbgLWkzRB0khSR4oL+xODmfVf9QUazcoocx7URODmiGhqF25JZ5LOsRotaTbparyTJU0ktXvMBP4LICLuknQ2cDfwCnBwRLya13MI6bjYCOCUiLirmXGaWXk++dj6o0yCep40ekRTRcReNYp/1UP9Y4Fja5RfAlzSxNDMzKyFyjTx/Y10/pGZmdmAK5OgvgasK+lzAxWMmZlZRZkmvknAr4ETJO1B7+dBndb/8Mxs6PG5XdYcZRLUVBadc7QN0NuIEk5QZh2pcm4XuFOE9UeZBHUaPpvQzMwGSZmhjvYfwDjMzMwWU3aoIzMzs0FRpolvMUpHQVfNDx+PiJqdJczMzPqi9B6UpB0lXUYah29enhZKulTSjs0O0MzMOlOpBCXpm6QBY3cElmPRMMzLATsBl0o6uskxmplZB2o4QUmaQjpZ93nSgLEbkhLTcsAGwP8CzwFfK16K3azTFAdINbO+K7MH9RngVWDXiDgyIv4ZES/n6f6IOAp4H2mP6jMDEazZULBogFSflWHWH2US1JbAtfkS7TXleX+jztVszczMGlUmQb2BNLRRbx7Jdc3MzPqsTIKaD7y1gXqbAgv6Fo6ZmVlSJkFNAzbpaTRzSZ8B3gL8pZ9xmZlZhytzou5xwEdIo5l/GDgVeJB0JHgdYF9gW+AFUo8+MzOzPiszFt/dkj4G/IaUiLapqiJgIbBPRNzdvBDNzKwTlRrqKCIulLQ+cCCwHTCWlJhmA1cBJ0fEvKZHaWZmHaf0WHw5AX1rAGIxMzN7nUczNzOztlRmqKNNJH1d0uY91Nki19mwOeGZmVmnKrMHdRBpLL6eznGaD3wd+O/+BGVmZlYmQb0HuC0i6o4mkefdCuzQ38DMzKyzlUlQY4EHGqj3YK5rZmbWZ2US1IgG6wsY1bdwzKyzjHr90iRdXd2tDsbaTJlu5g8BW0laqt7l3SUtRRrJfFYzgjOz4e5FKpclmTfP18+yxZXZg7oMWBM4ooc6h5Oa9y5rdKWSTpE0X9KdhbJVJF0h6f58u3Iul6QfS5oh6XZJWxSW2S/Xv1/SfiVel5mZtaEyCeoE0lBG35Z0uqTtJa2Zp/dIOh04Ntc5vsR6pwJTqsqOBK6MiPWAK/NjgF2A9fJ0IHAipIQGfIO097Yl8I1KUjMzs6Gp4QSVe+h9FHgW2Au4gtSUNwv4cy57FvhYRDxUYr1XA49XFe9GGoyWfLt7ofy0SG4AVpK0BrAzcEVEPB4RT+TYqpOemZkNIaVGkoiIy4FNgB8D/yQ1IL+Q7/8Y2DQiLm1CXKtHxNz8nHOB1XL5WBY/vjU7l9UrX4KkAyVNlzR9wQJftsrMrF31ZSy+WcChAxBLI2odRY0eypcsjDgJOAlg0qRJNeuYmVnrtetYfPNy0x35dn4unw2ML9QbR7rEfL1yMzMboto1QV0IVHri7Qf8oVC+b+7NtzXwVG4CvAzYSdLKuXPETpToSWhmZu2ndBNfs0k6E5gMjJY0m9Qb7zjgbEmfBB4mXckX4BJgV2AG8BzwCYCIeFzSt4Cbcr1jIqK644WZmQ0hLU9QEbFXnVlLjOcXEQEcXGc9pwCnNDE0MzNroXZt4jMzsw7nBGVmZm2pboKS9FZJ4wYzGDMzs4qe9qD+AXyz8iCPmXfAwIdkZmbWc4ISi58Auz+w7YBGY2ZmlvWUoBYCawxWIGZDVVdX9+vXNJJ8yQizZumpm/mdwPaSjiGddwTwZkn7NrLiiDitv8GZDQXz5j3E4iNrOUmZNUNPCeq7wDnAVwpl2+SpEU5QZmbWZ3UTVET8QdKWpEtdrEU6BvUv4NrBCc3MzDpZjyNJRMStwK0AkvYHrokI9+QzM7MBV2aoo2+Sup6bmZkNuIYTVER8s/daZmZmzVF6sFhJywB7kEYgr1y1dg4wDTg3Il5uVnBmZta5SiUoSW8j9exbmyX70n4K+Lakj0TELU2Kz8zMOlTDCSqPy3cZsAowCzid1KtPwATg/+XbyyRNjIg5zQ/XzMw6RZk9qCNJyenHwJeqm/IkfQP4HvC5XPczzQrSzMw6T5nLbUwBHgA+X+s4U0S8AhyW6+zanPDMzKxTlUlQY4Eb81Vta4qI14AbgTX7G5iZmXW2MgnqeVITX29WznXNzMz6rEyCuh2YLGnDehUkbQC8J9c1MzPrszIJ6lfASOAvkg6QNLIyQ9Iykj4BXAksA5zc3DDNzKzTNJygIuI3wJlAFykBPSvpYUkPAc8BvyQdezozIk4fiGDNbDgbtdh1tbq6ulsdkLVYmT0oIuLjwCHATGAEMA4Yn+8/CBwSEXs3OUYz6wgvkq6rlaZ0nS3rZKWHOoqInwE/kzSW1LNPwGyfmGtmZs1UOkFV5ITkpGQdqaur2//wzQZYqSY+M0sWXea97mmBZtZPTlBmZtaW2jpBSZop6Q5Jt0qanstWkXSFpPvz7cq5XJJ+LGmGpNslbdHa6M3MrD/aOkFl74mIiRExKT8+ErgyItYjnXd1ZC7fBVgvTwcCJw56pGZm1jRDIUFV2w04Nd8/Fdi9UH5aJDcAK0laoxUBmplZ/7V7ggrgckk3Szowl60eEXMB8u1quXws6TpVFbNZdMXf10k6UNJ0SdMXLFgwgKGbmVl/9Lmb+SDZJiIekbQacIWke3uoW32FX6jRxSoiTgJOApg0aZK7YJmZtam23oOKiEfy7XzgAmBLYF6l6S7fzs/VZ5NGtagYBzwyeNGamVkzlUpQkh6QdOxABVP1XCtIekPlPrATcCdwIbBfrrYf8Id8/0Jg39ybb2vgqUpToJkNRaM8Ll+HK9vE1w2MKRZIOgW4JiJOaVZQ2erABZIgxXlGRFwq6SbgbEmfBB4GPpLrX0K6ku8M0uC1n2hyPGY2qCpj88G8ebVa8G24q5ugJH0NuI50Fd2FPaxj/3zb1AQVEQ8Am9UofwzYoUZ5AAc3MwYzM2udnpr4vglcDjwh6TZJJ+XyEQMfllnrdXV1u4nJrIV6auJ7Z2HaGvhULt9f0jtJJ8n+dWDDM2udRePtwbx5y5Kbm81skNTdg4qIGyLihIjYIyLGARPyrNnA8sBBwNmkb/BOkr4naVdJKw541GaDbvFrFZnZwCtzRd3KtQUui4i1gfVJSUrAGsBhwB+BxyVd2+xAzcyss9RNUJJ+KekASRvVmh8RMyLiF/nhacBbgc8DFwMbND1SMzPrKD0dgzqA3FVb0lPADbl8JUlLR8QrhboREXeSzlP68YBEamZmHaWnBLUOqXNEZdqe1Pj+YVIz3lWkjhJmZmZNVzdBRcRMYCZwFoCkkcALwN3Ak8DOwPtISWt3ScsB04C/RsQ/BzJoMzMb/sp0kngp370uIrYFViVd6kLAMqQRHU4E7pE0u9mBmplZZ+nzYLERsTAiLswPzyElrA8BPwUeb0JsZmbWwZp2uY2IeAr4fZ7MzJpo1GInSq+++to8+ujM1oVjg6JUgoqItr48h5kNV4sGjgUPHtspmrEH9R7g0Sasx8zM7HX9TlARcVUzAjEzMytyk52ZmbUlJygzM2tLTlBmZtaWnKDMzKwtOUGZmVlbcoIyy4qXePfVc9vdqNffp66u7lYHYwOkaSNJmA11xUu8J05S7WvRibs+aXf48h6UmQ1xoxbb8/Ue1fDhPSgzG+I8DNJw5T0oMzNrS05QZjbMuMlvuHATn5kNM27yGy68B2Udrdi13IYrd0kfqoZdgpI0RdJ9kmZIOrLV8Vh7qT7XaVHX8uhtURuyKntUwbx5j7r5bwgZVk18kkaQLjm/IzAbuEnShRFxd2sjs1bp6urOSajI5zp1Ljf/DSXDbQ9qS2BGRDwQES8BZwG7tTimtlW9N1Hm32Rx2REjVqi7nt6eo6f1FB/3dd7ie0jeS7Jqoxr6HHlPqzUUMXy+tJL2AKZExKfy432ArSLikEKdA4ED88NNgTsHPdDmGA38u9VB9JFjH3xDNW5w7K0ykLGvHRFjeqs0rJr4qN1es1gGjoiTgJMAJE2PiEmDEVizOfbWGKqxD9W4wbG3SjvEPtya+GYD4wuPxwGPtCgWMzPrh+GWoG4C1pM0QdJIYE/gwhbHZGZmfTCsmvgi4hVJhwCXASOAUyLirh4WOWlwIhsQjr01hmrsQzVucOyt0vLYh1UnCTMzGz6GWxOfmZkNE05QZmbWljo2QQ3VIZEkjZf0V0n3SLpL0udaHVMZkkZI+oeki1odSxmSVpJ0rqR787Z/R6tjapTB+5qGAAARpUlEQVSkz+fPyp2SzpS0bKtjqkfSKZLmS7qzULaKpCsk3Z9vV25ljPXUif17+TNzu6QLJK3UyhjrqRV7Yd4XJYWk0YMdV0cmqMKQSLsAGwN7Sdq4tVE17BXgsIjYCNgaOHgIxQ7wOeCeVgfRBz8CLo2IDYHNGCKvQdJY4LPApIjYlNR5aM/WRtWjqcCUqrIjgSsjYj3gyvy4HU1lydivADaNiLcC/wSOGuygGjSVJWNH0njS0HEPD3ZA0KEJiiE8JFJEzI2IW/L9haQfyrGtjaoxksYB7wN+2epYypD0RmA74FcAEfFSRDzZ2qhKWRpYTtLSwPK08bmBEXE18HhV8W7Aqfn+qcDugxpUg2rFHhGXR8Qr+eENpHMz206d7Q7wA+BwWjROWKcmqLHArMLj2QyRH/kiSd3A5sDfWxtJw35I+rC/1upASloHWAD8OjdP/lLSCq0OqhERMQf4Pukf8FzgqYi4vLVRlbZ6RMyF9AcNWK3F8fTVAcCfWh1EoyR9AJgTEbe1KoZOTVC9DonU7iStCJwHHBoRT7c6nt5Iej8wPyJubnUsfbA0sAVwYkRsDjxL+zYzLSYfr9kNmACsCawgae/WRtV5JH2F1Dx/eqtjaYSk5YGvAF9vZRydmqCG9JBIkpYhJafTI+L8VsfToG2AD0iaSWpS3V7Sb1sbUsNmA7MjorKnei4pYQ0F7wUejIgFEfEycD7wzhbHVNY8SWsA5Nv5LY6nFEn7Ae8HPh5D58TTdUl/am7L39lxwC2SugYziE5NUEN2SCRJIh0LuSciTmh1PI2KiKMiYlxEdJO2918iYkj8k4+IR4FZkjbIRTsAQ+UaYw8DW0taPn92dmCIdPAouBDYL9/fD/hDC2MpRdIU4AjgAxHxXKvjaVRE3BERq0VEd/7Ozga2yN+FQdORCSoftKwMiXQPcHYvQyK1k22AfUh7ILfmaddWB9UBPgOcLul2YCLwnRbH05C813cucAtwB+k73/IhbOqRdCZwPbCBpNmSPgkcB+wo6X5Sj7LjWhljPXVi/z/gDcAV+bv685YGWUed2FvOQx2ZmVlb6sg9KDMza39OUGZm1pacoMzMrC05QZmZWVtygjIzs7bkBNULSTPzSL6TWx2LDR35M9PULrIDsc5WKXyvugfhuaa163dYUneObWarY2lHTlBmZtaWlm51AGbWkXYAlgHmtDqQFpsDbAS83OpA2pETlJkNuoj4V6tjaAd5fMR7Wx1Hu3IT3wCQtIKkr0i6TdIzkp7Nw5x8OY8SXG+5D0u6Ltd/QtLlkt4laXJup57Wh1jGSzpB0t15vU/nK8L+TNKmNeqvKunbku4oxH5LvirrMjXqT82x7S/pzZLOkDRP0ov5SqJHSFricyZpWUlH5nU/k+vPlXR9fv4lrvoqaRNJp0malev/W9Ilknap89qLsb1F0jmSHpX0qqRDq+puJemsPMzLS5IWSLpQ0rY9bNu3KF0l9fHCdvpUvfqN6Ms6c+zfkzQ9b/uXJD2idAXgrWvUPyVvl7ojskv6bK5zdqFshKRP58/oU/l55uUYj5c0psTrrHkMqni8SNLb8nvwmKTn8/epX0PwSHqPpD/n79czkq5RuqxEwzHWirWqfCVJ31G6ivFzOfbZuf5RVXXrHoNS4ZijpI/l78YzkhZKurKXz+YKkg6XdJPSd/75HM/RSldCqK5f6r2VtIGkUyU9lOsuzNvrAkkfrhdXaRHhqYcJmEm6FMfkBuuPBm7PyzwOXEAaQfqJXHYrsEqN5b6c578GXAOcAdxGGqL/hDxvWsnYdwKeysvOyXFUxmV7FTi6qv5bcr0gXS/rIuAS4LFc9mdgZNUyU/O8H+bn+hdptPK/kpotAvhJ1TJLka6MGsCTwMX59V5JGpQygK6qZT4AvJDn3ZnrX51fRwDfqvH6K7GdnJetxHYxcGCh3mF5u78GTAfOIY1L9mqe/rPGut8NPJfXfy9wJjAt16+8X1Hy/erTOvP78kr+vPwxv8d35PqvAB+pqj8xz3sQWKpOLHfnOu+usT2fAy7P78HlebsGsHUfvlfdVeXTcvlxwEv5dZwJXFt5/aQrSpfZrpV1/ihvy9tz7MV1fqHRGGusd3KhbHngrlw+jzTQbeV9nA+8ULWO7lx3Zo31V2I7Jsd9FfA70vihAbwIvKPGcuMKMcwnjTn6B9IVGyJ/Tlau813p9b0l/U48ncvvIV1Z4VzSd+Y50pWnm/P726wVDdeJ8gnq7Fz/amClQvnKhS/EmVXLvC1/AF8CplTN+2zhgzqtRNxrFT5EXwWWrjH/bYXHywEP5PpHFesDq5AuXR0smdSmFuI7msIPHukqtJUf+fFV5QHcDKxQtT6RBsRdvlDWxaJE+4Wq+pNJ12cKYOceYvs2NX6MSZe5riTwrarmbZOf9yVg/aptVUmk3yGPaZnnvbsQT5R4v/q8zvwaVq9R/h859seK2zPP+1te3/trLLd9nndnoWztXPZwneeaCKzWh+9Vd1X5tMJ7dkDVvL1z+VPVr6eX5yqu84s1ttHLpET+1kZirLHeyYWyfXPZRSz5nRsBbF9V1k3vCeoxFv+uVgb8DeCKGt+f6/K8n7D492g54Dd53tS+vrfAKbn+UTXqrkiNpNnXqSkrGc4TJRJUfqNfI/0gb1hj/sbU/sGuvOG/rLPeGyifoH6Ylzmrwfr/nev/rs78NUk/dgtY/Mdzal7uxmJ5Yf4lef6+hbKP5LIfNhjb13L9a+vM/986X9ZKbPcAI+os+/dcZ5c687+Y5x9fKNsnl82otV7g+MqPS4n3q+nrzMudnpd7X1X5R3P5JTWWOS/PO6hQ9vZc9vsyz99DXJXvVXdV+bRcfm6d5Sp7dtuVeK7KOqfXmX9qnn9yIzHWWO/kQtmXctmhDcbWTe8J6pAa81bL814AlimU75LLr6f2H7IVSHt2L5P3osq+t6QWiAAmNuOz0NPkY1DN9S7SP5gbImKJA58RcTfph3wp0l5Exbvz7Rl11ntmH2KZkm9/2WD9yiU7zqk1MyIeAe4nNWGuV6PKJZE/vVUq22HNQlmlifGTkg6StHovsVW2z9Q680/Jt9tKGlFj/h8i4tXqQkmjSV/Op0nNGbVclW/fUSOes2qtl/Qvtax+rVPS6Hys7ftKl6SfKmkqUDnOuH7VIueT9hp3lrROYT1jSc2pC6ue895c9j6lY6lrN/rC+uiiOuW1Pk+NqneBzMrrnNyHdVa7Md8eIWlvSSs1YZ1LbIuImE86bDAKWLUwq/I9Pi8iXqux3LOkZuylSZ99KP/eVl7jzyXtKGlUw6+kJCeo5hqbbx/soU6l99LYQlnl/kN1lqlX3pPKh6zRHkKVH6lzKgdnqyfSHiBArYPhD9dZb+Vy9K93eojUg+vzwEjgp8Cjkv4l6TeS9qiRZHrbrg+S9lyXZfEva0W97TeB9IfijcArdV5z5ctYfM3jeolnZp3ynvR5nZL+i/Qaf006nvZJ0oX99gPemqu9sbhMpGuinUj6Dfh0YdaBpB+v0yJiYaH+QuAA4HngWGBmPvB/Tk6MS3Rq6aeGP08l9LZtx9WZ37CIuAr4LmkP5zfA40odlE6StHMfV1tmW1S+x9/r4XtcSWJjcsxl39vvkY4Xb0X6Y/eUpBsk/a+kt/TxNdbkbubNpXxba0+iuk4t9ZZb4p9QA3qKoZZKUrgY+HcvdR+rUVYqxoj4iaRzgN2BbfO0d55ulfTuiKh8ARvZrj15vk555TU/Bfy+l3X0tk2K+hpn6XVKmkRKNK+Qmpf+SDqW9VxEhKTvkI4p1vrcnURqPj1A0tfJHULyvJ8tEUDEuZL+DOxGagHYBtgjT0dLeldEzOr7S1xMXz7z/VX2fav5Bz8ijlC6MOFupM/1NqTt+p+SLic1t77ScFA19oR6UPlMX0Xvf5Re/+NW5r2NdGXg90raitRSsw2phWEr4HBJ34iIY0rEXJcTVHPNzrfr9FBnQr4tnqD4SC5fm9RRoVp3H2J5GNggT7N7qQup194GwIkRcXEfnq+0SJeP/nmekLQZ6V/nROBIUs9GSPFvQNquV9ZYVTfpx+IFUs/JRlV+TF+OiP1LLFd577rrzJ9Qp3wg1rkHKfn8OCK+X2P+m+s9YUQskPQ70oH9j5K23xqkY501L2kfEU+SjtmcCiBpXVIvyfeQjgX+v3rP1wa6eyl/pKr8pXy7RLfsrG5TWEQ8SDoO/EOA3CX8TFLP2gMYuKsaVz7T50TET8ssWPa9jXS15r/nuiPz/JNJCe13EXFfP14H4Ca+Zqv0jNpaUnWbP5I2Iv3LeI3Uy6+icn+vOuvdsw+xXJZvGz0v50/59iN9eK6miIjbSF2BATYrzKocB9q3zqKfyLfXlPxnOofUjXl09bksvajEs2edY14fL7Gu/q5zlXy7xJ5LPndlx16e9yf59qA8QWp2bUhurj02P9ysp7ptoN42rJRPqyqv/GnYsHoBpXMIxzf6xBFxDYuOoQ7kdmra97jMexsRL0XEVFKHLrGoablfnKCaKCIeIvWAWgr4haQ3Veblg6W/yPPOrmoK+Skpae0nabEfFEkHsfgB+kadADxD+sE7qvpHT+kE3rcVik4i/cjtJ+mbqnFCsaRNJX2iurwsSdtL2lXS0lXlI1jUPl48bnQy6SDutpI+W7XMdsBn8sPj+xDO1/LtbyXtVCPWkZI+IKn4HpwLzCXtnRwtSYX625J6RJbV13VWjjHuq8IJmJLeQOo80uNB+oiYTvpR2YrUUeMRajR3Stpc6WTR5Wqs5j/ybV+OlQ6mt0v6fLFA0q6kZuVXgf+rql/ZWz9c0hsLy4wnJZslmk0lfVDSdqo6OT1vt/fmhwO5nX5POn3j3ZJ+LmmV6gqS1pF0cOFxqfdWqWPTBrXWC2xSXb9fBrqb4FCfWNTV9G7SF7netEauP5pFJ0k+RkpY55GanoL6J+pWulK/RtoTOz3XfZVFXcYvLxn7LqQf9iA1k1VOqLuZ+ifqPlSI/a+kZom/kA4wB6mHYnGZqbl8/zoxHE3V+VPAobnsybzu00knNFdOJJwLrF21nt1YdKJu5UTLaTR2om7N2Ar1vkA6hhPAfaSTK88lNV88mcs/XbXM9qRjW5Vu7Gfk7dWfE3VLr5N0ft3Ded58Uu+8C/L7Nxf4VfX2r/G8e1XWDXyjTp3d8/xn8+fzjLyNKidyPg1M6sP3qruqfBo9nNbR6HtaZ52VE3Vvy/FfU3jdX6qxXHHbzs3b9i+kP35/YdF5jZMLy1S+q5UTZH9LOi5YOdn9HuBNhfrd9NLNvA/bcByLBgt4Or9fZ5LOZbwvlz/a1/eW9LsUef4fSN/fK0knDgdV53n2Z2rKSobzVPgQ9DZ1F5ZZkXRy7O2kM6ufy1+KL1N1YmrVc32EdP7Cs6Qfxj+Tur5WTlA8ow/xTyD9M7yf9AP/FCnZ/h+wcY36byIdVL8h132RlNyuJ53RXn0y41TKJ6h1c/lfSD8AL5A6IdwCfB0YU2ddm5KOUc1h0QmofwJ2rVO/x9iq6m5G6pI/g5QkFgL/zF/AT1H7T8Vmef4T+T2+lZzI6OXHpZc4Sq2T1GPsF6Tjly+S9oRPJh1PWmL711h+TK7zMvmPVo06XaTjgn8i/Vl5Ln9G7wC+T9UfihLfq+6q8mkMXIKaTNqL+Uv+bD9LSjK797DseFKSWZC37T+Bb5B6oC4RK+n46XGk5DcnLzOP9GfnUOANVevvpskJKs9bFjiY1HT8OOn7MpfUxfx7wDv7+t4C7ycdN/5HYbs8TOrR9xHqjE7Sl0n5Ca2NSfoV6cDqFyOiL81YZnVJ+hzpn//ZEfGxVsdjVuEE1SZyp4r5kXrSVMoE7E9qpnkJWCfSCbNmTZGPrdxFahbaOlLPLLO24G7m7WNf4EuS/kFqolmOdGLsBNJxqc84OVmzSPoSqcl0O1JyOsfJydqNE1T7uIR0bGYrUk+YUaT23XNJY9Zd28LYbPh5H6nX3gLS8arDWhuO2ZLcxGdmZm3J50GZmVlbcoIyM7O25ARlZmZtyQnKzMzakhOUmZm1pf8PlxI6/PRdjbUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Graph log_censored_time_in_business:\n",
    "plt.hist(df.log_censored_tib, bins = 75,\n",
    "         color = 'blue', edgecolor = 'black')\n",
    "plt.axis([0, 15, 0, df.shape[0] / 15])\n",
    "plt.xlabel('Log censored days in business', size = 22)\n",
    "plt.ylabel('# of companies', size= 22)\n",
    "plt.title('Censored and uncensored')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This combined graph looks much less like a normal distribution than our prior graph of only entities that had gone out of business. Make a graph of the purely uncensored (entities still existing) data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inactive dates: 0    2005-10-07\n",
      "1           NaT\n",
      "2           NaT\n",
      "3    1998-11-10\n",
      "4    1989-09-21\n",
      "Name: InactiveDate, dtype: object\n",
      "\n",
      "Censored conclusion: 0    1\n",
      "1    0\n",
      "2    0\n",
      "3    1\n",
      "4    1\n",
      "Name: is_tib_censored, dtype: int32\n"
     ]
    }
   ],
   "source": [
    "# 1 == the event (company became inactive), 0 == censored (not inactive as of 3/1/2019)\n",
    "df['is_tib_censored'] = np.where(pd.notnull(df['InactiveDate']), 1, 0)\n",
    "\n",
    "print(\"Inactive dates:\", df.InactiveDate.head())\n",
    "print(\"\")\n",
    "print(\"Censored conclusion:\", df.is_tib_censored.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XmYHFW5x/HvjxAWcWFJIJBEhh0VBXUkKFyIKBAQBRUUFAFRcUPFDcEVcLm4gIpXURAMKItsKgKyiAYE2SbIvkgkgQRCEolA2Al57x/ntKl0ume6Znqme6Z/n+epp7tPnap+u3p5u06dOqWIwMzMrN2s0OoAzMzManGCMjOztuQEZWZmbckJyszM2pITlJmZtSUnKDMza0tOUGYtJmmypDlD9Fz/I+meXua/XNITkkYNRTx9GcptY+3HCcqGHUnvk9STf0jnSvqTpO1aHVc7khSSNq48joi/RcRmhfmzJL21MP+BiHhxRLww1LGaVXOCsmFF0ueAHwHfAdYBXg78DNijlXFVSFqx1TGYjRROUDZsSHoZcDTwyYg4PyKejIjnI+KPEfHFXGcFSYdL+pekRySdLWnNPK8r71EcIOkBSf+W9JXC+rfOe2aPS5on6bjCvHdIukPSo5KmSXpFYd4sSV+SdCvwpKQVJa0n6TxJCyTNlPTpQv1VJU2V9B9JdwJv6ON1by7pckkLJd0j6T2FeVMl/VTSRZIWSbpe0kZ53lW52i15b/O9xSYzSb8mJfg/5vmHFbbRipVtLunkvKf6oKRvVZr/JG0s6UpJj+Vt+dteXkNf2+8Lkm7N6/qtpFVqrOOLks6rKvuJpB/1tv1sGIsIT56GxQRMARYDK/ZS51DgOmACsDLwC+DMPK8LCOAkYFVgS+BZ4BV5/rXAB/L9FwPb5PubAk8COwGjgcOAGcBKef4s4GZgYl7vCsB04OvASsCGwH3ALrn+McDfgDXzMrcDc+q8ntWA2cAHgRWB1wH/Bl6V508FFgJb5/mnA2cVlg9g48LjycXnyrG/tfC4so1WzI9/n7fhasDawA3AR/O8M4Gv5Ne7CrBdndfQyPa7AVgvb5O7gI9Vxwusm9ezen68IjAfeH2rP5ueBmfyHpQNJ2sB/46Ixb3U+SjwlYiYExHPAkcCe1U1vR0VEU9HxC3ALaREBfA8sLGkMRHxRERcl8vfC1wUEZdHxPPAD0iJ6E2FdR4fEbMj4mnSHtHYiDg6Ip6LiPtISXGfXPc9wLcjYmFEzAaO7+X17A7MiohfRcTiiLgJOA/Yq1Dn/Ii4IW+X04GtellfwyStA+wKHBppb3U+8MPC63geWB9YLyKeiYir66yq0e33UEQsBP5Y6zVExFzgKmDvXDSF9HmYPqAXam3LCcqGk0eAMX0c51kf+F1uSnqU9G/8BdLxqoqHC/efIu0tAXyI9G//bkk3Sto9l68H3F9ZICKWkPZqxhfWM7sqhvUqMeQ4vlyIYb2q+vdT3/rApKp1vR8Y18DrGaj1SXs8cwvP/QvSnhSkPSEBN+Tmu4PqrKeR7dfoazgV2C/f3w/4deMvx4YbH9C14eRa4BlgT+DcOnVmAwdFxDXVMyR19bbyiLgX2FfSCsC7gHMlrQU8BLy6sB6RmuYeLC5eFcPMiNikzlPNzcvfkR+/vJewZgNXRsROvcU+AL1dzmA2qQl0TK291oh4GPgIQO5F+WdJV0XEjKqqjWy/Rv0eOEHSFqS9y8P6sQ4bJrwHZcNGRDxGOq7zU0l7SnqRpNGSdpX0vVzt58C3Ja0PIGmspIZ6+EnaT9LY/A//0Vz8AnA28DZJb5E0Gvg86Yf773VWdQPweO44saqkUZK2kFTpDHE2cISkNSRNAD7VS1gXAptK+kB+raMlvaHYyaAP80jHwErPz01qlwHHSnqpUgeUjSTtACBp7xw/wH9Iya5W9/Sy26+uiHiG9OfkDOCGiHig7Dps+HCCsmElIo4DPgd8FVhA+pd/COmfNcCPgQuAyyQtInWYmNTg6qcAd0h6Iq9nn3xs5R5Sc9JPSB0U3g68PSKeqxPjC7nOVsDMvMwvgZflKkeRmrxmkhJA3WaqiFgE7Ew67vMQqSnsu6QOII04Ejg1N9G9p8b8/wW+mud/ocb8/UkdPe4kJaFzSZ0VIB1ruz5vrwuAz0TEzBqvodT2a8CppD0yN++NcIrwBQvNbPiQ9HLgbmBcRDze6nhs8HgPysyGjXx88HOkrvROTiOcO0mY2bAgaTXSMbP7Sc2xNsK5ic/MzNqSm/jMzKwtdXQT35gxY6Krq6vVYZiZdZTp06f/OyLG9lWvoxNUV1cXPT09rQ7DzKyjSOpt9JT/chOfmZm1JScoMzNrS05QZmbWlpygzMysLTlBmZlZW3KCMjOztuQEZWZmbckJyszM2pITlJmZtSUnKDMza0tOUGZm1pacoMzMrC05QZmZWVtygjIzs7bkBGVmZm3JCcrMzNqSE5SZmbUlJygzM2tLTlBmZtaWVmzGSiRtArwGuD8iepqxTjMz62wN70FJepekiyVNqir/KnAXcDZwvaTfNDlGMzPrQGWa+PYDtgduqxRI2gI4GlgCXAM8Cuwr6V3NDNLMzDpPmQT1WuCWiHiqULYfEMCHI2J74A3A88BHmheimZl1ojIJai3gwaqyHYAngDMAIuI+4GrgFU2JzszMOlaZBLUyoMoDSSsBWwHXRsTiQr2HgXWaE56ZmXWqMglqLvDKwuPtSUnrmqp6LwYeb3SlkiZK+qukuyTdIekzufxISQ9KujlPuxWWOULSDEn3SNqlUD4ll82QdHiJ12ZmZm2mTDfzK4H9JB0GXAJ8k3T86ZKqelsAc0qsdzHw+Yi4SdJLgOmSLs/zfhgRPyhWlvRKYB/gVcB6wJ8lbZpn/xTYKT//jZIuiIg7S8RiZmZtoswe1LdJx5v+F/gHMAm4IiJurFTIiWJD4PpGVxoRcyPipnx/EanL+vheFtkDOCsino2ImcAMYOs8zYiI+yLiOeCsXNfMzIahhhNURPwT2BY4FfgTcCTLJ4C3ALcAF/YnGEldpN6ClQR3iKRbJZ0iaY1cNh6YXVhsTi6rV179HAdL6pHUs2DBgv6EaWZmQ6DUUEcRcXtEHBQRu0fE0RHxdNX8EyLitRFxcdlAJL0YOA84NCIeB04ANiJ1xJgLHFupWiu0XsqrX8OJEdEdEd1jx44tG6aZmQ2Rpgx1NFCSRpOS0+kRcT5ARMwrzD+JpXtlc4CJhcUnAA/l+/XKzcxsmCk9WKykjSV9X9LVucfc9wrztslNaKuXWJ+Ak4G7IuK4Qvm6hWrvBG7P9y8A9pG0sqQNgE2AG4AbgU0kbZC7wO+T65qZ2TBUag9K0odIPeVWykUBjClUGUtqmnse+FWDq90W+ABwm6Sbc9mXSUMmbZWfYxbwUYCIuEPS2cCdpB6An4yIF3J8hwCXAqOAUyLijjKvz8zM2ociljtMU7uitC2pq/kTpC7mV5E6M0yNiINyndHAfGBaRLxzUCJuou7u7ujp8eDrZmZDSdL0iOjuq16ZPajDSHszu0bEtflJlqkQEc9LugcPdWRmZgNU5hjUG4EbKsmpF7OBdfuoY2Zm1qsyCeplNDZCxEq0Se9AMzMbvsokqPnABg3U24zlRz03MzMrpUyCugZ4naS6B7Yk7QRsCkwbYFxmZtbhyiSoH5JGazhf0s6SlllW0vbAKaSu3z9pXohmZtaJyozFdz2pJ98E0lh8j5B69e0paR7wV9LYd4dFxG11V2RmZtaAsmPxHQvsBvQALyXtUa1OOkH3dmDPiPhRs4M0M7POU7q3XURcAlwiaS1Sp4lRwOyI8Lh3ZmbWNP3uDh4Rj5Ca+czMzJqu9GCxZmZmQ6HuHpSk/fPd30XEosLjhkTEaQOKzMzMOlpvTXxTSb30rgMWFR43ygnKzMz6rbcEdRopIT1W9djMzGzQ1U1QEXFgb4/NzMwGkztJmJlZW3KCMjOztlT6PChJqwDdwHrAKvXquRefmZkNRKkEJemzwNdJwxz1xQnKzMz6reEEJekg4Nj88C7gbuDxwQjKzMyszB7Up0ndzD8QEWcMUjxmZmZAuU4SmwJ/d3IyM7OhUCZBPQU8MFiBmJmZFZVJUH8HthisQMxsZBs3rgtJ/53GjetqdUjW5sokqKOAzSUdMFjBmNnINW/e/aTD2GlKj83qK9NJYjXgOOAUSbsBF5Ga/JbUqhwRVw08PDMz61RlEtQ00l8fAXvlqZ4ouW4zM7NllEkiVzEIo5lLmkg6qXccaW/sxIj4saQ1gd8CXcAs4D0R8R9JAn4M7EbquHFgRNyU13UA8NW86m9FxKnNjtfMzIZGwwkqIiYPUgyLgc9HxE2SXgJMl3Q5cCBwRUQcI+lw4HDgS8CuwCZ5mgScAEzKCe0bpGGYIq/ngoj4zyDFbWZmg6jlg8VGxNzKHlBELCKNUjEe2AOo7AGdCuyZ7+8BnBbJdcDqktYFdgEuj4iFOSldDkwZwpdiZmZN1PIEVSSpC3gtcD2wTkTMhZTEgLVztfHA7MJic3JZvfLq5zhYUo+kngULFjT7JZiZWZP0qyODpNWAjUmDxqpWnbK9+CS9GDgPODQiHk+HmmpXrfV0vZRXx3UicCJAd3e3rxBsZtamyo5mvjGpg8LO9L73VaoXn6TRpOR0ekScn4vnSVo3IubmJrz5uXwOMLGw+ATgoVw+uap8WqMxmJlZe2m4iU/SBNJoErsC84AFpL2W64B/s3QP5lrgbyXWK+Bk4K6IOK4w6wKgclLwAcAfCuX7K9kGeCw3AV4K7CxpDUlrkJLopY3GYWZm7aXMMajDgTHANyNiAvAnICJi24hYh9RJYSbwHCk5NGpb4APAjpJuztNuwDHATpLuBXbKjwEuBu4DZgAnAZ8gBbIQ+CZwY56OzmVmZjYMKaKxwzA5UawEbBARSyT9Ctg/IkYV6mwE3EFKYt8ejICbqbu7O3p6elodhllHSI0lxd8b0ejvj40skqZHRHdf9crsQU0Abo6IytBGS/ITja5UiIh/AVcC+5ZYr5mZ2XLKJKhngGcLj5/It2tX1VsIbDCQoMzMzMokqAeBlxcez8i3b6wU5A4PrwUeG3hoZmbWycp0M78B2EvSKhHxDHBJLv+hpCdJ3bw/ThqC6KLmhmlmZp2mzB7URcCqwO4AEXEvqXv4eOBC4GbgY8DzLB2w1czMrF/KDBZ7HjC6qvjjwD2kS2+sCdwN/G9E3Nq0CM3MrCMN6JpNEfECcGyezMzMmqatBos1MzOr6O9gseOB7UnnRgVpLLyrImJOE2MzM7MOVnaw2LHAT4B3s/zeV0g6HzgkIuYvt7CZmVkJZUYcX5M0COwmpFEk/k66FDuky7JvQ+ossaWkN3ocPDMzG4gye1BHApsCVwAfy8Ma/ZekDYCfA28lXXr9M02K0czMOlCZThJ7ki6xsWd1cgKIiJnAu0iX3nhnc8IzM7NOVSZBrQ1cGRFP1quQ510JjB1oYGZm1tnKjsW3UgP1ViL16jMzM+u3MgnqHNJFBcfVq5Dn7Ui6fLuZmVm/lUlQR5MuRvhXSbtWz5Q0hdSB4g5SJwkzG0bGjetC0jLTqFGrLfN43LiuVodpHaRML74LgReAzYALJT3Kst3MV8/3rwUuSlfe+K+IiLcMKFIzG1Tz5t3Psle8hSVLlr0K7rx5wmyolElQkwv3BayRp2pvqlHm6zqbmVkpZRLUmwctCjMzsyplLrdx5WAGYmZmVuTRzM3MrC05QZmZWVsqO5r5GsAnSMej1gNWqVM1ImKjAcZmZmYdrMxo5huThjEaR+rF1xv32jMzswEpswd1LLAu6ZIbPwTuBZ4YjKDMzMzKngc1C9gpIp4blGjMzMyyMp0kArih2clJ0imS5ku6vVB2pKQHJd2cp90K846QNEPSPZJ2KZRPyWUzJB3ezBjNzGzolUlQN5OOPzXbVGBKjfIfRsRWeboYQNIrgX2AV+VlfiZplKRRwE+BXYFXAvvmumZmNkyVSVA/ALaTVGsoo36LiKuARi8PvwdwVkQ8my+QOAPYOk8zIuK+vId3Vq5rZmbDVMMJKiIuBD5LGgj2m5K2k9Ql6eW1pibEdoikW3MTYGXMv/HA7EKdObmsXvlyJB0sqUdSz4IFC5oQplknWXm5Ec89wrkNlrIn6v4DmAd8mdTl/F/AzBrTfQOM6wRgI2ArYC6pByHU7t4evZQvXxhxYkR0R0T32LG+8K9ZOc+SvlpLpzQKulnzlTkPajJwCUuvqvsIg9TNPCLmFZ73JNKlPiDtGU0sVJ3A0qv31is3M7NhqEw382+SktP3gGMi4tHBCQkkrRsRc/PDdwKVHn4XAGdIOo40ksUmwA2kPahNJG1AujT9PsD7Bis+MzMbfGUS1FbA9IhoahduSWeSzrEaI2kO6Wq8kyVtRWpDmAV8FCAi7pB0NnAnsBj4ZES8kNdzCHApMAo4JSLuaGacZmY2tBTR2KhEkuYDf46IEbNn0t3dHT09Pa0Ow6wtpKtgV/8eVJfVrtPI78jy629sORt5JE2PiO6+6pXpJPE30vlHZmZmg65MgvoasJGkzwxWMGZmZhVljkF1A78CjpO0F+l4zxxgSa3KEXHawMMzM7NOVSZBTWXpOUfbAn2NKOEEZWZm/VYmQZ2Gr/NkZmZDpOEEFREHDmIcZmZmyyg71JGZmdmQKNPEtwylkxrWyg8XRkTNzhJmZmb9UXoPStJOki4ljcM3L0+LJF0iaadmB2hmZp2pVIKSdBRpwNidgFVZOqTxqsDOwCWSjmxyjGZm1oEaTlCSppBO1n2aNGDs5qTEtCqwGfBd4Cnga8VLsZuZmfVHmT2oTwEvALtFxOER8c+IeD5P90bEEcDbSHtUnxqMYM3MrHOUSVBbA9fkS7TXlOf9DZg00MDMzKyzlUlQLyENbdSXh3JdMzOzfiuToOYDr2mg3hbAgv6FY2ZmlpRJUNOAV/U2mrmkTwGvBv4ywLjMzKzDlTlR9xhgb9Jo5u8GTgVmkjpFbAjsD2wHPEPq0WdmZtZvZcbiu1PSe4FfkxLRtlVVBCwCPhARdzYvRDMz60SlhjqKiAskbQocDGwPjCclpjnAlcBJETGv6VGaWRtbOV/Ofal11lmfhx+e1ZpwbMQoPRZfTkDfHIRYzGxYepbqK/HMm6faVc1K8GjmZmbWlsoMdfQqSV+X9Npe6rwu19m8OeGZmVmnKrMH9QnSWHy9neM0H/g68PGBBGVmZlYmQb0ZuCUi6o4mkefdDLxloIGZmVlnK5OgxgP3NVBvZq5rZmbWb2US1KgG6wtYuX/hmJmZJWUS1P3AJEl1l8nzJgGzBxqYmZl1tjIJ6lJgPeBLvdQ5jNS8d2mjK5V0iqT5km4vlK0p6XJJ9+bbNXK5JB0vaYakWyW9rrDMAbn+vZIOKPG6zMysDZVJUMeRhjL6lqTTJe0oab08vVnS6cC3c51jS6x3KjClquxw4IqI2AS4Ij8G2BXYJE8HAydASmjAN0h7b1sD36gkNTMzG54aTlC5h957gCeBfYHLSU15s4E/57IngfdGxP0l1nsVsLCqeA/SYLTk2z0L5adFch2wuqR1gV2AyyNiYUT8J8dWnfTMzGwYKTWSRERcBrwKOB74J2mMk2fy/eOBLSLikibEtU5EzM3PORdYO5ePZ9njW3NyWb3y5Ug6WFKPpJ4FC3zZKjOzdtWfsfhmA4cOQiyNqDXAV/RSvnxhxInAiQDd3d0165jZQC0/gKxZWe06Ft+83HRHvp2fy+cAEwv1JpAuMV+v3MxaojKAbHEyK6ddE9QFQKUn3gHAHwrl++fefNsAj+UmwEuBnSWtkTtH7EyJnoRmZtZ+SjfxNZukM4HJwBhJc0i98Y4Bzpb0IeAB0pV8AS4GdgNmAE8BHwSIiIWSvgncmOsdHRHVHS/MzGwYUUTn7np3d3dHT09Pq8MwawvpmFH170F1WSN1Gi1bhdQUuJQvdNgZJE2PiO6+6rV8D8rMOpUvdGi9a9djUGZm1uF6G1fvNZImDGUwZmZmFb3tQf0DOKryII+Zd9Dgh2RmZtZ7ghLLngB7ILDdoEZjZmaW9ZagFgHrDlUgZmZmRb314rsd2FHS0aTzjgA2lrR/IyuOiNMGGpyZmXWu3hLU94BzgK8UyrbNUyOcoMzMrN/qJqiI+IOkrUmXung56RjUv4BrhiY0MzPrZL2eqBsRNwM3A0g6ELg6ItyTz8zMBl2ZkSSOInU9NzMzG3QNJ6iIOKrvWmZmZs1Reiw+SaOBvUgjkFeuWvsgMA04NyKeb1ZwZmbWuUolKEmvJ/XsW5/lr2L7YeBbkvaOiJuaFJ+ZmXWohhNUHpfvUmBNYDZwOqlXn4ANgPfl20slbRURDzY/XDMz6xRl9qAOJyWn44EvVjflSfoG8H3gM7nup5oVpJmZdZ4yl9uYAtwHfLbWcaaIWAx8PtfZrTnhmZlZpyqToMYDN0Qvl+CNiCXADcB6Aw3MzMw6W5kE9TSpia8va+S6ZmZm/VYmQd0KTJa0eb0KkjYD3pzrmpmZ9VuZBHUysBLwF0kHSVqpMkPSaEkfBK4ARgMnNTdMMzPrNGVGkvi1pCnAvqQE9AtJc4EgHXNagdTl/IyIOH0wgjUzs85RZg+KiHg/cAgwCxgFTAAm5vszgUMiYr8mx2hmZh2o9FBHEfEz4GeSxpN69gmY4xNzzcysmUonqIqckJyUzMxsUJRq4jMzMxsqTlBmZtaW2jpBSZol6TZJN0vqyWVrSrpc0r35do1cLknHS5oh6VZJr2tt9GZmNhBtnaCyN0fEVhHRnR8fDlwREZuQzrs6PJfvCmySp4OBE4Y8UjMza5rhkKCq7QGcmu+fCuxZKD8tkuuA1SWt24oAzay/VkbSMtO4cV2tDspapN0TVACXSZou6eBctk5EzAXIt2vn8vGk61RVzGHpFX//S9LBknok9SxYsGAQQzez8p4lfe2XTvPm3d/akKxl+t3NfIhsGxEPSVobuFzS3b3Urb7CL6RP+LIFEScCJwJ0d3fXHZndzMxaq633oCLioXw7H/gdsDUwr9J0l2/n5+pzSKNaVEwAHhq6aM3MrJlKJShJ90n69mAFU/Vcq0l6SeU+sDNwO3ABcECudgDwh3z/AmD/3JtvG+CxSlOgmZkNP2Wb+LqAscUCSacAV0fEKc0KKlsH+J0kSHGeERGXSLoROFvSh4AHgL1z/YtJV/KdATwFfLDJ8ZiZ2RCqm6AkfQ34O+kquot6WceB+bapCSoi7gO2rFH+CPCWGuUBfLKZMZiZWev0tgd1FLkbjaQ7gOtz+ahBj8rMzDpebwnqTYVpG+DDufxASW8inST718ENz8zMOpVSy1gDFaX1Sdd8qpxrNJGl3bjnAmeSEtZVEfFEk+McFN3d3dHT09PqMMzaQjreW/17UF3WSJ1GyxpfrtHfKRseJE0vjA5UV8O9+CKicrbcpRGxPrAp8AnSJ2pd4PPAH4GFkq4pH7KZmdlSdROUpF9KOkjSK2rNj4gZEfGL/PA04DXAZ4GLgM2aHqmZmXWU3o5BHUTuqi3pMeC6XL66pBUjYnGhbkTE7aTzlI4flEjNzKyj9JagNiR1jqhMO5Iah99Nasa7ktRRwszMrOnqJqiImAXMAs4CkLQS8AxwJ/AosAvwNlLS2lPSqsA04K8R8c/BDNrMzEa+Mp0knst3/x4R2wFrkS51IWA0aUSHE4C7JM1pdqBmZtZZ+j1YbEQsiogL8sNzSAnrXcBPgYVNiM3MDF8jqnM17XIbEfEY8Ps8mZk1SeUaUUvNm1fr6jo20pRKUBHR1pfnMDOzkaMZe1BvBh5uwnrMzMz+a8AJKiKubEYgZmZmRW6yMzOztuQEZWZmbckJyszM2pITlJmZtSUnKDMza0tOUGZm1pacoMxsGFrZQx91gKYNdWRmNnSWHf7IQx+NTN6DMjOztuQEZWYjgEc8H4ncxGdmI4BHPB+JvAdlZiOU96qGO+9BmdkI5b2q4W7E7UFJmiLpHkkzJB3e6njMrJ14r2o4GVF7UJJGkS45vxMwB7hR0gURcWdrIzOz9uC9quFkpO1BbQ3MiIj7IuI54CxgjxbHZGZtbfm9qlGjVvNeVhsYUXtQwHhgduHxHGBSsYKkg4GD88NnJd0+RLE12xjg360Oop8c+9BrMO5aexPVZY3Uaeq6xoBqxN7MuJa1ZMlTyzyeN+9+pH7taQ3XzwsMbuzrN1JppCWoWp+gZfbnI+JE4EQAST0R0T0UgTWbY2+N4Rr7cI0bHHurtEPsI62Jbw4wsfB4AvBQi2IxM7MBGGkJ6kZgE0kbSFoJ2Ae4oMUxmZlZP4yoJr6IWCzpEOBSYBRwSkTc0csiJw5NZIPCsbfGcI19uMYNjr1VWh67IqLvWmZmZkNspDXxmZnZCOEEZWZmbaljE9RwHRJJ0kRJf5V0l6Q7JH2m1TGVIWmUpH9IurDVsZQhaXVJ50q6O2/7N7Y6pkZJ+mz+rNwu6UxJq7Q6pnoknSJpfvH8RElrSrpc0r35do1WxlhPndi/nz8zt0r6naTVWxljPbViL8z7gqSQNGao4+rIBFUYEmlX4JXAvpJe2dqoGrYY+HxEvALYBvjkMIod4DPAXa0Ooh9+DFwSEZsDWzJMXoOk8cCnge6I2ILUeWif1kbVq6nAlKqyw4ErImIT4Ir8uB1NZfnYLwe2iIjXAP8EjhjqoBo0leVjR9JE0tBxDwx1QNChCYphPCRSRMyNiJvy/UWkH8rxrY2qMZImAG8DftnqWMqQ9FJge+BkgIh4LiIebW1UpawIrCppReBFtPG5gRFxFbCwqngP4NR8/1RgzyENqkG1Yo+IyyJicX54HenczLZTZ7sD/BA4jOoBDIdIpyaoWkMiDYsf+SJJXcBrgetbG0nDfkT6sC9pdSAlbQgsAH6Vmyd/KWm1VgfViIh4EPgB6R/wXOCxiListVGVtk5EzIX0Bw1Yu8Xx9NdBwJ9aHUSjJL0DeDAibmlVDJ2aoPocEqndSXoxcB5waEQ83up4+iJpd2B+RExvdSz9sCLwOuCEiHgt8CTt28y0jHy8Zg9gA2A9YDVJ+7U2qs4j6Suk5vnTWx1LIyS9CPgK8PVWxtGpCWpYD4kkaTQpOZ0eEee3Op4GbQsHinm5AAAPuUlEQVS8Q9IsUpPqjpJ+09qQGjYHmBMRlT3Vc0kJazh4KzAzIhZExPPA+cCbWhxTWfMkrQuQb+e3OJ5SJB0A7A68P4bPiacbkf7U3JK/sxOAmySNG8ogOjVBDdshkZSGVD4ZuCsijmt1PI2KiCMiYkJEdJG2918iYlj8k4+Ih4HZkjbLRW8Bhss1xh4AtpH0ovzZeQvDpINHwQXAAfn+AcAfWhhLKZKmAF8C3hERT/VVv11ExG0RsXZEdOXv7Bzgdfm7MGQ6MkHlg5aVIZHuAs7uY0ikdrIt8AHSHsjNedqt1UF1gE8Bp0u6FdgK+E6L42lI3us7F7gJuI30nW/5EDb1SDoTuBbYTNIcSR8CjgF2knQvqUfZMa2MsZ46sf8f8BLg8vxd/XlLg6yjTuwt56GOzMysLXXkHpSZmbU/JygzM2tLTlBmZtaWnKDMzKwtOUGZmVlbcoLqg6RZeSTfya2OxYaP/JlpahfZwVhnqxS+V11D8FzT2vU7LKkrxzar1bG0IycoMzNrSyu2OgAz60hvAUYDD7Y6kBZ7EHgF8HyrA2lHTlBmNuQi4l+tjqEd5PER7251HO3KTXyDQNJqkr4i6RZJT0h6Mg9z8uU8SnC95d4t6e+5/n8kXSbpfyRNzu3U0/oRy0RJx0m6M6/38XxF2J9J2qJG/bUkfUvSbYXYb8pXZR1do/7UHNuBkjaWdIakeZKezVcS/ZKk5T5nklaRdHhe9xO5/lxJ1+bnX+6qr5JeJek0SbNz/X9LuljSrnVeezG2V0s6R9LDkl6QdGhV3UmSzsrDvDwnaYGkCyRt18u2fbXSVVIXFrbTh+vVb0R/1plj/76knrztn5P0kNIVgLepUf+UvF3qjsgu6dO5ztmFslGSPpY/o4/l55mXYzxW0tgSr7PmMaji8SJJr8/vwSOSns7fpwENwSPpzZL+nL9fT0i6WumyEg3HWCvWqvLVJX1H6SrGT+XY5+T6R1TVrXsMSoVjjpLem78bT0haJOmKPj6bq0k6TNKNSt/5p3M8RypdCaG6fqn3VtJmkk6VdH+uuyhvr99Jene9uEqLCE+9TMAs0qU4JjdYfwxwa15mIfA70gjS/8llNwNr1ljuy3n+EuBq4AzgFtIQ/cfledNKxr4z8Fhe9sEcR2VctheAI6vqvzrXC9L1si4ELgYeyWV/BlaqWmZqnvej/Fz/Io1W/ldSs0UAP6laZgXSlVEDeBS4KL/eK0iDUgYwrmqZdwDP5Hm35/pX5dcRwDdrvP5KbCflZSuxXQQcXKj3+bzdlwA9wDmkccleyNNHaqx7B+CpvP67gTOBabl+5f2Kku9Xv9aZ35fF+fPyx/we35brLwb2rqq/VZ43E1ihTix35jo71NieTwGX5ffgsrxdA9imH9+rrqryabn8GOC5/DrOBK6pvH7SFaXLbNfKOn+ct+WtOfbiOj/XaIw11ju5UPYi4I5cPo800G3lfZwPPFO1jq5cd1aN9VdiOzrHfSXwW9L4oQE8C7yxxnITCjHMJ405+gfSFRsif07WqPNd6fO9Jf1OPJ7L7yJdWeFc0nfmKdKVp5vz+9usFY3UifIJ6uxc/ypg9UL5GoUvxJlVy7w+fwCfA6ZUzft04YM6rUTcLy98iL4KrFhj/usLj1cF7sv1jyjWB9YkXbo6WD6pTS3EdySFHzzSVWgrP/ITq8oDmA6sVrU+kQbEfVGhbBxLE+3nqupPJl2fKYBdeontW9T4MSZd5rqSwCdVzds2P+9zwKZV26qSSL9DHtMyz9uhEE+UeL/6vc78GtapUf72HPsjxe2Z5/0tr2/3GsvtmOfdXihbP5c9UOe5tgLW7sf3qquqfFrhPTuoat5+ufyx6tfTx3MV1/mFGtvoeVIif00jMdZY7+RC2f657EKW/86NAnasKuui7wT1CMt+VysD/gZweY3vz9/zvJ+w7PdoVeDXed7U/r63wCm5/hE16r6YGkmzv1NTVjKSJ0okqPxGLyH9IG9eY/4rqf2DXXnDf1lnvddRPkH9KC9zVoP1P57r/7bO/PVIP3YLWPbHc2pe7oZieWH+xXn+/oWyvXPZjxqM7Wu5/jV15n+3zpe1EttdwKg6y16f6+xaZ/4X8vxjC2UfyGUzaq0XOLby41Li/Wr6OvNyp+fl3lZV/p5cfnGNZc7L8z5RKHtDLvt9mefvJa7K96qrqnxaLj+3znKVPbvtSzxXZZ09deafmuef1EiMNdY7uVD2xVx2aIOxddF3gjqkxry187xngNGF8l1z+bXU/kO2GmnP7nnyXlTZ95bUAhHAVs34LPQ2+RhUc/0P6R/MdRGx3IHPiLiT9EO+AmkvomKHfHtGnfWe2Y9YpuTbXzZYv3LJjnNqzYyIh4B7SU2Ym9SocnHkT2+VynZYr1BWaWL8kKRPSFqnj9gq22dqnfmn5NvtJI2qMf8PEfFCdaGkMaQv5+Ok5oxarsy3b6wRz1m11kv6l1rWgNYpaUw+1vYDpUvST5U0FagcZ9y0apHzSXuNu0jasLCe8aTm1EVVz3l3Lnub0rHU9Rt9Yf10YZ3yWp+nRtW7QGbldU7uxzqr3ZBvvyRpP0mrN2Gdy22LiJhPOmywMrBWYVble3xeRCypsdyTpGbsFUmffSj/3lZe488l7SRp5YZfSUlOUM01Pt/O7KVOpffS+EJZ5f79dZapV96byoes0R5ClR+pcyoHZ6sn0h4gQK2D4Q/UWW/lcvT/7fQQqQfXZ4GVgJ8CD0v6l6RfS9qrRpLpa7vOJO25rsKyX9aKettvA9IfipcCi+u85sqXsfiaJ/QRz6w65b3p9zolfZT0Gn9FOp72IdKF/Q4AXpOrvbS4TKRrop1A+g34WGHWwaQfr9MiYlGh/iLgIOBp4NvArHzg/5ycGJfr1DJADX+eSuhr206oM79hEXEl8D3SHs6vgYVKHZROlLRLP1dbZltUvsff7+V7XEliY3PMZd/b75OOF08i/bF7TNJ1kr4r6dX9fI01uZt5cynf1tqTqK5TS73llvsn1IDeYqilkhQuAv7dR91HapSVijEifiLpHGBPYLs87ZenmyXtEBGVL2Aj27U3T9cpr7zmx4Df97GOvrZJUX/jLL1OSd2kRLOY1Lz0R9KxrKciIiR9h3RMsdbn7kRS8+lBkr5O7hCS5/1suQAizpX0Z2APUgvAtsBeeTpS0v9ExOz+v8Rl9OczP1Bl37eaf/Aj4ktKFybcg/S53pa0XT8i6TJSc+vihoOqsSfUi8pn+kr6/qP03z9uZd7bSFcGfqukSaSWmm1JLQyTgMMkfSMiji4Rc11OUM01J99u2EudDfJt8QTFh3L5+qSOCtW6+hHLA8BmeZrTR11IvfY2A06IiIv68XylRbp89M/zhKQtSf86twIOJ/VshBT/ZqTtekWNVXWRfiyeIfWcbFTlx/T5iDiwxHKV966rzvwN6pQPxjr3IiWf4yPiBzXmb1zvCSNigaTfkg7sv4e0/dYlHeuseUn7iHiUdMzmVABJG5F6Sb6ZdCzwffWerw109VH+UFX5c/l2uW7ZWd2msIiYSToO/COA3CX8TFLP2oMYvKsaVz7T50TET8ssWPa9jXS15utz3ZXy/JNICe23EXHPAF4H4Ca+Zqv0jNpGUnWbP5JeQfqXsYTUy6+icn/fOuvdpx+xXJpvGz0v50/5du9+PFdTRMQtpK7AAFsWZlWOA+1fZ9EP5turS/4zfZDUjXlM9bksfajEs0+dY17vL7Guga5zzXy73J5LPndlpz6e9yf59hN5gtTs2pDcXPvt/HDL3uq2gXrbsFI+raq88qdh8+oFlM4hnNjoE0fE1Sw9hjqY26lp3+My721EPBcRU0kdusTSpuUBcYJqooi4n9QDagXgF5JeVpmXD5b+Is87u6op5KekpHWApGV+UCR9gmUP0DfqOOAJ0g/eEdU/ekon8L6+UHQi6UfuAElHqcYJxZK2kPTB6vKyJO0oaTdJK1aVj2Jp+3jxuNFJpIO420n6dNUy2wOfyg+P7Uc4X8u3v5G0c41YV5L0DknF9+BcYC5p7+RISSrU347UI7Ks/q6zcoxxfxVOwJT0ElLnkV4P0kdED+lHZRKpo8ZD1GjulPRapZNFV62xmrfn2/4cKx1Kb5D02WKBpN1IzcovAP9XVb+yt36YpJcWlplISjbLNZtKeqek7VV1cnrebm/NDwdzO/2edPrGDpJ+LmnN6gqSNpT0ycLjUu+tUsemzWqtF3hVdf0BGexugsN9YmlX0ztJX+R607q5/hiWniT5CClhnUdqegrqn6hb6Uq9hLQndnqu+wJLu4xfVjL2XUk/7EFqJqucUDed+ifq3l+I/a+kZom/kA4wB6mHYnGZqbn8wDoxHEnV+VPAobns0bzu00knNFdOJJwLrF+1nj1YeqJu5UTLaTR2om7N2Ar1Pkc6hhPAPaSTK88lNV88mss/VrXMjqRjW5Vu7Gfk7TWQE3VLr5N0ft0Ded58Uu+83+X3by5wcvX2r/G8+1bWDXyjTp098/wn8+fzjLyNKidyPg509+N71VVVPo1eTuto9D2ts87Kibq35PivLrzuL9ZYrrht5+Zt+xfSH7+/sPS8xsmFZSrf1coJsr8hHResnOx+F/CyQv0u+uhm3o9tOIGlgwU8nt+vM0nnMt6Tyx/u73tL+l2KPP8PpO/vFaQTh4Oq8zwHMjVlJSN5KnwI+pq6Csu8mHRy7K2kM6ufyl+KL1N1YmrVc+1NOn/hSdIP459JXV8rJyie0Y/4NyD9M7yX9AP/GCnZ/h/wyhr1X0Y6qH5drvssKbldSzqjvfpkxqmUT1Ab5fK/kH4AniF1QrgJ+Dowts66tiAdo3qQpSeg/gnYrU79XmOrqrslqUv+DFKSWAT8M38BP0ztPxVb5vn/ye/xzeRERh8/Ln3EUWqdpB5jvyAdv3yWtCd8Eul40nLbv8byY3Od58l/tGrUGUc6Lvgn0p+Vp/Jn9DbgB1T9oSjxveqqKp/G4CWoyaS9mL/kz/aTpCSzZy/LTiQlmQV52/4T+AapB+pysZKOnx5DSn4P5mXmkf7sHAq8pGr9XTQ5QeV5qwCfJDUdLyR9X+aSuph/H3hTf99bYHfSceN/FLbLA6QefXtTZ3SS/kzKT2htTNLJpAOrX4iI/jRjmdUl6TOkf/5nR8R7Wx2PWYUTVJvInSrmR+pJUykTcCCpmeY5YMNIJ8yaNUU+tnIHqVlom0g9s8zagruZt4/9gS9K+gepiWZV0omxG5COS33KycmaRdIXSU2m25OS0zlOTtZunKDax8WkYzOTSD1hVia1755LGrPumhbGZiPP20i99haQjld9vrXhmC3PTXxmZtaWfB6UmZm1JScoMzNrS05QZmbWlpygzMysLTlBmZlZW/p/fO0p/Ih1wNwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_uncensored_only = df[df.is_tib_censored == 0]\n",
    "\n",
    "plt.hist(df_uncensored_only.log_censored_tib, bins = 15,\n",
    "         color = 'blue', edgecolor = 'black')\n",
    "plt.axis([0, 15, 0, df.shape[0] / 15])\n",
    "plt.xlabel('Log censored days in business', size = 22)\n",
    "plt.ylabel('# of companies', size= 22)\n",
    "plt.title('Censored entities only')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most recently organized: 2007-07-10\n",
      "Loggeg time_in_business for the most recently organized entity: 8.355\n"
     ]
    }
   ],
   "source": [
    "print(\"Most recently organized:\", df.OrganisationDate.max())\n",
    "print(\"Loggeg time_in_business for the most recently organized entity:\",\n",
    "      round(math.log((datetime.date(2019, 3, 1) - df.OrganisationDate.max()).days), 3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This makes sense. I pulled the data in 2019, but the data I was pulling from didn't include recently organized entities. Our uncensored data looks normally distributed, and it's reasonable to assume that our censored entities will become inactive following that same path."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make variables for the most common and meaningful words in entity's names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('inc', 24187), ('llc', 8618), ('&', 3230), ('corporation', 2117), ('of', 2005), ('the', 1943), ('realty', 1774), ('company', 1732), ('associates', 1604), ('co', 1506)]\n"
     ]
    }
   ],
   "source": [
    "# Remove all punctuation from the names of companies\n",
    "list_of_entity_names = [row[1] for row in df['EntityName'].iteritems()]\n",
    "\n",
    "list_of_names_w_o_punctuation = []\n",
    "for name in list_of_entity_names:\n",
    "    list_of_names_w_o_punctuation += [re.sub('\\,|\\.|\\!|\\;|\\:|\\?|', '', name)]\n",
    "\n",
    "# Break each name into a list of words so we can count the occurrence of each word\n",
    "adjusted_words_in_entity_names = []\n",
    "for words in list_of_names_w_o_punctuation:\n",
    "    for word in words.split():\n",
    "        adjusted_words_in_entity_names += [word.lower()]\n",
    "\n",
    "# Count the occurrence of each word:\n",
    "name_count = Counter(adjusted_words_in_entity_names)\n",
    "popular_names = name_count.most_common(300)\n",
    "print(popular_names[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('inc', 24187), ('llc', 8619), ('&', 3230), ('associ', 2492), ('corpor', 2158), ('servic', 2148), ('of', 2005), ('the', 1943), ('realti', 1774), ('compani', 1767)]\n"
     ]
    }
   ],
   "source": [
    "# Start by stemming entities names & descriptions. This makes words comparable across conjugations & forms,\n",
    "# but at a cost of occasionally making the resulting word less intelligible.\n",
    "stemmer = PorterStemmer()\n",
    "list_of_stems = [stemmer.stem(word) for word in adjusted_words_in_entity_names]\n",
    "\n",
    "# Count the occurrence of each word:\n",
    "stemmed_name_count = Counter(list_of_stems)\n",
    "stemmed_popular_names = stemmed_name_count.most_common(300)\n",
    "print(stemmed_popular_names[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('servic', 2148), ('realti', 1774), ('island', 1268), ('rhode', 1102), ('enterpris', 906), ('construct', 896), ('properti', 811), ('manag', 709), ('new', 610), ('develop', 550), ('invest', 535), ('limit', 519), ('system', 487), ('intern', 485), ('center', 483), ('england', 432), ('home', 422), ('auto', 421), ('provid', 408), ('product', 397)]\n"
     ]
    }
   ],
   "source": [
    "# Exclude words either associated with the entity's legal status (we already have a variable for that)\n",
    "# or that are otherwise meaningless by themselves. \n",
    "words_to_exclude = ['inc', 'llc', 'corporation', 'the', 'of', 'company', 'associate', 'co', 'ltd', 'corp', 'and',\n",
    "                    'enterprise', 'association', 'group', 'associates', 'enterprises', 'incorporated', 'limited',\n",
    "                    'partnership', 'club', 'in', 'for', 'lp', 'an', 'corpor', 'associ', 'compani', 'incorpor',\n",
    "                    'busi', 'foundat', 'llp', 'title', 'to', 'as', 'on', 'gl', 'at', 'business', 'operate', 'titl']\n",
    "\n",
    "worthwhile_entity_words = [item for item in stemmed_popular_names if \n",
    "                           item[0] not in words_to_exclude and \n",
    "                           len(item[0]) > 1]\n",
    "print(worthwhile_entity_words[:20])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Do a similar process to get the most common and important words in entity's descriptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The Comments variable is also called \"Purpose\" on Rhode Island's website. It's intended to be where companies describe\n",
    "# what kind of business they intend to do. Most companies give at least a brief answer, but plenty don't.\n",
    "# My favorite common fake response is \"Any Lawful Purpose.\"\n",
    "\n",
    "# Remove all punctuation and newline characters\n",
    "list_of_comments = [row[1] for row in df['Comments'].iteritems()]\n",
    "list_of_comments_w_o_punctuation = []\n",
    "for name in list_of_comments:\n",
    "    name = name.replace('\\n', ' ')\n",
    "    list_of_comments_w_o_punctuation += [re.sub('\\,|\\.|\\!|\\;|\\:|\\?|', '', name)]\n",
    "\n",
    "# Break each name into a list of words so we can count the occurrence of each word\n",
    "adjusted_words_in_entity_description = []\n",
    "for words in list_of_comments_w_o_punctuation:\n",
    "    for word in words.split():\n",
    "        adjusted_words_in_entity_description += [word.lower()]\n",
    "\n",
    "# Just like we did with EntityName, we need to stem the comments:\n",
    "list_of_stems = [stemmer.stem(word) for word in adjusted_words_in_entity_description]\n",
    "\n",
    "# Count the occurrence of each word:\n",
    "comment_count = Counter(list_of_stems)\n",
    "popular_comments = comment_count.most_common(300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('servic', 4384), ('real', 4380), ('estat', 3914), ('sale', 2844), ('oper', 2499), ('manag', 2056), ('provid', 1916), ('sell', 1866), ('develop', 1494), ('engag', 1435)]\n"
     ]
    }
   ],
   "source": [
    "# Remove words we don't want, including number clusters, which are references to the Rhode Island code, and aren't useful.\n",
    "def numbers_present(word):\n",
    "    return any(char.isdigit() for char in word[0])\n",
    "\n",
    "worthwhile_entity_descriptions = [item for item in popular_comments\n",
    "                                   if item[0] not in words_to_exclude\n",
    "                                   and len(item[0]) > 1\n",
    "                                   and not numbers_present(item)]\n",
    "print(worthwhile_entity_descriptions[:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make variables for whether a company's name or self-description contains the most popular words or not"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "396"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# These lists are based on the 300 most popular words in names or descriptions (before filtering out words).\n",
    "# That 300 number is very arbitrary.\n",
    "all_the_popular_words = [word[0] for word in worthwhile_entity_words]\n",
    "all_the_popular_words += [word[0] for word in worthwhile_entity_descriptions]\n",
    "new_variable_list = list(set(all_the_popular_words))\n",
    "len(new_variable_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(43658, 422)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Add all our new variables to the dataframe\n",
    "new_keywords = dict.fromkeys(new_variable_list, pd.Series([0] * len(df)))\n",
    "df = df.assign(**new_keywords)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0., nan])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.street.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For reasons I don't understand, this method is creating some values as 'nan' instead of '0'.\n",
    "# Fix that:\n",
    "for var in new_variable_list:\n",
    "    df[var] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0], dtype=int64)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.street.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Return a list of all words present in a company's name/description, scrubbed the same way we scrubbed them above\n",
    "def standardize_names_and_comments(entity_name, self_description):\n",
    "    combo_phrase = []\n",
    "    return_phrase = []\n",
    "\n",
    "    entity_name.replace('\\n', ' ')\n",
    "    self_description.replace('\\n', ' ')\n",
    "\n",
    "    for word in entity_name.split():\n",
    "        combo_phrase += [word.lower()]\n",
    "    for word in self_description.split():\n",
    "        combo_phrase += [word.lower()]\n",
    "\n",
    "    for word in combo_phrase:\n",
    "        return_phrase += [re.sub('\\,|\\.|\\!|\\;|\\:|\\?|', '', word)]\n",
    "\n",
    "    return list(set([stemmer.stem(word) for word in return_phrase if len(word) > 1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Iterate over each row in the df. Figure out which words they share with new_variable_list, and switch that value to 1\n",
    "for index, row in df.iterrows():\n",
    "    words_present = standardize_names_and_comments(row['EntityName'], row['Comments'])\n",
    "    for word in words_present:\n",
    "        if word in new_variable_list:\n",
    "            df.at[index, word] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1], dtype=int64)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# As long as there are 2 unique values (0 and 1), then we know things worked:\n",
    "df.real.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make variables for the type of legal entity, and the number of directors & officers listed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make dummy variables for all the types of legal entities that have been registered. We'll use all of them\n",
    "# but one in the analysis ('Foreign Corporation').\n",
    "\n",
    "entity_dummies = pd.get_dummies(df['EntityType'])\n",
    "df = pd.concat([df, entity_dummies], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    [PRESIDENT, JAMES H BOND , 2401 21ST AVENUE, S...\n",
       "1                                            [ ,  ,  ]\n",
       "2    [PRESIDENT, ETHAN ANDREW BLUMEN , 1881 BAY RD ...\n",
       "3    [PRESIDENT, CHARLES H RYBECK , 329 WICKENDEN S...\n",
       "4                                            [ ,  ,  ]\n",
       "Name: d_and_o, dtype: object"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.d_and_o.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x19999850>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPwAAADuCAYAAAAdtWr3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJztnXd4HNW5/z/vzKpLu5ar3CS5YbApBgwYDAYcSkD0yw0hIfiGQAgplABBqWwKxCm/VG7CzQ0kJAEuLYQEJZTQbIpNMcVg3C03yZas3nd35v39MWss27K1qzaS9nyeZx9pZ2fnfCXtV2fOOe95X1FVDAZDamD5LcBgMAwcxvAGQwphDG8wpBDG8AZDCmEMbzCkEMbwBkMKYQxvMKQQxvAGQwphDG8wpBDG8AZDCmEMbzCkEMbwBkMKYQxvMKQQxvAGQwphDG8wpBDG8AZDCmEMbzCkEMbwBkMKYQxv6DEi8nERWSMi60Wk1G89hu4Rk9PO0BNExAbWAmcC24A3gMtVdZWvwgwHxfTwhp5yPLBeVTeqagT4P+BCnzUZusEY3tBTJgJbOz3fFj9mGMQYwxt6inRxzIwPBznG8Iaesg2Y3On5JKDCJy2GBDGGN/SUN4AZIjJFRNKBTwJ/91mToRsCfgswDE1UNSYiXwaeBmzgXlX9wGdZhm4wy3IGQwphbukNhhTCGN5gSCGM4Q2GFMJM2qUQxaVl2cDITo/8+EOACBDt4msHsAuoKF9c0uSDbEMfYibthhHFpWU2MA2YBRwW/3ooXgRcPpDZyyZa8NbaNwOb4o8NwNvli0vW9fLahgHAGH6IUlxaZgFHAQvx4tpnAYcA6T5JqgPexFuffwN4o3xxyXaftBgOgDH8EKK4tGw2nsFPB07Fuy0fzFQArwFPAk+WLy7Z5bOelMcYfhBTXFqWDpwDfAL4GDDOX0W9wsUz/xPAE+WLS9b6rCclMYYfZBSXlglwMvBp4D8Z/L14T1mLZ/6/lC8uec9vMamCMfwgobi07HA8k18OFPksZ6BZBvwWeLh8cUm732KGM8bwPhLvzc8HbsXr1VOdWuA+4G5zy98/GMP7QHxsfgVwC97ymWFvFHgB+EX54pJ/+C1mOGEMP4AUl5YFgS8ANwATfJYzVHgd+Fb54pJn/RYyHDCGHwDiEW63Al8Fgj7LGaq8hGf8l/0WMpQxhu9nikvLLgd+xN7ZYQw952k847/pt5ChiDF8P1FcWnYs8AvMZFx/8SBwY/nikiq/hQwljOH7mOLSsnHAD4FFmN2I/U0tcEv54pI/+C1kqGAM30fEl9iuAxYDeT7LSTWeA64tX1yywW8hgx1j+D6guLRsEnAvXhUWgz+0AWHgZ+WLS2I+axm0GMP3kuLSsk8B/w2M8FuLAYC3gU+VLy5Z7beQwYgxfA8pLi3LxTP6lX5rMexHM3BV+eKSR/wWMtgwhu8BxaVlR+PVUjvEby2Gg/IL4FZzi78HY/gkKS4tuxT4E5DltxZDQrwMXFa+uMRUxcEsGyVFcWnZ14GHMWYfSpwMrCguLTvNbyGDAdPDJ0BxaVkacDdwld9aDD3GAb5Svrjkt34L8RNj+G4oLi0bATyGl1rKMPT5Zvnikjv9FuEXxvAHobi0bCpQhpf51TB8+Gn54pJb/RbhB8bwB6C4tGwm3g6toZxHznBg7gE+X764xPVbyEBiDN8FxaVl0/HMbvasD28eBT5dvrgk4reQgcIYfh+KS8um4JndbGdNDZ4BLipfXNLmt5CBwCzLdaK4tGwy8DzG7KnEWcAjxaVlKVF2zRg+TnFp2QS8PGrFPksxDDwlwL3xHY/DGmN4oLi0bCxezz7Nby0G3/gM8DO/RfQ3KW/4eAbZvwIz/dZi8J0bi0vLbvBbRH+S8oYHfg3M91uEYdDws+LSsgv9FtFfpPQsfXFp2XXAb/zW4SfqOlTedxOBvFGMvfR2VJX6pX+mdfXLIBZ5R59LcO4FtKx5hYal92Nl5TLmkm9hZwWJ1lVSv+RPjLnwNr9/jL6mFVhQvrjkLb+F9DUp28MXl5YtAH7ptw6/aXrz76SN2rMo0bLy3ziN1Uy45m4mXnM3OYct8M57/XEKPvNTcmYvpGXVSwDUL/0zI065whfd/Uw23sx9yG8hfU1KGr64tKwQL+gizW8tfhJr3EXbxjfIPeqsj441vfNPQvMvR8T7aNg58UQ+YqFOFI11IJZN+9b3sXPySRs50Q/pA8EU4Pd+i+hrUs7wxaVlWcDjwBi/tfhN3XO/Y8RpVyGyZzUqVreD1g+XUnnfjex8+HaitdsBCM2/nKqHv0N7+TvkzDqVhlcfIjT/cr+kDxSXxod9w4aUMzxeVtlj/BbhN63rX8fKGUFGwfS9jqsTRQJpjF/0C/KOOpuaf3mjnqwpRzP+v37J2Etvp3XtMrKmzSVWs43qx++k5l+/wo0O26KvPysuLTvKbxF9RUoZPj5u/4rfOgYDHdtX0bZuOdt+exXVf/8x7ZvfY9c/foqdN5rsmd6iRdYhJxKpKt/rfW60neb3nyPv6BLqltzHqHNvJL1gOi0fvDjwP8TAkAk8VFxaluO3kL4gJcIJAeJ/sD8Awz6aKhHyT/0v8k/9LwDat7xH4+uPM/r8W6h78Y+0b36X3CPPomPryv3G6I3LHyM49wLEDqDR+J4TsdBYxwD/BAPKTLzVnEV+C+ktqdTD/wiY6reIwU5o3qW0rnmVinu+RN1L9zHqnD03RLGmGiI71pM9Yx4AweMvZsefb6Hl/efImXWaT4oHjCuLS8su8ltEb0mJdfji0rLT8aqTmN7d0Bu2AIeVLy5p9VtITxn2PXw8f/y9GLMbek8h8G2/RfSGYW944E7MDjhD33FzcWnZkE15NqwNX1xadghegUeDoa9Iw6s4NCQZ1obHK9ucMisRhgFjYXFp2ZCMOhq2k3bFpWUnAq/6rcMwbKkEZpYvLmnyW0gyDOce/id+CzAMa8YDQ27v/LDs4YtLyy7GS2phMPQnNUBx+eKSZr+FJMqw6+HjyQh/6LcOQ0owCviS3yKSYdgZHi/80aSrMgwUNxeXlmX7LSJRhqPhb/RbgCGlGANc67eIRBlWY/ji0rKPAf/2W4ch5agEppYvLhn0e4SHWw8/5GZNDcOC8cDVfotIhIQNLyJTEjnmF/F6cCV+6zCkLEMiz0IyPfxjXRx7tK+E9AFfYfjdsRiGDocUl5YN+nTn3YadisihwGwgJCKXdHopiJcNxHeKS8uCwGf91mFIea4CXvFbxMFIpEecCZwHjADO7/Q4Brim/6QlxSIgz28RhpTnE4M9FVa3PbyqPgE8ISInquprA6CpJwzJjQyGYUcu8J/AH33WcUCS2Um2XkS+gbe3/KP3qepVfS0qGYpLyyYB8/zUYDB04ioGseGTmeR6AgjhrXOXdXr4zX9gstkYBg+nxFeMBiXJ9PDZqjoYi4hd6rcAg2EfPgV8z28RXZFMD/+kiJzbb0p6QHFp2QRM5VfD4GNQ+aQzyRj+BjzTt4lIo4g0iUhjfwlLkEswt/OGwcdxxaVlI/0W0RUJG15V81TVUtUsVQ3Gnwf7U1wCmNt5w2DEAs70W0RXJDyGF5EFXR1X1SV9Jydx4qV8T/GjbYMhAT4OPOS3iH1JZtLu1k7fZwLHA28BC/tUUeIswITSGgYvZ/stoCuSuaU/v9PjTOBwYGf/SeuWU31s22DojvGDsepsb3rIbXim9wtjeMNg5+N+C9iXZMbwvwZ2Z8uwgDnAu/0hqlvCoZx1GfaojTr+lRfcOc4/nRPGr9Qp0xTL3OIbBhODLgI0mTH8m52+jwEPqqpfO4OOTRNnykzZNmWmtY0vBJ5ElYYqRqxb5s5qLnNOCC11j5zZRsaQyTVmGJbM8VvAviSV4kpE0oFD4k/XqGq0X1R1Rzj0NbzyzwdElVgLmWvfc6dWP+Uel/6Mc9zUHYwcN0AKDYbdjChfXNLgt4jdJHNLfxpwH1COF+wyWUQW+bQsN7e7E0QI5NI+6yR7FSfZq/he2n1E1d66UcdvMcMAwwByJLDUbxG7SeaW/v8BZ6nqGgAROQR4EDi2P4R1Q4+qd6aJM3mmbJtshgGGAWQOQ9TwabvNDqCqa0UkrR80HZxwSIA+2Y0kQmgc9XMvtF/lQvvV3cOAVSvdKbuHAVMqGVXQF20ZUpZBtTSX8BheRO7Fm6X/c/zQp4GAqg5saqlwqBDYPFDNRdXetkkLNseHAQXv6dTpZhhgSII3yxeXHOe3iN0kY/gMvLI6J+ON4ZcAv1HVjv6T1wXh0BnAswPaZidUaahmxLpl7mG7hwGHtJI5qNMaGXylHcgpX1zi+i0EkjN8DtCuqk78uQ1kqGprP+rbn3Doi8B/D2ibB0GVWCuZ61bqlKqnnbnpTzvHFVcwerzfugyDinHli0uq/BYByY3hnwPOAHZXyswCngFO6mtR3XBI96cMHCIEcmg/bJ58eNg860NuT/szMbU6DQPmjXtXp84ww4CUpgAYcobPVNWPyuKqarOI+DGjXehDm0kREHfSDKmYNMOq4POBf6JKYzWhdcvcWU1mGJCSDJr4j2QM3yIix6jqCgARORZo6x9ZB2WUD232ChGCY2k49gL7NS6wX9s9DPjQDANShkGz0pOM4W8EHhGRivjz8cBlfS+pWwZlJpFkMMOAlGPoGV5V34hXoZmJN0u/unNorYicqaoDMXuePwBtDDgHGgYsdw9rKnNOCC51jzykhaxcv3UaesSQvKUnbvD3D/DyjxiY5bIh38Mnwu5hwPn2Ms63l6GK00rGh+/rlKqnnePSnnbmTtnOGDMMGBoMmh6+z+rDi8jbqnp0n1zsQIRDGXjrmgYgptb2ci3Y/KI7J1bmnDDuHZ2WksOAzT+5EFzno+dFtz3J5h+dt9dzIH7Moui2vw+0xGfLF5ec1d1JIjILr0PdnZhV8bIyP4Y3hN7Nw6p6Waf3FQKrgLCq/vRgbSTVw3dD3/znODgp0bsnSkDcidOlYuJ0q4KrvWFA0y5Ca5e7hzb905kXfCkFhgEtVRV7mR1gy11X7vW8dsn9RKo2xp/5Ev+SkeB517J3FmbBq+u4753cJ9h7/uznwL8SaaAvDT8QDIpqtYMVEfLG0HDsefZyzrOX7x4GrH5fp+wcrsOAXX/avyy7ttTu9bzpw5ehfisA6RN6tO+qtyTqs646tC8f7A0ichGwEWjpSyGJUN6H1zoQgyI8cagggp1Dx6EnyOpDT7BW8x1vNWCvYcC7Om26i2X7rbXHOAlEdjdUfPRtZuER/SjmgCT6+32vi2MH3KAWj369DS8l9i2JNJDMfvg04Dq8bLEALwF3756pV9VLDvTePsTp/hTDwTjQMOAV97D6Pzaf3dyRZWm7RFQitmZFI5qmUelwM8mIpaMZFlFxsIihVgBXXBARF3XBtRRXXcB1LbEEtRxHbFcRHCuAo6DqSBqq4CpYKoCoYGOrApa4tuDGUy2KJRq/vqgqYoGgmqYujgCoboaL8FKu7R5S7l+YRD/62GjjskekcdkjeuYtP3pCLVctAcsFRVQEFEVV4kcRsMB1VQERxVFLQbBFBHWxcC1Pr4uAWopru2qjoiKKqEpMpBxKEvnzdJVQph0vqnUvRESA7wI/jwfBJXL9pHr43+L9t/lN/Pln4seuTuIavcX08H3M7mHARfYyMoOrX6pYf3p11ohDNFbQaFWNzJMdAUfF2qFRaXMC0Q7R9kyibZkqHZZmRVo1tz1GXtQhQ5UMcUnTGNiiUcumjQwiaZmCBlw0XXBsTSMmBEBs3Jil4gbEcmwXVxwcibmiMbHcKILnPxtVcVFXLFRsLCxLxVIFXNf5qJppVmZ6i4hoa1tHLgeuRrT7uMhrD+efNf+4ba5rgYqCoI6FoliuEhAX1JUAjopg2eoAqpYolqgIriuqYCECqFhigboioggqiiO2q2KBaqKf2x93cWzEAc4dDZwAXCoiP46f54pIu6redaAGkjH8caraeW/v8yIy0EksTQ/fj3xc6k/95ryXXgy+uUJnvjI/1jrajZ6ek28Hc6bZDZmkbbNrtS2nIi0ttDMtM6++tSEnp2NXZr6WZxQEamL51LSNjDW0BbE7XEZGmq3xkV1OQaTKyXTqHdVmXG22XFo0J9JObnub5kQ6yG6NiB2zrajmOlHNc6OaKzHNIepkqKO2OI4lAUc03VW1HRfLVbEcLJsAlp0O8Z69rT2SHkCiHNjsGn9NAXnnvfUzrpl3XqPYrkqaq5btYgVcxVaNWQGiAZuoHcCxAxq1bIlZNtGALR1YGsW7rY0IbgyIqOB9RWNAVJUAUcDBJootsURv6bvy4wG7blX9qBCLiISB5oOZ/UANHAhHRKap6oZ4A1MZeAOaHr6fuaOm9rQrTkpbsrRhaeHt90vrjoklde+MTtNY9D1rYtbo9BnBYzry6sZM3GE31G+0d9bHrPrcdDpmFKXtdPLyVm0LBqsb8oLVbkZ2c2Z9IJixwyqQbVLUvJXZsUomWLU6MrO9IyM92m7btLkd0hZrt1ujkXGRmtiEaJU7PlbDeKdGxjq19liq7Sxpcp1Amxuz27U9EKXNjlpttmO3WEo0YsV4FBdvjJweQ9PjP4Z3e+Dd5O82uYP3eReAdDdNnI12lmpAXQscwY1Zqq6FxkTFFVdciYkrUXHFtZWYpRoVNOpmiWiuiAbEIiCWGxDbssXCFouA2BIQW2yxrYBlS0AyBUi0BuNO9l+zj9G1T3s075Js5ZkXRGQj3i+tCBjY5Bemhx8Q7qvcOf+cyRPeuuYG++ib/vb3V057wzph05Tz3tySc3ze9prX25zIKsZkTkyfGTxOFmTNy7csO1gTadqwqb2qYUtNjbVVWgpd0SIAy4q1FebUbTk8+HJNMFgdzcmti6Wnt46QDGdy/YiRjRVMrNpCUdNWijoqmWit5bDsZvLyo6QV4GJJu1MjbbF6aXNapDUWkbaYK+2OldXRZo1zazPgqi57QEvAOchC8ZxLdOcLJ96X1iCWSEQ00CEE2oX0diSjXayMDqy8Nty8VpxgG2S3o2lOhhtws9WSLLXIFsgWkSwbyRSVLCuWlq2xQJZ2BLKJBTIlZmeJI5YlbnskwV99V6tQG/CiW/dlLy+oajiRBpLNWpvB3qG1HZ1e6//Q2nAoAEQwFWP7nXaRttMLJ25otqzDj9jkvv/1h908S+3xm4pLlm+e9LFix9m0K9a2tF3d+rl5aaMqZgbnlk/OOXRkmpUxW0SsVjqqN9vVG8ut6vZqq3FkhNhMhPQ9LbhOdnbj1ry8XVXBUFVrbm5tIDOzOd+2o5NFCAI0EqypYGLVVooatlDUUcFEqhmb1URwRIT0cYiEqq75JLphzb7yNev4syraXn9mAgB2uosTBfSjXvETV1z+/vGFWQ3jpTZWILWMkQY7X5rS8mjNziKSa+MEmy3sBtturrWtlhrbbt1l2x3Vth3dZdturW1pnW1bDZYVaLYkvQ1JT49gZ3SIldmO5LYTy2ujI9hKND3Ka3fc/cFXu/udi8h6YNo+h9fTdUo3S3sQNdeXkXYrVPWYPrnYwQiHdjEEd8wNRWotq+aMwolNUZHijIi2fOdBZ8WMCk5xxY5sKi5ZvmXyGVNd2tNjbS9/4EQ+nAJuUaadUz097+g1U/KOyMiy844QkUwAB7ejwqpbs8naWVth12U30z4N6frvmJ7esjMvr6YiGKpqysvbRVZWY15aWmSCiO4Vk37H4trokhfqbScWn9a3LMV1BSDj9LMikXfeSteGOgikIRML0U3rIH6vP/Inf16fWTClTlqdDmmLOdLmCB1OukTdHBwNoYy2cTNG0lg3VuobCqS2ZYLUtE6Qmuh4qXHHUccoaQyMkObMHNqzM4kEbdwRIuQAtIm01ltWY51tNTda1mPzSqtKu/t9xwu2Pgud/zGyE2//SOdj7cAUVd2R6N/yozaGVGgtQDj0ATCr39sxALApLbD5wonjs1VkDMCCle4bXyxziyxlrCtWdFPxecu2TD5jiivWRDe69u1Y28sd6jbMBdJsSWspzp39/vTgMbFQ2uhZIrLXxqdaad5UblVt32Lvok5axju4U5ED373ZdqQhN69mWzBYXRcMVsc++GDruF27Wsb89127Rnuvg+vC7NkZlJdHycgUbW5RjUaVE64vWfXaz8tmi22jjiOjHyirtscWjOZg61kxt9kbUjgN0hprkdZYVNoclXbHloiTSUxzcTQfGC3xMXUmHW2jpbF+LHWN46W2eaLs6sgg8vjNd9xz0JDX3YjIFmBy/GkD3lLdJrzs0PV4wTmHq+oHiVxvv+sPwR7+eeD0fm/H8BFvZGasuqpgbDHxhCfBFq35wZ+cdQX1XimlzsZXy56kbnN1tO3lVW5k9VRwJwMIEpuQPf39Q4JzG0dnTpxqiT1p33baidRtsXetL7eqW3Za9fkdxA5B9l+D7kxtbS133XUXImheXkakqak9/cc/mfHO/z24fUZxcSDjiSca065clE9Hu8sjjzSSlSXYAeGeeyaRnpvZvpOCym0U1m2lsGUrhe5Oxgfqyc9rI2uMizUOke73Jqi6RNxaaXNqpS3WJK2xdmlzYtIWQzrcNCz545ZvnvW/3V1GRP6JV49u9z+hGF6MgTfl71VrPhkYq6q7utXVVRtD0PAPAp/s93YMe/FEbs4b3xo98hi8XIYAXPyq+8onX3KPELwxt2f8kmVbJp/pGV9V3eiaFbG2V6LxXv+jSeJRGRPWzgwdVzk+a2pBwErvalIKFze6w6pfu8mq2rXNqs1olrapKoztfE59fT333HMPbW1tBINBRo4cSXNzM1OmTCESieiqVe/rHXdc8s4vf/nktNGjrciqD5tGLViQE/3a18Z2G98ew45WM3bHdibv2kJRy1YKYzuYkFbLyJw2skc52OMRSWTi+5M7Tp/Tba14EXkZr8hKBrAdeAO4MP59fvxrsaomGpu/fxt9aPi/Dki0XTj0S+D6fm/HsB+/zA8t/f2I0Cmdj42t1+13/tGpDrbtqaO2r/EB1G2uirYtXeVG1kwHd6/ePScwYtvM4NyNk3MPy8uwso6Qg5ioQVq3lltVW7752E+mr92yfnRWVpYNYNs2M2bMYP369RQUFHDxxRfz85//nNmzZ1NXV8fmzZuZN28e1dXVzJ8/n8mTQ7vy8mq2B4NVDXnBXZqd3ZCbltZRADpBDjKs6IyL5dQwauc2Ju/aSlHTVgq9pUdGZ7eQMypGYDzeRPfJO06f020dRhHpPFxtAyrx9tI34s1bNQLZqtrj9GjdGl5EDmpiVf1rTxvvEeFQKfDDAW3T8BHXjx390gs52XuX6lbVq592l5z5ts6TTjvDXLGi5UXnLt9ceFbxR8ZXdd3I6hWx9lccdRuPZZ+l4XQrs25a3pxVU/OOsnMCoSPi8eL7sWzrO+SkZXHpA1/R9ljEW2sXOPeccyObt2zOyM3NZePGjWRmZuK6LpMmTWLFihUUFRVx+eWXH/Dns6xoS25u7ZZgqLo2mFcdy86pz8jIaB0j4haKHDiuvSsUtI6R1ZVMOOb6hU9sP9i5IjIReBn4PfAtvGSx7wN5wKOqulhESoGRqvq1ZHTs1U4Chv9D/NuxeBlqn48/Px14cYBi6PcQDl2Ctz/Y4BP/MaHg5bUZ6Sfve3xqpa4L3++4mdG91407Gb9ILXv3hBTqNu2Iti5d7UbXzACduO/1LLHbi3JmrZwePKY9P33soSLWmM6vb22o5NMP3UyancYTV/yWyx+6ib995rfO5Y9+tXVna40zumBs6/jCCVkr3l6Rf9VVV/HYY48xf/58CgqSz0ch4saysxu25AWrq4LBqo7c3LpARkZzvm3HCkU42BbkDiD7Yws3HDRoLG74tUATXm8ew9tM8yjeuL4Q2AL8p6rWHug63f4cSeSlfxK4RlUr48/HA//tg+FnAqsHtE3DXkQhetbkie/tCtj71RUMOBq5+TH31WM26AJv58keXLFi5UXnLNtcePbexld13ciHK6Ltrzq4jXPpOorMLcia8sEhweNqx2YVFttiF332sVJe2LAcERidPZJjJs6ivG47E4PjmJI/iY21W/nDpYspffanLc9vWi6jRo1svuxTl9fEcA5Behaptj+qGRktlXnBXTuCwaqmvLwaycpqCgYCHZNEGA18+LGFGxJaVRKRG4A7gVa83v4w4ERVresbrckZ/n1VPbzTcwt4r/OxASEcsvH2/vZ44sLQe5pFmhYWTtzeZlldbjCfs8F972uPuvkB96Mlpo84kPEB1G2sjLYuWeNG1x0COqGraz/0+rusqqh2ctKzO7LTsu0l1zyQ/sOX7pYXNi5n9tgZ/OK8b3Lrv37EpGABa3ZtQlU5dOxUbjhpEQBRYi3brdq1G+2qxh1WXU4rkRkIoT74texFINBel5NT//BNN939he7OjS9ZPgZk402CTsTrUL/Rl5qSSYf0oog8LSL/JSKLgDLghb4UkxDhBgfYL7TKMLDkqub9Y1tlvq26ravX35lmHXnVTXb+hgJe3vc1S93A1PKyk09deuP44vKyl8WNbdn9mljB8em5552WMeKGcYHss19H8t5gnzDSuVMmcc2px9kuHdltTkPGwxt+ufOZ9S81/P3Ku96KuTH3H6ufJ+Y6vLz5TXY0V/Pbi77LmupNbKr1kmCkEcgpdscevTB6+Kmf6jhl7uc6FuZd2HHcuiNjRUvy3ZxXLJUt9AGxWGZ+Q0NBdYKnX4lXeDITL5o0nT1b0fuMZLLWfjk+gbd7lvZ3qvp4XwtKkA/w6m4bfGSc44z7v4odGz4xoaBeRfbbxtmeLrlf/2zg5NPfdV+/9l/uFEvZawzuGf+fJxdvfipWXvTxlzcXnl2oVqAQQMSyAxmzjw9kzMZ1GipibUvWutH1M0HHTxszitqWPRXO2t2Wgoi28sSWu47d0LAq+rNXVjUfVTC7+bIjzh1x/7v/yAawxKIt1nWyDEGsMRqcMSYWnHF8PIq1hY4qLzS4qqPaahwV9YYB6V1e4OB0ldSiK5YBO/DmyTrwJu2e6UF7B6XPluUGlHDom8AP/JZh8FiSlfnul8aNOTS+BNUloWbdded9zoYxjZxwoHNcsWKbC89eVl708clqBYr2fV3VdZzIqrdiba/sE/CrAAAXGElEQVTKvUtemLt6R7UA5GVmUDhqBDXNrYzIzmRMbg5VTS1cc8oJkbJ3ymvX7KwKHllwqPU/F32/xynSYjjtXmhwVX2FXZvVQsd0JKEci9PD4fCGRNoQke/i5arLxLu1n9zXxVqTGcNfgpeKeixeJJAAqqrBvhSUEOHQecA/BrxdwwF5IC/3tR+Oyp930FBV4BNLnJf/4xU9Srzlpi45mPH/8uJPeH/zMjLSMmJoW+Tr5y5oePLdVePX7Khmwoggl58wh4ffeI/87Ex2NjbjKkwYkccZs2bo2MzCVYeE5u4qyJwyybYC+25SSQpFtU6ayzfZ1du2WLuok+ZJruiUfU6rC4fDSSdejZdmX9Hd3vaekIzh1wPnq+qHfS0iacKhEUANyc1BGPqZH4zKf+mhYN6p3Z1XUKtb77zPqcttP/iwzDP+WcvLi86ZtNv46yveIyMtk3v//QMCdho3XfBz51f/+ErrjWeetOb+114+dvaEcbK6spra1lZUlS8tPIm/vLaCsw+fyZi8PUv6wbRR5YeEjts8OXvmRzv8evvztxGp9UKDq1p3Wg0jo8Q23f7d8EXJXENE0oEKYLaq7uytpn1JZj/8zkFhdoBwQz3h0DtA/4fyGhLmWzV1p5anpb20PCvzoKbfMVImf+5Ge+IXytwXT1upJwldj40tdQNTNj81v2jLM7HNhWe9Ul50zuTpE44srGnas0nMsgK2HcjNywh9fm5E17e9uGZrR9GonLQTpk7OeW29N/cmIkSdvVMpNEZrit/c9VTxmzyFt8PvmDVT8g7fa4dfsmSRPnKmM+H4mc5HiwuP9OAy5+D17n1udkiuh/8lXjaOv+FNKgA+RNrtJhz6f0C3e4wNA4uCnj9p/LLNaWknJnL+9O265vYHHCsjxozury3O5sKzl928aukxayreyUIgmJVP8djDqG6sYGTuWMaEJrGzfotes/Cy5Q8uuWvK5l3V48aPyOPT8xLbyGlLWsuU3MNXTg8e7QTTRs+WLiYjk+D0SYtPebG7k+Jt/B44HC8H/a9V9Vu9aPfAbSVh+D90cVhV9aq+lZQg4dD5wICXEDF0TwQ6FhZOXN1g20d1fzYEYtpx2yPusiPL9ZR9g3W6Yls06lxZuavj21c8XL3vGP/+l37KgtkXsrV6LR9ue4sxwdGNH589YYUb3TQLdOyBrtkVe+/wmzTNEmu/aMCD0AbkT1p8SreTbiJyH7AUeADYChytqn2yNLhfW0Nylh4gHAoBtZhx/KCkwbLqF06eWBOxJOHJsblr3Xdu/qs7xlb2MlZlNMrXKyvZ5cQQ4Ky8PJ5tamJBTq4+1e60FRQc5V511u25W3et47FXf8PswhN4f/MybrrwF9z77+9z3nGfZUxwfMzpWPlmrH1ZOtp6ND3ImjQqY8KamaHjdozPmlYQsNK63OHXiWcmLT7l7O6uKSJB4F1gak8y2CRLMnnpM4HPAbPplHvLvx6+oYFw6G28xACGQUbIdUc8sb2ipWTShJ2uSELVU988xJpz1Y3S+L2/OK8UVTN/9/GACF8bO5ZZmZm0uA4XbdqEJcI77W3y78Ki7Fsq3tHIiv95+/Ftaw51XTdr3iEfZ2X5qwCIWERiHYjYgUDmnHmBzDm4Tu2WWNtLG91o+WzQMQcUtA81HRUzX616YiZAbmDEtkNCx20szDk0mO7t8Ns3VPfpBC87FagG/iAiR+Hteb9BVROqJJMsyfSOf8Ybw5+NV4RiEl6gv5+UdX+KwS8mxZyJf67cWY9qwp+TtkwJ3np1YP7vz7KWud4dHGMCAWZlen3M7Tt2UO04bI9Gea+tjUfr64ioKzXlLx4dql+XedHo8R8GM3M3FY+bxR2PXI0gTBq1902GZY8sTM+9+LSMEdePCGSdvgzJXkGStRGbY/WTVtQ8u+BvW349529bft2wsnbJK83R+mWqujsiKNHPZgBv8vm38YxRLUC36bB6SjJj+LdV9WgReU9Vj4xXonlaVRf2l7huCYcOB1b61r4hIZ7KyX7r1jGjjsT7zCRMfpNW3flHZ/OoZo7bfWx7NMJntmzh78VTeKi+niebGpmXnc1VI0dx+45KfjNpcnxy78xlm4pLJqgV2HdtvEtcp2ZzrPWlcjdWPgtIuNffF1sCbUW5s5+5+Pd3JLQcJyIFwDJVLY4/PwUoVdWEStUkSzKGf11VjxeRJcAX8cIAX1fVqf0hLGFMjrshwe9CwZd/PXLEfltqE+FTLzhLLlymx7a6bs6iLZu5dtRozszbO27n2zsquXxEPqva23mltYWZGRlcO2qMs2XyGcs3TjlvfKLGV3UiTse7b8Xal2eibXPoWYbkxTc/9OTXEz1ZRJYCV6vqmnhBiRxVvbUH7XZLMrf0v4vv6PkW3uz4KrzIO7/pyVqnYYD5fEPjySXNLS/15L0PnG4vuPGz1q5rKrY2nxcM7Wf2Ve3tABSnp/NEYwM/nzCRdR0dbI502EVbnz3ptCU3Fk7b8LdXxY1u6q4tETs9kHnMiZkjrjs6PbhosxUoegkvyCsZuv1MikimiLwer940Aa/mw3vAHLwtsv1CX6a4WqSq9/XJxZIhHJqFt5nGMAT41PhxS1ZmZiS1C0xV2f6/27GzLf1h3riX5q/S+dKpqup127by3YICMsXium3buL+oiFsrKvjcyJEcmrknhkYRd8vkjy3zevy0hHp8r30n4nS882asfXk22j6nm9PX3vzQk93N4O8uBpkTLwSZhrf//QZVXZaorp7Ql0taN/ThtRIn3LAKY/ghw58rd84viMVeT+Y9retaqX+1nubVLXL9m5tOm1e70f1XS1MlwL+bmjg8M4uxgTSCts1RWVlcuMnryDubHUBQq2jrv086bclNRdM2PJ5Qjw+7e/1jT8oc8cU56cErN1mBwpeITyh2wV8SuaZ6NMefpsUf/b4sN/Ty0ndFOPQt4Pu+tG1ImjaR1tMLJ25qsazZ+76mrrIhvIG0/DSKbipi691bad/WTt6cPAou9VJTVT1RRc74jMiP1+W+NmuLLpAeViLyevyFyzZOOb9ArbSk5qJUYx1Ox9tvxtrfyEXbdwcYKTD15oeeLE/kGvGlvLfwKsv8t6reloyGntCXPbyfETz34uUAMwwBslSzy7ZWjAuobt73tZpnasiY4O2ybd/qjc1n/GAGrWtbcVodovVR2ja2kXN8KP27n7ZP/dnF1tuOUNkTHV6P/9xJpy25qXjahr++Km50Y8LvlUBGIPO4+ZkjvnhUet5nNlqByS9B4G+Jmh1AVR1VnYO3xH28iPR79qi+NLx/9d7CDRXAE761b0iaUa47+rHtlSKdCipEa6M0vdtE/oJ4gRobNKqoq2hMwYKqv1Yx9pI9EbLLD7WOufoGO3vbKLpNA30gOht/+vrHXrXcaEL713djBcZMTc/7z1Mz86+/vyftq2o98CJessp+pdtIOxE56AYVVf1Z/Nse/8L7iN8A/+GzBkMSTI3GCn+/o+qDzxWMzUYku/KBSgouK8Bp83a2ZU7IJG1kGhtu38CIk0YQ2ekVYc0q2rsYTUuWhL76+cD8c193X130nDtLoEcbXgS1Crc9f9LkbS+4Wyed/urGqReMc620REODk+p0xCvdFVXVehHJAs5gAFa9EklTfXv825nAcezZsHI+sERVr+4/eUkSDr0HHOG3DENyPJ6b8/qN29LmNq9stiZcOYHmD5upeaqGopv2Tnqz+eebmfBfE6hbWkf71nZyZ+cy8rS980uMatQdd/7R2Z7f0vuQa0XcrZNOX75xygVjXbtb43/rS3cvvCPRa4vIkcB9eBl6LeBhVf1eL+Qm1m4SgTfPAP+h8TBJEckDHlHVfr8NSZhw6CrgHr9lGJLnpGedrW982DFZLEGjitPuEDw2yORrvaS2jSsaad/STmheiKrHq5h83WQ23rmR4puLsTL2GZmq6pXPuUtL3tDjhIPXpkuEBIzfChR+6e6F3a7Xi0g5Xki6A8RUdW5v9SVDMmP4QrxsmruJAMV9qqb33I9XXtcwxHj1THvyF35Q+OLM/zeTSddNIvew3I/MrjGl5tkaRp8zGjfi7pktUu+1/RCRP51hL7jlc/aOtnRW9VZb/Fb/xFOX3jRlxvpHX7Oc/cb49yZi9k6crqpzBtrskPzmmddFJBy/zV+Od0syeAg3dAA/9luGoWf8umrXaTMikf3SWtc8V8OI+SOwMiwyJ2eCwrpvrSN7RjZ2zoHrSWwdK1M+e5N9yGuHyovaB6s4glqTt73QyfiR9XiVXX/S22sPFEmtw4vIMexJU71EVd/uF1W9IRzKBNbhLXUYhhhRiJ5ROHFlrW33afqy2ZvdD77xkJud5pBwhF13KGhlwUk//NiL93wz0feIyCagzns7/6Oqv+srPQm1P2QTYByMcOjzwP/4LcPQM5pEGhcWTqxst6xuQ1STIT2qrd9+0Hlz5vY+K/AQAWYctvrDhLPTiMgEVa0QkbHAs8BXVHVJH+npluGaLeZeYL3fIhLBcZWj/6eZ8x7wtlFvqnM54ffNzPh1M5c92krE8f4h/3p5hMN/08y59+859vKWGF99ut037f1FnmrwH9sqQ7bqQSuuJkskTbK/fWVgwa8usN50pU/mev43GbMDqGpF/GsV8DhwfB/oSJjhafhwQwy4vdvzBgG/XB7hsNF7/gy3/budm+ZlsO4rueRnCvesiALw+7cjvHddDkcXWDy9Poaq8v0lHXx7wfAssVfgOAUPVuxoR7Whr6/98mxr7tXX2+mV+bzWi8s0kWQxFBHJia9uES+DfRZeSegBY3ga3uNBEi/z4wvbGl3K1sW4+hgvS7Oq8vwmh0tnefFQi45K429roh+dH3WgNQpptvDn96KcOz1AfpZ/AY79zWGR6LS7dlaX08fVVwCasyX/hi8ETrz/NOtVhZ78U7njsNUf7uj+tL0YB7wc3xL7OlCmqk/1oO0eM3wNH25Q/NrBlyA3PtXOj8/IxIp7tqZNGZEJgfiBSUGL7Y3e7fstJ6Yz754WqluV+ZNt7ns3yheP60mps6HFqW3tR91WW7+CfppseuJE66QvX2e3NGSTzAT0RuAXybalqhtV9aj4Y7aqJhyo01cMX8MDhBteBLpKr+07T66NMjZHOHbCnmWlrj7Suws3feaodN6+Npe/XJLFz16LcP3x6fxrfYxLH27lpqfacYfj5GucKxqbTvzPpuZ+m9iqHiETrrnenvPUMfKSQiKTItcftvrDhO46RMQWkbdF5MleyuwThrfhPW7Bywo6qHhli8Pf18Qo/kUTn3y0jec3xbjx6Xbq2yHmeubd1ugyIW/vW/aKJpc3KhwuPDSNHyzp4KFLs8gIwHMbna6aGTZ8p6bu1OPb2nuUMSchROTes+1Tb/usvb09jdUHOfOJw1Z/mEzy1BuAwVGxiVQwfLihlkFYoeaHZ2Sy7at5lN+Yx/9dmsXCKQHuvySb06fYPLrKixG5790oF87cO+/jt5/v4PunexN1bTHvDsASaI0O3x5+N/+7o+qUydFobybauqW8QKZ99iZ72hsz5EXdpy49Xgnn6xO9lohMAkrwqsoMCoa/4QHCDX8B/u23jET40RmZ/Oy1Dqb/qomaNuVzR+8x/NuV3ufv6PHeMOBzR6dxxG9bWFHp8PHpyZQJHJpYYD2+vfLooOP062SsY0vaTy61T7vjMmtVzKLznv1bk1yG+wXwNcDtW4U9Z3gG3nRFODQNL6V1rzdTGPylJ1VtekpmRJu/84Dz9vRK2g5b/WG3lWR2IyLnAeeq6hdF5DTgFlU9r9+EJkjqGB4gHLoa+F+/ZRh6z9ZAYNt5k8anJVrVppfUztiuR/71G+8nHAgkIj8EPoMXw58JBIG/quoV/aQxMV0pZXiAcOh+4FN+yzD0nncz0tdcMX7cBOLBLP3IxSsXrfxbT988mHr41BjD7821wFq/RRh6z1EdkZk/rq5Zi2p/5jO8uzdmH2yknuHDDc3AJ0hsvdUwyDmnpfXYL9c39Fcu9w/ogxUeVX1xMPTukIqGBwg3vAvc5LcMQ99wbX3jyef0sKrNQWgCLlu5aGVbH1/XV1JvDN+ZcOiPwCK/ZRj6hssnjFv6fkbGKd2f2S0OcP7KRSv/1QfXGlSkZg+/h2uA5/wWYegb/lKx86RxsdgbfXCpG4aj2SHVe3iAcCiIV9fLZLsdBrSKtCwsnLi5xbJ6WlH4VysXrRzUm656Q6r38BBuaATOBfo02YLBH7JVc57cVjGmq6o2CfAEw3xuxxgeINywDS/muclvKYbeM9pxxzy6vVJFNZlMsi/iTdINmjDY/sAYfjfezP3FwLCalU1VpkVjxb/bUVWJaiJ/z7eAC1YuWnnQLa8iMllEXhCRD0XkAxEZcrf+Zgy/L+HQQuAfQLbfUgy956+5Oa/fPnrksXiVWrtiDXDKykUru91CLSLjgfGquiKequot4CJV7XXu+4HC9PD7Em54Hq+oX3N3pxoGP5c0txy/qLHpQHUPVwNnJGJ2AFWtVNUV8e+b8Pa5T+wbpQODMXxXhBuW4iUYbPRbiqH33FJbv2BBa9uL+xx+A69n39aTa4pIMXA0XkGWIYMx/IEIN7yGV9Gz3m8pht5z187qU6dFIrt7+n8DC1cuWrnrYO85ECKSCzwG3KiqQ6pTMGP47giHjsAb0xd1d6phcBOF6CcmFty9Pj39lpWLVka6f8f+iEga8CTwdKdS6UMGY/hECIfG4hUNOMlvKYZe8RPgtnhG46QREcGrp1irqjf2qbIBwhg+UcKhDOB3wJV+SzEkjQN8mXDD3b25iIicDCzFy5y0e73+G6r6z17qGzCM4ZMlHLoNuBMz/zFUqAAuJ9wwYPXbBjPG8D0hHLoA+COQ77MSw8F5FriCcEOV30IGC6aX6gnhhr8DR+KFYxoGHw7wbeDjxux7Y3r43hAOWXiFLr4PDP+6T0ODSuBT8apDhn0whu8LwqGjgQeAQ/2WkuI8ANxIuGHQVRoaLBjD9xXhUBawGPgyZqg00JQD1xFuGNBKrEMRY/i+Jhw6FrgbmOu3lBTAwavu8h3CDa1+ixkKGMP3B97Y/vN4Y/vRPqsZrrwFXEu44S2/hQwljOH7k3AoHwgDXwSGf/G3gWE93gz8Qz2NmEtljOEHgnBoOvBN4AqM8XtKBfA94B7CDf1ZeGJYYww/kIRDU4Bv4KXGTuvmbINHHfAj4FeEG0w2ol5iDN8DRORe4DygSlUPT/oC4VAh8HXgKsz6/YFYB/wK+GO8WpChDzCG7wEisgAvI86femT43YRD4/FMfzVQ3Cfihj7PAL8E/mXG6H2PMXwPiWc8ebJXht+NN6t/Jt7M/gWk3ji/Hvg/vNv2D/0WM5wxhu8hfWr4zoRD44DPApcBc/r02oOLZuDveEZ/mnBDjxJSGJLDGL6H9JvhOxMOFQEXxh8LGPo9fxvwLzyTP2km4QYeY/geMiCG74y3pn8uXsGMk4HJA9Ju74jiJXl8HngBeI1ww0Fzvxv6F2P4HjLght+XcGgScCJe2q2T8DKo+r3UV4WXDeZNPIO/TLihxV9Jhs4Yw/cAEXkQOA0vbHYncLuq3uOrqHAoE5gFzOji0ZfhvS7ez7wFeB/P4CuB983e88GPMXwqEA6FgHF4GXrygRGdvo6In+UAsfhj9/cRoBaoBnYBO4AKE+k2dDGGNxhSCLNv22BIIYzhDYYUwhjeYEghjOENhhTCGN5gSCGM4Q2GFMIY3mBIIYzhDYYUwhjeYEghjOENhhTCGN5gSCGM4Q2GFMIY3mBIIYzhDYYUwhjeYEghjOENhhTCGN5gSCGM4Q2GFMIY3mBIIYzhDYYUwhjeYEghjOENhhTCGN5gSCGM4Q2GFMIY3mBIIYzhDYYUwhjeYEghjOENhhTCGN5gSCGM4Q2GFMIY3mBIIf4/EpR/k2mQikUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# The D&O variable is a list of lists, and 3 items in a list = 1 director/officer.\n",
    "# Not all entities list any directors or officers.\n",
    "def char_count_excluding_blanks(word):\n",
    "    # Empty lines often return '\\xa0' instead of being blank. Fix that.\n",
    "    return re.sub('\\xa0', '', word)\n",
    "\n",
    "def count_people(df_row):\n",
    "    # Return how many directors and officers are listed in the `d_and_o` variable\n",
    "    if len(df_row.d_and_o) > 3:\n",
    "        return int(len(df_row.d_and_o) / 3)\n",
    "    first_list = [len(char_count_excluding_blanks(item)) for item in df_row.d_and_o]\n",
    "    char_count = sum(first_list)\n",
    "    if char_count:\n",
    "        return 1\n",
    "    return 0\n",
    "\n",
    "df['d_and_o_count'] = df.apply(count_people, axis=1)\n",
    "df.d_and_o_count.value_counts().plot.pie(autopct='%1.0f%%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    43658.000000\n",
       "mean         0.134752\n",
       "std          0.341462\n",
       "min          0.000000\n",
       "25%          0.000000\n",
       "50%          0.000000\n",
       "75%          0.000000\n",
       "max          1.000000\n",
       "Name: gt_one_officer, dtype: float64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The number of officers listed is strange. It doesn't follow any discernable distribution,\n",
    "# and my guess for that is that different companies have different definitions of what it means\n",
    "# to be a director or officer. One company listed 120+ D&Os, while many only list senior officers.\n",
    "# There are several ways to explore this. For now I'm going to split it up as whether an entity\n",
    "# lists 1 or >1 D&O. It may be worthwhile to also have a variable for >2 & >3 D&Os, but I imagine\n",
    "# the variable's value falls of quickly around that point.\n",
    "df[\"gt_one_officer\"] = np.where(df.d_and_o_count > 1, 1, 0)\n",
    "df.gt_one_officer.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The dataframe is too big in terms of memory. Shrink the dataframe's memory requirement."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 43658 entries, 0 to 43683\n",
      "Columns: 443 entries, Comments to gt_one_officer\n",
      "dtypes: float64(2), int32(2), int64(397), object(23), uint8(19)\n",
      "memory usage: 173.0 MB\n"
     ]
    }
   ],
   "source": [
    "df.info(memory_usage='deep')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 43658 entries, 0 to 43683\n",
      "Columns: 443 entries, Comments to gt_one_officer\n",
      "dtypes: bool(415), float64(2), int32(2), int64(1), object(23)\n",
      "memory usage: 57.6 MB\n"
     ]
    }
   ],
   "source": [
    "# Convert the data types to types that use less memory\n",
    "for var in new_variable_list:\n",
    "    df[var] = df[var].astype('bool')\n",
    "for var in entity_dummies.columns:\n",
    "    df[var] = df[var].astype('bool')\n",
    "df.info(memory_usage='deep')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split into 2 dataframes: one where there was information given for their D&Os, and another where there wasn't"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "With officers: (23404, 443)\n",
      "No officer info: (20254, 443)\n"
     ]
    }
   ],
   "source": [
    "df_w_officers = df[df['d_and_o_count'] >= 1]\n",
    "df_no_officers = df[df['d_and_o_count'] == 0]\n",
    "print(\"With officers:\", df_w_officers.shape)\n",
    "print(\"No officer info:\", df_no_officers.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 2: regression time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "413\n"
     ]
    }
   ],
   "source": [
    "vars_to_exclude = ['EntityName', 'ConsentFlag', 'Comments', 'EffectiveDate', 'EntityType',\n",
    "                   'IDNumber', 'InactiveDate', 'NIACS', 'OrganisationDate', \n",
    "                   'PrincipleCity', 'PrincipleZip', 'PrincipleState', 'PrincipleCountry', 'PrincipleStreet',\n",
    "                   'ResidentAgentFlag', 'ResidentAgentName', 'ResidentCity', 'ResidentState',\n",
    "                   'ResidentCountry', 'ResidentZip', 'ResidentStreet',\n",
    "                   'poisson_d_and_o', 'd_and_o_count', 'officers_listed',\n",
    "                   'stock_info', 'd_and_o', '(ma)**', '(de)**', '**merg', '12/31/88',\n",
    "                   'log_time_in_business', 'time_in_business', 'log_censored_tib',\n",
    "                   'InactiveDate', 'OrganisationDate',\n",
    "                   'Foreign Corporation']\n",
    "                    ## **merg is definitely a variable to come back & look at\n",
    "\n",
    "explanatory_variables = list(set(df.columns) - set(vars_to_exclude))\n",
    "explanatory_variables.sort()\n",
    "print(len(explanatory_variables))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "With, w/o officers: 403 408\n"
     ]
    }
   ],
   "source": [
    "## There will be some variables that aren't in both datasets. Find them!\n",
    "explanatory_vars_w_officers = [item for item in explanatory_variables if len(df_w_officers[item].unique()) > 1]\n",
    "explanatory_vars_no_officers = [item for item in explanatory_variables if len(df_no_officers[item].unique()) > 1]\n",
    "\n",
    "print(\"With, w/o officers:\", len(explanatory_vars_w_officers), len(explanatory_vars_no_officers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_test_split\n",
    "X_train_officers, X_test_officers, y_train_officers, y_test_officers = train_test_split(\n",
    "                                                    df_w_officers[explanatory_vars_w_officers],\n",
    "                                                    df_w_officers['log_censored_tib'],\n",
    "                                                    test_size=0.2,\n",
    "                                                    random_state=5)\n",
    "\n",
    "X_train_no_officers, X_test_no_officers, y_train_no_officers, y_test_no_officers = train_test_split(\n",
    "                                                    df_no_officers[explanatory_vars_no_officers],\n",
    "                                                    df_no_officers['log_censored_tib'],\n",
    "                                                    test_size=0.2,\n",
    "                                                    random_state=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove 'is_tib_censored' from the test/train data in a way that keeps both sets usable\n",
    "X_train_officers2 = X_train_officers[X_train_officers.columns.difference(['is_tib_censored'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test on a very reduced dataset:\n",
    "smaller_officers = copy.deepcopy(df_w_officers)\n",
    "X_train_officers, X_test_officers, y_train_officers, y_test_officers = train_test_split(\n",
    "                                                    smaller_officers[explanatory_vars_w_officers],\n",
    "                                                    smaller_officers['log_censored_tib'],\n",
    "                                                    test_size=0.2,\n",
    "                                                    random_state=5)\n",
    "\n",
    "#smaller_vars_w_officers = [item for item in explanatory_vars_w_officers if len(X_train_officers[item].unique()) > 1]\n",
    "smaller_vars_w_officers = [item for item in explanatory_vars_w_officers \\\n",
    "                           if len(X_train_officers[:15000][item].unique()) > 1]\n",
    "\n",
    "\n",
    "\n",
    "### WHAT I'M DOING HERE DOESN'T MAKE SENSE.\n",
    "## Why would the random split be the same for the first X_train_ vs the 2nd?\n",
    "## Does the same random_state guarantee the same split? Seems it should...\n",
    "## Why is there often an issue with 'Bank', the first item?\n",
    "\n",
    "\n",
    "\n",
    "var_size = 15\n",
    "if 'is_tib_censored' not in smaller_vars_w_officers[:var_size]:\n",
    "    tiny_vars = smaller_vars_w_officers[:var_size] + ['is_tib_censored']\n",
    "else:\n",
    "    tiny_vars = smaller_vars_w_officers[:var_size]\n",
    "#tiny_vars += smaller_vars_w_officers[38:40]\n",
    "\n",
    "X_train_officers, X_test_officers, y_train_officers, y_test_officers = train_test_split(\n",
    "                                                    smaller_officers[tiny_vars],\n",
    "                                                    smaller_officers['log_censored_tib'],\n",
    "                                                    test_size=0.2,\n",
    "                                                    random_state=5)\n",
    "X_train_officers2 = X_train_officers[X_train_officers.columns.difference(['is_tib_censored'])]\n",
    "#X_train_officers2 = X_train_officers[smaller_vars_w_officers]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(18723, 20)\n",
      "(18723, 21)\n",
      "Bank\n",
      "Domestic Non-Profit Corporation\n",
      "Domestic Profit Corporation\n",
      "Foreign Non-Profit Corporation\n",
      "Insurance\n",
      "Loan and Investment\n",
      "Non-Profit Producers' Cooperative\n",
      "Professional Service Corporation\n",
      "Transportation\n",
      "accessori\n",
      "account\n",
      "acquir\n",
      "acquisit\n",
      "act\n",
      "activ\n",
      "administr\n",
      "advanc\n",
      "advertis\n",
      "agenc\n",
      "agent\n"
     ]
    }
   ],
   "source": [
    "#X_train_officers2[pd.isnull(X_train_officers).any(axis=1)]\n",
    "print(X_train_officers2.shape)\n",
    "print(X_train_officers.shape)\n",
    "\n",
    "for item in X_train_officers2.columns:\n",
    "#for item in explanatory_vars_w_officers:\n",
    "    print(item)\n",
    "\n",
    "### Try skipping this variable to confirm the problem is with the variable.\n",
    "### list_[0:35] + list_[37:40]\n",
    "\n",
    "#print(smaller_vars_w_officers[35])\n",
    "#smaller_officers.automot.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\aubre\\desktop\\github_math\\git_math\\lib\\site-packages\\statsmodels\\duration\\hazard_regression.py:811: RuntimeWarning: invalid value encountered in true_divide\n",
      "  hess += m*(xp2 / xp0  - np.outer(xp1, xp1) / xp0**2)\n",
      "c:\\users\\aubre\\desktop\\github_math\\git_math\\lib\\site-packages\\statsmodels\\duration\\hazard_regression.py:697: RuntimeWarning: invalid value encountered in true_divide\n",
      "  grad += (exog_s[ix,:] - xp1 / xp0).sum(0)\n",
      "c:\\users\\aubre\\desktop\\github_math\\git_math\\lib\\site-packages\\statsmodels\\base\\optimizer.py:271: RuntimeWarning: invalid value encountered in greater\n",
      "  oldparams) > tol)):\n",
      "c:\\users\\aubre\\desktop\\github_math\\git_math\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:877: RuntimeWarning: invalid value encountered in greater\n",
      "  return (self.a < x) & (x < self.b)\n",
      "c:\\users\\aubre\\desktop\\github_math\\git_math\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:877: RuntimeWarning: invalid value encountered in less\n",
      "  return (self.a < x) & (x < self.b)\n",
      "c:\\users\\aubre\\desktop\\github_math\\git_math\\lib\\site-packages\\scipy\\stats\\_distn_infrastructure.py:1831: RuntimeWarning: invalid value encountered in less_equal\n",
      "  cond2 = cond0 & (x <= self.a)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "        <td>Model:</td>             <td>PH Reg</td>      <td>Sample size:</td> <td>15000</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <td>Dependent variable:</td> <td>log_censored_tib</td> <td>Num. events:</td> <td>11197</td>\n",
       "</tr>\n",
       "<tr>\n",
       "         <td>Ties:</td>             <td>Breslow</td>           <td></td>         <td></td>   \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "                  <td></td>                  <th>log HR</th> <th>log HR SE</th> <th>HR</th>   <th>t</th>  <th>P>|t|</th> <th>[0.025</th> <th>0.975]</th>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Bank</th>                                <td>nan</td>     <td>nan</td>    <td>nan</td> <td>nan</td>  <td>nan</td>    <td>nan</td>    <td>nan</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Domestic Non-Profit Corporation</th>     <td>nan</td>     <td>nan</td>    <td>nan</td> <td>nan</td>  <td>nan</td>    <td>nan</td>    <td>nan</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Domestic Profit Corporation</th>         <td>nan</td>     <td>nan</td>    <td>nan</td> <td>nan</td>  <td>nan</td>    <td>nan</td>    <td>nan</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Foreign Non-Profit Corporation</th>      <td>nan</td>     <td>nan</td>    <td>nan</td> <td>nan</td>  <td>nan</td>    <td>nan</td>    <td>nan</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Insurance</th>                           <td>nan</td>     <td>nan</td>    <td>nan</td> <td>nan</td>  <td>nan</td>    <td>nan</td>    <td>nan</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Loan and Investment</th>                 <td>nan</td>     <td>nan</td>    <td>nan</td> <td>nan</td>  <td>nan</td>    <td>nan</td>    <td>nan</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Non-Profit Producers' Cooperative</th>   <td>nan</td>     <td>nan</td>    <td>nan</td> <td>nan</td>  <td>nan</td>    <td>nan</td>    <td>nan</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Professional Service Corporation</th>    <td>nan</td>     <td>nan</td>    <td>nan</td> <td>nan</td>  <td>nan</td>    <td>nan</td>    <td>nan</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Transportation</th>                      <td>nan</td>     <td>nan</td>    <td>nan</td> <td>nan</td>  <td>nan</td>    <td>nan</td>    <td>nan</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>accessori</th>                           <td>nan</td>     <td>nan</td>    <td>nan</td> <td>nan</td>  <td>nan</td>    <td>nan</td>    <td>nan</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>account</th>                             <td>nan</td>     <td>nan</td>    <td>nan</td> <td>nan</td>  <td>nan</td>    <td>nan</td>    <td>nan</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>acquir</th>                              <td>nan</td>     <td>nan</td>    <td>nan</td> <td>nan</td>  <td>nan</td>    <td>nan</td>    <td>nan</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>acquisit</th>                            <td>nan</td>     <td>nan</td>    <td>nan</td> <td>nan</td>  <td>nan</td>    <td>nan</td>    <td>nan</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>act</th>                                 <td>nan</td>     <td>nan</td>    <td>nan</td> <td>nan</td>  <td>nan</td>    <td>nan</td>    <td>nan</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>activ</th>                               <td>nan</td>     <td>nan</td>    <td>nan</td> <td>nan</td>  <td>nan</td>    <td>nan</td>    <td>nan</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>administr</th>                           <td>nan</td>     <td>nan</td>    <td>nan</td> <td>nan</td>  <td>nan</td>    <td>nan</td>    <td>nan</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>advanc</th>                              <td>nan</td>     <td>nan</td>    <td>nan</td> <td>nan</td>  <td>nan</td>    <td>nan</td>    <td>nan</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>advertis</th>                            <td>nan</td>     <td>nan</td>    <td>nan</td> <td>nan</td>  <td>nan</td>    <td>nan</td>    <td>nan</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>agenc</th>                               <td>nan</td>     <td>nan</td>    <td>nan</td> <td>nan</td>  <td>nan</td>    <td>nan</td>    <td>nan</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>agent</th>                               <td>nan</td>     <td>nan</td>    <td>nan</td> <td>nan</td>  <td>nan</td>    <td>nan</td>    <td>nan</td> \n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary2.Summary'>\n",
       "\"\"\"\n",
       "                                Results: PHReg\n",
       "==============================================================================\n",
       "Model:                       PH Reg                  Sample size:        15000\n",
       "Dependent variable:          log_censored_tib        Num. events:        11197\n",
       "Ties:                        Breslow                                          \n",
       "------------------------------------------------------------------------------\n",
       "                                  log HR log HR SE  HR  t  P>|t| [0.025 0.975]\n",
       "------------------------------------------------------------------------------\n",
       "Bank                                 nan       nan nan nan   nan    nan    nan\n",
       "Domestic Non-Profit Corporation      nan       nan nan nan   nan    nan    nan\n",
       "Domestic Profit Corporation          nan       nan nan nan   nan    nan    nan\n",
       "Foreign Non-Profit Corporation       nan       nan nan nan   nan    nan    nan\n",
       "Insurance                            nan       nan nan nan   nan    nan    nan\n",
       "Loan and Investment                  nan       nan nan nan   nan    nan    nan\n",
       "Non-Profit Producers' Cooperative    nan       nan nan nan   nan    nan    nan\n",
       "Professional Service Corporation     nan       nan nan nan   nan    nan    nan\n",
       "Transportation                       nan       nan nan nan   nan    nan    nan\n",
       "accessori                            nan       nan nan nan   nan    nan    nan\n",
       "account                              nan       nan nan nan   nan    nan    nan\n",
       "acquir                               nan       nan nan nan   nan    nan    nan\n",
       "acquisit                             nan       nan nan nan   nan    nan    nan\n",
       "act                                  nan       nan nan nan   nan    nan    nan\n",
       "activ                                nan       nan nan nan   nan    nan    nan\n",
       "administr                            nan       nan nan nan   nan    nan    nan\n",
       "advanc                               nan       nan nan nan   nan    nan    nan\n",
       "advertis                             nan       nan nan nan   nan    nan    nan\n",
       "agenc                                nan       nan nan nan   nan    nan    nan\n",
       "agent                                nan       nan nan nan   nan    nan    nan\n",
       "==============================================================================\n",
       "Confidence intervals are for the hazard ratios\n",
       "\"\"\""
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sf_officers = sm.duration.PHReg(\n",
    "    y_train_officers[:15000],\n",
    "    X_train_officers2[:15000].astype(float),\n",
    "    status=X_train_officers[:15000].is_tib_censored)\n",
    "officers_result = sf_officers.fit()\n",
    "officers_result.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sf_officers = sm.duration.PHReg(y_train_officers, X_train_officers2.astype(float), status=X_train_officers.is_tib_censored)\n",
    "officers_result = sf_officers.fit()\n",
    "officers_result.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for item in X_train_officers2.columns:\n",
    "    print(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df[df.isnull().any(axis=1)]\n",
    "#df_w_officers[explanatory_variables][df_w_officers[explanatory_variables].isnan().any(axis=1)]\n",
    "#df[pd.isnull(df).any(axis=1)]\n",
    "X_train_officers.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loss function: squared error isn't appropriate. What are my options for survival regressions?\n",
    "# Is that even a thing for survival regressions? If so, can I use quantile/absolute error?\n",
    "\n",
    "# Don't forget to make ROC curves for all the predictions!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}