FCPADataScrape.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "pd.set_option(\"display.max_colwidth\", 10000)\n",
    "pd.options.mode.chained_assignment = None  # to not make too many copies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from datetime import datetime\n",
    "from dateutil.parser import parse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import openpyxl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import PyPDF2 as pypdf2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scrape PDF report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "report_folder = \"\"\"C:\\\\Users\\\\tetyana.lookutova\\\\Documents\\\\Control Risks\\\\FCPA\\\\\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "report_path = report_folder + 'Shearman FCPA Doc.pdf' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "excel_path =report_folder + 'scrape.xlsx'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "report_folder = 'Report_Folder//'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(columns = ['Date_filed','Countries','Date_Interval','Amount_of_Value', 'Total_Sanction','FCPA_Statutory_Provision'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def find_between( s, first, last ):\n",
    "    try:\n",
    "        s = s.replace('\\n','')\n",
    "        start = s.lower().index( first.lower() ) + len( first )\n",
    "        end = s.lower().index( last.lower(), start )\n",
    "        return s[start:end]\n",
    "    except ValueError:\n",
    "        return \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def clean_up(s):\n",
    "    s.replace('\\n','').strip().strip('.')\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def split_up(s):\n",
    "    lst  = s.split(';')\n",
    "    return lst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pdfFileObj = open(report_path, 'rb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pdfReader = pypdf2.PdfFileReader(pdfFileObj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "865"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pdfReader.numPages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def find_row(s):\n",
    "    keyfacts = s\n",
    "    country_list = find_between(keyfacts, 'Country','Date' ).replace('\\n','').strip() \n",
    "    date_filed = find_between(keyfacts,'Date Filed.', 'Country' ).replace('\\n','').strip()\n",
    "    date_of_conduct = find_between(keyfacts,'Date of Conduct.', 'Amount of the Value' ).replace('\\n','').strip()\n",
    "    amount = find_between(keyfacts,'Amount of the Value', 'Amount of' ).replace('\\n','').strip()\n",
    "    total_sanction = find_between(keyfacts,'Total Sanction.', 'Compliance Monitor' ).replace('\\n','').strip()\n",
    "    if (total_sanction == ''):\n",
    "        total_sanction = find_between(keyfacts,'Total Sanction.', 'Related Enforcement' ).replace('\\n','').strip()\n",
    "    fcpa_provision = find_between(keyfacts,'FCPA Statutory Provision', 'Other Statutory Provision' ).replace('\\n','').strip()\n",
    "\n",
    "    a1 = list(zip(['Date_filed','Countries','Date_Interval','Amount_of_Value', 'Total_Sanction','FCPA_Statutory_Provision'],[date_filed,country_list,date_of_conduct,amount,total_sanction,fcpa_provision]))\n",
    "    dict1 = dict((k[0],k[1]) for k in a1)\n",
    "    return dict1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def reconcile_dicts(dict1, dict2):\n",
    "    if (dict1['Total_Sanction'] == '' and dict2['Total_Sanction'] != ''):\n",
    "        dict1['Total_Sanction'] = dict2['Total_Sanction'] \n",
    "    if (dict1['FCPA_Statutory_Provision'] == '' and dict2['FCPA_Statutory_Provision'] != ''):\n",
    "        dict1['FCPA_Statutory_Provision'] = dict2['FCPA_Statutory_Provision'] \n",
    "    if (dict1['Amount_of_Value'] == '' and dict2['Amount_of_Value'] != ''):\n",
    "        dict1['Amount_of_Value'] = dict2['Amount_of_Value'] \n",
    "    if (dict1['Date_Interval'] == '' and dict2['Date_Interval'] != ''):\n",
    "        dict1['Date_Interval'] = dict2['Date_Interval'] \n",
    "    if (dict1['Countries'] == '' and dict2['Countries'] != ''):\n",
    "        dict1['Countries'] = dict2['Countries'] \n",
    "    if (dict1['Date_filed'] == '' and dict2['Date_filed'] != ''):\n",
    "        dict1['Date_filed'] = dict2['Date_filed'] \n",
    "    return dict1    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "row_list = []\n",
    "for page in range(39,572):#39-572\n",
    "    pageObj = pdfReader.getPage(page)\n",
    "    full_str = str(pageObj.extractText())\n",
    "    ind_keyfacts = full_str.find('Key Facts')\n",
    "    if (ind_keyfacts > -1):\n",
    "        dict1 = find_row(full_str[ind_keyfacts:])\n",
    "        if (dict1['Total_Sanction'] == ''): #look ahead\n",
    "            pageObj = pdfReader.getPage(page + 1)\n",
    "            full_str = str(pageObj.extractText())\n",
    "            ind_keyfacts = full_str.find('Key Facts')\n",
    "            if (ind_keyfacts == -1):\n",
    "                dict2 = find_row(full_str)\n",
    "                # reconcile where dict1 is missing\n",
    "                dict1 = reconcile_dicts(dict1, dict2)\n",
    "                page +=1\n",
    "        row_list.append(dict1)\n",
    "        \n",
    "df3 = pd.DataFrame(row_list, columns=['Date_filed','Countries','Date_Interval','Amount_of_Value', 'Total_Sanction','FCPA_Statutory_Provision'])    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date_filed</th>\n",
       "      <th>Countries</th>\n",
       "      <th>Date_Interval</th>\n",
       "      <th>Amount_of_Value</th>\n",
       "      <th>Total_Sanction</th>\n",
       "      <th>FCPA_Statutory_Provision</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>August 29, 2017; December 22, 2017 (KOM, KOM U...</td>\n",
       "      <td>.  Brazil.</td>\n",
       "      <td>2001  2014.</td>\n",
       "      <td>.  $55,000,000.</td>\n",
       "      <td>$422,216,980 (Global Total);25 $105,554,245 (U...</td>\n",
       "      <td>.  Conspiracy (Anti-Bribery).</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>December 21, 2017.</td>\n",
       "      <td>.  Saudi Arabia.</td>\n",
       "      <td>2009  2011.</td>\n",
       "      <td>.  Approximately $1.5 million.</td>\n",
       "      <td>Pending.</td>\n",
       "      <td>.  Conspiracy (Anti-Bribery); Anti-Bribery.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>November 29, 2017.</td>\n",
       "      <td>.  Brazil; Angola; Equatorial Guinea; Kazakhst...</td>\n",
       "      <td>1996  2012.</td>\n",
       "      <td>.  At least $180 million.</td>\n",
       "      <td>$238,000,000.</td>\n",
       "      <td>.    SBM Offshore N.V.  Conspiracy (Anti-Bribe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>November 16, 2017.  Date Unsealed.  November 2...</td>\n",
       "      <td>.  Chad; Uganda.</td>\n",
       "      <td>2014  2017.</td>\n",
       "      <td>.  $2.5 million.</td>\n",
       "      <td>Chi Ping Patrick Ho.  Pending.  Cheikh Gadio. ...</td>\n",
       "      <td>.   Chi Ping Patrick Ho.  Conspiracy (Anti-Bri...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>October 6, 2017 (Zubiate); October 19, 2017 (M...</td>\n",
       "      <td>.  Angola, Brazil, and Equatorial Guinea.</td>\n",
       "      <td>2008  2011.</td>\n",
       "      <td>.  Not stated.</td>\n",
       "      <td>Pending.</td>\n",
       "      <td>.  Mace.  Conspiracy (Anti-Bribery).  Zubiate....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Keith Barnett.  December 20, 2016.  Andreas Ko...</td>\n",
       "      <td>.  China, Kazakhstan.</td>\n",
       "      <td>1999  2013.</td>\n",
       "      <td>.  $500,000.</td>\n",
       "      <td>Keith Barnett.  Pending.  Andreas Kohler.  Pen...</td>\n",
       "      <td>.    Keith Barnett.  Conspiracy (Anti-Bribery)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>October 4, 2017.</td>\n",
       "      <td>.  Haiti.</td>\n",
       "      <td>2014  2015.</td>\n",
       "      <td>.  $50,000.</td>\n",
       "      <td>Pending.</td>\n",
       "      <td>.  Conspiracy (Anti-Bribery).</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>September 21, 2017.</td>\n",
       "      <td>.  Uzbekistan.</td>\n",
       "      <td>2007  2010.</td>\n",
       "      <td>.  $330,000,000</td>\n",
       "      <td>$548,603,972.</td>\n",
       "      <td>.  Anti-Bribery; Conspiracy (Anti-Bribery).</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>June 29, 2017.</td>\n",
       "      <td>.  India.</td>\n",
       "      <td>2011  2015.</td>\n",
       "      <td>.  Not Stated.</td>\n",
       "      <td>$4,037,138.</td>\n",
       "      <td>.   Anti-Bribery.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>June 16, 2017.</td>\n",
       "      <td>.  Georgia.</td>\n",
       "      <td>2006  2009.</td>\n",
       "      <td>.  Not Stated.</td>\n",
       "      <td>$11,235,000.</td>\n",
       "      <td>.   Anti-Bribery.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>January 19, 2017.</td>\n",
       "      <td>.  China.</td>\n",
       "      <td>2006  2009.</td>\n",
       "      <td>.  Not Stated.</td>\n",
       "      <td>$6,960,000.</td>\n",
       "      <td>.   Books-and-Records; Internal Controls.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>December 20, 2016.</td>\n",
       "      <td>.  Angola; Azerbaijan; Brazil; Iraq; Kazakhsta...</td>\n",
       "      <td>2000  2013.</td>\n",
       "      <td>.  Approximately $35 million.</td>\n",
       "      <td>$195,496,880 (Total Criminal Penalty); $169,91...</td>\n",
       "      <td>.   Conspiracy (Anti-Bribery).</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>December 15, 2016 (Bahn; Sang; Harris); Januar...</td>\n",
       "      <td>.  Korea; Middle East.</td>\n",
       "      <td>2013  2015.</td>\n",
       "      <td>.  $500,000.</td>\n",
       "      <td>Dennis Bahn.  Pending.  Ban Ki Sang.  Pending....</td>\n",
       "      <td>.    Dennis Bahn.  Conspiracy (Anti-Bribery); ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>January 13, 2017.</td>\n",
       "      <td>.  Chile.</td>\n",
       "      <td>2008  2015.</td>\n",
       "      <td>.  Not Stated.</td>\n",
       "      <td>$15,487,500.</td>\n",
       "      <td>.  Books-and-Records; Internal Controls.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>January 12, 2017.</td>\n",
       "      <td>.  Brazil; Mexico.</td>\n",
       "      <td>2010  2013.</td>\n",
       "      <td>.  Not Stated.</td>\n",
       "      <td>$17,460,300.</td>\n",
       "      <td>.     Zimmer Biomet.  Internal Controls.   Jer...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>December 22, 2016.</td>\n",
       "      <td>.  Angola; Bangladesh; China; Indonesia; Thail...</td>\n",
       "      <td>2002  2013.</td>\n",
       "      <td>.  Approximately $13 million.</td>\n",
       "      <td>$20,469,694.80.</td>\n",
       "      <td>.  Anti-Bribery; Books and Records; Internal C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>September 15, 2016 (Ray); August 16, 2016 (Val...</td>\n",
       "      <td>.  Mexico.</td>\n",
       "      <td>2005  2016.</td>\n",
       "      <td>.  Approximately $2 million.</td>\n",
       "      <td>Douglas Ray.  18-Months Imprisonment; $589,698...</td>\n",
       "      <td>.    Douglas Ray.  Conspiracy (Anti-Bribery). ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Date_filed  \\\n",
       "0   August 29, 2017; December 22, 2017 (KOM, KOM U...   \n",
       "1                                  December 21, 2017.   \n",
       "2                                  November 29, 2017.   \n",
       "3   November 16, 2017.  Date Unsealed.  November 2...   \n",
       "4   October 6, 2017 (Zubiate); October 19, 2017 (M...   \n",
       "5   Keith Barnett.  December 20, 2016.  Andreas Ko...   \n",
       "6                                    October 4, 2017.   \n",
       "7                                 September 21, 2017.   \n",
       "8                                      June 29, 2017.   \n",
       "9                                      June 16, 2017.   \n",
       "10                                  January 19, 2017.   \n",
       "11                                 December 20, 2016.   \n",
       "12  December 15, 2016 (Bahn; Sang; Harris); Januar...   \n",
       "13                                  January 13, 2017.   \n",
       "14                                  January 12, 2017.   \n",
       "15                                 December 22, 2016.   \n",
       "16  September 15, 2016 (Ray); August 16, 2016 (Val...   \n",
       "\n",
       "                                            Countries Date_Interval  \\\n",
       "0                                          .  Brazil.   2001  2014.   \n",
       "1                                    .  Saudi Arabia.   2009  2011.   \n",
       "2   .  Brazil; Angola; Equatorial Guinea; Kazakhst...   1996  2012.   \n",
       "3                                    .  Chad; Uganda.   2014  2017.   \n",
       "4           .  Angola, Brazil, and Equatorial Guinea.   2008  2011.   \n",
       "5                               .  China, Kazakhstan.   1999  2013.   \n",
       "6                                           .  Haiti.   2014  2015.   \n",
       "7                                      .  Uzbekistan.   2007  2010.   \n",
       "8                                           .  India.   2011  2015.   \n",
       "9                                         .  Georgia.   2006  2009.   \n",
       "10                                          .  China.   2006  2009.   \n",
       "11  .  Angola; Azerbaijan; Brazil; Iraq; Kazakhsta...   2000  2013.   \n",
       "12                             .  Korea; Middle East.   2013  2015.   \n",
       "13                                          .  Chile.   2008  2015.   \n",
       "14                                 .  Brazil; Mexico.   2010  2013.   \n",
       "15  .  Angola; Bangladesh; China; Indonesia; Thail...   2002  2013.   \n",
       "16                                         .  Mexico.   2005  2016.   \n",
       "\n",
       "                   Amount_of_Value  \\\n",
       "0                  .  $55,000,000.   \n",
       "1   .  Approximately $1.5 million.   \n",
       "2        .  At least $180 million.   \n",
       "3                 .  $2.5 million.   \n",
       "4                   .  Not stated.   \n",
       "5                     .  $500,000.   \n",
       "6                      .  $50,000.   \n",
       "7                  .  $330,000,000   \n",
       "8                   .  Not Stated.   \n",
       "9                   .  Not Stated.   \n",
       "10                  .  Not Stated.   \n",
       "11   .  Approximately $35 million.   \n",
       "12                    .  $500,000.   \n",
       "13                  .  Not Stated.   \n",
       "14                  .  Not Stated.   \n",
       "15   .  Approximately $13 million.   \n",
       "16    .  Approximately $2 million.   \n",
       "\n",
       "                                       Total_Sanction  \\\n",
       "0   $422,216,980 (Global Total);25 $105,554,245 (U...   \n",
       "1                                            Pending.   \n",
       "2                                       $238,000,000.   \n",
       "3   Chi Ping Patrick Ho.  Pending.  Cheikh Gadio. ...   \n",
       "4                                            Pending.   \n",
       "5   Keith Barnett.  Pending.  Andreas Kohler.  Pen...   \n",
       "6                                            Pending.   \n",
       "7                                       $548,603,972.   \n",
       "8                                         $4,037,138.   \n",
       "9                                        $11,235,000.   \n",
       "10                                        $6,960,000.   \n",
       "11  $195,496,880 (Total Criminal Penalty); $169,91...   \n",
       "12  Dennis Bahn.  Pending.  Ban Ki Sang.  Pending....   \n",
       "13                                       $15,487,500.   \n",
       "14                                       $17,460,300.   \n",
       "15                                    $20,469,694.80.   \n",
       "16  Douglas Ray.  18-Months Imprisonment; $589,698...   \n",
       "\n",
       "                             FCPA_Statutory_Provision  \n",
       "0                       .  Conspiracy (Anti-Bribery).  \n",
       "1         .  Conspiracy (Anti-Bribery); Anti-Bribery.  \n",
       "2   .    SBM Offshore N.V.  Conspiracy (Anti-Bribe...  \n",
       "3   .   Chi Ping Patrick Ho.  Conspiracy (Anti-Bri...  \n",
       "4   .  Mace.  Conspiracy (Anti-Bribery).  Zubiate....  \n",
       "5   .    Keith Barnett.  Conspiracy (Anti-Bribery)...  \n",
       "6                       .  Conspiracy (Anti-Bribery).  \n",
       "7         .  Anti-Bribery; Conspiracy (Anti-Bribery).  \n",
       "8                                   .   Anti-Bribery.  \n",
       "9                                   .   Anti-Bribery.  \n",
       "10          .   Books-and-Records; Internal Controls.  \n",
       "11                     .   Conspiracy (Anti-Bribery).  \n",
       "12  .    Dennis Bahn.  Conspiracy (Anti-Bribery); ...  \n",
       "13           .  Books-and-Records; Internal Controls.  \n",
       "14  .     Zimmer Biomet.  Internal Controls.   Jer...  \n",
       "15  .  Anti-Bribery; Books and Records; Internal C...  \n",
       "16  .    Douglas Ray.  Conspiracy (Anti-Bribery). ...  "
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Key Facts\\n \\nCitation.  \\nIn re CDM Smith Inc. \\n(2017)\\n \\nDate\\n \\nFiled\\n.\\n  \\nJune 29, 2017.\\n \\nCountry\\n.\\n  \\nIndia.\\n \\nDate of Conduct.  \\n2011 \\n\\n \\n2015. \\n \\nAmount\\n \\nof the Value.\\n  \\nNot Stated.\\n \\nAmount of Business Related to the Payment.  \\nApproximately $4 million.\\n \\nIntermediary\\n.\\n  \\nSubsidiary; Subcontractors.\\n \\nForeign\\n \\nOfficial.\\n  \\nOfficials from the National Highways \\nAuthority of India; Unnamed officials from the Indian \\nstate of Goa.\\n \\nFCPA Statutory Provision.  \\n \\nAnti\\n-\\nBribery.\\n \\nOther Statutory Provi\\nsion.  \\nNone.\\n \\nDisposition.  \\nDeclination with Disgorgement.\\n \\nDefendant Jurisdictional Basis.  \\nDomestic Concern.\\n \\n\\n  \\nUnited States.\\n \\nTotal Sanction.  \\n$4,037,138.\\n \\nCompliance Monitor/Reporting Requirements.\\n  \\nNone.\\n \\nRelated Enforcement Actions\\n.\\n  \\nNone.\\n \\n \\n \\n \\n \\n'"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "writer= pd.ExcelWriter(excel_path)\n",
    "\n",
    "df3.to_excel(writer,'Sheet1')\n",
    "writer.save()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process what has been scraped\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "long_month_names = (\n",
    "    'January', 'February', 'March', 'April', 'May', 'June', 'July',\n",
    "    'August', 'September', 'October', 'November', 'December'\n",
    ")\n",
    "long_month_cap = '(?:' + '|'.join(long_month_names) + ')'\n",
    "long_year_cap = '(?:[0-9]{3}[1-9]|[0-9]{2}[1-9][0-9]|[0-9][1-9][0-9]{2}|[1-9][0-9]{3})'\n",
    "short_day_cap = '(?:[1-9]|[12][0-9]|3[01])'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "formats =  r'(?P<month_0>{lnm}) (?P<day_0>{sd}), (?P<year_0>{ly})'.format(\n",
    "        lnm=long_month_cap, sd=short_day_cap,\n",
    "        ly=long_year_cap\n",
    "    )\n",
    "\n",
    "# add random text before and after\n",
    "formats = r'\\b(?:{})\\b'.format(formats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\\\b(?:(?P<month_0>(?:January|February|March|April|May|June|July|August|September|October|November|December)) (?P<day_0>(?:[1-9]|[12][0-9]|3[01])), (?P<year_0>(?:[0-9]{3}[1-9]|[0-9]{2}[1-9][0-9]|[0-9][1-9][0-9]{2}|[1-9][0-9]{3})))\\\\b'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "formats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pattern = re.compile(formats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del(pattern)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_fields(match):\n",
    "    if not match:\n",
    "        return None\n",
    "    return match.groupdict()['month_0'] + ' ' + match.groupdict()['day_0'] + ' ' + match.groupdict()['year_0'] \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "isinstance(None, str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "tests = r'''August 29, 2017; December 22, 2017 (KOM, KOM USA).  Date Unsealed.  December 22, 2017 (Chow).'''\n",
    "def get_min_max_date_from_string(tests, is_min): \n",
    "    if not isinstance(tests,str):\n",
    "        return None, None\n",
    "    dates = []\n",
    "    for test in re.split('(;|\\.)+', tests):\n",
    "        date_str = '{!r}'.format(get_fields(pattern.search(test))).strip('\\'')\n",
    "        if (date_str != 'None'):\n",
    "            #print(date_str)\n",
    "            try:\n",
    "                dt = datetime.strptime(date_str, '%B %d %Y')  \n",
    "            except ValueError as e:\n",
    "                #print(e)\n",
    "                dt = None    \n",
    "            if dt:\n",
    "                #print(dt)\n",
    "                dates.append(dt)\n",
    "    if (len(dates) == 0):            \n",
    "        return None\n",
    "    else:\n",
    "        if is_min:            \n",
    "            return min(dates)\n",
    "        else:\n",
    "            return max(dates)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "get_min_max_date_from_string(tests, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_min_date_from_string(tests):\n",
    "    return get_min_max_date_from_string(tests, True)\n",
    "\n",
    "def get_max_date_from_string(tests):\n",
    "    return get_min_max_date_from_string(tests, False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_orig = pd.read_excel(excel_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_analysis = df_orig.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_analysis['date_filed_min'] = df_orig['Date_filed'].apply(get_min_date_from_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_analysis['date_filed_max'] = df_orig['Date_filed'].apply(get_max_date_from_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_analysis = df_analysis.dropna(subset = ['Countries', 'date_filed_max', 'date_filed_max'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_analysis['days_interval'] = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_analysis['days_interval'].loc[(df_analysis['date_filed_max'].notnull()) & (df_analysis['date_filed_min'].notnull())] = (df_analysis['date_filed_max']-df_analysis['date_filed_min'] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_analysis['date_commenced'] = df_analysis['Date_Interval'].str[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_analysis['date_finished'] = df_analysis['Date_Interval'].str[-5:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_analysis['Total_Sanction'] = df_analysis['Total_Sanction'].str.rstrip('.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "59            9-months Imprisonment; $3.82 million in Disgorgement\n",
       "60                                                     $17,280,000\n",
       "61                                                     $11,800,000\n",
       "62                                                     $16,800,000\n",
       "63                                                     $54,600,000\n",
       "64    Magyar Telekom.  $59,600,000.  Deutsche Telekom.  $4,360,000\n",
       "Name: Total_Sanction, dtype: object"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_analysis['Total_Sanction'].iloc[59:65]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process the money amounts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "money_str = ('|'.join([\n",
    "  r'\\$?((\\d|,)*\\.\\d{1,2})',  # e.g., $.50, .50, $1.50, $.5, .5\n",
    "  r'\\$?(\\d+)',           # e.g., $500, $5, 500, 5\n",
    "  r'\\$(\\d+\\.?)',         # e.g., $5.\n",
    "]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "money_str = r'\\b(?:{})\\b'.format(money_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "money = re.compile(money_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_l = '.  $204,537 (foreign officials) and $1,683,672 (private parties).'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_money(money_cell):\n",
    "    money_cell = str(money_cell)\n",
    "    if 'million' in money_cell: \n",
    "        money_cell = money_cell.replace(' million', '000000').replace('.','')\n",
    "    else:\n",
    "        money_cell= money_cell.replace(',','')\n",
    "    match = money.search(money_cell)\n",
    "    if match:\n",
    "        return float(match[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "204537.0"
      ]
     },
     "execution_count": 235,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_money(test_l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_analysis['Total_Sanction_Money'] =  df_analysis['Total_Sanction'].apply(get_money)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_analysis['Total_Amount_Money'] = df_analysis['Amount_of_Value'].apply(get_money)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "excel_date_money = pd.ExcelWriter(report_folder + 'Date_Money.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_analysis.to_excel(excel_date_money, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "excel_date_money.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Write countries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/content/datalab'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "report_folder = 'Report_Folder//'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_analysis = pd.read_excel(report_folder + 'Date_Money.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "l = list(df_analysis.columns)\n",
    "l.append('Country')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_analysis;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_analysis_countries = pd.DataFrame(columns = l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date_filed</th>\n",
       "      <th>Countries</th>\n",
       "      <th>Date_Interval</th>\n",
       "      <th>Amount_of_Value</th>\n",
       "      <th>Total_Sanction</th>\n",
       "      <th>FCPA_Statutory_Provision</th>\n",
       "      <th>date_filed_min</th>\n",
       "      <th>date_filed_max</th>\n",
       "      <th>days_interval</th>\n",
       "      <th>date_commenced</th>\n",
       "      <th>date_finished</th>\n",
       "      <th>Total_Sanction_Money</th>\n",
       "      <th>Total_Amount_Money</th>\n",
       "      <th>Country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Date_filed, Countries, Date_Interval, Amount_of_Value, Total_Sanction, FCPA_Statutory_Provision, date_filed_min, date_filed_max, days_interval, date_commenced, date_finished, Total_Sanction_Money, Total_Amount_Money, Country]\n",
       "Index: []"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_analysis_countries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in range(0,len(df_analysis)):\n",
    "    temp = list(df_analysis.iloc[i])\n",
    "    country_list = temp[1].strip('.').strip()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "',  rwanda'"
      ]
     },
     "execution_count": 363,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.sub(r'\\b(and)','', ', and rwanda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(0,len(df_analysis)):\n",
    "    temp = list(df_analysis.iloc[i])\n",
    "    country_list = temp[1].strip('.').strip()\n",
    "    country_list = country_list.replace('Iraq: Mexico', 'Iraq; Mexico')\n",
    "    country_list = country_list.replace('Serbia and Montenegro', 'Serbia; Montenegro')\n",
    "    countries = re.split('(;|,)+', country_list)\n",
    "    temp2 = []\n",
    "    for country in countries:\n",
    "        country = re.sub(r'\\b(and)','', country)\n",
    "        #country = country.replace('and','')\n",
    "        temp1 = temp.copy()\n",
    "        temp1.append(country.strip())\n",
    "        temp2.append(temp1)\n",
    "    df=pd.DataFrame(temp2,columns=l)    \n",
    "    \n",
    "    df_analysis_countries = df_analysis_countries.append(df)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_analysis_countries.reset_index(inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_analysis_countries.drop(columns=['index'], inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Int64Index([   3,    5,    7,    9,   12,   28,   30,   32,   34,   36,\n",
       "            ...\n",
       "             654,  659,  665,  881,  896,  899,  904,  930,  943, 1012],\n",
       "           dtype='int64', length=143)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_analysis_countries[df_analysis_countries['Country'] == ';'].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Int64Index([  15,   17,   20,   95,  226,  228,  230,  232,  234,  236,\n",
       "            ...\n",
       "            1027, 1029, 1034, 1036, 1041, 1053, 1055, 1057, 1059, 1061],\n",
       "           dtype='int64', length=221)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_analysis_countries[df_analysis_countries['Country'] == ','].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1064"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_analysis_countries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_analysis_countries = df_analysis_countries.drop(df_analysis_countries[df_analysis_countries['Country'] == ','].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_analysis_countries = df_analysis_countries.drop(df_analysis_countries[df_analysis_countries['Country'] == ';'].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "writer= pd.ExcelWriter(report_folder + 'Countries_analysis.xlsx')\n",
    "\n",
    "df_analysis_countries.to_excel(writer,'Sheet1', index=False)\n",
    "writer.save()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Select unique countries and count how many were implicated "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def replace_all(row):\n",
    "    elem = row['Country']\n",
    "    if elem == 'Trinidad  Tobago':\n",
    "        row['Country'] = 'Trinidad and Tobago'\n",
    "    elif elem == 'Uga':\n",
    "        row['Country'] = 'Uganda'\n",
    "    elif elem == 'At least 22 countries including China':\n",
    "        row['Country'] = 'China'    \n",
    "    elif elem == 'India (Books--Records)':\n",
    "        row['Country'] = 'India'\n",
    "    elif elem == 'South Korea  (Internal Controls)':\n",
    "        row['Country'] = 'South Korea'\n",
    "    elif elem == 'the Democratic Republic of the Congo':\n",
    "        row['Country'] = 'Congo (Kinshasa)'\n",
    "    elif elem == 'Democratic Republic of Congo':\n",
    "        row['Country'] = 'Congo (Kinshasa)'\n",
    "    elif elem == '22 countries including China':\n",
    "        row['Country'] = 'China'\n",
    "    elif elem == 'UAE':\n",
    "        row['Country'] = 'United Arab Emirates'\n",
    "    elif elem == 'the Philippines':\n",
    "        row['Country'] = 'Philippines'\n",
    "    elif elem == 'Antigua  Barbuda':\n",
    "        row['Country'] =  'Antigua and Barbuda'\n",
    "    elif elem == 'Vietnam  others': \n",
    "        row['Country'] = 'Vietnam'\n",
    "    elif elem == 'Macao':\n",
    "        row['Country'] = 'China'\n",
    "    elif elem == 'Korea':\n",
    "        row['Country'] = 'South Korea'\n",
    "    elif elem == 'Congo':\n",
    "        row['Country'] = 'Congo (Brazzaville)'    \n",
    "    elif elem == 'Bosnia':\n",
    "        row['Country'] = 'Bosnia and Herzegovina'\n",
    "    return row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_analysis_countries1 = df_analysis_countries.apply (lambda row: replace_all(row),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "700"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_analysis_countries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#df_analysis_countries = df_analysis_countries.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_analysis_countries1['Country'].to_csv(report_folder+'df_analysis_countries1.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### read back from \n",
    "df_analysis_countries1['Country'].to_csv(report_folder+'df_analysis_countries1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "full_list = df_analysis_countries1['Country'].str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "700"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(full_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "d = Counter(full_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_country_map = pd.DataFrame()\n",
    "df_country_map = df_country_map.from_dict(d, orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_country_map = df_country_map.reset_index(level=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Country</th>\n",
       "      <th>Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Malawi</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Country  Count\n",
       "0  Malawi      1"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_country_map.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "hiddenCell": true
   },
   "outputs": [],
   "source": [
    "df_country_map.columns=['Country', 'Count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "hiddenCell": true
   },
   "outputs": [],
   "source": [
    "df_country_map.to_csv(report_folder + 'country_counts.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_country_map = pd.read_csv(report_folder + 'country_counts.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Choropleth map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import plotly\n",
    "plotly.tools.set_credentials_file(username='tloskutova', api_key='6Go8dVHzYtonctgZ5vjo')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#mix dataframes with GDP and country codes\n",
    "df_countrycd_gdp = pd.read_excel(report_folder + 'CountryGDP.xlsx')\n",
    "df_country_gdp = pd.merge(left = df_country_map, right = df_countrycd_gdp, left_on = 'Country', right_on = 'COUNTRY', how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_country_gdp['text'] = df_country_gdp['Country'] + '<br>FCPA cases ' + df_country_gdp['Count'].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Country</th>\n",
       "      <th>Count</th>\n",
       "      <th>COUNTRY</th>\n",
       "      <th>GDP (BILLIONS)</th>\n",
       "      <th>CODE</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Unspecified Countries in Latin America</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Unspecified Countries in Latin America&lt;br&gt;FCPA cases 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Myanmar</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Myanmar&lt;br&gt;FCPA cases 2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>None (FBI Sting)</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None (FBI Sting)&lt;br&gt;FCPA cases 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>Middle East</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Middle East&lt;br&gt;FCPA cases 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>Ivory Coast</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Ivory Coast&lt;br&gt;FCPA cases 4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>Not Stated</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Not Stated&lt;br&gt;FCPA cases 3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>Multiple countries</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Multiple countries&lt;br&gt;FCPA cases 1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Country  Count COUNTRY  GDP (BILLIONS)  \\\n",
       "1    Unspecified Countries in Latin America      1     NaN             NaN   \n",
       "38                                  Myanmar      2     NaN             NaN   \n",
       "52                         None (FBI Sting)      1     NaN             NaN   \n",
       "53                              Middle East      1     NaN             NaN   \n",
       "75                              Ivory Coast      4     NaN             NaN   \n",
       "82                               Not Stated      3     NaN             NaN   \n",
       "100                      Multiple countries      1     NaN             NaN   \n",
       "\n",
       "    CODE                                                    text  \n",
       "1    NaN  Unspecified Countries in Latin America<br>FCPA cases 1  \n",
       "38   NaN                                 Myanmar<br>FCPA cases 2  \n",
       "52   NaN                        None (FBI Sting)<br>FCPA cases 1  \n",
       "53   NaN                             Middle East<br>FCPA cases 1  \n",
       "75   NaN                             Ivory Coast<br>FCPA cases 4  \n",
       "82   NaN                              Not Stated<br>FCPA cases 3  \n",
       "100  NaN                      Multiple countries<br>FCPA cases 1  "
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_country_gdp[pd.isna(df_country_gdp.COUNTRY)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import plotly.plotly as py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\" seamless=\"seamless\" src=\"https://plot.ly/~tloskutova/9.embed\" height=\"525px\" width=\"100%\"></iframe>"
      ],
      "text/plain": [
       "<plotly.tools.PlotlyDisplay object>"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = [ dict(\n",
    "        type = 'choropleth',\n",
    "        locations = df_country_gdp['CODE'],\n",
    "        z = df_country_gdp['Count'],\n",
    "        text = df_country_gdp['text'],\n",
    "        colorscale = [[0,\"rgb(5, 10, 172)\"],[0.35,\"rgb(40, 60, 190)\"],[0.5,\"rgb(70, 100, 245)\"],\\\n",
    "            [0.6,\"rgb(90, 120, 245)\"],[0.7,\"rgb(106, 137, 247)\"],[1,\"rgb(220, 220, 220)\"]],\n",
    "        autocolorscale = False,\n",
    "        reversescale = True,\n",
    "        marker = dict(\n",
    "            line = dict (\n",
    "                color = 'rgb(180,180,180)',\n",
    "                width = 0.5\n",
    "            ) ),\n",
    "        colorbar = dict(\n",
    "            autotick = False,\n",
    "            tickprefix = '',\n",
    "            title = 'FCPA Cases'),\n",
    "      ) ]\n",
    "\n",
    "layout = dict(\n",
    "    title = '2017 Global FCPA<br>Source:\\\n",
    "            <a href=\"https://www.justice.gov/criminal-fraud/related-enforcement-actions\">\\\n",
    "            DOJ</a>',\n",
    "    geo = dict(\n",
    "        showframe = False,\n",
    "        showcoastlines = False,\n",
    "        projection = dict(\n",
    "            type = 'Mercator'\n",
    "        )\n",
    "    )\n",
    ")\n",
    "\n",
    "fig = dict( data=data, layout=layout )\n",
    "py.iplot( fig, validate=False, filename='FCPA-world-map' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "writer= pd.ExcelWriter(report_folder + 'df_country_grp.xlsx')\n",
    "\n",
    "df_country_gdp.to_excel(writer)\n",
    "writer.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Correlations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.stats.stats import pearsonr   \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Correlate with World Freedom Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_orig = pd.read_excel(report_folder +  'Subcategory_Scores_FIW2017.xlsx', sheet_name='Countries FIW2017')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom = pd.merge(df_country_map, df_freedom_orig, left_on='Country', right_on='Country',\n",
    "      how='left', sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_freedom_an = df_freedom[['Country', 'Count','Total Aggr']]\n",
    "df_freedom_an = df_freedom_an.set_index('Country')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "writer = pd.ExcelWriter(report_folder + 'df_freedom.xlsx')\n",
    "df_freedom_an.to_excel(writer)\n",
    "writer.save()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Group by to defined average and cumulative 'Total' per 'Count' (count is a fk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "grouped = df_freedom_an.groupby(['Count'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped_df = grouped['Total Aggr'].aggregate(['mean','sum'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped_df1 = grouped_df.reset_index()\n",
    "#grouped_df1 = grouped_df1.columns=['Count', 'mean','sum']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Count</th>\n",
       "      <th>mean</th>\n",
       "      <th>sum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>58.12</td>\n",
       "      <td>1453.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Count   mean     sum\n",
       "0      1  58.12  1453.0"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grouped_df1.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import plotly.tools as tls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.12.1\n"
     ]
    }
   ],
   "source": [
    "# required to do iplot\n",
    "import cufflinks as cf\n",
    "\n",
    "print (cf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_an.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_an_aggr = pd.merge(left =df_freedom_an, right = grouped_df1, on = 'Count' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_an_aggr_plot = df_freedom_an_aggr[['Country','Count', 'mean', 'sum']]\n",
    "df_freedom_an_aggr_plot.set_index(keys='Country', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_freedom_an_aggr_plot = df_freedom_an_aggr_plot.sort_values(by=['Count','mean'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\" seamless=\"seamless\" src=\"https://plot.ly/~tloskutova/3.embed\" height=\"525px\" width=\"100%\"></iframe>"
      ],
      "text/plain": [
       "<plotly.tools.PlotlyDisplay object>"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_freedom_an_aggr_plot.iplot(kind='bar', subplots=True, shape=(3, 1), filename='df_freedom_an')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "writer = pd.ExcelWriter(report_folder + 'df_freedom_an_aggr_plot.xlsx')\n",
    "df_freedom_an_aggr_plot.to_excel(writer)\n",
    "writer.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 580,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.stats.stats import pearsonr   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_country_gdp_an = pd.merge(left = df_country_gdp, left_on = 'Country', right = df_freedom_an_aggr_plot, right_index = True, how = 'left')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Freedom of information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_right_inf_orig = pd.read_excel(report_folder + 'Right to information.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_right_inf = pd.merge(df_country_gdp_an, df_right_inf_orig, left_on='Country', right_on='COUNTRY',\n",
    "      how='left', sort=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Save freedom of information, freedom index, and GDP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "writer = pd.ExcelWriter(report_folder + 'freedom_gdp_freind.xlsx')\n",
    "df_right_inf.to_excel(writer)\n",
    "writer.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_right_inf_an = df_right_inf.dropna(subset = ['COUNTRY_y', 'GDP (BILLIONS)'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_gdp_freind.drop(columns = ['Count_x'], inplace = True);\n",
    "df_freedom_gdp_freind.drop(columns = ['COUNTRY_x'], inplace = True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_right_inf_an.set_index(keys = 'Country', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_counts.drop(columns=['Country'], inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_freedom_gdp_freind = pd.read_excel(report_folder + 'freedom_gdp_freind.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_freedom_gdp_freind.rename(columns = {'Count_y': 'Count'}, inplace= True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "hist, bin_edges = np.histogram(df_freedom_gdp_freind['Count'], bins=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.        , 13.83333333, 26.66666667, 39.5       , 52.33333333,\n",
       "       65.16666667, 78.        ])"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bin_edges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['APPEALS', 'CODE', 'COUNTRY_y', 'CSV', 'Count', 'Country', 'DATE',\n",
       "       'EXCEPTIONS & REFUSALS', 'GDP (BILLIONS)', 'LAW',\n",
       "       'PROMOTIONAL MEASURES', 'RANKING POSITION', 'REQUESTING PROCEDURES',\n",
       "       'RIGHT OF ACCESS', 'SANCTIONS & PROTECTIONS', 'SCOPE', 'TOTAL',\n",
       "       'label-H', 'label_num_h', 'mean', 'sum', 'text', 'label', 'label_num',\n",
       "       'Jurisdiction', 'corrupt perception', 'Country Name_x',\n",
       "       'corporate transparency', 'Country Name_y',\n",
       "       'Reliability of transparency', 'Country Name', 'disclosure_index',\n",
       "       'GINI'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_freedom_gdp_freind.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def label_rows_fibonacci(row):\n",
    "    seq =  [1 , 1 , 2 , 3 , 5 , 8 , 13 , 21 , 34 , 55 , 89 , 144, 233]\n",
    "    cnt = int(row['Count'])\n",
    "    if cnt < seq[3]:\n",
    "        row['label_h'] = '0-'+ str(seq[3])\n",
    "        row['label_num_h'] = 0\n",
    "    elif cnt < seq[6]:\n",
    "        row['label_h'] = str(seq[3]) + '-' + str(seq[6])\n",
    "        row['label_num_h'] = 1\n",
    "    elif cnt < seq[9]:\n",
    "        row['label_h'] = str(seq[6]) + '-' + str(9)\n",
    "        row['label_num_h'] = 2\n",
    "    else:    \n",
    "        row['label_h'] = str(seq[9]) + '-' + str(seq[12])\n",
    "        row['label_num_h'] = 3\n",
    "    return row    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def label_rows_hist(row, bin_edges):\n",
    "    cnt = int(row['Count'])\n",
    "    \n",
    "    if cnt < bin_edges[1]:\n",
    "        row['label-h'] = '0-'+ str(bin_edges[1])\n",
    "        row['label_num_h'] = 0\n",
    "    elif cnt < bin_edges[2]:\n",
    "        row['label_h'] = str(bin_edges[1]) + '-' + str(bin_edges[2])\n",
    "        row['label_num_h'] = 1\n",
    "    elif cnt < bin_edges[3]:\n",
    "        row['label_h'] = str(bin_edges[2]) + '-' + str(bin_edges[3])\n",
    "        row['label_num_h'] = 2\n",
    "    else:    \n",
    "        row['label_h'] = str(bin_edges[3]) + '-' + str(bin_edges[4])\n",
    "        row['label_num_h'] = 3\n",
    "    return row    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def label_rows(row): # just classifying as risk groups. Technically, it is a qualitative labelling\n",
    "    cnt = int(row['Count'])\n",
    "    \n",
    "    if cnt < 2:\n",
    "        row['label'] = '0-1'\n",
    "        row['label_num'] = 0\n",
    "    elif cnt < 6:\n",
    "        row['label'] = '2-5'\n",
    "        row['label_num'] = 1\n",
    "    elif cnt < 20:\n",
    "        row['label'] = '6-19'\n",
    "        row['label_num'] = 2\n",
    "    else:    \n",
    "        row['label'] = '20-80'\n",
    "        row['label_num'] = 3\n",
    "    return row    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "hist, bin_edges = np.histogram(df_freedom_gdp_freind['sum_duration'], bins=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  2. ,  41.9,  81.8, 121.7, 161.6, 201.5, 241.4, 281.3, 321.2,\n",
       "       361.1, 401. ])"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bin_edges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "def label_rows_duration(row): # just classifying as risk groups. Technically, it is a qualitative labelling\n",
    "    cnt = int(row['sum_duration'])\n",
    "    \n",
    "    if cnt < 101:\n",
    "        row['label_dur'] = '0-100'\n",
    "        row['label_dur_n'] = 0\n",
    "    elif cnt < 201:\n",
    "        row['label_dur'] = '101-200'\n",
    "        row['label_dur_n'] = 1\n",
    "    elif cnt < 301:\n",
    "        row['label_dur'] = '201-300'\n",
    "        row['label_dur_n'] = 2\n",
    "    else:    \n",
    "        row['label_dur'] = '300+'\n",
    "        row['label_dur_n'] = 3\n",
    "    return row    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_gdp_freind = df_freedom_gdp_freind.apply(lambda  row: label_rows_hist(row, bin_edges),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_gdp_freind = df_freedom_gdp_freind.apply(lambda  row: label_rows(row),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(df_freedom_gdp_freind['label_num_h'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "hiddenCell": true
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "labels ['label_h'] not contained in axis",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-a52b17557b05>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf_freedom_gdp_freind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'label_h'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minplace\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36mdrop\u001b[0;34m(self, labels, axis, index, columns, level, inplace, errors)\u001b[0m\n\u001b[1;32m   2528\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;32min\u001b[0m \u001b[0maxes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2529\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2530\u001b[0;31m                 \u001b[0mobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_drop_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlevel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2531\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2532\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m_drop_axis\u001b[0;34m(self, labels, axis, level, errors)\u001b[0m\n\u001b[1;32m   2560\u001b[0m                 \u001b[0mnew_axis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlevel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2561\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2562\u001b[0;31m                 \u001b[0mnew_axis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2563\u001b[0m             \u001b[0mdropped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreindex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0maxis_name\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mnew_axis\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2564\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mdrop\u001b[0;34m(self, labels, errors)\u001b[0m\n\u001b[1;32m   3742\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0merrors\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'ignore'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3743\u001b[0m                 raise ValueError('labels %s not contained in axis' %\n\u001b[0;32m-> 3744\u001b[0;31m                                  labels[mask])\n\u001b[0m\u001b[1;32m   3745\u001b[0m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m~\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3746\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdelete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: labels ['label_h'] not contained in axis"
     ]
    }
   ],
   "source": [
    "df_freedom_gdp_freind.drop(columns=['label_h'], inplace = True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "writer = pd.ExcelWriter(report_folder + 'df_freedom_gdp_freind.xlsx')\n",
    "df_freedom_gdp_freind.to_excel(writer)\n",
    "writer.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_gdp_freind = pd.read_excel(report_folder + 'df_freedom_gdp_freind.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>APPEALS</th>\n",
       "      <th>CODE</th>\n",
       "      <th>COUNTRY_y</th>\n",
       "      <th>CSV</th>\n",
       "      <th>Count</th>\n",
       "      <th>Country</th>\n",
       "      <th>DATE</th>\n",
       "      <th>EXCEPTIONS &amp; REFUSALS</th>\n",
       "      <th>GDP (BILLIONS)</th>\n",
       "      <th>LAW</th>\n",
       "      <th>...</th>\n",
       "      <th>SANCTIONS &amp; PROTECTIONS</th>\n",
       "      <th>SCOPE</th>\n",
       "      <th>TOTAL</th>\n",
       "      <th>label-H</th>\n",
       "      <th>label_num_h</th>\n",
       "      <th>mean</th>\n",
       "      <th>sum</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>label_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23.0</td>\n",
       "      <td>ALB</td>\n",
       "      <td>Albania</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>Albania</td>\n",
       "      <td>1999.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>13.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>127.0</td>\n",
       "      <td>0-20.25</td>\n",
       "      <td>0</td>\n",
       "      <td>60.562500</td>\n",
       "      <td>969</td>\n",
       "      <td>Albania&lt;br&gt;FCPA cases 2</td>\n",
       "      <td>0-2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>DZA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>Algeria</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>227.8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0-20.25</td>\n",
       "      <td>0</td>\n",
       "      <td>56.333333</td>\n",
       "      <td>845</td>\n",
       "      <td>Algeria&lt;br&gt;FCPA cases 3</td>\n",
       "      <td>2-5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>18.0</td>\n",
       "      <td>AGO</td>\n",
       "      <td>Angola</td>\n",
       "      <td>NaN</td>\n",
       "      <td>17</td>\n",
       "      <td>Angola</td>\n",
       "      <td>2002.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>131.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>0-20.25</td>\n",
       "      <td>0</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>24</td>\n",
       "      <td>Angola&lt;br&gt;FCPA cases 17</td>\n",
       "      <td>10-80</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   APPEALS CODE COUNTRY_y  CSV  Count  Country    DATE  EXCEPTIONS & REFUSALS  \\\n",
       "0     23.0  ALB   Albania  NaN      2  Albania  1999.0                   27.0   \n",
       "1      NaN  DZA       NaN  NaN      3  Algeria     NaN                    NaN   \n",
       "2     18.0  AGO    Angola  NaN     17   Angola  2002.0                   18.0   \n",
       "\n",
       "   GDP (BILLIONS)  LAW    ...      SANCTIONS & PROTECTIONS  SCOPE  TOTAL  \\\n",
       "0            13.4  NaN    ...                          6.0   29.0  127.0   \n",
       "1           227.8  NaN    ...                          NaN    NaN    NaN   \n",
       "2           131.4  NaN    ...                          0.0   20.0   76.0   \n",
       "\n",
       "   label-H  label_num_h       mean  sum                     text  label  \\\n",
       "0  0-20.25            0  60.562500  969  Albania<br>FCPA cases 2    0-2   \n",
       "1  0-20.25            0  56.333333  845  Algeria<br>FCPA cases 3    2-5   \n",
       "2  0-20.25            0  24.000000   24  Angola<br>FCPA cases 17  10-80   \n",
       "\n",
       "   label_num  \n",
       "0          0  \n",
       "1          1  \n",
       "2          3  \n",
       "\n",
       "[3 rows x 24 columns]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_freedom_gdp_freind.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_gdp_freind['PROMOTIONAL MEASURES'].fillna(0, inplace=True) #10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_gdp_freind['GDP (BILLIONS)'].fillna(0, inplace=True) #8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "writer = pd.ExcelWriter(report_folder + 'df_freedom_gdp_freind_.xlsx')\n",
    "df_freedom_gdp_freind.to_excel(writer)\n",
    "writer.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_gdp_freind = pd.read_excel(report_folder + 'df_freedom_gdp_freind_.xlsx')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add corruption perception"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd_corr_perc = pd.read_excel(report_folder + 'CorruptPerceptions.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd_corr_perc = pd_corr_perc[['Jurisdiction', 2015]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_gdp_freind_CP = pd.merge(left = df_freedom_gdp_freind, right =pd_corr_perc, left_on  = 'Country', right_on =  'Jurisdiction')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>APPEALS</th>\n",
       "      <th>CODE</th>\n",
       "      <th>COUNTRY_y</th>\n",
       "      <th>CSV</th>\n",
       "      <th>Count</th>\n",
       "      <th>Country</th>\n",
       "      <th>DATE</th>\n",
       "      <th>EXCEPTIONS &amp; REFUSALS</th>\n",
       "      <th>GDP (BILLIONS)</th>\n",
       "      <th>LAW</th>\n",
       "      <th>...</th>\n",
       "      <th>TOTAL</th>\n",
       "      <th>label-H</th>\n",
       "      <th>label_num_h</th>\n",
       "      <th>mean</th>\n",
       "      <th>sum</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>label_num</th>\n",
       "      <th>Jurisdiction</th>\n",
       "      <th>2015</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23.0</td>\n",
       "      <td>ALB</td>\n",
       "      <td>Albania</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>Albania</td>\n",
       "      <td>1999.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>13.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>127.0</td>\n",
       "      <td>0-20.25</td>\n",
       "      <td>0</td>\n",
       "      <td>60.562500</td>\n",
       "      <td>969</td>\n",
       "      <td>Albania&lt;br&gt;FCPA cases 2</td>\n",
       "      <td>0-2</td>\n",
       "      <td>0</td>\n",
       "      <td>Albania</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>DZA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>Algeria</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>227.8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0-20.25</td>\n",
       "      <td>0</td>\n",
       "      <td>56.333333</td>\n",
       "      <td>845</td>\n",
       "      <td>Algeria&lt;br&gt;FCPA cases 3</td>\n",
       "      <td>2-5</td>\n",
       "      <td>1</td>\n",
       "      <td>Algeria</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   APPEALS CODE COUNTRY_y  CSV  Count  Country    DATE  EXCEPTIONS & REFUSALS  \\\n",
       "0     23.0  ALB   Albania  NaN      2  Albania  1999.0                   27.0   \n",
       "1      NaN  DZA       NaN  NaN      3  Algeria     NaN                    NaN   \n",
       "\n",
       "   GDP (BILLIONS)  LAW  ...   TOTAL  label-H  label_num_h       mean  sum  \\\n",
       "0            13.4  NaN  ...   127.0  0-20.25            0  60.562500  969   \n",
       "1           227.8  NaN  ...     NaN  0-20.25            0  56.333333  845   \n",
       "\n",
       "                      text  label label_num  Jurisdiction  2015  \n",
       "0  Albania<br>FCPA cases 2    0-2         0       Albania    36  \n",
       "1  Algeria<br>FCPA cases 3    2-5         1       Algeria    36  \n",
       "\n",
       "[2 rows x 26 columns]"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_freedom_gdp_freind_CP.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_gdp_freind_CP.rename(columns={2015:'corrupt perception'}, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "writer = pd.ExcelWriter(report_folder + 'df_freedom_gdp_freind_CP.xlsx')\n",
    "df_freedom_gdp_freind_CP.to_excel(writer)\n",
    "writer.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_gdp_freind_CP = pd.read_excel(report_folder + 'df_freedom_gdp_freind_CP.xlsx')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Add corporate transparency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "#World bank indicators\n",
    "\n",
    "pd_indicators =pd.read_excel(report_folder + 'DBData-indicators.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd_indicators_cti = pd_indicators.loc[pd_indicators['Indicator Name'] == 'Extent of corporate transparency index (0-9)']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd_indicators_cti = pd_indicators_cti[['Country Name',2016]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_dgp_freind_cp_cti = pd.merge(left =df_freedom_gdp_freind_CP , right = pd_indicators_cti, left_on = 'Country', right_on='Country Name')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd_indicators_rel = pd_indicators.loc[pd_indicators['Indicator Name'] == 'Reliability of supply and transparency of tariff index (0-8)']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd_indicators_rel = pd_indicators_rel[['Country Name',2016]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_dgp_freind_cp_cti = pd.merge(left =df_freedom_dgp_freind_cp_cti , right = pd_indicators_rel, left_on = 'Country', right_on='Country Name')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['APPEALS', 'CODE', 'COUNTRY_y', 'CSV', 'Count', 'Country', 'DATE',\n",
       "       'EXCEPTIONS & REFUSALS', 'GDP (BILLIONS)', 'LAW',\n",
       "       'PROMOTIONAL MEASURES', 'RANKING POSITION', 'REQUESTING PROCEDURES',\n",
       "       'RIGHT OF ACCESS', 'SANCTIONS & PROTECTIONS', 'SCOPE', 'TOTAL',\n",
       "       'label-H', 'label_num_h', 'mean', 'sum', 'text', 'label', 'label_num',\n",
       "       'Jurisdiction', 'corrupt perception', 'Country Name_x', '2016_x',\n",
       "       'Country Name_y', '2016_y'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_freedom_dgp_freind_cp_cti.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_dgp_freind_cp_cti['corrupt perception'].replace(to_replace = '-', value = 0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_dgp_freind_cp_cti['2016_x'].fillna(0, inplace=True)\n",
    "df_freedom_dgp_freind_cp_cti['2016_y'].fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_dgp_freind_cp_cti.rename( columns = {'2016_x':'corporate transparency', '2016_y': 'Reliability of transparency'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Disclosure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd_indicators_discl = pd_indicators.loc[pd_indicators['Indicator Name'] == 'Extent of disclosure index (0 to 10)']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    " pd_indicators_discl = pd_indicators_discl[['Country Name', 2016]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_dgp_freind_cp_cti = pd.merge(left = df_freedom_dgp_freind_cp_cti, right = pd_indicators_discl, left_on = 'Country', right_on = 'Country Name')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_dgp_freind_cp_cti.rename( columns = {2016:'disclosure_index'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "writer = pd.ExcelWriter(report_folder + 'df_freedom_dgp_freind_cp_cti_rel.xlsx')\n",
    "df_freedom_dgp_freind_cp_cti.to_excel(writer)\n",
    "writer.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_dgp_freind_cp_cti = pd.read_excel(report_folder + 'df_freedom_dgp_freind_cp_cti_rel.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add GINI Index "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_gini = pd.read_excel(report_folder + 'Gini-index.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_dgp_freind_cp_cti_gini = pd.merge(left=df_freedom_dgp_freind_cp_cti, right = df_gini, left_on = 'Country', right_on = 'Country')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "writer = pd.ExcelWriter(report_folder + 'df_freedom_dgp_freind_cp_cti_gini.xlsx')\n",
    "df_freedom_dgp_freind_cp_cti_gini.to_excel(writer)\n",
    "writer.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_dgp_freind_cp_cti_gini = pd.read_excel(report_folder + 'df_freedom_dgp_freind_cp_cti_gini.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_dgp_freind_cp_cti_gini['disclosure_index'].fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_dgp_freind_cp_cti_gini['GINI'].fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_countrydate_rel_breach_an1 = pd.read_excel(report_folder + 'df_countrydate_rel_breach_an1.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_gdp_freind = df_countrydate_rel_breach_an1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "401"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_freedom_gdp_freind.sum_duration.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df_freedom_dgp_freind_cp_cti_gini' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-55-420398c08e6e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdf_freedom_gdp_freind\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdf_freedom_dgp_freind_cp_cti_gini\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'df_freedom_dgp_freind_cp_cti_gini' is not defined"
     ]
    }
   ],
   "source": [
    "df_freedom_gdp_freind = df_freedom_dgp_freind_cp_cti_gini"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Principal components\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. principal component (PC1)\n",
    "– the direction along which there is greatest variation\n",
    "• 2. principal component (PC2)\n",
    "– the direction with maximum variation left in data, orthogonal to\n",
    "the 1. PC\n",
    "• General about principal components\n",
    "– linear combinations of the original variables\n",
    "– uncorrelated with each other"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "import plotly.plotly as py\n",
    "from plotly.graph_objs import *\n",
    "import plotly.tools as tls\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_gdp_freind = df_freedom_gdp_freind.apply(lambda  row: label_rows(row),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_gdp_freind = df_freedom_gdp_freind.apply(lambda  row: label_rows_fibonacci(row),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_gdp_freind = df_freedom_gdp_freind.apply(lambda  row: label_rows_duration(row),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Count</th>\n",
       "      <th>GDP (BILLIONS)</th>\n",
       "      <th>PROMOTIONAL MEASURES</th>\n",
       "      <th>sum</th>\n",
       "      <th>label_num_h</th>\n",
       "      <th>label_num</th>\n",
       "      <th>corrupt perception</th>\n",
       "      <th>corporate transparency</th>\n",
       "      <th>Reliability of transparency</th>\n",
       "      <th>disclosure_index</th>\n",
       "      <th>GINI</th>\n",
       "      <th>sum_duration</th>\n",
       "      <th>Breaches_2017</th>\n",
       "      <th>label_dur</th>\n",
       "      <th>label_dur_n</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>13.4</td>\n",
       "      <td>10</td>\n",
       "      <td>969</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0-100</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>227.8</td>\n",
       "      <td>0</td>\n",
       "      <td>845</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>35.3</td>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "      <td>0-100</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>18</td>\n",
       "      <td>536.2</td>\n",
       "      <td>11</td>\n",
       "      <td>82</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>42.7</td>\n",
       "      <td>91</td>\n",
       "      <td>1</td>\n",
       "      <td>0-100</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Count  GDP (BILLIONS)  PROMOTIONAL MEASURES  sum  label_num_h  label_num  \\\n",
       "0      2            13.4                    10  969            0          1   \n",
       "1      3           227.8                     0  845            1          1   \n",
       "2     18           536.2                    11   82            2          2   \n",
       "\n",
       "   corrupt perception  corporate transparency  Reliability of transparency  \\\n",
       "0                  36                     7.0                          0.0   \n",
       "1                  36                     2.0                          5.0   \n",
       "2                  32                     7.0                          5.0   \n",
       "\n",
       "   disclosure_index  GINI  sum_duration  Breaches_2017 label_dur  label_dur_n  \n",
       "0               9.0  29.0            16              0     0-100            0  \n",
       "1               4.0  35.3            25              0     0-100            0  \n",
       "2               7.0  42.7            91              1     0-100            0  "
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df_freedom_gdp_freind.iloc[:,[4, 8,10, 20,18, 23, 25, 27, 29, 31, 32]].head(3)\n",
    "df_freedom_gdp_freind.iloc[:,[1,2, 3,4, 5,6, 7, 8, 9, 10, 11, 13,15,19]].head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "#X = df_freedom_gdp_freind.iloc[:,[ 10, 25, 27,29, 31, 32]].values \n",
    "#y = df_freedom_gdp_freind.iloc[:,18].values\n",
    "X = df_freedom_gdp_freind.iloc[:,[ 3, 7, 8, 9, 10, 11, 13,15]].values \n",
    "y = df_freedom_gdp_freind.iloc[:,5].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_std = StandardScaler().fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA as sklearnPCA\n",
    "sklearn_pca = sklearnPCA(n_components = 3 )\n",
    "Y_sklearn = sklearn_pca.fit_transform(X_std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "loadings = sklearn_pca.components_.T * np.sqrt(sklearn_pca.explained_variance_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.26052669,  0.54663111, -0.40254441],\n",
       "       [-0.67376105, -0.44362595, -0.28165875],\n",
       "       [-0.87449751,  0.11425073,  0.11307798],\n",
       "       [-0.74128254, -0.09625049,  0.2402753 ],\n",
       "       [-0.51331217,  0.3012411 ,  0.23287446],\n",
       "       [ 0.50835361,  0.46981603,  0.01677522],\n",
       "       [-0.27051014,  0.81001715, -0.06178442],\n",
       "       [ 0.08326586,  0.08398617,  0.92054869]])"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loadings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\" seamless=\"seamless\" src=\"https://plot.ly/~tloskutova/134.embed\" height=\"525px\" width=\"100%\"></iframe>"
      ],
      "text/plain": [
       "<plotly.tools.PlotlyDisplay object>"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 3-dimensional\n",
    "traces = []\n",
    "\n",
    "lbl_dict = {0:'0-1', \n",
    "          1:'2-5', \n",
    "          2:'6-9',\n",
    "          3:'10-..'}\n",
    "\n",
    "for name in [0,1,2,3]:\n",
    "\n",
    "    trace = Scatter3d(\n",
    "        x=Y_sklearn[y==name,0],\n",
    "        y=Y_sklearn[y==name,1],\n",
    "        z=Y_sklearn[y==name,2],\n",
    "        mode='markers',\n",
    "        name=lbl_dict[name],\n",
    "        marker=Marker(\n",
    "            size=12,\n",
    "            line=Line(\n",
    "                color='rgba(217, 217, 217, 0.14)',\n",
    "                width=0.5),\n",
    "            opacity=0.8))\n",
    "    traces.append(trace)\n",
    "\n",
    "\n",
    "data = Data(traces)\n",
    "layout = Layout(xaxis=XAxis(title='PC1', showline=False),\n",
    "                yaxis=YAxis(title='PC2', showline=False),)\n",
    "                #zaxis =ZAxis(title='PC3', showline=False))\n",
    "fig = Figure(data=data, layout=layout)\n",
    "py.iplot(fig, validate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2-dimensional\n",
    "from sklearn.decomposition import PCA as sklearnPCA\n",
    "sklearn_pca = sklearnPCA(n_components = 2 )\n",
    "Y_sklearn = sklearn_pca.fit_transform(X_std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\" seamless=\"seamless\" src=\"https://plot.ly/~tloskutova/132.embed\" height=\"525px\" width=\"100%\"></iframe>"
      ],
      "text/plain": [
       "<plotly.tools.PlotlyDisplay object>"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "traces = []\n",
    "\n",
    "lbl_dict = {0:'0-1', \n",
    "          1:'2-5', \n",
    "          2:'6-9',\n",
    "          3:'10-..'}\n",
    "\n",
    "for name in [0,1,2,3]:\n",
    "\n",
    "    trace = Scatter(\n",
    "        x=Y_sklearn[y==name,0],\n",
    "        y=Y_sklearn[y==name,1],\n",
    "        mode='markers',\n",
    "        name=lbl_dict[name],\n",
    "        marker=Marker(\n",
    "            size=12,\n",
    "            line=Line(\n",
    "                color='rgba(217, 217, 217, 0.14)',\n",
    "                width=0.5),\n",
    "            opacity=0.8))\n",
    "    traces.append(trace)\n",
    "\n",
    "\n",
    "data = Data(traces)\n",
    "layout = Layout(xaxis=XAxis(title='PC1', showline=False),\n",
    "                yaxis=YAxis(title='PC2', showline=False))\n",
    "fig = Figure(data=data, layout=layout)\n",
    "py.iplot(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "explained variance ratio (first two components): [0.30069845 0.18408508]\n"
     ]
    }
   ],
   "source": [
    "# With histogramming\n",
    "# X = df_freedom_gdp_freind.iloc[:,[4, 8,10, 20,18]].values # add 25 and 27\n",
    "# y = df_freedom_gdp_freind.iloc[:,18].values, explanation ratio is[0.41807147 0.1815613 ]\n",
    "\n",
    "\n",
    "### histogram\n",
    "# explanation ratio for [18, 10, 27, 31, 32]  is [0.34349439 0.25555496]\n",
    "\n",
    "### fibonacci\n",
    "# explanation ratio for [18, 10, 27, 31, 32]  is [0.34349439 0.25555496]\n",
    "# explanation ratio for [:,[10, 27,31, 32]] is [0.41548236 0.25665924]\n",
    "# explanation ratio for [:,[18, 10, 27,31, 32]] is [0.34797284 0.25060875]\n",
    "# explanation ratio for [:,[27,31, 32]] is [0.54065552 0.28529374] Just GINI\n",
    "\n",
    "#  explanation ratio for [:,[ 10, 25, 27,29, 31, 32]]. is [0.39249657 0.17561419 0.1552987 ]\n",
    "\n",
    "print('explained variance ratio (first two components): %s'\n",
    "      % str(sklearn_pca.explained_variance_ratio_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Duration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Country', 'Count', 'GDP (BILLIONS)', 'PROMOTIONAL MEASURES', 'sum',\n",
       "       'label_num_h', 'label_num', 'corrupt perception',\n",
       "       'corporate transparency', 'Reliability of transparency',\n",
       "       'disclosure_index', 'GINI', 'mean_duration', 'sum_duration',\n",
       "       'Reliability2016', 'Breaches_2017', 'label', 'label_h', 'label_dur',\n",
       "       'label_dur_n'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_freedom_gdp_freind.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "#X = df_freedom_gdp_freind.iloc[:,[ 10, 25, 27,29, 31, 32]].values \n",
    "#y = df_freedom_gdp_freind.iloc[:,18].values\n",
    "#X = df_freedom_gdp_freind.iloc[:,[ 3, 7, 8, 9, 10, 15]].values \n",
    "X = df_freedom_gdp_freind.iloc[:,[ 2, 3, 8, 10]].values \n",
    "y = df_freedom_gdp_freind.iloc[:,19].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_std = StandardScaler().fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "loadings = sklearn_pca.components_.T * np.sqrt(sklearn_pca.explained_variance_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.93639364,  0.1967306 ],\n",
       "       [ 0.00683774, -0.59411204],\n",
       "       [ 0.41656482, -0.72637559],\n",
       "       [ 0.41197799, -0.59155169],\n",
       "       [ 0.86184563,  0.42482538]])"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loadings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2-dimensional\n",
    "from sklearn.decomposition import PCA as sklearnPCA\n",
    "sklearn_pca = sklearnPCA(n_components = 2 )\n",
    "Y_sklearn = sklearn_pca.fit_transform(X_std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "explained variance ratio (first two components): [0.39897765 0.25517798]\n"
     ]
    }
   ],
   "source": [
    "print('explained variance ratio (first two components): %s'\n",
    "      % str(sklearn_pca.explained_variance_ratio_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\" seamless=\"seamless\" src=\"https://plot.ly/~tloskutova/140.embed\" height=\"525px\" width=\"100%\"></iframe>"
      ],
      "text/plain": [
       "<plotly.tools.PlotlyDisplay object>"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "traces = []\n",
    "\n",
    "lbl_dict = {0:'0-100', \n",
    "          1:'101-200', \n",
    "          2:'201-300',\n",
    "          3:'300-..'}\n",
    "\n",
    "for name in [0,1,2,3]:\n",
    "\n",
    "    trace = Scatter(\n",
    "        x=Y_sklearn[y==name,0],\n",
    "        y=Y_sklearn[y==name,1],\n",
    "        mode='markers',\n",
    "        name=lbl_dict[name],\n",
    "        marker=Marker(\n",
    "            size=12,\n",
    "            line=Line(\n",
    "                color='rgba(217, 217, 217, 0.14)',\n",
    "                width=0.5),\n",
    "            opacity=0.8))\n",
    "    traces.append(trace)\n",
    "\n",
    "\n",
    "data = Data(traces)\n",
    "layout = Layout(xaxis=XAxis(title='PC1', showline=False),\n",
    "                yaxis=YAxis(title='PC2', showline=False))\n",
    "fig = Figure(data=data, layout=layout)\n",
    "py.iplot(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "explained variance ratio (first two components): [0.38734754 0.28607484]\n"
     ]
    }
   ],
   "source": [
    "#  explanation ratio for [ 3, 7, 8, 9, 10, 11, 13,15]. is [0.30069845 0.18408508]\n",
    "# explanation ratio for [ 3, 7, 8, 9, 10, 11, 15]. is  [0.30069845 0.18408508 0.15017457]\n",
    "\n",
    "print('explained variance ratio (first two components): %s'\n",
    "      % str(sklearn_pca.explained_variance_ratio_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## With Fibonacci"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [4, 8,10, 20,18, 23, 25, 27, 29, 31, 32]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "4 Count\t8 GDP (BILLIONS)\t10 PROMOTIONAL MEASURES\t20 sum\t18label_num_h\t23label_num\t25corrupt perception\t27corporate transparency\t29Reliability of transparency\t31disclosure_index\t32GINI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "#X = df_freedom_gdp_freind.iloc[:,[27,31, 32]].values \n",
    "#y = df_freedom_gdp_freind.iloc[:,23].values\n",
    "X = df_freedom_gdp_freind.iloc[:,[ 3, 7, 8, 9, 10, 11, 13,15]].values \n",
    "y = df_freedom_gdp_freind.iloc[:,5].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "X_std = StandardScaler().fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA as sklearnPCA\n",
    "sklearn_pca = sklearnPCA(n_components = 3 )\n",
    "Y_sklearn = sklearn_pca.fit_transform(X_std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\" seamless=\"seamless\" src=\"https://plot.ly/~tloskutova/113.embed\" height=\"525px\" width=\"100%\"></iframe>"
      ],
      "text/plain": [
       "<plotly.tools.PlotlyDisplay object>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 3-dimensional\n",
    "traces = []\n",
    "\n",
    "lbl_dict = {0:'0-2', \n",
    "          1:'3-12', \n",
    "          2:'13-54',\n",
    "          3:'55-..'}\n",
    "\n",
    "for name in [0,1,2,3]:\n",
    "\n",
    "    trace = Scatter3d(\n",
    "        x=Y_sklearn[y==name,0],\n",
    "        y=Y_sklearn[y==name,1],\n",
    "        z=Y_sklearn[y==name,2],\n",
    "        mode='markers',\n",
    "        name=lbl_dict[name],\n",
    "        marker=Marker(\n",
    "            size=12,\n",
    "            line=Line(\n",
    "                color='rgba(217, 217, 217, 0.14)',\n",
    "                width=0.5),\n",
    "            opacity=0.8))\n",
    "    traces.append(trace)\n",
    "\n",
    "\n",
    "data = Data(traces)\n",
    "layout = Layout(xaxis=XAxis(title='PC1', showline=False),\n",
    "                yaxis=YAxis(title='PC2', showline=False),)\n",
    "                #zaxis =ZAxis(title='PC3', showline=False))\n",
    "fig = Figure(data=data, layout=layout)\n",
    "py.iplot(fig, validate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2-dimensional\n",
    "from sklearn.decomposition import PCA as sklearnPCA\n",
    "sklearn_pca = sklearnPCA(n_components = 2 )\n",
    "Y_sklearn = sklearn_pca.fit_transform(X_std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\" seamless=\"seamless\" src=\"https://plot.ly/~tloskutova/115.embed\" height=\"525px\" width=\"100%\"></iframe>"
      ],
      "text/plain": [
       "<plotly.tools.PlotlyDisplay object>"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "traces = []\n",
    "\n",
    "lbl_dict = {0:'0-2', \n",
    "          1:'3-12', \n",
    "          2:'13-54',\n",
    "          3:'55-..'}\n",
    "\n",
    "for name in [0,1,2,3]:\n",
    "\n",
    "    trace = Scatter(\n",
    "        x=Y_sklearn[y==name,0],\n",
    "        y=Y_sklearn[y==name,1],\n",
    "        mode='markers',\n",
    "        name=lbl_dict[name],\n",
    "        marker=Marker(\n",
    "            size=12,\n",
    "            line=Line(\n",
    "                color='rgba(217, 217, 217, 0.14)',\n",
    "                width=0.5),\n",
    "            opacity=0.8))\n",
    "    traces.append(trace)\n",
    "\n",
    "\n",
    "data = Data(traces)\n",
    "layout = Layout(xaxis=XAxis(title='PC1', showline=False),\n",
    "                yaxis=YAxis(title='PC2', showline=False))\n",
    "fig = Figure(data=data, layout=layout)\n",
    "py.iplot(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "loadings = sklearn_pca.components_.T * np.sqrt(sklearn_pca.explained_variance_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_gdp_freind.iloc[:,[ 8, 13]];"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.26052669,  0.54663111],\n",
       "       [-0.67376105, -0.44362595],\n",
       "       [-0.87449751,  0.11425073],\n",
       "       [-0.74128254, -0.09625049],\n",
       "       [-0.51331217,  0.3012411 ],\n",
       "       [ 0.50835361,  0.46981603],\n",
       "       [-0.27051014,  0.81001715],\n",
       "       [ 0.08326586,  0.08398617]])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df_freedom_gdp_freind.iloc[:,[ 3, 7, 8, 9, 10, 11, 13,15]].values \n",
    "loadings #in loadings for PC1 -var 2 (corporate transparency) is the most important and for PC2 -var 6 (sum_duration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "explained variance ratio (first two components): [0.30069845 0.18408508]\n"
     ]
    }
   ],
   "source": [
    "# Without histogramming, explanation ratio for [:,[4, 8,10, 20, 23]] is [0.50623105 0.2192345 ]\n",
    "# explanation ratio for [:,[4, 10, 20, 23]] is[0.62594315 0.24356224]\n",
    "# explanation ratio for [:,[8, 10, 23]] [0.38864472 0.32706249]\n",
    "# explanation ratio for [:,[8, 10, 23, 25, 27]] is[0.3138473  0.25207769]\n",
    "# explanation ratio for [:,[8, 10, 20, 23, 25, 27]] is [0.33988602 0.25764517]\n",
    "# explanation ratio for TI [8, 20, 23, 25] is [0.49793404 0.25904744]\n",
    "\n",
    "# explanation ratio for (openness of corporate info ) [8, 20, 23, 27]]  [0.48850924 0.29878065]\n",
    "# explanation ratio for (openness of corporate info ) and reliab [8, 20, 23, 27, 29 ]] is [0.40749641 0.32309442]\n",
    "# explanation ratio for [30, 20, 23,  29] (reliability) [0.47696987 0.3142275 ]\n",
    "\n",
    "# Swap GDP for GINI\n",
    "# explanation ratio for [30, 20, 23,  29] (reliability) [0.50225409 0.29457465]\n",
    "\n",
    "##################################\n",
    "#  No GDP-GINI \n",
    "# explanation ratio for[23, 10, 27, 31] [0.40943658 0.2414831 ] - does not cluster\n",
    "# explanation ratio for[23, 10, 27, 31, 32] [0.34349439 0.25555496] - does not cluster\n",
    "\n",
    "\n",
    "# explanation ratio for 27, 31, 32] is [0.54065552 0.28529374]\n",
    "print('explained variance ratio (first two components): %s'\n",
    "      % str(sklearn_pca.explained_variance_ratio_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cumulative explained variance ratio: [0.54065552 0.82594926 1.        ]\n"
     ]
    }
   ],
   "source": [
    "print('cumulative explained variance ratio: %s'\n",
    "      % str(sklearn_pca.explained_variance_ratio_.cumsum()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add which variables contribute to variation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Correlation matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import pearsonr\n",
    "import pandas as pd\n",
    "\n",
    "def calculate_pvalues(df):\n",
    "    df = df.dropna()._get_numeric_data()\n",
    "    dfcols = pd.DataFrame(columns=df.columns)\n",
    "    pvalues = dfcols.transpose().join(dfcols, how='outer')\n",
    "    for r in df.columns:\n",
    "        for c in df.columns:\n",
    "            pvalues[r][c] = round(pearsonr(df[r], df[c])[1], 4)\n",
    "    return pvalues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_final = df_freedom_gdp_freind.iloc[:,[4, 8,10, 20,18, 23, 25, 27, 29, 31, 32]]\n",
    "df_final = df_freedom_gdp_freind.iloc[:,[1,2, 3, 5,6,7,8,9,10,11,12,13,14, 15]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Count</th>\n",
       "      <th>GDP (BILLIONS)</th>\n",
       "      <th>PROMOTIONAL MEASURES</th>\n",
       "      <th>label_num_h</th>\n",
       "      <th>label_num</th>\n",
       "      <th>corrupt perception</th>\n",
       "      <th>corporate transparency</th>\n",
       "      <th>Reliability of transparency</th>\n",
       "      <th>disclosure_index</th>\n",
       "      <th>GINI</th>\n",
       "      <th>mean_duration</th>\n",
       "      <th>sum_duration</th>\n",
       "      <th>Reliability2016</th>\n",
       "      <th>Breaches_2017</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>13.4</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>16</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>227.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>35.3</td>\n",
       "      <td>8.333333</td>\n",
       "      <td>25</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>18</td>\n",
       "      <td>536.2</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>42.7</td>\n",
       "      <td>5.055556</td>\n",
       "      <td>91</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Count  GDP (BILLIONS)  PROMOTIONAL MEASURES  label_num_h  label_num  \\\n",
       "0      2            13.4                    10            0          1   \n",
       "1      3           227.8                     0            1          1   \n",
       "2     18           536.2                    11            2          2   \n",
       "\n",
       "   corrupt perception  corporate transparency  Reliability of transparency  \\\n",
       "0                  36                     7.0                          0.0   \n",
       "1                  36                     2.0                          5.0   \n",
       "2                  32                     7.0                          5.0   \n",
       "\n",
       "   disclosure_index  GINI  mean_duration  sum_duration  Reliability2016  \\\n",
       "0               9.0  29.0       8.000000            16              0.0   \n",
       "1               4.0  35.3       8.333333            25              5.0   \n",
       "2               7.0  42.7       5.055556            91              5.0   \n",
       "\n",
       "   Breaches_2017  \n",
       "0              0  \n",
       "1              0  \n",
       "2              1  "
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.set_printoptions(threshold=np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Count</th>\n",
       "      <th>GDP (BILLIONS)</th>\n",
       "      <th>PROMOTIONAL MEASURES</th>\n",
       "      <th>label_num_h</th>\n",
       "      <th>label_num</th>\n",
       "      <th>corrupt perception</th>\n",
       "      <th>corporate transparency</th>\n",
       "      <th>Reliability of transparency</th>\n",
       "      <th>disclosure_index</th>\n",
       "      <th>GINI</th>\n",
       "      <th>mean_duration</th>\n",
       "      <th>sum_duration</th>\n",
       "      <th>Reliability2016</th>\n",
       "      <th>Breaches_2017</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Count</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0019</td>\n",
       "      <td>0.0205</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.242</td>\n",
       "      <td>0.0323</td>\n",
       "      <td>0.4202</td>\n",
       "      <td>0.0265</td>\n",
       "      <td>0.0749</td>\n",
       "      <td>0.5321</td>\n",
       "      <td>0</td>\n",
       "      <td>0.4202</td>\n",
       "      <td>0.6082</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GDP (BILLIONS)</th>\n",
       "      <td>0.0019</td>\n",
       "      <td>0</td>\n",
       "      <td>0.9448</td>\n",
       "      <td>0.2897</td>\n",
       "      <td>0.8207</td>\n",
       "      <td>0.3425</td>\n",
       "      <td>0.0608</td>\n",
       "      <td>0.0192</td>\n",
       "      <td>0.1285</td>\n",
       "      <td>0.6633</td>\n",
       "      <td>0.0739</td>\n",
       "      <td>0.0048</td>\n",
       "      <td>0.0192</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PROMOTIONAL MEASURES</th>\n",
       "      <td>0.0205</td>\n",
       "      <td>0.9448</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0452</td>\n",
       "      <td>0.2033</td>\n",
       "      <td>0.9651</td>\n",
       "      <td>0.079</td>\n",
       "      <td>0.632</td>\n",
       "      <td>0.8664</td>\n",
       "      <td>0.7609</td>\n",
       "      <td>0.3276</td>\n",
       "      <td>0.0171</td>\n",
       "      <td>0.632</td>\n",
       "      <td>0.2672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label_num_h</th>\n",
       "      <td>0</td>\n",
       "      <td>0.2897</td>\n",
       "      <td>0.0452</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1885</td>\n",
       "      <td>0.0339</td>\n",
       "      <td>0.3268</td>\n",
       "      <td>0.0966</td>\n",
       "      <td>0.2007</td>\n",
       "      <td>0.1068</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3268</td>\n",
       "      <td>0.2581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label_num</th>\n",
       "      <td>0</td>\n",
       "      <td>0.8207</td>\n",
       "      <td>0.2033</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0759</td>\n",
       "      <td>0.209</td>\n",
       "      <td>0.3257</td>\n",
       "      <td>0.2051</td>\n",
       "      <td>0.0061</td>\n",
       "      <td>0.045</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3257</td>\n",
       "      <td>0.1474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>corrupt perception</th>\n",
       "      <td>0.242</td>\n",
       "      <td>0.3425</td>\n",
       "      <td>0.9651</td>\n",
       "      <td>0.1885</td>\n",
       "      <td>0.0759</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0009</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2781</td>\n",
       "      <td>0.008</td>\n",
       "      <td>0.8022</td>\n",
       "      <td>0.2656</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>corporate transparency</th>\n",
       "      <td>0.0323</td>\n",
       "      <td>0.0608</td>\n",
       "      <td>0.079</td>\n",
       "      <td>0.0339</td>\n",
       "      <td>0.209</td>\n",
       "      <td>0.0009</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0026</td>\n",
       "      <td>0.0004</td>\n",
       "      <td>0.2621</td>\n",
       "      <td>0.0152</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Reliability of transparency</th>\n",
       "      <td>0.4202</td>\n",
       "      <td>0.0192</td>\n",
       "      <td>0.632</td>\n",
       "      <td>0.3268</td>\n",
       "      <td>0.3257</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0.1163</td>\n",
       "      <td>0.4659</td>\n",
       "      <td>0.3254</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>disclosure_index</th>\n",
       "      <td>0.0265</td>\n",
       "      <td>0.1285</td>\n",
       "      <td>0.8664</td>\n",
       "      <td>0.0966</td>\n",
       "      <td>0.2051</td>\n",
       "      <td>0.2781</td>\n",
       "      <td>0.0026</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1877</td>\n",
       "      <td>0.6087</td>\n",
       "      <td>0.0166</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0.605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GINI</th>\n",
       "      <td>0.0749</td>\n",
       "      <td>0.6633</td>\n",
       "      <td>0.7609</td>\n",
       "      <td>0.2007</td>\n",
       "      <td>0.0061</td>\n",
       "      <td>0.008</td>\n",
       "      <td>0.0004</td>\n",
       "      <td>0.1163</td>\n",
       "      <td>0.1877</td>\n",
       "      <td>0</td>\n",
       "      <td>0.4101</td>\n",
       "      <td>0.1522</td>\n",
       "      <td>0.1163</td>\n",
       "      <td>0.5446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_duration</th>\n",
       "      <td>0.5321</td>\n",
       "      <td>0.0739</td>\n",
       "      <td>0.3276</td>\n",
       "      <td>0.1068</td>\n",
       "      <td>0.045</td>\n",
       "      <td>0.8022</td>\n",
       "      <td>0.2621</td>\n",
       "      <td>0.4659</td>\n",
       "      <td>0.6087</td>\n",
       "      <td>0.4101</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1095</td>\n",
       "      <td>0.4659</td>\n",
       "      <td>0.155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sum_duration</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0048</td>\n",
       "      <td>0.0171</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2656</td>\n",
       "      <td>0.0152</td>\n",
       "      <td>0.3254</td>\n",
       "      <td>0.0166</td>\n",
       "      <td>0.1522</td>\n",
       "      <td>0.1095</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3254</td>\n",
       "      <td>0.5627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Reliability2016</th>\n",
       "      <td>0.4202</td>\n",
       "      <td>0.0192</td>\n",
       "      <td>0.632</td>\n",
       "      <td>0.3268</td>\n",
       "      <td>0.3257</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0.1163</td>\n",
       "      <td>0.4659</td>\n",
       "      <td>0.3254</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Breaches_2017</th>\n",
       "      <td>0.6082</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2672</td>\n",
       "      <td>0.2581</td>\n",
       "      <td>0.1474</td>\n",
       "      <td>0.0148</td>\n",
       "      <td>0.8531</td>\n",
       "      <td>0.2392</td>\n",
       "      <td>0.605</td>\n",
       "      <td>0.5446</td>\n",
       "      <td>0.155</td>\n",
       "      <td>0.5627</td>\n",
       "      <td>0.2392</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              Count GDP (BILLIONS) PROMOTIONAL MEASURES  \\\n",
       "Count                             0         0.0019               0.0205   \n",
       "GDP (BILLIONS)               0.0019              0               0.9448   \n",
       "PROMOTIONAL MEASURES         0.0205         0.9448                    0   \n",
       "label_num_h                       0         0.2897               0.0452   \n",
       "label_num                         0         0.8207               0.2033   \n",
       "corrupt perception            0.242         0.3425               0.9651   \n",
       "corporate transparency       0.0323         0.0608                0.079   \n",
       "Reliability of transparency  0.4202         0.0192                0.632   \n",
       "disclosure_index             0.0265         0.1285               0.8664   \n",
       "GINI                         0.0749         0.6633               0.7609   \n",
       "mean_duration                0.5321         0.0739               0.3276   \n",
       "sum_duration                      0         0.0048               0.0171   \n",
       "Reliability2016              0.4202         0.0192                0.632   \n",
       "Breaches_2017                0.6082              0               0.2672   \n",
       "\n",
       "                            label_num_h label_num corrupt perception  \\\n",
       "Count                                 0         0              0.242   \n",
       "GDP (BILLIONS)                   0.2897    0.8207             0.3425   \n",
       "PROMOTIONAL MEASURES             0.0452    0.2033             0.9651   \n",
       "label_num_h                           0         0             0.1885   \n",
       "label_num                             0         0             0.0759   \n",
       "corrupt perception               0.1885    0.0759                  0   \n",
       "corporate transparency           0.0339     0.209             0.0009   \n",
       "Reliability of transparency      0.3268    0.3257                  0   \n",
       "disclosure_index                 0.0966    0.2051             0.2781   \n",
       "GINI                             0.2007    0.0061              0.008   \n",
       "mean_duration                    0.1068     0.045             0.8022   \n",
       "sum_duration                          0         0             0.2656   \n",
       "Reliability2016                  0.3268    0.3257                  0   \n",
       "Breaches_2017                    0.2581    0.1474             0.0148   \n",
       "\n",
       "                            corporate transparency  \\\n",
       "Count                                       0.0323   \n",
       "GDP (BILLIONS)                              0.0608   \n",
       "PROMOTIONAL MEASURES                         0.079   \n",
       "label_num_h                                 0.0339   \n",
       "label_num                                    0.209   \n",
       "corrupt perception                          0.0009   \n",
       "corporate transparency                           0   \n",
       "Reliability of transparency                      0   \n",
       "disclosure_index                            0.0026   \n",
       "GINI                                        0.0004   \n",
       "mean_duration                               0.2621   \n",
       "sum_duration                                0.0152   \n",
       "Reliability2016                                  0   \n",
       "Breaches_2017                               0.8531   \n",
       "\n",
       "                            Reliability of transparency disclosure_index  \\\n",
       "Count                                            0.4202           0.0265   \n",
       "GDP (BILLIONS)                                   0.0192           0.1285   \n",
       "PROMOTIONAL MEASURES                              0.632           0.8664   \n",
       "label_num_h                                      0.3268           0.0966   \n",
       "label_num                                        0.3257           0.2051   \n",
       "corrupt perception                                    0           0.2781   \n",
       "corporate transparency                                0           0.0026   \n",
       "Reliability of transparency                           0           0.3608   \n",
       "disclosure_index                                 0.3608                0   \n",
       "GINI                                             0.1163           0.1877   \n",
       "mean_duration                                    0.4659           0.6087   \n",
       "sum_duration                                     0.3254           0.0166   \n",
       "Reliability2016                                       0           0.3608   \n",
       "Breaches_2017                                    0.2392            0.605   \n",
       "\n",
       "                               GINI mean_duration sum_duration  \\\n",
       "Count                        0.0749        0.5321            0   \n",
       "GDP (BILLIONS)               0.6633        0.0739       0.0048   \n",
       "PROMOTIONAL MEASURES         0.7609        0.3276       0.0171   \n",
       "label_num_h                  0.2007        0.1068            0   \n",
       "label_num                    0.0061         0.045            0   \n",
       "corrupt perception            0.008        0.8022       0.2656   \n",
       "corporate transparency       0.0004        0.2621       0.0152   \n",
       "Reliability of transparency  0.1163        0.4659       0.3254   \n",
       "disclosure_index             0.1877        0.6087       0.0166   \n",
       "GINI                              0        0.4101       0.1522   \n",
       "mean_duration                0.4101             0       0.1095   \n",
       "sum_duration                 0.1522        0.1095            0   \n",
       "Reliability2016              0.1163        0.4659       0.3254   \n",
       "Breaches_2017                0.5446         0.155       0.5627   \n",
       "\n",
       "                            Reliability2016 Breaches_2017  \n",
       "Count                                0.4202        0.6082  \n",
       "GDP (BILLIONS)                       0.0192             0  \n",
       "PROMOTIONAL MEASURES                  0.632        0.2672  \n",
       "label_num_h                          0.3268        0.2581  \n",
       "label_num                            0.3257        0.1474  \n",
       "corrupt perception                        0        0.0148  \n",
       "corporate transparency                    0        0.8531  \n",
       "Reliability of transparency               0        0.2392  \n",
       "disclosure_index                     0.3608         0.605  \n",
       "GINI                                 0.1163        0.5446  \n",
       "mean_duration                        0.4659         0.155  \n",
       "sum_duration                         0.3254        0.5627  \n",
       "Reliability2016                           0        0.2392  \n",
       "Breaches_2017                        0.2392             0  "
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calculate_pvalues(df_final)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#corr_matrix = np.corrcoef(df_freedom_gdp_freind.iloc[:,[4, 8,10, 20, 18, 23, 25, 27, 29, 30]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- CORRELATIONS ---------------\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Count</th>\n",
       "      <th>GDP (BILLIONS)</th>\n",
       "      <th>PROMOTIONAL MEASURES</th>\n",
       "      <th>label_num_h</th>\n",
       "      <th>label_num</th>\n",
       "      <th>corrupt perception</th>\n",
       "      <th>corporate transparency</th>\n",
       "      <th>Reliability of transparency</th>\n",
       "      <th>disclosure_index</th>\n",
       "      <th>GINI</th>\n",
       "      <th>mean_duration</th>\n",
       "      <th>sum_duration</th>\n",
       "      <th>Reliability2016</th>\n",
       "      <th>Breaches_2017</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Count</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.355255</td>\n",
       "      <td>0.270428</td>\n",
       "      <td>0.739385</td>\n",
       "      <td>0.704295</td>\n",
       "      <td>-0.131057</td>\n",
       "      <td>0.248997</td>\n",
       "      <td>0.098424</td>\n",
       "      <td>0.255426</td>\n",
       "      <td>0.207678</td>\n",
       "      <td>0.072200</td>\n",
       "      <td>0.986011</td>\n",
       "      <td>0.095116</td>\n",
       "      <td>-0.060109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GDP (BILLIONS)</th>\n",
       "      <td>0.355255</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.003345</td>\n",
       "      <td>0.123195</td>\n",
       "      <td>0.027712</td>\n",
       "      <td>-0.104690</td>\n",
       "      <td>0.221086</td>\n",
       "      <td>0.273817</td>\n",
       "      <td>0.180857</td>\n",
       "      <td>0.050826</td>\n",
       "      <td>-0.210403</td>\n",
       "      <td>0.325120</td>\n",
       "      <td>0.271757</td>\n",
       "      <td>0.824477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PROMOTIONAL MEASURES</th>\n",
       "      <td>0.270428</td>\n",
       "      <td>-0.003345</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.226058</td>\n",
       "      <td>0.151793</td>\n",
       "      <td>0.016669</td>\n",
       "      <td>0.229670</td>\n",
       "      <td>0.075510</td>\n",
       "      <td>0.057571</td>\n",
       "      <td>0.033414</td>\n",
       "      <td>0.107545</td>\n",
       "      <td>0.277116</td>\n",
       "      <td>0.056590</td>\n",
       "      <td>-0.127463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label_num_h</th>\n",
       "      <td>0.739385</td>\n",
       "      <td>0.123195</td>\n",
       "      <td>0.226058</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.796237</td>\n",
       "      <td>-0.158651</td>\n",
       "      <td>0.230799</td>\n",
       "      <td>0.108476</td>\n",
       "      <td>0.175231</td>\n",
       "      <td>0.150950</td>\n",
       "      <td>0.190506</td>\n",
       "      <td>0.753384</td>\n",
       "      <td>0.115572</td>\n",
       "      <td>-0.133604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label_num</th>\n",
       "      <td>0.704295</td>\n",
       "      <td>0.027712</td>\n",
       "      <td>0.151793</td>\n",
       "      <td>0.796237</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.200490</td>\n",
       "      <td>0.149846</td>\n",
       "      <td>0.118496</td>\n",
       "      <td>0.150387</td>\n",
       "      <td>0.315164</td>\n",
       "      <td>0.232075</td>\n",
       "      <td>0.720580</td>\n",
       "      <td>0.115848</td>\n",
       "      <td>-0.169610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>corrupt perception</th>\n",
       "      <td>-0.131057</td>\n",
       "      <td>-0.104690</td>\n",
       "      <td>0.016669</td>\n",
       "      <td>-0.158651</td>\n",
       "      <td>-0.200490</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.400648</td>\n",
       "      <td>0.489273</td>\n",
       "      <td>0.167003</td>\n",
       "      <td>-0.304511</td>\n",
       "      <td>0.021244</td>\n",
       "      <td>-0.125331</td>\n",
       "      <td>0.476767</td>\n",
       "      <td>-0.276363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>corporate transparency</th>\n",
       "      <td>0.248997</td>\n",
       "      <td>0.221086</td>\n",
       "      <td>0.229670</td>\n",
       "      <td>0.230799</td>\n",
       "      <td>0.149846</td>\n",
       "      <td>0.400648</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.576450</td>\n",
       "      <td>0.389091</td>\n",
       "      <td>-0.391238</td>\n",
       "      <td>0.116547</td>\n",
       "      <td>0.279257</td>\n",
       "      <td>0.563356</td>\n",
       "      <td>0.024738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Reliability of transparency</th>\n",
       "      <td>0.098424</td>\n",
       "      <td>0.273817</td>\n",
       "      <td>0.075510</td>\n",
       "      <td>0.108476</td>\n",
       "      <td>0.118496</td>\n",
       "      <td>0.489273</td>\n",
       "      <td>0.576450</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.144745</td>\n",
       "      <td>-0.184512</td>\n",
       "      <td>0.077666</td>\n",
       "      <td>0.118291</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.139164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>disclosure_index</th>\n",
       "      <td>0.255426</td>\n",
       "      <td>0.180857</td>\n",
       "      <td>0.057571</td>\n",
       "      <td>0.175231</td>\n",
       "      <td>0.150387</td>\n",
       "      <td>0.167003</td>\n",
       "      <td>0.389091</td>\n",
       "      <td>0.144745</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.153062</td>\n",
       "      <td>0.043711</td>\n",
       "      <td>0.272990</td>\n",
       "      <td>0.107752</td>\n",
       "      <td>0.062757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GINI</th>\n",
       "      <td>0.207678</td>\n",
       "      <td>0.050826</td>\n",
       "      <td>0.033414</td>\n",
       "      <td>0.150950</td>\n",
       "      <td>0.315164</td>\n",
       "      <td>-0.304511</td>\n",
       "      <td>-0.391238</td>\n",
       "      <td>-0.184512</td>\n",
       "      <td>-0.153062</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.096227</td>\n",
       "      <td>0.167625</td>\n",
       "      <td>-0.184125</td>\n",
       "      <td>0.071301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_duration</th>\n",
       "      <td>0.072200</td>\n",
       "      <td>-0.210403</td>\n",
       "      <td>0.107545</td>\n",
       "      <td>0.190506</td>\n",
       "      <td>0.232075</td>\n",
       "      <td>0.021244</td>\n",
       "      <td>0.116547</td>\n",
       "      <td>0.077666</td>\n",
       "      <td>0.043711</td>\n",
       "      <td>-0.096227</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.186079</td>\n",
       "      <td>0.086070</td>\n",
       "      <td>-0.167510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sum_duration</th>\n",
       "      <td>0.986011</td>\n",
       "      <td>0.325120</td>\n",
       "      <td>0.277116</td>\n",
       "      <td>0.753384</td>\n",
       "      <td>0.720580</td>\n",
       "      <td>-0.125331</td>\n",
       "      <td>0.279257</td>\n",
       "      <td>0.118291</td>\n",
       "      <td>0.272990</td>\n",
       "      <td>0.167625</td>\n",
       "      <td>0.186079</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.115919</td>\n",
       "      <td>-0.067978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Reliability2016</th>\n",
       "      <td>0.095116</td>\n",
       "      <td>0.271757</td>\n",
       "      <td>0.056590</td>\n",
       "      <td>0.115572</td>\n",
       "      <td>0.115848</td>\n",
       "      <td>0.476767</td>\n",
       "      <td>0.563356</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.107752</td>\n",
       "      <td>-0.184125</td>\n",
       "      <td>0.086070</td>\n",
       "      <td>0.115919</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.138533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Breaches_2017</th>\n",
       "      <td>-0.060109</td>\n",
       "      <td>0.824477</td>\n",
       "      <td>-0.127463</td>\n",
       "      <td>-0.133604</td>\n",
       "      <td>-0.169610</td>\n",
       "      <td>-0.276363</td>\n",
       "      <td>0.024738</td>\n",
       "      <td>0.139164</td>\n",
       "      <td>0.062757</td>\n",
       "      <td>0.071301</td>\n",
       "      <td>-0.167510</td>\n",
       "      <td>-0.067978</td>\n",
       "      <td>0.138533</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                Count  GDP (BILLIONS)  PROMOTIONAL MEASURES  \\\n",
       "Count                        1.000000        0.355255              0.270428   \n",
       "GDP (BILLIONS)               0.355255        1.000000             -0.003345   \n",
       "PROMOTIONAL MEASURES         0.270428       -0.003345              1.000000   \n",
       "label_num_h                  0.739385        0.123195              0.226058   \n",
       "label_num                    0.704295        0.027712              0.151793   \n",
       "corrupt perception          -0.131057       -0.104690              0.016669   \n",
       "corporate transparency       0.248997        0.221086              0.229670   \n",
       "Reliability of transparency  0.098424        0.273817              0.075510   \n",
       "disclosure_index             0.255426        0.180857              0.057571   \n",
       "GINI                         0.207678        0.050826              0.033414   \n",
       "mean_duration                0.072200       -0.210403              0.107545   \n",
       "sum_duration                 0.986011        0.325120              0.277116   \n",
       "Reliability2016              0.095116        0.271757              0.056590   \n",
       "Breaches_2017               -0.060109        0.824477             -0.127463   \n",
       "\n",
       "                             label_num_h  label_num  corrupt perception  \\\n",
       "Count                           0.739385   0.704295           -0.131057   \n",
       "GDP (BILLIONS)                  0.123195   0.027712           -0.104690   \n",
       "PROMOTIONAL MEASURES            0.226058   0.151793            0.016669   \n",
       "label_num_h                     1.000000   0.796237           -0.158651   \n",
       "label_num                       0.796237   1.000000           -0.200490   \n",
       "corrupt perception             -0.158651  -0.200490            1.000000   \n",
       "corporate transparency          0.230799   0.149846            0.400648   \n",
       "Reliability of transparency     0.108476   0.118496            0.489273   \n",
       "disclosure_index                0.175231   0.150387            0.167003   \n",
       "GINI                            0.150950   0.315164           -0.304511   \n",
       "mean_duration                   0.190506   0.232075            0.021244   \n",
       "sum_duration                    0.753384   0.720580           -0.125331   \n",
       "Reliability2016                 0.115572   0.115848            0.476767   \n",
       "Breaches_2017                  -0.133604  -0.169610           -0.276363   \n",
       "\n",
       "                             corporate transparency  \\\n",
       "Count                                      0.248997   \n",
       "GDP (BILLIONS)                             0.221086   \n",
       "PROMOTIONAL MEASURES                       0.229670   \n",
       "label_num_h                                0.230799   \n",
       "label_num                                  0.149846   \n",
       "corrupt perception                         0.400648   \n",
       "corporate transparency                     1.000000   \n",
       "Reliability of transparency                0.576450   \n",
       "disclosure_index                           0.389091   \n",
       "GINI                                      -0.391238   \n",
       "mean_duration                              0.116547   \n",
       "sum_duration                               0.279257   \n",
       "Reliability2016                            0.563356   \n",
       "Breaches_2017                              0.024738   \n",
       "\n",
       "                             Reliability of transparency  disclosure_index  \\\n",
       "Count                                           0.098424          0.255426   \n",
       "GDP (BILLIONS)                                  0.273817          0.180857   \n",
       "PROMOTIONAL MEASURES                            0.075510          0.057571   \n",
       "label_num_h                                     0.108476          0.175231   \n",
       "label_num                                       0.118496          0.150387   \n",
       "corrupt perception                              0.489273          0.167003   \n",
       "corporate transparency                          0.576450          0.389091   \n",
       "Reliability of transparency                     1.000000          0.144745   \n",
       "disclosure_index                                0.144745          1.000000   \n",
       "GINI                                           -0.184512         -0.153062   \n",
       "mean_duration                                   0.077666          0.043711   \n",
       "sum_duration                                    0.118291          0.272990   \n",
       "Reliability2016                                 1.000000          0.107752   \n",
       "Breaches_2017                                   0.139164          0.062757   \n",
       "\n",
       "                                 GINI  mean_duration  sum_duration  \\\n",
       "Count                        0.207678       0.072200      0.986011   \n",
       "GDP (BILLIONS)               0.050826      -0.210403      0.325120   \n",
       "PROMOTIONAL MEASURES         0.033414       0.107545      0.277116   \n",
       "label_num_h                  0.150950       0.190506      0.753384   \n",
       "label_num                    0.315164       0.232075      0.720580   \n",
       "corrupt perception          -0.304511       0.021244     -0.125331   \n",
       "corporate transparency      -0.391238       0.116547      0.279257   \n",
       "Reliability of transparency -0.184512       0.077666      0.118291   \n",
       "disclosure_index            -0.153062       0.043711      0.272990   \n",
       "GINI                         1.000000      -0.096227      0.167625   \n",
       "mean_duration               -0.096227       1.000000      0.186079   \n",
       "sum_duration                 0.167625       0.186079      1.000000   \n",
       "Reliability2016             -0.184125       0.086070      0.115919   \n",
       "Breaches_2017                0.071301      -0.167510     -0.067978   \n",
       "\n",
       "                             Reliability2016  Breaches_2017  \n",
       "Count                               0.095116      -0.060109  \n",
       "GDP (BILLIONS)                      0.271757       0.824477  \n",
       "PROMOTIONAL MEASURES                0.056590      -0.127463  \n",
       "label_num_h                         0.115572      -0.133604  \n",
       "label_num                           0.115848      -0.169610  \n",
       "corrupt perception                  0.476767      -0.276363  \n",
       "corporate transparency              0.563356       0.024738  \n",
       "Reliability of transparency         1.000000       0.139164  \n",
       "disclosure_index                    0.107752       0.062757  \n",
       "GINI                               -0.184125       0.071301  \n",
       "mean_duration                       0.086070      -0.167510  \n",
       "sum_duration                        0.115919      -0.067978  \n",
       "Reliability2016                     1.000000       0.138533  \n",
       "Breaches_2017                       0.138533       1.000000  "
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corr_df = df_final.corr(method='pearson')\n",
    "print(\"--------------- CORRELATIONS ---------------\")\n",
    "corr_df.head(len(df_freedom_gdp_freind))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final = df_freedom_gdp_freind.iloc[:,[1,2, 3, 5,6,7,8,9,10,11,12,13,14, 15]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "abs_corr_df = abs(corr_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- CREATE A HEATMAP ---------------\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdcAAAF2CAYAAAA8xNKKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XecXGX5/vHPlQDSUToCEqSIgJQQEQQRBAQrVSkW4KvGAoIFe0P82RVE4ItGRBBjQYpERUAQpHwRk0AghADSCYKIioBSkuz1++OcDbOT2d3Z3TMzO5vr7WtemVPmuZ9l1733ec5z7iPbRERERHXGdboDERERY02Sa0RERMWSXCMiIiqW5BoREVGxJNeIiIiKJblGRERULMk1IiKWWJLOkPSIpFv6OS5J35V0p6SbJU1spt0k14iIWJKdCew9wPHXAZuUr8nAac00muQaERFLLNtXAf8c4JR9gB+78Cfg+ZLWGazdJNeIiIj+rQs8ULM9r9w3oKVa1p1Y0qSOZkQ0QyNt4Kd6SdO/b97GHe+lmM7tNcX2lCGEa9TfQeMnuUZERFcZN4Q5Vy/0FGAoybTePGD9mu31gL8O9qFMC0dERFcZN675VwWmAe8sVw3vAPzb9kODfSgj14iI6CoVJU0AJP0M2BVYXdI84AvA0gC2vwdcBLweuBP4L3BEM+0muUZERFepMrnaPmSQ4waOHGq7Sa4REdFVxo14SVTrJblGRERXWaoLMlcWNHUxSWtL+rmkuyTdKukiSZtW2P6ukl5ZVXsREVVo84KmYemC/B+NSBJwAXCW7YPLfdsAawF3VBRmV+BJ4P8qai8iYsQ6mTSb1QVdjH7sBswvV7MBYHsWcI2kb0q6RdJsSQfBolHob3rPlXSKpMPL9/dK+qKkG8rPbCZpAvA+4MOSZkl6VRu/toiIfmXkGq20JTCzwf79gW2ArYHVgemSrmqivUdtT5T0AeBY2++W9D3gSdvfqqzXEREjlJFrdMLOwM9sL7T9N+CPwMub+Nz55b8zgQnNBJI0WdIMSTOmTBlJAZSIiOZl5BqtNAc4sMH+/hapL6DvH1PL1h1/pvx3IU3+XJT1OXuzamoLR0RbjB8/+u/Fyci1e/0BeJ6k9/TukPRy4F/AQZLGS1oD2AX4M3AfsLmk50laBdi9iRhPACtV3/WIiOHLyDVaxrYl7Qd8R9IngaeBe4EPASsCN1GMJj9u+2EASecANwN/AW5sIsyvgXMl7QN80PbVlX8hERFD1A3XXFVUdooYsfwgRUQzRjyne92GmzX9+2bHe27ryBxyRq4REdFVumHkmuQaERFdpRvKH3ZBFyMiIp6TkWtERETF8lSciIiIimXkGhERUbEk14iIiIolucYS5ZqHPt/S9nde5/iWth8R3WH80qM/uya5RkREV1EXrGhKco2IiK6S5BoREVExdcFTcZJcIyKiq2TkGhERUbFuGLmO/iVXLSJpLUk/lXS3pJmSrisf4YakXSX9W9KNkm6XdJWkN9Z89jhJD0qaJekWSW/uJ8a+kj7f4DO3STpN0rjy2JmSDizfXylpUl07u0r6TYP2F50raRVJP5Z0V/n6cfncViRNkGRJH6z57CmSDi/f7yDp+rJvcyUdV+5/o6QvDv+/ckRE9cYvNa7pV6cskclVkoBfAVfZfrHt7YCDgfVqTrva9ra2XwIcDZwiqfYB4yfa3gZ4C3BGb6Ks83Hgfxt8ZnPgZcCrq/uq+CFwt+2NbG8E3AOcXnP8EeAYScs0+OxZwOSyb1sC55T7fwu8WdLyFfYzImJENF5NvzpliUyuwGuAZ21/r3eH7ftsn9zoZNuzgOOBoxocmwssAFav3S9pU+AZ2482aHIZYFngX8P+CvrG2hjYDvhSze7jgUmSNiq3/w5cDhzWoIk1gYcAbC+0fWv53sCVwBsbfCYioiM0blzTr05ZUpPrFsANQ/zMDcBm9TslvQLooUhetXZqEOPDkmZRJLI7yqRdhc2BWbYX9u4o38+i+Fp7fQ34qKTxdZ8/Ebhd0gWS3itp2ZpjM4BXVdTPiIgRy8i1S0g6VdJNkqYPdFrddm+i/BZwUDnKq7UOiyfc3mnhNYEVJB08oo737Vt9/MX2274H+DNwaO1Jto8HJgGXlscurjn8CPDChkGlyZJmSJoxZcqUEX0BERHN0jg1/eqUJXW18BzggN4N20dKWp1ilNafbYG5Ndsn2v7WAOc/BazS6IDt+ZIuBnYBft50r/s3B9hW0jjbPQDlNeCt6/oM8BXgXOCquj7dBZwm6QfA3yWtZvsfFNPXT/XzdUwBerOqW13+MCICYFwXlD8c/T1sjT8Ay0p6f82+fhftSNoK+Bxw6hBizAU27qc9Aa8E7hpCe/2yfSdwI/DZmt2fBW4oj9WeextwKzXXUSW9oewTwCbAQuCxcntT4JYq+hkRUYVx49T0q2N97FjkDiqncPcFXi3pHkl/plgx+4ma017VeysORVI92vblQwhzFcVosva72zuVfAvFrMH/Nvwk/FbSvPL1y3Lf7jX75knase4z7wI2lXSnpLsokuK7+mn/y/RdGf0Oimuus4CzgbfVXL/djWLVcETEqNAN08Ja/FJhVEXSScCvbV/W6b4Mh6S1gJ/a3n3Qk9swLZyn4kSMCSPOeI++baemE9fqU6/tSIZdIkeubfQVBphu7gIvAj7a6U5ERNTqhpHrkrqgqS1s/w2Y1ul+DJftgVZPR0R0RMofRkREVGzcUuObfjVD0t5lqds7JX2ywfEXSbqiXIdzs6TXD9ZmRq4REdFVxlU4ci2L6pwK7AnMA6ZLmtZbqa70WeAc26dJ2hy4CJgwULtJrlGZLDiKiHao+Frq9sCdtu8GkPRzYB+KWxZ7GVi5fL8K8NfBGk1yjYiIrjKUa66SJgOTa3ZNKQvg9FoXeKBmex7wirpmjgMuLZ8stgKwx2Bxk1wjIqKrDGXkWldJrmFzjT5Wt30IcKbtb5c1Bs6WtGVvRbxGklyjMt++4X0tbf+jExc9xIjxH3xly+IsPPn/WtZ2RIycqi1/OA9Yv2Z7PRaf9n0XsDeA7evKh5usTlF7vaGsFo6IiO4yflzzr8FNBzaRtGH5vOuDWfwWyvuB3QEkvZSi5nr9g1n6yMg1IiK6S4ULmmwvkHQUcAkwHjjD9hxJxwMzbE+jKKbzA0kfppgyPrzBk9D6SHKNiIiuUnURCdsXUdxeU7vv8zXvb6V4RnfTklwjIqK7dLCsYbOSXCMioruMhfKHkhZKmiXpFkm/lLR8g/2/lvT8ms9sIekPku6Q9BdJn+t99JqkwyVZ0u415+9X7juw3F5G0nck3VV+/kJJ60larYw5S9LDkh6s2V5G0pND6ENP+ZzW3vNvkTShZnvbsk971f33eJIBNPn1XVmW2urt+7l1bdwk6Wd1+3aQdH15/lxJx5X7j5N0bN2596p4+Hu/3ydJEyQ9VdOHWZLeWR77H0mzyzJft0jaZ6CvOSKinbT0+KZfndLMUqqnbG9je0vgWeB9Dfb/EzgSQNJyFCutvmZ7U2BrigeDf6CmzdkU9w31Ohi4qWb7K8BKwKa2NwF+BZwP/LOMuQ3wPeDE3m3bz/Z+uMk+zAM+M8DXfQhwTV0/mzXY1wfFM1N7+35gTd9fSvF92UXSCjXnnwVMLr/2LYFzmuxLw+9T6a6aPmxj+8eS1qP477Kz7a2AHYCbm4wVEdFy3fBUnKHeinM1sHGD/ddRVLkAOBS41valALb/CxwF1BZDvhrYXtLSklYs25wFUI6MjwA+3PvAbts/Ap4BXtNkP5vpw2+ALSS9pP7D5Qj3QOBw4LXlPU1D0e/X12TfzwYuBd5cs39N4CEA2wvr6l42q/b71J81gSeAJ8tYT9q+ZxixIiJaY7yaf3VI08lV0lLA6yhGZbX7x1Pc/9N7X9AWwMzac2zfBawoqbc2o4HLgL0oajjW3lO0MXC/7cfrujCjbLsZzfShB/gG8OkGn98JuKf8zJXAoE9AqDPQ19dras107Ddr9h8E/AL4GX1HvycCt0u6QNJ7h5rwG3yfADaqmxZ+FcUI+2/APZJ+JOlNA7Q5WdIMSTOmTBmoAEpERIXGSHJdTtIsiuR2P/DDuv3/AFYFfl/uF4uXjupVu//nFNOlB1Mkkl79fX6gdodybu3+nwI7SNqw7pxDyv719nM4U8P9fX29aqeFPwYg6eXA323fB1wOTJT0AgDbxwOTKEa0hwIXN/h6avXu7+/7BItPC19dzhbsTTFyvwM4sff67mIB7Cm2J9meNHny5EanRERUbqxMCz9V88v3gzXXNp8qr/9tACzDc9fy5lAkgUUkvRh40vYTvfts/5ni2uHqtu+oOf1OYANJK9X1YyJ9n1IwkGb7sAD4NvCJmvPGAwcAn5d0L3Ay8LoG/RnQAF/fQA4BNivj3kXxFIYDatq8y/ZpFCPQrSWtRpE0X1DXzkrAY+X7/r5PA/Xdtv9s+6sUfxwcMNhnIiLaZoyMXAdk+9/A0cCxkpYGpgI7S9oDFi0u+i7FFGy9T1E3LWv7PxSLd04oEx3lKtblgT802a2h9OFMiiccrFFu7wHcZHt92xNsbwCcB+zbZOxai319/ZE0DngLsFUZdwLFlPIh5fE39K52BjYBFlIk0KuAN/cmf0n7l/1fWNt+g+9Tf/14oaSJNbu2Ae5r5muIiGiHblgtXMl9rrZvlHQTcLDts8tbN06WdCpFOamzgVMafO53/TT5KeBbwB2SeoDbgP0GKzdV0+5TQ+jDs5K+C5xU7joEuKDutPOA95dtLC9pXs2xE2yf0E8/+vv6oLjm+lT5/lHg/wEP2n6w5pyrgM0lrQO8g2KK9r/AAopp5YXAzZJOAa6RZIpC0u/upz+Lvk8Ui642KqeMe50BXAh8S9ILgacp6me2tiJ/RMRQdEERCTWZryIG4zwVJyKaMOLMOP/0Q5pOXEu/+2cdycSp0BQREV2lkwuVmpXkGhER3aW5R8l1VJJrRER0lYoflt4SueYaVckPUkQ0Y8Rzugt/cVjTv2/GH3RWrrlGREQMKtdcIyIiqlX1w9JbIck1IiK6y7jRf801yTUq48d+0tL29fy3L3q/0nF7tCzOE8dd1md7tW8O9bkNzfvHxy5qWdsRY1amhSMiIiq21OhPXaO/hxEREbUyco2IiKhYrrlGRERULMk1IiKiYpkWjoiIqFgXjFxHfw9jWCStIOm3km6SdIukgyTdK2n18vgkSVeW74+TdJakS8tz9pf0DUmzJV080MPVIyLabqnxzb86JMl17Nob+KvtrW1vCVw8yPkbAW8A9gF+Alxh+2XAU+X+iIjRYdy45l+d6mLHIkerzQb2kPR1Sa+y/e9Bzv+d7fnl58bzXDKeDUxo9AFJkyXNkDRjypQpVfU7ImJgXZBcc811jLJ9h6TtgNcDX5V0KbCA5/6gWrbuI8+Un+uRNN/PPS6ph35+TmxPAaYs2mxxhaaICAApC5qiQyS9EPin7Z9IehI4HLgX2A74HXBA53oXETECXbCgKcl17HoZ8E1JPcB84P3AcsAPJX0auL6TnYuIGLYOLlRqVpLrGGX7EuCSBoc2bXDucXXbK/Z3LCKi4zJyjYiIqFiKSERERFSsC0auo7+HERERtSq+FUfS3pJul3SnpE/2c85bJd0qaY6knw7WZkauERHRXSqcFpY0HjgV2BOYB0yXNM32rTXnbAJ8CtjJ9r8krTlou8/dzhgxIvlBiohmjDgz+q6vN/37Rht9YsB4knYEjrO9V7n9KQDbX6055xvAHbZPbzZupoUjIqK7jFPzr8GtCzxQsz2v3FdrU2BTSddK+pOkvQdrNNPCERHRXYawoEnSZGByza4pZXW5Rac0+Fj9yHgpYBNgV2A94GpJW9p+rL+4Sa4REdFd1HxyrSvT2sg8YP2a7fWAvzY4509l/fV7JN1OkWyn99dokmtU5ur1N2tp+6964LZF72/+x1cHOHNktlrtU322v3vT+1sW6+itT+uz/au7P9yyWPu++MSWtR3RVkNIrk2YDmwiaUPgQeBg4NC6c34FHAKcWT62c1Pg7oEaTXKNiIjuUuF9rrYXSDqKoqLdeOAM23MkHQ/MsD2tPPZaSbcCC4GP2f7HQO0muUZERHcZV23qsn0RcFHdvs/XvDfwkfLVlCTXiIjoLtVOC7dEkmtERHSXLih/mOQaERHdpQtGrqO/h6NQ+fDxgY5PkHTLENs8U9KBI+tZNSQdJ+nYTvcjIqIhjWv+1SEZuUZERHepeEFTK2TkOgKSVpR0uaQbJM2WtE/N4aUknSXpZknnSlq+/Mx2kv4oaaakSySt02SseyV9sSbWZuX+PqNMSbeUI+cJkm6TdHq5b6qkPcryXX+RtP0gITeXdKWkuyUdPdT/NhERLVPxU3Fa0sWORR4bngb2sz0R2A34tqTeUlovoSiztRXwOPABSUsDJwMH2t4OOAP48hDiPVrGOg1oZtp2Y+AkYCtgM4obo3cuP/vpQT67GbAXsD3whbLvfUiaLGmGpBlTpgxUACUiojrS+KZfnTL6x9ajm4CvSNoF6KEo9rxWeewB29eW738CHA1cDGwJ/L7MweOBh4YQ7/zy35nA/k2cf4/t2QCS5gCX27ak2cCEQT77W9vPAM9IeoTi65pXe0JdWTFf/aUTmvsqIiJGogsWNCW5jszbgDWA7WzPl3QvsGx5rL7wsymS8RzbOw4z3jPlvwt57nu3gL4zEMs2OB+K5P9MzfvBvve1n62NFxHRWV2QXEd/D0e3VYBHysS6G7BBzbEXlc8JhKIm5TXA7cAavfslLS1pixH24V5gYtneRGDDEbYXETG6dcFq4STXkZkKTJI0g2IUe1vNsbnAYZJuBlYFTrP9LHAg8HVJNwGzgFeOsA/nAatKmgW8H7hjhO1FRIxu45dq/tUhmeobBtsrlv8+CvQ3xbt5P5+dBezSYP/hg8ScUPN+BsVzBbH9FPDafj62ZaP2bd9be6xBrOPqtvs9NyKi7bpgWjjJNSIiukuSawyVpAtY/LrpJ2xf0oJYRwDH1O2+1vaRVceKiKhMagvHUNner42xfgT8qF3xIiIq0QUjVxWPqYsYsfwgRUQzNPgpg1j4++Z/34zfc+TxhiEj14iI6C5dUFt49PcwIiKihocwFu3IsJUk16hQz6WtXQc17rWnLnr//C/v1bI4j32m79qxV/zs4JbFuv6Qn/fZvuPlDe/gqsSm029d9P6Rp37csjgAay73zpa2H0s2u6fpc9Wh7JrkGhERXaVnCMl1XJJrRETE4EzzybVTklwjIqKrLPSCps9d7FmZbZLkGhERXWUo08KdkuQaERFdZSgLmjolyTUiIrpKTxdccx39NaS6nKQnBzk+QdItQ2zzTEkHjqxnERHdye5p+tUpGblGRERX6YZrrhm5tomkFSVdLukGSbMl7VNzeClJZ0m6WdK5kpYvP7OdpD9KminpEknrNBnrXklfrIm1Wbn/OEnH1px3SzlyniDpNkmnl/umStpD0rWS/iJp+0r/Y0REjMBCL2j61SlJru3zNLCf7YnAbsC3pUW1Q14CTLG9FfA48AFJSwMnAwfa3g44A/jyEOI9WsY6DTh2sJOBjYGTgK2AzYBDgZ3Lz3660QckTZY0Q9KMKVOmDKFrERHDZxY2/eqUTAu3j4CvSNoF6AHWBdYqjz1g+9ry/U+Ao4GLgS2B35c5eDzw0BDinV/+OxPYv4nz77E9G0DSHOBy25Y0G5jQ6AO2pwC9WdWtLn8YEQHdMS2c5No+bwPWALazPV/SvcCy5bH6xyeZIhnPsb3jMOM9U/67kOe+zwvoO1uxbIPzoUj+z9S8z89JRIwa3XArTqaF22cV4JEyse4GbFBz7EWSepPoIcA1wO3AGr37JS0taYsR9uFeYGLZ3kRgwxG2FxHRdj1D+F+nJLm2z1RgkqQZFKPY22qOzQUOk3QzsCpwmu1ngQOBr0u6CZgFvHKEfTgPWFXSLOD9wB0jbC8iou0W9ixo+tUpme5rMdsrlv8+CvQ3xdvwOWO2ZwG7NNh/+CAxJ9S8nwHsWr5/CnhtPx/bslH7tu+tPRYR0Wkp3B8REVGxbljQlGnhLibpAkmz6l6te4p4RMQoYHqafjVD0t6Sbpd0p6RPDnDegZIsadJgbWbk2sVs79fpPkREtFuVI1dJ44FTgT2BecB0SdNs31p33koUt0le31S7dv1dIBHDkh+kiGiGBj9lYHf9+ztN/77ZaJUPDRivvCPjONt7ldufArD91brzvgNcRlFY59hyPUu/Mi0cERFdZSjlD2sryZWvyXXNrQs8ULM9r9y3iKRtgfVt/6bZPmZaOCIiukrPEGZc6yrJNdJoZLsogKRxwInA4U0HJck1KrTtTw5qafs3vv0Xi96vN6V1l5vnTb6gz/ZSx+zUslgLTrq2z/bHr63/o7o639jpud8va5+yzwBnjtzDR1246P1rz397S2Nduv9PWtp+jD4VrxaeB6xfs70e8Nea7ZUobke8sixFuzYwTdKbB5oaTnKNiIiuMpSRaxOmA5tI2hB4EDiY4sElANj+N7B677akK2nimmuSa0REdJWeCtdP2l4g6SjgEooHpJxhe46k44EZtqcNp90k14iI6CoVj1yxfRFwUd2+z/dz7q7NtJnkGhERXWVBT+ee09qsJNeIiOgqVY9cWyH3uY4RktaXdIWkuZLmSDqm3L+qpN9L+kv57wvK/ZtJuk7SM5KOrWvrXkmzy3KKA160j4hotx73NP3qlCTXsWMB8FHbLwV2AI6UtDnwSeBy25sAl5fbAP+kKOX1rX7a2832NrYHraEZEdFOPXbTr05Jch0jbD9k+4by/RMUz4hdF9gHOKs87Sxg3/KcR2xPB+Z3oLsREcPWfNn+ziXXXHMdgyRNALalKDC9lu2HoEjAktZsogkDl0oy8P2ywkmjOJOByQDf//73YfkKOh8RMYgsaIq2k7QicB7wIduPlxVFhmon238tE/HvJd1m+6r6k+rKivm0n1w+7H5HRDQrC5qirSQtTZFYp9o+v9z9N0nrlMfXAR4ZrB3bfy3/fQS4ANi+NT2OiBi6XHONtlExRP0hMNf2CTWHpgGHle8PAy6s/2xdOyuUzy1E0grAa4Fbqu9xRMTwdMNq4UwLjx07Ae8AZkuaVe77NPA14BxJ7wLuB94CIGltYAawMtAj6UPA5hQ1NC8op5OXAn5q++J2fiEREQPphmnhJNcxwvY19P8Q4t0bnP8wxdMf6j0ObF1h1yIiKpXkGhERUbEFzmrhiIiISmXkGhERUbFuSK5yF3QyukJ+kCKiGcO6+b7WuXce3fTvmwM3/u6I4w1HRq4REdFVumHkmuQaERFdJck1IiKiYvN7klxjCfKlP7+3pe1/bvvvL3p/7NXvaVmcb73qB322P3Pd5JbF+vKOfZ+JsN6U/VoWa97kCxa9/8AV725ZHID/3e30Re8/fFXrvlcAJ+7y3Pfrjsf6e4LiyG36/GMHPynaIiPXiIiIii0c/bk1yTUiIrpLF8wKJ7lGRER3WZhp4YiIiGrN79zDbpqW5BoREV2lG6aF8zzXUUzS+pKukDRX0hxJx5T7V5X0e0l/Kf99Qbl/M0nXSXpG0rF1bT1f0rmSbivb27ETX1NExEgttJt+dUqS6+i2APio7ZcCOwBHStoc+CRwue1NgMvLbYB/AkcDje5HOAm42PZmFI+Um9vqzkdEtEJPT/OvTklyHcVsP2T7hvL9ExQJcV1gH+Cs8rSzgH3Lcx6xPR2YX9uOpJWBXYAfluc9a/uxRjElLSVpuqRdy+2vSvpy1V9bRMRwLXTzr05Jcu0SkiYA2wLXA2vZfgiKBAysOcjHXwz8HfiRpBslnS5phUYn2l4AHA6cJmlPYG/gi/30abKkGZJmTJkypdEpERGV63Hzr05Jcu0CklYEzgM+ZPvxYTSxFDAROM32tsB/eG4qeTG25wBnA78G/sf2s/2cN8X2JNuTJk9uXRWjiIha83vc9KtTklxHOUlLUyTWqbbPL3f/TdI65fF1gEcGaWYeMM/29eX2uRTJdiAvAx4D1hpWxyMiWiTTwjEikkRxnXSu7RNqDk0DDivfHwZcOFA7th8GHpD0knLX7sCtA8TdH1iN4jrtdyU9f3hfQURE9Xrspl+dkvtcR7edgHcAsyXNKvd9GvgacI6kdwH3A28BkLQ2MANYGeiR9CFg83Iq+YPAVEnLAHcDRzQKKGn1sv3dbT8g6RSKlcaHNTo/IqLdUls4RsT2NYD6Obx7g/MfBtbrp61ZwKQmYj4KbFqz/d2mOhsR0SbdUEQiyTUiIrpKN4xcc811CSbpVEmz6l4Np4sjIkaL+Qvd9KsZkvaWdLukOyUtdieFpI9IulXSzZIul7TBYG1m5LoEs31kp/sQETFUVU4LSxoPnArsSXFnxXRJ02zXLvq8EZhk+7+S3g98AzhowHbdBY/uia6QH6SIaEZ/60iadtSV72n6980pu/5gwHhlnfXjbO9Vbn8KwPZX+zl/W+AU2zsN1G5GrhER0VUqvsVmXeCBmu15wCsGOP9dwO8GazTJNSIiuspQFjRJmgzUlpCbYru2XmujkW3DCJLeTnHXxasHi5vkGhERXeXZIWTXMpEOVPx8HrB+zfZ6wF/rT5K0B/AZ4NW2nxksbpJrVGad/923pe0/9IFfLXp/ys3vb1mco7Y6rc/2mie9qWWxHjnm1322V/vm61sW6x8fu2jR+2U+8qqWxQF49oSrF71/3QXvaGms3+139qL383suaVmcpcft1WfbD3+vZbG09vta1vZYUPFzWqcDm0jaEHgQOBg4tPaE8jrr94G9bQ9WbhZIco2IiC6zsMLlwrYXSDoKuAQYD5xhe46k44EZtqcB3wRWBH5ZVKXlfttvHqjdJNeIiOgqFY9csX0RcFHdvs/XvN9jqG0muUZERFdZ2NPpHgwuyTUiIrpK1SPXVkhyjYiIrvJsFwxdU1t4FJO0vqQrJM2VNEfSMeX+VSX9XtJfyn9fUO7fTNJ1kp6RdGxdW8+XdK6k28r2duzE1xQRMVILe5p/dUqS6+i2APio7ZcCOwBHStoc+CRwue1NgMvLbYB/AkcD32rQ1knAxbY3A7YG5ra68xERrbDQbvrVKUmuo5grK7IQAAAd1UlEQVTth2zfUL5/giIhrgvsA5xVnnYWsG95ziO2pwPza9uRtDKwC/DD8rxnbT/WKKakjSTdULO9iaSZlX5hEREjkOQalZE0AdgWuB5Yy/ZDUCRgYM1BPv5i4O/AjyTdKOl0SSs0OtH2XcC/JW1T7joCOLOfPk2WNEPSjClTBiqAEhFRnYU9bvrVKUmuXUDSisB5wIdsPz6MJpYCJgKn2d4W+A/PTSU3cjpwRPkopoOAnzY6yfYU25NsT5o8eXKjUyIiKvfsQjf96pQk11FO0tIUiXWq7fPL3X+TtE55fB1gsHJc84B5tq8vt8+lSLb9OQ94HfBGYKbtfwy3/xERVcu0cIyIijpbPwTm2j6h5tA04LDy/WHAhQO1Y/th4AFJLyl37Q7cOsD5T1OUAjsN+NHweh8R0RrdMC2c+1xHt52AdwCzJc0q930a+BpwjqR3AfcDbwGQtDYwA1gZ6JH0IWDzcir5g8BUScsAd1NcSx3IVGB/4NJqv6SIiJHp4Gxv05JcRzHb19D4WYNQjD7rz3+Y4nFJjdqaRfEcwmbtTFHAeuEQPhMR0XKdHJE2K8k1FiPpAmAj4DWd7ktERL2UP4xRTdKpFFPPtU6yvV8n+hMR0YxOrgJuVpLrEsz2kZ3uQ0TEUHXDyFXugk5GV8gPUkQ0o791JE3b4Af7N/375r73nD/ieMORkWtERHSVbhi5JrlGRERX6Rn9T5xLco3qHHXle1ra/im7/mDR+5W/uGfL4jz+hd/32X7esbu0LNYz37qqz/bn/9S6MpLH7/Bc/ef3/uHdLYsD8P3XnL7o/bFXt/bn4luveu7n4ud3HNWyOAdvekqf7Vv/+fWWxdp81U/02X7bxYPdlj58U/fuvjoxzq04ERER1epZMPqHrkmuERHRVbphIW6Sa0REdJVMC0dERFQsyTUiIqJi3ZBc88i5MUzSWpJ+KuluSTMlXSdpP0m7SvpNec7hknokbVXzuVskTSjf3ytp9c58BRERi+tZ0NP0q1OSXMeo8lmwvwKusv1i29sBB9P4qTnzgM+0s38REcPlHjf96pQk17HrNcCztr/Xu8P2fbZPbnDub4Atah6mHhExaiW5RidtAdzQ5Lk9wDcoHsQeETGqJbnGqCHpVEk3SZrezyk/BXaQtOEQ2pwsaYakGVOmTBn8AxERFbDd9KtTslp47JoDHNC7YfvIcmHSjEYn214g6dvAJxod7+czU4DerOqjruwvb0dEVCerhaOT/gAsK+n9NfuWH+QzZwJ7AGu0qlMRESOV1cLRMS7mQ/YFXi3pHkl/Bs5igJGp7WeB7wJrtqeXERFD1w3XXDMtPIbZfoji9ptGrizPOZNixNr7me9SJNje7Qmt6l9ExHB0w7RwkmtERHSVJNeIiIiKdUNyzTXXiIjoKlXfiiNpb0m3S7pT0icbHH+epF+Ux6/vLQ87kCTXiIjoKlWuFpY0HjgVeB2wOXCIpM3rTnsX8C/bGwMnAl8ftN1ueOhsdIX8IEVEMzTSBsYduWPTv296Tr1uwHiSdgSOs71Xuf0pANtfrTnnkvKc6yQtBTwMrOEBEmiuuUZVhvx/GEmTy0IULTcWY43Fr2msxhqLX1O7Y9UaLGHWkjQZmFyza0pdn9cFHqjZnge8oq6ZReeUBXf+DawGPNpf3EwLRydNHvyUxBoFcRKre+KM5VjDYnuK7Uk1r/o/Bhol6voRaTPn9JHkGhERS7J5wPo12+sBf+3vnHJaeBXgnwM1muQaERFLsunAJpI2lLQMReGdaXXnTAMOK98fCPxhoOutkGuu0VntvFYzFmONxa9prMYai19Tu2O1RHkN9SjgEmA8cIbtOZKOB2bYngb8EDhb0p0UI9b+Kt8tktXCERERFcu0cERERMWSXCMiIiqW5BoREVGxJNdoG0k7NbOvgjjLSjpQ0kmSfinpx5I+LmmLiuO8SdIGNdufl3STpGmSNqwyVkR0lyxoiraRdIPtiYPtG2GM44A3UTyvdibwCLAssCmwW/n+o7ZvriDWzcAOtv8r6Y3ACcAhwLbAW3rLqVWprIP6BmACNav9bZ9Qday6mGvVxbu/4hiTgM8AG5RxVITxVlXGqYm3M7A9cIvtS1sRo4zzJeCLtheU2ysDJ9k+ouI4M4AfAT+1/a8q266JsepAx20PeN/nkia34kTLlbU7XwmsIekjNYdWplj6XqXpto/r59gJktYEXlRRLNv+b/l+f+CHtmcCMyV9oKIY9X4NPA3MBgavSj5Ckj4IfAH4W008A1UnvanAx2jR1yXpz7a3L9+/BzgSuAD4gqSJtr9WdczSUsD1ko4A1gZOLl9VOxg4Aphek2gvHexezCGaSfG9769a0YsrjNX1MnKNlpP0amBX4H3A92oOPQH82vZfWhz/BcBjFf+i6R25vhL4L3APcIDtGeWxW23XP1mjkpitGs31E+9O4BW2/9HiONfY3rmF7d9oe9vy/XTg9bb/LmkF4E+2X9bC2HtQ/FH0L2AX23e2MNY44I3AaRR/pJxBMVLOqLLNMnKNlrP9R+CPks60fV8rY0n6PHCO7dskPQ+4GNgaWCDpUNuXVRjuO8As4HFgbk1i3RZ4qMI4tX4n6bWtnMqs8wDw7zbE+YKk04HLgWd6d9o+v6L2x5V/ZI2jGFT8vWz/P5IWVBRjMZJ2AU4CjgdeBpwi6X9s15fXqyLWVhSj19cD51HMBuwM/AHYpoL2B5zxqfpSQbdLco12ep6kKSx+vfA1FcY4CPhS+b63XNkaFNdczwIqS662zygfRbUmcFPNoYcpfsm1wp+AC8oRynyeuza5cpVBaqbv7waulPRb+ia9qq/xHgFsBixN3+nnqpLrKhTTmgIsaW3bD0takQoegTaAb1Fcf78VQNL+FMlusyqDSJoJPEZRSeiTtnu/V9dXuGjwtyw+LWyK/3+tSfWXeLpakmu00y8ppoVPBxa2KMazNdO/ewE/t70QmFsW3K6MpLfb/gnwYPkL7FoA2w+V5dROqTJe6dvAjsDsqqe566xU/nt/+VqmfLXK1q2cmrU9oZ9DPcB+rYoL7Fj+/PX243xJf+zdlnSY7bMqiPMW23c3OmB7/yri1H9/JE0APgHsAXxlJG2PRbnmGm0jaabt7Voc40/AuykW4NwObGf7nvLYbbYrGzHUrnSuX/Vc9SromnYvAV5nu+WLmZoh6WTbH6ygnR8AJ/aO8NpJ0oq2n2x33DJ2S35OWhlH0iYUK7tfQfHH3lm251fR9liSkWu006/LVbQX0HeKscrFFh8CzqWYqjqxJrG+HrixwjjQd3qsfmqxVVOND1FM0/6O1k7TNquqKcedgcMk3UPxdbX0Vpw6t1LdCvKhauWUdKVxJG1JkVS3AL4BvKt2VB59JblGO/VeA/1Yzb5Kl/Db/hMNrmfZvgi4qKo4vc32877RdlXuKV+tnqZtt71b2XjdLWB9DgErtjL2INo1dVhFnJsoFrj9luIe4e2l53K27aMriDFmJLlG29huedUiSe8cuAs+u8Jwm5W34wjYqHxPud2Se/5sf7EV7Y4CrU4yXwG+CTRaGdzJSnVdM3IF3kX7/hjoekmu0Tb9JT7bP64wzMsbhaao2rQuUGVyfWmFbTVF0hU0+AVX8YrroagqOdSuRF0W2JDimnlVJStvAH5VFvnoQ9K7K4oxHNd2SxzbZ1bQjyVGFjRF20iqrUyzLLA7cIPtA1sUT8DbKFY03gp8uYqyh03EHQ8cbHtqC9quXRC2LHAAsMD2x6uO1WR/Dm/FL11JE4H32n5vRe29BPhn7/2tdcfWsv23KuLUtPkKinufH5e0HPBJYCLFz+FXbFdy73C74pSxfs0AI1fbb64q1liQ5BodI2kV4Oyq/09Z3nJzOPBR4Hrgq7ZvrzJGGWdlijJ66wLTgN8DRwHHArNs71N1zH768Ufbr25R222t+VsXuy0raVtB0hyK24sWlPd2/5diod3u5f79uylOGavRz1hvAlFZLCZKmRaOTvovsEmVDUo6EjiGotLP3i2uCHU2RUm76yhu//kYxSKjfWzPakXAuuLp44BJFDVrW6WlNX971S04Gkcx+lpslDmC9lcBPgXsS7GSHIqHOlwIfM32Y1XFKo3rLdYPTKr5I+EaSVX+bLQrDsDzgfVsnwpFvWaK/5ammB2KGkmu0TZ100rjKa5ZnlNxmJMpfmnuTHHrz6LwVD/ienHvjfVl6b5HgRfZfqLCGPVqi6fPB+6lWGjSKn+3Pa2F7fdaqeb9AoprsOdV2P45FJWRdrX9MICktSlWsP8S2LPCWAC3SDrC9o+AmyRNsj1D0qYU37duiwPwcYoHBPRahuKPuxUoHhTwy4rjdbVMC0fb1E0rLQDusz2v4hgbDHS8ypFsuwpH1MV8K3BxeY3tcxQjvC/ZvqFF8XaneIxeq2r+toWk222/ZKjHRhBvFYqawq+i+KNrIsVtLA8AR9u+aYCPj7o4Zazptl9es32K7aPK93+yvUNVscaCJNdoK0lr8dyK3j/bfqRFcTakWGlqigUfDUvDjTDGQuA/vZvAchRT3S2p91vGvNn2ViqeR/oVigo5n7b9iqpjlfF+QnHf8Bxqav7a/p+K42xKca16Ai2oOy3pUoq60mf1Ll4qfxYPB/a0vUcVcRrEXYnitqylgHlVL5xqZxxJd9reuJ9jd9neqOqY3SzJNdqmHHV9k+JB5qL4a/tjts+tMMbKFLWLJ1E8sUYUT8WZSVFR5vGqYnWCykenSfoqRX3hn6rmcWotiDe7lTV/a+LcRFF3eiY1dacb3TozzPZfQLGSdh+KB7+bokTmNODrFVcJG6wvbSm3WHUcSVOBK23/oG7/eymm2w+pKtZYkOQabVP+At2zd7QqaQ3gMttbVxjjTIrrkMf31t8tb8n5HLCx7YGKTAw11qp1u0wLnhtbF/M3wIMUxdK3A56imAGo7L9hXby21PxtU93pzYD1KJ7f+mTN/r1tX9zK2HX9uN92y8stVh1H0prAryguD/RehtgOeB6wb6tG5d0qyTXapn4UpOKxaTdVOTKS9BfbDVcgD3RsmLHuYfFHcK1EMWJ+t+17q4pVE3N5ilKBs23/RdI6wMvcoue7SpoLbERRcrFlNX8lHUexEK0ldaclHU1x29RcimebHmP7wvJY5dfKBym3+Bnb9X+Yjeo4dTFfw3PFPebY/kPVMcaCJNdoG0nfBLYCflbuOgi42XZly/gHuS5UaXIdoA/7A5Ntt7Rebjv0t0Cs6lucyj9UGoRxJWUkJc2mePzbkyoelXYuxT3WJ7ViWl3S0/RfbvHDtp/fTXFi6JJco+UkbQysZfvaMvHsTPGX9b+AqbbvqjDWWcBdFCtoXbP/c8Cmtt9RVaxB+tG1BRBqSWo4rWj7/jb3Y0/bvx/B52+1vXnN9ooUCfZW4DW2t6mgm7Xx/g/4YKNrxpIesL1+N8WJoUtyjZYrrxN+2nWlB8vqP1+w/aYKY60M/JDiloRZFNO221I8bu7dLSgW0KgPKwLXVP0LuxPKEd9iNX9tV1Xzt9l+jOiPFUl/AD5SW9yjrOR1BvA22+Mr6GZtvLaUW2xXnBi6JNdoOUm32N6yn2MtWY0qaSNgc4qkMKfK0XFNjEbXu14AvBk4pX5V5Vigimv+DiHuiKZuJa1HUYP54QbHdrLdrgL6sYRIhaZoh2UHOLZclYHKX/69Hiz/XaV3f8XFFlaq2zbwMPB227MrjDNq2L5BUqMnD7U89Ig+PECxklYk1naVW2xXnBi6JNdoh+mS3tPg/rh3UdzXWKVvD3DMQGWPZnODZ6uW91OOmV9oanHN3zGsXeUW213WMZqUaeFoubISzgXAszyXTCdR1Cbdr9FUXYv6sYPtP1XY3ueBc2zfJul5wO8obvNYABxq+7KqYnWKpC/UbC6guIf4PNtPt7kf57vCJ7y0WrvKLba7rGM0L8k12kbSbkDvtde23x/Xgpvq5wBb2rakycChFI/62pSizN72VcUaq8rV4/1yl9Uw7tWucoudKusYg8u0cLSN7SuAKzrYBQ1+ypA8W3O7z17Az2wvBOaWK1G7Xqtr/gIDrRQ30JXJleIe7k8CfyyTXW25xbd2YZwYooxcY4nRgpHrnyie4/o34HZgO9v3lMdus71ZVbE6pdU1f8eydpVbHC1lHaOvJNcYU9T3mbF9DlEUC1ihwlivAM6iWKX5HdtfKve/HniHx0Ah83bU/C3jrEXxlJ8X2n6dpM0pKir9sNWxW6Fd5RbbXdYxmpfkGmOK+j4zdjG2/9imfoyJG/hbXfO3Js7vKB64/RnbW5fT6je24h7odmhXucV2l3WM5o2J60IRvQZKnpJ+AbQsuZb3HB5AsbDppcC6rYrVRoeV/36sZp8pnh1apdVtnyPpUwC2F6h4Xm63Gt87RWv7Xkm7AueWtZqrvPbfrjgxROM63YGINtqx6gYlLSfpIEkXArcAJwD/DxgTNV1tb9jgtSixSqrqPsr/SFqNckpf0g7AvytquxMelrSo/GWZAN8IrA5UORpvV5wYokwLxxKjBQuapgK7AJcCP6e4mf9O2xtWFWO0q+q6XllB62SKW7VuobiOfWB9Pepu0a5yiynrOHplWjjGlLryh30OAUtXHG5Liif7zAVus71Q0pL212olU49lWcVXAy8p27zd9vwq2u6EdpVbbHdZx2hekmuMNQOVP7ytykDlwpvNKK6xXibpEWAlSWu3q+rUKFDJHxOSlgU+QPE4QgNXS/peuytBRVQl08IRFSkfoXcI8BZgnu1XdrhLLVfhtPA5wBPAT8pdhwAvsP2WkbYd0QlJrjGmjIZyepIE7NKu2346qaqav5Jusr31YPsiukWmhWOsaVs5PUnn2H5r+f7rtj8BUNYa/gwtvO2n1Zr9I6XCYvo31j5YoSzQkWuG0bWSXGNMsX1EG8NtUvN+T+ATNdtr0N3a8kdKWQTBFIvN3inp/nJ7A+DWKmJEdEKSa4w55arTf9m+WdJbKW6XuQv4X9vPDPzpIRnomkpXX29p4x8pb2xTnIi2SnKNMUXSqcBWwLKSbgdWBC4GXgmcAbytwnDLS9qWohjLcuV7la/lKozTMa2u+Wv7vrp4awLLVtF2RCdlQVOMKZJutb15eWvHg8Ca5f2nAm6uslatpCsZYIRqe7eqYnVKu2r+SnozxW1UL6SoZbwBMNf2FlXGiWiXjFxjrHkawPbTku4rn6/au8io0qIEtnetsr1Rql01f78E7ABcZntbSbtR3I4T0ZWSXGOsWVPSRyimZnvfU25XushoNNz20wbtqvk73/Y/JI2TNM72FZK+3oI4EW2R5BpjzQ+AlRq8Bzi94ljnArPKF/QtBVjpbT8d9BFgGrCRpGspa/62IM5jklYErgKmltWuFrQgTkRb5JprxDBJ2g84CNgYuBD4me07O9ur6pXXWVta81fSChRT+qJYdLYKMNX2P6qOFdEOSa4xpkj6LHCq7X/1c/w1wPK2f1NhzBWAfSgS7WoUi3+6toBErUY1f4HU/I0YRKaFY6yZDfxG0tPADcDfKW7t2ATYBriM4taSKj1NcR3yceBFjK1bSX5MUfP35HL7EOBsivrJIybpCRqvuBbFOrSVq4gT0W4ZucaYJGkTYCdgHeApisfCXWX7qQpj9K5o3Z4iaf/c9oyq2h8NUvM3YniSXCOGSVIPcDNwDcXoq8//mWwf3Yl+VUnSmRTTwLU1fw+z/YGOdixilMu0cMTwtbOOcVul5m/EyGTkGhGLkbTBQMfryxZGRF8ZucaYJGl12492uh/dKjV/I0ZmXKc7EFElSW+S9HdgtqR5kl7Z6T51M0lvlvQX4B6K59PeC/yuo52K6AJJrjHWfBl4le11gAOAr3aiE5I+1Im4LdBb8/cO2xsCu5OHmEcMKsk1xpoFtm8DsH09fcsfttNHBj+lK8wvqyQtqvlLcb9wRAwg11xjrKkt1r/Ytu0T2tQPDX5KV0jN34hhyGrhGFMkfWGg47a/2KZ+3G/7Re2I1Uqp+RsxPEmuEcM0SOm+5W2Pb3OXImKUSHKNMacsS3gUsFm5ay5wiu0rO9apLpOavxEjk+QaY4qkNwCnAMdTFO4XMBH4LHCU7YtaHH8FYF/gUNtvaGWsiBi9klxjTJF0JXCM7Zvq9m8FnGz71S2IuQzweuBQYG/gPOB827+uOlZEdIesFo6xZu36xApg+2ZJa1UZSNKeFE/F2Qu4guJRbNvbHrM1hyOiOUmuMdb8Z5jHhuMSioeH72z7HgBJJ1UcIyK6UJJrjDUbSZrWYL+AF1ccazvgYOAySXcDPweyQjgics01xhZJA15Ttf3HFsXdiWKK+ABgFnCB7SmtiBURo1+Sa0SFJI0D9gQOzrXXiCVXkmuMKZL2AdazfWq5fT2wRnn447bPrTDWgBWYbN9fVayI6C5JrjGmSLqWYtT4QLk9i+JJLisAP7K9e4WxZlMUWqitI2yKZL5mKjRFLLmyoCnGmmV6E2vpmrIO7j/KAg+Vsf2y2m1JE4BPAHsAX6kyVkR0lzxyLsaaF9Ru2D6qZnMNWkDSJpLOpHiI+Exgc9sntyJWRHSHJNcYa66X9J76nZLeC/y5ykCStpT0M4qKTJcBW9o+3fb8KuNERPfJNdcYUyStCfwKeIaitjAU96M+D9jX9t8qjLUQeAD4LbCw/rjto6uKFRHdJck1xiRJrwG2KDfn2P5DC2IcNtBx22dVHTMiukOSa0QFJK1I8Si2qkssRkQXyjXXiBGQ9H5J9wP3AfdLuk/SBzrdr4jorCTXiGGS9FngTcCutlezvRqwG/C68lhELKEyLRwxTJJuB7a2/XTd/uWAm2xv2pmeRUSnZeQaMQL1ibXc9xTQ04HuRMQokeQaMXzzJC1WTrFcqfxQB/oTEaNEpoUjhknSFsCFwDUUlZkMvBzYCdjH9pwOdi8iOijJNWIEJC0LHEpxT62AOcDURtPFEbHkSHKNiIioWJ6KEzFMkp6gmApe7BBFQYmV29yliBglMnKNGCZJS6dIf0Q0ktXCEcN3fac7EBGjU5JrxPCp0x2IiNEp11wjhm8NSR/p76DtE9rZmYgYPZJcI4ZvPLAiGcFGRJ0saIoYJkk32J7Y6X5ExOiTa64Rw5cRa0Q0lJFrxDBJWtX2PyXtRlGhycCttq/ocNciosOSXCOGSdK6wPnA0xS1hQVMBJYD9rP9YAe7FxEdlOQaMUySLgAutH1m3f53AgfY3qcjHYuIjktyjRgmSbfbfslQj0XE2JcFTRHDN77RTknj+jsWEUuGJNeI4fu1pB9IWqF3R/n+e8BFnetWRHRakmvE8H0c+Ddwn6SZkmYC9wKPA8d2smMR0Vm55hoxQpKWAzamWC18p+3/drhLEdFhSa4RwyRp/4GO2z6/XX2JiNElyTVimCT1ALPKF/St2GTb/9P+XkXEaJDkGjFMkvYDDqKYEr4Q+JntOzvbq4gYDZJcI0aoXCG8D0WiXQ34jO0/drZXEdFJWS0cMXJPU6wafhxYAVi2s92JiE7LyDVimMqC/YcA2wOXAT+3PaOzvYqI0SDJNWKYygVNNwPXUDwRp8//mWwf3Yl+RUTnLdXpDkR0sSM63YGIGJ0yco2ogKQVKW6/+U+n+xIRnZcFTREjIOn9ku4H7gPul3SfpA90ul8R0VlJrhHDJOmzwJuAXW2vZns1YDfgdeWxiFhCZVo4Ypgk3Q5sbfvpuv3LATfZ3rQzPYuITsvINWIE6hNrue8poKcD3YmIUSLJNWL45knavX6npNcAD3WgPxExSmRaOGKYJG1BUVP4GmAmxX2uLwd2AvaxPaeD3YuIDkpyjRgBScsChwJbUDwVZw4wtdF0cUQsOZJcIyomaTxwsO2pne5LRHRGrrlGDJOklSV9StIpkvZU4SjgbuCtne5fRHRORq4RwyTpQuBfwHXA7sALgGWAY2zPGuizETG2JblGDJOk2bZfVr4fDzwKvMj2E53tWUR0WqaFI4Zvfu8b2wuBe5JYIwIyco0YNkkLgf9QrBIGWA74b7lt2yt3qm8R0VlJrhERERXL81wjhqm8x/V9wMYUD00/w/aCzvYqIkaDjFwjhknSLyiuu14NvA64z/Yxne1VRIwGSa4Rw1S3Wngp4M+2J3a4WxExCmS1cMTw1a4WznRwRCySkWvEMNWsFoZihXBWC0cEkOQaERFRuUwLR0REVCzJNSIiomJJrhERERVLco2IiKhYkmtERETFklwjIiIq9v8BX+8YJFVxTH8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x15739358>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"--------------- CREATE A HEATMAP ---------------\")\n",
    "# Create a mask to display only the lower triangle of the matrix (since it's mirrored around its \n",
    "# top-left to bottom-right diagonal).\n",
    "mask = np.zeros_like(abs_corr_df)\n",
    "mask[np.triu_indices_from(mask)] = True\n",
    "# Create the heatmap using seaborn library. \n",
    "# List if colormaps (parameter 'cmap') is available here: http://matplotlib.org/examples/color/colormaps_reference.html\n",
    "seaborn.heatmap(abs_corr_df, cmap='RdYlGn_r', vmax=1.0, vmin=0 , mask = mask, linewidths=2.5)\n",
    " \n",
    "# Show the plot we reorient the labels for each column and row to make them easier to read.\n",
    "plt.yticks(rotation=0) \n",
    "plt.xticks(rotation=90) \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Open Data Set Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "## Procurement Open Data Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Does information reliability impacts the duration of the case? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_countrydate = pd.read_excel(report_folder + 'Countries_count.xlsx') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_countrydate[df_countrydate['date_commenced']=='Mid-'] = 2002"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_countrydate['duration'] =  df_countrydate['date_finished'] - pd.to_numeric(df_countrydate['date_commenced'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "group_country = df_countrydate.groupby('Country')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped_countrydate = group_country['duration'].aggregate(['mean','sum'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped_countrydate = grouped_countrydate.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_countrydate_rel = pd.merge(left = grouped_countrydate, right = pd_indicators_rel, left_on = 'Country', right_on = 'Country Name')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_countrydate_rel.rename(columns={2016:'Reliability2016'}, inplace = True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_countrydate_rel_an = df_countrydate_rel[['Country','mean', 'sum','Reliability2016']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_countrydate_rel_an.rename(columns= {'mean': 'mean_duration', 'sum': 'sum_duration'}, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>sum</th>\n",
       "      <th>Reliability2016</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0.1021</td>\n",
       "      <td>0.9792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sum</th>\n",
       "      <td>0.1021</td>\n",
       "      <td>0</td>\n",
       "      <td>0.7361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Reliability2016</th>\n",
       "      <td>0.9792</td>\n",
       "      <td>0.7361</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   mean     sum Reliability2016\n",
       "mean                  0  0.1021          0.9792\n",
       "sum              0.1021       0          0.7361\n",
       "Reliability2016  0.9792  0.7361               0"
      ]
     },
     "execution_count": 261,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calculate_pvalues(df_countrydate_rel_an) #very bad :( "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- CORRELATIONS ---------------\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>sum</th>\n",
       "      <th>Reliability2016</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.177494</td>\n",
       "      <td>-0.002870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sum</th>\n",
       "      <td>0.177494</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.037001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Reliability2016</th>\n",
       "      <td>-0.002870</td>\n",
       "      <td>0.037001</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     mean       sum  Reliability2016\n",
       "mean             1.000000  0.177494        -0.002870\n",
       "sum              0.177494  1.000000         0.037001\n",
       "Reliability2016 -0.002870  0.037001         1.000000"
      ]
     },
     "execution_count": 262,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corr_df = df_countrydate_rel_an.corr(method='pearson')\n",
    "print(\"--------------- CORRELATIONS ---------------\")\n",
    "corr_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add security breaches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_breaches = pd.read_excel(report_folder + \"Breach-index.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final_w_breaches = df_freedom_gdp_freind.iloc[:,[5, 4, 8,10, 20,18, 23, 25, 27, 29, 31, 32]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_countrydate_rel_breach_an = pd.merge(left = df_countrydate_rel_an, right = df_breaches, on = 'Country')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_countrydate_rel_breach_an1 = pd.merge(left = df_final_w_breaches, right = df_countrydate_rel_breach_an, on = 'Country')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Country</th>\n",
       "      <th>Count</th>\n",
       "      <th>GDP (BILLIONS)</th>\n",
       "      <th>PROMOTIONAL MEASURES</th>\n",
       "      <th>sum</th>\n",
       "      <th>label_num_h</th>\n",
       "      <th>label_num</th>\n",
       "      <th>corrupt perception</th>\n",
       "      <th>corporate transparency</th>\n",
       "      <th>Reliability of transparency</th>\n",
       "      <th>disclosure_index</th>\n",
       "      <th>GINI</th>\n",
       "      <th>mean_duration</th>\n",
       "      <th>sum_duration</th>\n",
       "      <th>Reliability2016</th>\n",
       "      <th>Breaches_2017</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Albania</td>\n",
       "      <td>2</td>\n",
       "      <td>13.4</td>\n",
       "      <td>10</td>\n",
       "      <td>969</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>36</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Algeria</td>\n",
       "      <td>3</td>\n",
       "      <td>227.8</td>\n",
       "      <td>0</td>\n",
       "      <td>845</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>35.3</td>\n",
       "      <td>8.333333</td>\n",
       "      <td>25.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Argentina</td>\n",
       "      <td>18</td>\n",
       "      <td>536.2</td>\n",
       "      <td>11</td>\n",
       "      <td>82</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>32</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>42.7</td>\n",
       "      <td>5.055556</td>\n",
       "      <td>91.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Country  Count  GDP (BILLIONS)  PROMOTIONAL MEASURES  sum  label_num_h  \\\n",
       "0    Albania      2            13.4                    10  969            0   \n",
       "1    Algeria      3           227.8                     0  845            1   \n",
       "2  Argentina     18           536.2                    11   82            2   \n",
       "\n",
       "   label_num  corrupt perception  corporate transparency  \\\n",
       "0          0                  36                     7.0   \n",
       "1          1                  36                     2.0   \n",
       "2          3                  32                     7.0   \n",
       "\n",
       "   Reliability of transparency  disclosure_index  GINI  mean_duration  \\\n",
       "0                          0.0               9.0  29.0       8.000000   \n",
       "1                          5.0               4.0  35.3       8.333333   \n",
       "2                          5.0               7.0  42.7       5.055556   \n",
       "\n",
       "   sum_duration  Reliability2016  Breaches_2017  \n",
       "0          16.0              0.0              0  \n",
       "1          25.0              5.0              0  \n",
       "2          91.0              5.0              1  "
      ]
     },
     "execution_count": 283,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_countrydate_rel_breach_an1.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [],
   "source": [
    "writer = pd.ExcelWriter(report_folder + 'df_countrydate_rel_breach_an1.xlsx')\n",
    "df_countrydate_rel_breach_an1.to_excel (writer)\n",
    "writer.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_countrydate_rel_breach_an1 = pd.read_excel(report_folder + 'df_countrydate_rel_breach_an1.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Count</th>\n",
       "      <th>GDP (BILLIONS)</th>\n",
       "      <th>PROMOTIONAL MEASURES</th>\n",
       "      <th>sum</th>\n",
       "      <th>label_num_h</th>\n",
       "      <th>label_num</th>\n",
       "      <th>corrupt perception</th>\n",
       "      <th>corporate transparency</th>\n",
       "      <th>Reliability of transparency</th>\n",
       "      <th>disclosure_index</th>\n",
       "      <th>GINI</th>\n",
       "      <th>mean_duration</th>\n",
       "      <th>sum_duration</th>\n",
       "      <th>Reliability2016</th>\n",
       "      <th>Breaches_2017</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Count</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0019</td>\n",
       "      <td>0.0205</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.242</td>\n",
       "      <td>0.0323</td>\n",
       "      <td>0.4202</td>\n",
       "      <td>0.0265</td>\n",
       "      <td>0.0749</td>\n",
       "      <td>0.5321</td>\n",
       "      <td>0</td>\n",
       "      <td>0.4202</td>\n",
       "      <td>0.6082</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GDP (BILLIONS)</th>\n",
       "      <td>0.0019</td>\n",
       "      <td>0</td>\n",
       "      <td>0.9448</td>\n",
       "      <td>0.6369</td>\n",
       "      <td>0.2897</td>\n",
       "      <td>0.6397</td>\n",
       "      <td>0.3425</td>\n",
       "      <td>0.0608</td>\n",
       "      <td>0.0192</td>\n",
       "      <td>0.1285</td>\n",
       "      <td>0.6633</td>\n",
       "      <td>0.0739</td>\n",
       "      <td>0.0048</td>\n",
       "      <td>0.0192</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PROMOTIONAL MEASURES</th>\n",
       "      <td>0.0205</td>\n",
       "      <td>0.9448</td>\n",
       "      <td>0</td>\n",
       "      <td>0.4099</td>\n",
       "      <td>0.0452</td>\n",
       "      <td>0.1818</td>\n",
       "      <td>0.9651</td>\n",
       "      <td>0.079</td>\n",
       "      <td>0.632</td>\n",
       "      <td>0.8664</td>\n",
       "      <td>0.7609</td>\n",
       "      <td>0.3276</td>\n",
       "      <td>0.0171</td>\n",
       "      <td>0.632</td>\n",
       "      <td>0.2672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sum</th>\n",
       "      <td>0</td>\n",
       "      <td>0.6369</td>\n",
       "      <td>0.4099</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0471</td>\n",
       "      <td>0.3126</td>\n",
       "      <td>0.4427</td>\n",
       "      <td>0.2871</td>\n",
       "      <td>0.0457</td>\n",
       "      <td>0.0165</td>\n",
       "      <td>0</td>\n",
       "      <td>0.4427</td>\n",
       "      <td>0.1245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label_num_h</th>\n",
       "      <td>0</td>\n",
       "      <td>0.2897</td>\n",
       "      <td>0.0452</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1885</td>\n",
       "      <td>0.0339</td>\n",
       "      <td>0.3268</td>\n",
       "      <td>0.0966</td>\n",
       "      <td>0.2007</td>\n",
       "      <td>0.1068</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3268</td>\n",
       "      <td>0.2581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label_num</th>\n",
       "      <td>0</td>\n",
       "      <td>0.6397</td>\n",
       "      <td>0.1818</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1624</td>\n",
       "      <td>0.0502</td>\n",
       "      <td>0.1392</td>\n",
       "      <td>0.0779</td>\n",
       "      <td>0.0619</td>\n",
       "      <td>0.1999</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1392</td>\n",
       "      <td>0.2911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>corrupt perception</th>\n",
       "      <td>0.242</td>\n",
       "      <td>0.3425</td>\n",
       "      <td>0.9651</td>\n",
       "      <td>0.0471</td>\n",
       "      <td>0.1885</td>\n",
       "      <td>0.1624</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0009</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2781</td>\n",
       "      <td>0.008</td>\n",
       "      <td>0.8022</td>\n",
       "      <td>0.2656</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>corporate transparency</th>\n",
       "      <td>0.0323</td>\n",
       "      <td>0.0608</td>\n",
       "      <td>0.079</td>\n",
       "      <td>0.3126</td>\n",
       "      <td>0.0339</td>\n",
       "      <td>0.0502</td>\n",
       "      <td>0.0009</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0026</td>\n",
       "      <td>0.0004</td>\n",
       "      <td>0.2621</td>\n",
       "      <td>0.0152</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Reliability of transparency</th>\n",
       "      <td>0.4202</td>\n",
       "      <td>0.0192</td>\n",
       "      <td>0.632</td>\n",
       "      <td>0.4427</td>\n",
       "      <td>0.3268</td>\n",
       "      <td>0.1392</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0.1163</td>\n",
       "      <td>0.4659</td>\n",
       "      <td>0.3254</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>disclosure_index</th>\n",
       "      <td>0.0265</td>\n",
       "      <td>0.1285</td>\n",
       "      <td>0.8664</td>\n",
       "      <td>0.2871</td>\n",
       "      <td>0.0966</td>\n",
       "      <td>0.0779</td>\n",
       "      <td>0.2781</td>\n",
       "      <td>0.0026</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1877</td>\n",
       "      <td>0.6087</td>\n",
       "      <td>0.0166</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0.605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GINI</th>\n",
       "      <td>0.0749</td>\n",
       "      <td>0.6633</td>\n",
       "      <td>0.7609</td>\n",
       "      <td>0.0457</td>\n",
       "      <td>0.2007</td>\n",
       "      <td>0.0619</td>\n",
       "      <td>0.008</td>\n",
       "      <td>0.0004</td>\n",
       "      <td>0.1163</td>\n",
       "      <td>0.1877</td>\n",
       "      <td>0</td>\n",
       "      <td>0.4101</td>\n",
       "      <td>0.1522</td>\n",
       "      <td>0.1163</td>\n",
       "      <td>0.5446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_duration</th>\n",
       "      <td>0.5321</td>\n",
       "      <td>0.0739</td>\n",
       "      <td>0.3276</td>\n",
       "      <td>0.0165</td>\n",
       "      <td>0.1068</td>\n",
       "      <td>0.1999</td>\n",
       "      <td>0.8022</td>\n",
       "      <td>0.2621</td>\n",
       "      <td>0.4659</td>\n",
       "      <td>0.6087</td>\n",
       "      <td>0.4101</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1095</td>\n",
       "      <td>0.4659</td>\n",
       "      <td>0.155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sum_duration</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0048</td>\n",
       "      <td>0.0171</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2656</td>\n",
       "      <td>0.0152</td>\n",
       "      <td>0.3254</td>\n",
       "      <td>0.0166</td>\n",
       "      <td>0.1522</td>\n",
       "      <td>0.1095</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3254</td>\n",
       "      <td>0.5627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Reliability2016</th>\n",
       "      <td>0.4202</td>\n",
       "      <td>0.0192</td>\n",
       "      <td>0.632</td>\n",
       "      <td>0.4427</td>\n",
       "      <td>0.3268</td>\n",
       "      <td>0.1392</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0.1163</td>\n",
       "      <td>0.4659</td>\n",
       "      <td>0.3254</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Breaches_2017</th>\n",
       "      <td>0.6082</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2672</td>\n",
       "      <td>0.1245</td>\n",
       "      <td>0.2581</td>\n",
       "      <td>0.2911</td>\n",
       "      <td>0.0148</td>\n",
       "      <td>0.8531</td>\n",
       "      <td>0.2392</td>\n",
       "      <td>0.605</td>\n",
       "      <td>0.5446</td>\n",
       "      <td>0.155</td>\n",
       "      <td>0.5627</td>\n",
       "      <td>0.2392</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              Count GDP (BILLIONS) PROMOTIONAL MEASURES  \\\n",
       "Count                             0         0.0019               0.0205   \n",
       "GDP (BILLIONS)               0.0019              0               0.9448   \n",
       "PROMOTIONAL MEASURES         0.0205         0.9448                    0   \n",
       "sum                               0         0.6369               0.4099   \n",
       "label_num_h                       0         0.2897               0.0452   \n",
       "label_num                         0         0.6397               0.1818   \n",
       "corrupt perception            0.242         0.3425               0.9651   \n",
       "corporate transparency       0.0323         0.0608                0.079   \n",
       "Reliability of transparency  0.4202         0.0192                0.632   \n",
       "disclosure_index             0.0265         0.1285               0.8664   \n",
       "GINI                         0.0749         0.6633               0.7609   \n",
       "mean_duration                0.5321         0.0739               0.3276   \n",
       "sum_duration                      0         0.0048               0.0171   \n",
       "Reliability2016              0.4202         0.0192                0.632   \n",
       "Breaches_2017                0.6082              0               0.2672   \n",
       "\n",
       "                                sum label_num_h label_num corrupt perception  \\\n",
       "Count                             0           0         0              0.242   \n",
       "GDP (BILLIONS)               0.6369      0.2897    0.6397             0.3425   \n",
       "PROMOTIONAL MEASURES         0.4099      0.0452    0.1818             0.9651   \n",
       "sum                               0           0         0             0.0471   \n",
       "label_num_h                       0           0         0             0.1885   \n",
       "label_num                         0           0         0             0.1624   \n",
       "corrupt perception           0.0471      0.1885    0.1624                  0   \n",
       "corporate transparency       0.3126      0.0339    0.0502             0.0009   \n",
       "Reliability of transparency  0.4427      0.3268    0.1392                  0   \n",
       "disclosure_index             0.2871      0.0966    0.0779             0.2781   \n",
       "GINI                         0.0457      0.2007    0.0619              0.008   \n",
       "mean_duration                0.0165      0.1068    0.1999             0.8022   \n",
       "sum_duration                      0           0         0             0.2656   \n",
       "Reliability2016              0.4427      0.3268    0.1392                  0   \n",
       "Breaches_2017                0.1245      0.2581    0.2911             0.0148   \n",
       "\n",
       "                            corporate transparency  \\\n",
       "Count                                       0.0323   \n",
       "GDP (BILLIONS)                              0.0608   \n",
       "PROMOTIONAL MEASURES                         0.079   \n",
       "sum                                         0.3126   \n",
       "label_num_h                                 0.0339   \n",
       "label_num                                   0.0502   \n",
       "corrupt perception                          0.0009   \n",
       "corporate transparency                           0   \n",
       "Reliability of transparency                      0   \n",
       "disclosure_index                            0.0026   \n",
       "GINI                                        0.0004   \n",
       "mean_duration                               0.2621   \n",
       "sum_duration                                0.0152   \n",
       "Reliability2016                                  0   \n",
       "Breaches_2017                               0.8531   \n",
       "\n",
       "                            Reliability of transparency disclosure_index  \\\n",
       "Count                                            0.4202           0.0265   \n",
       "GDP (BILLIONS)                                   0.0192           0.1285   \n",
       "PROMOTIONAL MEASURES                              0.632           0.8664   \n",
       "sum                                              0.4427           0.2871   \n",
       "label_num_h                                      0.3268           0.0966   \n",
       "label_num                                        0.1392           0.0779   \n",
       "corrupt perception                                    0           0.2781   \n",
       "corporate transparency                                0           0.0026   \n",
       "Reliability of transparency                           0           0.3608   \n",
       "disclosure_index                                 0.3608                0   \n",
       "GINI                                             0.1163           0.1877   \n",
       "mean_duration                                    0.4659           0.6087   \n",
       "sum_duration                                     0.3254           0.0166   \n",
       "Reliability2016                                       0           0.3608   \n",
       "Breaches_2017                                    0.2392            0.605   \n",
       "\n",
       "                               GINI mean_duration sum_duration  \\\n",
       "Count                        0.0749        0.5321            0   \n",
       "GDP (BILLIONS)               0.6633        0.0739       0.0048   \n",
       "PROMOTIONAL MEASURES         0.7609        0.3276       0.0171   \n",
       "sum                          0.0457        0.0165            0   \n",
       "label_num_h                  0.2007        0.1068            0   \n",
       "label_num                    0.0619        0.1999            0   \n",
       "corrupt perception            0.008        0.8022       0.2656   \n",
       "corporate transparency       0.0004        0.2621       0.0152   \n",
       "Reliability of transparency  0.1163        0.4659       0.3254   \n",
       "disclosure_index             0.1877        0.6087       0.0166   \n",
       "GINI                              0        0.4101       0.1522   \n",
       "mean_duration                0.4101             0       0.1095   \n",
       "sum_duration                 0.1522        0.1095            0   \n",
       "Reliability2016              0.1163        0.4659       0.3254   \n",
       "Breaches_2017                0.5446         0.155       0.5627   \n",
       "\n",
       "                            Reliability2016 Breaches_2017  \n",
       "Count                                0.4202        0.6082  \n",
       "GDP (BILLIONS)                       0.0192             0  \n",
       "PROMOTIONAL MEASURES                  0.632        0.2672  \n",
       "sum                                  0.4427        0.1245  \n",
       "label_num_h                          0.3268        0.2581  \n",
       "label_num                            0.1392        0.2911  \n",
       "corrupt perception                        0        0.0148  \n",
       "corporate transparency                    0        0.8531  \n",
       "Reliability of transparency               0        0.2392  \n",
       "disclosure_index                     0.3608         0.605  \n",
       "GINI                                 0.1163        0.5446  \n",
       "mean_duration                        0.4659         0.155  \n",
       "sum_duration                         0.3254        0.5627  \n",
       "Reliability2016                           0        0.2392  \n",
       "Breaches_2017                        0.2392             0  "
      ]
     },
     "execution_count": 285,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calculate_pvalues(df_countrydate_rel_breach_an1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- CORRELATIONS ---------------\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Count</th>\n",
       "      <th>GDP (BILLIONS)</th>\n",
       "      <th>PROMOTIONAL MEASURES</th>\n",
       "      <th>sum</th>\n",
       "      <th>label_num_h</th>\n",
       "      <th>label_num</th>\n",
       "      <th>corrupt perception</th>\n",
       "      <th>corporate transparency</th>\n",
       "      <th>Reliability of transparency</th>\n",
       "      <th>disclosure_index</th>\n",
       "      <th>GINI</th>\n",
       "      <th>mean_duration</th>\n",
       "      <th>sum_duration</th>\n",
       "      <th>Reliability2016</th>\n",
       "      <th>Breaches_2017</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Count</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.355255</td>\n",
       "      <td>0.270428</td>\n",
       "      <td>-0.551167</td>\n",
       "      <td>0.739385</td>\n",
       "      <td>0.674832</td>\n",
       "      <td>-0.131057</td>\n",
       "      <td>0.248997</td>\n",
       "      <td>0.098424</td>\n",
       "      <td>0.255426</td>\n",
       "      <td>0.207678</td>\n",
       "      <td>0.072200</td>\n",
       "      <td>0.986011</td>\n",
       "      <td>0.095116</td>\n",
       "      <td>-0.060109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GDP (BILLIONS)</th>\n",
       "      <td>0.355255</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.003345</td>\n",
       "      <td>0.056230</td>\n",
       "      <td>0.123195</td>\n",
       "      <td>0.055594</td>\n",
       "      <td>-0.104690</td>\n",
       "      <td>0.221086</td>\n",
       "      <td>0.273817</td>\n",
       "      <td>0.180857</td>\n",
       "      <td>0.050826</td>\n",
       "      <td>-0.210403</td>\n",
       "      <td>0.325120</td>\n",
       "      <td>0.271757</td>\n",
       "      <td>0.824477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PROMOTIONAL MEASURES</th>\n",
       "      <td>0.270428</td>\n",
       "      <td>-0.003345</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.094457</td>\n",
       "      <td>0.226058</td>\n",
       "      <td>0.156707</td>\n",
       "      <td>0.016669</td>\n",
       "      <td>0.229670</td>\n",
       "      <td>0.075510</td>\n",
       "      <td>0.057571</td>\n",
       "      <td>0.033414</td>\n",
       "      <td>0.107545</td>\n",
       "      <td>0.277116</td>\n",
       "      <td>0.056590</td>\n",
       "      <td>-0.127463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sum</th>\n",
       "      <td>-0.551167</td>\n",
       "      <td>0.056230</td>\n",
       "      <td>-0.094457</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.811996</td>\n",
       "      <td>-0.891605</td>\n",
       "      <td>0.230770</td>\n",
       "      <td>-0.112371</td>\n",
       "      <td>-0.087514</td>\n",
       "      <td>-0.116132</td>\n",
       "      <td>-0.233265</td>\n",
       "      <td>-0.278168</td>\n",
       "      <td>-0.571965</td>\n",
       "      <td>-0.090594</td>\n",
       "      <td>0.180331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label_num_h</th>\n",
       "      <td>0.739385</td>\n",
       "      <td>0.123195</td>\n",
       "      <td>0.226058</td>\n",
       "      <td>-0.811996</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.914758</td>\n",
       "      <td>-0.158651</td>\n",
       "      <td>0.230799</td>\n",
       "      <td>0.108476</td>\n",
       "      <td>0.175231</td>\n",
       "      <td>0.150950</td>\n",
       "      <td>0.190506</td>\n",
       "      <td>0.753384</td>\n",
       "      <td>0.115572</td>\n",
       "      <td>-0.133604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label_num</th>\n",
       "      <td>0.674832</td>\n",
       "      <td>0.055594</td>\n",
       "      <td>0.156707</td>\n",
       "      <td>-0.891605</td>\n",
       "      <td>0.914758</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.160468</td>\n",
       "      <td>0.224208</td>\n",
       "      <td>0.172937</td>\n",
       "      <td>0.200183</td>\n",
       "      <td>0.217927</td>\n",
       "      <td>0.150073</td>\n",
       "      <td>0.687387</td>\n",
       "      <td>0.173532</td>\n",
       "      <td>-0.124203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>corrupt perception</th>\n",
       "      <td>-0.131057</td>\n",
       "      <td>-0.104690</td>\n",
       "      <td>0.016669</td>\n",
       "      <td>0.230770</td>\n",
       "      <td>-0.158651</td>\n",
       "      <td>-0.160468</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.400648</td>\n",
       "      <td>0.489273</td>\n",
       "      <td>0.167003</td>\n",
       "      <td>-0.304511</td>\n",
       "      <td>0.021244</td>\n",
       "      <td>-0.125331</td>\n",
       "      <td>0.476767</td>\n",
       "      <td>-0.276363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>corporate transparency</th>\n",
       "      <td>0.248997</td>\n",
       "      <td>0.221086</td>\n",
       "      <td>0.229670</td>\n",
       "      <td>-0.112371</td>\n",
       "      <td>0.230799</td>\n",
       "      <td>0.224208</td>\n",
       "      <td>0.400648</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.576450</td>\n",
       "      <td>0.389091</td>\n",
       "      <td>-0.391238</td>\n",
       "      <td>0.116547</td>\n",
       "      <td>0.279257</td>\n",
       "      <td>0.563356</td>\n",
       "      <td>0.024738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Reliability of transparency</th>\n",
       "      <td>0.098424</td>\n",
       "      <td>0.273817</td>\n",
       "      <td>0.075510</td>\n",
       "      <td>-0.087514</td>\n",
       "      <td>0.108476</td>\n",
       "      <td>0.172937</td>\n",
       "      <td>0.489273</td>\n",
       "      <td>0.576450</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.144745</td>\n",
       "      <td>-0.184512</td>\n",
       "      <td>0.077666</td>\n",
       "      <td>0.118291</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.139164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>disclosure_index</th>\n",
       "      <td>0.255426</td>\n",
       "      <td>0.180857</td>\n",
       "      <td>0.057571</td>\n",
       "      <td>-0.116132</td>\n",
       "      <td>0.175231</td>\n",
       "      <td>0.200183</td>\n",
       "      <td>0.167003</td>\n",
       "      <td>0.389091</td>\n",
       "      <td>0.144745</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.153062</td>\n",
       "      <td>0.043711</td>\n",
       "      <td>0.272990</td>\n",
       "      <td>0.107752</td>\n",
       "      <td>0.062757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GINI</th>\n",
       "      <td>0.207678</td>\n",
       "      <td>0.050826</td>\n",
       "      <td>0.033414</td>\n",
       "      <td>-0.233265</td>\n",
       "      <td>0.150950</td>\n",
       "      <td>0.217927</td>\n",
       "      <td>-0.304511</td>\n",
       "      <td>-0.391238</td>\n",
       "      <td>-0.184512</td>\n",
       "      <td>-0.153062</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.096227</td>\n",
       "      <td>0.167625</td>\n",
       "      <td>-0.184125</td>\n",
       "      <td>0.071301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_duration</th>\n",
       "      <td>0.072200</td>\n",
       "      <td>-0.210403</td>\n",
       "      <td>0.107545</td>\n",
       "      <td>-0.278168</td>\n",
       "      <td>0.190506</td>\n",
       "      <td>0.150073</td>\n",
       "      <td>0.021244</td>\n",
       "      <td>0.116547</td>\n",
       "      <td>0.077666</td>\n",
       "      <td>0.043711</td>\n",
       "      <td>-0.096227</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.186079</td>\n",
       "      <td>0.086070</td>\n",
       "      <td>-0.167510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sum_duration</th>\n",
       "      <td>0.986011</td>\n",
       "      <td>0.325120</td>\n",
       "      <td>0.277116</td>\n",
       "      <td>-0.571965</td>\n",
       "      <td>0.753384</td>\n",
       "      <td>0.687387</td>\n",
       "      <td>-0.125331</td>\n",
       "      <td>0.279257</td>\n",
       "      <td>0.118291</td>\n",
       "      <td>0.272990</td>\n",
       "      <td>0.167625</td>\n",
       "      <td>0.186079</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.115919</td>\n",
       "      <td>-0.067978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Reliability2016</th>\n",
       "      <td>0.095116</td>\n",
       "      <td>0.271757</td>\n",
       "      <td>0.056590</td>\n",
       "      <td>-0.090594</td>\n",
       "      <td>0.115572</td>\n",
       "      <td>0.173532</td>\n",
       "      <td>0.476767</td>\n",
       "      <td>0.563356</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.107752</td>\n",
       "      <td>-0.184125</td>\n",
       "      <td>0.086070</td>\n",
       "      <td>0.115919</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.138533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Breaches_2017</th>\n",
       "      <td>-0.060109</td>\n",
       "      <td>0.824477</td>\n",
       "      <td>-0.127463</td>\n",
       "      <td>0.180331</td>\n",
       "      <td>-0.133604</td>\n",
       "      <td>-0.124203</td>\n",
       "      <td>-0.276363</td>\n",
       "      <td>0.024738</td>\n",
       "      <td>0.139164</td>\n",
       "      <td>0.062757</td>\n",
       "      <td>0.071301</td>\n",
       "      <td>-0.167510</td>\n",
       "      <td>-0.067978</td>\n",
       "      <td>0.138533</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                Count  GDP (BILLIONS)  PROMOTIONAL MEASURES  \\\n",
       "Count                        1.000000        0.355255              0.270428   \n",
       "GDP (BILLIONS)               0.355255        1.000000             -0.003345   \n",
       "PROMOTIONAL MEASURES         0.270428       -0.003345              1.000000   \n",
       "sum                         -0.551167        0.056230             -0.094457   \n",
       "label_num_h                  0.739385        0.123195              0.226058   \n",
       "label_num                    0.674832        0.055594              0.156707   \n",
       "corrupt perception          -0.131057       -0.104690              0.016669   \n",
       "corporate transparency       0.248997        0.221086              0.229670   \n",
       "Reliability of transparency  0.098424        0.273817              0.075510   \n",
       "disclosure_index             0.255426        0.180857              0.057571   \n",
       "GINI                         0.207678        0.050826              0.033414   \n",
       "mean_duration                0.072200       -0.210403              0.107545   \n",
       "sum_duration                 0.986011        0.325120              0.277116   \n",
       "Reliability2016              0.095116        0.271757              0.056590   \n",
       "Breaches_2017               -0.060109        0.824477             -0.127463   \n",
       "\n",
       "                                  sum  label_num_h  label_num  \\\n",
       "Count                       -0.551167     0.739385   0.674832   \n",
       "GDP (BILLIONS)               0.056230     0.123195   0.055594   \n",
       "PROMOTIONAL MEASURES        -0.094457     0.226058   0.156707   \n",
       "sum                          1.000000    -0.811996  -0.891605   \n",
       "label_num_h                 -0.811996     1.000000   0.914758   \n",
       "label_num                   -0.891605     0.914758   1.000000   \n",
       "corrupt perception           0.230770    -0.158651  -0.160468   \n",
       "corporate transparency      -0.112371     0.230799   0.224208   \n",
       "Reliability of transparency -0.087514     0.108476   0.172937   \n",
       "disclosure_index            -0.116132     0.175231   0.200183   \n",
       "GINI                        -0.233265     0.150950   0.217927   \n",
       "mean_duration               -0.278168     0.190506   0.150073   \n",
       "sum_duration                -0.571965     0.753384   0.687387   \n",
       "Reliability2016             -0.090594     0.115572   0.173532   \n",
       "Breaches_2017                0.180331    -0.133604  -0.124203   \n",
       "\n",
       "                             corrupt perception  corporate transparency  \\\n",
       "Count                                 -0.131057                0.248997   \n",
       "GDP (BILLIONS)                        -0.104690                0.221086   \n",
       "PROMOTIONAL MEASURES                   0.016669                0.229670   \n",
       "sum                                    0.230770               -0.112371   \n",
       "label_num_h                           -0.158651                0.230799   \n",
       "label_num                             -0.160468                0.224208   \n",
       "corrupt perception                     1.000000                0.400648   \n",
       "corporate transparency                 0.400648                1.000000   \n",
       "Reliability of transparency            0.489273                0.576450   \n",
       "disclosure_index                       0.167003                0.389091   \n",
       "GINI                                  -0.304511               -0.391238   \n",
       "mean_duration                          0.021244                0.116547   \n",
       "sum_duration                          -0.125331                0.279257   \n",
       "Reliability2016                        0.476767                0.563356   \n",
       "Breaches_2017                         -0.276363                0.024738   \n",
       "\n",
       "                             Reliability of transparency  disclosure_index  \\\n",
       "Count                                           0.098424          0.255426   \n",
       "GDP (BILLIONS)                                  0.273817          0.180857   \n",
       "PROMOTIONAL MEASURES                            0.075510          0.057571   \n",
       "sum                                            -0.087514         -0.116132   \n",
       "label_num_h                                     0.108476          0.175231   \n",
       "label_num                                       0.172937          0.200183   \n",
       "corrupt perception                              0.489273          0.167003   \n",
       "corporate transparency                          0.576450          0.389091   \n",
       "Reliability of transparency                     1.000000          0.144745   \n",
       "disclosure_index                                0.144745          1.000000   \n",
       "GINI                                           -0.184512         -0.153062   \n",
       "mean_duration                                   0.077666          0.043711   \n",
       "sum_duration                                    0.118291          0.272990   \n",
       "Reliability2016                                 1.000000          0.107752   \n",
       "Breaches_2017                                   0.139164          0.062757   \n",
       "\n",
       "                                 GINI  mean_duration  sum_duration  \\\n",
       "Count                        0.207678       0.072200      0.986011   \n",
       "GDP (BILLIONS)               0.050826      -0.210403      0.325120   \n",
       "PROMOTIONAL MEASURES         0.033414       0.107545      0.277116   \n",
       "sum                         -0.233265      -0.278168     -0.571965   \n",
       "label_num_h                  0.150950       0.190506      0.753384   \n",
       "label_num                    0.217927       0.150073      0.687387   \n",
       "corrupt perception          -0.304511       0.021244     -0.125331   \n",
       "corporate transparency      -0.391238       0.116547      0.279257   \n",
       "Reliability of transparency -0.184512       0.077666      0.118291   \n",
       "disclosure_index            -0.153062       0.043711      0.272990   \n",
       "GINI                         1.000000      -0.096227      0.167625   \n",
       "mean_duration               -0.096227       1.000000      0.186079   \n",
       "sum_duration                 0.167625       0.186079      1.000000   \n",
       "Reliability2016             -0.184125       0.086070      0.115919   \n",
       "Breaches_2017                0.071301      -0.167510     -0.067978   \n",
       "\n",
       "                             Reliability2016  Breaches_2017  \n",
       "Count                               0.095116      -0.060109  \n",
       "GDP (BILLIONS)                      0.271757       0.824477  \n",
       "PROMOTIONAL MEASURES                0.056590      -0.127463  \n",
       "sum                                -0.090594       0.180331  \n",
       "label_num_h                         0.115572      -0.133604  \n",
       "label_num                           0.173532      -0.124203  \n",
       "corrupt perception                  0.476767      -0.276363  \n",
       "corporate transparency              0.563356       0.024738  \n",
       "Reliability of transparency         1.000000       0.139164  \n",
       "disclosure_index                    0.107752       0.062757  \n",
       "GINI                               -0.184125       0.071301  \n",
       "mean_duration                       0.086070      -0.167510  \n",
       "sum_duration                        0.115919      -0.067978  \n",
       "Reliability2016                     1.000000       0.138533  \n",
       "Breaches_2017                       0.138533       1.000000  "
      ]
     },
     "execution_count": 286,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corr_df = df_countrydate_rel_breach_an1.corr(method='pearson')\n",
    "print(\"--------------- CORRELATIONS ---------------\")\n",
    "corr_df.head(len(df_countrydate_rel_breach_an1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analysis without the US "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final_wo_us = df_freedom_gdp_freind.iloc[:,[0, 1,2, 3, 5,6,7,8,9,10,11,12,13,14, 15]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final_wo_us = df_final_wo_us.drop(df_final_wo_us[df_final_wo_us['Country'] == 'United States'].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final_wo_us = df_final_wo_us.iloc[:,[1,2, 3, 4,5,6,7,8,9,10,11,12,13, 14]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Count</th>\n",
       "      <th>GDP (BILLIONS)</th>\n",
       "      <th>PROMOTIONAL MEASURES</th>\n",
       "      <th>label_num_h</th>\n",
       "      <th>label_num</th>\n",
       "      <th>corrupt perception</th>\n",
       "      <th>corporate transparency</th>\n",
       "      <th>Reliability of transparency</th>\n",
       "      <th>disclosure_index</th>\n",
       "      <th>GINI</th>\n",
       "      <th>mean_duration</th>\n",
       "      <th>sum_duration</th>\n",
       "      <th>Reliability2016</th>\n",
       "      <th>Breaches_2017</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Count</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0243</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1661</td>\n",
       "      <td>0.0326</td>\n",
       "      <td>0.3808</td>\n",
       "      <td>0.0252</td>\n",
       "      <td>0.069</td>\n",
       "      <td>0.5855</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3808</td>\n",
       "      <td>0.8803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GDP (BILLIONS)</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1202</td>\n",
       "      <td>0.0004</td>\n",
       "      <td>0.0119</td>\n",
       "      <td>0.0405</td>\n",
       "      <td>0.0016</td>\n",
       "      <td>0.0124</td>\n",
       "      <td>0.0433</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.247</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0124</td>\n",
       "      <td>0.0146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PROMOTIONAL MEASURES</th>\n",
       "      <td>0.0243</td>\n",
       "      <td>0.1202</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0621</td>\n",
       "      <td>0.276</td>\n",
       "      <td>0.6804</td>\n",
       "      <td>0.0755</td>\n",
       "      <td>0.5243</td>\n",
       "      <td>0.8188</td>\n",
       "      <td>0.6921</td>\n",
       "      <td>0.422</td>\n",
       "      <td>0.0208</td>\n",
       "      <td>0.5243</td>\n",
       "      <td>0.2007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label_num_h</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0004</td>\n",
       "      <td>0.0621</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0824</td>\n",
       "      <td>0.0323</td>\n",
       "      <td>0.2558</td>\n",
       "      <td>0.0845</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.1464</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2558</td>\n",
       "      <td>0.6709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label_num</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0119</td>\n",
       "      <td>0.276</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0186</td>\n",
       "      <td>0.1986</td>\n",
       "      <td>0.234</td>\n",
       "      <td>0.1754</td>\n",
       "      <td>0.0038</td>\n",
       "      <td>0.0714</td>\n",
       "      <td>0</td>\n",
       "      <td>0.234</td>\n",
       "      <td>0.9775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>corrupt perception</th>\n",
       "      <td>0.1661</td>\n",
       "      <td>0.0405</td>\n",
       "      <td>0.6804</td>\n",
       "      <td>0.0824</td>\n",
       "      <td>0.0186</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0004</td>\n",
       "      <td>0</td>\n",
       "      <td>0.205</td>\n",
       "      <td>0.0106</td>\n",
       "      <td>0.871</td>\n",
       "      <td>0.1793</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>corporate transparency</th>\n",
       "      <td>0.0326</td>\n",
       "      <td>0.0016</td>\n",
       "      <td>0.0755</td>\n",
       "      <td>0.0323</td>\n",
       "      <td>0.1986</td>\n",
       "      <td>0.0004</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0028</td>\n",
       "      <td>0.0004</td>\n",
       "      <td>0.2516</td>\n",
       "      <td>0.0153</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Reliability of transparency</th>\n",
       "      <td>0.3808</td>\n",
       "      <td>0.0124</td>\n",
       "      <td>0.5243</td>\n",
       "      <td>0.2558</td>\n",
       "      <td>0.234</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3899</td>\n",
       "      <td>0.0962</td>\n",
       "      <td>0.3585</td>\n",
       "      <td>0.2881</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>disclosure_index</th>\n",
       "      <td>0.0252</td>\n",
       "      <td>0.0433</td>\n",
       "      <td>0.8188</td>\n",
       "      <td>0.0845</td>\n",
       "      <td>0.1754</td>\n",
       "      <td>0.205</td>\n",
       "      <td>0.0028</td>\n",
       "      <td>0.3899</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1778</td>\n",
       "      <td>0.5579</td>\n",
       "      <td>0.0156</td>\n",
       "      <td>0.3899</td>\n",
       "      <td>0.0787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GINI</th>\n",
       "      <td>0.069</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.6921</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.0038</td>\n",
       "      <td>0.0106</td>\n",
       "      <td>0.0004</td>\n",
       "      <td>0.0962</td>\n",
       "      <td>0.1778</td>\n",
       "      <td>0</td>\n",
       "      <td>0.468</td>\n",
       "      <td>0.1404</td>\n",
       "      <td>0.0962</td>\n",
       "      <td>0.3801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_duration</th>\n",
       "      <td>0.5855</td>\n",
       "      <td>0.247</td>\n",
       "      <td>0.422</td>\n",
       "      <td>0.1464</td>\n",
       "      <td>0.0714</td>\n",
       "      <td>0.871</td>\n",
       "      <td>0.2516</td>\n",
       "      <td>0.3585</td>\n",
       "      <td>0.5579</td>\n",
       "      <td>0.468</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1287</td>\n",
       "      <td>0.3585</td>\n",
       "      <td>0.2157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sum_duration</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0208</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1793</td>\n",
       "      <td>0.0153</td>\n",
       "      <td>0.2881</td>\n",
       "      <td>0.0156</td>\n",
       "      <td>0.1404</td>\n",
       "      <td>0.1287</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2881</td>\n",
       "      <td>0.9172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Reliability2016</th>\n",
       "      <td>0.3808</td>\n",
       "      <td>0.0124</td>\n",
       "      <td>0.5243</td>\n",
       "      <td>0.2558</td>\n",
       "      <td>0.234</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3899</td>\n",
       "      <td>0.0962</td>\n",
       "      <td>0.3585</td>\n",
       "      <td>0.2881</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Breaches_2017</th>\n",
       "      <td>0.8803</td>\n",
       "      <td>0.0146</td>\n",
       "      <td>0.2007</td>\n",
       "      <td>0.6709</td>\n",
       "      <td>0.9775</td>\n",
       "      <td>0.0029</td>\n",
       "      <td>0.1161</td>\n",
       "      <td>0.0776</td>\n",
       "      <td>0.0787</td>\n",
       "      <td>0.3801</td>\n",
       "      <td>0.2157</td>\n",
       "      <td>0.9172</td>\n",
       "      <td>0.0776</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              Count GDP (BILLIONS) PROMOTIONAL MEASURES  \\\n",
       "Count                             0              0               0.0243   \n",
       "GDP (BILLIONS)                    0              0               0.1202   \n",
       "PROMOTIONAL MEASURES         0.0243         0.1202                    0   \n",
       "label_num_h                       0         0.0004               0.0621   \n",
       "label_num                         0         0.0119                0.276   \n",
       "corrupt perception           0.1661         0.0405               0.6804   \n",
       "corporate transparency       0.0326         0.0016               0.0755   \n",
       "Reliability of transparency  0.3808         0.0124               0.5243   \n",
       "disclosure_index             0.0252         0.0433               0.8188   \n",
       "GINI                          0.069           0.87               0.6921   \n",
       "mean_duration                0.5855          0.247                0.422   \n",
       "sum_duration                      0              0               0.0208   \n",
       "Reliability2016              0.3808         0.0124               0.5243   \n",
       "Breaches_2017                0.8803         0.0146               0.2007   \n",
       "\n",
       "                            label_num_h label_num corrupt perception  \\\n",
       "Count                                 0         0             0.1661   \n",
       "GDP (BILLIONS)                   0.0004    0.0119             0.0405   \n",
       "PROMOTIONAL MEASURES             0.0621     0.276             0.6804   \n",
       "label_num_h                           0         0             0.0824   \n",
       "label_num                             0         0             0.0186   \n",
       "corrupt perception               0.0824    0.0186                  0   \n",
       "corporate transparency           0.0323    0.1986             0.0004   \n",
       "Reliability of transparency      0.2558     0.234                  0   \n",
       "disclosure_index                 0.0845    0.1754              0.205   \n",
       "GINI                               0.17    0.0038             0.0106   \n",
       "mean_duration                    0.1464    0.0714              0.871   \n",
       "sum_duration                          0         0             0.1793   \n",
       "Reliability2016                  0.2558     0.234                  0   \n",
       "Breaches_2017                    0.6709    0.9775             0.0029   \n",
       "\n",
       "                            corporate transparency  \\\n",
       "Count                                       0.0326   \n",
       "GDP (BILLIONS)                              0.0016   \n",
       "PROMOTIONAL MEASURES                        0.0755   \n",
       "label_num_h                                 0.0323   \n",
       "label_num                                   0.1986   \n",
       "corrupt perception                          0.0004   \n",
       "corporate transparency                           0   \n",
       "Reliability of transparency                      0   \n",
       "disclosure_index                            0.0028   \n",
       "GINI                                        0.0004   \n",
       "mean_duration                               0.2516   \n",
       "sum_duration                                0.0153   \n",
       "Reliability2016                                  0   \n",
       "Breaches_2017                               0.1161   \n",
       "\n",
       "                            Reliability of transparency disclosure_index  \\\n",
       "Count                                            0.3808           0.0252   \n",
       "GDP (BILLIONS)                                   0.0124           0.0433   \n",
       "PROMOTIONAL MEASURES                             0.5243           0.8188   \n",
       "label_num_h                                      0.2558           0.0845   \n",
       "label_num                                         0.234           0.1754   \n",
       "corrupt perception                                    0            0.205   \n",
       "corporate transparency                                0           0.0028   \n",
       "Reliability of transparency                           0           0.3899   \n",
       "disclosure_index                                 0.3899                0   \n",
       "GINI                                             0.0962           0.1778   \n",
       "mean_duration                                    0.3585           0.5579   \n",
       "sum_duration                                     0.2881           0.0156   \n",
       "Reliability2016                                       0           0.3899   \n",
       "Breaches_2017                                    0.0776           0.0787   \n",
       "\n",
       "                               GINI mean_duration sum_duration  \\\n",
       "Count                         0.069        0.5855            0   \n",
       "GDP (BILLIONS)                 0.87         0.247            0   \n",
       "PROMOTIONAL MEASURES         0.6921         0.422       0.0208   \n",
       "label_num_h                    0.17        0.1464            0   \n",
       "label_num                    0.0038        0.0714            0   \n",
       "corrupt perception           0.0106         0.871       0.1793   \n",
       "corporate transparency       0.0004        0.2516       0.0153   \n",
       "Reliability of transparency  0.0962        0.3585       0.2881   \n",
       "disclosure_index             0.1778        0.5579       0.0156   \n",
       "GINI                              0         0.468       0.1404   \n",
       "mean_duration                 0.468             0       0.1287   \n",
       "sum_duration                 0.1404        0.1287            0   \n",
       "Reliability2016              0.0962        0.3585       0.2881   \n",
       "Breaches_2017                0.3801        0.2157       0.9172   \n",
       "\n",
       "                            Reliability2016 Breaches_2017  \n",
       "Count                                0.3808        0.8803  \n",
       "GDP (BILLIONS)                       0.0124        0.0146  \n",
       "PROMOTIONAL MEASURES                 0.5243        0.2007  \n",
       "label_num_h                          0.2558        0.6709  \n",
       "label_num                             0.234        0.9775  \n",
       "corrupt perception                        0        0.0029  \n",
       "corporate transparency                    0        0.1161  \n",
       "Reliability of transparency               0        0.0776  \n",
       "disclosure_index                     0.3899        0.0787  \n",
       "GINI                                 0.0962        0.3801  \n",
       "mean_duration                        0.3585        0.2157  \n",
       "sum_duration                         0.2881        0.9172  \n",
       "Reliability2016                           0        0.0776  \n",
       "Breaches_2017                        0.0776             0  "
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calculate_pvalues(df_final_wo_us)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- CORRELATIONS ---------------\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Count</th>\n",
       "      <th>GDP (BILLIONS)</th>\n",
       "      <th>PROMOTIONAL MEASURES</th>\n",
       "      <th>label_num_h</th>\n",
       "      <th>label_num</th>\n",
       "      <th>corrupt perception</th>\n",
       "      <th>corporate transparency</th>\n",
       "      <th>Reliability of transparency</th>\n",
       "      <th>disclosure_index</th>\n",
       "      <th>GINI</th>\n",
       "      <th>mean_duration</th>\n",
       "      <th>sum_duration</th>\n",
       "      <th>Reliability2016</th>\n",
       "      <th>Breaches_2017</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Count</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.704973</td>\n",
       "      <td>0.265105</td>\n",
       "      <td>0.739134</td>\n",
       "      <td>0.705479</td>\n",
       "      <td>-0.156000</td>\n",
       "      <td>0.250466</td>\n",
       "      <td>0.107430</td>\n",
       "      <td>0.259514</td>\n",
       "      <td>0.213361</td>\n",
       "      <td>0.063323</td>\n",
       "      <td>0.985973</td>\n",
       "      <td>0.104105</td>\n",
       "      <td>0.018694</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GDP (BILLIONS)</th>\n",
       "      <td>0.704973</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.187385</td>\n",
       "      <td>0.403899</td>\n",
       "      <td>0.293767</td>\n",
       "      <td>0.243881</td>\n",
       "      <td>0.361438</td>\n",
       "      <td>0.294232</td>\n",
       "      <td>0.239122</td>\n",
       "      <td>-0.020207</td>\n",
       "      <td>-0.138962</td>\n",
       "      <td>0.661961</td>\n",
       "      <td>0.291465</td>\n",
       "      <td>0.285522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PROMOTIONAL MEASURES</th>\n",
       "      <td>0.265105</td>\n",
       "      <td>0.187385</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.212106</td>\n",
       "      <td>0.131976</td>\n",
       "      <td>-0.024012</td>\n",
       "      <td>0.233980</td>\n",
       "      <td>0.094618</td>\n",
       "      <td>0.065412</td>\n",
       "      <td>0.044244</td>\n",
       "      <td>0.087799</td>\n",
       "      <td>0.271102</td>\n",
       "      <td>0.075717</td>\n",
       "      <td>0.153513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label_num_h</th>\n",
       "      <td>0.739134</td>\n",
       "      <td>0.403899</td>\n",
       "      <td>0.212106</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.792204</td>\n",
       "      <td>-0.207838</td>\n",
       "      <td>0.234949</td>\n",
       "      <td>0.127643</td>\n",
       "      <td>0.184046</td>\n",
       "      <td>0.162859</td>\n",
       "      <td>0.173127</td>\n",
       "      <td>0.752741</td>\n",
       "      <td>0.134715</td>\n",
       "      <td>-0.051461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label_num</th>\n",
       "      <td>0.705479</td>\n",
       "      <td>0.293767</td>\n",
       "      <td>0.131976</td>\n",
       "      <td>0.792204</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.265472</td>\n",
       "      <td>0.154820</td>\n",
       "      <td>0.143783</td>\n",
       "      <td>0.161985</td>\n",
       "      <td>0.333966</td>\n",
       "      <td>0.210510</td>\n",
       "      <td>0.721224</td>\n",
       "      <td>0.141033</td>\n",
       "      <td>0.004110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>corrupt perception</th>\n",
       "      <td>-0.156000</td>\n",
       "      <td>0.243881</td>\n",
       "      <td>-0.024012</td>\n",
       "      <td>-0.207838</td>\n",
       "      <td>-0.265472</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.423856</td>\n",
       "      <td>0.555856</td>\n",
       "      <td>0.191110</td>\n",
       "      <td>-0.295942</td>\n",
       "      <td>-0.027276</td>\n",
       "      <td>-0.152034</td>\n",
       "      <td>0.544357</td>\n",
       "      <td>0.342890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>corporate transparency</th>\n",
       "      <td>0.250466</td>\n",
       "      <td>0.361438</td>\n",
       "      <td>0.233980</td>\n",
       "      <td>0.234949</td>\n",
       "      <td>0.154820</td>\n",
       "      <td>0.423856</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.579339</td>\n",
       "      <td>0.388844</td>\n",
       "      <td>-0.393650</td>\n",
       "      <td>0.120685</td>\n",
       "      <td>0.281005</td>\n",
       "      <td>0.566442</td>\n",
       "      <td>0.186246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Reliability of transparency</th>\n",
       "      <td>0.107430</td>\n",
       "      <td>0.294232</td>\n",
       "      <td>0.094618</td>\n",
       "      <td>0.127643</td>\n",
       "      <td>0.143783</td>\n",
       "      <td>0.555856</td>\n",
       "      <td>0.579339</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.139246</td>\n",
       "      <td>-0.196618</td>\n",
       "      <td>0.100616</td>\n",
       "      <td>0.128398</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.209268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>disclosure_index</th>\n",
       "      <td>0.259514</td>\n",
       "      <td>0.239122</td>\n",
       "      <td>0.065412</td>\n",
       "      <td>0.184046</td>\n",
       "      <td>0.161985</td>\n",
       "      <td>0.191110</td>\n",
       "      <td>0.388844</td>\n",
       "      <td>0.139246</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.157794</td>\n",
       "      <td>0.052999</td>\n",
       "      <td>0.277592</td>\n",
       "      <td>0.102124</td>\n",
       "      <td>0.205684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GINI</th>\n",
       "      <td>0.213361</td>\n",
       "      <td>-0.020207</td>\n",
       "      <td>0.044244</td>\n",
       "      <td>0.162859</td>\n",
       "      <td>0.333966</td>\n",
       "      <td>-0.295942</td>\n",
       "      <td>-0.393650</td>\n",
       "      <td>-0.196618</td>\n",
       "      <td>-0.157794</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.085308</td>\n",
       "      <td>0.173681</td>\n",
       "      <td>-0.196203</td>\n",
       "      <td>-0.104644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_duration</th>\n",
       "      <td>0.063323</td>\n",
       "      <td>-0.138962</td>\n",
       "      <td>0.087799</td>\n",
       "      <td>0.173127</td>\n",
       "      <td>0.210510</td>\n",
       "      <td>-0.027276</td>\n",
       "      <td>0.120685</td>\n",
       "      <td>0.100616</td>\n",
       "      <td>0.052999</td>\n",
       "      <td>-0.085308</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.177944</td>\n",
       "      <td>0.109033</td>\n",
       "      <td>-0.147641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sum_duration</th>\n",
       "      <td>0.985973</td>\n",
       "      <td>0.661961</td>\n",
       "      <td>0.271102</td>\n",
       "      <td>0.752741</td>\n",
       "      <td>0.721224</td>\n",
       "      <td>-0.152034</td>\n",
       "      <td>0.281005</td>\n",
       "      <td>0.128398</td>\n",
       "      <td>0.277592</td>\n",
       "      <td>0.173681</td>\n",
       "      <td>0.177944</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.126002</td>\n",
       "      <td>-0.011717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Reliability2016</th>\n",
       "      <td>0.104105</td>\n",
       "      <td>0.291465</td>\n",
       "      <td>0.075717</td>\n",
       "      <td>0.134715</td>\n",
       "      <td>0.141033</td>\n",
       "      <td>0.544357</td>\n",
       "      <td>0.566442</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.102124</td>\n",
       "      <td>-0.196203</td>\n",
       "      <td>0.109033</td>\n",
       "      <td>0.126002</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.207866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Breaches_2017</th>\n",
       "      <td>0.018694</td>\n",
       "      <td>0.285522</td>\n",
       "      <td>0.153513</td>\n",
       "      <td>-0.051461</td>\n",
       "      <td>0.004110</td>\n",
       "      <td>0.342890</td>\n",
       "      <td>0.186246</td>\n",
       "      <td>0.209268</td>\n",
       "      <td>0.205684</td>\n",
       "      <td>-0.104644</td>\n",
       "      <td>-0.147641</td>\n",
       "      <td>-0.011717</td>\n",
       "      <td>0.207866</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                Count  GDP (BILLIONS)  PROMOTIONAL MEASURES  \\\n",
       "Count                        1.000000        0.704973              0.265105   \n",
       "GDP (BILLIONS)               0.704973        1.000000              0.187385   \n",
       "PROMOTIONAL MEASURES         0.265105        0.187385              1.000000   \n",
       "label_num_h                  0.739134        0.403899              0.212106   \n",
       "label_num                    0.705479        0.293767              0.131976   \n",
       "corrupt perception          -0.156000        0.243881             -0.024012   \n",
       "corporate transparency       0.250466        0.361438              0.233980   \n",
       "Reliability of transparency  0.107430        0.294232              0.094618   \n",
       "disclosure_index             0.259514        0.239122              0.065412   \n",
       "GINI                         0.213361       -0.020207              0.044244   \n",
       "mean_duration                0.063323       -0.138962              0.087799   \n",
       "sum_duration                 0.985973        0.661961              0.271102   \n",
       "Reliability2016              0.104105        0.291465              0.075717   \n",
       "Breaches_2017                0.018694        0.285522              0.153513   \n",
       "\n",
       "                             label_num_h  label_num  corrupt perception  \\\n",
       "Count                           0.739134   0.705479           -0.156000   \n",
       "GDP (BILLIONS)                  0.403899   0.293767            0.243881   \n",
       "PROMOTIONAL MEASURES            0.212106   0.131976           -0.024012   \n",
       "label_num_h                     1.000000   0.792204           -0.207838   \n",
       "label_num                       0.792204   1.000000           -0.265472   \n",
       "corrupt perception             -0.207838  -0.265472            1.000000   \n",
       "corporate transparency          0.234949   0.154820            0.423856   \n",
       "Reliability of transparency     0.127643   0.143783            0.555856   \n",
       "disclosure_index                0.184046   0.161985            0.191110   \n",
       "GINI                            0.162859   0.333966           -0.295942   \n",
       "mean_duration                   0.173127   0.210510           -0.027276   \n",
       "sum_duration                    0.752741   0.721224           -0.152034   \n",
       "Reliability2016                 0.134715   0.141033            0.544357   \n",
       "Breaches_2017                  -0.051461   0.004110            0.342890   \n",
       "\n",
       "                             corporate transparency  \\\n",
       "Count                                      0.250466   \n",
       "GDP (BILLIONS)                             0.361438   \n",
       "PROMOTIONAL MEASURES                       0.233980   \n",
       "label_num_h                                0.234949   \n",
       "label_num                                  0.154820   \n",
       "corrupt perception                         0.423856   \n",
       "corporate transparency                     1.000000   \n",
       "Reliability of transparency                0.579339   \n",
       "disclosure_index                           0.388844   \n",
       "GINI                                      -0.393650   \n",
       "mean_duration                              0.120685   \n",
       "sum_duration                               0.281005   \n",
       "Reliability2016                            0.566442   \n",
       "Breaches_2017                              0.186246   \n",
       "\n",
       "                             Reliability of transparency  disclosure_index  \\\n",
       "Count                                           0.107430          0.259514   \n",
       "GDP (BILLIONS)                                  0.294232          0.239122   \n",
       "PROMOTIONAL MEASURES                            0.094618          0.065412   \n",
       "label_num_h                                     0.127643          0.184046   \n",
       "label_num                                       0.143783          0.161985   \n",
       "corrupt perception                              0.555856          0.191110   \n",
       "corporate transparency                          0.579339          0.388844   \n",
       "Reliability of transparency                     1.000000          0.139246   \n",
       "disclosure_index                                0.139246          1.000000   \n",
       "GINI                                           -0.196618         -0.157794   \n",
       "mean_duration                                   0.100616          0.052999   \n",
       "sum_duration                                    0.128398          0.277592   \n",
       "Reliability2016                                 1.000000          0.102124   \n",
       "Breaches_2017                                   0.209268          0.205684   \n",
       "\n",
       "                                 GINI  mean_duration  sum_duration  \\\n",
       "Count                        0.213361       0.063323      0.985973   \n",
       "GDP (BILLIONS)              -0.020207      -0.138962      0.661961   \n",
       "PROMOTIONAL MEASURES         0.044244       0.087799      0.271102   \n",
       "label_num_h                  0.162859       0.173127      0.752741   \n",
       "label_num                    0.333966       0.210510      0.721224   \n",
       "corrupt perception          -0.295942      -0.027276     -0.152034   \n",
       "corporate transparency      -0.393650       0.120685      0.281005   \n",
       "Reliability of transparency -0.196618       0.100616      0.128398   \n",
       "disclosure_index            -0.157794       0.052999      0.277592   \n",
       "GINI                         1.000000      -0.085308      0.173681   \n",
       "mean_duration               -0.085308       1.000000      0.177944   \n",
       "sum_duration                 0.173681       0.177944      1.000000   \n",
       "Reliability2016             -0.196203       0.109033      0.126002   \n",
       "Breaches_2017               -0.104644      -0.147641     -0.011717   \n",
       "\n",
       "                             Reliability2016  Breaches_2017  \n",
       "Count                               0.104105       0.018694  \n",
       "GDP (BILLIONS)                      0.291465       0.285522  \n",
       "PROMOTIONAL MEASURES                0.075717       0.153513  \n",
       "label_num_h                         0.134715      -0.051461  \n",
       "label_num                           0.141033       0.004110  \n",
       "corrupt perception                  0.544357       0.342890  \n",
       "corporate transparency              0.566442       0.186246  \n",
       "Reliability of transparency         1.000000       0.209268  \n",
       "disclosure_index                    0.102124       0.205684  \n",
       "GINI                               -0.196203      -0.104644  \n",
       "mean_duration                       0.109033      -0.147641  \n",
       "sum_duration                        0.126002      -0.011717  \n",
       "Reliability2016                     1.000000       0.207866  \n",
       "Breaches_2017                       0.207866       1.000000  "
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corr_df = df_final_wo_us.corr(method='pearson')\n",
    "print(\"--------------- CORRELATIONS ---------------\")\n",
    "corr_df.head(len(df_final_wo_us))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Does duration correlate with anything at all?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_freedom_gdp_freind_an1 = df_freedom_gdp_freind.iloc[:,[5, 4, 8,10, 20,18, 23, 25, 27, 29, 31, 32]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final_countrydate_rel = pd.merge(left = df_freedom_gdp_freind_an1, right = df_countrydate_rel_an, on = 'Country' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Count</th>\n",
       "      <th>GDP (BILLIONS)</th>\n",
       "      <th>PROMOTIONAL MEASURES</th>\n",
       "      <th>sum</th>\n",
       "      <th>label_num_h</th>\n",
       "      <th>label_num</th>\n",
       "      <th>corrupt perception</th>\n",
       "      <th>corporate transparency</th>\n",
       "      <th>Reliability of transparency</th>\n",
       "      <th>disclosure_index</th>\n",
       "      <th>GINI</th>\n",
       "      <th>mean_duration</th>\n",
       "      <th>sum_duration</th>\n",
       "      <th>Reliability2016</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Count</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0019</td>\n",
       "      <td>0.0205</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.242</td>\n",
       "      <td>0.0323</td>\n",
       "      <td>0.4202</td>\n",
       "      <td>0.0265</td>\n",
       "      <td>0.0749</td>\n",
       "      <td>0.5321</td>\n",
       "      <td>0</td>\n",
       "      <td>0.4202</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GDP (BILLIONS)</th>\n",
       "      <td>0.0019</td>\n",
       "      <td>0</td>\n",
       "      <td>0.9448</td>\n",
       "      <td>0.6369</td>\n",
       "      <td>0.2897</td>\n",
       "      <td>0.6397</td>\n",
       "      <td>0.3425</td>\n",
       "      <td>0.0608</td>\n",
       "      <td>0.0192</td>\n",
       "      <td>0.1285</td>\n",
       "      <td>0.6633</td>\n",
       "      <td>0.0739</td>\n",
       "      <td>0.0048</td>\n",
       "      <td>0.0192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PROMOTIONAL MEASURES</th>\n",
       "      <td>0.0205</td>\n",
       "      <td>0.9448</td>\n",
       "      <td>0</td>\n",
       "      <td>0.4099</td>\n",
       "      <td>0.0452</td>\n",
       "      <td>0.1818</td>\n",
       "      <td>0.9651</td>\n",
       "      <td>0.079</td>\n",
       "      <td>0.632</td>\n",
       "      <td>0.8664</td>\n",
       "      <td>0.7609</td>\n",
       "      <td>0.3276</td>\n",
       "      <td>0.0171</td>\n",
       "      <td>0.632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sum</th>\n",
       "      <td>0</td>\n",
       "      <td>0.6369</td>\n",
       "      <td>0.4099</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0471</td>\n",
       "      <td>0.3126</td>\n",
       "      <td>0.4427</td>\n",
       "      <td>0.2871</td>\n",
       "      <td>0.0457</td>\n",
       "      <td>0.0165</td>\n",
       "      <td>0</td>\n",
       "      <td>0.4427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label_num_h</th>\n",
       "      <td>0</td>\n",
       "      <td>0.2897</td>\n",
       "      <td>0.0452</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1885</td>\n",
       "      <td>0.0339</td>\n",
       "      <td>0.3268</td>\n",
       "      <td>0.0966</td>\n",
       "      <td>0.2007</td>\n",
       "      <td>0.1068</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label_num</th>\n",
       "      <td>0</td>\n",
       "      <td>0.6397</td>\n",
       "      <td>0.1818</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1624</td>\n",
       "      <td>0.0502</td>\n",
       "      <td>0.1392</td>\n",
       "      <td>0.0779</td>\n",
       "      <td>0.0619</td>\n",
       "      <td>0.1999</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>corrupt perception</th>\n",
       "      <td>0.242</td>\n",
       "      <td>0.3425</td>\n",
       "      <td>0.9651</td>\n",
       "      <td>0.0471</td>\n",
       "      <td>0.1885</td>\n",
       "      <td>0.1624</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0009</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2781</td>\n",
       "      <td>0.008</td>\n",
       "      <td>0.8022</td>\n",
       "      <td>0.2656</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>corporate transparency</th>\n",
       "      <td>0.0323</td>\n",
       "      <td>0.0608</td>\n",
       "      <td>0.079</td>\n",
       "      <td>0.3126</td>\n",
       "      <td>0.0339</td>\n",
       "      <td>0.0502</td>\n",
       "      <td>0.0009</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0026</td>\n",
       "      <td>0.0004</td>\n",
       "      <td>0.2621</td>\n",
       "      <td>0.0152</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Reliability of transparency</th>\n",
       "      <td>0.4202</td>\n",
       "      <td>0.0192</td>\n",
       "      <td>0.632</td>\n",
       "      <td>0.4427</td>\n",
       "      <td>0.3268</td>\n",
       "      <td>0.1392</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0.1163</td>\n",
       "      <td>0.4659</td>\n",
       "      <td>0.3254</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>disclosure_index</th>\n",
       "      <td>0.0265</td>\n",
       "      <td>0.1285</td>\n",
       "      <td>0.8664</td>\n",
       "      <td>0.2871</td>\n",
       "      <td>0.0966</td>\n",
       "      <td>0.0779</td>\n",
       "      <td>0.2781</td>\n",
       "      <td>0.0026</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1877</td>\n",
       "      <td>0.6087</td>\n",
       "      <td>0.0166</td>\n",
       "      <td>0.3608</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GINI</th>\n",
       "      <td>0.0749</td>\n",
       "      <td>0.6633</td>\n",
       "      <td>0.7609</td>\n",
       "      <td>0.0457</td>\n",
       "      <td>0.2007</td>\n",
       "      <td>0.0619</td>\n",
       "      <td>0.008</td>\n",
       "      <td>0.0004</td>\n",
       "      <td>0.1163</td>\n",
       "      <td>0.1877</td>\n",
       "      <td>0</td>\n",
       "      <td>0.4101</td>\n",
       "      <td>0.1522</td>\n",
       "      <td>0.1163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean_duration</th>\n",
       "      <td>0.5321</td>\n",
       "      <td>0.0739</td>\n",
       "      <td>0.3276</td>\n",
       "      <td>0.0165</td>\n",
       "      <td>0.1068</td>\n",
       "      <td>0.1999</td>\n",
       "      <td>0.8022</td>\n",
       "      <td>0.2621</td>\n",
       "      <td>0.4659</td>\n",
       "      <td>0.6087</td>\n",
       "      <td>0.4101</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1095</td>\n",
       "      <td>0.4659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sum_duration</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0048</td>\n",
       "      <td>0.0171</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2656</td>\n",
       "      <td>0.0152</td>\n",
       "      <td>0.3254</td>\n",
       "      <td>0.0166</td>\n",
       "      <td>0.1522</td>\n",
       "      <td>0.1095</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Reliability2016</th>\n",
       "      <td>0.4202</td>\n",
       "      <td>0.0192</td>\n",
       "      <td>0.632</td>\n",
       "      <td>0.4427</td>\n",
       "      <td>0.3268</td>\n",
       "      <td>0.1392</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0.1163</td>\n",
       "      <td>0.4659</td>\n",
       "      <td>0.3254</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              Count GDP (BILLIONS) PROMOTIONAL MEASURES  \\\n",
       "Count                             0         0.0019               0.0205   \n",
       "GDP (BILLIONS)               0.0019              0               0.9448   \n",
       "PROMOTIONAL MEASURES         0.0205         0.9448                    0   \n",
       "sum                               0         0.6369               0.4099   \n",
       "label_num_h                       0         0.2897               0.0452   \n",
       "label_num                         0         0.6397               0.1818   \n",
       "corrupt perception            0.242         0.3425               0.9651   \n",
       "corporate transparency       0.0323         0.0608                0.079   \n",
       "Reliability of transparency  0.4202         0.0192                0.632   \n",
       "disclosure_index             0.0265         0.1285               0.8664   \n",
       "GINI                         0.0749         0.6633               0.7609   \n",
       "mean_duration                0.5321         0.0739               0.3276   \n",
       "sum_duration                      0         0.0048               0.0171   \n",
       "Reliability2016              0.4202         0.0192                0.632   \n",
       "\n",
       "                                sum label_num_h label_num corrupt perception  \\\n",
       "Count                             0           0         0              0.242   \n",
       "GDP (BILLIONS)               0.6369      0.2897    0.6397             0.3425   \n",
       "PROMOTIONAL MEASURES         0.4099      0.0452    0.1818             0.9651   \n",
       "sum                               0           0         0             0.0471   \n",
       "label_num_h                       0           0         0             0.1885   \n",
       "label_num                         0           0         0             0.1624   \n",
       "corrupt perception           0.0471      0.1885    0.1624                  0   \n",
       "corporate transparency       0.3126      0.0339    0.0502             0.0009   \n",
       "Reliability of transparency  0.4427      0.3268    0.1392                  0   \n",
       "disclosure_index             0.2871      0.0966    0.0779             0.2781   \n",
       "GINI                         0.0457      0.2007    0.0619              0.008   \n",
       "mean_duration                0.0165      0.1068    0.1999             0.8022   \n",
       "sum_duration                      0           0         0             0.2656   \n",
       "Reliability2016              0.4427      0.3268    0.1392                  0   \n",
       "\n",
       "                            corporate transparency  \\\n",
       "Count                                       0.0323   \n",
       "GDP (BILLIONS)                              0.0608   \n",
       "PROMOTIONAL MEASURES                         0.079   \n",
       "sum                                         0.3126   \n",
       "label_num_h                                 0.0339   \n",
       "label_num                                   0.0502   \n",
       "corrupt perception                          0.0009   \n",
       "corporate transparency                           0   \n",
       "Reliability of transparency                      0   \n",
       "disclosure_index                            0.0026   \n",
       "GINI                                        0.0004   \n",
       "mean_duration                               0.2621   \n",
       "sum_duration                                0.0152   \n",
       "Reliability2016                                  0   \n",
       "\n",
       "                            Reliability of transparency disclosure_index  \\\n",
       "Count                                            0.4202           0.0265   \n",
       "GDP (BILLIONS)                                   0.0192           0.1285   \n",
       "PROMOTIONAL MEASURES                              0.632           0.8664   \n",
       "sum                                              0.4427           0.2871   \n",
       "label_num_h                                      0.3268           0.0966   \n",
       "label_num                                        0.1392           0.0779   \n",
       "corrupt perception                                    0           0.2781   \n",
       "corporate transparency                                0           0.0026   \n",
       "Reliability of transparency                           0           0.3608   \n",
       "disclosure_index                                 0.3608                0   \n",
       "GINI                                             0.1163           0.1877   \n",
       "mean_duration                                    0.4659           0.6087   \n",
       "sum_duration                                     0.3254           0.0166   \n",
       "Reliability2016                                       0           0.3608   \n",
       "\n",
       "                               GINI mean_duration sum_duration Reliability2016  \n",
       "Count                        0.0749        0.5321            0          0.4202  \n",
       "GDP (BILLIONS)               0.6633        0.0739       0.0048          0.0192  \n",
       "PROMOTIONAL MEASURES         0.7609        0.3276       0.0171           0.632  \n",
       "sum                          0.0457        0.0165            0          0.4427  \n",
       "label_num_h                  0.2007        0.1068            0          0.3268  \n",
       "label_num                    0.0619        0.1999            0          0.1392  \n",
       "corrupt perception            0.008        0.8022       0.2656               0  \n",
       "corporate transparency       0.0004        0.2621       0.0152               0  \n",
       "Reliability of transparency  0.1163        0.4659       0.3254               0  \n",
       "disclosure_index             0.1877        0.6087       0.0166          0.3608  \n",
       "GINI                              0        0.4101       0.1522          0.1163  \n",
       "mean_duration                0.4101             0       0.1095          0.4659  \n",
       "sum_duration                 0.1522        0.1095            0          0.3254  \n",
       "Reliability2016              0.1163        0.4659       0.3254               0  "
      ]
     },
     "execution_count": 272,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calculate_pvalues(df_final_countrydate_rel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# split by ; and strip \\n\n",
    "country_list = find_between(keyfacts, '\\nCountry.\\n  \\n','\\nDate' ).replace('\\n','').strip() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "date_filed = find_between(keyfacts,'\\nDate Filed.\\n', '\\nCountry.\\n  \\n' ).replace('\\n','').strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "date_of_conduct = find_between(keyfacts,'Date of Conduct.\\n  \\n', '\\nAmount of the Value' ).replace('\\n','').strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "amount = find_between(keyfacts,'\\nAmount of the Value', '\\nAmount of Business Related' ).replace('\\n','').strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "total_sanction = find_between(keyfacts,'\\nTotal Sanction.\\n', '\\nCompliance Monitor' ).replace('\\n','').strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fcpa_provision = find_between(keyfacts,'\\nFCPA Statutory Provision', 'nOther Statutory Provision' ).replace('\\n','').strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'November 29, 2017'"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "date_filed.replace('\\n','').strip().strip('.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "startTime = time.perf_counter()\n",
    "row_list = []\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a1 = list(zip(['a','b','c'],[ 1,2,3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'a': 1, 'b': 2, 'c': 3}"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict((k[0],k[1]) for k in a1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "RuntimeWarning: '<' not supported between instances of 'str' and 'int', sort order is undefined for incomparable objects [api.py:87]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date_filed</th>\n",
       "      <th>Countries</th>\n",
       "      <th>Date_Interval</th>\n",
       "      <th>Amount_of_Value</th>\n",
       "      <th>Total_Sanction</th>\n",
       "      <th>FCPA_Statutory_Provision</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>\\nNovember 29, 2017.\\n</td>\n",
       "      <td>Brazil; Angola; Equatorial Guinea; Kazakhstan;...</td>\n",
       "      <td>1996  2012.</td>\n",
       "      <td>.  At least $180 million.</td>\n",
       "      <td>$238,000,000.</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Date_filed Countries Date_Interval Amount_of_Value Total_Sanction  \\\n",
       "0        NaN       NaN           NaN             NaN            NaN   \n",
       "\n",
       "  FCPA_Statutory_Provision                          0  \\\n",
       "0                      NaN    \\nNovember 29, 2017.\\n    \n",
       "\n",
       "                                                   1            2  \\\n",
       "0  Brazil; Angola; Equatorial Guinea; Kazakhstan;...  1996  2012.   \n",
       "\n",
       "                           3              4 5  \n",
       "0  .  At least $180 million.  $238,000,000.    "
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.append([[date_filed, country_list, date_of_conduct, amount, total_sanction, fcpa_provision]] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df3 = pandas.DataFrame(row_list, columns=['A','B','C','D','E'])\n",
    "print('Elapsed time: {:6.3f} seconds for {:d} rows'.format(time.perf_counter() - startTime, numOfRows))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#how to get all of the necessary parts together?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Brazil', ' Angola', ' Equatorial Guinea', ' Kazakhstan', ' Iraq.']"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "split_up(country_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}