Training and Feature Extraction with Reassigned Labels - ICI Mode, Peak2Peak, RMSE, Eigen.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: scipy in /mnt/anaconda/lib/python3.6/site-packages\r\n"
     ]
    }
   ],
   "source": [
    "!pip install scipy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing require dependencies and starting a spark context."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "from pyspark import SparkContext\n",
    "sc = SparkContext(pyFiles=['lib/numpy_pack.py','lib/row_parser.py','lib/spark_PCA.py'])\n",
    "\n",
    "from pyspark.sql import Row, SQLContext,DataFrame\n",
    "from pyspark.sql.types import *\n",
    "\n",
    "sqlContext = SQLContext(sc)\n",
    "\n",
    "%pylab inline\n",
    "\n",
    "#!pip install pandas\n",
    "#!pip install scipy\n",
    "\n",
    "import pandas as pd\n",
    "import datetime as dt\n",
    "\n",
    "from scipy.io import loadmat,savemat,whosmat\n",
    "\n",
    "from string import split\n",
    "from collections import Counter\n",
    "import re\n",
    "import numpy as np\n",
    "from numpy import shape\n",
    "\n",
    "from glob import glob\n",
    "from time import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the Dataset and Parsing the Rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('time', 'datetime') {'start': 0, 'parser': <function parse_date at 0x7fc72dd8a668>, 'end': 1, 'name': 'time'}\n",
      "('species', 'str') {'start': 1, 'parser': <function parse_string at 0x7fc72dd8a848>, 'end': 2, 'name': 'species'}\n",
      "('site', 'str') {'start': 2, 'parser': <function parse_string at 0x7fc72dd8a848>, 'end': 3, 'name': 'site'}\n",
      "('rec_no', 'str') {'start': 3, 'parser': <function parse_string at 0x7fc72dd8a848>, 'end': 4, 'name': 'rec_no'}\n",
      "('bout_i', 'int') {'start': 4, 'parser': <function parse_int at 0x7fc72dd8a758>, 'end': 5, 'name': 'bout_i'}\n",
      "('peak2peak', 'float') {'start': 5, 'parser': <function parse_float at 0x7fc72dd8a7d0>, 'end': 6, 'name': 'peak2peak'}\n",
      "('MSN', 'array', 202) {'start': 6, 'parser': <function parse_array at 0x7fc72dd8a6e0>, 'end': 208, 'name': 'MSN'}\n",
      "('MSP', 'array', 101) {'start': 208, 'parser': <function parse_array at 0x7fc72dd8a6e0>, 'end': 309, 'name': 'MSP'}\n",
      "('TPWS1', 'bool') {'start': 309, 'parser': <function parse_int at 0x7fc72dd8a758>, 'end': 310, 'name': 'TPWS1'}\n",
      "('MD1', 'bool') {'start': 310, 'parser': <function parse_int at 0x7fc72dd8a758>, 'end': 311, 'name': 'MD1'}\n",
      "('FD1', 'bool') {'start': 311, 'parser': <function parse_int at 0x7fc72dd8a758>, 'end': 312, 'name': 'FD1'}\n",
      "('TPWS2', 'bool') {'start': 312, 'parser': <function parse_int at 0x7fc72dd8a758>, 'end': 313, 'name': 'TPWS2'}\n",
      "('MD2', 'bool') {'start': 313, 'parser': <function parse_int at 0x7fc72dd8a758>, 'end': 314, 'name': 'MD2'}\n",
      "('FD2', 'bool') {'start': 314, 'parser': <function parse_int at 0x7fc72dd8a758>, 'end': 315, 'name': 'FD2'}\n",
      "('TPWS3', 'bool') {'start': 315, 'parser': <function parse_int at 0x7fc72dd8a758>, 'end': 316, 'name': 'TPWS3'}\n",
      "('MD3', 'bool') {'start': 316, 'parser': <function parse_int at 0x7fc72dd8a758>, 'end': 317, 'name': 'MD3'}\n",
      "('FD3', 'bool') {'start': 317, 'parser': <function parse_int at 0x7fc72dd8a758>, 'end': 318, 'name': 'FD3'}\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.append('lib')\n",
    "from row_parser import *\n",
    "\n",
    "Parse_rules,field_names,RowObject = init_parser_parameters()\n",
    "\n",
    "from pyspark.sql import DataFrame\n",
    "\n",
    "CVS_Data=sc.textFile(\"/CVS/*/\")\n",
    "\n",
    "\n",
    "def parse(row):\n",
    "    items=row.split(',')\n",
    "    D=[]\n",
    "    for pr in Parse_rules:\n",
    "        start=pr['start']\n",
    "        end=pr['end']\n",
    "        parser=pr['parser']\n",
    "        if end-start==1:\n",
    "            D.append(parser(items[start]))\n",
    "        else:\n",
    "            D.append(parser(items[start:end]))\n",
    "    return RowObject(*D)\n",
    "\n",
    "\n",
    "RDD=CVS_Data.map(parse)\n",
    "\n",
    "df=sqlContext.createDataFrame(RDD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4144531"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Caching the Dataframe\n",
    "df = df.filter(df.TPWS2==1).filter(df.MD2==0).filter(df.FD2==0)\n",
    "df.cache().count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions to Read MSP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from spark_PCA import *\n",
    "from row_parser import unpackArray\n",
    "\n",
    "def trimmed_spectra(row):\n",
    "    return unpackArray(row.MSP,data_type=numpy.float64)[12:85]\n",
    "\n",
    "def unpackArray(x,data_type=numpy.int16):\n",
    "    return np.frombuffer(x,dtype=data_type)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Eigen Vector Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of E= (73,) shape of NE= (73,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fc7297e2f10>]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VfWd//HXJwmELWFL2AIhBAMIqCyRRRZ3pFrFTheX\nWlttRW2daTu102Xm9/jNtI/f49HWmenP/mqraF3GfdrRwrQOqF0ERJAtyG5CCAkJIQlJCBBClvv5\n/ZFbGyPKRW5ybu59Px8PHnDOPefyvgfyzsn33vM95u6IiEjiSAo6gIiIdC8Vv4hIglHxi4gkGBW/\niEiCUfGLiCQYFb+ISIJR8YuIJBgVv4hIglHxi4gkmJSgA5xORkaG5+TkBB1DRKTH2Lx5c427Z0ay\nbUwWf05ODps2bQo6hohIj2FmByLdVkM9IiIJRsUvIpJgVPwiIglGxS8ikmBU/CIiCUbFLyKSYFT8\nIiIJJiY/xy8ikihCIae45jhbDtRT29jMPZeO7/K/U8UvItKNjja2sLWsjq2l9WwpraOgrJ5jTa0A\nDE9PZemCXJKSrEszqPhFRLpIW8jZW3nsfUVfXH0CgCSDCcPT+OSFo5iRPYjp2YPJzejf5aUPERa/\nmS0GHgSSgcfc/UedHh8IPANkh5/zX939ifBjJcAxoA1odff8qKUXEYkhNcdPURAu+K2l9Ww7WE9j\ncxsAQ/r3Zkb2ID49YzTTswdx4ehBDEgN5tz7jH+rmSUDDwFXAweBjWa2wt13ddjsa8Aud7/ezDKB\nvWb2rLs3hx+/3N1roh1eRCQoLW0hdh9qYGtpPVtL69hSWk9pbSMAKUnG+SPT+ezM0UzPHsz07EFk\nD+mHWdefzUcikm83s4Aidy8GMLMXgCVAx+J3IM3aX9UAoBZojXJWEZHAHG5oYsuBOraWtRf9OweP\ncqo1BMCwtFRmZA/m87OzmTF2MFNHDaRv7+SAE3+4SIo/CyjrsHwQmN1pm58DK4AKIA24yd1D4ccc\neN3M2oBH3H3Z6f4SM1sKLAXIzs6O+AWIiERbU0sbOysa2BoestlaWkfF0SYAeicnMTUrndvmjGV6\neGx+1MA+MXM2H4loDTBdAxQAVwDjgdfMbI27NwDz3b3czIaF1+9x99WdnyD8DWEZQH5+vkcpl4jI\nR3J3yutPsiVc8FtL69lV0UBzW/u5a9agvswYO5ivhIdsJo9KJzUlds/mIxFJ8ZcDYzosjw6v6+gO\n4Efu7kCRme0HJgFvu3s5gLtXmdnLtA8dfaD4RUS6g7tTVnuS9cVH3vv1l7P5Pr2SuHD0IO6Yn8P0\nMYOZkT2IYel9Ak4cfZEU/0Ygz8zG0V74NwO3dtqmFLgSWGNmw4GJQLGZ9QeS3P1Y+M+LgB9ELb2I\nyBl8VNEP7d+bOblDuXvcEGaOHczEEWn0So7/CQ3OWPzu3mpm9wGraP845+PuvtPM7gk//jDwQ+BJ\nM9sOGPAdd68xs1zg5fDYVwrwnLuv7KLXIiKCu1Na2xgu+VrWFx/hUKeivzd3CHNyh3LesAE9amw+\nWqx9dCa25Ofnu269KCKR+KiizxjQm9m5Q5mTO5S5uUMYnxm/RW9mmyO9TkpX7opIj6KiP3cqfhGJ\nae7OgSPtRb9hf+eiT2VO7hBmq+jPiopfRGJKx6L/y1l9ZcP7i35O+Kx+fGZ/Ff3HoOIXkUCp6Luf\nil9Eul3l0SbWFFbzZlHN+4o+My01XPLtZZ+boaLvCip+Eelyjc2tbCiuZU1hDWsKqymsOg60n9HP\nHa+i724qfhGJulDI2VFx9L2i33ygjpY2JzUliVnjhvC5/DHMz8tg0og0FX0AVPwiEhXl9SdZW1jN\nmsIa3iyqoa6xBYDJI9O5c944FuRlkp8zmD69evY8N/FAxS8iH8vxU62s33eEtUU1rC6sfu/OUsPS\nUrli0nAW5GUw77wMMtNSA04qnan4RSQibSHnnYP1rC2sYU1RDVsO1NEacvr0SmL2uKHcOiubhRMy\nyUvQaRB6EhW/iHyostpG1hTWsLaomjeLjnD0ZPvwzdSsdO5amMuC8zKYmTO4x09TnGhU/CLynoam\nFt7ad6T9rL6wmpIj7bcSHDmwD4smD2fBhEzmjR/K0AEavunJVPwiCay1LcS2g0dZU1jN2sIatpbV\n0xZy+vVOZk7uUL54SQ4L8jI0FUKcUfGLJJiqhiZe232Y1e9Ws27fEY41tWIGF2YN5J5Lc1mQl8mM\n7MH0Ton/eekTlYpfJAEcOHKCVTsrWbmjkq1l9bi331LwugtGMj8vg3njMxjcv3fQMaWbqPhF4pC7\ns6fy2Htlv6fyGABTRqXzzasmcM2UEUwYruGbRKXiF4kToZCztayeVTsrWbWzkgNHGjGD/LGD+afr\nzueaKSMYM6Rf0DElBqj4RXqwlrYQG4prWbnzEK/uPEzVsVP0Sjbmjs/g7oXjuWryMIalxd/NwuXc\nqPhFepimljZWv1vNyp2V/GF3FUdPttC3VzKXTczkmikjuHzSMAb27RV0TIlhKn6RHuDoyRb+tKeK\nVTsr+fPeak62tJHeJ4WrJg/nmikjWJiXSd/euohKIqPiF4lR1cdO8dquw6zcWclb+2poaXOGpaXy\n6ZlZLJ4yktm5Q+iVrI9cytlT8YvEkLLaxvfenN10oA53GDu0H3fOG8eiKSOYPmYQSUn6JI6cGxW/\nSIDcncKq46zc0V72OysaAJg0Io2vX5nHNVNGaM56iToVv0g3C4Wcd8qPsnJHJa/urKS4pn0645lj\nB/P9aydxzZQRjB3aP+CUEs9U/CLdwN0pKKtneUEFK3dUUtnQREqSMXf8UO6YP45Fk4czPF0fu5Tu\nEVHxm9li4EEgGXjM3X/U6fGBwDNAdvg5/9Xdn4hkX5F4VlR1jOUFFSwvqKC0tpHeKUlcNiGTf5g6\nkSsnDWdgP33sUrrfGYvfzJKBh4CrgYPARjNb4e67Omz2NWCXu19vZpnAXjN7FmiLYF+RuFJRf5L/\n3tZe9rsONZBkMO+8DP7uyjwWTRlOeh+VvQQrkjP+WUCRuxcDmNkLwBKgY3k7kGbt70ANAGqBVmB2\nBPuK9Hh1J5p5ZcchlhdU8Pb+WgCmjRnE/75+MtddOFJXz0pMiaT4s4CyDssHaS/0jn4OrAAqgDTg\nJncPmVkk+4r0SI3Nrby+u4oVBeW88W41LW3O+Mz+fOvqCdwwbZTeoJWYFa03d68BCoArgPHAa2a2\n5myewMyWAksBsrOzoxRLJLpa2kKsLaxheUE5r+46TGNzGyPS+3DHvHHccNEopoxK10cvJeZFUvzl\nwJgOy6PD6zq6A/iRuztQZGb7gUkR7guAuy8DlgHk5+d7ROlFukEo5GwpreO3BeX8/p1D1DW2MLBv\nL5ZMy2LJtFHMyhmii6qkR4mk+DcCeWY2jvbSvhm4tdM2pcCVwBozGw5MBIqB+gj2FYlJeyobWF5Q\nwYqCCsrrT9KnVxJXnT+cG6dlsXBCpu5QJT3WGYvf3VvN7D5gFe0fyXzc3Xea2T3hxx8Gfgg8aWbb\nAQO+4+41AKfbt2teisi5K6ttZMW29rLfe/gYyUnGgrwM7r9mAldPHsGAVF36Ij2ftY/OxJb8/Hzf\ntGlT0DEkQRw5fopXtrd/ImfTgTqg/eYlS6aN4toLRjJ0QGrACUXOzMw2u3t+JNvq9EUS0vFTrby2\nq5LlBRWsKayhLeRMHJ7GPyyeyPUXjtKdqiSuqfglYTS3hnjj3WqWF5Tz+u7DNLWEyBrUl6ULc1ky\nbRSTRqQHHVGkW6j4Je4VVR3n6bdK+G1BBUdPtjCkf28+O3MMS6aNYkb2YH0iRxKOil/iUlvI+dOe\nKp56q4Q1hTX0Tk5i8dQRfGp6FvPzMnQDE0loKn6JK0cbW/jPTWU8vf4ApbWNjEjvw/2LJnDzrGwy\n9CatCKDilzixt/IYT64r4bdbyznZ0sasnCF8Z/EkFk0ZrrN7kU5U/NJjtbaFeH13FU+u28/64lpS\nU5K4cVoWt18ylimjBgYdTyRmqfilx6k70cwLG8t4Zv0ByutPkjWoL9/9xCRuyh/D4P69g44nEvNU\n/NJj7Kw4ylPrSlheUMGp1hBzc4fyvz45mavOH0aKhnNEIqbil5jW0hZi1c5KnlpXwsaSOvr2SubT\nM0fzxbk5TByRFnQ8kR5JxS8xqeb4KZ7fUMqzG0qpbGgie0g//um68/nszDG6XaHIOVLxS0zZVlbP\nU+tK+N07h2huC7EgL4P/86mpXDZxGMm60EokKlT8Erjm1hD/s+MQT64rYWtpPf17J3PLrDF8YW4O\n5w0bEHQ8kbij4pfAVDU08eyGUp57u5TqY6fIzejPP18/mU/PHE2abkgu0mVU/NKt3J0tpe3DOa9s\nP0RryLl8YiZfvCSHhXmZmjdHpBuo+KVbNLW08bt3DvHUuhK2lx8lLTWF2+fmcPvcseRk6KbkIt1J\nxS9dqupYE0+tK+H5t8uoPdHMecMG8MMbp/I307Por7tZiQRCX3nSJeobm3n4jWKeXLefU60hrjp/\nOF+6JIdLxg/FTMM5IkFS8UtUHT/VyuNr9/Po6mKON7dyw0Wj+MZVExin4RyRmKHil6hoamnjmfUH\n+MWf91F7oplFk4fz94sm6K5WIjFIxS/npKUtxH9uKuP//aGIyoYmFuRl8K1FE5k2ZlDQ0UTkQ6j4\n5WNpCzkrtpXz09cKKa1tZObYwfz0pmnMHT806GgicgYqfjkr7s6qnYf599f28u7h40wemc7jX8rn\n8onD9KatSA+h4peIuDtrCmv4t1f3su3gUXIz+/PzW6dz7dSRuuhKpIdR8csZbSqp5YFVe9mwv5as\nQX35yWcu5G+mZ2kOfJEeKqLiN7PFwINAMvCYu/+o0+PfBj7f4TnPBzLdvdbMSoBjQBvQ6u75Ucou\nXWxH+VH+7dW9/GlvNRkDUvmXG6Zw86wxpKYkBx1NRM7BGYvfzJKBh4CrgYPARjNb4e67/rKNuz8A\nPBDe/nrgm+5e2+FpLnf3mqgmly5TVHWcn772Lr/ffoiBfXvxncWT+OIlY+nXWz8gisSDSL6SZwFF\n7l4MYGYvAEuAXR+y/S3A89GJJ92prLaRB/9QyEtbDtK3VzJ/d8V5fGVhLumaKVMkrkRS/FlAWYfl\ng8Ds021oZv2AxcB9HVY78LqZtQGPuPuyj5lVukhVQxM//1MRz79diplx57xx3HvZeIYOSA06moh0\ngWj/7H498GanYZ757l5uZsOA18xsj7uv7ryjmS0FlgJkZ2dHOZacTt2JZh5evY+n1pXQ2uZ87uIx\n/O0V5zFyYN+go4lIF4qk+MuBMR2WR4fXnc7NdBrmcffy8O9VZvYy7UNHHyj+8E8CywDy8/M9glzy\nMR1rauHxtSU8tqZ9Pp0bp2XxjavyGDtU8+mIJIJIin8jkGdm42gv/JuBWztvZGYDgUuB2zqs6w8k\nufux8J8XAT+IRnA5e00tbTz91gF+8eci6hpbuGbKcP7+6olMHJEWdDQR6UZnLH53bzWz+4BVtH+c\n83F332lm94Qffzi86aeAV939RIfdhwMvh6/oTAGec/eV0XwBcmbNrSFe3FTGz/9YyOGGUyzIy+D+\nRRO5SPPpiCQkc4+9UZX8/HzftGlT0DF6vLaQs7ygnJ++/i5ltSfJHzuY+6+ZyJxczacjEm/MbHOk\n10npg9lxqrj6OHc/vZnCquNMGZXOE3dM5bIJmZpPR0RU/PHoxKlW7n56M0dONPOLz89g8ZQRmk9H\nRN6j4o8z7s73XtrOvurjPP3l2cw7LyPoSCISYzTLVpx5ev0BVmyr4FuLJqr0ReS0VPxxZEtpHT/8\n3S6unDSMey8dH3QcEYlRKv44ceT4Kb727BaGp/fh3z83TWP6IvKhNMYfB9pCzjdeLODIiWZeuvcS\nBvbTpGoi8uF0xh8HHnz9XdYU1vDDJVOYmjUw6DgiEuNU/D3cH/cc5md/LOJz+aO56WJNbiciZ6bi\n78HKahv55ovbmDwynR8smRp0HBHpIVT8PVRTSxtffXYLIXd+edsM+vTS7RBFJDJ6c7eH+pf/3sX2\n8qM8enu+plMWkbOiM/4e6Nebynj+7VK+etl4rp48POg4ItLDqPh7mF0VDfzTb3cwN3cof3/1hKDj\niEgPpOLvQY6ebOHeZzczqF8vfnbLdFKS9c8nImdPY/w9hLtz/6+3UV53kheWziEzTTdCF5GPR6eM\nPcQjq4t5bddhvn/t+eTnDAk6joj0YCr+HuCtfUf4yco9XHfhSO6YlxN0HBHp4VT8Me5wQxN/+/xW\nxmX058efvlB30BKRc6Yx/hjW0hbivue20NjcyvN3zWZAqv65ROTcqUli2I//Zw8bS+p48OZp5A1P\nCzqOiMQJDfXEqFe2H+Kxtfv50iU5LJmWFXQcEYkjKv4YtK/6ON/+9TamZw/i+9eeH3QcEYkzKv4Y\n09jcyr3PbCa1VzIP3TqD3in6JxKR6NIYfwxxd7730nYKq47z9J2zGTWob9CRRCQORXQ6aWaLzWyv\nmRWZ2XdP8/i3zawg/GuHmbWZ2ZBI9pW/emb9AZYXVPCtqycwPy8j6DgiEqfOWPxmlgw8BHwCmAzc\nYmaTO27j7g+4+zR3nwZ8D3jD3Wsj2VfabS2t4we/28UVk4bx1cvOCzqOiMSxSM74ZwFF7l7s7s3A\nC8CSj9j+FuD5j7lvQqo90czXnt3C8PQ+/PRz00hK0kVaItJ1Iin+LKCsw/LB8LoPMLN+wGLgv852\n30TVFnK+/sJWak4088vPz2Rgv15BRxKROBftj4xcD7zp7rVnu6OZLTWzTWa2qbq6OsqxYteDfyhk\nTWENP7hhCheMHhh0HBFJAJEUfzkwpsPy6PC607mZvw7znNW+7r7M3fPdPT8zMzOCWD3fn/ZW8bM/\nFPLZmaO56eIxZ95BRCQKIin+jUCemY0zs960l/uKzhuZ2UDgUmD52e6biMpqG/nmiwWcPzKdH944\nVZOviUi3OePn+N291czuA1YBycDj7r7TzO4JP/5weNNPAa+6+4kz7RvtF9HTNLW08dVnt9AWcn75\n+Rn06ZUcdCQRSSDm7kFn+ID8/HzftGlT0DG6zPdf3s5zG0pZ9oWZLJoyIug4IhIHzGyzu+dHsq3m\nA+hmv9l8kOc2lHLvZeNV+iISCBV/N9p9qIF/fHk7c3OH8q2rJwQdR0QSlIq/mzQ0tXDvM5sZ2LcX\nP7tlOinJOvQiEgxN0tYN3J37/3MbB+tO8sLSOWSmpQYdSUQSmE47u8Gy1cW8uusw37v2fPJzhgQd\nR0QSnIq/i60vPsKPV+7hugtGcue8nKDjiIio+LtSVUMT9z23lZyM/vz4MxfqIi0RiQka4+8iLW0h\nvvbcFk6cauW5u2YzIFWHWkRig9qoi/xk5R42ltTx4M3TmDA8Leg4IiLv0VBPF/if7Yd4dM1+vjh3\nLEumaRZqEYktKv4oq2po4tu/eYdpYwbxj9fpZmMiEntU/FH2xLoSGptb+elN0+idosMrIrFHzRRF\nx0+18sz6AyyeOoJxGf2DjiMicloq/ih64e1SjjW1snTh+KCjiIh8KBV/lLS0hXjizRJmjRvCtDGD\ngo4jIvKhVPxR8sr2Q5TXn+TuhblBRxER+Ugq/ihwdx55o5jxmf25fOKwoOOIiHwkFX8UvFl0hF2H\nGli6MJekJE3LICKxTcUfBY+s3kdmWio3TtfFWiIS+1T852hXRQNrCmv40iU5pKbopukiEvtU/Ofo\nsTXF9OudzG2zxwYdRUQkIir+c1BRf5IV2yq4+eJsBvbrFXQcEZGIqPjPwRNv7seBO+fnBB1FRCRi\nKv6PqaGpheffLuO6C0YyenC/oOOIiERMxf8xPbehlOOnWlmqC7ZEpIeJqPjNbLGZ7TWzIjP77ods\nc5mZFZjZTjN7o8P6EjPbHn5sU7SCB6m5NcQTb+5n3nlDmZo1MOg4IiJn5Yx34DKzZOAh4GrgILDR\nzFa4+64O2wwCfgEsdvdSM+t8+erl7l4TxdyBWrGtgsMNp/jJZy4KOoqIyFmL5Ix/FlDk7sXu3gy8\nACzptM2twEvuXgrg7lXRjRk73J1HVxczaUQaC/Mygo4jInLWIin+LKCsw/LB8LqOJgCDzezPZrbZ\nzG7v8JgDr4fXLz23uMH787vV7D18jKULczHT9Awi0vNE62brKcBM4EqgL/CWma1393eB+e5eHh7+\nec3M9rj76s5PEP6msBQgOzs7SrGib9kbxYxI78MnLxwVdBQRkY8lkjP+cmBMh+XR4XUdHQRWufuJ\n8Fj+auAiAHcvD/9eBbxM+9DRB7j7MnfPd/f8zMzMs3sV3WT7waO8VXyEO+fn6LaKItJjRdJeG4E8\nMxtnZr2Bm4EVnbZZDsw3sxQz6wfMBnabWX8zSwMws/7AImBH9OJ3r0dW7yMtNYVbZsXuTyQiImdy\nxqEed281s/uAVUAy8Li77zSze8KPP+zuu81sJfAOEAIec/cdZpYLvBweC08BnnP3lV31YrpSWW0j\nr2w/xF0Lcknro+kZRKTnimiM391fAV7ptO7hTssPAA90WldMeMinp/vV2v0kJxl3zBsXdBQRkXOi\ngeoI1Dc28+LGMm64KIsRA/sEHUdE5Jyo+CPwzPoDnGxp466FOtsXkZ5PxX8GTS1tPLnuAJdOyGTS\niPSg44iInDMV/xm8vLWcmuOnuFuTsYlInFDxf4RQyHl0TTFTs9KZO35o0HFERKJCxf8R/rCniuLq\nEyxdOF7TM4hI3FDxf4Rlq/eRNagv104dEXQUEZGoUfF/iC2ldWwsqePL88eRkqzDJCLxQ432IZa9\nUczAvr246eIxZ95YRKQHUfGfxv6aE6zaVcltc7LpnxqtCUxFRGKDiv80frW2mF5JSXzxkpygo4iI\nRJ2Kv5Mjx0/x600H+ZsZWQxL0/QMIhJ/VPyd/MdbBzjVGuIrC3TBlojEJxV/Byeb2/iPt0q46vxh\nnDdsQNBxRES6hIq/g99sLqOusYWlC8cHHUVEpMuo+MPaQs5ja/czbcwgLs4ZHHQcEZEuo+IPe3Vn\nJQeONHL3wlxNzyAicU3FD7g7j6wuJmdoPxZN0fQMIhLfVPzAxpI6Csrq+fKCXJKTdLYvIvFNxU/7\nZGxD+vfmMzNGBx1FRKTLJXzxF1Ud4/XdVXxhzlj69k4OOo6ISJdL+OJ/dPV+UlOSuH3u2KCjiIh0\ni4Qu/qpjTby8tZzP5o9m6IDUoOOIiHSLhC7+p9aV0BIK8ZX5mp5BRBJHwhb/iVOtPLO+lMVTRpCT\n0T/oOCIi3SZhi//FjWUcPdnCXQt1ti8iiSWi4jezxWa218yKzOy7H7LNZWZWYGY7zeyNs9m3u7W2\nhfjV2v1cnDOYGdmankFEEssZi9/MkoGHgE8Ak4FbzGxyp20GAb8AbnD3KcBnI903CL/ffojy+pOa\njE1EElIkZ/yzgCJ3L3b3ZuAFYEmnbW4FXnL3UgB3rzqLfbuVu/PommLGZ/bnyknDgowiIhKISIo/\nCyjrsHwwvK6jCcBgM/uzmW02s9vPYl8AzGypmW0ys03V1dWRpf8Y3tp3hB3lDdy1IJckTc8gIgko\nWncSTwFmAlcCfYG3zGz92TyBuy8DlgHk5+d7lHJ9wCOri8kYkMqN00/7/UdEJO5FUvzlwJgOy6PD\n6zo6CBxx9xPACTNbDVwUXn+mfbvNnsoG3ni3mvsXTaBPL03PICKJKZKhno1AnpmNM7PewM3Aik7b\nLAfmm1mKmfUDZgO7I9y32yxbXUy/3sncNkfTM4hI4jrjGb+7t5rZfcAqIBl43N13mtk94ccfdvfd\nZrYSeAcIAY+5+w6A0+3bRa/lIx06epIVBRXcNmcsg/r1DiKCiEhMiGiM391fAV7ptO7hTssPAA9E\nsm8QnnyzBAe+PH9c0FFERAKVEFfuHmtq4bkNpVx7wUjGDOkXdBwRkUAlRPE//3Ypx061snSBpmcQ\nEYn74m9uDfH42hLm5g7lgtEDg44jIhK4uC/+/95WQWVDE0sv1dm+iAjEefH/ZXqGicPTuGxCZtBx\nRERiQlwX/+rCGvZUHuOuhbmYaXoGERGI8+JftnofI9L7cMNFo4KOIiISM+K2+HeUH+XNoiPcMS+H\n3ilx+zJFRM5a3DbistXFDEhN4ZbZ2UFHERGJKXFZ/AfrGvn99kPcMmsM6X16BR1HRCSmxGXx/2rt\nfgy4Y56mZxAR6Szuiv9oYwsvbizjhotGMWpQ36DjiIjEnLgr/mc2HKCxuY27FuqCLRGR04mr4j/V\n2saT60pYOCGT80emBx1HRCQmxVXx/3ZrOdXHTmkyNhGRjxA3xR8KOctWFzN5ZDrzzhsadBwRkZgV\nrZutB66xpY2Lc4awIC9T0zOIiHyEuCn+Aakp/OjTFwYdQ0Qk5sXNUI+IiERGxS8ikmBU/CIiCUbF\nLyKSYFT8IiIJRsUvIpJgVPwiIglGxS8ikmDM3YPO8AFmVg0c+Ji7ZwA1UYzTk+lYvJ+Ox/vpePxV\nPByLse6eGcmGMVn858LMNrl7ftA5YoGOxfvpeLyfjsdfJdqx0FCPiEiCUfGLiCSYeCz+ZUEHiCE6\nFu+n4/F+Oh5/lVDHIu7G+EVE5KPF4xm/iIh8hLgpfjNbbGZ7zazIzL4bdJ4gmdkYM/uTme0ys51m\n9vWgMwXNzJLNbKuZ/S7oLEEzs0Fm9hsz22Nmu81sbtCZgmRm3wx/newws+fNrE/QmbpaXBS/mSUD\nDwGfACYDt5jZ5GBTBaoV+Ja7TwbmAF9L8OMB8HVgd9AhYsSDwEp3nwRcRAIfFzPLAv4OyHf3qUAy\ncHOwqbpeXBQ/MAsocvdid28GXgCWBJwpMO5+yN23hP98jPYv7KxgUwXHzEYD1wGPBZ0laGY2EFgI\n/ArA3ZvdvT7YVIFLAfqaWQrQD6gIOE+Xi5fizwLKOiwfJIGLriMzywGmAxuCTRKo/wv8AxAKOkgM\nGAdUA0+Eh74eM7P+QYcKiruXA/8KlAKHgKPu/mqwqbpevBS/nIaZDQD+C/iGuzcEnScIZvZJoMrd\nNwedJUakADOAX7r7dOAEkLDviZnZYNpHB8YBo4D+ZnZbsKm6XrwUfzkwpsPy6PC6hGVmvWgv/Wfd\n/aWg8wRgNOWhAAABAklEQVRoHnCDmZXQPgR4hZk9E2ykQB0EDrr7X34C/A3t3wgS1VXAfnevdvcW\n4CXgkoAzdbl4Kf6NQJ6ZjTOz3rS/ObMi4EyBMTOjfQx3t7v/e9B5guTu33P30e6eQ/v/iz+6e9yf\n0X0Yd68EysxsYnjVlcCuACMFrRSYY2b9wl83V5IAb3anBB0gGty91czuA1bR/q784+6+M+BYQZoH\nfAHYbmYF4XXfd/dXAswkseNvgWfDJ0nFwB0B5wmMu28ws98AW2j/NNxWEuAqXl25KyKSYOJlqEdE\nRCKk4hcRSTAqfhGRBKPiFxFJMCp+EZEEo+IXEUkwKn4RkQSj4hcRSTD/HwQqVvf/UwmRAAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc7682d5610>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## Get Spectra\n",
    "spectra=df.rdd.map(trimmed_spectra)\n",
    "\n",
    "## Extract Eigen Vectors\n",
    "COV=computeCov(spectra)\n",
    "eigval,eigvec=LA.eig(COV['Cov'])\n",
    "eigval=eigval/sum(eigval)\n",
    "\n",
    "## Check the Reconstruction\n",
    "plot(cumsum(eigval[:10]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Using First 5 Eigen Vectors\n",
    "V=eigvec[:,0:5]\n",
    "\n",
    "## Defining a function to return Projection\n",
    "def project(array):\n",
    "    return np.dot(array, V)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Interclick Interval"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Writing a UDF to get timestamp with higher precision to calculate Interclick Interval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.sql.window import Window\n",
    "from pyspark.sql.functions import udf, lag, collect_list, col\n",
    "from pyspark.sql.types import DoubleType\n",
    "\n",
    "def unix_ts(x):\n",
    "    from datetime import datetime\n",
    "    y = datetime.utcfromtimestamp(0)\n",
    "    return (x-y).total_seconds()\n",
    "\n",
    "unix_ts_udf = udf(unix_ts, DoubleType())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculating ICI for a given window of bout by sorting them by unix_ts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_with_unix_ts = df.withColumn(\"unix_ts\", unix_ts_udf(df.time))\n",
    "df_with_ici = df_with_unix_ts.withColumn(\"ici\",\n",
    "                                         df_with_unix_ts.unix_ts - lag(df_with_unix_ts.unix_ts, 1).over(\n",
    "                                             Window.partitionBy(\"bout_i\").orderBy(\"unix_ts\")\n",
    "                                         ))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ICI for Gervais"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD8CAYAAACLrvgBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFwRJREFUeJzt3X+s3fV93/Hna7ihpCnEwK3n2rDrFrcdoEYNnsPaqmLz\nBm6oaiYBddYWN7OwKlibTZUSk0ljSmTJaNNoUAuTFTwMiwKWmw5vCU09UJZNnSEmv8yPUu4CBLsm\ndrAHWypoTd7743xucnxz7fvlnnvv8fV9PqSj+z3v7/fzPZ+PbJ/X/Xw/33OcqkKSpC7+1rA7IEma\nPwwNSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzhYNuwMz7cILL6zR0dFhd0OS\n5pUnn3zy21U1MtVxZ1xojI6Osm/fvmF3Q5LmlSQvdTnOy1OSpM4MDUlSZ4aGJKkzQ0OS1JmhIUnq\nzNCQJHVmaEiSOjM0JEmdGRqSpM7OuE+E6/Q2uvmz39t+ceu1Q+yJpOlwpiFJ6syZhmZd/+xC0vzm\nTEOS1JmhIUnqzNCQJHVmaEiSOjM0JEmdGRqSpM4MDUlSZ4aGJKkzQ0OS1NmUoZFke5LDSZ6aZN/v\nJakkF/bVbksyluS5JNf01a9Isr/tuytJWv3sJA+1+uNJRvvabEjyfHtsGHSwkqTBdJlp3AesnVhM\nchFwNfDNvtqlwHrgstbm7iRntd33ADcDK9tj/JwbgWNVdQlwJ3BHO9f5wO3A+4DVwO1JFr+94UmS\nZtKUoVFVXwSOTrLrTuDDQPXV1gEPVtWbVfUCMAasTrIUOLeq9lZVAfcD1/W12dG2dwFr2izkGmBP\nVR2tqmPAHiYJL0nS3JnWmkaSdcDBqvrahF3LgJf7nh9otWVte2L9hDZVdRx4DbjgFOeSJA3J2/6W\n2yTvBD5K79LUaSHJJmATwMUXXzzk3kjSmWs6M42fBFYAX0vyIrAc+HKSvw0cBC7qO3Z5qx1s2xPr\n9LdJsgg4D3j1FOf6AVW1rapWVdWqkZGRaQxJktTF2w6NqtpfVT9WVaNVNUrvstF7q+oVYDewvt0R\ntYLegvcTVXUIeD3JlW294ibg4XbK3cD4nVHXA4+1dY/PA1cnWdwWwK9uNUnSkEx5eSrJp4GrgAuT\nHABur6p7Jzu2qp5OshN4BjgO3FpVb7Xdt9C7E+sc4JH2ALgXeCDJGL0F9/XtXEeTfBz4UjvuY1U1\n2YK8JGmOTBkaVfWBKfaPTni+BdgyyXH7gMsnqb8B3HCSc28Htk/VR0nS3PAT4ZKkzgwNSVJnhoYk\nqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnb/ur0aWZMrr5s9/bfnHrtUPsiaSu\nnGlIkjpzpqFZ0T+LkHTmcKYhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzqYMjSTbkxxO8lRf7d8m+fMk\nX0/yx0ne3bfvtiRjSZ5Lck1f/Yok+9u+u5Kk1c9O8lCrP55ktK/NhiTPt8eGmRq0JGl6usw07gPW\nTqjtAS6vqp8F/gK4DSDJpcB64LLW5u4kZ7U29wA3AyvbY/ycG4FjVXUJcCdwRzvX+cDtwPuA1cDt\nSRa//SFKkmbKlKFRVV8Ejk6o/WlVHW9P9wLL2/Y64MGqerOqXgDGgNVJlgLnVtXeqirgfuC6vjY7\n2vYuYE2bhVwD7Kmqo1V1jF5QTQwvSdIcmok1jX8GPNK2lwEv9+070GrL2vbE+gltWhC9BlxwinNJ\nkoZkoNBI8q+A48CnZqY70+7HpiT7kuw7cuTIMLsiSWe0aYdGkt8CfgX49XbJCeAgcFHfYctb7SDf\nv4TVXz+hTZJFwHnAq6c41w+oqm1VtaqqVo2MjEx3SJKkKUwrNJKsBT4M/GpV/VXfrt3A+nZH1Ap6\nC95PVNUh4PUkV7b1ipuAh/vajN8ZdT3wWAuhzwNXJ1ncFsCvbjVJ0pBM+YWFST4NXAVcmOQAvTua\nbgPOBva0O2f3VtVvV9XTSXYCz9C7bHVrVb3VTnULvTuxzqG3BjK+DnIv8ECSMXoL7usBqupoko8D\nX2rHfayqTliQlyTNrSlDo6o+MEn53lMcvwXYMkl9H3D5JPU3gBtOcq7twPap+ihJmht+IlyS1Jmh\nIUnqzNCQJHVmaEiSOjM0JEmdGRqSpM4MDUlSZ4aGJKkzQ0OS1JmhIUnqzNCQJHVmaEiSOjM0JEmd\nGRqSpM4MDUlSZ4aGJKkzQ0OS1JmhIUnqzNCQJHU2ZWgk2Z7kcJKn+mrnJ9mT5Pn2c3HfvtuSjCV5\nLsk1ffUrkuxv++5KklY/O8lDrf54ktG+NhvaazyfZMNMDVqSND1dZhr3AWsn1DYDj1bVSuDR9pwk\nlwLrgctam7uTnNXa3APcDKxsj/FzbgSOVdUlwJ3AHe1c5wO3A+8DVgO394eTJGnuTRkaVfVF4OiE\n8jpgR9veAVzXV3+wqt6sqheAMWB1kqXAuVW1t6oKuH9Cm/Fz7QLWtFnINcCeqjpaVceAPfxgeEmS\n5tB01zSWVNWhtv0KsKRtLwNe7jvuQKsta9sT6ye0qarjwGvABac41w9IsinJviT7jhw5Ms0hSZKm\nMvBCeJs51Az0ZZA+bKuqVVW1amRkZJhdkaQz2qJptvtWkqVVdahdejrc6geBi/qOW95qB9v2xHp/\nmwNJFgHnAa+2+lUT2nxhmv3VHBjd/Nlhd0HSLJvuTGM3MH430wbg4b76+nZH1Ap6C95PtEtZrye5\nsq1X3DShzfi5rgcea7OXzwNXJ1ncFsCvbjVJ0pBMOdNI8ml6v/FfmOQAvTuatgI7k2wEXgJuBKiq\np5PsBJ4BjgO3VtVb7VS30LsT6xzgkfYAuBd4IMkYvQX39e1cR5N8HPhSO+5jVTVxQV6SNIemDI2q\n+sBJdq05yfFbgC2T1PcBl09SfwO44STn2g5sn6qPkqS54SfCJUmdTXchXJpR/YvoL269dog9kXQq\nzjQkSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYk\nqTNDQ5LUmaEhSerM0JAkdWZoSJI6Gyg0kvzLJE8neSrJp5P8cJLzk+xJ8nz7ubjv+NuSjCV5Lsk1\nffUrkuxv++5KklY/O8lDrf54ktFB+itJGsy0QyPJMuB3gVVVdTlwFrAe2Aw8WlUrgUfbc5Jc2vZf\nBqwF7k5yVjvdPcDNwMr2WNvqG4FjVXUJcCdwx3T7K0ka3KCXpxYB5yRZBLwT+EtgHbCj7d8BXNe2\n1wEPVtWbVfUCMAasTrIUOLeq9lZVAfdPaDN+rl3AmvFZiCRp7k07NKrqIPDvgG8Ch4DXqupPgSVV\ndagd9gqwpG0vA17uO8WBVlvWtifWT2hTVceB14ALpttnSdJgBrk8tZjeTGAF8OPAjyT5jf5j2syh\nBupht75sSrIvyb4jR47M9stJ0oI1yOWpfwS8UFVHqupvgM8APw98q11yov083I4/CFzU1355qx1s\n2xPrJ7Rpl8DOA16d2JGq2lZVq6pq1cjIyABDkiSdyiCh8U3gyiTvbOsMa4Bngd3AhnbMBuDhtr0b\nWN/uiFpBb8H7iXYp6/UkV7bz3DShzfi5rgcea7MXSdIQLJpuw6p6PMku4MvAceArwDbgXcDOJBuB\nl4Ab2/FPJ9kJPNOOv7Wq3mqnuwW4DzgHeKQ9AO4FHkgyBhyld/eVJGlIph0aAFV1O3D7hPKb9GYd\nkx2/BdgySX0fcPkk9TeAGwbpoyRp5viJcElSZ4aGJKkzQ0OS1NlAaxrS6ObPDrsLkuaQMw1JUmeG\nhiSpM0NDktSZoSFJ6szQkCR1ZmhIkjozNCRJnRkakqTODA1JUmeGhiSpM0NDktSZoSFJ6swvLNRp\np/9LEF/ceu0QeyJpImcakqTOBgqNJO9OsivJnyd5NsnfT3J+kj1Jnm8/F/cdf1uSsSTPJbmmr35F\nkv1t311J0upnJ3mo1R9PMjpIfyVJgxl0pvEJ4E+q6meA9wDPApuBR6tqJfBoe06SS4H1wGXAWuDu\nJGe189wD3AysbI+1rb4ROFZVlwB3AncM2F9J0gCmHRpJzgN+CbgXoKr+uqr+D7AO2NEO2wFc17bX\nAQ9W1ZtV9QIwBqxOshQ4t6r2VlUB909oM36uXcCa8VmIJGnuDTLTWAEcAf5jkq8k+WSSHwGWVNWh\ndswrwJK2vQx4ua/9gVZb1rYn1k9oU1XHgdeACwbosyRpAIOExiLgvcA9VfVzwHdol6LGtZlDDfAa\nnSTZlGRfkn1HjhyZ7ZeTpAVrkNA4AByoqsfb8130QuRb7ZIT7efhtv8gcFFf++WtdrBtT6yf0CbJ\nIuA84NWJHamqbVW1qqpWjYyMDDAkSdKpTDs0quoV4OUkP91Ka4BngN3AhlbbADzctncD69sdUSvo\nLXg/0S5lvZ7kyrZecdOENuPnuh54rM1eJElDMOiH+34H+FSSdwDfAD5IL4h2JtkIvATcCFBVTyfZ\nSS9YjgO3VtVb7Ty3APcB5wCPtAf0FtkfSDIGHKV395UkaUgGCo2q+iqwapJda05y/BZgyyT1fcDl\nk9TfAG4YpI+SpJnjJ8IlSZ353VN62/q/G0rSwuJMQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKk\nzgwNSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqTP/EyZ1Mqz/\neKn/dV/ceu1Q+iDp+waeaSQ5K8lXkvzX9vz8JHuSPN9+Lu479rYkY0meS3JNX/2KJPvbvruSpNXP\nTvJQqz+eZHTQ/kqSpm8mLk99CHi27/lm4NGqWgk82p6T5FJgPXAZsBa4O8lZrc09wM3AyvZY2+ob\ngWNVdQlwJ3DHDPRXkjRNA4VGkuXAtcAn+8rrgB1tewdwXV/9wap6s6peAMaA1UmWAudW1d6qKuD+\nCW3Gz7ULWDM+C5Ekzb1BZxq/D3wY+G5fbUlVHWrbrwBL2vYy4OW+4w602rK2PbF+QpuqOg68Blww\nsRNJNiXZl2TfkSNHBhqQJOnkph0aSX4FOFxVT57smDZzqOm+RldVta2qVlXVqpGRkdl+OUlasAa5\ne+oXgF9N8n7gh4Fzk/wn4FtJllbVoXbp6XA7/iBwUV/75a12sG1PrPe3OZBkEXAe8OoAfZYkDWDa\nM42quq2qllfVKL0F7seq6jeA3cCGdtgG4OG2vRtY3+6IWkFvwfuJdinr9SRXtvWKmya0GT/X9e01\nZn3mIkma3Gx8TmMrsDPJRuAl4EaAqno6yU7gGeA4cGtVvdXa3ALcB5wDPNIeAPcCDyQZA47SCydJ\n0pDMSGhU1ReAL7TtV4E1JzluC7Blkvo+4PJJ6m8AN8xEH2eSHziTtFD5iXCd1LA+BS7p9OV3T0mS\nOjM0JEmdGRqSpM4MDUlSZ4aGJKkzQ0OS1JmhIUnqzM9p6AR+NkPSqRgamjf8JL40fF6ekiR15kxj\nlvhbsaQzkaExoDMhHFzHkNSVoTGDTvbmeyYEiySBaxqSpLfBmcYC5OUoSdNlaMwxL1VJms8MjQXC\n2YWkmTDt0EhyEXA/sAQoYFtVfSLJ+cBDwCjwInBjVR1rbW4DNgJvAb9bVZ9v9Sv4/v8R/jngQ1VV\nSc5ur3EF8Crwa1X14nT7fLo52Ru5MxBJp6tBZhrHgd+rqi8n+VHgySR7gN8CHq2qrUk2A5uBjyS5\nFFgPXAb8OPDfkvxUVb0F3APcDDxOLzTWAo/QC5hjVXVJkvXAHcCvDdDneWGmwsTZhaSZNu3QqKpD\nwKG2/X+TPAssA9YBV7XDdgBfAD7S6g9W1ZvAC0nGgNVJXgTOraq9AEnuB66jFxrrgH/TzrUL+IMk\nqaqabr/nM0NA0rDNyJpGklHg5+jNFJa0QAF4hd7lK+gFyt6+Zgda7W/a9sT6eJuXAarqeJLXgAuA\nb094/U3AJoCLL754Joak05w3FEjDMfDnNJK8C/gj4F9U1ev9+9qMYNZnBVW1rapWVdWqkZGR2X45\nSVqwBgqNJD9ELzA+VVWfaeVvJVna9i8FDrf6QeCivubLW+1g255YP6FNkkXAefQWxCVJQzDt0EgS\n4F7g2ar69327dgMb2vYG4OG++vokZydZAawEnmiXsl5PcmU7500T2oyf63rgsYW6niFJp4NB1jR+\nAfhNYH+Sr7baR4GtwM4kG4GXgBsBqurpJDuBZ+jdeXVru3MK4Ba+f8vtI+0BvVB6oC2aH6V395Uk\naUgGuXvqfwI5ye41J2mzBdgySX0fcPkk9TeAG6bbR0nSzPILCyVJnRkakqTO/O4pzXt+ZkOaO840\nJEmdGRqSpM4MDUlSZ4aGJKkzF8J1RnFRXJpdzjQkSZ0ZGpKkzgwNSVJnrmnojOX6hjTznGlIkjpz\npqEFwVmHNDMMDS04Bog0fYaGFrT+AIH5HyIGomaboSH1OZ3fdCcG3HSPP93GpfnF0JBOYlgB8nbD\nYabOb5ioi3kRGknWAp8AzgI+WVVbh9wlLTCDvJH3vxnPdiAM4lR9M1A07rQPjSRnAX8I/GPgAPCl\nJLur6pnh9kzq5nQOiq66jMFgWRhO+9AAVgNjVfUNgCQPAusAQ0M6jcxGOJ5sljZfZm8z6XQJ5fkQ\nGsuAl/ueHwDeN9edWCh/MaXTycn+3S3Ef4+ny2xvPoTGlJJsAja1p/8vyXMDnO5C4NuD92peWWhj\nXmjjBce8IOSOgcb8d7ocNB9C4yBwUd/z5a32PVW1Ddg2Ey+WZF9VrZqJc80XC23MC2284JgXirkY\n83z47qkvASuTrEjyDmA9sHvIfZKkBem0n2lU1fEk/xz4PL1bbrdX1dND7pYkLUinfWgAVNXngM/N\n0cvNyGWueWahjXmhjRcc80Ix62NOVc32a0iSzhDzYU1DknSaWJChkWRtkueSjCXZPMn+JLmr7f96\nkvcOo58zqcOYf72NdX+SP0vynmH0cyZNNea+4/5ekuNJrp/L/s2GLmNOclWSryZ5Osl/n+s+zrQO\nf7fPS/JfknytjfmDw+jnTEmyPcnhJE+dZP/svn9V1YJ60FtM/9/ATwDvAL4GXDrhmPcDjwABrgQe\nH3a/52DMPw8sbtu/vBDG3HfcY/TWzK4fdr/n4M/53fS+TeHi9vzHht3vORjzR4E72vYIcBR4x7D7\nPsCYfwl4L/DUSfbP6vvXQpxpfO9rSarqr4HxryXptw64v3r2Au9OsnSuOzqDphxzVf1ZVR1rT/fS\n+zzMfNblzxngd4A/Ag7PZedmSZcx/1PgM1X1TYCqmu/j7jLmAn40SYB30QuN43PbzZlTVV+kN4aT\nmdX3r4UYGpN9LcmyaRwzn7zd8Wyk95vKfDblmJMsA/4JcM8c9ms2dflz/ilgcZIvJHkyyU1z1rvZ\n0WXMfwD8XeAvgf3Ah6rqu3PTvaGY1feveXHLreZOkn9ALzR+cdh9mQO/D3ykqr7b+yV0QVgEXAGs\nAc4B/leSvVX1F8Pt1qy6Bvgq8A+BnwT2JPkfVfX6cLs1Py3E0Jjya0k6HjOfdBpPkp8FPgn8clW9\nOkd9my1dxrwKeLAFxoXA+5Mcr6r/PDddnHFdxnwAeLWqvgN8J8kXgfcA8zU0uoz5g8DW6l3wH0vy\nAvAzwBNz08U5N6vvXwvx8lSXryXZDdzU7kK4Enitqg7NdUdn0JRjTnIx8BngN8+Q3zqnHHNVraiq\n0aoaBXYBt8zjwIBuf7cfBn4xyaIk76T3jdHPznE/Z1KXMX+T3syKJEuAnwa+Mae9nFuz+v614GYa\ndZKvJUny223/f6B3J837gTHgr+j9pjJvdRzzvwYuAO5uv3kfr3n8ZW8dx3xG6TLmqno2yZ8AXwe+\nS+9/wpz01s35oOOf88eB+5Lsp3dH0Ueqat5++22STwNXARcmOQDcDvwQzM37l58IlyR1thAvT0mS\npsnQkCR1ZmhIkjozNCRJnRkakqTODA1JUmeGhiSpM0NDktTZ/weg5Xmqlkwz1wAAAABJRU5ErkJg\ngg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc729773810>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ici_ger=df_with_ici.filter(df_with_ici.species==\"Gervais\").select(\"ici\").rdd.flatMap(lambda x: x).collect()\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "plt.hist([x for x in ici_ger if x], bins=100, range=(0, 1))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ICI with Cuviers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFNlJREFUeJzt3X+s3fV93/HnqzihrA2EH67l2Hhmw20HaKThjrE1qtJa\nG05azUwC6v0oXmRhTbAskyYtpn+smyZL8E9oaQsTChmGNQWLpsNrQypqmmVTY1OzkhBDGXcBij3A\nxBBYO0Fr570/zsft8f365p5777nn3B/Ph3R0vudzvp/v+XwwPq/z+Xy+369TVUiS1O/7xt0ASdLi\nYzhIkjoMB0lSh+EgSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1LFq3A2Yq4suuqg2btw47mZI0pLy\n1FNPfbuqVs+035INh40bN3Lo0KFxN0OSlpQkLw+yn9NKkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwk\nSR2GgySpw3CQJHUYDpKkjiV7hbS01G3c9dtnLH/p9p8ecUukLsNBGqHpAkFabJxWkiR1GA6SpI6B\nwiHJB5M8kuSPkjyX5O8kuSDJ40leaM/n9+1/W5LJJM8nubav/Kokz7T37kqSVn52kodb+cEkG4fd\nUUnS4AYdOfwS8OWq+lHgSuA5YBewv6o2Afvba5JcBmwDLge2AHcnOasd5x7gZmBTe2xp5TuAt6rq\nUuBO4I559kuSNA8zhkOS84CfAO4DqKo/q6rvAFuBPW23PcB1bXsr8FBVvVdVLwKTwNVJ1gLnVtWB\nqirggSl1Th3rEWDzqVGFJGn0Bhk5XAK8AfynJH+Y5HNJfgBYU1Wvtn1eA9a07XXAK331j7SydW17\navlpdarqBPA2cOHsuyNJGoZBTmVdBXwE+FRVHUzyS7QppFOqqpLUQjSwX5KdwE6ADRs2LPTHSWPR\nf7qr1zxoXAYZORwBjlTVwfb6EXph8XqbKqI9H2vvHwUu7qu/vpUdbdtTy0+rk2QVcB5wfGpDqure\nqpqoqonVq2f8J1AlSXM048ihql5L8kqSH6mq54HNwLPtsR24vT0/2qrsA76Q5LPAh+gtPD9ZVSeT\nvJPkGuAgcBPwy311tgNfA64HnmjrEtKS54VvWooGvUL6U8CvJXk/8C3gk/RGHXuT7ABeBm4EqKrD\nSfbSC48TwK1VdbId5xbgfuAc4LH2gN5i94NJJoE36Z3tJEkak4HCoaqeBibO8NbmafbfDew+Q/kh\n4IozlL8L3DBIWyRJC88rpCVJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6/JfgpEXMW2loXAwHaci8\nIlrLgdNKkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgwH\nSVKH4SBJ6vCurNIQeCdWLTeGg7RE+G87aJScVpIkdQwUDkleSvJMkqeTHGplFyR5PMkL7fn8vv1v\nSzKZ5Pkk1/aVX9WOM5nkriRp5WcnebiVH0yycbjdlCTNxmxGDj9ZVR+uqon2ehewv6o2Afvba5Jc\nBmwDLge2AHcnOavVuQe4GdjUHlta+Q7graq6FLgTuGPuXZIkzdd8ppW2Anva9h7gur7yh6rqvap6\nEZgErk6yFji3qg5UVQEPTKlz6liPAJtPjSokSaM3aDgU8LtJnkqys5WtqapX2/ZrwJq2vQ54pa/u\nkVa2rm1PLT+tTlWdAN4GLpxFPyRJQzTo2UofraqjSX4IeDzJH/W/WVWVpIbfvNO1YNoJsGHDhoX+\nOElasQYaOVTV0fZ8DPhN4Grg9TZVRHs+1nY/ClzcV319KzvatqeWn1YnySrgPOD4Gdpxb1VNVNXE\n6tWrB2m6JGkOZgyHJD+Q5AOntoG/D3wT2Adsb7ttBx5t2/uAbe0MpEvoLTw/2aag3klyTVtPuGlK\nnVPHuh54oq1LSJLGYJBppTXAb7b14VXAF6rqy0n+ANibZAfwMnAjQFUdTrIXeBY4AdxaVSfbsW4B\n7gfOAR5rD4D7gAeTTAJv0jvbSZI0JjOGQ1V9C7jyDOXHgc3T1NkN7D5D+SHgijOUvwvcMEB7JUkj\n4BXSkqQOw0GS1GE4SJI6DAdJUoe37JbmaJz/hoO379ZCc+QgSeowHCRJHYaDJKnDcJAkdRgOkqQO\nw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHV44z1pFsZ5sz1plBw5SJI6HDlIS5y379ZC\ncOQgSeowHCRJHYaDJKnDcJAkdQwcDknOSvKHSX6rvb4gyeNJXmjP5/fte1uSySTPJ7m2r/yqJM+0\n9+5KklZ+dpKHW/nBJBuH10VJ0mzNZuTwaeC5vte7gP1VtQnY316T5DJgG3A5sAW4O8lZrc49wM3A\npvbY0sp3AG9V1aXAncAdc+qNJGkoBgqHJOuBnwY+11e8FdjTtvcA1/WVP1RV71XVi8AkcHWStcC5\nVXWgqgp4YEqdU8d6BNh8alQhSRq9QUcOvwj8G+C7fWVrqurVtv0asKZtrwNe6dvvSCtb17anlp9W\np6pOAG8DFw7YNknSkM0YDkl+BjhWVU9Nt08bCdQwGzZNW3YmOZTk0BtvvLHQHydJK9YgV0j/OPAP\nknwC+H7g3CT/GXg9ydqqerVNGR1r+x8FLu6rv76VHW3bU8v76xxJsgo4Dzg+tSFVdS9wL8DExMSC\nh5EE3k9JK9OMI4equq2q1lfVRnoLzU9U1T8F9gHb227bgUfb9j5gWzsD6RJ6C89Ptimod5Jc09YT\nbppS59Sxrm+f4Ze/JI3JfO6tdDuwN8kO4GXgRoCqOpxkL/AscAK4tapOtjq3APcD5wCPtQfAfcCD\nSSaBN+mFkKRZ8j5LGpZZhUNVfQX4Sts+DmyeZr/dwO4zlB8CrjhD+bvADbNpiyRp4XiFtCSpw3CQ\nJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6pjPRXDSsuUtM7TSOXKQJHUYDpKkDqeVpGXK+yxpPhw5\nSJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOrxCWmqW8832vFpas+XIQZLU\nYThIkjoMB0lSh+EgSeqYMRySfH+SJ5N8PcnhJP++lV+Q5PEkL7Tn8/vq3JZkMsnzSa7tK78qyTPt\nvbuSpJWfneThVn4wycbhd1WSNKhBRg7vAT9VVVcCHwa2JLkG2AXsr6pNwP72miSXAduAy4EtwN1J\nzmrHuge4GdjUHlta+Q7graq6FLgTuGMIfZMkzdGM4VA9f9Jevq89CtgK7Gnle4Dr2vZW4KGqeq+q\nXgQmgauTrAXOraoDVVXAA1PqnDrWI8DmU6MKSdLoDXSdQ/vl/xRwKfCrVXUwyZqqerXt8hqwpm2v\nAw70VT/Syv68bU8tP1XnFYCqOpHkbeBC4NtT2rET2AmwYcOGQZouaQqvedAgBlqQrqqTVfVhYD29\nUcAVU94veqOJBVVV91bVRFVNrF69eqE/TpJWrFmdrVRV3wF+j95awettqoj2fKztdhS4uK/a+lZ2\ntG1PLT+tTpJVwHnA8dm0TZI0PDNOKyVZDfx5VX0nyTnA36O3YLwP2A7c3p4fbVX2AV9I8lngQ/QW\nnp+sqpNJ3mmL2QeBm4Bf7quzHfgacD3wRBuNSAtqOd8yQ5qPQdYc1gJ72rrD9wF7q+q3knwN2Jtk\nB/AycCNAVR1Oshd4FjgB3FpVJ9uxbgHuB84BHmsPgPuAB5NMAm/SO9tJkjQmM4ZDVX0D+LEzlB8H\nNk9TZzew+wzlh4ArzlD+LnDDAO2VJI2AV0hLkjq8Zbe0gnlaq6bjyEGS1GE4SJI6DAdJUofhIEnq\ncEFaK44XvkkzMxwkAZ65pNM5rSRJ6jAcJEkdhoMkqcNwkCR1GA6SpA7PVpLU4ZlLcuQgSeowHCRJ\nHYaDJKnDNQetCN4yQ5odRw6SpA5HDpK+J89cWpkMh2n4F0LSSua0kiSpw3CQJHU4rSRpYE63rhwz\njhySXJzk95I8m+Rwkk+38guSPJ7khfZ8fl+d25JMJnk+ybV95Vcleaa9d1eStPKzkzzcyg8m2Tj8\nrkqSBjXIyOEE8K+r6n8m+QDwVJLHgX8G7K+q25PsAnYBn0lyGbANuBz4EPC7SX64qk4C9wA3AweB\nLwFbgMeAHcBbVXVpkm3AHcDPDrOjWnm8tkGauxnDoapeBV5t2/83yXPAOmAr8LG22x7gK8BnWvlD\nVfUe8GKSSeDqJC8B51bVAYAkDwDX0QuHrcC/a8d6BPiVJKmqmn8XJS0Ep5iWt1ktSLfpnh+j98t/\nTQsOgNeANW17HfBKX7UjrWxd255aflqdqjoBvA1ceIbP35nkUJJDb7zxxmyaLkmahYHDIckPAr8B\n/Kuqeqf/vfYLf8F/5VfVvVU1UVUTq1evXuiPk6QVa6BwSPI+esHwa1X1xVb8epK17f21wLFWfhS4\nuK/6+lZ2tG1PLT+tTpJVwHnA8dl2RpI0HIOcrRTgPuC5qvps31v7gO1tezvwaF/5tnYG0iXAJuDJ\nNgX1TpJr2jFvmlLn1LGuB55wvUGSxmeQs5V+HPg54JkkT7eynwduB/Ym2QG8DNwIUFWHk+wFnqV3\nptOt7UwlgFuA+4Fz6C1EP9bK7wMebIvXb9I720nSEuHi9PIzyNlK/wPING9vnqbObmD3GcoPAVec\nofxd4IaZ2iLNxNNXpeHw9hmSpA5vnyFpqJxiWh4cOUiSOgwHSVKH00qSFoxTTEuXIwdJUocjBy15\nnr4qDZ/hMICpXz79w2OHzdJgvtffIy0+TitJkjocOczBdNMYjiIkLReGg6Sx8MfU4ua0kiSpw5GD\npLFzFLH4GA5akjx9VVpYhsMC8ZeQpKXMcJC0qPjDanEwHEbA/9mHw6kkaXQMhxEzKCQtBYbDGBkU\n0vfm35HxMRwWiemmTPwLIWkcDActaq4z6BRHEaNlOCxyjigkjYPhsETNNjQG+QW+WALH0YI0fqmq\ncbdhTiYmJurQoUMLdny/oP7SXEJjtlMA/vfWXC2WHzVLRZKnqmpipv1mHDkk+TzwM8CxqrqilV0A\nPAxsBF4Cbqyqt9p7twE7gJPAv6yq32nlVwH3A+cAXwI+XVWV5GzgAeAq4Djws1X10iz6qgU23y9u\nv/ilpWeQu7LeD2yZUrYL2F9Vm4D97TVJLgO2AZe3OncnOavVuQe4GdjUHqeOuQN4q6ouBe4E7phr\nZyStPBt3/fZfPDQ8M4ZDVX0VeHNK8VZgT9veA1zXV/5QVb1XVS8Ck8DVSdYC51bVgerNYz0wpc6p\nYz0CbE6SuXZIkjR/c/33HNZU1att+zVgTdteB7zSt9+RVraubU8tP61OVZ0A3gYunGO7JElDMO+z\nldq6wUhWtZPsBHYCbNiwYRQfKWkJ8VqI4ZnryOH1NlVEez7Wyo8CF/ftt76VHW3bU8tPq5NkFXAe\nvYXpjqq6t6omqmpi9erVc2y6JGkmcw2HfcD2tr0deLSvfFuSs5NcQm/h+ck2BfVOkmvaesJNU+qc\nOtb1wBO1VM+vlbRouFA9P4OcyvrrwMeAi5IcAX4BuB3Ym2QH8DJwI0BVHU6yF3gWOAHcWlUn26Fu\n4S9PZX2sPQDuAx5MMklv4XvbUHomSZozL4Kbhr82pOVppa9FDHoR3FynlSRJy5j3VpK0onhG02Ac\nOUiSOhw5SFqxHEVMz5GDJKnDcJAkdTitJEk4xTSV4SBJUxgUTitJks7AkYMkfQ8rdRRhOEjSgFZS\nUDitJEnqcOQgSXOw3EcRhoMkzdNyDAqnlSRJHY4cJGmIlssownCQpAWylIPCcJCkEZjuX5dcrKFh\nOEjSGE0NjcUSFoaDJC0ii2WEYThI0hIw6vULT2WVJHUYDpKkDsNBktSxaMIhyZYkzyeZTLJr3O2R\npJVsUYRDkrOAXwU+DlwG/KMkl423VZK0ci2Ws5WuBiar6lsASR4CtgLPjrIR051CJkkrzaIYOQDr\ngFf6Xh9pZZKkMVgsI4eBJNkJ7Gwv/yTJ83M81EXAt4fTqiXDPq8M9nkFyB3z6vNfHWSnxRIOR4GL\n+16vb2Wnqap7gXvn+2FJDlXVxHyPs5TY55XBPq8Mo+jzYplW+gNgU5JLkrwf2AbsG3ObJGnFWhQj\nh6o6keRfAL8DnAV8vqoOj7lZkrRiLYpwAKiqLwFfGtHHzXtqagmyzyuDfV4ZFrzPqaqF/gxJ0hKz\nWNYcJEmLyLIOh5luyZGeu9r730jykXG0c5gG6PM/aX19JsnvJ7lyHO0cpkFvvZLkbyU5keT6UbZv\nIQzS5yQfS/J0ksNJ/tuo2zhMA/x/fV6S/5rk662/nxxHO4cpyeeTHEvyzWneX9jvr6palg96C9v/\nG/hrwPuBrwOXTdnnE8BjQIBrgIPjbvcI+vx3gfPb9sdXQp/79nuC3rrW9eNu9wj+nD9I7w4DG9rr\nHxp3uxe4vz8P3NG2VwNvAu8fd9vn2e+fAD4CfHOa9xf0+2s5jxz+4pYcVfVnwKlbcvTbCjxQPQeA\nDyZZO+qGDtGMfa6q36+qt9rLA/SuKVnKBvlzBvgU8BvAsVE2boEM0ud/DHyxqv4YoKqWcr8H6W8B\nH0gS4AfphcOJ0TZzuKrqq/T6MZ0F/f5azuEwyC05ltttO2bbnx30fnksZTP2Ock64B8C94ywXQtp\nkD/nHwbOT/KVJE8luWlkrRu+Qfr7K8DfAP4P8Azw6ar67miaNzYL+v21aE5l1Wgl+Ul64fDRcbdl\nBH4R+ExVfbf3w3JFWAVcBWwGzgG+luRAVf2v8TZrwVwLPA38FPDXgceT/Peqeme8zVq6lnM4DHJL\njoFu27GEDNSfJH8T+Bzw8ao6PqK2LZRB+jwBPNSC4SLgE0lOVNV/GU0Th26QPh8BjlfVnwJ/muSr\nwJXAUgyHQfr7SeD26k3GTyZ5EfhR4MnRNHEsFvT7azlPKw1yS459wE1t1f8a4O2qenXUDR2iGfuc\nZAPwReDnlsmvyBn7XFWXVNXGqtoIPALcsoSDAQb7f/tR4KNJViX5K8DfBp4bcTuHZZD+/jG9URJJ\n1gA/AnxrpK0cvQX9/lq2I4ea5pYcSf55e/8/0jtz5RPAJPD/6P36WLIG7PO/BS4E7m6/pE/UEr5p\n2YB9XlYG6XNVPZfky8A3gO8Cn6uqM54SudgN+Gf8H4D7kzxD7+ydz1TVkr5Ta5JfBz4GXJTkCPAL\nwPtgNN9fXiEtSepYztNKkqQ5MhwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgwHSVLH/wfUJcUS\nf4rzSgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc72e81a790>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ici_cur=df_with_ici.filter(df_with_ici.species==\"Cuviers\").select(\"ici\").rdd.flatMap(lambda x: x).collect()\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "plt.hist([x for x in ici_cur if x], bins=100, range=(0, 1))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ICI Mode per Bout"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculating ICI per bout by taking mode of all the ICI's in the given bout."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "821"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Trim ICI to Two Decimal Places\n",
    "lookup_grouped = df_with_ici.withColumn('ici_temp', df_with_ici.ici * 100).drop('ici')\n",
    "lookup_grouped = lookup_grouped.withColumn('ici_100', lookup_grouped.ici_temp.cast('bigint')).drop('ici_temp')\n",
    "\n",
    "## Calculate Mode\n",
    "lookup_grouped = lookup_grouped.groupBy(['bout_i', 'ici_100']).count()\n",
    "lookup_grouped_max = lookup_grouped.groupBy('bout_i').agg({'count':'max'}).withColumnRenamed(\"max(count)\", \"max_ct\")\n",
    "lookup_mode = lookup_grouped_max.alias('a').join(lookup_grouped.alias('b'), (col('a.bout_i') == col('b.bout_i')) & (col('a.max_ct') == col('b.count')) , 'left')\n",
    "lookup_mode = lookup_mode.groupBy('a.bout_i').agg({'ici_100':'avg'}).withColumnRenamed(\"avg(ici_100)\", \"mode_ici\")\n",
    "lookup_mode.cache().count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ICI Median"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[bout_i: bigint, median_ici: float]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def median(values_list):\n",
    "    med = np.median(values_list)\n",
    "    return float(med)\n",
    "\n",
    "udf_median = udf(median, FloatType())\n",
    "\n",
    "lookup_grouped_median = df_with_ici.groupby(['bout_i'])\n",
    "lookup_median = lookup_grouped_median.agg(udf_median(collect_list('ici')).alias('median_ici'))\n",
    "lookup_median.cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4144531"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final = df_with_ici.alias('a').join(lookup_mode.alias('b'), 'bout_i')\n",
    "\n",
    "df_final.cache().count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def features(row):\n",
    "    array = trimmed_spectra(row)\n",
    "    rmse = np.std(array)\n",
    "    projections = project(array)\n",
    "    ici = row.mode_ici\n",
    "    peak2peak = row.peak2peak\n",
    "    return np.hstack((projections, [rmse, ici, peak2peak]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating Training and Test Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filtering Samples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Only taking the samples which were neither Misclassified (MS = 0) nor Falsely Detected (FD = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Cuviers=df_final.filter(df.species==u'Cuviers')\n",
    "Gervais=df_final.filter(df.species==u'Gervais')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cuv_bout=Cuviers.groupBy(\"bout_i\").count()\n",
    "ger_bout=Gervais.groupBy(\"bout_i\").count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mode_bout = cuv_bout.alias('c').join(ger_bout.alias('d'), (col('c.bout_i') == col('d.bout_i')),'full')\n",
    "mode_bout_list=mode_bout.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Labelling as Cuviers: 1 and Gervais: 0 for each bout. Labeled if ratio is greater than 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_dict={}\n",
    "for i in mode_bout_list:\n",
    "    bout=i[0]\n",
    "    cuv_count = i[1] if i[1] else 0\n",
    "    ger_count = i[3] if i[3] else 0\n",
    "    ratio_cuv= 1.0 * cuv_count / (cuv_count + ger_count)\n",
    "    label_dict[bout]=(1 if ratio_cuv>=0.65 else (0 if ratio_cuv <=0.35 else 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEEFJREFUeJzt3XGM33V9x/Hna4BM0c2ynrWWskJWsxWj6M7GqFkwbAMh\nprIZVrZos5HULcxooovgH+qyNGHJ1G3JcKlKrImDNVGkUXRD1BEnggepSAvMTkDaFHqKG+IWZst7\nf9wX+Vl79/v97u53v7uPz0dyue/v8/187/u6b3qv+/b7+/5+l6pCktSuXxh3AEnSaFn0ktQ4i16S\nGmfRS1LjLHpJapxFL0mNs+glqXEWvSQ1rm/RJ/nFJHck+WaSfUn+shs/PcnNSb7dfV7Vs81VSQ4k\nuT/JBaP8BiRJc0u/V8YmCXBaVT2R5BTgq8Dbgd8DHquqq5NcCayqqncn2QRcB2wGXgR8EXhxVR2b\nbR+rV6+uDRs2LMo3JEk/L+68887vVdVEv3kn95tQM78JnugentJ9FLAFOK8b3wV8BXh3N359VT0J\nPJDkADOlf9ts+9iwYQNTU1P9okiSeiR5aJB5A12jT3JSkr3AEeDmqrodWFNVh7spjwBruuV1wMM9\nmx/sxiRJYzBQ0VfVsao6FzgD2JzkJcetL2bO8geWZHuSqSRT09PTw2wqSRrCUHfdVNV/AV8GLgQe\nTbIWoPt8pJt2CFjfs9kZ3djxX2tnVU1W1eTERN9LTJKkeRrkrpuJJM/vlp8N/A5wH7AH2NZN2wbc\n2C3vAbYmOTXJWcBG4I7FDi5JGkzfJ2OBtcCuJCcx84thd1V9NsltwO4klwMPAZcCVNW+JLuB/cBR\n4Iq57riRJI1W39srl8Lk5GR5140kDSfJnVU12W+er4yVpMZZ9JLUOItekho3yJOxkn7ObLjyc2PZ\n74NXXzyW/bbOM3pJapxFL0mNs+glqXEWvSQ1zqKXpMZZ9JLUOItekhpn0UtS4yx6SWqcRS9JjbPo\nJalxFr0kNc6il6TGWfSS1DiLXpIaZ9FLUuMseklqnEUvSY2z6CWpcRa9JDXOopekxln0ktQ4i16S\nGte36JOsT/LlJPuT7Evy9m78/UkOJdnbfVzUs81VSQ4kuT/JBaP8BiRJczt5gDlHgXdW1V1Jngfc\nmeTmbt2Hqupveicn2QRsBc4BXgR8McmLq+rYYgaXJA2m7xl9VR2uqru65R8C9wLr5thkC3B9VT1Z\nVQ8AB4DNixFWkjS8oa7RJ9kAvBy4vRt6W5K7k1ybZFU3tg54uGezg8z9i0GSNEIDF32S5wKfAt5R\nVY8DHwbOBs4FDgMfGGbHSbYnmUoyNT09PcymkqQhDFT0SU5hpuQ/WVWfBqiqR6vqWFU9BXyEZy7P\nHALW92x+Rjf2U6pqZ1VNVtXkxMTEQr4HSdIcBrnrJsDHgHur6oM942t7pl0C3NMt7wG2Jjk1yVnA\nRuCOxYssSRrGIHfdvAZ4M/CtJHu7sfcAlyU5FyjgQeCtAFW1L8luYD8zd+xc4R03kjQ+fYu+qr4K\n5ASrbppjmx3AjgXkkiQtEl8ZK0mNs+glqXEWvSQ1zqKXpMZZ9JLUOItekhpn0UtS4yx6SWqcRS9J\njbPoJalxFr0kNc6il6TGWfSS1DiLXpIaZ9FLUuMseklqnEUvSY2z6CWpcRa9JDXOopekxln0ktQ4\ni16SGmfRS1LjLHpJapxFL0mNs+glqXEWvSQ1rm/RJ1mf5MtJ9ifZl+Tt3fjpSW5O8u3u86qeba5K\nciDJ/UkuGOU3IEma2yBn9EeBd1bVJuBVwBVJNgFXArdU1Ubglu4x3bqtwDnAhcA1SU4aRXhJUn99\ni76qDlfVXd3yD4F7gXXAFmBXN20X8MZueQtwfVU9WVUPAAeAzYsdXJI0mKGu0SfZALwcuB1YU1WH\nu1WPAGu65XXAwz2bHezGjv9a25NMJZmanp4eMrYkaVADF32S5wKfAt5RVY/3rquqAmqYHVfVzqqa\nrKrJiYmJYTaVJA1hoKJPcgozJf/Jqvp0N/xokrXd+rXAkW78ELC+Z/MzujFJ0hgMctdNgI8B91bV\nB3tW7QG2dcvbgBt7xrcmOTXJWcBG4I7FiyxJGsbJA8x5DfBm4FtJ9nZj7wGuBnYnuRx4CLgUoKr2\nJdkN7Gfmjp0rqurYoieXJA2kb9FX1VeBzLL6/Fm22QHsWEAuSdIi8ZWxktQ4i16SGmfRS1LjLHpJ\napxFL0mNs+glqXEWvSQ1zqKXpMZZ9JLUOItekhpn0UtS4yx6SWqcRS9JjbPoJalxFr0kNc6il6TG\nWfSS1DiLXpIaZ9FLUuMseklqnEUvSY2z6CWpcRa9JDXOopekxln0ktQ4i16SGmfRS1Lj+hZ9kmuT\nHElyT8/Y+5McSrK3+7ioZ91VSQ4kuT/JBaMKLkkazCBn9B8HLjzB+Ieq6tzu4yaAJJuArcA53TbX\nJDlpscJKkobXt+ir6lbgsQG/3hbg+qp6sqoeAA4AmxeQT5K0QAu5Rv+2JHd3l3ZWdWPrgId75hzs\nxn5Gku1JppJMTU9PLyCGJGku8y36DwNnA+cCh4EPDPsFqmpnVU1W1eTExMQ8Y0iS+plX0VfVo1V1\nrKqeAj7CM5dnDgHre6ae0Y1JksZkXkWfZG3Pw0uAp+/I2QNsTXJqkrOAjcAdC4soSVqIk/tNSHId\ncB6wOslB4H3AeUnOBQp4EHgrQFXtS7Ib2A8cBa6oqmOjiS5JGkTfoq+qy04w/LE55u8AdiwklCRp\n8fjKWElqnEUvSY2z6CWpcRa9JDXOopekxln0ktQ4i16SGmfRS1LjLHpJapxFL0mNs+glqXEWvSQ1\nzqKXpMZZ9JLUOItekhpn0UtS4yx6SWqcRS9JjbPoJalxFr0kNc6il6TGWfSS1DiLXpIaZ9FLUuMs\neklqnEUvSY3rW/RJrk1yJMk9PWOnJ7k5ybe7z6t61l2V5ECS+5NcMKrgkqTBDHJG/3HgwuPGrgRu\nqaqNwC3dY5JsArYC53TbXJPkpEVLK0kaWt+ir6pbgceOG94C7OqWdwFv7Bm/vqqerKoHgAPA5kXK\nKkmah/leo19TVYe75UeANd3yOuDhnnkHuzFJ0pgs+MnYqiqght0uyfYkU0mmpqenFxpDkjSL+Rb9\no0nWAnSfj3Tjh4D1PfPO6MZ+RlXtrKrJqpqcmJiYZwxJUj/zLfo9wLZueRtwY8/41iSnJjkL2Ajc\nsbCIkqSFOLnfhCTXAecBq5McBN4HXA3sTnI58BBwKUBV7UuyG9gPHAWuqKpjI8ouSRpA36Kvqstm\nWXX+LPN3ADsWEkqStHh8ZawkNc6il6TGWfSS1DiLXpIaZ9FLUuMseklqnEUvSY2z6CWpcRa9JDXO\nopekxln0ktQ4i16SGmfRS1LjLHpJapxFL0mNs+glqXEWvSQ1zqKXpMZZ9JLUuL5/M1bSeGy48nPj\njqBGeEYvSY2z6CWpcRa9JDXOopekxln0ktQ4i16SGmfRS1LjFnQffZIHgR8Cx4CjVTWZ5HTgn4EN\nwIPApVX1g4XFlCTN12Kc0b+uqs6tqsnu8ZXALVW1EbileyxJGpNRXLrZAuzqlncBbxzBPiRJA1po\n0RfwxSR3Jtneja2pqsPd8iPAmgXuQ5K0AAt9r5vXVtWhJC8Abk5yX+/KqqokdaINu18M2wHOPPPM\nBcaQJM1mQWf0VXWo+3wEuAHYDDyaZC1A9/nILNvurKrJqpqcmJhYSAxJ0hzmXfRJTkvyvKeXgd8F\n7gH2ANu6aduAGxcaUpI0fwu5dLMGuCHJ01/nn6rqC0m+AexOcjnwEHDpwmNKkuZr3kVfVd8BXnaC\n8e8D5y8klCRp8fiHRyQtG+P6YysPXn3xWPa7VHwLBElqnGf0GopnXNLK4xm9JDXOopekxln0ktQ4\ni16SGmfRS1LjmrjrxjtBJGl2ntFLUuMseklqnEUvSY2z6CWpcRa9JDXOopekxln0ktQ4i16SGmfR\nS1LjLHpJapxFL0mNs+glqXEWvSQ1zqKXpMZZ9JLUOItekhrXxB8eUfvG9cdlpBZ4Ri9JjRvZGX2S\nC4G/A04CPlpVV49qXz9vPLuVNIyRnNEnOQn4B+D1wCbgsiSbRrEvSdLcRnVGvxk4UFXfAUhyPbAF\n2D+i/Y2FZ9ZSG8b5s/zg1RePfB+juka/Dni45/HBbkyStMTGdtdNku3A9u7hE0nuB1YD3xtXpnla\niZlhZeZeiZlhZeY28xLJXy8o968OMmlURX8IWN/z+Ixu7Ceqaiews3csyVRVTY4o00isxMywMnOv\nxMywMnObeeksRe5RXbr5BrAxyVlJngVsBfaMaF+SpDmM5Iy+qo4m+XPgX5i5vfLaqto3in1JkuY2\nsmv0VXUTcNOQm+3sP2XZWYmZYWXmXomZYWXmNvPSGXnuVNWo9yFJGiPfAkGSGjeWok9yYZL7kxxI\ncuUJ1ifJ33fr707yinHkPC5Tv8y/nuS2JE8medc4Mh5vgMx/1B3fbyX5WpKXjSPn8QbIvaXLvTfJ\nVJLXjiPncZnmzNwz75VJjiZ501Lmm80Ax/q8JP/dHeu9Sd47jpzHZep7rLvce5PsS/JvS53xRAY4\n1n/Rc5zvSXIsyemLsvOqWtIPZp6c/U/gbOBZwDeBTcfNuQj4PBDgVcDtS51zHplfALwS2AG8a5x5\nh8j8amBVt/z6cR/nIXI/l2cuO74UuG+5Z+6Z9yVmnrt60wo51ucBnx131iEzP5+ZV+Gf2T1+wUrI\nfdz8NwBfWqz9j+OM/idvj1BV/wc8/fYIvbYAn6gZXween2TtUgft0TdzVR2pqm8APx5HwBMYJPPX\nquoH3cOvM/N6h3EbJPcT1f00AKcB436iaZB/0wBvAz4FHFnKcHMYNPdyMkjmPwQ+XVXfhZmfzSXO\neCLDHuvLgOsWa+fjKPpB3h5hub2FwnLLM4hhM1/OzP+ixm2g3EkuSXIf8DngT5Yo22z6Zk6yDrgE\n+PAS5upn0H8jr+4ulX0+yTlLE21Wg2R+MbAqyVeS3JnkLUuWbnYD/zwmeQ5wITMnBYvCPzwikryO\nmaIf+7XuQVXVDcANSX4L+Cvgt8ccqZ+/Bd5dVU8lGXeWYdzFzCWQJ5JcBHwG2DjmTP2cDPwmcD7w\nbOC2JF+vqv8Yb6yBvQH496p6bLG+4DiKvu/bIww4ZykttzyDGChzkpcCHwVeX1XfX6JscxnqWFfV\nrUnOTrK6qsb1PieDZJ4Eru9KfjVwUZKjVfWZpYl4QoO8VcnjPcs3JblmBRzrg8D3q+pHwI+S3Aq8\nDBhn0Q/z73ori3jZBhjLk7EnA98BzuKZJyXOOW7Oxfz0k7F3jOtJlEEz98x9P8vjydhBjvOZwAHg\n1ePOO2TuX+OZJ2NfwcwPTJZz5uPmf5zl8WTsIMf6hT3HejPw3eV+rIHfAG7p5j4HuAd4yXI/1t28\nXwYeA05bzP0v+Rl9zfL2CEn+tFv/j8zclXARMyX0P8AfL3XOXoNkTvJCYAr4JeCpJO9g5ln1x2f9\nwmPODLwX+BXgmu5M82iN+U2hBsz9+8BbkvwY+F/gD6r7KVnGmZedAXO/CfizJEeZOdZbl/uxrqp7\nk3wBuBt4ipm/cHfPuDJ3uQb9N3IJ8K8187+RReMrYyWpcb4yVpIaZ9FLUuMseklqnEUvSY2z6CWp\ncRa9JDXOopekxln0ktS4/wfF/P7FeByPdAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc7282acdd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADwVJREFUeJzt3X+s3XV9x/Hny4LdIi6D9Vq7tu6WrIspi9bl2hn1DxzZ\nqPpHISOsZHFNRlK3MaeJJhb/EJOlCUumblkGSxVi/3DWRmA0ijPYmRCjAy+kU1pgdlJCm0KvPzYg\nWTpb3vvjftUjtvece885PfTT5yM5Od/v5/v5nu/7ky+8+s3nfM/3pqqQJLXrFZMuQJI0Xga9JDXO\noJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEXTboAgBUrVtT09PSky5Ck88rDDz/8/aqa\n6tevb9An+SXgAWB51/8LVXVLksuAzwPTwBHg+qr6UbfPzcCNwGngr6rqKwsdY3p6mtnZ2X6lSJJ6\nJHlqkH6DTN2cBH6vqt4IbAQ2J3kLsAPYX1Xrgf3dOkk2AFuBK4DNwG1Jli1+CJKkUegb9DXvhW71\n4u5VwBZgd9e+G7imW94C7Kmqk1X1JHAY2DTSqiVJAxvoy9gky5IcAE4A91fVg8DKqjredXkGWNkt\nrwae7tn9aNcmSZqAgYK+qk5X1UZgDbApyW+/ZHsxf5U/sCTbk8wmmZ2bm1vMrpKkRVjU7ZVV9d/A\n15ife382ySqA7v1E1+0YsLZntzVd20s/a1dVzVTVzNRU3y+NJUlL1Dfok0wl+dVu+ZeB3wceB/YB\n27pu24B7u+V9wNYky5OsA9YDD426cEnSYAa5j34VsLu7c+YVwN6q+mKSbwJ7k9wIPAVcD1BVB5Ps\nBQ4Bp4Cbqur0eMqXJPWTl8OfEpyZmSnvo5ekxUnycFXN9OvnIxAkqXEvi0cgSP1M7/jSxI595NZ3\nT+zY0ih4RS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXO\noJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6\nSWqcQS9Jjesb9EnWJvlakkNJDiZ5f9f+sSTHkhzoXu/q2efmJIeTPJHk6nEOQJK0sIsG6HMK+GBV\nPZLk1cDDSe7vtn2yqv62t3OSDcBW4Arg14GvJvmtqjo9ysIlSYPpe0VfVcer6pFu+XngMWD1Arts\nAfZU1cmqehI4DGwaRbGSpMVb1Bx9kmngTcCDXdP7knw7yZ1JLu3aVgNP9+x2lIX/YZAkjdHAQZ/k\nEuAu4ANV9RxwO3A5sBE4Dnx8MQdOsj3JbJLZubm5xewqSVqEgYI+ycXMh/xnq+pugKp6tqpOV9WL\nwKf42fTMMWBtz+5rurafU1W7qmqmqmampqaGGYMkaQGD3HUT4A7gsar6RE/7qp5u1wKPdsv7gK1J\nlidZB6wHHhpdyZKkxRjkrpu3Ae8BvpPkQNf2EeCGJBuBAo4A7wWoqoNJ9gKHmL9j5ybvuJGkyekb\n9FX1dSBn2HTfAvvsBHYOUZckaUT8ZawkNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWp\ncQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn\n0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TG9Q36JGuTfC3JoSQHk7y/a78syf1Jvtu9X9qzz81J\nDid5IsnV4xyAJGlhg1zRnwI+WFUbgLcANyXZAOwA9lfVemB/t063bStwBbAZuC3JsnEUL0nqr2/Q\nV9XxqnqkW34eeAxYDWwBdnfddgPXdMtbgD1VdbKqngQOA5tGXbgkaTCLmqNPMg28CXgQWFlVx7tN\nzwAru+XVwNM9ux3t2iRJEzBw0Ce5BLgL+EBVPde7raoKqMUcOMn2JLNJZufm5hazqyRpEQYK+iQX\nMx/yn62qu7vmZ5Os6ravAk507ceAtT27r+nafk5V7aqqmaqamZqaWmr9kqQ+BrnrJsAdwGNV9Yme\nTfuAbd3yNuDenvatSZYnWQesBx4aXcmSpMW4aIA+bwPeA3wnyYGu7SPArcDeJDcCTwHXA1TVwSR7\ngUPM37FzU1WdHnnlkqSB9A36qvo6kLNsvuos++wEdg5RlyRpRPxlrCQ1zqCXpMYZ9JLUOINekhpn\n0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9\nJDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUuL5Bn+TOJCeS\nPNrT9rEkx5Ic6F7v6tl2c5LDSZ5IcvW4CpckDWaQK/rPAJvP0P7JqtrYve4DSLIB2Apc0e1zW5Jl\noypWkrR4fYO+qh4Afjjg520B9lTVyap6EjgMbBqiPknSkIaZo39fkm93UzuXdm2rgad7+hzt2n5B\nku1JZpPMzs3NDVGGJGkhSw3624HLgY3AceDji/2AqtpVVTNVNTM1NbXEMiRJ/Swp6Kvq2ao6XVUv\nAp/iZ9Mzx4C1PV3XdG2SpAlZUtAnWdWzei3wkzty9gFbkyxPsg5YDzw0XImSpGFc1K9Dks8BVwIr\nkhwFbgGuTLIRKOAI8F6AqjqYZC9wCDgF3FRVp8dTuiRpEH2DvqpuOEPzHQv03wnsHKYoSdLo+MtY\nSWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJek\nxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqc\nQS9Jjesb9EnuTHIiyaM9bZcluT/Jd7v3S3u23ZzkcJInklw9rsIlSYMZ5Ir+M8Dml7TtAPZX1Xpg\nf7dOkg3AVuCKbp/bkiwbWbWSpEXrG/RV9QDww5c0bwF2d8u7gWt62vdU1cmqehI4DGwaUa2SpCVY\n6hz9yqo63i0/A6zsllcDT/f0O9q1SZImZOgvY6uqgFrsfkm2J5lNMjs3NzdsGZKks1hq0D+bZBVA\n936iaz8GrO3pt6Zr+wVVtauqZqpqZmpqaollSJL6WWrQ7wO2dcvbgHt72rcmWZ5kHbAeeGi4EiVJ\nw7ioX4cknwOuBFYkOQrcAtwK7E1yI/AUcD1AVR1Mshc4BJwCbqqq02OqXZI0gL5BX1U3nGXTVWfp\nvxPYOUxRkqTR8ZexktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn\n0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9\nJDXOoJekxhn0ktQ4g16SGmfQS1LjLhpm5yRHgOeB08CpqppJchnweWAaOAJcX1U/Gq5MSdJSjeKK\n/h1VtbGqZrr1HcD+qloP7O/WJUkTMo6pmy3A7m55N3DNGI4hSRrQsEFfwFeTPJxke9e2sqqOd8vP\nACvPtGOS7Ulmk8zOzc0NWYYk6WyGmqMH3l5Vx5K8Brg/yeO9G6uqktSZdqyqXcAugJmZmTP2kSQN\nb6gr+qo61r2fAO4BNgHPJlkF0L2fGLZISdLSLTnok7wqyat/sgz8AfAosA/Y1nXbBtw7bJGSpKUb\nZupmJXBPkp98zj9X1b8m+RawN8mNwFPA9cOXKUlaqiUHfVV9D3jjGdp/AFw1TFGSpNHxl7GS1DiD\nXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+gl\nqXEGvSQ1zqCXpMYZ9JLUuGH+lKB0QZje8aWJHPfIre+eyHHVHq/oJalxBr0kNc6gl6TGGfSS1DiD\nXpIaZ9BLUuMMeklqXBP30XufsySd3diCPslm4O+BZcCnq+rWcR1LkoYxqYtFODcXjGOZukmyDPhH\n4J3ABuCGJBvGcSxJ0sLGNUe/CThcVd+rqv8D9gBbxnQsSdICxjV1sxp4umf9KPC7YzqW1KRJTidM\nit97jcfEvoxNsh3Y3q2+kOSJSdWyVPmbRXVfAXx/PJW8rFwo44QLZ6znbJyL/H9qHM75OR1yzL8x\nSKdxBf0xYG3P+pqu7aeqahewa0zHf9lJMltVM5OuY9wulHHChTPWC2Wc0O5YxzVH/y1gfZJ1SV4J\nbAX2jelYkqQFjOWKvqpOJflL4CvM3155Z1UdHMexJEkLG9scfVXdB9w3rs8/D10o01QXyjjhwhnr\nhTJOaHSsqapJ1yBJGiOfdSNJjTPoRyjJ5iRPJDmcZMcZtr8+yTeTnEzyoUnUOCoDjPWPk3w7yXeS\nfCPJGydR57AGGOeWbpwHkswmefsk6hyFfmPt6ffmJKeSXHcu6xuVAc7plUn+pzunB5J8dBJ1jlRV\n+RrBi/kvnf8LuBx4JfAfwIaX9HkN8GZgJ/ChSdc85rG+Fbi0W34n8OCk6x7TOC/hZ1OgbwAen3Td\n4xprT79/Y/77t+smXfeYzumVwBcnXesoX17Rj07fxz5U1Ymq+hbw40kUOEKDjPUbVfWjbvXfmf8t\nxflmkHG+UF06AK8CztcvvQZ9bMn7gLuAE+eyuBG6IB/PYtCPzpke+7B6QrWM22LHeiPw5bFWNB4D\njTPJtUkeB74E/Ok5qm3U+o41yWrgWuD2c1jXqA363+5buym5Lye54tyUNj4GvcYqyTuYD/oPT7qW\ncamqe6rq9cA1wF9Pup4x+jvgw1X14qQLGbNHgNdV1RuAfwD+ZcL1DM2gH52+j31oyEBjTfIG4NPA\nlqr6wTmqbZQWdU6r6gHg8iQrxl3YGAwy1hlgT5IjwHXAbUmuOTfljcwgj2d5rqpe6JbvAy4+T8/p\nTxn0o3MhPfah71iTvA64G3hPVf3nBGochUHG+ZtJ0i3/DrAcOB//Ues71qpaV1XTVTUNfAH4i6o6\n3652Bzmnr+05p5uYz8nz8Zz+VBN/SvDloM7y2Ickf9Zt/6ckrwVmgV8BXkzyAea/8X9uYoUvwSBj\nBT4K/BrzV30Ap+o8e1jUgOP8Q+BPkvwY+F/gj3q+nD1vDDjW896A47wO+PMkp5g/p1vPx3Pay1/G\nSlLjnLqRpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNe7/ARbnGdKr5jPeAAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc72944bf50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ici_cuv = []\n",
    "ici_ger = []\n",
    "for i in lookup_mode.collect():\n",
    "    bout=i[0]\n",
    "    if label_dict[bout] == 1:\n",
    "        ici_cuv.append(1.0 * i[1] / 100)\n",
    "    else:\n",
    "        ici_ger.append(1.0 * i[1] / 100)  \n",
    "        \n",
    "plt.hist(ici_cuv, bins=10)\n",
    "plt.show()\n",
    "\n",
    "plt.hist(ici_ger, bins=10)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### median\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADwRJREFUeJzt3X+sX3V9x/Hna6AkohMYtaulrpjVYDGK865bxD8wZAMl\nprAZVrYIQbK6hRlNzGIxmZiYJjWbbsscLPVHwMTBmiizG+imVUecUbywIm2B2UGRdkirkAFmYWt5\n74/vYXwtbe/33u/9/uDT5yO5uef7OZ9zz6un977uuef7veemqpAktevnJh1AkjRaFr0kNc6il6TG\nWfSS1DiLXpIaZ9FLUuPmLPokK5J8I8muJDuTvK8b/0iSfUm2d29v79vmmiS7k9yf5IJR/gMkSceW\nuV5Hn2QZsKyq7kryMuBO4GLgUuCpqvqzw+avBm4C1gCvBL4GvKaqDo0gvyRpDnOe0VfVI1V1V7f8\nJHAvsPwYm6wFbq6qp6vqQWA3vdKXJE3AifOZnGQl8Ebgu8C5wHuTXA7MAh+oqsfpfRP4Tt9meznC\nN4Yk64H1ACeffPKbzjrrrAXEl6Tj15133vnjqloy17yBiz7JS4EvAO+vqieSXA98FKju/ceBdw/6\n8apqM7AZYGZmpmZnZwfdVJIEJHlokHkDveomyYvolfznq+qLAFX1aFUdqqpngE/x3OWZfcCKvs3P\n6MYkSRMwyKtuAnwGuLeqPtE3vqxv2iXAjm55K7AuyUlJzgRWAXcsXmRJ0nwMcunmXOBdwD1Jtndj\nHwIuS3IOvUs3e4D3AFTVziRbgF3AQeBqX3EjSZMzZ9FX1beAHGHVbcfYZiOwcYhckqRF4m/GSlLj\nLHpJapxFL0mNs+glqXEWvSQ1bl63QJCORys33DqR/e7ZdNFE9qv2eEYvSY2z6CWpcRa9JDXOopek\nxln0ktQ4i16SGmfRS1LjLHpJapxFL0mNs+glqXEWvSQ1zqKXpMZZ9JLUOItekhpn0UtS4yx6SWqc\nRS9JjbPoJalxFr0kNc6il6TGWfSS1DiLXpIaZ9FLUuMseklqnEUvSY2z6CWpcRa9JDXOopekxln0\nktS4OYs+yYok30iyK8nOJO/rxk9L8tUkP+jen9q3zTVJdie5P8kFo/wHSJKObZAz+oPAB6pqNfDr\nwNVJVgMbgG1VtQrY1j2mW7cOOBu4ELguyQmjCC9JmtucRV9Vj1TVXd3yk8C9wHJgLXBjN+1G4OJu\neS1wc1U9XVUPAruBNYsdXJI0mHldo0+yEngj8F1gaVU90q36EbC0W14OPNy32d5u7PCPtT7JbJLZ\nAwcOzDO2JGlQAxd9kpcCXwDeX1VP9K+rqgJqPjuuqs1VNVNVM0uWLJnPppKkeRio6JO8iF7Jf76q\nvtgNP5pkWbd+GbC/G98HrOjb/IxuTJI0AYO86ibAZ4B7q+oTfau2Ald0y1cAX+obX5fkpCRnAquA\nOxYvsiRpPk4cYM65wLuAe5Js78Y+BGwCtiS5CngIuBSgqnYm2QLsoveKnaur6tCiJ5ckDWTOoq+q\nbwE5yurzj7LNRmDjELkkSYvE34yVpMZZ9JLUOItekhpn0UtS4yx6SWqcRS9JjbPoJalxFr0kNc6i\nl6TGWfSS1DiLXpIaZ9FLUuMseklqnEUvSY2z6CWpcRa9JDXOopekxln0ktQ4i16SGmfRS1LjLHpJ\napxFL0mNs+glqXEWvSQ1zqKXpMZZ9JLUOItekhpn0UtS4yx6SWqcRS9JjbPoJalxFr0kNc6il6TG\nWfSS1Lg5iz7JZ5PsT7Kjb+wjSfYl2d69vb1v3TVJdie5P8kFowouSRrMIGf0NwAXHmH8z6vqnO7t\nNoAkq4F1wNndNtclOWGxwkqS5m/Ooq+q24HHBvx4a4Gbq+rpqnoQ2A2sGSKfJGlIw1yjf2+S73eX\ndk7txpYDD/fN2duNPU+S9Ulmk8weOHBgiBiSpGNZaNFfD7waOAd4BPj4fD9AVW2uqpmqmlmyZMkC\nY0iS5rKgoq+qR6vqUFU9A3yK5y7P7ANW9E09oxuTJE3Igoo+ybK+h5cAz74iZyuwLslJSc4EVgF3\nDBdRkjSME+eakOQm4Dzg9CR7gWuB85KcAxSwB3gPQFXtTLIF2AUcBK6uqkOjiS5JGsScRV9Vlx1h\n+DPHmL8R2DhMKEnS4vE3YyWpcXOe0UvTYOWGWycdQXrB8oxekhpn0UtS4yx6SWqcRS9JjbPoJalx\nFr0kNc6il6TGWfSS1DiLXpIaZ9FLUuMseklqnEUvSY2z6CWpcRa9JDXOopekxln0ktQ4i16SGmfR\nS1LjLHpJapxFL0mNs+glqXEWvSQ1zqKXpMZZ9JLUOItekhpn0UtS4yx6SWqcRS9JjbPoJalxFr0k\nNc6il6TGWfSS1DiLXpIaN2fRJ/lskv1JdvSNnZbkq0l+0L0/tW/dNUl2J7k/yQWjCi5JGswgZ/Q3\nABceNrYB2FZVq4Bt3WOSrAbWAWd321yX5IRFSytJmrc5i76qbgceO2x4LXBjt3wjcHHf+M1V9XRV\nPQjsBtYsUlZJ0gIs9Br90qp6pFv+EbC0W14OPNw3b2839jxJ1ieZTTJ74MCBBcaQJM1l6Cdjq6qA\nWsB2m6tqpqpmlixZMmwMSdJRLLToH02yDKB7v78b3wes6Jt3RjcmSZqQhRb9VuCKbvkK4Et94+uS\nnJTkTGAVcMdwESVJwzhxrglJbgLOA05Pshe4FtgEbElyFfAQcClAVe1MsgXYBRwErq6qQyPKLkka\nwJxFX1WXHWXV+UeZvxHYOEwoSdLi8TdjJalxFr0kNc6il6TGWfSS1DiLXpIaZ9FLUuMseklqnEUv\nSY2z6CWpcRa9JDXOopekxln0ktQ4i16SGmfRS1LjLHpJapxFL0mNs+glqXEWvSQ1zqKXpMZZ9JLU\nOItekhpn0UtS4yx6SWqcRS9JjbPoJalxFr0kNc6il6TGWfSS1DiLXpIaZ9FLUuMseklqnEUvSY2z\n6CWpcRa9JDXuxGE2TrIHeBI4BBysqpkkpwF/B6wE9gCXVtXjw8WUJC3UYpzRv7Wqzqmqme7xBmBb\nVa0CtnWPJUkTMtQZ/VGsBc7rlm8Evgl8cAT7kZq2csOtE9v3nk0XTWzfWnzDntEX8LUkdyZZ340t\nrapHuuUfAUuPtGGS9Ulmk8weOHBgyBiSpKMZ9oz+LVW1L8krgK8mua9/ZVVVkjrShlW1GdgMMDMz\nc8Q5kqThDXVGX1X7uvf7gVuANcCjSZYBdO/3DxtSkrRwCy76JCcnedmzy8BvAjuArcAV3bQrgC8N\nG1KStHDDXLpZCtyS5NmP87dV9ZUk3wO2JLkKeAi4dPiYkqSFWnDRV9UDwBuOMP4T4PxhQkmSFo+/\nGStJjbPoJalxFr0kNc6il6TGWfSS1DiLXpIaZ9FLUuMseklq3ChuU6yGTfLWuZIWxjN6SWqcRS9J\njbPoJalxFr0kNc6il6TGWfSS1DiLXpIaZ9FLUuMseklqnEUvSY2z6CWpcRa9JDXOopekxln0ktQ4\ni16SGmfRS1LjLHpJapx/YeoFyL/yJGk+PKOXpMZZ9JLUOItekhpn0UtS4yx6SWqcr7qR9DyTemXX\nnk0XTWS/rfOMXpIa5xn9EHw9u6QXgpEVfZILgb8ETgA+XVWbRrUvC1fSMCbZIeO4XDWSok9yAvDX\nwG8Ae4HvJdlaVbtGsT9JbfCkbTRGdY1+DbC7qh6oqv8BbgbWjmhfkqRjGNWlm+XAw32P9wK/1j8h\nyXpgfffwqST3jyjLsZwO/HgC+x2U+YY37RnNN5wXfL58bKiP/0uDTJrYk7FVtRnYPKn9AySZraqZ\nSWY4FvMNb9ozmm845hvMqC7d7ANW9D0+oxuTJI3ZqIr+e8CqJGcmeTGwDtg6on1Jko5hJJduqupg\nkj8C/oneyys/W1U7R7GvIU300tEAzDe8ac9ovuGYbwCpqklnkCSNkLdAkKTGWfSS1LjjouiTXJjk\n/iS7k2w4wvrfS/L9JPck+XaSN0xZvrVdvu1JZpO8ZZry9c371SQHk7xzmvIlOS/Jf3XHb3uSD09T\nvr6M25PsTPIv48w3SMYkf9x3/HYkOZTktCnK9/Ik/5Dk7u4YXjmubAPmOzXJLd3X8R1JXjfOfFRV\n02/0ngz+D+DVwIuBu4HVh815M3Bqt/w24LtTlu+lPPd8yuuB+6YpX9+8rwO3Ae+cpnzAecA/TvHn\n3ynALuBV3eNXTFvGw+a/A/j6NOUDPgR8rFteAjwGvHiK8v0pcG23fBawbZz/x8fDGf2ct2Ooqm9X\n1ePdw+/Qe93/NOV7qrrPEOBkYJzPoA96O4v3Al8A9o8xG0z/7TYGyfe7wBer6ocAVTXtx/Ay4Kax\nJOsZJF8BL0sSeidGjwEHpyjfanonQlTVfcDKJEvHlO+4KPoj3Y5h+THmXwV8eaSJftZA+ZJckuQ+\n4Fbg3WPKBgPkS7IcuAS4foy5njXo/++bux+bv5zk7PFEAwbL9xrg1CTfTHJnksvHlq5n4K+RJC8B\nLqT3TX1cBsn3SeC1wH8C9wDvq6pnxhNvoHx3A78FkGQNvVsXjO2E8ngo+oEleSu9ov/gpLMcrqpu\nqaqzgIuBj046z2H+AvjgGL+w5usuepdFXg/8FfD3E85zuBOBNwEXARcAf5LkNZONdFTvAP61qh6b\ndJDDXABsB14JnAN8MsnPTzbSz9gEnJJkO72ffv8NODSunR8Pf3hkoNsxJHk98GngbVX1kzFlg3ne\nLqKqbk/y6iSnV9U4buY0SL4Z4ObeT82cDrw9ycGqGkehzpmvqp7oW74tyXVTdvz2Aj+pqp8CP01y\nO/AG4N/HkA/m9zm4jvFetoHB8l0JbOouce5O8iC9a+F3TEO+7nPwSoDu8tKDwANjyPb/AZp+o/fN\n7AHgTJ57ouTsw+a8CtgNvHlK8/0yzz0Z+yv0PokyLfkOm38D430ydpDj94t9x28N8MNpOn70Ljls\n6+a+BNgBvG6ajmE37+X0rn2fPK5s8ziG1wMf6ZaXdl8jp09RvlPonhwGfh/43DiPYfNn9HWU2zEk\n+YNu/d8AHwZ+AbiuOys9WGO649yA+X4buDzJ/wL/DfxOdZ8xU5JvYgbM907gD5McpHf81k3T8auq\ne5N8Bfg+8Ay9v8i2Yxz5Bs3YTb0E+Ofq/eQxNgPm+yhwQ5J7gNC7lDiW2xcPmO+1wI1JCthJ7xLx\n2HgLBElqnE/GSlLjLHpJapxFL0mNs+glqXEWvSQ1zqKXpMZZ9JLUuP8DllqC6o3Q004AAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc7292a95d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD8CAYAAACfF6SlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADw9JREFUeJzt3X+s3Xddx/HnyxUmDCOdvdS5DTti/dERCHhdcBAyLYbh\nCB1xmUWRKksaDCIaE+w0YX+YJTUaA38IphlIEybLAsw1bPyoRVwMbngHE7Z1cxW6rbNbL6Dg0ACF\nt3/cr+6u9Lb3nO/33HPG5/lIyPn++tzvi09PXv3ue+73NFWFJKktPzDtAJKktWf5S1KDLH9JapDl\nL0kNsvwlqUGWvyQ1yPKXpAZZ/pLUIMtfkhq0btoBADZs2FCbNm2adgxJekq58847v1xVc+OMnYny\n37RpEwsLC9OOIUlPKUkeHHest30kqUGWvyQ1yPKXpAZZ/pLUIMtfkhpk+UtSgyx/SWqQ5S9JDbL8\nJalBM/GEr0azadctUzv34d2XTe3ckobjlb8kNcjyl6QGnbb8k7w3ybEkdy/b9mdJ7kvy+SQ3JXn2\nsn1XJzmU5P4kr5xUcEnS+FZz5f8+4NITtu0Hnl9VLwD+FbgaIMkWYDtwYTfmXUnOGCytJGkQpy3/\nqroN+OoJ2z5RVce71duB87rlbcANVfXNqvoScAi4aMC8kqQBDHHP/43AR7vlc4GHl+070m37Hkl2\nJllIsrC4uDhADEnSavUq/yR/DBwHrh91bFXtqar5qpqfmxvrH6KRJI1p7N/zT/KbwKuBrVVV3eZH\ngPOXHXZet02SNEPGuvJPcinwNuA1VfXfy3btA7YnOTPJBcBm4DP9Y0qShnTaK/8kHwAuATYkOQJc\nw9Jv95wJ7E8CcHtVvamq7klyI3AvS7eD3lxV35lUeEnSeE5b/lX1upNsfs8pjr8WuLZPKEnSZPmE\nryQ1yPKXpAZZ/pLUIMtfkhpk+UtSgyx/SWqQ5S9JDbL8JalBlr8kNcjyl6QGWf6S1CDLX5IaZPlL\nUoMsf0lqkOUvSQ2y/CWpQZa/JDXI8pekBln+ktQgy1+SGmT5S1KDLH9JapDlL0kNsvwlqUGnLf8k\n701yLMndy7adnWR/kge61/XL9l2d5FCS+5O8clLBJUnjW82V//uAS0/Ytgs4UFWbgQPdOkm2ANuB\nC7sx70pyxmBpJUmDOG35V9VtwFdP2LwN2Nst7wUuX7b9hqr6ZlV9CTgEXDRQVknSQMa957+xqo52\ny48CG7vlc4GHlx13pNsmSZohvT/wraoCatRxSXYmWUiysLi42DeGJGkE45b/Y0nOAehej3XbHwHO\nX3bced2271FVe6pqvqrm5+bmxowhSRrHuOW/D9jRLe8Abl62fXuSM5NcAGwGPtMvoiRpaOtOd0CS\nDwCXABuSHAGuAXYDNya5CngQuBKgqu5JciNwL3AceHNVfWdC2SVJYzpt+VfV61bYtXWF468Fru0T\nSpI0WT7hK0kNsvwlqUGWvyQ1yPKXpAZZ/pLUIMtfkhpk+UtSgyx/SWqQ5S9JDbL8JalBlr8kNcjy\nl6QGWf6S1CDLX5IaZPlLUoMsf0lqkOUvSQ2y/CWpQZa/JDXI8pekBln+ktQgy1+SGmT5S1KDLH9J\napDlL0kN6lX+SX4/yT1J7k7ygSQ/mOTsJPuTPNC9rh8qrCRpGGOXf5Jzgd8F5qvq+cAZwHZgF3Cg\nqjYDB7p1SdIM6XvbZx3wjCTrgGcC/w5sA/Z2+/cCl/c8hyRpYGOXf1U9Avw58BBwFPhaVX0C2FhV\nR7vDHgU2nmx8kp1JFpIsLC4ujhtDkjSGPrd91rN0lX8B8GPAWUlev/yYqiqgTja+qvZU1XxVzc/N\nzY0bQ5I0hj63fV4BfKmqFqvq28CHgYuBx5KcA9C9HusfU5I0pD7l/xDwkiTPTBJgK3AQ2Afs6I7Z\nAdzcL6IkaWjrxh1YVXck+SDwWeA48DlgD/As4MYkVwEPAlcOEVSSNJyxyx+gqq4Brjlh8zdZ+q8A\nSdKM8glfSWqQ5S9JDbL8JalBlr8kNcjyl6QGWf6S1CDLX5IaZPlLUoMsf0lqkOUvSQ2y/CWpQZa/\nJDXI8pekBln+ktQgy1+SGmT5S1KDLH9JapDlL0kNsvwlqUGWvyQ1yPKXpAZZ/pLUIMtfkhpk+UtS\ng3qVf5JnJ/lgkvuSHEzy80nOTrI/yQPd6/qhwkqShtH3yv+dwMeq6qeBFwIHgV3AgaraDBzo1iVJ\nM2Ts8k/yw8DLgfcAVNW3quo/gW3A3u6wvcDlfUNKkobV58r/AmAR+Oskn0tyXZKzgI1VdbQ75lFg\nY9+QkqRh9Sn/dcCLgXdX1YuAb3DCLZ6qKqBONjjJziQLSRYWFxd7xJAkjapP+R8BjlTVHd36B1n6\ny+CxJOcAdK/HTja4qvZU1XxVzc/NzfWIIUka1djlX1WPAg8n+alu01bgXmAfsKPbtgO4uVdCSdLg\n1vUc/xbg+iRPB74I/BZLf6HcmOQq4EHgyp7nkCQNrFf5V9VdwPxJdm3t83MlSZPlE76S1CDLX5Ia\nZPlLUoMsf0lqkOUvSQ2y/CWpQZa/JDXI8pekBln+ktQgy1+SGmT5S1KDLH9JapDlL0kNsvwlqUGW\nvyQ1yPKXpAZZ/pLUIMtfkhpk+UtSgyx/SWqQ5S9JDVo37QB6atm065apnPfw7sumcl7p+5VX/pLU\nIMtfkhrUu/yTnJHkc0k+0q2fnWR/kge61/X9Y0qShjTElf9bgYPL1ncBB6pqM3CgW5ckzZBe5Z/k\nPOAy4Lplm7cBe7vlvcDlfc4hSRpe3yv/dwBvA767bNvGqjraLT8KbOx5DknSwMYu/ySvBo5V1Z0r\nHVNVBdQK43cmWUiysLi4OG4MSdIY+lz5vxR4TZLDwA3ALyZ5P/BYknMAutdjJxtcVXuqar6q5ufm\n5nrEkCSNauzyr6qrq+q8qtoEbAc+WVWvB/YBO7rDdgA3904pSRrUJH7PfzfwS0keAF7RrUuSZsgg\nX+9QVZ8CPtUtfwXYOsTPlSRNhk/4SlKDLH9JapDlL0kNsvwlqUGWvyQ1yPKXpAZZ/pLUIMtfkhpk\n+UtSgyx/SWqQ5S9JDbL8JalBlr8kNWiQb/WUpCFs2nXLVM57ePdlUznvNHnlL0kNsvwlqUGWvyQ1\nyHv+0oya1v1vaPMeeGssf0nfY5p/8WhteNtHkhpk+UtSgyx/SWqQ5S9JDfID3x78UGzt+Jsv0rC8\n8pekBo1d/knOT/L3Se5Nck+St3bbz06yP8kD3ev64eJKkobQ58r/OPAHVbUFeAnw5iRbgF3Agara\nDBzo1iVJM2Ts8q+qo1X12W75v4CDwLnANmBvd9he4PK+ISVJwxrknn+STcCLgDuAjVV1tNv1KLBx\nhTE7kywkWVhcXBwihiRplXqXf5JnAR8Cfq+qvr58X1UVUCcbV1V7qmq+qubn5ub6xpAkjaBX+Sd5\nGkvFf31Vfbjb/FiSc7r95wDH+kWUJA2tz2/7BHgPcLCq/mLZrn3Ajm55B3Dz+PEkSZPQ5yGvlwK/\nAXwhyV3dtj8CdgM3JrkKeBC4sl9Eabp8mE/fj8Yu/6r6RyAr7N467s+VJE2eT/hKUoMsf0lqkOUv\nSQ2y/CWpQZa/JDXI8pekBln+ktQgy1+SGmT5S1KDLH9JapDlL0kNsvwlqUGWvyQ1yPKXpAb1+T5/\nSfq+MM1/s+Hw7sumcl6v/CWpQZa/JDXI8pekBln+ktQgy1+SGmT5S1KDLH9JapDlL0kNsvwlqUET\ne8I3yaXAO4EzgOuqavekzjXNp/Mk6aloIlf+Sc4A/hJ4FbAFeF2SLZM4lyRpdJO67XMRcKiqvlhV\n3wJuALZN6FySpBFNqvzPBR5etn6k2yZJmgFT+1bPJDuBnd3q40nuP+GQDcCX1zbVyMw4DDMOw4zD\nWNOM+dOxhv1fxh8f97yTKv9HgPOXrZ/Xbft/VbUH2LPSD0iyUFXzk4k3DDMOw4zDMOMwWsk4qds+\n/wxsTnJBkqcD24F9EzqXJGlEE7nyr6rjSX4H+DhLv+r53qq6ZxLnkiSNbmL3/KvqVuDWHj9ixVtC\nM8SMwzDjMMw4jCYypqqGCCJJegrx6x0kqUFTKf8klya5P8mhJLtOsv/Xk3w+yReSfDrJC1c7dkYy\nHu6235VkYUr5tnX57kqykORlqx07IxknPoerybjsuJ9LcjzJFaOOnXLGmZjHJJck+VqX464kb1/t\n2BnJOBPzuCznXUnuSfIPo4x9kqpa0/+x9AHwvwHPA54O/Auw5YRjLgbWd8uvAu5Y7dhpZ+zWDwMb\npjyHz+KJ23ovAO6bwTk8aca1mMNR5qI77pMsfYZ1xazN40oZZ2kegUuAj4z7/2+aGWdsHp8N3As8\nt1t/zrjzOI0r/9N+9UNVfbqq/qNbvZ2l5wRWNXYGMq6F1eR7vLp3BXAWUKsdOwMZ18pq5+ItwIeA\nY2OMnWbGtdJnLmZtHqdpNRl/DfhwVT0EUFXHRhj7JNMo/1G/+uEq4KNjjh1Xn4ywVGJ/l+TO7knm\nqeRL8tok9wG3AG8cZeyUM8Lk53BVGZOcC7wWePeoY2cgI8zIPHYu7m7zfTTJhSOOnWZGmJ15/Elg\nfZJPdVneMMLYJ5na1zusRpJfYKlYX3a6Y6dlhYwvq6pHkjwH2J/kvqq6ba2zVdVNwE1JXg78CfCK\ntc5wOqfIOBNzCLwD+MOq+m6SKZx+VU6VcVbm8bMs3ap4PMkvA38LbJ5CjlM5VcZZmcd1wM8CW4Fn\nAP+U5PZxftA0rvxP+9UPAEleAFwHbKuqr4wydsoZqapHutdjwE0s/SfZmudbluc24HlJNow6dkoZ\n12IOV5txHrghyWHgCuBdSS5f5dhpZ5yZeayqr1fV493yrcDTZu39eIqMMzOPLF3Rf7yqvlFVXwZu\nA164yrFPNskPMFb4UGMd8EXgAp74YOLCE455LnAIuHjUsTOQ8Szgh5Ytfxq4dAr5foInPkx9cfdG\nyIzN4UoZJz6H47yfgPfxxAe+MzOPp8g4M/MI/OiyP+uLgIdm8P24UsZZmsefAQ50xz4TuBt4/jjz\nuOa3fWqFr35I8qZu/18Bbwd+hKUrGIDjVTW/0thZyghsZOk2Biz9gfxNVX1sCvl+BXhDkm8D/wP8\nai29e2ZpDk+aMcnE53CEjCONnaWMrMF7cYSMVwC/neQ4S3/W22fw/XjSjLP0fqyqg0k+Bnwe+C5L\n/0ri3QCjzqNP+EpSg3zCV5IaZPlLUoMsf0lqkOUvSQ2y/CWpQZa/JDXI8pekBln+ktSg/wVFN7Lx\nDbkwYQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc7282bbb10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ici_cuv = []\n",
    "ici_ger = []\n",
    "for i in lookup_median.collect():\n",
    "    bout=i[0]\n",
    "    if label_dict[bout] == 1:\n",
    "        ici_cuv.append(1.0 * i[1])\n",
    "    else:\n",
    "        ici_ger.append(1.0 * i[1])  \n",
    "        \n",
    "plt.hist(ici_cuv, bins=10)\n",
    "plt.show()\n",
    "\n",
    "plt.hist(ici_ger, bins=10)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reasigning Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def convert_to_list(dict_1):\n",
    "    dictlist = []\n",
    "    for key, value in dict_1.iteritems():\n",
    "        temp = [key,value]\n",
    "        dictlist.append(temp)\n",
    "    return dictlist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "l = convert_to_list(label_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.sql.types import StructType\n",
    "from pyspark.sql.types import StructField\n",
    "from pyspark.sql.types import StringType\n",
    "\n",
    "schema = StructType([StructField(\"bout_i\", LongType(), True), StructField(\"bout_label\", IntegerType(), True)])\n",
    "rdd1 = sc.parallelize(l)\n",
    "df_new1 = sqlContext.createDataFrame(rdd1, schema)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df_final_rename = df_final.alias('e').join(df_new1.alias('f'), 'bout_i')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Cuviers=df_final_rename.filter(df_final_rename.bout_label==1)\n",
    "Gervais=df_final_rename.filter(df_final_rename.bout_label==0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFoNJREFUeJzt3X+QXeV93/H3J5JNqBMwPxSNLKEuLapTYOofUinTeDJO\nNC6ynYnoDBClbdC4GtQONHFmOlOL/NG009GM+MckOIUOAw6CxgaV2EWNjT1UxHU7sYRFgo0Fpd4Y\nCFIFkgWGxhlIJH/7x33Wvtqzm7272r27q32/Zu7c537vec4+z0izn/ucc+7ZVBWSJPX7sfkegCRp\n4TEcJEkdhoMkqcNwkCR1GA6SpA7DQZLUYThIkjoMB0lSh+EgSepYPt8DmKmLL764RkZG5nsYkrSo\nPPnkk9+tqhVTbbdow2FkZISDBw/O9zAkaVFJ8uIg23lYSZLUYThIkjoMB0lSh+EgSeowHCRJHYaD\nJKnDcJAkdRgOkqQOw0GS1LFovyGthW1kxxd+2H5h10fncSSSZsKVgySpw3CQJHV4WEmzpv9QkqTF\nzZWDJKljynBI8u4kT/U93kjy60kuTPJYkm+35wv6+tyaZDTJc0mu6auvT/J0e++OJGn1c5I81OoH\nkozMxWQlSYOZMhyq6rmqem9VvRdYD/wF8HlgB7CvqtYB+9prklwObAGuADYBdyZZ1nZ3F3ATsK49\nNrX6NuC1qroMuB24bXamJ0maiekeVtoI/GlVvQhsBna3+m7g2tbeDDxYVW9V1fPAKHBVklXAeVW1\nv6oKuH9cn7F9PQxsHFtVSJKGb7rhsAX4bGuvrKqjrf0ysLK1VwMv9fU53GqrW3t8/bQ+VXUSeB24\naPwPT7I9ycEkB48fPz7NoUuSBjVwOCR5O/CLwH8Z/15bCdQsjmtCVXV3VW2oqg0rVkz5J1AlSTM0\nnZXDh4E/rqpX2utX2qEi2vOxVj8CXNLXb02rHWnt8fXT+iRZDpwPnJjG2CRJs2g64fDL/OiQEsBe\nYGtrbwUe6atvaVcgXUrvxPMT7RDUG0mubucTbhzXZ2xf1wGPt9WIJGkeDPQluCTvAD4E/Iu+8i5g\nT5JtwIvADQBVdSjJHuAZ4CRwS1Wdan1uBu4DzgUebQ+Ae4EHkowCr9I7tyFJmicDhUNVfZ9xJ4ir\n6gS9q5cm2n4nsHOC+kHgygnqbwLXDzIWSdLc8xvSkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2G\ngySpw3CQJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqWOgv+cgnYmRHV/4YfuFXR+d\nx5FIGpQrB0lSh+EgSeowHCRJHQOdc0jyTuAeen//uYB/DjwHPASMAC8AN1TVa237W4FtwCng16rq\ny62+HrgPOBf4IvDxqqok5wD3A+uBE8AvVdULszFBzZ3+cwmSzi6Drhx+G/hSVf008B7gWWAHsK+q\n1gH72muSXA5sAa4ANgF3JlnW9nMXcBOwrj02tfo24LWqugy4HbjtDOclSToDU4ZDkvOBnwXuBaiq\nv6yq7wGbgd1ts93Ata29GXiwqt6qqueBUeCqJKuA86pqf1UVvZVCf5+xfT0MbEySM56dJGlGBlk5\nXAocB343yZ8kuSfJO4CVVXW0bfMysLK1VwMv9fU/3GqrW3t8/bQ+VXUSeB24aPrTkSTNhkHCYTnw\nfuCuqnof8H3aIaQxbSVQsz+80yXZnuRgkoPHjx+f6x8nSUvWIOFwGDhcVQfa64fphcUr7VAR7flY\ne/8IcElf/zWtdqS1x9dP65NkOXA+vRPTp6mqu6tqQ1VtWLFixQBDlyTNxJThUFUvAy8leXcrbQSe\nAfYCW1ttK/BIa+8FtiQ5J8ml9E48P9EOQb2R5Op2PuHGcX3G9nUd8HhbjUiS5sGgt8/4VeD3krwd\n+A7wMXrBsifJNuBF4AaAqjqUZA+9ADkJ3FJVp9p+buZHl7I+2h7QO9n9QJJR4FV6VztJkubJQOFQ\nVU8BGyZ4a+Mk2+8Edk5QP0jvuxLj628C1w8yFknS3PMb0pKkDsNBktRhOEiSOgwHSVKH4SBJ6jAc\nJEkdhoMkqcNwkCR1GA6SpA7DQZLUYThIkjoMB0lSh+EgSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS\n1GE4SJI6BgqHJC8keTrJU0kOttqFSR5L8u32fEHf9rcmGU3yXJJr+urr235Gk9yRJK1+TpKHWv1A\nkpHZnaYkaTqms3L4uap6b1VtaK93APuqah2wr70myeXAFuAKYBNwZ5Jlrc9dwE3AuvbY1OrbgNeq\n6jLgduC2mU9JknSmzuSw0mZgd2vvBq7tqz9YVW9V1fPAKHBVklXAeVW1v6oKuH9cn7F9PQxsHFtV\nSJKGb9BwKOC/J3kyyfZWW1lVR1v7ZWBla68GXurre7jVVrf2+PppfarqJPA6cNE05iFJmkXLB9zu\nA1V1JMlPAY8l+d/9b1ZVJanZH97pWjBtB1i7du1c/zhJWrIGWjlU1ZH2fAz4PHAV8Eo7VER7PtY2\nPwJc0td9Tasdae3x9dP6JFkOnA+cmGAcd1fVhqrasGLFikGGLkmagSnDIck7kvzkWBv4R8C3gL3A\n1rbZVuCR1t4LbGlXIF1K78TzE+0Q1BtJrm7nE24c12dsX9cBj7fzEpKkeTDIYaWVwOfb+eHlwGeq\n6ktJvg7sSbINeBG4AaCqDiXZAzwDnARuqapTbV83A/cB5wKPtgfAvcADSUaBV+ld7SRJmidThkNV\nfQd4zwT1E8DGSfrsBHZOUD8IXDlB/U3g+gHGK0kaAr8hLUnqGPRqJQmAkR1fmO8hSBoCVw6SpA5X\nDhqq/pXHC7s+Oo8jkfTXceUgSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnq\nMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgwHSVLHwOGQZFmSP0nyB+31hUkeS/Lt9nxB37a3\nJhlN8lySa/rq65M83d67I0la/ZwkD7X6gSQjszdFSdJ0TWfl8HHg2b7XO4B9VbUO2Ndek+RyYAtw\nBbAJuDPJstbnLuAmYF17bGr1bcBrVXUZcDtw24xmI0maFQOFQ5I1wEeBe/rKm4Hdrb0buLav/mBV\nvVVVzwOjwFVJVgHnVdX+qirg/nF9xvb1MLBxbFUhSRq+QVcOvwX8G+AHfbWVVXW0tV8GVrb2auCl\nvu0Ot9rq1h5fP61PVZ0EXgcuGj+IJNuTHExy8Pjx4wMOXZI0XVOGQ5JfAI5V1ZOTbdNWAjWbA5vk\n59xdVRuqasOKFSvm+sdJ0pK1fIBtfgb4xSQfAX4cOC/JfwZeSbKqqo62Q0bH2vZHgEv6+q9ptSOt\nPb7e3+dwkuXA+cCJGc5JknSGplw5VNWtVbWmqkbonWh+vKr+GbAX2No22wo80tp7gS3tCqRL6Z14\nfqIdgnojydXtfMKN4/qM7eu69jPmfCUiSZrYICuHyewC9iTZBrwI3ABQVYeS7AGeAU4Ct1TVqdbn\nZuA+4Fzg0fYAuBd4IMko8Cq9EJIkzZNphUNVfQX4SmufADZOst1OYOcE9YPAlRPU3wSun85YJElz\nx29IS5I6DAdJUofhIEnqOJMT0loiRnZ8Yb6HIGnIXDlIkjoMB0lSh+EgSeowHCRJHYaDJKnDcJAk\ndRgOkqQOw0GS1GE4SJI6DAdJUoe3z9C86b8txwu7PjqPI5E0nisHSVKH4SBJ6jAcJEkdhoMkqWPK\ncEjy40meSPKNJIeS/PtWvzDJY0m+3Z4v6Otza5LRJM8luaavvj7J0+29O5Kk1c9J8lCrH0gyMvtT\nlSQNapCVw1vAz1fVe4D3ApuSXA3sAPZV1TpgX3tNksuBLcAVwCbgziTL2r7uAm4C1rXHplbfBrxW\nVZcBtwO3zcLcJEkzNGU4VM+ft5dva48CNgO7W303cG1rbwYerKq3qup5YBS4Kskq4Lyq2l9VBdw/\nrs/Yvh4GNo6tKiRJwzfQOYcky5I8BRwDHquqA8DKqjraNnkZWNnaq4GX+rofbrXVrT2+flqfqjoJ\nvA5cNO3ZSJJmxUDhUFWnquq9wBp6q4Arx71f9FYTcyrJ9iQHkxw8fvz4XP84SVqypnW1UlV9D/hD\neucKXmmHimjPx9pmR4BL+rqtabUjrT2+flqfJMuB84ETE/z8u6tqQ1VtWLFixXSGLkmahilvn5Fk\nBfBXVfW9JOcCH6J3wngvsBXY1Z4faV32Ap9J8kngXfROPD9RVaeSvNFOZh8AbgQ+1ddnK/A14Drg\n8bYa0Tzpv7WFpKVnkHsrrQJ2tyuOfgzYU1V/kORrwJ4k24AXgRsAqupQkj3AM8BJ4JaqOtX2dTNw\nH3Au8Gh7ANwLPJBkFHiV3tVOkqR5MmU4VNU3gfdNUD8BbJykz05g5wT1g8CVE9TfBK4fYLySpCHw\nG9KSpA7DQZLUYThIkjoMB0lSh38JbhL+lTJJS5krB0lSh+EgSeowHCRJHYaDJKnDcJAkdRgOkqQO\nw0GS1GE4SJI6/BKcfsi/4SBpjCsHSVKH4SBJ6vCwkhYE72UlLSyuHCRJHYaDJKljynBIckmSP0zy\nTJJDST7e6hcmeSzJt9vzBX19bk0ymuS5JNf01dcnebq9d0eStPo5SR5q9QNJRmZ/qpKkQQ2ycjgJ\n/Ouquhy4GrglyeXADmBfVa0D9rXXtPe2AFcAm4A7kyxr+7oLuAlY1x6bWn0b8FpVXQbcDtw2C3OT\nJM3QlOFQVUer6o9b+/8BzwKrgc3A7rbZbuDa1t4MPFhVb1XV88AocFWSVcB5VbW/qgq4f1yfsX09\nDGwcW1VIkoZvWucc2uGe9wEHgJVVdbS99TKwsrVXAy/1dTvcaqtbe3z9tD5VdRJ4HbhoOmOTJM2e\ngS9lTfITwO8Dv15Vb/R/sK+qSlJzML7xY9gObAdYu3btXP+4gSz2SzD9VrSkiQy0ckjyNnrB8HtV\n9blWfqUdKqI9H2v1I8Alfd3XtNqR1h5fP61PkuXA+cCJ8eOoqrurakNVbVixYsUgQ5ckzcCUK4d2\n7P9e4Nmq+mTfW3uBrcCu9vxIX/0zST4JvIveiecnqupUkjeSXE3vsNSNwKfG7etrwHXA4+28xIIw\n/tP1YlwhSNJ0DHJY6WeAXwGeTvJUq/0GvVDYk2Qb8CJwA0BVHUqyB3iG3pVOt1TVqdbvZuA+4Fzg\n0faAXvg8kGQUeJXe1U4LlodiJJ3tpgyHqvpfwGRXDm2cpM9OYOcE9YPAlRPU3wSun2osC91iP/8g\nSWP8hrQkqcMb7y1BHhaTNBVXDpKkDsNBktRhOEiSOjznMEe8cknSYmY4LBGehJY0HYbDELiKkLTY\nGA5DNsygWKyrBcNUmn+ekJYkdbhymEeTfbI/k0/Li3W1IGlhceUgSepw5bAA+elf0nxz5SBJ6jAc\nJEkdhoMkqcNwkCR1GA6SpA7DQZLU4aWsWtC8lYY0P6YMhySfBn4BOFZVV7bahcBDwAjwAnBDVb3W\n3rsV2AacAn6tqr7c6uuB+4BzgS8CH6+qSnIOcD+wHjgB/FJVvTBrM5QWkEG+w2IIaiEY5LDSfcCm\ncbUdwL6qWgfsa69JcjmwBbii9bkzybLW5y7gJmBde4ztcxvwWlVdBtwO3DbTyUhng5EdX/jhQ5ov\nU64cquqrSUbGlTcDH2zt3cBXgE+0+oNV9RbwfJJR4KokLwDnVdV+gCT3A9cCj7Y+/67t62Hgd5Kk\nqmqmk5IWkjP5Je9hNc2XmZ5zWFlVR1v7ZWBla68G9vdtd7jV/qq1x9fH+rwEUFUnk7wOXAR8d/wP\nTbId2A6wdu3aGQ5dmlt+4tfZ4IxPSLfzBkP5lF9VdwN3A2zYsMGVhZYUVxEappmGwytJVlXV0SSr\ngGOtfgS4pG+7Na12pLXH1/v7HE6yHDif3olpSZMwKDTXZhoOe4GtwK72/Ehf/TNJPgm8i96J5yeq\n6lSSN5JcDRwAbgQ+NW5fXwOuAx73fIMWGw8l6WwzyKWsn6V38vniJIeB36QXCnuSbANeBG4AqKpD\nSfYAzwAngVuq6lTb1c386FLWR9sD4F7ggXby+lV6VztJHX5aloZnkKuVfnmStzZOsv1OYOcE9YPA\nlRPU3wSun2oc0kKzUFYLhqbmgt+Qls4iBoVmi+EgTcNCWS1Ic81w0KI0zE/IizUQXEXoTHhXVklS\nhysHaQKLdbUwGVcRmi7DQYvebP3iO9sCQToThoPOKpMFhb/4f8RVhAZhOOisZSBMzaDQZDwhLUnq\ncOUgCZh8peWKYmkyHCT9tTz0tDQZDpIGZlAsHYaDpBkxKM5uhoOkM+b5irOP4SBpzri6WLwMB0lD\nMej3TgyRhcFwkLSgTPfLi4bJ3DAcJC1qhsncMBwkLSmGyWAWTDgk2QT8NrAMuKeqds3zkCTpjO7R\ntZiDZUGEQ5JlwH8EPgQcBr6eZG9VPTO/I5OkmZvNmz8OO2gWRDgAVwGjVfUdgCQPApsBw0GSGP5l\nwQslHFYDL/W9Pgz8g2EPwls8S1LPQgmHgSTZDmxvL/88yXMz3NXFwHdnZ1SLhnNeGpzzEpDbzmjO\nf3OQjRZKOBwBLul7vabVTlNVdwN3n+kPS3Kwqjac6X4WE+e8NDjnpWEYc14of+zn68C6JJcmeTuw\nBdg7z2OSpCVrQawcqupkkn8FfJnepayfrqpD8zwsSVqyFkQ4AFTVF4EvDunHnfGhqUXIOS8Nznlp\nmPM5p6rm+mdIkhaZhXLOQZK0gJzV4ZBkU5Lnkowm2THB+0lyR3v/m0nePx/jnE0DzPmftrk+neSP\nkrxnPsY5m6aac992fz/JySTXDXN8c2GQOSf5YJKnkhxK8j+GPcbZNMD/6/OT/Lck32jz/dh8jHM2\nJfl0kmNJvjXJ+3P7+6uqzsoHvRPbfwr8LeDtwDeAy8dt8xHgUSDA1cCB+R73EOb8D4ELWvvDS2HO\nfds9Tu+81nXzPe4h/Du/k94dBta21z813+Oe4/n+BnBba68AXgXePt9jP8N5/yzwfuBbk7w/p7+/\nzuaVww9vyVFVfwmM3ZKj32bg/urZD7wzyaphD3QWTTnnqvqjqnqtvdxP7zsli9kg/84Avwr8PnBs\nmIObI4PM+Z8An6uqPwOoqsU870HmW8BPJgnwE/TC4eRwhzm7quqr9OYxmTn9/XU2h8NEt+RYPYNt\nFpPpzmcbvU8ei9mUc06yGvjHwF1DHNdcGuTf+e8AFyT5SpInk9w4tNHNvkHm+zvA3wX+L/A08PGq\n+sFwhjdv5vT314K5lFXDleTn6IXDB+Z7LEPwW8AnquoHvQ+WS8JyYD2wETgX+FqS/VX1f+Z3WHPm\nGuAp4OeBvw08luR/VtUb8zusxetsDodBbskx0G07FpGB5pPk7wH3AB+uqhNDGttcGWTOG4AHWzBc\nDHwkycmq+q/DGeKsG2TOh4ETVfV94PtJvgq8B1iM4TDIfD8G7KrewfjRJM8DPw08MZwhzos5/f11\nNh9WGuSWHHuBG9tZ/6uB16vq6LAHOoumnHOStcDngF85Sz5FTjnnqrq0qkaqagR4GLh5EQcDDPZ/\n+xHgA0mWJ/kb9O5y/OyQxzlbBpnvn9FbJZFkJfBu4DtDHeXwzenvr7N25VCT3JIjyb9s7/8neleu\nfAQYBf6C3qePRWvAOf9b4CLgzvZJ+mQt4puWDTjns8ogc66qZ5N8Cfgm8AN6f11xwksiF7oB/43/\nA3BfkqfpXb3ziapa1HdqTfJZ4IPAxUkOA78JvA2G8/vLb0hLkjrO5sNKkqQZMhwkSR2GgySpw3CQ\nJHUYDpKkDsNBktRhOEiSOgwHSVLH/wcQK1rhseID6QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc729222650>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ici_ger=Gervais.select(\"ici\").rdd.flatMap(lambda x: x).collect()\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "plt.hist([x for x in ici_ger if x], bins=100, range=(0, 1))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFIZJREFUeJzt3XGs3eV93/H3p5AwthRCwLU8AzMZ7jaDFlLfedYaTbSo\ni5NWMpEIcTYFVFm4EyyhUv8o5I+l04QEfzRs2QqbWxAGtSEWSYfXQKMU2qZTY+ilIoChrF4xxZ4D\nDjBoO4XF5rs/zmNyfH/X3ON7zz33nnveL+mn+zvP+T2/8zy69vmc53l+53dTVUiS1O9HlroBkqTl\nx3CQJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqeP0pW7AfJ133nm1bt26pW6GJI2V\nJ5544ntVtWqu48Y2HNatW8f09PRSN0OSxkqSFwc5zmklSVKH4SBJ6jAcJEkdhoMkqcNwkCR1GA6S\npA7DQZLUYThIkjrmDIckfyvJ40m+k2Rfkn/Xyj+Q5JtJ/rz9PKevzs1J9id5PslH+8o3Jnm6Pfel\nJGnlZyT5Sit/LMm64XdVkjSoQUYObwE/XVUfAi4DtiTZDNwEPFJV64FH2mOSbAC2AZcAW4A7kpzW\nznUncB2wvm1bWvl24PWquhi4HbhtCH2Txsa6m77+ziYtB3OGQ/X8dXv4nrYVsBXY1cp3AVe2/a3A\n/VX1VlW9AOwHNiVZA5xVVXurqoB7Z9Q5fq4HgCuOjyokSaM30L2V2if/J4CLgV+rqseSrK6qw+2Q\n7wKr2/5aYG9f9YOt7Adtf2b58TovAVTV0SRvAOcC35vRjh3ADoALL7xwkKZLy5ajBC1nAy1IV9Wx\nqroMOJ/eKODSGc8XvdHEoqqqnVU1VVVTq1bNeVNBSdI8ndLVSlX1f4Dfp7dW8HKbKqL9fKUddgi4\noK/a+a3sUNufWX5CnSSnA2cDr55K2yRJwzPI1Uqrkry/7Z8J/AzwZ8Ae4Np22LXAg21/D7CtXYF0\nEb2F58fbFNSbSTa39YRrZtQ5fq6rgEfbaESStAQGWXNYA+xq6w4/Auyuqt9J8m1gd5LtwIvA1QBV\ntS/JbuBZ4ChwQ1Uda+e6HrgHOBN4uG0AdwH3JdkPvEbvaidpIvWvRRy49WeXsCWaZHOGQ1U9BXx4\nlvJXgStOUucW4JZZyqeBS2cp/z7wyQHaK0kaAb8hLUnqGNs/EyqNIy9f1bhw5CBJ6jAcJEkdhoMk\nqcNwkCR1GA6SpA7DQZLUYThIkjr8noO0yBby3QZvpaGl4shBktRhOEiSOgwHSVKH4SBJ6jAcJEkd\nhoMkqcNwkCR1GA6SpA7DQZLU4TekpSHzr71pJXDkIEnqMBwkSR2GgySpw3CQJHUYDpKkjjnDIckF\nSX4/ybNJ9iW5sZX/SpJDSZ5s28f76tycZH+S55N8tK98Y5Kn23NfSpJWfkaSr7Tyx5KsG35XpfG2\n7qavv7NJi22QkcNR4JeqagOwGbghyYb23O1VdVnbHgJoz20DLgG2AHckOa0dfydwHbC+bVta+Xbg\n9aq6GLgduG3hXZMkzdec4VBVh6vqT9v+XwHPAWvfpcpW4P6qequqXgD2A5uSrAHOqqq9VVXAvcCV\nfXV2tf0HgCuOjyokSaN3SmsObbrnw8BjreizSZ5KcneSc1rZWuClvmoHW9natj+z/IQ6VXUUeAM4\n91TaJkkanoHDIcn7gK8Cv1hVb9KbIvogcBlwGPjVRWnhiW3YkWQ6yfSRI0cW++UkaWINFA5J3kMv\nGH6zqr4GUFUvV9Wxqnob+HVgUzv8EHBBX/XzW9mhtj+z/IQ6SU4HzgZendmOqtpZVVNVNbVq1arB\neihJOmWDXK0U4C7guar6Yl/5mr7DPgE80/b3ANvaFUgX0Vt4fryqDgNvJtncznkN8GBfnWvb/lXA\no21dQpK0BAa58d5PAp8Bnk7yZCv7PPDpJJcBBRwAfgGgqvYl2Q08S+9Kpxuq6lirdz1wD3Am8HDb\noBc+9yXZD7xG72onaWx4ealWmjnDoar+BzDblUMPvUudW4BbZimfBi6dpfz7wCfnaoskaTT8hrQk\nqcNwkCR1GA6SpA7DQZLUYThIkjr8G9LSGOq/dPbArT+7hC3RSuXIQZLUYThIkjoMB0lSh+EgSeow\nHCRJHYaDJKnDcJAkdfg9B2mevE23VjJHDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqcPv\nOUhjzr/toMXgyEGS1GE4SJI6DAdJUsecaw5JLgDuBVYDBeysqv+Y5APAV4B1wAHg6qp6vdW5GdgO\nHAM+V1XfaOUbgXuAM4GHgBurqpKc0V5jI/Aq8KmqOjC0XkpD4v2UNCkGGTkcBX6pqjYAm4EbkmwA\nbgIeqar1wCPtMe25bcAlwBbgjiSntXPdCVwHrG/blla+HXi9qi4GbgduG0LfJEnzNGc4VNXhqvrT\ntv9XwHPAWmArsKsdtgu4su1vBe6vqreq6gVgP7ApyRrgrKraW1VFb6TQX+f4uR4ArkiSBfdOkjQv\np7TmkGQd8GHgMWB1VR1uT32X3rQT9ILjpb5qB1vZ2rY/s/yEOlV1FHgDOPdU2iZJGp6BwyHJ+4Cv\nAr9YVW/2P9dGAjXkts3Whh1JppNMHzlyZLFfTpIm1kDhkOQ99ILhN6vqa6345TZVRPv5Sis/BFzQ\nV/38Vnao7c8sP6FOktOBs+ktTJ+gqnZW1VRVTa1atWqQpkuS5mHOcGhz/3cBz1XVF/ue2gNc2/av\nBR7sK9+W5IwkF9FbeH68TUG9mWRzO+c1M+ocP9dVwKNtNCJJWgKD3D7jJ4HPAE8nebKVfR64Fdid\nZDvwInA1QFXtS7IbeJbelU43VNWxVu96fngp68Ntg1743JdkP/AavaudJElLJOP6AX1qaqqmp6eX\nuhmaMOP0PQfvs6TZJHmiqqbmOs5vSEuSOrwrqzSHcRotSMPiyEGS1GE4SJI6DAdJUofhIEnqMBwk\nSR2GgySpw3CQJHX4PQdpher/fobfltapcuQgSeowHCRJHU4raWhOdpuJcZzS8JYZmnSOHCRJHYaD\nJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUoffc5AmgLfS0Kly5CBJ6jAcJEkdhoMkqcM1B82b9x+S\nVq45wyHJ3cDPAa9U1aWt7FeA64Aj7bDPV9VD7bmbge3AMeBzVfWNVr4RuAc4E3gIuLGqKskZwL3A\nRuBV4FNVdWBI/dMyMC6LoYad9EODTCvdA2yZpfz2qrqsbceDYQOwDbik1bkjyWnt+DvpBcr6th0/\n53bg9aq6GLgduG2efZEkDcmc4VBV3wJeG/B8W4H7q+qtqnoB2A9sSrIGOKuq9lZV0RspXNlXZ1fb\nfwC4IklOpROSpOFayIL0Z5M8leTuJOe0srXAS33HHGxla9v+zPIT6lTVUeAN4NwFtEuStEDzDYc7\ngQ8ClwGHgV8dWoveRZIdSaaTTB85cmTuCpI61t309Xc26WTmFQ5V9XJVHauqt4FfBza1pw4BF/Qd\nen4rO9T2Z5afUCfJ6cDZ9BamZ3vdnVU1VVVTq1atmk/TJUkDmFc4tDWE4z4BPNP29wDbkpyR5CJ6\nC8+PV9Vh4M0km9t6wjXAg311rm37VwGPtnUJSdISGeRS1i8DlwPnJTkIfAG4PMllQAEHgF8AqKp9\nSXYDzwJHgRuq6lg71fX88FLWh9sGcBdwX5L99Ba+tw2jY5Kk+ZszHKrq07MU3/Uux98C3DJL+TRw\n6Szl3wc+OVc7JEmj4+0zJEkd3j5DE80rdqTZGQ7SBBuXW5to9JxWkiR1GA6SpA6nlXRKnKOXJoMj\nB0lShyMHTRxHP9LcDAdJgFcu6UROK0mSOhw5aKT8dCqNB8PhJHwTkzTJnFaSJHUYDpKkDqeVJHU4\nrSpHDpKkDsNBktRhOEiSOlxz0ETwlhnSqXHkIEnqMBwkSR1OK2lOTslMNi9rnUyOHCRJHYaDJKnD\naSWtWE6HSfM3ZzgkuRv4OeCVqrq0lX0A+AqwDjgAXF1Vr7fnbga2A8eAz1XVN1r5RuAe4EzgIeDG\nqqokZwD3AhuBV4FPVdWBofVQy5Zz2ePH39nkGGRa6R5gy4yym4BHqmo98Eh7TJINwDbgklbnjiSn\ntTp3AtcB69t2/Jzbgder6mLgduC2+XZGkjQcc4ZDVX0LeG1G8VZgV9vfBVzZV35/Vb1VVS8A+4FN\nSdYAZ1XV3qoqeiOFK2c51wPAFUky3w5JkhZuvmsOq6vqcNv/LrC67a8F9vYdd7CV/aDtzyw/Xucl\ngKo6muQN4Fzge/Nsm6QRcIppZVvw1UptJFBDaMuckuxIMp1k+siRI6N4SUmaSPMdObycZE1VHW5T\nRq+08kPABX3Hnd/KDrX9meX9dQ4mOR04m97CdEdV7QR2AkxNTY0kkObipydJK9F8w2EPcC1wa/v5\nYF/5byX5IvB36S08P15Vx5K8mWQz8BhwDfCfZpzr28BVwKNtNKIl5GWg0mQb5FLWLwOXA+clOQh8\ngV4o7E6yHXgRuBqgqvYl2Q08CxwFbqiqY+1U1/PDS1kfbhvAXcB9SfbTW/jeNpSeaSIZatJwzBkO\nVfXpkzx1xUmOvwW4ZZbyaeDSWcq/D3xyrnYspZlvOE4fSSdyenXl8RvSQ+R/EEkrheEwD05dDJ/B\nKi0vhoOkoTLoVwbDQWPPkZw0fN6yW5LU4chB7/ATuIbNKabxZTho2fENRVp6hsMiGZc3OEcLGhW/\nLzReDActaycLWUNNWlyGwwiMyyhiuTMQpNExHCaQb7KS5mI4jNjJ3pgdUWjSOKJe3gyHZcj/NJKW\nmuGwTCz2VI9TSZJOheGwzA0yivCKHo07R8vLj+EwRgZ5szcQJA2D91aSJHU4cpC0rDjFtDw4cpAk\ndThykLRsOYpYOo4cJEkdhoMkqcNpJUljwSmm0XLkIEnqcOQgaew4ilh8Cxo5JDmQ5OkkTyaZbmUf\nSPLNJH/efp7Td/zNSfYneT7JR/vKN7bz7E/ypSRZSLskSQszjGmln6qqy6pqqj2+CXikqtYDj7TH\nJNkAbAMuAbYAdyQ5rdW5E7gOWN+2LUNol6QJsO6mr7+zaXgWY81hK7Cr7e8Cruwrv7+q3qqqF4D9\nwKYka4CzqmpvVRVwb18dSdISWOiaQwG/l+QY8F+raiewuqoOt+e/C6xu+2uBvX11D7ayH7T9meWS\ndEpcixiehYbDR6rqUJIfA76Z5M/6n6yqSlILfI13JNkB7AC48MILh3VaSdIMC5pWqqpD7ecrwG8D\nm4CX21QR7ecr7fBDwAV91c9vZYfa/szy2V5vZ1VNVdXUqlWrFtJ0SdK7mHc4JPk7SX70+D7wL4Bn\ngD3Ate2wa4EH2/4eYFuSM5JcRG/h+fE2BfVmks3tKqVr+upI0ry4UL0wC5lWWg38drvq9HTgt6rq\nd5P8CbA7yXbgReBqgKral2Q38CxwFLihqo61c10P3AOcCTzcNknSEknvAqHxMzU1VdPT04t2fj9t\nSCvTpC9UJ3mi76sHJ+XtMyRJHd4+Q9JE8XLXwThykCR1OHKQNLEcRZycIwdJUofhIEnqcFpJknCK\naSbDQZJmMCicVpIkzcKRgyS9i0kdRRgOkjSgSQoKp5UkSR2OHCRpHlb6KMJwkKQFWolB4bSSJKnD\nkYMkDdFKGUUYDpK0SMY5KAwHSRqBcQsKw0GSRmwcgsJwkKQl9G5/r34pg8NwkKRl6mTBMYrQ8FJW\nSVKH4SBJ6jAcJEkdhoMkqWPZhEOSLUmeT7I/yU1L3R5JmmTLIhySnAb8GvAxYAPw6SQblrZVkjS5\nlsulrJuA/VX1FwBJ7ge2As+OshHvdr2xJE2SZTFyANYCL/U9PtjKJElLYLmMHAaSZAewoz386yTP\nz/NU5wHfG06rxoZ9ngz2eQLktgX1+e8NctByCYdDwAV9j89vZSeoqp3AzoW+WJLpqppa6HnGiX2e\nDPZ5Moyiz8tlWulPgPVJLkryXmAbsGeJ2yRJE2tZjByq6miSfwN8AzgNuLuq9i1xsyRpYi2LcACo\nqoeAh0b0cguemhpD9nky2OfJsOh9TlUt9mtIksbMcllzkCQtIys6HOa6JUd6vtSefyrJTyxFO4dp\ngD7/q9bXp5P8cZIPLUU7h2nQW68k+SdJjia5apTtWwyD9DnJ5UmeTLIvyR+Ouo3DNMC/67OT/Pck\n32n9/fmlaOcwJbk7yStJnjnJ84v7/lVVK3Kjt7D9v4APAu8FvgNsmHHMx4GHgQCbgceWut0j6PM/\nA85p+x+bhD73HfcovXWtq5a63SP4Pb+f3h0GLmyPf2yp273I/f08cFvbXwW8Brx3qdu+wH7/c+An\ngGdO8vyivn+t5JHDO7fkqKr/Bxy/JUe/rcC91bMXeH+SNaNu6BDN2eeq+uOqer093EvvOyXjbJDf\nM8Bnga8Cr4yycYtkkD7/S+BrVfWXAFU1zv0epL8F/GiSAO+jFw5HR9vM4aqqb9Hrx8ks6vvXSg6H\nQW7JsdJu23Gq/dlO75PHOJuzz0nWAp8A7hxhuxbTIL/nHwfOSfIHSZ5Ics3IWjd8g/T3PwP/CPjf\nwNPAjVX19miat2QW9f1r2VzKqtFK8lP0wuEjS92WEfgPwC9X1du9D5YT4XRgI3AFcCbw7SR7q+p/\nLm2zFs1HgSeBnwb+PvDNJH9UVW8ubbPG10oOh0FuyTHQbTvGyED9SfKPgd8APlZVr46obYtlkD5P\nAfe3YDgP+HiSo1X130bTxKEbpM8HgVer6m+Av0nyLeBDwDiGwyD9/Xng1upNxu9P8gLwD4HHR9PE\nJbGo718reVppkFty7AGuaav+m4E3qurwqBs6RHP2OcmFwNeAz6yQT5Fz9rmqLqqqdVW1DngAuH6M\ngwEG+7f9IPCRJKcn+dvAPwWeG3E7h2WQ/v4lvVESSVYD/wD4i5G2cvQW9f1rxY4c6iS35Ejyr9vz\n/4XelSsfB/YD/5fep4+xNWCf/y1wLnBH+yR9tMb4pmUD9nlFGaTPVfVckt8FngLeBn6jqma9JHK5\nG/B3/O+Be5I8Te/qnV+uqrG+U2uSLwOXA+clOQh8AXgPjOb9y29IS5I6VvK0kiRpngwHSVKH4SBJ\n6jAcJEkdhoMkqcNwkCR1GA6SpA7DQZLU8f8Bq87GPruQ6nAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc7292131d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ici_ger=Cuviers.select(\"ici\").rdd.flatMap(lambda x: x).collect()\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "plt.hist([x for x in ici_ger if x], bins=100, range=(0, 1))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD8CAYAAACLrvgBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFyxJREFUeJzt3X+snuV93/H3p3ZCaTKITTyL2Wy2hJUKkEKKZehSVVm8\n2u4SxfwBzJEyrM6DSbAtmTpVpvvDKsgSSFNpkQYSCi6GpgGPJsNKSphnUnWThuGQZCNAkM9CKPYA\nn3IcaDNBZvrdH891wuMTH5/r2Mc++Pj9kh491/297+t6rish55P7x/OQqkKSpB6/MNcTkCSdOQwN\nSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndFs71BGbbRz/60VqxYsVcT0OSzijP\nPPPMX1XVkumOm3ehsWLFCkZGRuZ6GpJ0Rknycs9xXp6SJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQk\nSd0MDUlSN0NDktTN0JAkdZt33wjX+9uKrd/8WftHt39mDmci6UR4piFJ6jZtaCT5WJLvDb3eSvKl\nJIuT7Emyv70vGupzS5LRJC8mWT9UvyLJs23fXUnS6uckebjV9yVZMdRnc/uM/Uk2z+7yJUkzMW1o\nVNWLVXV5VV0OXAH8X+DrwFZgb1WtAva2bZJcAmwCLgU2AHcnWdCGuwe4AVjVXhtafQtwuKouBu4E\n7mhjLQa2AVcCa4Btw+EkSTq9Znp5ai3wv6vqZWAjsLPVdwJXt/ZG4KGqeqeqXgJGgTVJLgTOq6on\nq6qAByb1mRjrEWBtOwtZD+ypqvGqOgzs4b2gkSSdZjMNjU3AV1t7aVW92tqvAUtbexnwylCfA622\nrLUn14/qU1VHgDeBC44zliRpDnSHRpIPAp8D/tPkfe3MoWZxXjOS5MYkI0lGxsbG5moakjTvzeRM\n4zeB71TV62379XbJifZ+qNUPAhcN9Vveagdbe3L9qD5JFgLnA28cZ6yjVNW9VbW6qlYvWTLtv3hK\nknSCZhIan+e9S1MAu4GJp5k2A48O1Te1J6JWMrjh/VS7lPVWkqva/YrrJ/WZGOsa4Il29vI4sC7J\nonYDfF2rSZLmQNeX+5J8CPgN4F8OlW8HdiXZArwMXAdQVc8l2QU8DxwBbq6qd1ufm4D7gXOBx9oL\n4D7gwSSjwDiDeydU1XiS24Cn23G3VtX4CaxTkjQLukKjqn7C4Mb0cO0NBk9THev47cD2Y9RHgMuO\nUX8buHaKsXYAO3rmKUk6tfxGuCSpm6EhSepmaEiSuhkakqRuhoYkqZuhIUnqZmhIkroZGpKkboaG\nJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuhIUnqZmhIkroZGpKkbl2h\nkeQjSR5J8oMkLyT51SSLk+xJsr+9Lxo6/pYko0leTLJ+qH5FkmfbvruSpNXPSfJwq+9LsmKoz+b2\nGfuTbJ69pUuSZqr3TOMPgW9V1S8DHwdeALYCe6tqFbC3bZPkEmATcCmwAbg7yYI2zj3ADcCq9trQ\n6luAw1V1MXAncEcbazGwDbgSWANsGw4nSdLpNW1oJDkf+HXgPoCq+mlV/RjYCOxsh+0Erm7tjcBD\nVfVOVb0EjAJrklwInFdVT1ZVAQ9M6jMx1iPA2nYWsh7YU1XjVXUY2MN7QSNJOs16zjRWAmPAHyX5\nbpIvJ/kQsLSqXm3HvAYsbe1lwCtD/Q+02rLWnlw/qk9VHQHeBC44zliSpDnQExoLgV8B7qmqTwA/\noV2KmtDOHGr2p9cnyY1JRpKMjI2NzdU0JGne6wmNA8CBqtrXth9hECKvt0tOtPdDbf9B4KKh/stb\n7WBrT64f1SfJQuB84I3jjHWUqrq3qlZX1eolS5Z0LEmSdCKmDY2qeg14JcnHWmkt8DywG5h4mmkz\n8Ghr7wY2tSeiVjK44f1Uu5T1VpKr2v2K6yf1mRjrGuCJdvbyOLAuyaJ2A3xdq0mS5sDCzuP+NfCV\nJB8Efgj8FoPA2ZVkC/AycB1AVT2XZBeDYDkC3FxV77ZxbgLuB84FHmsvGNxkfzDJKDDO4Okrqmo8\nyW3A0+24W6tq/ATXKkk6SV2hUVXfA1YfY9faKY7fDmw/Rn0EuOwY9beBa6cYawewo2eekqRTy2+E\nS5K6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuhIUnqZmhIkroZGpKkboaGJKmboSFJ6mZo\nSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuhIUnq1hUaSX6U5Nkk30sy0mqLk+xJsr+9\nLxo6/pYko0leTLJ+qH5FG2c0yV1J0urnJHm41fclWTHUZ3P7jP1JNs/WwiVJMzeTM41/VFWXV9Xq\ntr0V2FtVq4C9bZsklwCbgEuBDcDdSRa0PvcANwCr2mtDq28BDlfVxcCdwB1trMXANuBKYA2wbTic\nJEmn18lcntoI7GztncDVQ/WHquqdqnoJGAXWJLkQOK+qnqyqAh6Y1GdirEeAte0sZD2wp6rGq+ow\nsIf3gkaSdJr1hkYB/zXJM0lubLWlVfVqa78GLG3tZcArQ30PtNqy1p5cP6pPVR0B3gQuOM5YR0ly\nY5KRJCNjY2OdS5IkzdTCzuN+raoOJvm7wJ4kPxjeWVWVpGZ/en2q6l7gXoDVq1fP2Twkab7rOtOo\nqoPt/RDwdQb3F15vl5xo74fa4QeBi4a6L2+1g609uX5UnyQLgfOBN44zliRpDkwbGkk+lOTvTLSB\ndcD3gd3AxNNMm4FHW3s3sKk9EbWSwQ3vp9qlrLeSXNXuV1w/qc/EWNcAT7T7Ho8D65IsajfA17Wa\nJGkO9FyeWgp8vT0duxD4k6r6VpKngV1JtgAvA9cBVNVzSXYBzwNHgJur6t021k3A/cC5wGPtBXAf\n8GCSUWCcwdNXVNV4ktuAp9txt1bV+EmsV5J0EqYNjar6IfDxY9TfANZO0Wc7sP0Y9RHgsmPU3wau\nnWKsHcCO6eYpSTr1/Ea4JKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuh\nIUnqZmhIkroZGpKkboaGJKmboSFJ6tb7r3uVTtiKrd+c6ylImiWeaUiSuhkakqRuhoYkqZuhIUnq\n1h0aSRYk+W6Sb7TtxUn2JNnf3hcNHXtLktEkLyZZP1S/Ismzbd9dSdLq5yR5uNX3JVkx1Gdz+4z9\nSTbPxqIlSSdmJmcaXwReGNreCuytqlXA3rZNkkuATcClwAbg7iQLWp97gBuAVe21odW3AIer6mLg\nTuCONtZiYBtwJbAG2DYcTpKk06srNJIsBz4DfHmovBHY2do7gauH6g9V1TtV9RIwCqxJciFwXlU9\nWVUFPDCpz8RYjwBr21nIemBPVY1X1WFgD+8FjSTpNOs90/gD4HeAvx2qLa2qV1v7NWBpay8DXhk6\n7kCrLWvtyfWj+lTVEeBN4ILjjCVJmgPThkaSzwKHquqZqY5pZw41mxObiSQ3JhlJMjI2NjZX05Ck\nea/nTOOTwOeS/Ah4CPh0kj8GXm+XnGjvh9rxB4GLhvovb7WDrT25flSfJAuB84E3jjPWUarq3qpa\nXVWrlyxZ0rEkSdKJmDY0quqWqlpeVSsY3OB+oqq+AOwGJp5m2gw82tq7gU3tiaiVDG54P9UuZb2V\n5Kp2v+L6SX0mxrqmfUYBjwPrkixqN8DXtZokaQ6czG9P3Q7sSrIFeBm4DqCqnkuyC3geOALcXFXv\ntj43AfcD5wKPtRfAfcCDSUaBcQbhRFWNJ7kNeLodd2tVjZ/EnCVJJ2FGoVFVfw78eWu/Aayd4rjt\nwPZj1EeAy45Rfxu4doqxdgA7ZjJPSdKp4TfCJUndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS\n1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS\n1M3QkCR1mzY0kvxikqeS/M8kzyX5vVZfnGRPkv3tfdFQn1uSjCZ5Mcn6ofoVSZ5t++5KklY/J8nD\nrb4vyYqhPpvbZ+xPsnk2Fy9JmpmeM413gE9X1ceBy4ENSa4CtgJ7q2oVsLdtk+QSYBNwKbABuDvJ\ngjbWPcANwKr22tDqW4DDVXUxcCdwRxtrMbANuBJYA2wbDidJ0uk1bWjUwN+0zQ+0VwEbgZ2tvhO4\nurU3Ag9V1TtV9RIwCqxJciFwXlU9WVUFPDCpz8RYjwBr21nIemBPVY1X1WFgD+8FjSTpNOu6p5Fk\nQZLvAYcY/BHfByytqlfbIa8BS1t7GfDKUPcDrbastSfXj+pTVUeAN4ELjjOWJGkOdIVGVb1bVZcD\nyxmcNVw2aX8xOPuYE0luTDKSZGRsbGyupiFJ896Mnp6qqh8D32Zwiej1dsmJ9n6oHXYQuGio2/JW\nO9jak+tH9UmyEDgfeOM4Y02e171VtbqqVi9ZsmQmS5IkzUDP01NLknyktc8FfgP4AbAbmHiaaTPw\naGvvBja1J6JWMrjh/VS7lPVWkqva/YrrJ/WZGOsa4Il29vI4sC7JonYDfF2rSZLmwMKOYy4EdrYn\noH4B2FVV30jyP4BdSbYALwPXAVTVc0l2Ac8DR4Cbq+rdNtZNwP3AucBj7QVwH/BgklFgnMHTV1TV\neJLbgKfbcbdW1fjJLFiSdOKmDY2q+l/AJ45RfwNYO0Wf7cD2Y9RHgMuOUX8buHaKsXYAO6abpyTp\n1PMb4ZKkboaGJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRuhoYkqZuhIUnqZmhI\nkroZGpKkboaGJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuk0bGkkuSvLtJM8neS7JF1t9\ncZI9Sfa390VDfW5JMprkxSTrh+pXJHm27bsrSVr9nCQPt/q+JCuG+mxun7E/yebZXLwkaWZ6zjSO\nAL9dVZcAVwE3J7kE2ArsrapVwN62Tdu3CbgU2ADcnWRBG+se4AZgVXttaPUtwOGquhi4E7ijjbUY\n2AZcCawBtg2HkyTp9Jo2NKrq1ar6Tmv/NfACsAzYCOxsh+0Erm7tjcBDVfVOVb0EjAJrklwInFdV\nT1ZVAQ9M6jMx1iPA2nYWsh7YU1XjVXUY2MN7QSNJOs1mdE+jXTb6BLAPWFpVr7ZdrwFLW3sZ8MpQ\ntwOttqy1J9eP6lNVR4A3gQuOM9bked2YZCTJyNjY2EyWJEmage7QSPJh4E+BL1XVW8P72plDzfLc\nulXVvVW1uqpWL1myZK6mIUnzXldoJPkAg8D4SlV9rZVfb5ecaO+HWv0gcNFQ9+WtdrC1J9eP6pNk\nIXA+8MZxxpIkzYGep6cC3Ae8UFW/P7RrNzDxNNNm4NGh+qb2RNRKBje8n2qXst5KclUb8/pJfSbG\nugZ4op29PA6sS7Ko3QBf12qSpDmwsOOYTwL/DHg2yfda7XeB24FdSbYALwPXAVTVc0l2Ac8zePLq\n5qp6t/W7CbgfOBd4rL1gEEoPJhkFxhk8fUVVjSe5DXi6HXdrVY2f4FolSSdp2tCoqv8OZIrda6fo\nsx3Yfoz6CHDZMepvA9dOMdYOYMd085QknXp+I1yS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0\nJEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVI3Q0OS1M3QkCR1MzQkSd0MDUlSN0NDktTN0JAkdTM0\nJEndpg2NJDuSHEry/aHa4iR7kuxv74uG9t2SZDTJi0nWD9WvSPJs23dXkrT6OUkebvV9SVYM9dnc\nPmN/ks2ztWhJ0onpOdO4H9gwqbYV2FtVq4C9bZsklwCbgEtbn7uTLGh97gFuAFa118SYW4DDVXUx\ncCdwRxtrMbANuBJYA2wbDidJ0uk3bWhU1V8A45PKG4Gdrb0TuHqo/lBVvVNVLwGjwJokFwLnVdWT\nVVXAA5P6TIz1CLC2nYWsB/ZU1XhVHQb28PPhJUk6jU70nsbSqnq1tV8Dlrb2MuCVoeMOtNqy1p5c\nP6pPVR0B3gQuOM5YkqQ5ctI3wtuZQ83CXE5YkhuTjCQZGRsbm8upSNK8dqKh8Xq75ER7P9TqB4GL\nho5b3moHW3ty/ag+SRYC5wNvHGesn1NV91bV6qpavWTJkhNckiRpOicaGruBiaeZNgOPDtU3tSei\nVjK44f1Uu5T1VpKr2v2K6yf1mRjrGuCJdvbyOLAuyaJ2A3xdq0mS5sjC6Q5I8lXgU8BHkxxg8ETT\n7cCuJFuAl4HrAKrquSS7gOeBI8DNVfVuG+omBk9inQs81l4A9wEPJhllcMN9UxtrPMltwNPtuFur\navINeUnSaTRtaFTV56fYtXaK47cD249RHwEuO0b9beDaKcbaAeyYbo6SpNPDb4RLkroZGpKkboaG\nJKmboSFJ6mZoSJK6GRqSpG6GhiSpm6EhSepmaEiSuhkakqRu0/6MiH7eiq3f/Fn7R7d/Zg5nIkmn\nl6FxihgskuYjQ0NzxmCVzjze05AkdTM0JEndDA1JUjdDQ5LUzdCQJHUzNCRJ3QwNSVK3M+J7Gkk2\nAH8ILAC+XFW3z/GUNI3h72BImj/e92caSRYA/xH4TeAS4PNJLpnbWUnS2el9HxrAGmC0qn5YVT8F\nHgI2zvGcJOmsdCZcnloGvDK0fQC4co7mIvz5D+lsdiaExrSS3Ajc2Db/JsmLJzHcR4G/6v7sO2bn\nmDk2ozUPm621neb/jE54vWcw13x2OJk1/4Oeg86E0DgIXDS0vbzVfqaq7gXunY0PSzJSVatnY6wz\nxdm25rNtveCazxanY81nwj2Np4FVSVYm+SCwCdg9x3OSpLPS+/5Mo6qOJPlXwOMMHrndUVXPzfG0\nJOms9L4PDYCq+jPgz07Tx83KZa4zzNm25rNtveCazxanfM2pqlP9GZKkeeJMuKchSXqfMDSaJBuS\nvJhkNMnWuZ7PqZDkoiTfTvJ8kueSfLHVFyfZk2R/e18013OdTUkWJPlukm+07Xm9XoAkH0nySJIf\nJHkhya/O53Un+bftn+nvJ/lqkl+cj+tNsiPJoSTfH6pNuc4kt7S/aS8mWT8bczA0OKt+quQI8NtV\ndQlwFXBzW+dWYG9VrQL2tu355IvAC0Pb8329MPittm9V1S8DH2ew/nm57iTLgH8DrK6qyxg8MLOJ\n+bne+4ENk2rHXGf73/Ym4NLW5+72t+6kGBoDZ8VPlVTVq1X1ndb+awZ/SJYxWOvOdthO4Oq5meHs\nS7Ic+Azw5aHyvF0vQJLzgV8H7gOoqp9W1Y+Z3+teCJybZCHwS8D/YR6ut6r+AhifVJ5qnRuBh6rq\nnap6CRhl8LfupBgaA8f6qZJlczSX0yLJCuATwD5gaVW92na9Biydo2mdCn8A/A7wt0O1+bxegJXA\nGPBH7bLcl5N8iHm67qo6CPwH4C+BV4E3q+q/ME/XewxTrfOU/F0zNM5CST4M/Cnwpap6a3hfDR6n\nmxeP1CX5LHCoqp6Z6pj5tN4hC4FfAe6pqk8AP2HSpZn5tO52DX8jg7D8e8CHknxh+Jj5tN7jOR3r\nNDQGpv2pkvkiyQcYBMZXquprrfx6kgvb/guBQ3M1v1n2SeBzSX7E4JLjp5P8MfN3vRMOAAeqal/b\nfoRBiMzXdf9j4KWqGquq/wd8DfiHzN/1TjbVOk/J3zVDY+Cs+KmSJGFwnfuFqvr9oV27gc2tvRl4\n9HTP7VSoqluqanlVrWDw3+kTVfUF5ul6J1TVa8ArST7WSmuB55m/6/5L4Kokv9T+GV/L4H7dfF3v\nZFOtczewKck5SVYCq4CnTvbD/HJfk+SfMLj+PfFTJdvneEqzLsmvAf8NeJb3rvH/LoP7GruAvw+8\nDFxXVZNvtp3RknwK+HdV9dkkFzD/13s5g5v/HwR+CPwWg/+TOC/XneT3gH/K4AnB7wL/Avgw82y9\nSb4KfIrBr9m+DmwD/jNTrDPJvwf+OYP/XL5UVY+d9BwMDUlSLy9PSZK6GRqSpG6GhiSpm6EhSepm\naEiSuhkakqRuhoYkqZuhIUnq9v8BLJmhN87V/zoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc7292a9190>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ici_ger=Gervais.select(\"mode_ici\").rdd.flatMap(lambda x: x).collect()\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "plt.hist([x for x in ici_ger if x], bins=100, range=(0, 100))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD8CAYAAACLrvgBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFthJREFUeJzt3XGsnfV93/H3Z3ZDSCKIAc+itrNrDSuVQesSLMdtpgrN\nHfZKFPMHMEdLcVsXNEHXpMqU2ekfbIksgVaVFm0gWcHF0Aiw3GxYTWlimUbZ/sBwk7QDQxhXAYJd\ng28xg6oTpKbf/XF+lxzfXPs+3HPtY9/7fklH9znf5/k95/dzbu6H3/N7zjmpKiRJ6uIfDbsDkqRz\nh6EhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LU2cJhd2C2XXLJJTUyMjLsbkjS\nOeW73/3u31TV4umOm3OhMTIywujo6LC7IUnnlCQvdTnOy1OSpM4MDUlSZ4aGJKkzQ0OS1Nm0oZFk\nZ5KjSZ6eYt8XklSSS/pq25KMJXkuyfq++pVJnmr77kqSVj8vycOtfiDJSF+bzUmeb4/Ngw5WkjSY\nLjON+4ANk4tJlgNXAz/qq60CNgGXtzZ3J1nQdt8D3ASsbI+Jc24BXq+qy4A7gTvauS4CbgM+AawB\nbkuy6L0NT5I0m6YNjar6DnBsil13Al8E+r/6byPwUFW9XVUvAGPAmiSXAhdU1ePV+6rA+4Fr+9rs\natt7gHVtFrIe2FdVx6rqdWAfU4SXJOnMmdGaRpKNwOGq+qtJu5YCL/c9P9RqS9v25PoJbarqOPAG\ncPEpziVJGpL3/Oa+JB8AvkTv0tRZIcnNwM0AH/nIR4bcG0mau2byjvB/CqwA/qqtZS8DvpdkDXAY\nWN537LJWO9y2J9fpa3MoyULgQuC1Vr9qUptvT9WhqtoB7ABYvXp1TXWMdK4Y2fqNd7dfvP2aIfZE\n+mnv+fJUVT1VVf+4qkaqaoTeZaOPV9UrwF5gU7sjagW9Be8nquoI8GaStW294kbgkXbKvcDEnVHX\nAY+1dY9vAlcnWdQWwK9uNUnSkEw700jyIL3/4r8kySHgtqq6d6pjq+pgkt3AM8Bx4NaqeqftvoXe\nnVjnA4+2B8C9wANJxugtuG9q5zqW5CvAk+24L1fVVAvykqQzZNrQqKrPTLN/ZNLz7cD2KY4bBa6Y\nov4WcP1Jzr0T2DldHyVJZ4bvCJckdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSerM\n0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKk\nzgwNSVJn04ZGkp1JjiZ5uq/2X5L8IMn/TvLfk3y4b9+2JGNJnkuyvq9+ZZKn2r67kqTVz0vycKsf\nSDLS12ZzkufbY/NsDVqSNDNdZhr3ARsm1fYBV1TVPwP+D7ANIMkqYBNweWtzd5IFrc09wE3AyvaY\nOOcW4PWqugy4E7ijnesi4DbgE8Aa4LYki977ECVJs2Xa0Kiq7wDHJtW+VVXH29PHgWVteyPwUFW9\nXVUvAGPAmiSXAhdU1eNVVcD9wLV9bXa17T3AujYLWQ/sq6pjVfU6vaCaHF6SpDNoNtY0fgN4tG0v\nBV7u23eo1Za27cn1E9q0IHoDuPgU5/opSW5OMppkdHx8fKDBSJJObqDQSPK7wHHga7PTnZmpqh1V\ntbqqVi9evHiYXZGkOW3GoZHk14BPAf+2XXICOAws7ztsWasd5ieXsPrrJ7RJshC4EHjtFOeSJA3J\njEIjyQbgi8Cnq+r/9e3aC2xqd0StoLfg/URVHQHeTLK2rVfcCDzS12bizqjrgMdaCH0TuDrJorYA\nfnWrSZKGZOF0ByR5ELgKuCTJIXp3NG0DzgP2tTtnH6+qf1dVB5PsBp6hd9nq1qp6p53qFnp3Yp1P\nbw1kYh3kXuCBJGP0Ftw3AVTVsSRfAZ5sx325qk5YkJfmupGt33h3+8XbrxliT6SeaUOjqj4zRfne\nUxy/Hdg+RX0UuGKK+lvA9Sc5105g53R9lCSdGb4jXJLUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKk\nzgwNSVJnhoYkqTNDQ5LU2bTvCJd0+vV/XIh0NnOmIUnqzNCQJHVmaEiSOnNNYwb8uGoNg793Ohs4\n05AkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LU2bShkWRnkqNJnu6rXZRkX5Ln289Fffu2\nJRlL8lyS9X31K5M81fbdlSStfl6Sh1v9QJKRvjab22s8n2TzbA1akjQzXWYa9wEbJtW2AvuraiWw\nvz0nySpgE3B5a3N3kgWtzT3ATcDK9pg45xbg9aq6DLgTuKOd6yLgNuATwBrgtv5wkiSdedOGRlV9\nBzg2qbwR2NW2dwHX9tUfqqq3q+oFYAxYk+RS4IKqeryqCrh/UpuJc+0B1rVZyHpgX1Udq6rXgX38\ndHhJks6gma5pLKmqI237FWBJ214KvNx33KFWW9q2J9dPaFNVx4E3gItPca6fkuTmJKNJRsfHx2c4\nJEnSdAZeCG8zh5qFvgzShx1VtbqqVi9evHiYXZGkOW2mofFqu+RE+3m01Q8Dy/uOW9Zqh9v25PoJ\nbZIsBC4EXjvFuSRJQzLT0NgLTNzNtBl4pK++qd0RtYLegvcT7VLWm0nWtvWKGye1mTjXdcBjbfby\nTeDqJIvaAvjVrSZJGpJpPxo9yYPAVcAlSQ7Ru6PpdmB3ki3AS8ANAFV1MMlu4BngOHBrVb3TTnUL\nvTuxzgcebQ+Ae4EHkozRW3Df1M51LMlXgCfbcV+uqskL8pKkM2ja0Kiqz5xk17qTHL8d2D5FfRS4\nYor6W8D1JznXTmDndH2UJJ0ZfgmTTju/PEiaO/wYEUlSZ4aGJKkzQ0OS1JmhIUnqzNCQJHVmaEiS\nOjM0JEmdGRqSpM4MDUlSZ4aGJKkzQ0OS1JmhIUnqzNCQJHVmaEiSOjM0JEmdGRqSpM4MDUlSZ35z\nn3QO8tsQNSzONCRJnQ0UGkl+J8nBJE8neTDJ+5NclGRfkufbz0V9x29LMpbkuSTr++pXJnmq7bsr\nSVr9vCQPt/qBJCOD9FeSNJgZh0aSpcBvA6ur6gpgAbAJ2Arsr6qVwP72nCSr2v7LgQ3A3UkWtNPd\nA9wErGyPDa2+BXi9qi4D7gTumGl/JUmDG/Ty1ELg/CQLgQ8Afw1sBHa1/buAa9v2RuChqnq7ql4A\nxoA1SS4FLqiqx6uqgPsntZk41x5g3cQsRJJ05s04NKrqMPB7wI+AI8AbVfUtYElVHWmHvQIsadtL\ngZf7TnGo1Za27cn1E9pU1XHgDeDimfZZkjSYQS5PLaI3E1gB/CzwwSSf7T+mzRxqoB5268vNSUaT\njI6Pj5/ul5OkeWuQy1O/DLxQVeNV9ffA14FfBF5tl5xoP4+24w8Dy/vaL2u1w217cv2ENu0S2IXA\na5M7UlU7qmp1Va1evHjxAEOSJJ3KIKHxI2Btkg+0dYZ1wLPAXmBzO2Yz8Ejb3gtsandEraC34P1E\nu5T1ZpK17Tw3Tmozca7rgMfa7EWSNAQzfnNfVR1Isgf4HnAc+D6wA/gQsDvJFuAl4IZ2/MEku4Fn\n2vG3VtU77XS3APcB5wOPtgfAvcADScaAY/TuvpIkDclA7wivqtuA2yaV36Y365jq+O3A9inqo8AV\nU9TfAq4fpI+SpNnjO8IlSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0Z\nGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0N9M190ns1svUb726/ePs1Q+yJ\npJlwpiFJ6szQkCR15uUpaUj6L9VJ54qBZhpJPpxkT5IfJHk2yS8kuSjJviTPt5+L+o7flmQsyXNJ\n1vfVr0zyVNt3V5K0+nlJHm71A0lGBumvJGkwg16e+kPgz6vq54CfB54FtgL7q2olsL89J8kqYBNw\nObABuDvJgnaee4CbgJXtsaHVtwCvV9VlwJ3AHQP2V5I0gBlfnkpyIfBLwK8BVNWPgR8n2Qhc1Q7b\nBXwb+I/ARuChqnobeCHJGLAmyYvABVX1eDvv/cC1wKOtzX9q59oD/Nckqaqaab+lYfKSlM51g8w0\nVgDjwB8l+X6Sryb5ILCkqo60Y14BlrTtpcDLfe0PtdrStj25fkKbqjoOvAFcPECfJUkDGCQ0FgIf\nB+6pqo8Bf0e7FDWhzQhO+6wgyc1JRpOMjo+Pn+6Xk6R5a5DQOAQcqqoD7fkeeiHyapJLAdrPo23/\nYWB5X/tlrXa4bU+un9AmyULgQuC1yR2pqh1VtbqqVi9evHiAIUmSTmXGoVFVrwAvJ/loK60DngH2\nAptbbTPwSNveC2xqd0StoLfg/US7lPVmkrXtrqkbJ7WZONd1wGOuZ0jS8Az6Po1/D3wtyfuAHwK/\nTi+IdifZArwE3ABQVQeT7KYXLMeBW6vqnXaeW4D7gPPpLYA/2ur3Ag+0RfNj9O6+kiQNyUChUVV/\nCayeYte6kxy/Hdg+RX0UuGKK+lvA9YP0UZI0e/wYEUlSZ4aGJKkzQ0OS1JmhIUnqzNCQJHVmaEiS\nOjM0JEmdGRqSpM4MDUlSZ4aGJKkzQ0OS1JmhIUnqzNCQJHVmaEiSOjM0JEmdGRqSpM4MDUlSZ4aG\nJKkzQ0OS1JmhIUnqzNCQJHU2cGgkWZDk+0n+tD2/KMm+JM+3n4v6jt2WZCzJc0nW99WvTPJU23dX\nkrT6eUkebvUDSUYG7a8kaeZmY6bxOeDZvudbgf1VtRLY356TZBWwCbgc2ADcnWRBa3MPcBOwsj02\ntPoW4PWqugy4E7hjFvorSZqhgUIjyTLgGuCrfeWNwK62vQu4tq/+UFW9XVUvAGPAmiSXAhdU1eNV\nVcD9k9pMnGsPsG5iFiJJOvMGnWn8AfBF4B/6akuq6kjbfgVY0raXAi/3HXeo1Za27cn1E9pU1XHg\nDeDiyZ1IcnOS0SSj4+PjAw1IknRyMw6NJJ8CjlbVd092TJs51Exfo6uq2lFVq6tq9eLFi0/3y0nS\nvLVwgLafBD6d5FeA9wMXJPlj4NUkl1bVkXbp6Wg7/jCwvK/9slY73LYn1/vbHEqyELgQeG2APkuS\nBjDjmUZVbauqZVU1Qm+B+7Gq+iywF9jcDtsMPNK29wKb2h1RK+gteD/RLmW9mWRtW6+4cVKbiXNd\n117jtM9cpHPJyNZvvPuQTrdBZhonczuwO8kW4CXgBoCqOphkN/AMcBy4tareaW1uAe4DzgcebQ+A\ne4EHkowBx+iFkyRpSGYlNKrq28C32/ZrwLqTHLcd2D5FfRS4Yor6W8D1s9FHnVn+V680N/mOcElS\nZ4aGJKkzQ0OS1JmhIUnqzNCQJHVmaEiSOjM0JEmdGRqSpM5OxzvCpU763wD44u3XDLEnkrpypiFJ\n6szQkCR1ZmhIkjozNCRJnRkakqTODA1JUmeGhiSpM9+nIZ1mfiGV5hJnGpKkzgwNSVJnhoYkqbMZ\nh0aS5Un+IskzSQ4m+VyrX5RkX5Ln289FfW22JRlL8lyS9X31K5M81fbdlSStfl6Sh1v9QJKRmQ9V\nkjSoQWYax4EvVNUqYC1wa5JVwFZgf1WtBPa357R9m4DLgQ3A3UkWtHPdA9wErGyPDa2+BXi9qi4D\n7gTuGKC/kqQBzTg0qupIVX2vbf8t8CywFNgI7GqH7QKubdsbgYeq6u2qegEYA9YkuRS4oKoer6oC\n7p/UZuJce4B1E7MQSdKZNytrGu2y0ceAA8CSqjrSdr0CLGnbS4GX+5odarWlbXty/YQ2VXUceAO4\neDb6LEl67wYOjSQfAv4E+HxVvdm/r80catDX6NCHm5OMJhkdHx8/3S8nSfPWQKGR5GfoBcbXqurr\nrfxqu+RE+3m01Q8Dy/uaL2u1w217cv2ENkkWAhcCr03uR1XtqKrVVbV68eLFgwxJknQKg9w9FeBe\n4Nmq+v2+XXuBzW17M/BIX31TuyNqBb0F7yfapaw3k6xt57xxUpuJc10HPNZmL5KkIRjkY0Q+Cfwq\n8FSSv2y1LwG3A7uTbAFeAm4AqKqDSXYDz9C78+rWqnqntbsFuA84H3i0PaAXSg8kGQOO0bv7SpI0\nJDMOjar6X8DJ7mRad5I224HtU9RHgSumqL8FXD/TPkqSZpfvCJckdWZoSJI6MzQkSZ35fRqnSf93\nKLx4+zVD7InmK38HdTo405AkdeZMQ7PGb6iT5j5nGpKkzpxpSKeBsy7NVYaG9B65wKz5zNCQZsnZ\nMLs4G/qguc01DUlSZ4aGJKkzQ0OS1JmhIUnqzNCQJHXm3VNngLdoSporDA1pAN7iqvnGy1OSpM4M\nDUlSZ16eGpDrFbPPf1Pp7GVoDJF/HHWm+Lum2XJOhEaSDcAfAguAr1bV7UPu0jnBPxTTm7yQ7b+T\ndGpnfWgkWQD8N+BfAYeAJ5Psrapnhtszne1mMzS9S0rqOetDA1gDjFXVDwGSPARsBAyNZtA/js5I\nfsJ/C+nUzoXQWAq83Pf8EPCJIfXlnDVf/hjO5ozA2cXU5svvkqaWqhp2H04pyXXAhqr6zfb8V4FP\nVNVv9R1zM3Bze/pR4LkBXvIS4G8GaH8umm9jnm/jBcc8Xwwy5n9SVYunO+hcmGkcBpb3PV/Wau+q\nqh3Ajtl4sSSjVbV6Ns51rphvY55v4wXHPF+ciTGfC2/uexJYmWRFkvcBm4C9Q+6TJM1LZ/1Mo6qO\nJ/kt4Jv0brndWVUHh9wtSZqXzvrQAKiqPwP+7Ay93Kxc5jrHzLcxz7fxgmOeL077mM/6hXBJ0tnj\nXFjTkCSdJQyNJsmGJM8lGUuyddj9OR2SLE/yF0meSXIwyeda/aIk+5I8334uGnZfZ1OSBUm+n+RP\n2/M5PV6AJB9OsifJD5I8m+QX5vK4k/xO+51+OsmDSd4/F8ebZGeSo0me7quddJxJtrW/ac8lWT8b\nfTA0OOGjSv41sAr4TJJVw+3VaXEc+EJVrQLWAre2cW4F9lfVSmB/ez6XfA54tu/5XB8v9D6r7c+r\n6ueAn6c3/jk57iRLgd8GVlfVFfRumNnE3BzvfcCGSbUpx9n+v70JuLy1ubv9rRuIodHz7keVVNWP\ngYmPKplTqupIVX2vbf8tvT8kS+mNdVc7bBdw7XB6OPuSLAOuAb7aV56z4wVIciHwS8C9AFX146r6\nv8ztcS8Ezk+yEPgA8NfMwfFW1XeAY5PKJxvnRuChqnq7ql4Axuj9rRuIodEz1UeVLB1SX86IJCPA\nx4ADwJKqOtJ2vQIsGVK3Toc/AL4I/ENfbS6PF2AFMA78Ubss99UkH2SOjruqDgO/B/wIOAK8UVXf\nYo6OdwonG+dp+btmaMxDST4E/Anw+ap6s39f9W6nmxO31CX5FHC0qr57smPm0nj7LAQ+DtxTVR8D\n/o5Jl2bm0rjbNfyN9MLyZ4EPJvls/zFzabyncibGaWj0TPtRJXNFkp+hFxhfq6qvt/KrSS5t+y8F\njg6rf7Psk8Cnk7xI75Ljv0zyx8zd8U44BByqqgPt+R56ITJXx/3LwAtVNV5Vfw98HfhF5u54JzvZ\nOE/L3zVDo2defFRJktC7zv1sVf1+3669wOa2vRl45Ez37XSoqm1VtayqRuj9b/pYVX2WOTreCVX1\nCvByko+20jp6XyUwV8f9I2Btkg+03/F19Nbr5up4JzvZOPcCm5Kcl2QFsBJ4YtAX8819TZJfoXf9\ne+KjSrYPuUuzLsm/AP4n8BQ/ucb/JXrrGruBjwAvATdU1eTFtnNakquA/1BVn0pyMXN/vP+c3uL/\n+4AfAr9O7z8S5+S4k/xn4N/Qu0Pw+8BvAh9ijo03yYPAVfQ+zfZV4Dbgf3CScSb5XeA36P27fL6q\nHh24D4aGJKkrL09JkjozNCRJnRkakqTODA1JUmeGhiSpM0NDktSZoSFJ6szQkCR19v8BREFH7DBS\n07UAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc72913d810>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ici_ger=Cuviers.select(\"mode_ici\").rdd.flatMap(lambda x: x).collect()\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "plt.hist([x for x in ici_ger if x], bins=100, range=(0, 100))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Cuvier_projections=Cuviers.rdd.map(features)\n",
    "Gervais_projections=Gervais.rdd.map(features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adding Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.mllib.regression import LabeledPoint\n",
    "Cuvier_samples = Cuvier_projections.map(lambda features:LabeledPoint(1, features))\n",
    "Gervais_samples = Gervais_projections.map(lambda features:LabeledPoint(0, features))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sample Check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[LabeledPoint(1.0, [-214.951054829,18.6825179931,65.4334582605,4.30022217798,1.31022351216,9.59520608467,56.0,46.326])]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Cuvier_samples.take(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[LabeledPoint(0.0, [-222.609747255,-23.3386866141,108.94848139,1.08543350696,4.88404630163,9.91000418559,29.0,46.3157])]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Gervais_samples.take(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating Training and Test Data\n",
    "\n",
    "Splitting the randomly shuffled dataset in 80/20 split for traning and testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PythonRDD[207] at RDD at PythonRDD.scala:48"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "samples = Cuvier_samples.union(Gervais_samples)\n",
    "[training_data, test_data] = samples.randomSplit([0.8, 0.2])\n",
    "training_data.cache()\n",
    "test_data.cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining Score Metrics\n",
    "\n",
    "Accuracy is being used as a scoring function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def score(data, model):\n",
    "    predictions = model.predict(data.map(lambda x: x.features))\n",
    "    labels_and_preds = data.map(lambda x: x.label).zip(predictions)\n",
    "    accuracy = labels_and_preds.filter(lambda x: x[0] == x[1]).count() / float(data.count())\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Logistic Regression Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/spark/python/pyspark/mllib/classification.py:313: UserWarning: Deprecated in 2.0.0. Use ml.classification.LogisticRegression or LogisticRegressionWithLBFGS.\n",
      "  \"Deprecated in 2.0.0. Use ml.classification.LogisticRegression or \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy:  0.678841929423 , Test Accuracy:  0.678695035912\n"
     ]
    }
   ],
   "source": [
    "from pyspark.mllib.classification import LogisticRegressionWithSGD\n",
    "\n",
    "algo = LogisticRegressionWithSGD()\n",
    "model = algo.train(training_data)\n",
    "print \"Training Accuracy: \", score(training_data, model), \", Test Accuracy: \", score(test_data, model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVM Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy:  0.651530864259 , Test Accuracy:  0.651154099234\n"
     ]
    }
   ],
   "source": [
    "from pyspark.mllib.classification import SVMWithSGD\n",
    "\n",
    "algo = SVMWithSGD()\n",
    "model = algo.train(training_data)\n",
    "print \"Training Accuracy: \", score(training_data, model), \", Test Accuracy: \", score(test_data, model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy:  0.895139211223 , Test Accuracy:  0.894605324104\n"
     ]
    }
   ],
   "source": [
    "from pyspark.mllib.tree import DecisionTree\n",
    "\n",
    "algo = DecisionTree()\n",
    "model = algo.trainClassifier(training_data,numClasses=2,categoricalFeaturesInfo={})\n",
    "print \"Training Accuracy: \", score(training_data, model), \", Test Accuracy: \", score(test_data, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'DecisionTreeModel classifier of depth 5 with 33 nodes\\n  If (feature 6 <= 30.0)\\n   If (feature 6 <= 27.0)\\n    If (feature 1 <= -20.19542795232453)\\n     If (feature 0 <= -106.61047144163801)\\n      If (feature 6 <= 4.0)\\n       Predict: 1.0\\n      Else (feature 6 > 4.0)\\n       Predict: 0.0\\n     Else (feature 0 > -106.61047144163801)\\n      If (feature 6 <= 24.0)\\n       Predict: 0.0\\n      Else (feature 6 > 24.0)\\n       Predict: 1.0\\n    Else (feature 1 > -20.19542795232453)\\n     If (feature 6 <= 4.0)\\n      If (feature 1 <= -6.266544095925244)\\n       Predict: 1.0\\n      Else (feature 1 > -6.266544095925244)\\n       Predict: 1.0\\n     Else (feature 6 > 4.0)\\n      If (feature 6 <= 24.0)\\n       Predict: 1.0\\n      Else (feature 6 > 24.0)\\n       Predict: 1.0\\n   Else (feature 6 > 27.0)\\n    If (feature 1 <= -10.57972142009637)\\n     If (feature 6 <= 29.0)\\n      If (feature 6 <= 28.5)\\n       Predict: 0.0\\n      Else (feature 6 > 28.5)\\n       Predict: 0.0\\n     Else (feature 6 > 29.0)\\n      If (feature 1 <= -23.126932610621683)\\n       Predict: 0.0\\n      Else (feature 1 > -23.126932610621683)\\n       Predict: 0.0\\n    Else (feature 1 > -10.57972142009637)\\n     If (feature 6 <= 29.0)\\n      If (feature 6 <= 28.5)\\n       Predict: 0.0\\n      Else (feature 6 > 28.5)\\n       Predict: 0.0\\n     Else (feature 6 > 29.0)\\n      If (feature 0 <= -117.39776281856301)\\n       Predict: 0.0\\n      Else (feature 0 > -117.39776281856301)\\n       Predict: 1.0\\n  Else (feature 6 > 30.0)\\n   Predict: 1.0\\n'"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.toDebugString()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy:  0.889506722896 , Test Accuracy:  0.889257762413\n"
     ]
    }
   ],
   "source": [
    "from pyspark.mllib.tree import RandomForest\n",
    "\n",
    "algo = RandomForest()\n",
    "model = algo.trainClassifier(training_data,numClasses=2, maxDepth=3, categoricalFeaturesInfo={},numTrees=100)\n",
    "print \"Training Accuracy: \", score(training_data, model), \", Test Accuracy: \", score(test_data, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy:  0.888854911052 , Test Accuracy:  0.888724994185\n"
     ]
    }
   ],
   "source": [
    "from pyspark.mllib.tree import RandomForest\n",
    "\n",
    "algo = RandomForest()\n",
    "model = algo.trainClassifier(training_data,numClasses=2, categoricalFeaturesInfo={},numTrees=100)\n",
    "print \"Training Accuracy: \", score(training_data, model), \", Test Accuracy: \", score(test_data, model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GB Trees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy:  0.899980887467 , Test Accuracy:  0.899362467274\n"
     ]
    }
   ],
   "source": [
    "from pyspark.mllib.tree import GradientBoostedTrees\n",
    "\n",
    "algo = GradientBoostedTrees()\n",
    "model = algo.trainClassifier(training_data,categoricalFeaturesInfo={}, numIterations=100, maxDepth=3)\n",
    "print \"Training Accuracy: \", score(training_data, model), \", Test Accuracy: \", score(test_data, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy:  0.899980887467 , Test Accuracy:  0.899362467274\n"
     ]
    }
   ],
   "source": [
    "from pyspark.mllib.tree import GradientBoostedTrees\n",
    "\n",
    "algo = GradientBoostedTrees()\n",
    "model = algo.trainClassifier(training_data,categoricalFeaturesInfo={}, numIterations=100)\n",
    "print \"Training Accuracy: \", score(training_data, model), \", Test Accuracy: \", score(test_data, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}