Lol Data Addict.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LoL dataCrawler Script \"lol_addict\"\n",
    "\n",
    "### Created by Lucas Scoppio\n",
    "* Twitter @developercoppio\n",
    "* developercoppio@gmail.com \n",
    "\n",
    " Do what you want cuz a pirate is free! YARRR!!!\n",
    " \n",
    " But don't forget to be nice, the world is lacking nice people"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "from slugify import slugify\n",
    "from urllib import request\n",
    "from time import strptime, strftime\n",
    "import argparse\n",
    "import shelve\n",
    "import pickle\n",
    "import time\n",
    "import os\n",
    "\n",
    "BASE_PATH = os.path.dirname(os.path.abspath('__file__'))\n",
    "DOWNLOADS_PATH = os.path.join(BASE_PATH, 'downloads')\n",
    "if not os.path.isdir(DOWNLOADS_PATH):\n",
    "    print ('CREATING DOWNLOADS_PATH ({})'.format(DOWNLOADS_PATH))\n",
    "    os.mkdir(DOWNLOADS_PATH)\n",
    "    \n",
    "data_download_path = os.path.join(DOWNLOADS_PATH, \"lol_data\")\n",
    "VERBOSE = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_page(url):\n",
    "    \"\"\" loads a webpage into temporary file to a string and read it  and returns the string\"\"\"\n",
    "    local_filename = \"01.html\"\n",
    "    header = \"\"\n",
    "    html = \"\"\n",
    "    \n",
    "    try:\n",
    "        local_filename, headers = request.urlretrieve(url)                \n",
    "        print( \"Capturing page:\", url)\n",
    "    except Exception as e:\n",
    "        print (\"ERROR 405\", e)\n",
    "        return 0   \n",
    "    \n",
    "    html = open(local_filename, encoding=\"utf-8\")\n",
    "    \n",
    "    if VERBOSE:\n",
    "        print (\"oppening file\", local_filename, header,)    \n",
    "    \n",
    "    return html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class Team(object):    \n",
    "    def __init__(self, team_id=\"\", name=\"\", country=\"\", rank=0, rating=0, url=\"\"):\n",
    "        self.id_ = team_id\n",
    "        self.name = name\n",
    "        self.country = country\n",
    "        self.rank = rank\n",
    "        self.rating = rating\n",
    "        self.url = url\n",
    "    \n",
    "    def get_entry(self):\n",
    "        return \"{0} {1} from {2}, is rank {3} and rating {4}\".format(self.id_, self.name, self.country, self.rank, self.rating)\n",
    "\n",
    "    entry = property(get_entry)\n",
    "\n",
    "\n",
    "class Tournament(Team):    \n",
    "    def __init__(self, div, date, TeamBlue, TeamPurple, MPB, MPP, MR, TRB, TRP, FRR, RB, RP, W10B, L10B, W10P, L10P, HWB, HWP, GBB, GBP, BO, url):\n",
    "        \"\"\"\n",
    "        MPB/MPP = Match Points Blue/Purple\n",
    "        MR = Match Result\n",
    "        TRB/TRP = Team Rank Blue/Purple\n",
    "        FRR = First Round Result (B/P)\n",
    "        RB/RP = Rating of team Blue/Purple\n",
    "        W10B/W10P = Wins in the last 10 months team Blue/Purple\n",
    "        L10B/L10P = Losses in the last 10 months team Blue/Purple\n",
    "        HWB/HWP = Historic Wins by Blue/Purple against this opponent\n",
    "        GBB/GBP = GosuBet on Blue/Purple\n",
    "        BO = Best of 1, 3 or 5\n",
    "        url = url of the match\n",
    "        \"\"\"\n",
    "        self.div = div\n",
    "        self.date = date\n",
    "        self.TeamBlue = TeamBlue\n",
    "        self.TeamPurple = TeamPurple\n",
    "        self.MPB = MPB\n",
    "        self.MPP = MPP\n",
    "        self.MR = MR\n",
    "        self.TRB = TRB\n",
    "        self.TRP = TRP\n",
    "        self.FRR = FRR\n",
    "        self.RB = RB #team rating\n",
    "        self.RP = RP #team rating\n",
    "        self.W10B = W10B \n",
    "        self.L10B = L10B\n",
    "        self.W10P = W10P\n",
    "        self.L10P = L10P\n",
    "        self.HWB = HWB\n",
    "        self.HWP = HWP\n",
    "        self.GBB = GBB\n",
    "        self.GBP = GBP\n",
    "        self.BO = BO\n",
    "        self.url = url\n",
    "\n",
    "    def get_entry(self): \n",
    "        return {\"Tournament{0}, {1} vs {2}.\".format(self.div, self.TeamBlue, self.TeamPurple)}\n",
    "\n",
    "    def get_results(self):\n",
    "        if self.MR is \"B\":\n",
    "            return {\"Winner is {0}\".format(self.TeamBlue)}\n",
    "        else:\n",
    "            return {\"Winner is {0}\".format(self.TeamPurple)}\n",
    "\n",
    "    entry = property(get_entry)\n",
    "    result = property(get_results)\n",
    "    \n",
    "class TeamData(Team):    \n",
    "    def __init__(self, team_name, team_country, rank, rating, wins_last, losses_last, wins, draws, losses, players, heroes):\n",
    "        self.team_name = team_name\n",
    "        self.team_country = team_country\n",
    "        self.rank = rank\n",
    "        self.rating = rating\n",
    "        self.wins_last = wins_last\n",
    "        self.losses_last = losses_last\n",
    "        self.balance = wins_last-losses_last\n",
    "        self.wins = wins\n",
    "        self.draws = draws\n",
    "        self.losses = losses\n",
    "        if (wins+draws+losses) > 0:\n",
    "            self.win_percentage = (100.0 / (wins+draws+losses)) *wins\n",
    "            self.draw_percentage = (100.0 / (wins+draws+losses)) *draws\n",
    "            self.losses_percentage = (100.0 / (wins+draws+losses)) *losses\n",
    "        else:\n",
    "            self.win_percentage = 0.0\n",
    "            self.draw_percentage = 0.0\n",
    "            self.losses_percentage = 0.0\n",
    "        self.matches_played = wins+draws+losses\n",
    "        self.players = players\n",
    "        self.heroes = heroes\n",
    "    \n",
    "    def get_entry(self): \n",
    "        return {\"{0}, from {1}, has {2} players. Rank: {3} and rating {4}.\".format(self.team_name, self.team_country, len(self.players), self.rank, self.rating)}\n",
    "    \n",
    "    def get_results(selft):    \n",
    "        return {\"Win% {0:.2f}, draw% {1:.2f}, losses% {2:.2f}, matches played {3}\".format(self.win_percentage, self.draw_percentage, self.losses_percentage, self.matches_played)}\n",
    "\n",
    "    def get_chance(self): \n",
    "        return {\"win\": self.win_percentage, \"draw\": self.draw_percentage, \"losses\": self.losses_percentage, \"matches\": self.matches_played}\n",
    "    \n",
    "    def get_chance10(self): \n",
    "        return {\"win\":(100 / (self.wins_last+self.losses_last)) * self.wins_last , \"losses\": (100 / (self.wins_last+self.losses_last)) * self.losses_last, \"matches\": self.wins_last+self.losses_last}\n",
    "\n",
    "    def get_number_of_players(self): \n",
    "        return len(self.players)\n",
    "    \n",
    "    number_of_players = property(get_number_of_players)\n",
    "    entry = property(get_entry)\n",
    "    chance = property(get_chance)\n",
    "    chance10 = property(get_chance10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def save_obj(obj, name ):\n",
    "    with open(DOWNLOADS_PATH + \"/\" + name + '.pkl', 'wb') as f:\n",
    "        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "def load_obj(name ):\n",
    "    with open(DOWNLOADS_PATH + \"/\" + name + '.pkl', 'rb') as f:\n",
    "        return pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getData(url_root, url, callback=True):\n",
    "    # first it will drop \"http[s]://\" and \"index.html\", if present:\n",
    "    *rest, page_url = url.split(url_root)\n",
    "    if VERBOSE:\n",
    "        print( \"page url:\", page_url)\n",
    "        \n",
    "    if not os.path.isdir(data_download_path):\n",
    "        os.mkdir(data_download_path)\n",
    "        if VERBOSE:\n",
    "            print ('CREATING data_download_path ({})'.format(data_download_path))\n",
    "\n",
    "    resp = get_page(url_root+url)\n",
    "    soup = BeautifulSoup(resp, \"lxml\", from_encoding=\"UTF-8\")\n",
    "    links = []    \n",
    "    teams_set = {}  \n",
    "    try:\n",
    "        teams_set = load_obj (\"data_11\")\n",
    "    except:\n",
    "        print('nothing to load')\n",
    "    \n",
    "    if callback:\n",
    "        for link in soup.table.find_all(\"a\"):\n",
    "            links.append(link.get('href'))\n",
    "    \n",
    "        links = list(set(links))\n",
    "        links.remove(page_url)\n",
    "    \n",
    "        if len(links):\n",
    "            for link in links:\n",
    "                print (link)\n",
    "                    \n",
    "    rank = soup.table.find_all(\"tr\")\n",
    "\n",
    "    for entry in rank[:-1]:\n",
    "        try:            \n",
    "            TEAM_ID = entry[\"data-id\"]\n",
    "            TEAM_NAME = entry.find_all('span')[2].getText()\n",
    "            TEAM_COUNTRY = entry.find_all('span')[1]['title']\n",
    "            TEAM_RANK = int(entry.find('div').getText())\n",
    "            TEAM_RATING = int(entry.find(\"td\", class_=\"numbers\").getText().replace(\",\",\"\"))\n",
    "            print (TEAM_ID, TEAM_NAME + \",\", TEAM_COUNTRY,\"-\", TEAM_RANK, TEAM_RATING)\n",
    "            t_url = \"http://www.gosugamers.net/lol/teams/\"+entry[\"data-id\"]+\"-\"+slugify(TEAM_NAME)\n",
    "            \n",
    "            teams_set[TEAM_ID] = Team(TEAM_ID, TEAM_NAME, TEAM_COUNTRY, TEAM_RANK, TEAM_RATING, t_url)\n",
    "            \n",
    "            print (teams_set[TEAM_ID].url)\n",
    "            \n",
    "        except Exception as e:\n",
    "            print (\"ERROR 400\", e)\n",
    "            \n",
    "    print (\"set:\", len(teams_set))\n",
    "    \n",
    "    save_obj( teams_set, \"data_11\")\n",
    "    \n",
    "    if len(links):\n",
    "        for link in links:\n",
    "            getData(url_root, link, False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def getTeam(url_root, url, id_):\n",
    "    print( \"team id:\", id_)\n",
    "    if not os.path.isdir(data_download_path):\n",
    "        os.mkdir(data_download_path)\n",
    "        if VERBOSE:\n",
    "            print ('CREATING data_download_path ({})'.format(data_download_path))\n",
    "\n",
    "    resp = get_page(url)\n",
    "    if (resp == 0):\n",
    "        print(\"ERROR 402\", id_)\n",
    "        return 0\n",
    "    \n",
    "    soup = BeautifulSoup(resp, \"lxml\", from_encoding=\"UTF-8\")\n",
    "    team_data = {}\n",
    "    \n",
    "    try:\n",
    "        team_data = load_obj (\"data_12\")\n",
    "        if type(team_data[id_].wins) is not None:\n",
    "            if VERBOSE:\n",
    "                print(\"data loaded\")\n",
    "        else:\n",
    "            if VERBOSE:\n",
    "                print(\"data empty\")\n",
    "    except:\n",
    "        if VERBOSE:\n",
    "            print(\"nothing to load\")\n",
    "            \n",
    "    try:\n",
    "        score = soup.table.find_all(\"span\", class_='score')\n",
    "        if VERBOSE:\n",
    "            print (\"Team Stats:\", score[-3].getText(), score[-2].getText(), score[-1].getText())\n",
    "\n",
    "        #win_lose rate\n",
    "        gameScore = soup.find(\"div\", class_=\"months-wrap\")\n",
    "        wins = 0\n",
    "        losses = 0\n",
    "        rank_ = 0\n",
    "        \n",
    "        for i in gameScore.find_all(\"div\", class_=\"wins\"):\n",
    "            wins += int(i['style'].split(\":\")[1].split(\"px;\")[0]) / 12        \n",
    "        if VERBOSE:\n",
    "            print (\"wins %i\" % wins)        \n",
    "        for i in gameScore.find_all(\"div\", class_=\"losses\"):\n",
    "            losses += int(i['style'].split(\":\")[1].split(\"px;\")[0]) / 12        \n",
    "        if VERBOSE:\n",
    "            print (\"losses %i\" % losses)    \n",
    "            \n",
    "        header = soup.find(\"div\", class_=\"teamNameHolder\")\n",
    "        t_name = header.h1.getText().rstrip().lstrip().split(\" - \")[0]\n",
    "        t_country = header.div.span[\"title\"]\n",
    "        rating_ = int(soup.find_all(\"span\", class_=\"tooltip\")[0].getText().replace(\",\",\"\"))\n",
    "        print (rating_)\n",
    "        try:\n",
    "            rank_ = int(soup.find_all(\"span\", class_=\"number\")[1].getText())\n",
    "        except:\n",
    "            rank_ = int(soup.find_all(\"span\", class_=\"number\")[0].getText())\n",
    "        \n",
    "        if rank_ == 0:\n",
    "            return 0\n",
    "\n",
    "        if VERBOSE:\n",
    "            print (t_name, t_country, rank, rating)\n",
    "         \n",
    "        PLAYER_ROSTER = soup.find_all('div', class_=\"roster\")\n",
    "        #pegar match history pelo site, considerando que cada 12 pixels das barras são 1 vitória ou derrota        \n",
    "        for a in PLAYER_ROSTER:            \n",
    "            j = a.find_all(\"a\", class_=\"player\")            \n",
    "            player = []\n",
    "            heroes = []\n",
    "            for i in j:\n",
    "                try:\n",
    "                    heroes.append(i.find_all(\"img\", class_=\"icon\")[0]['alt'])\n",
    "                    heroes.append(i.find_all(\"img\", class_=\"icon\")[1]['alt'])\n",
    "                except:\n",
    "                    pass                \n",
    "                player.append(i['href'])\n",
    "\n",
    "            heroes = list(set(heroes))\n",
    "            team_data[id_] = TeamData(\n",
    "                                        team_name = t_name,\n",
    "                                        team_country = t_country,\n",
    "                                        rank = rank_,\n",
    "                                        rating = rating_,\n",
    "                                        wins_last = wins,\n",
    "                                        losses_last = losses,\n",
    "                                        wins = int(score[-3].getText()),\n",
    "                                        draws = int(score[-2].getText()), \n",
    "                                        losses = int(score[-1].getText()),\n",
    "                                        heroes = heroes,\n",
    "                                        players = player\n",
    "                                        )\n",
    "            \n",
    "            if VERBOSE:\n",
    "                print(team_data[id_].entry)\n",
    "                \n",
    "            #Com o player, devo pegar seus resultados com herois\n",
    "        #players = PLAYER_ROSTER.find_all(\"a\", class_=\"player\").getText()        \n",
    "    except Exception as e:\n",
    "        print (\"ERROR 401\", id_, e)\n",
    "        return 0\n",
    "    \n",
    "    print(team_data[id_].entry)\n",
    "    \n",
    "    save_obj( team_data, \"data_12\")\n",
    "    return -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getTeamData(url):\n",
    "    if not os.path.isdir(data_download_path):\n",
    "        os.mkdir(data_download_path)\n",
    "        if VERBOSE:\n",
    "            print ('CREATING data_download_path ({})'.format(data_download_path))\n",
    "\n",
    "    resp = get_page(url)\n",
    "    if (resp == 0):\n",
    "        print(\"getTeamData - ERROR 402\", url)\n",
    "        return 0\n",
    "    \n",
    "    soup = BeautifulSoup(resp, \"lxml\", from_encoding=\"UTF-8\")\n",
    "    team_data = {}\n",
    "    \n",
    "  \n",
    "    try:\n",
    "        score = soup.table.find_all(\"span\", class_='score')\n",
    "        if VERBOSE:\n",
    "            print (\"Team Stats:\", score[-3].getText(), score[-2].getText(), score[-1].getText())\n",
    "\n",
    "        #win_lose rate\n",
    "        gameScore = soup.find(\"div\", class_=\"months-wrap\")\n",
    "        wins = 0\n",
    "        losses = 0\n",
    "        rank_ = 0\n",
    "        \n",
    "        for i in gameScore.find_all(\"div\", class_=\"wins\"):\n",
    "            wins += int(i['style'].split(\":\")[1].split(\"px;\")[0]) / 12        \n",
    "        if VERBOSE:\n",
    "            print (\"wins %i\" % wins)        \n",
    "        for i in gameScore.find_all(\"div\", class_=\"losses\"):\n",
    "            losses += int(i['style'].split(\":\")[1].split(\"px;\")[0]) / 12        \n",
    "        if VERBOSE:\n",
    "            print (\"losses %i\" % losses)    \n",
    "            \n",
    "        header = soup.find(\"div\", class_=\"teamNameHolder\")\n",
    "        t_name = header.h1.getText().rstrip().lstrip().split(\" - \")[0]\n",
    "        t_country = header.div.span[\"title\"]\n",
    "        rating_ = int(soup.find_all(\"span\", class_=\"tooltip\")[0].getText().replace(\",\",\"\"))\n",
    "        print (rating_)\n",
    "        try:\n",
    "            rank_ = int(soup.find_all(\"span\", class_=\"number\")[1].getText())\n",
    "        except:\n",
    "            rank_ = int(soup.find_all(\"span\", class_=\"number\")[0].getText())\n",
    "        \n",
    "        if rank_ == 0:\n",
    "            return 0\n",
    "\n",
    "        if VERBOSE:\n",
    "            print (t_name, t_country, rank_, rating_)\n",
    "         \n",
    "        PLAYER_ROSTER = soup.find_all('div', class_=\"roster\")\n",
    "        #pegar match history pelo site, considerando que cada 12 pixels das barras são 1 vitória ou derrota        \n",
    "        for a in PLAYER_ROSTER:            \n",
    "            j = a.find_all(\"a\", class_=\"player\")            \n",
    "            player = []\n",
    "            heroes = []\n",
    "            for i in j:\n",
    "                try:\n",
    "                    heroes.append(i.find_all(\"img\", class_=\"icon\")[0]['alt'])\n",
    "                    heroes.append(i.find_all(\"img\", class_=\"icon\")[1]['alt'])\n",
    "                except:\n",
    "                    pass                \n",
    "                player.append(i['href'])\n",
    "\n",
    "            heroes = list(set(heroes))\n",
    "            \n",
    "            team_data = TeamData(\n",
    "                                        team_name = t_name,\n",
    "                                        team_country = t_country,\n",
    "                                        rank = rank_,\n",
    "                                        rating = rating_,\n",
    "                                        wins_last = wins,\n",
    "                                        losses_last = losses,\n",
    "                                        wins = int(score[-3].getText()),\n",
    "                                        draws = int(score[-2].getText()), \n",
    "                                        losses = int(score[-1].getText()),\n",
    "                                        heroes = heroes,\n",
    "                                        players = player\n",
    "                                        )\n",
    "            \n",
    "            if VERBOSE:\n",
    "                print(team_data.entry)                \n",
    "                \n",
    "            #Com o player, devo pegar seus resultados com herois\n",
    "        #players = PLAYER_ROSTER.find_all(\"a\", class_=\"player\").getText()        \n",
    "    except Exception as e:\n",
    "        print (\"ERROR 401\", e)\n",
    "        return 0\n",
    "    \n",
    "    return {\"wins\": team_data.wins_last, \"losses\": team_data.losses_last, \"rating\": team_data.rating ,\"heroes\" : team_data.heroes, \"players\": team_data.players, \"Total_Wins\": team_data.wins, \"Total_Losses\": team_data.losses}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#team = load_obj (\"data_12\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checking values inside the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DATA NOT FOUND\n",
      "Tournament_data - looking for url: http://www.gosugamers.net/lol/events/431-2016-na-lcs-spring\n",
      "Capturing page: http://www.gosugamers.net/lol/events/431-2016-na-lcs-spring\n",
      "oppening file C:\\Users\\usuario\\AppData\\Local\\Temp\\tmpzvp74umb \n",
      "<_io.TextIOWrapper name='C:\\\\Users\\\\usuario\\\\AppData\\\\Local\\\\Temp\\\\tmpzvp74umb' mode='r' encoding='utf-8'>\n",
      "http://www.gosugamers.net/lol/tournaments/8554-2016-na-lcs-spring/matches/98262-counter-logic-gaming-lol-vs-renegades-lol\n",
      "Capturing page: http://www.gosugamers.net/lol/tournaments/8554-2016-na-lcs-spring/matches/98262-counter-logic-gaming-lol-vs-renegades-lol\n",
      "oppening file C:\\Users\\usuario\\AppData\\Local\\Temp\\tmp8dv6xr5c \n",
      "Hist B/P: 0.0 100.0\n",
      "Capturing page: http://www.gosugamers.net/lol/teams/2799-counter-logic-gaming-lol\n",
      "oppening file C:\\Users\\usuario\\AppData\\Local\\Temp\\tmp53hbo_77 \n",
      "Team Stats: 124 0 101\n",
      "wins 27\n",
      "losses 15\n",
      "1221\n",
      "Counter Logic Gaming LoL United States 11 1221\n",
      "{'Counter Logic Gaming LoL, from United States, has 7 players. Rank: 11 and rating 1221.'}\n",
      "Capturing page: http://www.gosugamers.net/lol/teams/14253-renegades-lol\n",
      "oppening file C:\\Users\\usuario\\AppData\\Local\\Temp\\tmpdrc71z0r \n",
      "Team Stats: 7 0 13\n",
      "wins 7\n",
      "losses 13\n",
      "1052\n",
      "Renegades United States 33 1052\n",
      "{'Renegades, from United States, has 11 players. Rank: 33 and rating 1052.'}\n",
      "11 Counter Logic Gaming LoL 1 vs 0 33 Renegades - LoL - First round winner is B Odds are 0.0 100.0 best of 1\n",
      "http://www.gosugamers.net/lol/tournaments/8554-2016-na-lcs-spring/matches/98327-renegades-lol-vs-team-dignitas-lol\n",
      "Capturing page: http://www.gosugamers.net/lol/tournaments/8554-2016-na-lcs-spring/matches/98327-renegades-lol-vs-team-dignitas-lol\n",
      "oppening file C:\\Users\\usuario\\AppData\\Local\\Temp\\tmpgtwkirv1 \n",
      "Hist B/P: 100.0 0.0\n",
      "Capturing page: http://www.gosugamers.net/lol/teams/14253-renegades-lol\n",
      "oppening file C:\\Users\\usuario\\AppData\\Local\\Temp\\tmpsu7xxz06 \n",
      "Team Stats: 7 0 13\n",
      "wins 7\n",
      "losses 13\n",
      "1052\n",
      "Renegades United States 33 1052\n",
      "{'Renegades, from United States, has 11 players. Rank: 33 and rating 1052.'}\n",
      "Capturing page: http://www.gosugamers.net/lol/teams/2973-team-dignitas-lol\n",
      "oppening file C:\\Users\\usuario\\AppData\\Local\\Temp\\tmpie3z503x \n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-69-2e9dbf941771>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m    121\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    122\u001b[0m \u001b[0mVERBOSE\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 123\u001b[1;33m \u001b[0mTournament_Data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"http://www.gosugamers.net/lol/events/431-2016-na-lcs-spring\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"lcs-s-16\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mDEV\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-69-2e9dbf941771>\u001b[0m in \u001b[0;36mTournament_Data\u001b[1;34m(url, event_name, DEV, year)\u001b[0m\n\u001b[0;32m     91\u001b[0m             \u001b[0mopp_2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0ma\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"a\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"href\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     92\u001b[0m             \u001b[0mTEAM_BLUE\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgetTeamData\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mURL_ROOT\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mopp_1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 93\u001b[1;33m             \u001b[0mTEAM_PURPLE\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgetTeamData\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mURL_ROOT\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mopp_2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     94\u001b[0m             \u001b[1;31m#print (TEAM_BLUE)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     95\u001b[0m             \u001b[1;31m#print (TEAM_PURPLE)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-63-f2750fadd7fc>\u001b[0m in \u001b[0;36mgetTeamData\u001b[1;34m(url)\u001b[0m\n\u001b[0;32m     10\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 12\u001b[1;33m     \u001b[0msoup\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBeautifulSoup\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"lxml\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfrom_encoding\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"UTF-8\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     13\u001b[0m     \u001b[0mteam_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\lib\\site-packages\\bs4\\__init__.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, markup, features, builder, parse_only, from_encoding, exclude_encodings, **kwargs)\u001b[0m\n\u001b[0;32m    213\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    214\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 215\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_feed\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    216\u001b[0m                 \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    217\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mParserRejectedMarkup\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\lib\\site-packages\\bs4\\__init__.py\u001b[0m in \u001b[0;36m_feed\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    237\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuilder\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    238\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 239\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuilder\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfeed\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmarkup\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    240\u001b[0m         \u001b[1;31m# Close out any unfinished strings and close all the open tags.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    241\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mendData\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\lib\\site-packages\\bs4\\builder\\_lxml.py\u001b[0m in \u001b[0;36mfeed\u001b[1;34m(self, markup)\u001b[0m\n\u001b[0;32m    238\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    239\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparser\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparser_for\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mencoding\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 240\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparser\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfeed\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmarkup\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    241\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparser\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    242\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mUnicodeDecodeError\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mLookupError\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0metree\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mParserError\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mparser.pxi\u001b[0m in \u001b[0;36mlxml.etree._FeedParser.feed (src\\lxml\\lxml.etree.c:111941)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mparser.pxi\u001b[0m in \u001b[0;36mlxml.etree._FeedParser.feed (src\\lxml\\lxml.etree.c:111816)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mparsertarget.pxi\u001b[0m in \u001b[0;36mlxml.etree._TargetParserContext._handleParseResult (src\\lxml\\lxml.etree.c:128265)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mparsertarget.pxi\u001b[0m in \u001b[0;36mlxml.etree._TargetParserContext._handleParseResult (src\\lxml\\lxml.etree.c:128135)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mlxml.etree.pyx\u001b[0m in \u001b[0;36mlxml.etree._ExceptionContext._raise_if_stored (src\\lxml\\lxml.etree.c:10656)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32msaxparser.pxi\u001b[0m in \u001b[0;36mlxml.etree._handleSaxTargetStartNoNs (src\\lxml\\lxml.etree.c:120804)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32msaxparser.pxi\u001b[0m in \u001b[0;36mlxml.etree._callTargetSaxStart (src\\lxml\\lxml.etree.c:120998)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mparsertarget.pxi\u001b[0m in \u001b[0;36mlxml.etree._PythonSaxParserTarget._handleSaxStart (src\\lxml\\lxml.etree.c:127247)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\lib\\site-packages\\bs4\\builder\\_lxml.py\u001b[0m in \u001b[0;36mstart\u001b[1;34m(self, name, attrs, nsmap)\u001b[0m\n\u001b[0;32m    134\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnsmaps\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDEFAULT_NSMAPS\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    135\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 136\u001b[1;33m     \u001b[1;32mdef\u001b[0m \u001b[0mstart\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnsmap\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    137\u001b[0m         \u001b[1;31m# Make sure attrs is a mutable dict--lxml may send an immutable dictproxy.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    138\u001b[0m         \u001b[0mattrs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mattrs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "def Tournament_Data(url, event_name, DEV = True, year = 2016):\n",
    "    #Will capture information from game chance\n",
    "    tourney_data = {}    \n",
    "    #event_name = url.split(\"/\")[-1]\n",
    "    URL_ROOT = \"http://www.gosugamers.net\"\n",
    "    try:\n",
    "        tourney_data = load_obj (\"tournament_data\")\n",
    "        if len(tourney_data):\n",
    "            if VERBOSE:\n",
    "                print(\"data loaded\")\n",
    "        else:\n",
    "            if VERBOSE:\n",
    "                print(\"data empty\")\n",
    "    except:\n",
    "        if VERBOSE:\n",
    "            print(\"DATA NOT FOUND\")\n",
    "\n",
    "    print (\"Tournament_data - looking for url:\", url)\n",
    "    resp = get_page(url)\n",
    "    print (resp)\n",
    "    if (resp == 0):\n",
    "        print(\"ERROR 404\", url)\n",
    "        return 0\n",
    "    \n",
    "    soup = BeautifulSoup(resp, \"lxml\", from_encoding=\"UTF-8\")                        \n",
    "\n",
    "    all_urls = []\n",
    "    for link in soup.find(\"table\", class_=\"schedule\").find_all('a'):\n",
    "        all_urls.append(link.get('href'))\n",
    "        \n",
    "    i = 0\n",
    "    all_urls = list(set(all_urls))\n",
    "    for link in all_urls:       \n",
    "        if \"tournament\" in link:\n",
    "            if VERBOSE:\n",
    "                print (URL_ROOT+link)\n",
    "                \n",
    "            res = get_page(URL_ROOT+link)            \n",
    "            if (res == 0):\n",
    "                print(\"ERROR 402\", event_name)\n",
    "                return 0\n",
    "    \n",
    "            soup_match = BeautifulSoup(res, \"lxml\", from_encoding=\"UTF-8\")\n",
    "            #div\n",
    "            div = event_name\n",
    "            #TeamBlue TeamPurple\n",
    "            TeamBlue = soup_match.find(\"div\", class_=\"opponent1\").h3.getText()\n",
    "            TeamPurple = soup_match.find(\"div\", class_=\"opponent2\").h3.getText()\n",
    "            #date\n",
    "            date = soup_match.find(\"p\", class_=\"datetime\").getText()\n",
    "            date = str(year) + \" \" + date.strip()[:-5]\n",
    "            date = strptime(date, \"%Y %B %d, %A, %H:%M\") #\"January 16, Saturday, 21:00 CEST\"\n",
    "            #MPB MPP\n",
    "            MPB = soup_match.find(\"span\", class_=\"hidden results btn-2\").find_all(\"span\")[0].getText()\n",
    "            MPP = soup_match.find(\"span\", class_=\"hidden results btn-2\").find_all(\"span\")[1].getText()\n",
    "            \n",
    "            #TRB TRP\n",
    "            TRB = int(soup_match.find_all(\"p\", class_=\"ranked\")[0].getText().replace(\"Ranked #\", \"\"))\n",
    "            TRP = int(soup_match.find_all(\"p\", class_=\"ranked\")[1].getText().replace(\"Ranked #\", \"\"))\n",
    "            \n",
    "            #MR\n",
    "            MR = \"D\"\n",
    "            if MPB > MPP:\n",
    "                MR = \"B\"\n",
    "            elif MPP > MPB:\n",
    "                MR = \"P\"\n",
    "            \n",
    "            #FRR            \n",
    "            FRR = MR #can't get it right now                        \n",
    "\n",
    "            #Historic matches is a little bit adhoc            \n",
    "            home = len(soup_match.find_all(\"div\", class_=\"matchup away\"))\n",
    "            away = len(soup_match.find_all(\"div\", class_=\"matchup home\"))\n",
    "            \n",
    "            HWB = (100/(home+away))*home\n",
    "            HWP = (100/(home+away))*away\n",
    "            print(\"Hist B/P:\", HWB, HWP)\n",
    "            \n",
    "            #GosuBet\n",
    "            GBB = float(soup_match.find(\"div\", class_=\"bet-opp1\").span['val'])\n",
    "            GBP = float(soup_match.find(\"div\", class_=\"bet-opp2\").span['val'])\n",
    "\n",
    "            if (GBB < 1) and (GBP < 1):\n",
    "                GBB = HWB\n",
    "                GBP = HWP\n",
    "\n",
    "            #I need data from other games, so I'll take the \"all time winrate\" and last two months win-rate\n",
    "            a = soup_match.find(\"div\", class_=\"opponent1\")\n",
    "            opp_1 = a.find(\"a\")[\"href\"]\n",
    "            a = soup_match.find(\"div\", class_=\"opponent2\")\n",
    "            opp_2 = a.find(\"a\")[\"href\"]\n",
    "            TEAM_BLUE = getTeamData(URL_ROOT+opp_1)\n",
    "            TEAM_PURPLE = getTeamData(URL_ROOT+opp_2)\n",
    "            #print (TEAM_BLUE)\n",
    "            #print (TEAM_PURPLE)\n",
    "            \n",
    "            RB = TEAM_BLUE[\"rating\"]\n",
    "            RP = TEAM_PURPLE[\"rating\"]\n",
    "            W10B = TEAM_BLUE[\"wins\"]\n",
    "            L10B = TEAM_PURPLE[\"losses\"]\n",
    "            W10P = TEAM_BLUE[\"wins\"]\n",
    "            L10P = TEAM_PURPLE[\"losses\"]\n",
    "            \n",
    "            #Best of!\n",
    "            BO = int(soup_match.find(\"p\", class_=\"bestof\").getText().replace(\"Best of \", \"\"))\n",
    "            if VERBOSE:\n",
    "                print (TRB, TeamBlue, MPB ,\"vs\", MPP, TRP, TeamPurple, \"- First round winner is\", FRR, \"Odds are\", GBB, GBP, \"best of\", BO)\n",
    "                \n",
    "            tourney_data[i] = Tournament(\n",
    "                                        div, date, TeamBlue, TeamPurple, MPB, MPP, \n",
    "                                        MR, TRB, TRP, FRR, RB, RP, W10B, L10B, \n",
    "                                        W10P, L10P, HWB, HWP, GBB, GBP, BO, \n",
    "                                        \"http://www.gosugamers.net\"+link\n",
    "                                        )\n",
    "            i = i + 1\n",
    "\n",
    "    if not DEV:\n",
    "        save_obj( tourney_data, \"data_13\")\n",
    "    if VERBOSE:\n",
    "        print (\"fin\")\n",
    "\n",
    "VERBOSE = True        \n",
    "Tournament_Data(\"http://www.gosugamers.net/lol/events/431-2016-na-lcs-spring\", \"lcs-s-16\", DEV = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "+\n",
      "fin\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "\n",
    "with open('data/lol-2016/lcs-s-16.csv', 'w', newline='') as csvfile:\n",
    "    writer = csv.writer(csvfile, delimiter=' ',\n",
    "                            quotechar='\"', quoting=csv.QUOTE_MINIMAL)\n",
    "    \n",
    "    a = load_obj(\"data_13\")\n",
    "    fieldnames = [\n",
    "                  'div', 'data', 'TeamBlue', 'TeamPurple', 'MPB', 'MPP', \n",
    "                  'MR', 'TRB', 'TRP', 'FRR', 'RB', 'RP', 'W10B', 'L10B', \n",
    "                  'W10P', 'L10P', 'HWB', 'HWP', 'GBB', 'GBP', 'BO', 'url'\n",
    "                 ]\n",
    "    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n",
    "    writer.writeheader()\n",
    "    \n",
    "    for data in a:\n",
    "        writer.writerow({\n",
    "                         'div': a[data].div, 'data': strftime(\"%Y-%m-%d\", a[data].date), 'TeamBlue': a[data].TeamBlue, \n",
    "                         'TeamPurple': a[data].TeamPurple, 'MPB': a[data].MPB, 'MPP': a[data].MPP,\n",
    "                         'MR': a[data].MR, 'TRB': a[data].TRB, 'TRP': a[data].TRP, 'FRR': a[data].FRR,\n",
    "                         'RB': a[data].RB, 'RP': a[data].RP, 'W10B': a[data].W10B, 'L10B': a[data].L10B,\n",
    "                         'W10P': a[data].W10P, 'L10P': a[data].L10P, 'HWB': a[data].HWB, 'HWP': a[data].HWP, \n",
    "                         'GBB': a[data].GBB, 'GBP': a[data].GBP, 'BO': a[data].BO, 'url': a[data].url\n",
    "                        })\n",
    "        print (\"+\")\n",
    "    print (\"fin\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feature_columns = ['MPB', 'MPP', \n",
    "                  'MR', 'TRB', 'TRP', 'RB', 'RP', 'W10B', 'L10B', \n",
    "                  'W10P', 'L10P', 'HWB', 'HWP', 'GBB', 'GBP']"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}