notebooks/data_collection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mvfuskeNLVkb"
   },
   "source": [
    "Data Source: https://www.uci.org/mountain-bike/results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 579,
     "status": "ok",
     "timestamp": 1594887984073,
     "user": {
      "displayName": "Andrew Mackay",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GipvhJwlVpg1-ArYl_wNJCOJchIEp9jy27NBVfoXg=s64",
      "userId": "06804410358976473893"
     },
     "user_tz": -60
    },
    "id": "U4eU3G7qovF_"
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import json\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6136,
     "status": "ok",
     "timestamp": 1594887990690,
     "user": {
      "displayName": "Andrew Mackay",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GipvhJwlVpg1-ArYl_wNJCOJchIEp9jy27NBVfoXg=s64",
      "userId": "06804410358976473893"
     },
     "user_tz": -60
    },
    "id": "LJ3KZ6r7-Q5T"
   },
   "outputs": [],
   "source": [
    "URL_BASE = 'https://dataride.uci.ch/iframe/'\n",
    "\n",
    "# All competitions\n",
    "URL_COMPETITIONS = URL_BASE + 'Competitions/' \n",
    "\n",
    "# Races in a competition\n",
    "URL_RACES = URL_BASE + 'Races/'\n",
    "\n",
    "# Events in a race\n",
    "URL_EVENTS = URL_BASE + 'Events/'\n",
    "\n",
    "# Results for event\n",
    "URL_RESULTS = URL_BASE + 'Results/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 590,
     "status": "ok",
     "timestamp": 1594887992434,
     "user": {
      "displayName": "Andrew Mackay",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GipvhJwlVpg1-ArYl_wNJCOJchIEp9jy27NBVfoXg=s64",
      "userId": "06804410358976473893"
     },
     "user_tz": -60
    },
    "id": "yF4XA1jOAhhX"
   },
   "outputs": [],
   "source": [
    "DICIPLINE_ID_MOUNTAIN_BIKE = '7'\n",
    "\n",
    "RACE_TYPE_ID_DOWNHILL = '19'\n",
    "RACE_TYPE_ID_ENDURO = '122'\n",
    "RACE_TYPE_ID_CROSS_COUNTRY_OLYMPIC = '92'\n",
    "\n",
    "SEASON_ID_YEAR_MAP = {\n",
    "    2020: '129',\n",
    "    2019: '128',\n",
    "    2018: '123',\n",
    "    2017: '22',\n",
    "    2016: '12',\n",
    "    2015: '4',\n",
    "    2014: '102',\n",
    "    2013: '103',\n",
    "    2012: '104',\n",
    "    2011: '105',\n",
    "    2010: '106',\n",
    "    2009: '107',\n",
    "}\n",
    "\n",
    "COMPETITION_CLASS_CODE_WORLD_CHAMPS = 'CM'\n",
    "COMPETITION_CLASS_CODE_WORLD_CUP = 'CDM'\n",
    "COMPETITION_CLASS_CODE_ENDURO_WORLD_SERIES = '3'\n",
    "\n",
    "CATEGORY_CODE_MEN_ELITE = 'Men Elite'\n",
    "CATEGORY_CODE_WOMEN_ELITE = 'Women Elite'\n",
    "\n",
    "RACE_TYPE_CODE_DHI = 'DHI'\n",
    "RACE_TYPE_CODE_ENDURO = 'END'\n",
    "RACE_TYPE_CODE_XCO = 'XCO'\n",
    "\n",
    "RACE_TYPE_ID_TO_CODE_MAP = {\n",
    "    RACE_TYPE_ID_DOWNHILL: RACE_TYPE_CODE_DHI,\n",
    "    RACE_TYPE_ID_ENDURO: RACE_TYPE_CODE_ENDURO,\n",
    "    RACE_TYPE_ID_CROSS_COUNTRY_OLYMPIC: RACE_TYPE_CODE_XCO\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.mkdir('../data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GMtc79BKANvs"
   },
   "outputs": [],
   "source": [
    "def getCompetitions(race_type_id: str, year: int):\n",
    "    request_body = {\n",
    "        \"disciplineId\": DICIPLINE_ID_MOUNTAIN_BIKE,\n",
    "        \"take\":\"400\",\n",
    "        \"skip\":\"0\",\n",
    "        \"page\":\"1\",\n",
    "        \"pageSize\":\"400\",\n",
    "        \"sort\": [{\"field\": \"StartDate\", \"dir\": \"desc\"}],\n",
    "        \"filter\": {\n",
    "            \"filters\": [\n",
    "                {\"field\": \"RaceTypeId\", \"value\": race_type_id},\n",
    "                {\"field\": \"SeasonId\", \"value\": SEASON_ID_YEAR_MAP[year]}\n",
    "           ]\n",
    "        }\n",
    "    }\n",
    "    response = requests.post(URL_COMPETITIONS, json=request_body).json()\n",
    "    if len(response['data']) < response['total']:\n",
    "        print('DID NOT GET ALL COMPETITIONS')\n",
    "    return response['data']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Tc7oCjCpHm64"
   },
   "outputs": [],
   "source": [
    "def getRaces(competition_id: str):\n",
    "    request_body = {\n",
    "        \"disciplineId\": DICIPLINE_ID_MOUNTAIN_BIKE,\n",
    "        \"competitionId\": competition_id,\n",
    "        \"take\":\"400\",\n",
    "        \"skip\":\"0\",\n",
    "        \"page\":\"1\",\n",
    "        \"pageSize\":\"400\"\n",
    "    }\n",
    "    response = requests.post(URL_RACES, json=request_body).json()\n",
    "    if len(response['data']) < response['total']:\n",
    "        print('DID NOT GET ALL RACES')\n",
    "    return response['data']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "c9xMPMxwIwvX"
   },
   "outputs": [],
   "source": [
    "def getEvents(race_id: str):\n",
    "    request_body = {\n",
    "        \"disciplineId\": DICIPLINE_ID_MOUNTAIN_BIKE,\n",
    "        \"raceId\": race_id\n",
    "    }\n",
    "    return requests.post(URL_EVENTS, json=request_body).json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 528,
     "status": "ok",
     "timestamp": 1594888003856,
     "user": {
      "displayName": "Andrew Mackay",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GipvhJwlVpg1-ArYl_wNJCOJchIEp9jy27NBVfoXg=s64",
      "userId": "06804410358976473893"
     },
     "user_tz": -60
    },
    "id": "BQjYD7ZorKhq"
   },
   "outputs": [],
   "source": [
    "def getResults(event_id: str):\n",
    "    request_body = {\n",
    "        \"disciplineId\": DICIPLINE_ID_MOUNTAIN_BIKE,\n",
    "        \"eventId\": event_id,\n",
    "        \"take\":\"400\",\n",
    "        \"skip\":\"0\",\n",
    "        \"page\":\"1\",\n",
    "        \"pageSize\":\"400\"\n",
    "    }\n",
    "    response = requests.post(URL_RESULTS, json=request_body).json()\n",
    "    if len(response['data']) < response['total']:\n",
    "        print('DID NOT GET ALL RESULTS')\n",
    "    return response['data']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For each year, retrieve the competitions for each of our chosen mountain bike disciplines (downhill, enduro and cross country olympic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Y-l96M7j5AJG"
   },
   "outputs": [],
   "source": [
    "competitions = {\n",
    "    RACE_TYPE_ID_DOWNHILL: {},\n",
    "    RACE_TYPE_ID_ENDURO: {},\n",
    "    RACE_TYPE_ID_CROSS_COUNTRY_OLYMPIC: {}\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "w1HkF3dqAKCb"
   },
   "outputs": [],
   "source": [
    "for year in SEASON_ID_YEAR_MAP:\n",
    "    competitions[RACE_TYPE_ID_DOWNHILL][year] = getCompetitions(RACE_TYPE_ID_DOWNHILL, year)\n",
    "    competitions[RACE_TYPE_ID_ENDURO][year] = getCompetitions(RACE_TYPE_ID_ENDURO, year)\n",
    "    competitions[RACE_TYPE_ID_CROSS_COUNTRY_OLYMPIC][year] = getCompetitions(RACE_TYPE_ID_CROSS_COUNTRY_OLYMPIC, year)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OpzF-sgluFY4"
   },
   "outputs": [],
   "source": [
    "competitions_json = json.dumps(competitions)\n",
    "\n",
    "with open(\"../data/competitions.json\",\"w\") as f:\n",
    "    f.write(competitions_json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ydUbnB0HgKdI"
   },
   "outputs": [],
   "source": [
    "with open('../data/competitions.json') as f:\n",
    "    competitions = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1P1gPpo9Dd0z"
   },
   "source": [
    "Filter the competitions:\n",
    "\n",
    "For downhill and XCO we filter for only world cup and world championship competitions.\n",
    "\n",
    "For enduro we filter for EWS competitions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HnmalNxI8rF5"
   },
   "outputs": [],
   "source": [
    "filtered_competitions = {\n",
    "    RACE_TYPE_ID_DOWNHILL: {},\n",
    "    RACE_TYPE_ID_ENDURO: {},\n",
    "    RACE_TYPE_ID_CROSS_COUNTRY_OLYMPIC: {}\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EOThpxuSra_y"
   },
   "outputs": [],
   "source": [
    "for race_type in filtered_competitions:\n",
    "    for year in competitions[race_type]:\n",
    "        filtered_competitions[race_type][year] = []\n",
    "        for competition in competitions[race_type][year]:\n",
    "            if race_type in [RACE_TYPE_ID_DOWNHILL, RACE_TYPE_ID_CROSS_COUNTRY_OLYMPIC]:\n",
    "                if competition['ClassCode'] in [COMPETITION_CLASS_CODE_WORLD_CHAMPS, COMPETITION_CLASS_CODE_WORLD_CUP]:\n",
    "                    filtered_competitions[race_type][year].append(competition)\n",
    "            if race_type == RACE_TYPE_ID_ENDURO:\n",
    "                if competition['ClassCode'] == COMPETITION_CLASS_CODE_ENDURO_WORLD_SERIES:\n",
    "                    filtered_competitions[RACE_TYPE_ID_ENDURO][year].append(competition)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "HwtUWCKuDmiq"
   },
   "source": [
    "For each filtered competition, fetch the races"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "X0bD50fc2APQ"
   },
   "outputs": [],
   "source": [
    "for race_type in filtered_competitions:\n",
    "    for year in filtered_competitions[race_type]:\n",
    "        for competition in filtered_competitions[race_type][year]:\n",
    "            races = getRaces(competition['CompetitionId'])\n",
    "            competition['races'] = races"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vERqMVNL72uq"
   },
   "outputs": [],
   "source": [
    "competitions_with_races_json = json.dumps(filtered_competitions)\n",
    "\n",
    "with open(\"../data/competitions_with_races.json\",\"w\") as f:\n",
    "    f.write(competitions_with_races_json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WQBQXvJ68j_0"
   },
   "outputs": [],
   "source": [
    "with open('../data/competitions_with_races.json') as f:\n",
    "    competitions_with_races = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uWdiI0VaDsH8"
   },
   "source": [
    "For each race (excluding qualifying), fetch the events"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "oPt2dM-R8_qB"
   },
   "outputs": [],
   "source": [
    "for race_type in competitions_with_races:\n",
    "    for year in competitions_with_races[race_type]:\n",
    "        for competition in competitions_with_races[race_type][year]:\n",
    "            for race in competition['races']:\n",
    "                race['events'] = {}\n",
    "                for category_code in [CATEGORY_CODE_MEN_ELITE, CATEGORY_CODE_WOMEN_ELITE]:\n",
    "                    if race['CategoryCode'] == category_code and race['RaceTypeCode'] == RACE_TYPE_ID_TO_CODE_MAP[race_type]:\n",
    "                        if (race_type == RACE_TYPE_ID_DOWNHILL and 'qualifying' not in race['RaceName'].lower()) or race_type == RACE_TYPE_ID_CROSS_COUNTRY_OLYMPIC or race_type == RACE_TYPE_ID_ENDURO:\n",
    "                            events = getEvents(race['Id'])\n",
    "                            if (len(events) > 1):\n",
    "                                print('MORE THAN ONE EVENT')\n",
    "                            race['events'][category_code] = events[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aHQ3RREEAufW"
   },
   "outputs": [],
   "source": [
    "competitions_with_races_and_events_json = json.dumps(competitions_with_races)\n",
    "\n",
    "with open(\"../data/competitions_with_races_and_events.json\",\"w\") as f:\n",
    "    f.write(competitions_with_races_and_events_json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 525,
     "status": "ok",
     "timestamp": 1594888015290,
     "user": {
      "displayName": "Andrew Mackay",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GipvhJwlVpg1-ArYl_wNJCOJchIEp9jy27NBVfoXg=s64",
      "userId": "06804410358976473893"
     },
     "user_tz": -60
    },
    "id": "CQH2u5kt7pE-"
   },
   "outputs": [],
   "source": [
    "with open('../data/competitions_with_races_and_events.json') as f:\n",
    "    competitions_with_races_and_events = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For each event, fetch the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 227044,
     "status": "ok",
     "timestamp": 1594888269952,
     "user": {
      "displayName": "Andrew Mackay",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GipvhJwlVpg1-ArYl_wNJCOJchIEp9jy27NBVfoXg=s64",
      "userId": "06804410358976473893"
     },
     "user_tz": -60
    },
    "id": "kEDGYi1gB0TQ"
   },
   "outputs": [],
   "source": [
    "for race_type in competitions_with_races_and_events:\n",
    "    for year in competitions_with_races_and_events[race_type]:\n",
    "        for competition in competitions_with_races_and_events[race_type][year]:\n",
    "            for race in competition['races']:\n",
    "                for category_code in [CATEGORY_CODE_MEN_ELITE, CATEGORY_CODE_WOMEN_ELITE]:\n",
    "                    if category_code in race['events']:\n",
    "                        event_id = race['events'][category_code]['EventId']\n",
    "                        results = getResults(event_id)\n",
    "                        race['events'][category_code]['results'] = results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 226556,
     "status": "ok",
     "timestamp": 1594888270738,
     "user": {
      "displayName": "Andrew Mackay",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GipvhJwlVpg1-ArYl_wNJCOJchIEp9jy27NBVfoXg=s64",
      "userId": "06804410358976473893"
     },
     "user_tz": -60
    },
    "id": "qegc1UC2uj8A"
   },
   "outputs": [],
   "source": [
    "competitions_with_races_and_events_and_results_json = json.dumps(competitions_with_races_and_events)\n",
    "\n",
    "with open(\"../data/competitions_with_races_and_events_and_results.json\",\"w\") as f:\n",
    "    f.write(competitions_with_races_and_events_and_results_json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iDvkn-1ktdEW"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyMUcbCGUb2/oLMw2+JXT+MX",
   "collapsed_sections": [],
   "mount_file_id": "1o2w4U4KTbOwds3tRPkghthoX4WCBAmUh",
   "name": "data_collection.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}