notebooks/upcoming_web_scrape.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "115d2fb2-3d87-4272-8f55-d6f052aa1d48",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> file management func:\n",
      "\n",
      "    get_content(directory)\n",
      "    get_last_update(file_path)\n",
      "\n",
      "> Updating lists func:\n",
      "\n",
      "    get_participants(event_link)\n",
      "\n",
      "> fighters_stats_scrape func:\n",
      "\n",
      "    get_xml(url, data_type='xml', headers = {'User-Agent':'Moz...}) # Return web page data in .xml format.\n",
      "    get_links(url)\n",
      "    get_fighter_data(url) # Retrive data as list\n",
      "    get_current_date()\n",
      "\n",
      "> matches_stats_web_scrape func:\n",
      "\n",
      "    get_event_date(xml)\n",
      "    get_event_data(event_link)\n",
      "    get_later_date(date1, date2)\n",
      "\n",
      "> upcoming_web_scrape func:\n",
      "\n",
      "    get_upcoming_event_url(xml)\n",
      "    get_upcoming_event_data(upcoming_event_url, event_name=None, event_time=None)\n",
      "\n",
      "> fighters_stats_full_update func:\n",
      "\n",
      "    get_fighters_links(page_url)\n",
      "\n",
      "> draftking_odds func:\n",
      "\n",
      "    get_table_odds(table)\n",
      "    get_odds(data, fighter, opponent)\n"
     ]
    }
   ],
   "source": [
    "%run web_scrape_func_def.ipynb"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0baf5e5f-a25a-4932-92c0-e7d586090da2",
   "metadata": {},
   "source": [
    "# Main( )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "e5b4a753-46c7-4ad9-958d-76bd7fd724fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "CARDS_PATH = '../cards/'\n",
    "MAIN_URL = 'http://www.ufcstats.com/statistics/events/upcoming'\n",
    "#MAIN_URL = 'http://www.ufcstats.com/statistics/events/completed'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0e8b0df-224c-4587-9780-62de2c7ab10e",
   "metadata": {},
   "source": [
    "### Get next events links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "9366e296-56b2-4284-987e-da0edc179f53",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event_name</th>\n",
       "      <th>url</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>UFC 290: Volkanovski vs. Rodriguez</td>\n",
       "      <td>http://www.ufcstats.com/event-details/6085ceb5...</td>\n",
       "      <td>2023-07-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>UFC Fight Night: Holm vs. Silva</td>\n",
       "      <td>http://www.ufcstats.com/event-details/c9885b1b...</td>\n",
       "      <td>2023-07-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>UFC Fight Night: Aspinall vs. Tybura</td>\n",
       "      <td>http://www.ufcstats.com/event-details/1174782e...</td>\n",
       "      <td>2023-07-22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>UFC 291: Poirier vs. Gaethje 2</td>\n",
       "      <td>http://www.ufcstats.com/event-details/ccd58ff7...</td>\n",
       "      <td>2023-07-29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>UFC Fight Night: Sandhagen vs. Nurmagomedov</td>\n",
       "      <td>http://www.ufcstats.com/event-details/6f81b6de...</td>\n",
       "      <td>2023-08-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>UFC Fight Night: Luque vs. Dos Anjos</td>\n",
       "      <td>http://www.ufcstats.com/event-details/d2fa318f...</td>\n",
       "      <td>2023-08-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>UFC 292: Sterling vs. O'Malley</td>\n",
       "      <td>http://www.ufcstats.com/event-details/2719f300...</td>\n",
       "      <td>2023-08-19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>UFC Fight Night: Holloway vs. The Korean Zombie</td>\n",
       "      <td>http://www.ufcstats.com/event-details/89a40703...</td>\n",
       "      <td>2023-08-26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>UFC Fight Night: Gane vs. Spivac</td>\n",
       "      <td>http://www.ufcstats.com/event-details/ef61d9f5...</td>\n",
       "      <td>2023-09-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>UFC 293</td>\n",
       "      <td>http://www.ufcstats.com/event-details/ece28074...</td>\n",
       "      <td>2023-09-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>UFC Fight Night: Grasso vs. Shevchenko 2</td>\n",
       "      <td>http://www.ufcstats.com/event-details/8fa2b065...</td>\n",
       "      <td>2023-09-16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>UFC 294</td>\n",
       "      <td>http://www.ufcstats.com/event-details/13a0fb8f...</td>\n",
       "      <td>2023-10-21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         event_name  \\\n",
       "0                UFC 290: Volkanovski vs. Rodriguez   \n",
       "1                   UFC Fight Night: Holm vs. Silva   \n",
       "2              UFC Fight Night: Aspinall vs. Tybura   \n",
       "3                    UFC 291: Poirier vs. Gaethje 2   \n",
       "4       UFC Fight Night: Sandhagen vs. Nurmagomedov   \n",
       "5              UFC Fight Night: Luque vs. Dos Anjos   \n",
       "6                    UFC 292: Sterling vs. O'Malley   \n",
       "7   UFC Fight Night: Holloway vs. The Korean Zombie   \n",
       "8                  UFC Fight Night: Gane vs. Spivac   \n",
       "9                                           UFC 293   \n",
       "10         UFC Fight Night: Grasso vs. Shevchenko 2   \n",
       "11                                          UFC 294   \n",
       "\n",
       "                                                  url       date  \n",
       "0   http://www.ufcstats.com/event-details/6085ceb5... 2023-07-08  \n",
       "1   http://www.ufcstats.com/event-details/c9885b1b... 2023-07-15  \n",
       "2   http://www.ufcstats.com/event-details/1174782e... 2023-07-22  \n",
       "3   http://www.ufcstats.com/event-details/ccd58ff7... 2023-07-29  \n",
       "4   http://www.ufcstats.com/event-details/6f81b6de... 2023-08-05  \n",
       "5   http://www.ufcstats.com/event-details/d2fa318f... 2023-08-12  \n",
       "6   http://www.ufcstats.com/event-details/2719f300... 2023-08-19  \n",
       "7   http://www.ufcstats.com/event-details/89a40703... 2023-08-26  \n",
       "8   http://www.ufcstats.com/event-details/ef61d9f5... 2023-09-02  \n",
       "9   http://www.ufcstats.com/event-details/ece28074... 2023-09-09  \n",
       "10  http://www.ufcstats.com/event-details/8fa2b065... 2023-09-16  \n",
       "11  http://www.ufcstats.com/event-details/13a0fb8f... 2023-10-21  "
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "upcoming_events = {'event_name':[],'url':[],'date':[]}\n",
    "xml = get_xml(MAIN_URL)                  \n",
    "for item,date_item in zip(xml.find_all('a',{'class':'b-link b-link_style_black'}),xml.find_all('span',{'class':'b-statistics__date'})):\n",
    "    upcoming_events['event_name'].append(item.text.strip())\n",
    "    upcoming_events['url'].append(item['href'])\n",
    "    upcoming_events['date'].append(datetime.strptime(date_item.text.strip(),'%B %d, %Y'))\n",
    "    \n",
    "pd.DataFrame(upcoming_events)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "3fe518b8-01b1-49be-91c1-5ccd6c8c2bab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "UFC 290: Volkanovski vs. Rodriguez:http://www.ufcstats.com/event-details/6085ceb59087514b\n"
     ]
    }
   ],
   "source": [
    "EVENT_NUM = 0\n",
    "upcoming_event_url = upcoming_events['url'][EVENT_NUM]\n",
    "upcoming_event_name = upcoming_events['event_name'][EVENT_NUM]\n",
    "print(f\"{upcoming_event_name}:{upcoming_event_url}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "5202e668-176b-4d93-a9ff-18a7e4f37e26",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12it [00:06,  1.91it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fighter</th>\n",
       "      <th>opponent</th>\n",
       "      <th>weight_class</th>\n",
       "      <th>event_name</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>Carlos Ulberg</td>\n",
       "      <td>Da Woon Jung</td>\n",
       "      <td>Light Heavyweight</td>\n",
       "      <td>UFC 293</td>\n",
       "      <td>2023-09-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>Alexa Grasso</td>\n",
       "      <td>Valentina Shevchenko</td>\n",
       "      <td>Women's Flyweight</td>\n",
       "      <td>UFC Fight Night: Grasso vs. Shevchenko 2</td>\n",
       "      <td>2023-09-16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>Shavkat Rakhmonov</td>\n",
       "      <td>Kelvin Gastelum</td>\n",
       "      <td>Welterweight</td>\n",
       "      <td>UFC Fight Night: Grasso vs. Shevchenko 2</td>\n",
       "      <td>2023-09-16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              fighter              opponent       weight_class  \\\n",
       "90      Carlos Ulberg          Da Woon Jung  Light Heavyweight   \n",
       "91       Alexa Grasso  Valentina Shevchenko  Women's Flyweight   \n",
       "92  Shavkat Rakhmonov       Kelvin Gastelum       Welterweight   \n",
       "\n",
       "                                  event_name       date  \n",
       "90                                   UFC 293 2023-09-09  \n",
       "91  UFC Fight Night: Grasso vs. Shevchenko 2 2023-09-16  \n",
       "92  UFC Fight Night: Grasso vs. Shevchenko 2 2023-09-16  "
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.DataFrame()\n",
    "for event_name,url,date in tqdm(zip(upcoming_events['event_name'],upcoming_events['url'],upcoming_events['date'])):\n",
    "    data = pd.concat([data,pd.DataFrame(get_upcoming_event_data(url,event_name,date))]).reset_index(drop=True)\n",
    "    \n",
    "data.tail(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "d20ec10c-3a0a-4fb2-9348-62981dda3f20",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Next event Schedualed time: 07-08-2023\n"
     ]
    }
   ],
   "source": [
    "scheduled_time = get_event_date(xml=get_xml(url=upcoming_event_url))\n",
    "print(f\"Next event Schedualed time: {scheduled_time}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "bdabf2c7-f2a4-40a9-80a5-5513620f4bb3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../cards/.ipynb_checkpoints/',\n",
       " '../cards/darftking_odds_06-29-2023.csv',\n",
       " '../cards/darftking_odds_07-02-2023.csv',\n",
       " '../cards/darftking_odds_07-03-2023.csv',\n",
       " '../cards/ufc_card_06-17-2023.csv',\n",
       " '../cards/ufc_card_07-01-2023.csv',\n",
       " '../cards/ufc_card_07-08-2023.csv',\n",
       " '../cards/ufc_card_07-15-2023.csv']"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.to_csv(CARDS_PATH + 'ufc_card_' + scheduled_time + '.csv', index=0)\n",
    "get_content(CARDS_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "069bbeb5-a931-4cc3-9dcd-722014d02ca2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../cards/darftking_odds_06-29-2023.csv\n",
      "../cards/darftking_odds_07-02-2023.csv\n",
      "../cards/darftking_odds_07-03-2023.csv\n"
     ]
    }
   ],
   "source": [
    "for item in get_content(CARDS_PATH):\n",
    "    if 'odds' in item:\n",
    "        print(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36365a83-a973-4d3e-9527-3fa18591efb8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}