05_other_scrapers.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Other scrapers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notes: Be careful when re-run this script as many face images are replaced with current office holder \n",
    "on relevant official websites"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import bs4 as bs \n",
    "import urllib.request\n",
    "import requests\n",
    "import re "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load wmpids "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "person = pd.read_csv(\"../datasets/people/person_2022.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### International leaders"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### G20 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code is simple but scrape smaller thumb images; instead, we use larger profile\n",
    "images from their personal pages for better face recognition results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# url = 'https://en.wikipedia.org/wiki/G20'\n",
    "# source = req.urlopen(url).read()\n",
    "# soup = bs.BeautifulSoup(source)\n",
    "\n",
    "# image_css = '.gallerybox .image img'\n",
    "# images = soup.select(image_css)\n",
    "\n",
    "# images_src = [i['src'][2:] for i in images]\n",
    "# print(images_src)\n",
    "\n",
    "# df = pd.DataFrame()\n",
    "# df['url'] = images_src\n",
    "\n",
    "# df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Download the website's html\n",
    "#The English page does not have the pictures anymore, you can try using the French page: https://fr.wikipedia.org/wiki/Groupe_des_vingt\n",
    "source = urllib.request.urlopen(\"https://en.wikipedia.org/wiki/G20\").read()\n",
    "soup = bs.BeautifulSoup(source)\n",
    "urls_css = \".center p a\"\n",
    "urls = soup.select(urls_css)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "l1 = urls[1:14:3]\n",
    "l2 = urls[17::3]\n",
    "leaders = l1+l2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "l_names = [url.text for url in leaders]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "extracted_urls = [url[\"href\"] for url in urls]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "w1 = extracted_urls[1:14:3]\n",
    "w2 = extracted_urls[17::3]\n",
    "pages = w1+w2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "page_urls = [\"https://en.wikipedia.org/\" + url_suffix for url_suffix in pages]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "g20 = []\n",
    "\n",
    "for i in page_urls[:]:\n",
    "  source = urllib.request.urlopen(i).read()\n",
    "  soup = bs.BeautifulSoup(source)\n",
    "  img_css = \".infobox-image a img\"  \n",
    "  img = soup.select(img_css)\n",
    "  g20.extend(list(set([i['src'] for i in img])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "g20_face_urls = [\"https:\" + url_suffix for url_suffix in g20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(g20_face_urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = pd.DataFrame(g20_face_urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = t.loc[t[0].str.contains(\"220px\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "l_urls = t[0].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(l_urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://upload.wikimedia.org/wikipedia/commons/thumb/9/9e/Ursula_Von_der_Leyen_-_April_2022.jpg/220px-Ursula_Von_der_Leyen_-_April_2022.jpg'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l_urls.pop(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://upload.wikimedia.org/wikipedia/commons/thumb/f/fb/President_of_South_Africa_MC_Ramaphosa_speaks_to_Russian_President_%2828_June_2019%29.jpg/220px-President_of_South_Africa_MC_Ramaphosa_speaks_to_Russian_President_%2828_June_2019%29.jpg'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l_urls.pop(16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "g20 = pd.DataFrame()\n",
    "g20['face_name']=l_names\n",
    "g20['face_url']=l_urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To make name consistent with names in the person file:\n",
    "g20.loc[g20['face_name']=='Salman', 'face_name'] = 'Mohammed bin Salman'\n",
    "# Replace the president of Brazil\n",
    "g20.loc[g20['face_name']=='Luiz Inácio Lula da Silva', 'face_name'] = 'Jair Bolsonaro'\n",
    "g20.loc[g20['face_name']=='Luiz Inácio Lula da Silva', 'face_url'] = 'https://upload.wikimedia.org/wikipedia/commons/thumb/8/8a/Jair_Bolsonaro_em_24_de_abril_de_2019_%281%29_%28cropped%29.jpg/220px-Jair_Bolsonaro_em_24_de_abril_de_2019_%281%29_%28cropped%29.jpg'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(21, 2)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g20.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "l1 = {'face_name': 'António Guterres', 'face_url': \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/d1/Ant%C3%B3nio_Guterres_2021.jpg/220px-Ant%C3%B3nio_Guterres_2021.jpg\"}\n",
    "il = g20.append(l1, ignore_index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "il['face_category']  = 'IntLeader'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22, 3)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "il.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get wmp ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "wmp_il = person.loc[person['intl_leaders']==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22, 30)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wmp_il.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "wmp_il = wmp_il[['full_name', 'wmpid']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "il = il.merge(wmp_il, how='left', left_on='face_name', right_on='full_name', indicator=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "il.loc[il['face_name']=='Joe Biden', 'wmpid']='WMPID1289'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>face_name</th>\n",
       "      <th>face_url</th>\n",
       "      <th>face_category</th>\n",
       "      <th>full_name</th>\n",
       "      <th>wmpid</th>\n",
       "      <th>_merge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Joe Biden</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>NaN</td>\n",
       "      <td>WMPID1289</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    face_name                                           face_url  \\\n",
       "18  Joe Biden  https://upload.wikimedia.org/wikipedia/commons...   \n",
       "\n",
       "   face_category full_name      wmpid     _merge  \n",
       "18     IntLeader       NaN  WMPID1289  left_only  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "il.loc[il._merge =='left_only']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>face_name</th>\n",
       "      <th>face_url</th>\n",
       "      <th>face_category</th>\n",
       "      <th>full_name</th>\n",
       "      <th>wmpid</th>\n",
       "      <th>_merge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Alberto Fernández</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Alberto Fernández</td>\n",
       "      <td>WMPID5195</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Anthony Albanese</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Anthony Albanese</td>\n",
       "      <td>WMPID5217</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Jair Bolsonaro</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Jair Bolsonaro</td>\n",
       "      <td>WMPID5182</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Justin Trudeau</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Justin Trudeau</td>\n",
       "      <td>WMPID5261</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Xi Jinping</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Xi Jinping</td>\n",
       "      <td>WMPID5255</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Emmanuel Macron</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Emmanuel Macron</td>\n",
       "      <td>WMPID5260</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Olaf Scholz</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Olaf Scholz</td>\n",
       "      <td>WMPID5173</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Narendra Modi</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Narendra Modi</td>\n",
       "      <td>WMPID5258</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Joko Widodo</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Joko Widodo</td>\n",
       "      <td>WMPID5249</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Giorgia Meloni</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Giorgia Meloni</td>\n",
       "      <td>WMPID5188</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Fumio Kishida</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Fumio Kishida</td>\n",
       "      <td>WMPID5169</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Andrés Manuel López Obrador</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Andrés Manuel López Obrador</td>\n",
       "      <td>WMPID5177</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Yoon Suk-yeol</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Yoon Suk-yeol</td>\n",
       "      <td>WMPID5190</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Vladimir Putin</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Vladimir Putin</td>\n",
       "      <td>WMPID5183</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Mohammed bin Salman</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Mohammed bin Salman</td>\n",
       "      <td>WMPID5187</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Cyril Ramaphosa</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Cyril Ramaphosa</td>\n",
       "      <td>WMPID5170</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Recep Tayyip Erdoğan</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Recep Tayyip Erdoğan</td>\n",
       "      <td>WMPID5246</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Rishi Sunak</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Rishi Sunak</td>\n",
       "      <td>WMPID5205</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Joe Biden</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>NaN</td>\n",
       "      <td>WMPID1289</td>\n",
       "      <td>left_only</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Charles Michel</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Charles Michel</td>\n",
       "      <td>WMPID5218</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Ursula von der Leyen</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Ursula von der Leyen</td>\n",
       "      <td>WMPID5211</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>António Guterres</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>António Guterres</td>\n",
       "      <td>WMPID5248</td>\n",
       "      <td>both</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      face_name  \\\n",
       "0             Alberto Fernández   \n",
       "1              Anthony Albanese   \n",
       "2                Jair Bolsonaro   \n",
       "3                Justin Trudeau   \n",
       "4                    Xi Jinping   \n",
       "5               Emmanuel Macron   \n",
       "6                   Olaf Scholz   \n",
       "7                 Narendra Modi   \n",
       "8                   Joko Widodo   \n",
       "9                Giorgia Meloni   \n",
       "10                Fumio Kishida   \n",
       "11  Andrés Manuel López Obrador   \n",
       "12                Yoon Suk-yeol   \n",
       "13               Vladimir Putin   \n",
       "14          Mohammed bin Salman   \n",
       "15              Cyril Ramaphosa   \n",
       "16         Recep Tayyip Erdoğan   \n",
       "17                  Rishi Sunak   \n",
       "18                    Joe Biden   \n",
       "19               Charles Michel   \n",
       "20         Ursula von der Leyen   \n",
       "21             António Guterres   \n",
       "\n",
       "                                             face_url face_category  \\\n",
       "0   https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "1   https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "2   https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "3   https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "4   https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "5   https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "6   https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "7   https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "8   https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "9   https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "10  https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "11  https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "12  https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "13  https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "14  https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "15  https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "16  https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "17  https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "18  https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "19  https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "20  https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "21  https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "\n",
       "                      full_name      wmpid     _merge  \n",
       "0             Alberto Fernández  WMPID5195       both  \n",
       "1              Anthony Albanese  WMPID5217       both  \n",
       "2                Jair Bolsonaro  WMPID5182       both  \n",
       "3                Justin Trudeau  WMPID5261       both  \n",
       "4                    Xi Jinping  WMPID5255       both  \n",
       "5               Emmanuel Macron  WMPID5260       both  \n",
       "6                   Olaf Scholz  WMPID5173       both  \n",
       "7                 Narendra Modi  WMPID5258       both  \n",
       "8                   Joko Widodo  WMPID5249       both  \n",
       "9                Giorgia Meloni  WMPID5188       both  \n",
       "10                Fumio Kishida  WMPID5169       both  \n",
       "11  Andrés Manuel López Obrador  WMPID5177       both  \n",
       "12                Yoon Suk-yeol  WMPID5190       both  \n",
       "13               Vladimir Putin  WMPID5183       both  \n",
       "14          Mohammed bin Salman  WMPID5187       both  \n",
       "15              Cyril Ramaphosa  WMPID5170       both  \n",
       "16         Recep Tayyip Erdoğan  WMPID5246       both  \n",
       "17                  Rishi Sunak  WMPID5205       both  \n",
       "18                          NaN  WMPID1289  left_only  \n",
       "19               Charles Michel  WMPID5218       both  \n",
       "20         Ursula von der Leyen  WMPID5211       both  \n",
       "21             António Guterres  WMPID5248       both  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "il"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "il = il[['wmpid', 'face_url', 'face_category']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22, 3)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "il.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wmpid</th>\n",
       "      <th>face_url</th>\n",
       "      <th>face_category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>WMPID5195</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WMPID5217</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       wmpid                                           face_url face_category\n",
       "0  WMPID5195  https://upload.wikimedia.org/wikipedia/commons...     IntLeader\n",
       "1  WMPID5217  https://upload.wikimedia.org/wikipedia/commons...     IntLeader"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "il.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Former presidents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = requests.get(\"https://en.wikipedia.org/wiki/List_of_presidents_of_the_United_States\")\n",
    "soup = bs.BeautifulSoup(res.content, \"html.parser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "names =[]\n",
    "\n",
    "for name in soup.find_all('b')[0:46]:\n",
    "    names.append(name.get_text())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls=[]\n",
    "for link in soup.find_all('img')[6:52]:\n",
    "    urls.append(link.get('src'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = ['https:' + str(i).replace('160px', '320px') for i in urls]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp = pd.DataFrame(columns=('face_name', 'face_url'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp['face_name']=names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp['face_url']=urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(46, 2)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fp.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>face_name</th>\n",
       "      <th>face_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>George Washington</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>John Adams</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Thomas Jefferson</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>James Madison</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>James Monroe</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           face_name                                           face_url\n",
       "0  George Washington  https://upload.wikimedia.org/wikipedia/commons...\n",
       "1         John Adams  https://upload.wikimedia.org/wikipedia/commons...\n",
       "2   Thomas Jefferson  https://upload.wikimedia.org/wikipedia/commons...\n",
       "3      James Madison  https://upload.wikimedia.org/wikipedia/commons...\n",
       "4       James Monroe  https://upload.wikimedia.org/wikipedia/commons..."
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "wmp_fp= person.loc[person['former_uspres']==1]\n",
    "wmp_fp =wmp_fp[['wmpid', 'full_name']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp = fp.merge(wmp_fp, left_on='face_name', right_on='full_name', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>face_name</th>\n",
       "      <th>face_url</th>\n",
       "      <th>wmpid</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>James A. Garfield</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Richard Nixon</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Gerald Ford</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Jimmy Carter</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>George W. Bush</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>Barack Obama</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            face_name                                           face_url  \\\n",
       "19  James A. Garfield  https://upload.wikimedia.org/wikipedia/commons...   \n",
       "36      Richard Nixon  https://upload.wikimedia.org/wikipedia/commons...   \n",
       "37        Gerald Ford  https://upload.wikimedia.org/wikipedia/commons...   \n",
       "38       Jimmy Carter  https://upload.wikimedia.org/wikipedia/commons...   \n",
       "42     George W. Bush  https://upload.wikimedia.org/wikipedia/commons...   \n",
       "43       Barack Obama  https://upload.wikimedia.org/wikipedia/commons...   \n",
       "\n",
       "   wmpid full_name  \n",
       "19   NaN       NaN  \n",
       "36   NaN       NaN  \n",
       "37   NaN       NaN  \n",
       "38   NaN       NaN  \n",
       "42   NaN       NaN  \n",
       "43   NaN       NaN  "
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fp.loc[fp.wmpid.isna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wmpid</th>\n",
       "      <th>full_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2122</th>\n",
       "      <td>WMPID5180</td>\n",
       "      <td>James Carter</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          wmpid     full_name\n",
       "2122  WMPID5180  James Carter"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wmp_fp.loc[wmp_fp['full_name'].str.contains(\"Carter\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp.loc[fp['face_name']=='James A. Garfield', 'wmpid'] ='WMPID5184'\n",
    "fp.loc[fp['face_name']=='George W. Bush', 'wmpid'] ='WMPID5'\n",
    "fp.loc[fp['face_name']=='Barack Obama', 'wmpid'] ='WMPID3'\n",
    "fp.loc[fp['face_name']=='Richard Nixon', 'wmpid'] ='WMPID5196'\n",
    "fp.loc[fp['face_name']=='Gerald Ford', 'wmpid'] ='WMPID5197'\n",
    "fp.loc[fp['face_name']=='Jimmy Carter', 'wmpid'] ='WMPID5180'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp['face_category']='FormerPres'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Drop duplicated Grover Cleveland"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp = fp.drop_duplicates(keep='last')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp = fp[['wmpid', 'face_url', 'face_category']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wmpid</th>\n",
       "      <th>face_url</th>\n",
       "      <th>face_category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>WMPID5206</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>FormerPres</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WMPID5241</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>FormerPres</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       wmpid                                           face_url face_category\n",
       "0  WMPID5206  https://upload.wikimedia.org/wikipedia/commons...    FormerPres\n",
       "1  WMPID5241  https://upload.wikimedia.org/wikipedia/commons...    FormerPres"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fp.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(45, 3)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fp.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Supreme Court Justices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "All justices who took judicial oath after 1980"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "source = urllib.request.urlopen(\"https://www.supremecourt.gov/about/biographies.aspx\").read()\n",
    "soup = bs.BeautifulSoup(source)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls_css = \".media-object\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = soup.select(urls_css)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = [url[\"alt\"] for url in urls]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = [url[\"src\"] for url in urls]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = [\"https://www.supremecourt.gov/about/\" + url_suffix for url_suffix in urls]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = pd.DataFrame()\n",
    "s['face_name'] = names\n",
    "s['face_url']=urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = s['face_name'].str.split(pat = ',', expand = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "s['face_name'] = n[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "s2 = {'face_name': 'Ruth Bader Ginsburg', 'face_url': \"https://www.supremecourt.gov/about/justice_pictures/Official_Justice_Ginsburg_sm.jpg\"}\n",
    "s = s.append(s2, ignore_index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "s3 = {'face_name': 'William Rehnquist', \n",
    "      'face_url':'https://upload.wikimedia.org/wikipedia/commons/thumb/3/31/William_Rehnquist.jpg/220px-William_Rehnquist.jpg'}\n",
    "s = s.append(s3, ignore_index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "s4 = {'face_name': 'Antonin Scalia', \n",
    "      'face_url': \"https://upload.wikimedia.org/wikipedia/commons/thumb/c/c5/Antonin_Scalia_Official_SCOTUS_Portrait.jpg/220px-Antonin_Scalia_Official_SCOTUS_Portrait.jpg\"}\n",
    "s = s.append(s4, ignore_index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16, 2)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "s.loc[s['face_name']==\"Sandra Day O'Connor (Retired)\", 'face_name']=\"Sandra Day O'Connor\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get wmpid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "wmp_s = person[['full_name', 'wmpid']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = s.merge(wmp_s, left_on='face_name', right_on='full_name', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "s.loc[s['face_name']==\"John G. Roberts\", 'wmpid'] ='WMPID10'\n",
    "s.loc[s['face_name']==\"Samuel A. Alito\", 'wmpid'] ='WMPID9'\n",
    "s.loc[s['face_name']==\"Neil M. Gorsuch\", 'wmpid'] ='WMPID15'\n",
    "s.loc[s['face_name']==\"Brett M. Kavanaugh\", 'wmpid'] ='WMPID16'\n",
    "s.loc[s['face_name']==\"Amy Coney Barrett\", 'wmpid'] ='WMPID17'\n",
    "s.loc[s['face_name']==\"Stephen G. Breyer\", 'wmpid'] ='WMPID12'\n",
    "s.loc[s['face_name']==\"Ruth Bader Ginsburg\", 'wmpid'] ='WMPID18'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "s['face_category']='SupremeCourt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>face_name</th>\n",
       "      <th>face_url</th>\n",
       "      <th>full_name</th>\n",
       "      <th>wmpid</th>\n",
       "      <th>face_category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>John G. Roberts</td>\n",
       "      <td>https://www.supremecourt.gov/about/justice_pic...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>WMPID10</td>\n",
       "      <td>SupremeCourt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Clarence Thomas</td>\n",
       "      <td>https://www.supremecourt.gov/about/justice_pic...</td>\n",
       "      <td>Clarence Thomas</td>\n",
       "      <td>WMPID11</td>\n",
       "      <td>SupremeCourt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Samuel A. Alito</td>\n",
       "      <td>https://www.supremecourt.gov/about/justice_pic...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>WMPID9</td>\n",
       "      <td>SupremeCourt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Sonia Sotomayor</td>\n",
       "      <td>https://www.supremecourt.gov/about/justice_pic...</td>\n",
       "      <td>Sonia Sotomayor</td>\n",
       "      <td>WMPID14</td>\n",
       "      <td>SupremeCourt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Elena Kagan</td>\n",
       "      <td>https://www.supremecourt.gov/about/justice_pic...</td>\n",
       "      <td>Elena Kagan</td>\n",
       "      <td>WMPID13</td>\n",
       "      <td>SupremeCourt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Neil M. Gorsuch</td>\n",
       "      <td>https://www.supremecourt.gov/about/justice_pic...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>WMPID15</td>\n",
       "      <td>SupremeCourt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Brett M. Kavanaugh</td>\n",
       "      <td>https://www.supremecourt.gov/about/justice_pic...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>WMPID16</td>\n",
       "      <td>SupremeCourt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Amy Coney Barrett</td>\n",
       "      <td>https://www.supremecourt.gov/about/justice_pic...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>WMPID17</td>\n",
       "      <td>SupremeCourt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Ketanji Brown Jackson</td>\n",
       "      <td>https://www.supremecourt.gov/about/justice_pic...</td>\n",
       "      <td>Ketanji Brown Jackson</td>\n",
       "      <td>WMPID5166</td>\n",
       "      <td>SupremeCourt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Sandra Day O'Connor</td>\n",
       "      <td>https://www.supremecourt.gov/about/justice_pic...</td>\n",
       "      <td>Sandra Day O'Connor</td>\n",
       "      <td>WMPID5252</td>\n",
       "      <td>SupremeCourt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Anthony M. Kennedy</td>\n",
       "      <td>https://www.supremecourt.gov/about/justice_pic...</td>\n",
       "      <td>Anthony M. Kennedy</td>\n",
       "      <td>WMPID5224</td>\n",
       "      <td>SupremeCourt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>David Hackett Souter</td>\n",
       "      <td>https://www.supremecourt.gov/about/justice_pic...</td>\n",
       "      <td>David Hackett Souter</td>\n",
       "      <td>WMPID5202</td>\n",
       "      <td>SupremeCourt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Stephen G. Breyer</td>\n",
       "      <td>https://www.supremecourt.gov/about/justice_pic...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>WMPID12</td>\n",
       "      <td>SupremeCourt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Ruth Bader Ginsburg</td>\n",
       "      <td>https://www.supremecourt.gov/about/justice_pic...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>WMPID18</td>\n",
       "      <td>SupremeCourt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>William Rehnquist</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>William Rehnquist</td>\n",
       "      <td>WMPID5227</td>\n",
       "      <td>SupremeCourt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Antonin Scalia</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>Antonin Scalia</td>\n",
       "      <td>WMPID19</td>\n",
       "      <td>SupremeCourt</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                face_name                                           face_url  \\\n",
       "0         John G. Roberts  https://www.supremecourt.gov/about/justice_pic...   \n",
       "1         Clarence Thomas  https://www.supremecourt.gov/about/justice_pic...   \n",
       "2         Samuel A. Alito  https://www.supremecourt.gov/about/justice_pic...   \n",
       "3         Sonia Sotomayor  https://www.supremecourt.gov/about/justice_pic...   \n",
       "4             Elena Kagan  https://www.supremecourt.gov/about/justice_pic...   \n",
       "5         Neil M. Gorsuch  https://www.supremecourt.gov/about/justice_pic...   \n",
       "6      Brett M. Kavanaugh  https://www.supremecourt.gov/about/justice_pic...   \n",
       "7       Amy Coney Barrett  https://www.supremecourt.gov/about/justice_pic...   \n",
       "8   Ketanji Brown Jackson  https://www.supremecourt.gov/about/justice_pic...   \n",
       "9     Sandra Day O'Connor  https://www.supremecourt.gov/about/justice_pic...   \n",
       "10     Anthony M. Kennedy  https://www.supremecourt.gov/about/justice_pic...   \n",
       "11   David Hackett Souter  https://www.supremecourt.gov/about/justice_pic...   \n",
       "12      Stephen G. Breyer  https://www.supremecourt.gov/about/justice_pic...   \n",
       "13    Ruth Bader Ginsburg  https://www.supremecourt.gov/about/justice_pic...   \n",
       "14      William Rehnquist  https://upload.wikimedia.org/wikipedia/commons...   \n",
       "15         Antonin Scalia  https://upload.wikimedia.org/wikipedia/commons...   \n",
       "\n",
       "                full_name      wmpid face_category  \n",
       "0                     NaN    WMPID10  SupremeCourt  \n",
       "1         Clarence Thomas    WMPID11  SupremeCourt  \n",
       "2                     NaN     WMPID9  SupremeCourt  \n",
       "3         Sonia Sotomayor    WMPID14  SupremeCourt  \n",
       "4             Elena Kagan    WMPID13  SupremeCourt  \n",
       "5                     NaN    WMPID15  SupremeCourt  \n",
       "6                     NaN    WMPID16  SupremeCourt  \n",
       "7                     NaN    WMPID17  SupremeCourt  \n",
       "8   Ketanji Brown Jackson  WMPID5166  SupremeCourt  \n",
       "9     Sandra Day O'Connor  WMPID5252  SupremeCourt  \n",
       "10     Anthony M. Kennedy  WMPID5224  SupremeCourt  \n",
       "11   David Hackett Souter  WMPID5202  SupremeCourt  \n",
       "12                    NaN    WMPID12  SupremeCourt  \n",
       "13                    NaN    WMPID18  SupremeCourt  \n",
       "14      William Rehnquist  WMPID5227  SupremeCourt  \n",
       "15         Antonin Scalia    WMPID19  SupremeCourt  "
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = s[['wmpid', 'face_url', 'face_category']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wmpid</th>\n",
       "      <th>face_url</th>\n",
       "      <th>face_category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>WMPID10</td>\n",
       "      <td>https://www.supremecourt.gov/about/justice_pic...</td>\n",
       "      <td>SupremeCourt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WMPID11</td>\n",
       "      <td>https://www.supremecourt.gov/about/justice_pic...</td>\n",
       "      <td>SupremeCourt</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     wmpid                                           face_url face_category\n",
       "0  WMPID10  https://www.supremecourt.gov/about/justice_pic...  SupremeCourt\n",
       "1  WMPID11  https://www.supremecourt.gov/about/justice_pic...  SupremeCourt"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cabinet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "source = urllib.request.urlopen(\"https://www.whitehouse.gov/administration/cabinet/\").read()\n",
    "soup = bs.BeautifulSoup(source)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls_css = \".persongrid__item__inner__content .h4alt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = soup.select(urls_css)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "c_names = [url.text for url in urls]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "cab_names = []\n",
    "for sub in c_names:\n",
    "    cab_names.append(sub.replace(\"\\n\", \"\").replace(\"\\t\", \"\").replace(\"Dr. \", \"\")), "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls_css = \".persongrid__item__inner__image>div img\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = soup.select(urls_css)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = [url[\"src\"] for url in urls]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = pd.DataFrame()\n",
    "c['face_name'] = cab_names\n",
    "c['face_url']=urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace chief of staff Jess Zients with his predecessor Ron Klain in 2022\n",
    "c.loc[c['face_name']=='Jeff Zients', 'face_name']='Ron Klain'\n",
    "c.loc[c['face_name']=='Jeff Zients', 'face_url']='https://upload.wikimedia.org/wikipedia/commons/thumb/e/e8/Ron_Klain%2C_White_House_Chief_of_Staff.jpg/220px-Ron_Klain%2C_White_House_Chief_of_Staff.jpg'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get wmpid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "wmp = person[['full_name', 'wmpid']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = c.merge(wmp, left_on='face_name', right_on='full_name', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>face_name</th>\n",
       "      <th>face_url</th>\n",
       "      <th>full_name</th>\n",
       "      <th>wmpid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Deb Haaland</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Marcia Fudge</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Shalanda Young</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          face_name                                           face_url  \\\n",
       "5       Deb Haaland  https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "10     Marcia Fudge  https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "22  Shalanda Young   https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "\n",
       "   full_name wmpid  \n",
       "5        NaN   NaN  \n",
       "10       NaN   NaN  \n",
       "22       NaN   NaN  "
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.loc[c['wmpid'].isna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "c.loc[c['face_name']==\"Deb Haaland\", 'wmpid'] ='WMPID1183'\n",
    "c.loc[c['face_name']==\"Marcia Fudge\", 'wmpid'] ='WMPID1206'\n",
    "c.loc[c['face_name']==\"Shalanda Young \", 'wmpid'] ='WMPID5185'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "c['face_category']='Cabinet'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>face_name</th>\n",
       "      <th>face_url</th>\n",
       "      <th>full_name</th>\n",
       "      <th>wmpid</th>\n",
       "      <th>face_category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Kamala Harris</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Kamala Harris</td>\n",
       "      <td>WMPID2</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Antony Blinken</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Antony Blinken</td>\n",
       "      <td>WMPID5223</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Janet Yellen</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Janet Yellen</td>\n",
       "      <td>WMPID5214</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Lloyd Austin</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Lloyd Austin</td>\n",
       "      <td>WMPID5228</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Merrick Garland</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Merrick Garland</td>\n",
       "      <td>WMPID20</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Deb Haaland</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>WMPID1183</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Tom Vilsack</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Tom Vilsack</td>\n",
       "      <td>WMPID5237</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Gina Raimondo</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Gina Raimondo</td>\n",
       "      <td>WMPID5198</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Marty Walsh</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Marty Walsh</td>\n",
       "      <td>WMPID5234</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Xavier Becerra</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Xavier Becerra</td>\n",
       "      <td>WMPID5236</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Marcia Fudge</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>WMPID1206</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Pete Buttigieg</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Pete Buttigieg</td>\n",
       "      <td>WMPID5181</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Jennifer Granholm</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Jennifer Granholm</td>\n",
       "      <td>WMPID5231</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Miguel Cardona</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Miguel Cardona</td>\n",
       "      <td>WMPID5176</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Denis McDonough</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Denis McDonough</td>\n",
       "      <td>WMPID5209</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Alejandro Mayorkas</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Alejandro Mayorkas</td>\n",
       "      <td>WMPID5203</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Michael Regan</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Michael Regan</td>\n",
       "      <td>WMPID5243</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Avril Haines</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Avril Haines</td>\n",
       "      <td>WMPID5254</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Katherine Tai</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Katherine Tai</td>\n",
       "      <td>WMPID5229</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Linda Thomas-Greenfield</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Linda Thomas-Greenfield</td>\n",
       "      <td>WMPID5210</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Cecilia Rouse</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Cecilia Rouse</td>\n",
       "      <td>WMPID5171</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Isabel Guzman</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Isabel Guzman</td>\n",
       "      <td>WMPID5245</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Shalanda Young</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>WMPID5185</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Arati Prabhakar</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Arati Prabhakar</td>\n",
       "      <td>WMPID5186</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Ron Klain</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Ron Klain</td>\n",
       "      <td>WMPID5244</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  face_name  \\\n",
       "0             Kamala Harris   \n",
       "1            Antony Blinken   \n",
       "2              Janet Yellen   \n",
       "3              Lloyd Austin   \n",
       "4           Merrick Garland   \n",
       "5               Deb Haaland   \n",
       "6               Tom Vilsack   \n",
       "7             Gina Raimondo   \n",
       "8               Marty Walsh   \n",
       "9            Xavier Becerra   \n",
       "10             Marcia Fudge   \n",
       "11           Pete Buttigieg   \n",
       "12        Jennifer Granholm   \n",
       "13           Miguel Cardona   \n",
       "14          Denis McDonough   \n",
       "15       Alejandro Mayorkas   \n",
       "16            Michael Regan   \n",
       "17             Avril Haines   \n",
       "18            Katherine Tai   \n",
       "19  Linda Thomas-Greenfield   \n",
       "20            Cecilia Rouse   \n",
       "21            Isabel Guzman   \n",
       "22          Shalanda Young    \n",
       "23          Arati Prabhakar   \n",
       "24                Ron Klain   \n",
       "\n",
       "                                             face_url  \\\n",
       "0   https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "1   https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "2   https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "3   https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "4   https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "5   https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "6   https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "7   https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "8   https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "9   https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "10  https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "11  https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "12  https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "13  https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "14  https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "15  https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "16  https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "17  https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "18  https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "19  https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "20  https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "21  https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "22  https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "23  https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "24  https://www.whitehouse.gov/wp-content/uploads/...   \n",
       "\n",
       "                  full_name      wmpid face_category  \n",
       "0             Kamala Harris     WMPID2       Cabinet  \n",
       "1            Antony Blinken  WMPID5223       Cabinet  \n",
       "2              Janet Yellen  WMPID5214       Cabinet  \n",
       "3              Lloyd Austin  WMPID5228       Cabinet  \n",
       "4           Merrick Garland    WMPID20       Cabinet  \n",
       "5                       NaN  WMPID1183       Cabinet  \n",
       "6               Tom Vilsack  WMPID5237       Cabinet  \n",
       "7             Gina Raimondo  WMPID5198       Cabinet  \n",
       "8               Marty Walsh  WMPID5234       Cabinet  \n",
       "9            Xavier Becerra  WMPID5236       Cabinet  \n",
       "10                      NaN  WMPID1206       Cabinet  \n",
       "11           Pete Buttigieg  WMPID5181       Cabinet  \n",
       "12        Jennifer Granholm  WMPID5231       Cabinet  \n",
       "13           Miguel Cardona  WMPID5176       Cabinet  \n",
       "14          Denis McDonough  WMPID5209       Cabinet  \n",
       "15       Alejandro Mayorkas  WMPID5203       Cabinet  \n",
       "16            Michael Regan  WMPID5243       Cabinet  \n",
       "17             Avril Haines  WMPID5254       Cabinet  \n",
       "18            Katherine Tai  WMPID5229       Cabinet  \n",
       "19  Linda Thomas-Greenfield  WMPID5210       Cabinet  \n",
       "20            Cecilia Rouse  WMPID5171       Cabinet  \n",
       "21            Isabel Guzman  WMPID5245       Cabinet  \n",
       "22                      NaN  WMPID5185       Cabinet  \n",
       "23          Arati Prabhakar  WMPID5186       Cabinet  \n",
       "24                Ron Klain  WMPID5244       Cabinet  "
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = c[['wmpid', 'face_url', 'face_category']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wmpid</th>\n",
       "      <th>face_url</th>\n",
       "      <th>face_category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>WMPID2</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WMPID5223</td>\n",
       "      <td>https://www.whitehouse.gov/wp-content/uploads/...</td>\n",
       "      <td>Cabinet</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       wmpid                                           face_url face_category\n",
       "0     WMPID2  https://www.whitehouse.gov/wp-content/uploads/...       Cabinet\n",
       "1  WMPID5223  https://www.whitehouse.gov/wp-content/uploads/...       Cabinet"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Public health related"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "ph = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "ph1 = {'face_name': 'Anthony Fauci', \n",
    "      'face_url': \"https://upload.wikimedia.org/wikipedia/commons/thumb/6/61/Anthony_Fauci_2020.jpg/220px-Anthony_Fauci_2020.jpg\"}\n",
    "ph = ph.append(ph1, ignore_index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "ph2 = {'face_name': 'Tedros Adhanom Ghebreyesus', \n",
    "      'face_url': \"https://upload.wikimedia.org/wikipedia/commons/thumb/0/0b/Mukhisa_Kituyi%2C_Houlin_Zhao%2C_Tedros_Adhanom_Ghebreyesus_with_Sophia_-_AI_for_Good_Global_Summit_2018_%2841223188035%29_%28cropped%29.jpg/220px-Mukhisa_Kituyi%2C_Houlin_Zhao%2C_Tedros_Adhanom_Ghebreyesus_with_Sophia_-_AI_for_Good_Global_Summit_2018_%2841223188035%29_%28cropped%29.jpg\"}\n",
    "ph = ph.append(ph2, ignore_index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "#The Secretary of the Department of Health and Human Services\n",
    "ph3 = {'face_name': 'Rachel L. Levine', \n",
    "      'face_url': \"https://upload.wikimedia.org/wikipedia/commons/thumb/2/28/Admiral_Rachel_L._Levine.jpg/220px-Admiral_Rachel_L._Levine.jpg\"}\n",
    "ph = ph.append(ph3, ignore_index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "#The Surgeon General\n",
    "ph4 = {'face_name': 'Vivek H. Murthy', \n",
    "      'face_url': \"https://upload.wikimedia.org/wikipedia/commons/thumb/7/7b/Vivek_Murthy%2C_Surgeon_General_%28profile%29.jpg/220px-Vivek_Murthy%2C_Surgeon_General_%28profile%29.jpg\"}\n",
    "ph = ph.append(ph4, ignore_index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Director of CDC\n",
    "ph5 = {'face_name': 'Rochelle Walensky', \n",
    "      'face_url': \"https://upload.wikimedia.org/wikipedia/commons/thumb/b/b9/Rochelle_Walensky%2C_CDC_Director_%28cropped%29.jpg/220px-Rochelle_Walensky%2C_CDC_Director_%28cropped%29.jpg\"}\n",
    "ph = ph.append(ph5, ignore_index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Director of NIH\n",
    "ph6 = {'face_name': 'Lawrence A. Tabak', \n",
    "      'face_url': \"https://upload.wikimedia.org/wikipedia/commons/thumb/a/a3/Lawrence_A._Tabak_official_portrait.jpg/220px-Lawrence_A._Tabak_official_portrait.jpg\"}\n",
    "ph = ph.append(ph6, ignore_index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Director of FDA\n",
    "ph7 = {'face_name': 'Robert Califf', \n",
    "      'face_url': \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Robert_M._Califf%2C_FDA_Commissioner.jpg/220px-Robert_M._Califf%2C_FDA_Commissioner.jpg\"}\n",
    "ph = ph.append(ph7, ignore_index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "ph['face_category']='PublicHealth'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>face_name</th>\n",
       "      <th>face_url</th>\n",
       "      <th>face_category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Anthony Fauci</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>PublicHealth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tedros Adhanom Ghebreyesus</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>PublicHealth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Rachel L. Levine</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>PublicHealth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Vivek H. Murthy</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>PublicHealth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Rochelle Walensky</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>PublicHealth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Lawrence A. Tabak</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>PublicHealth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Robert Califf</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>PublicHealth</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    face_name  \\\n",
       "0               Anthony Fauci   \n",
       "1  Tedros Adhanom Ghebreyesus   \n",
       "2            Rachel L. Levine   \n",
       "3             Vivek H. Murthy   \n",
       "4           Rochelle Walensky   \n",
       "5           Lawrence A. Tabak   \n",
       "6               Robert Califf   \n",
       "\n",
       "                                            face_url face_category  \n",
       "0  https://upload.wikimedia.org/wikipedia/commons...  PublicHealth  \n",
       "1  https://upload.wikimedia.org/wikipedia/commons...  PublicHealth  \n",
       "2  https://upload.wikimedia.org/wikipedia/commons...  PublicHealth  \n",
       "3  https://upload.wikimedia.org/wikipedia/commons...  PublicHealth  \n",
       "4  https://upload.wikimedia.org/wikipedia/commons...  PublicHealth  \n",
       "5  https://upload.wikimedia.org/wikipedia/commons...  PublicHealth  \n",
       "6  https://upload.wikimedia.org/wikipedia/commons...  PublicHealth  "
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "ph.loc[ph['face_name']=='Tedros Adhanom Ghebreyesus', 'face_category']='IntLeader'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "get wmpid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "ph = ph.merge(wmp, left_on='face_name', right_on='full_name', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>face_name</th>\n",
       "      <th>face_url</th>\n",
       "      <th>face_category</th>\n",
       "      <th>full_name</th>\n",
       "      <th>wmpid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Anthony Fauci</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>PublicHealth</td>\n",
       "      <td>Anthony Fauci</td>\n",
       "      <td>WMPID5215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tedros Adhanom Ghebreyesus</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "      <td>Tedros Adhanom Ghebreyesus</td>\n",
       "      <td>WMPID5242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Rachel L. Levine</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>PublicHealth</td>\n",
       "      <td>Rachel L. Levine</td>\n",
       "      <td>WMPID5207</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Vivek H. Murthy</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>PublicHealth</td>\n",
       "      <td>Vivek H. Murthy</td>\n",
       "      <td>WMPID5216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Rochelle Walensky</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>PublicHealth</td>\n",
       "      <td>Rochelle Walensky</td>\n",
       "      <td>WMPID5199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Lawrence A. Tabak</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>PublicHealth</td>\n",
       "      <td>Lawrence A. Tabak</td>\n",
       "      <td>WMPID5247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Robert Califf</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>PublicHealth</td>\n",
       "      <td>Robert Califf</td>\n",
       "      <td>WMPID5175</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    face_name  \\\n",
       "0               Anthony Fauci   \n",
       "1  Tedros Adhanom Ghebreyesus   \n",
       "2            Rachel L. Levine   \n",
       "3             Vivek H. Murthy   \n",
       "4           Rochelle Walensky   \n",
       "5           Lawrence A. Tabak   \n",
       "6               Robert Califf   \n",
       "\n",
       "                                            face_url face_category  \\\n",
       "0  https://upload.wikimedia.org/wikipedia/commons...  PublicHealth   \n",
       "1  https://upload.wikimedia.org/wikipedia/commons...     IntLeader   \n",
       "2  https://upload.wikimedia.org/wikipedia/commons...  PublicHealth   \n",
       "3  https://upload.wikimedia.org/wikipedia/commons...  PublicHealth   \n",
       "4  https://upload.wikimedia.org/wikipedia/commons...  PublicHealth   \n",
       "5  https://upload.wikimedia.org/wikipedia/commons...  PublicHealth   \n",
       "6  https://upload.wikimedia.org/wikipedia/commons...  PublicHealth   \n",
       "\n",
       "                    full_name      wmpid  \n",
       "0               Anthony Fauci  WMPID5215  \n",
       "1  Tedros Adhanom Ghebreyesus  WMPID5242  \n",
       "2            Rachel L. Levine  WMPID5207  \n",
       "3             Vivek H. Murthy  WMPID5216  \n",
       "4           Rochelle Walensky  WMPID5199  \n",
       "5           Lawrence A. Tabak  WMPID5247  \n",
       "6               Robert Califf  WMPID5175  "
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "ph = ph[['wmpid', 'face_url', 'face_category']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wmpid</th>\n",
       "      <th>face_url</th>\n",
       "      <th>face_category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>WMPID5215</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>PublicHealth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WMPID5242</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       wmpid                                           face_url face_category\n",
       "0  WMPID5215  https://upload.wikimedia.org/wikipedia/commons...  PublicHealth\n",
       "1  WMPID5242  https://upload.wikimedia.org/wikipedia/commons...     IntLeader"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ph.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Historical figures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "h = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "h1 = {'face_name': 'Martin Luther King Jr.', \n",
    "      'face_url': \"https://upload.wikimedia.org/wikipedia/commons/thumb/0/05/Martin_Luther_King%2C_Jr..jpg/220px-Martin_Luther_King%2C_Jr..jpg\",\n",
    "     'face_category': 'HistFigure'}\n",
    "h = h.append(h1, ignore_index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>full_name</th>\n",
       "      <th>wmpid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3458</th>\n",
       "      <td>Martin Luther King Jr.</td>\n",
       "      <td>WMPID5221</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   full_name      wmpid\n",
       "3458  Martin Luther King Jr.  WMPID5221"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wmp.loc[wmp['full_name'].str.contains('Martin Luther')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "h['wmpid']='WMPID5221'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>face_category</th>\n",
       "      <th>face_name</th>\n",
       "      <th>face_url</th>\n",
       "      <th>wmpid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HistFigure</td>\n",
       "      <td>Martin Luther King Jr.</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>WMPID5221</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  face_category               face_name  \\\n",
       "0    HistFigure  Martin Luther King Jr.   \n",
       "\n",
       "                                            face_url      wmpid  \n",
       "0  https://upload.wikimedia.org/wikipedia/commons...  WMPID5221  "
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "h = h[['wmpid', 'face_url', 'face_category']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wmpid</th>\n",
       "      <th>face_url</th>\n",
       "      <th>face_category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>WMPID5221</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>HistFigure</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       wmpid                                           face_url face_category\n",
       "0  WMPID5221  https://upload.wikimedia.org/wikipedia/commons...    HistFigure"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combine all results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([il, fp, s, c, ph, h])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(116, 3)"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wmpid</th>\n",
       "      <th>face_url</th>\n",
       "      <th>face_category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>WMPID5195</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WMPID5217</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>WMPID5182</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>WMPID5261</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>WMPID5255</td>\n",
       "      <td>https://upload.wikimedia.org/wikipedia/commons...</td>\n",
       "      <td>IntLeader</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       wmpid                                           face_url face_category\n",
       "0  WMPID5195  https://upload.wikimedia.org/wikipedia/commons...     IntLeader\n",
       "1  WMPID5217  https://upload.wikimedia.org/wikipedia/commons...     IntLeader\n",
       "2  WMPID5182  https://upload.wikimedia.org/wikipedia/commons...     IntLeader\n",
       "3  WMPID5261  https://upload.wikimedia.org/wikipedia/commons...     IntLeader\n",
       "4  WMPID5255  https://upload.wikimedia.org/wikipedia/commons...     IntLeader"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FormerPres      45\n",
       "Cabinet         25\n",
       "IntLeader       23\n",
       "SupremeCourt    16\n",
       "PublicHealth     6\n",
       "HistFigure       1\n",
       "Name: face_category, dtype: int64"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['face_category'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"./data/other_face_urls_2022.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}