scrap_imgur_image_links.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import time\n",
    "\n",
    "from selenium.webdriver.chrome import options\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "from selenium.webdriver.common.by import By\n",
    "\n",
    "# PS: this is chrome version 109, so if ur chrome is using another version, go to download an updated one\n",
    "path = 'C:\\Program Files (x86)\\chromedriver.exe'\n",
    "url = 'https://imgur.com/a/yODQvDV'\n",
    "\n",
    "driver = webdriver.Chrome(service=Service(path))\n",
    "driver.get(url)\n",
    "\n",
    "# waits maximum 3 seconds for element's presence. If it is found after 2 seconds then code execution will be continued without wait for more 1 seconds.\n",
    "driver.implicitly_wait(3)\n",
    "\n",
    "# use if because no everytime need to click\n",
    "\n",
    "# click yes\n",
    "if (driver.find_element(By.CSS_SELECTOR, '#root .WallPage .Wall .Wall-choices div').click() == True):\n",
    "    pass\n",
    "\n",
    "driver.implicitly_wait(3)\n",
    "\n",
    "# choose load more\n",
    "if (driver.find_element(By.CSS_SELECTOR, '#root .loadMoreContainer').click() == True):\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(86,\n",
       " ['https://i.imgur.com/LRYQtIo.jpeg',\n",
       "  'https://i.imgur.com/UGrNcyi.jpeg',\n",
       "  'https://i.imgur.com/1eNUh0f.jpeg',\n",
       "  'https://i.imgur.com/wPugOJi.jpeg',\n",
       "  'https://i.imgur.com/DPTrJeo.jpeg',\n",
       "  'https://i.imgur.com/ok5hwWm.jpeg',\n",
       "  'https://i.imgur.com/VHjTBI2.jpeg',\n",
       "  'https://i.imgur.com/deUafCv.jpeg',\n",
       "  'https://i.imgur.com/MIvIJx2.jpeg',\n",
       "  'https://i.imgur.com/a0j3XPw.jpeg',\n",
       "  'https://i.imgur.com/jFCnuHK.jpeg',\n",
       "  'https://i.imgur.com/zB4njHx.jpeg',\n",
       "  'https://i.imgur.com/7AJOmSe.jpeg',\n",
       "  'https://i.imgur.com/vaJ2Jpk.jpeg',\n",
       "  'https://i.imgur.com/sFOWMTX.jpeg',\n",
       "  'https://i.imgur.com/YRfGKMl.jpeg',\n",
       "  'https://i.imgur.com/y82fnHE.jpeg',\n",
       "  'https://i.imgur.com/Fc6kkqR.jpeg',\n",
       "  'https://i.imgur.com/DDNgF1k.jpeg',\n",
       "  'https://i.imgur.com/fwCBv3x.jpeg',\n",
       "  'https://i.imgur.com/zrJlaPO.jpeg',\n",
       "  'https://i.imgur.com/yE8mQ85.jpeg',\n",
       "  'https://i.imgur.com/rzQOalR.jpeg',\n",
       "  'https://i.imgur.com/UsZArhT.jpeg',\n",
       "  'https://i.imgur.com/v0r4i8M.jpeg',\n",
       "  'https://i.imgur.com/26rQZtp.jpeg',\n",
       "  'https://i.imgur.com/hslp4ol.jpeg',\n",
       "  'https://i.imgur.com/C9sK1b3.jpeg',\n",
       "  'https://i.imgur.com/IRgqD83.jpeg',\n",
       "  'https://i.imgur.com/6OmNBCP.jpeg',\n",
       "  'https://i.imgur.com/l2Grszl.jpeg',\n",
       "  'https://i.imgur.com/lIwovr0.jpeg',\n",
       "  'https://i.imgur.com/8q8ZgTw.jpeg',\n",
       "  'https://i.imgur.com/4BsmunE.jpeg',\n",
       "  'https://i.imgur.com/Mr5dlzV.jpeg',\n",
       "  'https://i.imgur.com/RVCMOLa.jpeg',\n",
       "  'https://i.imgur.com/KGu5WaQ.jpeg',\n",
       "  'https://i.imgur.com/fZIMZGj.jpeg',\n",
       "  'https://i.imgur.com/qdWKiso.jpeg',\n",
       "  'https://i.imgur.com/e6BysHY.jpeg',\n",
       "  'https://i.imgur.com/k2KFIH2.jpeg',\n",
       "  'https://i.imgur.com/jm0mVzZ.jpeg',\n",
       "  'https://i.imgur.com/6z1zFKx.jpeg',\n",
       "  'https://i.imgur.com/ZEjWEQ4.jpeg',\n",
       "  'https://i.imgur.com/pONBl3A.jpeg',\n",
       "  'https://i.imgur.com/5Sd6pam.jpeg',\n",
       "  'https://i.imgur.com/ZC2USLb.jpeg',\n",
       "  'https://i.imgur.com/vZZOQ5G.jpeg',\n",
       "  'https://i.imgur.com/W6NuR6e.jpeg',\n",
       "  'https://i.imgur.com/DUWuGiP.jpeg',\n",
       "  'https://i.imgur.com/XA6gZjT.jpeg',\n",
       "  'https://i.imgur.com/ICu1XFx.jpeg',\n",
       "  'https://i.imgur.com/upUKijo.jpeg',\n",
       "  'https://i.imgur.com/HsijiLm.jpeg',\n",
       "  'https://i.imgur.com/gHXP3XE.jpeg',\n",
       "  'https://i.imgur.com/uN69Pty.jpeg',\n",
       "  'https://i.imgur.com/O10Olyo.jpeg',\n",
       "  'https://i.imgur.com/M2qe75r.jpeg',\n",
       "  'https://i.imgur.com/8UkIUCV.jpeg',\n",
       "  'https://i.imgur.com/ZSDa9S2.jpeg',\n",
       "  'https://i.imgur.com/iGT34Pi.jpeg',\n",
       "  'https://i.imgur.com/82T4206.jpeg',\n",
       "  'https://i.imgur.com/bYsdH3Q.jpeg',\n",
       "  'https://i.imgur.com/XlLTepx.jpeg',\n",
       "  'https://i.imgur.com/weYR3Kd.jpeg',\n",
       "  'https://i.imgur.com/U5Y8swK.jpeg',\n",
       "  'https://i.imgur.com/zN0ub7b.jpeg',\n",
       "  'https://i.imgur.com/dMm8fDE.jpeg',\n",
       "  'https://i.imgur.com/UAISDbK.jpeg',\n",
       "  'https://i.imgur.com/9i8WQnX.jpeg',\n",
       "  'https://i.imgur.com/yMAO1xp.jpeg',\n",
       "  'https://i.imgur.com/ye288tC.jpeg',\n",
       "  'https://i.imgur.com/okv0cBJ.jpeg',\n",
       "  'https://i.imgur.com/mNdBotW.jpeg',\n",
       "  'https://i.imgur.com/ZzHzdQU.jpeg',\n",
       "  'https://i.imgur.com/t2OJAhZ.jpeg',\n",
       "  'https://i.imgur.com/vTYr9ST.jpeg',\n",
       "  'https://i.imgur.com/rzFkhAc.jpeg',\n",
       "  'https://i.imgur.com/G02CKOk.jpeg',\n",
       "  'https://i.imgur.com/cb74elw.jpeg',\n",
       "  'https://i.imgur.com/KPv7Zgg.jpeg',\n",
       "  'https://i.imgur.com/mfvAL9K.jpeg',\n",
       "  'https://i.imgur.com/cpzgPRa.jpeg',\n",
       "  'https://i.imgur.com/gXCn0k9.jpeg',\n",
       "  'https://i.imgur.com/2LBlKrd.jpeg',\n",
       "  'https://i.imgur.com/GmE2tJR.jpeg'])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# after pressing that button, we need to go back to the top of the page, preparing for the scrap below(need to start at top)\n",
    "driver.execute_script(\"window.scrollTo(0, 0)\")\n",
    "# u can also find any element then send keys(home), both ok, but u need to import Keys first\n",
    "\n",
    "links = []\n",
    "y = 700\n",
    "loop =50  # cant think of a perfect way, but just choose a optimal number depending on the # of pics. fyi:86 is loop=70\n",
    "# the reason for us to loop is that, the f12 will change according to where u scroll\n",
    "# for example, if you are at the top of the page, and u parse the page_source to bs4, u can only get the first n < total pic links\n",
    "\n",
    "# PS: dont scroll too much everytime, ow u will miss some pics\n",
    "\n",
    "for j in range(loop):\n",
    "    # seems that is easier to pass whole page source(but need scrolling) then extract links\n",
    "    soup = BeautifulSoup(driver.page_source, 'lxml')\n",
    "    preprocessed_links = soup.find_all('img', class_=\"image-placeholder\")\n",
    "    # links are in this form 'https://i.imgur.com/LRYQtIo_d.webp?maxwidth=760&fidelity=grand'\n",
    "    # so i need to extract unitl _ and add jpeg\n",
    "    # use find() because we only have one _\n",
    "\n",
    "    for i in preprocessed_links:\n",
    "        index = i['src'].find('_')\n",
    "        links.append(i['src'][:index] + '.jpeg')\n",
    "        links = list(dict.fromkeys(links))\n",
    "    # driver.execute_script('window.scrollBy(0,1000)')\n",
    "    driver.execute_script('window.scrollBy(0,' + str(y) + ')')\n",
    "    # scroll_down()\n",
    "    time.sleep(0.1)\n",
    "\n",
    "# i put length here for u to check whether u are getting all the links\n",
    "len(links), links\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write list to txt file, with \\n\n",
    "# just the classic way\n",
    "# save in cd\n",
    "def write(txt_name):\n",
    "    with open(txt_name + '.txt', 'w') as f:\n",
    "        for i in links:\n",
    "            f.write(f'{i}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "write('takahashirie')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.1 (tags/v3.11.1:a7a450f, Dec  6 2022, 19:58:39) [MSC v.1934 64 bit (AMD64)]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "7fe533f6ece8544fc76406e01ecae658d35d2ac7df2e167be82a4eee1a8767ee"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}