web_scraper_template.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import bs4\n",
    "from typing import List, Optional\n",
    "from urllib.parse import urlparse\n",
    "import logging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Scraper:\n",
    "    \"\"\"Base class for web scraping implementations\"\"\"\n",
    "    \n",
    "    def __init__(self, base_url: str):\n",
    "        \"\"\"\n",
    "        Initialize the scraper with a base URL\n",
    "        \n",
    "        Args:\n",
    "            base_url (str): The base URL for the scraper\n",
    "        \"\"\"\n",
    "        self.base_url = base_url\n",
    "        self.session = requests.Session()\n",
    "        self.soup = None\n",
    "        self._setup_logging()\n",
    "    \n",
    "    def _setup_logging(self):\n",
    "        \"\"\"Configure logging for the scraper\"\"\"\n",
    "        logging.basicConfig(\n",
    "            level=logging.INFO,\n",
    "            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'\n",
    "        )\n",
    "        self.logger = logging.getLogger(self.__class__.__name__)\n",
    "    \n",
    "    def _validate_url(self, url: str) -> bool:\n",
    "        \"\"\"\n",
    "        Validate if the URL is properly formatted\n",
    "        \n",
    "        Args:\n",
    "            url (str): URL to validate\n",
    "            \n",
    "        Returns:\n",
    "            bool: True if valid, False otherwise\n",
    "        \"\"\"\n",
    "        try:\n",
    "            result = urlparse(url)\n",
    "            return all([result.scheme, result.netloc])\n",
    "        except Exception as e:\n",
    "            self.logger.error(f\"Invalid URL: {str(e)}\")\n",
    "            return False\n",
    "    \n",
    "    def fetch_page(self, url: str) -> Optional[bs4.BeautifulSoup]:\n",
    "        \"\"\"\n",
    "        Fetch and parse a webpage\n",
    "        \n",
    "        Args:\n",
    "            url (str): URL to fetch\n",
    "            \n",
    "        Returns:\n",
    "            Optional[BeautifulSoup]: Parsed page or None if failed\n",
    "        \"\"\"\n",
    "        if not self._validate_url(url):\n",
    "            return None\n",
    "            \n",
    "        try:\n",
    "            response = self.session.get(url)\n",
    "            response.raise_for_status()\n",
    "            self.soup = bs4.BeautifulSoup(response.text, 'lxml')\n",
    "            return self.soup\n",
    "        except requests.exceptions.RequestException as e:\n",
    "            self.logger.error(f\"Failed to fetch page: {str(e)}\")\n",
    "            return None\n",
    "    \n",
    "    def get_elements_by_tag(self, tag: str) -> List[bs4.element.Tag]:\n",
    "        \"\"\"\n",
    "        Get all elements of a specific HTML tag\n",
    "        \n",
    "        Args:\n",
    "            tag (str): HTML tag to search for\n",
    "            \n",
    "        Returns:\n",
    "            List[Tag]: List of matching elements\n",
    "        \"\"\"\n",
    "        if self.soup is None:\n",
    "            self.logger.warning(\"No page has been fetched yet\")\n",
    "            return []\n",
    "        return self.soup.find_all(tag)\n",
    "    \n",
    "    def get_elements_by_selector(self, selector: str) -> List[bs4.element.Tag]:\n",
    "        \"\"\"\n",
    "        Get all elements matching a CSS selector\n",
    "        \n",
    "        Args:\n",
    "            selector (str): CSS selector to search for\n",
    "            \n",
    "        Returns:\n",
    "            List[Tag]: List of matching elements\n",
    "        \"\"\"\n",
    "        if self.soup is None:\n",
    "            self.logger.warning(\"No page has been fetched yet\")\n",
    "            return []\n",
    "        return self.soup.select(selector)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class WikiScraper(Scraper):\n",
    "    \"\"\"Specialized scraper for Wikipedia pages\"\"\"\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__(\"https://wikipedia.org\")\n",
    "        \n",
    "    def get_page_title(self) -> Optional[str]:\n",
    "        \"\"\"\n",
    "        Get the main title of the Wikipedia page\n",
    "        \n",
    "        Returns:\n",
    "            Optional[str]: Page title or None if not found\n",
    "        \"\"\"\n",
    "        title_elem = self.get_elements_by_selector(\"title\")\n",
    "        if title_elem:\n",
    "            return title_elem[0].getText().replace(\" - Wikipedia\", \"\")\n",
    "        return None\n",
    "    \n",
    "    def get_main_heading(self) -> Optional[str]:\n",
    "        \"\"\"\n",
    "        Get the main heading (h1) of the article\n",
    "        \n",
    "        Returns:\n",
    "            Optional[str]: Main heading or None if not found\n",
    "        \"\"\"\n",
    "        heading = self.get_elements_by_selector(\"h1\")\n",
    "        if heading:\n",
    "            return heading[0].getText()\n",
    "        return None\n",
    "    \n",
    "    def get_table_of_contents(self) -> List[str]:\n",
    "        \"\"\"\n",
    "        Get all table of contents entries\n",
    "        \n",
    "        Returns:\n",
    "            List[str]: List of table of contents entries\n",
    "        \"\"\"\n",
    "        toc = self.get_elements_by_selector(\"div#toc ul li\")\n",
    "        return [item.getText().strip() for item in toc]\n",
    "    \n",
    "    def get_paragraphs(self, limit: Optional[int] = None) -> List[str]:\n",
    "        \"\"\"\n",
    "        Get main article paragraphs\n",
    "        \n",
    "        Args:\n",
    "            limit (Optional[int]): Maximum number of paragraphs to return\n",
    "            \n",
    "        Returns:\n",
    "            List[str]: List of paragraph texts\n",
    "        \"\"\"\n",
    "        paragraphs = self.get_elements_by_selector(\"div#mw-content-text p\")\n",
    "        texts = [p.getText().strip() for p in paragraphs if p.getText().strip()]\n",
    "        return texts[:limit] if limit else texts"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nassau",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}