feat: add comet scraper

Author: g0ldyy

.env-sample

@@ -19,6 +19,8 @@ INDEXER_MANAGER_TIMEOUT=60 # maximum time to obtain search results from indexer
 INDEXER_MANAGER_INDEXERS='["EXAMPLE1_CHANGETHIS", "EXAMPLE2_CHANGETHIS"]' # for jackett, get the names from https://github.com/Jackett/Jackett/tree/master/src/Jackett.Common/Definitions - for prowlarr you can write them like on the web dashboard
 GET_TORRENT_TIMEOUT=5 # maximum time to obtain the torrent info hash in seconds
 DOWNLOAD_TORRENT_FILES=False # set to True to enable torrent file retrieval instead of using only magnet link info (infohash and sources, ensuring file index is included in results for Jackett and Prowlarr torrents)
+SCRAPE_COMET=False # scrape another Comet instance
+COMET_URL=https://comet.elfhosted.com # Allows you to scrape custom instances of Comet
 SCRAPE_ZILEAN=False # scrape Zilean/DMM
 ZILEAN_URL=https://zilean.elfhosted.com # for DMM search - https://github.com/iPromKnight/zilean - ex: http://127.0.0.1:8181
 SCRAPE_TORRENTIO=False # scrape Torrentio

comet/api/stream.py

@@ -304,7 +304,9 @@ async def stream(
         return {"streams": cached_results + non_cached_results}
 
 
-@streams.get("/{b64config}/playback/{hash}/{index}/{name}/{season}/{episode}/{torrent_name}")
+@streams.get(
+    "/{b64config}/playback/{hash}/{index}/{name}/{season}/{episode}/{torrent_name}"
+)
 async def playback(
     request: Request,
     b64config: str,
@@ -313,7 +315,7 @@ async def playback(
     name: str,
     season: str,
     episode: str,
-    torrent_name: str
+    torrent_name: str,
 ):
     config = config_check(b64config)
 

comet/debrid/stremthru.py

@@ -169,7 +169,13 @@ async def get_availability(
         return files
 
     async def generate_download_link(
-        self, hash: str, index: str, name: str, torrent_name: str, season: int, episode: int
+        self,
+        hash: str,
+        index: str,
+        name: str,
+        torrent_name: str,
+        season: int,
+        episode: int,
     ):
         try:
             magnet = await self.session.post(

comet/main.py

@@ -131,6 +131,12 @@ def start_log():
     else:
         logger.log("COMET", "Indexer Manager: False")
 
+    comet_url = f" - {settings.COMET_URL}"
+    logger.log(
+        "COMET",
+        f"Comet Scraper: {bool(settings.SCRAPE_COMET)}{comet_url if settings.SCRAPE_COMET else ''}",
+    )
+
     zilean_url = f" - {settings.ZILEAN_URL}"
     logger.log(
         "COMET",

comet/scrapers/comet.py

@@ -0,0 +1,53 @@
+from curl_cffi import requests
+
+from comet.utils.models import settings
+from comet.utils.logger import logger
+
+
+async def get_comet(manager, media_type: str, media_id: str):
+    torrents = []
+    try:
+        try:
+            get_comet = requests.get(
+                f"{settings.COMET_URL}/stream/{media_type}/{media_id}.json"
+            ).json()
+        except Exception as e:
+            logger.warning(
+                f"Failed to get Comet results without proxy for {media_id}: {e}"
+            )
+
+            get_comet = requests.get(
+                f"{settings.COMET_URL}/stream/{media_type}/{media_id}.json",
+                proxies={
+                    "http": settings.DEBRID_PROXY_URL,
+                    "https": settings.DEBRID_PROXY_URL,
+                },
+            ).json()
+
+        for torrent in get_comet["streams"]:
+            title_full = torrent["description"]
+            title = title_full.split("\n")[0]
+
+            seeders = (
+                title_full.split("👤 ")[1].split(" ")[0] if "👤" in title_full else None
+            )
+            tracker = title_full.split("🔎 ")[1].split("\n")[0]
+
+            torrents.append(
+                {
+                    "title": title,
+                    "infoHash": torrent["infoHash"].lower(),
+                    "fileIndex": torrent["fileIdx"] if "fileIdx" in torrent else None,
+                    "seeders": seeders,
+                    "size": torrent["behaviorHints"]["videoSize"],
+                    "tracker": f"Comet|{tracker}",
+                    "sources": torrent["sources"] if "sources" in torrent else [],
+                }
+            )
+    except Exception as e:
+        logger.warning(
+            f"Exception while getting torrents for {media_id} with Comet, your IP is most likely blacklisted (you should try proxying Comet): {e}"
+        )
+        pass
+
+    await manager.filter_manager(torrents)

comet/scrapers/manager.py

@@ -23,6 +23,7 @@
 from .mediafusion import get_mediafusion
 from .jackett import get_jackett
 from .prowlarr import get_prowlarr
+from .comet import get_comet
 
 
 class TorrentManager:
@@ -66,6 +67,8 @@ async def scrape_torrents(
         session: aiohttp.ClientSession,
     ):
         tasks = []
+        if settings.SCRAPE_COMET:
+            tasks.append(get_comet(self, self.media_type, self.media_id))
         if settings.SCRAPE_TORRENTIO:
             tasks.append(get_torrentio(self, self.media_type, self.media_id))
         if settings.SCRAPE_MEDIAFUSION:

comet/utils/models.py

@@ -49,7 +49,8 @@ class AppSettings(BaseSettings):
     INDEXER_MANAGER_INDEXERS: List[str] = []
     GET_TORRENT_TIMEOUT: Optional[int] = 5
     DOWNLOAD_TORRENT_FILES: Optional[bool] = False
-    SCRAPE_MEDIAFUSION: Optional[bool] = False
+    SCRAPE_COMET: Optional[bool] = False
+    COMET_URL: Optional[str] = "https://comet.elfhosted.com"
     SCRAPE_ZILEAN: Optional[bool] = False
     ZILEAN_URL: Optional[str] = "https://zilean.elfhosted.com"
     SCRAPE_TORRENTIO: Optional[bool] = False
@@ -72,6 +73,7 @@ class AppSettings(BaseSettings):
         "ZILEAN_URL",
         "TORRENTIO_URL",
         "MEDIAFUSION_URL",
+        "COMET_URL",
         "STREMTHRU_URL",
     )
     def remove_trailing_slash(cls, v):