Rearrange code to get rid of the Global class

.gitignore

@@ -210,3 +210,4 @@ pyrightconfig.json
 
 # ignore all vscode, this is not standard configuration in this place
 .vscode
+output

Dockerfile

@@ -5,7 +5,6 @@ LABEL org.opencontainers.image.source https://github.com/openzim/ifixit
 # TODO: do we really need all these packages?
 RUN apt-get update \
  && apt-get install -y --no-install-recommends \
-      # locales required if tool has any i18n support
       locales \
       locales-all \
       libmagic1 \

README.md

@@ -77,15 +77,15 @@ Docker container as explained below.
 
 First, build the Docker image (to be ran in the main folder of this repo):
 ```
-docker build -t ghcr.io/openzim/ifixit:local .
+docker build -t local-ifixit .
 ```
 
 Then run the scraper with CLI arguments needed for your test (everything after `ifixit2zim` in the example below).
 
 For instance, if you want to run a scrape of only the `Apple_PDA` category, including its guides,
 in French :
 ```
-docker run -it -v $(pwd)/output:/output --rm ghcr.io/openzim/fixit:local ifixit2zim --language fr --output /output --tmp-dir /tmp --category Apple_PDA
+docker run -it -v $(pwd)/output:/output --rm local-ifixit ifixit2zim --language fr --output /output --tmp-dir /tmp --category Apple_PDA
 ```
 
 This will produce a ZIM in the output folder of your current directory.

src/ifixit2zim/constants.py

@@ -1,7 +1,6 @@
 import pathlib
 import tempfile
 import urllib.parse
-from dataclasses import dataclass
 
 from zimscraperlib.i18n import get_language_details
 
@@ -144,6 +143,24 @@
 # https://www.ifixit.com/Guide/MacBook+Air+11-Inch+Late+2010+Battery+Replacement/4384
 # https://www.ifixit.com/Teardown/Apple+Watch+Teardown/40655
 
+TITLE = {
+    "en": {
+        "title_en": "iFixit in English",
+        "title_fr": "iFixit in French",
+        "title_pt": "iFixit in Portuguese",
+        "title_de": "iFixit in German",
+        "title_ko": "iFixit in Korean",
+        "title_zh": "iFixit in Chinese",
+        "title_ru": "iFixit in Russian",
+        "title_nl": "iFixit in Dutch",
+        "title_ja": "iFixit in Japanese",
+        "title_tr": "iFixit in Turkish",
+        "title_es": "iFixit in Spanish",
+        "title_it": "iFixit in Italian",
+    },
+    "fr": {"title_fr": "iFixit en Français"},
+}
+
 HOME_LABELS = {
     "en": {"top_title": "Repair guides for every thing, written by everyone."},
     "fr": {"top_title": "Tutoriels de réparation pour tout, écrits par tous."},
@@ -800,7 +817,6 @@
 UNAVAILABLE_OFFLINE_INFOS = ["toolkits"]
 
 
-@dataclass
 class Configuration:
     fpath: pathlib.Path
 
@@ -815,14 +831,14 @@ class Configuration:
     tag: list[str]
 
     # filesystem
-    _output_dir: str  # TODO: rename output_name
-    _tmp_dir: str  # IDEM
-    output_dir: pathlib.Path  # TODO: rename output_path
-    tmp_dir: pathlib.Path  # IDEM
+    _output_name: str
+    _tmp_name: str
+    output_path: pathlib.Path
+    tmp_path: pathlib.Path
 
     required = (
         "lang_code",
-        "output_dir",
+        "output_path",
     )
 
     lang_code: str
@@ -859,35 +875,21 @@ class Configuration:
     stats_filename: str | None
     skip_checks: bool
 
-    @staticmethod
-    def get_url(lang_code: str) -> urllib.parse.ParseResult:
-        return urllib.parse.urlparse(URLS[lang_code])
-
-    @property
-    def domain(self) -> str:
-        return self.main_url.netloc
-
-    @property
-    def api_url(self) -> str:
-        return self.main_url.geturl() + API_PREFIX
-
-    @property
-    def s3_url(self) -> str | None:
-        return self.s3_url_with_credentials
-
-    def __post_init__(self):
+    def __init__(self, **kwargs):
+        for key, value in kwargs.items():
+            self.__setattr__(key, value)
         self.main_url = Configuration.get_url(self.lang_code)
         self.language = get_language_details(self.lang_code)
-        self.output_dir = pathlib.Path(self._output_dir).expanduser().resolve()
-        self.output_dir.mkdir(parents=True, exist_ok=True)
+        self.output_path = pathlib.Path(self._output_name).expanduser().resolve()
+        self.output_path.mkdir(parents=True, exist_ok=True)
 
-        self.tmp_dir = pathlib.Path(self._tmp_dir).expanduser().resolve()
-        self.tmp_dir.mkdir(parents=True, exist_ok=True)
+        self.tmp_path = pathlib.Path(self._tmp_name).expanduser().resolve()
+        self.tmp_path.mkdir(parents=True, exist_ok=True)
         if self.build_dir_is_tmp_dir:
-            self.build_dir = self.tmp_dir
+            self.build_path = self.tmp_path
         else:
-            self.build_dir = pathlib.Path(
-                tempfile.mkdtemp(prefix=f"ifixit_{self.lang_code}_", dir=self.tmp_dir)
+            self.build_path = pathlib.Path(
+                tempfile.mkdtemp(prefix=f"ifixit_{self.lang_code}_", dir=self.tmp_path)
             )
 
         self.stats_path = None
@@ -901,3 +903,19 @@ def __post_init__(self):
                 if ";" in tag:
                     self.tag += [p.strip() for p in tag.split(";")]
                     self.tag.remove(tag)
+
+    @staticmethod
+    def get_url(lang_code: str) -> urllib.parse.ParseResult:
+        return urllib.parse.urlparse(URLS[lang_code])
+
+    @property
+    def domain(self) -> str:
+        return self.main_url.netloc
+
+    @property
+    def api_url(self) -> str:
+        return self.main_url.geturl() + API_PREFIX
+
+    @property
+    def s3_url(self) -> str | None:
+        return self.s3_url_with_credentials

src/ifixit2zim/context.py

@@ -0,0 +1,21 @@
+import threading
+from dataclasses import dataclass
+from typing import Any
+
+from jinja2 import Environment
+from zimscraperlib.zim.creator import Creator
+
+from ifixit2zim.processor import Processor
+from ifixit2zim.scraper import Configuration
+from ifixit2zim.utils import Utils
+
+
+@dataclass
+class Context:
+    lock: threading.Lock
+    configuration: Configuration
+    creator: Creator
+    utils: Utils
+    metadata: dict[str, Any]
+    env: Environment
+    processor: Processor

src/ifixit2zim/entrypoint.py

@@ -26,7 +26,7 @@ def main():
         "--output",
         help="Output folder for ZIM file",
         default="/output",
-        dest="_output_dir",
+        dest="_output_name",
     )
 
     parser.add_argument(
@@ -103,7 +103,7 @@ def main():
         "--tmp-dir",
         help="Path to create temp folder in. Used for building ZIM file.",
         default=os.getenv("TMPDIR", "."),
-        dest="_tmp_dir",
+        dest="_tmp_name",
     )
 
     parser.add_argument(
@@ -274,7 +274,7 @@ def main():
         scraper = IFixit2Zim(**dict(args._get_kwargs()))
         sys.exit(scraper.run())
     except Exception as exc:
-        logger.error(f"FAILED. An error occurred: {exc}")
+        logger.error("FAILED. An error occurred", exc_info=exc)
         if args.debug:
             logger.exception(exc)
         raise SystemExit(1) from None

src/ifixit2zim/imager.py

@@ -5,27 +5,42 @@
 import io
 import pathlib
 import re
+import threading
 import urllib.parse
 
 from kiwixstorage import KiwixStorage, NotFoundError
 from PIL import Image
 from zimscraperlib.download import stream_file
 from zimscraperlib.image.optimization import optimize_webp
+from zimscraperlib.zim.creator import Creator
 
 from ifixit2zim.constants import IMAGES_ENCODER_VERSION
-from ifixit2zim.scraper import IFixit2Zim
+from ifixit2zim.executor import Executor
+from ifixit2zim.scraper import Configuration
 from ifixit2zim.shared import logger
+from ifixit2zim.utils import Utils
 
 
 class Imager:
-    def __init__(self, scraper: IFixit2Zim):
+    def __init__(
+        self,
+        img_executor: Executor,
+        lock: threading.Lock,
+        creator: Creator,
+        utils: Utils,
+        configuration: Configuration,
+    ):
         self.aborted = False
         # list of source URLs that we've processed and added to ZIM
         self.handled = set()
         self.dedup_items = {}
-        self.scraper = scraper
+        self.img_executor = img_executor
+        self.lock = lock
+        self.creator = creator
+        self.utils = utils
+        self.configuration = configuration
 
-        self.scraper.img_executor.start()
+        self.img_executor.start()
 
     def abort(self):
         """request imager to cancel processing of futures"""
@@ -70,7 +85,7 @@ def defer(self, url: str) -> str | None:
 
         # find actual URL should it be from a provider
         try:
-            parsed_url = urllib.parse.urlparse(self.scraper.utils.to_url(url))
+            parsed_url = urllib.parse.urlparse(self.utils.to_url(url))
         except Exception:
             logger.warning(f"Can't parse image URL `{url}`. Skipping")
             return
@@ -89,7 +104,7 @@ def defer(self, url: str) -> str | None:
         # record that we are processing this one
         self.handled.add(path)
 
-        self.scraper.img_executor.submit(
+        self.img_executor.submit(
             self.process_image,
             url=parsed_url,
             path=path,
@@ -108,22 +123,22 @@ def check_for_duplicate(self, path, content):
 
     def add_image_to_zim(self, path, content, mimetype):
         duplicate_path = self.check_for_duplicate(path, content)
-        with self.scraper.lock:
+        with self.lock:
             if duplicate_path:
-                self.scraper.creator.add_redirect(
+                self.creator.add_redirect(
                     path=path,
                     target_path=duplicate_path,
                 )
             else:
-                self.scraper.creator.add_item_for(
+                self.creator.add_item_for(
                     path=path,
                     content=content,
                     mimetype=mimetype,
                 )
 
     def add_missing_image_to_zim(self, path):
-        with self.scraper.lock:
-            self.scraper.creator.add_redirect(
+        with self.lock:
+            self.creator.add_redirect(
                 path=path,
                 target_path="assets/NoImage_300x225.jpg",
             )
@@ -137,7 +152,7 @@ def process_image(
             return
 
         # just download, optimize and add to ZIM if not using S3
-        if not self.scraper.configuration.s3_url:
+        if not self.configuration.s3_url:
             try:
                 fileobj = self.get_image_data(url.geturl())
             except Exception as exc:
@@ -159,7 +174,7 @@ def process_image(
             return path
 
         # we are using S3 cache
-        ident = self.scraper.utils.get_version_ident_for(url.geturl())
+        ident = self.utils.get_version_ident_for(url.geturl())
         if ident is None:
             logger.error(f"Unable to query {url.geturl()}. Skipping")
             self.add_missing_image_to_zim(
@@ -168,7 +183,7 @@ def process_image(
             return path
 
         # key = self.get_s3_key_for(url.geturl())
-        s3_storage = KiwixStorage(self.scraper.configuration.s3_url)
+        s3_storage = KiwixStorage(self.configuration.s3_url)
         meta = {"ident": ident, "encoder_version": str(IMAGES_ENCODER_VERSION)}
 
         download_failed = False  # useful to trigger reupload or not