From 48c654653c6d2585ba0cc4a7069590e3defe809c Mon Sep 17 00:00:00 2001 From: Aryan Arora Date: Tue, 20 Feb 2024 13:59:53 +0530 Subject: [PATCH] migrate to generic syntax in all std collections --- contrib/encode_video.py | 5 +++-- src/zimscraperlib/download.py | 16 +++++++------- src/zimscraperlib/filesystem.py | 3 ++- src/zimscraperlib/fix_ogvjs_dist.py | 6 ++++-- src/zimscraperlib/i18n.py | 14 +++++++------ src/zimscraperlib/image/optimization.py | 5 +++-- src/zimscraperlib/image/presets.py | 28 +++++++++++++------------ src/zimscraperlib/image/probing.py | 10 +++++---- src/zimscraperlib/inputs.py | 6 ++++-- src/zimscraperlib/logging.py | 3 ++- src/zimscraperlib/types.py | 6 ++++-- src/zimscraperlib/video/config.py | 10 +++++---- src/zimscraperlib/video/encoding.py | 7 ++++--- src/zimscraperlib/video/presets.py | 14 +++++++------ src/zimscraperlib/zim/_libkiwix.py | 10 +++++---- src/zimscraperlib/zim/archive.py | 11 ++++++---- src/zimscraperlib/zim/creator.py | 12 ++++++----- src/zimscraperlib/zim/filesystem.py | 9 +++++--- src/zimscraperlib/zim/metadata.py | 8 ++++--- tests/download/test_download.py | 6 ++++-- tests/video/test_encoding.py | 8 ++++--- tests/zim/test_metadata.py | 5 ++++- 22 files changed, 121 insertions(+), 81 deletions(-) diff --git a/contrib/encode_video.py b/contrib/encode_video.py index 8b256e68..d4bb8570 100644 --- a/contrib/encode_video.py +++ b/contrib/encode_video.py @@ -1,6 +1,7 @@ +from __future__ import annotations + import sys from pathlib import Path -from typing import List from zimscraperlib import logger from zimscraperlib.video import presets, reencode @@ -25,7 +26,7 @@ def encode_video(src_path: Path, dst_path: Path, preset: str): logger.error(f"conversion failed:\n{process.stdout}") -def run(args: List[str] = sys.argv): +def run(args: list[str] = sys.argv): if len(args) < 4: # noqa: PLR2004 print(f"Usage: {args[0]} ") # noqa: T201 print( # noqa: T201 diff --git a/src/zimscraperlib/download.py b/src/zimscraperlib/download.py index 2b8268f6..43c00913 100644 --- a/src/zimscraperlib/download.py +++ b/src/zimscraperlib/download.py @@ -7,7 +7,7 @@ import pathlib import subprocess from concurrent.futures import Future, ThreadPoolExecutor -from typing import ClassVar, Dict, Optional, Union +from typing import ClassVar, Optional, Union import requests import yt_dlp as youtube_dl @@ -34,14 +34,14 @@ def shutdown(self) -> None: """shuts down the executor, awaiting completion""" self.executor.shutdown(wait=True) - def _run_youtube_dl(self, url: str, options: Dict) -> None: + def _run_youtube_dl(self, url: str, options: dict) -> None: with youtube_dl.YoutubeDL(options) as ydl: ydl.download([url]) def download( self, url: str, - options: Optional[Dict], + options: Optional[dict], wait: Optional[bool] = True, # noqa: FBT002 ) -> Union[bool, Future]: """Downloads video using initialized executor. @@ -65,8 +65,8 @@ def download( class YoutubeConfig(dict): - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {} - defaults: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {} + defaults: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "writethumbnail": True, "write_all_thumbnails": True, "writesubtitles": True, @@ -109,14 +109,14 @@ def get_options( class BestWebm(YoutubeConfig): - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "preferredcodec": "webm", "format": "best[ext=webm]/bestvideo[ext=webm]+bestaudio[ext=webm]/best", } class BestMp4(YoutubeConfig): - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "preferredcodec": "mp4", "format": "best[ext=mp4]/bestvideo[ext=mp4]+bestaudio[ext=m4a]/best", } @@ -179,7 +179,7 @@ def stream_file( proxies: Optional[dict] = None, only_first_block: Optional[bool] = False, # noqa: FBT002 max_retries: Optional[int] = 5, - headers: Optional[Dict[str, str]] = None, + headers: Optional[dict[str, str]] = None, session: Optional[requests.Session] = None, ) -> tuple[int, requests.structures.CaseInsensitiveDict]: # pyright: ignore """Stream data from a URL to either a BytesIO object or a file diff --git a/src/zimscraperlib/filesystem.py b/src/zimscraperlib/filesystem.py index 7b22dfe3..d94d979f 100644 --- a/src/zimscraperlib/filesystem.py +++ b/src/zimscraperlib/filesystem.py @@ -7,7 +7,8 @@ import os import pathlib -from typing import Any, Callable, Optional, Union +from collections.abc import Callable +from typing import Any, Optional, Union import magic diff --git a/src/zimscraperlib/fix_ogvjs_dist.py b/src/zimscraperlib/fix_ogvjs_dist.py index 72cb8972..082e4187 100755 --- a/src/zimscraperlib/fix_ogvjs_dist.py +++ b/src/zimscraperlib/fix_ogvjs_dist.py @@ -4,10 +4,12 @@ """ quick script to fix videojs-ogvjs so that it triggers on webm mimetype """ +from __future__ import annotations + import logging import pathlib import sys -from typing import List, Union +from typing import Union logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.DEBUG) logger = logging.getLogger(__name__) @@ -33,7 +35,7 @@ def fix_source_dir(source_vendors_path: Union[pathlib.Path, str]): logger.info("all done.") -def run(args: List[str] = sys.argv): +def run(args: list[str] = sys.argv): if len(args) < 2: # noqa: PLR2004 print(f"Usage: {args[0]} ") # noqa: T201 print( # noqa: T201 diff --git a/src/zimscraperlib/i18n.py b/src/zimscraperlib/i18n.py index 0c9e1948..badc5df8 100644 --- a/src/zimscraperlib/i18n.py +++ b/src/zimscraperlib/i18n.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 # vim: ai ts=4 sts=4 et sw=4 nu +from __future__ import annotations + import gettext import locale import pathlib import re -from typing import Dict, Optional, Tuple, Union +from typing import Optional, Union import babel import iso639 @@ -58,7 +60,7 @@ def setlocale(root_dir: pathlib.Path, locale_name: str): return Locale.setup(root_dir / "locale", locale_name) -def get_iso_lang_data(lang: str) -> Tuple[Dict, Union[Dict, None]]: +def get_iso_lang_data(lang: str) -> tuple[dict, Union[dict, None]]: """ISO-639-x languages details for lang. Raises NotFound Included keys: iso-639-1, iso-639-2b, iso-639-2t, iso-639-3, iso-639-5 @@ -112,8 +114,8 @@ def replace_types(new_type: str) -> str: def find_language_names( - query: str, lang_data: Optional[Dict] = None -) -> Tuple[str, str]: + query: str, lang_data: Optional[dict] = None +) -> tuple[str, str]: """(native, english) language names for lang with help from language_details dict Falls back to English name if available or query if not""" @@ -140,7 +142,7 @@ def find_language_names( return default, default -def update_with_macro(lang_data: Dict, macro_data: Dict): +def update_with_macro(lang_data: dict, macro_data: dict): """update empty keys from lang_data with ones of macro_data""" if macro_data: for key, value in macro_data.items(): @@ -151,7 +153,7 @@ def update_with_macro(lang_data: Dict, macro_data: Dict): def get_language_details( query: str, failsafe: Optional[bool] = False # noqa: FBT002 -) -> Dict: +) -> dict: """language details dict from query. Raises NotFound or return `und` language details if failsafe diff --git a/src/zimscraperlib/image/optimization.py b/src/zimscraperlib/image/optimization.py index bed4c8a5..ec90f533 100644 --- a/src/zimscraperlib/image/optimization.py +++ b/src/zimscraperlib/image/optimization.py @@ -22,12 +22,13 @@ can still run on default settings which give a bit less size than the original images but maintain a high quality. """ +from __future__ import annotations import io import os import pathlib import subprocess -from typing import Optional, Tuple, Union +from typing import Optional, Union import piexif from optimize_images.img_aux_processing import do_reduce_colors, rebuild_palette @@ -57,7 +58,7 @@ def optimize_png( max_colors: Optional[int] = 256, fast_mode: Optional[bool] = True, # noqa: FBT002 remove_transparency: Optional[bool] = False, # noqa: FBT002 - background_color: Optional[Tuple[int, int, int]] = (255, 255, 255), + background_color: Optional[tuple[int, int, int]] = (255, 255, 255), **options, # noqa: ARG001 ) -> Union[pathlib.Path, io.BytesIO]: """method to optimize PNG files using a pure python external optimizer diff --git a/src/zimscraperlib/image/presets.py b/src/zimscraperlib/image/presets.py index 8eab6991..8d06cadc 100644 --- a/src/zimscraperlib/image/presets.py +++ b/src/zimscraperlib/image/presets.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 # vim: ai ts=4 sts=4 et sw=4 nu -from typing import ClassVar, Dict, Optional, Union +from __future__ import annotations + +from typing import ClassVar, Optional, Union """ presets for ImageOptimizer in zimscraperlib.image.optimization module """ @@ -20,7 +22,7 @@ class WebpLow: ext = "webp" mimetype = f"{preset_type}/webp" - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "lossless": False, "quality": 40, "method": 6, @@ -39,7 +41,7 @@ class WebpMedium: ext = "webp" mimetype = f"{preset_type}/webp" - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "lossless": False, "quality": 50, "method": 6, @@ -58,7 +60,7 @@ class WebpHigh: ext = "webp" mimetype = f"{preset_type}/webp" - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "lossless": False, "quality": 90, "method": 6, @@ -79,7 +81,7 @@ class GifLow: ext = "gif" mimetype = f"{preset_type}/gif" - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "optimize_level": 3, "max_colors": 256, "lossiness": 80, @@ -102,7 +104,7 @@ class GifMedium: ext = "gif" mimetype = f"{preset_type}/gif" - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "optimize_level": 3, "lossiness": 20, "no_extensions": True, @@ -124,7 +126,7 @@ class GifHigh: ext = "gif" mimetype = f"{preset_type}/gif" - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "optimize_level": 2, "lossiness": None, "no_extensions": True, @@ -143,7 +145,7 @@ class PngLow: ext = "png" mimetype = f"{preset_type}/png" - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "reduce_colors": True, "remove_transparency": False, "max_colors": 256, @@ -162,7 +164,7 @@ class PngMedium: ext = "png" mimetype = f"{preset_type}/png" - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "reduce_colors": False, "remove_transparency": False, "fast_mode": False, @@ -180,7 +182,7 @@ class PngHigh: ext = "png" mimetype = f"{preset_type}/png" - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "reduce_colors": False, "remove_transparency": False, "fast_mode": True, @@ -199,7 +201,7 @@ class JpegLow: ext = "png" mimetype = f"{preset_type}/png" - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "quality": 45, "keep_exif": False, "fast_mode": True, @@ -218,7 +220,7 @@ class JpegMedium: ext = "jpg" mimetype = f"{preset_type}/jpeg" - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "quality": 65, "keep_exif": False, "fast_mode": True, @@ -237,7 +239,7 @@ class JpegHigh: ext = "jpg" mimetype = f"{preset_type}/jpeg" - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "quality": 80, "keep_exif": True, "fast_mode": True, diff --git a/src/zimscraperlib/image/probing.py b/src/zimscraperlib/image/probing.py index ac3badca..a314d09e 100644 --- a/src/zimscraperlib/image/probing.py +++ b/src/zimscraperlib/image/probing.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 # vim: ai ts=4 sts=4 et sw=4 nu +from __future__ import annotations + import colorsys import io import pathlib import re -from typing import Optional, Tuple, Union +from typing import Optional, Union import colorthief import PIL.Image @@ -13,14 +15,14 @@ def get_colors( src: pathlib.Path, use_palette: Optional[bool] = True # noqa: FBT002 -) -> Tuple[str, str]: +) -> tuple[str, str]: """(main, secondary) HTML color codes from an image path""" def rgb_to_hex(r: int, g: int, b: int) -> str: """hexadecimal HTML-friendly color code for RGB tuple""" return "#{}{}{}".format(*[str(hex(x)[2:]).zfill(2) for x in (r, g, b)]).upper() - def solarize(r: int, g: int, b: int) -> Tuple[int, int, int]: + def solarize(r: int, g: int, b: int) -> tuple[int, int, int]: # calculate solarized color for main h, l, s = colorsys.rgb_to_hls( # noqa: E741 float(r) / 256, float(g) / 256, float(b) / 256 @@ -71,7 +73,7 @@ def format_for( def is_valid_image( image: Union[pathlib.Path, io.IOBase, bytes], imformat: str, - size: Optional[Tuple[int, int]] = None, + size: Optional[tuple[int, int]] = None, ) -> bool: """whether image is a valid imformat (PNG) image, optionnaly of requested size""" if isinstance(image, bytes): diff --git a/src/zimscraperlib/inputs.py b/src/zimscraperlib/inputs.py index 4ab9e6ef..e0bbe859 100644 --- a/src/zimscraperlib/inputs.py +++ b/src/zimscraperlib/inputs.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 # vim: ai ts=4 sts=4 et sw=4 nu +from __future__ import annotations + import pathlib import shutil import tempfile -from typing import Optional, Tuple, Union +from typing import Optional, Union from zimscraperlib import logger from zimscraperlib.constants import ( @@ -60,7 +62,7 @@ def compute_descriptions( default_description: str, user_description: Optional[str], user_long_description: Optional[str], -) -> Tuple[str, Optional[str]]: +) -> tuple[str, Optional[str]]: """Computes short and long descriptions compliant with ZIM standard. Based on provided parameters, the function computes a short and a long description diff --git a/src/zimscraperlib/logging.py b/src/zimscraperlib/logging.py index 4a0f1881..f4a0c4ac 100644 --- a/src/zimscraperlib/logging.py +++ b/src/zimscraperlib/logging.py @@ -5,8 +5,9 @@ import logging import pathlib import sys +from collections.abc import Iterable from logging.handlers import RotatingFileHandler -from typing import Iterable, Optional +from typing import Optional from zimscraperlib.constants import NAME diff --git a/src/zimscraperlib/types.py b/src/zimscraperlib/types.py index de142111..81e8f049 100644 --- a/src/zimscraperlib/types.py +++ b/src/zimscraperlib/types.py @@ -15,13 +15,15 @@ Should your scraper need additional mapping, use mimetypes.add_type() and it will be automatically used. """ +from __future__ import annotations + import mimetypes import pathlib -from typing import List, Optional, Union +from typing import Optional, Union ARTICLE_MIME: str = "text/html" FALLBACK_MIME: str = "application/octet-stream" -FONT_MIMES: List[str] = [ +FONT_MIMES: list[str] = [ "font/ttf", "application/font-ttf", "font/sfnt", diff --git a/src/zimscraperlib/video/config.py b/src/zimscraperlib/video/config.py index 82d8ed54..2ad7b820 100644 --- a/src/zimscraperlib/video/config.py +++ b/src/zimscraperlib/video/config.py @@ -1,18 +1,20 @@ #!/usr/bin/env python3 # vim: ai ts=4 sts=4 et sw=4 nu -from typing import ClassVar, Dict, Optional, Union +from __future__ import annotations + +from typing import ClassVar, Optional, Union class Config(dict): VERSION = 1 ext = "dat" mimetype = "application/data" - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {} - defaults: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {} + defaults: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "-max_muxing_queue_size": "9999" } - mapping: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + mapping: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "video_codec": "-codec:v", "audio_codec": "-codec:a", "max_video_bitrate": "-maxrate", diff --git a/src/zimscraperlib/video/encoding.py b/src/zimscraperlib/video/encoding.py index f66618af..91d3873f 100644 --- a/src/zimscraperlib/video/encoding.py +++ b/src/zimscraperlib/video/encoding.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 # vim: ai ts=4 sts=4 et sw=4 nu +from __future__ import annotations import pathlib import shutil import subprocess import tempfile -from typing import List, Optional +from typing import Optional from zimscraperlib import logger from zimscraperlib.logging import nicer_args_join @@ -15,9 +16,9 @@ def _build_ffmpeg_args( src_path: pathlib.Path, tmp_path: pathlib.Path, - ffmpeg_args: List[str], + ffmpeg_args: list[str], threads: Optional[int], -) -> List[str]: +) -> list[str]: if threads: if "-threads" in ffmpeg_args: raise AttributeError("Cannot set the number of threads, already set") diff --git a/src/zimscraperlib/video/presets.py b/src/zimscraperlib/video/presets.py index 37b7bbba..f04e0a11 100644 --- a/src/zimscraperlib/video/presets.py +++ b/src/zimscraperlib/video/presets.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 # vim: ai ts=4 sts=4 et sw=4 nu -from typing import ClassVar, Dict, Optional, Union +from __future__ import annotations + +from typing import ClassVar, Optional, Union from zimscraperlib.video.config import Config @@ -20,7 +22,7 @@ class VoiceMp3Low(Config): ext = "mp3" mimetype = "audio/mp3" - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "-vn": "", # remove video stream "-codec:a": "mp3", # audio codec "-ar": "44100", # audio sampling rate @@ -40,7 +42,7 @@ class VideoWebmLow(Config): ext = "webm" mimetype = f"{preset_type}/webm" - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "-codec:v": "libvpx", # video codec "-quality": "best", # codec preset "-b:v": "128k", # Adjust quantizer within min/max to target this bitrate @@ -66,7 +68,7 @@ class VideoMp4Low(Config): ext = "mp4" mimetype = f"{preset_type}/mp4" - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "-codec:v": "h264", # video codec "-b:v": "300k", # target video bitrate "-maxrate": "300k", # max video bitrate @@ -91,7 +93,7 @@ class VideoWebmHigh(Config): ext = "webm" mimetype = f"{preset_type}/webm" - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "-codec:v": "libvpx", # video codec "-codec:a": "libvorbis", # audio codec "-crf": "25", # constant quality, lower value gives better qual and larger size @@ -109,7 +111,7 @@ class VideoMp4High(Config): ext = "mp4" mimetype = f"{preset_type}/mp4" - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = { + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = { "-codec:v": "h264", # video codec "-codec:a": "aac", # audio codec "-crf": "20", # constant quality, lower value gives better qual and larger size diff --git a/src/zimscraperlib/zim/_libkiwix.py b/src/zimscraperlib/zim/_libkiwix.py index 117b5b20..b4e3938d 100644 --- a/src/zimscraperlib/zim/_libkiwix.py +++ b/src/zimscraperlib/zim/_libkiwix.py @@ -12,9 +12,11 @@ https://github.com/kiwix/libkiwix/blob/master/src/tools/otherTools.cpp """ +from __future__ import annotations + import io from collections import namedtuple -from typing import Dict, List, Optional, Tuple +from typing import Dict, Optional MimetypeAndCounter = namedtuple("MimetypeAndCounter", ["mimetype", "value"]) CounterMap = Dict[ @@ -22,7 +24,7 @@ ] -def getline(src: io.StringIO, delim: Optional[bool] = None) -> Tuple[bool, str]: +def getline(src: io.StringIO, delim: Optional[bool] = None) -> tuple[bool, str]: """C++ stdlib getline() ~clone Reads `src` until it finds `delim`. @@ -42,7 +44,7 @@ def getline(src: io.StringIO, delim: Optional[bool] = None) -> Tuple[bool, str]: def readFullMimetypeAndCounterString( src: io.StringIO, -) -> Tuple[bool, str]: +) -> tuple[bool, str]: """read a single mimetype-and-counter string from source Returns whether the source is EOF and the extracted string (or empty one)""" @@ -87,7 +89,7 @@ def parseMimetypeCounter( return counters -def convertTags(tags_str: str) -> List[str]: +def convertTags(tags_str: str) -> list[str]: """List of tags expanded with libkiwix's additional hints for pic/vid/det/index""" tags = tags_str.split(";") tagsList = [] diff --git a/src/zimscraperlib/zim/archive.py b/src/zimscraperlib/zim/archive.py index 7d69ee9a..3c7caba1 100644 --- a/src/zimscraperlib/zim/archive.py +++ b/src/zimscraperlib/zim/archive.py @@ -9,7 +9,10 @@ - direct access to search results and number of results - public Entry access by Id""" -from typing import Dict, Iterable, List, Optional +from __future__ import annotations + +from collections.abc import Iterable +from typing import Optional import libzim.reader # pyright: ignore import libzim.search # Query, Searcher # pyright: ignore @@ -27,7 +30,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): pass @property - def metadata(self) -> Dict[str, str]: + def metadata(self) -> dict[str, str]: """key: value for all non-illustration metadata listed in .metadata_keys""" return { key: self.get_text_metadata(key) @@ -39,7 +42,7 @@ def metadata(self) -> Dict[str, str]: def tags(self): return self.get_tags() - def get_tags(self, libkiwix: bool = False) -> List[str]: # noqa: FBT001, FBT002 + def get_tags(self, libkiwix: bool = False) -> list[str]: # noqa: FBT001, FBT002 """List of ZIM tags, optionnaly expanded with libkiwix's hints""" try: tags_meta = self.get_text_metadata("Tags") @@ -100,7 +103,7 @@ def get_search_results_count(self, query: str) -> int: return search.getEstimatedMatches() @property - def counters(self) -> Dict[str, int]: + def counters(self) -> dict[str, int]: try: return parseMimetypeCounter(self.get_text_metadata("Counter")) except RuntimeError: # pragma: no cover (no ZIM avail to test itl) diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index 70ae11ed..d918e348 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -17,12 +17,14 @@ - content stored on object - can be used to store a filepath and content read from it (not stored) """ -import collections.abc +from __future__ import annotations + import datetime import pathlib import re import weakref -from typing import Any, Callable, Iterable, Optional, Tuple, Union +from collections.abc import Callable, Iterable +from typing import Any, Optional, Union import libzim.writer # pyright: ignore @@ -203,7 +205,7 @@ def add_metadata( name == "Tags" and not isinstance(content, str) and not isinstance(content, bytes) - and isinstance(content, collections.abc.Iterable) + and isinstance(content, Iterable) ): content = ";".join(content) super().add_metadata(name, content, mimetype) @@ -269,7 +271,7 @@ def add_item_for( delete_fpath: Optional[bool] = False, # noqa: FBT002 duplicate_ok: Optional[bool] = None, callback: Optional[ - Union[callable, Tuple[callable, Any]] # pyright: ignore + Union[callable, tuple[callable, Any]] # pyright: ignore ] = None, ): """Add a File or content at a specified path and get its path @@ -329,7 +331,7 @@ def add_item( self, item: libzim.writer.Item, duplicate_ok: Optional[bool] = None, - callback: Optional[Union[Callable, Tuple[Callable, Any]]] = None, + callback: Optional[Union[Callable, tuple[Callable, Any]]] = None, ): """Add a libzim.writer.Item diff --git a/src/zimscraperlib/zim/filesystem.py b/src/zimscraperlib/zim/filesystem.py index 0a81a7ee..15d9d2b7 100644 --- a/src/zimscraperlib/zim/filesystem.py +++ b/src/zimscraperlib/zim/filesystem.py @@ -26,10 +26,13 @@ Meaning you should exit right after an exception in your code (during zim creation) Use workaround_nocancel=False to disable the workaround. """ +from __future__ import annotations + import datetime import pathlib import re -from typing import Optional, Sequence, Tuple +from collections.abc import Sequence +from typing import Optional from zimscraperlib import logger from zimscraperlib.filesystem import get_file_mimetype @@ -90,7 +93,7 @@ def add_to_zim( def add_redirects_to_zim( zim_file: Creator, - redirects: Optional[Sequence[Tuple[str, str, Optional[str]]]] = None, + redirects: Optional[Sequence[tuple[str, str, Optional[str]]]] = None, redirects_file: Optional[pathlib.Path] = None, ): """add redirects from list of source/target or redirects file to zim""" @@ -128,7 +131,7 @@ def make_zim_file( scraper: str = None, # noqa: RUF013 # pyright: ignore long_description: str = None, # noqa: RUF013 # pyright: ignore without_fulltext_index: bool = False, # noqa: FBT001, FBT002, ARG001 - redirects: Sequence[Tuple[str, str, str]] = None, # noqa: RUF013 # pyright: ignore + redirects: Sequence[tuple[str, str, str]] = None, # noqa: RUF013 # pyright: ignore redirects_file: pathlib.Path = None, # noqa: RUF013 # pyright: ignore rewrite_links: bool = True, # noqa: FBT001, FBT002, ARG001 workaround_nocancel: bool = True, # noqa: FBT001, FBT002 diff --git a/src/zimscraperlib/zim/metadata.py b/src/zimscraperlib/zim/metadata.py index 66e92f13..430c9e8d 100644 --- a/src/zimscraperlib/zim/metadata.py +++ b/src/zimscraperlib/zim/metadata.py @@ -1,7 +1,9 @@ +from __future__ import annotations + import datetime import re -from collections.abc import Iterable as IterableT -from typing import Any, Iterable, Union +from collections.abc import Iterable +from typing import Any, Union from zimscraperlib.constants import ( ILLUSTRATIONS_METADATA_RE, @@ -97,7 +99,7 @@ def validate_longdescription(name: str, value: str): def validate_tags(name: str, value: Union[Iterable[str], str]): """ensures Tags metadata is either one or a list of strings""" if name == "Tags" and ( - not isinstance(value, IterableT) + not isinstance(value, Iterable) or not all(isinstance(tag, str) for tag in value) ): raise ValueError(f"Invalid type(s) for {name}") diff --git a/tests/download/test_download.py b/tests/download/test_download.py index 2c10093f..440dda79 100644 --- a/tests/download/test_download.py +++ b/tests/download/test_download.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 # vim: ai ts=4 sts=4 et sw=4 nu +from __future__ import annotations + import concurrent.futures import io import pathlib import re -from typing import ClassVar, Dict, Optional, Union +from typing import ClassVar, Optional, Union import pytest import requests @@ -232,7 +234,7 @@ def custom_outtmpl() -> str: class WrongOuttmplType(BestWebm): - options: ClassVar[Dict[str, Optional[Union[str, bool, int]]]] = {"outtmpl": 123} + options: ClassVar[dict[str, Optional[Union[str, bool, int]]]] = {"outtmpl": 123} def test_get_options_wrong_outtmpl_type(): diff --git a/tests/video/test_encoding.py b/tests/video/test_encoding.py index 292f660a..8f46af9b 100644 --- a/tests/video/test_encoding.py +++ b/tests/video/test_encoding.py @@ -1,6 +1,8 @@ +from __future__ import annotations + import re from pathlib import Path -from typing import List, Optional +from typing import Optional import pytest @@ -67,9 +69,9 @@ def test_build_ffmpeg_args( src_path: Path, tmp_path: Path, - ffmpeg_args: List[str], + ffmpeg_args: list[str], threads: Optional[int], - expected: Optional[List[str]], + expected: Optional[list[str]], ): if expected: assert ( diff --git a/tests/zim/test_metadata.py b/tests/zim/test_metadata.py index c4d96255..462fb202 100644 --- a/tests/zim/test_metadata.py +++ b/tests/zim/test_metadata.py @@ -1,5 +1,8 @@ +from __future__ import annotations + import re -from typing import Iterable, Union +from collections.abc import Iterable +from typing import Union import pytest