Switch to ruff and pyproject.toml

nabla-c0d3 · Nov 24, 2024 · 0882f35 · 0882f35
1 parent 2853f0e
commit 0882f35
Show file tree

Hide file tree

Showing 18 changed files with 183 additions and 69 deletions.
diff --git a/main.py b/main.py
@@ -73,7 +73,12 @@ def refresh_trust_stores() -> None:
         if has_store_changed:
             print(f"Detected changes for {platform.name}; updating store...")
             with open(store_path, mode="w") as store_file:
-                yaml.dump(fetched_store, store_file, encoding="utf-8", default_flow_style=False)
+                yaml.dump(
+                    fetched_store,
+                    store_file,
+                    encoding="utf-8",
+                    default_flow_style=False,
+                )
         else:
             print(f"No changes detected for {platform.name}")
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,7 @@
+[tool.ruff]
+line-length = 120
+
+[[tool.mypy.overrides]]
+module = "jks.*"
+ignore_missing_imports = true
+
diff --git a/requirements.txt b/requirements.txt
@@ -6,7 +6,7 @@ pyyaml==6.0.1
 # Dev dependencies
 mypy==1.0.1
 types-PyYAML
-flake8
 pytest
-black==22.6.0
+ruff==0.8.0
 invoke
+types-beautifulsoup4
diff --git a/setup.cfg b/setup.cfg
diff --git a/tasks.py b/tasks.py
@@ -7,9 +7,8 @@
 @task
 def test(ctx):
     # Run linters
-    ctx.run("flake8 .")
+    ctx.run("ruff check")
     ctx.run("mypy trust_stores_observatory")
-    ctx.run("black -l 120 . --check")
 
     # Run the test suite
     ctx.run("pytest")
diff --git a/trust_stores_observatory/certificates_repository.py b/trust_stores_observatory/certificates_repository.py
@@ -40,7 +40,9 @@ def get_all_certificates(self) -> List[Certificate]:
         return self._all_certificates
 
     def lookup_certificate_with_fingerprint(
-        self, fingerprint: bytes, hash_algorithm: Union[hashes.SHA1, hashes.SHA256] = hashes.SHA256()
+        self,
+        fingerprint: bytes,
+        hash_algorithm: Union[hashes.SHA1, hashes.SHA256] = hashes.SHA256(),
     ) -> Certificate:
         hex_fingerprint = fingerprint.hex()
         if isinstance(hash_algorithm, hashes.SHA1):

diff --git a/trust_stores_observatory/root_record.py b/trust_stores_observatory/root_record.py
@@ -7,7 +7,9 @@
 from trust_stores_observatory.certificate_utils import CertificateUtils
 
 if TYPE_CHECKING:
-    from trust_stores_observatory.store_fetcher.scraped_root_record import ScrapedRootCertificateRecord  # noqa: F401
+    from trust_stores_observatory.store_fetcher.scraped_root_record import (
+        ScrapedRootCertificateRecord,
+    )  # noqa: F401
 
 
 class RootCertificateRecord:

diff --git a/trust_stores_observatory/store_fetcher/__init__.py b/trust_stores_observatory/store_fetcher/__init__.py
@@ -1,13 +1,25 @@
 from typing import Dict, Type
 
 from trust_stores_observatory.certificates_repository import RootCertificatesRepository
-from trust_stores_observatory.store_fetcher.apple_store_fetcher import AppleTrustStoreFetcher
-from trust_stores_observatory.store_fetcher.google_aosp_fetcher import AospTrustStoreFetcher
-from trust_stores_observatory.store_fetcher.microsoft_fetcher import MicrosoftTrustStoreFetcher
-from trust_stores_observatory.store_fetcher.mozilla_fetcher import MozillaTrustStoreFetcher
+from trust_stores_observatory.store_fetcher.apple_store_fetcher import (
+    AppleTrustStoreFetcher,
+)
+from trust_stores_observatory.store_fetcher.google_aosp_fetcher import (
+    AospTrustStoreFetcher,
+)
+from trust_stores_observatory.store_fetcher.microsoft_fetcher import (
+    MicrosoftTrustStoreFetcher,
+)
+from trust_stores_observatory.store_fetcher.mozilla_fetcher import (
+    MozillaTrustStoreFetcher,
+)
 from trust_stores_observatory.store_fetcher.java_fetcher import JavaTrustStoreFetcher
-from trust_stores_observatory.store_fetcher.openjdk_fetcher import OpenJDKTrustStoreFetcher
-from trust_stores_observatory.store_fetcher.store_fetcher_interface import StoreFetcherInterface
+from trust_stores_observatory.store_fetcher.openjdk_fetcher import (
+    OpenJDKTrustStoreFetcher,
+)
+from trust_stores_observatory.store_fetcher.store_fetcher_interface import (
+    StoreFetcherInterface,
+)
 from trust_stores_observatory.trust_store import PlatformEnum, TrustStore
 
 
@@ -24,6 +36,9 @@ class TrustStoreFetcher:
     }
 
     def fetch(
-        self, platform: PlatformEnum, certs_repo: RootCertificatesRepository, should_update_repo: bool = True
+        self,
+        platform: PlatformEnum,
+        certs_repo: RootCertificatesRepository,
+        should_update_repo: bool = True,
     ) -> TrustStore:
         return self._FETCHER_CLS[platform]().fetch(certs_repo, should_update_repo)
diff --git a/trust_stores_observatory/store_fetcher/apple_store_fetcher.py b/trust_stores_observatory/store_fetcher/apple_store_fetcher.py
@@ -8,17 +8,22 @@
 from cryptography.hazmat.primitives import hashes
 
 from trust_stores_observatory.certificates_repository import RootCertificatesRepository
-from trust_stores_observatory.store_fetcher.root_records_validator import RootRecordsValidator
-from trust_stores_observatory.store_fetcher.scraped_root_record import ScrapedRootCertificateRecord
-from trust_stores_observatory.store_fetcher.store_fetcher_interface import StoreFetcherInterface
+from trust_stores_observatory.store_fetcher.root_records_validator import (
+    RootRecordsValidator,
+)
+from trust_stores_observatory.store_fetcher.scraped_root_record import (
+    ScrapedRootCertificateRecord,
+)
+from trust_stores_observatory.store_fetcher.store_fetcher_interface import (
+    StoreFetcherInterface,
+)
 from trust_stores_observatory.trust_store import TrustStore, PlatformEnum
 
 
 _logger = logging.getLogger(__file__)
 
 
 class AppleTrustStoreFetcher(StoreFetcherInterface):
-
     _INDEX_PAGE_URL = "https://support.apple.com/en-us/103272"
 
     def fetch(self, certs_repo: RootCertificatesRepository, should_update_repo: bool = True) -> TrustStore:
@@ -43,11 +48,17 @@ def fetch(self, certs_repo: RootCertificatesRepository, should_update_repo: bool
         validated_trusted_certs = RootRecordsValidator.validate_with_repository(certs_repo, parsed_trusted_certs)
 
         return TrustStore(
-            PlatformEnum.APPLE, os_version, trust_store_url, datetime.now(UTC).date(), validated_trusted_certs
+            PlatformEnum.APPLE,
+            os_version,
+            trust_store_url,
+            datetime.now(UTC).date(),
+            validated_trusted_certs,
         )
 
     @staticmethod
-    def _parse_root_records_in_div(certs_section: BeautifulSoup) -> List[ScrapedRootCertificateRecord]:
+    def _parse_root_records_in_div(
+        certs_section: BeautifulSoup,
+    ) -> List[ScrapedRootCertificateRecord]:
         # Look for each certificate entry in the table
         root_records = []
         for tr_tag in certs_section.find_all("tr"):
@@ -71,8 +82,13 @@ def _find_latest_root_certificates_page(cls) -> Tuple[str, str]:
         parsed_page = BeautifulSoup(page_content, "html.parser")
 
         # The page contains links to the root certificates page for each version of iOS/macOS - find the latest one
-        section_current = parsed_page.find("h2", text="Current Root Store").parent
-        for p_tag in section_current.find_all("p"):
+        current_root_store_h2_tag = parsed_page.find("h2", text="Current Root Store")
+        assert current_root_store_h2_tag
+
+        current_section = current_root_store_h2_tag.parent
+        assert current_section
+
+        for p_tag in current_section.find_all("p"):
             if "List of available root certificates in" in p_tag.text:
                 os_and_version = p_tag.text.split("List of available root certificates in")[1].strip()
                 trust_store_path = p_tag.a["href"]

diff --git a/trust_stores_observatory/store_fetcher/google_aosp_fetcher.py b/trust_stores_observatory/store_fetcher/google_aosp_fetcher.py
@@ -13,14 +13,19 @@
 
 from trust_stores_observatory.certificate_utils import CertificateUtils
 from trust_stores_observatory.certificates_repository import RootCertificatesRepository
-from trust_stores_observatory.store_fetcher.root_records_validator import RootRecordsValidator
-from trust_stores_observatory.store_fetcher.scraped_root_record import ScrapedRootCertificateRecord
-from trust_stores_observatory.store_fetcher.store_fetcher_interface import StoreFetcherInterface
+from trust_stores_observatory.store_fetcher.root_records_validator import (
+    RootRecordsValidator,
+)
+from trust_stores_observatory.store_fetcher.scraped_root_record import (
+    ScrapedRootCertificateRecord,
+)
+from trust_stores_observatory.store_fetcher.store_fetcher_interface import (
+    StoreFetcherInterface,
+)
 from trust_stores_observatory.trust_store import TrustStore, PlatformEnum
 
 
 class AospTrustStoreFetcher(StoreFetcherInterface):
-
     _REPO_URL = "https://android.googlesource.com/platform/system/ca-certificates"
 
     _GIT_CMD = 'git clone --branch master {repo_url} "{local_path}"'
@@ -41,13 +46,19 @@ def fetch(self, certs_repo: RootCertificatesRepository, should_update_repo: bool
 
                 # Find the latest tag that looks like android-8XXX - we don't care about android-iot or android-wear
                 tag_list = subprocess.check_output(
-                    self._GIT_FIND_TAG_CMD, shell=True, cwd=temp_dir.name, stderr=dev_null
+                    self._GIT_FIND_TAG_CMD,
+                    shell=True,
+                    cwd=temp_dir.name,
+                    stderr=dev_null,
                 ).decode("ascii")
                 last_tag = tag_list.strip().rsplit("\n", 1)[1].strip()
 
                 # Switch to this tag
                 subprocess.check_output(
-                    self._GIT_CHECKOUT_TAG_CMD.format(tag=last_tag), shell=True, cwd=temp_dir.name, stderr=dev_null
+                    self._GIT_CHECKOUT_TAG_CMD.format(tag=last_tag),
+                    shell=True,
+                    cwd=temp_dir.name,
+                    stderr=dev_null,
                 )
 
                 # Inspect each certificate
@@ -80,4 +91,10 @@ def fetch(self, certs_repo: RootCertificatesRepository, should_update_repo: bool
 
         date_fetched = datetime.now(UTC).date()
         version = last_tag.split("android-")[1]
-        return TrustStore(PlatformEnum.GOOGLE_AOSP, version, self._REPO_URL, date_fetched, trusted_cert_records)
+        return TrustStore(
+            PlatformEnum.GOOGLE_AOSP,
+            version,
+            self._REPO_URL,
+            date_fetched,
+            trusted_cert_records,
+        )
diff --git a/trust_stores_observatory/store_fetcher/java_fetcher.py b/trust_stores_observatory/store_fetcher/java_fetcher.py
@@ -9,15 +9,18 @@
 
 from tempfile import NamedTemporaryFile
 from trust_stores_observatory.certificates_repository import RootCertificatesRepository
-from trust_stores_observatory.store_fetcher.root_records_validator import RootRecordsValidator
-from trust_stores_observatory.store_fetcher.store_fetcher_interface import StoreFetcherInterface
+from trust_stores_observatory.store_fetcher.root_records_validator import (
+    RootRecordsValidator,
+)
+from trust_stores_observatory.store_fetcher.store_fetcher_interface import (
+    StoreFetcherInterface,
+)
 from trust_stores_observatory.trust_store import PlatformEnum, TrustStore
 
 from trust_stores_observatory.store_fetcher.jdk_helper import JdkPackage
 
 
 class JavaTrustStoreFetcher(StoreFetcherInterface):
-
     _BASE_URL = "https://www.oracle.com"
     _DOWNLOADS_INDEX = "/technetwork/java/javase/downloads/index.html"
 
@@ -56,7 +59,12 @@ def fetch(self, cert_repo: RootCertificatesRepository, should_update_repo: bool
         blacklisted_records = RootRecordsValidator.validate_with_repository(cert_repo, scraped_blacklisted_records)
 
         return TrustStore(
-            PlatformEnum.ORACLE_JAVA, version, final_url, datetime.utcnow().date(), trusted_records, blacklisted_records
+            PlatformEnum.ORACLE_JAVA,
+            version,
+            final_url,
+            datetime.utcnow().date(),
+            trusted_records,
+            blacklisted_records,
         )
 
     @classmethod
@@ -74,8 +82,14 @@ def _get_latest_download_url(cls) -> str:
                 pass
 
         # Find the link to the latest JRE's download page
-        href = main_page.find("img", alt="Download JDK").parent
-        latest_download_link = href.get("href")
+        download_jdk_img_tag = main_page.find("img", alt="Download JDK")
+        assert download_jdk_img_tag
+
+        href_tag = download_jdk_img_tag.parent
+        assert href_tag
+
+        latest_download_link = href_tag.get("href")
+        assert isinstance(latest_download_link, str)
 
         with urlopen(cls._BASE_URL + latest_download_link) as download_page:
             latest_download_page = download_page.read().decode("utf-8")

diff --git a/trust_stores_observatory/store_fetcher/jdk_helper.py b/trust_stores_observatory/store_fetcher/jdk_helper.py
@@ -8,7 +8,9 @@
 import tarfile
 from trust_stores_observatory.certificate_utils import CertificateUtils
 from trust_stores_observatory.certificates_repository import RootCertificatesRepository
-from trust_stores_observatory.store_fetcher.scraped_root_record import ScrapedRootCertificateRecord
+from trust_stores_observatory.store_fetcher.scraped_root_record import (
+    ScrapedRootCertificateRecord,
+)
 
 
 class JdkPackage:
@@ -30,7 +32,10 @@ def __enter__(self) -> "JdkPackage":
         return self
 
     def __exit__(
-        self, exc_type: Optional[Type[BaseException]], exc_value: Optional[BaseException], traceback: Optional[Any]
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_value: Optional[BaseException],
+        traceback: Optional[Any],
     ) -> None:
         if self._tar_file:
             self._tar_file.close()
@@ -62,7 +67,9 @@ def get_cacerts_password(self) -> str:
 
     @staticmethod
     def extract_trusted_root_records(
-        key_store: jks.KeyStore, should_update_repo: bool, cert_repo: RootCertificatesRepository
+        key_store: jks.KeyStore,
+        should_update_repo: bool,
+        cert_repo: RootCertificatesRepository,
     ) -> List[ScrapedRootCertificateRecord]:
         root_records = []
         for alias, item in key_store.certs.items():
@@ -81,14 +88,20 @@ def extract_trusted_root_records(
         return root_records
 
     @staticmethod
-    def extract_blacklisted_root_records(blacklisted_certs_content: str) -> List[ScrapedRootCertificateRecord]:
+    def extract_blacklisted_root_records(
+        blacklisted_certs_content: str,
+    ) -> List[ScrapedRootCertificateRecord]:
         # The file only contains a list of SHA-256 fingerprints
         blacklisted_records = []
         for fingerprint in blacklisted_certs_content.split("\n")[1:]:
             if not fingerprint:
                 continue
             blacklisted_records.append(
-                ScrapedRootCertificateRecord("Blacklisted", bytes(bytearray.fromhex(fingerprint)), hashes.SHA256())
+                ScrapedRootCertificateRecord(
+                    "Blacklisted",
+                    bytes(bytearray.fromhex(fingerprint)),
+                    hashes.SHA256(),
+                )
             )
 
         return blacklisted_records
diff --git a/trust_stores_observatory/store_fetcher/microsoft_fetcher.py b/trust_stores_observatory/store_fetcher/microsoft_fetcher.py
@@ -5,9 +5,15 @@
 from cryptography.hazmat.primitives import hashes
 
 from trust_stores_observatory.certificates_repository import RootCertificatesRepository
-from trust_stores_observatory.store_fetcher.root_records_validator import RootRecordsValidator
-from trust_stores_observatory.store_fetcher.scraped_root_record import ScrapedRootCertificateRecord
-from trust_stores_observatory.store_fetcher.store_fetcher_interface import StoreFetcherInterface
+from trust_stores_observatory.store_fetcher.root_records_validator import (
+    RootRecordsValidator,
+)
+from trust_stores_observatory.store_fetcher.scraped_root_record import (
+    ScrapedRootCertificateRecord,
+)
+from trust_stores_observatory.store_fetcher.store_fetcher_interface import (
+    StoreFetcherInterface,
+)
 from trust_stores_observatory.trust_store import TrustStore, PlatformEnum