From f6c134b59286fff6320feacc2690bcf07012bfe7 Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Mon, 8 Aug 2022 15:53:33 +0200 Subject: [PATCH] Fix the TUF Repository Service This commit fixes some bug implementation from the last commit. This improves the internal functions that require the role names to work correctly once the role object has no explicit type (name), adds the SPEC_VERSION in the service, and the JSONSerializer for persisting the files with a better appearance. Signed-off-by: Kairo de Araujo --- requirements/main.in | 2 +- requirements/main.txt | 11 +++++--- warehouse/tuf/__init__.py | 2 +- warehouse/tuf/services.py | 56 ++++++++++++++++++++------------------- 4 files changed, 39 insertions(+), 32 deletions(-) diff --git a/requirements/main.in b/requirements/main.in index 6974135c528d..b0682cc75112 100644 --- a/requirements/main.in +++ b/requirements/main.in @@ -62,7 +62,7 @@ stdlib-list structlog transaction trove-classifiers -tuf==1.0.0 +tuf==1.1.0 typeguard webauthn>=1.0.0,<2.0.0 whitenoise diff --git a/requirements/main.txt b/requirements/main.txt index afa8decba0a5..eef73875e81e 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -1141,6 +1141,7 @@ requests==2.28.1 \ # google-cloud-storage # premailer # requests-aws4auth + # tuf requests-aws4auth==1.1.2 \ --hash=sha256:23b7a054326f80f86caf87e3eaf54ea41aa27adbed4297bd3456b1fa38f06a52 \ --hash=sha256:ebde0662dccda5023546055ec4cbe4470cae017ecbfce8d368b80b5e4a94d619 @@ -1167,6 +1168,10 @@ securesystemslib==0.22.0 \ # via # -r requirements/main.in # tuf +sentry-sdk==1.6.0 \ + --hash=sha256:b82ad57306d5546713f15d5d70daea0408cf7f998c7566db16e0e6257e51e561 \ + --hash=sha256:ddbd191b6f4e696b7845b4d87389898ae1207981faf114f968a57363aa6be03c + # via -r requirements/main.in six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 @@ -1265,9 +1270,9 @@ trove-classifiers==2022.8.7 \ --hash=sha256:10053f2df40092c2d65d750cf12fab6df28714e422114fed91465814939a3284 \ --hash=sha256:c0618efe27904e272de1212143c8554657eef8adcf9800f1d2a1cf7164cc185b # via -r requirements/main.in -tuf==1.0.0 \ - --hash=sha256:6d967f992d494678b684a6f5f03d948130e2d9956310c7af74fa2f0296e8ec4d \ - --hash=sha256:ec74150954d56e206fa72f3b0234ea802649370e528dea5a83e507bd408fda82 +tuf==1.1.0 \ + --hash=sha256:28cd35eafa5aa4223eba03a397d14acb57c522381180db9ff3b54477dcbe1b73 \ + --hash=sha256:512a864789e291b5e8f5a5ace0e87b2d158303364a77ad6e53ffd042ba2b4933 # via -r requirements/main.in typeguard==2.13.3 \ --hash=sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4 \ diff --git a/warehouse/tuf/__init__.py b/warehouse/tuf/__init__.py index a82a44734ba5..19dd21ccb1c7 100644 --- a/warehouse/tuf/__init__.py +++ b/warehouse/tuf/__init__.py @@ -13,7 +13,7 @@ from celery.schedules import crontab from warehouse.tuf.interfaces import IKeyService, IRepositoryService, IStorageService -from warehouse.tuf.repository import SPEC_VERSION +from warehouse.tuf.services import SPEC_VERSION from warehouse.tuf.tasks import bump_bin_n_roles, bump_snapshot diff --git a/warehouse/tuf/services.py b/warehouse/tuf/services.py index b520083b710c..c418024adea0 100644 --- a/warehouse/tuf/services.py +++ b/warehouse/tuf/services.py @@ -26,6 +26,7 @@ ) from securesystemslib.signer import SSlibSigner # type: ignore from tuf.api.metadata import ( + SPECIFICATION_VERSION, TOP_LEVEL_ROLE_NAMES, DelegatedRole, Delegations, @@ -39,6 +40,7 @@ Targets, Timestamp, ) +from tuf.api.serialization.json import JSONSerializer from zope.interface import implementer from warehouse.config import Environment @@ -46,6 +48,8 @@ from warehouse.tuf.hash_bins import HashBins from warehouse.tuf.interfaces import IKeyService, IRepositoryService, IStorageService +SPEC_VERSION: str = ".".join(SPECIFICATION_VERSION) + class InsecureKeyWarning(UserWarning): pass @@ -212,30 +216,30 @@ def _load(self, role_name: str) -> Metadata: """ return Metadata.from_file(role_name, None, self._storage_backend) - def _sign(self, role: Metadata, key_id: Optional[str] = None) -> None: + def _sign(self, role: Metadata, role_name: str) -> None: """Re-signs metadata with role-specific key from global key store. The metadata role type is used as default key id. This is only allowed for top-level roles. """ role.signatures.clear() - for signer in self._key_storage_backend.get(key_id or role.type): + for signer in self._key_storage_backend.get(role_name): role.sign(signer, append=True) - def _persist(self, role: Metadata, role_name: Optional[str] = None) -> None: + def _persist(self, role: Metadata, role_name: str) -> None: """Persists metadata using the configured storage backend. The metadata role type is used as default role name. This is only allowed for top-level roles. All names but 'timestamp' are prefixed with a version number. """ - filename = f"{role_name or role.type}.json" + filename = f"{role_name}.json" - if not isinstance(role, Timestamp): + if role_name != RoleType.TIMESTAMP.value: filename = f"{role.signed.version}.{filename}" - Metadata.to_file(filename, None, self._storage_backend) + role.to_file(filename, JSONSerializer(), self._storage_backend) - def _bump_expiry(self, role: Metadata, expiry_id: Optional[str] = None) -> None: + def _bump_expiry(self, role: Metadata, expiry_id: str) -> None: """Bumps metadata expiration date by role-specific interval. The metadata role type is used as default expiry id. This is only allowed for @@ -247,9 +251,7 @@ def _bump_expiry(self, role: Metadata, expiry_id: Optional[str] = None) -> None: # consistent snapshot only 'timestamp' is bumped: # https://www.python.org/dev/peps/pep-0458/#producing-consistent-snapshots role.signed.expires = datetime.now().replace(microsecond=0) + timedelta( - seconds=self._request.registry.settings[ - f"tuf.{expiry_id or role.type}.expiry" - ] + seconds=self._request.registry.settings[f"tuf.{expiry_id}.expiry"] ) def _bump_version(self, role: Metadata) -> None: @@ -263,9 +265,9 @@ def _update_timestamp(self, snapshot_version: int) -> Metadata[Timestamp]: timestamp.signed.snapshot_meta = MetaFile(version=snapshot_version) self._bump_version(timestamp) - self._bump_expiry(timestamp) - self._sign(timestamp) - self._persist(timestamp) + self._bump_expiry(timestamp, RoleType.TIMESTAMP.value) + self._sign(timestamp, RoleType.TIMESTAMP.value) + self._persist(timestamp, RoleType.TIMESTAMP.value) def _update_snapshot( self, targets_meta: List[Tuple[str, int]] @@ -278,10 +280,10 @@ def _update_snapshot( for name, version in targets_meta: snapshot.signed.meta[f"{name}.json"] = MetaFile(version=version) - self._bump_expiry(snapshot) + self._bump_expiry(snapshot, RoleType.SNAPSHOT.value) self._bump_version(snapshot) - self._sign(snapshot) - self._persist(snapshot) + self._sign(snapshot, RoleType.SNAPSHOT.value) + self._persist(snapshot, RoleType.SNAPSHOT.value) return snapshot.signed.version @@ -325,9 +327,9 @@ def init_dev_repository(self): # Add signature wrapper, bump expiration, and sign and persist for role in [targets, snapshot, timestamp, root]: metadata = Metadata(role) - self._bump_expiry(metadata) - self._sign(metadata) - self._persist(metadata) + self._bump_expiry(metadata, role.type) + self._sign(metadata, role.type) + self._persist(metadata, role.type) def init_targets_delegation(self): """ @@ -370,11 +372,11 @@ def init_targets_delegation(self): # Bump version and expiration, and sign and persist updated 'targets'. self._bump_version(targets) - self._bump_expiry(targets) - self._sign(targets) - self._persist(targets) + self._bump_expiry(targets, RoleType.TARGETS.value) + self._sign(targets, RoleType.TARGETS.value) + self._persist(targets, RoleType.TARGETS.value) - targets_meta.append((Targets.type, targets.signed.version)) + targets_meta.append((RoleType.TARGETS.value, targets.signed.version)) # Create new 'bins' role and delegate trust from 'bins' for all target files to # 'bin-n' roles based on file path hash prefixes, a.k.a hash bin delegation. @@ -399,9 +401,9 @@ def init_targets_delegation(self): # Create new empty 'bin-n' roles, bump expiration, and sign and persist bin_n = Metadata(Targets()) - self._bump_expiry(bin_n) - self._sign(bin_n) - self._persist(bin_n) + self._bump_expiry(bin_n, RoleType.BIN_N.value) + self._sign(bin_n, RoleType.BIN_N.value) + self._persist(bin_n, bin_n_name) # FIXME: Possible performance gain by updating 'snapshot' right here, to # omit creation of massive list and iterating over all 'bin-n' roles twice. @@ -440,7 +442,7 @@ def add_hashed_targets(self, targets): # Update target file info in responsible 'bin-n' roles, bump version and expiry # and sign and persist targets_meta = [] - for bin_n_name, target_files in bin_n_target_groups: + for bin_n_name, target_files in bin_n_target_groups.items(): bin_n = self._load(bin_n_name) for target_file in target_files: