From d8f51344b3b3457415cede67ab3232165e035961 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 21 Aug 2024 16:25:47 -0400 Subject: [PATCH 01/34] Reapply "Store attestations for PEP740 (#16302)" (#16545) This reverts commit da7e1ed8cb087f3128b5d23dadc2f9ab63373625. --- requirements/main.in | 4 +- requirements/main.txt | 12 +- tests/common/db/attestation.py | 28 ++ tests/common/db/packaging.py | 7 + tests/unit/api/test_simple.py | 25 ++ tests/unit/attestations/__init__.py | 11 + tests/unit/attestations/test_init.py | 31 ++ tests/unit/attestations/test_services.py | 419 +++++++++++++++++ tests/unit/forklift/test_legacy.py | 420 +++--------------- tests/unit/packaging/test_utils.py | 36 ++ warehouse/attestations/__init__.py | 31 ++ .../errors.py} | 19 +- warehouse/attestations/interfaces.py | 54 +++ warehouse/attestations/models.py | 55 +++ warehouse/attestations/services.py | 231 ++++++++++ warehouse/forklift/legacy.py | 124 ++---- .../7f0c9f105f44_create_attestations_table.py | 1 + warehouse/packaging/models.py | 8 + warehouse/packaging/utils.py | 8 +- warehouse/templates/api/simple/detail.html | 2 +- 20 files changed, 1036 insertions(+), 490 deletions(-) create mode 100644 tests/common/db/attestation.py create mode 100644 tests/unit/attestations/__init__.py create mode 100644 tests/unit/attestations/test_init.py create mode 100644 tests/unit/attestations/test_services.py create mode 100644 warehouse/attestations/__init__.py rename warehouse/{migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py => attestations/errors.py} (66%) create mode 100644 warehouse/attestations/interfaces.py create mode 100644 warehouse/attestations/models.py create mode 100644 warehouse/attestations/services.py diff --git a/requirements/main.in b/requirements/main.in index 068b7ff21a4c..6fa06b782f43 100644 --- a/requirements/main.in +++ b/requirements/main.in @@ -62,8 +62,8 @@ redis>=2.8.0,<6.0.0 rfc3986 sentry-sdk setuptools -sigstore~=3.0.0 -pypi-attestations==0.0.9 +sigstore~=3.2.0 +pypi-attestations==0.0.11 sqlalchemy[asyncio]>=2.0,<3.0 stdlib-list stripe diff --git a/requirements/main.txt b/requirements/main.txt index 922852b57c3e..e1ff17e074af 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -1770,9 +1770,9 @@ pyparsing==3.1.2 \ --hash=sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad \ --hash=sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742 # via linehaul -pypi-attestations==0.0.9 \ - --hash=sha256:3bfc07f64a8db0d6e2646720e70df7c7cb01a2936056c764a2cc3268969332f2 \ - --hash=sha256:4b38cce5d221c8145cac255bfafe650ec0028d924d2b3572394df8ba8f07a609 +pypi-attestations==0.0.11 \ + --hash=sha256:b730e6b23874d94da0f3817b1f9dd3ecb6a80d685f62a18ad96e5b0396149ded \ + --hash=sha256:e74329074f049568591e300373e12fcd46a35e21723110856546e33bf2949efa # via -r requirements/main.in pyqrcode==1.2.1 \ --hash=sha256:1b2812775fa6ff5c527977c4cd2ccb07051ca7d0bc0aecf937a43864abe5eff6 \ @@ -2079,9 +2079,9 @@ sentry-sdk==2.13.0 \ --hash=sha256:6beede8fc2ab4043da7f69d95534e320944690680dd9a963178a49de71d726c6 \ --hash=sha256:8d4a576f7a98eb2fdb40e13106e41f330e5c79d72a68be1316e7852cf4995260 # via -r requirements/main.in -sigstore==3.0.0 \ - --hash=sha256:6cc7dc92607c2fd481aada0f3c79e710e4c6086e3beab50b07daa9a50a79d109 \ - --hash=sha256:a6a9538a648e112a0c3d8092d3f73a351c7598164764f1e73a6b5ba406a3a0bd +sigstore==3.2.0 \ + --hash=sha256:25c8a871a3a6adf959c0cde598ea8bef8794f1a29277d067111eb4ded4ba7f65 \ + --hash=sha256:d18508f34febb7775065855e92557fa1c2c16580df88f8e8903b9514438bad44 # via # -r requirements/main.in # pypi-attestations diff --git a/tests/common/db/attestation.py b/tests/common/db/attestation.py new file mode 100644 index 000000000000..2080519a806f --- /dev/null +++ b/tests/common/db/attestation.py @@ -0,0 +1,28 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import hashlib + +import factory + +from warehouse.attestations.models import Attestation + +from .base import WarehouseFactory + + +class AttestationFactory(WarehouseFactory): + class Meta: + model = Attestation + + file = factory.SubFactory("tests.common.db.packaging.FileFactory") + attestation_file_blake2_digest = factory.LazyAttribute( + lambda o: hashlib.blake2b(o.file.filename.encode("utf8")).hexdigest() + ) diff --git a/tests/common/db/packaging.py b/tests/common/db/packaging.py index 2a12379da170..369dc9f092d0 100644 --- a/tests/common/db/packaging.py +++ b/tests/common/db/packaging.py @@ -34,6 +34,7 @@ from warehouse.utils import readme from .accounts import UserFactory +from .attestation import AttestationFactory from .base import WarehouseFactory from .observations import ObserverFactory @@ -140,6 +141,12 @@ class Meta: ) ) + attestations = factory.RelatedFactoryList( + AttestationFactory, + factory_related_name="file", + size=1, + ) + class FileEventFactory(WarehouseFactory): class Meta: diff --git a/tests/unit/api/test_simple.py b/tests/unit/api/test_simple.py index 9937038dd145..5f8003d50eb5 100644 --- a/tests/unit/api/test_simple.py +++ b/tests/unit/api/test_simple.py @@ -18,6 +18,7 @@ from pyramid.testing import DummyRequest from warehouse.api import simple +from warehouse.attestations import IIntegrityService from warehouse.packaging.utils import API_VERSION from ...common.db.accounts import UserFactory @@ -87,6 +88,16 @@ def test_selects(self, header, expected): class TestSimpleIndex: + + @pytest.fixture + def db_request(self, db_request): + """Override db_request to add the Release Verification service""" + db_request.find_service = lambda svc, name=None, context=None: { + IIntegrityService: pretend.stub(), + }.get(svc) + + return db_request + @pytest.mark.parametrize( ("content_type", "renderer_override"), CONTENT_TYPE_PARAMS, @@ -185,6 +196,17 @@ def test_quarantined_project_omitted_from_index(self, db_request): class TestSimpleDetail: + @pytest.fixture + def db_request(self, db_request): + """Override db_request to add the Release Verification service""" + db_request.find_service = lambda svc, name=None, context=None: { + IIntegrityService: pretend.stub( + get_provenance_digest=lambda *args, **kwargs: None, + ), + }.get(svc) + + return db_request + def test_redirects(self, pyramid_request): project = pretend.stub(normalized_name="foo") @@ -286,6 +308,7 @@ def test_with_files_no_serial(self, db_request, content_type, renderer_override) "upload-time": f.upload_time.isoformat() + "Z", "data-dist-info-metadata": False, "core-metadata": False, + "provenance": None, } for f in files ], @@ -334,6 +357,7 @@ def test_with_files_with_serial(self, db_request, content_type, renderer_overrid "upload-time": f.upload_time.isoformat() + "Z", "data-dist-info-metadata": False, "core-metadata": False, + "provenance": None, } for f in files ], @@ -427,6 +451,7 @@ def test_with_files_with_version_multi_digit( if f.metadata_file_sha256_digest is not None else False ), + "provenance": None, } for f in files ], diff --git a/tests/unit/attestations/__init__.py b/tests/unit/attestations/__init__.py new file mode 100644 index 000000000000..164f68b09175 --- /dev/null +++ b/tests/unit/attestations/__init__.py @@ -0,0 +1,11 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/unit/attestations/test_init.py b/tests/unit/attestations/test_init.py new file mode 100644 index 000000000000..c59080b7c5a3 --- /dev/null +++ b/tests/unit/attestations/test_init.py @@ -0,0 +1,31 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pretend + +from warehouse import attestations +from warehouse.attestations.interfaces import IIntegrityService +from warehouse.attestations.services import IntegrityService + + +def test_includeme(): + config = pretend.stub( + register_service_factory=pretend.call_recorder( + lambda factory, iface, name=None: None + ), + ) + + attestations.includeme(config) + + assert config.register_service_factory.calls == [ + pretend.call(IntegrityService.create_service, IIntegrityService), + ] diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py new file mode 100644 index 000000000000..a83a0c302b00 --- /dev/null +++ b/tests/unit/attestations/test_services.py @@ -0,0 +1,419 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import hashlib +import tempfile + +import pretend +import pytest + +from pydantic import TypeAdapter +from pypi_attestations import ( + Attestation, + AttestationBundle, + AttestationType, + Envelope, + GitHubPublisher, + GitLabPublisher, + Provenance, + VerificationError, + VerificationMaterial, +) +from sigstore.verify import Verifier +from zope.interface.verify import verifyClass + +from tests.common.db.oidc import GitHubPublisherFactory, GitLabPublisherFactory +from tests.common.db.packaging import FileEventFactory, FileFactory +from warehouse.attestations import ( + Attestation as DatabaseAttestation, + AttestationUploadError, + IIntegrityService, + IntegrityService, + UnsupportedPublisherError, + services, +) +from warehouse.events.tags import EventTag +from warehouse.metrics import IMetricsService +from warehouse.packaging import File, IFileStorage + +VALID_ATTESTATION = Attestation( + version=1, + verification_material=VerificationMaterial( + certificate="somebase64string", transparency_entries=[dict()] + ), + envelope=Envelope( + statement="somebase64string", + signature="somebase64string", + ), +) + + +class TestAttestationsService: + def test_interface_matches(self): + assert verifyClass(IIntegrityService, IntegrityService) + + def test_create_service(self): + request = pretend.stub( + find_service=pretend.call_recorder( + lambda svc, context=None, name=None: None + ), + ) + + assert IntegrityService.create_service(None, request) is not None + assert not set(request.find_service.calls) ^ { + pretend.call(IFileStorage), + pretend.call(IMetricsService), + } + + def test_persist_attestations(self, db_request, monkeypatch): + @pretend.call_recorder + def storage_service_store(path: str, file_path, *_args, **_kwargs): + expected = VALID_ATTESTATION.model_dump_json().encode("utf-8") + with open(file_path, "rb") as fp: + assert fp.read() == expected + + assert path.endswith(".attestation") + + integrity_service = IntegrityService( + storage=pretend.stub( + store=storage_service_store, + ), + metrics=pretend.stub(), + ) + + file = FileFactory.create(attestations=[]) + + integrity_service.persist_attestations([VALID_ATTESTATION], file) + + attestations_db = ( + db_request.db.query(DatabaseAttestation) + .join(DatabaseAttestation.file) + .filter(File.filename == file.filename) + .all() + ) + assert len(attestations_db) == 1 + assert len(file.attestations) == 1 + + def test_parse_no_publisher(self, db_request): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=pretend.stub(), + ) + + db_request.oidc_publisher = None + with pytest.raises( + AttestationUploadError, + match="Attestations are only supported when using Trusted", + ): + integrity_service.parse_attestations(db_request, pretend.stub()) + + def test_parse_unsupported_publisher(self, db_request): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=pretend.stub(), + ) + db_request.oidc_publisher = pretend.stub(publisher_name="not-existing") + with pytest.raises( + AttestationUploadError, + match="Attestations are only supported when using Trusted", + ): + integrity_service.parse_attestations(db_request, pretend.stub()) + + def test_parse_malformed_attestation(self, metrics, db_request): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=metrics, + ) + + db_request.oidc_publisher = pretend.stub(publisher_name="GitHub") + db_request.POST["attestations"] = "{'malformed-attestation'}" + with pytest.raises( + AttestationUploadError, + match="Error while decoding the included attestation", + ): + integrity_service.parse_attestations(db_request, pretend.stub()) + + assert ( + pretend.call("warehouse.upload.attestations.malformed") + in metrics.increment.calls + ) + + def test_parse_multiple_attestations(self, metrics, db_request): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=metrics, + ) + + db_request.oidc_publisher = pretend.stub(publisher_name="GitHub") + db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( + [VALID_ATTESTATION, VALID_ATTESTATION] + ) + with pytest.raises( + AttestationUploadError, match="Only a single attestation per file" + ): + integrity_service.parse_attestations( + db_request, + pretend.stub(), + ) + + assert ( + pretend.call("warehouse.upload.attestations.failed_multiple_attestations") + in metrics.increment.calls + ) + + @pytest.mark.parametrize( + ("verify_exception", "expected_message"), + [ + ( + VerificationError, + "Could not verify the uploaded", + ), + ( + ValueError, + "Unknown error while", + ), + ], + ) + def test_parse_failed_verification( + self, metrics, monkeypatch, db_request, verify_exception, expected_message + ): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=metrics, + ) + + db_request.oidc_publisher = pretend.stub( + publisher_name="GitHub", + publisher_verification_policy=pretend.call_recorder(lambda c: None), + ) + db_request.oidc_claims = {"sha": "somesha"} + db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( + [VALID_ATTESTATION] + ) + + def failing_verify(_self, _verifier, _policy, _dist): + raise verify_exception("error") + + monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) + monkeypatch.setattr(Attestation, "verify", failing_verify) + + with pytest.raises(AttestationUploadError, match=expected_message): + integrity_service.parse_attestations( + db_request, + pretend.stub(), + ) + + def test_parse_wrong_predicate(self, metrics, monkeypatch, db_request): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=metrics, + ) + db_request.oidc_publisher = pretend.stub( + publisher_name="GitHub", + publisher_verification_policy=pretend.call_recorder(lambda c: None), + ) + db_request.oidc_claims = {"sha": "somesha"} + db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( + [VALID_ATTESTATION] + ) + + monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) + monkeypatch.setattr( + Attestation, "verify", lambda *args: ("wrong-predicate", {}) + ) + + with pytest.raises( + AttestationUploadError, match="Attestation with unsupported predicate" + ): + integrity_service.parse_attestations( + db_request, + pretend.stub(), + ) + + assert ( + pretend.call( + "warehouse.upload.attestations.failed_unsupported_predicate_type" + ) + in metrics.increment.calls + ) + + def test_parse_succeed(self, metrics, monkeypatch, db_request): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=metrics, + ) + db_request.oidc_publisher = pretend.stub( + publisher_name="GitHub", + publisher_verification_policy=pretend.call_recorder(lambda c: None), + ) + db_request.oidc_claims = {"sha": "somesha"} + db_request.POST["attestations"] = TypeAdapter(list[Attestation]).dump_json( + [VALID_ATTESTATION] + ) + + monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) + monkeypatch.setattr( + Attestation, "verify", lambda *args: (AttestationType.PYPI_PUBLISH_V1, {}) + ) + + attestations = integrity_service.parse_attestations( + db_request, + pretend.stub(), + ) + assert attestations == [VALID_ATTESTATION] + + def test_generate_provenance_unsupported_publisher(self): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=pretend.stub(), + ) + + oidc_publisher = pretend.stub(publisher_name="not-existing") + + assert ( + integrity_service.generate_provenance(oidc_publisher, pretend.stub()) + is None + ) + + @pytest.mark.parametrize( + "publisher_name", + [ + "github", + "gitlab", + ], + ) + def test_generate_provenance_succeeds(self, publisher_name: str, monkeypatch): + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=pretend.stub(), + ) + + if publisher_name == "github": + publisher = GitHubPublisher( + repository="fake-repository", + workflow="fake-workflow", + ) + else: + publisher = GitLabPublisher( + repository="fake-repository", + environment="fake-env", + ) + + monkeypatch.setattr( + services, + "_publisher_from_oidc_publisher", + lambda s: publisher, + ) + + provenance = integrity_service.generate_provenance( + pretend.stub(), + [VALID_ATTESTATION], + ) + + assert provenance == Provenance( + attestation_bundles=[ + AttestationBundle( + publisher=publisher, + attestations=[VALID_ATTESTATION], + ) + ] + ) + + def test_persist_provenance_succeeds(self, db_request): + provenance = Provenance( + attestation_bundles=[ + AttestationBundle( + publisher=GitHubPublisher( + repository="fake-repository", + workflow="fake-workflow", + ), + attestations=[VALID_ATTESTATION], + ) + ] + ) + + @pretend.call_recorder + def storage_service_store(path, file_path, *_args, **_kwargs): + expected = provenance.model_dump_json().encode("utf-8") + with open(file_path, "rb") as fp: + assert fp.read() == expected + + assert path.suffix == ".provenance" + + integrity_service = IntegrityService( + storage=pretend.stub(store=storage_service_store), + metrics=pretend.stub(), + ) + assert ( + integrity_service.persist_provenance(provenance, FileFactory.create()) + is None + ) + + def test_get_provenance_digest(self, db_request): + file = FileFactory.create() + FileEventFactory.create( + source=file, + tag=EventTag.File.FileAdd, + additional={"publisher_url": "fake-publisher-url"}, + ) + + with tempfile.NamedTemporaryFile() as f: + integrity_service = IntegrityService( + storage=pretend.stub(get=pretend.call_recorder(lambda p: f)), + metrics=pretend.stub(), + ) + + assert ( + integrity_service.get_provenance_digest(file) + == hashlib.file_digest(f, "sha256").hexdigest() + ) + + def test_get_provenance_digest_fails_no_attestations(self, db_request): + # If the attestations are missing, there is no provenance file + file = FileFactory.create() + file.attestations = [] + FileEventFactory.create( + source=file, + tag=EventTag.File.FileAdd, + additional={"publisher_url": "fake-publisher-url"}, + ) + integrity_service = IntegrityService( + storage=pretend.stub(), + metrics=pretend.stub(), + ) + + assert integrity_service.get_provenance_digest(file) is None + + +def test_publisher_from_oidc_publisher_github(db_request): + publisher = GitHubPublisherFactory.create() + + attestation_publisher = services._publisher_from_oidc_publisher(publisher) + assert isinstance(attestation_publisher, GitHubPublisher) + assert attestation_publisher.repository == publisher.repository + assert attestation_publisher.workflow == publisher.workflow_filename + assert attestation_publisher.environment == publisher.environment + + +def test_publisher_from_oidc_publisher_gitlab(db_request): + publisher = GitLabPublisherFactory.create() + + attestation_publisher = services._publisher_from_oidc_publisher(publisher) + assert isinstance(attestation_publisher, GitLabPublisher) + assert attestation_publisher.repository == publisher.project_path + assert attestation_publisher.environment == publisher.environment + + +def test_publisher_from_oidc_publisher_fails(): + publisher = pretend.stub(publisher_name="not-existing") + + with pytest.raises(UnsupportedPublisherError): + services._publisher_from_oidc_publisher(publisher) diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index e127299ca204..6e062ef1f0f2 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -23,15 +23,8 @@ import pretend import pytest -from pypi_attestations import ( - Attestation, - Distribution, - Envelope, - VerificationError, - VerificationMaterial, -) +from pypi_attestations import Attestation, Envelope, VerificationMaterial from pyramid.httpexceptions import HTTPBadRequest, HTTPForbidden, HTTPTooManyRequests -from sigstore.verify import Verifier from sqlalchemy import and_, exists from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import joinedload @@ -42,6 +35,11 @@ from warehouse.accounts.utils import UserContext from warehouse.admin.flags import AdminFlag, AdminFlagValue +from warehouse.attestations import ( + Attestation as DatabaseAttestation, + AttestationUploadError, + IIntegrityService, +) from warehouse.classifiers.models import Classifier from warehouse.forklift import legacy, metadata from warehouse.macaroons import IMacaroonService, caveats, security_policy @@ -63,6 +61,7 @@ from warehouse.packaging.tasks import sync_file_to_cache, update_bigquery_release_files from ...common.db.accounts import EmailFactory, UserFactory +from ...common.db.attestation import AttestationFactory from ...common.db.classifiers import ClassifierFactory from ...common.db.oidc import GitHubPublisherFactory from ...common.db.packaging import ( @@ -2420,85 +2419,6 @@ def test_upload_fails_without_oidc_publisher_permission( "See /the/help/url/ for more information." ).format(project.name) - def test_upload_attestation_fails_without_oidc_publisher( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - project_service, - macaroon_service, - ): - project = ProjectFactory.create() - owner = UserFactory.create() - maintainer = UserFactory.create() - RoleFactory.create(user=owner, project=project, role_name="Owner") - RoleFactory.create(user=maintainer, project=project, role_name="Maintainer") - - EmailFactory.create(user=maintainer) - db_request.user = maintainer - raw_macaroon, macaroon = macaroon_service.create_macaroon( - "fake location", - "fake description", - [caveats.RequestUser(user_id=str(maintainer.id))], - user_id=maintainer.id, - ) - identity = UserContext(maintainer, macaroon) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="some_cert", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": f"[{attestation.model_dump_json()}]", - "version": "1.0", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - extract_http_macaroon = pretend.call_recorder(lambda r, _: raw_macaroon) - monkeypatch.setattr( - security_policy, "_extract_http_macaroon", extract_http_macaroon - ) - - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMacaroonService: macaroon_service, - IMetricsService: metrics, - IProjectService: project_service, - }.get(svc) - db_request.user_agent = "warehouse-tests/6.6.6" - - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status == ( - "400 Attestations are currently only supported when using Trusted " - "Publishing with GitHub Actions." - ) - @pytest.mark.parametrize( "plat", [ @@ -3408,8 +3328,10 @@ def test_upload_succeeds_creates_release( ), ] - def test_upload_with_valid_attestation_succeeds( + @pytest.mark.parametrize("provenance_rv", [None, "fake-provenance-object"]) + def test_upload_succeeds_with_valid_attestation( self, + provenance_rv, monkeypatch, pyramid_config, db_request, @@ -3464,296 +3386,68 @@ def test_upload_with_valid_attestation_succeeds( } ) - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - - record_event = pretend.call_recorder( - lambda self, *, tag, request=None, additional: None - ) - monkeypatch.setattr(HasEvents, "record_event", record_event) - - verify = pretend.call_recorder( - lambda _self, _verifier, _policy, _dist: ( - "https://docs.pypi.org/attestations/publish/v1", - None, - ) - ) - monkeypatch.setattr(Attestation, "verify", verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - - resp = legacy.file_upload(db_request) - - assert resp.status_code == 200 - - assert len(verify.calls) == 1 - verified_distribution = verify.calls[0].args[3] - assert verified_distribution == Distribution( - name=filename, digest=_TAR_GZ_PKG_SHA256 - ) + def persist_attestations(attestations, file): + file.attestations.append(AttestationFactory.create(file=file)) - def test_upload_with_invalid_attestation_predicate_type_fails( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - ): - from warehouse.events.models import HasEvents + def persist_provenance(provenance_object, file): + assert provenance_object == provenance_rv - project = ProjectFactory.create() - version = "1.0" - publisher = GitHubPublisherFactory.create(projects=[project]) - claims = { - "sha": "somesha", - "repository": f"{publisher.repository_owner}/{publisher.repository_name}", - "workflow": "workflow_name", - } - identity = PublisherTokenContext(publisher, SignedClaims(claims)) - db_request.oidc_publisher = identity.publisher - db_request.oidc_claims = identity.claims - - db_request.db.add(Classifier(classifier="Environment :: Other Environment")) - db_request.db.add(Classifier(classifier="Programming Language :: Python")) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", + storage_service = pretend.stub(store=lambda path, filepath, *, meta=None: None) + integrity_service = pretend.stub( + parse_attestations=lambda *args, **kwargs: [attestation], + persist_attestations=persist_attestations, + generate_provenance=pretend.call_recorder( + lambda oidc_publisher, attestations: provenance_rv ), + persist_provenance=persist_provenance, ) - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.user = None - db_request.user_agent = "warehouse-tests/6.6.6" - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": f"[{attestation.model_dump_json()}]", - "version": version, - "summary": "This is my summary!", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) db_request.find_service = lambda svc, name=None, context=None: { IFileStorage: storage_service, IMetricsService: metrics, + IIntegrityService: integrity_service, }.get(svc) record_event = pretend.call_recorder( lambda self, *, tag, request=None, additional: None ) monkeypatch.setattr(HasEvents, "record_event", record_event) + resp = legacy.file_upload(db_request) - invalid_predicate_type = "Unsupported predicate type" - verify = pretend.call_recorder( - lambda _self, _verifier, _policy, _dist: (invalid_predicate_type, None) - ) - monkeypatch.setattr(Attestation, "verify", verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status.startswith( - f"400 Attestation with unsupported predicate type: {invalid_predicate_type}" - ) - - def test_upload_with_multiple_attestations_fails( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - ): - from warehouse.events.models import HasEvents - - project = ProjectFactory.create() - version = "1.0" - publisher = GitHubPublisherFactory.create(projects=[project]) - claims = { - "sha": "somesha", - "repository": f"{publisher.repository_owner}/{publisher.repository_name}", - "workflow": "workflow_name", - } - identity = PublisherTokenContext(publisher, SignedClaims(claims)) - db_request.oidc_publisher = identity.publisher - db_request.oidc_claims = identity.claims - - db_request.db.add(Classifier(classifier="Environment :: Other Environment")) - db_request.db.add(Classifier(classifier="Programming Language :: Python")) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) - - pyramid_config.testing_securitypolicy(identity=identity) - db_request.user = None - db_request.user_agent = "warehouse-tests/6.6.6" - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": f"[{attestation.model_dump_json()}," - f" {attestation.model_dump_json()}]", - "version": version, - "summary": "This is my summary!", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } - ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - - record_event = pretend.call_recorder( - lambda self, *, tag, request=None, additional: None - ) - monkeypatch.setattr(HasEvents, "record_event", record_event) - - verify = pretend.call_recorder( - lambda _self, _verifier, _policy, _dist: ( - "https://docs.pypi.org/attestations/publish/v1", - None, - ) - ) - monkeypatch.setattr(Attestation, "verify", verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status.startswith( - "400 Only a single attestation per-file is supported at the moment." - ) - - def test_upload_with_malformed_attestation_fails( - self, - monkeypatch, - pyramid_config, - db_request, - metrics, - ): - from warehouse.events.models import HasEvents - - project = ProjectFactory.create() - version = "1.0" - publisher = GitHubPublisherFactory.create(projects=[project]) - claims = { - "sha": "somesha", - "repository": f"{publisher.repository_owner}/{publisher.repository_name}", - "workflow": "workflow_name", - } - identity = PublisherTokenContext(publisher, SignedClaims(claims)) - db_request.oidc_publisher = identity.publisher - db_request.oidc_claims = identity.claims - - db_request.db.add(Classifier(classifier="Environment :: Other Environment")) - db_request.db.add(Classifier(classifier="Programming Language :: Python")) - - filename = "{}-{}.tar.gz".format(project.name, "1.0") + assert resp.status_code == 200 - pyramid_config.testing_securitypolicy(identity=identity) - db_request.user = None - db_request.user_agent = "warehouse-tests/6.6.6" - db_request.POST = MultiDict( - { - "metadata_version": "1.2", - "name": project.name, - "attestations": "[{'a_malformed_attestation': 3}]", - "version": version, - "summary": "This is my summary!", - "filetype": "sdist", - "md5_digest": _TAR_GZ_PKG_MD5, - "content": pretend.stub( - filename=filename, - file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), - type="application/tar", - ), - } + assert ( + pretend.call("warehouse.upload.attestations.ok") in metrics.increment.calls ) - - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - }.get(svc) - - record_event = pretend.call_recorder( - lambda self, *, tag, request=None, additional: None + attestations_db = ( + db_request.db.query(DatabaseAttestation) + .join(DatabaseAttestation.file) + .filter(File.filename == filename) + .all() ) - monkeypatch.setattr(HasEvents, "record_event", record_event) + assert len(attestations_db) == 1 - with pytest.raises(HTTPBadRequest) as excinfo: - legacy.file_upload(db_request) - - resp = excinfo.value - - assert resp.status_code == 400 - assert resp.status.startswith( - "400 Error while decoding the included attestation:" - ) + assert integrity_service.generate_provenance.calls == [ + pretend.call(db_request.oidc_publisher, [attestation]) + ] @pytest.mark.parametrize( - ("verify_exception", "expected_msg"), + "expected_message", [ - ( - VerificationError, - "400 Could not verify the uploaded artifact using the included " - "attestation", - ), - ( - ValueError, - "400 Unknown error while trying to verify included attestations", - ), + "Attestations are only supported when using", + "Error while decoding the included attestation", + "Only a single attestation", + "Could not verify the uploaded", + "Unknown error while trying", + "Attestation with unsupported predicate", ], ) - def test_upload_with_failing_attestation_verification( + def test_upload_fails_attestation_error( self, monkeypatch, pyramid_config, db_request, metrics, - verify_exception, - expected_msg, + expected_message, ): from warehouse.events.models import HasEvents @@ -3773,16 +3467,6 @@ def test_upload_with_failing_attestation_verification( db_request.db.add(Classifier(classifier="Programming Language :: Python")) filename = "{}-{}.tar.gz".format(project.name, "1.0") - attestation = Attestation( - version=1, - verification_material=VerificationMaterial( - certificate="somebase64string", transparency_entries=[dict()] - ), - envelope=Envelope( - statement="somebase64string", - signature="somebase64string", - ), - ) pyramid_config.testing_securitypolicy(identity=identity) db_request.user = None @@ -3791,7 +3475,7 @@ def test_upload_with_failing_attestation_verification( { "metadata_version": "1.2", "name": project.name, - "attestations": f"[{attestation.model_dump_json()}]", + "attestations": "", "version": version, "summary": "This is my summary!", "filetype": "sdist", @@ -3805,9 +3489,15 @@ def test_upload_with_failing_attestation_verification( ) storage_service = pretend.stub(store=lambda path, filepath, meta: None) + + def stub_parse(*_args, **_kwargs): + raise AttestationUploadError(expected_message) + + integrity_service = pretend.stub(parse_attestations=stub_parse) db_request.find_service = lambda svc, name=None, context=None: { IFileStorage: storage_service, IMetricsService: metrics, + IIntegrityService: integrity_service, }.get(svc) record_event = pretend.call_recorder( @@ -3815,19 +3505,13 @@ def test_upload_with_failing_attestation_verification( ) monkeypatch.setattr(HasEvents, "record_event", record_event) - def failing_verify(_self, _verifier, _policy, _dist): - raise verify_exception("error") - - monkeypatch.setattr(Attestation, "verify", failing_verify) - monkeypatch.setattr(Verifier, "production", lambda: pretend.stub()) - with pytest.raises(HTTPBadRequest) as excinfo: legacy.file_upload(db_request) resp = excinfo.value assert resp.status_code == 400 - assert resp.status.startswith(expected_msg) + assert resp.status.startswith(f"400 {expected_message}") @pytest.mark.parametrize( ("url", "expected"), diff --git a/tests/unit/packaging/test_utils.py b/tests/unit/packaging/test_utils.py index afa7bd2056fe..8065c8b9ad8c 100644 --- a/tests/unit/packaging/test_utils.py +++ b/tests/unit/packaging/test_utils.py @@ -15,6 +15,7 @@ import pretend +from warehouse.attestations import IIntegrityService from warehouse.packaging.interfaces import ISimpleStorage from warehouse.packaging.utils import _simple_detail, render_simple_detail @@ -27,11 +28,37 @@ def test_simple_detail_empty_string(db_request): FileFactory.create(release=release) db_request.route_url = lambda *a, **kw: "the-url" + db_request.find_service = lambda svc, name=None, context=None: { + IIntegrityService: pretend.stub( + get_provenance_digest=pretend.call_recorder(lambda f: None), + ), + }.get(svc) + expected_content = _simple_detail(project, db_request) assert expected_content["files"][0]["requires-python"] is None +def test_simple_detail_with_provenance(db_request): + project = ProjectFactory.create() + release = ReleaseFactory.create(project=project, version="1.0") + FileFactory.create(release=release) + + hash_digest = "deadbeefdeadbeefdeadbeefdeadbeef" + + db_request.route_url = lambda *a, **kw: "the-url" + db_request.find_service = pretend.call_recorder( + lambda svc, name=None, context=None: { + IIntegrityService: pretend.stub( + get_provenance_digest=pretend.call_recorder(lambda f: hash_digest), + ), + }.get(svc) + ) + + expected_content = _simple_detail(project, db_request) + assert expected_content["files"][0]["provenance"] == hash_digest + + def test_render_simple_detail(db_request, monkeypatch, jinja): project = ProjectFactory.create() release1 = ReleaseFactory.create(project=project, version="1.0") @@ -49,6 +76,12 @@ def test_render_simple_detail(db_request, monkeypatch, jinja): monkeypatch.setattr(hashlib, "blake2b", fakeblake2b) db_request.route_url = lambda *a, **kw: "the-url" + db_request.find_service = lambda svc, name=None, context=None: { + IIntegrityService: pretend.stub( + get_provenance_digest=pretend.call_recorder(lambda f: None), + ), + }.get(svc) + template = jinja.get_template("templates/api/simple/detail.html") expected_content = template.render( **_simple_detail(project, db_request), request=db_request @@ -78,6 +111,9 @@ def test_render_simple_detail_with_store(db_request, monkeypatch, jinja): db_request.find_service = pretend.call_recorder( lambda svc, name=None, context=None: { ISimpleStorage: storage_service, + IIntegrityService: pretend.stub( + get_provenance_digest=pretend.call_recorder(lambda f: None), + ), }.get(svc) ) diff --git a/warehouse/attestations/__init__.py b/warehouse/attestations/__init__.py new file mode 100644 index 000000000000..208684d87587 --- /dev/null +++ b/warehouse/attestations/__init__.py @@ -0,0 +1,31 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from warehouse.attestations.errors import ( + AttestationUploadError, + UnsupportedPublisherError, +) +from warehouse.attestations.interfaces import IIntegrityService +from warehouse.attestations.models import Attestation +from warehouse.attestations.services import IntegrityService + +__all__ = [ + "Attestation", + "AttestationUploadError", + "IIntegrityService", + "IntegrityService", + "UnsupportedPublisherError", +] + + +def includeme(config): + config.register_service_factory(IntegrityService.create_service, IIntegrityService) diff --git a/warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py b/warehouse/attestations/errors.py similarity index 66% rename from warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py rename to warehouse/attestations/errors.py index 31a1b5636bf4..463a34a4da69 100644 --- a/warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py +++ b/warehouse/attestations/errors.py @@ -9,24 +9,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -Rollback attestation migration -Revision ID: 7ca0f1f5e7b3 -Revises: 7f0c9f105f44 -Create Date: 2024-08-21 19:52:40.084048 -""" - -from alembic import op - -revision = "7ca0f1f5e7b3" -down_revision = "7f0c9f105f44" - - -def upgrade(): - op.drop_table("attestation") +class UnsupportedPublisherError(Exception): + pass -def downgrade(): +class AttestationUploadError(Exception): pass diff --git a/warehouse/attestations/interfaces.py b/warehouse/attestations/interfaces.py new file mode 100644 index 000000000000..4055e92055a5 --- /dev/null +++ b/warehouse/attestations/interfaces.py @@ -0,0 +1,54 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pypi_attestations import Attestation, Distribution, Provenance +from pyramid.request import Request +from zope.interface import Interface + + +class IIntegrityService(Interface): + + def create_service(context, request): + """ + Create the service, given the context and request for which it is being + created for. + """ + + def persist_attestations(attestations: list[Attestation], file): + """ + ̦Persist attestations in storage. + """ + pass + + def parse_attestations( + request: Request, distribution: Distribution + ) -> list[Attestation]: + """ + Process any attestations included in a file upload request + """ + + def generate_provenance( + oidc_publisher, attestations: list[Attestation] + ) -> Provenance | None: + """ + Generate a Provenance object from an OIDCPublisher and its attestations. + """ + + def persist_provenance(provenance: Provenance, file) -> None: + """ + Persist a Provenance object in storage. + """ + + def get_provenance_digest(file) -> str | None: + """ + Compute a provenance file digest for a `File` if it exists. + """ diff --git a/warehouse/attestations/models.py b/warehouse/attestations/models.py new file mode 100644 index 000000000000..9b95bfb0d7ad --- /dev/null +++ b/warehouse/attestations/models.py @@ -0,0 +1,55 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import typing + +from pathlib import Path +from uuid import UUID + +from sqlalchemy import ForeignKey, orm +from sqlalchemy.dialects.postgresql import CITEXT +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.orm import Mapped, mapped_column + +from warehouse import db + +if typing.TYPE_CHECKING: + from warehouse.packaging.models import File + + +class Attestation(db.Model): + """ + Table used to store Attestations. + + Attestations are stored on disk. We keep in database only the attestation hash. + """ + + __tablename__ = "attestation" + + file_id: Mapped[UUID] = mapped_column( + ForeignKey("release_files.id", onupdate="CASCADE", ondelete="CASCADE"), + ) + file: Mapped[File] = orm.relationship(back_populates="attestations") + + attestation_file_blake2_digest: Mapped[str] = mapped_column(CITEXT) + + @hybrid_property + def attestation_path(self): + return "/".join( + [ + self.attestation_file_blake2_digest[:2], + self.attestation_file_blake2_digest[2:4], + self.attestation_file_blake2_digest[4:], + f"{Path(self.file.path).name}.attestation", + ] + ) diff --git a/warehouse/attestations/services.py b/warehouse/attestations/services.py new file mode 100644 index 000000000000..056c47c4894e --- /dev/null +++ b/warehouse/attestations/services.py @@ -0,0 +1,231 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import hashlib +import tempfile +import typing + +from pathlib import Path + +import sentry_sdk + +from pydantic import TypeAdapter, ValidationError +from pypi_attestations import ( + Attestation, + AttestationBundle, + AttestationType, + Distribution, + GitHubPublisher, + GitLabPublisher, + Provenance, + Publisher, + VerificationError, +) +from pyramid.request import Request +from sigstore.verify import Verifier +from zope.interface import implementer + +from warehouse.attestations.errors import ( + AttestationUploadError, + UnsupportedPublisherError, +) +from warehouse.attestations.interfaces import IIntegrityService +from warehouse.attestations.models import Attestation as DatabaseAttestation +from warehouse.metrics.interfaces import IMetricsService +from warehouse.oidc.models import ( + GitHubPublisher as GitHubOIDCPublisher, + GitLabPublisher as GitLabOIDCPublisher, + OIDCPublisher, +) +from warehouse.packaging.interfaces import IFileStorage +from warehouse.packaging.models import File + + +def _publisher_from_oidc_publisher(publisher: OIDCPublisher) -> Publisher: + """ + Convert an OIDCPublisher object in a pypi-attestations Publisher. + """ + match publisher.publisher_name: + case "GitLab": + publisher = typing.cast(GitLabOIDCPublisher, publisher) + return GitLabPublisher( + repository=publisher.project_path, environment=publisher.environment + ) + case "GitHub": + publisher = typing.cast(GitHubOIDCPublisher, publisher) + return GitHubPublisher( + repository=publisher.repository, + workflow=publisher.workflow_filename, + environment=publisher.environment, + ) + case _: + raise UnsupportedPublisherError + + +@implementer(IIntegrityService) +class IntegrityService: + + def __init__( + self, + storage: IFileStorage, + metrics: IMetricsService, + ): + self.storage: IFileStorage = storage + self.metrics: IMetricsService = metrics + + @classmethod + def create_service(cls, _context, request: Request): + return cls( + storage=request.find_service(IFileStorage), + metrics=request.find_service(IMetricsService), + ) + + def persist_attestations(self, attestations: list[Attestation], file: File) -> None: + for attestation in attestations: + with tempfile.NamedTemporaryFile() as tmp_file: + tmp_file.write(attestation.model_dump_json().encode("utf-8")) + + attestation_digest = hashlib.file_digest( + tmp_file, "blake2b" + ).hexdigest() + database_attestation = DatabaseAttestation( + file=file, attestation_file_blake2_digest=attestation_digest + ) + + self.storage.store( + database_attestation.attestation_path, + tmp_file.name, + meta=None, + ) + + file.attestations.append(database_attestation) + + def parse_attestations( + self, request: Request, distribution: Distribution + ) -> list[Attestation]: + """ + Process any attestations included in a file upload request + + Attestations, if present, will be parsed and verified against the uploaded + artifact. Attestations are only allowed when uploading via a Trusted + Publisher, because a Trusted Publisher provides the identity that will be + used to verify the attestations. + Only GitHub Actions Trusted Publishers are supported. + """ + publisher: OIDCPublisher | None = request.oidc_publisher + if not publisher or not publisher.publisher_name == "GitHub": + raise AttestationUploadError( + "Attestations are only supported when using Trusted " + "Publishing with GitHub Actions.", + ) + + try: + attestations = TypeAdapter(list[Attestation]).validate_json( + request.POST["attestations"] + ) + except ValidationError as e: + # Log invalid (malformed) attestation upload + self.metrics.increment("warehouse.upload.attestations.malformed") + raise AttestationUploadError( + f"Error while decoding the included attestation: {e}", + ) + + if len(attestations) > 1: + self.metrics.increment( + "warehouse.upload.attestations.failed_multiple_attestations" + ) + + raise AttestationUploadError( + "Only a single attestation per file is supported.", + ) + + verification_policy = publisher.publisher_verification_policy( + request.oidc_claims + ) + for attestation_model in attestations: + try: + predicate_type, _ = attestation_model.verify( + Verifier.production(), + verification_policy, + distribution, + ) + except VerificationError as e: + # Log invalid (failed verification) attestation upload + self.metrics.increment("warehouse.upload.attestations.failed_verify") + raise AttestationUploadError( + f"Could not verify the uploaded artifact using the included " + f"attestation: {e}", + ) + except Exception as e: + with sentry_sdk.push_scope() as scope: + scope.fingerprint = [e] + sentry_sdk.capture_message( + f"Unexpected error while verifying attestation: {e}" + ) + + raise AttestationUploadError( + f"Unknown error while trying to verify included attestations: {e}", + ) + + if predicate_type != AttestationType.PYPI_PUBLISH_V1: + self.metrics.increment( + "warehouse.upload.attestations.failed_unsupported_predicate_type" + ) + raise AttestationUploadError( + f"Attestation with unsupported predicate type: {predicate_type}", + ) + + return attestations + + def generate_provenance( + self, oidc_publisher: OIDCPublisher, attestations: list[Attestation] + ) -> Provenance | None: + try: + publisher: Publisher = _publisher_from_oidc_publisher(oidc_publisher) + except UnsupportedPublisherError: + sentry_sdk.capture_message( + f"Unsupported OIDCPublisher found {oidc_publisher.publisher_name}" + ) + + return None + + attestation_bundle = AttestationBundle( + publisher=publisher, + attestations=attestations, + ) + + return Provenance(attestation_bundles=[attestation_bundle]) + + def persist_provenance( + self, + provenance: Provenance, + file: File, + ) -> None: + """ + Persist a Provenance object in storage. + """ + provenance_file_path = Path(f"{file.path}.provenance") + with tempfile.NamedTemporaryFile() as f: + f.write(provenance.model_dump_json().encode("utf-8")) + f.flush() + + self.storage.store( + provenance_file_path, + f.name, + ) + + def get_provenance_digest(self, file: File) -> str | None: + """Returns the sha256 digest of the provenance file for the release.""" + if not file.attestations: + return None + + provenance_file = self.storage.get(f"{file.path}.provenance") + return hashlib.file_digest(provenance_file, "sha256").hexdigest() diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 89010bbaaee0..baae8da64e84 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -30,13 +30,7 @@ import wtforms import wtforms.validators -from pydantic import TypeAdapter, ValidationError -from pypi_attestations import ( - Attestation, - AttestationType, - Distribution, - VerificationError, -) +from pypi_attestations import Attestation, Distribution from pyramid.httpexceptions import ( HTTPBadRequest, HTTPException, @@ -48,11 +42,11 @@ ) from pyramid.request import Request from pyramid.view import view_config -from sigstore.verify import Verifier from sqlalchemy import and_, exists, func, orm from sqlalchemy.exc import MultipleResultsFound, NoResultFound from warehouse.admin.flags import AdminFlagValue +from warehouse.attestations import AttestationUploadError, IIntegrityService from warehouse.authnz import Permissions from warehouse.classifiers.models import Classifier from warehouse.constants import MAX_FILESIZE, MAX_PROJECT_SIZE, ONE_GIB, ONE_MIB @@ -377,88 +371,6 @@ def _is_duplicate_file(db_session, filename, hashes): return None -def _process_attestations(request, distribution: Distribution): - """ - Process any attestations included in a file upload request - - Attestations, if present, will be parsed and verified against the uploaded - artifact. Attestations are only allowed when uploading via a Trusted - Publisher, because a Trusted Publisher provides the identity that will be - used to verify the attestations. - Currently, only GitHub Actions Trusted Publishers are supported, and - attestations are discarded after verification. - """ - - metrics = request.find_service(IMetricsService, context=None) - - publisher = request.oidc_publisher - if not publisher or not publisher.publisher_name == "GitHub": - raise _exc_with_message( - HTTPBadRequest, - "Attestations are currently only supported when using Trusted " - "Publishing with GitHub Actions.", - ) - try: - attestations = TypeAdapter(list[Attestation]).validate_json( - request.POST["attestations"] - ) - except ValidationError as e: - # Log invalid (malformed) attestation upload - metrics.increment("warehouse.upload.attestations.malformed") - raise _exc_with_message( - HTTPBadRequest, - f"Error while decoding the included attestation: {e}", - ) - - if len(attestations) > 1: - metrics.increment("warehouse.upload.attestations.failed_multiple_attestations") - raise _exc_with_message( - HTTPBadRequest, - "Only a single attestation per-file is supported at the moment.", - ) - - verification_policy = publisher.publisher_verification_policy(request.oidc_claims) - for attestation_model in attestations: - try: - # For now, attestations are not stored, just verified - predicate_type, _ = attestation_model.verify( - Verifier.production(), - verification_policy, - distribution, - ) - except VerificationError as e: - # Log invalid (failed verification) attestation upload - metrics.increment("warehouse.upload.attestations.failed_verify") - raise _exc_with_message( - HTTPBadRequest, - f"Could not verify the uploaded artifact using the included " - f"attestation: {e}", - ) - except Exception as e: - with sentry_sdk.push_scope() as scope: - scope.fingerprint = [e] - sentry_sdk.capture_message( - f"Unexpected error while verifying attestation: {e}" - ) - - raise _exc_with_message( - HTTPBadRequest, - f"Unknown error while trying to verify included attestations: {e}", - ) - - if predicate_type != AttestationType.PYPI_PUBLISH_V1: - metrics.increment( - "warehouse.upload.attestations.failed_unsupported_predicate_type" - ) - raise _exc_with_message( - HTTPBadRequest, - f"Attestation with unsupported predicate type: {predicate_type}", - ) - - # Log successful attestation upload - metrics.increment("warehouse.upload.attestations.ok") - - _pypi_project_urls = [ "https://pypi.org/project/", "https://pypi.org/p/", @@ -1269,12 +1181,6 @@ def file_upload(request): k: h.hexdigest().lower() for k, h in metadata_file_hashes.items() } - if "attestations" in request.POST: - _process_attestations( - request=request, - distribution=Distribution(name=filename, digest=file_hashes["sha256"]), - ) - # TODO: This should be handled by some sort of database trigger or a # SQLAlchemy hook or the like instead of doing it inline in this # view. @@ -1371,6 +1277,32 @@ def file_upload(request): }, ) + # If the user provided attestations, verify and store them + if "attestations" in request.POST: + integrity_service = request.find_service(IIntegrityService, context=None) + + try: + attestations: list[Attestation] = integrity_service.parse_attestations( + request, + Distribution(name=filename, digest=file_hashes["sha256"]), + ) + except AttestationUploadError as e: + raise _exc_with_message( + HTTPBadRequest, + str(e), + ) + + integrity_service.persist_attestations(attestations, file_) + + provenance = integrity_service.generate_provenance( + request.oidc_publisher, attestations + ) + if provenance: + integrity_service.persist_provenance(provenance, file_) + + # Log successful attestation upload + metrics.increment("warehouse.upload.attestations.ok") + # For existing releases, we check if any of the existing project URLs are unverified # and have been verified in the current upload. In that case, we mark them as # verified. diff --git a/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py b/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py index c86d454ddc5a..2b15277127f6 100644 --- a/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py +++ b/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py @@ -11,6 +11,7 @@ # limitations under the License. """ create Attestations table + Revision ID: 7f0c9f105f44 Revises: 26455e3712a2 Create Date: 2024-07-25 15:49:01.993869 diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 344689b36d0b..615739f3a1b8 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -81,6 +81,7 @@ from warehouse.utils.db.types import bool_false, datetime_now if typing.TYPE_CHECKING: + from warehouse.attestations.models import Attestation from warehouse.oidc.models import OIDCPublisher _MONOTONIC_SEQUENCE = 42 @@ -852,6 +853,13 @@ def __table_args__(cls): # noqa comment="If True, the metadata for the file cannot be backfilled.", ) + # PEP 740 attestations + attestations: Mapped[list[Attestation]] = orm.relationship( + cascade="all, delete-orphan", + lazy="joined", + passive_deletes=True, + ) + @property def uploaded_via_trusted_publisher(self) -> bool: """Return True if the file was uploaded via a trusted publisher.""" diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 30c85b3feb50..5fc67bc29761 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -19,10 +19,12 @@ from pyramid_jinja2 import IJinja2Environment from sqlalchemy.orm import joinedload +from warehouse.attestations import IIntegrityService +from warehouse.attestations.models import Attestation from warehouse.packaging.interfaces import ISimpleStorage from warehouse.packaging.models import File, LifecycleStatus, Project, Release -API_VERSION = "1.1" +API_VERSION = "1.2" def _simple_index(request, serial): @@ -51,6 +53,7 @@ def _simple_detail(project, request): request.db.query(File) .options(joinedload(File.release)) .join(Release) + .join(Attestation) .filter(Release.project == project) # Exclude projects that are in the `quarantine-enter` lifecycle status. .join(Project) @@ -64,6 +67,8 @@ def _simple_detail(project, request): {f.release.version for f in files}, key=packaging_legacy.version.parse ) + integrity_service = request.find_service(IIntegrityService, context=None) + return { "meta": {"api-version": API_VERSION, "_last-serial": project.last_serial}, "name": project.normalized_name, @@ -97,6 +102,7 @@ def _simple_detail(project, request): if file.metadata_file_sha256_digest else False ), + "provenance": integrity_service.get_provenance_digest(file), } for file in files ], diff --git a/warehouse/templates/api/simple/detail.html b/warehouse/templates/api/simple/detail.html index 24b0042c5863..05e0221a5612 100644 --- a/warehouse/templates/api/simple/detail.html +++ b/warehouse/templates/api/simple/detail.html @@ -20,7 +20,7 @@

Links for {{ name }}

{% for file in files -%} - {{ file.filename }}
+ {{ file.filename }}
{% endfor -%} From 20a4e4e6543b39b3f6a8272a30d15f776a2cf017 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 21 Aug 2024 16:38:55 -0400 Subject: [PATCH 02/34] migrations: re-roll migration history Signed-off-by: William Woodruff --- ...037669366ca_recreate_attestations_table.py | 47 +++++++++++++++++++ ...f1f5e7b3_rollback_attestation_migration.py | 31 ++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 warehouse/migrations/versions/4037669366ca_recreate_attestations_table.py create mode 100644 warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py diff --git a/warehouse/migrations/versions/4037669366ca_recreate_attestations_table.py b/warehouse/migrations/versions/4037669366ca_recreate_attestations_table.py new file mode 100644 index 000000000000..315b4e4acc95 --- /dev/null +++ b/warehouse/migrations/versions/4037669366ca_recreate_attestations_table.py @@ -0,0 +1,47 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +recreate attestations table + +Revision ID: 4037669366ca +Revises: 7ca0f1f5e7b3 +Create Date: 2024-08-21 20:33:53.489489 +""" + +import sqlalchemy as sa + +from alembic import op +from sqlalchemy.dialects import postgresql + +revision = "4037669366ca" +down_revision = "7ca0f1f5e7b3" + + +def upgrade(): + op.create_table( + "attestation", + sa.Column("file_id", sa.UUID(), nullable=False), + sa.Column( + "attestation_file_blake2_digest", postgresql.CITEXT(), nullable=False + ), + sa.Column( + "id", sa.UUID(), server_default=sa.text("gen_random_uuid()"), nullable=False + ), + sa.ForeignKeyConstraint( + ["file_id"], ["release_files.id"], onupdate="CASCADE", ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + + +def downgrade(): + op.drop_table("attestation") diff --git a/warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py b/warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py new file mode 100644 index 000000000000..eb36da1dfdd9 --- /dev/null +++ b/warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py @@ -0,0 +1,31 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Rollback attestation migration + +Revision ID: 7ca0f1f5e7b3 +Revises: 7f0c9f105f44 +Create Date: 2024-08-21 19:52:40.084048 +""" + +from alembic import op + +revision = "7ca0f1f5e7b3" +down_revision = "7f0c9f105f44" + + +def upgrade(): + op.drop_table("attestation") + + +def downgrade(): + pass From a058c41a7ba8f8c0407e49124801eb98190ccaf1 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 21 Aug 2024 17:08:13 -0400 Subject: [PATCH 03/34] config: register .attestations for inclusion Signed-off-by: William Woodruff --- warehouse/config.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/warehouse/config.py b/warehouse/config.py index 5c656fd11bc7..feb21bd6d1e4 100644 --- a/warehouse/config.py +++ b/warehouse/config.py @@ -454,12 +454,14 @@ def configure(settings=None): "VERIFY_EMAIL_RATELIMIT_STRING", default="3 per 6 hours", ) - maybe_set( - settings, - "warehouse.account.accounts_search_ratelimit_string", - "ACCOUNTS_SEARCH_RATELIMIT_STRING", - default="100 per hour", - ), + ( + maybe_set( + settings, + "warehouse.account.accounts_search_ratelimit_string", + "ACCOUNTS_SEARCH_RATELIMIT_STRING", + default="100 per hour", + ), + ) maybe_set( settings, "warehouse.account.password_reset_ratelimit_string", @@ -739,6 +741,9 @@ def configure(settings=None): # Register support for OIDC based authentication config.include(".oidc") + # Register support for attestations + config.include(".attestations") + # Register logged-in views config.include(".manage") From 19b773976d2ac4e5394041ad08282fe6cbff44d4 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 21 Aug 2024 17:23:13 -0400 Subject: [PATCH 04/34] attestations: request the appropriate IFileStorage service IFileStorage requires a name to disambiguate it. Signed-off-by: William Woodruff --- warehouse/attestations/services.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/warehouse/attestations/services.py b/warehouse/attestations/services.py index 056c47c4894e..169b3a3fb085 100644 --- a/warehouse/attestations/services.py +++ b/warehouse/attestations/services.py @@ -72,7 +72,6 @@ def _publisher_from_oidc_publisher(publisher: OIDCPublisher) -> Publisher: @implementer(IIntegrityService) class IntegrityService: - def __init__( self, storage: IFileStorage, @@ -84,7 +83,7 @@ def __init__( @classmethod def create_service(cls, _context, request: Request): return cls( - storage=request.find_service(IFileStorage), + storage=request.find_service(IFileStorage, name="archive"), metrics=request.find_service(IMetricsService), ) From f4a40ad198e71cd4809edf2a57285ff73a1fd689 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 21 Aug 2024 17:28:22 -0400 Subject: [PATCH 05/34] conftest: add archive_files.path to get_app_config Signed-off-by: William Woodruff --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index 4f276ee53f07..8b371456fd8e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -315,6 +315,7 @@ def get_app_config(database, nondefaults=None): "opensearch.url": "https://localhost/warehouse", "files.backend": "warehouse.packaging.services.LocalFileStorage", "archive_files.backend": "warehouse.packaging.services.LocalArchiveFileStorage", + "archive_files.path": "/var/opt/warehouse/packages-archive/", "simple.backend": "warehouse.packaging.services.LocalSimpleStorage", "docs.backend": "warehouse.packaging.services.LocalDocsStorage", "sponsorlogos.backend": "warehouse.admin.services.LocalSponsorLogoStorage", From b151b821271882715432ec2922b230db55fa542b Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 21 Aug 2024 18:21:14 -0400 Subject: [PATCH 06/34] test, warehouse: remove problematic mocks This removes two mocked `db_request`s from the simple index tests. These mocks were masking larger architectural issues with both attestations and our test scaffolding for attestations. This isn't quite complete yet, since it does a nasty thing (uses a file storage with a tmpdir) to get IntegrityService initialization working. Signed-off-by: William Woodruff --- tests/common/db/packaging.py | 3 ++- tests/conftest.py | 16 ++++++++++++++-- tests/unit/api/test_simple.py | 22 ---------------------- tests/unit/attestations/test_services.py | 4 +++- tests/unit/test_config.py | 1 + warehouse/packaging/utils.py | 4 +--- 6 files changed, 21 insertions(+), 29 deletions(-) diff --git a/tests/common/db/packaging.py b/tests/common/db/packaging.py index 369dc9f092d0..4085ee617dac 100644 --- a/tests/common/db/packaging.py +++ b/tests/common/db/packaging.py @@ -141,10 +141,11 @@ class Meta: ) ) + # Empty attestations by default. attestations = factory.RelatedFactoryList( AttestationFactory, factory_related_name="file", - size=1, + size=0, ) diff --git a/tests/conftest.py b/tests/conftest.py index 8b371456fd8e..a85bfe5b0f75 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -44,6 +44,8 @@ from warehouse.accounts import services as account_services from warehouse.accounts.interfaces import ITokenService, IUserService from warehouse.admin.flags import AdminFlag, AdminFlagValue +from warehouse.attestations import services as attestations_services +from warehouse.attestations.interfaces import IIntegrityService from warehouse.email import services as email_services from warehouse.email.interfaces import IEmailSender from warehouse.macaroons import services as macaroon_services @@ -171,6 +173,7 @@ def pyramid_services( project_service, github_oidc_service, activestate_oidc_service, + integrity_service, macaroon_service, ): services = _Services() @@ -191,6 +194,7 @@ def pyramid_services( services.register_service( activestate_oidc_service, IOIDCPublisherService, None, name="activestate" ) + services.register_service(integrity_service, IIntegrityService, None, name="") services.register_service(macaroon_service, IMacaroonService, None, name="") return services @@ -382,13 +386,11 @@ def get_db_session_for_app_config(app_config): @pytest.fixture(scope="session") def app_config(database): - return get_app_config(database) @pytest.fixture(scope="session") def app_config_dbsession_from_env(database): - nondefaults = { "warehouse.db_create_session": lambda r: r.environ.get("warehouse.db_session") } @@ -534,6 +536,16 @@ def activestate_oidc_service(db_session): ) +@pytest.fixture +def local_file_storage(tmp_path): + return packaging_services.GenericLocalBlobStorage(tmp_path) + + +@pytest.fixture +def integrity_service(db_session, local_file_storage, metrics): + return attestations_services.IntegrityService(local_file_storage, metrics) + + @pytest.fixture def macaroon_service(db_session): return macaroon_services.DatabaseMacaroonService(db_session) diff --git a/tests/unit/api/test_simple.py b/tests/unit/api/test_simple.py index 5f8003d50eb5..86b517d07d18 100644 --- a/tests/unit/api/test_simple.py +++ b/tests/unit/api/test_simple.py @@ -18,7 +18,6 @@ from pyramid.testing import DummyRequest from warehouse.api import simple -from warehouse.attestations import IIntegrityService from warehouse.packaging.utils import API_VERSION from ...common.db.accounts import UserFactory @@ -88,16 +87,6 @@ def test_selects(self, header, expected): class TestSimpleIndex: - - @pytest.fixture - def db_request(self, db_request): - """Override db_request to add the Release Verification service""" - db_request.find_service = lambda svc, name=None, context=None: { - IIntegrityService: pretend.stub(), - }.get(svc) - - return db_request - @pytest.mark.parametrize( ("content_type", "renderer_override"), CONTENT_TYPE_PARAMS, @@ -196,17 +185,6 @@ def test_quarantined_project_omitted_from_index(self, db_request): class TestSimpleDetail: - @pytest.fixture - def db_request(self, db_request): - """Override db_request to add the Release Verification service""" - db_request.find_service = lambda svc, name=None, context=None: { - IIntegrityService: pretend.stub( - get_provenance_digest=lambda *args, **kwargs: None, - ), - }.get(svc) - - return db_request - def test_redirects(self, pyramid_request): project = pretend.stub(normalized_name="foo") diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py index a83a0c302b00..1e2a6b106d74 100644 --- a/tests/unit/attestations/test_services.py +++ b/tests/unit/attestations/test_services.py @@ -30,6 +30,7 @@ from sigstore.verify import Verifier from zope.interface.verify import verifyClass +from tests.common.db.attestation import AttestationFactory from tests.common.db.oidc import GitHubPublisherFactory, GitLabPublisherFactory from tests.common.db.packaging import FileEventFactory, FileFactory from warehouse.attestations import ( @@ -69,7 +70,7 @@ def test_create_service(self): assert IntegrityService.create_service(None, request) is not None assert not set(request.find_service.calls) ^ { - pretend.call(IFileStorage), + pretend.call(IFileStorage, name="archive"), pretend.call(IMetricsService), } @@ -359,6 +360,7 @@ def storage_service_store(path, file_path, *_args, **_kwargs): def test_get_provenance_digest(self, db_request): file = FileFactory.create() + AttestationFactory.create(file=file) FileEventFactory.create( source=file, tag=EventTag.File.FileAdd, diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 71cd6f2e660c..fd3254b1a33c 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -348,6 +348,7 @@ def __init__(self): pretend.call(".accounts"), pretend.call(".macaroons"), pretend.call(".oidc"), + pretend.call(".attestations"), pretend.call(".manage"), pretend.call(".organizations"), pretend.call(".subscriptions"), diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 5fc67bc29761..03c33a94a844 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -20,7 +20,6 @@ from sqlalchemy.orm import joinedload from warehouse.attestations import IIntegrityService -from warehouse.attestations.models import Attestation from warehouse.packaging.interfaces import ISimpleStorage from warehouse.packaging.models import File, LifecycleStatus, Project, Release @@ -53,7 +52,6 @@ def _simple_detail(project, request): request.db.query(File) .options(joinedload(File.release)) .join(Release) - .join(Attestation) .filter(Release.project == project) # Exclude projects that are in the `quarantine-enter` lifecycle status. .join(Project) @@ -67,7 +65,7 @@ def _simple_detail(project, request): {f.release.version for f in files}, key=packaging_legacy.version.parse ) - integrity_service = request.find_service(IIntegrityService, context=None) + integrity_service = request.find_service(IIntegrityService) return { "meta": {"api-version": API_VERSION, "_last-serial": project.last_serial}, From 64ed3e5a36e3e6a9d9c55aa413f88ddecb49b55c Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 21 Aug 2024 18:41:33 -0400 Subject: [PATCH 07/34] test_services: rename test class Signed-off-by: William Woodruff --- tests/unit/attestations/test_services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py index 1e2a6b106d74..0cbd92a191fe 100644 --- a/tests/unit/attestations/test_services.py +++ b/tests/unit/attestations/test_services.py @@ -57,7 +57,7 @@ ) -class TestAttestationsService: +class TestIntegrityService: def test_interface_matches(self): assert verifyClass(IIntegrityService, IntegrityService) From e19be6c30d07e0d652b513b3d9128bc61e60dee5 Mon Sep 17 00:00:00 2001 From: Alexis Date: Thu, 22 Aug 2024 15:55:55 +0200 Subject: [PATCH 08/34] Try to clean a bit the mess with the migrations. --- ...037669366ca_recreate_attestations_table.py | 47 ------------------- ...f1f5e7b3_rollback_attestation_migration.py | 31 ------------ ...9091e5fb04a4_create_attestations_table.py} | 6 +-- 3 files changed, 3 insertions(+), 81 deletions(-) delete mode 100644 warehouse/migrations/versions/4037669366ca_recreate_attestations_table.py delete mode 100644 warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py rename warehouse/migrations/versions/{7f0c9f105f44_create_attestations_table.py => 9091e5fb04a4_create_attestations_table.py} (93%) diff --git a/warehouse/migrations/versions/4037669366ca_recreate_attestations_table.py b/warehouse/migrations/versions/4037669366ca_recreate_attestations_table.py deleted file mode 100644 index 315b4e4acc95..000000000000 --- a/warehouse/migrations/versions/4037669366ca_recreate_attestations_table.py +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -recreate attestations table - -Revision ID: 4037669366ca -Revises: 7ca0f1f5e7b3 -Create Date: 2024-08-21 20:33:53.489489 -""" - -import sqlalchemy as sa - -from alembic import op -from sqlalchemy.dialects import postgresql - -revision = "4037669366ca" -down_revision = "7ca0f1f5e7b3" - - -def upgrade(): - op.create_table( - "attestation", - sa.Column("file_id", sa.UUID(), nullable=False), - sa.Column( - "attestation_file_blake2_digest", postgresql.CITEXT(), nullable=False - ), - sa.Column( - "id", sa.UUID(), server_default=sa.text("gen_random_uuid()"), nullable=False - ), - sa.ForeignKeyConstraint( - ["file_id"], ["release_files.id"], onupdate="CASCADE", ondelete="CASCADE" - ), - sa.PrimaryKeyConstraint("id"), - ) - - -def downgrade(): - op.drop_table("attestation") diff --git a/warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py b/warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py deleted file mode 100644 index eb36da1dfdd9..000000000000 --- a/warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Rollback attestation migration - -Revision ID: 7ca0f1f5e7b3 -Revises: 7f0c9f105f44 -Create Date: 2024-08-21 19:52:40.084048 -""" - -from alembic import op - -revision = "7ca0f1f5e7b3" -down_revision = "7f0c9f105f44" - - -def upgrade(): - op.drop_table("attestation") - - -def downgrade(): - pass diff --git a/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py b/warehouse/migrations/versions/9091e5fb04a4_create_attestations_table.py similarity index 93% rename from warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py rename to warehouse/migrations/versions/9091e5fb04a4_create_attestations_table.py index 2b15277127f6..378e4c888b5e 100644 --- a/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py +++ b/warehouse/migrations/versions/9091e5fb04a4_create_attestations_table.py @@ -12,9 +12,9 @@ """ create Attestations table -Revision ID: 7f0c9f105f44 +Revision ID: 9091e5fb04a4 Revises: 26455e3712a2 -Create Date: 2024-07-25 15:49:01.993869 +Create Date: 2024-08-22 13:54:03.141492 """ import sqlalchemy as sa @@ -22,7 +22,7 @@ from alembic import op from sqlalchemy.dialects import postgresql -revision = "7f0c9f105f44" +revision = "9091e5fb04a4" down_revision = "26455e3712a2" From 98b833dec88e2fe7226c70f2f99e6af49aff989b Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 22 Aug 2024 11:37:18 -0400 Subject: [PATCH 09/34] begin refactoring IntegrityService This reduces the overall API surface for IIntegrityService implementers, and adds an initial NullIntegrityService to make unit-level testing simpler. Signed-off-by: William Woodruff --- tests/conftest.py | 9 +- tests/unit/attestations/test_services.py | 8 +- tests/unit/forklift/test_legacy.py | 21 +-- warehouse/attestations/interfaces.py | 23 +-- warehouse/attestations/services.py | 182 ++++++++++++++++------- warehouse/forklift/legacy.py | 8 +- warehouse/utils/exceptions.py | 4 + 7 files changed, 147 insertions(+), 108 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index a85bfe5b0f75..ae715a999d09 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -537,13 +537,8 @@ def activestate_oidc_service(db_session): @pytest.fixture -def local_file_storage(tmp_path): - return packaging_services.GenericLocalBlobStorage(tmp_path) - - -@pytest.fixture -def integrity_service(db_session, local_file_storage, metrics): - return attestations_services.IntegrityService(local_file_storage, metrics) +def integrity_service(db_session): + return attestations_services.NullIntegrityService() @pytest.fixture diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py index 0cbd92a191fe..a344ac9e9afb 100644 --- a/tests/unit/attestations/test_services.py +++ b/tests/unit/attestations/test_services.py @@ -92,7 +92,7 @@ def storage_service_store(path: str, file_path, *_args, **_kwargs): file = FileFactory.create(attestations=[]) - integrity_service.persist_attestations([VALID_ATTESTATION], file) + integrity_service._persist_attestations([VALID_ATTESTATION], file) attestations_db = ( db_request.db.query(DatabaseAttestation) @@ -280,7 +280,7 @@ def test_generate_provenance_unsupported_publisher(self): oidc_publisher = pretend.stub(publisher_name="not-existing") assert ( - integrity_service.generate_provenance(oidc_publisher, pretend.stub()) + integrity_service._generate_provenance(oidc_publisher, pretend.stub()) is None ) @@ -314,7 +314,7 @@ def test_generate_provenance_succeeds(self, publisher_name: str, monkeypatch): lambda s: publisher, ) - provenance = integrity_service.generate_provenance( + provenance = integrity_service._generate_provenance( pretend.stub(), [VALID_ATTESTATION], ) @@ -354,7 +354,7 @@ def storage_service_store(path, file_path, *_args, **_kwargs): metrics=pretend.stub(), ) assert ( - integrity_service.persist_provenance(provenance, FileFactory.create()) + integrity_service._persist_provenance(provenance, FileFactory.create()) is None ) diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index 6e062ef1f0f2..4480d9284a54 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -3328,14 +3328,13 @@ def test_upload_succeeds_creates_release( ), ] - @pytest.mark.parametrize("provenance_rv", [None, "fake-provenance-object"]) def test_upload_succeeds_with_valid_attestation( self, - provenance_rv, monkeypatch, pyramid_config, db_request, metrics, + integrity_service, ): from warehouse.events.models import HasEvents @@ -3386,21 +3385,7 @@ def test_upload_succeeds_with_valid_attestation( } ) - def persist_attestations(attestations, file): - file.attestations.append(AttestationFactory.create(file=file)) - - def persist_provenance(provenance_object, file): - assert provenance_object == provenance_rv - storage_service = pretend.stub(store=lambda path, filepath, *, meta=None: None) - integrity_service = pretend.stub( - parse_attestations=lambda *args, **kwargs: [attestation], - persist_attestations=persist_attestations, - generate_provenance=pretend.call_recorder( - lambda oidc_publisher, attestations: provenance_rv - ), - persist_provenance=persist_provenance, - ) db_request.find_service = lambda svc, name=None, context=None: { IFileStorage: storage_service, IMetricsService: metrics, @@ -3426,10 +3411,6 @@ def persist_provenance(provenance_object, file): ) assert len(attestations_db) == 1 - assert integrity_service.generate_provenance.calls == [ - pretend.call(db_request.oidc_publisher, [attestation]) - ] - @pytest.mark.parametrize( "expected_message", [ diff --git a/warehouse/attestations/interfaces.py b/warehouse/attestations/interfaces.py index 4055e92055a5..46d13995ffca 100644 --- a/warehouse/attestations/interfaces.py +++ b/warehouse/attestations/interfaces.py @@ -16,19 +16,6 @@ class IIntegrityService(Interface): - - def create_service(context, request): - """ - Create the service, given the context and request for which it is being - created for. - """ - - def persist_attestations(attestations: list[Attestation], file): - """ - ̦Persist attestations in storage. - """ - pass - def parse_attestations( request: Request, distribution: Distribution ) -> list[Attestation]: @@ -37,15 +24,11 @@ def parse_attestations( """ def generate_provenance( - oidc_publisher, attestations: list[Attestation] + request, file, attestations: list[Attestation] ) -> Provenance | None: """ - Generate a Provenance object from an OIDCPublisher and its attestations. - """ - - def persist_provenance(provenance: Provenance, file) -> None: - """ - Persist a Provenance object in storage. + Generate and persist a Provenance object for the given file and list of + associated attestations. """ def get_provenance_digest(file) -> str | None: diff --git a/warehouse/attestations/services.py b/warehouse/attestations/services.py index 169b3a3fb085..fa99e595145b 100644 --- a/warehouse/attestations/services.py +++ b/warehouse/attestations/services.py @@ -12,11 +12,10 @@ import hashlib import tempfile import typing - +import warnings from pathlib import Path import sentry_sdk - from pydantic import TypeAdapter, ValidationError from pypi_attestations import ( Attestation, @@ -42,11 +41,16 @@ from warehouse.metrics.interfaces import IMetricsService from warehouse.oidc.models import ( GitHubPublisher as GitHubOIDCPublisher, +) +from warehouse.oidc.models import ( GitLabPublisher as GitLabOIDCPublisher, +) +from warehouse.oidc.models import ( OIDCPublisher, ) from warehouse.packaging.interfaces import IFileStorage from warehouse.packaging.models import File +from warehouse.utils.exceptions import InsecureIntegrityServiceWarning def _publisher_from_oidc_publisher(publisher: OIDCPublisher) -> Publisher: @@ -70,6 +74,83 @@ def _publisher_from_oidc_publisher(publisher: OIDCPublisher) -> Publisher: raise UnsupportedPublisherError +def _extract_attestations_from_request(request: Request) -> list[Attestation]: + """ + Extract well-formed attestation objects from the given request's payload. + """ + + metrics = request.find_service(IMetricsService, context=None) + + try: + attestations = TypeAdapter(list[Attestation]).validate_json( + request.POST["attestations"] + ) + except ValidationError as e: + # Log invalid (malformed) attestation upload + metrics.increment("warehouse.upload.attestations.malformed") + raise AttestationUploadError( + f"Error while decoding the included attestation: {e}", + ) + + if len(attestations) > 1: + metrics.increment("warehouse.upload.attestations.failed_multiple_attestations") + + raise AttestationUploadError( + "Only a single attestation per file is supported.", + ) + + return attestations + + +@implementer(IIntegrityService) +class NullIntegrityService: + def __init__(self): + warnings.warn( + "NullIntegrityService is intended only for use in development, " + "you should not use it in production due to the lack of actual " + "attestation verification.", + InsecureIntegrityServiceWarning, + ) + + def parse_attestations( + self, request: Request, _distribution: Distribution + ) -> list[Attestation]: + return _extract_attestations_from_request(request) + + def generate_provenance( + self, request: Request, file: File, attestations: list[Attestation] + ) -> Provenance | None: + publisher = _publisher_from_oidc_publisher(request.oidc_publisher) + attestation_bundle = AttestationBundle( + publisher=publisher, + attestations=attestations, + ) + provenance = Provenance(attestation_bundles=[attestation_bundle]) + + for attestation in attestations: + db_attestation = DatabaseAttestation( + file=file, + attestation_file_blake2_digest=hashlib.blake2b( + attestation.model_dump_json().encode("utf-8") + ).hexdigest(), + ) + file.attestations.append(db_attestation) + + return provenance + + def get_provenance_digest(self, file: File) -> str | None: + if not file.attestations: + return None + + # For the null service, our "provenance digest" is just the digest + # of the release file's name merged with the number of attestations. + # We do this because there's no verification involved; we just need + # a unique value to preserve invariants. + return hashlib.sha256( + f"{file.filename}:{len(file.attestations)}".encode() + ).hexdigest() + + @implementer(IIntegrityService) class IntegrityService: def __init__( @@ -87,26 +168,6 @@ def create_service(cls, _context, request: Request): metrics=request.find_service(IMetricsService), ) - def persist_attestations(self, attestations: list[Attestation], file: File) -> None: - for attestation in attestations: - with tempfile.NamedTemporaryFile() as tmp_file: - tmp_file.write(attestation.model_dump_json().encode("utf-8")) - - attestation_digest = hashlib.file_digest( - tmp_file, "blake2b" - ).hexdigest() - database_attestation = DatabaseAttestation( - file=file, attestation_file_blake2_digest=attestation_digest - ) - - self.storage.store( - database_attestation.attestation_path, - tmp_file.name, - meta=None, - ) - - file.attestations.append(database_attestation) - def parse_attestations( self, request: Request, distribution: Distribution ) -> list[Attestation]: @@ -126,25 +187,7 @@ def parse_attestations( "Publishing with GitHub Actions.", ) - try: - attestations = TypeAdapter(list[Attestation]).validate_json( - request.POST["attestations"] - ) - except ValidationError as e: - # Log invalid (malformed) attestation upload - self.metrics.increment("warehouse.upload.attestations.malformed") - raise AttestationUploadError( - f"Error while decoding the included attestation: {e}", - ) - - if len(attestations) > 1: - self.metrics.increment( - "warehouse.upload.attestations.failed_multiple_attestations" - ) - - raise AttestationUploadError( - "Only a single attestation per file is supported.", - ) + attestations = _extract_attestations_from_request(request) verification_policy = publisher.publisher_verification_policy( request.oidc_claims @@ -185,6 +228,53 @@ def parse_attestations( return attestations def generate_provenance( + self, request: Request, file: File, attestations: list[Attestation] + ) -> Provenance | None: + # Generate the provenance object. + provenance = self._generate_provenance(request.oidc_publisher, attestations) + + if not provenance: + return None + + # Persist the attestations and provenance objects. We only do this + # after generating the provenance above, to prevent orphaned artifacts + # on any generation failures. + self._persist_attestations(attestations, file) + self._persist_provenance(provenance, file) + + return provenance + + def get_provenance_digest(self, file: File) -> str | None: + """Returns the sha256 digest of the provenance file for the release.""" + if not file.attestations: + return None + + provenance_file = self.storage.get(f"{file.path}.provenance") + return hashlib.file_digest(provenance_file, "sha256").hexdigest() + + def _persist_attestations( + self, attestations: list[Attestation], file: File + ) -> None: + for attestation in attestations: + with tempfile.NamedTemporaryFile() as tmp_file: + tmp_file.write(attestation.model_dump_json().encode("utf-8")) + + attestation_digest = hashlib.file_digest( + tmp_file, "blake2b" + ).hexdigest() + database_attestation = DatabaseAttestation( + file=file, attestation_file_blake2_digest=attestation_digest + ) + + self.storage.store( + database_attestation.attestation_path, + tmp_file.name, + meta=None, + ) + + file.attestations.append(database_attestation) + + def _generate_provenance( self, oidc_publisher: OIDCPublisher, attestations: list[Attestation] ) -> Provenance | None: try: @@ -203,7 +293,7 @@ def generate_provenance( return Provenance(attestation_bundles=[attestation_bundle]) - def persist_provenance( + def _persist_provenance( self, provenance: Provenance, file: File, @@ -220,11 +310,3 @@ def persist_provenance( provenance_file_path, f.name, ) - - def get_provenance_digest(self, file: File) -> str | None: - """Returns the sha256 digest of the provenance file for the release.""" - if not file.attestations: - return None - - provenance_file = self.storage.get(f"{file.path}.provenance") - return hashlib.file_digest(provenance_file, "sha256").hexdigest() diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index baae8da64e84..0f71ad51f376 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -1292,13 +1292,7 @@ def file_upload(request): str(e), ) - integrity_service.persist_attestations(attestations, file_) - - provenance = integrity_service.generate_provenance( - request.oidc_publisher, attestations - ) - if provenance: - integrity_service.persist_provenance(provenance, file_) + integrity_service.generate_provenance(request, file_, attestations) # Log successful attestation upload metrics.increment("warehouse.upload.attestations.ok") diff --git a/warehouse/utils/exceptions.py b/warehouse/utils/exceptions.py index 238c9222de1b..2b4b3c674e5d 100644 --- a/warehouse/utils/exceptions.py +++ b/warehouse/utils/exceptions.py @@ -13,3 +13,7 @@ class InsecureOIDCPublisherWarning(UserWarning): pass + + +class InsecureIntegrityServiceWarning(UserWarning): + pass From 4549711fb90695e5afefc1fda529cae6303e56ce Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 22 Aug 2024 11:41:15 -0400 Subject: [PATCH 10/34] Revert "Try to clean a bit the mess with the migrations." This reverts commit e19be6c30d07e0d652b513b3d9128bc61e60dee5. --- ...037669366ca_recreate_attestations_table.py | 47 +++++++++++++++++++ ...f1f5e7b3_rollback_attestation_migration.py | 31 ++++++++++++ ...7f0c9f105f44_create_attestations_table.py} | 6 +-- 3 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 warehouse/migrations/versions/4037669366ca_recreate_attestations_table.py create mode 100644 warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py rename warehouse/migrations/versions/{9091e5fb04a4_create_attestations_table.py => 7f0c9f105f44_create_attestations_table.py} (93%) diff --git a/warehouse/migrations/versions/4037669366ca_recreate_attestations_table.py b/warehouse/migrations/versions/4037669366ca_recreate_attestations_table.py new file mode 100644 index 000000000000..315b4e4acc95 --- /dev/null +++ b/warehouse/migrations/versions/4037669366ca_recreate_attestations_table.py @@ -0,0 +1,47 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +recreate attestations table + +Revision ID: 4037669366ca +Revises: 7ca0f1f5e7b3 +Create Date: 2024-08-21 20:33:53.489489 +""" + +import sqlalchemy as sa + +from alembic import op +from sqlalchemy.dialects import postgresql + +revision = "4037669366ca" +down_revision = "7ca0f1f5e7b3" + + +def upgrade(): + op.create_table( + "attestation", + sa.Column("file_id", sa.UUID(), nullable=False), + sa.Column( + "attestation_file_blake2_digest", postgresql.CITEXT(), nullable=False + ), + sa.Column( + "id", sa.UUID(), server_default=sa.text("gen_random_uuid()"), nullable=False + ), + sa.ForeignKeyConstraint( + ["file_id"], ["release_files.id"], onupdate="CASCADE", ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + + +def downgrade(): + op.drop_table("attestation") diff --git a/warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py b/warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py new file mode 100644 index 000000000000..eb36da1dfdd9 --- /dev/null +++ b/warehouse/migrations/versions/7ca0f1f5e7b3_rollback_attestation_migration.py @@ -0,0 +1,31 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Rollback attestation migration + +Revision ID: 7ca0f1f5e7b3 +Revises: 7f0c9f105f44 +Create Date: 2024-08-21 19:52:40.084048 +""" + +from alembic import op + +revision = "7ca0f1f5e7b3" +down_revision = "7f0c9f105f44" + + +def upgrade(): + op.drop_table("attestation") + + +def downgrade(): + pass diff --git a/warehouse/migrations/versions/9091e5fb04a4_create_attestations_table.py b/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py similarity index 93% rename from warehouse/migrations/versions/9091e5fb04a4_create_attestations_table.py rename to warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py index 378e4c888b5e..2b15277127f6 100644 --- a/warehouse/migrations/versions/9091e5fb04a4_create_attestations_table.py +++ b/warehouse/migrations/versions/7f0c9f105f44_create_attestations_table.py @@ -12,9 +12,9 @@ """ create Attestations table -Revision ID: 9091e5fb04a4 +Revision ID: 7f0c9f105f44 Revises: 26455e3712a2 -Create Date: 2024-08-22 13:54:03.141492 +Create Date: 2024-07-25 15:49:01.993869 """ import sqlalchemy as sa @@ -22,7 +22,7 @@ from alembic import op from sqlalchemy.dialects import postgresql -revision = "9091e5fb04a4" +revision = "7f0c9f105f44" down_revision = "26455e3712a2" From 77d08a263a39881b3c9088bed1375df7f1a1b27c Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 22 Aug 2024 12:01:43 -0400 Subject: [PATCH 11/34] tests, warehouse: more error tests, remove more stubs Signed-off-by: William Woodruff --- tests/unit/forklift/test_legacy.py | 47 +++++++++++++++++------------- warehouse/attestations/services.py | 19 ++++++++---- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index 4480d9284a54..56ad792ecff8 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -12,6 +12,7 @@ import hashlib import io +import json import re import tarfile import tempfile @@ -35,11 +36,7 @@ from warehouse.accounts.utils import UserContext from warehouse.admin.flags import AdminFlag, AdminFlagValue -from warehouse.attestations import ( - Attestation as DatabaseAttestation, - AttestationUploadError, - IIntegrityService, -) +from warehouse.attestations import Attestation as DatabaseAttestation, IIntegrityService from warehouse.classifiers.models import Classifier from warehouse.forklift import legacy, metadata from warehouse.macaroons import IMacaroonService, caveats, security_policy @@ -61,7 +58,6 @@ from warehouse.packaging.tasks import sync_file_to_cache, update_bigquery_release_files from ...common.db.accounts import EmailFactory, UserFactory -from ...common.db.attestation import AttestationFactory from ...common.db.classifiers import ClassifierFactory from ...common.db.oidc import GitHubPublisherFactory from ...common.db.packaging import ( @@ -3412,14 +3408,28 @@ def test_upload_succeeds_with_valid_attestation( assert len(attestations_db) == 1 @pytest.mark.parametrize( - "expected_message", + "invalid_attestations", [ - "Attestations are only supported when using", - "Error while decoding the included attestation", - "Only a single attestation", - "Could not verify the uploaded", - "Unknown error while trying", - "Attestation with unsupported predicate", + # Bad top-level types. + "", + {}, + 1, + # Empty attestation sets not permitted. + [], + # Wrong version number. + [ + { + "version": 2, + "verification_material": { + "certificate": "somebase64string", + "transparency_entries": [{}], + }, + "envelope": { + "statement": "somebase64string", + "signature": "somebase64string", + }, + }, + ], ], ) def test_upload_fails_attestation_error( @@ -3428,7 +3438,8 @@ def test_upload_fails_attestation_error( pyramid_config, db_request, metrics, - expected_message, + integrity_service, + invalid_attestations, ): from warehouse.events.models import HasEvents @@ -3456,7 +3467,7 @@ def test_upload_fails_attestation_error( { "metadata_version": "1.2", "name": project.name, - "attestations": "", + "attestations": json.dumps(invalid_attestations), "version": version, "summary": "This is my summary!", "filetype": "sdist", @@ -3471,10 +3482,6 @@ def test_upload_fails_attestation_error( storage_service = pretend.stub(store=lambda path, filepath, meta: None) - def stub_parse(*_args, **_kwargs): - raise AttestationUploadError(expected_message) - - integrity_service = pretend.stub(parse_attestations=stub_parse) db_request.find_service = lambda svc, name=None, context=None: { IFileStorage: storage_service, IMetricsService: metrics, @@ -3492,7 +3499,7 @@ def stub_parse(*_args, **_kwargs): resp = excinfo.value assert resp.status_code == 400 - assert resp.status.startswith(f"400 {expected_message}") + assert resp.status.startswith("400 Malformed attestations") @pytest.mark.parametrize( ("url", "expected"), diff --git a/warehouse/attestations/services.py b/warehouse/attestations/services.py index fa99e595145b..47e74d3ac9c5 100644 --- a/warehouse/attestations/services.py +++ b/warehouse/attestations/services.py @@ -13,9 +13,11 @@ import tempfile import typing import warnings + from pathlib import Path import sentry_sdk + from pydantic import TypeAdapter, ValidationError from pypi_attestations import ( Attestation, @@ -41,11 +43,7 @@ from warehouse.metrics.interfaces import IMetricsService from warehouse.oidc.models import ( GitHubPublisher as GitHubOIDCPublisher, -) -from warehouse.oidc.models import ( GitLabPublisher as GitLabOIDCPublisher, -) -from warehouse.oidc.models import ( OIDCPublisher, ) from warehouse.packaging.interfaces import IFileStorage @@ -89,14 +87,23 @@ def _extract_attestations_from_request(request: Request) -> list[Attestation]: # Log invalid (malformed) attestation upload metrics.increment("warehouse.upload.attestations.malformed") raise AttestationUploadError( - f"Error while decoding the included attestation: {e}", + f"Malformed attestations: {e}", + ) + + # Empty attestation sets are not permitted; users should omit `attestations` + # entirely to upload without attestations. + if not attestations: + raise AttestationUploadError( + "Malformed attestations: an empty attestation set is not permitted" ) + # This is a temporary constraint; multiple attestations per file will + # be supported in the future. if len(attestations) > 1: metrics.increment("warehouse.upload.attestations.failed_multiple_attestations") raise AttestationUploadError( - "Only a single attestation per file is supported.", + "Only a single attestation per file is supported", ) return attestations From 014f6492d5a7643bc62d6b13ccbe69b20a160e07 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 22 Aug 2024 12:05:19 -0400 Subject: [PATCH 12/34] test_services: fix match Signed-off-by: William Woodruff --- tests/unit/attestations/test_services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py index a344ac9e9afb..ccca1b7fe8fa 100644 --- a/tests/unit/attestations/test_services.py +++ b/tests/unit/attestations/test_services.py @@ -138,7 +138,7 @@ def test_parse_malformed_attestation(self, metrics, db_request): db_request.POST["attestations"] = "{'malformed-attestation'}" with pytest.raises( AttestationUploadError, - match="Error while decoding the included attestation", + match="Malformed attestations", ): integrity_service.parse_attestations(db_request, pretend.stub()) From 46f32e264ec410d3a1a0eef9e813295fbb7fb9d9 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 22 Aug 2024 12:27:31 -0400 Subject: [PATCH 13/34] remove more implicit file service deps Signed-off-by: William Woodruff --- dev/environment | 1 + tests/conftest.py | 2 +- tests/unit/attestations/test_init.py | 11 +++++++++-- tests/unit/attestations/test_services.py | 5 +++++ tests/unit/test_config.py | 1 + warehouse/attestations/__init__.py | 7 ++++++- warehouse/attestations/interfaces.py | 5 +++++ warehouse/attestations/services.py | 4 ++++ warehouse/config.py | 6 ++++++ 9 files changed, 38 insertions(+), 4 deletions(-) diff --git a/dev/environment b/dev/environment index 787b083958ac..d376f9d4e222 100644 --- a/dev/environment +++ b/dev/environment @@ -46,6 +46,7 @@ BREACHED_EMAILS=warehouse.accounts.NullEmailBreachedService BREACHED_PASSWORDS=warehouse.accounts.NullPasswordBreachedService OIDC_BACKEND=warehouse.oidc.services.NullOIDCPublisherService +ATTESTATIONS_BACKEND=warehouse.attestations.services.NullIntegrityService METRICS_BACKEND=warehouse.metrics.DataDogMetrics host=notdatadog diff --git a/tests/conftest.py b/tests/conftest.py index ae715a999d09..0ef1569592c7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -319,11 +319,11 @@ def get_app_config(database, nondefaults=None): "opensearch.url": "https://localhost/warehouse", "files.backend": "warehouse.packaging.services.LocalFileStorage", "archive_files.backend": "warehouse.packaging.services.LocalArchiveFileStorage", - "archive_files.path": "/var/opt/warehouse/packages-archive/", "simple.backend": "warehouse.packaging.services.LocalSimpleStorage", "docs.backend": "warehouse.packaging.services.LocalDocsStorage", "sponsorlogos.backend": "warehouse.admin.services.LocalSponsorLogoStorage", "billing.backend": "warehouse.subscriptions.services.MockStripeBillingService", + "attestations.backend": "warehouse.attestations.services.NullIntegrityService", "billing.api_base": "http://stripe:12111", "billing.api_version": "2020-08-27", "mail.backend": "warehouse.email.services.SMTPEmailSender", diff --git a/tests/unit/attestations/test_init.py b/tests/unit/attestations/test_init.py index c59080b7c5a3..6c5e26a8776f 100644 --- a/tests/unit/attestations/test_init.py +++ b/tests/unit/attestations/test_init.py @@ -14,11 +14,17 @@ from warehouse import attestations from warehouse.attestations.interfaces import IIntegrityService -from warehouse.attestations.services import IntegrityService def test_includeme(): + fake_service_klass = pretend.stub(create_service=pretend.stub()) config = pretend.stub( + registry=pretend.stub( + settings={"attestations.backend": "fake.path.to.backend"} + ), + maybe_dotted=pretend.call_recorder( + lambda attr: fake_service_klass, + ), register_service_factory=pretend.call_recorder( lambda factory, iface, name=None: None ), @@ -26,6 +32,7 @@ def test_includeme(): attestations.includeme(config) + assert config.maybe_dotted.calls == [pretend.call("fake.path.to.backend")] assert config.register_service_factory.calls == [ - pretend.call(IntegrityService.create_service, IIntegrityService), + pretend.call(fake_service_klass.create_service, IIntegrityService), ] diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py index ccca1b7fe8fa..fc54969a8647 100644 --- a/tests/unit/attestations/test_services.py +++ b/tests/unit/attestations/test_services.py @@ -57,6 +57,11 @@ ) +class TestNullIntegrityService: + def test_interface_matches(self): + assert verifyClass(IIntegrityService, services.NullIntegrityService) + + class TestIntegrityService: def test_interface_matches(self): assert verifyClass(IIntegrityService, IntegrityService) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index fd3254b1a33c..ccdf189ec57a 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -250,6 +250,7 @@ def __init__(self): "warehouse.packaging.project_create_user_ratelimit_string": "20 per hour", "warehouse.packaging.project_create_ip_ratelimit_string": "40 per hour", "oidc.backend": "warehouse.oidc.services.OIDCPublisherService", + "attestations.backend": "warehouse.attestations.services.IntegrityService", "warehouse.organizations.max_undecided_organization_applications": 3, "reconcile_file_storages.batch_size": 100, "metadata_backfill.batch_size": 500, diff --git a/warehouse/attestations/__init__.py b/warehouse/attestations/__init__.py index 208684d87587..783b03e02fcd 100644 --- a/warehouse/attestations/__init__.py +++ b/warehouse/attestations/__init__.py @@ -28,4 +28,9 @@ def includeme(config): - config.register_service_factory(IntegrityService.create_service, IIntegrityService) + integrity_service_class = config.maybe_dotted( + config.registry.settings["attestations.backend"] + ) + config.register_service_factory( + integrity_service_class.create_service, IIntegrityService + ) diff --git a/warehouse/attestations/interfaces.py b/warehouse/attestations/interfaces.py index 46d13995ffca..c6fa9ef4c86b 100644 --- a/warehouse/attestations/interfaces.py +++ b/warehouse/attestations/interfaces.py @@ -16,6 +16,11 @@ class IIntegrityService(Interface): + def create_service(context, request): + """ + Create the service for the given context and request. + """ + def parse_attestations( request: Request, distribution: Distribution ) -> list[Attestation]: diff --git a/warehouse/attestations/services.py b/warehouse/attestations/services.py index 47e74d3ac9c5..6c48884d1da5 100644 --- a/warehouse/attestations/services.py +++ b/warehouse/attestations/services.py @@ -119,6 +119,10 @@ def __init__(self): InsecureIntegrityServiceWarning, ) + @classmethod + def create_service(cls, _context, _request): + return cls() + def parse_attestations( self, request: Request, _distribution: Distribution ) -> list[Attestation]: diff --git a/warehouse/config.py b/warehouse/config.py index feb21bd6d1e4..443a413aab86 100644 --- a/warehouse/config.py +++ b/warehouse/config.py @@ -405,6 +405,12 @@ def configure(settings=None): "OIDC_BACKEND", default="warehouse.oidc.services.OIDCPublisherService", ) + maybe_set( + settings, + "attestations.backend", + "ATTESTATIONS_BACKEND", + default="warehouse.attestations.services.IntegrityService", + ) # Pythondotorg integration settings maybe_set( From 7e7ea8c214de6adbaef856529f1cbf85edebff35 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 22 Aug 2024 12:56:56 -0400 Subject: [PATCH 14/34] continue to burn down coverage Remove more ad-hoc stubs as well. Signed-off-by: William Woodruff --- tests/unit/attestations/test_services.py | 66 ++++++++++++++---------- warehouse/attestations/services.py | 4 +- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py index fc54969a8647..b130528c052e 100644 --- a/tests/unit/attestations/test_services.py +++ b/tests/unit/attestations/test_services.py @@ -61,6 +61,23 @@ class TestNullIntegrityService: def test_interface_matches(self): assert verifyClass(IIntegrityService, services.NullIntegrityService) + def test_get_provenance_digest(self, db_request): + db_request.oidc_publisher = pretend.stub( + publisher_name="GitHub", + repository="fake/fake", + workflow_filename="fake.yml", + environment="fake", + ) + + file = FileFactory.create() + service = services.NullIntegrityService() + + provenance = service.generate_provenance(db_request, file, [VALID_ATTESTATION]) + assert isinstance(provenance, Provenance) + + provenance_digest = service.get_provenance_digest(file) + assert isinstance(provenance_digest, str) + class TestIntegrityService: def test_interface_matches(self): @@ -285,55 +302,48 @@ def test_generate_provenance_unsupported_publisher(self): oidc_publisher = pretend.stub(publisher_name="not-existing") assert ( - integrity_service._generate_provenance(oidc_publisher, pretend.stub()) + integrity_service._build_provenance_object(oidc_publisher, pretend.stub()) is None ) @pytest.mark.parametrize( - "publisher_name", + "publisher_factory", [ - "github", - "gitlab", + GitHubPublisherFactory, + GitLabPublisherFactory, ], ) - def test_generate_provenance_succeeds(self, publisher_name: str, monkeypatch): + def test_generate_provenance_succeeds(self, db_request, metrics, publisher_factory): integrity_service = IntegrityService( - storage=pretend.stub(), - metrics=pretend.stub(), + storage=pretend.stub(store=pretend.call_recorder(lambda *a, **kw: None)), + metrics=metrics, ) - if publisher_name == "github": - publisher = GitHubPublisher( - repository="fake-repository", - workflow="fake-workflow", - ) - else: - publisher = GitLabPublisher( - repository="fake-repository", - environment="fake-env", - ) - - monkeypatch.setattr( - services, - "_publisher_from_oidc_publisher", - lambda s: publisher, - ) + request = pretend.stub(oidc_publisher=publisher_factory.create()) + file = FileFactory.create() - provenance = integrity_service._generate_provenance( - pretend.stub(), + provenance = integrity_service.generate_provenance( + request, + file, [VALID_ATTESTATION], ) assert provenance == Provenance( attestation_bundles=[ AttestationBundle( - publisher=publisher, + publisher=services._publisher_from_oidc_publisher( + request.oidc_publisher + ), attestations=[VALID_ATTESTATION], ) ] ) - def test_persist_provenance_succeeds(self, db_request): + # We call `storage.store` twice: once for the attestation, and once + # for the provenance. + assert len(integrity_service.storage.store.calls) == 2 + + def test_persist_provenance_succeeds(self, db_request, metrics): provenance = Provenance( attestation_bundles=[ AttestationBundle( @@ -356,7 +366,7 @@ def storage_service_store(path, file_path, *_args, **_kwargs): integrity_service = IntegrityService( storage=pretend.stub(store=storage_service_store), - metrics=pretend.stub(), + metrics=metrics, ) assert ( integrity_service._persist_provenance(provenance, FileFactory.create()) diff --git a/warehouse/attestations/services.py b/warehouse/attestations/services.py index 6c48884d1da5..fbe9590aed8e 100644 --- a/warehouse/attestations/services.py +++ b/warehouse/attestations/services.py @@ -242,7 +242,7 @@ def generate_provenance( self, request: Request, file: File, attestations: list[Attestation] ) -> Provenance | None: # Generate the provenance object. - provenance = self._generate_provenance(request.oidc_publisher, attestations) + provenance = self._build_provenance_object(request.oidc_publisher, attestations) if not provenance: return None @@ -285,7 +285,7 @@ def _persist_attestations( file.attestations.append(database_attestation) - def _generate_provenance( + def _build_provenance_object( self, oidc_publisher: OIDCPublisher, attestations: list[Attestation] ) -> Provenance | None: try: From fd2a3a8d2f8464b3f6f08d9858c63103dffc074a Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 22 Aug 2024 13:03:39 -0400 Subject: [PATCH 15/34] full coverage Signed-off-by: William Woodruff --- tests/unit/attestations/test_services.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py index b130528c052e..901af2f028a7 100644 --- a/tests/unit/attestations/test_services.py +++ b/tests/unit/attestations/test_services.py @@ -293,16 +293,20 @@ def test_parse_succeed(self, metrics, monkeypatch, db_request): ) assert attestations == [VALID_ATTESTATION] - def test_generate_provenance_unsupported_publisher(self): + def test_generate_provenance_unsupported_publisher(self, metrics): integrity_service = IntegrityService( storage=pretend.stub(), metrics=pretend.stub(), ) - oidc_publisher = pretend.stub(publisher_name="not-existing") + request = pretend.stub( + oidc_publisher=pretend.stub(publisher_name="not-existing") + ) assert ( - integrity_service._build_provenance_object(oidc_publisher, pretend.stub()) + integrity_service.generate_provenance( + request, pretend.stub(), pretend.stub() + ) is None ) From 899f065fddc3c3bb98b86a94fdb9c103e55a4537 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 22 Aug 2024 13:31:28 -0400 Subject: [PATCH 16/34] test_simple: positive provenance test for /simple Signed-off-by: William Woodruff --- tests/unit/api/test_simple.py | 60 +++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/unit/api/test_simple.py b/tests/unit/api/test_simple.py index 86b517d07d18..ba8980762c21 100644 --- a/tests/unit/api/test_simple.py +++ b/tests/unit/api/test_simple.py @@ -17,6 +17,7 @@ from pyramid.httpexceptions import HTTPMovedPermanently from pyramid.testing import DummyRequest +from tests.common.db.attestation import AttestationFactory from warehouse.api import simple from warehouse.packaging.utils import API_VERSION @@ -442,6 +443,65 @@ def test_with_files_with_version_multi_digit( if renderer_override is not None: assert db_request.override_renderer == renderer_override + def test_with_files_varying_provenance(self, db_request, integrity_service): + project = ProjectFactory.create() + release = ReleaseFactory.create(project=project, version="1.0.0") + + # wheel with provenance, sdist with no provenance + wheel = FileFactory.create( + release=release, + filename=f"{project.name}-1.0.0.whl", + packagetype="bdist_wheel", + metadata_file_sha256_digest="deadbeefdeadbeefdeadbeefdeadbeef", + ) + AttestationFactory.create(file=wheel) + sdist = FileFactory.create( + release=release, + filename=f"{project.name}-1.0.0.tar.gz", + packagetype="sdist", + ) + + files = [sdist, wheel] + + urls_iter = (f"/file/{f.filename}" for f in files) + db_request.matchdict["name"] = project.normalized_name + db_request.route_url = lambda *a, **kw: next(urls_iter) + user = UserFactory.create() + je = JournalEntryFactory.create(name=project.name, submitted_by=user) + + assert simple.simple_detail(project, db_request) == { + "meta": {"_last-serial": je.id, "api-version": API_VERSION}, + "name": project.normalized_name, + "versions": ["1.0.0"], + "files": [ + { + "filename": f.filename, + "url": f"/file/{f.filename}", + "hashes": {"sha256": f.sha256_digest}, + "requires-python": f.requires_python, + "yanked": False, + "size": f.size, + "upload-time": f.upload_time.isoformat() + "Z", + "data-dist-info-metadata": ( + {"sha256": "deadbeefdeadbeefdeadbeefdeadbeef"} + if f.metadata_file_sha256_digest is not None + else False + ), + "core-metadata": ( + {"sha256": "deadbeefdeadbeefdeadbeefdeadbeef"} + if f.metadata_file_sha256_digest is not None + else False + ), + "provenance": integrity_service.get_provenance_digest(f), + } + for f in files + ], + } + + # Backstop: assert that we're testing at least provenance above + # by confirming that the wheel has one. + assert integrity_service.get_provenance_digest(wheel) is not None + def test_with_files_quarantined_omitted_from_index(self, db_request): db_request.accept = "text/html" project = ProjectFactory.create(lifecycle_status="quarantine-enter") From 7b0200d4c61181df24c1281f406410c17ca4ed53 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 22 Aug 2024 13:58:07 -0400 Subject: [PATCH 17/34] tests: minimize, increase confidence in behavior Signed-off-by: William Woodruff --- tests/conftest.py | 9 ++++ tests/unit/attestations/test_services.py | 62 ++++++++++++------------ 2 files changed, 41 insertions(+), 30 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0ef1569592c7..d81009d1669a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -112,6 +112,15 @@ def metrics(): ) +@pytest.fixture +def storage_service(tmp_path): + """ + A good-enough local file storage service. + """ + + return packaging_services.LocalArchiveFileStorage(tmp_path) + + @pytest.fixture def remote_addr(): return "1.2.3.4" diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py index 901af2f028a7..32749fefdfbe 100644 --- a/tests/unit/attestations/test_services.py +++ b/tests/unit/attestations/test_services.py @@ -10,7 +10,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import hashlib -import tempfile import pretend import pytest @@ -30,9 +29,8 @@ from sigstore.verify import Verifier from zope.interface.verify import verifyClass -from tests.common.db.attestation import AttestationFactory from tests.common.db.oidc import GitHubPublisherFactory, GitLabPublisherFactory -from tests.common.db.packaging import FileEventFactory, FileFactory +from tests.common.db.packaging import FileFactory from warehouse.attestations import ( Attestation as DatabaseAttestation, AttestationUploadError, @@ -41,7 +39,6 @@ UnsupportedPublisherError, services, ) -from warehouse.events.tags import EventTag from warehouse.metrics import IMetricsService from warehouse.packaging import File, IFileStorage @@ -317,9 +314,11 @@ def test_generate_provenance_unsupported_publisher(self, metrics): GitLabPublisherFactory, ], ) - def test_generate_provenance_succeeds(self, db_request, metrics, publisher_factory): + def test_generate_provenance_succeeds( + self, db_request, metrics, storage_service, publisher_factory + ): integrity_service = IntegrityService( - storage=pretend.stub(store=pretend.call_recorder(lambda *a, **kw: None)), + storage=storage_service, metrics=metrics, ) @@ -343,9 +342,18 @@ def test_generate_provenance_succeeds(self, db_request, metrics, publisher_facto ] ) - # We call `storage.store` twice: once for the attestation, and once - # for the provenance. - assert len(integrity_service.storage.store.calls) == 2 + # We can round-trip the provenance object out of storage. + provenance_from_store = Provenance.model_validate_json( + storage_service.get(f"{file.path}.provenance").read() + ) + provenance_from_store.attestation_bundles == [ + AttestationBundle( + publisher=services._publisher_from_oidc_publisher( + request.oidc_publisher + ), + attestations=[VALID_ATTESTATION], + ) + ] def test_persist_provenance_succeeds(self, db_request, metrics): provenance = Provenance( @@ -377,35 +385,29 @@ def storage_service_store(path, file_path, *_args, **_kwargs): is None ) - def test_get_provenance_digest(self, db_request): + def test_get_provenance_digest(self, db_request, metrics, storage_service): file = FileFactory.create() - AttestationFactory.create(file=file) - FileEventFactory.create( - source=file, - tag=EventTag.File.FileAdd, - additional={"publisher_url": "fake-publisher-url"}, + + integrity_service = IntegrityService( + storage=storage_service, + metrics=metrics, ) - with tempfile.NamedTemporaryFile() as f: - integrity_service = IntegrityService( - storage=pretend.stub(get=pretend.call_recorder(lambda p: f)), - metrics=pretend.stub(), - ) + db_request.oidc_publisher = GitHubPublisherFactory.create() - assert ( - integrity_service.get_provenance_digest(file) - == hashlib.file_digest(f, "sha256").hexdigest() - ) + integrity_service.generate_provenance(db_request, file, [VALID_ATTESTATION]) + provenance_file = f"{file.path}.provenance" + + assert ( + integrity_service.get_provenance_digest(file) + == hashlib.file_digest( + storage_service.get(provenance_file), "sha256" + ).hexdigest() + ) def test_get_provenance_digest_fails_no_attestations(self, db_request): # If the attestations are missing, there is no provenance file file = FileFactory.create() - file.attestations = [] - FileEventFactory.create( - source=file, - tag=EventTag.File.FileAdd, - additional={"publisher_url": "fake-publisher-url"}, - ) integrity_service = IntegrityService( storage=pretend.stub(), metrics=pretend.stub(), From 9767c790b207d6fa1d105c5c4a887854112cf390 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 26 Aug 2024 11:17:51 -0400 Subject: [PATCH 18/34] Update warehouse/config.py Co-authored-by: dm --- warehouse/config.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/warehouse/config.py b/warehouse/config.py index 443a413aab86..e9d71ed888c7 100644 --- a/warehouse/config.py +++ b/warehouse/config.py @@ -460,13 +460,11 @@ def configure(settings=None): "VERIFY_EMAIL_RATELIMIT_STRING", default="3 per 6 hours", ) - ( - maybe_set( - settings, - "warehouse.account.accounts_search_ratelimit_string", - "ACCOUNTS_SEARCH_RATELIMIT_STRING", - default="100 per hour", - ), + maybe_set( + settings, + "warehouse.account.accounts_search_ratelimit_string", + "ACCOUNTS_SEARCH_RATELIMIT_STRING", + default="100 per hour", ) maybe_set( settings, From d92f057604f1861bf412b85fee2f64dc0978a2ba Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 26 Aug 2024 11:48:08 -0400 Subject: [PATCH 19/34] packaging/test_utils: remove another mock Signed-off-by: William Woodruff --- tests/unit/packaging/test_utils.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/unit/packaging/test_utils.py b/tests/unit/packaging/test_utils.py index 8065c8b9ad8c..c1b55f247b39 100644 --- a/tests/unit/packaging/test_utils.py +++ b/tests/unit/packaging/test_utils.py @@ -15,6 +15,7 @@ import pretend +from tests.common.db.attestation import AttestationFactory from warehouse.attestations import IIntegrityService from warehouse.packaging.interfaces import ISimpleStorage from warehouse.packaging.utils import _simple_detail, render_simple_detail @@ -39,24 +40,23 @@ def test_simple_detail_empty_string(db_request): assert expected_content["files"][0]["requires-python"] is None -def test_simple_detail_with_provenance(db_request): +def test_simple_detail_with_provenance(db_request, integrity_service): project = ProjectFactory.create() release = ReleaseFactory.create(project=project, version="1.0") - FileFactory.create(release=release) - - hash_digest = "deadbeefdeadbeefdeadbeefdeadbeef" + file = FileFactory.create(release=release) + AttestationFactory.create(file=file) db_request.route_url = lambda *a, **kw: "the-url" db_request.find_service = pretend.call_recorder( - lambda svc, name=None, context=None: { - IIntegrityService: pretend.stub( - get_provenance_digest=pretend.call_recorder(lambda f: hash_digest), - ), - }.get(svc) + lambda svc, name=None, context=None: {IIntegrityService: integrity_service}.get( + svc + ) ) expected_content = _simple_detail(project, db_request) - assert expected_content["files"][0]["provenance"] == hash_digest + assert expected_content["files"][0][ + "provenance" + ] == integrity_service.get_provenance_digest(file) def test_render_simple_detail(db_request, monkeypatch, jinja): From f1e0a278377bde18a91b3f6e80f3a28be41ce4a2 Mon Sep 17 00:00:00 2001 From: Alexis Date: Wed, 28 Aug 2024 11:47:55 +0200 Subject: [PATCH 20/34] Remove even more mocks --- tests/conftest.py | 4 +- tests/unit/attestations/test_services.py | 126 ++++++++++------------- tests/unit/forklift/test_legacy.py | 20 +--- tests/unit/packaging/test_utils.py | 24 +---- 4 files changed, 64 insertions(+), 110 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d81009d1669a..92071834b097 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,7 +57,7 @@ from warehouse.organizations import services as organization_services from warehouse.organizations.interfaces import IOrganizationService from warehouse.packaging import services as packaging_services -from warehouse.packaging.interfaces import IProjectService +from warehouse.packaging.interfaces import IFileStorage, IProjectService from warehouse.subscriptions import services as subscription_services from warehouse.subscriptions.interfaces import IBillingService, ISubscriptionService @@ -183,6 +183,7 @@ def pyramid_services( github_oidc_service, activestate_oidc_service, integrity_service, + storage_service, macaroon_service, ): services = _Services() @@ -205,6 +206,7 @@ def pyramid_services( ) services.register_service(integrity_service, IIntegrityService, None, name="") services.register_service(macaroon_service, IMacaroonService, None, name="") + services.register_service(storage_service, IFileStorage, None, "archive") return services diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py index 32749fefdfbe..0fbb1d9651ff 100644 --- a/tests/unit/attestations/test_services.py +++ b/tests/unit/attestations/test_services.py @@ -39,8 +39,7 @@ UnsupportedPublisherError, services, ) -from warehouse.metrics import IMetricsService -from warehouse.packaging import File, IFileStorage +from warehouse.packaging import File VALID_ATTESTATION = Attestation( version=1, @@ -80,37 +79,18 @@ class TestIntegrityService: def test_interface_matches(self): assert verifyClass(IIntegrityService, IntegrityService) - def test_create_service(self): - request = pretend.stub( - find_service=pretend.call_recorder( - lambda svc, context=None, name=None: None - ), - ) - - assert IntegrityService.create_service(None, request) is not None - assert not set(request.find_service.calls) ^ { - pretend.call(IFileStorage, name="archive"), - pretend.call(IMetricsService), - } - - def test_persist_attestations(self, db_request, monkeypatch): - @pretend.call_recorder - def storage_service_store(path: str, file_path, *_args, **_kwargs): - expected = VALID_ATTESTATION.model_dump_json().encode("utf-8") - with open(file_path, "rb") as fp: - assert fp.read() == expected - - assert path.endswith(".attestation") + def test_create_service(self, db_request): + service = IntegrityService.create_service(None, db_request) + assert service is not None + assert service.storage.base.exists() + def test_persist_attestations_succeeds(self, db_request, storage_service): integrity_service = IntegrityService( - storage=pretend.stub( - store=storage_service_store, - ), + storage=storage_service, metrics=pretend.stub(), ) - file = FileFactory.create(attestations=[]) - + file = FileFactory.create() integrity_service._persist_attestations([VALID_ATTESTATION], file) attestations_db = ( @@ -122,7 +102,15 @@ def storage_service_store(path: str, file_path, *_args, **_kwargs): assert len(attestations_db) == 1 assert len(file.attestations) == 1 - def test_parse_no_publisher(self, db_request): + attestation_path = attestations_db[0].attestation_path + + assert attestation_path.endswith(".attestation") + assert ( + storage_service.get(attestation_path).read() + == VALID_ATTESTATION.model_dump_json().encode() + ) + + def test_parse_attestations_fails_no_publisher(self, db_request): integrity_service = IntegrityService( storage=pretend.stub(), metrics=pretend.stub(), @@ -135,7 +123,7 @@ def test_parse_no_publisher(self, db_request): ): integrity_service.parse_attestations(db_request, pretend.stub()) - def test_parse_unsupported_publisher(self, db_request): + def test_parse_attestations_fails_unsupported_publisher(self, db_request): integrity_service = IntegrityService( storage=pretend.stub(), metrics=pretend.stub(), @@ -147,7 +135,7 @@ def test_parse_unsupported_publisher(self, db_request): ): integrity_service.parse_attestations(db_request, pretend.stub()) - def test_parse_malformed_attestation(self, metrics, db_request): + def test_parse_attestations_fails_malformed_attestation(self, metrics, db_request): integrity_service = IntegrityService( storage=pretend.stub(), metrics=metrics, @@ -166,7 +154,7 @@ def test_parse_malformed_attestation(self, metrics, db_request): in metrics.increment.calls ) - def test_parse_multiple_attestations(self, metrics, db_request): + def test_parse_attestations_fails_multiple_attestations(self, metrics, db_request): integrity_service = IntegrityService( storage=pretend.stub(), metrics=metrics, @@ -202,7 +190,7 @@ def test_parse_multiple_attestations(self, metrics, db_request): ), ], ) - def test_parse_failed_verification( + def test_parse_attestations_fails_verification( self, metrics, monkeypatch, db_request, verify_exception, expected_message ): integrity_service = IntegrityService( @@ -231,7 +219,9 @@ def failing_verify(_self, _verifier, _policy, _dist): pretend.stub(), ) - def test_parse_wrong_predicate(self, metrics, monkeypatch, db_request): + def test_parse_attestations_fails_wrong_predicate( + self, metrics, monkeypatch, db_request + ): integrity_service = IntegrityService( storage=pretend.stub(), metrics=metrics, @@ -265,7 +255,7 @@ def test_parse_wrong_predicate(self, metrics, monkeypatch, db_request): in metrics.increment.calls ) - def test_parse_succeed(self, metrics, monkeypatch, db_request): + def test_parse_attestations_succeeds(self, metrics, monkeypatch, db_request): integrity_service = IntegrityService( storage=pretend.stub(), metrics=metrics, @@ -290,23 +280,23 @@ def test_parse_succeed(self, metrics, monkeypatch, db_request): ) assert attestations == [VALID_ATTESTATION] - def test_generate_provenance_unsupported_publisher(self, metrics): + def test_generate_provenance_fails_unsupported_publisher(self, db_request, metrics): integrity_service = IntegrityService( storage=pretend.stub(), metrics=pretend.stub(), ) - request = pretend.stub( - oidc_publisher=pretend.stub(publisher_name="not-existing") - ) + db_request.oidc_publisher = pretend.stub(publisher_name="not-existing") + file = FileFactory.create() assert ( - integrity_service.generate_provenance( - request, pretend.stub(), pretend.stub() - ) + integrity_service.generate_provenance(db_request, file, [VALID_ATTESTATION]) is None ) + # If the generate provenance fails, verify that no attestations are stored + assert not file.attestations + @pytest.mark.parametrize( "publisher_factory", [ @@ -322,40 +312,41 @@ def test_generate_provenance_succeeds( metrics=metrics, ) - request = pretend.stub(oidc_publisher=publisher_factory.create()) file = FileFactory.create() + db_request.oidc_publisher = publisher_factory.create() provenance = integrity_service.generate_provenance( - request, + db_request, file, [VALID_ATTESTATION], ) - assert provenance == Provenance( + expected_provenance = Provenance( attestation_bundles=[ AttestationBundle( publisher=services._publisher_from_oidc_publisher( - request.oidc_publisher + db_request.oidc_publisher ), attestations=[VALID_ATTESTATION], ) ] ) + assert provenance == expected_provenance + # We can round-trip the provenance object out of storage. provenance_from_store = Provenance.model_validate_json( storage_service.get(f"{file.path}.provenance").read() ) - provenance_from_store.attestation_bundles == [ - AttestationBundle( - publisher=services._publisher_from_oidc_publisher( - request.oidc_publisher - ), - attestations=[VALID_ATTESTATION], - ) - ] + assert provenance_from_store == expected_provenance == provenance + + # Generate provenance also persist attestations + assert ( + storage_service.get(file.attestations[0].attestation_path).read() + == VALID_ATTESTATION.model_dump_json().encode() + ) - def test_persist_provenance_succeeds(self, db_request, metrics): + def test_persist_provenance_succeeds(self, db_request, storage_service, metrics): provenance = Provenance( attestation_bundles=[ AttestationBundle( @@ -368,24 +359,19 @@ def test_persist_provenance_succeeds(self, db_request, metrics): ] ) - @pretend.call_recorder - def storage_service_store(path, file_path, *_args, **_kwargs): - expected = provenance.model_dump_json().encode("utf-8") - with open(file_path, "rb") as fp: - assert fp.read() == expected - - assert path.suffix == ".provenance" - integrity_service = IntegrityService( - storage=pretend.stub(store=storage_service_store), + storage=storage_service, metrics=metrics, ) + file = FileFactory.create() + assert integrity_service._persist_provenance(provenance, file) is None + assert ( - integrity_service._persist_provenance(provenance, FileFactory.create()) - is None + storage_service.get(f"{file.path}.provenance").read() + == provenance.model_dump_json().encode() ) - def test_get_provenance_digest(self, db_request, metrics, storage_service): + def test_get_provenance_digest_succeeds(self, db_request, metrics, storage_service): file = FileFactory.create() integrity_service = IntegrityService( @@ -416,7 +402,7 @@ def test_get_provenance_digest_fails_no_attestations(self, db_request): assert integrity_service.get_provenance_digest(file) is None -def test_publisher_from_oidc_publisher_github(db_request): +def test_publisher_from_oidc_publisher_succeeds_github(db_request): publisher = GitHubPublisherFactory.create() attestation_publisher = services._publisher_from_oidc_publisher(publisher) @@ -426,7 +412,7 @@ def test_publisher_from_oidc_publisher_github(db_request): assert attestation_publisher.environment == publisher.environment -def test_publisher_from_oidc_publisher_gitlab(db_request): +def test_publisher_from_oidc_publisher_succeeds_gitlab(db_request): publisher = GitLabPublisherFactory.create() attestation_publisher = services._publisher_from_oidc_publisher(publisher) @@ -435,7 +421,7 @@ def test_publisher_from_oidc_publisher_gitlab(db_request): assert attestation_publisher.environment == publisher.environment -def test_publisher_from_oidc_publisher_fails(): +def test_publisher_from_oidc_publisher_fails_unsupported(): publisher = pretend.stub(publisher_name="not-existing") with pytest.raises(UnsupportedPublisherError): diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index 56ad792ecff8..992a28c88926 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -36,7 +36,7 @@ from warehouse.accounts.utils import UserContext from warehouse.admin.flags import AdminFlag, AdminFlagValue -from warehouse.attestations import Attestation as DatabaseAttestation, IIntegrityService +from warehouse.attestations import Attestation as DatabaseAttestation from warehouse.classifiers.models import Classifier from warehouse.forklift import legacy, metadata from warehouse.macaroons import IMacaroonService, caveats, security_policy @@ -3330,7 +3330,6 @@ def test_upload_succeeds_with_valid_attestation( pyramid_config, db_request, metrics, - integrity_service, ): from warehouse.events.models import HasEvents @@ -3381,13 +3380,6 @@ def test_upload_succeeds_with_valid_attestation( } ) - storage_service = pretend.stub(store=lambda path, filepath, *, meta=None: None) - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - IIntegrityService: integrity_service, - }.get(svc) - record_event = pretend.call_recorder( lambda self, *, tag, request=None, additional: None ) @@ -3437,8 +3429,6 @@ def test_upload_fails_attestation_error( monkeypatch, pyramid_config, db_request, - metrics, - integrity_service, invalid_attestations, ): from warehouse.events.models import HasEvents @@ -3480,14 +3470,6 @@ def test_upload_fails_attestation_error( } ) - storage_service = pretend.stub(store=lambda path, filepath, meta: None) - - db_request.find_service = lambda svc, name=None, context=None: { - IFileStorage: storage_service, - IMetricsService: metrics, - IIntegrityService: integrity_service, - }.get(svc) - record_event = pretend.call_recorder( lambda self, *, tag, request=None, additional: None ) diff --git a/tests/unit/packaging/test_utils.py b/tests/unit/packaging/test_utils.py index c1b55f247b39..92edf774d92d 100644 --- a/tests/unit/packaging/test_utils.py +++ b/tests/unit/packaging/test_utils.py @@ -29,12 +29,6 @@ def test_simple_detail_empty_string(db_request): FileFactory.create(release=release) db_request.route_url = lambda *a, **kw: "the-url" - db_request.find_service = lambda svc, name=None, context=None: { - IIntegrityService: pretend.stub( - get_provenance_digest=pretend.call_recorder(lambda f: None), - ), - }.get(svc) - expected_content = _simple_detail(project, db_request) assert expected_content["files"][0]["requires-python"] is None @@ -47,11 +41,6 @@ def test_simple_detail_with_provenance(db_request, integrity_service): AttestationFactory.create(file=file) db_request.route_url = lambda *a, **kw: "the-url" - db_request.find_service = pretend.call_recorder( - lambda svc, name=None, context=None: {IIntegrityService: integrity_service}.get( - svc - ) - ) expected_content = _simple_detail(project, db_request) assert expected_content["files"][0][ @@ -76,11 +65,6 @@ def test_render_simple_detail(db_request, monkeypatch, jinja): monkeypatch.setattr(hashlib, "blake2b", fakeblake2b) db_request.route_url = lambda *a, **kw: "the-url" - db_request.find_service = lambda svc, name=None, context=None: { - IIntegrityService: pretend.stub( - get_provenance_digest=pretend.call_recorder(lambda f: None), - ), - }.get(svc) template = jinja.get_template("templates/api/simple/detail.html") expected_content = template.render( @@ -100,7 +84,9 @@ def test_render_simple_detail(db_request, monkeypatch, jinja): ) -def test_render_simple_detail_with_store(db_request, monkeypatch, jinja): +def test_render_simple_detail_with_store( + db_request, monkeypatch, jinja, integrity_service +): project = ProjectFactory.create() storage_service = pretend.stub( @@ -111,9 +97,7 @@ def test_render_simple_detail_with_store(db_request, monkeypatch, jinja): db_request.find_service = pretend.call_recorder( lambda svc, name=None, context=None: { ISimpleStorage: storage_service, - IIntegrityService: pretend.stub( - get_provenance_digest=pretend.call_recorder(lambda f: None), - ), + IIntegrityService: integrity_service, }.get(svc) ) From 5e5f91ebf3e2b79446f215b9633c98d4fe049bde Mon Sep 17 00:00:00 2001 From: dm Date: Wed, 28 Aug 2024 17:07:37 +0200 Subject: [PATCH 21/34] Update tests/conftest.py Co-authored-by: William Woodruff --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 92071834b097..4d93a23f97b1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -206,7 +206,7 @@ def pyramid_services( ) services.register_service(integrity_service, IIntegrityService, None, name="") services.register_service(macaroon_service, IMacaroonService, None, name="") - services.register_service(storage_service, IFileStorage, None, "archive") + services.register_service(storage_service, IFileStorage, None, name="archive") return services From 226204147a7c743b447a8b6014f955266b26824f Mon Sep 17 00:00:00 2001 From: Alexis Date: Wed, 28 Aug 2024 17:10:20 +0200 Subject: [PATCH 22/34] Update test_create_service --- tests/unit/attestations/test_services.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py index 0fbb1d9651ff..6654feeab832 100644 --- a/tests/unit/attestations/test_services.py +++ b/tests/unit/attestations/test_services.py @@ -81,8 +81,7 @@ def test_interface_matches(self): def test_create_service(self, db_request): service = IntegrityService.create_service(None, db_request) - assert service is not None - assert service.storage.base.exists() + assert isinstance(service, IntegrityService) def test_persist_attestations_succeeds(self, db_request, storage_service): integrity_service = IntegrityService( From 17e9d45f9c66f4781cf5dc29c6fdc854f9719437 Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Thu, 29 Aug 2024 21:04:55 +0000 Subject: [PATCH 23/34] Add a functional test --- ...leproject-3.0.0.tar.gz.publish.attestation | 1 + tests/functional/api/test_simple.py | 86 ++++++++++++++++++- 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation diff --git a/tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation b/tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation new file mode 100644 index 000000000000..9f7a4eb28f53 --- /dev/null +++ b/tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation @@ -0,0 +1 @@ +{"version":1,"verification_material":{"certificate":"MIIC6zCCAnGgAwIBAgIUFgmhIYx8gvBGePCTacG/4kbBdRwwCgYIKoZIzj0EAwMwNzEVMBMGA1UE\nChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjQwODI5\nMTcwOTM5WhcNMjQwODI5MTcxOTM5WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEtGrMPml4\nOtsRJ3Z6qRahs0kHCZxP4n9fvrJE957WVxgAGg4k6a1PbRJY9nT9wKpRrZmKV++AgA9ndhdruXXa\nAKOCAZAwggGMMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAdBgNVHQ4EFgQU\nosNvhYEuTPfgyU/dZfu93lFGRNswHwYDVR0jBBgwFoAU39Ppz1YkEZb5qNjpKFWixi4YZD8wQAYD\nVR0RAQH/BDYwNIEyOTE5NDM2MTU4MjM2LWNvbXB1dGVAZGV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3Vu\ndC5jb20wKQYKKwYBBAGDvzABAQQbaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tMCsGCisGAQQB\ng78wAQgEHQwbaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tMIGKBgorBgEEAdZ5AgQCBHwEegB4\nAHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGRnx0/aQAABAMARzBFAiBogvcK\nHIIR9FcX1vQgDhGtAl0XQoMRiEB3OdUWO94P1gIhANdJlyISdtvVrHes25dWKTLepy+IzQmzfQU/\nS7cxWHmOMAoGCCqGSM49BAMDA2gAMGUCMGe2xTiuenbjdt1d2e4IaCiwRh2G4KAtyujRESSSUbpu\nGme/o9ouiApeONBv2CvvGAIxAOEkAGFO3aALE3IPNosxqaz9MbqJOdmYhB1Cz1D7xbFc/m243VxJ\nWxaC/uOFEpyiYQ==\n","transparency_entries":[{"logIndex":"125970014","logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="},"kindVersion":{"kind":"dsse","version":"0.0.1"},"integratedTime":"1724951379","inclusionPromise":{"signedEntryTimestamp":"MEUCIQCHrKFTeXNY432S0bUSBS69S8d5JnNcDXa41q6OEvxEwgIgaZstc5Jpm0IgwFC7RDTXYEAKk+3aG/MkRkaPdJdyn8U="},"inclusionProof":{"logIndex":"4065752","rootHash":"7jVDF3UNUZVEU85ffETQ3WKfXhOoMi4cgytJM250HTk=","treeSize":"4065754","hashes":["NwJgWJoxjearbnEIT9bnWXpzo0LGNrR1cpWId0g66rE=","kLjpW3Eh7pQJNOvyntghzF57tcfqk2IzX7cqiBDgGf8=","FW8y9LQ1i3q+MnbeGJipKGl4VfX1zRBOD7TmhbEw7uI=","mKcbGJDJ/+buNbXy9Eyv94nVoAyUauuIlN3cJg3qSBY=","5VytqqAHhfRkRWMrY43UXWCnRBb7JwElMlKpY5JueBc=","mZJnD39LTKdis2wUTz1OOMx3r7HwgJh9rnb2VwiPzts=","MXZOQFJFiOjREF0xwMOCXu29HwTchjTtl/BeFoI51wY=","g8zCkHnLwO3LojK7g5AnqE8ezSNRnCSz9nCL5GD3a8A=","RrZsD/RSxNoujlvq/MsCEvLSkKZfv0jmQM9Kp7qbJec=","QxmVWsbTp4cClxuAkuT51UH2EY7peHMVGKq7+b+cGwQ=","Q2LAtNzOUh+3PfwfMyNxYb06fTQmF3VeTT6Fr6Upvfc=","ftwAu6v62WFDoDmcZ1JKfrRPrvuiIw5v3BvRsgQj7N8="],"checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n4065754\n7jVDF3UNUZVEU85ffETQ3WKfXhOoMi4cgytJM250HTk=\n\n— rekor.sigstore.dev wNI9ajBGAiEAhMomhZHOTNB5CVPO98CMXCv01ZlIF+C+CgzraAB01r8CIQCEuXbv6aqguUpB/ig5eXRIbarvxLXkg3nX48DzambktQ==\n"}},"canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiZHNzZSIsInNwZWMiOnsiZW52ZWxvcGVIYXNoIjp7ImFsZ29yaXRobSI6InNoYTI1NiIsInZhbHVlIjoiOWRiNGJjMzE3MTgyZWI3NzljNDIyY2Q0NGI2ZDdlYTk5ZWM1M2Q3M2JiY2ZjZWVmZTIyNWVlYjQ3NTQyMjc4OCJ9LCJwYXlsb2FkSGFzaCI6eyJhbGdvcml0aG0iOiJzaGEyNTYiLCJ2YWx1ZSI6IjlkYjY0MjlhOTkzZGFiYTI4NzAwODk2ZTY2MzNjNzkxYWE0MDM3ODQ4NjJiYzY2MDBkM2E4NjYwMGQzYjA1NjMifSwic2lnbmF0dXJlcyI6W3sic2lnbmF0dXJlIjoiTUVVQ0lCaGlOL25NR0w3aHpZQk9QQjlUTGtuaEdTZEtuQ0Q0ekI3TDV5ZXc0QmJ3QWlFQXJzOHl6MCtCT2NnSEtzS0JzTXVOeVlhREdaRTBVV0JuMEdwNVpGMzUvU2M9IiwidmVyaWZpZXIiOiJMUzB0TFMxQ1JVZEpUaUJEUlZKVVNVWkpRMEZVUlMwdExTMHRDazFKU1VNMmVrTkRRVzVIWjBGM1NVSkJaMGxWUm1kdGFFbFplRGhuZGtKSFpWQkRWR0ZqUnk4MGEySkNaRkozZDBObldVbExiMXBKZW1vd1JVRjNUWGNLVG5wRlZrMUNUVWRCTVZWRlEyaE5UV015Ykc1ak0xSjJZMjFWZFZwSFZqSk5ValIzU0VGWlJGWlJVVVJGZUZaNllWZGtlbVJIT1hsYVV6RndZbTVTYkFwamJURnNXa2RzYUdSSFZYZElhR05PVFdwUmQwOUVTVFZOVkdOM1QxUk5OVmRvWTA1TmFsRjNUMFJKTlUxVVkzaFBWRTAxVjJwQlFVMUdhM2RGZDFsSUNrdHZXa2w2YWpCRFFWRlpTVXR2V2tsNmFqQkVRVkZqUkZGblFVVjBSM0pOVUcxc05FOTBjMUpLTTFvMmNWSmhhSE13YTBoRFduaFFORzQ1Wm5aeVNrVUtPVFUzVjFaNFowRkhaelJyTm1FeFVHSlNTbGs1YmxRNWQwdHdVbkphYlV0V0t5dEJaMEU1Ym1Sb1pISjFXRmhoUVV0UFEwRmFRWGRuWjBkTlRVRTBSd3BCTVZWa1JIZEZRaTkzVVVWQmQwbElaMFJCVkVKblRsWklVMVZGUkVSQlMwSm5aM0pDWjBWR1FsRmpSRUY2UVdSQ1owNVdTRkUwUlVablVWVnZjMDUyQ21oWlJYVlVVR1puZVZVdlpGcG1kVGt6YkVaSFVrNXpkMGgzV1VSV1VqQnFRa0puZDBadlFWVXpPVkJ3ZWpGWmEwVmFZalZ4VG1wd1MwWlhhWGhwTkZrS1drUTRkMUZCV1VSV1VqQlNRVkZJTDBKRVdYZE9TVVY1VDFSRk5VNUVUVEpOVkZVMFRXcE5Na3hYVG5aaVdFSXhaRWRXUVZwSFZqSmFWM2gyWTBkV2VRcE1iV1I2V2xoS01tRlhUbXhaVjA1cVlqTldkV1JETldwaU1qQjNTMUZaUzB0M1dVSkNRVWRFZG5wQlFrRlJVV0poU0ZJd1kwaE5Oa3g1T1doWk1rNTJDbVJYTlRCamVUVnVZakk1Ym1KSFZYVlpNamwwVFVOelIwTnBjMGRCVVZGQ1p6YzRkMEZSWjBWSVVYZGlZVWhTTUdOSVRUWk1lVGxvV1RKT2RtUlhOVEFLWTNrMWJtSXlPVzVpUjFWMVdUSTVkRTFKUjB0Q1oyOXlRbWRGUlVGa1dqVkJaMUZEUWtoM1JXVm5RalJCU0ZsQk0xUXdkMkZ6WWtoRlZFcHFSMUkwWXdwdFYyTXpRWEZLUzFoeWFtVlFTek12YURSd2VXZERPSEEzYnpSQlFVRkhVbTU0TUM5aFVVRkJRa0ZOUVZKNlFrWkJhVUp2WjNaalMwaEpTVkk1Um1OWUNqRjJVV2RFYUVkMFFXd3dXRkZ2VFZKcFJVSXpUMlJWVjA4NU5GQXhaMGxvUVU1a1NteDVTVk5rZEhaV2NraGxjekkxWkZkTFZFeGxjSGtyU1hwUmJYb0tabEZWTDFNM1kzaFhTRzFQVFVGdlIwTkRjVWRUVFRRNVFrRk5SRUV5WjBGTlIxVkRUVWRsTW5oVWFYVmxibUpxWkhReFpESmxORWxoUTJsM1VtZ3lSd28wUzBGMGVYVnFVa1ZUVTFOVlluQjFSMjFsTDI4NWIzVnBRWEJsVDA1Q2RqSkRkblpIUVVsNFFVOUZhMEZIUms4ellVRk1SVE5KVUU1dmMzaHhZWG81Q2sxaWNVcFBaRzFaYUVJeFEzb3hSRGQ0WWtaakwyMHlORE5XZUVwWGVHRkRMM1ZQUmtWd2VXbFpVVDA5Q2kwdExTMHRSVTVFSUVORlVsUkpSa2xEUVZSRkxTMHRMUzBLIn1dfX0="}]},"envelope":{"statement":"eyJfdHlwZSI6Imh0dHBzOi8vaW4tdG90by5pby9TdGF0ZW1lbnQvdjEiLCJzdWJqZWN0IjpbeyJu\nYW1lIjoic2FtcGxlcHJvamVjdC0zLjAuMC50YXIuZ3oiLCJkaWdlc3QiOnsic2hhMjU2IjoiMTE3\nZWQ4OGU1ZGIwNzNiYjkyOTY5YTc1NDU3NDVmZDk3N2VlODViNzAxOTcwNmRkMjU2YTY0MDU4Zjcw\nOTYzZCJ9fV0sInByZWRpY2F0ZVR5cGUiOiJodHRwczovL2RvY3MucHlwaS5vcmcvYXR0ZXN0YXRp\nb25zL3B1Ymxpc2gvdjEiLCJwcmVkaWNhdGUiOm51bGx9\n","signature":"MEUCIBhiN/nMGL7hzYBOPB9TLknhGSdKnCD4zB7L5yew4BbwAiEArs8yz0+BOcgHKsKBsMuNyYaD\nGZE0UWBn0Gp5ZF35/Sc=\n"}} \ No newline at end of file diff --git a/tests/functional/api/test_simple.py b/tests/functional/api/test_simple.py index 636530b0233b..f81dc880f588 100644 --- a/tests/functional/api/test_simple.py +++ b/tests/functional/api/test_simple.py @@ -10,9 +10,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +import base64 +import hashlib from http import HTTPStatus -from ...common.db.packaging import ProjectFactory, ReleaseFactory +from ...common.db.packaging import ( + ProjectFactory, + ReleaseFactory, +) +from ...common.db.accounts import EmailFactory, UserFactory +from ...common.db.packaging import RoleFactory +from ...common.db.macaroons import MacaroonFactory +from ...common.db.oidc import GitHubPublisherFactory +from warehouse.macaroons import caveats + +import pymacaroons def test_simple_api_html(webtest): @@ -31,3 +43,75 @@ def test_simple_api_detail(webtest): assert resp.content_type == "text/html" assert "X-PyPI-Last-Serial" in resp.headers assert f"Links for {project.normalized_name}" in resp.text + + +def test_simple_attestations_from_upload(webtest): + user = UserFactory.create( + password="$argon2id$v=19$m=1024,t=6,p=6$EiLE2Nsbo9S6N+acs/beGw$ccyZDCZstr1/+Y/1s3BVZHOJaqfBroT0JCieHug281c" # 'password' + ) + EmailFactory.create(user=user, verified=True) + project = ProjectFactory.create(name="sampleproject") + RoleFactory.create(user=user, project=project, role_name="Owner") + publisher = GitHubPublisherFactory.create(projects=[project]) + + # Construct the macaroon. This needs to be based on a Trusted Publisher, which is + # required to upload attestations + dm = MacaroonFactory.create( + oidc_publisher_id=publisher.id, + caveats=[ + caveats.OIDCPublisher(oidc_publisher_id=str(publisher.id)), + caveats.ProjectID(project_ids=[str(p.id) for p in publisher.projects]), + ], + additional={"oidc": {"ref": "someref", "sha": "somesha"}}, + ) + + m = pymacaroons.Macaroon( + location="localhost", + identifier=str(dm.id), + key=dm.key, + version=pymacaroons.MACAROON_V2, + ) + for caveat in dm.caveats: + m.add_first_party_caveat(caveats.serialize(caveat)) + serialized_macaroon = f"pypi-{m.serialize()}" + + credentials = base64.b64encode( + f"__token__:{serialized_macaroon}".encode("utf-8") + ).decode("utf-8") + + with open("./tests/functional/_fixtures/sampleproject-3.0.0.tar.gz", "rb") as f: + content = f.read() + + with open( + "./tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation", + "r", + ) as f: + attestation = f.read() + + with open( + "./tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation", + "rb", + ) as f: + digest = hashlib.file_digest(f, "sha256") + + expected_hash = digest.hexdigest() + + webtest.post( + "/legacy/?:action=file_upload", + headers={"Authorization": f"Basic {credentials}"}, + params={ + "name": "sampleproject", + "sha256_digest": "117ed88e5db073bb92969a7545745fd977ee85b7019706dd256a64058f70963d", + "filetype": "sdist", + "metadata_version": "2.1", + "version": "3.0.0", + "attestations": f"[{attestation}]", + }, + upload_files=[("content", "sampleproject-3.0.0.tar.gz", content)], + status=HTTPStatus.OK, + ) + + response = webtest.get("/simple/sampleproject/", status=HTTPStatus.OK) + link = response.html.find("a", text="sampleproject-3.0.0.tar.gz") + assert "data-provenance" in link.attrs + assert link.get("data-provenance") == expected_hash From b8c5423a195e6edd43573ac82bdedbfeeafa5bda Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Thu, 29 Aug 2024 21:11:34 +0000 Subject: [PATCH 24/34] Linting --- tests/functional/api/test_simple.py | 30 +++++++++++++++-------------- warehouse/forklift/legacy.py | 9 ++++++++- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/tests/functional/api/test_simple.py b/tests/functional/api/test_simple.py index f81dc880f588..493dd75a4a44 100644 --- a/tests/functional/api/test_simple.py +++ b/tests/functional/api/test_simple.py @@ -12,19 +12,17 @@ import base64 import hashlib + from http import HTTPStatus -from ...common.db.packaging import ( - ProjectFactory, - ReleaseFactory, -) +import pymacaroons + +from warehouse.macaroons import caveats + from ...common.db.accounts import EmailFactory, UserFactory -from ...common.db.packaging import RoleFactory from ...common.db.macaroons import MacaroonFactory from ...common.db.oidc import GitHubPublisherFactory -from warehouse.macaroons import caveats - -import pymacaroons +from ...common.db.packaging import ProjectFactory, ReleaseFactory, RoleFactory def test_simple_api_html(webtest): @@ -47,7 +45,10 @@ def test_simple_api_detail(webtest): def test_simple_attestations_from_upload(webtest): user = UserFactory.create( - password="$argon2id$v=19$m=1024,t=6,p=6$EiLE2Nsbo9S6N+acs/beGw$ccyZDCZstr1/+Y/1s3BVZHOJaqfBroT0JCieHug281c" # 'password' + password=( # 'password' + "$argon2id$v=19$m=1024,t=6,p=6$EiLE2Nsbo9S6N+acs/beGw$ccyZDCZstr1/+Y/1s3BVZ" + "HOJaqfBroT0JCieHug281c" + ) ) EmailFactory.create(user=user, verified=True) project = ProjectFactory.create(name="sampleproject") @@ -75,16 +76,15 @@ def test_simple_attestations_from_upload(webtest): m.add_first_party_caveat(caveats.serialize(caveat)) serialized_macaroon = f"pypi-{m.serialize()}" - credentials = base64.b64encode( - f"__token__:{serialized_macaroon}".encode("utf-8") - ).decode("utf-8") + credentials = base64.b64encode(f"__token__:{serialized_macaroon}".encode()).decode( + "utf-8" + ) with open("./tests/functional/_fixtures/sampleproject-3.0.0.tar.gz", "rb") as f: content = f.read() with open( "./tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation", - "r", ) as f: attestation = f.read() @@ -101,7 +101,9 @@ def test_simple_attestations_from_upload(webtest): headers={"Authorization": f"Basic {credentials}"}, params={ "name": "sampleproject", - "sha256_digest": "117ed88e5db073bb92969a7545745fd977ee85b7019706dd256a64058f70963d", + "sha256_digest": ( + "117ed88e5db073bb92969a7545745fd977ee85b7019706dd256a64058f70963d" + ), "filetype": "sdist", "metadata_version": "2.1", "version": "3.0.0", diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index b506142c88ca..daeaa84adf3f 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -29,7 +29,13 @@ import wtforms import wtforms.validators -from pypi_attestations import Attestation, Distribution +from pydantic import TypeAdapter, ValidationError +from pypi_attestations import ( + Attestation, + AttestationType, + Distribution, + VerificationError, +) from pyramid.httpexceptions import ( HTTPBadRequest, HTTPException, @@ -41,6 +47,7 @@ ) from pyramid.request import Request from pyramid.view import view_config +from sigstore.verify import Verifier from sqlalchemy import and_, exists, func, orm from sqlalchemy.exc import MultipleResultsFound, NoResultFound From 866b0a7a62099d83250656585a76a769f861d24c Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Thu, 29 Aug 2024 21:22:57 +0000 Subject: [PATCH 25/34] Fixup migration --- .../versions/4037669366ca_recreate_attestations_table.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/warehouse/migrations/versions/4037669366ca_recreate_attestations_table.py b/warehouse/migrations/versions/4037669366ca_recreate_attestations_table.py index 315b4e4acc95..959bb011ef32 100644 --- a/warehouse/migrations/versions/4037669366ca_recreate_attestations_table.py +++ b/warehouse/migrations/versions/4037669366ca_recreate_attestations_table.py @@ -13,7 +13,7 @@ recreate attestations table Revision ID: 4037669366ca -Revises: 7ca0f1f5e7b3 +Revises: 606abd3b8e7f Create Date: 2024-08-21 20:33:53.489489 """ @@ -23,7 +23,7 @@ from sqlalchemy.dialects import postgresql revision = "4037669366ca" -down_revision = "7ca0f1f5e7b3" +down_revision = "606abd3b8e7f" def upgrade(): From f4fc53ce04fa2ab52e1742ac909c08390fc27692 Mon Sep 17 00:00:00 2001 From: Alexis Date: Fri, 30 Aug 2024 15:36:29 +0200 Subject: [PATCH 26/34] Fix test error --- tests/functional/api/test_simple.py | 19 ++++++++++--------- warehouse/attestations/services.py | 5 +---- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/tests/functional/api/test_simple.py b/tests/functional/api/test_simple.py index 493dd75a4a44..bdcea6043c38 100644 --- a/tests/functional/api/test_simple.py +++ b/tests/functional/api/test_simple.py @@ -14,6 +14,7 @@ import hashlib from http import HTTPStatus +from pathlib import Path import pymacaroons @@ -24,6 +25,9 @@ from ...common.db.oidc import GitHubPublisherFactory from ...common.db.packaging import ProjectFactory, ReleaseFactory, RoleFactory +_HERE = Path(__file__).parent +_ASSETS = _HERE.parent / "_fixtures" + def test_simple_api_html(webtest): resp = webtest.get("/simple/", status=HTTPStatus.OK) @@ -80,21 +84,18 @@ def test_simple_attestations_from_upload(webtest): "utf-8" ) - with open("./tests/functional/_fixtures/sampleproject-3.0.0.tar.gz", "rb") as f: + with open(_ASSETS / "sampleproject-3.0.0.tar.gz", "rb") as f: content = f.read() with open( - "./tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation", + _ASSETS / "sampleproject-3.0.0.tar.gz.publish.attestation", ) as f: attestation = f.read() - with open( - "./tests/functional/_fixtures/sampleproject-3.0.0.tar.gz.publish.attestation", - "rb", - ) as f: - digest = hashlib.file_digest(f, "sha256") - - expected_hash = digest.hexdigest() + expected_hash = hashlib.sha256( + # Filename:len(attestations) + b"sampleproject-3.0.0.tar.gz:1" + ).hexdigest() webtest.post( "/legacy/?:action=file_upload", diff --git a/warehouse/attestations/services.py b/warehouse/attestations/services.py index fbe9590aed8e..8fa3509761a7 100644 --- a/warehouse/attestations/services.py +++ b/warehouse/attestations/services.py @@ -139,13 +139,12 @@ def generate_provenance( provenance = Provenance(attestation_bundles=[attestation_bundle]) for attestation in attestations: - db_attestation = DatabaseAttestation( + DatabaseAttestation( file=file, attestation_file_blake2_digest=hashlib.blake2b( attestation.model_dump_json().encode("utf-8") ).hexdigest(), ) - file.attestations.append(db_attestation) return provenance @@ -283,8 +282,6 @@ def _persist_attestations( meta=None, ) - file.attestations.append(database_attestation) - def _build_provenance_object( self, oidc_publisher: OIDCPublisher, attestations: list[Attestation] ) -> Provenance | None: From e72b746d034b32f19c20eeefb11d5edf606d1011 Mon Sep 17 00:00:00 2001 From: Alexis Date: Fri, 30 Aug 2024 17:46:50 +0200 Subject: [PATCH 27/34] Revert change --- warehouse/attestations/services.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/warehouse/attestations/services.py b/warehouse/attestations/services.py index 8fa3509761a7..52f9f69ab5af 100644 --- a/warehouse/attestations/services.py +++ b/warehouse/attestations/services.py @@ -139,13 +139,15 @@ def generate_provenance( provenance = Provenance(attestation_bundles=[attestation_bundle]) for attestation in attestations: - DatabaseAttestation( + database_attestation = DatabaseAttestation( file=file, attestation_file_blake2_digest=hashlib.blake2b( attestation.model_dump_json().encode("utf-8") ).hexdigest(), ) + file.attestations.append(database_attestation) + return provenance def get_provenance_digest(self, file: File) -> str | None: @@ -282,6 +284,8 @@ def _persist_attestations( meta=None, ) + file.attestations.append(database_attestation) + def _build_provenance_object( self, oidc_publisher: OIDCPublisher, attestations: list[Attestation] ) -> Provenance | None: From 52931a14b9c27e5dc5d450a51500bfc646295e28 Mon Sep 17 00:00:00 2001 From: dm Date: Fri, 30 Aug 2024 18:02:48 +0200 Subject: [PATCH 28/34] Apply suggestions from code review Rename key Co-authored-by: Dustin Ingram --- warehouse/packaging/utils.py | 2 +- warehouse/templates/api/simple/detail.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 03c33a94a844..60455ad826a1 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -100,7 +100,7 @@ def _simple_detail(project, request): if file.metadata_file_sha256_digest else False ), - "provenance": integrity_service.get_provenance_digest(file), + "provenance_digest": integrity_service.get_provenance_digest(file), } for file in files ], diff --git a/warehouse/templates/api/simple/detail.html b/warehouse/templates/api/simple/detail.html index 05e0221a5612..d3ff9dc27723 100644 --- a/warehouse/templates/api/simple/detail.html +++ b/warehouse/templates/api/simple/detail.html @@ -20,7 +20,7 @@

Links for {{ name }}

{% for file in files -%} - {{ file.filename }}
+ {{ file.filename }}
{% endfor -%} From c9a774f8731887b8df47b7c128016aaecfc12d64 Mon Sep 17 00:00:00 2001 From: Alexis Date: Fri, 30 Aug 2024 18:18:46 +0200 Subject: [PATCH 29/34] Revert "Apply suggestions from code review " This reverts commit 52931a14b9c27e5dc5d450a51500bfc646295e28. --- warehouse/packaging/utils.py | 2 +- warehouse/templates/api/simple/detail.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 60455ad826a1..03c33a94a844 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -100,7 +100,7 @@ def _simple_detail(project, request): if file.metadata_file_sha256_digest else False ), - "provenance_digest": integrity_service.get_provenance_digest(file), + "provenance": integrity_service.get_provenance_digest(file), } for file in files ], diff --git a/warehouse/templates/api/simple/detail.html b/warehouse/templates/api/simple/detail.html index d3ff9dc27723..05e0221a5612 100644 --- a/warehouse/templates/api/simple/detail.html +++ b/warehouse/templates/api/simple/detail.html @@ -20,7 +20,7 @@

Links for {{ name }}

{% for file in files -%} - {{ file.filename }}
+ {{ file.filename }}
{% endfor -%} From f7e277e02c1f44874765d79c4ae32c7a6f684098 Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Fri, 30 Aug 2024 18:28:49 +0000 Subject: [PATCH 30/34] Give the IntegrityService access to the session --- tests/conftest.py | 2 +- tests/unit/attestations/test_services.py | 15 ++++++++++++++- warehouse/attestations/services.py | 17 ++++++++--------- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5aae840297fb..ebee90a5f93a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -555,7 +555,7 @@ def activestate_oidc_service(db_session): @pytest.fixture def integrity_service(db_session): - return attestations_services.NullIntegrityService() + return attestations_services.NullIntegrityService(db_session) @pytest.fixture diff --git a/tests/unit/attestations/test_services.py b/tests/unit/attestations/test_services.py index 6654feeab832..3750a669adb0 100644 --- a/tests/unit/attestations/test_services.py +++ b/tests/unit/attestations/test_services.py @@ -66,7 +66,7 @@ def test_get_provenance_digest(self, db_request): ) file = FileFactory.create() - service = services.NullIntegrityService() + service = services.NullIntegrityService(session=db_request.db) provenance = service.generate_provenance(db_request, file, [VALID_ATTESTATION]) assert isinstance(provenance, Provenance) @@ -87,6 +87,7 @@ def test_persist_attestations_succeeds(self, db_request, storage_service): integrity_service = IntegrityService( storage=storage_service, metrics=pretend.stub(), + session=db_request.db, ) file = FileFactory.create() @@ -113,6 +114,7 @@ def test_parse_attestations_fails_no_publisher(self, db_request): integrity_service = IntegrityService( storage=pretend.stub(), metrics=pretend.stub(), + session=db_request.db, ) db_request.oidc_publisher = None @@ -126,6 +128,7 @@ def test_parse_attestations_fails_unsupported_publisher(self, db_request): integrity_service = IntegrityService( storage=pretend.stub(), metrics=pretend.stub(), + session=db_request.db, ) db_request.oidc_publisher = pretend.stub(publisher_name="not-existing") with pytest.raises( @@ -138,6 +141,7 @@ def test_parse_attestations_fails_malformed_attestation(self, metrics, db_reques integrity_service = IntegrityService( storage=pretend.stub(), metrics=metrics, + session=db_request.db, ) db_request.oidc_publisher = pretend.stub(publisher_name="GitHub") @@ -157,6 +161,7 @@ def test_parse_attestations_fails_multiple_attestations(self, metrics, db_reques integrity_service = IntegrityService( storage=pretend.stub(), metrics=metrics, + session=db_request.db, ) db_request.oidc_publisher = pretend.stub(publisher_name="GitHub") @@ -195,6 +200,7 @@ def test_parse_attestations_fails_verification( integrity_service = IntegrityService( storage=pretend.stub(), metrics=metrics, + session=db_request.db, ) db_request.oidc_publisher = pretend.stub( @@ -224,6 +230,7 @@ def test_parse_attestations_fails_wrong_predicate( integrity_service = IntegrityService( storage=pretend.stub(), metrics=metrics, + session=db_request.db, ) db_request.oidc_publisher = pretend.stub( publisher_name="GitHub", @@ -258,6 +265,7 @@ def test_parse_attestations_succeeds(self, metrics, monkeypatch, db_request): integrity_service = IntegrityService( storage=pretend.stub(), metrics=metrics, + session=db_request.db, ) db_request.oidc_publisher = pretend.stub( publisher_name="GitHub", @@ -283,6 +291,7 @@ def test_generate_provenance_fails_unsupported_publisher(self, db_request, metri integrity_service = IntegrityService( storage=pretend.stub(), metrics=pretend.stub(), + session=db_request.db, ) db_request.oidc_publisher = pretend.stub(publisher_name="not-existing") @@ -309,6 +318,7 @@ def test_generate_provenance_succeeds( integrity_service = IntegrityService( storage=storage_service, metrics=metrics, + session=db_request.db, ) file = FileFactory.create() @@ -361,6 +371,7 @@ def test_persist_provenance_succeeds(self, db_request, storage_service, metrics) integrity_service = IntegrityService( storage=storage_service, metrics=metrics, + session=db_request.db, ) file = FileFactory.create() assert integrity_service._persist_provenance(provenance, file) is None @@ -376,6 +387,7 @@ def test_get_provenance_digest_succeeds(self, db_request, metrics, storage_servi integrity_service = IntegrityService( storage=storage_service, metrics=metrics, + session=db_request.db, ) db_request.oidc_publisher = GitHubPublisherFactory.create() @@ -396,6 +408,7 @@ def test_get_provenance_digest_fails_no_attestations(self, db_request): integrity_service = IntegrityService( storage=pretend.stub(), metrics=pretend.stub(), + session=db_request.db, ) assert integrity_service.get_provenance_digest(file) is None diff --git a/warehouse/attestations/services.py b/warehouse/attestations/services.py index 52f9f69ab5af..1c2098363052 100644 --- a/warehouse/attestations/services.py +++ b/warehouse/attestations/services.py @@ -111,17 +111,18 @@ def _extract_attestations_from_request(request: Request) -> list[Attestation]: @implementer(IIntegrityService) class NullIntegrityService: - def __init__(self): + def __init__(self, session): warnings.warn( "NullIntegrityService is intended only for use in development, " "you should not use it in production due to the lack of actual " "attestation verification.", InsecureIntegrityServiceWarning, ) + self.db = session @classmethod - def create_service(cls, _context, _request): - return cls() + def create_service(cls, _context, request): + return cls(session=request.db) def parse_attestations( self, request: Request, _distribution: Distribution @@ -165,19 +166,17 @@ def get_provenance_digest(self, file: File) -> str | None: @implementer(IIntegrityService) class IntegrityService: - def __init__( - self, - storage: IFileStorage, - metrics: IMetricsService, - ): + def __init__(self, storage: IFileStorage, metrics: IMetricsService, session): self.storage: IFileStorage = storage self.metrics: IMetricsService = metrics + self.db = session @classmethod - def create_service(cls, _context, request: Request): + def create_service(cls, _context, request): return cls( storage=request.find_service(IFileStorage, name="archive"), metrics=request.find_service(IMetricsService), + session=request.db, ) def parse_attestations( From fa1bd58a4ac11d38b87df25048fb8caf6c5735fa Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Fri, 30 Aug 2024 18:30:31 +0000 Subject: [PATCH 31/34] Add the Attestation object to the session --- warehouse/attestations/services.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/warehouse/attestations/services.py b/warehouse/attestations/services.py index 1c2098363052..31dc8c1cbf60 100644 --- a/warehouse/attestations/services.py +++ b/warehouse/attestations/services.py @@ -140,15 +140,15 @@ def generate_provenance( provenance = Provenance(attestation_bundles=[attestation_bundle]) for attestation in attestations: - database_attestation = DatabaseAttestation( - file=file, - attestation_file_blake2_digest=hashlib.blake2b( - attestation.model_dump_json().encode("utf-8") - ).hexdigest(), + self.db.add( + DatabaseAttestation( + file=file, + attestation_file_blake2_digest=hashlib.blake2b( + attestation.model_dump_json().encode("utf-8") + ).hexdigest(), + ) ) - file.attestations.append(database_attestation) - return provenance def get_provenance_digest(self, file: File) -> str | None: @@ -241,6 +241,7 @@ def parse_attestations( def generate_provenance( self, request: Request, file: File, attestations: list[Attestation] ) -> Provenance | None: + # Generate the provenance object. provenance = self._build_provenance_object(request.oidc_publisher, attestations) @@ -276,6 +277,7 @@ def _persist_attestations( database_attestation = DatabaseAttestation( file=file, attestation_file_blake2_digest=attestation_digest ) + self.db.add(database_attestation) self.storage.store( database_attestation.attestation_path, @@ -283,8 +285,6 @@ def _persist_attestations( meta=None, ) - file.attestations.append(database_attestation) - def _build_provenance_object( self, oidc_publisher: OIDCPublisher, attestations: list[Attestation] ) -> Provenance | None: From 4ca217b806b4b9bd7bb427b363e60661e37bd68a Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Fri, 30 Aug 2024 18:30:40 +0000 Subject: [PATCH 32/34] Update the functional test with more assertions --- tests/functional/api/test_simple.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/functional/api/test_simple.py b/tests/functional/api/test_simple.py index bdcea6043c38..4e0f8510c7cc 100644 --- a/tests/functional/api/test_simple.py +++ b/tests/functional/api/test_simple.py @@ -92,11 +92,6 @@ def test_simple_attestations_from_upload(webtest): ) as f: attestation = f.read() - expected_hash = hashlib.sha256( - # Filename:len(attestations) - b"sampleproject-3.0.0.tar.gz:1" - ).hexdigest() - webtest.post( "/legacy/?:action=file_upload", headers={"Authorization": f"Basic {credentials}"}, @@ -114,7 +109,15 @@ def test_simple_attestations_from_upload(webtest): status=HTTPStatus.OK, ) + assert len(project.releases) == 1 + assert project.releases[0].files.count() == 1 + assert len(project.releases[0].files[0].attestations) == 1 + response = webtest.get("/simple/sampleproject/", status=HTTPStatus.OK) link = response.html.find("a", text="sampleproject-3.0.0.tar.gz") + assert "data-provenance" in link.attrs - assert link.get("data-provenance") == expected_hash + assert ( + link.get("data-provenance") + == hashlib.sha256(b"sampleproject-3.0.0.tar.gz:1").hexdigest() + ) From 9497124cb831f7a4d52b24651c1fd5d048413b71 Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Fri, 30 Aug 2024 19:14:25 +0000 Subject: [PATCH 33/34] Remove vestigial helper --- warehouse/forklift/legacy.py | 91 +----------------------------------- 1 file changed, 1 insertion(+), 90 deletions(-) diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index daeaa84adf3f..233a0b50177f 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -29,13 +29,7 @@ import wtforms import wtforms.validators -from pydantic import TypeAdapter, ValidationError -from pypi_attestations import ( - Attestation, - AttestationType, - Distribution, - VerificationError, -) +from pypi_attestations import Attestation, Distribution from pyramid.httpexceptions import ( HTTPBadRequest, HTTPException, @@ -47,7 +41,6 @@ ) from pyramid.request import Request from pyramid.view import view_config -from sigstore.verify import Verifier from sqlalchemy import and_, exists, func, orm from sqlalchemy.exc import MultipleResultsFound, NoResultFound @@ -377,88 +370,6 @@ def _is_duplicate_file(db_session, filename, hashes): return None -def _process_attestations(request, distribution: Distribution): - """ - Process any attestations included in a file upload request - - Attestations, if present, will be parsed and verified against the uploaded - artifact. Attestations are only allowed when uploading via a Trusted - Publisher, because a Trusted Publisher provides the identity that will be - used to verify the attestations. - Currently, only GitHub Actions Trusted Publishers are supported, and - attestations are discarded after verification. - """ - - metrics = request.find_service(IMetricsService, context=None) - - publisher = request.oidc_publisher - if not publisher or not publisher.publisher_name == "GitHub": - raise _exc_with_message( - HTTPBadRequest, - "Attestations are currently only supported when using Trusted " - "Publishing with GitHub Actions.", - ) - try: - attestations = TypeAdapter(list[Attestation]).validate_json( - request.POST["attestations"] - ) - except ValidationError as e: - # Log invalid (malformed) attestation upload - metrics.increment("warehouse.upload.attestations.malformed") - raise _exc_with_message( - HTTPBadRequest, - f"Error while decoding the included attestation: {e}", - ) - - if len(attestations) > 1: - metrics.increment("warehouse.upload.attestations.failed_multiple_attestations") - raise _exc_with_message( - HTTPBadRequest, - "Only a single attestation per-file is supported at the moment.", - ) - - verification_policy = publisher.publisher_verification_policy(request.oidc_claims) - for attestation_model in attestations: - try: - # For now, attestations are not stored, just verified - predicate_type, _ = attestation_model.verify( - Verifier.production(), - verification_policy, - distribution, - ) - except VerificationError as e: - # Log invalid (failed verification) attestation upload - metrics.increment("warehouse.upload.attestations.failed_verify") - raise _exc_with_message( - HTTPBadRequest, - f"Could not verify the uploaded artifact using the included " - f"attestation: {e}", - ) - except Exception as e: - with sentry_sdk.push_scope() as scope: - scope.fingerprint = [e] - sentry_sdk.capture_message( - f"Unexpected error while verifying attestation: {e}" - ) - - raise _exc_with_message( - HTTPBadRequest, - f"Unknown error while trying to verify included attestations: {e}", - ) - - if predicate_type != AttestationType.PYPI_PUBLISH_V1: - metrics.increment( - "warehouse.upload.attestations.failed_unsupported_predicate_type" - ) - raise _exc_with_message( - HTTPBadRequest, - f"Attestation with unsupported predicate type: {predicate_type}", - ) - - # Log successful attestation upload - metrics.increment("warehouse.upload.attestations.ok") - - def _sort_releases(request: Request, project: Project): releases = ( request.db.query(Release) From ac783a838ad2b8bdf7e4894b9669688724f31ad6 Mon Sep 17 00:00:00 2001 From: Alexis Date: Tue, 3 Sep 2024 15:54:47 +0200 Subject: [PATCH 34/34] Update functional test --- tests/functional/api/test_simple.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/tests/functional/api/test_simple.py b/tests/functional/api/test_simple.py index 4e0f8510c7cc..377563c7dd02 100644 --- a/tests/functional/api/test_simple.py +++ b/tests/functional/api/test_simple.py @@ -10,7 +10,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import base64 import hashlib from http import HTTPStatus @@ -80,10 +79,6 @@ def test_simple_attestations_from_upload(webtest): m.add_first_party_caveat(caveats.serialize(caveat)) serialized_macaroon = f"pypi-{m.serialize()}" - credentials = base64.b64encode(f"__token__:{serialized_macaroon}".encode()).decode( - "utf-8" - ) - with open(_ASSETS / "sampleproject-3.0.0.tar.gz", "rb") as f: content = f.read() @@ -92,9 +87,9 @@ def test_simple_attestations_from_upload(webtest): ) as f: attestation = f.read() + webtest.set_authorization(("Basic", ("__token__", serialized_macaroon))) webtest.post( "/legacy/?:action=file_upload", - headers={"Authorization": f"Basic {credentials}"}, params={ "name": "sampleproject", "sha256_digest": ( @@ -113,11 +108,24 @@ def test_simple_attestations_from_upload(webtest): assert project.releases[0].files.count() == 1 assert len(project.releases[0].files[0].attestations) == 1 + expected_provenance = hashlib.sha256(b"sampleproject-3.0.0.tar.gz:1").hexdigest() + expected_filename = "sampleproject-3.0.0.tar.gz" + response = webtest.get("/simple/sampleproject/", status=HTTPStatus.OK) - link = response.html.find("a", text="sampleproject-3.0.0.tar.gz") + link = response.html.find("a", text=expected_filename) assert "data-provenance" in link.attrs - assert ( - link.get("data-provenance") - == hashlib.sha256(b"sampleproject-3.0.0.tar.gz:1").hexdigest() + assert link.get("data-provenance") == expected_provenance + + response = webtest.get( + "/simple/sampleproject/", + headers={"Accept": "application/vnd.pypi.simple.v1+json"}, + status=HTTPStatus.OK, ) + + assert response.content_type == "application/vnd.pypi.simple.v1+json" + + json_content = response.json + assert len(json_content["files"]) == 1 + assert json_content["files"][0]["filename"] == expected_filename + assert json_content["files"][0]["provenance"] == expected_provenance