Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reapply "Store attestations for PEP740 (#16302)" (#16545) #16546

Merged
merged 39 commits into from
Sep 3, 2024
Merged
Show file tree
Hide file tree
Changes from 31 commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
d8f5134
Reapply "Store attestations for PEP740 (#16302)" (#16545)
woodruffw Aug 21, 2024
20a4e4e
migrations: re-roll migration history
woodruffw Aug 21, 2024
a058c41
config: register .attestations for inclusion
woodruffw Aug 21, 2024
19b7739
attestations: request the appropriate IFileStorage service
woodruffw Aug 21, 2024
19b4ef8
Merge remote-tracking branch 'upstream/main' into ww/pep740-persisten…
woodruffw Aug 21, 2024
f4a40ad
conftest: add archive_files.path to get_app_config
woodruffw Aug 21, 2024
b151b82
test, warehouse: remove problematic mocks
woodruffw Aug 21, 2024
64ed3e5
test_services: rename test class
woodruffw Aug 21, 2024
e19be6c
Try to clean a bit the mess with the migrations.
DarkaMaul Aug 22, 2024
98b833d
begin refactoring IntegrityService
woodruffw Aug 22, 2024
4549711
Revert "Try to clean a bit the mess with the migrations."
woodruffw Aug 22, 2024
77d08a2
tests, warehouse: more error tests, remove more stubs
woodruffw Aug 22, 2024
014f649
test_services: fix match
woodruffw Aug 22, 2024
46f32e2
remove more implicit file service deps
woodruffw Aug 22, 2024
7e7ea8c
continue to burn down coverage
woodruffw Aug 22, 2024
fd2a3a8
full coverage
woodruffw Aug 22, 2024
899f065
test_simple: positive provenance test for /simple
woodruffw Aug 22, 2024
7b0200d
tests: minimize, increase confidence in behavior
woodruffw Aug 22, 2024
bbb33f5
Merge remote-tracking branch 'upstream/main' into ww/pep740-persisten…
woodruffw Aug 22, 2024
9767c79
Update warehouse/config.py
woodruffw Aug 26, 2024
d92f057
packaging/test_utils: remove another mock
woodruffw Aug 26, 2024
f1e0a27
Remove even more mocks
DarkaMaul Aug 28, 2024
5e5f91e
Update tests/conftest.py
DarkaMaul Aug 28, 2024
2262041
Update test_create_service
DarkaMaul Aug 28, 2024
0467f5c
Merge branch 'main' into ww/pep740-persistence-take-2
DarkaMaul Aug 28, 2024
26e48d2
Merge branch 'main' into ww/pep740-persistence-take-2
di Aug 29, 2024
17e9d45
Add a functional test
di Aug 29, 2024
b8c5423
Linting
di Aug 29, 2024
866b0a7
Fixup migration
di Aug 29, 2024
f4fc53c
Fix test error
DarkaMaul Aug 30, 2024
e72b746
Revert change
DarkaMaul Aug 30, 2024
52931a1
Apply suggestions from code review
DarkaMaul Aug 30, 2024
c9a774f
Revert "Apply suggestions from code review "
DarkaMaul Aug 30, 2024
f7e277e
Give the IntegrityService access to the session
di Aug 30, 2024
fa1bd58
Add the Attestation object to the session
di Aug 30, 2024
4ca217b
Update the functional test with more assertions
di Aug 30, 2024
9497124
Remove vestigial helper
di Aug 30, 2024
ac783a8
Update functional test
DarkaMaul Sep 3, 2024
caa181f
Merge branch 'main' into ww/pep740-persistence-take-2
di Sep 3, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions dev/environment
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ BREACHED_EMAILS=warehouse.accounts.NullEmailBreachedService
BREACHED_PASSWORDS=warehouse.accounts.NullPasswordBreachedService

OIDC_BACKEND=warehouse.oidc.services.NullOIDCPublisherService
ATTESTATIONS_BACKEND=warehouse.attestations.services.NullIntegrityService

METRICS_BACKEND=warehouse.metrics.DataDogMetrics host=notdatadog

Expand Down
4 changes: 2 additions & 2 deletions requirements/main.in
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,8 @@ redis>=2.8.0,<6.0.0
rfc3986
sentry-sdk
setuptools
sigstore~=3.0.0
pypi-attestations==0.0.9
sigstore~=3.2.0
pypi-attestations==0.0.11
sqlalchemy[asyncio]>=2.0,<3.0
stdlib-list
stripe
Expand Down
12 changes: 6 additions & 6 deletions requirements/main.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1776,9 +1776,9 @@ pyparsing==3.1.4 \
--hash=sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c \
--hash=sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032
# via linehaul
pypi-attestations==0.0.9 \
--hash=sha256:3bfc07f64a8db0d6e2646720e70df7c7cb01a2936056c764a2cc3268969332f2 \
--hash=sha256:4b38cce5d221c8145cac255bfafe650ec0028d924d2b3572394df8ba8f07a609
pypi-attestations==0.0.11 \
--hash=sha256:b730e6b23874d94da0f3817b1f9dd3ecb6a80d685f62a18ad96e5b0396149ded \
--hash=sha256:e74329074f049568591e300373e12fcd46a35e21723110856546e33bf2949efa
# via -r requirements/main.in
pyqrcode==1.2.1 \
--hash=sha256:1b2812775fa6ff5c527977c4cd2ccb07051ca7d0bc0aecf937a43864abe5eff6 \
Expand Down Expand Up @@ -2091,9 +2091,9 @@ sentry-sdk==2.13.0 \
--hash=sha256:6beede8fc2ab4043da7f69d95534e320944690680dd9a963178a49de71d726c6 \
--hash=sha256:8d4a576f7a98eb2fdb40e13106e41f330e5c79d72a68be1316e7852cf4995260
# via -r requirements/main.in
sigstore==3.0.0 \
--hash=sha256:6cc7dc92607c2fd481aada0f3c79e710e4c6086e3beab50b07daa9a50a79d109 \
--hash=sha256:a6a9538a648e112a0c3d8092d3f73a351c7598164764f1e73a6b5ba406a3a0bd
sigstore==3.2.0 \
--hash=sha256:25c8a871a3a6adf959c0cde598ea8bef8794f1a29277d067111eb4ded4ba7f65 \
--hash=sha256:d18508f34febb7775065855e92557fa1c2c16580df88f8e8903b9514438bad44
# via
# -r requirements/main.in
# pypi-attestations
Expand Down
28 changes: 28 additions & 0 deletions tests/common/db/attestation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import hashlib

import factory

from warehouse.attestations.models import Attestation

from .base import WarehouseFactory


class AttestationFactory(WarehouseFactory):
class Meta:
model = Attestation

file = factory.SubFactory("tests.common.db.packaging.FileFactory")
attestation_file_blake2_digest = factory.LazyAttribute(
lambda o: hashlib.blake2b(o.file.filename.encode("utf8")).hexdigest()
)
8 changes: 8 additions & 0 deletions tests/common/db/packaging.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
from warehouse.utils import readme

from .accounts import UserFactory
from .attestation import AttestationFactory
from .base import WarehouseFactory
from .observations import ObserverFactory

Expand Down Expand Up @@ -140,6 +141,13 @@ class Meta:
)
)

# Empty attestations by default.
attestations = factory.RelatedFactoryList(
AttestationFactory,
factory_related_name="file",
size=0,
)


class FileEventFactory(WarehouseFactory):
class Meta:
Expand Down
25 changes: 22 additions & 3 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@
from warehouse.accounts import services as account_services
from warehouse.accounts.interfaces import ITokenService, IUserService
from warehouse.admin.flags import AdminFlag, AdminFlagValue
from warehouse.attestations import services as attestations_services
from warehouse.attestations.interfaces import IIntegrityService
from warehouse.email import services as email_services
from warehouse.email.interfaces import IEmailSender
from warehouse.helpdesk import services as helpdesk_services
Expand All @@ -57,7 +59,7 @@
from warehouse.organizations import services as organization_services
from warehouse.organizations.interfaces import IOrganizationService
from warehouse.packaging import services as packaging_services
from warehouse.packaging.interfaces import IProjectService
from warehouse.packaging.interfaces import IFileStorage, IProjectService
from warehouse.subscriptions import services as subscription_services
from warehouse.subscriptions.interfaces import IBillingService, ISubscriptionService

Expand Down Expand Up @@ -112,6 +114,15 @@ def metrics():
)


@pytest.fixture
def storage_service(tmp_path):
"""
A good-enough local file storage service.
"""

return packaging_services.LocalArchiveFileStorage(tmp_path)
woodruffw marked this conversation as resolved.
Show resolved Hide resolved


@pytest.fixture
def remote_addr():
return "1.2.3.4"
Expand Down Expand Up @@ -173,6 +184,8 @@ def pyramid_services(
project_service,
github_oidc_service,
activestate_oidc_service,
integrity_service,
storage_service,
macaroon_service,
helpdesk_service,
):
Expand All @@ -194,7 +207,9 @@ def pyramid_services(
services.register_service(
activestate_oidc_service, IOIDCPublisherService, None, name="activestate"
)
services.register_service(integrity_service, IIntegrityService, None, name="")
services.register_service(macaroon_service, IMacaroonService, None, name="")
services.register_service(storage_service, IFileStorage, None, name="archive")
services.register_service(helpdesk_service, IHelpDeskService, None)

return services
Expand Down Expand Up @@ -324,6 +339,7 @@ def get_app_config(database, nondefaults=None):
"docs.backend": "warehouse.packaging.services.LocalDocsStorage",
"sponsorlogos.backend": "warehouse.admin.services.LocalSponsorLogoStorage",
"billing.backend": "warehouse.subscriptions.services.MockStripeBillingService",
"attestations.backend": "warehouse.attestations.services.NullIntegrityService",
"billing.api_base": "http://stripe:12111",
"billing.api_version": "2020-08-27",
"mail.backend": "warehouse.email.services.SMTPEmailSender",
Expand Down Expand Up @@ -387,13 +403,11 @@ def get_db_session_for_app_config(app_config):

@pytest.fixture(scope="session")
def app_config(database):

return get_app_config(database)


@pytest.fixture(scope="session")
def app_config_dbsession_from_env(database):

nondefaults = {
"warehouse.db_create_session": lambda r: r.environ.get("warehouse.db_session")
}
Expand Down Expand Up @@ -539,6 +553,11 @@ def activestate_oidc_service(db_session):
)


@pytest.fixture
def integrity_service(db_session):
return attestations_services.NullIntegrityService()


@pytest.fixture
def macaroon_service(db_session):
return macaroon_services.DatabaseMacaroonService(db_session)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"version":1,"verification_material":{"certificate":"MIIC6zCCAnGgAwIBAgIUFgmhIYx8gvBGePCTacG/4kbBdRwwCgYIKoZIzj0EAwMwNzEVMBMGA1UE\nChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjQwODI5\nMTcwOTM5WhcNMjQwODI5MTcxOTM5WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEtGrMPml4\nOtsRJ3Z6qRahs0kHCZxP4n9fvrJE957WVxgAGg4k6a1PbRJY9nT9wKpRrZmKV++AgA9ndhdruXXa\nAKOCAZAwggGMMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAdBgNVHQ4EFgQU\nosNvhYEuTPfgyU/dZfu93lFGRNswHwYDVR0jBBgwFoAU39Ppz1YkEZb5qNjpKFWixi4YZD8wQAYD\nVR0RAQH/BDYwNIEyOTE5NDM2MTU4MjM2LWNvbXB1dGVAZGV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3Vu\ndC5jb20wKQYKKwYBBAGDvzABAQQbaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tMCsGCisGAQQB\ng78wAQgEHQwbaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tMIGKBgorBgEEAdZ5AgQCBHwEegB4\nAHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGRnx0/aQAABAMARzBFAiBogvcK\nHIIR9FcX1vQgDhGtAl0XQoMRiEB3OdUWO94P1gIhANdJlyISdtvVrHes25dWKTLepy+IzQmzfQU/\nS7cxWHmOMAoGCCqGSM49BAMDA2gAMGUCMGe2xTiuenbjdt1d2e4IaCiwRh2G4KAtyujRESSSUbpu\nGme/o9ouiApeONBv2CvvGAIxAOEkAGFO3aALE3IPNosxqaz9MbqJOdmYhB1Cz1D7xbFc/m243VxJ\nWxaC/uOFEpyiYQ==\n","transparency_entries":[{"logIndex":"125970014","logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="},"kindVersion":{"kind":"dsse","version":"0.0.1"},"integratedTime":"1724951379","inclusionPromise":{"signedEntryTimestamp":"MEUCIQCHrKFTeXNY432S0bUSBS69S8d5JnNcDXa41q6OEvxEwgIgaZstc5Jpm0IgwFC7RDTXYEAKk+3aG/MkRkaPdJdyn8U="},"inclusionProof":{"logIndex":"4065752","rootHash":"7jVDF3UNUZVEU85ffETQ3WKfXhOoMi4cgytJM250HTk=","treeSize":"4065754","hashes":["NwJgWJoxjearbnEIT9bnWXpzo0LGNrR1cpWId0g66rE=","kLjpW3Eh7pQJNOvyntghzF57tcfqk2IzX7cqiBDgGf8=","FW8y9LQ1i3q+MnbeGJipKGl4VfX1zRBOD7TmhbEw7uI=","mKcbGJDJ/+buNbXy9Eyv94nVoAyUauuIlN3cJg3qSBY=","5VytqqAHhfRkRWMrY43UXWCnRBb7JwElMlKpY5JueBc=","mZJnD39LTKdis2wUTz1OOMx3r7HwgJh9rnb2VwiPzts=","MXZOQFJFiOjREF0xwMOCXu29HwTchjTtl/BeFoI51wY=","g8zCkHnLwO3LojK7g5AnqE8ezSNRnCSz9nCL5GD3a8A=","RrZsD/RSxNoujlvq/MsCEvLSkKZfv0jmQM9Kp7qbJec=","QxmVWsbTp4cClxuAkuT51UH2EY7peHMVGKq7+b+cGwQ=","Q2LAtNzOUh+3PfwfMyNxYb06fTQmF3VeTT6Fr6Upvfc=","ftwAu6v62WFDoDmcZ1JKfrRPrvuiIw5v3BvRsgQj7N8="],"checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n4065754\n7jVDF3UNUZVEU85ffETQ3WKfXhOoMi4cgytJM250HTk=\n\n— rekor.sigstore.dev wNI9ajBGAiEAhMomhZHOTNB5CVPO98CMXCv01ZlIF+C+CgzraAB01r8CIQCEuXbv6aqguUpB/ig5eXRIbarvxLXkg3nX48DzambktQ==\n"}},"canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiZHNzZSIsInNwZWMiOnsiZW52ZWxvcGVIYXNoIjp7ImFsZ29yaXRobSI6InNoYTI1NiIsInZhbHVlIjoiOWRiNGJjMzE3MTgyZWI3NzljNDIyY2Q0NGI2ZDdlYTk5ZWM1M2Q3M2JiY2ZjZWVmZTIyNWVlYjQ3NTQyMjc4OCJ9LCJwYXlsb2FkSGFzaCI6eyJhbGdvcml0aG0iOiJzaGEyNTYiLCJ2YWx1ZSI6IjlkYjY0MjlhOTkzZGFiYTI4NzAwODk2ZTY2MzNjNzkxYWE0MDM3ODQ4NjJiYzY2MDBkM2E4NjYwMGQzYjA1NjMifSwic2lnbmF0dXJlcyI6W3sic2lnbmF0dXJlIjoiTUVVQ0lCaGlOL25NR0w3aHpZQk9QQjlUTGtuaEdTZEtuQ0Q0ekI3TDV5ZXc0QmJ3QWlFQXJzOHl6MCtCT2NnSEtzS0JzTXVOeVlhREdaRTBVV0JuMEdwNVpGMzUvU2M9IiwidmVyaWZpZXIiOiJMUzB0TFMxQ1JVZEpUaUJEUlZKVVNVWkpRMEZVUlMwdExTMHRDazFKU1VNMmVrTkRRVzVIWjBGM1NVSkJaMGxWUm1kdGFFbFplRGhuZGtKSFpWQkRWR0ZqUnk4MGEySkNaRkozZDBObldVbExiMXBKZW1vd1JVRjNUWGNLVG5wRlZrMUNUVWRCTVZWRlEyaE5UV015Ykc1ak0xSjJZMjFWZFZwSFZqSk5ValIzU0VGWlJGWlJVVVJGZUZaNllWZGtlbVJIT1hsYVV6RndZbTVTYkFwamJURnNXa2RzYUdSSFZYZElhR05PVFdwUmQwOUVTVFZOVkdOM1QxUk5OVmRvWTA1TmFsRjNUMFJKTlUxVVkzaFBWRTAxVjJwQlFVMUdhM2RGZDFsSUNrdHZXa2w2YWpCRFFWRlpTVXR2V2tsNmFqQkVRVkZqUkZGblFVVjBSM0pOVUcxc05FOTBjMUpLTTFvMmNWSmhhSE13YTBoRFduaFFORzQ1Wm5aeVNrVUtPVFUzVjFaNFowRkhaelJyTm1FeFVHSlNTbGs1YmxRNWQwdHdVbkphYlV0V0t5dEJaMEU1Ym1Sb1pISjFXRmhoUVV0UFEwRmFRWGRuWjBkTlRVRTBSd3BCTVZWa1JIZEZRaTkzVVVWQmQwbElaMFJCVkVKblRsWklVMVZGUkVSQlMwSm5aM0pDWjBWR1FsRmpSRUY2UVdSQ1owNVdTRkUwUlVablVWVnZjMDUyQ21oWlJYVlVVR1puZVZVdlpGcG1kVGt6YkVaSFVrNXpkMGgzV1VSV1VqQnFRa0puZDBadlFWVXpPVkJ3ZWpGWmEwVmFZalZ4VG1wd1MwWlhhWGhwTkZrS1drUTRkMUZCV1VSV1VqQlNRVkZJTDBKRVdYZE9TVVY1VDFSRk5VNUVUVEpOVkZVMFRXcE5Na3hYVG5aaVdFSXhaRWRXUVZwSFZqSmFWM2gyWTBkV2VRcE1iV1I2V2xoS01tRlhUbXhaVjA1cVlqTldkV1JETldwaU1qQjNTMUZaUzB0M1dVSkNRVWRFZG5wQlFrRlJVV0poU0ZJd1kwaE5Oa3g1T1doWk1rNTJDbVJYTlRCamVUVnVZakk1Ym1KSFZYVlpNamwwVFVOelIwTnBjMGRCVVZGQ1p6YzRkMEZSWjBWSVVYZGlZVWhTTUdOSVRUWk1lVGxvV1RKT2RtUlhOVEFLWTNrMWJtSXlPVzVpUjFWMVdUSTVkRTFKUjB0Q1oyOXlRbWRGUlVGa1dqVkJaMUZEUWtoM1JXVm5RalJCU0ZsQk0xUXdkMkZ6WWtoRlZFcHFSMUkwWXdwdFYyTXpRWEZLUzFoeWFtVlFTek12YURSd2VXZERPSEEzYnpSQlFVRkhVbTU0TUM5aFVVRkJRa0ZOUVZKNlFrWkJhVUp2WjNaalMwaEpTVkk1Um1OWUNqRjJVV2RFYUVkMFFXd3dXRkZ2VFZKcFJVSXpUMlJWVjA4NU5GQXhaMGxvUVU1a1NteDVTVk5rZEhaV2NraGxjekkxWkZkTFZFeGxjSGtyU1hwUmJYb0tabEZWTDFNM1kzaFhTRzFQVFVGdlIwTkRjVWRUVFRRNVFrRk5SRUV5WjBGTlIxVkRUVWRsTW5oVWFYVmxibUpxWkhReFpESmxORWxoUTJsM1VtZ3lSd28wUzBGMGVYVnFVa1ZUVTFOVlluQjFSMjFsTDI4NWIzVnBRWEJsVDA1Q2RqSkRkblpIUVVsNFFVOUZhMEZIUms4ellVRk1SVE5KVUU1dmMzaHhZWG81Q2sxaWNVcFBaRzFaYUVJeFEzb3hSRGQ0WWtaakwyMHlORE5XZUVwWGVHRkRMM1ZQUmtWd2VXbFpVVDA5Q2kwdExTMHRSVTVFSUVORlVsUkpSa2xEUVZSRkxTMHRMUzBLIn1dfX0="}]},"envelope":{"statement":"eyJfdHlwZSI6Imh0dHBzOi8vaW4tdG90by5pby9TdGF0ZW1lbnQvdjEiLCJzdWJqZWN0IjpbeyJu\nYW1lIjoic2FtcGxlcHJvamVjdC0zLjAuMC50YXIuZ3oiLCJkaWdlc3QiOnsic2hhMjU2IjoiMTE3\nZWQ4OGU1ZGIwNzNiYjkyOTY5YTc1NDU3NDVmZDk3N2VlODViNzAxOTcwNmRkMjU2YTY0MDU4Zjcw\nOTYzZCJ9fV0sInByZWRpY2F0ZVR5cGUiOiJodHRwczovL2RvY3MucHlwaS5vcmcvYXR0ZXN0YXRp\nb25zL3B1Ymxpc2gvdjEiLCJwcmVkaWNhdGUiOm51bGx9\n","signature":"MEUCIBhiN/nMGL7hzYBOPB9TLknhGSdKnCD4zB7L5yew4BbwAiEArs8yz0+BOcgHKsKBsMuNyYaD\nGZE0UWBn0Gp5ZF35/Sc=\n"}}
89 changes: 88 additions & 1 deletion tests/functional/api/test_simple.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import base64
import hashlib

from http import HTTPStatus
from pathlib import Path

import pymacaroons

from warehouse.macaroons import caveats

from ...common.db.accounts import EmailFactory, UserFactory
from ...common.db.macaroons import MacaroonFactory
from ...common.db.oidc import GitHubPublisherFactory
from ...common.db.packaging import ProjectFactory, ReleaseFactory, RoleFactory

from ...common.db.packaging import ProjectFactory, ReleaseFactory
_HERE = Path(__file__).parent
_ASSETS = _HERE.parent / "_fixtures"


def test_simple_api_html(webtest):
Expand All @@ -31,3 +45,76 @@ def test_simple_api_detail(webtest):
assert resp.content_type == "text/html"
assert "X-PyPI-Last-Serial" in resp.headers
assert f"Links for {project.normalized_name}" in resp.text


def test_simple_attestations_from_upload(webtest):
user = UserFactory.create(
password=( # 'password'
"$argon2id$v=19$m=1024,t=6,p=6$EiLE2Nsbo9S6N+acs/beGw$ccyZDCZstr1/+Y/1s3BVZ"
"HOJaqfBroT0JCieHug281c"
)
)
EmailFactory.create(user=user, verified=True)
project = ProjectFactory.create(name="sampleproject")
RoleFactory.create(user=user, project=project, role_name="Owner")
publisher = GitHubPublisherFactory.create(projects=[project])

# Construct the macaroon. This needs to be based on a Trusted Publisher, which is
# required to upload attestations
dm = MacaroonFactory.create(
oidc_publisher_id=publisher.id,
caveats=[
caveats.OIDCPublisher(oidc_publisher_id=str(publisher.id)),
caveats.ProjectID(project_ids=[str(p.id) for p in publisher.projects]),
],
additional={"oidc": {"ref": "someref", "sha": "somesha"}},
)

m = pymacaroons.Macaroon(
location="localhost",
identifier=str(dm.id),
key=dm.key,
version=pymacaroons.MACAROON_V2,
)
for caveat in dm.caveats:
m.add_first_party_caveat(caveats.serialize(caveat))
serialized_macaroon = f"pypi-{m.serialize()}"

credentials = base64.b64encode(f"__token__:{serialized_macaroon}".encode()).decode(
"utf-8"
)

with open(_ASSETS / "sampleproject-3.0.0.tar.gz", "rb") as f:
content = f.read()

with open(
_ASSETS / "sampleproject-3.0.0.tar.gz.publish.attestation",
) as f:
attestation = f.read()

expected_hash = hashlib.sha256(
# Filename:len(attestations)
b"sampleproject-3.0.0.tar.gz:1"
).hexdigest()

webtest.post(
"/legacy/?:action=file_upload",
headers={"Authorization": f"Basic {credentials}"},
params={
"name": "sampleproject",
"sha256_digest": (
"117ed88e5db073bb92969a7545745fd977ee85b7019706dd256a64058f70963d"
),
"filetype": "sdist",
"metadata_version": "2.1",
"version": "3.0.0",
"attestations": f"[{attestation}]",
},
upload_files=[("content", "sampleproject-3.0.0.tar.gz", content)],
status=HTTPStatus.OK,
)

response = webtest.get("/simple/sampleproject/", status=HTTPStatus.OK)
link = response.html.find("a", text="sampleproject-3.0.0.tar.gz")
assert "data-provenance" in link.attrs
assert link.get("data-provenance") == expected_hash
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test fails here because the hash we get here is

hashlib.sha256(
    b"sampleproject-3.0.0.tar.gz:2"   # notice the 2 here
).hexdigest()

However, the len(attestations) should be 1.

The problem lies in generate_provenance of NullService that creates the DatabaseAttestation and adds it to the file.attestations (like in _persist_attestations )

Because the relationship is noted as back_populates, sqlalchemy automatically adds the newly created DatabaseAttestation to the file instance.

The statement below is thus redundant and creates add a second time the attestation to the file instance.

file.attestations.append(database_attestation)

However, if we look at test_persist_attestations_succeeds, the test passes with the following statements :

        assert len(attestations_db) == 1
        assert len(file.attestations) == 1

I've observed the state in a debugger using sqlalchemy tools and they looked appropriate.

from sqlachemy import inspect
state = inspect(database_attestation)

Any idea here of the reason for this behavior ?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The issue is that the Attestation objects weren't actually being added to the session, the test was emitting a warning like:

tests/unit/attestations/test_services.py::TestIntegrityService::test_generate_provenance_succeeds[GitHubPublisherFactory]
  /opt/warehouse/src/warehouse/attestations/models.py:53: SAWarning: Object of type <Attestation> not in session, add operation along 'File.attestations' will not proceed (This warning originated from the Session 'autoflush' process, which was invoked automatically in response to a user-initiated operation.)
    f"{Path(self.file.path).name}.attestation",

I think f7e277e and fa1bd58 should resolve this.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for sorting this out - I missed the warning in the logs.

63 changes: 63 additions & 0 deletions tests/unit/api/test_simple.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from pyramid.httpexceptions import HTTPMovedPermanently
from pyramid.testing import DummyRequest

from tests.common.db.attestation import AttestationFactory
from warehouse.api import simple
from warehouse.packaging.utils import API_VERSION

Expand Down Expand Up @@ -286,6 +287,7 @@ def test_with_files_no_serial(self, db_request, content_type, renderer_override)
"upload-time": f.upload_time.isoformat() + "Z",
"data-dist-info-metadata": False,
"core-metadata": False,
"provenance": None,
}
for f in files
],
Expand Down Expand Up @@ -334,6 +336,7 @@ def test_with_files_with_serial(self, db_request, content_type, renderer_overrid
"upload-time": f.upload_time.isoformat() + "Z",
"data-dist-info-metadata": False,
"core-metadata": False,
"provenance": None,
}
for f in files
],
Expand Down Expand Up @@ -427,6 +430,7 @@ def test_with_files_with_version_multi_digit(
if f.metadata_file_sha256_digest is not None
else False
),
"provenance": None,
}
for f in files
],
Expand All @@ -439,6 +443,65 @@ def test_with_files_with_version_multi_digit(
if renderer_override is not None:
assert db_request.override_renderer == renderer_override

def test_with_files_varying_provenance(self, db_request, integrity_service):
project = ProjectFactory.create()
release = ReleaseFactory.create(project=project, version="1.0.0")

# wheel with provenance, sdist with no provenance
wheel = FileFactory.create(
release=release,
filename=f"{project.name}-1.0.0.whl",
packagetype="bdist_wheel",
metadata_file_sha256_digest="deadbeefdeadbeefdeadbeefdeadbeef",
)
AttestationFactory.create(file=wheel)
sdist = FileFactory.create(
release=release,
filename=f"{project.name}-1.0.0.tar.gz",
packagetype="sdist",
)

files = [sdist, wheel]

urls_iter = (f"/file/{f.filename}" for f in files)
db_request.matchdict["name"] = project.normalized_name
db_request.route_url = lambda *a, **kw: next(urls_iter)
user = UserFactory.create()
je = JournalEntryFactory.create(name=project.name, submitted_by=user)

assert simple.simple_detail(project, db_request) == {
"meta": {"_last-serial": je.id, "api-version": API_VERSION},
"name": project.normalized_name,
"versions": ["1.0.0"],
"files": [
{
"filename": f.filename,
"url": f"/file/{f.filename}",
"hashes": {"sha256": f.sha256_digest},
"requires-python": f.requires_python,
"yanked": False,
"size": f.size,
"upload-time": f.upload_time.isoformat() + "Z",
"data-dist-info-metadata": (
{"sha256": "deadbeefdeadbeefdeadbeefdeadbeef"}
if f.metadata_file_sha256_digest is not None
else False
),
"core-metadata": (
{"sha256": "deadbeefdeadbeefdeadbeefdeadbeef"}
if f.metadata_file_sha256_digest is not None
else False
),
"provenance": integrity_service.get_provenance_digest(f),
}
for f in files
],
}

# Backstop: assert that we're testing at least provenance above
# by confirming that the wheel has one.
assert integrity_service.get_provenance_digest(wheel) is not None

def test_with_files_quarantined_omitted_from_index(self, db_request):
db_request.accept = "text/html"
project = ProjectFactory.create(lifecycle_status="quarantine-enter")
Expand Down
Loading
Loading