[pull] main from pypi:main

.dockerignore

@@ -5,3 +5,4 @@ dev/*
 **/*.pyc
 htmlcov
 warehouse/static/dist
+.venv

.editorconfig

@@ -10,6 +10,11 @@ insert_final_newline = true
 
 [*.py]
 indent_size = 4
+# handled by black config
+max_line_length = off
+
+[*.md]
+trim_trailing_whitespace = false
 
 [Makefile]
 indent_style = tab

.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

.github/dependabot.yml

@@ -11,14 +11,26 @@ updates:
   - dependency-type: indirect
   rebase-strategy: "disabled"
   ignore:
-  # Always ignore elasticsearch, future versions are always incompatible with our provider
-  - dependency-name: "elasticsearch"
   # These update basically every day, and 99.9% of the time we don't care
   - dependency-name: "boto3"
   - dependency-name: "boto3-stubs"
   - dependency-name: "botocore"
   - dependency-name: "botocore-stubs"
+  groups:
+    celery:
+      # Keep both celery and kombu together
+      patterns:
+        - "celery"
+        - "kombu"
+    psycopg:
+      # Keep both psycopg and psycopg-c together
+      patterns:
+        - "psycopg*"
 - package-ecosystem: github-actions
   directory: "/"
   schedule:
     interval: "daily"
+- package-ecosystem: docker
+  directory: "/"
+  schedule:
+    interval: "daily"

.github/workflows/ci.yml

@@ -6,71 +6,122 @@ on:
   pull_request:
   merge_group:
     types: [checks_requested]
+  workflow_dispatch:  # generally only for the "combine-prs" workflow
 permissions:
+  id-token: write
   contents: read
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
   cancel-in-progress: true
 jobs:
+  build:
+    if: github.repository == 'pypi/warehouse'
+    runs-on: depot-ubuntu-22.04-arm
+    outputs:
+      buildId: ${{ steps.build.outputs.build-id}}
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+      - name: Set up Depot CLI
+        uses: depot/setup-action@v1
+      - name: Build image
+        id: build
+        uses: depot/build-push-action@v1
+        with:
+          save: true
+          build-args: |
+            DEVEL=yes
+            CI=yes
+          tags: pypi/warehouse:ci-${{ github.run_id }}
   test:
+    # Time out if our test suite has gotten hung
+    timeout-minutes: 15
+    needs: build
     strategy:
       matrix:
         include:
           - name: Tests
-            command: bin/tests --postgresql-host localhost
+            command: bin/tests --postgresql-host postgres
           - name: Lint
             command: bin/lint
-          - name: Documentation
-            command: bin/docs
+          - name: User Documentation
+            command: bin/user-docs
+          - name: Developer Documentation
+            command: bin/dev-docs
           - name: Dependencies
-            command: bin/github-actions-deps
+            command: bin/deps
           - name: Licenses
             command: bin/licenses
           - name: Translations
             command: bin/translations
-    runs-on: ubuntu-latest
+    runs-on: depot-ubuntu-22.04-arm
+    container:
+      image: registry.depot.dev/rltf7cln5v:${{ needs.build.outputs.buildId }}
+      env:
+        BILLING_BACKEND: warehouse.subscriptions.services.MockStripeBillingService api_base=http://stripe:12111 api_version=2020-08-27
     services:
       postgres:
-        image: postgres:14.4
+        image: ${{ (matrix.name == 'Tests') && 'postgres:16.1' || '' }}
         ports:
           - 5432:5432
         env:
           POSTGRES_HOST_AUTH_METHOD: trust  # never do this in production!
+          POSTGRES_INITDB_ARGS: '--no-sync --set fsync=off --set full_page_writes=off'
         # Set health checks to wait until postgres has started
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+        options: --health-cmd "pg_isready --username=postgres --dbname=postgres" --health-interval 10s --health-timeout 5s --health-retries 5
+      redis:
+        image: ${{ (matrix.name == 'Tests') && 'redis:7.0' || '' }}
+        ports:
+          - 6379:6379
       stripe:
-        image: stripe/stripe-mock:v0.140.0
+        image: ${{ (matrix.name == 'Tests') && 'stripe/stripe-mock:v0.162.0' || '' }}
         ports:
           - 12111:12111
     name: ${{ matrix.name }}
-    env:
-      BILLING_BACKEND: warehouse.subscriptions.services.MockStripeBillingService api_base=http://localhost:12111 api_version=2020-08-27
     steps:
       - name: Check out repository
-        uses: actions/checkout@v3
-      - name: Install platform dependencies
-        run: |
-          sudo apt -y update
-          sudo apt -y install libcurl4-openssl-dev libssl-dev pkg-config
-      - uses: actions/setup-python@v4
-        with:
-          python-version-file: '.python-version'
-          cache: 'pip'
-          cache-dependency-path: |
-            requirements.txt
-            requirements/*.txt
-      - name: Cache common Python cache paths
-        uses: actions/cache@v3
+        uses: actions/checkout@v4
+      - name: Cache mypy results
+        if: ${{ (matrix.name == 'Lint') }}
+        uses: actions/cache@v4
         with:
           path: |
-            .cache
-            .mypy_cache
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt', 'requirements/lint.txt') }}
-      - name: Install Python dependencies
-        run: |
-          pip install -U pip setuptools wheel
-          pip install -r requirements.txt --no-deps
-          pip install -r requirements/dev.txt
-          pip check
+              dev/.mypy_cache
+          key: ${{ runner.os }}-mypy-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }}
       - name: Run ${{ matrix.name }}
         run: ${{ matrix.command }}
+
+  check_db:
+    name: Check Database Consistency
+    needs: build
+    runs-on: depot-ubuntu-22.04-arm
+    continue-on-error: true
+    container:
+      image: registry.depot.dev/rltf7cln5v:${{ needs.build.outputs.buildId }}
+    services:
+      postgres:
+        image: postgres:16.1
+        ports:
+        - 5432:5432
+        env:
+          POSTGRES_DB: warehouse
+          POSTGRES_HOST_AUTH_METHOD: trust  # never do this in production!
+          POSTGRES_INITDB_ARGS: '--no-sync --set fsync=off --set full_page_writes=off'
+        # Set health checks to wait until postgres has started
+        options: --health-cmd "pg_isready --username=postgres --dbname=postgres" --health-interval 10s --health-timeout 5s --health-retries 5
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+      - name: Dotenv Action
+        # We need to load the environment variables to run the CLI
+        id: dotenv
+        uses: falti/dotenv-action@v1
+        with:
+          path: dev/environment
+          export-variables: true
+          keys-case: upper
+      - name: Check Database
+        run: bin/db-check
+        env:
+          # override the hostname set in `dev/environment`
+          DATABASE_URL: 'postgresql+psycopg://postgres@postgres/warehouse'

.github/workflows/codeql-analysis.yml

@@ -38,39 +38,49 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
-
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      if: matrix.language == 'python'
+      uses: actions/setup-python@v5
+      with:
+        python-version-file: '.python-version'
+
     - name: Install dependencies
       # Needed for pycurl
-      run: sudo apt install libcurl4-openssl-dev libssl-dev
+      run: |
+        sudo apt-get update
+        sudo apt install libcurl4-openssl-dev libssl-dev
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
         # By default, queries listed here will override any specified in a config file.
         # Prefix the list here with "+" to use these queries and those in the config file.
-        
+
         # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
         # queries: security-extended,security-and-quality
 
-        
+
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
 
-    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
     #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
 
     # - run: |
     #   echo "Run, Build Application using script"
     #   ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"