From 02f978e11b9b5e12ef134737d98a992604ddd8c7 Mon Sep 17 00:00:00 2001 From: bart-maykin Date: Fri, 26 Apr 2024 18:33:20 +0200 Subject: [PATCH] :construction_worker: added github workflows --- .github/workflows/ci.yml | 183 ++++++++++++++++++ .github/workflows/code_quality.yml | 105 ++++++++++ .github/workflows/codeql-analysis.yml | 62 ++++++ requirements/base.txt | 3 +- requirements/ci.txt | 151 ++------------- requirements/dev.txt | 73 ++----- requirements/test-tools.in | 16 +- requirements/test.txt | 1 - src/referentielijsten/api/tests/test_tabel.py | 1 + src/referentielijsten/conf/ci.py | 63 ++++++ src/referentielijsten/conf/jenkins.py | 82 -------- 11 files changed, 452 insertions(+), 288 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/code_quality.yml create mode 100644 .github/workflows/codeql-analysis.yml delete mode 100644 requirements/test.txt create mode 100644 src/referentielijsten/conf/ci.py delete mode 100644 src/referentielijsten/conf/jenkins.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ef3d1aa --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,183 @@ +name: ci + +# Run this workflow every time a new commit pushed to your repository +on: + push: + branches: + - main + tags: + - '*' + pull_request: + workflow_dispatch: + +env: + IMAGE_NAME: maykinmedia/referentielijsten + DJANGO_SETTINGS_MODULE: referentielijsten.conf.ci + DB_PASSWORD: '' + DB_USER: postgres + +jobs: + # determine changed files to decide if certain jobs can be skipped or not + changed-files: + runs-on: ubuntu-latest # windows-latest | macos-latest + name: Determine changed files + steps: + + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Get changed PY files + id: changed-py-files + uses: tj-actions/changed-files@v41 + with: + files: | + ^src/.+\.py + - name: Get changed JS files + id: changed-js-files + uses: tj-actions/changed-files@v41 + with: + files: | + ^src/.+\.js + - name: Get changed requirements files + id: changed-requirements + uses: tj-actions/changed-files@v41 + with: + files: ^requirements/.+\.txt$ + + outputs: + changed-py-files: ${{ steps.changed-py-files.outputs.any_changed }} + changed-js-files: ${{ steps.changed-js-files.outputs.any_changed }} + changed-requirements: ${{ steps.changed-requirements.outputs.any_changed }} + + tests: + runs-on: ubuntu-latest + # needs: + # - changed-files + + # # only run tests if source files have changed (e.g. skip for PRs that only update docs) + # if: ${{ needs.changed-files.outputs.changed-py-files == 'true'|| needs.changed-files.outputs.changed-requirements == 'true'|| github.event_name == 'push' }} + + strategy: + matrix: + postgres: ['15', '16'] + + name: Tests (PG ${{ matrix.postgres }}) + + services: + postgres: + image: postgres:${{ matrix.postgres }} + env: + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + # Needed because the postgres container does not provide a healthcheck + options: + --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - uses: actions/setup-node@v4 + with: + node-version: '18' + + - name: Install system packages + run: | + sudo apt-get update \ + && sudo apt-get install -y --no-install-recommends + - name: Install dependencies + run: pip install -r requirements/dev.txt codecov + - name: Build frontend + run: | + npm ci + npm run build + - name: Run tests + run: | + python src/manage.py collectstatic --noinput --link + coverage run src/manage.py test src + env: + DJANGO_SETTINGS_MODULE: referentielijsten.conf.ci + SECRET_KEY: dummy + DB_USER: postgres + DB_PASSWORD: '' + + - name: Publish coverage report + uses: codecov/codecov-action@v3 + + docker: + needs: tests + name: Docker image build + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Determine tag/commit hash + id: vars + run: | + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + # Strip "v" prefix from tag name (if present at all) + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + # Use Docker `latest` tag convention + [ "$VERSION" == "master" ] && VERSION=latest + echo "tag=${VERSION}" >> $GITHUB_OUTPUT + echo "git_hash=${GITHUB_SHA}" >> $GITHUB_OUTPUT + - name: Build the Docker image + run: | + docker build . \ + --tag $IMAGE_NAME:$RELEASE_VERSION \ + --build-arg COMMIT_HASH=${{ steps.vars.outputs.git_hash }} \ + --build-arg RELEASE=${{ steps.vars.outputs.tag }} \ + env: + RELEASE_VERSION: ${{ steps.vars.outputs.tag }} + + - run: docker image save -o image.tar $IMAGE_NAME:${{ steps.vars.outputs.tag }} + - name: Store image artifact + uses: actions/upload-artifact@v3 + with: + name: docker-image + path: image.tar + retention-days: 1 + + publish: + needs: + - tests + - docker + + name: Push Docker image + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.repository_owner == 'referentielijsten' # exclude PRs/forks + + steps: + - uses: actions/checkout@v4 + - name: Download built image + uses: actions/download-artifact@v3 + with: + name: docker-image + + - name: Determine tag/commit hash + id: vars + run: | + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + + # Strip "v" prefix from tag name (if present at all) + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + + # Use Docker `latest` tag convention + [ "$VERSION" == "main" ] && VERSION=latest + + echo "tag=${VERSION}" >> $GITHUB_OUTPUT + + - name: Load image + run: | + docker image load -i image.tar + + - name: Log into registry + run: echo "${{ secrets.DOCKER_TOKEN }}" | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + + - name: Push the Docker image + run: docker push $IMAGE_NAME:${{ steps.vars.outputs.tag }} diff --git a/.github/workflows/code_quality.yml b/.github/workflows/code_quality.yml new file mode 100644 index 0000000..2bc3cf1 --- /dev/null +++ b/.github/workflows/code_quality.yml @@ -0,0 +1,105 @@ +name: Code quality checks + +# Run this workflow every time a new commit pushed to your repository +on: + push: + branches: + - main + paths: + - '**.py' + - '**.yml' + pull_request: + paths: + - '**.py' + - '**.yml' + workflow_dispatch: + +jobs: + isort: + name: Code imports + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: 'requirements/*.txt' + - name: Install dependencies + run: pip install -r requirements/ci.txt + - name: Run isort + run: isort --check-only --diff . + + black: + name: Code format + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: 'requirements/*.txt' + - name: Install dependencies + run: pip install -r requirements/ci.txt + - name: Run black + run: black --check --diff src docs + + flake8: + name: Code style + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: 'requirements/*.txt' + - name: Install dependencies + run: pip install -r requirements/ci.txt + - name: Run flake8 + run: flake8 src + + migrations: + name: Check for model changes not present in the migrations + runs-on: ubuntu-latest + + services: + postgres: + image: postgis/postgis:12-2.5 + env: + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + # Needed because the postgres container does not provide a healthcheck + options: + --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: 'requirements/*.txt' + - name: Install system packages + run: | + sudo apt-get update \ + && sudo apt-get install -y --no-install-recommends \ + libgdal-dev \ + gdal-bin + + - name: Install dependencies + run: pip install -r requirements/ci.txt + + - name: Check for missing migrations + run: src/manage.py makemigrations --check --dry-run + env: + DJANGO_SETTINGS_MODULE: referentielijsten.conf.ci + SECRET_KEY: dummy + DB_USER: postgres + DB_NAME: postgres + DB_PASSWORD: '' diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..6367046 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,62 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +name: "CodeQL" + +on: + push: + branches: [main] + pull_request: + # The branches below must be a subset of the branches above + branches: [main] + schedule: + - cron: '0 23 * * 6' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + # Override automatic language detection by changing the below list + # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] + language: ['python', 'javascript'] + # Learn more... + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/requirements/base.txt b/requirements/base.txt index c807beb..c62a681 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # ./bin/compile_dependencies.sh @@ -302,7 +302,6 @@ tornado==6.4 # via flower typing-extensions==4.11.0 # via - # asgiref # qrcode # zgw-consumers tzdata==2024.1 diff --git a/requirements/ci.txt b/requirements/ci.txt index 1896b5e..8737f96 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,40 +1,32 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # ./bin/compile_dependencies.sh # amqp==5.2.0 # via - # -c requirements/base.txt # -r requirements/base.txt # kombu ape-pie==0.1.0 # via - # -c requirements/base.txt # -r requirements/base.txt # zgw-consumers asgiref==3.8.1 # via - # -c requirements/base.txt # -r requirements/base.txt # django # django-cors-headers asn1crypto==1.5.1 # via - # -c requirements/base.txt # -r requirements/base.txt # webauthn -astroid==3.1.0 - # via pylint async-timeout==4.0.3 # via - # -c requirements/base.txt # -r requirements/base.txt # redis attrs==23.2.0 # via - # -c requirements/base.txt # -r requirements/base.txt # glom # jsonschema @@ -43,105 +35,88 @@ beautifulsoup4==4.12.3 # via webtest billiard==4.2.0 # via - # -c requirements/base.txt # -r requirements/base.txt # celery +black==24.4.2 + # via -r requirements/test-tools.in bleach==6.1.0 # via - # -c requirements/base.txt # -r requirements/base.txt # open-api-framework boltons==24.0.0 # via - # -c requirements/base.txt # -r requirements/base.txt # face # glom cbor2==5.6.3 # via - # -c requirements/base.txt # -r requirements/base.txt # webauthn celery==5.4.0 # via - # -c requirements/base.txt # -r requirements/base.txt # flower # notifications-api-common # open-api-framework certifi==2024.2.2 # via - # -c requirements/base.txt # -r requirements/base.txt # elastic-apm # requests # sentry-sdk cffi==1.16.0 # via - # -c requirements/base.txt # -r requirements/base.txt # cryptography charset-normalizer==3.3.2 # via - # -c requirements/base.txt # -r requirements/base.txt # requests click==8.1.7 # via - # -c requirements/base.txt # -r requirements/base.txt + # -r requirements/test-tools.in + # black # celery # click-didyoumean # click-plugins # click-repl click-didyoumean==0.3.1 # via - # -c requirements/base.txt # -r requirements/base.txt # celery click-plugins==1.1.1 # via - # -c requirements/base.txt # -r requirements/base.txt # celery click-repl==0.3.0 # via - # -c requirements/base.txt # -r requirements/base.txt # celery commonground-api-common==1.13.0 # via - # -c requirements/base.txt # -r requirements/base.txt # open-api-framework coreapi==2.3.3 # via - # -c requirements/base.txt # -r requirements/base.txt # commonground-api-common coreschema==0.0.4 # via - # -c requirements/base.txt # -r requirements/base.txt # coreapi coverage==4.5.4 # via -r requirements/test-tools.in cryptography==42.0.5 # via - # -c requirements/base.txt # -r requirements/base.txt # django-simple-certmanager # josepy # mozilla-django-oidc # pyopenssl # webauthn -cssselect==1.2.0 - # via pyquery -dill==0.3.8 - # via pylint django==4.2.11 # via - # -c requirements/base.txt # -r requirements/base.txt # commonground-api-common # django-admin-index @@ -149,7 +124,6 @@ django==4.2.11 # django-cors-headers # django-filter # django-formtools - # django-jenkins # django-jsonform # django-markup # django-otp @@ -175,91 +149,72 @@ django==4.2.11 # zgw-consumers django-admin-index==3.1.1 # via - # -c requirements/base.txt # -r requirements/base.txt # open-api-framework django-axes==6.4.0 # via - # -c requirements/base.txt # -r requirements/base.txt # open-api-framework django-cors-headers==4.3.1 # via - # -c requirements/base.txt # -r requirements/base.txt # open-api-framework django-filter==24.2 # via - # -c requirements/base.txt # -r requirements/base.txt # commonground-api-common # open-api-framework django-formtools==2.5.1 # via - # -c requirements/base.txt # -r requirements/base.txt # django-two-factor-auth -django-jenkins==0.110.0 - # via -r requirements/test-tools.in django-jsonform==2.22.0 # via - # -c requirements/base.txt # -r requirements/base.txt # mozilla-django-oidc-db # open-api-framework django-markup==1.8.1 # via - # -c requirements/base.txt # -r requirements/base.txt # open-api-framework django-ordered-model==3.7.4 # via - # -c requirements/base.txt # -r requirements/base.txt # django-admin-index django-otp==1.5.0 # via - # -c requirements/base.txt # -r requirements/base.txt # django-two-factor-auth django-phonenumber-field==7.3.0 # via - # -c requirements/base.txt # -r requirements/base.txt # django-two-factor-auth django-privates==2.0.0.post1 # via - # -c requirements/base.txt # -r requirements/base.txt # django-simple-certmanager django-redis==5.4.0 # via - # -c requirements/base.txt # -r requirements/base.txt # open-api-framework django-relativedelta==2.0.0 # via - # -c requirements/base.txt # -r requirements/base.txt # zgw-consumers django-rest-framework-condition==0.1.1 # via - # -c requirements/base.txt # -r requirements/base.txt # commonground-api-common django-sendfile2==0.7.1 # via - # -c requirements/base.txt # -r requirements/base.txt # django-privates django-simple-certmanager==2.0.0 # via - # -c requirements/base.txt # -r requirements/base.txt # zgw-consumers django-solo==2.2.0 # via - # -c requirements/base.txt # -r requirements/base.txt # commonground-api-common # mozilla-django-oidc-db @@ -267,14 +222,12 @@ django-solo==2.2.0 # zgw-consumers django-two-factor-auth[phonenumberslite,webauthn]==1.16.0 # via - # -c requirements/base.txt # -r requirements/base.txt # maykin-2fa django-webtest==1.9.11 # via -r requirements/test-tools.in djangorestframework==3.15.1 # via - # -c requirements/base.txt # -r requirements/base.txt # commonground-api-common # djangorestframework-gis @@ -286,48 +239,39 @@ djangorestframework==3.15.1 # open-api-framework djangorestframework-camel-case==1.4.2 # via - # -c requirements/base.txt # -r requirements/base.txt # commonground-api-common # notifications-api-common djangorestframework-gis==1.0 # via - # -c requirements/base.txt # -r requirements/base.txt # open-api-framework djangorestframework-inclusions==1.2.0 # via - # -c requirements/base.txt # -r requirements/base.txt # open-api-framework drf-nested-routers==0.93.5 # via - # -c requirements/base.txt # -r requirements/base.txt # commonground-api-common drf-spectacular==0.27.2 # via - # -c requirements/base.txt # -r requirements/base.txt # open-api-framework drf-yasg==1.21.7 # via - # -c requirements/base.txt # -r requirements/base.txt # commonground-api-common ecs-logging==2.1.0 # via - # -c requirements/base.txt # -r requirements/base.txt # elastic-apm elastic-apm==6.22.0 # via - # -c requirements/base.txt # -r requirements/base.txt # open-api-framework face==20.1.1 # via - # -c requirements/base.txt # -r requirements/base.txt # glom factory-boy==3.3.0 @@ -338,191 +282,154 @@ flake8==7.0.0 # via -r requirements/test-tools.in flower==2.0.1 # via - # -c requirements/base.txt # -r requirements/base.txt # open-api-framework freezegun==1.5.0 # via -r requirements/test-tools.in furl==2.1.3 # via - # -c requirements/base.txt # -r requirements/base.txt # ape-pie gemma-zds-client==2.0.0 # via - # -c requirements/base.txt # -r requirements/base.txt # commonground-api-common # notifications-api-common glom==23.5.0 # via - # -c requirements/base.txt # -r requirements/base.txt # mozilla-django-oidc-db humanize==4.9.0 # via - # -c requirements/base.txt # -r requirements/base.txt # flower idna==3.7 # via - # -c requirements/base.txt # -r requirements/base.txt # requests inflection==0.5.1 # via - # -c requirements/base.txt # -r requirements/base.txt # drf-spectacular # drf-yasg iso-639==0.4.5 # via - # -c requirements/base.txt # -r requirements/base.txt # commonground-api-common isodate==0.6.1 # via - # -c requirements/base.txt # -r requirements/base.txt # commonground-api-common isort==5.13.2 - # via pylint + # via -r requirements/test-tools.in itypes==1.2.0 # via - # -c requirements/base.txt # -r requirements/base.txt # coreapi jinja2==3.1.3 # via - # -c requirements/base.txt # -r requirements/base.txt # coreschema josepy==1.14.0 # via - # -c requirements/base.txt # -r requirements/base.txt # mozilla-django-oidc jsonschema==4.21.1 # via - # -c requirements/base.txt # -r requirements/base.txt # drf-spectacular jsonschema-specifications==2023.12.1 # via - # -c requirements/base.txt # -r requirements/base.txt # jsonschema kombu==5.3.7 # via - # -c requirements/base.txt # -r requirements/base.txt # celery -lxml==5.2.1 - # via pyquery markupsafe==2.1.5 # via - # -c requirements/base.txt # -r requirements/base.txt # jinja2 maykin-2fa==1.0.0 # via - # -c requirements/base.txt # -r requirements/base.txt # open-api-framework mccabe==0.7.0 - # via - # flake8 - # pylint + # via flake8 mozilla-django-oidc==4.0.1 # via - # -c requirements/base.txt # -r requirements/base.txt # mozilla-django-oidc-db mozilla-django-oidc-db==0.15.0 # via - # -c requirements/base.txt # -r requirements/base.txt # open-api-framework +mypy-extensions==1.0.0 + # via black notifications-api-common==0.2.2 # via - # -c requirements/base.txt # -r requirements/base.txt # commonground-api-common open-api-framework==0.2.0 - # via - # -c requirements/base.txt - # -r requirements/base.txt + # via -r requirements/base.txt orderedmultidict==1.0.1 # via - # -c requirements/base.txt # -r requirements/base.txt # furl oyaml==1.0 # via - # -c requirements/base.txt # -r requirements/base.txt # commonground-api-common packaging==24.0 # via - # -c requirements/base.txt # -r requirements/base.txt + # black # drf-yasg +pathspec==0.12.1 + # via black phonenumberslite==8.13.35 # via - # -c requirements/base.txt # -r requirements/base.txt # django-two-factor-auth platformdirs==4.2.1 - # via pylint + # via black prometheus-client==0.20.0 # via - # -c requirements/base.txt # -r requirements/base.txt # flower prompt-toolkit==3.0.43 # via - # -c requirements/base.txt # -r requirements/base.txt # click-repl psycopg2==2.9.9 # via - # -c requirements/base.txt # -r requirements/base.txt # open-api-framework pycodestyle==2.11.1 # via flake8 pycparser==2.22 # via - # -c requirements/base.txt # -r requirements/base.txt # cffi pyflakes==3.2.0 # via flake8 pyjwt==2.8.0 # via - # -c requirements/base.txt # -r requirements/base.txt # commonground-api-common # gemma-zds-client # zgw-consumers -pylint==3.1.0 - # via -r requirements/test-tools.in pyopenssl==24.1.0 # via - # -c requirements/base.txt # -r requirements/base.txt # josepy # webauthn pypng==0.20220715.0 # via - # -c requirements/base.txt # -r requirements/base.txt # qrcode -pyquery==2.0.0 - # via -r requirements/test-tools.in python-dateutil==2.9.0.post0 # via - # -c requirements/base.txt # -r requirements/base.txt # celery # django-relativedelta @@ -530,23 +437,19 @@ python-dateutil==2.9.0.post0 # freezegun python-decouple==3.8 # via - # -c requirements/base.txt # -r requirements/base.txt # open-api-framework python-dotenv==1.0.1 # via - # -c requirements/base.txt # -r requirements/base.txt # open-api-framework pytz==2024.1 # via - # -c requirements/base.txt # -r requirements/base.txt # drf-yasg # flower pyyaml==6.0.1 # via - # -c requirements/base.txt # -r requirements/base.txt # drf-spectacular # drf-yasg @@ -554,23 +457,19 @@ pyyaml==6.0.1 # oyaml qrcode==7.4.2 # via - # -c requirements/base.txt # -r requirements/base.txt # django-two-factor-auth redis==5.0.4 # via - # -c requirements/base.txt # -r requirements/base.txt # django-redis referencing==0.35.0 # via - # -c requirements/base.txt # -r requirements/base.txt # jsonschema # jsonschema-specifications requests==2.31.0 # via - # -c requirements/base.txt # -r requirements/base.txt # ape-pie # commonground-api-common @@ -584,18 +483,15 @@ requests-mock==1.12.1 # via -r requirements/test-tools.in rpds-py==0.18.0 # via - # -c requirements/base.txt # -r requirements/base.txt # jsonschema # referencing sentry-sdk==2.0.0 # via - # -c requirements/base.txt # -r requirements/base.txt # open-api-framework six==1.16.0 # via - # -c requirements/base.txt # -r requirements/base.txt # bleach # furl @@ -606,55 +502,41 @@ soupsieve==2.5 # via beautifulsoup4 sqlparse==0.5.0 # via - # -c requirements/base.txt # -r requirements/base.txt # django tblib==3.0.0 # via -r requirements/test-tools.in -tomli==2.0.1 - # via pylint -tomlkit==0.12.4 - # via pylint tornado==6.4 # via - # -c requirements/base.txt # -r requirements/base.txt # flower typing-extensions==4.11.0 # via - # -c requirements/base.txt # -r requirements/base.txt - # asgiref - # astroid # qrcode # zgw-consumers tzdata==2024.1 # via - # -c requirements/base.txt # -r requirements/base.txt # celery uritemplate==4.1.1 # via - # -c requirements/base.txt # -r requirements/base.txt # coreapi # drf-spectacular # drf-yasg urllib3==2.2.1 # via - # -c requirements/base.txt # -r requirements/base.txt # elastic-apm # requests # sentry-sdk uwsgi==2.0.25.1 # via - # -c requirements/base.txt # -r requirements/base.txt # open-api-framework vine==5.1.0 # via - # -c requirements/base.txt # -r requirements/base.txt # amqp # celery @@ -663,17 +545,14 @@ waitress==3.0.0 # via webtest wcwidth==0.2.13 # via - # -c requirements/base.txt # -r requirements/base.txt # prompt-toolkit webauthn==2.1.0 # via - # -c requirements/base.txt # -r requirements/base.txt # django-two-factor-auth webencodings==0.5.1 # via - # -c requirements/base.txt # -r requirements/base.txt # bleach webob==1.8.7 @@ -682,12 +561,10 @@ webtest==3.0.0 # via django-webtest wrapt==1.14.1 # via - # -c requirements/base.txt # -r requirements/base.txt # elastic-apm zgw-consumers==0.33.0 # via - # -c requirements/base.txt # -r requirements/base.txt # notifications-api-common # open-api-framework diff --git a/requirements/dev.txt b/requirements/dev.txt index 47dcb75..0129666 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # ./bin/compile_dependencies.sh @@ -27,11 +27,6 @@ asn1crypto==1.5.1 # -c requirements/ci.txt # -r requirements/ci.txt # webauthn -astroid==3.1.0 - # via - # -c requirements/ci.txt - # -r requirements/ci.txt - # pylint async-timeout==4.0.3 # via # -c requirements/ci.txt @@ -56,8 +51,11 @@ billiard==4.2.0 # -c requirements/ci.txt # -r requirements/ci.txt # celery -black==24.4.1 - # via -r requirements/dev.in +black==24.4.2 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # -r requirements/dev.in bleach==6.1.0 # via # -c requirements/ci.txt @@ -157,16 +155,6 @@ cryptography==42.0.5 # mozilla-django-oidc # pyopenssl # webauthn -cssselect==1.2.0 - # via - # -c requirements/ci.txt - # -r requirements/ci.txt - # pyquery -dill==0.3.8 - # via - # -c requirements/ci.txt - # -r requirements/ci.txt - # pylint django==4.2.11 # via # -c requirements/ci.txt @@ -179,7 +167,6 @@ django==4.2.11 # django-extensions # django-filter # django-formtools - # django-jenkins # django-jsonform # django-markup # django-otp @@ -233,10 +220,6 @@ django-formtools==2.5.1 # -c requirements/ci.txt # -r requirements/ci.txt # django-two-factor-auth -django-jenkins==0.110.0 - # via - # -c requirements/ci.txt - # -r requirements/ci.txt django-jsonform==2.22.0 # via # -c requirements/ci.txt @@ -447,7 +430,6 @@ isort[colors]==5.13.2 # -c requirements/ci.txt # -r requirements/ci.txt # -r requirements/dev.in - # pylint itypes==1.2.0 # via # -c requirements/ci.txt @@ -479,11 +461,6 @@ kombu==5.3.7 # -c requirements/ci.txt # -r requirements/ci.txt # celery -lxml==5.2.1 - # via - # -c requirements/ci.txt - # -r requirements/ci.txt - # pyquery markupsafe==2.1.5 # via # -c requirements/ci.txt @@ -499,7 +476,6 @@ mccabe==0.7.0 # -c requirements/ci.txt # -r requirements/ci.txt # flake8 - # pylint mozilla-django-oidc==4.0.1 # via # -c requirements/ci.txt @@ -511,7 +487,10 @@ mozilla-django-oidc-db==0.15.0 # -r requirements/ci.txt # open-api-framework mypy-extensions==1.0.0 - # via black + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # black notifications-api-common==0.2.2 # via # -c requirements/ci.txt @@ -540,7 +519,10 @@ packaging==24.0 # drf-yasg # sphinx pathspec==0.12.1 - # via black + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # black phonenumberslite==8.13.35 # via # -c requirements/ci.txt @@ -553,7 +535,6 @@ platformdirs==4.2.1 # -c requirements/ci.txt # -r requirements/ci.txt # black - # pylint prometheus-client==0.20.0 # via # -c requirements/ci.txt @@ -593,10 +574,6 @@ pyjwt==2.8.0 # commonground-api-common # gemma-zds-client # zgw-consumers -pylint==3.1.0 - # via - # -c requirements/ci.txt - # -r requirements/ci.txt pyopenssl==24.1.0 # via # -c requirements/ci.txt @@ -612,10 +589,6 @@ pyproject-hooks==1.0.0 # via # build # pip-tools -pyquery==2.0.0 - # via - # -c requirements/ci.txt - # -r requirements/ci.txt python-dateutil==2.9.0.post0 # via # -c requirements/ci.txt @@ -741,21 +714,6 @@ tblib==3.0.0 # via # -c requirements/ci.txt # -r requirements/ci.txt -tomli==2.0.1 - # via - # -c requirements/ci.txt - # -r requirements/ci.txt - # black - # build - # pip-tools - # pylint - # pyproject-hooks - # sphinx -tomlkit==0.12.4 - # via - # -c requirements/ci.txt - # -r requirements/ci.txt - # pylint tornado==6.4 # via # -c requirements/ci.txt @@ -765,9 +723,6 @@ typing-extensions==4.11.0 # via # -c requirements/ci.txt # -r requirements/ci.txt - # asgiref - # astroid - # black # qrcode # zgw-consumers tzdata==2024.1 diff --git a/requirements/test-tools.in b/requirements/test-tools.in index 13c90e9..7bacb7c 100644 --- a/requirements/test-tools.in +++ b/requirements/test-tools.in @@ -1,13 +1,15 @@ --c base.txt - # Dependencies only relevant for (unit) testing -coverage < 5.0 -django-jenkins +coverage django-webtest factory-boy freezegun -flake8 -pylint -pyquery # integrates with webtest requests-mock tblib + +# Code formatting +isort +black +flake8 + +# SPDX header checks +click diff --git a/requirements/test.txt b/requirements/test.txt deleted file mode 100644 index 0ceea45..0000000 --- a/requirements/test.txt +++ /dev/null @@ -1 +0,0 @@ --r production.txt diff --git a/src/referentielijsten/api/tests/test_tabel.py b/src/referentielijsten/api/tests/test_tabel.py index 4d2cc26..49f44a9 100644 --- a/src/referentielijsten/api/tests/test_tabel.py +++ b/src/referentielijsten/api/tests/test_tabel.py @@ -5,6 +5,7 @@ from vng_api_common.tests import reverse from referentielijsten.token.tests.api_testcase import APITestCase + from ..models import Tabel from .factories import ItemFactory, TabelFactory diff --git a/src/referentielijsten/conf/ci.py b/src/referentielijsten/conf/ci.py new file mode 100644 index 0000000..385c687 --- /dev/null +++ b/src/referentielijsten/conf/ci.py @@ -0,0 +1,63 @@ +""" +Continuous integration settings module. +""" + +import os +import warnings + +# Importing the idna module has an IO side-effect to load the data, which is a rather +# big file. Pre-loading this in the settings file populates the python module cache, +# preventing flakiness in hypothesis tests that hit this code path. +import idna # noqa: F401 + +os.environ.setdefault("IS_HTTPS", "no") +os.environ.setdefault("SECRET_KEY", "dummy") +# Do not log requests in CI/tests: +# +# * overhead making tests slower +# * it conflicts with SimpleTestCase in some cases when the run-time configuration is +# looked up from the django-solo model +os.environ.setdefault("LOG_REQUESTS", "no") + +from .base import * # noqa isort:skip + +CACHES.update( + { + "default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}, + # See: https://github.com/jazzband/django-axes/blob/master/docs/configuration.rst#cache-problems + "axes": {"BACKEND": "django.core.cache.backends.dummy.DummyCache"}, + "oidc": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}, + } +) + +# shut up logging +LOGGING["loggers"].update( + { + "openforms.api.exception_handling": { + "handlers": ["console"], + "level": "CRITICAL", + "propagate": False, + }, + } +) + +# don't spend time on password hashing in tests/user factories +PASSWORD_HASHERS = ["django.contrib.auth.hashers.UnsaltedMD5PasswordHasher"] + +ENVIRONMENT = "CI" + +# +# Django-axes +# +AXES_BEHIND_REVERSE_PROXY = False + +# Django privates +SENDFILE_BACKEND = "django_sendfile.backends.development" + +# THOU SHALT NOT USE NAIVE DATETIMES +warnings.filterwarnings( + "error", + r"DateTimeField .* received a naive datetime", + RuntimeWarning, + r"django\.db\.models\.fields", +) diff --git a/src/referentielijsten/conf/jenkins.py b/src/referentielijsten/conf/jenkins.py deleted file mode 100644 index 58b9d40..0000000 --- a/src/referentielijsten/conf/jenkins.py +++ /dev/null @@ -1,82 +0,0 @@ -import os -import warnings - -os.environ.setdefault("DEBUG", "no") -os.environ.setdefault("SECRET_KEY", "for-testing-purposes-only") -os.environ.setdefault("IS_HTTPS", "no") -os.environ.setdefault("ALLOWED_HOSTS", "") - -os.environ.setdefault("DB_USER", "jenkins") -os.environ.setdefault("DB_PASSWORD", "jenkins") -# PostgreSQL 9.6: 5432 (default for Jenkins) -os.environ.setdefault("DB_PORT", "5432") - -os.environ.setdefault("ENVIRONMENT", "jenkins") - -from .base import * # noqa isort:skip - - -def get_db_name(prefix): - """ - get a reasonable name below Postgres' 63 char name limit - """ - job = os.getenv("JOB_NAME", default="").lower().rsplit("/", 1)[-1] - build = os.getenv("BUILD_NUMBER", default="0") - lim = 63 - 2 - len(prefix) - len(build) - return "{}_{}_{}".format(prefix, job[:lim], build) - - -DATABASES["default"]["TEST"] = {"NAME": get_db_name("test_referentielijsten")} - -LOGGING["loggers"].update( - { - "django": { - "handlers": ["django"], - "level": "WARNING", - "propagate": True, - }, - } -) - -# -# Django-axes -# -AXES_BEHIND_REVERSE_PROXY = ( - False # Required to allow FakeRequest and the like to work correctly. -) - -# in memory cache and django-axes don't get along. -# https://django-axes.readthedocs.io/en/latest/configuration.html#known-configuration-problems -CACHES = { - "default": { - "BACKEND": "django.core.cache.backends.locmem.LocMemCache", - }, - "axes": { - "BACKEND": "django.core.cache.backends.dummy.DummyCache", - }, -} - -ELASTIC_APM["DEBUG"] = True - -# disable 2 FA in tests -MAYKIN_2FA_ALLOW_MFA_BYPASS_BACKENDS = AUTHENTICATION_BACKENDS - -# -# Jenkins settings -# -INSTALLED_APPS += [ - "django_jenkins", -] -PROJECT_APPS = [app for app in INSTALLED_APPS if app.startswith("referentielijsten.")] -JENKINS_TASKS = ( - # 'django_jenkins.tasks.run_pylint', # Pylint < 2.0 does not run on Python 3.7+ - # "django_jenkins.tasks.run_pep8", # -> renamed to pycodestyle, but django-jenkins hasn't been updated in 6 years -) - -# THOU SHALT NOT USE NAIVE DATETIMES -warnings.filterwarnings( - "error", - r"DateTimeField .* received a naive datetime", - RuntimeWarning, - r"django\.db\.models\.fields", -)