Merge pull request #1662 from alphagov/upgrade-ckan-2.10.3-tests

Upgrade to ckan 2.10.4
alphagov · Jun 11, 2024 · da48eb5 · da48eb5
2 parents fa15041 + 75e849e
commit da48eb5
Show file tree

Hide file tree

Showing 112 changed files with 1,614 additions and 1,088 deletions.
diff --git a/.github/workflows/build-image-on-tags.yaml b/.github/workflows/build-image-on-tags.yaml
@@ -9,16 +9,6 @@ jobs:
   build_and_push:
     name: ${{ matrix.app.name }} (${{ matrix.arch }})
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        app:
-          - name: ckan
-            version: 2.9.9
-          - name: pycsw
-            version: 2.4.2
-          - name: solr
-            version: 8-2.9f
-        arch: [ amd64 ]
     permissions:
       packages: write
     steps:
@@ -33,8 +23,8 @@ jobs:
           ref: ${{ inputs.gitRef }}
       - name: Build and push images
         env:
-          APP: ${{ matrix.app.name }}
-          VERSION: ${{ matrix.app.version }}
-          ARCH: ${{ matrix.arch }}
+          APP: ckan
+          VERSION: 2.10.4
+          ARCH: amd64
           GH_REF: ${{ github.ref_name }}
         run: ./docker/build-image.sh
diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml
@@ -8,10 +8,11 @@ on:
         required: true
         type: choice
         options:
-          - build_push
+          - build_push_ckan
           - build_only
           - build_push_base
           - build_push_pycsw
+          - build_push_solr
           - build_push_test_ckan
   push:
     branches:
@@ -28,14 +29,13 @@ jobs:
         # remember to update versions on build-image-on-tags.yaml
         app:
           - name: ckan
-            version: 2.9.9
+            version: 2.10.4
+            patch: ""
           - name: pycsw
             version: 2.6.1
-            skip: true
-          - name: pycsw
-            version: 2.4.2
+            patch: e
           - name: solr
-            version: 8-2.9f
+            version: "2.10"
         arch: [ amd64 ]
     permissions:
       packages: write
@@ -57,15 +57,14 @@ jobs:
           VERSION: ${{ matrix.app.version }}
           ARCH: ${{ matrix.arch }}
         run: ./docker/build-image.sh
-      - name: Build and push images
-        if: ${{ inputs.buildType == 'build_push' || github.ref == 'refs/heads/main' }}
+      - name: Build and push CKAN image
+        if: ${{ (inputs.buildType == 'build_push_ckan' || github.ref == 'refs/heads/main') && matrix.app.name == 'ckan' }}
         env:
           APP: ${{ matrix.app.name }}
           VERSION: ${{ matrix.app.version }}
           ARCH: ${{ matrix.arch }}
-          SKIP: ${{ matrix.app.skip }}
         run: ./docker/build-image.sh
-      - name: Build and push CKAN base image
+      - name: Build and push CKAN base and core image
         if: ${{ inputs.buildType == 'build_push_base' && matrix.app.name == 'ckan' }}
         env:
           APP: ${{ matrix.app.name }}
@@ -78,9 +77,18 @@ jobs:
         env:
           APP: ${{ matrix.app.name }}
           VERSION: ${{ matrix.app.version }}
+          PATCH: ${{ matrix.app.patch }}
           ARCH: ${{ matrix.arch }}
           BUILD_BASE: true
-          SKIP: ${{ matrix.app.skip }}
+        run: ./docker/build-image.sh
+      - name: Build and push solr image
+        if: ${{ inputs.buildType == 'build_push_solr'  && matrix.app.name == 'solr' }}
+        env:
+          APP: ${{ matrix.app.name }}
+          VERSION: ${{ matrix.app.version }}
+          ARCH: ${{ matrix.arch }}
+          GH_REF: ${{ matrix.app.version }}
+          BUILD_BASE: false
         run: ./docker/build-image.sh
       - name: Build and push CKAN test image
         if: ${{ inputs.buildType == 'build_push_test_ckan' && matrix.app.name == 'ckan' }}

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,13 +1,13 @@
 name: ckanext-datagovuk ci
 on:
+  workflow_dispatch:
   push:
   pull_request_target:
     branches:
       - main
     types:
       - closed
 
-
 jobs:
   codeql-sast:
     name: CodeQL SAST scan
@@ -23,11 +23,11 @@ jobs:
     name: test
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/alphagov/ckan:2.9.9-base
+      image: ghcr.io/alphagov/ckan:2.10-base
       options: --user root
     services:
       solr:
-        image: ghcr.io/alphagov/solr:8-2.9f
+        image: ghcr.io/alphagov/solr:2.10
       postgres:
         image: postgis/postgis:13-3.1-alpine
         env:
@@ -67,7 +67,7 @@ jobs:
           pip install -e .
       - name: Install dev requirements
         run: |
-          pip install factory-boy==2.12.0 pytest-ckan==0.0.12 pytest-cov==4.0.0 mock==2.0.0
+          pip install -r dev-requirements.txt
       - name: Update config
         run: |
           # Replace default path to CKAN core config file with the one on the container
@@ -78,4 +78,4 @@ jobs:
         run: |
           ckan db init
       - name: Run tests
-        run: pytest --ckan-ini=test.ini tests --cov=ckanext.datagovuk --cov-report=term-missing --disable-pytest-warnings -v
+        run: pytest --ckan-ini=test.ini ckanext/datagovuk/tests --cov=ckanext.datagovuk --cov-report=term-missing --disable-pytest-warnings -v
diff --git a/.github/workflows/create-pr.yaml b/.github/workflows/create-pr.yaml
@@ -6,8 +6,9 @@ on:
     workflows: [ "Build and push images" ]
     types:
       - completed
+  push:
     branches:
-      - main
+      - main-2.10
 
 jobs:
   create_pr:

diff --git a/.gitignore b/.gitignore
@@ -48,3 +48,4 @@ docs/_build/
 environment.sh
 .DS_Store
 .env*
+docker/src
diff --git a/README.rst b/README.rst
@@ -95,3 +95,11 @@ Then run the paster command:
 In order to remove the test data:
 
    ckan datagovuk remove-dgu-test-data
+
+-----------------
+Deploying CKAN, PyCSW and Solr
+-----------------
+
+By default merging a PR into main will trigger a build of the CKAN image and create PRs on the govuk-dgu-charts repo. Updating the tag will build and push a tagged image and create the PR for Staging and Production environments in the govuk-dgu-charts repo.
+
+If you want to deploy updates to PyCSW or Solr then you will need to trigger the build manually via the github actions tab and manually update the pycsw or solr tag in govuk-dgu-charts repo for each environment.
diff --git a/bin/setup_ckan.sh b/bin/setup_ckan.sh
@@ -11,6 +11,8 @@ if [ ${CKAN_DB_INIT:-} = "true" ]; then
     ckan db init
 fi
 
+ckan db upgrade
+
 if [ ! -z $CREATE_CKAN_ADMIN ]; then
     if (ckan user show ckan_admin | grep -q "User: None"); then
         echo "===== adding sysadmin user ====="

diff --git a/bin/setup_pycsw.sh b/bin/setup_pycsw.sh
@@ -9,6 +9,7 @@ done
 
 # only run the setup script if the records table has not been created
 if !(psql $CKAN_SQLALCHEMY_URL -c "select exists (SELECT * FROM information_schema.tables where table_name = 'records');" | tr -d '\n' | grep -q "exists -------- t"); then 
+  sed "s|database=\${CKAN_SQLALCHEMY_URL}|database=$CKAN_SQLALCHEMY_URL|g" -i /config/pycsw.cfg
   ckan ckan-pycsw setup -p $CKAN_CONFIG/pycsw.cfg
 
   echo "Drop ix_records_abstract if exists and create pycsw abstract index to allow for larger records"