From 61477fc2aa57cb1acd1db26a276d87e301d95144 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Thu, 2 Jan 2025 13:03:05 -0500 Subject: [PATCH] integrate with renovate (#358) * Add renovate * Migrate config renovate.json * Do not match on type * hmm * Add solr * Make sha update script more generic * Add composer * Add solr-ocrhighlighting * fixup * Use advanced capture for github releases * Update matchPackageNames * Use advanced capture for apache repos * Add cantaloupe * Add FITS * Add blazegraph log4j * Add some docs around renovate * matchDepNames * match all * Remove double quotes around version * Fix semver for some repos and better multi file support * put blazegraph log4j hash back * Update README.md * Add renovate validation action * Also update version in README * fix typo * Explicitly enable managers we want instead of disabling ones renovate auto-detected --- .github/workflows/validate-renovate.yml | 21 +++++ README.md | 37 ++++++++ activemq/Dockerfile | 3 +- blazegraph/Dockerfile | 4 +- cantaloupe/Dockerfile | 3 +- ci/update-sha.sh | 116 ++++++++++++++++++++++++ code-server/Dockerfile | 3 +- fcrepo6/Dockerfile | 7 +- fcrepo6/README.md | 2 +- fits/Dockerfile | 10 +- nginx/Dockerfile | 1 + renovate.json | 80 ++++++++++++++++ solr/Dockerfile | 2 + tomcat/Dockerfile | 3 +- 14 files changed, 282 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/validate-renovate.yml create mode 100755 ci/update-sha.sh create mode 100644 renovate.json diff --git a/.github/workflows/validate-renovate.yml b/.github/workflows/validate-renovate.yml new file mode 100644 index 00000000..4958b154 --- /dev/null +++ b/.github/workflows/validate-renovate.yml @@ -0,0 +1,21 @@ +name: validate renovate.json + +on: + pull_request: + +env: + LOG_LEVEL: debug + +jobs: + renovate-config-validator: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4 + with: + node-version: 20 + + - run: npx -p renovate renovate-config-validator renovate.json diff --git a/README.md b/README.md index 36c460d2..84be80f1 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ - [Docker Images](#docker-images) - [Updating Dependencies](#updating-dependencies) - [Updating Composer](#updating-composer) + - [Renovate](#renovate) - [Updating Configuration](#updating-configuration) - [Design Considerations](#design-considerations) - [Confd](#confd) @@ -518,6 +519,42 @@ shasum -a 256 ${ALPACA_FILE} > N.B. Please read the release notes the new version and account for any changes > to configuration that are required, as well as test locally. +#### Renovate + +Several dependencies in this repo can be automatically updating using [renovate](https://www.mend.io/renovate/). + +Currently these docker images have some depenencies managed by renovate: + +``` +activemq +blazegraph +cantaloupe +code-server +fcrepo6 +fits +nginx +solr +tomcat +``` + +Since renovate does not natively support the ability to extract a sha256 from a file, we need [a custom shell script](./ci/update-sha.sh) in the [postUpgradeTasks](https://docs.renovatebot.com/configuration-options/#postupgradetasks) to calculate the sha256 of our files and update our Dockerfile accordingly. + +Post upgrade tasks can only run on self-hosted Renovate instances, so this forces us to run renovate on a properly configured runner (instead of using mend.io's free GitHub app to manage our dependencies). Getting renovate setup locally looks like + +``` +npm install -g renovate +export RENOVATE_REPOSITORIES=islandora-devops/isle-buildkit +export RENOVATE_TOKEN=your-gh-pat +export RENOVATE_ALLOWED_POST_UPGRADE_COMMANDS='["bash ci/update-sha.sh \"{{{depName}}}\" \"{{{currentVersion}}}\" \"{{{newVersion}}}\""]' + +# this will issue a PR with any updates to +# our docker images that have renovate configured properly +renovate --platform=github +``` + +Versions listed in GitHub tags or releases can use [advanced capture](https://docs.renovatebot.com/modules/manager/regex/#advanced-capture) in the Dockerfile to update the pinned version. + + #### Updating Composer A number of images like [crayfish] provide a `composer.lock` file to pin them to diff --git a/activemq/Dockerfile b/activemq/Dockerfile index 6538c8e2..75a39d16 100644 --- a/activemq/Dockerfile +++ b/activemq/Dockerfile @@ -2,7 +2,8 @@ FROM java ARG TARGETARCH -ARG ACTIVEMQ_VERSION="5.18.5" +# renovate: datasource=github-tags depName=apache-activemq packageName=apache/activemq +ARG ACTIVEMQ_VERSION=5.18.5 ARG ACTIVEMQ_FILE="apache-activemq-${ACTIVEMQ_VERSION}-bin.tar.gz" ARG ACTIVEMQ_URL="https://archive.apache.org/dist/activemq/${ACTIVEMQ_VERSION}/${ACTIVEMQ_FILE}" ARG ACTIVEMQ_FILE_SHA256="1190b809a35ceb5fb0273f4a47d4584684d75c43360942be10d9af1fc0a39394" diff --git a/blazegraph/Dockerfile b/blazegraph/Dockerfile index 2f1a90e6..2198a604 100644 --- a/blazegraph/Dockerfile +++ b/blazegraph/Dockerfile @@ -6,7 +6,9 @@ ARG BLAZEGRAPH_VERSION="CANDIDATE_2_1_5" ARG BLAZEGRAPH_FILE="blazegraph.war" ARG BLAZEGRAPH_URL="https://github.com/blazegraph/database/releases/download/BLAZEGRAPH_RELEASE_${BLAZEGRAPH_VERSION}/${BLAZEGRAPH_FILE}" ARG BLAZEGRAPH_SHA256="b22f1a1aa8e536443db9a57da63720813374ef59e4021cfa9ad0e98f9a420e85" -ARG LOG4J_VERSION="2.22.0" + +# renovate: datasource=github-releases depName=apache-log4j packageName=apache/logging-log4j2 +ARG LOG4J_VERSION=2.22.0 ARG LOG4J_FILE="apache-log4j-${LOG4J_VERSION}-bin.zip" ARG LOG4J_URL="https://archive.apache.org/dist/logging/log4j/${LOG4J_VERSION}/${LOG4J_FILE}" ARG LOG4J_FILE_SHA256="c6d61ecf2563b1200e02587b89b7c75b58b6e62e6a16cdb6f333c2482167c2dc" diff --git a/cantaloupe/Dockerfile b/cantaloupe/Dockerfile index 947e728b..6d001bc1 100644 --- a/cantaloupe/Dockerfile +++ b/cantaloupe/Dockerfile @@ -2,7 +2,8 @@ FROM java ARG TARGETARCH -ARG CANTALOUPE_VERSION="5.0.6" +# renovate: datasource=github-releases depName=cantaloupe packageName=cantaloupe-project/cantaloupe +ARG CANTALOUPE_VERSION=5.0.6 ARG CANTALOUPE_FILE="cantaloupe-${CANTALOUPE_VERSION}.zip" ARG CANTALOUPE_URL="https://github.com/cantaloupe-project/cantaloupe/releases/download/v${CANTALOUPE_VERSION}/${CANTALOUPE_FILE}" ARG CANTALOUPE_SHA256="35311eb0d4d6f0578cab42fd5e51d6150e62821cb3b4ee3a265e2befbeeb5897" diff --git a/ci/update-sha.sh b/ci/update-sha.sh new file mode 100755 index 00000000..c4d0aa8d --- /dev/null +++ b/ci/update-sha.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash + +set -eou pipefail + +DEP=$1 +OLD_VERSION=$2 +NEW_VERSION=$3 +URL="" +ARG="" +DOCKERFILES=() +README="" + +echo "Updating SHA for $DEP@$NEW_VERSION" + +if [ "$DEP" = "apache-tomcat" ]; then + URL="https://downloads.apache.org/tomcat/tomcat-9/v$NEW_VERSION/bin/apache-tomcat-$NEW_VERSION.tar.gz" + ARG="TOMCAT_FILE_SHA256" + DOCKERFILES=("tomcat/Dockerfile") + README="tomcat/README.md" + +elif [ "$DEP" = "apache-activemq" ]; then + URL="https://downloads.apache.org/activemq/$NEW_VERSION/apache-activemq-$NEW_VERSION-bin.tar.gz" + ARG="ACTIVEMQ_FILE_SHA256" + DOCKERFILES=("activemq/Dockerfile") + README="activemq/README.md" + +elif [ "$DEP" = "apache-solr" ]; then + URL="https://downloads.apache.org/solr/solr/$NEW_VERSION/solr-$NEW_VERSION.tgz" + ARG="SOLR_FILE_SHA256" + DOCKERFILES=("solr/Dockerfile") + README="solr/README.md" + +elif [ "$DEP" = "custom-composer" ]; then + URL="https://getcomposer.org/download/${NEW_VERSION}/composer.phar" + ARG="COMPOSER_SHA256" + DOCKERFILES=("nginx/Dockerfile") + +elif [ "$DEP" = "solr-ocrhighlighting" ]; then + URL=https://github.com/dbmdz/solr-ocrhighlighting/releases/download/${NEW_VERSION}/solr-ocrhighlighting-${NEW_VERSION}.jar + ARG="OCRHIGHLIGHT_FILE_SHA256" + DOCKERFILES=("solr/Dockerfile") + +elif [ "$DEP" = "alpine-pkg-glibc" ]; then + URL="https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${NEW_VERSION}/glibc-${NEW_VERSION}.apk" + ARG="GLIBC_SHA256" + DOCKERFILES=("code-server/Dockerfile") + +elif [ "$DEP" = "fcrepo" ]; then + URL="https://github.com/fcrepo/fcrepo/releases/download/fcrepo-${NEW_VERSION}/fcrepo-webapp-${NEW_VERSION}.war" + ARG="FCREPO_SHA256" + DOCKERFILES=("fcrepo6/Dockerfile") + README="fcrepo6/README.md" + +elif [ "$DEP" = "islandora-syn" ]; then + URL="https://github.com/Islandora/Syn/releases/download/v${NEW_VERSION}/islandora-syn-${NEW_VERSION}-all.jar" + ARG="SYN_SHA256" + DOCKERFILES=("fcrepo6/Dockerfile") + +elif [ "$DEP" = "fcrepo-import-export" ]; then + URL="https://github.com/fcrepo-exts/fcrepo-import-export/releases/download/fcrepo-import-export-${NEW_VERSION}/fcrepo-import-export-${NEW_VERSION}.jar" + ARG="IMPORT_EXPORT_SHA256" + DOCKERFILES=("fcrepo6/Dockerfile") + +elif [ "$DEP" = "fcrepo-upgrade-utils" ]; then + URL="https://github.com/fcrepo-exts/fcrepo-upgrade-utils/releases/download/fcrepo-upgrade-utils-${NEW_VERSION}/fcrepo-upgrade-utils-${NEW_VERSION}.jar" + ARG="UPGRADE_UTILS_SHA256" + DOCKERFILES=("fcrepo6/Dockerfile") + +elif [ "$DEP" = "cantaloupe" ]; then + URL="https://github.com/cantaloupe-project/cantaloupe/releases/download/v${NEW_VERSION}/cantaloupe-${NEW_VERSION}.zip" + ARG="CANTALOUPE_SHA256" + DOCKERFILES=("cantaloupe/Dockerfile") + README="cantaloupe/README.md" + +elif [ "$DEP" = "fits-servlet" ]; then + URL="https://github.com/harvard-lts/FITSservlet/releases/download/${NEW_VERSION}/fits-service-${NEW_VERSION}.war" + ARG="FITSSERVLET_SHA256" + DOCKERFILES=("fits/Dockerfile") + +elif [ "$DEP" = "fits" ]; then + URL="https://github.com/harvard-lts/fits/releases/download/${NEW_VERSION}/fits-${NEW_VERSION}.zip" + ARG="FITS_SHA256" + DOCKERFILES=("fits/Dockerfile") + README="fits/README.md" + +elif [ "$DEP" = "apache-log4j" ]; then + URL="https://archive.apache.org/dist/logging/log4j/${NEW_VERSION}/apache-log4j-${NEW_VERSION}-bin.zip" + ARG="LOG4J_FILE_SHA256" + DOCKERFILES=( + "blazegraph/Dockerfile" + "fits/Dockerfile" + ) + +else + echo "DEP not found" + exit 0 +fi + +# update the Dockerfile(s) SHA256 with the file we're downloading +SHA=$(curl -s "$URL" \ + | shasum -a 256 \ + | awk '{print $1}') +if [[ "$OSTYPE" == "darwin"* ]]; then + sed -i '' 's|^ARG '"$ARG"'=.*|ARG '"$ARG"'="'"$SHA"'"|g' "${DOCKERFILES[@]}" +else + sed -i 's|^ARG '"$ARG"'=.*|ARG '"$ARG"'="'"$SHA"'"|g' "${DOCKERFILES[@]}" +fi + +# update the README to specify the new version +if [ "$README" != "" ]; then + if [[ "$OSTYPE" == "darwin"* ]]; then + sed -i '' "s/${OLD_VERSION}\.$/${NEW_VERSION}\./" "$README" + else + sed -i "s/${OLD_VERSION}\.$/${NEW_VERSION}\./" "$README" + fi +fi diff --git a/code-server/Dockerfile b/code-server/Dockerfile index 64243b7f..9dd2fc28 100644 --- a/code-server/Dockerfile +++ b/code-server/Dockerfile @@ -4,7 +4,8 @@ FROM drupal ARG TARGETARCH -ARG GLIBC_VERSION="2.35-r1" +# renovate: datasource=github-releases depName=alpine-pkg-glibc packageName=sgerrand/alpine-pkg-glibc +ARG GLIBC_VERSION=2.35-r1 ARG GLIBC_FILE="glibc-${GLIBC_VERSION}.apk" ARG GLIBC_URL="https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/${GLIBC_FILE}" ARG GLIBC_SHA256="276f43ce9b2d5878422bca94ca94e882a7eb263abe171d233ac037201ffcaf06" diff --git a/fcrepo6/Dockerfile b/fcrepo6/Dockerfile index b22e264f..2417abce 100644 --- a/fcrepo6/Dockerfile +++ b/fcrepo6/Dockerfile @@ -2,21 +2,26 @@ FROM tomcat ARG TARGETARCH + +# renovate: datasource=github-releases depName=fcrepo packageName=fcrepo/fcrepo ARG FCREPO_VERSION=6.5.0 ARG FCREPO_FILE=fcrepo-webapp-${FCREPO_VERSION}.war ARG FCREPO_URL=https://github.com/fcrepo/fcrepo/releases/download/fcrepo-${FCREPO_VERSION}/${FCREPO_FILE} ARG FCREPO_SHA256=dd0ac889f253dbe9fb09aa3f1fe4444fd57b51af34ebaaf23daec72fbdc5cf55 +# renovate: datasource=github-releases depName=islandora-syn packageName=Islandora/Syn ARG SYN_VERSION=1.1.1 ARG SYN_FILE=islandora-syn-${SYN_VERSION}-all.jar -ARG SYN_URL=https://github.com/Islandora-CLAW/Syn/releases/download/v${SYN_VERSION}/${SYN_FILE} +ARG SYN_URL=https://github.com/Islandora/Syn/releases/download/v${SYN_VERSION}/${SYN_FILE} ARG SYN_SHA256=24c589d16d579c48361d14c346e0d633a6985642f1c116295ed91eb32b283085 +# renovate: datasource=github-releases depName=fcrepo-import-export packageName=fcrepo-exts/fcrepo-import-export ARG IMPORT_EXPORT_VERSION=1.2.0 ARG IMPORT_EXPORT_FILE=fcrepo-import-export-${IMPORT_EXPORT_VERSION}.jar ARG IMPORT_EXPORT_URL=https://github.com/fcrepo-exts/fcrepo-import-export/releases/download/fcrepo-import-export-${IMPORT_EXPORT_VERSION}/${IMPORT_EXPORT_FILE} ARG IMPORT_EXPORT_SHA256=e767f43b0b4f639042b0e399ebfebdf61957515d585b7f0d2d8aeb9f877b12a0 +# renovate: datasource=github-releases depName=fcrepo-upgrade-utils packageName=fcrepo-exts/fcrepo-upgrade-utils ARG UPGRADE_UTILS_VERSION=6.3.0 ARG UPGRADE_UTILS_FILE=fcrepo-upgrade-utils-${UPGRADE_UTILS_VERSION}.jar ARG UPGRADE_UTILS_URL=https://github.com/fcrepo-exts/fcrepo-upgrade-utils/releases/download/fcrepo-upgrade-utils-${UPGRADE_UTILS_VERSION}/${UPGRADE_UTILS_FILE} diff --git a/fcrepo6/README.md b/fcrepo6/README.md index e93f5961..5e3398e9 100644 --- a/fcrepo6/README.md +++ b/fcrepo6/README.md @@ -1,6 +1,6 @@ # Fcrepo -Docker image for [fcrepo] version 6.5.0 +Docker image for [fcrepo] version 6.5.0. Please refer to the [Fcrepo Documentation] for more in-depth information. diff --git a/fits/Dockerfile b/fits/Dockerfile index 3a2885bf..ebce7387 100644 --- a/fits/Dockerfile +++ b/fits/Dockerfile @@ -2,17 +2,21 @@ FROM tomcat ARG TARGETARCH -ARG FITSSERVLET_VERSION="2.1.0" + +# renovate: datasource=github-releases depName=fits-servlet packageName=harvard-lts/FITSservlet +ARG FITSSERVLET_VERSION=2.1.0 ARG FITSSERVLET_FILE="fits-service-${FITSSERVLET_VERSION}.war" ARG FITSSERVLET_URL="https://github.com/harvard-lts/FITSservlet/releases/download/${FITSSERVLET_VERSION}/${FITSSERVLET_FILE}" ARG FITSSERVLET_SHA256="ba3c4d5af4dfbe42a32b4321eb96fad8db1379c8b00cd2bc7b7b0e4f59cb7e67" -ARG FITS_VERSION="1.6.0" +# renovate: datasource=github-releases depName=fits packageName=harvard-lts/fits +ARG FITS_VERSION=1.6.0 ARG FITS_FILE="fits-${FITS_VERSION}.zip" ARG FITS_URL="https://github.com/harvard-lts/fits/releases/download/${FITS_VERSION}/${FITS_FILE}" ARG FITS_SHA256="32e436effe7251c5b067ec3f02321d5baf4944b3f0d1010fb8ec42039d9e3b73" -ARG LOG4J_VERSION="2.23.1" +# renovate: datasource=github-releases depName=apache-log4j packageName=apache/logging-log4j2 +ARG LOG4J_VERSION=2.23.1 ARG LOG4J_FILE="apache-log4j-${LOG4J_VERSION}-bin.zip" ARG LOG4J_URL="https://archive.apache.org/dist/logging/log4j/${LOG4J_VERSION}/${LOG4J_FILE}" ARG LOG4J_FILE_SHA256="c53645414ce68f21dd347439df5852ad16b38cd7227a9d417ebaf411a1c09501" diff --git a/nginx/Dockerfile b/nginx/Dockerfile index a901f7de..be65ab71 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -2,6 +2,7 @@ FROM base ARG TARGETARCH +# renovate: datasource=github-releases depName=custom-composer packageName=composer/composer ARG COMPOSER_VERSION=2.8.2 ARG COMPOSER_FILE=composer.phar ARG COMPOSER_URL=https://getcomposer.org/download/${COMPOSER_VERSION}/${COMPOSER_FILE} diff --git a/renovate.json b/renovate.json new file mode 100644 index 00000000..0f288d41 --- /dev/null +++ b/renovate.json @@ -0,0 +1,80 @@ +{ + "extends": [ + "config:best-practices", + ":rebaseStalePrs" + ], + "enabledManagers": [ + "custom.regex" + ], + "packageRules": [ + { + "matchPackageNames": [ + "apache/activemq", + "apache/logging-log4j2", + "fcrepo/fcrepo", + "fcrepo-exts/fcrepo-import-export", + "fcrepo-exts/fcrepo-upgrade-utils" + ], + "extractVersion": "^(fcrepo|activemq|rel|fcrepo-import-export|fcrepo-upgrade-utils)(\/|-)(?\\d+\\.\\d+\\.\\d+)$", + "versioning": "semver" + }, + { + "packagePatterns": ["^apache/solr$"], + "extractVersion": "^releases/solr/(?\\d+\\.\\d+\\.\\d+)$", + "versioning": "semver" + }, + { + "matchUpdateTypes": [ + "major" + ], + "enabled": false + }, + { + "groupName": "all non-major dependencies", + "matchUpdateTypes": [ + "minor", + "patch", + "digest" + ], + "matchDepNames": [ + "alpine-pkg-glibc", + "apache-activemq", + "apache-log4j", + "apache-solr", + "apache-tomcat", + "cantaloupe", + "custom-composer", + "fcrepo", + "fcrepo-import-export", + "fcrepo-upgrade-utils", + "fits", + "fits-servlet", + "islandora-syn", + "solr-ocrhighlighting" + ], + "postUpgradeTasks": { + "commands": [ + "bash ci/update-sha.sh \"{{{depName}}}\" \"{{{currentVersion}}}\" \"{{{newVersion}}}\"" + ] + } + } + ], + "customManagers": [ + { + "customType": "regex", + "description": "Update _VERSION variables in Dockerfiles", + "fileMatch": ["(^|/|\\.)Dockerfile$", "(^|/)Dockerfile\\.[^/]*$"], + "matchStrings": [ + "# renovate: datasource=(?[a-z-]+?)(?: depName=(?.+?))? packageName=(?.+?)(?: versioning=(?[a-z-]+?))?\\s(?:ENV|ARG) .+?_VERSION=(?.+?)(\\s|$)" + ] + } + ], + "labels": [ + "dependencies" + ], + "osvVulnerabilityAlerts": true, + "dependencyDashboardOSVVulnerabilitySummary": "unresolved", + "vulnerabilityAlerts": { + "enabled": true + } +} diff --git a/solr/Dockerfile b/solr/Dockerfile index 130dc741..8e32653e 100644 --- a/solr/Dockerfile +++ b/solr/Dockerfile @@ -2,10 +2,12 @@ FROM java ARG TARGETARCH +# renovate: datasource=github-tags depName=apache-solr packageName=apache/solr ARG SOLR_VERSION=9.5.0 ARG SOLR_FILE=solr-${SOLR_VERSION}.tgz ARG SOLR_URL=https://archive.apache.org/dist/solr/solr/${SOLR_VERSION}/solr-${SOLR_VERSION}.tgz ARG SOLR_FILE_SHA256=d8538502019af1945e0b124a4613b46ca43aedcf3f20e9912c482c080407ea21 +# renovate: datasource=github-releases depName=solr-ocrhighlighting packageName=dbmdz/solr-ocrhighlighting ARG OCRHIGHLIGHT_VERSION=0.9.0 ARG OCRHIGHLIGHT_FILE=solr-ocrhighlighting-${OCRHIGHLIGHT_VERSION}.jar ARG OCRHIGHLIGHT_URL=https://github.com/dbmdz/solr-ocrhighlighting/releases/download/${OCRHIGHLIGHT_VERSION}/solr-ocrhighlighting-${OCRHIGHLIGHT_VERSION}.jar diff --git a/tomcat/Dockerfile b/tomcat/Dockerfile index e5e90e31..aa863857 100644 --- a/tomcat/Dockerfile +++ b/tomcat/Dockerfile @@ -2,7 +2,8 @@ FROM java ARG TARGETARCH -ARG TOMCAT_VERSION="9.0.98" +# renovate: datasource=github-tags depName=apache-tomcat packageName=apache/tomcat +ARG TOMCAT_VERSION=9.0.98 ARG TOMCAT_FILE_SHA256="1d9a1104c2e235a5baff6e9ca8e28be3d864803fafc478f5c13c2ae6a5ed3d6f" EXPOSE 8080