Skip to content

Commit

Permalink
Merge remote-tracking branch 'IQSS/develop' into QDR-CVoc_updates_for…
Browse files Browse the repository at this point in the history
…_author
  • Loading branch information
qqmyers committed Sep 25, 2024
2 parents b8878b0 + d40ce32 commit f1218a3
Show file tree
Hide file tree
Showing 128 changed files with 5,937 additions and 1,373 deletions.
37 changes: 37 additions & 0 deletions .github/actions/setup-maven/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
---
name: "Setup Maven and Caches"
description: "Determine Java version and setup Maven, including necessary caches."
inputs:
git-reference:
description: 'The git reference (branch/tag) to check out'
required: false
default: '${{ github.ref }}'
pom-paths:
description: "List of paths to Maven POM(s) for cache dependency setup"
required: false
default: 'pom.xml'
runs:
using: composite
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ inputs.git-reference }}
- name: Determine Java version by reading the Maven property
shell: bash
run: |
echo "JAVA_VERSION=$(grep '<target.java.version>' ${GITHUB_WORKSPACE}/modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" | tee -a ${GITHUB_ENV}
- name: Set up JDK ${{ env.JAVA_VERSION }}
id: setup-java
uses: actions/setup-java@v4
with:
java-version: ${{ env.JAVA_VERSION }}
distribution: 'temurin'
cache: 'maven'
cache-dependency-path: ${{ inputs.pom-paths }}
- name: Download common cache on branch cache miss
if: ${{ steps.setup-java.outputs.cache-hit != 'true' }}
uses: actions/cache/restore@v4
with:
key: dataverse-maven-cache
path: ~/.m2/repository
41 changes: 27 additions & 14 deletions .github/workflows/container_app_push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@ on:
# We are deliberately *not* running on push events here to avoid double runs.
# Instead, push events will trigger from the base image and maven unit tests via workflow_call.
workflow_call:
inputs:
base-image-ref:
type: string
description: "Reference of the base image to build on in full qualified form [<registry>/]<namespace>/<repo>:<tag>"
required: false
default: "gdcc/base:unstable"
pull_request:
branches:
- develop
Expand All @@ -16,7 +22,6 @@ on:

env:
IMAGE_TAG: unstable
BASE_IMAGE_TAG: unstable
REGISTRY: "" # Empty means default to Docker Hub
PLATFORMS: "linux/amd64,linux/arm64"
MASTER_BRANCH_TAG: alpha
Expand All @@ -33,20 +38,24 @@ jobs:
if: ${{ github.repository_owner == 'IQSS' }}

steps:
- name: Checkout repository
uses: actions/checkout@v3

- name: Set up JDK
uses: actions/setup-java@v3
- name: Checkout and Setup Maven
uses: IQSS/dataverse/.github/actions/setup-maven@develop
with:
java-version: "17"
distribution: temurin
cache: maven
pom-paths: |
pom.xml
modules/container-configbaker/pom.xml
modules/dataverse-parent/pom.xml
# TODO: Add a filter step here, that avoids building the image if this is a PR and there are other files touched than declared above.
# Use https://github.com/dorny/paths-filter to solve this. This will ensure we do not run this twice if this workflow
# will be triggered by the other workflows already (base image or java changes)
# To become a part of #10618.

- name: Build app and configbaker container image with local architecture and submodules (profile will skip tests)
run: >
mvn -B -f modules/dataverse-parent
-P ct -pl edu.harvard.iq:dataverse -am
$( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" )
install
# TODO: add smoke / integration testing here (add "-Pct -DskipIntegrationTests=false")
Expand Down Expand Up @@ -106,11 +115,13 @@ jobs:
if: needs.check-secrets.outputs.available == 'true' &&
( github.event_name != 'push' || ( github.event_name == 'push' && contains(fromJSON('["develop", "master"]'), github.ref_name)))
steps:
- uses: actions/checkout@v3
- uses: actions/setup-java@v3
- name: Checkout and Setup Maven
uses: IQSS/dataverse/.github/actions/setup-maven@develop
with:
java-version: "17"
distribution: temurin
pom-paths: |
pom.xml
modules/container-configbaker/pom.xml
modules/dataverse-parent/pom.xml
# Depending on context, we push to different targets. Login accordingly.
- if: github.event_name != 'pull_request'
Expand Down Expand Up @@ -146,11 +157,13 @@ jobs:
run: >
mvn -B -f modules/dataverse-parent
-P ct -pl edu.harvard.iq:dataverse -am
$( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" )
install
- name: Deploy multi-arch application and configbaker container image
run: >
mvn
-Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }}
-Dapp.image.tag=${{ env.IMAGE_TAG }}
$( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" )
${{ env.REGISTRY }} -Ddocker.platforms=${{ env.PLATFORMS }}
-P ct deploy
Expand Down
153 changes: 92 additions & 61 deletions .github/workflows/container_base_push.yml
Original file line number Diff line number Diff line change
@@ -1,99 +1,130 @@
---
name: Base Container Image
name: Container Images Releasing

on:
push:
tags:
- 'v[6-9].**'
branches:
- 'develop'
- 'master'
# "Path filters are not evaluated for pushes of tags" https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#onpushpull_requestpull_request_targetpathspaths-ignore
paths:
- 'modules/container-base/**'
- '!modules/container-base/src/backports/**'
- '!modules/container-base/README.md'
- 'modules/dataverse-parent/pom.xml'
- '.github/workflows/container_base_push.yml'
pull_request:
branches:
- 'develop'
- 'master'
paths:
- 'modules/container-base/**'
- 'modules/dataverse-parent/pom.xml'
- '.github/workflows/container_base_push.yml'
schedule:
- cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC

# These TODOs are left for #10618
# TODO: we are missing a workflow_call option here, so we can trigger this flow from pr comments and maven tests (keep the secrets availability in mind!)
# TODO: we are missing a pull_request option here (filter for stuff that would trigger the maven runs!) so we can trigger preview builds for them when coming from the main repo (keep the secrets availability in mind!)

env:
IMAGE_TAG: unstable
PLATFORMS: linux/amd64,linux/arm64
DEVELOPMENT_BRANCH: develop

jobs:
build:
name: Build image
name: Base Image
runs-on: ubuntu-latest
permissions:
contents: read
packages: read
strategy:
matrix:
jdk: [ '17' ]
# Only run in upstream repo - avoid unnecessary runs in forks
if: ${{ github.repository_owner == 'IQSS' }}
outputs:
base-image-ref: ${{ steps.finalize.outputs.base-image-ref }}

steps:
- name: Checkout repository
uses: actions/checkout@v3

- name: Set up JDK ${{ matrix.jdk }}
uses: actions/setup-java@v3
- name: Checkout and Setup Maven
uses: IQSS/dataverse/.github/actions/setup-maven@develop
with:
java-version: ${{ matrix.jdk }}
distribution: 'adopt'
- name: Cache Maven packages
uses: actions/cache@v3
with:
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-m2

- name: Build base container image with local architecture
run: mvn -f modules/container-base -Pct package
pom-paths: modules/container-base/pom.xml

# Run anything below only if this is not a pull request.
# Accessing, pushing tags etc. to DockerHub will only succeed in upstream because secrets.

- if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }}
name: Push description to DockerHub
uses: peter-evans/dockerhub-description@v3
# Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and
# on events in context of upstream because secrets. PRs run in context of forks by default!
- name: Log in to the Container registry
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
repository: gdcc/base
short-description: "Dataverse Base Container image providing Payara application server and optimized configuration"
readme-filepath: ./modules/container-base/README.md

- if: ${{ github.event_name != 'pull_request' }}
name: Log in to the Container registry
uses: docker/login-action@v2
# In case this is a push to develop, we care about buildtime.
# Configure a remote ARM64 build host in addition to the local AMD64 in two steps.
- name: Setup SSH agent
if: ${{ github.event_name != 'schedule' }}
uses: webfactory/ssh-agent@v0.9.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- if: ${{ github.event_name != 'pull_request' }}
name: Set up QEMU for multi-arch builds
uses: docker/setup-qemu-action@v2
- name: Re-set image tag based on branch
if: ${{ github.ref_name == 'master' }}
run: echo "IMAGE_TAG=alpha" >> $GITHUB_ENV
- if: ${{ github.event_name != 'pull_request' }}
name: Deploy multi-arch base container image to Docker Hub
run: mvn -f modules/container-base -Pct deploy -Dbase.image.tag=${{ env.IMAGE_TAG }} -Ddocker.platforms=${{ env.PLATFORMS }}
ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }}
- name: Provide the known hosts key and the builder config
if: ${{ github.event_name != 'schedule' }}
run: |
echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts
mkdir -p modules/container-base/target/buildx-state/buildx/instances
cat > modules/container-base/target/buildx-state/buildx/instances/maven << EOF
{ "Name": "maven",
"Driver": "docker-container",
"Dynamic": false,
"Nodes": [{"Name": "maven0",
"Endpoint": "unix:///var/run/docker.sock",
"Platforms": [{"os": "linux", "architecture": "amd64"}],
"DriverOpts": null,
"Flags": ["--allow-insecure-entitlement=network.host"],
"Files": null},
{"Name": "maven1",
"Endpoint": "ssh://${{ secrets.BUILDER_ARM64_SSH_CONNECTION }}",
"Platforms": [{"os": "linux", "architecture": "arm64"}],
"DriverOpts": null,
"Flags": ["--allow-insecure-entitlement=network.host"],
"Files": null}]}
EOF
# Determine the base image name we are going to use from here on
- name: Determine base image name
run: |
if [[ "${{ github.ref_name }}" = "${{ env.DEVELOPMENT_BRANCH }}" ]]; then
echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" | tee -a "${GITHUB_ENV}"
echo "BASE_IMAGE_UPCOMING=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}"
else
echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}"
fi
- name: Calculate revision number for immutable tag (on release branches only)
if: ${{ github.ref_name != env.DEVELOPMENT_BRANCH }}
id: revision-tag
uses: ./.github/actions/get-image-revision
with:
image-ref: ${{ env.BASE_IMAGE }}
tag-options-prefix: "-Dbase.image.tag.suffix='' -Ddocker.tags.revision="
- name: Configure update of "latest" tag for development branch
id: develop-tag
if: ${{ github.ref_name == env.DEVELOPMENT_BRANCH }}
run: |
echo "tag-options=-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=${BASE_IMAGE_UPCOMING#*:}" | tee -a "${GITHUB_OUTPUT}"
- name: Deploy multi-arch base container image to Docker Hub
id: build
run: |
mvn -f modules/container-base -Pct deploy -Ddocker.noCache -Ddocker.platforms=${{ env.PLATFORMS }} \
-Ddocker.imagePropertyConfiguration=override ${{ steps.develop-tag.outputs.tag-options }} ${{ steps.revision-tag.outputs.tag-options }}
- name: Determine appropriate base image ref for app image
id: finalize
run: |
if [[ "${{ github.ref_name }}" = "${{ env.DEVELOPMENT_BRANCH }}" ]]; then
echo "base-image-ref=${BASE_IMAGE_UPCOMING}" | tee -a "$GITHUB_OUTPUT"
else
echo "base-image-ref=gdcc/base:${{ steps.revision-tag.outputs.revision-tag }}" | tee -a "$GITHUB_OUTPUT"
fi
push-app-img:
name: "Rebase & Publish App Image"
permissions:
contents: read
packages: write
pull-requests: write
needs: build
# We do not release a new base image for pull requests, so do not trigger.
if: ${{ github.event_name != 'pull_request' }}
uses: ./.github/workflows/container_app_push.yml
secrets: inherit
needs:
- build
uses: ./.github/workflows/container_app_push.yml
with:
base-image-ref: ${{ needs.build.outputs.base-image-ref }}
Loading

0 comments on commit f1218a3

Please sign in to comment.