sv_pipeline_docker.yml

name: Docker Images

on:
  push:
    branches:
      - main
    paths:
      - 'src/**'
      - 'dockerfiles/**'
      - 'scripts/docker/build_docker.py'
      - '.github/workflows/sv_pipeline_docker.yml'
  pull_request:
    branches:
      - main
    paths:
      - 'src/**'
      - 'dockerfiles/**'
      - 'scripts/docker/build_docker.py'
      - '.github/workflows/sv_pipeline_docker.yml'

jobs:
  build_args_job:
    runs-on: ubuntu-20.04
    name: Determine Build Args
    env:
      GITHUB_CONTEXT: ${{ toJson(github) }}
    outputs:
      base_sha: ${{ steps.commit_sha.outputs.BASE_SHA }}
      head_sha: ${{ steps.commit_sha.outputs.HEAD_SHA }}
      image_tag: ${{ steps.image_tag.outputs.IMAGE_TAG }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v3
        with:
          # By default, this checks out only the current commit;
          # however, since a diff between the current commit and
          # the base commit is required to determined which docker
          # images to rebuild, we use the following to check out
          # the complete git history.
          fetch-depth: 0

      - name: Determine Commit SHAs
        id: commit_sha
        # This action determines the SHA of two commits:
        # - BASE (BASE_SHA): The commit SHA of the base branch (e.g.,
        #   broadinstitute/gatk-sv:main) which the feature branch targets.
        # - HEAD (HEAD_SHA): The commit SHA of the latest commit on the
        #   feature branch.
        #
        # In the following example, BASE_SHA=B and HEAD_SHA=Z
        #
        #     	    X---Y---Z     feature
        #          /         \
        #     A---B---C---D---E   main
        #
        # 'E' is the merge commit (e.g., 'Merge pull request #0').
        #
        # This action can be invoked as a result of (a) pushing commits X,
        # Y, or Z, or (b) pushing merge commit E (i.e., merging the PR).
        # Depending on (a) and (b) the commit SHAs are determined differently.
        # In case of (a), the commit SHAs are extracted from the
        # 'event.pull_request' key in the github's context JSON. In case of
        # (b), the commit SHAs are extracted from the list of commits recorded
        # under the 'event' key.
        #
        # Note: Github's context JSON is printed in the action's debug page.
        #
        run: |
          echo "::debug::EVENT_NAME: ${{ github.event_name }}"
          if [[ ${{ github.event_name }} == "pull_request" ]]; then
            BASE_SHA=${{ github.event.pull_request.base.sha }}
            HEAD_SHA=${{ github.event.pull_request.head.sha }}
          else
            BASE_SHA=${{ github.event.before }}
            HEAD_SHA=$(echo "$GITHUB_CONTEXT" | jq '.event.commits[].id' | tail -2 | head -1 | sed 's/\"//g')
          fi

          echo "::debug::BASE_SHA: $BASE_SHA"
          echo "::debug::HEAD_SHA: $HEAD_SHA"

          # Avail the determined commit SHAs to other steps.
          echo "BASE_SHA=$BASE_SHA" >> $GITHUB_OUTPUT
          echo "HEAD_SHA=$HEAD_SHA" >> $GITHUB_OUTPUT

      - name: Compose Image Tag
        id: image_tag
        # This step composes a tag to be used for all the images created by
        # the build_docker.py script. The tag follows the following template:
        #
        #   DATE-RELEAST_TAG-HEAD_SHA_8
        #
        # where 'DATE' is YYYY-MM-DD extracted from the time stamp of the last
        # commit on the feature branch (HEAD), `RELEASE_TAG` is extracted from
        # the latest [pre-]release on Github, and the 'HEAD_SHA_8' is the first
        # eight letters of the SHA of the last commit on the feature branch (HEAD).
        run: |          
          # Extract the time stamp of COMMIT_SHA in YYYY-MM-DD format.
          # See git-show documentation available at:
          # http://schacon.github.io/git/git-show
          DATE=$(git show -s --format=%ad --date=format:'%Y-%m-%d' $COMMIT_SHA)
          
          # Get latest [pre-]release tag.
          RELEASE_TAG=$(jq -r '.[0] | .tag_name' <<< $(curl --silent https://api.github.com/repos/broadinstitute/gatk-sv/releases))
          
          COMMIT_SHA=${{ steps.commit_sha.outputs.HEAD_SHA }}
          
          IMAGE_TAG=$DATE-$RELEASE_TAG-${COMMIT_SHA::8}
          echo "::debug::Image tag: $IMAGE_TAG"
          echo "IMAGE_TAG=$IMAGE_TAG" >> $GITHUB_OUTPUT

  build_job:
    runs-on: ubuntu-20.04
    name: Test Images Build
    if: github.event_name == 'pull_request'
    needs: build_args_job
    strategy:
      matrix:
        python-version: ['3.8']
    steps:
      - name: Checkout code
        uses: actions/checkout@v3
        with:
          # See the comment on build_args_job.
          fetch-depth: 0

      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}

      - name: Run build_docker.py
        run: |
          cd ./scripts/docker/
          python build_docker.py \
            --base-git-commit ${{ needs.build_args_job.outputs.base_sha }} \
            --current-git-commit ${{ needs.build_args_job.outputs.head_sha }} \
            --image-tag ${{ needs.build_args_job.outputs.image_tag }} \
            --prune-after-each-image

  publish_job:
    # This job first configures gcloud with the authentication of a
    # service account. It then uses gcloud to configure the docker
    # credentials. Finally, it rebuilds the docker images (targets are
    # determined by the `build_args_job`) and pushes them to GCR.
    name: Publish
    runs-on: ubuntu-20.04
    environment: Deploy
    if: github.event_name == 'push'
    needs: build_args_job
    strategy:
      matrix:
        python-version: ['3.8']
    env:
      DOCKERS_AZURE: "./inputs/values/dockers_azure.json"
      DOCKERS_GCP: "./inputs/values/dockers.json"
    steps:
      - name: Checkout code
        uses: actions/checkout@v3
        with:
          # See the comment on build_args_job.
          fetch-depth: 0
          # Authenticates git using the bot's access token.
          token: ${{ secrets.BOT_PAT }}

      - name: Setup Python
        uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-version }}

      - name: Azure login
        uses: azure/docker-login@v1
        with:
          login-server: ${{ secrets.AZ_CR }}
          username: ${{ secrets.AZ_USERNAME }}
          password: ${{ secrets.AZ_PASSWORD }}

      - name: Setup gcloud CLI
        uses: google-github-actions/setup-gcloud@v0.3.0
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}
          service_account_key: ${{ secrets.GCP_GCR_SA_KEY }}
          # xref: https://github.com/google-github-actions/setup-gcloud#inputs
          # If you need to set `export_default_credentials: true`
          # make sure to pass the `--disable-git-protect` flag to
          # build_docker.py; because the setting export_default_credentials
          # to true will cause creating an untracked file in the root of
          # GATK-SV cloned directory which will cause build_docker.py to
          # raise an exception about the uncommitted file.

      - name: Configure Docker using gcloud
        # This step uses gcloud to configure docker credentials to access GCR
        # using a service account.
        # See: https://cloud.google.com/container-registry/docs/advanced-authentication
        #
        # The build_docker.py uses the `--squash` flag when building the
        # images to be pushed to GCR. This flag is only available when
        # experimental features are enabled, hence the features are enabled
        # in this flag.
        run: |
          gcloud auth configure-docker
          tmp=$(mktemp)
          sudo jq '.+{experimental:true}' /etc/docker/daemon.json > "$tmp"
          sudo mv "$tmp" /etc/docker/daemon.json
          sudo systemctl restart docker.service

      - name: Build and Publish Docker Images to ACR & GCR
        id: build_and_publish
        run: |
          python ./scripts/docker/build_docker.py \
            --base-git-commit ${{ needs.build_args_job.outputs.base_sha }} \
            --current-git-commit ${{ needs.build_args_job.outputs.head_sha }} \
            --docker-repo ${{ secrets.AZ_CR }} us.gcr.io/${{ secrets.GCP_PROJECT_ID }}/gatk-sv \
            --image-tag ${{ needs.build_args_job.outputs.image_tag }} \
            --input-json $DOCKERS_AZURE $DOCKERS_GCP \
            --output-json $DOCKERS_AZURE $DOCKERS_GCP \
            --disable-git-protect \
            --prune-after-each-image
          
          CHANGED=$(git diff --quiet $DOCKERS_GCP || echo True)
          echo "CHANGED=$CHANGED" >> $GITHUB_OUTPUT

      - name: Commit Changes to dockers_*.json
        if: steps.build_and_publish.outputs.CHANGED
        run: |
          COMMIT_SHA=${{ needs.build_args_job.outputs.head_sha }}
          git config --global user.name 'gatk-sv-bot'
          git config --global user.email '101641599+gatk-sv-bot@users.noreply.github.com'
          git commit $DOCKERS_AZURE $DOCKERS_GCP -m "Update docker images list, triggered by "${COMMIT_SHA::8}
          git pull --rebase origin main
          
          # In the following, force-push is required when the above rebase updates the branch;
          # otherwise, the push will be rejected with the following error: 
          # > Updates were rejected because the tip of your current branch is behind ts remote counterpart.
          # See this thread for details: https://stackoverflow.com/q/39399804
          git push -f