From ac90cf38a94267e6b648164a7021d65e792824db Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 27 Jul 2022 16:57:31 +0000 Subject: [PATCH] refactor(ci): refactor Docker build-and-push workflows (#5494) --- .../docker-custom-build-and-push/action.yml | 92 +++++ .github/workflows/docker-unified.yml | 345 +++--------------- 2 files changed, 133 insertions(+), 304 deletions(-) create mode 100644 .github/actions/docker-custom-build-and-push/action.yml diff --git a/.github/actions/docker-custom-build-and-push/action.yml b/.github/actions/docker-custom-build-and-push/action.yml new file mode 100644 index 00000000000000..af172a082f39a7 --- /dev/null +++ b/.github/actions/docker-custom-build-and-push/action.yml @@ -0,0 +1,92 @@ +name: Custom Docker build and push +description: "Build and push a Docker image to Docker Hub" + +inputs: + username: + description: "Docker Hub username" + password: + description: "Docker Hub password" + publish: + description: "Set to true to actually publish the image to Docker Hub" + + context: + description: "Same as docker/build-push-action" + required: false + file: + description: "Same as docker/build-push-action" + required: false + platforms: + description: "Same as docker/build-push-action" + required: false + + images: + # e.g. linkedin/datahub-gms + description: "List of Docker images to use as base name for tags" + required: true + tags: + # e.g. latest,head,sha12345 + description: "List of tags to use for the Docker image" + required: true +outputs: + image_tag: + description: "Docker image tags" + value: ${{ steps.docker_meta.outputs.tags }} + # image_name: ${{ env.DATAHUB_GMS_IMAGE }} + +runs: + using: "composite" + + steps: + - name: Docker meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v1 + with: + # list of Docker images to use as base name for tags + images: ${{ inputs.images }} + # add git short SHA as Docker tag + tag-custom: ${{ inputs.tags }} + tag-custom-only: true + + # Code for testing the build when not pushing to Docker Hub. + - name: Build and Load image for testing (if not publishing) + uses: docker/build-push-action@v2 + if: ${{ inputs.publish != 'true' }} + with: + context: ${{ inputs.context }} + file: ${{ inputs.file }} + # TODO this only does single-platform builds in testing? + # leaving it for now since it matches the previous behavior + platforms: linux/amd64 + tags: ${{ steps.docker_meta.outputs.tags }} + load: true + push: false + - name: Upload image locally for testing (if not publishing) + uses: ishworkh/docker-image-artifact-upload@v1 + if: ${{ inputs.publish != 'true' }} + with: + image: ${{ steps.docker_meta.outputs.tags }} + + # Code for building multi-platform images and pushing to Docker Hub. + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + if: ${{ inputs.publish == 'true' }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + if: ${{ inputs.publish == 'true' }} + - name: Login to DockerHub + uses: docker/login-action@v1 + if: ${{ inputs.publish == 'true' }} + with: + username: ${{ inputs.username }} + password: ${{ inputs.password }} + - name: Build and Push Multi-Platform image + uses: docker/build-push-action@v2 + if: ${{ inputs.publish == 'true' }} + with: + context: ${{ inputs.context }} + file: ${{ inputs.file }} + platforms: ${{ inputs.platforms }} + tags: ${{ steps.docker_meta.outputs.tags }} + push: true + + # TODO add code for vuln scanning? diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index 5d479b0d627217..744203364e8ef2 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -58,6 +58,7 @@ jobs: run: | echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}" echo "::set-output name=publish::${{ env.ENABLE_PUBLISH != '' }}" + gms_build: name: Build and Push DataHub GMS Docker Image runs-on: ubuntu-latest @@ -70,52 +71,18 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + - name: Build and push + uses: ./.github/actions/docker-custom-build-and-push with: - # list of Docker images to use as base name for tags images: | ${{ env.DATAHUB_GMS_IMAGE }} - # add git short SHA as Docker tag - tag-custom: ${{ needs.setup.outputs.tag }} - tag-custom-only: true - - name: Build and Load image for testing (if not publishing) - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - context: . - file: ./docker/datahub-gms/Dockerfile - platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - load: true - push: false - - name: Upload image locally for testing (if not publishing) - uses: ishworkh/docker-image-artifact-upload@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - image: ${{ steps.docker_meta.outputs.tags }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + tags: ${{ needs.setup.outputs.tag }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and Push MultiPlatform image - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + publish: ${{ needs.setup.outputs.publish }} context: . file: ./docker/datahub-gms/Dockerfile platforms: linux/amd64,linux/arm64 - tags: ${{ steps.docker_meta.outputs.tags }} - push: true gms_scan: name: "[Monitoring] Scan GMS images for vulnerabilities" runs-on: ubuntu-latest @@ -134,6 +101,7 @@ jobs: ignore-unfixed: true vuln-type: 'os,library' severity: 'CRITICAL,HIGH' + mae_consumer_build: name: Build and Push DataHub MAE Consumer Docker Image runs-on: ubuntu-latest @@ -146,52 +114,18 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + - name: Build and push + uses: ./.github/actions/docker-custom-build-and-push with: - # list of Docker images to use as base name for tags images: | ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }} - # add git short SHA as Docker tag - tag-custom: ${{ needs.setup.outputs.tag }} - tag-custom-only: true - - name: Build and Load image for testing (if not publishing) - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - context: . - file: ./docker/datahub-mae-consumer/Dockerfile - platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - load: true - push: false - - name: Upload image locally for testing (if not publishing) - uses: ishworkh/docker-image-artifact-upload@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - image: ${{ steps.docker_meta.outputs.tags }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + tags: ${{ needs.setup.outputs.tag }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and Push Platform image - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + publish: ${{ needs.setup.outputs.publish }} context: . file: ./docker/datahub-mae-consumer/Dockerfile platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - push: true mae_consumer_scan: name: "[Monitoring] Scan MAE consumer images for vulnerabilities" runs-on: ubuntu-latest @@ -210,6 +144,7 @@ jobs: ignore-unfixed: true vuln-type: 'os,library' severity: 'CRITICAL,HIGH' + mce_consumer_build: name: Build and Push DataHub MCE Consumer Docker Image runs-on: ubuntu-latest @@ -222,52 +157,18 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + - name: Build and push + uses: ./.github/actions/docker-custom-build-and-push with: - # list of Docker images to use as base name for tags images: | ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }} - # add git short SHA as Docker tag - tag-custom: ${{ needs.setup.outputs.tag }} - tag-custom-only: true - - name: Build and Load image for testing (if not publishing) - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - context: . - file: ./docker/datahub-mce-consumer/Dockerfile - platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - load: true - push: false - - name: Upload image locally for testing (if not publishing) - uses: ishworkh/docker-image-artifact-upload@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - image: ${{ steps.docker_meta.outputs.tags }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + tags: ${{ needs.setup.outputs.tag }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and Push Platform image - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + publish: ${{ needs.setup.outputs.publish }} context: . file: ./docker/datahub-mce-consumer/Dockerfile platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - push: true mce_consumer_scan: name: "[Monitoring] Scan MCE consumer images for vulnerabilities" runs-on: ubuntu-latest @@ -286,6 +187,7 @@ jobs: ignore-unfixed: true vuln-type: 'os,library' severity: 'CRITICAL,HIGH' + datahub_upgrade_build: name: Build and Push DataHub Upgrade Docker Image runs-on: ubuntu-latest @@ -298,52 +200,18 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + - name: Build and push + uses: ./.github/actions/docker-custom-build-and-push with: - # list of Docker images to use as base name for tags images: | ${{ env.DATAHUB_UPGRADE_IMAGE }} - # add git short SHA as Docker tag - tag-custom: ${{ needs.setup.outputs.tag }} - tag-custom-only: true - - name: Build and Load image for testing (if not publishing) - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - context: . - file: ./docker/datahub-upgrade/Dockerfile - platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - load: true - push: false - - name: Upload image locally for testing (if not publishing) - uses: ishworkh/docker-image-artifact-upload@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - image: ${{ steps.docker_meta.outputs.tags }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + tags: ${{ needs.setup.outputs.tag }} username: ${{ secrets.ACRYL_DOCKER_USERNAME }} password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - - name: Build and Push Platform image - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + publish: ${{ needs.setup.outputs.publish }} context: . file: ./docker/datahub-upgrade/Dockerfile platforms: linux/amd64,linux/arm64 - tags: ${{ steps.docker_meta.outputs.tags }} - push: true datahub_upgrade_scan: name: "[Monitoring] Scan DataHub Upgrade images for vulnerabilities" runs-on: ubuntu-latest @@ -362,6 +230,7 @@ jobs: ignore-unfixed: true vuln-type: 'os,library' severity: 'CRITICAL,HIGH' + frontend_build: name: Build and Push DataHub Frontend Docker Image runs-on: ubuntu-latest @@ -374,52 +243,18 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + - name: Build and push + uses: ./.github/actions/docker-custom-build-and-push with: - # list of Docker images to use as base name for tags images: | ${{ env.DATAHUB_FRONTEND_IMAGE }} - # add git short SHA as Docker tag - tag-custom: ${{ needs.setup.outputs.tag }} - tag-custom-only: true - - name: Build and Load image for testing (if not publishing) - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - context: . - file: ./docker/datahub-frontend/Dockerfile - platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - load: true - push: false - - name: Upload image locally for testing (if not publishing) - uses: ishworkh/docker-image-artifact-upload@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - image: ${{ steps.docker_meta.outputs.tags }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + tags: ${{ needs.setup.outputs.tag }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and Push MultiPlatform image - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + publish: ${{ needs.setup.outputs.publish }} context: . file: ./docker/datahub-frontend/Dockerfile platforms: linux/amd64,linux/arm64 - tags: ${{ steps.docker_meta.outputs.tags }} - push: true frontend_scan: name: "[Monitoring] Scan Frontend images for vulnerabilities" runs-on: ubuntu-latest @@ -438,6 +273,7 @@ jobs: ignore-unfixed: true vuln-type: 'os,library' severity: 'CRITICAL,HIGH' + kafka_setup_build: name: Build and Push DataHub Kafka Setup Docker Image runs-on: ubuntu-latest @@ -450,52 +286,19 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + - name: Build and push + uses: ./.github/actions/docker-custom-build-and-push with: - # list of Docker images to use as base name for tags images: | ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }} - # add git short SHA as Docker tag - tag-custom: ${{ needs.setup.outputs.tag }} - tag-custom-only: true - - name: Build and Load image for testing (if not publishing) - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - context: ./docker/kafka-setup - file: ./docker/kafka-setup/Dockerfile - platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - load: true - push: false - - name: Upload image locally for testing (if not publishing) - uses: ishworkh/docker-image-artifact-upload@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - image: ${{ steps.docker_meta.outputs.tags }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + tags: ${{ needs.setup.outputs.tag }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and Push MultiPlatform image - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + publish: ${{ needs.setup.outputs.publish }} context: ./docker/kafka-setup file: ./docker/kafka-setup/Dockerfile platforms: linux/amd64,linux/arm64 - tags: ${{ steps.docker_meta.outputs.tags }} - push: true + mysql_setup_build: name: Build and Push DataHub MySQL Setup Docker Image runs-on: ubuntu-latest @@ -508,52 +311,19 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + - name: Build and push + uses: ./.github/actions/docker-custom-build-and-push with: - # list of Docker images to use as base name for tags images: | ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }} - # add git short SHA as Docker tag - tag-custom: ${{ needs.setup.outputs.tag }} - tag-custom-only: true - - name: Build and Load image for testing (if not publishing) - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - context: . - file: ./docker/mysql-setup/Dockerfile - platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - load: true - push: false - - name: Upload image locally for testing (if not publishing) - uses: ishworkh/docker-image-artifact-upload@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - image: ${{ steps.docker_meta.outputs.tags }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + tags: ${{ needs.setup.outputs.tag }} username: ${{ secrets.ACRYL_DOCKER_USERNAME }} password: ${{ secrets.ACRYL_DOCKER_PASSWORD }} - - name: Build and Push MultiPlatform image - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + publish: ${{ needs.setup.outputs.publish }} context: . file: ./docker/mysql-setup/Dockerfile platforms: linux/amd64,linux/arm64 - tags: ${{ steps.docker_meta.outputs.tags }} - push: true + elasticsearch_setup_build: name: Build and Push DataHub Elasticsearch Setup Docker Image runs-on: ubuntu-latest @@ -566,52 +336,19 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + - name: Build and push + uses: ./.github/actions/docker-custom-build-and-push with: - # list of Docker images to use as base name for tags images: | ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }} - # add git short SHA as Docker tag - tag-custom: ${{ needs.setup.outputs.tag }} - tag-custom-only: true - - name: Build and Load image for testing (if not publishing) - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - context: . - file: ./docker/elasticsearch-setup/Dockerfile - platforms: linux/amd64 - tags: ${{ steps.docker_meta.outputs.tags }} - load: true - push: false - - name: Upload image locally for testing (if not publishing) - uses: ishworkh/docker-image-artifact-upload@v1 - if: ${{ needs.setup.outputs.publish != 'true' }} - with: - image: ${{ steps.docker_meta.outputs.tags }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + tags: ${{ needs.setup.outputs.tag }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and Push MultiPlatform image - uses: docker/build-push-action@v2 - if: ${{ needs.setup.outputs.publish == 'true' }} - with: + publish: ${{ needs.setup.outputs.publish }} context: . file: ./docker/elasticsearch-setup/Dockerfile platforms: linux/amd64,linux/arm64 - tags: ${{ steps.docker_meta.outputs.tags }} - push: true + smoke_test: name: Run Smoke Tests runs-on: ubuntu-latest