From e22d41362dc2d30642ac4975f61b7b74c38059f6 Mon Sep 17 00:00:00 2001 From: Ying Chun Guo Date: Wed, 7 Aug 2024 15:57:29 +0800 Subject: [PATCH] Refactor build image workflows with common action.yml (#525) Signed-off-by: Yingchun Guo --- .../docker/compose/AudioQnA-compose.yaml | 10 +++ .../docker/compose/ChatQnA-compose.yaml | 20 +++++ .../docker/compose/CodeGen-compose.yaml | 20 +++++ .../docker/compose/CodeTrans-compose.yaml | 15 ++++ .../docker/compose/DocSum-compose.yaml | 20 +++++ .../docker/compose/FaqGen-compose.yaml | 20 +++++ .../docker/compose/SearchQnA-compose.yaml | 15 ++++ .../docker/compose/Translation-compose.yaml | 15 ++++ .github/workflows/image-build-on-manual.yml | 78 +++++++++++++++++++ .github/workflows/image-build-on-push.yml | 36 +++++++-- 10 files changed, 243 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/docker/compose/AudioQnA-compose.yaml create mode 100644 .github/workflows/docker/compose/ChatQnA-compose.yaml create mode 100644 .github/workflows/docker/compose/CodeGen-compose.yaml create mode 100644 .github/workflows/docker/compose/CodeTrans-compose.yaml create mode 100644 .github/workflows/docker/compose/DocSum-compose.yaml create mode 100644 .github/workflows/docker/compose/FaqGen-compose.yaml create mode 100644 .github/workflows/docker/compose/SearchQnA-compose.yaml create mode 100644 .github/workflows/docker/compose/Translation-compose.yaml create mode 100644 .github/workflows/image-build-on-manual.yml diff --git a/.github/workflows/docker/compose/AudioQnA-compose.yaml b/.github/workflows/docker/compose/AudioQnA-compose.yaml new file mode 100644 index 000000000..d78017717 --- /dev/null +++ b/.github/workflows/docker/compose/AudioQnA-compose.yaml @@ -0,0 +1,10 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +services: + audioqna: + build: + context: docker + dockerfile: ./Dockerfile + image: ${REGISTRY}opea/audioqna:${TAG:-latest} diff --git a/.github/workflows/docker/compose/ChatQnA-compose.yaml b/.github/workflows/docker/compose/ChatQnA-compose.yaml new file mode 100644 index 000000000..e8124436d --- /dev/null +++ b/.github/workflows/docker/compose/ChatQnA-compose.yaml @@ -0,0 +1,20 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +services: + chatqna: + build: + context: docker + dockerfile: ./Dockerfile + image: ${REGISTRY}opea/chatqna:${TAG:-latest} + chatqna-ui: + build: + context: docker/ui + dockerfile: ./docker/Dockerfile + image: ${REGISTRY}opea/chatqna-ui:${TAG:-latest} + chatqna-conversation-ui: + build: + context: docker/ui + dockerfile: ./docker/Dockerfile.react + image: ${REGISTRY}opea/chatqna-conversation-ui:${TAG:-latest} diff --git a/.github/workflows/docker/compose/CodeGen-compose.yaml b/.github/workflows/docker/compose/CodeGen-compose.yaml new file mode 100644 index 000000000..f362e37b8 --- /dev/null +++ b/.github/workflows/docker/compose/CodeGen-compose.yaml @@ -0,0 +1,20 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +services: + codegen: + build: + context: docker + dockerfile: ./Dockerfile + image: ${REGISTRY}opea/codegen:${TAG:-latest} + codegen-ui: + build: + context: docker/ui + dockerfile: ./docker/Dockerfile + image: ${REGISTRY}opea/codegen-ui:${TAG:-latest} + codegen-react-ui: + build: + context: docker/ui + dockerfile: ./docker/Dockerfile.react + image: ${REGISTRY}opea/codegen-conversation-ui:${TAG:-latest} diff --git a/.github/workflows/docker/compose/CodeTrans-compose.yaml b/.github/workflows/docker/compose/CodeTrans-compose.yaml new file mode 100644 index 000000000..4d1f4eeec --- /dev/null +++ b/.github/workflows/docker/compose/CodeTrans-compose.yaml @@ -0,0 +1,15 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +services: + codetrans: + build: + context: docker + dockerfile: ./Dockerfile + image: ${REGISTRY}opea/codetrans:${TAG:-latest} + codetrans-ui: + build: + context: docker/ui + dockerfile: ./docker/Dockerfile + image: ${REGISTRY}opea/codetrans-ui:${TAG:-latest} diff --git a/.github/workflows/docker/compose/DocSum-compose.yaml b/.github/workflows/docker/compose/DocSum-compose.yaml new file mode 100644 index 000000000..494e318fb --- /dev/null +++ b/.github/workflows/docker/compose/DocSum-compose.yaml @@ -0,0 +1,20 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +services: + docsum: + build: + context: docker + dockerfile: ./Dockerfile + image: ${REGISTRY}opea/docsum:${TAG:-latest} + docsum-ui: + build: + context: docker/ui + dockerfile: ./docker/Dockerfile + image: ${REGISTRY}opea/docsum-ui:${TAG:-latest} + docsum-react-ui: + build: + context: docker/ui + dockerfile: ./docker/Dockerfile.react + image: ${REGISTRY}opea/docsum-react-ui:${TAG:-latest} diff --git a/.github/workflows/docker/compose/FaqGen-compose.yaml b/.github/workflows/docker/compose/FaqGen-compose.yaml new file mode 100644 index 000000000..724268b9a --- /dev/null +++ b/.github/workflows/docker/compose/FaqGen-compose.yaml @@ -0,0 +1,20 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +services: + faqgen: + build: + context: docker + dockerfile: ./Dockerfile + image: ${REGISTRY}opea/faqgen:${TAG:-latest} + faqgen-ui: + build: + context: docker/ui + dockerfile: ./docker/Dockerfile + image: ${REGISTRY}opea/faqgen-ui:${TAG:-latest} + faqgen-react-ui: + build: + context: docker/ui + dockerfile: ./docker/Dockerfile.react + image: ${REGISTRY}opea/faqgen-react-ui:${TAG:-latest} diff --git a/.github/workflows/docker/compose/SearchQnA-compose.yaml b/.github/workflows/docker/compose/SearchQnA-compose.yaml new file mode 100644 index 000000000..217f2f0e9 --- /dev/null +++ b/.github/workflows/docker/compose/SearchQnA-compose.yaml @@ -0,0 +1,15 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +services: + searchqna: + build: + context: docker + dockerfile: ./Dockerfile + image: ${REGISTRY}opea/searchqna:${TAG:-latest} + searchqna-ui: + build: + context: docker/ui + dockerfile: ./docker/Dockerfile + image: ${REGISTRY}opea/searchqna-ui:${TAG:-latest} diff --git a/.github/workflows/docker/compose/Translation-compose.yaml b/.github/workflows/docker/compose/Translation-compose.yaml new file mode 100644 index 000000000..2c8a4476a --- /dev/null +++ b/.github/workflows/docker/compose/Translation-compose.yaml @@ -0,0 +1,15 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +services: + translation: + build: + context: docker + dockerfile: ./Dockerfile + image: ${REGISTRY}opea/translation:${TAG:-latest} + translation-ui: + build: + context: docker/ui + dockerfile: ./docker/Dockerfile + image: ${REGISTRY}opea/translation-ui:${TAG:-latest} diff --git a/.github/workflows/image-build-on-manual.yml b/.github/workflows/image-build-on-manual.yml new file mode 100644 index 000000000..3c184ba17 --- /dev/null +++ b/.github/workflows/image-build-on-manual.yml @@ -0,0 +1,78 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: Build latest images on manual event + +on: + workflow_dispatch: + inputs: + registry: + default: "" + description: "Registry to store images,e.g., docker.io, default is empty" + required: false + type: string + services: + default: "AudioQnA,ChatQnA,CodeGen,CodeTrans,DocSum,FaqGen,SearchQnA,Translation" + description: "List of examples to build" + required: true + type: string + tag: + default: "latest" + description: "Tag to apply to images" + required: true + type: string + nodes: + default: "docker-build-xeon,docker-build-gaudi" + description: "List of node to run the build on" + required: true + type: string + +jobs: + get-build-matrix: + runs-on: ubuntu-latest + outputs: + services: ${{ steps.get-services.outputs.services }} + nodes: ${{ steps.get-services.outputs.nodes }} + steps: + - name: Get test Services + id: get-services + run: | + set -x + service_list=($(echo ${{ github.event.inputs.services }} | tr ',' ' ')) + services=$(printf '%s\n' "${service_list[@]}" | sort -u | jq -R '.' | jq -sc '.') + echo "services=$services" >> $GITHUB_OUTPUT + node_list=($(echo ${{ github.event.inputs.nodes }} | tr ',' ' ')) + nodes=$(printf '%s\n' "${node_list[@]}" | sort -u | jq -R '.' | jq -sc '.') + echo "nodes=$nodes" >> $GITHUB_OUTPUT + + image-build: + needs: get-build-matrix + strategy: + matrix: + service: ${{ fromJSON(needs.get-build-matrix.outputs.services) }} + node: ${{ fromJSON(needs.get-build-matrix.outputs.nodes) }} + runs-on: ${{ matrix.node }} + continue-on-error: true + steps: + - name: Clean Up Working Directory + run: | + sudo rm -rf ${{github.workspace}}/* + + - name: Checkout out Repo + uses: actions/checkout@v4 + + - name: Config image repo + run: | + if [[ -z "${{ github.event.inputs.registry }}" ]]; then + echo "image_repo=${OPEA_IMAGE_REPO}" >> $GITHUB_ENV + else + echo "image_repo=${{ github.event.inputs.registry }}/" >> $GITHUB_ENV + fi + + - name: Build image + uses: opea-project/validation/actions/image-build@main + with: + work_dir: ${{ github.workspace }}/${{ matrix.service }} + docker_compose_path: ${{ github.workspace }}/.github/workflows/docker/compose/${{ matrix.service }}-compose.yaml + registry: ${{ env.image_repo }} + tag: ${{ github.event.inputs.tag }} diff --git a/.github/workflows/image-build-on-push.yml b/.github/workflows/image-build-on-push.yml index f18633336..76e160b90 100644 --- a/.github/workflows/image-build-on-push.yml +++ b/.github/workflows/image-build-on-push.yml @@ -5,7 +5,7 @@ name: Build latest images on push event on: push: - branches: [ 'main' ] + branches: [ 'main','daisyonpush' ] paths: - "**/docker/*.py" - "**/docker/Dockerfile" @@ -26,8 +26,32 @@ jobs: matrix: workload: ${{ fromJSON(needs.job1.outputs.run_matrix).include.*.example }} hardware: ["gaudi","xeon"] - uses: ./.github/workflows/reuse-image-build.yml - with: - image_tag: latest - mega_service: "${{ matrix.workload }}" - runner_label: docker-build-${{ matrix.hardware }} + runs-on: docker-build-${{ matrix.hardware }} + steps: + - name: Clean up Working Directory + run: | + sudo rm -rf ${{github.workspace}}/* + + - name: Checkout out Repo + uses: actions/checkout@v4 + + - name: Check Docker Compose File Exists + env: + service: ${{ matrix.workload }} + run: | + docker_compose_path="${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose.yaml" + if [ -e $docker_compose_path ]; then + echo "file_exists=true" >> $GITHUB_ENV + echo "docker_compose_path=${docker_compose_path}" >> $GITHUB_ENV + else + echo "file_exists=false" >> $GITHUB_ENV + echo "docker_compose_path=${docker_compose_path} for this service does not exist, so skipping image build for this service!!!" + fi + + - name: Build Image + if: env.file_exists == 'true' + uses: opea-project/validation/actions/image-build@main + with: + work_dir: ${{ github.workspace }}/${{ matrix.workload }} + docker_compose_path: ${{ env.docker_compose_path }} + registry: ${OPEA_IMAGE_REPO}