Improvements to Container CD

.github/workflows/composite/docker-build/action.yml

@@ -0,0 +1,80 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# Source: https://github.com/intel/ai-containers/blob/main/.github/action.yml
+
+---
+name: Build Container Group
+description: Given the inputs found below, build all containers found in a docker-compose.yaml file for a given configuration
+author: tyler.titsworth@intel.com
+inputs:
+  group_dir:
+    description: Directory with docker-compose.yaml to build
+    required: true
+    type: string
+  env_overrides:
+    description: Bash Env Variable Overrides in `KEY=VAL && KEY2=VAL2` format
+    required: false
+    type: string
+  registry:
+    description: Container Registry URL
+    required: false
+    default: 'opea-project'
+    type: string
+outputs:
+  container-group:
+    description: "Container Group"
+    value: ${{ steps.container-output.outputs.group }}
+runs:
+  using: composite
+  steps:
+      # This step generates a random number to use as the project number
+      # which can help avoid collisions with parallel builds on the same system
+    - name: Generate Project Number
+      shell: bash
+      run: echo "project-number=$(shuf -i 0-10000 -n1)" >> $GITHUB_ENV
+    - name: Build Containers
+      shell: bash
+      run: |
+        REGISTRY=${{ inputs.registry }} \
+        COMPOSE_PROJECT_NAME=${{ env.project-number }} \
+        ${{ inputs.env_overrides }} docker compose -p ${{ env.project-number }} up --build --force-recreate --always-recreate-deps --no-start
+      working-directory: ${{ inputs.group_dir }}
+    - name: Print Containers
+      id: container-output
+      shell: bash
+      run: |
+        mkdir matrix
+        images=$(REGISTRY=${{ inputs.registry }} \
+        COMPOSE_PROJECT_NAME=${{ env.project-number }} \
+        ${{ inputs.env_overrides }} docker compose -p ${{ env.project-number }} images --format json)
+        for image in $(echo $images | jq -r --arg registry "$REGISTRY" '.[] | select(.Repository | contains($registry)) | .Tag'); do
+          echo "$image" > matrix/$image.txt
+        done
+        echo "group=${{ inputs.group_dir }}" | tr '/' '_' >> $GITHUB_OUTPUT
+      working-directory: ${{ inputs.group_dir }}
+    - uses: actions/upload-artifact@v4
+      with:
+        name: ${{ env.project-number }}-${{ steps.container-output.outputs.group }}
+        path: ${{ inputs.group_dir }}/matrix/*
+        retention-days: 1
+        overwrite: true
+    - name: Push Containers
+      shell: bash
+      run: |
+        REGISTRY=${{ inputs.registry }} \
+        COMPOSE_PROJECT_NAME=${{ env.project-number }} \
+        ${{ inputs.env_overrides }} docker compose -p ${{ env.project-number }} push
+      working-directory: ${{ inputs.group_dir }}
+    - name: Un-Tag Containers
+      if: ${{ always() }}
+      shell: bash
+      run: |
+        REGISTRY=${{ inputs.registry }} \
+        COMPOSE_PROJECT_NAME=${{ env.project-number }} \
+        ${{ inputs.env_overrides }} docker compose -p ${{ env.project-number }} down --rmi all
+      working-directory: ${{ inputs.group_dir }}
+    - name: Remove Containers
+      if: ${{ always() }}
+      shell: bash
+      run: docker system prune --force

.github/workflows/composite/scan/action.yml

@@ -0,0 +1,26 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# Source: https://github.com/intel/ai-containers/blob/main/.github/scan/action.yml
+
+name: 'Aqua Security Trivy'
+description: 'Scans container images for vulnerabilities with Trivy without building the image. For use behind firewalls.'
+author: 'tyler.titsworth@intel.com'
+inputs:
+  image-ref:
+    description: 'image reference(for backward compatibility)'
+    required: true
+  output:
+    description: 'writes results to a file with the specified file name'
+    required: true
+runs:
+  using: 'docker'
+  image: "docker://ghcr.io/aquasecurity/trivy"
+  entrypoint: trivy
+  args:
+    - '--timeout=30m'
+    - image
+    - '--format=sarif'
+    - '--no-progress'
+    - '--output=${{ inputs.output }}'
+    - ${{ inputs.image-ref }}

.github/workflows/container-ci.yml

@@ -0,0 +1,77 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Container Integration Tests
+on:
+  pull_request
+permissions: read-all
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+jobs:
+  group-diff:
+    runs-on: ubuntu-latest
+    outputs:
+      groups: ${{ steps.group-list.outputs.FOLDERS }}
+    steps:
+    - name: Harden Runner
+      uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
+      with:
+        egress-policy: audit
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      with:
+        fetch-depth: 0
+    - name: Output Modified Group Directories
+      id: group-list
+      run: |
+        # Get diff array filtered by specific filetypes
+        DIFF=$(git diff --diff-filter=d \
+            --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \
+            -- '*/*Dockerfile' '*.py' '*.yaml' '*.yml' '*.sh' '*/*requirements.txt' '*.json' '*.ts' '*.js' | \
+            jq -R '.' | jq -sc '.' \
+        )
+        # Search for compose files in each file to determine the container groups
+        DOCKER_COMPOSE_PATHS=()
+        for path in $(echo $DIFF | jq -r '.[]'); do
+          while [[ "$path" != "." ]]; do
+            DIR_PATH=$(dirname "$path")
+            if [ -n "$(find "$DIR_PATH" -maxdepth 1 -name 'docker-compose.yaml' -print -quit)" ] && [ "$DIR_PATH" != "." ]; then
+              DOCKER_COMPOSE_PATHS+=("$DIR_PATH")
+              path="."
+            else
+              path="$DIR_PATH"
+            fi
+          done
+        done
+        # Convert the array to a JSON array
+        DOCKER_COMPOSE_PATHS_JSON=$(printf '%s\n' "${DOCKER_COMPOSE_PATHS[@]}" | jq -R '.' | jq -sc 'unique_by(.)')
+        echo "FOLDERS=$DOCKER_COMPOSE_PATHS_JSON" >> $GITHUB_OUTPUT
+  pipeline-ci:
+    needs: group-diff
+    if: needs.group-diff.outputs.groups != '[""]'
+    strategy:
+      matrix:
+        group: ${{ fromJson(needs.group-diff.outputs.groups) }}
+        experimental: [true]
+      fail-fast: false
+    uses: opea/genaiexamples/.github/workflows/reuse-container-ci.yaml@main
+    with:
+      group_dir: ${{ matrix.group }}
+    secrets: inherit
+  status-check:
+    needs: [group-diff, pipeline-ci]
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+    - name: Harden Runner
+      uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
+      with:
+        egress-policy: audit
+    - run: exit 1
+      if: >-
+        ${{
+             contains(needs.*.result, 'failure')
+          || contains(needs.*.result, 'cancelled')
+          || contains(needs.*.result, 'skipped')
+          && needs.group-diff.outputs.groups != '[""]'
+        }}