simplify smoke-test action (#5311)

nginx · Mar 27, 2024 · f8b79bc · f8b79bc
1 parent 0651f25
commit f8b79bc
Show file tree

Hide file tree

Showing 4 changed files with 84 additions and 122 deletions.
diff --git a/.github/actions/smoke-tests/action.yaml b/.github/actions/smoke-tests/action.yaml
@@ -2,51 +2,32 @@ name: Run Smoke Tests
 description: Run Smoke Tests for the project
 
 inputs:
-  go-md5:
-    required: true
-    type: string
-  base-image-md5:
-    required: true
-    type: string
   k8s-version:
     description: Kubernetes version to use
     required: false
   k8s-timeout:
     description: Timeout to use
     default: 75s
     required: false
-  image:
-    description: Docker image to use
-    default: debian
-    required: false
+  image-type:
+    description: Image type to test
+    required: true
+  image-name:
+    description: Docker image name to test
+    required: true
+  tag:
+    description: Docker image tag to test
+    required: true
   test-image:
     description: Test Docker image to use
     default: gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:latest
     required: false
   marker:
     description: Marker to use
     required: false
-  nginx-key:
-    description: Nginx key to use
-    required: false
-  nginx-crt:
-    description: Nginx cert to use
-    required: false
   azure-ad-secret:
     description: Azure Active Directory secret for JWKs
     required: false
-  rhel-license:
-    description: RHEL license for UBI builds
-    required: false
-  gcr-workload-identity-secret:
-    description: Google Workflow Identity secret
-    required: false
-  gcr-service-account-secret:
-    description: Google Service Account secret
-    required: false
-  forked-workflow:
-    description: Does this workflow full access to repo resources?
-    required: false
 
 outputs:
   test-results-name:
@@ -56,90 +37,24 @@ outputs:
 runs:
   using: composite
   steps:
-    - name: Fetch Cached Artifacts
-      uses: actions/cache@v3
-      with:
-        path: ${{ github.workspace }}/dist
-        key: nginx-ingress-${{ inputs.go-md5 }}
-
-    - name: Ingress type
-      id: ingress-type
-      run: |
-        echo "name=nginx${{ contains(inputs.image, 'plus') && '-plus' || '' }}-ingress" >> $GITHUB_OUTPUT
-        echo "tag=${{ inputs.image }}${{ contains(inputs.marker, 'dos') && '-dos' || '' }}${{ contains(inputs.marker, 'appprotect') && '-nap' || '' }}-${{ github.sha }}" >> $GITHUB_OUTPUT
-        echo "modules=${{ contains(inputs.marker, 'appprotect') && 'waf' || '' }}${{ contains(inputs.marker, 'dos') && 'dos' || '' }}" >> $GITHUB_OUTPUT
-      shell: bash
-
-    - name: Docker Buildx
-      uses: docker/setup-buildx-action@v2
-
-    - name: Authenticate to Google Cloud
-      id: auth
-      uses: google-github-actions/auth@v2
-      with:
-        token_format: access_token
-        workload_identity_provider: ${{ inputs.gcr-workload-identity-secret }}
-        service_account: ${{ inputs.gcr-service-account-secret }}
-      if: ${{ inputs.forked-workflow == 'false' }}
-
-    - name: Login to GCR
-      uses: docker/login-action@v3
-      with:
-        registry: gcr.io
-        username: oauth2accesstoken
-        password: ${{ steps.auth.outputs.access_token }}
-      if: ${{ inputs.forked-workflow == 'false' }}
-
-    - name: Build ${{ inputs.image }} Container
-      uses: docker/build-push-action@v3
-      with:
-        file: build/Dockerfile
-        context: "."
-        cache-from: type=gha,scope=${{ inputs.image }}${{ contains(inputs.marker, 'dos') && '-dos' || '' }}${{ contains(inputs.marker, 'appprotect') && '-nap' || '' }}
-        target: goreleaser${{ inputs.forked-workflow == 'false' && '-prebuilt' || '' }}
-        tags: "docker.io/nginx/${{ steps.ingress-type.outputs.name }}:${{ steps.ingress-type.outputs.tag }}"
-        load: true
-        pull: true
-        build-args: |
-          BUILD_OS=${{ inputs.image }}
-          PREBUILT_BASE_IMG=gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ contains(inputs.image, 'plus') && 'plus' || 'oss' }}:${{ inputs.base-image-md5 }}-${{ inputs.image }}${{ contains(inputs.marker, 'appprotect') && '-waf' || '' }}${{ contains(inputs.marker, 'dos') && '-dos' || '' }}
-          IC_VERSION=CI
-          ${{ steps.ingress-type.outputs.modules != '' && format('NAP_MODULES={0}', steps.ingress-type.outputs.modules) || '' }}
-          ${{ contains(inputs.marker, 'appprotect') && 'DEBIAN_VERSION=buster-slim' || '' }}
-        secrets: |
-          ${{ contains(inputs.image, 'plus') && format('"nginx-repo.crt={0}"', inputs.nginx-crt) || '' }}
-          ${{ contains(inputs.image, 'plus') && format('"nginx-repo.key={0}"', inputs.nginx-key) || '' }}
-          ${{ contains(inputs.image, 'ubi') && format('"rhel_license={0}"', inputs.rhel-license) || '' }}
-
     - name: Deploy Kubernetes
       id: k8s
       run: |
         make -f tests/Makefile create-kind-cluster K8S_CLUSTER_NAME=${{ github.run_id }} K8S_CLUSTER_VERSION=${{ inputs.k8s-version }} K8S_TIMEOUT=${{ inputs.k8s-timeout }}
-        make -f tests/Makefile image-load PREFIX=nginx/${{ steps.ingress-type.outputs.name }} TAG=${{ steps.ingress-type.outputs.tag }} K8S_CLUSTER_NAME=${{ github.run_id }}
+        make -f tests/Makefile image-load REGISTRY="" PREFIX=${{ inputs.image-name }} TAG=${{ inputs.tag }} K8S_CLUSTER_NAME=${{ github.run_id }}
         marker="${{ inputs.marker }}"
         nospaces="${marker// /_}"
         sanitized_marker="${nospaces//\'/}"
         name="${sanitized_marker:-${{ inputs.k8s-version }}}"
         echo "cluster_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${{ github.run_id }}-control-plane)" >> $GITHUB_OUTPUT
-        echo "cluster=$(echo nginx-${{ inputs.image }}-$name)" >> $GITHUB_OUTPUT
+        echo "cluster=$(echo nginx-${{ inputs.image-type }}-$name)" >> $GITHUB_OUTPUT
       shell: bash
 
     - name: Setup Kubeconfig
       run: |
         sed -i 's|server:.*|server: https://${{ steps.k8s.outputs.cluster_ip }}:6443|' ~/.kube/kind/config
       shell: bash
 
-    - name: Build Test-Runner Container
-      uses: docker/build-push-action@v3
-      with:
-        file: tests/Dockerfile
-        context: "."
-        cache-from: type=gha,scope=test-runner
-        tags: ${{ inputs.test-image }}
-        pull: true
-        load: true
-      if: ${{ inputs.forked-workflow == 'true' }}
-
     - name: Run Smoke Tests
       run: |
         touch tests-${{ steps.k8s.outputs.cluster }}.html
@@ -149,9 +64,9 @@ runs:
         -v ${{ github.workspace }}/tests:/workspace/tests \
         -v ~/.kube/kind/config:/root/.kube/config ${{ inputs.test-image }} \
         --context=kind-${{ github.run_id }} \
-        --image=docker.io/nginx/${{ steps.ingress-type.outputs.name }}:${{ steps.ingress-type.outputs.tag }} \
+        --image=${{ inputs.image-name }}:${{ inputs.tag }} \
         --image-pull-policy=Never \
-        --ic-type=${{ steps.ingress-type.outputs.name }} \
+        --ic-type=nginx${{ contains(inputs.image-type, 'plus') && '-plus' || '' }}-ingress \
         --service=nodeport --node-ip=${{ steps.k8s.outputs.cluster_ip }} \
         --html=tests-${{ steps.k8s.outputs.cluster }}.html \
         --self-contained-html \

diff --git a/.github/workflows/build-plus.yml b/.github/workflows/build-plus.yml
@@ -175,7 +175,7 @@ jobs:
           build-args: |
             BUILD_OS=${{ inputs.image }}
             IC_VERSION=${{ github.ref_type == 'tag' && steps.meta.outputs.version || 'CI' }}
-            ${{ steps.nap_modules.outputs.modules != '' && format('NAP_MODULES_AWS={0}', steps.nap_modules.outputs.modules) || '' }}
+            ${{ steps.nap_modules.outputs.modules != '' && format('NAP_MODULES={0}', steps.nap_modules.outputs.modules) || '' }}
           secrets: |
             "nginx-repo.crt=${{ inputs.nap_modules != '' && secrets.NGINX_AP_CRT || secrets.NGINX_CRT }}"
             "nginx-repo.key=${{ inputs.nap_modules != '' && secrets.NGINX_AP_KEY || secrets.NGINX_KEY }}"
@@ -204,8 +204,8 @@ jobs:
             BUILD_OS=${{ inputs.image }}
             PREBUILT_BASE_IMG=gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/plus:${{ inputs.base-image-md5 }}-${{ inputs.image }}${{ steps.nap_modules.outputs.name != '' && format('-{0}', steps.nap_modules.outputs.name) || '' }}
             IC_VERSION=${{ github.ref_type == 'tag' && steps.meta.outputs.version || 'CI' }}
-            ${{ inputs.nap_modules != '' && format('NAP_MODULES={0}', inputs.nap_modules) || '' }}
-            ${{ steps.nap_modules.outputs.modules != '' && format('NAP_MODULES_AWS={0}', steps.nap_modules.outputs.modules) || '' }}
+            ${{ inputs.nap_modules != '' && format('NAP_MODULES={0}', steps.nap_modules.outputs.modules) || '' }}
+            ${{ (contains(inputs.target, 'aws') && inputs.nap_modules != '') && format('NAP_MODULES_AWS={0}', steps.nap_modules.outputs.modules) || '' }}
           secrets: |
             "nginx-repo.crt=${{ inputs.nap_modules != '' && secrets.NGINX_AP_CRT || secrets.NGINX_CRT }}"
             "nginx-repo.key=${{ inputs.nap_modules != '' && secrets.NGINX_AP_KEY || secrets.NGINX_KEY }}"

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -436,11 +436,11 @@ jobs:
           tags: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"
           pull: true
           push: ${{ needs.checks.outputs.forked_workflow == 'false' }}
-          load: ${{ needs.checks.outputs.forked_workflow == 'true' }}
-        if: ${{ steps.check-image.outcome == 'failure' || needs.checks.outputs.forked_workflow == 'true' }}
+          load: false
+        if: ${{ steps.check-image.outcome == 'failure' }}
 
   smoke-tests:
-    name: ${{ matrix.images.label }} ${{ matrix.images.image }} smoke tests
+    name: ${{ matrix.images.label }} ${{ matrix.images.image }} ${{ matrix.k8s }} smoke tests
     runs-on: ubuntu-22.04
     needs: [checks, setup-matrix]
     strategy:
@@ -456,8 +456,10 @@ jobs:
       - name: NAP modules
         id: nap_modules
         run: |
-          [[ "${{ matrix.images.nap_modules }}" == "waf,dos" ]] && modules="waf-dos" || modules="${{ matrix.images.nap_modules }}"
+          [[ "${{ matrix.images.nap_modules }}" == "waf,dos" ]] && modules="waf-dos" || name="${{ matrix.images.nap_modules }}"
+          [[ "${{ matrix.images.nap_modules }}" == "waf,dos" ]] && modules="both" || modules="${{ matrix.images.nap_modules }}"
           echo "modules=${modules}" >> $GITHUB_OUTPUT
+          echo "name=${name}" >> $GITHUB_OUTPUT
         if: ${{ matrix.images.nap_modules }}
 
       - name: Authenticate to Google Cloud
@@ -480,7 +482,7 @@ jobs:
       - name: Check if base images exist
         id: base_exists
         run: |
-          if docker manifest inspect gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ matrix.images.type }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.modules) || '' }}; then
+          if docker manifest inspect gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ matrix.images.type }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.name) || '' }}; then
             echo "exists=0" >> $GITHUB_OUTPUT
           else
             echo "exists=1" >> $GITHUB_OUTPUT
@@ -489,53 +491,93 @@ jobs:
 
       - name: Docker Buildx
         uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0
-        if: ${{ needs.checks.outputs.forked_workflow == 'false' && steps.base_exists.outputs.exists != 0 }}
 
       - name: Setup QEMU
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
         with:
           platforms: ${{ matrix.platforms }}
-        if: ${{ needs.checks.outputs.forked_workflow == 'false' && steps.base_exists.outputs.exists != 0 }}
 
       - name: Build Base Container
         uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
         with:
           file: build/Dockerfile
           context: "."
-          cache-from: type=gha,scope=${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.modules) || '' }}
-          cache-to: type=gha,scope=${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.modules) || '' }},mode=max
+          cache-from: type=gha,scope=${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.name) || '' }}
+          cache-to: type=gha,scope=${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.name) || '' }},mode=max
           target: common
-          tags: gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ matrix.images.type }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.modules) || '' }}
+          tags: gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ matrix.images.type }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.name) || '' }}
           platforms: ${{ matrix.images.platforms }}
           pull: true
           push: true
           build-args: |
             BUILD_OS=${{ matrix.images.image }}
             IC_VERSION=${{ needs.checks.outputs.ic_version }}
-            ${{ contains(matrix.images.image, 'nap') && format('NAP_MODULES={0}', matrix.images.nap_modules) || '' }}
+            ${{ steps.nap_modules.outputs.modules != '' && format('NAP_MODULES={0}', steps.nap_modules.outputs.modules) || '' }}
           secrets: |
             ${{ contains(matrix.images.image, 'nap') && format('"nginx-repo.crt={0}"', secrets.NGINX_AP_CRT) || format('"nginx-repo.crt={0}"', secrets.NGINX_CRT) }}
             ${{ contains(matrix.images.image, 'nap') && format('"nginx-repo.key={0}"', secrets.NGINX_AP_KEY) || format('"nginx-repo.key={0}"', secrets.NGINX_KEY) }}
             ${{ contains(matrix.images.image, 'ubi') && format('"rhel_license={0}"', secrets.RHEL_LICENSE) || '' }}
         if: ${{ needs.checks.outputs.forked_workflow == 'false' && steps.base_exists.outputs.exists != 0 }}
 
+      - name: Fetch Cached Artifacts
+        uses: actions/cache@v3
+        with:
+          path: ${{ github.workspace }}/dist
+          key: nginx-ingress-${{ needs.checks.outputs.go_code_md5 }}
+
+      - name: Check if test image exists
+        id: check-image
+        run: |
+          docker manifest inspect "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"
+        shell: bash
+        continue-on-error: true
+        if: ${{ needs.checks.outputs.forked_workflow == 'false' }}
+
+      - name: Build Test-Runner Container
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
+        with:
+          file: tests/Dockerfile
+          context: "."
+          cache-from: type=gha,scope=test-runner
+          tags: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"
+          pull: true
+          push: ${{ needs.checks.outputs.forked_workflow == 'false' }}
+          load: ${{ needs.checks.outputs.forked_workflow == 'true' }}
+        if: ${{ needs.checks.outputs.forked_workflow == 'true' || steps.check-image.outcome == 'failure' }}
+
+      - name: Build ${{ matrix.images.image }} Container
+        uses: docker/build-push-action@v3
+        with:
+          file: build/Dockerfile
+          context: "."
+          cache-from: type=gha,scope=${{ matrix.images.image }}${{ steps.nap_modules.outputs.name != '' && format('-{0}', steps.nap_modules.outputs.name) || '' }}
+          cache-to: type=gha,scope=${{ matrix.images.image }}${{ steps.nap_modules.outputs.name != '' && format('-{0}', steps.nap_modules.outputs.name) || '' }},mode=max
+          target: goreleaser${{ needs.checks.outputs.forked_workflow == 'false' && '-prebuilt' || '' }}
+          tags: "docker.io/nginx/nginx${{ contains(matrix.images.image, 'plus') && '-plus' || '' }}-ingress:${{ matrix.images.image }}${{ contains(matrix.images.marker, 'dos') && '-dos' || '' }}${{ contains(matrix.images.marker, 'appprotect') && '-nap' || '' }}-${{ github.sha }}"
+          load: true
+          pull: true
+          build-args: |
+            BUILD_OS=${{ matrix.images.image }}
+            IC_VERSION=CI
+            PREBUILT_BASE_IMG=gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ matrix.images.type }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.images.image }}${{ steps.nap_modules.outputs.name != '' && format('-{0}', steps.nap_modules.outputs.name) || '' }}
+            ${{ contains(matrix.images.image, 'nap') && format('NAP_MODULES={0}', steps.nap_modules.outputs.modules) || '' }}
+            ${{ contains(matrix.images.marker, 'appprotect') && 'DEBIAN_VERSION=buster-slim' || '' }}
+          secrets: |
+            ${{ contains(matrix.images.image, 'nap') && format('"nginx-repo.crt={0}"', secrets.NGINX_AP_CRT) || format('"nginx-repo.crt={0}"', secrets.NGINX_CRT) }}
+            ${{ contains(matrix.images.image, 'nap') && format('"nginx-repo.key={0}"', secrets.NGINX_AP_KEY) || format('"nginx-repo.key={0}"', secrets.NGINX_KEY) }}
+            ${{ contains(matrix.images.image, 'ubi') && format('"rhel_license={0}"', secrets.RHEL_LICENSE) || '' }}
+
       - name: Run Smoke Tests
         id: smoke-tests
         uses: ./.github/actions/smoke-tests
         with:
-          image: ${{ matrix.images.image != '' && matrix.images.image || 'debian' }}
+          image-type: ${{ matrix.images.image }}
+          image-name: docker.io/nginx/nginx${{ contains(matrix.images.image, 'plus') && '-plus' || '' }}-ingress
+          tag: ${{ matrix.images.image }}${{ contains(matrix.images.marker, 'dos') && '-dos' || '' }}${{ contains(matrix.images.marker, 'appprotect') && '-nap' || '' }}-${{ github.sha }}
           marker: ${{ matrix.images.marker != '' && matrix.images.marker || '' }}
           k8s-version: ${{ matrix.k8s }}
-          nginx-crt: ${{ contains(matrix.images.image, 'nap') && secrets.NGINX_AP_CRT || secrets.NGINX_CRT }}
-          nginx-key: ${{ contains(matrix.images.image, 'nap') && secrets.NGINX_AP_KEY || secrets.NGINX_KEY }}
           azure-ad-secret: ${{ secrets.AZURE_AD_AUTOMATION }}
-          gcr-workload-identity-secret: ${{ secrets.GCR_WORKLOAD_IDENTITY }}
-          gcr-service-account-secret: ${{ secrets.GCR_SERVICE_ACCOUNT }}
-          rhel-license: ${{ contains(matrix.images.image, 'ubi') && secrets.RHEL_LICENSE || '' }}
-          go-md5: ${{ needs.checks.outputs.go_code_md5 }}
-          base-image-md5: ${{ needs.checks.outputs.docker_md5 }}
           test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"
-          forked-workflow: ${{ needs.checks.outputs.forked_workflow == 'true' }}
 
       - name: Upload Test Results
         uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1