diff --git a/.github/actions/smoke-tests/action.yaml b/.github/actions/smoke-tests/action.yaml index d12257e9b3..a85aba1415 100644 --- a/.github/actions/smoke-tests/action.yaml +++ b/.github/actions/smoke-tests/action.yaml @@ -2,12 +2,6 @@ name: Run Smoke Tests description: Run Smoke Tests for the project inputs: - go-md5: - required: true - type: string - base-image-md5: - required: true - type: string k8s-version: description: Kubernetes version to use required: false @@ -15,10 +9,15 @@ inputs: description: Timeout to use default: 75s required: false - image: - description: Docker image to use - default: debian - required: false + image-type: + description: Image type to test + required: true + image-name: + description: Docker image name to test + required: true + tag: + description: Docker image tag to test + required: true test-image: description: Test Docker image to use default: gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:latest @@ -26,27 +25,9 @@ inputs: marker: description: Marker to use required: false - nginx-key: - description: Nginx key to use - required: false - nginx-crt: - description: Nginx cert to use - required: false azure-ad-secret: description: Azure Active Directory secret for JWKs required: false - rhel-license: - description: RHEL license for UBI builds - required: false - gcr-workload-identity-secret: - description: Google Workflow Identity secret - required: false - gcr-service-account-secret: - description: Google Service Account secret - required: false - forked-workflow: - description: Does this workflow full access to repo resources? - required: false outputs: test-results-name: @@ -56,72 +37,17 @@ outputs: runs: using: composite steps: - - name: Fetch Cached Artifacts - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/dist - key: nginx-ingress-${{ inputs.go-md5 }} - - - name: Ingress type - id: ingress-type - run: | - echo "name=nginx${{ contains(inputs.image, 'plus') && '-plus' || '' }}-ingress" >> $GITHUB_OUTPUT - echo "tag=${{ inputs.image }}${{ contains(inputs.marker, 'dos') && '-dos' || '' }}${{ contains(inputs.marker, 'appprotect') && '-nap' || '' }}-${{ github.sha }}" >> $GITHUB_OUTPUT - echo "modules=${{ contains(inputs.marker, 'appprotect') && 'waf' || '' }}${{ contains(inputs.marker, 'dos') && 'dos' || '' }}" >> $GITHUB_OUTPUT - shell: bash - - - name: Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Authenticate to Google Cloud - id: auth - uses: google-github-actions/auth@v2 - with: - token_format: access_token - workload_identity_provider: ${{ inputs.gcr-workload-identity-secret }} - service_account: ${{ inputs.gcr-service-account-secret }} - if: ${{ inputs.forked-workflow == 'false' }} - - - name: Login to GCR - uses: docker/login-action@v3 - with: - registry: gcr.io - username: oauth2accesstoken - password: ${{ steps.auth.outputs.access_token }} - if: ${{ inputs.forked-workflow == 'false' }} - - - name: Build ${{ inputs.image }} Container - uses: docker/build-push-action@v3 - with: - file: build/Dockerfile - context: "." - cache-from: type=gha,scope=${{ inputs.image }}${{ contains(inputs.marker, 'dos') && '-dos' || '' }}${{ contains(inputs.marker, 'appprotect') && '-nap' || '' }} - target: goreleaser${{ inputs.forked-workflow == 'false' && '-prebuilt' || '' }} - tags: "docker.io/nginx/${{ steps.ingress-type.outputs.name }}:${{ steps.ingress-type.outputs.tag }}" - load: true - pull: true - build-args: | - BUILD_OS=${{ inputs.image }} - PREBUILT_BASE_IMG=gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ contains(inputs.image, 'plus') && 'plus' || 'oss' }}:${{ inputs.base-image-md5 }}-${{ inputs.image }}${{ contains(inputs.marker, 'appprotect') && '-waf' || '' }}${{ contains(inputs.marker, 'dos') && '-dos' || '' }} - IC_VERSION=CI - ${{ steps.ingress-type.outputs.modules != '' && format('NAP_MODULES={0}', steps.ingress-type.outputs.modules) || '' }} - ${{ contains(inputs.marker, 'appprotect') && 'DEBIAN_VERSION=buster-slim' || '' }} - secrets: | - ${{ contains(inputs.image, 'plus') && format('"nginx-repo.crt={0}"', inputs.nginx-crt) || '' }} - ${{ contains(inputs.image, 'plus') && format('"nginx-repo.key={0}"', inputs.nginx-key) || '' }} - ${{ contains(inputs.image, 'ubi') && format('"rhel_license={0}"', inputs.rhel-license) || '' }} - - name: Deploy Kubernetes id: k8s run: | make -f tests/Makefile create-kind-cluster K8S_CLUSTER_NAME=${{ github.run_id }} K8S_CLUSTER_VERSION=${{ inputs.k8s-version }} K8S_TIMEOUT=${{ inputs.k8s-timeout }} - make -f tests/Makefile image-load PREFIX=nginx/${{ steps.ingress-type.outputs.name }} TAG=${{ steps.ingress-type.outputs.tag }} K8S_CLUSTER_NAME=${{ github.run_id }} + make -f tests/Makefile image-load REGISTRY="" PREFIX=${{ inputs.image-name }} TAG=${{ inputs.tag }} K8S_CLUSTER_NAME=${{ github.run_id }} marker="${{ inputs.marker }}" nospaces="${marker// /_}" sanitized_marker="${nospaces//\'/}" name="${sanitized_marker:-${{ inputs.k8s-version }}}" echo "cluster_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${{ github.run_id }}-control-plane)" >> $GITHUB_OUTPUT - echo "cluster=$(echo nginx-${{ inputs.image }}-$name)" >> $GITHUB_OUTPUT + echo "cluster=$(echo nginx-${{ inputs.image-type }}-$name)" >> $GITHUB_OUTPUT shell: bash - name: Setup Kubeconfig @@ -129,17 +55,6 @@ runs: sed -i 's|server:.*|server: https://${{ steps.k8s.outputs.cluster_ip }}:6443|' ~/.kube/kind/config shell: bash - - name: Build Test-Runner Container - uses: docker/build-push-action@v3 - with: - file: tests/Dockerfile - context: "." - cache-from: type=gha,scope=test-runner - tags: ${{ inputs.test-image }} - pull: true - load: true - if: ${{ inputs.forked-workflow == 'true' }} - - name: Run Smoke Tests run: | touch tests-${{ steps.k8s.outputs.cluster }}.html @@ -149,9 +64,9 @@ runs: -v ${{ github.workspace }}/tests:/workspace/tests \ -v ~/.kube/kind/config:/root/.kube/config ${{ inputs.test-image }} \ --context=kind-${{ github.run_id }} \ - --image=docker.io/nginx/${{ steps.ingress-type.outputs.name }}:${{ steps.ingress-type.outputs.tag }} \ + --image=${{ inputs.image-name }}:${{ inputs.tag }} \ --image-pull-policy=Never \ - --ic-type=${{ steps.ingress-type.outputs.name }} \ + --ic-type=nginx${{ contains(inputs.image-type, 'plus') && '-plus' || '' }}-ingress \ --service=nodeport --node-ip=${{ steps.k8s.outputs.cluster_ip }} \ --html=tests-${{ steps.k8s.outputs.cluster }}.html \ --self-contained-html \ diff --git a/.github/workflows/build-plus.yml b/.github/workflows/build-plus.yml index 725e7d2947..ed8d14bd3b 100644 --- a/.github/workflows/build-plus.yml +++ b/.github/workflows/build-plus.yml @@ -175,7 +175,7 @@ jobs: build-args: | BUILD_OS=${{ inputs.image }} IC_VERSION=${{ github.ref_type == 'tag' && steps.meta.outputs.version || 'CI' }} - ${{ steps.nap_modules.outputs.modules != '' && format('NAP_MODULES_AWS={0}', steps.nap_modules.outputs.modules) || '' }} + ${{ steps.nap_modules.outputs.modules != '' && format('NAP_MODULES={0}', steps.nap_modules.outputs.modules) || '' }} secrets: | "nginx-repo.crt=${{ inputs.nap_modules != '' && secrets.NGINX_AP_CRT || secrets.NGINX_CRT }}" "nginx-repo.key=${{ inputs.nap_modules != '' && secrets.NGINX_AP_KEY || secrets.NGINX_KEY }}" @@ -204,8 +204,8 @@ jobs: BUILD_OS=${{ inputs.image }} PREBUILT_BASE_IMG=gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/plus:${{ inputs.base-image-md5 }}-${{ inputs.image }}${{ steps.nap_modules.outputs.name != '' && format('-{0}', steps.nap_modules.outputs.name) || '' }} IC_VERSION=${{ github.ref_type == 'tag' && steps.meta.outputs.version || 'CI' }} - ${{ inputs.nap_modules != '' && format('NAP_MODULES={0}', inputs.nap_modules) || '' }} - ${{ steps.nap_modules.outputs.modules != '' && format('NAP_MODULES_AWS={0}', steps.nap_modules.outputs.modules) || '' }} + ${{ inputs.nap_modules != '' && format('NAP_MODULES={0}', steps.nap_modules.outputs.modules) || '' }} + ${{ (contains(inputs.target, 'aws') && inputs.nap_modules != '') && format('NAP_MODULES_AWS={0}', steps.nap_modules.outputs.modules) || '' }} secrets: | "nginx-repo.crt=${{ inputs.nap_modules != '' && secrets.NGINX_AP_CRT || secrets.NGINX_CRT }}" "nginx-repo.key=${{ inputs.nap_modules != '' && secrets.NGINX_AP_KEY || secrets.NGINX_KEY }}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6d8dcf9f7..5ffb5e0589 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -436,11 +436,11 @@ jobs: tags: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}" pull: true push: ${{ needs.checks.outputs.forked_workflow == 'false' }} - load: ${{ needs.checks.outputs.forked_workflow == 'true' }} - if: ${{ steps.check-image.outcome == 'failure' || needs.checks.outputs.forked_workflow == 'true' }} + load: false + if: ${{ steps.check-image.outcome == 'failure' }} smoke-tests: - name: ${{ matrix.images.label }} ${{ matrix.images.image }} smoke tests + name: ${{ matrix.images.label }} ${{ matrix.images.image }} ${{ matrix.k8s }} smoke tests runs-on: ubuntu-22.04 needs: [checks, setup-matrix] strategy: @@ -456,8 +456,10 @@ jobs: - name: NAP modules id: nap_modules run: | - [[ "${{ matrix.images.nap_modules }}" == "waf,dos" ]] && modules="waf-dos" || modules="${{ matrix.images.nap_modules }}" + [[ "${{ matrix.images.nap_modules }}" == "waf,dos" ]] && modules="waf-dos" || name="${{ matrix.images.nap_modules }}" + [[ "${{ matrix.images.nap_modules }}" == "waf,dos" ]] && modules="both" || modules="${{ matrix.images.nap_modules }}" echo "modules=${modules}" >> $GITHUB_OUTPUT + echo "name=${name}" >> $GITHUB_OUTPUT if: ${{ matrix.images.nap_modules }} - name: Authenticate to Google Cloud @@ -480,7 +482,7 @@ jobs: - name: Check if base images exist id: base_exists run: | - if docker manifest inspect gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ matrix.images.type }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.modules) || '' }}; then + if docker manifest inspect gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ matrix.images.type }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.name) || '' }}; then echo "exists=0" >> $GITHUB_OUTPUT else echo "exists=1" >> $GITHUB_OUTPUT @@ -489,53 +491,93 @@ jobs: - name: Docker Buildx uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0 - if: ${{ needs.checks.outputs.forked_workflow == 'false' && steps.base_exists.outputs.exists != 0 }} - name: Setup QEMU uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 with: platforms: ${{ matrix.platforms }} - if: ${{ needs.checks.outputs.forked_workflow == 'false' && steps.base_exists.outputs.exists != 0 }} - name: Build Base Container uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 with: file: build/Dockerfile context: "." - cache-from: type=gha,scope=${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.modules) || '' }} - cache-to: type=gha,scope=${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.modules) || '' }},mode=max + cache-from: type=gha,scope=${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.name) || '' }} + cache-to: type=gha,scope=${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.name) || '' }},mode=max target: common - tags: gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ matrix.images.type }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.modules) || '' }} + tags: gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ matrix.images.type }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.images.image }}${{ matrix.images.nap_modules && format('-{0}', steps.nap_modules.outputs.name) || '' }} platforms: ${{ matrix.images.platforms }} pull: true push: true build-args: | BUILD_OS=${{ matrix.images.image }} IC_VERSION=${{ needs.checks.outputs.ic_version }} - ${{ contains(matrix.images.image, 'nap') && format('NAP_MODULES={0}', matrix.images.nap_modules) || '' }} + ${{ steps.nap_modules.outputs.modules != '' && format('NAP_MODULES={0}', steps.nap_modules.outputs.modules) || '' }} secrets: | ${{ contains(matrix.images.image, 'nap') && format('"nginx-repo.crt={0}"', secrets.NGINX_AP_CRT) || format('"nginx-repo.crt={0}"', secrets.NGINX_CRT) }} ${{ contains(matrix.images.image, 'nap') && format('"nginx-repo.key={0}"', secrets.NGINX_AP_KEY) || format('"nginx-repo.key={0}"', secrets.NGINX_KEY) }} ${{ contains(matrix.images.image, 'ubi') && format('"rhel_license={0}"', secrets.RHEL_LICENSE) || '' }} if: ${{ needs.checks.outputs.forked_workflow == 'false' && steps.base_exists.outputs.exists != 0 }} + - name: Fetch Cached Artifacts + uses: actions/cache@v3 + with: + path: ${{ github.workspace }}/dist + key: nginx-ingress-${{ needs.checks.outputs.go_code_md5 }} + + - name: Check if test image exists + id: check-image + run: | + docker manifest inspect "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}" + shell: bash + continue-on-error: true + if: ${{ needs.checks.outputs.forked_workflow == 'false' }} + + - name: Build Test-Runner Container + uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 + with: + file: tests/Dockerfile + context: "." + cache-from: type=gha,scope=test-runner + tags: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}" + pull: true + push: ${{ needs.checks.outputs.forked_workflow == 'false' }} + load: ${{ needs.checks.outputs.forked_workflow == 'true' }} + if: ${{ needs.checks.outputs.forked_workflow == 'true' || steps.check-image.outcome == 'failure' }} + + - name: Build ${{ matrix.images.image }} Container + uses: docker/build-push-action@v3 + with: + file: build/Dockerfile + context: "." + cache-from: type=gha,scope=${{ matrix.images.image }}${{ steps.nap_modules.outputs.name != '' && format('-{0}', steps.nap_modules.outputs.name) || '' }} + cache-to: type=gha,scope=${{ matrix.images.image }}${{ steps.nap_modules.outputs.name != '' && format('-{0}', steps.nap_modules.outputs.name) || '' }},mode=max + target: goreleaser${{ needs.checks.outputs.forked_workflow == 'false' && '-prebuilt' || '' }} + tags: "docker.io/nginx/nginx${{ contains(matrix.images.image, 'plus') && '-plus' || '' }}-ingress:${{ matrix.images.image }}${{ contains(matrix.images.marker, 'dos') && '-dos' || '' }}${{ contains(matrix.images.marker, 'appprotect') && '-nap' || '' }}-${{ github.sha }}" + load: true + pull: true + build-args: | + BUILD_OS=${{ matrix.images.image }} + IC_VERSION=CI + PREBUILT_BASE_IMG=gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ matrix.images.type }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.images.image }}${{ steps.nap_modules.outputs.name != '' && format('-{0}', steps.nap_modules.outputs.name) || '' }} + ${{ contains(matrix.images.image, 'nap') && format('NAP_MODULES={0}', steps.nap_modules.outputs.modules) || '' }} + ${{ contains(matrix.images.marker, 'appprotect') && 'DEBIAN_VERSION=buster-slim' || '' }} + secrets: | + ${{ contains(matrix.images.image, 'nap') && format('"nginx-repo.crt={0}"', secrets.NGINX_AP_CRT) || format('"nginx-repo.crt={0}"', secrets.NGINX_CRT) }} + ${{ contains(matrix.images.image, 'nap') && format('"nginx-repo.key={0}"', secrets.NGINX_AP_KEY) || format('"nginx-repo.key={0}"', secrets.NGINX_KEY) }} + ${{ contains(matrix.images.image, 'ubi') && format('"rhel_license={0}"', secrets.RHEL_LICENSE) || '' }} + - name: Run Smoke Tests id: smoke-tests uses: ./.github/actions/smoke-tests with: - image: ${{ matrix.images.image != '' && matrix.images.image || 'debian' }} + image-type: ${{ matrix.images.image }} + image-name: docker.io/nginx/nginx${{ contains(matrix.images.image, 'plus') && '-plus' || '' }}-ingress + tag: ${{ matrix.images.image }}${{ contains(matrix.images.marker, 'dos') && '-dos' || '' }}${{ contains(matrix.images.marker, 'appprotect') && '-nap' || '' }}-${{ github.sha }} marker: ${{ matrix.images.marker != '' && matrix.images.marker || '' }} k8s-version: ${{ matrix.k8s }} - nginx-crt: ${{ contains(matrix.images.image, 'nap') && secrets.NGINX_AP_CRT || secrets.NGINX_CRT }} - nginx-key: ${{ contains(matrix.images.image, 'nap') && secrets.NGINX_AP_KEY || secrets.NGINX_KEY }} azure-ad-secret: ${{ secrets.AZURE_AD_AUTOMATION }} - gcr-workload-identity-secret: ${{ secrets.GCR_WORKLOAD_IDENTITY }} - gcr-service-account-secret: ${{ secrets.GCR_SERVICE_ACCOUNT }} - rhel-license: ${{ contains(matrix.images.image, 'ubi') && secrets.RHEL_LICENSE || '' }} - go-md5: ${{ needs.checks.outputs.go_code_md5 }} - base-image-md5: ${{ needs.checks.outputs.docker_md5 }} test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}" - forked-workflow: ${{ needs.checks.outputs.forked_workflow == 'true' }} - name: Upload Test Results uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 diff --git a/tests/Makefile b/tests/Makefile index 3d01992835..11c557a0dd 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -13,6 +13,7 @@ IP_FAMILY = dual IC_TYPE ?= nginx-ingress ## The Ingress Controller type to use, "nginx-ingress" or "nginx-plus-ingress". Defaults to "nginx-ingress" SHOW_IC_LOGS ?= no ## Should the tests show the Ingress Controller logs on failure, "yes" or "no". Defaults to "no" TEST_TAG ?= latest ## The Tag to use for the test image. e.g. commitsha +REGISTRY ?= docker.io ## The registry where the image is located. For example, docker.io PREFIX ?= nginx/nginx-ingress ## The name of the image. For example, nginx/nginx-ingress TAG ?= edge ## The tag of the image. For example, edge K8S_CLUSTER_NAME ?= local ## The name used when creating/using a Kind Kubernetes cluster @@ -20,7 +21,11 @@ K8S_CLUSTER_VERSION ?= $(shell grep -m1 'FROM kindest/node' < ${DOCKERFILEPA K8S_TIMEOUT ?= 75s ## The timeout used when creating a Kind Kubernetes cluster AD_SECRET ?= PYTEST_ARGS ?= +ifeq (${REGISTRY},) BUILD_IMAGE := $(strip $(PREFIX)):$(strip $(TAG)) +else +BUILD_IMAGE := $(strip $(REGISTRY))/$(strip $(PREFIX)):$(strip $(TAG)) +endif .PHONY: help ## Show this help help: ## Show available make targets @@ -79,7 +84,7 @@ delete-kind-cluster: ## Delete a Kind K8S cluster .PHONY: image-load image-load: ## Load the image into the Kind K8S cluster - @kind load docker-image docker.io/$(strip $(PREFIX)):$(strip $(TAG)) --name $(K8S_CLUSTER_NAME) + @kind load docker-image $(BUILD_IMAGE) --name $(K8S_CLUSTER_NAME) .PHONY: test-lint