From e4b6d3e0a5c71a1a8a2c6ef04f87b565aea92d97 Mon Sep 17 00:00:00 2001 From: Michael Wilkerson Date: Mon, 31 Oct 2022 14:47:59 -0700 Subject: [PATCH] added nightly acceptance tests - New scheduled nightly that will run on release branches - Separate scheduled nightly that will run on main branch, this is so the release branch and main branch nightlies can run at a separate cadence (maybe we'll want to run release nightlies one night a week? a "weekly"?). For now both main and release will run nightly - Nightlies now also run the kind-tproxy acceptance tests --- .circleci/config.yml | 231 +++++++++++++++++++++++++++++-------------- 1 file changed, 157 insertions(+), 74 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2c9147e5bd..cb6be09a1d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,6 @@ cli-path: &cli-path cli acceptance-mod-path: &acceptance-mod-path acceptance acceptance-test-path: &acceptance-test-path acceptance/tests acceptance-framework-path: &acceptance-framework-path acceptance/framework -charts-consul-path: &charts-consul-path charts/consul helm-gen-path: &helm-gen-path hack/helm-reference-gen gke-terraform-path: &gke-terraform-path charts/consul/test/terraform/gke eks-terraform-path: &eks-terraform-path charts/consul/test/terraform/eks @@ -25,6 +24,10 @@ openshift-terraform-path: &openshift-terraform-path charts/consul/test/terraform # This image is built from test/docker/Test.dockerfile consul-helm-test-image: &consul-helm-test-image docker.mirror.hashicorp.services/hashicorpdev/consul-helm-test:0.15.0 +######################## +# COMMANDS +######################## +# Commands define a sequence of steps as a map to be executed and reused in jobs commands: install-prereqs: steps: @@ -51,6 +54,48 @@ commands: wget https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz tar -zxvf helm-v3.9.4-linux-amd64.tar.gz sudo mv linux-amd64/helm /usr/local/bin/helm + custom-checkout: + description: | + custom-checkout will perform a custom checkout procedure if provided with an alternative git reference, + otherwise it will run the CircleCI defined checkout command. This is needed as the CircleCI defined checkout + command does not support subsequent git actions after being called. + parameters: + git-ref: + type: string + default: "" + steps: + - when: + condition: << parameters.git-ref >> + steps: + - run: + name: Checkout code + command: | + ssh-keyscan github.com >> ~/.ssh/known_hosts + if [ -e '/home/circleci/project/.git' ] ; then + echo 'Fetching into existing repository' + existing_repo='true' + cd '/home/circleci/project' + git remote set-url origin "$CIRCLE_REPOSITORY_URL" || true + else + echo 'Cloning git repository' + existing_repo='false' + mkdir -p '/home/circleci/project' + cd '/home/circleci/project' + git clone --no-checkout "$CIRCLE_REPOSITORY_URL" . + fi + + if [ "$existing_repo" = 'true' ] || [ 'false' = 'true' ]; then + echo 'Fetching from remote repository' + git fetch --force origin + git fetch --force --tags origin + fi + + echo 'Checking out branch/tag' + git checkout --force "<< parameters.git-ref >>" + - unless: + condition: << parameters.git-ref >> + steps: + - checkout create-kind-clusters: parameters: version: @@ -84,6 +129,9 @@ commands: consul-k8s version run-acceptance-tests: + description: | + Runs the Kind acceptance tests using a provided consul-k8s image, or else attempts to use the image referenced by the + branch name and git reference of the current git commit parameters: failfast: type: boolean @@ -158,6 +206,11 @@ commands: -debug-directory="$TEST_RESULTS/debug" \ -consul-k8s-image=<< parameters.consul-k8s-image >> +######################## +# JOBS +######################## +# Jobs are a collection of steps. These are used in the workflows to define +# what gets run in the pipeline jobs: go-fmt-and-vet-control-plane: executor: go @@ -281,7 +334,6 @@ jobs: GOXPARALLEL: 2 # CircleCI containers are 2 CPU x 4GB RAM steps: - checkout - # Restore go module cache if there is one - restore_cache: keys: @@ -507,6 +559,9 @@ jobs: working_directory: charts/consul command: bats --jobs 4 ./test/unit + ########################### + # KIND ACCEPTANCE TEST JOBS + ########################### acceptance: environment: - TEST_RESULTS: /tmp/test-results @@ -606,9 +661,9 @@ jobs: - store_artifacts: path: /tmp/test-results - ########################## - # CLEANUP CLOUD RESOURCES - ########################## + ############################## + # CLEANUP CLOUD RESOURCES JOBS + ############################## cleanup-gcp-resources: docker: - image: *consul-helm-test-image @@ -665,9 +720,9 @@ jobs: fail_only: true failure_message: "EKS cleanup failed" - ######################## - # ACCEPTANCE TESTS - ######################## + ############################# + # CLOUD ACCEPTANCE TEST JOBS + ############################# acceptance-gke-1-23: parallelism: 2 environment: @@ -1099,7 +1154,7 @@ jobs: fail_only: true failure_message: "OpenShift acceptance tests failed. Check the logs at: ${CIRCLE_BUILD_URL}" - acceptance-kind-1-23-consul-nightly-1-12: + acceptance-kind-1-23-consul-compat-nightly-1-12: environment: - TEST_RESULTS: /tmp/test-results - CONSUL_IMAGE: "docker.mirror.hashicorp.services/hashicorppreview/consul-enterprise:1.12-dev" @@ -1110,29 +1165,8 @@ jobs: image: ubuntu-2004:202010-01 resource_class: xlarge steps: - - run: - name: checkout code - command: | - if [ -e '/home/circleci/project/.git' ] ; then - echo 'Fetching into existing repository' - existing_repo='true' - cd '/home/circleci/project' - git remote set-url origin "$CIRCLE_REPOSITORY_URL" || true - else - echo 'Cloning git repository' - existing_repo='false' - mkdir -p '/home/circleci/project' - cd '/home/circleci/project' - git clone --no-checkout "$CIRCLE_REPOSITORY_URL" . - fi - - if [ "$existing_repo" = 'true' ] || [ 'false' = 'true' ]; then - echo 'Fetching from remote repository' - git fetch --force --tags origin - fi - - echo 'Checking out tag' - git checkout --force "v$HELM_CHART_VERSION" + - custom-checkout: + git-ref: "v$HELM_CHART_VERSION" - install-prereqs - create-kind-clusters: version: "v1.23.0" @@ -1161,7 +1195,7 @@ jobs: fail_only: true failure_message: "Acceptance tests against Kind with Kubernetes v1.23 with Consul 1.12 nightly failed. Check the logs at: ${CIRCLE_BUILD_URL}" - acceptance-kind-1-23-consul-nightly-1-13: + acceptance-kind-1-23-consul-compat-nightly-1-13: environment: - TEST_RESULTS: /tmp/test-results - CONSUL_IMAGE: "docker.mirror.hashicorp.services/hashicorppreview/consul-enterprise:1.13-dev" @@ -1172,29 +1206,8 @@ jobs: image: ubuntu-2004:202010-01 resource_class: xlarge steps: - - run: - name: checkout code - command: | - if [ -e '/home/circleci/project/.git' ] ; then - echo 'Fetching into existing repository' - existing_repo='true' - cd '/home/circleci/project' - git remote set-url origin "$CIRCLE_REPOSITORY_URL" || true - else - echo 'Cloning git repository' - existing_repo='false' - mkdir -p '/home/circleci/project' - cd '/home/circleci/project' - git clone --no-checkout "$CIRCLE_REPOSITORY_URL" . - fi - - if [ "$existing_repo" = 'true' ] || [ 'false' = 'true' ]; then - echo 'Fetching from remote repository' - git fetch --force --tags origin - fi - - echo 'Checking out tag' - git checkout --force "v$HELM_CHART_VERSION" + - custom-checkout: + git-ref: "v$HELM_CHART_VERSION" - install-prereqs - create-kind-clusters: version: "v1.23.0" @@ -1223,15 +1236,23 @@ jobs: fail_only: true failure_message: "Acceptance tests against Kind with Kubernetes v1.23 with Consul 1.13 nightly failed. Check the logs at: ${CIRCLE_BUILD_URL}" +######################## +# WORKFLOWS +######################## +# Workflows are a set of rules for defining a collection of jobs and their run order. +# This is where the pipeline tests and builds are constructed and triggers for running these tests +# are defined workflows: version: 2 test-and-build: jobs: # Build this one control-plane binary so that acceptance and acceptance-tproxy will run - # The rest of these CircleCI jobs have been migrated to Github Actions. We need to wait until - # the summer of 2022 for larger puplic Github Action VMs be available before the acceptance tests can + # The rest of these CircleCI jobs have been migrated to GitHub Actions. We need to wait until + # there is support for a larger pool of runners before the acceptance tests can # be moved - # Run acceptance tests using the docker image built for the control plane + # Run acceptance tests using the docker image built for the control plane for this particular + # branch + # This is run on every PR - build-distro: OS: "linux" ARCH: "amd64 arm64" @@ -1252,10 +1273,73 @@ workflows: context: consul-ci requires: - dev-upload-docker - nightly-acceptance-tests: + + nightly-cleanup: triggers: - schedule: - cron: "0 0 * * *" + cron: "0 12 * * *" # Run at 12 pm UTC (5 am PST) + filters: + branches: + only: + - main + jobs: + - cleanup-gcp-resources + - cleanup-azure-resources + - cleanup-eks-resources + + + nightly-acceptance-tests-release: + description: | + Tests which run on a release branch nightly. These exist separate from the main + acceptance tests so that they can run at their own cadence, but + contains the same sequence of jobs. + triggers: + - schedule: + cron: "0 0 * * *" # Run at 12 am UTC (5 pm PST) + filters: + branches: + only: + - release/0.49.x + jobs: + - build-distro: + OS: "linux" + ARCH: "amd64 arm64" + name: build-distros-linux + - dev-upload-docker: + requires: + - build-distros-linux + # Disable until we can use UBI images. + # - acceptance-openshift + - acceptance-gke-1-23: + requires: + - dev-upload-docker + - acceptance-gke-cni-1-23: + requires: + - acceptance-gke-1-23 + - acceptance-eks-1-21: + requires: + - dev-upload-docker + - acceptance-eks-cni-1-21: + requires: + - acceptance-eks-1-21 + - acceptance-aks-1-22: + requires: + - dev-upload-docker + - acceptance-aks-cni-1-22: + requires: + - acceptance-aks-1-22 + - acceptance-tproxy: + requires: + - dev-upload-docker + + nightly-acceptance-tests-main: + description: | + Tests which run on the main branch nightly. These exist separate from the release + acceptance tests so that they can run at their own cadence, but + contains the same sequence of jobs. + triggers: + - schedule: + cron: "0 0 * * *" # Run at 12 am UTC (5 pm PST) filters: branches: only: @@ -1268,43 +1352,42 @@ workflows: - dev-upload-docker: requires: - build-distros-linux - - cleanup-gcp-resources - - cleanup-azure-resources - - cleanup-eks-resources # Disable until we can use UBI images. - # - acceptance-openshift: - # requires: - # - cleanup-azure-resources + # - acceptance-openshift - acceptance-gke-1-23: requires: - - cleanup-gcp-resources - dev-upload-docker - acceptance-gke-cni-1-23: requires: - acceptance-gke-1-23 - acceptance-eks-1-21: requires: - - cleanup-eks-resources - dev-upload-docker - acceptance-eks-cni-1-21: requires: - acceptance-eks-1-21 - acceptance-aks-1-22: requires: - - cleanup-azure-resources - dev-upload-docker - acceptance-aks-cni-1-22: requires: - acceptance-aks-1-22 + - acceptance-tproxy: + requires: + - dev-upload-docker - nightly-acceptance-tests-consul: + nightly-kind-acceptance-tests-consul-compatability: + description: | + Acceptance tests which run nightly to verify the compatibility between + a consul-k8s binary and it's consul version pair. Tests will be conducted + for up to n-2 previous Consul-k8s releases. triggers: - schedule: - cron: "0 0 * * *" + cron: "0 0 * * *" # Run at 12 am UTC (5 pm PST) filters: branches: only: - main jobs: - - acceptance-kind-1-23-consul-nightly-1-12 - - acceptance-kind-1-23-consul-nightly-1-13 + - acceptance-kind-1-23-consul-compat-nightly-1-12 + - acceptance-kind-1-23-consul-compat-nightly-1-13