Skip to content

Commit

Permalink
added nightly acceptance tests
Browse files Browse the repository at this point in the history 
- New scheduled nightly that will run on release branches
- Separate scheduled nightly that will run on main branch, this is so the release branch and main branch nightlies can run at a separate cadence (maybe we'll want to run release nightlies one night a week? a "weekly"?). For now both main and release will run nightly
- Nightlies now also run the kind-tproxy acceptance tests
  • Loading branch information
@wilkermichael
wilkermichael committed Nov 3, 2022
1 parent edb10f9 commit e4b6d3e
Showing 1 changed file with 157 additions and 74 deletions.
231 changes: 157 additions & 74 deletions .circleci/config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ cli-path: &cli-path cli
acceptance-mod-path: &acceptance-mod-path acceptance
acceptance-test-path: &acceptance-test-path acceptance/tests
acceptance-framework-path: &acceptance-framework-path acceptance/framework
charts-consul-path: &charts-consul-path charts/consul
helm-gen-path: &helm-gen-path hack/helm-reference-gen
gke-terraform-path: &gke-terraform-path charts/consul/test/terraform/gke
eks-terraform-path: &eks-terraform-path charts/consul/test/terraform/eks
Expand All @@ -25,6 +24,10 @@ openshift-terraform-path: &openshift-terraform-path charts/consul/test/terraform
# This image is built from test/docker/Test.dockerfile
consul-helm-test-image: &consul-helm-test-image docker.mirror.hashicorp.services/hashicorpdev/consul-helm-test:0.15.0

########################
# COMMANDS
########################
# Commands define a sequence of steps as a map to be executed and reused in jobs
commands:
install-prereqs:
steps:
Expand All @@ -51,6 +54,48 @@ commands:
wget https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz
tar -zxvf helm-v3.9.4-linux-amd64.tar.gz
sudo mv linux-amd64/helm /usr/local/bin/helm
custom-checkout:
description: |
custom-checkout will perform a custom checkout procedure if provided with an alternative git reference,
otherwise it will run the CircleCI defined checkout command. This is needed as the CircleCI defined checkout
command does not support subsequent git actions after being called.
parameters:
git-ref:
type: string
default: ""
steps:
- when:
condition: << parameters.git-ref >>
steps:
- run:
name: Checkout code
command: |
ssh-keyscan github.com >> ~/.ssh/known_hosts
if [ -e '/home/circleci/project/.git' ] ; then
echo 'Fetching into existing repository'
existing_repo='true'
cd '/home/circleci/project'
git remote set-url origin "$CIRCLE_REPOSITORY_URL" || true
else
echo 'Cloning git repository'
existing_repo='false'
mkdir -p '/home/circleci/project'
cd '/home/circleci/project'
git clone --no-checkout "$CIRCLE_REPOSITORY_URL" .
fi
if [ "$existing_repo" = 'true' ] || [ 'false' = 'true' ]; then
echo 'Fetching from remote repository'
git fetch --force origin
git fetch --force --tags origin
fi
echo 'Checking out branch/tag'
git checkout --force "<< parameters.git-ref >>"
- unless:
condition: << parameters.git-ref >>
steps:
- checkout
create-kind-clusters:
parameters:
version:
Expand Down Expand Up @@ -84,6 +129,9 @@ commands:
consul-k8s version
run-acceptance-tests:
description: |
Runs the Kind acceptance tests using a provided consul-k8s image, or else attempts to use the image referenced by the
branch name and git reference of the current git commit
parameters:
failfast:
type: boolean
Expand Down Expand Up @@ -158,6 +206,11 @@ commands:
-debug-directory="$TEST_RESULTS/debug" \
-consul-k8s-image=<< parameters.consul-k8s-image >>
########################
# JOBS
########################
# Jobs are a collection of steps. These are used in the workflows to define
# what gets run in the pipeline
jobs:
go-fmt-and-vet-control-plane:
executor: go
Expand Down Expand Up @@ -281,7 +334,6 @@ jobs:
GOXPARALLEL: 2 # CircleCI containers are 2 CPU x 4GB RAM
steps:
- checkout

# Restore go module cache if there is one
- restore_cache:
keys:
Expand Down Expand Up @@ -507,6 +559,9 @@ jobs:
working_directory: charts/consul
command: bats --jobs 4 ./test/unit

###########################
# KIND ACCEPTANCE TEST JOBS
###########################
acceptance:
environment:
- TEST_RESULTS: /tmp/test-results
Expand Down Expand Up @@ -606,9 +661,9 @@ jobs:
- store_artifacts:
path: /tmp/test-results

##########################
# CLEANUP CLOUD RESOURCES
##########################
##############################
# CLEANUP CLOUD RESOURCES JOBS
##############################
cleanup-gcp-resources:
docker:
- image: *consul-helm-test-image
Expand Down Expand Up @@ -665,9 +720,9 @@ jobs:
fail_only: true
failure_message: "EKS cleanup failed"

########################
# ACCEPTANCE TESTS
########################
#############################
# CLOUD ACCEPTANCE TEST JOBS
#############################
acceptance-gke-1-23:
parallelism: 2
environment:
Expand Down Expand Up @@ -1099,7 +1154,7 @@ jobs:
fail_only: true
failure_message: "OpenShift acceptance tests failed. Check the logs at: ${CIRCLE_BUILD_URL}"

acceptance-kind-1-23-consul-nightly-1-12:
acceptance-kind-1-23-consul-compat-nightly-1-12:
environment:
- TEST_RESULTS: /tmp/test-results
- CONSUL_IMAGE: "docker.mirror.hashicorp.services/hashicorppreview/consul-enterprise:1.12-dev"
Expand All @@ -1110,29 +1165,8 @@ jobs:
image: ubuntu-2004:202010-01
resource_class: xlarge
steps:
- run:
name: checkout code
command: |
if [ -e '/home/circleci/project/.git' ] ; then
echo 'Fetching into existing repository'
existing_repo='true'
cd '/home/circleci/project'
git remote set-url origin "$CIRCLE_REPOSITORY_URL" || true
else
echo 'Cloning git repository'
existing_repo='false'
mkdir -p '/home/circleci/project'
cd '/home/circleci/project'
git clone --no-checkout "$CIRCLE_REPOSITORY_URL" .
fi
if [ "$existing_repo" = 'true' ] || [ 'false' = 'true' ]; then
echo 'Fetching from remote repository'
git fetch --force --tags origin
fi
echo 'Checking out tag'
git checkout --force "v$HELM_CHART_VERSION"
- custom-checkout:
git-ref: "v$HELM_CHART_VERSION"
- install-prereqs
- create-kind-clusters:
version: "v1.23.0"
Expand Down Expand Up @@ -1161,7 +1195,7 @@ jobs:
fail_only: true
failure_message: "Acceptance tests against Kind with Kubernetes v1.23 with Consul 1.12 nightly failed. Check the logs at: ${CIRCLE_BUILD_URL}"

acceptance-kind-1-23-consul-nightly-1-13:
acceptance-kind-1-23-consul-compat-nightly-1-13:
environment:
- TEST_RESULTS: /tmp/test-results
- CONSUL_IMAGE: "docker.mirror.hashicorp.services/hashicorppreview/consul-enterprise:1.13-dev"
Expand All @@ -1172,29 +1206,8 @@ jobs:
image: ubuntu-2004:202010-01
resource_class: xlarge
steps:
- run:
name: checkout code
command: |
if [ -e '/home/circleci/project/.git' ] ; then
echo 'Fetching into existing repository'
existing_repo='true'
cd '/home/circleci/project'
git remote set-url origin "$CIRCLE_REPOSITORY_URL" || true
else
echo 'Cloning git repository'
existing_repo='false'
mkdir -p '/home/circleci/project'
cd '/home/circleci/project'
git clone --no-checkout "$CIRCLE_REPOSITORY_URL" .
fi
if [ "$existing_repo" = 'true' ] || [ 'false' = 'true' ]; then
echo 'Fetching from remote repository'
git fetch --force --tags origin
fi
echo 'Checking out tag'
git checkout --force "v$HELM_CHART_VERSION"
- custom-checkout:
git-ref: "v$HELM_CHART_VERSION"
- install-prereqs
- create-kind-clusters:
version: "v1.23.0"
Expand Down Expand Up @@ -1223,15 +1236,23 @@ jobs:
fail_only: true
failure_message: "Acceptance tests against Kind with Kubernetes v1.23 with Consul 1.13 nightly failed. Check the logs at: ${CIRCLE_BUILD_URL}"

########################
# WORKFLOWS
########################
# Workflows are a set of rules for defining a collection of jobs and their run order.
# This is where the pipeline tests and builds are constructed and triggers for running these tests
# are defined
workflows:
version: 2
test-and-build:
jobs:
# Build this one control-plane binary so that acceptance and acceptance-tproxy will run
# The rest of these CircleCI jobs have been migrated to Github Actions. We need to wait until
# the summer of 2022 for larger puplic Github Action VMs be available before the acceptance tests can
# The rest of these CircleCI jobs have been migrated to GitHub Actions. We need to wait until
# there is support for a larger pool of runners before the acceptance tests can
# be moved
# Run acceptance tests using the docker image built for the control plane
# Run acceptance tests using the docker image built for the control plane for this particular
# branch
# This is run on every PR
- build-distro:
OS: "linux"
ARCH: "amd64 arm64"
Expand All @@ -1252,10 +1273,73 @@ workflows:
context: consul-ci
requires:
- dev-upload-docker
nightly-acceptance-tests:

nightly-cleanup:
triggers:
- schedule:
cron: "0 0 * * *"
cron: "0 12 * * *" # Run at 12 pm UTC (5 am PST)
filters:
branches:
only:
- main
jobs:
- cleanup-gcp-resources
- cleanup-azure-resources
- cleanup-eks-resources


nightly-acceptance-tests-release:
description: |
Tests which run on a release branch nightly. These exist separate from the main
acceptance tests so that they can run at their own cadence, but
contains the same sequence of jobs.
triggers:
- schedule:
cron: "0 0 * * *" # Run at 12 am UTC (5 pm PST)
filters:
branches:
only:
- release/0.49.x
jobs:
- build-distro:
OS: "linux"
ARCH: "amd64 arm64"
name: build-distros-linux
- dev-upload-docker:
requires:
- build-distros-linux
# Disable until we can use UBI images.
# - acceptance-openshift
- acceptance-gke-1-23:
requires:
- dev-upload-docker
- acceptance-gke-cni-1-23:
requires:
- acceptance-gke-1-23
- acceptance-eks-1-21:
requires:
- dev-upload-docker
- acceptance-eks-cni-1-21:
requires:
- acceptance-eks-1-21
- acceptance-aks-1-22:
requires:
- dev-upload-docker
- acceptance-aks-cni-1-22:
requires:
- acceptance-aks-1-22
- acceptance-tproxy:
requires:
- dev-upload-docker

nightly-acceptance-tests-main:
description: |
Tests which run on the main branch nightly. These exist separate from the release
acceptance tests so that they can run at their own cadence, but
contains the same sequence of jobs.
triggers:
- schedule:
cron: "0 0 * * *" # Run at 12 am UTC (5 pm PST)
filters:
branches:
only:
Expand All @@ -1268,43 +1352,42 @@ workflows:
- dev-upload-docker:
requires:
- build-distros-linux
- cleanup-gcp-resources
- cleanup-azure-resources
- cleanup-eks-resources
# Disable until we can use UBI images.
# - acceptance-openshift:
# requires:
# - cleanup-azure-resources
# - acceptance-openshift
- acceptance-gke-1-23:
requires:
- cleanup-gcp-resources
- dev-upload-docker
- acceptance-gke-cni-1-23:
requires:
- acceptance-gke-1-23
- acceptance-eks-1-21:
requires:
- cleanup-eks-resources
- dev-upload-docker
- acceptance-eks-cni-1-21:
requires:
- acceptance-eks-1-21
- acceptance-aks-1-22:
requires:
- cleanup-azure-resources
- dev-upload-docker
- acceptance-aks-cni-1-22:
requires:
- acceptance-aks-1-22
- acceptance-tproxy:
requires:
- dev-upload-docker

nightly-acceptance-tests-consul:
nightly-kind-acceptance-tests-consul-compatability:
description: |
Acceptance tests which run nightly to verify the compatibility between
a consul-k8s binary and it's consul version pair. Tests will be conducted
for up to n-2 previous Consul-k8s releases.
triggers:
- schedule:
cron: "0 0 * * *"
cron: "0 0 * * *" # Run at 12 am UTC (5 pm PST)
filters:
branches:
only:
- main
jobs:
- acceptance-kind-1-23-consul-nightly-1-12
- acceptance-kind-1-23-consul-nightly-1-13
- acceptance-kind-1-23-consul-compat-nightly-1-12
- acceptance-kind-1-23-consul-compat-nightly-1-13

0 comments on commit e4b6d3e

Please sign in to comment.