diff --git a/.ci/jobs/e2e-custom.yml b/.ci/jobs/e2e-custom.yml new file mode 100644 index 0000000000..146f5c1b17 --- /dev/null +++ b/.ci/jobs/e2e-custom.yml @@ -0,0 +1,30 @@ +--- +- job: + description: Job that runs e2e tests against custom ECK image running in a dedicated k8s cluster in GKE. This Job is managed by JJB. + logrotate: + daysToKeep: 7 + numToKeep: 100 + artifactDaysToKeep: 5 + artifactNumToKeep: 10 + name: cloud-on-k8s-e2e-tests-custom + project-type: pipeline + parameters: + - string: + name: IMAGE + description: "Docker image with ECK" + - string: + name: VERSION + default: 1.12 + description: "Kubernetes version, default is 1.12" + concurrent: true + pipeline-scm: + scm: + - git: + url: https://github.com/elastic/cloud-on-k8s + branches: + - master + credentials-id: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba' + script-path: build/ci/e2e/custom_operator_image.jenkinsfile + lightweight-checkout: true + wrappers: + - ansicolor diff --git a/build/ci/Makefile b/build/ci/Makefile index a0ec04ea30..c30f5d378b 100644 --- a/build/ci/Makefile +++ b/build/ci/Makefile @@ -130,7 +130,26 @@ ci-e2e: vault-gke-creds -e "TESTS_MATCH=$(TESTS_MATCH)" \ -e "GKE_CLUSTER_VERSION=$(GKE_CLUSTER_VERSION)" \ cloud-on-k8s-ci-e2e \ - bash -c "make -C operators ci-e2e GKE_MACHINE_TYPE=n1-standard-8" + bash -c "make -C operators ci-e2e" + +# Run e2e tests in GKE against provided ECK image +ci-e2e-rc: vault-gke-creds + docker build -f Dockerfile -t cloud-on-k8s-ci-e2e . + docker run --rm -t \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v $(ROOT_DIR):$(GO_MOUNT_PATH) \ + -w $(GO_MOUNT_PATH) \ + -e "IMG_SUFFIX=-ci" \ + -e "GCLOUD_PROJECT=$(GCLOUD_PROJECT)" \ + -e "REGISTRY=$(REGISTRY)" \ + -e "REPOSITORY=$(GCLOUD_PROJECT)" \ + -e "GKE_CLUSTER_NAME=$(GKE_CLUSTER_NAME)" \ + -e "GKE_SERVICE_ACCOUNT_KEY_FILE=$(GO_MOUNT_PATH)/build/ci/$(GKE_CREDS_FILE)" \ + -e "TESTS_MATCH=$(TESTS_MATCH)" \ + -e "GKE_CLUSTER_VERSION=$(GKE_CLUSTER_VERSION)" \ + -e "OPERATOR_IMAGE=$(OPERATOR_IMAGE)" \ + cloud-on-k8s-ci-e2e \ + bash -c "make -C operators ci-e2e-rc" # Remove k8s cluster ci-e2e-delete-cluster: vault-gke-creds diff --git a/build/ci/e2e/custom_operator_image.jenkinsfile b/build/ci/e2e/custom_operator_image.jenkinsfile new file mode 100644 index 0000000000..7bbfe0af60 --- /dev/null +++ b/build/ci/e2e/custom_operator_image.jenkinsfile @@ -0,0 +1,53 @@ +pipeline { + + agent { + label 'linux' + } + + options { + timeout(time: 150, unit: 'MINUTES') + } + + environment { + VAULT_ADDR = credentials('vault-addr') + VAULT_ROLE_ID = credentials('vault-role-id') + VAULT_SECRET_ID = credentials('vault-secret-id') + REGISTRY = "eu.gcr.io" + GCLOUD_PROJECT = credentials('k8s-operators-gcloud-project') + GKE_CLUSTER_VERSION = "${VERSION}" + GKE_CLUSTER_NAME = "${BUILD_TAG}" + OPERATOR_IMAGE = "${IMAGE}" + LATEST_RELEASED_IMG = "${IMAGE}" + } + + stages { + stage('Checkout from GitHub') { + steps { + checkout scm + } + } + stage("Run E2E tests") { + steps { + sh 'make -C build/ci ci-e2e-rc' + } + } + } + + post { + unsuccessful { + script { + def msg = "E2E tests failed!\r\n" + env.BUILD_URL + slackSend botUser: true, + channel: '#cloud-k8s', + color: 'danger', + message: msg, + tokenCredentialId: 'cloud-ci-slack-integration-token' + } + } + cleanup { + sh 'make -C build/ci ci-e2e-delete-cluster' + cleanWs() + } + } + +} diff --git a/operators/Makefile b/operators/Makefile index 50ec5fcc09..9dd1587672 100644 --- a/operators/Makefile +++ b/operators/Makefile @@ -46,6 +46,9 @@ GO_LDFLAGS := -X github.com/elastic/cloud-on-k8s/operators/pkg/about.version=$(V -X github.com/elastic/cloud-on-k8s/operators/pkg/about.buildDate=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ') \ -X github.com/elastic/cloud-on-k8s/operators/pkg/about.buildSnapshot=$(SNAPSHOT) +# Setting for CI, if set to true will prevent building and using local Docker image +SKIP_DOCKER_COMMAND ?= false + ## -- Namespaces # namespace in which the global operator is deployed (see config/global-operator) @@ -146,7 +149,11 @@ endif endif # Deploy both the global and namespace operators against the current k8s cluster -deploy: check-gke install-crds docker-build docker-push apply-operators +deploy: check-gke install-crds +ifeq ($(SKIP_DOCKER_COMMAND), false) + $(MAKE) docker-build docker-push +endif + $(MAKE) apply-operators apply-operators: OPERATOR_IMAGE=$(OPERATOR_IMAGE) \ @@ -235,8 +242,10 @@ bootstrap-gke: require-gcloud-project ifeq ($(PSP), 1) kubectl apply -f config/dev/elastic-psp.yaml endif +ifeq ($(SKIP_DOCKER_COMMAND), false) # push "latest" operator image to be used for init containers when running the operator locally $(MAKE) docker-build docker-push OPERATOR_IMAGE=$(OPERATOR_IMAGE_LATEST) +endif delete-gke: require-gcloud-project GKE_CLUSTER_VERSION=$(GKE_CLUSTER_VERSION) ./hack/gke-cluster.sh delete @@ -341,6 +350,10 @@ ci: dep-vendor-only check-fmt generate check-local-changes unit integration e2e- # Let's use n1-standard-8 machine to have enough room for multiple pods on a single node. ci-e2e: ci-bootstrap-gke e2e +# Run e2e tests in gke using custom operator image +ci-e2e-rc: export SKIP_DOCKER_COMMAND=true +ci-e2e-rc: ci-bootstrap-gke e2e + ci-bootstrap-gke: PSP=1 GKE_MACHINE_TYPE=n1-standard-8 $(MAKE) bootstrap-gke