From 439db1d00e139b186ebad0edd99424fb60bbf695 Mon Sep 17 00:00:00 2001 From: Jacob Salway Date: Fri, 30 Aug 2024 11:01:52 +1000 Subject: [PATCH] Run e2e-test on Kind in CI Signed-off-by: Jacob Salway --- .github/workflows/integration.yaml | 33 ++++++++++-------------------- Makefile | 33 +----------------------------- test/e2e/suit_test.go | 7 ++++++- 3 files changed, 18 insertions(+), 55 deletions(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 9380dfb2d..2379f6ad9 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -91,7 +91,7 @@ jobs: done build-helm-chart: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Determine branch name id: get_branch @@ -163,37 +163,26 @@ jobs: minikube image load docker.io/kubeflow/spark-operator:local ct install - integration-test: - runs-on: ubuntu-22.04 + e2e-test: + runs-on: ubuntu-latest steps: - name: Checkout source code uses: actions/checkout@v4 with: - fetch-depth: "0" + fetch-depth: 0 - name: Set up Go uses: actions/setup-go@v5 with: - go-version-file: "go.mod" + go-version-file: go.mod - - name: setup minikube - uses: manusa/actions-setup-minikube@v2.11.0 - with: - minikube version: v1.33.0 - kubernetes version: v1.30.0 - start args: --memory 6g --cpus=2 --addons ingress - github token: ${{ inputs.github-token }} + - name: Create a Kind cluster + uses: helm/kind-action@v1 - - name: Build local spark-operator docker image for minikube testing + - name: Build and load image to Kind cluster run: | docker build -t docker.io/kubeflow/spark-operator:local . - minikube image load docker.io/kubeflow/spark-operator:local - - # The integration tests are currently broken see: https://github.com/kubeflow/spark-operator/issues/1416 - # - name: Run chart-testing (integration test) - # run: make integration-test + kind load docker-image docker.io/kubeflow/spark-operator:local - - name: Setup tmate session - if: failure() - uses: mxschmitt/action-tmate@v3 - timeout-minutes: 15 + - name: Run e2e tests + run: make e2e-test IMAGE_TAG=local diff --git a/Makefile b/Makefile index e76f1dc2f..b1141992f 100644 --- a/Makefile +++ b/Makefile @@ -40,18 +40,12 @@ IMAGE_REPOSITORY ?= kubeflow/spark-operator IMAGE_TAG ?= $(VERSION) IMAGE ?= $(IMAGE_REGISTRY)/$(IMAGE_REPOSITORY):$(IMAGE_TAG) -# Kind cluster -KIND_CLUSTER_NAME ?= spark-operator -KIND_CONFIG_FILE ?= charts/spark-operator-chart/ci/kind-config.yaml -KIND_KUBE_CONFIG ?= $(HOME)/.kube/config - ## Location to install binaries LOCALBIN ?= $(shell pwd)/bin ## Versions KUSTOMIZE_VERSION ?= v5.4.1 CONTROLLER_TOOLS_VERSION ?= v0.15.0 -KIND_VERSION ?= v0.23.0 ENVTEST_VERSION ?= release-0.18 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION ?= 1.29.3 @@ -67,7 +61,6 @@ SPARKCTL ?= $(LOCALBIN)/sparkctl KUBECTL ?= kubectl KUSTOMIZE ?= $(LOCALBIN)/kustomize-$(KUSTOMIZE_VERSION) CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen-$(CONTROLLER_TOOLS_VERSION) -KIND ?= $(LOCALBIN)/kind-$(KIND_VERSION) ENVTEST ?= $(LOCALBIN)/setup-envtest-$(ENVTEST_VERSION) GOLANGCI_LINT ?= $(LOCALBIN)/golangci-lint-$(GOLANGCI_LINT_VERSION) GEN_CRD_API_REFERENCE_DOCS ?= $(LOCALBIN)/gen-crd-api-reference-docs-$(GEN_CRD_API_REFERENCE_DOCS_VERSION) @@ -153,7 +146,7 @@ unit-test: envtest ## Run unit tests. go test $(shell go list ./... | grep -v /e2e) -coverprofile cover.out .PHONY: e2e-test -e2e-test: envtest ## Run the e2e tests against a Kind k8s instance that is spun up. +e2e-test: envtest ## Run the e2e tests. This uses the current kubectl context. @echo "Running e2e tests..." go test ./test/e2e/ -v -ginkgo.v -timeout 30m @@ -239,25 +232,6 @@ helm-docs: helm-docs-plugin ## Generates markdown documentation for helm charts ##@ Deployment -ifndef ignore-not-found - ignore-not-found = false -endif - -.PHONY: kind-create-cluster -kind-create-cluster: kind ## Create a kind cluster for integration tests. - if ! $(KIND) get clusters 2>/dev/null | grep -q "^$(KIND_CLUSTER_NAME)$$"; then \ - kind create cluster --name $(KIND_CLUSTER_NAME) --config $(KIND_CONFIG_FILE) --kubeconfig $(KIND_KUBE_CONFIG); \ - fi - -.PHONY: kind-load-image -kind-load-image: kind-create-cluster docker-build ## Load the image into the kind cluster. - $(KIND) load docker-image --name $(KIND_CLUSTER_NAME) $(IMAGE) - -.PHONY: kind-delete-custer -kind-delete-custer: kind ## Delete the created kind cluster. - $(KIND) delete cluster --name $(KIND_CLUSTER_NAME) && \ - rm -f $(KIND_KUBE_CONFIG) - .PHONY: install install-crd: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f - @@ -290,11 +264,6 @@ controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessar $(CONTROLLER_GEN): $(LOCALBIN) $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) -.PHONY: kind -kind: $(KIND) ## Download kind locally if necessary. -$(KIND): $(LOCALBIN) - $(call go-install-tool,$(KIND),sigs.k8s.io/kind,$(KIND_VERSION)) - .PHONY: envtest envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. $(ENVTEST): $(LOCALBIN) diff --git a/test/e2e/suit_test.go b/test/e2e/suit_test.go index 4c60f9762..c70fa4495 100644 --- a/test/e2e/suit_test.go +++ b/test/e2e/suit_test.go @@ -128,7 +128,12 @@ var _ = BeforeSuite(func() { chart, err := loader.Load(chartPath) Expect(err).NotTo(HaveOccurred()) Expect(chart).NotTo(BeNil()) - release, err := installAction.Run(chart, nil) + + values := make(map[string]interface{}) + if imageTag := os.Getenv("IMAGE_TAG"); imageTag != "" { + values["image.tag"] = imageTag + } + release, err := installAction.Run(chart, values) Expect(err).NotTo(HaveOccurred()) Expect(release).NotTo(BeNil()) })