From a4c25aff4d7377939f0f1d78fb31153fe13db052 Mon Sep 17 00:00:00 2001 From: Naadir Jeewa Date: Wed, 8 Jul 2020 12:54:53 +0100 Subject: [PATCH] e2e framework: Add conformance testing to test framework --- Makefile | 107 ++++-- go.mod | 1 + go.sum | 1 + test/e2e/Makefile | 8 + test/e2e/common.go | 58 +--- test/e2e/config/docker-ci.yaml | 1 + test/e2e/config/docker-dev-conformance.yaml | 124 +++++++ test/e2e/config/docker-dev.yaml | 1 + test/e2e/conformance/ci_artifacts.go | 137 ++++++++ test/e2e/conformance/ci_artifacts_test.go | 39 +++ test/e2e/conformance/conformance.go | 139 ++++++++ .../e2e/conformance/conformance_suite_test.go | 179 ++++++++++ test/e2e/conformance/conformance_test.go | 39 +++ .../cluster-template-ci.yaml | 2 +- .../cluster-template-kcp-adoption.yaml | 2 +- .../cluster-template.yaml | 2 +- .../platform-kustomization.yaml | 18 + test/e2e/data/kubetest/conformance-fast.yaml | 8 + test/e2e/data/kubetest/conformance.yaml | 7 + test/e2e/e2e_suite_test.go | 158 ++++----- test/e2e/internal/setup/setup.go | 230 +++++++++++++ test/e2e/kcp_adoption.go | 24 +- test/e2e/kcp_upgrade.go | 24 +- test/e2e/kcp_upgrade_ci_artifacts.go | 213 ++++++++++++ test/e2e/kcp_upgrade_ci_artifacts_test.go | 39 +++ test/e2e/md_upgrades.go | 24 +- test/e2e/mhc_remediations.go | 24 +- test/e2e/quick_start.go | 24 +- test/e2e/self_hosted.go | 29 +- test/framework/alltypes_helpers.go | 16 + test/framework/bootstrap/kind_provider.go | 2 +- test/framework/bootstrap/kind_util.go | 2 +- test/framework/cluster_helpers.go | 10 +- test/framework/cluster_proxy.go | 2 +- test/framework/clusterctl/client.go | 2 +- .../clusterctl/clusterctl_helpers.go | 2 +- test/framework/clusterctl/e2e_config.go | 13 +- test/framework/config.go | 14 + test/framework/control_plane.go | 2 +- test/framework/controlpane_helpers.go | 10 +- test/framework/convenience.go | 30 ++ test/framework/daemonset_helpers.go | 2 +- test/framework/deployment_helpers.go | 6 +- test/framework/deprecated.go | 2 +- .../log/log.go => ginkgoextensions/output.go} | 18 +- .../log.go => framework/kubetest/bindata.go} | 12 +- .../data/debian_injection_script.envsubst.sh | 90 +++++ .../kubetest/data/kustomization.yaml | 8 + test/framework/kubetest/run.go | 211 ++++++++++++ test/framework/kubetest/setup.go | 60 ++++ test/framework/kubetest/template.go | 177 ++++++++++ .../kubetest/zz_generated.bindata.go | 308 ++++++++++++++++++ test/framework/log/log.go | 185 +++++++++++ test/framework/machine_helpers.go | 4 +- test/framework/machinedeployment_helpers.go | 2 +- test/framework/machines.go | 4 +- test/framework/namespace_helpers.go | 2 +- test/framework/pod_helpers.go | 2 +- test/framework/suite_helpers.go | 46 +++ .../config/manager/manager_image_patch.yaml | 2 +- .../controllers/dockermachine_controller.go | 2 +- 61 files changed, 2660 insertions(+), 250 deletions(-) create mode 100644 test/e2e/config/docker-dev-conformance.yaml create mode 100644 test/e2e/conformance/ci_artifacts.go create mode 100644 test/e2e/conformance/ci_artifacts_test.go create mode 100644 test/e2e/conformance/conformance.go create mode 100644 test/e2e/conformance/conformance_suite_test.go create mode 100644 test/e2e/conformance/conformance_test.go create mode 100644 test/e2e/data/infrastructure-docker/platform-kustomization.yaml create mode 100644 test/e2e/data/kubetest/conformance-fast.yaml create mode 100644 test/e2e/data/kubetest/conformance.yaml create mode 100644 test/e2e/internal/setup/setup.go create mode 100644 test/e2e/kcp_upgrade_ci_artifacts.go create mode 100644 test/e2e/kcp_upgrade_ci_artifacts_test.go rename test/framework/{internal/log/log.go => ginkgoextensions/output.go} (65%) rename test/{e2e/internal/log/log.go => framework/kubetest/bindata.go} (68%) create mode 100644 test/framework/kubetest/data/debian_injection_script.envsubst.sh create mode 100644 test/framework/kubetest/data/kustomization.yaml create mode 100644 test/framework/kubetest/run.go create mode 100644 test/framework/kubetest/setup.go create mode 100644 test/framework/kubetest/template.go create mode 100644 test/framework/kubetest/zz_generated.bindata.go create mode 100644 test/framework/log/log.go create mode 100644 test/framework/suite_helpers.go diff --git a/Makefile b/Makefile index a16ceef5d800..c39f94506b87 100644 --- a/Makefile +++ b/Makefile @@ -44,25 +44,38 @@ TOOLS_BIN_DIR := $(TOOLS_DIR)/bin BIN_DIR := bin E2E_FRAMEWORK_DIR := test/framework CAPD_DIR := test/infrastructure/docker -RELEASE_NOTES_BIN := bin/release-notes -RELEASE_NOTES := $(TOOLS_DIR)/$(RELEASE_NOTES_BIN) -LINK_CHECKER_BIN := bin/liche -LINK_CHECKER := $(TOOLS_DIR)/$(LINK_CHECKER_BIN) -GO_APIDIFF_BIN := bin/go-apidiff -GO_APIDIFF := $(TOOLS_DIR)/$(GO_APIDIFF_BIN) -ENVSUBST_BIN := bin/envsubst -ENVSUBST := $(TOOLS_DIR)/$(ENVSUBST_BIN) + +PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH) +export PATH # Binaries. # Need to use abspath so we can invoke these from subdirectories -KUSTOMIZE := $(abspath $(TOOLS_BIN_DIR)/kustomize) -CONTROLLER_GEN := $(abspath $(TOOLS_BIN_DIR)/controller-gen) -GOLANGCI_LINT := $(abspath $(TOOLS_BIN_DIR)/golangci-lint) -CONVERSION_GEN := $(abspath $(TOOLS_BIN_DIR)/conversion-gen) -ENVSUBST := $(abspath $(TOOLS_BIN_DIR)/envsubst) + +KUSTOMIZE_BIN := $(TOOLS_BIN_DIR)/kustomize +CONTROLLER_GEN_BIN := $(TOOLS_BIN_DIR)/controller-gen +GOLANGCI_LINT_BIN := $(TOOLS_BIN_DIR)/golangci-lint +CONVERSION_GEN_BIN := $(TOOLS_BIN_DIR)/conversion-gen +DEFAULTER_GEN_BIN := $(TOOLS_BIN_DIR)/defaulter-gen +GINKGO_BIN := $(TOOLS_BIN_DIR)/ginkgo +ENVSUBST_BIN := $(TOOLS_BIN_DIR)/envsubst +GOBINDATA_BIN := $(TOOLS_BIN_DIR)/go-bindata +RELEASE_NOTES_BIN := $(TOOLS_BIN_DIR)/release-notes +GO_APIDIFF_BIN := $(TOOLS_BIN_DIR)/go-apidiff +LINK_CHECKER_BIN := $(TOOLS_BIN_DIR)/link-checker + +KUSTOMIZE := $(abspath $(KUSTOMIZE_BIN)) +CONTROLLER_GEN := $(abspath $(CONTROLLER_GEN)) +GOLANGCI_LINT := $(abspath $(GOLANGCI_LINT)) +CONVERSION_GEN := $(abspath $(CONVERSION_GEN)) +DEFAULTER_GEN := $(abspath $(DEFAULTER_GEN)) +GINKGO := $(abspath $(GINKGO_BIN)) +ENVSUBST := $(abspath $(ENVSUBST_BIN)) +GOBINDATA := $(abspath $(GOBINDATA)) +RELEASE_NOTES := $(abspath $(RELEASE_NOTES_BIN)) +GO_APIDIFF := $(abspath $(GO_APIDIFF_BIN)) +LINK_CHECKER := $(abspath $(LINK_CHECKER_BIN)) # Bindata. -GOBINDATA := $(abspath $(TOOLS_BIN_DIR)/go-bindata) GOBINDATA_CLUSTERCTL_DIR := cmd/clusterctl/config CLOUDINIT_PKG_DIR := bootstrap/kubeadm/internal/cloudinit CLOUDINIT_GENERATED := $(CLOUDINIT_PKG_DIR)/zz_generated.bindata.go @@ -127,9 +140,13 @@ docker-build-e2e: ## Rebuild all Cluster API provider images to be used in the e $(MAKE) -C test/infrastructure/docker docker-build REGISTRY=gcr.io/k8s-staging-cluster-api .PHONY: test-e2e -test-e2e: ## Run the e2e tests +test-e2e: $(GINKGO_BIN) ## Run the e2e tests $(MAKE) -C test/e2e run +.PHONY: test-conformance +test-conformance: $(GINKGO_BIN) $(KUSTOMIZE_BIN) ## Run the e2e tests + $(MAKE) -C test/e2e conformance + ## -------------------------------------- ## Binaries ## -------------------------------------- @@ -156,32 +173,51 @@ managers: ## Build all managers clusterctl: ## Build clusterctl binary go build -ldflags "$(LDFLAGS)" -o bin/clusterctl sigs.k8s.io/cluster-api/cmd/clusterctl -$(KUSTOMIZE): $(TOOLS_DIR)/go.mod # Build kustomize from tools folder. - cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/kustomize sigs.k8s.io/kustomize/kustomize/v3 +## -------------------------------------- +## Tooling Binaries +## -------------------------------------- + +$(TOOLS_BIN_DIR): + mkdir -p $@ -$(CONTROLLER_GEN): $(TOOLS_DIR)/go.mod # Build controller-gen from tools folder. - cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/controller-gen sigs.k8s.io/controller-tools/cmd/controller-gen +$(TOOLS_SHARE_DIR): + mkdir -p $@ -$(GOLANGCI_LINT): $(TOOLS_DIR)/go.mod # Build golangci-lint from tools folder. - cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/golangci-lint github.com/golangci/golangci-lint/cmd/golangci-lint +$(GOBINDATA_BIN): $(TOOLS_DIR)/go.mod # Build go-bindata from tools folder. + cd $(TOOLS_DIR); go build -tags=tools -o $(subst hack/tools/,,$@) github.com/go-bindata/go-bindata/go-bindata -$(CONVERSION_GEN): $(TOOLS_DIR)/go.mod - cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/conversion-gen k8s.io/code-generator/cmd/conversion-gen +$(LINK_CHECKER_BIN): $(TOOLS_DIR)/go.mod + cd $(TOOLS_DIR) && go build -tags=tools -o $(subst hack/tools/,,$@) github.com/raviqqe/liche -$(GOBINDATA): $(TOOLS_DIR)/go.mod # Build go-bindata from tools folder. - cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/go-bindata github.com/go-bindata/go-bindata/go-bindata +$(GO_APIDIFF_BIN): $(TOOLS_DIR)/go.mod + cd $(TOOLS_DIR) && go build -tags=tools -o $(subst hack/tools/,,$@) github.com/joelanford/go-apidiff -$(RELEASE_NOTES): $(TOOLS_DIR)/go.mod - cd $(TOOLS_DIR) && go build -tags=tools -o $(RELEASE_NOTES_BIN) ./release +$(CONTROLLER_GEN_BIN): $(TOOLS_DIR)/go.mod # Build controller-gen from tools folder. + cd $(TOOLS_DIR); go build -tags=tools -o $(subst hack/tools/,,$@) sigs.k8s.io/controller-tools/cmd/controller-gen -$(LINK_CHECKER): $(TOOLS_DIR)/go.mod - cd $(TOOLS_DIR) && go build -tags=tools -o $(LINK_CHECKER_BIN) github.com/raviqqe/liche +$(ENVSUBST_BIN): $(TOOLS_DIR)/go.mod # Build envsubst from tools folder. + cd $(TOOLS_DIR); go build -tags=tools -o $(subst hack/tools/,,$@) github.com/a8m/envsubst/cmd/envsubst -$(GO_APIDIFF): $(TOOLS_DIR)/go.mod - cd $(TOOLS_DIR) && go build -tags=tools -o $(GO_APIDIFF_BIN) github.com/joelanford/go-apidiff +$(GOLANGCI_LINT_BIN): $(TOOLS_DIR)/go.mod # Build golangci-lint from tools folder. + cd $(TOOLS_DIR); go build -tags=tools -o $(subst hack/tools/,,$@) github.com/golangci/golangci-lint/cmd/golangci-lint -$(ENVSUBST): $(TOOLS_DIR)/go.mod - cd $(TOOLS_DIR) && go build -tags=tools -o $(ENVSUBST_BIN) github.com/drone/envsubst/cmd/envsubst +$(MOCKGEN_BIN): $(TOOLS_DIR)/go.mod # Build mockgen from tools folder. + cd $(TOOLS_DIR); go build -tags=tools -o $(subst hack/tools/,,$@) github.com/golang/mock/mockgen + +$(CONVERSION_GEN_BIN): $(TOOLS_DIR)/go.mod + cd $(TOOLS_DIR); go build -tags=tools -o $(subst hack/tools/,,$@) k8s.io/code-generator/cmd/conversion-gen + +$(DEFAULTER_GEN_BIN): $(TOOLS_DIR)/go.mod + cd $(TOOLS_DIR); go build -tags=tools -o $(subst hack/tools/,,$@) k8s.io/code-generator/cmd/defaulter-gen + +$(KUSTOMIZE_BIN): $(TOOLS_DIR)/go.mod # Build kustomize from tools folder. + cd $(TOOLS_DIR); go build -tags=tools -o $(subst hack/tools/,,$@) sigs.k8s.io/kustomize/kustomize/v3 + +$(RELEASE_NOTES_BIN): $(TOOLS_DIR)/go.mod + cd $(TOOLS_DIR) && go build -tags tools -o $(subst hack/tools/,,$@) sigs.k8s.io/cluster-api/hack/tools/release + +$(GINKGO_BIN): $(TOOLS_DIR)/go.mod + cd $(TOOLS_DIR) && go build -tags=tools -o $(subst hack/tools/,,$@) github.com/onsi/ginkgo/ginkgo envsubst: $(ENVSUBST) ## Build a local copy of envsubst. kustomize: $(KUSTOMIZE) ## Build a local copy of kustomize. @@ -529,6 +565,11 @@ docker-build-example-provider: ## Build the docker image for example provider clean: ## Remove all generated files $(MAKE) clean-bin $(MAKE) clean-book + $(MAKE) clean-artifacts + +.PHONY: clean-artifacts +clean-bin: ## Remove all test artifacts + rm -rf _artifacts .PHONY: clean-bin clean-bin: ## Remove all generated binaries diff --git a/go.mod b/go.mod index c08240be84db..ef22e86fe20b 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,7 @@ require ( k8s.io/cluster-bootstrap v0.17.8 k8s.io/component-base v0.17.9 k8s.io/klog v1.0.0 + k8s.io/klog/v2 v2.0.0 k8s.io/utils v0.0.0-20200619165400-6e3d28b6ed19 sigs.k8s.io/controller-runtime v0.5.10 sigs.k8s.io/kind v0.7.1-0.20200303021537-981bd80d3802 diff --git a/go.sum b/go.sum index 2405ecdce4af..fe63c8ddfadf 100644 --- a/go.sum +++ b/go.sum @@ -661,6 +661,7 @@ modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= +sigs.k8s.io/cluster-api/test/framework v0.0.0-20200304170348-97097699f713 h1:5xCiWE7khnVT5JhwKd2XnJP71wxlnX1aRaWQsNJd1xw= sigs.k8s.io/controller-runtime v0.5.10 h1:IJ2zO+BeNvTJEo2W1Kho1+X756QroQjYCKIzEYYqsI8= sigs.k8s.io/controller-runtime v0.5.10/go.mod h1:OTqxLuz7gVcrq+BHGUgedRu6b2VIKCEc7Pu4Jbwui0A= sigs.k8s.io/kind v0.7.1-0.20200303021537-981bd80d3802 h1:L6/8hETA7jvdx3xBcbDifrIN2xaYHE7tA58n+Kdp2Zw= diff --git a/test/e2e/Makefile b/test/e2e/Makefile index cd74c81632f7..89c9f8abbe5e 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -52,6 +52,7 @@ TEST_E2E_DIR := $(REPO_ROOT)/test/e2e GINKGO_FOCUS ?= GINKGO_NODES ?= 1 E2E_CONF_FILE ?= ${REPO_ROOT}/test/e2e/config/docker-dev.yaml +CONFORMANCE_CONF_FILE ?= ${REPO_ROOT}/test/e2e/config/docker-dev-conformance.yaml ARTIFACTS ?= ${REPO_ROOT}/_artifacts SKIP_RESOURCE_CLEANUP ?= false USE_EXISTING_CLUSTER ?= false @@ -63,3 +64,10 @@ run: ginkgo ## Run the end-to-end tests -e2e.artifacts-folder="$(ARTIFACTS)" \ -e2e.config="$(E2E_CONF_FILE)" \ -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) + +.PHONY: conformance +conformance: + cd $(TEST_E2E_DIR); $(GINKGO) -v -trace -tags=e2e -focus=$(GINKGO_FOCUS) -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) ./conformance -- \ + -e2e.artifacts-folder="$(ARTIFACTS)" \ + -e2e.config="$(CONFORMANCE_CONF_FILE)" \ + -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) diff --git a/test/e2e/common.go b/test/e2e/common.go index 896a1d9f6d92..dbd70b77ccce 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -17,18 +17,11 @@ limitations under the License. package e2e import ( - "context" "fmt" - "path/filepath" - - . "github.com/onsi/ginkgo" "github.com/blang/semver" "github.com/onsi/gomega/types" - corev1 "k8s.io/api/core/v1" - clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" - "sigs.k8s.io/cluster-api/test/framework" - "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" ) // Test suite constants for e2e config variables @@ -42,54 +35,9 @@ const ( CoreDNSVersionUpgradeTo = "COREDNS_VERSION_UPGRADE_TO" ) +// Byf is deprecated. Use "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" as dot import instead. func Byf(format string, a ...interface{}) { - By(fmt.Sprintf(format, a...)) -} - -func setupSpecNamespace(ctx context.Context, specName string, clusterProxy framework.ClusterProxy, artifactFolder string) (*corev1.Namespace, context.CancelFunc) { - Byf("Creating a namespace for hosting the %q test spec", specName) - namespace, cancelWatches := framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ - Creator: clusterProxy.GetClient(), - ClientSet: clusterProxy.GetClientSet(), - Name: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), - LogFolder: filepath.Join(artifactFolder, "clusters", clusterProxy.GetName()), - }) - - return namespace, cancelWatches -} - -func dumpSpecResourcesAndCleanup(ctx context.Context, specName string, clusterProxy framework.ClusterProxy, artifactFolder string, namespace *corev1.Namespace, cancelWatches context.CancelFunc, cluster *clusterv1.Cluster, intervalsGetter func(spec, key string) []interface{}, skipCleanup bool) { - Byf("Dumping logs from the %q workload cluster", cluster.Name) - - // Dump all the logs from the workload cluster before deleting them. - clusterProxy.CollectWorkloadClusterLogs(ctx, cluster.Namespace, cluster.Name, filepath.Join(artifactFolder, "clusters", cluster.Name, "machines")) - - Byf("Dumping all the Cluster API resources in the %q namespace", namespace.Name) - - // Dump all Cluster API related resources to artifacts before deleting them. - framework.DumpAllResources(ctx, framework.DumpAllResourcesInput{ - Lister: clusterProxy.GetClient(), - Namespace: namespace.Name, - LogPath: filepath.Join(artifactFolder, "clusters", clusterProxy.GetName(), "resources"), - }) - - if !skipCleanup { - Byf("Deleting cluster %s/%s", cluster.Namespace, cluster.Name) - // While https://github.com/kubernetes-sigs/cluster-api/issues/2955 is addressed in future iterations, there is a chance - // that cluster variable is not set even if the cluster exists, so we are calling DeleteAllClustersAndWait - // instead of DeleteClusterAndWait - framework.DeleteAllClustersAndWait(ctx, framework.DeleteAllClustersAndWaitInput{ - Client: clusterProxy.GetClient(), - Namespace: namespace.Name, - }, intervalsGetter(specName, "wait-delete-cluster")...) - - Byf("Deleting namespace used for hosting the %q test spec", specName) - framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{ - Deleter: clusterProxy.GetClient(), - Name: namespace.Name, - }) - } - cancelWatches() + ginkgoextensions.Byf(format, a...) } // HaveValidVersion succeeds if version is a valid semver version diff --git a/test/e2e/config/docker-ci.yaml b/test/e2e/config/docker-ci.yaml index 54f469ef52ff..d69a868b3502 100644 --- a/test/e2e/config/docker-ci.yaml +++ b/test/e2e/config/docker-ci.yaml @@ -79,6 +79,7 @@ variables: COREDNS_VERSION_UPGRADE_TO: "1.6.7" KUBERNETES_VERSION_UPGRADE_TO: "v1.18.2" KUBERNETES_VERSION_UPGRADE_FROM: "v1.17.2" + KUBERNETES_CI_ARTIFACTS_START_VERSION: "v1.19.0" DOCKER_SERVICE_DOMAIN: "cluster.local" DOCKER_SERVICE_CIDRS: "10.128.0.0/12" # IMPORTANT! This values should match the one used by the CNI provider diff --git a/test/e2e/config/docker-dev-conformance.yaml b/test/e2e/config/docker-dev-conformance.yaml new file mode 100644 index 000000000000..b14182c35cb7 --- /dev/null +++ b/test/e2e/config/docker-dev-conformance.yaml @@ -0,0 +1,124 @@ +--- +# E2E test scenario using local dev images and manifests built from the source tree for following providers: +# - cluster-api +# - bootstrap kubeadm +# - control-plane kubeadm +# - docker + +# For creating local dev images built from the source tree; +# - `make docker-build REGISTRY=gcr.io/k8s-staging-cluster-api` to build the cluster-api, bootstrap kubeadm, control-plane kubeadm provider images. +# - `make -C test/infrastructure/docker docker-build REGISTRY=gcr.io/k8s-staging-cluster-api` to build the docker provider images. + +images: +# Use local dev images built source tree; +- name: gcr.io/k8s-staging-cluster-api/cluster-api-controller-amd64:dev + loadBehavior: mustLoad +- name: gcr.io/k8s-staging-cluster-api/kubeadm-bootstrap-controller-amd64:dev + loadBehavior: mustLoad +- name: gcr.io/k8s-staging-cluster-api/kubeadm-control-plane-controller-amd64:dev + loadBehavior: mustLoad +- name: gcr.io/k8s-staging-cluster-api/capd-manager-amd64:dev + loadBehavior: mustLoad +- name: quay.io/jetstack/cert-manager-cainjector:v0.16.1 + loadBehavior: tryLoad +- name: quay.io/jetstack/cert-manager-webhook:v0.16.1 + loadBehavior: tryLoad +- name: quay.io/jetstack/cert-manager-controller:v0.16.1 + loadBehavior: tryLoad +# If using Calico uncomment following lines to speed up test by pre-loading required images on nodes +# - name: calico/kube-controllers:v3.13.1 +# loadBehavior: tryLoad +# - name: calico/cni:v3.13.1 +# loadBehavior: tryLoad +# - name: calico/pod2daemon-flexvol:v3.13.1 +# loadBehavior: tryLoad +# - name: calico/node:v3.13.1 +# loadBehavior: tryLoad + +providers: + +- name: cluster-api + type: CoreProvider + versions: + - name: v0.3.0 + # Use manifest from source files + value: ../../../config + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: "--enable-leader-election" + new: "--enable-leader-election=false" + - old: --metrics-addr=127.0.0.1:8080 + new: --metrics-addr=:8080 + +- name: kubeadm + type: BootstrapProvider + versions: + - name: v0.3.0 + # Use manifest from source files + value: ../../../bootstrap/kubeadm/config + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: "--enable-leader-election" + new: "--enable-leader-election=false" + - old: --metrics-addr=127.0.0.1:8080 + new: --metrics-addr=:8080 + +- name: kubeadm + type: ControlPlaneProvider + versions: + - name: v0.3.0 + # Use manifest from source files + value: ../../../controlplane/kubeadm/config + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: "--enable-leader-election" + new: "--enable-leader-election=false" + - old: --metrics-addr=127.0.0.1:8080 + new: --metrics-addr=:8080 + +- name: docker + type: InfrastructureProvider + versions: + - name: v0.3.0 + # Use manifest from source files + value: ../../../test/infrastructure/docker/config + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: "--enable-leader-election" + new: "--enable-leader-election=false" + - old: --metrics-addr=127.0.0.1:8080 + new: --metrics-addr=:8080 + files: + # Add a metadata for docker provider + - sourcePath: "../data/infrastructure-docker/metadata.yaml" + # Add cluster templates + - sourcePath: "../data/infrastructure-docker/cluster-template.yaml" + - sourcePath: "../data/infrastructure-docker/cluster-template-kcp-adoption.yaml" + +variables: + KUBERNETES_VERSION: "v1.19.0" + ETCD_VERSION_UPGRADE_TO: "3.4.3-0" + COREDNS_VERSION_UPGRADE_TO: "1.6.7" + KUBERNETES_VERSION_UPGRADE_TO: "v1.18.2" + KUBERNETES_VERSION_UPGRADE_FROM: "v1.17.2" + KUBERNETES_CI_ARTIFACTS_START_VERSION: "v1.19.0" + DOCKER_SERVICE_DOMAIN: "cluster.local" + DOCKER_SERVICE_CIDRS: "10.128.0.0/12" + # IMPORTANT! This values should match the one used by the CNI provider + DOCKER_POD_CIDRS: "192.168.0.0/16" + #CNI: "./data/cni/calico/calico.yaml" + CNI: "../data/cni/kindnet/kindnet.yaml" + EXP_CLUSTER_RESOURCE_SET: "true" + +intervals: + default/wait-controllers: ["3m", "10s"] + default/wait-cluster: ["3m", "10s"] + default/wait-control-plane: ["10m", "10s"] + default/wait-worker-nodes: ["5m", "10s"] + default/wait-delete-cluster: ["3m", "10s"] + default/wait-machine-upgrade: ["20m", "10s"] + default/wait-machine-remediation: ["5m", "10s"] diff --git a/test/e2e/config/docker-dev.yaml b/test/e2e/config/docker-dev.yaml index bd9775a2c9d7..f186d02368b6 100644 --- a/test/e2e/config/docker-dev.yaml +++ b/test/e2e/config/docker-dev.yaml @@ -105,6 +105,7 @@ variables: COREDNS_VERSION_UPGRADE_TO: "1.6.7" KUBERNETES_VERSION_UPGRADE_TO: "v1.18.2" KUBERNETES_VERSION_UPGRADE_FROM: "v1.17.2" + KUBERNETES_CI_ARTIFACTS_START_VERSION: "v1.19.0" DOCKER_SERVICE_DOMAIN: "cluster.local" DOCKER_SERVICE_CIDRS: "10.128.0.0/12" # IMPORTANT! This values should match the one used by the CNI provider diff --git a/test/e2e/conformance/ci_artifacts.go b/test/e2e/conformance/ci_artifacts.go new file mode 100644 index 000000000000..4cdba47343e1 --- /dev/null +++ b/test/e2e/conformance/ci_artifacts.go @@ -0,0 +1,137 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conformance + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/test/framework/kubetest" + "sigs.k8s.io/cluster-api/util" +) + +// ConformanceCIArtifactsSpecInput is the input for ConformanceSpec. +type ConformanceCIArtifactsSpecInput struct { + E2EConfig *clusterctl.E2EConfig + ClusterctlConfigPath string + BootstrapClusterProxy framework.ClusterProxy + ArtifactFolder string + SkipCleanup bool +} + +// ConformanceSpec implements a spec that mimics the operation described in the Cluster API quick start, that is +// creating a workload cluster. +// This test is meant to provide a first, fast signal to detect regression; it is recommended to use it as a PR blocker test. +func ConformanceCIArtifactsSpec(ctx context.Context, inputGetter func() ConformanceCIArtifactsSpecInput) { + var ( + specName = "conformance-with-ci-artifacts" + input ConformanceCIArtifactsSpecInput + namespace *corev1.Namespace + cancelWatches context.CancelFunc + cluster *clusterv1.Cluster + ) + + BeforeEach(func() { + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + input = inputGetter() + Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName) + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) + + // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. + namespace, cancelWatches = setup.CreateSpecNamespace( + ctx, + setup.CreateSpecNamespaceInput{ + ArtifactsDirectory: input.ArtifactFolder, + ClusterProxy: input.BootstrapClusterProxy, + SpecName: specName, + }, + ) + }) + + It("Should create a workload cluster", func() { + + By("Creating a workload cluster") + name := fmt.Sprintf("cluster-%s", util.RandomString(6)) + kubernetesVersion, err := kubetest.FetchKubernetesCIVersion() + Expect(err).NotTo(HaveOccurred()) + cluster, _, _ = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "with-ci-artifacts", + Namespace: namespace.Name, + ClusterName: name, + KubernetesVersion: kubernetesVersion, + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }) + + workloadProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, name) + By("Running conformance suite") + + kubetestPath, err := filepath.Abs(path.Join("..", "data", "kubetest", "conformance.yaml")) + Expect(err).ToNot(HaveOccurred()) + + Expect(kubetest.Run(kubetest.RunInput{ + ClusterProxy: workloadProxy, + KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion), + ConfigFilePath: kubetestPath, + })).To(Succeed()) + + By("PASSED!") + }) + + AfterEach(func() { + // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. + setup.DumpSpecResourcesAndCleanup( + ctx, + setup.DumpSpecResourcesAndCleanupInput{ + SpecName: specName, + ClusterProxy: input.BootstrapClusterProxy, + ArtifactsDirectory: input.ArtifactFolder, + Namespace: namespace, + CancelWatches: cancelWatches, + Cluster: cluster, + IntervalsGetter: input.E2EConfig.GetIntervals, + SkipCleanup: input.SkipCleanup, + }, + ) + }) +} diff --git a/test/e2e/conformance/ci_artifacts_test.go b/test/e2e/conformance/ci_artifacts_test.go new file mode 100644 index 000000000000..429792ee9465 --- /dev/null +++ b/test/e2e/conformance/ci_artifacts_test.go @@ -0,0 +1,39 @@ +// +build e2e + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conformance + +import ( + "context" + + . "github.com/onsi/ginkgo" +) + +var _ = FDescribe("[Conformance] When running conformance with CI Artifacts", func() { + + ConformanceCIArtifactsSpec(context.TODO(), func() ConformanceCIArtifactsSpecInput { + return ConformanceCIArtifactsSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + } + }) + +}) diff --git a/test/e2e/conformance/conformance.go b/test/e2e/conformance/conformance.go new file mode 100644 index 000000000000..b50bb3b18198 --- /dev/null +++ b/test/e2e/conformance/conformance.go @@ -0,0 +1,139 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conformance + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/test/framework/kubetest" + "sigs.k8s.io/cluster-api/util" +) + +const ( + KubernetesVersion = "KUBERNETES_VERSION" +) + +// ConformanceSpecInput is the input for ConformanceSpec. +type ConformanceSpecInput struct { + E2EConfig *clusterctl.E2EConfig + ClusterctlConfigPath string + BootstrapClusterProxy framework.ClusterProxy + ArtifactFolder string + SkipCleanup bool +} + +// ConformanceSpec implements a spec that mimics the operation described in the Cluster API quick start, that is +// creating a workload cluster. +// This test is meant to provide a first, fast signal to detect regression; it is recommended to use it as a PR blocker test. +func ConformanceSpec(ctx context.Context, inputGetter func() ConformanceSpecInput) { + var ( + specName = "conformance" + input ConformanceSpecInput + namespace *corev1.Namespace + cancelWatches context.CancelFunc + cluster *clusterv1.Cluster + ) + + BeforeEach(func() { + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + input = inputGetter() + Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName) + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) + + // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. + namespace, cancelWatches = setup.CreateSpecNamespace( + ctx, + setup.CreateSpecNamespaceInput{ + ArtifactsDirectory: input.ArtifactFolder, + ClusterProxy: input.BootstrapClusterProxy, + SpecName: specName, + }, + ) + }) + + It("Should create a workload cluster", func() { + + By("Creating a workload cluster") + name := fmt.Sprintf("cluster-%s", util.RandomString(6)) + cluster, _, _ = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: clusterctl.DefaultFlavor, + Namespace: namespace.Name, + ClusterName: name, + KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }) + + workloadProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, name) + By("Running conformance suite") + + kubetestPath, err := filepath.Abs(path.Join("..", "data", "kubetest", "conformance.yaml")) + Expect(err).ToNot(HaveOccurred()) + + Expect(kubetest.Run(kubetest.RunInput{ + ClusterProxy: workloadProxy, + KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion), + ConfigFilePath: kubetestPath, + })).To(Succeed()) + + By("PASSED!") + }) + + AfterEach(func() { + // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. + setup.DumpSpecResourcesAndCleanup( + ctx, + setup.DumpSpecResourcesAndCleanupInput{ + SpecName: specName, + ClusterProxy: input.BootstrapClusterProxy, + ArtifactsDirectory: input.ArtifactFolder, + Namespace: namespace, + CancelWatches: cancelWatches, + Cluster: cluster, + IntervalsGetter: input.E2EConfig.GetIntervals, + SkipCleanup: input.SkipCleanup, + }, + ) + }) +} diff --git a/test/e2e/conformance/conformance_suite_test.go b/test/e2e/conformance/conformance_suite_test.go new file mode 100644 index 000000000000..351485f6389b --- /dev/null +++ b/test/e2e/conformance/conformance_suite_test.go @@ -0,0 +1,179 @@ +// +build e2e + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conformance + +import ( + "flag" + "os" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/bootstrap" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" + "sigs.k8s.io/yaml" +) + +// Test suite flags +var ( + // configPath is the path to the e2e config file. + configPath string + + // useExistingCluster instructs the test to use the current cluster instead of creating a new one (default discovery rules apply). + useExistingCluster bool + + // artifactFolder is the folder to store e2e test artifacts. + artifactFolder string + + // skipCleanup prevents cleanup of test resources e.g. for debug purposes. + skipCleanup bool +) + +// Test suite global vars +var ( + // e2eConfig to be used for this test, read from configPath. + e2eConfig *clusterctl.E2EConfig + + // clusterctlConfigPath to be used for this test, created by generating a clusterctl local repository + // with the providers specified in the configPath. + clusterctlConfigPath string + + // bootstrapClusterProvider manages provisioning of the the bootstrap cluster to be used for the e2e tests. + // Please note that provisioning will be skipped if e2e.use-existing-cluster is provided. + bootstrapClusterProvider bootstrap.ClusterProvider + + // bootstrapClusterProxy allows to interact with the bootstrap cluster to be used for the e2e tests. + bootstrapClusterProxy framework.ClusterProxy +) + +func init() { + flag.StringVar(&configPath, "e2e.config", "", "path to the e2e config file") + flag.StringVar(&artifactFolder, "e2e.artifacts-folder", "", "folder where e2e test artifact should be stored") + flag.BoolVar(&skipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped") + flag.BoolVar(&useExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)") +} + +func TestE2E(t *testing.T) { + artifactFolder := framework.ResolveArtifactsDirectory(artifactFolder) + RegisterFailHandler(Fail) + RunSpecsWithDefaultAndCustomReporters(t, "capi-conformance", []Reporter{framework.CreateJUnitReporterForProw(artifactFolder)}) +} + +type suiteConfig struct { + ArtifactFolder string `json:"artifactFolder,omitempty"` + ConfigPath string `json:"configPath,omitempty"` + ClusterctlConfigPath string `json:"clusterctlConfigPath,omitempty"` + KubeconfigPath string `json:"kubeconfigPath,omitempty"` +} + +// Using a SynchronizedBeforeSuite for controlling how to create resources shared across ParallelNodes (~ginkgo threads). +// The local clusterctl repository & the bootstrap cluster are created once and shared across all the tests. +var _ = SynchronizedBeforeSuite(func() []byte { + // Before all ParallelNodes. + + Expect(configPath).To(BeAnExistingFile(), "Invalid test suite argument. e2e.config should be an existing file.") + artifactFolder := framework.ResolveArtifactsDirectory(artifactFolder) + log := ginkgoextensions.Log.WithValues("config-path", configPath, "artifacts-directory", artifactFolder) + Expect(os.MkdirAll(artifactFolder, 0755)).To(Succeed(), "Invalid test suite argument. Can't create e2e.artifacts-folder %q", artifactFolder) + + By("Initializing a runtime.Scheme with all the GVK relevant for this test") + scheme := initScheme() + + log.Info("Loading the e2e test configuration") + e2eConfig = setup.LoadE2EConfig(configPath) + + log.Info("Creating a clusterctl local repository in artifacts directory") + + setup.CreateCIArtifactsTemplate(artifactFolder, "..", e2eConfig) + + clusterctlConfigPath = setup.CreateClusterctlLocalRepository(e2eConfig, filepath.Join(artifactFolder, "repository")) + + log.Info("Setting up the bootstrap cluster") + // e2eConfig, scheme, useExistingCluster + bootstrapClusterProvider, bootstrapClusterProxy = setup.CreateBootstrapCluster( + setup.CreateBootstrapClusterInput{ + E2EConfig: e2eConfig, + Scheme: scheme, + UseExistingCluster: useExistingCluster, + }, + ) + + log.Info("Initializing the bootstrap cluster") + // setup.InitBootstrapCluster(bootstrapClusterProxy, e2eConfig, clusterctlConfigPath, artifactFolder) + setup.InitBootstrapCluster( + setup.InitBootstrapClusterInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + E2EConfig: e2eConfig, + ClusterctlConfig: clusterctlConfigPath, + ArtifactsDirectory: artifactFolder, + }, + ) + + conf := suiteConfig{ + ArtifactFolder: artifactFolder, + ConfigPath: configPath, + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + } + + data, err := yaml.Marshal(conf) + Expect(err).NotTo(HaveOccurred()) + return data + +}, func(data []byte) { + // Before each ParallelNode. + + conf := &suiteConfig{} + err := yaml.UnmarshalStrict(data, conf) + Expect(err).NotTo(HaveOccurred()) + + artifactFolder = conf.ArtifactFolder + configPath = conf.ConfigPath + clusterctlConfigPath = conf.ClusterctlConfigPath + kubeconfigPath := conf.KubeconfigPath + + e2eConfig = setup.LoadE2EConfig(configPath) + bootstrapClusterProxy = framework.NewClusterProxy("bootstrap", kubeconfigPath, initScheme()) +}) + +// Using a SynchronizedAfterSuite for controlling how to delete resources shared across ParallelNodes (~ginkgo threads). +// The bootstrap cluster is shared across all the tests, so it should be deleted only after all ParallelNodes completes. +// The local clusterctl repository is preserved like everything else created into the artifact folder. +var _ = SynchronizedAfterSuite(func() { + // After each ParallelNode. +}, func() { + // After all ParallelNodes. + log := ginkgoextensions.Log.WithValues("config-path", configPath, "artifacts-directory", artifactFolder) + log.Info("Tearing down the management cluster") + if !skipCleanup { + setup.TearDown(bootstrapClusterProvider, bootstrapClusterProxy) + } +}) + +func initScheme() *runtime.Scheme { + sc := runtime.NewScheme() + framework.TryAddDefaultSchemes(sc) + return sc +} diff --git a/test/e2e/conformance/conformance_test.go b/test/e2e/conformance/conformance_test.go new file mode 100644 index 000000000000..4de215981062 --- /dev/null +++ b/test/e2e/conformance/conformance_test.go @@ -0,0 +1,39 @@ +// +build e2e + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conformance + +import ( + "context" + + . "github.com/onsi/ginkgo" +) + +var _ = Describe("[Conformance] When running conformance", func() { + + ConformanceSpec(context.TODO(), func() ConformanceSpecInput { + return ConformanceSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + } + }) + +}) diff --git a/test/e2e/data/infrastructure-docker/cluster-template-ci.yaml b/test/e2e/data/infrastructure-docker/cluster-template-ci.yaml index ce66062a76e1..5cc5964cf1e5 100644 --- a/test/e2e/data/infrastructure-docker/cluster-template-ci.yaml +++ b/test/e2e/data/infrastructure-docker/cluster-template-ci.yaml @@ -51,7 +51,7 @@ spec: controllerManager: extraArgs: {enable-hostpath-provisioner: 'true'} apiServer: - certSANs: [localhost, 127.0.0.1] + certSANs: [localhost, "host.docker.internal", 127.0.0.1] initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock diff --git a/test/e2e/data/infrastructure-docker/cluster-template-kcp-adoption.yaml b/test/e2e/data/infrastructure-docker/cluster-template-kcp-adoption.yaml index 0ed2d0f33c4d..e6b44fd41b52 100644 --- a/test/e2e/data/infrastructure-docker/cluster-template-kcp-adoption.yaml +++ b/test/e2e/data/infrastructure-docker/cluster-template-kcp-adoption.yaml @@ -54,7 +54,7 @@ spec: controllerManager: extraArgs: {enable-hostpath-provisioner: 'true'} apiServer: - certSANs: [localhost, 127.0.0.1] + certSANs: [localhost, "host.docker.internal", 127.0.0.1] initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock diff --git a/test/e2e/data/infrastructure-docker/cluster-template.yaml b/test/e2e/data/infrastructure-docker/cluster-template.yaml index ce66062a76e1..5cc5964cf1e5 100644 --- a/test/e2e/data/infrastructure-docker/cluster-template.yaml +++ b/test/e2e/data/infrastructure-docker/cluster-template.yaml @@ -51,7 +51,7 @@ spec: controllerManager: extraArgs: {enable-hostpath-provisioner: 'true'} apiServer: - certSANs: [localhost, 127.0.0.1] + certSANs: [localhost, "host.docker.internal", 127.0.0.1] initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock diff --git a/test/e2e/data/infrastructure-docker/platform-kustomization.yaml b/test/e2e/data/infrastructure-docker/platform-kustomization.yaml new file mode 100644 index 000000000000..60802c2563db --- /dev/null +++ b/test/e2e/data/infrastructure-docker/platform-kustomization.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: DockerMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + customImage: kindest/node:v1.18.2 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: DockerMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + customImage: kindest/node:v1.18.2 diff --git a/test/e2e/data/kubetest/conformance-fast.yaml b/test/e2e/data/kubetest/conformance-fast.yaml new file mode 100644 index 000000000000..6f7936378b14 --- /dev/null +++ b/test/e2e/data/kubetest/conformance-fast.yaml @@ -0,0 +1,8 @@ +ginkgo.focus: \[Conformance\] +ginkgo.skip: \[sig-scheduling\].*\[Serial\] +disable-log-dump: true +ginkgo.progress: true +ginkgo.slowSpecThreshold: 120.0 +ginkgo.flakeAttempts: 3 +ginkgo.trace: true +ginkgo.v: true diff --git a/test/e2e/data/kubetest/conformance.yaml b/test/e2e/data/kubetest/conformance.yaml new file mode 100644 index 000000000000..d748f432888b --- /dev/null +++ b/test/e2e/data/kubetest/conformance.yaml @@ -0,0 +1,7 @@ +ginkgo.focus: \[Conformance\] +disable-log-dump: true +ginkgo.progress: true +ginkgo.slowSpecThreshold: 120.0 +ginkgo.flakeAttempts: 3 +ginkgo.trace: true +ginkgo.v: true diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 0c77876e4223..d794722cf9b6 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -19,23 +19,21 @@ limitations under the License. package e2e import ( - "context" "flag" - "fmt" "os" "path/filepath" - "strings" "testing" . "github.com/onsi/ginkgo" - "github.com/onsi/ginkgo/config" - "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/bootstrap" "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" + "sigs.k8s.io/yaml" ) // Test suite flags @@ -78,15 +76,16 @@ func init() { } func TestE2E(t *testing.T) { - // If running in prow, make sure to use the artifacts folder that will be reported in test grid (ignoring the value provided by flag). - if prowArtifactFolder, exists := os.LookupEnv("ARTIFACTS"); exists { - artifactFolder = prowArtifactFolder - } - + artifactFolder := framework.ResolveArtifactsDirectory(artifactFolder) RegisterFailHandler(Fail) - junitPath := filepath.Join(artifactFolder, fmt.Sprintf("junit.e2e_suite.%d.xml", config.GinkgoConfig.ParallelNode)) - junitReporter := reporters.NewJUnitReporter(junitPath) - RunSpecsWithDefaultAndCustomReporters(t, "capi-e2e", []Reporter{junitReporter}) + RunSpecsWithDefaultAndCustomReporters(t, "capi-e2e", []Reporter{framework.CreateJUnitReporterForProw(artifactFolder)}) +} + +type suiteConfig struct { + ArtifactFolder string `json:"artifactFolder,omitempty"` + ConfigPath string `json:"configPath,omitempty"` + ClusterctlConfigPath string `json:"clusterctlConfigPath,omitempty"` + KubeconfigPath string `json:"kubeconfigPath,omitempty"` } // Using a SynchronizedBeforeSuite for controlling how to create resources shared across ParallelNodes (~ginkgo threads). @@ -95,43 +94,65 @@ var _ = SynchronizedBeforeSuite(func() []byte { // Before all ParallelNodes. Expect(configPath).To(BeAnExistingFile(), "Invalid test suite argument. e2e.config should be an existing file.") + artifactFolder := framework.ResolveArtifactsDirectory(artifactFolder) + log := ginkgoextensions.Log.WithValues("config-path", configPath, "artifacts-directory", artifactFolder) Expect(os.MkdirAll(artifactFolder, 0755)).To(Succeed(), "Invalid test suite argument. Can't create e2e.artifacts-folder %q", artifactFolder) By("Initializing a runtime.Scheme with all the GVK relevant for this test") scheme := initScheme() - Byf("Loading the e2e test configuration from %q", configPath) - e2eConfig = loadE2EConfig(configPath) + log.Info("Loading the e2e test configuration") + e2eConfig = setup.LoadE2EConfig(configPath) + + log.Info("Creating a clusterctl local repository in artifacts directory") + setup.CreateCIArtifactsTemplate(artifactFolder, ".", e2eConfig) + clusterctlConfigPath = setup.CreateClusterctlLocalRepository(e2eConfig, filepath.Join(artifactFolder, "repository")) + + log.Info("Setting up the bootstrap cluster") + // e2eConfig, scheme, useExistingCluster + bootstrapClusterProvider, bootstrapClusterProxy = setup.CreateBootstrapCluster( + setup.CreateBootstrapClusterInput{ + E2EConfig: e2eConfig, + Scheme: scheme, + UseExistingCluster: useExistingCluster, + }, + ) - Byf("Creating a clusterctl local repository into %q", artifactFolder) - clusterctlConfigPath = createClusterctlLocalRepository(e2eConfig, filepath.Join(artifactFolder, "repository")) + log.Info("Initializing the bootstrap cluster") + // setup.InitBootstrapCluster(bootstrapClusterProxy, e2eConfig, clusterctlConfigPath, artifactFolder) + setup.InitBootstrapCluster( + setup.InitBootstrapClusterInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + E2EConfig: e2eConfig, + ClusterctlConfig: clusterctlConfigPath, + ArtifactsDirectory: artifactFolder, + }, + ) - By("Setting up the bootstrap cluster") - bootstrapClusterProvider, bootstrapClusterProxy = setupBootstrapCluster(e2eConfig, scheme, useExistingCluster) + conf := suiteConfig{ + ArtifactFolder: artifactFolder, + ConfigPath: configPath, + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + } - By("Initializing the bootstrap cluster") - initBootstrapCluster(bootstrapClusterProxy, e2eConfig, clusterctlConfigPath, artifactFolder) + data, err := yaml.Marshal(conf) + Expect(err).NotTo(HaveOccurred()) + return data - return []byte( - strings.Join([]string{ - artifactFolder, - configPath, - clusterctlConfigPath, - bootstrapClusterProxy.GetKubeconfigPath(), - }, ","), - ) }, func(data []byte) { // Before each ParallelNode. - parts := strings.Split(string(data), ",") - Expect(parts).To(HaveLen(4)) + conf := &suiteConfig{} + err := yaml.UnmarshalStrict(data, conf) + Expect(err).NotTo(HaveOccurred()) - artifactFolder = parts[0] - configPath = parts[1] - clusterctlConfigPath = parts[2] - kubeconfigPath := parts[3] + artifactFolder = conf.ArtifactFolder + configPath = conf.ConfigPath + clusterctlConfigPath = conf.ClusterctlConfigPath + kubeconfigPath := conf.KubeconfigPath - e2eConfig = loadE2EConfig(configPath) + e2eConfig = setup.LoadE2EConfig(configPath) bootstrapClusterProxy = framework.NewClusterProxy("bootstrap", kubeconfigPath, initScheme(), framework.WithMachineLogCollector(framework.DockerLogCollector{})) }) @@ -142,10 +163,10 @@ var _ = SynchronizedAfterSuite(func() { // After each ParallelNode. }, func() { // After all ParallelNodes. - - By("Tearing down the management cluster") + log := ginkgoextensions.Log.WithValues("config-path", configPath, "artifacts-directory", artifactFolder) + log.Info("Tearing down the management cluster") if !skipCleanup { - tearDown(bootstrapClusterProvider, bootstrapClusterProxy) + setup.TearDown(bootstrapClusterProvider, bootstrapClusterProxy) } }) @@ -154,62 +175,3 @@ func initScheme() *runtime.Scheme { framework.TryAddDefaultSchemes(sc) return sc } - -func loadE2EConfig(configPath string) *clusterctl.E2EConfig { - config := clusterctl.LoadE2EConfig(context.TODO(), clusterctl.LoadE2EConfigInput{ConfigPath: configPath}) - Expect(config).ToNot(BeNil(), "Failed to load E2E config from %s", configPath) - - // Read CNI file and set CNI_RESOURCES environmental variable - Expect(config.Variables).To(HaveKey(CNIPath), "Missing %s variable in the config", CNIPath) - clusterctl.SetCNIEnvVar(config.GetVariable(CNIPath), CNIResources) - - return config -} - -func createClusterctlLocalRepository(config *clusterctl.E2EConfig, repositoryFolder string) string { - clusterctlConfig := clusterctl.CreateRepository(context.TODO(), clusterctl.CreateRepositoryInput{ - E2EConfig: config, - RepositoryFolder: repositoryFolder, - }) - Expect(clusterctlConfig).To(BeAnExistingFile(), "The clusterctl config file does not exists in the local repository %s", repositoryFolder) - return clusterctlConfig -} - -func setupBootstrapCluster(config *clusterctl.E2EConfig, scheme *runtime.Scheme, useExistingCluster bool) (bootstrap.ClusterProvider, framework.ClusterProxy) { - var clusterProvider bootstrap.ClusterProvider - kubeconfigPath := "" - if !useExistingCluster { - clusterProvider = bootstrap.CreateKindBootstrapClusterAndLoadImages(context.TODO(), bootstrap.CreateKindBootstrapClusterAndLoadImagesInput{ - Name: config.ManagementClusterName, - RequiresDockerSock: config.HasDockerProvider(), - Images: config.Images, - }) - Expect(clusterProvider).ToNot(BeNil(), "Failed to create a bootstrap cluster") - - kubeconfigPath = clusterProvider.GetKubeconfigPath() - Expect(kubeconfigPath).To(BeAnExistingFile(), "Failed to get the kubeconfig file for the bootstrap cluster") - } - - clusterProxy := framework.NewClusterProxy("bootstrap", kubeconfigPath, scheme) - Expect(clusterProxy).ToNot(BeNil(), "Failed to get a bootstrap cluster proxy") - - return clusterProvider, clusterProxy -} - -func initBootstrapCluster(bootstrapClusterProxy framework.ClusterProxy, config *clusterctl.E2EConfig, clusterctlConfig, artifactFolder string) { - clusterctl.InitManagementClusterAndWatchControllerLogs(context.TODO(), clusterctl.InitManagementClusterAndWatchControllerLogsInput{ - ClusterProxy: bootstrapClusterProxy, - ClusterctlConfigPath: clusterctlConfig, - InfrastructureProviders: config.InfrastructureProviders(), - LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), - }, config.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) -} - -func tearDown(bootstrapClusterProvider bootstrap.ClusterProvider, bootstrapClusterProxy framework.ClusterProxy) { - if bootstrapClusterProxy != nil { - bootstrapClusterProxy.Dispose(context.TODO()) - } - if bootstrapClusterProvider != nil { - bootstrapClusterProvider.Dispose(context.TODO()) - } -} diff --git a/test/e2e/internal/setup/setup.go b/test/e2e/internal/setup/setup.go new file mode 100644 index 000000000000..bb4c127055cf --- /dev/null +++ b/test/e2e/internal/setup/setup.go @@ -0,0 +1,230 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package setup + +import ( + "context" + "fmt" + "io/ioutil" + "path/filepath" + + . "github.com/onsi/gomega" + . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" + "sigs.k8s.io/cluster-api/test/framework/kubetest" + + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/bootstrap" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/util" + + corev1 "k8s.io/api/core/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" +) + +// Test suite constants for e2e config variables +const ( + CNIPath = "CNI" + CNIResources = "CNI_RESOURCES" +) + +func initScheme() *runtime.Scheme { + sc := runtime.NewScheme() + framework.TryAddDefaultSchemes(sc) + return sc +} + +func LoadE2EConfig(configPath string) *clusterctl.E2EConfig { + config := clusterctl.LoadE2EConfig(context.TODO(), clusterctl.LoadE2EConfigInput{ConfigPath: configPath}) + Expect(config).ToNot(BeNil(), "Failed to load E2E config from %s", configPath) + + // Read CNI file and set CNI_RESOURCES environmental variable + Expect(config.Variables).To(HaveKey(CNIPath), "Missing %s variable in the config", CNIPath) + clusterctl.SetCNIEnvVar(config.GetVariable(CNIPath), CNIResources) + + return config +} + +func CreateClusterctlLocalRepository(config *clusterctl.E2EConfig, repositoryFolder string) string { + clusterctlConfig := clusterctl.CreateRepository(context.TODO(), clusterctl.CreateRepositoryInput{ + E2EConfig: config, + RepositoryFolder: repositoryFolder, + }) + Expect(clusterctlConfig).To(BeAnExistingFile(), "The clusterctl config file does not exists in the local repository %s", repositoryFolder) + return clusterctlConfig +} + +type CreateBootstrapClusterInput struct { + E2EConfig *clusterctl.E2EConfig + Scheme *runtime.Scheme + UseExistingCluster bool +} + +func CreateCIArtifactsTemplate(artifactFolder string, srcFolder string, e2eConfig *clusterctl.E2EConfig) string { + template, err := ioutil.ReadFile(srcFolder + "/data/infrastructure-docker/cluster-template.yaml") + Expect(err).NotTo(HaveOccurred()) + + platformKustomization, err := ioutil.ReadFile(srcFolder + "/data/infrastructure-docker/platform-kustomization.yaml") + Expect(err).NotTo(HaveOccurred()) + + ciTemplate, err := kubetest.GenerateCIArtifactsInjectedTemplateForDebian(kubetest.GenerateCIArtifactsInjectedTemplateForDebianInput{ + ArtifactsDirectory: artifactFolder, + SourceTemplate: template, + PlatformKustomization: platformKustomization, + }) + + Expect(err).NotTo(HaveOccurred()) + for i, p := range e2eConfig.Providers { + if p.Name != "docker" { + continue + } + e2eConfig.Providers[i].Files = append(e2eConfig.Providers[i].Files, clusterctl.Files{ + SourcePath: ciTemplate, + TargetName: "cluster-template-with-ci-artifacts.yaml", + }) + } + return ciTemplate +} + +func CreateBootstrapCluster(input CreateBootstrapClusterInput) (bootstrap.ClusterProvider, framework.ClusterProxy) { + var clusterProvider bootstrap.ClusterProvider + kubeconfigPath := "" + Expect(input.E2EConfig).ToNot(BeNil(), "E2EConfig must be provided") + Expect(input.Scheme).ToNot(BeNil(), "Scheme must be provided") + if !input.UseExistingCluster { + clusterProvider = bootstrap.CreateKindBootstrapClusterAndLoadImages(context.TODO(), bootstrap.CreateKindBootstrapClusterAndLoadImagesInput{ + Name: input.E2EConfig.ManagementClusterName, + RequiresDockerSock: input.E2EConfig.HasDockerProvider(), + Images: input.E2EConfig.Images, + }) + Expect(clusterProvider).ToNot(BeNil(), "Failed to create a bootstrap cluster") + + kubeconfigPath = clusterProvider.GetKubeconfigPath() + Expect(kubeconfigPath).To(BeAnExistingFile(), "Failed to get the kubeconfig file for the bootstrap cluster") + } + + clusterProxy := framework.NewClusterProxy("bootstrap", kubeconfigPath, input.Scheme) + Expect(clusterProxy).ToNot(BeNil(), "Failed to get a bootstrap cluster proxy") + + return clusterProvider, clusterProxy +} + +type InitBootstrapClusterInput struct { + BootstrapClusterProxy framework.ClusterProxy + E2EConfig *clusterctl.E2EConfig + ClusterctlConfig string + ArtifactsDirectory string +} + +func InitBootstrapCluster(input InitBootstrapClusterInput) { + input.ArtifactsDirectory = framework.ResolveArtifactsDirectory(input.ArtifactsDirectory) + Expect(input.E2EConfig).ToNot(BeNil(), "E2EConfig must be provided") + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "BootstrapClusterProxy must be provided") + + clusterctl.InitManagementClusterAndWatchControllerLogs(context.TODO(), clusterctl.InitManagementClusterAndWatchControllerLogsInput{ + ClusterProxy: input.BootstrapClusterProxy, + ClusterctlConfigPath: input.ClusterctlConfig, + InfrastructureProviders: input.E2EConfig.InfrastructureProviders(), + LogFolder: filepath.Join(input.ArtifactsDirectory, "clusters", input.BootstrapClusterProxy.GetName()), + }, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-controllers")...) +} + +func TearDown(bootstrapClusterProvider bootstrap.ClusterProvider, bootstrapClusterProxy framework.ClusterProxy) { + if bootstrapClusterProxy != nil { + bootstrapClusterProxy.Dispose(context.TODO()) + } + if bootstrapClusterProvider != nil { + bootstrapClusterProvider.Dispose(context.TODO()) + } +} + +type CreateSpecNamespaceInput struct { + SpecName string + ClusterProxy framework.ClusterProxy + ArtifactsDirectory string +} + +func CreateSpecNamespace(ctx context.Context, input CreateSpecNamespaceInput) (*corev1.Namespace, context.CancelFunc) { + name := fmt.Sprintf("%s-%s", input.SpecName, util.RandomString(6)) + log := Log.WithValues("spec-name", input.SpecName, "namespace", name) + log.Info("Creating a namespace for hosting the %q test spec", input.SpecName) + namespace, cancelWatches := framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ + Creator: input.ClusterProxy.GetClient(), + ClientSet: input.ClusterProxy.GetClientSet(), + Name: name, + LogFolder: filepath.Join(framework.ResolveArtifactsDirectory(input.ArtifactsDirectory), "clusters", input.ClusterProxy.GetName()), + }) + + return namespace, cancelWatches +} + +type DumpSpecResourcesAndCleanupInput struct { + SpecName string + ClusterProxy framework.ClusterProxy + ArtifactsDirectory string + Namespace *corev1.Namespace + CancelWatches context.CancelFunc + Cluster *clusterv1.Cluster + IntervalsGetter func(spec, key string) []interface{} + SkipCleanup bool +} + +func DumpSpecResourcesAndCleanup(ctx context.Context, input DumpSpecResourcesAndCleanupInput) { + Expect(input.ClusterProxy).ToNot(BeNil(), "ClusterProxy must be provided") + Expect(input.Namespace).ToNot(BeNil(), "Namespace must be provided") + Expect(input.IntervalsGetter).ToNot(BeNil(), "IntervalsGetter must be provided") + logPath := filepath.Join(framework.ResolveArtifactsDirectory(input.ArtifactsDirectory), "clusters", input.ClusterProxy.GetName(), "resources") + log := Log.WithValues( + "namespace", input.Namespace.Name, + "cluster-namespace", input.Cluster.Namespace, + "cluster-name", input.Cluster.Name, + "spec-name", input.SpecName, + "log-path", logPath, + ) + + // Dump all the logs from the workload cluster before deleting them. + input.ClusterProxy.CollectWorkloadClusterLogs(ctx, + input.Cluster.Namespace, + input.Cluster.Name, + filepath.Join(input.ArtifactsDirectory, "clusters", input.Cluster.Name, "machines")) + + log.Info("Dumping all the Cluster API resources in namespace") + // Dump all Cluster API related resources to artifacts before deleting them. + framework.DumpAllResources(ctx, framework.DumpAllResourcesInput{ + Lister: input.ClusterProxy.GetClient(), + Namespace: input.Namespace.Name, + LogPath: logPath, + }) + + if !input.SkipCleanup { + log.Info("Deleting cluster") + // While https://github.com/kubernetes-sigs/cluster-api/issues/2955 is addressed in future iterations, there is a chance + // that cluster variable is not set even if the cluster exists, so we are calling DeleteAllClustersAndWait + // instead of DeleteClusterAndWait + framework.DeleteAllClustersAndWait(ctx, framework.DeleteAllClustersAndWaitInput{ + Client: input.ClusterProxy.GetClient(), + Namespace: input.Namespace.Name, + }, input.IntervalsGetter(input.SpecName, "wait-delete-cluster")...) + + log.Info("Deleting namespace used for hosting the test spec") + framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{ + Deleter: input.ClusterProxy.GetClient(), + Name: input.Namespace.Name, + }) + } + input.CancelWatches() +} diff --git a/test/e2e/kcp_adoption.go b/test/e2e/kcp_adoption.go index 628446efafe6..fbb97450a9a8 100644 --- a/test/e2e/kcp_adoption.go +++ b/test/e2e/kcp_adoption.go @@ -33,6 +33,7 @@ import ( "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" @@ -78,7 +79,14 @@ func KCPAdoptionSpec(ctx context.Context, inputGetter func() KCPAdoptionSpecInpu Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. - namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) + namespace, cancelWatches = setup.CreateSpecNamespace( + ctx, + setup.CreateSpecNamespaceInput{ + ArtifactsDirectory: input.ArtifactFolder, + ClusterProxy: input.BootstrapClusterProxy, + SpecName: specName, + }, + ) }) It("Should adopt up-to-date control plane Machines without modification", func() { @@ -222,6 +230,18 @@ func KCPAdoptionSpec(ctx context.Context, inputGetter func() KCPAdoptionSpecInpu AfterEach(func() { // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. - dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + setup.DumpSpecResourcesAndCleanup( + ctx, + setup.DumpSpecResourcesAndCleanupInput{ + SpecName: specName, + ClusterProxy: input.BootstrapClusterProxy, + ArtifactsDirectory: input.ArtifactFolder, + Namespace: namespace, + CancelWatches: cancelWatches, + Cluster: cluster, + IntervalsGetter: input.E2EConfig.GetIntervals, + SkipCleanup: input.SkipCleanup, + }, + ) }) } diff --git a/test/e2e/kcp_upgrade.go b/test/e2e/kcp_upgrade.go index ab8a69bff8a4..1f3a6aebc7a8 100644 --- a/test/e2e/kcp_upgrade.go +++ b/test/e2e/kcp_upgrade.go @@ -29,6 +29,7 @@ import ( "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" @@ -67,7 +68,14 @@ func KCPUpgradeSpec(ctx context.Context, inputGetter func() KCPUpgradeSpecInput) Expect(input.E2EConfig.Variables).To(HaveKey(CoreDNSVersionUpgradeTo)) // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. - namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) + namespace, cancelWatches = setup.CreateSpecNamespace( + ctx, + setup.CreateSpecNamespaceInput{ + ArtifactsDirectory: input.ArtifactFolder, + ClusterProxy: input.BootstrapClusterProxy, + SpecName: specName, + }, + ) }) It("Should successfully upgrade Kubernetes, DNS, kube-proxy, and etcd in a single control plane cluster", func() { @@ -149,6 +157,18 @@ func KCPUpgradeSpec(ctx context.Context, inputGetter func() KCPUpgradeSpecInput) AfterEach(func() { // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. - dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + setup.DumpSpecResourcesAndCleanup( + ctx, + setup.DumpSpecResourcesAndCleanupInput{ + SpecName: specName, + ClusterProxy: input.BootstrapClusterProxy, + ArtifactsDirectory: input.ArtifactFolder, + Namespace: namespace, + CancelWatches: cancelWatches, + Cluster: cluster, + IntervalsGetter: input.E2EConfig.GetIntervals, + SkipCleanup: input.SkipCleanup, + }, + ) }) } diff --git a/test/e2e/kcp_upgrade_ci_artifacts.go b/test/e2e/kcp_upgrade_ci_artifacts.go new file mode 100644 index 000000000000..a32ce771a701 --- /dev/null +++ b/test/e2e/kcp_upgrade_ci_artifacts.go @@ -0,0 +1,213 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/test/framework/kubetest" + "sigs.k8s.io/cluster-api/util" +) + +// KCPUpgradeCIArtifactsSpecInput is the input for KCPUpgradeSpec. +type KCPUpgradeCIArtifactsSpecInput struct { + E2EConfig *clusterctl.E2EConfig + ClusterctlConfigPath string + BootstrapClusterProxy framework.ClusterProxy + ArtifactFolder string + SkipCleanup bool +} + +// KCPUpgradeCIArtifactsSpec implements a test that verifies KCP to properly upgrade a control plane with 3 machines. +func KCPUpgradeCIArtifactsSpec(ctx context.Context, inputGetter func() KCPUpgradeCIArtifactsSpecInput) { + var ( + specName = "kcp-upgrade-ci-artifacts" + input KCPUpgradeCIArtifactsSpecInput + namespace *corev1.Namespace + cancelWatches context.CancelFunc + cluster *clusterv1.Cluster + controlPlane *controlplanev1.KubeadmControlPlane + ciVersion string + ) + + BeforeEach(func() { + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + input = inputGetter() + os.Setenv("USE_CI_ARTIFACTS", "false") + Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName) + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + Expect(input.E2EConfig.Variables).To(HaveKey("KUBERNETES_CI_ARTIFACTS_START_VERSION")) + var err error + ciVersion, err = kubetest.FetchKubernetesCIVersion() + Expect(err).NotTo(HaveOccurred()) + // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. + namespace, cancelWatches = setup.CreateSpecNamespace( + ctx, + setup.CreateSpecNamespaceInput{ + ArtifactsDirectory: input.ArtifactFolder, + ClusterProxy: input.BootstrapClusterProxy, + SpecName: specName, + }, + ) + }) + + It("Should successfully upgrade Kubernetes to the latest main branch version", func() { + clusterName := fmt.Sprintf("cluster-%s", util.RandomString(6)) + By("Creating a workload cluster") + cluster, controlPlane, _ = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "with-ci-artifacts", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: input.E2EConfig.GetVariable("KUBERNETES_CI_ARTIFACTS_START_VERSION"), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }) + + os.Setenv("USE_CI_ARTIFACTS", "true") + + By("Upgrading Kubernetes") + cluster, _, _ = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "with-ci-artifacts", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: ciVersion, + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), + }) + + By("Waiting for control plane to be up to date") + framework.WaitForControlPlaneMachinesToBeUpgraded(ctx, framework.WaitForControlPlaneMachinesToBeUpgradedInput{ + Cluster: cluster, + MachineCount: int(*controlPlane.Spec.Replicas), + KubernetesUpgradeVersion: ciVersion, + Lister: input.BootstrapClusterProxy.GetClient(), + }, + input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade")..., + ) + + By("PASSED!") + }) + + It("Should successfully upgrade Kubernetes to the latest main branch version in a HA cluster", func() { + clusterName := fmt.Sprintf("clusterha-upgrade-%s", util.RandomString(6)) + By("Creating a workload cluster") + + cluster, _, _ = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "with-ci-artifacts", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: input.E2EConfig.GetVariable("KUBERNETES_CI_ARTIFACTS_START_VERSION"), + ControlPlaneMachineCount: pointer.Int64Ptr(3), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }) + + By("Upgrading Kubernetes") + + cluster, _, _ = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "with-ci-artifacts", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: ciVersion, + ControlPlaneMachineCount: pointer.Int64Ptr(3), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }) + + By("Waiting for control plane to be up to date") + framework.WaitForControlPlaneMachinesToBeUpgraded(ctx, framework.WaitForControlPlaneMachinesToBeUpgradedInput{ + Cluster: cluster, + MachineCount: int(*controlPlane.Spec.Replicas), + KubernetesUpgradeVersion: ciVersion, + Lister: input.BootstrapClusterProxy.GetClient(), + }, + "20m", + ) + + By("PASSED!") + }) + + AfterEach(func() { + // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. + setup.DumpSpecResourcesAndCleanup( + ctx, + setup.DumpSpecResourcesAndCleanupInput{ + SpecName: specName, + ClusterProxy: input.BootstrapClusterProxy, + ArtifactsDirectory: input.ArtifactFolder, + Namespace: namespace, + CancelWatches: cancelWatches, + Cluster: cluster, + IntervalsGetter: input.E2EConfig.GetIntervals, + SkipCleanup: input.SkipCleanup, + }, + ) + }) +} diff --git a/test/e2e/kcp_upgrade_ci_artifacts_test.go b/test/e2e/kcp_upgrade_ci_artifacts_test.go new file mode 100644 index 000000000000..61f20c7a9ff5 --- /dev/null +++ b/test/e2e/kcp_upgrade_ci_artifacts_test.go @@ -0,0 +1,39 @@ +// +build e2e + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo" +) + +var _ = Describe("When testing KCP upgrade with CI artifacts", func() { + + KCPUpgradeCIArtifactsSpec(context.TODO(), func() KCPUpgradeCIArtifactsSpecInput { + return KCPUpgradeCIArtifactsSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + } + }) + +}) diff --git a/test/e2e/md_upgrades.go b/test/e2e/md_upgrades.go index d33bcf7398f4..9e895b089ae7 100644 --- a/test/e2e/md_upgrades.go +++ b/test/e2e/md_upgrades.go @@ -28,6 +28,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" @@ -65,7 +66,14 @@ func MachineDeploymentUpgradesSpec(ctx context.Context, inputGetter func() Machi Expect(input.E2EConfig.Variables).To(HaveValidVersion(input.E2EConfig.GetVariable(KubernetesVersionUpgradeFrom))) // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. - namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) + namespace, cancelWatches = setup.CreateSpecNamespace( + ctx, + setup.CreateSpecNamespaceInput{ + ArtifactsDirectory: input.ArtifactFolder, + ClusterProxy: input.BootstrapClusterProxy, + SpecName: specName, + }, + ) }) It("Should successfully upgrade Machines upon changes in relevant MachineDeployment fields", func() { @@ -113,6 +121,18 @@ func MachineDeploymentUpgradesSpec(ctx context.Context, inputGetter func() Machi AfterEach(func() { // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. - dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + setup.DumpSpecResourcesAndCleanup( + ctx, + setup.DumpSpecResourcesAndCleanupInput{ + SpecName: specName, + ClusterProxy: input.BootstrapClusterProxy, + ArtifactsDirectory: input.ArtifactFolder, + Namespace: namespace, + CancelWatches: cancelWatches, + Cluster: cluster, + IntervalsGetter: input.E2EConfig.GetIntervals, + SkipCleanup: input.SkipCleanup, + }, + ) }) } diff --git a/test/e2e/mhc_remediations.go b/test/e2e/mhc_remediations.go index 0d5b76dc72fd..c19ffc5dbb06 100644 --- a/test/e2e/mhc_remediations.go +++ b/test/e2e/mhc_remediations.go @@ -28,6 +28,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" @@ -62,7 +63,14 @@ func MachineRemediationSpec(ctx context.Context, inputGetter func() MachineRemed Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. - namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) + namespace, cancelWatches = setup.CreateSpecNamespace( + ctx, + setup.CreateSpecNamespaceInput{ + ArtifactsDirectory: input.ArtifactFolder, + ClusterProxy: input.BootstrapClusterProxy, + SpecName: specName, + }, + ) }) It("Should successfully remediate unhealthy machines with MachineHealthCheck", func() { @@ -100,6 +108,18 @@ func MachineRemediationSpec(ctx context.Context, inputGetter func() MachineRemed AfterEach(func() { // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. - dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + setup.DumpSpecResourcesAndCleanup( + ctx, + setup.DumpSpecResourcesAndCleanupInput{ + SpecName: specName, + ClusterProxy: input.BootstrapClusterProxy, + ArtifactsDirectory: input.ArtifactFolder, + Namespace: namespace, + CancelWatches: cancelWatches, + Cluster: cluster, + IntervalsGetter: input.E2EConfig.GetIntervals, + SkipCleanup: input.SkipCleanup, + }, + ) }) } diff --git a/test/e2e/quick_start.go b/test/e2e/quick_start.go index 5faea5972c6c..cf8b5dc6dac9 100644 --- a/test/e2e/quick_start.go +++ b/test/e2e/quick_start.go @@ -28,6 +28,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" @@ -65,7 +66,14 @@ func QuickStartSpec(ctx context.Context, inputGetter func() QuickStartSpecInput) Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. - namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) + namespace, cancelWatches = setup.CreateSpecNamespace( + ctx, + setup.CreateSpecNamespaceInput{ + ArtifactsDirectory: input.ArtifactFolder, + ClusterProxy: input.BootstrapClusterProxy, + SpecName: specName, + }, + ) }) It("Should create a workload cluster", func() { @@ -96,6 +104,18 @@ func QuickStartSpec(ctx context.Context, inputGetter func() QuickStartSpecInput) AfterEach(func() { // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. - dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + setup.DumpSpecResourcesAndCleanup( + ctx, + setup.DumpSpecResourcesAndCleanupInput{ + SpecName: specName, + ClusterProxy: input.BootstrapClusterProxy, + ArtifactsDirectory: input.ArtifactFolder, + Namespace: namespace, + CancelWatches: cancelWatches, + Cluster: cluster, + IntervalsGetter: input.E2EConfig.GetIntervals, + SkipCleanup: input.SkipCleanup, + }, + ) }) } diff --git a/test/e2e/self_hosted.go b/test/e2e/self_hosted.go index 06bfc173fe62..1f12a33f6ee0 100644 --- a/test/e2e/self_hosted.go +++ b/test/e2e/self_hosted.go @@ -28,7 +28,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" - "sigs.k8s.io/cluster-api/test/e2e/internal/log" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/bootstrap" "sigs.k8s.io/cluster-api/test/framework/clusterctl" @@ -69,7 +69,14 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput) Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. - namespace, cancelWatches = setupSpecNamespace(context.TODO(), specName, input.BootstrapClusterProxy, input.ArtifactFolder) + namespace, cancelWatches = setup.CreateSpecNamespace( + ctx, + setup.CreateSpecNamespaceInput{ + ArtifactsDirectory: input.ArtifactFolder, + ClusterProxy: input.BootstrapClusterProxy, + SpecName: specName, + }, + ) }) It("Should pivot the bootstrap cluster to a self-hosted cluster", func() { @@ -137,7 +144,7 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput) Namespace: namespace.Name, }) - log.Logf("Waiting for the cluster infrastructure to be provisioned") + By("Waiting for the cluster infrastructure to be provisioned") selfHostedCluster = framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{ Getter: selfHostedClusterProxy.GetClient(), Namespace: selfHostedNamespace.Name, @@ -174,7 +181,7 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput) Namespace: selfHostedNamespace.Name, }) - log.Logf("Waiting for the cluster infrastructure to be provisioned") + By("Waiting for the cluster infrastructure to be provisioned") cluster = framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{ Getter: input.BootstrapClusterProxy.GetClient(), Namespace: namespace.Name, @@ -186,6 +193,18 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput) } // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. - dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + setup.DumpSpecResourcesAndCleanup( + ctx, + setup.DumpSpecResourcesAndCleanupInput{ + SpecName: specName, + ClusterProxy: input.BootstrapClusterProxy, + ArtifactsDirectory: input.ArtifactFolder, + Namespace: namespace, + CancelWatches: cancelWatches, + Cluster: cluster, + IntervalsGetter: input.E2EConfig.GetIntervals, + SkipCleanup: input.SkipCleanup, + }, + ) }) } diff --git a/test/framework/alltypes_helpers.go b/test/framework/alltypes_helpers.go index b947a22b2cc6..fecae7e26c11 100644 --- a/test/framework/alltypes_helpers.go +++ b/test/framework/alltypes_helpers.go @@ -22,11 +22,14 @@ import ( "fmt" "io/ioutil" "os" + "os/exec" "path" "path/filepath" + "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -182,3 +185,16 @@ func PrettyPrint(v interface{}) string { } return string(b) } + +// CompleteCommand prints a command before running it. Acts as a helper function. +// privateArgs when true will not print arguments. +func CompleteCommand(cmd *exec.Cmd, desc string, privateArgs bool) *exec.Cmd { + cmd.Stderr = ginkgo.GinkgoWriter + cmd.Stdout = ginkgo.GinkgoWriter + if privateArgs { + Byf("%s: dir=%s, command=%s", desc, cmd.Dir, cmd) + } else { + Byf("%s: dir=%s, command=%s", desc, cmd.Dir, cmd.String()) + } + return cmd +} diff --git a/test/framework/bootstrap/kind_provider.go b/test/framework/bootstrap/kind_provider.go index 4f766b5ef334..4bbb6b21af1f 100644 --- a/test/framework/bootstrap/kind_provider.go +++ b/test/framework/bootstrap/kind_provider.go @@ -23,7 +23,7 @@ import ( . "github.com/onsi/gomega" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" kindv1 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4" kind "sigs.k8s.io/kind/pkg/cluster" ) diff --git a/test/framework/bootstrap/kind_util.go b/test/framework/bootstrap/kind_util.go index 6839b3bb7cf1..d5532c164572 100644 --- a/test/framework/bootstrap/kind_util.go +++ b/test/framework/bootstrap/kind_util.go @@ -27,7 +27,7 @@ import ( "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/exec" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" kind "sigs.k8s.io/kind/pkg/cluster" kindnodes "sigs.k8s.io/kind/pkg/cluster/nodes" kindnodesutils "sigs.k8s.io/kind/pkg/cluster/nodeutils" diff --git a/test/framework/cluster_helpers.go b/test/framework/cluster_helpers.go index da1514f9938f..3f9cc178152f 100644 --- a/test/framework/cluster_helpers.go +++ b/test/framework/cluster_helpers.go @@ -26,7 +26,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" "sigs.k8s.io/cluster-api/test/framework/options" "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/controller-runtime/pkg/client" @@ -41,13 +41,13 @@ type CreateClusterInput struct { // CreateCluster will create the Cluster and InfraCluster objects. func CreateCluster(ctx context.Context, input CreateClusterInput, intervals ...interface{}) { - By("creating an InfrastructureCluster resource") + By("Creating an InfrastructureCluster resource") Expect(input.Creator.Create(ctx, input.InfraCluster)).To(Succeed()) // This call happens in an eventually because of a race condition with the // webhook server. If the latter isn't fully online then this call will // fail. - By("creating a Cluster resource linked to the InfrastructureCluster resource") + By("Creating a Cluster resource linked to the InfrastructureCluster resource") Eventually(func() error { if err := input.Creator.Create(ctx, input.Cluster); err != nil { log.Logf("Failed to create a cluster: %+v", err) @@ -122,7 +122,7 @@ type WaitForClusterToProvisionInput struct { // WaitForClusterToProvision will wait for a cluster to have a phase status of provisioned. func WaitForClusterToProvision(ctx context.Context, input WaitForClusterToProvisionInput, intervals ...interface{}) { - By("waiting for cluster to enter the provisioned phase") + By("Waiting for cluster to enter the provisioned phase") Eventually(func() (string, error) { cluster := &clusterv1.Cluster{} key := client.ObjectKey{ @@ -162,7 +162,7 @@ func WaitForClusterDeleted(ctx context.Context, input WaitForClusterDeletedInput if options.SkipResourceCleanup { return } - By(fmt.Sprintf("waiting for cluster %s to be deleted", input.Cluster.GetName())) + By(fmt.Sprintf("Waiting for cluster %s to be deleted", input.Cluster.GetName())) Eventually(func() bool { cluster := &clusterv1.Cluster{} key := client.ObjectKey{ diff --git a/test/framework/cluster_proxy.go b/test/framework/cluster_proxy.go index 48d2ce145b8d..ede8be30e37b 100644 --- a/test/framework/cluster_proxy.go +++ b/test/framework/cluster_proxy.go @@ -36,7 +36,7 @@ import ( "k8s.io/client-go/tools/clientcmd/api" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/cluster-api/test/framework/exec" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/test/framework/clusterctl/client.go b/test/framework/clusterctl/client.go index 2e5cd0741355..3b93b3f010c7 100644 --- a/test/framework/clusterctl/client.go +++ b/test/framework/clusterctl/client.go @@ -28,7 +28,7 @@ import ( clusterctlclient "sigs.k8s.io/cluster-api/cmd/clusterctl/client" clusterctllog "sigs.k8s.io/cluster-api/cmd/clusterctl/log" "sigs.k8s.io/cluster-api/test/framework/clusterctl/logger" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" ) // Provide E2E friendly wrappers for the clusterctl client library. diff --git a/test/framework/clusterctl/clusterctl_helpers.go b/test/framework/clusterctl/clusterctl_helpers.go index 181c16584ab6..065ba5f3001c 100644 --- a/test/framework/clusterctl/clusterctl_helpers.go +++ b/test/framework/clusterctl/clusterctl_helpers.go @@ -27,7 +27,7 @@ import ( "sigs.k8s.io/cluster-api/cmd/clusterctl/client/config" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3" "sigs.k8s.io/cluster-api/test/framework" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" ) // InitManagementClusterAndWatchControllerLogsInput is the input type for InitManagementClusterAndWatchControllerLogs. diff --git a/test/framework/clusterctl/e2e_config.go b/test/framework/clusterctl/e2e_config.go index 6efd49a56c19..44e1a1bafcff 100644 --- a/test/framework/clusterctl/e2e_config.go +++ b/test/framework/clusterctl/e2e_config.go @@ -385,9 +385,18 @@ func (c *E2EConfig) GetIntervals(spec, key string) []interface{} { // GetVariable returns a variable from the e2e config file. func (c *E2EConfig) GetVariable(varName string) string { - version, ok := c.Variables[varName] + variable, ok := c.Variables[varName] Expect(ok).NotTo(BeFalse()) - return version + return variable +} + +// GetStringVariableWithDefault returns a variable from the e2e config file, but can return a default string. +func (c *E2EConfig) GetStringVariableWithDefault(varName string, defaultValue string) string { + variable, ok := c.Variables[varName] + if !ok { + return defaultValue + } + return variable } // GetVariable returns an Int64Ptr variable from the e2e config file. diff --git a/test/framework/config.go b/test/framework/config.go index 99ee1f7994a6..2d2ef7c5fcdb 100644 --- a/test/framework/config.go +++ b/test/framework/config.go @@ -22,6 +22,8 @@ import ( "io" "io/ioutil" "net/http" + "os" + "path" "regexp" "github.com/pkg/errors" @@ -363,3 +365,15 @@ func (g componentSourceGenerator) GetName() string { func (g componentSourceGenerator) Manifests(ctx context.Context) ([]byte, error) { return YAMLForComponentSource(ctx, g.ComponentSource) } + +// DumpToFile will dump the running e2e config to a file +func (c *Config) DumpToFile(filename string) error { + yaml, err := yaml.Marshal(c) + if err != nil { + return err + } + if err := os.MkdirAll(path.Dir(filename), 0o700); err != nil { + return err + } + return ioutil.WriteFile(filename, yaml, 0o600) +} diff --git a/test/framework/control_plane.go b/test/framework/control_plane.go index 3607e123b533..941c426891ec 100644 --- a/test/framework/control_plane.go +++ b/test/framework/control_plane.go @@ -34,7 +34,7 @@ type WaitForControlPlaneToBeUpToDateInput struct { // WaitForControlPlaneToBeUpToDate will wait for a control plane to be fully up-to-date. func WaitForControlPlaneToBeUpToDate(ctx context.Context, input WaitForControlPlaneToBeUpToDateInput, intervals ...interface{}) { - By("waiting for the control plane to be ready") + By("Waiting for the control plane to be ready") Eventually(func() (int32, error) { controlplane := &controlplanev1.KubeadmControlPlane{} key := client.ObjectKey{ diff --git a/test/framework/controlpane_helpers.go b/test/framework/controlpane_helpers.go index 521ba96452b2..1c23e5dbc479 100644 --- a/test/framework/controlpane_helpers.go +++ b/test/framework/controlpane_helpers.go @@ -27,7 +27,7 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/v1beta1" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -41,10 +41,10 @@ type CreateKubeadmControlPlaneInput struct { // CreateKubeadmControlPlane creates the control plane object and necessary dependencies. func CreateKubeadmControlPlane(ctx context.Context, input CreateKubeadmControlPlaneInput, intervals ...interface{}) { - By("creating the machine template") + By("Creating the machine template") Expect(input.Creator.Create(ctx, input.MachineTemplate)).To(Succeed()) - By("creating a KubeadmControlPlane") + By("Creating a KubeadmControlPlane") Eventually(func() error { err := input.Creator.Create(ctx, input.ControlPlane) if err != nil { @@ -120,7 +120,7 @@ func WaitForOneKubeadmControlPlaneMachineToExist(ctx context.Context, input Wait Expect(input.Lister).ToNot(BeNil(), "Invalid argument. input.Getter can't be nil when calling WaitForOneKubeadmControlPlaneMachineToExist") Expect(input.ControlPlane).ToNot(BeNil(), "Invalid argument. input.ControlPlane can't be nil when calling WaitForOneKubeadmControlPlaneMachineToExist") - By("waiting for one control plane node to exist") + By("Waiting for one control plane node to exist") inClustersNamespaceListOption := client.InNamespace(input.Cluster.Namespace) // ControlPlane labels matchClusterListOption := client.MatchingLabels{ @@ -152,7 +152,7 @@ type WaitForControlPlaneToBeReadyInput struct { // WaitForControlPlaneToBeReady will wait for a control plane to be ready. func WaitForControlPlaneToBeReady(ctx context.Context, input WaitForControlPlaneToBeReadyInput, intervals ...interface{}) { - By("waiting for the control plane to be ready") + By("Waiting for the control plane to be ready") Eventually(func() (bool, error) { controlplane := &controlplanev1.KubeadmControlPlane{} key := client.ObjectKey{ diff --git a/test/framework/convenience.go b/test/framework/convenience.go index 3819601261e4..2032b34c9602 100644 --- a/test/framework/convenience.go +++ b/test/framework/convenience.go @@ -17,8 +17,16 @@ limitations under the License. package framework import ( + "fmt" + "os" + "os/exec" + "path" + "path/filepath" "reflect" + "strings" + "github.com/onsi/ginkgo/config" + "github.com/onsi/ginkgo/reporters" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -77,3 +85,25 @@ func TypeToKind(i interface{}) string { func ObjectToKind(i runtime.Object) string { return reflect.ValueOf(i).Elem().Type().Name() } + +func ResolveArtifactsDirectory(input string) string { + if input != "" { + return input + } + if dir, ok := os.LookupEnv("ARTIFACTS"); ok { + return dir + } + + findRootCmd := exec.Command("git", "rev-parse", "--show-toplevel") + out, err := findRootCmd.Output() + if err != nil { + return "_artifacts" + } + rootDir := strings.TrimSpace(string(out)) + return path.Join(rootDir, "_artifacts") +} + +func CreateJUnitReporterForProw(artifactsDirectory string) *reporters.JUnitReporter { + junitPath := filepath.Join(artifactsDirectory, fmt.Sprintf("junit.e2e_suite.%d.xml", config.GinkgoConfig.ParallelNode)) + return reporters.NewJUnitReporter(junitPath) +} diff --git a/test/framework/daemonset_helpers.go b/test/framework/daemonset_helpers.go index 7466165e8f16..364b08a270fb 100644 --- a/test/framework/daemonset_helpers.go +++ b/test/framework/daemonset_helpers.go @@ -35,7 +35,7 @@ type WaitForKubeProxyUpgradeInput struct { // WaitForKubeProxyUpgrade waits until kube-proxy version matches with the kubernetes version. This is called during KCP upgrade. func WaitForKubeProxyUpgrade(ctx context.Context, input WaitForKubeProxyUpgradeInput, intervals ...interface{}) { - By("ensuring kube-proxy has the correct image") + By("Ensuring kube-proxy has the correct image") Eventually(func() (bool, error) { ds := &appsv1.DaemonSet{} diff --git a/test/framework/deployment_helpers.go b/test/framework/deployment_helpers.go index 7e9b9bbc36ea..348d2e78f8cc 100644 --- a/test/framework/deployment_helpers.go +++ b/test/framework/deployment_helpers.go @@ -35,7 +35,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -49,7 +49,7 @@ type WaitForDeploymentsAvailableInput struct { // all the desired replicas are in place. // This can be used to check if Cluster API controllers installed in the management cluster are working. func WaitForDeploymentsAvailable(ctx context.Context, input WaitForDeploymentsAvailableInput, intervals ...interface{}) { - By(fmt.Sprintf("waiting for deployment %s/%s to be available", input.Deployment.GetNamespace(), input.Deployment.GetName())) + By(fmt.Sprintf("Waiting for deployment %s/%s to be available", input.Deployment.GetNamespace(), input.Deployment.GetName())) deployment := &appsv1.Deployment{} Eventually(func() bool { key := client.ObjectKey{ @@ -233,7 +233,7 @@ type WaitForDNSUpgradeInput struct { // WaitForDNSUpgrade waits until CoreDNS version matches with the CoreDNS upgrade version. This is called during KCP upgrade. func WaitForDNSUpgrade(ctx context.Context, input WaitForDNSUpgradeInput, intervals ...interface{}) { - By("ensuring CoreDNS has the correct image") + By("Ensuring CoreDNS has the correct image") Eventually(func() (bool, error) { d := &appsv1.Deployment{} diff --git a/test/framework/deprecated.go b/test/framework/deprecated.go index 6ab9fd8f55ad..d82103ea4d5a 100644 --- a/test/framework/deprecated.go +++ b/test/framework/deprecated.go @@ -33,7 +33,7 @@ import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/client-go/kubernetes" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" "sigs.k8s.io/cluster-api/test/framework/management/kind" "sigs.k8s.io/cluster-api/test/framework/options" diff --git a/test/framework/internal/log/log.go b/test/framework/ginkgoextensions/output.go similarity index 65% rename from test/framework/internal/log/log.go rename to test/framework/ginkgoextensions/output.go index c672fcb413d6..88e4e4c16105 100644 --- a/test/framework/internal/log/log.go +++ b/test/framework/ginkgoextensions/output.go @@ -14,14 +14,24 @@ See the License for the specific language governing permissions and limitations under the License. */ -package log +package ginkgoextensions import ( "fmt" - . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo" + "k8s.io/klog" + "sigs.k8s.io/cluster-api/test/framework/log" ) -func Logf(format string, a ...interface{}) { - fmt.Fprintf(GinkgoWriter, "INFO: "+format+"\n", a...) +var Log log.Logger + +func init() { + klog.InitFlags(nil) + Log = log.Logger{} + klog.SetOutput(ginkgo.GinkgoWriter) +} + +func Byf(format string, a ...interface{}) { + ginkgo.By(fmt.Sprintf(format, a...)) } diff --git a/test/e2e/internal/log/log.go b/test/framework/kubetest/bindata.go similarity index 68% rename from test/e2e/internal/log/log.go rename to test/framework/kubetest/bindata.go index c672fcb413d6..3702181f05a9 100644 --- a/test/e2e/internal/log/log.go +++ b/test/framework/kubetest/bindata.go @@ -14,14 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -package log +package kubetest -import ( - "fmt" - - . "github.com/onsi/ginkgo" -) - -func Logf(format string, a ...interface{}) { - fmt.Fprintf(GinkgoWriter, "INFO: "+format+"\n", a...) -} +//go:generate sh -c "go-bindata -nometadata -pkg kubetest -o zz_generated.bindata.go.tmp data && cat ../../../hack/boilerplate/boilerplate.generatego.txt zz_generated.bindata.go.tmp > zz_generated.bindata.go && rm zz_generated.bindata.go.tmp" diff --git a/test/framework/kubetest/data/debian_injection_script.envsubst.sh b/test/framework/kubetest/data/debian_injection_script.envsubst.sh new file mode 100644 index 000000000000..d066fd0179f4 --- /dev/null +++ b/test/framework/kubetest/data/debian_injection_script.envsubst.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +## Please note that this file needs to be escaped for envsubst to function + +set -o nounset +set -o pipefail +set -o errexit + +[[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + +USE_CI_ARTIFACTS=${USE_CI_ARTIFACTS:=false} + +if [ ! "${USE_CI_ARTIFACTS}" = true ]; then + echo "No CI Artifacts installation, exiting" + exit 0 +fi + +GSUTIL=gsutil + +if ! command -v $${GSUTIL} >/dev/null; then + apt-get update + apt-get install -y apt-transport-https ca-certificates gnupg curl + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | $${SUDO} tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | $${SUDO} apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - + apt-get update + apt-get install -y google-cloud-sdk +fi + +$${GSUTIL} version + +# This test installs release packages or binaries that are a result of the CI and release builds. +# It runs '... --version' commands to verify that the binaries are correctly installed +# and finally uninstalls the packages. +# For the release packages it tests all versions in the support skew. +LINE_SEPARATOR="*************************************************" +echo "$${LINE_SEPARATOR}" + +## Clusterctl set variables +## +# $${KUBERNETES_VERSION} will be replaced by clusterctl +KUBERNETES_VERSION=${KUBERNETES_VERSION} +## +## End clusterctl set variables + +if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $${KUBERNETES_VERSION}" + # Check for semver + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="gs://kubernetes-release/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" + DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' >/etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" + DEBIAN_FRONTEND=noninteractive apt-get install -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" + done + else + CI_URL="gs://kubernetes-release-dev/ci/$${KUBERNETES_VERSION}-bazel/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + $${GSUTIL} cp "$${CI_URL}/$${CI_PACKAGE}" "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" + done + systemctl restart kubelet + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${GSUTIL} cp "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "k8s.gcr.io/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/kubernetes-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done +fi +echo "* checking binary versions" +echo "ctr version: " "$(ctr version)" +echo "kubeadm version: " "$(kubeadm version -o=short)" +echo "kubectl version: " "$(kubectl version --client=true --short=true)" +echo "kubelet version: " "$(kubelet --version)" +echo "$${LINE_SEPARATOR}" diff --git a/test/framework/kubetest/data/kustomization.yaml b/test/framework/kubetest/data/kustomization.yaml new file mode 100644 index 000000000000..e5db20f339ab --- /dev/null +++ b/test/framework/kubetest/data/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: + - ci-artifacts-source-template.yaml +patchesStrategicMerge: + - kustomizeversions.yaml + - platform-kustomization.yaml diff --git a/test/framework/kubetest/run.go b/test/framework/kubetest/run.go new file mode 100644 index 000000000000..ed5a7e86e91f --- /dev/null +++ b/test/framework/kubetest/run.go @@ -0,0 +1,211 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubetest + +import ( + "io/ioutil" + "os" + "os/exec" + "os/user" + "path" + "runtime" + "strconv" + "strings" + + "github.com/pkg/errors" + corev1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/cluster-api/test/framework" +) + +const ( + standardImage = "us.gcr.io/k8s-artifacts-prod/conformance" + ciArtifactImage = "gcr.io/kubernetes-ci-images/conformance" +) + +const ( + DefaultGinkgoNodes = 1 + DefaultGinkoSlowSpecThreshold = 120 +) + +type RunInput struct { + // ClusterProxy is a clusterctl test framework proxy for the workload cluster + // for which to run kubetest against + ClusterProxy framework.ClusterProxy + // NumberOfNodes is the number of cluster nodes that exist for kubetest + // to be aware of + NumberOfNodes int + // UseCIArtifacts will fetch the latest build from the main branch of kubernetes/kubernetes + UseCIArtifacts bool + // ArtifactsDirectory is where conformance suite output will go + ArtifactsDirectory string + // Path to the kubetest e2e config file + ConfigFilePath string + // GinkgoNodes is the number of Ginkgo nodes to use + GinkgoNodes int + // GinkgoSlowSpecThreshold is time in s before spec is marked as slow + GinkgoSlowSpecThreshold int + // KubernetesVersion is the version of Kubernetes to test + KubernetesVersion string + // ConformanceImage is an optional field to specify an exact conformance image + ConformanceImage string +} + +// Run executes kube-test given an artifact directory, and sets settings +// required for kubetest to work with Cluster API. JUnit files are +// also gathered for inclusion in Prow. +func Run(input RunInput) error { + if input.GinkgoNodes == 0 { + input.GinkgoNodes = DefaultGinkgoNodes + } + if input.GinkgoSlowSpecThreshold == 0 { + input.GinkgoSlowSpecThreshold = 120 + } + if input.NumberOfNodes == 0 { + numNodes, err := countClusterNodes(input.ClusterProxy) + if err != nil { + return err + } + input.NumberOfNodes = numNodes + } + input.ArtifactsDirectory = framework.ResolveArtifactsDirectory(input.ArtifactsDirectory) + reportDir := path.Join(input.ArtifactsDirectory, "kubetest") + outputDir := path.Join(reportDir, "e2e-output") + kubetestConfigDir := path.Join(reportDir, "config") + if err := os.MkdirAll(outputDir, 0o750); err != nil { + return err + } + if err := os.MkdirAll(kubetestConfigDir, 0o750); err != nil { + return err + } + ginkgoVars := map[string]string{ + "nodes": strconv.Itoa(input.GinkgoNodes), + "slowSpecThreshold": strconv.Itoa(input.GinkgoSlowSpecThreshold), + } + + // Copy configuration files for kubetest into the artifacts directory + // to avoid issues with volume mounts on MacOS + tmpConfigFilePath := path.Join(kubetestConfigDir, "viper-config.yaml") + if err := copy(input.ConfigFilePath, tmpConfigFilePath); err != nil { + return err + } + tmpKubeConfigPath, err := dockeriseKubeconfig(kubetestConfigDir, input.ClusterProxy.GetKubeconfigPath()) + if err != nil { + return err + } + + e2eVars := map[string]string{ + "kubeconfig": "/tmp/kubeconfig", + "provider": "skeleton", + "report-dir": "/output", + "e2e-output-dir": "/output/e2e-output", + "dump-logs-on-failure": "false", + "report-prefix": "kubetest.", + "num-nodes": strconv.FormatInt(int64(input.NumberOfNodes), 10), + "viper-config": "/tmp/viper-config.yaml", + } + ginkgoArgs := buildArgs(ginkgoVars, "-") + e2eArgs := buildArgs(e2eVars, "--") + if input.ConformanceImage == "" { + input.ConformanceImage = versionToConformanceImage(input.KubernetesVersion, input.UseCIArtifacts) + } + kubeConfigVolumeMount := volumeArg(tmpKubeConfigPath, "/tmp/kubeconfig") + outputVolumeMount := volumeArg(reportDir, "/output") + viperVolumeMount := volumeArg(tmpConfigFilePath, "/tmp/viper-config.yaml") + user, err := user.Current() + if err != nil { + return errors.Wrap(err, "unable to determine current user") + } + userArg := user.Uid + ":" + user.Gid + e2eCmd := exec.Command("docker", "run", "--user", userArg, kubeConfigVolumeMount, outputVolumeMount, viperVolumeMount, "-t", input.ConformanceImage) + e2eCmd.Args = append(e2eCmd.Args, "/usr/local/bin/ginkgo") + e2eCmd.Args = append(e2eCmd.Args, ginkgoArgs...) + e2eCmd.Args = append(e2eCmd.Args, "/usr/local/bin/e2e.test") + e2eCmd.Args = append(e2eCmd.Args, "--") + e2eCmd.Args = append(e2eCmd.Args, e2eArgs...) + e2eCmd = framework.CompleteCommand(e2eCmd, "Running e2e test", false) + if err := e2eCmd.Run(); err != nil { + return errors.Wrap(err, "Unable to run conformance tests") + } + if err := framework.GatherJUnitReports(reportDir, input.ArtifactsDirectory); err != nil { + return err + } + return nil +} + +func dockeriseKubeconfig(kubetestConfigDir string, kubeConfigPath string) (string, error) { + kubeConfig, err := clientcmd.LoadFromFile(kubeConfigPath) + if err != nil { + return "", err + } + newPath := path.Join(kubetestConfigDir, "kubeconfig") + + // On CAPD, if not running on Linux, we need to use Docker's proxy to connect back to the host + // to the CAPD cluster. Moby on Linux doesn't use the host.docker.internal DNS name. + if runtime.GOOS != "linux" { + for i := range kubeConfig.Clusters { + kubeConfig.Clusters[i].Server = strings.ReplaceAll(kubeConfig.Clusters[i].Server, "127.0.0.1", "host.docker.internal") + } + } + if err := clientcmd.WriteToFile(*kubeConfig, newPath); err != nil { + return "", err + } + return newPath, nil +} + +func countClusterNodes(proxy framework.ClusterProxy) (int, error) { + nodeList, err := proxy.GetClientSet().CoreV1().Nodes().List(corev1.ListOptions{}) + if err != nil { + return 0, errors.Wrap(err, "Unable to count nodes") + } + return len(nodeList.Items), nil +} + +func isSELinuxEnforcing() bool { + dat, err := ioutil.ReadFile("/sys/fs/selinux/enforce") + if err != nil { + return false + } + return string(dat) == "1" +} + +func volumeArg(src, dest string) string { + volumeArg := "-v" + src + ":" + dest + if isSELinuxEnforcing() { + return volumeArg + ":z" + } + return volumeArg +} + +func versionToConformanceImage(kubernetesVersion string, usingCIArtifacts bool) string { + k8sVersion := strings.ReplaceAll(kubernetesVersion, "+", "_") + if usingCIArtifacts { + return ciArtifactImage + ":" + k8sVersion + } + return standardImage + ":" + k8sVersion +} + +// buildArgs converts a string map to the format --key=value +func buildArgs(kv map[string]string, flagMarker string) []string { + args := make([]string, len(kv)) + i := 0 + for k, v := range kv { + args[i] = flagMarker + k + "=" + v + i++ + } + return args +} diff --git a/test/framework/kubetest/setup.go b/test/framework/kubetest/setup.go new file mode 100644 index 000000000000..95dc81b35e87 --- /dev/null +++ b/test/framework/kubetest/setup.go @@ -0,0 +1,60 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubetest + +import ( + "io" + "io/ioutil" + "net/http" + "os" + "path" + "strings" +) + +const ( + ciVersionURL = "https://dl.k8s.io/ci/latest.txt" +) + +// FetchKubernetesCIVersion fetches the latest main branch Kubernetes version +func FetchKubernetesCIVersion() (string, error) { + resp, err := http.Get(ciVersionURL) + if err != nil { + return "", err + } + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + return strings.TrimSpace(string(b)), nil +} + +func copy(src, dest string) error { + os.MkdirAll(path.Dir(dest), 0o750) + srcFile, err := os.Open(src) + if err != nil { + return err + } + destFile, err := os.Create(dest) + if err != nil { + return err + } + if _, err := io.Copy(destFile, srcFile); err != nil { + return err + } + return nil +} diff --git a/test/framework/kubetest/template.go b/test/framework/kubetest/template.go new file mode 100644 index 000000000000..53329bc8cbae --- /dev/null +++ b/test/framework/kubetest/template.go @@ -0,0 +1,177 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubetest + +import ( + "errors" + "io/ioutil" + "os" + "os/exec" + "path" + + "sigs.k8s.io/yaml" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha3" + kcpv1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/framework" +) + +type GenerateCIArtifactsInjectedTemplateForDebianInput struct { + ArtifactsDirectory string + SourceTemplate []byte + PlatformKustomization []byte + KubeadmConfigTemplateName string + KubeadmControlPlaneName string +} + +func GenerateCIArtifactsInjectedTemplateForDebian(input GenerateCIArtifactsInjectedTemplateForDebianInput) (string, error) { + if input.SourceTemplate == nil { + return "", errors.New("SourceTemplate must be provided") + } + input.ArtifactsDirectory = framework.ResolveArtifactsDirectory(input.ArtifactsDirectory) + if input.KubeadmConfigTemplateName == "" { + input.KubeadmConfigTemplateName = "${ CLUSTER_NAME }-md-0" + } + if input.KubeadmControlPlaneName == "" { + input.KubeadmControlPlaneName = "${ CLUSTER_NAME }-control-plane" + } + templateDir := path.Join(input.ArtifactsDirectory, "templates") + overlayDir := path.Join(input.ArtifactsDirectory, "overlay") + + if err := os.MkdirAll(templateDir, 0o750); err != nil { + return "", err + } + if err := os.MkdirAll(overlayDir, 0o750); err != nil { + return "", err + } + + kustomizedTemplate := path.Join(templateDir, "cluster-template-conformance-ci-artifacts.yaml") + + kustomization, err := dataKustomizationYamlBytes() + if err != nil { + return "", err + } + + if err := ioutil.WriteFile(path.Join(overlayDir, "kustomization.yaml"), kustomization, 0o600); err != nil { + return "", err + } + + kustomizeVersions, err := generateKustomizeVersionsYaml(input.KubeadmControlPlaneName, input.KubeadmConfigTemplateName) + if err != nil { + return "", err + } + + if err := ioutil.WriteFile(path.Join(overlayDir, "kustomizeversions.yaml"), kustomizeVersions, 0o600); err != nil { + return "", err + } + if err := ioutil.WriteFile(path.Join(overlayDir, "ci-artifacts-source-template.yaml"), input.SourceTemplate, 0o600); err != nil { + return "", err + } + if err := ioutil.WriteFile(path.Join(overlayDir, "platform-kustomization.yaml"), input.PlatformKustomization, 0o600); err != nil { + return "", err + } + cmd := exec.Command("kustomize", "build", overlayDir) + data, err := cmd.CombinedOutput() + if err != nil { + return "", err + } + if err := ioutil.WriteFile(kustomizedTemplate, data, 0o640); err != nil { + return "", err + } + return kustomizedTemplate, nil +} + +func generateKustomizeVersionsYaml(kcpName, kubeadmName string) ([]byte, error) { + kcp, err := generateKubeadmControlPlane(kcpName) + if err != nil { + return nil, err + } + kubeadm, err := generateKubeadmConfigTemplate(kubeadmName) + if err != nil { + return nil, err + } + kcpYaml, err := yaml.Marshal(kcp) + if err != nil { + return nil, err + } + kubeadmYaml, err := yaml.Marshal(kubeadm) + if err != nil { + return nil, err + } + fileStr := string(kcpYaml) + "\n---\n" + string(kubeadmYaml) + return []byte(fileStr), nil +} + +func generateKubeadmConfigTemplate(name string) (*cabpkv1.KubeadmConfigTemplate, error) { + kubeadmSpec, err := generateKubeadmConfigSpec() + if err != nil { + return nil, err + } + return &cabpkv1.KubeadmConfigTemplate{ + TypeMeta: metav1.TypeMeta{ + Kind: "KubeadmConfigTemplate", + APIVersion: cabpkv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: cabpkv1.KubeadmConfigTemplateSpec{ + Template: cabpkv1.KubeadmConfigTemplateResource{ + Spec: *kubeadmSpec, + }, + }, + }, nil +} + +func generateKubeadmControlPlane(name string) (*kcpv1.KubeadmControlPlane, error) { + kubeadmSpec, err := generateKubeadmConfigSpec() + if err != nil { + return nil, err + } + return &kcpv1.KubeadmControlPlane{ + TypeMeta: metav1.TypeMeta{ + Kind: "KubeadmControlPlane", + APIVersion: kcpv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: kcpv1.KubeadmControlPlaneSpec{ + KubeadmConfigSpec: *kubeadmSpec, + Version: "${ KUBERNETES_VERSION }", + }, + }, nil +} + +func generateKubeadmConfigSpec() (*cabpkv1.KubeadmConfigSpec, error) { + data, err := dataDebian_injection_scriptEnvsubstShBytes() + if err != nil { + return nil, err + } + return &cabpkv1.KubeadmConfigSpec{ + Files: []cabpkv1.File{ + { + Path: "/usr/local/bin/ci-artifacts.sh", + Content: string(data), + Owner: "root:root", + Permissions: "0750", + }, + }, + PreKubeadmCommands: []string{"/usr/local/bin/ci-artifacts.sh"}, + }, nil +} diff --git a/test/framework/kubetest/zz_generated.bindata.go b/test/framework/kubetest/zz_generated.bindata.go new file mode 100644 index 000000000000..35f7bb35adf7 --- /dev/null +++ b/test/framework/kubetest/zz_generated.bindata.go @@ -0,0 +1,308 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated for package kubetest by go-bindata DO NOT EDIT. (@generated) +// sources: +// data/debian_injection_script.envsubst.sh +// data/ds.sh +// data/kustomization.yaml +package kubetest + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" +) + +func bindataRead(data []byte, name string) ([]byte, error) { + gz, err := gzip.NewReader(bytes.NewBuffer(data)) + if err != nil { + return nil, fmt.Errorf("Read %q: %v", name, err) + } + + var buf bytes.Buffer + _, err = io.Copy(&buf, gz) + clErr := gz.Close() + + if err != nil { + return nil, fmt.Errorf("Read %q: %v", name, err) + } + if clErr != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +type asset struct { + bytes []byte + info os.FileInfo +} + +type bindataFileInfo struct { + name string + size int64 + mode os.FileMode + modTime time.Time +} + +// Name return file name +func (fi bindataFileInfo) Name() string { + return fi.name +} + +// Size return file size +func (fi bindataFileInfo) Size() int64 { + return fi.size +} + +// Mode return file mode +func (fi bindataFileInfo) Mode() os.FileMode { + return fi.mode +} + +// Mode return file modify time +func (fi bindataFileInfo) ModTime() time.Time { + return fi.modTime +} + +// IsDir return file whether a directory +func (fi bindataFileInfo) IsDir() bool { + return fi.mode&os.ModeDir != 0 +} + +// Sys return file is sys mode +func (fi bindataFileInfo) Sys() interface{} { + return nil +} + +var _dataDebian_injection_scriptEnvsubstSh = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xbc\x57\x61\x6f\xe3\xb8\x11\xfd\xae\x5f\x31\x2b\x07\xe7\xcd\x2d\x28\x6d\x8b\xa2\xb8\xe6\xa0\x43\x73\x5e\xed\xd6\xb8\x45\xb2\xb0\x9d\xf6\x00\xaf\x6b\xd0\xe4\x58\x26\x4c\x91\x02\x49\x79\xe3\x6e\xdc\xdf\x5e\x90\x96\x1c\xdb\x71\x72\xde\x2b\x70\xf9\x12\x6b\xc4\x79\x33\x7c\x9c\x37\x43\x75\x5e\xa5\x33\xa1\xd2\x19\xb5\x8b\x28\xea\x74\xe0\x93\x44\x6a\x11\x94\x76\x08\x6e\x41\x1d\xb8\x85\xb0\x30\x17\x12\x41\x21\x72\x0b\x4e\xc3\x0c\x01\x2d\xa3\x15\x72\x98\x6b\x03\xa8\x56\xb6\x9e\x59\xe7\x5f\xcd\x6b\xc5\x9c\xd0\x2a\x8a\x2c\x3a\x20\x1a\x94\xae\x95\x45\xd7\x3e\x56\xa2\xc2\x39\x15\xb2\x7d\x46\x63\xf0\x5e\xb8\x28\x1a\x8f\xe1\xe2\xb5\xe0\x40\xea\x4b\x78\x95\xc1\x5b\x98\x4c\xe0\xbb\xef\x60\x78\xf7\xee\x36\x8b\x6d\xcd\x75\x0c\x0f\x0f\xcd\x63\x1c\x45\x77\xc3\x7c\xda\xeb\x4f\xaf\x07\xa3\xfe\xfb\xeb\xde\x68\x98\x5d\x7c\x3d\x36\x5d\x65\x73\x2a\x2d\x6e\xa2\x48\xcc\x61\x0c\xaf\x20\x7e\xba\x66\x13\x43\x06\xce\xd4\x08\x93\x1f\xc1\x2d\x50\x45\x00\x3e\x1f\x78\x1b\xcd\x45\x14\x7d\x18\xde\x8d\xfa\x1f\xb3\xc2\xd6\x4e\xc8\x80\xf3\x0a\x98\x2e\x4b\xaa\x38\x90\x15\x5c\x5c\x7c\xdd\xae\xd8\xc0\x4f\x29\xc7\x55\xaa\x6a\x29\x77\x38\xb4\x72\xa4\x40\x07\x75\xc5\xa9\xc3\x3d\x83\x50\xd6\x51\x29\x81\xac\x83\xc9\x19\xaa\x6c\xa5\x8d\x23\x0b\xe7\x2a\x0b\x8c\x12\x86\xc6\x89\xb9\x60\xd4\xa1\x85\x42\xd5\x55\x01\xac\x36\xd2\x27\xc7\x16\x1a\x62\x8e\x33\x18\x5b\x51\x28\xe4\x64\xb6\xce\xd2\xda\x9a\xd4\x2e\xa8\xc1\x74\x89\x6b\x23\x54\x61\x53\x26\x75\xcd\x93\x42\xeb\x42\x62\x52\x54\xc5\x04\x02\xfa\x55\x9a\x56\x94\x2d\x69\x81\x36\x39\x58\xc2\x74\x99\xd2\xca\x41\x30\x12\xcb\x97\x50\x52\xa1\x62\x78\xf0\xbb\xf4\xb4\x6f\xc0\x21\x02\xa1\x90\xa2\x63\x7e\x69\x6a\x75\x6d\x18\xda\x44\x0a\xeb\x12\x9e\x6e\x81\xc8\x0e\x20\xd8\x23\x08\x99\x9f\x17\x3c\xe5\x3a\x20\x93\x25\xae\x7d\xce\xfb\xc1\x1b\x33\x10\xd2\x6c\x11\xce\xd9\x35\x50\xce\x81\x9c\x79\x18\xc7\x1b\x08\x25\xb0\x77\xc6\x2b\x34\x36\x94\x76\x07\x46\x5e\x14\x0e\xed\xce\xdf\x82\xc1\xad\x72\xda\x1d\x82\x36\x30\x13\x8a\x1a\x81\x76\xab\x24\x6a\x10\x28\x18\xb4\xb5\x74\xa0\xe7\xbe\x4e\xa0\xd7\x07\x5f\x4c\xad\xf3\xac\x16\x92\xdb\x24\xea\x40\xdf\x81\xa9\x95\x85\x6e\x92\x24\x40\x48\x13\xbb\xdb\x96\x5f\xd0\xe1\x0a\x8d\x98\xaf\x5b\x99\xe2\x63\x38\x1f\x89\x69\x63\x90\x39\xb9\x6e\x53\x44\x1e\x75\x42\xb0\xb9\x50\x54\xca\x35\xd4\x6a\x97\xbc\xf7\xde\x1d\x4d\xd4\x81\xf7\xda\x04\xdb\x93\x4d\x09\x17\xb6\x6d\xc1\x73\xd6\x24\x65\x41\xa8\xb0\xda\xd6\x95\xaf\x63\xb0\x4b\xfc\x92\x44\x1f\xfb\x37\xf9\x74\x98\x7f\xba\x1e\x5c\x8f\x6e\x07\x59\xfc\xfd\xb7\xfe\xc5\xd1\xb6\xda\x2f\x2e\xbe\x1e\x62\x6d\xe2\xd0\xaa\x7a\xb2\xb6\x0e\x0d\x73\x12\x7c\x37\x59\x51\x23\xe8\x4c\xa2\x8d\x3a\x9d\xa8\xe3\x2b\xe7\x97\xbb\x9f\xf3\xc1\x4d\x3e\xca\x87\xd3\x7f\xe6\x83\x61\xff\xf6\x66\x03\x5f\x84\x94\xbe\x81\x19\xac\x24\x65\xc8\x61\xb6\x06\xb6\x03\x8a\x9e\xba\x64\x27\x71\x42\x8c\x0e\xe4\x8a\xef\x79\x1f\xa5\x11\xfa\xce\x38\xe4\x7f\x02\x21\xf6\x6d\x2e\x8e\x61\xf2\xd8\x79\x7a\xfd\xe9\xbb\xfe\x20\x4b\x5d\x59\xa5\xcb\x1f\x2c\x61\x22\x02\x28\x97\x5c\x18\x20\x55\xc0\xd9\xae\xd8\xc4\x11\x00\x47\x26\xfd\x39\x13\x0a\x9f\xae\x7b\xbf\x5c\x7f\xc8\x87\xd3\xd1\xed\x74\x94\x0f\x47\xd9\xeb\x78\x59\xcf\xfc\xe1\xc7\x10\x7e\x49\x74\xcd\x2f\xca\xcb\xf8\xf2\xd0\xbb\x77\x7b\x33\xba\xee\xdf\xe4\x83\x63\x7f\x42\x2b\x61\xd1\xac\xd0\x34\xce\x84\x69\xe5\x8c\x96\x12\x0d\x29\xa9\xa2\xc5\xe3\x9b\xca\xe8\xfb\x75\xfb\x60\xd9\x02\x79\x2d\xd1\x84\x50\x3b\xfc\x69\xfe\xeb\x28\x8b\x1d\x35\xf1\xae\x91\x7d\x1f\xaa\xc9\xab\xb9\xd7\x6f\xcb\xe9\x99\x93\xf3\x4e\x1d\xe8\x2d\x90\x2d\xc3\xd4\xb1\x58\xae\xd0\x44\x00\xbf\xc1\x72\xf6\x5f\xf8\xf7\x6a\xfc\x96\xfc\x6d\xf2\xe6\x73\x72\xf8\xff\x62\x9f\xfc\x40\xff\xdd\xe0\x63\x16\x17\xbe\x51\xf9\x8d\x18\x85\x0e\x2d\x69\x44\x90\xb6\xff\x4f\x47\x0a\x63\x54\x0a\x55\xdf\xa7\xb4\xe4\x7f\xfd\x4b\x1c\x30\x9b\xb7\xd3\x7f\xf5\x47\xff\xb8\xbd\x1b\x4d\x3f\x0d\xf2\xf7\xfd\x5f\xb3\xd3\xd9\x76\x56\x9b\xad\xd7\xbb\xfc\xe7\xfe\xf5\xcd\xf4\xfd\xe0\xf6\x66\x94\xdf\xbc\xcb\x94\x56\x42\x39\x34\x94\x39\xb1\xc2\xb3\x07\xc9\x76\x64\x34\x1d\x98\xd8\xdf\xdd\x84\xdb\xde\xdb\x36\xd3\xe6\xf4\xba\x7e\x0c\xb5\x98\xb4\x72\xc9\x23\x67\x89\xd0\x29\xec\x51\x78\x8f\x4a\x50\x19\x46\x4a\x17\x7e\x7a\x6e\x82\xec\xf9\x37\xb3\xe3\x44\xe3\xf6\x55\xd0\x88\x17\x12\xf8\x22\xdc\x02\x3e\x27\x07\x5c\x0f\xf2\x0f\xf9\x96\xe2\xd3\xec\xa7\x69\x92\x7e\xfe\x9c\x34\x5c\x37\xe2\xd9\xc9\x3d\xbe\x78\xed\x63\x32\xca\x16\x08\x25\xe5\xc2\x6a\x05\x8d\x88\xe0\x01\x0a\x83\x5b\x29\x1e\x44\xdb\x10\x3f\x29\x17\x48\x39\x10\xf5\x27\x78\x00\x56\x3b\x20\x1c\xba\x0f\x5d\x20\x73\xf8\x33\x3c\x80\x33\xc1\x30\xbe\xb2\x15\x65\x78\x35\xe9\x5e\x6e\xe3\xfb\x5a\xee\xf5\xa7\x4d\x1a\xbe\x95\x7a\xf4\x63\x49\x8f\xff\x3e\xd9\xc4\x3f\x02\xd7\xc1\xe7\x51\x3e\x4d\x09\x78\x05\x35\xa7\x7a\x05\xdb\x3e\xd1\x20\x6c\xe0\x11\xed\x40\x4b\xbf\xaf\xce\xe2\x43\xf0\xec\x59\x70\xae\x95\x3f\x2d\x94\x16\xcf\x11\x17\xf1\xd7\x27\x26\x9e\xd1\x16\x99\xd1\xff\xa0\x3c\xad\xb0\xff\x8f\x3f\xae\xbf\x28\xa9\x29\xf7\x04\x86\xf1\xb9\x6e\xf9\xbb\x1b\x7c\xdc\xa4\x87\xbb\x6d\x69\xdb\xbb\x13\xb0\x5d\x5f\x3e\xb5\x7e\xbf\x67\x3f\x83\xc5\x16\xa5\xe6\xf0\xe6\xfe\x8c\xa5\xe5\xea\xa5\x45\x10\x87\x0b\x91\xe7\xe8\x94\x7b\x73\x1e\x00\x76\x6d\x1d\x96\x7e\x5a\x19\xb4\x8e\x1a\xd7\x16\x77\x04\x30\xf7\x03\xa7\x21\x74\xd7\xb6\x5b\x4a\x9f\xce\x89\x43\x52\x4f\x51\x7a\x54\x93\x7b\x1c\xed\xd0\x36\xc9\x3e\xb6\x9f\x11\x4d\xc6\xbf\xc5\xf2\x8b\x08\x27\x98\x3a\x2b\xe2\xf6\xb2\xc9\xbc\x58\x15\x2c\x7f\xf0\x8d\x0c\x44\xb9\xbd\xfa\x94\xe1\x72\xf3\x8d\xc0\xfe\xab\x65\x27\xd6\x42\xe9\x70\x75\xc5\xfb\x0a\x99\x43\x0e\x5d\x1f\xe9\x00\xbf\xdb\x5c\x10\xcf\xc9\xc8\xd1\x02\x62\x6f\x2a\x98\xf1\x1d\xf7\x38\x1d\x12\x64\x72\x75\x52\x53\x69\xfa\x26\x9d\x7a\x9e\x5e\xf0\x7f\xd1\xf3\x8f\xca\xaf\xf1\xdd\x6b\x19\x4c\x90\x6d\x80\x6f\x4e\x38\x88\x60\x2e\xa2\xf6\x40\x98\xbf\x4d\x3c\x4a\x7f\x77\xa1\x6d\x6f\x9e\x7e\x57\x8d\xed\x0a\x7c\x4d\xbd\xde\xb3\x5c\xb6\xab\x9a\x7b\xd5\xd1\xca\x23\x2b\x10\x9d\xd9\x85\x36\xee\xc0\xcd\xcb\xf0\xa9\xdb\x9e\x15\x08\x61\x52\xa0\x72\x59\xf8\x4a\x25\x24\x60\x84\x87\x03\x20\x3f\x9c\x9e\x02\x79\xeb\xee\xdb\xe1\xf2\xa5\xfb\xf4\xff\x02\x00\x00\xff\xff\x44\x7b\xf8\x41\x07\x10\x00\x00") + +func dataDebian_injection_scriptEnvsubstShBytes() ([]byte, error) { + return bindataRead( + _dataDebian_injection_scriptEnvsubstSh, + "data/debian_injection_script.envsubst.sh", + ) +} + +func dataDebian_injection_scriptEnvsubstSh() (*asset, error) { + bytes, err := dataDebian_injection_scriptEnvsubstShBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "data/debian_injection_script.envsubst.sh", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _dataDsSh = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xbc\x57\x6d\x6f\xe3\xb8\x11\xfe\xae\x5f\x31\x2b\x1b\xe7\xcd\x2d\x28\x6d\x8b\xa2\xb8\xe6\xa0\x43\x73\x5e\xed\xd6\xb8\x45\xb2\xb0\x9d\xf6\x80\xac\x6b\xd0\xe4\x58\x26\x4c\x91\x02\x49\x79\xe3\x26\xee\x6f\x2f\x48\x4b\x7e\x8b\x37\xcd\x6e\x81\xe6\x4b\xac\x11\xe7\x99\xe1\xcc\x3c\x33\xa3\xce\xab\x74\x26\x54\x3a\xa3\x76\x11\x45\x9d\x0e\x7c\x92\x48\x2d\x82\xd2\x0e\xc1\x2d\xa8\x03\xb7\x10\x16\xe6\x42\x22\x28\x44\x6e\xc1\x69\x98\x21\xa0\x65\xb4\x42\x0e\x73\x6d\x00\xd5\xca\xd6\x33\xeb\xfc\xab\x79\xad\x98\x13\x5a\x45\x91\x45\x07\x44\x83\xd2\xb5\xb2\xe8\xda\xc7\x4a\x54\x38\xa7\x42\xb6\xcf\x68\x0c\xde\x0b\x17\x45\x77\x77\xd0\x7d\x2d\x38\x90\xfa\x02\x5e\x65\xf0\x16\x26\x13\xf8\xe1\x07\x18\xdd\xbe\xbb\xc9\x62\x5b\x73\x1d\xc3\xe3\x63\xf3\x18\x47\xd1\xed\x28\x9f\xf6\x07\xd3\xab\xe1\x78\xf0\xfe\xaa\x3f\x1e\x65\xdd\x87\x53\xd1\x65\x36\xa7\xd2\xe2\x26\x8a\xc4\x1c\xee\xe0\x15\xc4\x4f\xcf\x6c\x62\xc8\xc0\x99\x1a\x61\xf2\x33\xb8\x05\xaa\x08\xc0\xfb\x03\x6f\xa3\xb9\x88\xa2\x0f\xa3\xdb\xf1\xe0\x63\x56\xd8\xda\x09\x19\x70\x5e\x01\xd3\x65\x49\x15\x07\xb2\x82\xee\xc3\xf6\xc0\x06\x7e\x49\x39\xae\x52\x55\x4b\xb9\x83\xa1\x95\x23\x05\x3a\xa8\x2b\x4e\x1d\x1e\x08\x84\xb2\x8e\x4a\x09\x64\x1d\x44\xce\x50\x65\x2b\x6d\x1c\x59\x38\x57\x59\x60\x94\x30\x34\x4e\xcc\x05\xa3\x0e\x2d\x14\xaa\xae\x0a\x60\xb5\x91\xde\x37\xb6\xd0\x10\x73\x9c\xc1\x9d\x15\x85\x42\x4e\x66\xeb\x2c\xad\xad\x49\xed\x82\x1a\x4c\x97\xb8\x36\x42\x15\x36\x65\x52\xd7\x3c\x29\xb4\x2e\x24\x26\x45\x55\x4c\x20\xa0\x5f\xa6\x69\x45\xd9\x92\x16\x68\x93\xa3\x23\x4c\x97\x29\xad\x1c\x04\x21\xb1\x7c\x09\x25\x15\x2a\x86\x47\xe8\x3e\xf8\xa0\x6f\xc0\x21\x02\xa1\x90\xa2\x63\xfe\x64\x6a\x75\x6d\x18\xda\x44\x0a\xeb\x12\x9e\x6e\x71\xc8\x4e\x3f\xc8\x23\x08\x8e\xbf\xcc\x76\xca\x75\x40\x26\x4b\x5c\x7b\x97\x0f\x6c\x37\x52\x20\xa4\xb9\x20\xbc\xe4\xce\x40\x39\x07\xf2\xc2\x54\x9c\xfa\x1f\xf2\xbf\x4f\xf0\x0a\x8d\x0d\x65\xdd\x81\xb1\x27\x84\x43\xbb\x53\xb7\x60\x70\xcb\x9a\xf6\x7e\xa0\x0d\xcc\x84\xa2\x46\xa0\xdd\xb2\x88\x1a\x04\x0a\x06\x6d\x2d\x1d\xe8\xb9\x2f\x12\xe8\x0f\xc0\x17\x52\xab\x3c\xab\x85\xe4\x36\x89\x3a\x30\x70\x60\x6a\x65\xa1\x97\x24\x09\x10\xd2\xd8\xee\xb5\xa5\x17\x38\xb8\x42\x23\xe6\xeb\x96\xa2\xb8\x37\xe7\x2d\x31\x6d\x0c\x32\x27\xd7\xad\x8b\xc8\xa3\x4e\x30\x36\x17\x8a\x4a\xb9\x86\x5a\xed\x9c\xf7\xda\xbb\xc4\x44\x1d\x78\xaf\x4d\x90\x3d\xb9\x94\x70\xe1\xda\x16\x7c\xc8\x1a\xa7\x2c\x08\x15\x4e\xdb\xba\xf2\x45\x0c\x76\x89\x5f\x92\xe8\xe3\xe0\x3a\x9f\x8e\xf2\x4f\x57\xc3\xab\xf1\xcd\x30\x8b\x7f\xfc\xd6\xbf\x38\xda\x96\x7a\xf7\xe1\x18\x6a\x13\x87\x2e\xd5\x97\xb5\x75\x68\x98\x93\xe0\x1b\xc9\x8a\x1a\x41\x67\x12\x6d\xd4\xe9\x44\x1d\xe8\x3e\xfc\x76\xfb\x6b\x3e\xbc\xce\xc7\xf9\x68\xfa\xf7\x7c\x38\x1a\xdc\x5c\x6f\xe0\x8b\x90\xd2\xb7\x2e\x83\x95\xa4\x0c\x39\xcc\xd6\xc0\x76\x38\xd1\x53\x95\xec\x2c\x4e\x30\xd1\x81\x5c\xf1\x03\xed\x13\x2f\x42\xc7\xb9\xf3\xde\x9f\x01\x88\x7d\x7f\x8b\x63\x98\xec\x5b\x4e\x7f\x30\x7d\x37\x18\x66\xa9\x2b\xab\x74\xf9\x93\x25\x4c\x44\x00\xe5\x92\x0b\x03\xa4\xf2\x30\xdb\x03\x9b\x38\x02\xe0\xc8\xa4\xcf\x31\xa1\xf0\xe9\xaa\xff\xdb\xd5\x87\x7c\x34\x1d\xdf\x4c\xc7\xf9\x68\x9c\xbd\x8e\x97\xf5\xcc\x27\x3e\x86\xf0\x4b\xa2\x6b\x7e\x51\x5e\xc6\x17\xc7\xda\xfd\x9b\xeb\xf1\xd5\xe0\x3a\x1f\x9e\xea\x13\x5a\x09\x8b\x66\x85\xa6\x51\x26\x4c\x2b\x67\xb4\x94\x68\x48\x49\x15\x2d\xf6\x6f\x2a\xa3\xef\xd7\xed\x83\x65\x0b\xe4\xb5\x44\x13\x4c\xed\xf0\xa7\xf9\xef\xe3\x2c\x76\xd4\xc4\xbb\x0e\xf6\x63\xa8\x24\x4f\xe4\xfe\xa0\x2d\xa5\xf3\x69\xf3\x3a\x1d\xe8\x2f\x90\x2d\xc3\xb0\xb1\x58\xae\xd0\x44\x00\xcf\x87\x38\xfb\x37\xfc\x73\x75\xf7\x96\xfc\x65\xf2\xe6\x73\x72\xfc\xbf\x7b\x18\xf9\x10\xfb\xdb\xe1\xc7\x2c\x2e\x7c\x83\xf2\xd7\x30\x0a\x1d\x5a\xd2\x94\x7f\xda\xfe\x3f\x6b\x28\xcc\x4e\x29\x54\x7d\x9f\xd2\x92\xff\xf9\x4f\x71\x80\x6c\xde\x4e\xff\x31\x18\xff\xed\xe6\x76\x3c\xfd\x34\xcc\xdf\x0f\x7e\xcf\xce\xfa\xda\x59\x6d\xb6\x4a\xef\xf2\x5f\x07\x57\xd7\xd3\xf7\xc3\x9b\xeb\x71\x7e\xfd\x2e\x53\x5a\x09\xe5\xd0\x50\xe6\xc4\x0a\x5f\x3c\x3d\xb6\x73\xa2\xe9\xbb\xc4\x7e\x77\xeb\x6d\x5b\x6e\xdb\x43\x9b\xcc\xf5\xfc\xec\x69\x31\x69\xe5\x92\x7d\xc4\x12\xa1\x53\x38\x08\xe0\x3d\x2a\x41\x65\x98\x23\x3d\xf8\xe5\x6b\x73\xe3\x40\xbf\x99\x18\x67\xfa\xb5\x2f\x81\x86\xb6\x90\xc0\x17\xe1\x16\xf0\x39\x39\x0a\xf5\x30\xff\x90\x87\x08\x9f\x8f\x7d\x9a\x26\xe9\xe7\xcf\x49\x13\xea\x86\x37\x3b\x9e\xc7\xdd\xd7\xde\x24\xa3\x6c\x81\x50\x52\x2e\xac\x56\xd0\xf0\x07\x1e\xa1\x30\x18\x48\x78\x64\x6b\x43\xfc\x70\x5c\x20\xe5\x40\xd4\x1f\xe0\x11\x58\xed\x80\x70\xe8\x3d\xf6\x80\xcc\xe1\x8f\xf0\x08\xce\x04\xc1\xdd\xa5\xad\x28\xc3\xcb\x49\xef\x62\x6b\xde\x97\x71\x7f\x30\x6d\xbc\xf0\x0d\x34\xee\x3e\x9c\x72\xf9\xee\xaf\x93\x4d\xfc\x33\x70\x1d\x54\xf6\xbc\x69\xf2\xef\xa9\xd3\xa4\xf4\x12\x42\x7f\x68\x00\x36\xb0\xc3\x3a\xe2\xd0\xf7\x95\x58\x7c\x04\x9d\x7d\x0d\x9a\x6b\xe5\xd3\x84\xd2\xe2\x4b\x38\x45\xfc\xb2\xc4\xc4\x79\x4a\x91\x19\xfd\x17\xca\xf3\xc4\xfa\x9f\x22\xc7\xf5\x17\x25\x35\xe5\x3e\x74\x61\x5a\xae\x9b\xc8\xdd\x0e\x3f\x6e\xd2\xa3\x8b\xb6\x01\xdb\xcf\x7f\xd6\xb6\xe1\x33\x87\x0f\x1a\xf4\x79\x18\xb6\x28\x35\x87\x37\xf7\xff\xf5\x60\xb9\xfa\xfa\x11\x88\xc3\xce\xe3\xe3\x72\x46\xb7\xc9\x00\x80\x5d\x5b\x87\xa5\x1f\x49\x06\xad\xa3\xc6\xb5\x85\x1c\x01\xcc\xfd\x58\x69\x62\xb8\xeb\xce\x4d\x14\x9f\x4e\x83\xe3\x38\x9e\x8b\xe2\x71\x01\xee\x23\xb3\xc3\xda\x24\x07\xc0\x7e\x0c\x34\xde\x3e\x1b\xd7\xe7\xb4\x9f\x84\xe7\x05\xa6\xb6\x2b\x24\xf3\x6c\x54\xb0\xfc\xc9\xf7\x29\x10\xe5\x76\xa3\x29\xc3\xce\xf2\x2d\xa0\xfe\x2b\x64\xc7\xc5\x42\xe9\xb0\x8d\xe2\x7d\x85\xcc\x21\x87\x9e\x37\x73\x04\xde\x6b\x96\xbe\x17\x78\xe3\x68\x01\xb1\x17\x15\xcc\xf8\x66\x7a\xe2\x0b\x09\x44\xb8\x3c\x47\x9a\x34\x7d\x93\x4e\x7d\x74\xbe\xae\xfd\x9c\xde\xff\xc5\xb5\x46\xf3\xa0\x1b\x30\x41\xb6\xf0\xdf\xe8\x6b\xa8\xf5\xb9\x88\xda\x34\x30\xbf\x1c\xec\x59\xbd\x5b\x4d\xdb\x1d\xd2\xdf\xa8\x91\x5d\x82\x2f\xa1\xd7\x07\x92\x8b\xf6\x54\xb3\x25\x9d\x9c\x3c\x91\x02\xd1\x99\x5d\x68\xe3\x8e\xd4\x3c\xdb\x9e\xaa\x1d\x48\x81\x10\x26\x05\x2a\x97\x85\x6f\x4d\x42\x02\x46\x78\x38\x02\xf2\xf3\xe6\x29\x90\x97\xee\xbe\x02\x2e\x9e\xd9\x8c\xff\x13\x00\x00\xff\xff\xeb\xe0\xca\xb8\xcc\x0f\x00\x00") + +func dataDsShBytes() ([]byte, error) { + return bindataRead( + _dataDsSh, + "data/ds.sh", + ) +} + +func dataDsSh() (*asset, error) { + bytes, err := dataDsShBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "data/ds.sh", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _dataKustomizationYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x4c\x8e\x4d\x0a\xc2\x30\x10\x46\xf7\x39\x45\x2e\x90\x48\x77\x92\x2b\x88\x2b\xc1\xfd\x98\x4e\xea\x90\xe6\x87\x99\x69\x41\x4f\x2f\x25\x22\xae\xdf\xf7\x3e\x1e\x74\xba\x23\x0b\xb5\x1a\x6c\xde\x44\x5b\xa1\x37\xfa\xd8\x6a\xa2\xc5\xe7\xb3\x78\x6a\xa7\x7d\x7a\xa0\xc2\x64\x32\xd5\x39\xd8\xcb\x77\x05\x4a\xad\x9a\x0a\x05\xa5\x43\xc4\x60\x67\x4c\xb0\xad\x6a\x18\xa5\x6d\x1c\x51\x82\xb1\xd6\xd9\x48\x0e\x58\x29\x41\x54\x71\x83\x38\xc5\xd2\x57\x50\xf4\x2f\x28\xab\xe9\xa0\xf1\x89\x72\x53\x06\xc5\x85\xe2\x15\x79\xc1\x21\xff\x92\xf6\x11\x29\xc3\x38\xd0\x71\x90\x1a\x17\x97\xff\x83\x06\xff\x04\x00\x00\xff\xff\x16\x92\x86\x00\xd6\x00\x00\x00") + +func dataKustomizationYamlBytes() ([]byte, error) { + return bindataRead( + _dataKustomizationYaml, + "data/kustomization.yaml", + ) +} + +func dataKustomizationYaml() (*asset, error) { + bytes, err := dataKustomizationYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "data/kustomization.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +// Asset loads and returns the asset for the given name. +// It returns an error if the asset could not be found or +// could not be loaded. +func Asset(name string) ([]byte, error) { + cannonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[cannonicalName]; ok { + a, err := f() + if err != nil { + return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err) + } + return a.bytes, nil + } + return nil, fmt.Errorf("Asset %s not found", name) +} + +// MustAsset is like Asset but panics when Asset would return an error. +// It simplifies safe initialization of global variables. +func MustAsset(name string) []byte { + a, err := Asset(name) + if err != nil { + panic("asset: Asset(" + name + "): " + err.Error()) + } + + return a +} + +// AssetInfo loads and returns the asset info for the given name. +// It returns an error if the asset could not be found or +// could not be loaded. +func AssetInfo(name string) (os.FileInfo, error) { + cannonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[cannonicalName]; ok { + a, err := f() + if err != nil { + return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err) + } + return a.info, nil + } + return nil, fmt.Errorf("AssetInfo %s not found", name) +} + +// AssetNames returns the names of the assets. +func AssetNames() []string { + names := make([]string, 0, len(_bindata)) + for name := range _bindata { + names = append(names, name) + } + return names +} + +// _bindata is a table, holding each asset generator, mapped to its name. +var _bindata = map[string]func() (*asset, error){ + "data/debian_injection_script.envsubst.sh": dataDebian_injection_scriptEnvsubstSh, + "data/ds.sh": dataDsSh, + "data/kustomization.yaml": dataKustomizationYaml, +} + +// AssetDir returns the file names below a certain +// directory embedded in the file by go-bindata. +// For example if you run go-bindata on data/... and data contains the +// following hierarchy: +// data/ +// foo.txt +// img/ +// a.png +// b.png +// then AssetDir("data") would return []string{"foo.txt", "img"} +// AssetDir("data/img") would return []string{"a.png", "b.png"} +// AssetDir("foo.txt") and AssetDir("notexist") would return an error +// AssetDir("") will return []string{"data"}. +func AssetDir(name string) ([]string, error) { + node := _bintree + if len(name) != 0 { + cannonicalName := strings.Replace(name, "\\", "/", -1) + pathList := strings.Split(cannonicalName, "/") + for _, p := range pathList { + node = node.Children[p] + if node == nil { + return nil, fmt.Errorf("Asset %s not found", name) + } + } + } + if node.Func != nil { + return nil, fmt.Errorf("Asset %s not found", name) + } + rv := make([]string, 0, len(node.Children)) + for childName := range node.Children { + rv = append(rv, childName) + } + return rv, nil +} + +type bintree struct { + Func func() (*asset, error) + Children map[string]*bintree +} + +var _bintree = &bintree{nil, map[string]*bintree{ + "data": &bintree{nil, map[string]*bintree{ + "debian_injection_script.envsubst.sh": &bintree{dataDebian_injection_scriptEnvsubstSh, map[string]*bintree{}}, + "ds.sh": &bintree{dataDsSh, map[string]*bintree{}}, + "kustomization.yaml": &bintree{dataKustomizationYaml, map[string]*bintree{}}, + }}, +}} + +// RestoreAsset restores an asset under the given directory +func RestoreAsset(dir, name string) error { + data, err := Asset(name) + if err != nil { + return err + } + info, err := AssetInfo(name) + if err != nil { + return err + } + err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755)) + if err != nil { + return err + } + err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode()) + if err != nil { + return err + } + err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) + if err != nil { + return err + } + return nil +} + +// RestoreAssets restores an asset under the given directory recursively +func RestoreAssets(dir, name string) error { + children, err := AssetDir(name) + // File + if err != nil { + return RestoreAsset(dir, name) + } + // Dir + for _, child := range children { + err = RestoreAssets(dir, filepath.Join(name, child)) + if err != nil { + return err + } + } + return nil +} + +func _filePath(dir, name string) string { + cannonicalName := strings.Replace(name, "\\", "/", -1) + return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) +} diff --git a/test/framework/log/log.go b/test/framework/log/log.go new file mode 100644 index 000000000000..f88f7e7840fd --- /dev/null +++ b/test/framework/log/log.go @@ -0,0 +1,185 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package log + +import ( + "fmt" + "strings" + "time" + + "bytes" + "encoding/json" + "runtime" + "sort" + + "github.com/onsi/ginkgo" +) + +func nowStamp() string { + return time.Now().UTC().Format(time.RFC3339) +} + +func log(level string, format string, args ...interface{}) { + timeStr := flatten("time", nowStamp()) + fmt.Fprintf(ginkgo.GinkgoWriter, level+": "+format+" "+timeStr+"\n", args...) +} + +// Logf logs the info. +func Logf(format string, args ...interface{}) { + log("INFO", format, args...) +} + +type Logger struct { + level int + prefix string + values []interface{} +} + +func (l Logger) clone() Logger { + return Logger{ + level: l.level, + prefix: l.prefix, + values: copySlice(l.values), + } +} + +func copySlice(in []interface{}) []interface{} { + out := make([]interface{}, len(in)) + copy(out, in) + return out +} + +// Magic string for intermediate frames that we should ignore. +const autogeneratedFrameName = "" + +// Discover how many frames we need to climb to find the caller. This approach +// was suggested by Ian Lance Taylor of the Go team, so it *should* be safe +// enough (famous last words). +func framesToCaller() int { + // 1 is the immediate caller. 3 should be too many. + for i := 1; i < 3; i++ { + _, file, _, _ := runtime.Caller(i + 1) // +1 for this function's frame + if file != autogeneratedFrameName { + return i + } + } + return 1 // something went wrong, this is safe +} + +// trimDuplicates will deduplicates elements provided in multiple KV tuple +// slices, whilst maintaining the distinction between where the items are +// contained. +func trimDuplicates(kvLists ...[]interface{}) [][]interface{} { + // maintain a map of all seen keys + seenKeys := map[interface{}]struct{}{} + // build the same number of output slices as inputs + outs := make([][]interface{}, len(kvLists)) + // iterate over the input slices backwards, as 'later' kv specifications + // of the same key will take precedence over earlier ones + for i := len(kvLists) - 1; i >= 0; i-- { + // initialise this output slice + outs[i] = []interface{}{} + // obtain a reference to the kvList we are processing + kvList := kvLists[i] + + // start iterating at len(kvList) - 2 (i.e. the 2nd last item) for + // slices that have an even number of elements. + // We add (len(kvList) % 2) here to handle the case where there is an + // odd number of elements in a kvList. + // If there is an odd number, then the last element in the slice will + // have the value 'null'. + for i2 := len(kvList) - 2 + (len(kvList) % 2); i2 >= 0; i2 -= 2 { + k := kvList[i2] + // if we have already seen this key, do not include it again + if _, ok := seenKeys[k]; ok { + continue + } + // make a note that we've observed a new key + seenKeys[k] = struct{}{} + // attempt to obtain the value of the key + var v interface{} + // i2+1 should only ever be out of bounds if we handling the first + // iteration over a slice with an odd number of elements + if i2+1 < len(kvList) { + v = kvList[i2+1] + } + // add this KV tuple to the *start* of the output list to maintain + // the original order as we are iterating over the slice backwards + outs[i] = append([]interface{}{k, v}, outs[i]...) + } + } + return outs +} + +func flatten(kvList ...interface{}) string { + keys := make([]string, 0, len(kvList)) + vals := make(map[string]interface{}, len(kvList)) + for i := 0; i < len(kvList); i += 2 { + k, ok := kvList[i].(string) + if !ok { + panic(fmt.Sprintf("key is not a string: %s", pretty(kvList[i]))) + } + var v interface{} + if i+1 < len(kvList) { + v = kvList[i+1] + } + keys = append(keys, k) + vals[k] = v + } + sort.Strings(keys) + buf := bytes.Buffer{} + for i, k := range keys { + v := vals[k] + if i > 0 { + buf.WriteRune(' ') + } + buf.WriteString(pretty(k)) + buf.WriteString("=") + buf.WriteString(pretty(v)) + } + return buf.String() +} + +func pretty(value interface{}) string { + jb, _ := json.Marshal(value) + return string(jb) +} + +func (l Logger) Info(msg string, kvList ...interface{}) { + trimmed := trimDuplicates(l.values, kvList) + fixedStr := flatten(trimmed[0]...) + userStr := flatten(trimmed[1]...) + log("INFO", l.prefix+strings.Join([]string{msg + ":", fixedStr, userStr}, " ")) +} + +func (l Logger) Error(err error, msg string, kvList ...interface{}) { + var loggableErr interface{} + if err != nil { + loggableErr = err.Error() + } + errStr := flatten("error", loggableErr) + trimmed := trimDuplicates(l.values, kvList) + fixedStr := flatten(trimmed[0]...) + userStr := flatten(trimmed[1]...) + log("ERROR", l.prefix+strings.Join([]string{msg + ":", errStr, fixedStr, userStr}, " ")) +} + +func (l Logger) WithValues(kvList ...interface{}) Logger { + new := l.clone() + new.values = append(new.values, kvList...) + return new +} diff --git a/test/framework/machine_helpers.go b/test/framework/machine_helpers.go index 62234eafab75..d9be430c8bb9 100644 --- a/test/framework/machine_helpers.go +++ b/test/framework/machine_helpers.go @@ -26,7 +26,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -120,7 +120,7 @@ func WaitForControlPlaneMachinesToBeUpgraded(ctx context.Context, input WaitForC Expect(input.KubernetesUpgradeVersion).ToNot(BeEmpty(), "Invalid argument. input.KubernetesUpgradeVersion can't be empty when calling WaitForControlPlaneMachinesToBeUpgraded") Expect(input.MachineCount).To(BeNumerically(">", 0), "Invalid argument. input.MachineCount can't be smaller than 1 when calling WaitForControlPlaneMachinesToBeUpgraded") - By("ensuring all machines have upgraded kubernetes version") + By("Ensuring all machines have upgraded kubernetes version") log.Logf("Ensuring all MachineDeployment Machines have upgraded kubernetes version %s", input.KubernetesUpgradeVersion) Eventually(func() (int, error) { diff --git a/test/framework/machinedeployment_helpers.go b/test/framework/machinedeployment_helpers.go index 31a1a1f09d66..982bbb523b55 100644 --- a/test/framework/machinedeployment_helpers.go +++ b/test/framework/machinedeployment_helpers.go @@ -29,7 +29,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/test/framework/machines.go b/test/framework/machines.go index b19c2230140e..8c12109dc692 100644 --- a/test/framework/machines.go +++ b/test/framework/machines.go @@ -35,7 +35,7 @@ type WaitForClusterMachineNodeRefsInput struct { // WaitForClusterMachineNodesRefs waits until all nodes associated with a machine deployment exist. func WaitForClusterMachineNodeRefs(ctx context.Context, input WaitForClusterMachineNodeRefsInput, intervals ...interface{}) { - By("waiting for the machines' nodes to exist") + By("Waiting for the machines' nodes to exist") machines := &clusterv1.MachineList{} Expect(input.GetLister.List(ctx, machines, byClusterOptions(input.Cluster.Name, input.Cluster.Namespace)...)).To(Succeed(), "Failed to get Cluster machines %s/%s", input.Cluster.Namespace, input.Cluster.Name) @@ -61,7 +61,7 @@ type WaitForClusterMachinesReadyInput struct { } func WaitForClusterMachinesReady(ctx context.Context, input WaitForClusterMachinesReadyInput, intervals ...interface{}) { - By("waiting for the machines' nodes to be ready") + By("Waiting for the machines' nodes to be ready") machines := &clusterv1.MachineList{} Expect(input.GetLister.List(ctx, machines, byClusterOptions(input.Cluster.Name, input.Cluster.Namespace)...)).To(Succeed(), "Failed to get Cluster machines %s/%s", input.Cluster.Namespace, input.Cluster.Name) diff --git a/test/framework/namespace_helpers.go b/test/framework/namespace_helpers.go index be46ba2d18e9..6b4069b070e3 100644 --- a/test/framework/namespace_helpers.go +++ b/test/framework/namespace_helpers.go @@ -33,7 +33,7 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/test/framework/pod_helpers.go b/test/framework/pod_helpers.go index 70ec68a140fc..79151c24da6e 100644 --- a/test/framework/pod_helpers.go +++ b/test/framework/pod_helpers.go @@ -55,7 +55,7 @@ func WaitForPodListCondition(ctx context.Context, input WaitForPodListConditionI } return true, nil }, intervals...).Should(BeTrue()) - By("pod condition satisfied") + By("Pod condition satisfied") } // EtcdImageTagCondition returns a podListCondition that ensures the pod image diff --git a/test/framework/suite_helpers.go b/test/framework/suite_helpers.go new file mode 100644 index 000000000000..48e12e73f5b1 --- /dev/null +++ b/test/framework/suite_helpers.go @@ -0,0 +1,46 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "os" + "path" + "path/filepath" + "strings" +) + +func GatherJUnitReports(srcDir string, destDir string) error { + if err := os.MkdirAll(srcDir, 0o700); err != nil { + return err + } + return filepath.Walk(srcDir, func(p string, info os.FileInfo, err error) error { + if info.IsDir() && p != srcDir { + return filepath.SkipDir + } + if filepath.Ext(p) != ".xml" { + return nil + } + base := filepath.Base(p) + if strings.HasPrefix(base, "junit") { + newName := strings.ReplaceAll(base, "_", ".") + if err := os.Rename(p, path.Join(destDir, newName)); err != nil { + return err + } + } + return nil + }) +} diff --git a/test/infrastructure/docker/config/manager/manager_image_patch.yaml b/test/infrastructure/docker/config/manager/manager_image_patch.yaml index 0fee551ee40d..078b7b393ae4 100644 --- a/test/infrastructure/docker/config/manager/manager_image_patch.yaml +++ b/test/infrastructure/docker/config/manager/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: gcr.io/k8s-staging-cluster-api/capd-manager:dev + - image: gcr.io/k8s-staging-cluster-api/capd-manager-amd64:dev name: manager diff --git a/test/infrastructure/docker/controllers/dockermachine_controller.go b/test/infrastructure/docker/controllers/dockermachine_controller.go index 7b0b733948aa..38794d20689e 100644 --- a/test/infrastructure/docker/controllers/dockermachine_controller.go +++ b/test/infrastructure/docker/controllers/dockermachine_controller.go @@ -276,7 +276,7 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, machine * return ctrl.Result{}, err } - timeoutctx, cancel := context.WithTimeout(ctx, 3*time.Minute) + timeoutctx, cancel := context.WithTimeout(ctx, 6*time.Minute) defer cancel() // Run the bootstrap script. Simulates cloud-init. if err := externalMachine.ExecBootstrap(timeoutctx, bootstrapData); err != nil {