diff --git a/Makefile b/Makefile index d573a9523d97..b8c0e903c9a0 100644 --- a/Makefile +++ b/Makefile @@ -53,6 +53,8 @@ GO_APIDIFF := $(TOOLS_DIR)/$(GO_APIDIFF_BIN) ENVSUBST_BIN := bin/envsubst ENVSUBST := $(TOOLS_DIR)/$(ENVSUBST_BIN) +export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH) + # Binaries. # Need to use abspath so we can invoke these from subdirectories KUSTOMIZE := $(abspath $(TOOLS_BIN_DIR)/kustomize) @@ -220,7 +222,8 @@ generate: ## Generate code $(MAKE) -C test/infrastructure/docker generate .PHONY: generate-go -generate-go: ## Runs Go related generate targets +generate-go: $(GOBINDATA) ## Runs Go related generate targets + go generate ./... $(MAKE) generate-go-core $(MAKE) generate-go-kubeadm-bootstrap $(MAKE) generate-go-kubeadm-control-plane diff --git a/test/framework/cluster_proxy.go b/test/framework/cluster_proxy.go index 48d2ce145b8d..2a343fac3e1c 100644 --- a/test/framework/cluster_proxy.go +++ b/test/framework/cluster_proxy.go @@ -60,6 +60,9 @@ type ClusterProxy interface { // GetClientSet returns a client-go client to the Kubernetes cluster. GetClientSet() *kubernetes.Clientset + // GetRESTConfig returns the REST config for direct use with client-go if needed. + GetRESTConfig() *rest.Config + // Apply to apply YAML to the Kubernetes cluster, `kubectl apply`. Apply(context.Context, []byte) error @@ -158,7 +161,7 @@ func (p *clusterProxy) GetScheme() *runtime.Scheme { // GetClient returns a controller-runtime client for the cluster. func (p *clusterProxy) GetClient() client.Client { - config := p.getConfig() + config := p.GetRESTConfig() c, err := client.New(config, client.Options{Scheme: p.scheme}) Expect(err).ToNot(HaveOccurred(), "Failed to get controller-runtime client") @@ -168,7 +171,7 @@ func (p *clusterProxy) GetClient() client.Client { // GetClientSet returns a client-go client for the cluster. func (p *clusterProxy) GetClientSet() *kubernetes.Clientset { - restConfig := p.getConfig() + restConfig := p.GetRESTConfig() cs, err := kubernetes.NewForConfig(restConfig) Expect(err).ToNot(HaveOccurred(), "Failed to get client-go client") @@ -192,7 +195,7 @@ func (p *clusterProxy) ApplyWithArgs(ctx context.Context, resources []byte, args return exec.KubectlApplyWithArgs(ctx, p.kubeconfigPath, resources, args...) } -func (p *clusterProxy) getConfig() *rest.Config { +func (p *clusterProxy) GetRESTConfig() *rest.Config { config, err := clientcmd.LoadFromFile(p.kubeconfigPath) Expect(err).ToNot(HaveOccurred(), "Failed to load Kubeconfig file from %q", p.kubeconfigPath) @@ -308,6 +311,7 @@ func findLoadBalancerPort(ctx context.Context, name string) (string, error) { if err != nil { return "", err } + return strings.TrimSpace(string(stdout)), nil } diff --git a/test/framework/ginkgoextensions/output.go b/test/framework/ginkgoextensions/output.go new file mode 100644 index 000000000000..0c9a487b6e26 --- /dev/null +++ b/test/framework/ginkgoextensions/output.go @@ -0,0 +1,27 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ginkgoextensions + +import ( + "fmt" + + "github.com/onsi/ginkgo" +) + +func Byf(format string, a ...interface{}) { + ginkgo.By(fmt.Sprintf(format, a...)) +} diff --git a/test/framework/kubernetesversions/bindata.go b/test/framework/kubernetesversions/bindata.go new file mode 100644 index 000000000000..2938e2350bc3 --- /dev/null +++ b/test/framework/kubernetesversions/bindata.go @@ -0,0 +1,19 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetesversions + +//go:generate sh -c "go-bindata -nometadata -pkg kubernetesversions -o zz_generated.bindata.go.tmp data && cat ../../../hack/boilerplate/boilerplate.generatego.txt zz_generated.bindata.go.tmp > zz_generated.bindata.go && rm zz_generated.bindata.go.tmp" diff --git a/test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh b/test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh new file mode 100644 index 000000000000..6763815822ea --- /dev/null +++ b/test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Please note that this file needs to be escaped for envsubst to function + +# shellcheck disable=SC1083,SC2034,SC2066,SC2193 + +set -o nounset +set -o pipefail +set -o errexit + +[[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + +USE_CI_ARTIFACTS=${USE_CI_ARTIFACTS:=false} + +if [ ! "${USE_CI_ARTIFACTS}" = true ]; then + echo "No CI Artifacts installation, exiting" + exit 0 +fi + +GSUTIL=gsutil + +if ! command -v $${GSUTIL} >/dev/null; then + apt-get update + apt-get install -y apt-transport-https ca-certificates gnupg curl + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | $${SUDO} tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | $${SUDO} apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - + apt-get update + apt-get install -y google-cloud-sdk +fi + +$${GSUTIL} version + +# This test installs release packages or binaries that are a result of the CI and release builds. +# It runs '... --version' commands to verify that the binaries are correctly installed +# and finally uninstalls the packages. +# For the release packages it tests all versions in the support skew. +LINE_SEPARATOR="*************************************************" +echo "$${LINE_SEPARATOR}" + +## Clusterctl set variables +## +# $${KUBERNETES_VERSION} will be replaced by clusterctl +KUBERNETES_VERSION=${KUBERNETES_VERSION} +## +## End clusterctl set variables + +if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $${KUBERNETES_VERSION}" + # Check for semver + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="gs://kubernetes-release/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" + DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' >/etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" + DEBIAN_FRONTEND=noninteractive apt-get install -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" + done + else + CI_URL="gs://kubernetes-release-dev/ci/$${KUBERNETES_VERSION}-bazel/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + $${GSUTIL} cp "$${CI_URL}/$${CI_PACKAGE}" "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" + done + systemctl restart kubelet + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${GSUTIL} cp "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "k8s.gcr.io/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/kubernetes-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done +fi +echo "* checking binary versions" +echo "ctr version: " "$(ctr version)" +echo "kubeadm version: " "$(kubeadm version -o=short)" +echo "kubectl version: " "$(kubectl version --client=true --short=true)" +echo "kubelet version: " "$(kubelet --version)" +echo "$${LINE_SEPARATOR}" diff --git a/test/framework/kubernetesversions/data/kustomization.yaml b/test/framework/kubernetesversions/data/kustomization.yaml new file mode 100644 index 000000000000..e5db20f339ab --- /dev/null +++ b/test/framework/kubernetesversions/data/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: + - ci-artifacts-source-template.yaml +patchesStrategicMerge: + - kustomizeversions.yaml + - platform-kustomization.yaml diff --git a/test/framework/kubernetesversions/template.go b/test/framework/kubernetesversions/template.go new file mode 100644 index 000000000000..94da870183fa --- /dev/null +++ b/test/framework/kubernetesversions/template.go @@ -0,0 +1,190 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetesversions + +import ( + "errors" + "io/ioutil" + "os" + "os/exec" + "path" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha3" + kcpv1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/yaml" +) + +type GenerateCIArtifactsInjectedTemplateForDebianInput struct { + // ArtifactsDirectory is where conformance suite output will go. Defaults to _artifacts + ArtifactsDirectory string + // SourceTemplate is an input YAML clusterctl template which is to have + // the CI artifact script injection + SourceTemplate []byte + // PlatformKustomization is an SMP (strategic-merge-style) patch for adding + // platform specific kustomizations required for use with CI, such as + // referencing a specific image + PlatformKustomization []byte + // KubeadmConfigTemplateName is the name of the KubeadmConfigTemplate resource + // that needs to have the Debian install script injected. Defaults to "${CLUSTER_NAME}-md-0". + KubeadmConfigTemplateName string + // KubeadmControlPlaneName is the name of the KubeadmControlPlane resource + // that needs to have the Debian install script injected. Defaults to "${CLUSTER_NAME}-control-plane". + KubeadmControlPlaneName string +} + +// GenerateCIArtifactsInjectedTemplateForDebian takes a source clusterctl template +// and a platform-specific Kustomize SMP patch and injects a bash script to download +// and install the debian packages for the given Kubernetes version, returning the +// location of the outputted file. +func GenerateCIArtifactsInjectedTemplateForDebian(input GenerateCIArtifactsInjectedTemplateForDebianInput) (string, error) { + if input.SourceTemplate == nil { + return "", errors.New("SourceTemplate must be provided") + } + input.ArtifactsDirectory = framework.ResolveArtifactsDirectory(input.ArtifactsDirectory) + if input.KubeadmConfigTemplateName == "" { + input.KubeadmConfigTemplateName = "${CLUSTER_NAME}-md-0" + } + if input.KubeadmControlPlaneName == "" { + input.KubeadmControlPlaneName = "${CLUSTER_NAME}-control-plane" + } + templateDir := path.Join(input.ArtifactsDirectory, "templates") + overlayDir := path.Join(input.ArtifactsDirectory, "overlay") + + if err := os.MkdirAll(templateDir, 0o750); err != nil { + return "", err + } + if err := os.MkdirAll(overlayDir, 0o750); err != nil { + return "", err + } + + kustomizedTemplate := path.Join(templateDir, "cluster-template-conformance-ci-artifacts.yaml") + + kustomization, err := dataKustomizationYamlBytes() + if err != nil { + return "", err + } + + if err := ioutil.WriteFile(path.Join(overlayDir, "kustomization.yaml"), kustomization, 0o600); err != nil { + return "", err + } + + kustomizeVersions, err := generateKustomizeVersionsYaml(input.KubeadmControlPlaneName, input.KubeadmConfigTemplateName) + if err != nil { + return "", err + } + + if err := ioutil.WriteFile(path.Join(overlayDir, "kustomizeversions.yaml"), kustomizeVersions, 0o600); err != nil { + return "", err + } + if err := ioutil.WriteFile(path.Join(overlayDir, "ci-artifacts-source-template.yaml"), input.SourceTemplate, 0o600); err != nil { + return "", err + } + if err := ioutil.WriteFile(path.Join(overlayDir, "platform-kustomization.yaml"), input.PlatformKustomization, 0o600); err != nil { + return "", err + } + cmd := exec.Command("kustomize", "build", overlayDir) + data, err := cmd.CombinedOutput() + if err != nil { + return "", err + } + if err := ioutil.WriteFile(kustomizedTemplate, data, 0o600); err != nil { + return "", err + } + return kustomizedTemplate, nil +} + +func generateKustomizeVersionsYaml(kcpName, kubeadmName string) ([]byte, error) { + kcp, err := generateKubeadmControlPlane(kcpName) + if err != nil { + return nil, err + } + kubeadm, err := generateKubeadmConfigTemplate(kubeadmName) + if err != nil { + return nil, err + } + kcpYaml, err := yaml.Marshal(kcp) + if err != nil { + return nil, err + } + kubeadmYaml, err := yaml.Marshal(kubeadm) + if err != nil { + return nil, err + } + fileStr := string(kcpYaml) + "\n---\n" + string(kubeadmYaml) + return []byte(fileStr), nil +} + +func generateKubeadmConfigTemplate(name string) (*cabpkv1.KubeadmConfigTemplate, error) { + kubeadmSpec, err := generateKubeadmConfigSpec() + if err != nil { + return nil, err + } + return &cabpkv1.KubeadmConfigTemplate{ + TypeMeta: metav1.TypeMeta{ + Kind: "KubeadmConfigTemplate", + APIVersion: cabpkv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: cabpkv1.KubeadmConfigTemplateSpec{ + Template: cabpkv1.KubeadmConfigTemplateResource{ + Spec: *kubeadmSpec, + }, + }, + }, nil +} + +func generateKubeadmControlPlane(name string) (*kcpv1.KubeadmControlPlane, error) { + kubeadmSpec, err := generateKubeadmConfigSpec() + if err != nil { + return nil, err + } + return &kcpv1.KubeadmControlPlane{ + TypeMeta: metav1.TypeMeta{ + Kind: "KubeadmControlPlane", + APIVersion: kcpv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: kcpv1.KubeadmControlPlaneSpec{ + KubeadmConfigSpec: *kubeadmSpec, + Version: "${KUBERNETES_VERSION}", + }, + }, nil +} + +func generateKubeadmConfigSpec() (*cabpkv1.KubeadmConfigSpec, error) { + data, err := dataDebian_injection_scriptEnvsubstShBytes() + if err != nil { + return nil, err + } + return &cabpkv1.KubeadmConfigSpec{ + Files: []cabpkv1.File{ + { + Path: "/usr/local/bin/ci-artifacts.sh", + Content: string(data), + Owner: "root:root", + Permissions: "0750", + }, + }, + PreKubeadmCommands: []string{"/usr/local/bin/ci-artifacts.sh"}, + }, nil +} diff --git a/test/framework/kubernetesversions/versions.go b/test/framework/kubernetesversions/versions.go new file mode 100644 index 000000000000..231a329be780 --- /dev/null +++ b/test/framework/kubernetesversions/versions.go @@ -0,0 +1,78 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetesversions + +import ( + "fmt" + "io/ioutil" + "net/http" + "strings" + + "github.com/blang/semver" +) + +const ( + ciVersionURL = "https://dl.k8s.io/ci/latest.txt" + stableVersionURL = "https://storage.googleapis.com/kubernetes-release/release/stable-%d.%d.txt" + tagPrefix = "v" +) + +// LatestCIRelease fetches the latest main branch Kubernetes version +func LatestCIRelease() (string, error) { + resp, err := http.Get(ciVersionURL) + if err != nil { + return "", err + } + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + return strings.TrimSpace(string(b)), nil +} + +// LatestPatchRelease returns the latest patch release matching +func LatestPatchRelease(searchVersion string) (string, error) { + searchSemVer, err := semver.Make(strings.TrimPrefix(searchVersion, tagPrefix)) + if err != nil { + return "", err + } + resp, err := http.Get(fmt.Sprintf(stableVersionURL, searchSemVer.Major, searchSemVer.Minor)) + if err != nil { + return "", err + } + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + return strings.TrimSpace(string(b)), nil +} + +// PreviousMinorRelease returns the latest patch release for the previous version +// of Kubernetes, e.g. v1.19.1 returns v1.18.8 as of Sep 2020. +func PreviousMinorRelease(searchVersion string) (string, error) { + semVer, err := semver.Make(strings.TrimPrefix(searchVersion, tagPrefix)) + if err != nil { + return "", err + } + semVer.Minor-- + + return LatestPatchRelease(semVer.String()) +} diff --git a/test/framework/kubernetesversions/zz_generated.bindata.go b/test/framework/kubernetesversions/zz_generated.bindata.go new file mode 100644 index 000000000000..85c01480ce85 --- /dev/null +++ b/test/framework/kubernetesversions/zz_generated.bindata.go @@ -0,0 +1,285 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated for package kubernetesversions by go-bindata DO NOT EDIT. (@generated) +// sources: +// data/debian_injection_script.envsubst.sh +// data/kustomization.yaml +package kubernetesversions + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" +) + +func bindataRead(data []byte, name string) ([]byte, error) { + gz, err := gzip.NewReader(bytes.NewBuffer(data)) + if err != nil { + return nil, fmt.Errorf("Read %q: %v", name, err) + } + + var buf bytes.Buffer + _, err = io.Copy(&buf, gz) + clErr := gz.Close() + + if err != nil { + return nil, fmt.Errorf("Read %q: %v", name, err) + } + if clErr != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +type asset struct { + bytes []byte + info os.FileInfo +} + +type bindataFileInfo struct { + name string + size int64 + mode os.FileMode + modTime time.Time +} + +// Name return file name +func (fi bindataFileInfo) Name() string { + return fi.name +} + +// Size return file size +func (fi bindataFileInfo) Size() int64 { + return fi.size +} + +// Mode return file mode +func (fi bindataFileInfo) Mode() os.FileMode { + return fi.mode +} + +// Mode return file modify time +func (fi bindataFileInfo) ModTime() time.Time { + return fi.modTime +} + +// IsDir return file whether a directory +func (fi bindataFileInfo) IsDir() bool { + return fi.mode&os.ModeDir != 0 +} + +// Sys return file is sys mode +func (fi bindataFileInfo) Sys() interface{} { + return nil +} + +var _dataDebian_injection_scriptEnvsubstSh = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xbc\x57\x6d\x6f\xdb\x38\x12\xfe\xae\x5f\x31\x95\x83\x4d\xb3\x0d\xa5\x34\x5d\x14\xdd\x14\x5e\x9c\xeb\xa8\x3d\xa1\x81\x5d\xd8\xce\xbe\x20\xcd\x19\x34\x35\x96\x09\xd3\xa4\x8e\xa4\x9c\xf8\x1a\xdf\x6f\x3f\x90\x92\x1c\x3b\x71\xbb\xe9\x1e\xb0\xf9\xe2\x88\xe2\xbc\xf0\x19\x3e\xcf\x8c\x5a\xcf\xe2\x09\x97\xf1\x84\x9a\x59\x10\xb4\xa0\xab\x8a\x95\xe6\xf9\xcc\xc2\xe9\xc9\xe9\x09\x8c\x66\x08\x1f\xcb\x09\x6a\x89\x16\x0d\x74\x4a\x3b\x53\xda\x44\x41\x2b\x68\xc1\x05\x67\x28\x0d\x66\x50\xca\x0c\x35\xd8\x19\x42\xa7\xa0\x6c\x86\xcd\x9b\x63\xf8\x15\xb5\xe1\x4a\xc2\x69\x74\x02\xcf\xdd\x86\xb0\x7e\x15\x1e\xbd\x0d\x5a\xb0\x52\x25\x2c\xe8\x0a\xa4\xb2\x50\x1a\x04\x3b\xe3\x06\xa6\x5c\x20\xe0\x2d\xc3\xc2\x02\x97\xc0\xd4\xa2\x10\x9c\x4a\x86\x70\xc3\xed\xcc\x87\xa9\x9d\x44\x41\x0b\xfe\xa8\x5d\xa8\x89\xa5\x5c\x02\x05\xa6\x8a\x15\xa8\xe9\xf6\x3e\xa0\xd6\x27\xec\xfe\x66\xd6\x16\x67\x71\x7c\x73\x73\x13\x51\x9f\x6c\xa4\x74\x1e\x8b\x6a\xa3\x89\x2f\xd2\x6e\xd2\x1b\x26\xe4\x34\x3a\xf1\x26\x97\x52\xa0\x31\xa0\xf1\xdf\x25\xd7\x98\xc1\x64\x05\xb4\x28\x04\x67\x74\x22\x10\x04\xbd\x01\xa5\x81\xe6\x1a\x31\x03\xab\x5c\xbe\x37\x9a\x5b\x2e\xf3\x63\x30\x6a\x6a\x6f\xa8\xc6\xa0\x05\x19\x37\x56\xf3\x49\x69\x77\xc0\x6a\xb2\xe3\x66\x67\x83\x92\x40\x25\x84\x9d\x21\xa4\xc3\x10\xde\x75\x86\xe9\xf0\x38\x68\xc1\x6f\xe9\xe8\x9f\xfd\xcb\x11\xfc\xd6\x19\x0c\x3a\xbd\x51\x9a\x0c\xa1\x3f\x80\x6e\xbf\x77\x9e\x8e\xd2\x7e\x6f\x08\xfd\xf7\xd0\xe9\xfd\x01\x1f\xd3\xde\xf9\x31\x20\xb7\x33\xd4\x80\xb7\x85\x76\xf9\x2b\x0d\xdc\xc1\x88\x99\xc3\x6c\x88\xb8\x93\xc0\x54\x55\x09\x99\x02\x19\x9f\x72\x06\x82\xca\xbc\xa4\x39\x42\xae\x96\xa8\x25\x97\x39\x14\xa8\x17\xdc\xb8\x62\x1a\xa0\x32\x0b\x5a\x20\xf8\x82\x5b\x6a\xfd\xca\xa3\x43\x45\x41\xd0\x6a\xc1\x27\x81\xd4\xa0\x2b\xaf\x0b\x48\xed\x56\x81\x25\x62\x66\x1c\x64\x13\x04\x34\x8c\x16\x98\xf9\x3c\x50\x2e\x4d\x39\x31\xd6\xbd\x9a\x96\x92\x39\xff\xee\x5e\x9a\x19\x0a\xc1\x66\xc8\xe6\x0e\x2c\x87\x7e\x7b\xd8\x7d\x79\xf2\xe6\xd5\xf1\xb0\x7b\x7a\xf2\xea\x27\xff\xf3\xfa\xb5\xfb\x79\xf9\xf3\xab\x20\x30\x68\x81\x28\x90\xaa\x94\x06\x6d\xf3\x58\xf0\x02\xa7\x94\x8b\xe6\x19\xb5\xc6\x5b\x6e\x83\xe0\xea\x0a\x0e\x9e\xf3\x0c\x48\x79\x04\xcf\xda\x70\x02\xd7\xd7\xf0\xc3\x0f\x30\xbc\x3c\xef\xb7\x43\x53\x66\x2a\x84\xbb\xbb\xfa\x31\x0c\x82\xcb\x61\x32\xee\xa6\xe3\xce\x60\x94\xbe\xef\x74\x47\xc3\xf6\xc1\x97\x87\x4b\x67\xed\x29\x15\x06\xd7\x41\xc0\xa7\x70\x05\xcf\x20\x7c\xbc\x67\x1d\x42\x1b\xac\x2e\x11\xae\xdf\x3a\xf4\x64\x00\x80\x6c\xa6\x20\xec\x29\xe8\xa6\xd0\xd1\x96\x4f\x29\xb3\x06\xb8\x34\x96\x0a\xe1\xe1\x3e\x06\x97\x33\x97\x79\xe8\xb6\xdf\x72\x0b\x27\xc1\x94\x07\xc1\x87\xe1\xe5\x28\xbd\x68\xe7\xa6\xb4\x5c\xf8\xb0\xcf\x1c\x7b\x16\x54\x66\x40\x96\x70\x70\xf0\xa5\xda\xb1\x86\x5f\xe2\x0c\x97\xb1\x2c\x85\xd8\x84\xa5\x85\x25\x39\x5a\x28\x8b\x8c\x5a\xdc\x5a\xa8\x23\x03\x59\xf9\x25\xab\xa9\x34\x85\xd2\x96\x38\x26\x19\x60\x94\x30\x74\x69\x72\x46\x9d\x42\xe4\xb2\x2c\x72\x60\xa5\x16\x9b\xb3\x64\x38\x81\x2b\xc3\x73\x89\x19\x99\xac\xda\x71\x69\x74\x6c\x66\x54\x63\x3c\xc7\x95\xe6\x32\x37\x31\x13\xaa\xcc\xa2\x5c\xa9\x5c\x60\x94\x17\xf9\xb5\xe7\xa9\x39\x8b\xe3\x82\xb2\x39\xcd\xd1\x44\x3b\x5b\x98\x5a\xc4\xb4\xb0\xe0\x17\x89\xc9\xe6\xb0\xa0\x5c\x86\x70\xe7\x4e\xe9\xaa\xb4\x06\x8b\x08\x84\x42\x8c\x96\xb9\xad\xb1\x51\xa5\x66\x68\x22\xc1\x8d\x8d\xb2\xb8\x72\x44\x36\x0e\xfc\x7a\x00\x3e\xf3\xa7\x05\x8f\x33\xe5\x3d\x93\x39\xae\x5c\xce\xdb\xc1\xeb\x65\x20\xa4\x3e\x22\x3c\xe5\xd4\x40\xb3\x0c\xc8\x13\x8b\xf1\xf0\x00\xfe\x0a\x6c\xd5\x78\x59\x49\xaf\xe3\xce\xc8\xb1\xce\xa2\xd9\xd8\x3b\x41\xab\xa8\xd9\x9c\xd0\x29\xc4\x84\x4b\xaa\x39\x9a\x8a\xaa\x54\x23\x50\xd0\x68\x4a\x61\x1b\x3d\xed\xa6\x8e\xfb\x1b\xe3\x49\xc9\x45\xe6\x9a\x01\xa4\x16\x74\x29\x0d\x1c\x46\x51\x04\x84\xd4\xb1\x0f\x9b\xeb\xe7\x89\xbe\x44\xcd\xa7\xab\x46\x07\xf0\x3e\x9c\x8b\xc4\x94\xd6\xc8\xac\x58\x35\x29\xa2\x93\x18\x17\x6c\xca\x25\x15\x62\x05\xa5\xdc\x24\xef\xac\x37\xa5\x09\x5a\xf0\xbe\xd6\xaf\x47\x87\xe2\xd6\x1f\xdb\x80\xc3\xac\x4e\xca\x71\xa9\x52\xbb\xb2\x70\xf7\x18\xcc\x1c\x6f\xa2\xe0\x22\xed\x25\xe3\x61\xf2\xa9\x33\xe8\x8c\xfa\x83\x76\xf8\xe3\xf7\xfe\x85\x41\x75\xdb\x0f\x0e\xbe\xec\xfa\x5a\x87\x5e\x0b\xbb\xa2\x34\x16\x35\xb3\x02\x9c\xf8\x2c\xa9\xe6\x4e\xc5\x4c\xd0\x72\x7d\xe6\xe0\xe0\xcb\xc7\xcb\x77\xc9\xa0\x97\x8c\x92\xe1\xf8\xd7\x64\x30\x4c\xfb\xbd\x35\xdc\x70\x21\x9c\x42\x6a\x2c\x04\x65\x55\xff\x61\x1b\x47\xc1\x63\x93\xf6\x5e\x3f\x3e\x46\x0b\x12\x99\x6d\x59\x3f\x48\xc3\xcb\xd4\x95\xcf\x7f\x8f\x87\xd0\xa9\x62\x18\xc2\xf5\xbd\x50\x75\xd3\xf1\x79\x3a\x68\xc7\x76\x51\xc4\xf3\x37\x86\x30\x1e\x00\x2c\xe6\x19\xd7\x40\x0a\xef\xa7\xda\xb1\x76\x32\x95\x21\x13\xae\xce\x84\xc2\xa7\x4e\xf7\x63\xe7\x43\x32\x1c\x8f\xfa\xe3\x51\x32\x1c\xb5\x9f\x87\xf3\x72\xe2\x8a\x1f\x82\xff\x4f\xa0\xad\xff\xa3\xd9\x22\x3c\xda\xb5\xee\xf6\x7b\xa3\x4e\xda\x4b\x06\x0f\xed\x09\x2d\xb8\x41\xbd\x44\x5d\x1b\x13\xa6\xa4\xd5\x4a\x08\xd4\x64\x41\x25\xcd\xef\xdf\x14\x5a\xdd\xae\x9a\x07\xc3\x66\x98\x95\x02\xb5\x0f\xb5\xf1\x3f\x4e\x7e\x1f\xb5\x43\x4b\x75\xb8\x11\xb2\x1f\xfd\x6d\x72\x6c\xee\xa6\xcd\x75\xfa\x4a\xe5\x9c\x51\x0b\xba\xbe\x5f\xb9\xb6\x66\x70\xb1\x44\x1d\x00\xfc\x09\xca\xed\xff\xc2\xbf\x96\x57\x27\xe4\xe7\xeb\x17\x9f\xa3\xdd\xdf\x83\x6d\xf0\x3d\xfc\x97\x83\x8b\x76\x98\x3b\xa1\x9a\x6f\x46\x34\x52\x93\x20\x6e\x7e\xf7\x47\xf2\x33\x9f\xe0\xb2\xbc\x8d\xe9\x22\x7b\xfd\x53\xe8\x7d\xd6\x6f\xc7\xf5\xac\x31\xfe\x34\x48\xde\xa7\xbf\xb7\xf7\x67\xdb\x5a\xae\x2b\xab\xf3\xe4\x5d\xda\xe9\x8d\xdf\x0f\xfa\xbd\x51\xd2\x3b\x6f\x4b\x25\xb9\xb4\xa8\x29\xb3\x7c\x89\x4f\x6e\x24\x55\xcb\xa8\x15\x98\x98\xbf\x2c\xc2\x8d\xf6\x36\x62\x5a\x57\xef\xd0\xb5\xa1\xc6\x27\x2d\x6c\x74\x8f\x59\xc4\x55\x0c\x5b\x10\xde\xa2\xe4\x54\xf8\x96\x72\x08\xbf\x7c\xad\x83\x6c\xd9\xd7\xbd\x63\x8f\x70\xbb\x5b\x50\x93\x17\xa2\x6a\x80\xfd\x1c\xed\x60\x3d\x48\x3e\x24\x15\xc4\xfb\xd1\x8f\xe3\x28\xfe\xfc\x39\xaa\xb1\xae\xc9\xb3\xa1\x7b\x78\xf0\xdc\xc5\x64\x7e\xec\x5e\xd0\x8c\x1b\x25\xa1\x26\x11\xdc\x41\xae\xb1\xa2\xe2\x4e\xb4\x35\x71\x9d\x72\x86\x34\x03\x22\x5f\xc2\x1d\xb0\xd2\x02\xc9\xe0\xf0\xee\x10\xc8\x14\x4e\xe1\x0e\xac\xf6\x0b\x57\x67\xa6\xa0\x0c\xcf\xae\x0f\x8f\xaa\xf8\xee\x2e\x77\xd3\x71\x9d\x86\x93\x52\xe7\xfd\x21\xa5\xaf\xfe\x71\xbd\x0e\xdf\x42\xa6\xbc\xcd\x3d\x7d\xea\x2b\xe0\xe7\xc9\xaa\xaa\x67\x50\xe9\x44\xed\x61\x0d\xf7\xde\x76\xb8\xf4\xd7\xee\x59\xb8\xeb\xbc\xfd\x55\xe7\x99\x92\xae\x5a\x28\x0c\x3e\x85\x5c\xc4\x8d\x4f\x8c\x7f\x85\x5b\x64\x42\xff\x83\x62\x3f\xc3\xfe\x3f\xfc\x32\x75\x23\x85\xa2\x99\x03\xd0\xb7\xcf\x55\x83\xdf\xe5\xe0\x62\x1d\xef\x9e\xb6\x81\x6d\x6b\x26\x60\x1b\x5d\xde\xb7\x7f\x5b\xb3\xbf\xe2\x8b\xcd\x16\x2a\x83\x17\xb7\x4f\xd8\xba\x58\x7e\x6b\x13\x84\x7e\x20\x72\x18\xed\x33\xaf\xeb\x01\x60\x56\xc6\xe2\xc2\x75\x2b\x8d\xc6\x52\x6d\x9b\xcb\x1d\x00\x4c\x5d\xc3\xa9\x01\xdd\xc8\x76\x03\xe9\xe3\x3e\xb1\x0b\xea\x3e\x48\x1f\xdc\xc9\x2d\x8c\x36\xde\xd6\xd1\xb6\x6f\xd7\x23\xea\x8c\xff\x0c\xe5\x6f\x7a\xd8\x83\xd4\x93\x22\x56\xc3\x26\x73\x64\x95\x30\x7f\xe3\x84\x0c\xf8\xa2\x1a\x7d\x16\x7e\xb8\xf9\x4e\xc7\xee\x23\x67\x43\xd6\x5c\x2a\x3f\xba\xe2\x6d\x81\xcc\x7d\x99\x1e\xba\x48\x3b\xfe\x0f\xeb\x01\xf1\x29\x19\x59\x9a\x43\xe8\x96\x72\xa6\x9d\xe2\x3e\x4c\x87\x78\x9a\x9c\xed\xe5\x54\x1c\xbf\x88\xc7\x0e\xa7\x6f\xd8\x7f\xd3\xf2\xef\xca\xaf\xb6\xdd\x92\x0c\xc6\x49\x15\xe0\xbb\x13\xf6\x24\x98\xf2\xa0\x29\x88\xff\xfa\xbd\xa7\xfe\x66\xa0\x6d\x26\x4f\x77\xaa\x7a\xed\x0c\xdc\x9d\x7a\xbe\xb5\x72\xd4\xec\xaa\xe7\xaa\x07\x3b\x1f\xac\x02\x51\x6d\x33\x53\xda\xee\x98\x39\x1a\x3e\x36\xdb\x5a\x05\x42\x98\xe0\x28\x6d\xdb\x7f\xd4\x12\xe2\x7d\xf8\x87\x1d\x47\xae\x39\x3d\x76\xe4\x56\x37\xdf\x0e\x47\xdf\x9a\xa7\xff\x17\x00\x00\xff\xff\x3f\x8e\x7f\x9b\xb4\x12\x00\x00") + +func dataDebian_injection_scriptEnvsubstShBytes() ([]byte, error) { + return bindataRead( + _dataDebian_injection_scriptEnvsubstSh, + "data/debian_injection_script.envsubst.sh", + ) +} + +func dataDebian_injection_scriptEnvsubstSh() (*asset, error) { + bytes, err := dataDebian_injection_scriptEnvsubstShBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "data/debian_injection_script.envsubst.sh", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _dataKustomizationYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x4c\x8e\x4d\x0a\xc2\x30\x10\x46\xf7\x39\x45\x2e\x90\x48\x77\x92\x2b\x88\x2b\xc1\xfd\x98\x4e\xea\x90\xe6\x87\x99\x69\x41\x4f\x2f\x25\x22\xae\xdf\xf7\x3e\x1e\x74\xba\x23\x0b\xb5\x1a\x6c\xde\x44\x5b\xa1\x37\xfa\xd8\x6a\xa2\xc5\xe7\xb3\x78\x6a\xa7\x7d\x7a\xa0\xc2\x64\x32\xd5\x39\xd8\xcb\x77\x05\x4a\xad\x9a\x0a\x05\xa5\x43\xc4\x60\x67\x4c\xb0\xad\x6a\x18\xa5\x6d\x1c\x51\x82\xb1\xd6\xd9\x48\x0e\x58\x29\x41\x54\x71\x83\x38\xc5\xd2\x57\x50\xf4\x2f\x28\xab\xe9\xa0\xf1\x89\x72\x53\x06\xc5\x85\xe2\x15\x79\xc1\x21\xff\x92\xf6\x11\x29\xc3\x38\xd0\x71\x90\x1a\x17\x97\xff\x83\x06\xff\x04\x00\x00\xff\xff\x16\x92\x86\x00\xd6\x00\x00\x00") + +func dataKustomizationYamlBytes() ([]byte, error) { + return bindataRead( + _dataKustomizationYaml, + "data/kustomization.yaml", + ) +} + +func dataKustomizationYaml() (*asset, error) { + bytes, err := dataKustomizationYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "data/kustomization.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +// Asset loads and returns the asset for the given name. +// It returns an error if the asset could not be found or +// could not be loaded. +func Asset(name string) ([]byte, error) { + cannonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[cannonicalName]; ok { + a, err := f() + if err != nil { + return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err) + } + return a.bytes, nil + } + return nil, fmt.Errorf("Asset %s not found", name) +} + +// MustAsset is like Asset but panics when Asset would return an error. +// It simplifies safe initialization of global variables. +func MustAsset(name string) []byte { + a, err := Asset(name) + if err != nil { + panic("asset: Asset(" + name + "): " + err.Error()) + } + + return a +} + +// AssetInfo loads and returns the asset info for the given name. +// It returns an error if the asset could not be found or +// could not be loaded. +func AssetInfo(name string) (os.FileInfo, error) { + cannonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[cannonicalName]; ok { + a, err := f() + if err != nil { + return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err) + } + return a.info, nil + } + return nil, fmt.Errorf("AssetInfo %s not found", name) +} + +// AssetNames returns the names of the assets. +func AssetNames() []string { + names := make([]string, 0, len(_bindata)) + for name := range _bindata { + names = append(names, name) + } + return names +} + +// _bindata is a table, holding each asset generator, mapped to its name. +var _bindata = map[string]func() (*asset, error){ + "data/debian_injection_script.envsubst.sh": dataDebian_injection_scriptEnvsubstSh, + "data/kustomization.yaml": dataKustomizationYaml, +} + +// AssetDir returns the file names below a certain +// directory embedded in the file by go-bindata. +// For example if you run go-bindata on data/... and data contains the +// following hierarchy: +// data/ +// foo.txt +// img/ +// a.png +// b.png +// then AssetDir("data") would return []string{"foo.txt", "img"} +// AssetDir("data/img") would return []string{"a.png", "b.png"} +// AssetDir("foo.txt") and AssetDir("notexist") would return an error +// AssetDir("") will return []string{"data"}. +func AssetDir(name string) ([]string, error) { + node := _bintree + if len(name) != 0 { + cannonicalName := strings.Replace(name, "\\", "/", -1) + pathList := strings.Split(cannonicalName, "/") + for _, p := range pathList { + node = node.Children[p] + if node == nil { + return nil, fmt.Errorf("Asset %s not found", name) + } + } + } + if node.Func != nil { + return nil, fmt.Errorf("Asset %s not found", name) + } + rv := make([]string, 0, len(node.Children)) + for childName := range node.Children { + rv = append(rv, childName) + } + return rv, nil +} + +type bintree struct { + Func func() (*asset, error) + Children map[string]*bintree +} + +var _bintree = &bintree{nil, map[string]*bintree{ + "data": &bintree{nil, map[string]*bintree{ + "debian_injection_script.envsubst.sh": &bintree{dataDebian_injection_scriptEnvsubstSh, map[string]*bintree{}}, + "kustomization.yaml": &bintree{dataKustomizationYaml, map[string]*bintree{}}, + }}, +}} + +// RestoreAsset restores an asset under the given directory +func RestoreAsset(dir, name string) error { + data, err := Asset(name) + if err != nil { + return err + } + info, err := AssetInfo(name) + if err != nil { + return err + } + err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755)) + if err != nil { + return err + } + err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode()) + if err != nil { + return err + } + err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) + if err != nil { + return err + } + return nil +} + +// RestoreAssets restores an asset under the given directory recursively +func RestoreAssets(dir, name string) error { + children, err := AssetDir(name) + // File + if err != nil { + return RestoreAsset(dir, name) + } + // Dir + for _, child := range children { + err = RestoreAssets(dir, filepath.Join(name, child)) + if err != nil { + return err + } + } + return nil +} + +func _filePath(dir, name string) string { + cannonicalName := strings.Replace(name, "\\", "/", -1) + return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) +} diff --git a/test/framework/kubetest/run.go b/test/framework/kubetest/run.go new file mode 100644 index 000000000000..880a349bc0e3 --- /dev/null +++ b/test/framework/kubetest/run.go @@ -0,0 +1,238 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubetest + +import ( + "io/ioutil" + "os" + "os/exec" + "os/user" + "path" + "runtime" + "strconv" + "strings" + + "github.com/pkg/errors" + corev1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/discovery" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/cluster-api/test/framework" +) + +const ( + standardImage = "us.gcr.io/k8s-artifacts-prod/conformance" + ciArtifactImage = "gcr.io/kubernetes-ci-images/conformance" +) + +const ( + DefaultGinkgoNodes = 1 + DefaultGinkoSlowSpecThreshold = 120 +) + +type RunInput struct { + // ClusterProxy is a clusterctl test framework proxy for the workload cluster + // for which to run kubetest against + ClusterProxy framework.ClusterProxy + // NumberOfNodes is the number of cluster nodes that exist for kubetest + // to be aware of + NumberOfNodes int + // ArtifactsDirectory is where conformance suite output will go + ArtifactsDirectory string + // Path to the kubetest e2e config file + ConfigFilePath string + // GinkgoNodes is the number of Ginkgo nodes to use + GinkgoNodes int + // GinkgoSlowSpecThreshold is time in s before spec is marked as slow + GinkgoSlowSpecThreshold int + // KubernetesVersion is the version of Kubernetes to test (if not specified, then an attempt to discover the server version is made) + KubernetesVersion string + // ConformanceImage is an optional field to specify an exact conformance image + ConformanceImage string +} + +// Run executes kube-test given an artifact directory, and sets settings +// required for kubetest to work with Cluster API. JUnit files are +// also gathered for inclusion in Prow. +func Run(input RunInput) error { + if input.ClusterProxy == nil { + return errors.New("ClusterProxy must be provided") + } + if input.GinkgoNodes == 0 { + input.GinkgoNodes = DefaultGinkgoNodes + } + if input.GinkgoSlowSpecThreshold == 0 { + input.GinkgoSlowSpecThreshold = 120 + } + if input.NumberOfNodes == 0 { + numNodes, err := countClusterNodes(input.ClusterProxy) + if err != nil { + return errors.Wrap(err, "Unable to count number of cluster nodes") + } + input.NumberOfNodes = numNodes + } + if input.KubernetesVersion == "" && input.ConformanceImage == "" { + discoveredVersion, err := discoverClusterKubernetesVersion(input.ClusterProxy) + if err != nil { + return errors.Wrap(err, "Unable to discover server's Kubernetes version") + } + input.KubernetesVersion = discoveredVersion + } + input.ArtifactsDirectory = framework.ResolveArtifactsDirectory(input.ArtifactsDirectory) + reportDir := path.Join(input.ArtifactsDirectory, "kubetest") + outputDir := path.Join(reportDir, "e2e-output") + kubetestConfigDir := path.Join(reportDir, "config") + if err := os.MkdirAll(outputDir, 0o750); err != nil { + return err + } + if err := os.MkdirAll(kubetestConfigDir, 0o750); err != nil { + return err + } + ginkgoVars := map[string]string{ + "nodes": strconv.Itoa(input.GinkgoNodes), + "slowSpecThreshold": strconv.Itoa(input.GinkgoSlowSpecThreshold), + } + + // Copy configuration files for kubetest into the artifacts directory + // to avoid issues with volume mounts on MacOS + tmpConfigFilePath := path.Join(kubetestConfigDir, "viper-config.yaml") + if err := copyFile(input.ConfigFilePath, tmpConfigFilePath); err != nil { + return err + } + tmpKubeConfigPath, err := dockeriseKubeconfig(kubetestConfigDir, input.ClusterProxy.GetKubeconfigPath()) + if err != nil { + return err + } + + e2eVars := map[string]string{ + "kubeconfig": "/tmp/kubeconfig", + "provider": "skeleton", + "report-dir": "/output", + "e2e-output-dir": "/output/e2e-output", + "dump-logs-on-failure": "false", + "report-prefix": "kubetest.", + "num-nodes": strconv.FormatInt(int64(input.NumberOfNodes), 10), + "viper-config": "/tmp/viper-config.yaml", + } + ginkgoArgs := buildArgs(ginkgoVars, "-") + e2eArgs := buildArgs(e2eVars, "--") + if input.ConformanceImage == "" { + input.ConformanceImage = versionToConformanceImage(input.KubernetesVersion) + } + kubeConfigVolumeMount := volumeArg(tmpKubeConfigPath, "/tmp/kubeconfig") + outputVolumeMount := volumeArg(reportDir, "/output") + viperVolumeMount := volumeArg(tmpConfigFilePath, "/tmp/viper-config.yaml") + user, err := user.Current() + if err != nil { + return errors.Wrap(err, "unable to determine current user") + } + userArg := user.Uid + ":" + user.Gid + e2eCmd := exec.Command("docker", "run", "--user", userArg, kubeConfigVolumeMount, outputVolumeMount, viperVolumeMount, "-t", input.ConformanceImage) + e2eCmd.Args = append(e2eCmd.Args, "/usr/local/bin/ginkgo") + e2eCmd.Args = append(e2eCmd.Args, ginkgoArgs...) + e2eCmd.Args = append(e2eCmd.Args, "/usr/local/bin/e2e.test") + e2eCmd.Args = append(e2eCmd.Args, "--") + e2eCmd.Args = append(e2eCmd.Args, e2eArgs...) + e2eCmd = framework.CompleteCommand(e2eCmd, "Running e2e test", false) + if err := e2eCmd.Run(); err != nil { + return errors.Wrap(err, "Unable to run conformance tests") + } + if err := framework.GatherJUnitReports(reportDir, input.ArtifactsDirectory); err != nil { + return err + } + return nil +} + +func isUsingCIArtifactsVersion(k8sVersion string) bool { + return strings.Contains(k8sVersion, "-") +} + +func discoverClusterKubernetesVersion(proxy framework.ClusterProxy) (string, error) { + config := proxy.GetRESTConfig() + discoverClient, err := discovery.NewDiscoveryClientForConfig(config) + if err != nil { + return "", err + } + serverVersionInfo, err := discoverClient.ServerVersion() + if err != nil { + return "", err + } + + return serverVersionInfo.String(), nil +} + +func dockeriseKubeconfig(kubetestConfigDir string, kubeConfigPath string) (string, error) { + kubeConfig, err := clientcmd.LoadFromFile(kubeConfigPath) + if err != nil { + return "", err + } + newPath := path.Join(kubetestConfigDir, "kubeconfig") + + // On CAPD, if not running on Linux, we need to use Docker's proxy to connect back to the host + // to the CAPD cluster. Moby on Linux doesn't use the host.docker.internal DNS name. + if runtime.GOOS != "linux" { + for i := range kubeConfig.Clusters { + kubeConfig.Clusters[i].Server = strings.ReplaceAll(kubeConfig.Clusters[i].Server, "127.0.0.1", "host.docker.internal") + } + } + if err := clientcmd.WriteToFile(*kubeConfig, newPath); err != nil { + return "", err + } + return newPath, nil +} + +func countClusterNodes(proxy framework.ClusterProxy) (int, error) { + nodeList, err := proxy.GetClientSet().CoreV1().Nodes().List(corev1.ListOptions{}) + if err != nil { + return 0, errors.Wrap(err, "Unable to count nodes") + } + return len(nodeList.Items), nil +} + +func isSELinuxEnforcing() bool { + dat, err := ioutil.ReadFile("/sys/fs/selinux/enforce") + if err != nil { + return false + } + return string(dat) == "1" +} + +func volumeArg(src, dest string) string { + volumeArg := "-v" + src + ":" + dest + if isSELinuxEnforcing() { + return volumeArg + ":z" + } + return volumeArg +} + +func versionToConformanceImage(kubernetesVersion string) string { + k8sVersion := strings.ReplaceAll(kubernetesVersion, "+", "_") + if isUsingCIArtifactsVersion(kubernetesVersion) { + return ciArtifactImage + ":" + k8sVersion + } + return standardImage + ":" + k8sVersion +} + +// buildArgs converts a string map to the format --key=value +func buildArgs(kv map[string]string, flagMarker string) []string { + args := make([]string, len(kv)) + i := 0 + for k, v := range kv { + args[i] = flagMarker + k + "=" + v + i++ + } + return args +} diff --git a/test/framework/kubetest/setup.go b/test/framework/kubetest/setup.go new file mode 100644 index 000000000000..8302000d1a45 --- /dev/null +++ b/test/framework/kubetest/setup.go @@ -0,0 +1,42 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubetest + +import ( + "io" + "os" + "path" +) + +func copyFile(srcFilePath, destFilePath string) error { + err := os.MkdirAll(path.Dir(destFilePath), 0o750) + if err != nil { + return err + } + srcFile, err := os.Open(srcFilePath) + if err != nil { + return err + } + destFile, err := os.Create(destFilePath) + if err != nil { + return err + } + if _, err := io.Copy(destFile, srcFile); err != nil { + return err + } + return nil +} diff --git a/test/framework/suite_helpers.go b/test/framework/suite_helpers.go new file mode 100644 index 000000000000..6e7c7a0aa24f --- /dev/null +++ b/test/framework/suite_helpers.go @@ -0,0 +1,99 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + + "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/config" + "github.com/onsi/ginkgo/reporters" + . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" +) + +// GatherJUnitReports will move JUnit files from one directory to another, +// renaming them in a format expected by Prow. +func GatherJUnitReports(srcDir string, destDir string) error { + if err := os.MkdirAll(srcDir, 0o700); err != nil { + return err + } + + return filepath.Walk(srcDir, func(p string, info os.FileInfo, err error) error { + if info.IsDir() && p != srcDir { + return filepath.SkipDir + } + if filepath.Ext(p) != ".xml" { + return nil + } + base := filepath.Base(p) + if strings.HasPrefix(base, "junit") { + newName := strings.ReplaceAll(base, "_", ".") + if err := os.Rename(p, path.Join(destDir, newName)); err != nil { + return err + } + } + + return nil + }) +} + +// ResolveArtifactsDirectory attempts to resolve a directory to store test +// outputs, using either that provided by Prow, or defaulting to _artifacts +func ResolveArtifactsDirectory(input string) string { + if input != "" { + return input + } + if dir, ok := os.LookupEnv("ARTIFACTS"); ok { + return dir + } + + findRootCmd := exec.Command("git", "rev-parse", "--show-toplevel") + out, err := findRootCmd.Output() + if err != nil { + return "_artifacts" + } + rootDir := strings.TrimSpace(string(out)) + + return path.Join(rootDir, "_artifacts") +} + +// CreateJUnitReporterForProw sets up Ginkgo to create JUnit outputs compatible +// with Prow. +func CreateJUnitReporterForProw(artifactsDirectory string) *reporters.JUnitReporter { + junitPath := filepath.Join(artifactsDirectory, fmt.Sprintf("junit.e2e_suite.%d.xml", config.GinkgoConfig.ParallelNode)) + + return reporters.NewJUnitReporter(junitPath) +} + +// CompleteCommand prints a command before running it. Acts as a helper function. +// privateArgs when true will not print arguments. +func CompleteCommand(cmd *exec.Cmd, desc string, privateArgs bool) *exec.Cmd { + cmd.Stderr = ginkgo.GinkgoWriter + cmd.Stdout = ginkgo.GinkgoWriter + if privateArgs { + Byf("%s: dir=%s, command=%s", desc, cmd.Dir, cmd) + } else { + Byf("%s: dir=%s, command=%s", desc, cmd.Dir, cmd.String()) + } + + return cmd +}