test framework: Add kubetest, kubernetesversions, ginkgo extensions a…

…nd additional suite_helpers

Mostly upstreaming from CAPA.

The kubetest package allows a consumer to run kubetest (i.e. Kubernetes conformance) against
given clusters.

The kubernetesversions package provides helpers to determine the latest CI release of Kubernetes
and adds debian/ubuntu injection script kustomizations for  based on prior work by dims.

ginkgoextensions are common helpers that were added to CAPA, and are also present in the test/e2e
package.

Additionally add helpers to log running commands as well as gather JUnit files in a way that
is compatible with Prow.

Future PRs will add consumption of this in an e2e test and make CAPA consume these.

Signed-off-by: Naadir Jeewa <jeewan@vmware.com>

test/framework/ginkgoextensions/output.go

@@ -0,0 +1,37 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ginkgoextensions
+
+import (
+	"fmt"
+
+	"github.com/onsi/ginkgo"
+	"k8s.io/klog"
+	"sigs.k8s.io/cluster-api/test/framework/log"
+)
+
+var Log log.Logger
+
+func init() {
+	klog.InitFlags(nil)
+	Log = log.Logger{}
+	klog.SetOutput(ginkgo.GinkgoWriter)
+}
+
+func Byf(format string, a ...interface{}) {
+	ginkgo.By(fmt.Sprintf(format, a...))
+}

test/framework/kubernetesversions/bindata.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubernetesversions
+
+//go:generate sh -c "go-bindata -nometadata -pkg kubernetesversions -o zz_generated.bindata.go.tmp data && cat ../../../hack/boilerplate/boilerplate.generatego.txt zz_generated.bindata.go.tmp > zz_generated.bindata.go && rm zz_generated.bindata.go.tmp"

test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh

@@ -0,0 +1,106 @@
+#!/bin/bash
+
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## Please note that this file needs to be escaped for envsubst to function
+
+# shellcheck disable=SC1083,SC2034,SC2066,SC2193
+
+set -o nounset
+set -o pipefail
+set -o errexit
+
+[[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
+
+USE_CI_ARTIFACTS=${USE_CI_ARTIFACTS:=false}
+
+if [ ! "${USE_CI_ARTIFACTS}" = true ]; then
+  echo "No CI Artifacts installation, exiting"
+  exit 0
+fi
+
+GSUTIL=gsutil
+
+if ! command -v $${GSUTIL} >/dev/null; then
+  apt-get update
+  apt-get install -y apt-transport-https ca-certificates gnupg curl
+  echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | $${SUDO} tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+  curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | $${SUDO} apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
+  apt-get update
+  apt-get install -y google-cloud-sdk
+fi
+
+$${GSUTIL} version
+
+# This test installs release packages or binaries that are a result of the CI and release builds.
+# It runs '... --version' commands to verify that the binaries are correctly installed
+# and finally uninstalls the packages.
+# For the release packages it tests all versions in the support skew.
+LINE_SEPARATOR="*************************************************"
+echo "$${LINE_SEPARATOR}"
+
+## Clusterctl set variables
+##
+# $${KUBERNETES_VERSION} will be replaced by clusterctl
+KUBERNETES_VERSION=${KUBERNETES_VERSION}
+##
+## End clusterctl set variables
+
+if [[ "$${KUBERNETES_VERSION}" != "" ]]; then
+  CI_DIR=/tmp/k8s-ci
+  mkdir -p "$${CI_DIR}"
+  declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
+  declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
+  CONTAINER_EXT="tar"
+  echo "* testing CI version $${KUBERNETES_VERSION}"
+  # Check for semver
+  if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    CI_URL="gs://kubernetes-release/release/$${KUBERNETES_VERSION}/bin/linux/amd64"
+    VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}"
+    DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
+    curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+    echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' >/etc/apt/sources.list.d/kubernetes.list
+    apt-get update
+    # replace . with \.
+    VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}"
+    PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
+    for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
+      echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}"
+      DEBIAN_FRONTEND=noninteractive apt-get install -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}"
+    done
+  else
+    CI_URL="gs://kubernetes-release-dev/ci/$${KUBERNETES_VERSION}-bazel/bin/linux/amd64"
+    for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
+      echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}"
+      $${GSUTIL} cp "$${CI_URL}/$${CI_PACKAGE}" "$${CI_DIR}/$${CI_PACKAGE}"
+      chmod +x "$${CI_DIR}/$${CI_PACKAGE}"
+      mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}"
+    done
+    systemctl restart kubelet
+  fi
+  for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
+    echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}"
+    $${GSUTIL} cp "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}"
+    $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result"
+    $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "k8s.gcr.io/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}"
+    $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/kubernetes-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}"
+  done
+fi
+echo "* checking binary versions"
+echo "ctr version: " "$(ctr version)"
+echo "kubeadm version: " "$(kubeadm version -o=short)"
+echo "kubectl version: " "$(kubectl version --client=true --short=true)"
+echo "kubelet version: " "$(kubelet --version)"
+echo "$${LINE_SEPARATOR}"

test/framework/kubernetesversions/data/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: default
+resources:
+  - ci-artifacts-source-template.yaml
+patchesStrategicMerge:
+  - kustomizeversions.yaml
+  - platform-kustomization.yaml

test/framework/kubernetesversions/template.go

@@ -0,0 +1,177 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubernetesversions
+
+import (
+	"errors"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+
+	"sigs.k8s.io/yaml"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha3"
+	kcpv1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3"
+	"sigs.k8s.io/cluster-api/test/framework"
+)
+
+type GenerateCIArtifactsInjectedTemplateForDebianInput struct {
+	ArtifactsDirectory        string
+	SourceTemplate            []byte
+	PlatformKustomization     []byte
+	KubeadmConfigTemplateName string
+	KubeadmControlPlaneName   string
+}
+
+func GenerateCIArtifactsInjectedTemplateForDebian(input GenerateCIArtifactsInjectedTemplateForDebianInput) (string, error) {
+	if input.SourceTemplate == nil {
+		return "", errors.New("SourceTemplate must be provided")
+	}
+	input.ArtifactsDirectory = framework.ResolveArtifactsDirectory(input.ArtifactsDirectory)
+	if input.KubeadmConfigTemplateName == "" {
+		input.KubeadmConfigTemplateName = "${ CLUSTER_NAME }-md-0"
+	}
+	if input.KubeadmControlPlaneName == "" {
+		input.KubeadmControlPlaneName = "${ CLUSTER_NAME }-control-plane"
+	}
+	templateDir := path.Join(input.ArtifactsDirectory, "templates")
+	overlayDir := path.Join(input.ArtifactsDirectory, "overlay")
+
+	if err := os.MkdirAll(templateDir, 0o750); err != nil {
+		return "", err
+	}
+	if err := os.MkdirAll(overlayDir, 0o750); err != nil {
+		return "", err
+	}
+
+	kustomizedTemplate := path.Join(templateDir, "cluster-template-conformance-ci-artifacts.yaml")
+
+	kustomization, err := dataKustomizationYamlBytes()
+	if err != nil {
+		return "", err
+	}
+
+	if err := ioutil.WriteFile(path.Join(overlayDir, "kustomization.yaml"), kustomization, 0o600); err != nil {
+		return "", err
+	}
+
+	kustomizeVersions, err := generateKustomizeVersionsYaml(input.KubeadmControlPlaneName, input.KubeadmConfigTemplateName)
+	if err != nil {
+		return "", err
+	}
+
+	if err := ioutil.WriteFile(path.Join(overlayDir, "kustomizeversions.yaml"), kustomizeVersions, 0o600); err != nil {
+		return "", err
+	}
+	if err := ioutil.WriteFile(path.Join(overlayDir, "ci-artifacts-source-template.yaml"), input.SourceTemplate, 0o600); err != nil {
+		return "", err
+	}
+	if err := ioutil.WriteFile(path.Join(overlayDir, "platform-kustomization.yaml"), input.PlatformKustomization, 0o600); err != nil {
+		return "", err
+	}
+	cmd := exec.Command("kustomize", "build", overlayDir)
+	data, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", err
+	}
+	if err := ioutil.WriteFile(kustomizedTemplate, data, 0o600); err != nil {
+		return "", err
+	}
+	return kustomizedTemplate, nil
+}
+
+func generateKustomizeVersionsYaml(kcpName, kubeadmName string) ([]byte, error) {
+	kcp, err := generateKubeadmControlPlane(kcpName)
+	if err != nil {
+		return nil, err
+	}
+	kubeadm, err := generateKubeadmConfigTemplate(kubeadmName)
+	if err != nil {
+		return nil, err
+	}
+	kcpYaml, err := yaml.Marshal(kcp)
+	if err != nil {
+		return nil, err
+	}
+	kubeadmYaml, err := yaml.Marshal(kubeadm)
+	if err != nil {
+		return nil, err
+	}
+	fileStr := string(kcpYaml) + "\n---\n" + string(kubeadmYaml)
+	return []byte(fileStr), nil
+}
+
+func generateKubeadmConfigTemplate(name string) (*cabpkv1.KubeadmConfigTemplate, error) {
+	kubeadmSpec, err := generateKubeadmConfigSpec()
+	if err != nil {
+		return nil, err
+	}
+	return &cabpkv1.KubeadmConfigTemplate{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "KubeadmConfigTemplate",
+			APIVersion: cabpkv1.GroupVersion.String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Spec: cabpkv1.KubeadmConfigTemplateSpec{
+			Template: cabpkv1.KubeadmConfigTemplateResource{
+				Spec: *kubeadmSpec,
+			},
+		},
+	}, nil
+}
+
+func generateKubeadmControlPlane(name string) (*kcpv1.KubeadmControlPlane, error) {
+	kubeadmSpec, err := generateKubeadmConfigSpec()
+	if err != nil {
+		return nil, err
+	}
+	return &kcpv1.KubeadmControlPlane{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "KubeadmControlPlane",
+			APIVersion: kcpv1.GroupVersion.String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Spec: kcpv1.KubeadmControlPlaneSpec{
+			KubeadmConfigSpec: *kubeadmSpec,
+			Version:           "${ KUBERNETES_VERSION }",
+		},
+	}, nil
+}
+
+func generateKubeadmConfigSpec() (*cabpkv1.KubeadmConfigSpec, error) {
+	data, err := dataDebian_injection_scriptEnvsubstShBytes()
+	if err != nil {
+		return nil, err
+	}
+	return &cabpkv1.KubeadmConfigSpec{
+		Files: []cabpkv1.File{
+			{
+				Path:        "/usr/local/bin/ci-artifacts.sh",
+				Content:     string(data),
+				Owner:       "root:root",
+				Permissions: "0750",
+			},
+		},
+		PreKubeadmCommands: []string{"/usr/local/bin/ci-artifacts.sh"},
+	}, nil
+}