test framework: Add kubetest, kubernetesversions, ginkgo extensions a…

…nd additional suite_helpers

Mostly upstreaming from CAPA.

The kubetest package allows a consumer to run kubetest (i.e. Kubernetes conformance) against
given clusters.

The kubernetesversions package provides helpers to determine the latest CI release of Kubernetes
and adds debian/ubuntu injection script kustomizations for  based on prior work by dims.

ginkgoextensions are common helpers that were added to CAPA, and are also present in the test/e2e
package.

Additionally add helpers to log running commands as well as gather JUnit files in a way that
is compatible with Prow.

Future PRs will add consumption of this in an e2e test and make CAPA consume these.

Signed-off-by: Naadir Jeewa <jeewan@vmware.com>

Makefile

@@ -53,6 +53,8 @@ GO_APIDIFF := $(TOOLS_DIR)/$(GO_APIDIFF_BIN)
 ENVSUBST_BIN := bin/envsubst
 ENVSUBST := $(TOOLS_DIR)/$(ENVSUBST_BIN)
 
+export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH)
+
 # Binaries.
 # Need to use abspath so we can invoke these from subdirectories
 KUSTOMIZE := $(abspath $(TOOLS_BIN_DIR)/kustomize)
@@ -220,7 +222,8 @@ generate: ## Generate code
 	$(MAKE) -C test/infrastructure/docker generate
 
 .PHONY: generate-go
-generate-go: ## Runs Go related generate targets
+generate-go: $(GOBINDATA) ## Runs Go related generate targets
+	go generate ./...
 	$(MAKE) generate-go-core
 	$(MAKE) generate-go-kubeadm-bootstrap
 	$(MAKE) generate-go-kubeadm-control-plane

test/framework/cluster_proxy.go

@@ -60,6 +60,9 @@ type ClusterProxy interface {
 	// GetClientSet returns a client-go client to the Kubernetes cluster.
 	GetClientSet() *kubernetes.Clientset
 
+	// GetRESTConfig returns the REST config for direct use with client-go if needed.
+	GetRESTConfig() *rest.Config
+
 	// Apply to apply YAML to the Kubernetes cluster, `kubectl apply`.
 	Apply(context.Context, []byte) error
 
@@ -158,7 +161,7 @@ func (p *clusterProxy) GetScheme() *runtime.Scheme {
 
 // GetClient returns a controller-runtime client for the cluster.
 func (p *clusterProxy) GetClient() client.Client {
-	config := p.getConfig()
+	config := p.GetRESTConfig()
 
 	c, err := client.New(config, client.Options{Scheme: p.scheme})
 	Expect(err).ToNot(HaveOccurred(), "Failed to get controller-runtime client")
@@ -168,7 +171,7 @@ func (p *clusterProxy) GetClient() client.Client {
 
 // GetClientSet returns a client-go client for the cluster.
 func (p *clusterProxy) GetClientSet() *kubernetes.Clientset {
-	restConfig := p.getConfig()
+	restConfig := p.GetRESTConfig()
 
 	cs, err := kubernetes.NewForConfig(restConfig)
 	Expect(err).ToNot(HaveOccurred(), "Failed to get client-go client")
@@ -192,7 +195,7 @@ func (p *clusterProxy) ApplyWithArgs(ctx context.Context, resources []byte, args
 	return exec.KubectlApplyWithArgs(ctx, p.kubeconfigPath, resources, args...)
 }
 
-func (p *clusterProxy) getConfig() *rest.Config {
+func (p *clusterProxy) GetRESTConfig() *rest.Config {
 	config, err := clientcmd.LoadFromFile(p.kubeconfigPath)
 	Expect(err).ToNot(HaveOccurred(), "Failed to load Kubeconfig file from %q", p.kubeconfigPath)
 
@@ -308,6 +311,7 @@ func findLoadBalancerPort(ctx context.Context, name string) (string, error) {
 	if err != nil {
 		return "", err
 	}
+
 	return strings.TrimSpace(string(stdout)), nil
 }
 

test/framework/ginkgoextensions/output.go

@@ -0,0 +1,27 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ginkgoextensions
+
+import (
+	"fmt"
+
+	"github.com/onsi/ginkgo"
+)
+
+func Byf(format string, a ...interface{}) {
+	ginkgo.By(fmt.Sprintf(format, a...))
+}

test/framework/kubernetesversions/bindata.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubernetesversions
+
+//go:generate sh -c "go-bindata -nometadata -pkg kubernetesversions -o zz_generated.bindata.go.tmp data && cat ../../../hack/boilerplate/boilerplate.generatego.txt zz_generated.bindata.go.tmp > zz_generated.bindata.go && rm zz_generated.bindata.go.tmp"

test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh

@@ -0,0 +1,106 @@
+#!/bin/bash
+
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## Please note that this file needs to be escaped for envsubst to function
+
+# shellcheck disable=SC1083,SC2034,SC2066,SC2193
+
+set -o nounset
+set -o pipefail
+set -o errexit
+
+[[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
+
+USE_CI_ARTIFACTS=${USE_CI_ARTIFACTS:=false}
+
+if [ ! "${USE_CI_ARTIFACTS}" = true ]; then
+  echo "No CI Artifacts installation, exiting"
+  exit 0
+fi
+
+GSUTIL=gsutil
+
+if ! command -v $${GSUTIL} >/dev/null; then
+  apt-get update
+  apt-get install -y apt-transport-https ca-certificates gnupg curl
+  echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | $${SUDO} tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+  curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | $${SUDO} apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
+  apt-get update
+  apt-get install -y google-cloud-sdk
+fi
+
+$${GSUTIL} version
+
+# This test installs release packages or binaries that are a result of the CI and release builds.
+# It runs '... --version' commands to verify that the binaries are correctly installed
+# and finally uninstalls the packages.
+# For the release packages it tests all versions in the support skew.
+LINE_SEPARATOR="*************************************************"
+echo "$${LINE_SEPARATOR}"
+
+## Clusterctl set variables
+##
+# $${KUBERNETES_VERSION} will be replaced by clusterctl
+KUBERNETES_VERSION=${KUBERNETES_VERSION}
+##
+## End clusterctl set variables
+
+if [[ "$${KUBERNETES_VERSION}" != "" ]]; then
+  CI_DIR=/tmp/k8s-ci
+  mkdir -p "$${CI_DIR}"
+  declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
+  declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
+  CONTAINER_EXT="tar"
+  echo "* testing CI version $${KUBERNETES_VERSION}"
+  # Check for semver
+  if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    CI_URL="gs://kubernetes-release/release/$${KUBERNETES_VERSION}/bin/linux/amd64"
+    VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}"
+    DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
+    curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+    echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' >/etc/apt/sources.list.d/kubernetes.list
+    apt-get update
+    # replace . with \.
+    VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}"
+    PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
+    for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
+      echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}"
+      DEBIAN_FRONTEND=noninteractive apt-get install -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}"
+    done
+  else
+    CI_URL="gs://kubernetes-release-dev/ci/$${KUBERNETES_VERSION}-bazel/bin/linux/amd64"
+    for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
+      echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}"
+      $${GSUTIL} cp "$${CI_URL}/$${CI_PACKAGE}" "$${CI_DIR}/$${CI_PACKAGE}"
+      chmod +x "$${CI_DIR}/$${CI_PACKAGE}"
+      mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}"
+    done
+    systemctl restart kubelet
+  fi
+  for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
+    echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}"
+    $${GSUTIL} cp "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}"
+    $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result"
+    $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "k8s.gcr.io/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}"
+    $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/kubernetes-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}"
+  done
+fi
+echo "* checking binary versions"
+echo "ctr version: " "$(ctr version)"
+echo "kubeadm version: " "$(kubeadm version -o=short)"
+echo "kubectl version: " "$(kubectl version --client=true --short=true)"
+echo "kubelet version: " "$(kubelet --version)"
+echo "$${LINE_SEPARATOR}"

test/framework/kubernetesversions/data/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: default
+resources:
+  - ci-artifacts-source-template.yaml
+patchesStrategicMerge:
+  - kustomizeversions.yaml
+  - platform-kustomization.yaml