From 9f41d571e07258bdbb8252b22fdf7f4de2884798 Mon Sep 17 00:00:00 2001 From: Yecheng Fu Date: Tue, 18 Feb 2020 20:44:11 +0800 Subject: [PATCH] use kubetest2 to run our e2e and support GKE provider (#1716) --- go.sum | 4 + hack/e2e.sh | 309 +++++++++++++++++++++----------------------- hack/lib.sh | 34 +++++ hack/prepare-e2e.sh | 1 + hack/run-e2e.sh | 172 ++++++++++++++++++++++-- tests/e2e/e2e.go | 7 + 6 files changed, 355 insertions(+), 172 deletions(-) diff --git a/go.sum b/go.sum index 8e1ada0a58..5c41eb686a 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,7 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20190605020000-c4ba1fdf4d36/go.mod h1:aJ4qN3TfrelA6NZ6AXsXRfmEVaYin3EDbSPJrKS8OXo= +github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534 h1:N7lSsF+R7wSulUADi36SInSQA3RvfO/XclHQfedr0qk= github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14= github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA= github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= @@ -1001,6 +1002,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.0 h1:0HIbH907iBTAntm+88IJV2qmJALDAh8sPekI9Vc1fm0= gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= @@ -1018,6 +1020,7 @@ gopkg.in/square/go-jose.v2 v2.2.2 h1:orlkJ3myw8CN1nVQHBFfloD+L3egixIa4FvUP6RosSA gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.1 h1:XM28wIgFzaBmeZ5dNHIpWLQpt/9DGKxk+rCg/22nnYE= gopkg.in/warnings.v0 v0.1.1/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.0.0/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= @@ -1083,6 +1086,7 @@ k8s.io/kubectl v0.0.0-20190918164019-21692a0861df/go.mod h1:AjffgL1ZYSrbpRJHER9v k8s.io/kubelet v0.0.0-20190918162654-250a1838aa2c/go.mod h1:LGhpyzd/3AkWcFcQJ3yO1UxMnJ6urMkCYfCp4iVxhjs= k8s.io/kubernetes v1.16.0 h1:WPaqle2JWogVzLxhN6IK67u62IHKKrtYF7MS4FVR4/E= k8s.io/kubernetes v1.16.0/go.mod h1:nlP2zevWKRGKuaaVbKIwozU0Rjg9leVDXkL4YTtjmVs= +k8s.io/legacy-cloud-providers v0.0.0-20190918163543-cfa506e53441 h1:JkEasocl8SM6+H65kaEUjtLAOFYzwaQOVTDdy5DLOXk= k8s.io/legacy-cloud-providers v0.0.0-20190918163543-cfa506e53441/go.mod h1:Phw/j+7dcoTPXRkv9Nyi3RJuA6SVSoHlc7M5K1pHizM= k8s.io/metrics v0.0.0-20190918162108-227c654b2546/go.mod h1:XUFuIsGbIqaUga6Ivs02cCzxNjY4RPRvYnW0KhmnpQY= k8s.io/repo-infra v0.0.0-20181204233714-00fe14e3d1a3/go.mod h1:+G1xBfZDfVFsm1Tj/HNCvg4QqWx8rJ2Fxpqr1rqp/gQ= diff --git a/hack/e2e.sh b/hack/e2e.sh index df5ace923b..7411a8133e 100755 --- a/hack/e2e.sh +++ b/hack/e2e.sh @@ -46,19 +46,28 @@ Usage: hack/e2e.sh [-h] -- [extra test args] Environments: + PROVIDER Kubernetes provider, e.g. kind, gke, defaults: kind DOCKER_REGISTRY image docker registry IMAGE_TAG image tag + CLUSTER the name of e2e cluster, defaults: tidb-operator + KUBECONFIG path to the kubeconfig file, defaults: ~/.kube/config SKIP_BUILD skip building binaries SKIP_IMAGE_BUILD skip build and push images SKIP_UP skip starting the cluster SKIP_DOWN skip shutting down the cluster - REUSE_CLUSTER reuse existing cluster if found + SKIP_TEST skip running the test KUBE_VERSION the version of Kubernetes to test against KUBE_WORKERS the number of worker nodes (excludes master nodes), defaults: 3 DOCKER_IO_MIRROR configure mirror for docker.io GCR_IO_MIRROR configure mirror for gcr.io QUAY_IO_MIRROR configure mirror for quay.io - KIND_DATA_HOSTPATH (for kind) the host path of data directory for kind cluster, defaults: none + KIND_DATA_HOSTPATH (kind only) the host path of data directory for kind cluster, defaults: none + GCP_PROJECT (gke only) the GCP project to run in + GCP_SERVICE_ACCOUNT (gke only) the GCP service account to use + GCP_REGION (gke only) the GCP region, if specified a regional cluster is creaetd + GCP_ZONE (gke only) the GCP zone, if specified a zonal cluster is created + GCP_SSH_PRIVATE_KEY (gke only) the path to the private ssh key + GCP_SSH_PUBLIC_KEY (gke only) the path to the public ssh key GINKGO_NODES ginkgo nodes to run specs, defaults: 1 GINKGO_PARALLEL if set to `y`, will run specs in parallel, the number of nodes will be the number of cpus GINKGO_NO_COLOR if set to `y`, suppress color output in default reporter @@ -83,12 +92,41 @@ Examples: 3) reuse the cluster and don't tear down it after the testing - REUSE_CLUSTER=y SKIP_DOWN=y ./hack/e2e.sh -- + # for the first time, skip the down phase + SKIP_DOWN=y ./hack/e2e.sh -- + # then skip both the up/down phase in subsequent tests + SKIP_UP=y SKIP_DOWN=y ./hack/e2e.sh -- 4) use registry mirrors DOCKER_IO_MIRROR=https://dockerhub.azk8s.cn QUAY_IO_MIRROR=https://quay.azk8s.cn GCR_IO_MIRROR=https://gcr.azk8s.cn ./hack/e2e.sh -- +5) run e2e with gke provider locally + + You need install Google Cloud SDK first, then prepare GCP servie account + and configure ssh key pairs + + GCP service account must be created with following permissions: + + - Compute Network Admin + - Kubernetes Engine Admin + - Service Account User + - Storage Admin + + You can create ssh keypair with ssh-keygen at ~/.ssh/google_compute_engine + or specifc existing ssh keypair with following environments: + + export GCP_SSH_PRIVATE_KEY= + export GCP_SSH_PUBLIC_KEY= + + Then run with following additional GCP-specific environments: + + export GCP_PROJECT= + export GCP_SERVICE_ACCOUNT= + export GCP_ZONE=us-central1-b + + ./hack/e2e.sh -- + EOF } @@ -110,34 +148,46 @@ hack::ensure_kind hack::ensure_kubectl hack::ensure_helm +PROVIDER=${PROVIDER:-kind} DOCKER_REGISTRY=${DOCKER_REGISTRY:-localhost:5000} IMAGE_TAG=${IMAGE_TAG:-latest} CLUSTER=${CLUSTER:-tidb-operator} KUBECONFIG=${KUBECONFIG:-~/.kube/config} -KUBECONTEXT=kind-$CLUSTER SKIP_BUILD=${SKIP_BUILD:-} SKIP_IMAGE_BUILD=${SKIP_IMAGE_BUILD:-} SKIP_UP=${SKIP_UP:-} SKIP_DOWN=${SKIP_DOWN:-} +SKIP_TEST=${SKIP_TEST:-} REUSE_CLUSTER=${REUSE_CLUSTER:-} KIND_DATA_HOSTPATH=${KIND_DATA_HOSTPATH:-none} +GCP_PROJECT=${GCP_PROJECT:-} +GCP_SERVICE_ACCOUNT=${GCP_SERVICE_ACCOUNT:-} +GCP_REGION=${GCP_REGION:-} +GCP_ZONE=${GCP_ZONE:-} +GCP_SSH_PRIVATE_KEY=${GCP_SSH_PRIVATE_KEY:-} +GCP_SSH_PUBLIC_KEY=${GCP_SSH_PUBLIC_KEY:-} KUBE_VERSION=${KUBE_VERSION:-v1.12.10} KUBE_WORKERS=${KUBE_WORKERS:-3} DOCKER_IO_MIRROR=${DOCKER_IO_MIRROR:-} GCR_IO_MIRROR=${GCR_IO_MIRROR:-} QUAY_IO_MIRROR=${QUAY_IO_MIRROR:-} +echo "PROVIDER: $PROVIDER" echo "DOCKER_REGISTRY: $DOCKER_REGISTRY" echo "IMAGE_TAG: $IMAGE_TAG" echo "CLUSTER: $CLUSTER" echo "KUBECONFIG: $KUBECONFIG" -echo "KUBECONTEXT: $KUBECONTEXT" echo "SKIP_BUILD: $SKIP_BUILD" echo "SKIP_IMAGE_BUILD: $SKIP_IMAGE_BUILD" echo "SKIP_UP: $SKIP_UP" echo "SKIP_DOWN: $SKIP_DOWN" echo "KIND_DATA_HOSTPATH: $KIND_DATA_HOSTPATH" +echo "GCP_PROJECT: $GCP_PROJECT" +echo "GCP_SERVICE_ACCOUNT: $GCP_SERVICE_ACCOUNT" +echo "GCP_REGION: $GCP_REGION" +echo "GCP_ZONE: $GCP_ZONE" echo "KUBE_VERSION: $KUBE_VERSION" +echo "KUBE_WORKERS: $KUBE_WORKERS" echo "DOCKER_IO_MIRROR: $DOCKER_IO_MIRROR" echo "GCR_IO_MIRROR: $GCR_IO_MIRROR" echo "QUAY_IO_MIRROR: $QUAY_IO_MIRROR" @@ -165,21 +215,6 @@ function e2e::image_build() { DOCKER_REGISTRY=$DOCKER_REGISTRY IMAGE_TAG=$IMAGE_TAG make e2e-docker } -function e2e::image_load() { - local names=( - pingcap/tidb-operator - pingcap/tidb-operator-e2e - ) - for n in ${names[@]}; do - $KIND_BIN load docker-image --name $CLUSTER $DOCKER_REGISTRY/$n:$IMAGE_TAG - done -} - -function e2e::cluster_exists() { - local name="$1" - $KIND_BIN get clusters | grep $CLUSTER &>/dev/null -} - function e2e::__restart_docker() { echo "info: restarting docker" service docker restart @@ -201,14 +236,6 @@ function e2e::__restart_docker() { echo "info: done restarting docker" } -# e2e::__cluster_is_alive checks if the cluster is alive or not -function e2e::__cluster_is_alive() { - local ret=0 - echo "info: checking the cluster version" - $KUBECTL_BIN --context $KUBECONTEXT version --short || ret=$? - return $ret -} - function e2e::__configure_docker_mirror_for_dind() { echo "info: configure docker.io mirror '$DOCKER_IO_MIRROR' for DinD" cat < /etc/docker/daemon.json.tmp @@ -225,29 +252,8 @@ EOF fi } -function e2e::up() { - if [ -n "$SKIP_UP" ]; then - echo "info: skip starting a new cluster" - return - fi - if [ -n "$DOCKER_IO_MIRROR" -a -n "${DOCKER_IN_DOCKER_ENABLED:-}" ]; then - e2e::__configure_docker_mirror_for_dind - fi - if e2e::cluster_exists $CLUSTER; then - if [ -n "$REUSE_CLUSTER" ]; then - if e2e::__cluster_is_alive; then - echo "info: REUSE_CLUSTER is enabled and the cluster is alive, reusing it" - return - else - echo "info: REUSE_CLUSTER is enabled but the cluster is not alive, trying to recreate it" - fi - fi - echo "info: deleting the cluster '$CLUSTER'" - $KIND_BIN delete cluster --name $CLUSTER - fi - echo "info: starting a new cluster" - tmpfile=$(mktemp) - trap "test -f $tmpfile && rm $tmpfile" RETURN +function e2e::create_kindconfig() { + local tmpfile=${1} cat < $tmpfile kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 @@ -315,137 +321,114 @@ EOF EOF fi } +} + +e2e::image_build + +if [ -n "$DOCKER_IO_MIRROR" -a -n "${DOCKER_IN_DOCKER_ENABLED:-}" ]; then + e2e::__configure_docker_mirror_for_dind +fi + +kubetest2_args=( + $PROVIDER +) + +if [ -z "$SKIP_UP" ]; then + kubetest2_args+=(--up) +fi + +if [ -z "$SKIP_DOWN" ]; then + kubetest2_args+=(--down) +fi + +if [ -z "$SKIP_TEST" ]; then + kubetest2_args+=(--test exec) +fi + +if [ "$PROVIDER" == "kind" ]; then + tmpfile=$(mktemp) + trap "test -f $tmpfile && rm $tmpfile" EXIT + e2e::create_kindconfig $tmpfile echo "info: print the contents of kindconfig" cat $tmpfile - echo "info: end of the contents of kindconfig" - echo "info: creating the cluster '$CLUSTER'" - local image="" + image="" for v in ${!kind_node_images[*]}; do if [[ "$KUBE_VERSION" == "$v" ]]; then image=${kind_node_images[$v]} echo "info: image for $KUBE_VERSION: $image" - break fi done if [ -z "$image" ]; then echo "error: no image for $KUBE_VERSION, exit" exit 1 fi - # Retry on error. Sometimes, kind will fail with the following error: - # - # OCI runtime create failed: container_linux.go:346: starting container process caused "process_linux.go:319: getting the final child's pid from pipe caused \"EOF\"": unknown - # - # TODO this error should be related to docker or linux kernel, find the root cause. - hack::wait_for_success 120 5 "$KIND_BIN create cluster --config $KUBECONFIG --name $CLUSTER --image $image --config $tmpfile -v 4" - # make it able to schedule pods on control-plane, then less resources we required - # This is disabled because when hostNetwork is used, pd requires 2379/2780 - # which may conflict with etcd on control-plane. - #echo "info: remove 'node-role.kubernetes.io/master' taint from $CLUSTER-control-plane" - #kubectl taint nodes $CLUSTER-control-plane node-role.kubernetes.io/master- -} - -function e2e::__wait_for_ds() { - local ns="$1" - local name="$2" - local retries="${3:-300}" - echo "info: waiting for pods of daemonset $ns/$name are ready (retries: $retries, interval: 1s)" - for ((i = 0; i < retries; i++)) { - read a b <<<$($KUBECTL_BIN --context $KUBECONTEXT -n $ns get ds/$name -ojsonpath='{.status.desiredNumberScheduled} {.status.numberReady}{"\n"}') - if [[ "$a" -gt 0 && "$a" -eq "$b" ]]; then - echo "info: all pods of daemonset $ns/$name are ready (desired: $a, ready: $b)" - return 0 - fi - echo "info: pods of daemonset $ns/$name (desired: $a, ready: $b)" - sleep 1 - } - echo "info: timed out waiting for pods of daemonset $ns/$name are ready" - return 1 -} - -function e2e::__wait_for_deploy() { - local ns="$1" - local name="$2" - local retries="${3:-300}" - echo "info: waiting for pods of deployment $ns/$name are ready (retries: $retries, interval: 1s)" - for ((i = 0; i < retries; i++)) { - read a b <<<$($KUBECTL_BIN --context $KUBECONTEXT -n $ns get deploy/$name -ojsonpath='{.spec.replicas} {.status.readyReplicas}{"\n"}') - if [[ "$a" -gt 0 && "$a" -eq "$b" ]]; then - echo "info: all pods of deployment $ns/$name are ready (desired: $a, ready: $b)" - return 0 - fi - echo "info: pods of deployment $ns/$name (desired: $a, ready: $b)" - sleep 1 - } - echo "info: timed out waiting for pods of deployment $ns/$name are ready" - return 1 -} - -function e2e::setup_local_pvs() { - echo "info: preparing disks" - for n in $($KIND_BIN get nodes --name=$CLUSTER); do - docker exec -i $n bash <<'EOF' -test -d /mnt/disks || mkdir -p /mnt/disks -df -h /mnt/disks -if mountpoint /mnt/disks &>/dev/null; then - echo "info: /mnt/disks is a mountpoint" -else - echo "info: /mnt/disks is not a mountpoint, creating local volumes on the rootfs" -fi -cd /mnt/disks -for ((i = 1; i <= 32; i++)) { - if [ ! -d vol$i ]; then - mkdir vol$i + kubetest2_args+=(--image-name $image) + kubetest2_args+=( + --cluster-name "$CLUSTER" + --config "$tmpfile" + --verbosity 4 + ) +elif [ "$PROVIDER" == "gke" ]; then + if [ -z "$GCP_PROJECT" ]; then + echo "error: GCP_PROJECT is required" + exit 1 fi - if ! mountpoint vol$i &>/dev/null; then - mount --bind vol$i vol$i + if [ -z "$GCP_SERVICE_ACCOUNT" ]; then + echo "error: GCP_SERVICE_ACCOUNT is required" + exit 1 fi -} -EOF - done - echo "info: installing local-volume-provisioner" - $KUBECTL_BIN --context $KUBECONTEXT apply -f ${ROOT}/manifests/local-dind/local-volume-provisioner.yaml - e2e::__wait_for_ds kube-system local-volume-provisioner -} - -function e2e::setup_helm_server() { - $KUBECTL_BIN --context $KUBECONTEXT apply -f ${ROOT}/manifests/tiller-rbac.yaml - if hack::version_ge $KUBE_VERSION "v1.16.0"; then - # workaround for https://github.com/helm/helm/issues/6374 - # TODO remove this when we can upgrade to helm 2.15+, see https://github.com/helm/helm/pull/6462 - $HELM_BIN init --service-account tiller --output yaml \ - | sed 's@apiVersion: extensions/v1beta1@apiVersion: apps/v1@' \ - | sed 's@ replicas: 1@ replicas: 1\n selector: {"matchLabels": {"app": "helm", "name": "tiller"}}@' \ - | $KUBECTL_BIN --context $KUBECONTEXT apply -f - - echo "info: wait for tiller to be ready" - e2e::__wait_for_deploy kube-system tiller-deploy - else - $HELM_BIN init --service-account=tiller --wait + if [ -z "$GCP_REGION" -a -z "$GCP_ZONE" ]; then + echo "error: either GCP_REGION or GCP_ZONE must be specified" + exit 1 + elif [ -n "$GCP_REGION" -a -n "$GCP_ZONE" ]; then + echo "error: GCP_REGION or GCP_ZONE cannot be both set" + exit 1 fi - $HELM_BIN version -} - -function e2e::down() { - if [ -n "$SKIP_DOWN" ]; then - echo "info: skip shutting down the cluster '$CLUSTER'" - return + echo "info: preparing ssh keypairs for GCP" + if [ ! -d ~/.ssh ]; then + mkdir ~/.ssh fi - if ! e2e::cluster_exists $CLUSTER; then - echo "info: cluster '$CLUSTER' does not exist, skip shutting down the cluster" - return + if [ ! -e ~/.ssh/google_compute_engine -o -n "$GCP_SSH_PRIVATE_KEY" ]; then + echo "Copying $GCP_SSH_PRIVATE_KEY to ~/.ssh/google_compute_engine" >&2 + cp $GCP_SSH_PRIVATE_KEY ~/.ssh/google_compute_engine + chmod 0600 ~/.ssh/google_compute_engine fi - $KIND_BIN delete cluster --name $CLUSTER -} - -trap "e2e::down" EXIT -e2e::up -e2e::setup_local_pvs -e2e::setup_helm_server -e2e::image_build -e2e::image_load + if [ ! -e ~/.ssh/google_compute_engine.pub -o -n "$GCP_SSH_PUBLIC_KEY" ]; then + echo "Copying $GCP_SSH_PUBLIC_KEY to ~/.ssh/google_compute_engine.pub" >&2 + cp $GCP_SSH_PUBLIC_KEY ~/.ssh/google_compute_engine.pub + chmod 0600 ~/.ssh/google_compute_engine.pub + fi + kubetest2_args+=( + --cluster-name "$CLUSTER" + --project "$GCP_PROJECT" + --gcp-service-account "$GCP_SERVICE_ACCOUNT" + --environment prod + ) + if [ -n "$GCP_REGION" ]; then + kubetest2_args+=( + --region "$GCP_REGION" + ) + fi + if [ -n "$GCP_ZONE" ]; then + kubetest2_args+=( + --zone "$GCP_ZONE" + ) + fi +else + echo "error: unsupported provider '$PROVIDER'" + exit 1 +fi +# Environments for hack/run-e2e.sh +export PROVIDER +export CLUSTER export KUBECONFIG -export KUBECONTEXT +export GCP_PROJECT +export IMAGE_TAG export TIDB_OPERATOR_IMAGE=$DOCKER_REGISTRY/pingcap/tidb-operator:${IMAGE_TAG} export E2E_IMAGE=$DOCKER_REGISTRY/pingcap/tidb-operator-e2e:${IMAGE_TAG} +export PATH=$PATH:$OUTPUT_BIN -hack/run-e2e.sh "$@" +hack::ensure_kubetest2 +echo "info: run 'kubetest2 ${kubetest2_args[@]} -- hack/run-e2e.sh $@'" +$KUBETSTS2_BIN ${kubetest2_args[@]} -- hack/run-e2e.sh "$@" diff --git a/hack/lib.sh b/hack/lib.sh index 8bca616633..d753775a94 100644 --- a/hack/lib.sh +++ b/hack/lib.sh @@ -34,6 +34,10 @@ HELM_BIN=$OUTPUT_BIN/helm HELM_VERSION=${HELM_VERSION:-2.9.1} KIND_VERSION=${KIND_VERSION:-0.7.0} KIND_BIN=$OUTPUT_BIN/kind +KUBETEST2_VERSION=v0.0.1+a810685993a3e100f4c51bc346cdc05eaf753922 +KUBETEST2_GKE_VERSION=v0.0.1+a3755779de7f745733de10f9bf63e01cf0864f9d +KUBETEST2_KIND_VERSION=v0.0.1+d8d70a33d2cc5df85786b7724ac61c221bad3e18 +KUBETSTS2_BIN=$OUTPUT_BIN/kubetest2 test -d "$OUTPUT_BIN" || mkdir -p "$OUTPUT_BIN" @@ -146,3 +150,33 @@ function hack::wait_for_success() { done return 1 } + +function hack::__verify_kubetest2() { + local n="$1" + local h="$2" + if test -x "$OUTPUT_BIN/$n"; then + local tmph=$(sha1sum $OUTPUT_BIN/$n | awk '{print $1}') + [[ "$tmph" == "$h" ]] + return + fi + return 1 +} + +function hack::__ensure_kubetest2() { + local n="$1" + IFS=+ read -r v h <<<"$2" + if hack::__verify_kubetest2 $n $h; then + return 0 + fi + tmpfile=$(mktemp) + trap "test -f $tmpfile && rm $tmpfile" RETURN + curl --retry 10 -L -o - https://github.com/cofyc/kubetest2/releases/download/$v/$n.gz | gunzip > $tmpfile + mv $tmpfile $OUTPUT_BIN/$n + chmod +x $OUTPUT_BIN/$n +} + +function hack::ensure_kubetest2() { + hack::__ensure_kubetest2 kubetest2 $KUBETEST2_VERSION + hack::__ensure_kubetest2 kubetest2-gke $KUBETEST2_GKE_VERSION + hack::__ensure_kubetest2 kubetest2-kind $KUBETEST2_KIND_VERSION +} diff --git a/hack/prepare-e2e.sh b/hack/prepare-e2e.sh index fb32d6c531..1f35640779 100755 --- a/hack/prepare-e2e.sh +++ b/hack/prepare-e2e.sh @@ -30,3 +30,4 @@ source "${ROOT}/hack/lib.sh" hack::ensure_kind hack::ensure_kubectl hack::ensure_helm +hack::ensure_kubetest2 diff --git a/hack/run-e2e.sh b/hack/run-e2e.sh index e8470676ad..c4b056a77e 100755 --- a/hack/run-e2e.sh +++ b/hack/run-e2e.sh @@ -22,15 +22,22 @@ cd $ROOT source $ROOT/hack/lib.sh -hack::ensure_kubectl -hack::ensure_helm - +PROVIDER=${PROVIDER:-} +CLUSTER=${CLUSTER:-} +GCP_PROJECT=${GCP_PROJECT:-} +IMAGE_TAG=${IMAGE_TAG:-} TIDB_OPERATOR_IMAGE=${TIDB_OPERATOR_IMAGE:-localhost:5000/pingcap/tidb-operator:latest} E2E_IMAGE=${E2E_IMAGE:-localhost:5000/pingcap/tidb-operator-e2e:latest} KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config} KUBECONTEXT=${KUBECONTEXT:-} REPORT_DIR=${REPORT_DIR:-} REPORT_PREFIX=${REPORT_PREFIX:-} +GINKGO_NODES=${GINKGO_NODES:-} +GINKGO_PARALLEL=${GINKGO_PARALLEL:-n} # set to 'y' to run tests in parallel +# If 'y', Ginkgo's reporter will not print out in color when tests are run +# in parallel +GINKGO_NO_COLOR=${GINKGO_NO_COLOR:-n} +GINKGO_STREAM=${GINKGO_STREAM:-y} if [ -z "$KUBECONFIG" ]; then echo "error: KUBECONFIG is required" @@ -43,12 +50,160 @@ echo "KUBECONFIG: $KUBECONFIG" echo "KUBECONTEXT: $KUBECONTEXT" echo "REPORT_DIR: $REPORT_DIR" echo "REPORT_PREFIX: $REPORT_PREFIX" +echo "GINKGO_NODES: $GINKGO_NODES" +echo "GINKGO_PARALLEL: $GINKGO_PARALLEL" +echo "GINKGO_NO_COLOR: $GINKGO_NO_COLOR" +echo "GINKGO_STREAM: $GINKGO_STREAM" -GINKGO_PARALLEL=${GINKGO_PARALLEL:-n} # set to 'y' to run tests in parallel -# If 'y', Ginkgo's reporter will not print out in color when tests are run -# in parallel -GINKGO_NO_COLOR=${GINKGO_NO_COLOR:-n} -GINKGO_STREAM=${GINKGO_STREAM:-y} +function e2e::__wait_for_ds() { + local ns="$1" + local name="$2" + local retries="${3:-300}" + echo "info: waiting for pods of daemonset $ns/$name are ready (retries: $retries, interval: 1s)" + for ((i = 0; i < retries; i++)) { + read a b <<<$($KUBECTL_BIN --context $KUBECONTEXT -n $ns get ds/$name -ojsonpath='{.status.desiredNumberScheduled} {.status.numberReady}{"\n"}') + if [[ "$a" -gt 0 && "$a" -eq "$b" ]]; then + echo "info: all pods of daemonset $ns/$name are ready (desired: $a, ready: $b)" + return 0 + fi + echo "info: pods of daemonset $ns/$name (desired: $a, ready: $b)" + sleep 1 + } + echo "info: timed out waiting for pods of daemonset $ns/$name are ready" + return 1 +} + +function e2e::__wait_for_deploy() { + local ns="$1" + local name="$2" + local retries="${3:-300}" + echo "info: waiting for pods of deployment $ns/$name are ready (retries: $retries, interval: 1s)" + for ((i = 0; i < retries; i++)) { + read a b <<<$($KUBECTL_BIN --context $KUBECONTEXT -n $ns get deploy/$name -ojsonpath='{.spec.replicas} {.status.readyReplicas}{"\n"}') + if [[ "$a" -gt 0 && "$a" -eq "$b" ]]; then + echo "info: all pods of deployment $ns/$name are ready (desired: $a, ready: $b)" + return 0 + fi + echo "info: pods of deployment $ns/$name (desired: $a, ready: $b)" + sleep 1 + } + echo "info: timed out waiting for pods of deployment $ns/$name are ready" + return 1 +} + +function e2e::setup_local_pvs() { + echo "info: preparing local disks" + if [ "$PROVIDER" == "kind" ]; then + for n in $($KIND_BIN get nodes --name=$CLUSTER); do + docker exec -i $n bash <<'EOF' +test -d /mnt/disks || mkdir -p /mnt/disks +df -h /mnt/disks +if mountpoint /mnt/disks &>/dev/null; then + echo "info: /mnt/disks is a mountpoint" +else + echo "info: /mnt/disks is not a mountpoint, creating local volumes on the rootfs" +fi +cd /mnt/disks +for ((i = 1; i <= 32; i++)) { + if [ ! -d vol$i ]; then + mkdir vol$i + fi + if ! mountpoint vol$i &>/dev/null; then + mount --bind vol$i vol$i + fi +} +EOF + done + else + # disks are created under /mnt/stateful_partition directory + # https://cloud.google.com/container-optimized-os/docs/concepts/disks-and-filesystem + for n in $($KUBECTL_BIN --context $KUBECONTEXT get nodes -ojsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do + gcloud compute ssh $n --command 'sudo bash -c '"'"' +test -d /mnt/stateful_partition/disks || mkdir -p /mnt/stateful_partition/disks +df -h /mnt/stateful_partition/disks +test -d /mnt/disks || mkdir -p /mnt/disks +cd /mnt/disks +for ((i = 1; i <= 32; i++)) { + if [ ! -d vol$i ]; then + mkdir vol$i + fi + if ! mountpoint vol$i &>/dev/null; then + if [ ! -d /mnt/stateful_partition/disks/vol$i ]; then + mkdir /mnt/stateful_partition/disks/vol$i + fi + mount --bind /mnt/stateful_partition/disks/vol$i vol$i + fi +} +'"'" + done + fi + echo "info: installing local-volume-provisioner" + $KUBECTL_BIN --context $KUBECONTEXT apply -f ${ROOT}/manifests/local-dind/local-volume-provisioner.yaml + e2e::__wait_for_ds kube-system local-volume-provisioner +} + +function e2e::get_kube_version() { + $KUBECTL_BIN --context $KUBECONTEXT version --short | awk '/Server Version:/ {print $3}' +} + +function e2e::setup_helm_server() { + $KUBECTL_BIN --context $KUBECONTEXT apply -f ${ROOT}/manifests/tiller-rbac.yaml + if hack::version_ge $(e2e::get_kube_version) "v1.16.0"; then + # workaround for https://github.com/helm/helm/issues/6374 + # TODO remove this when we can upgrade to helm 2.15+, see https://github.com/helm/helm/pull/6462 + $HELM_BIN init --service-account tiller --output yaml \ + | sed 's@apiVersion: extensions/v1beta1@apiVersion: apps/v1@' \ + | sed 's@ replicas: 1@ replicas: 1\n selector: {"matchLabels": {"app": "helm", "name": "tiller"}}@' \ + | $KUBECTL_BIN --context $KUBECONTEXT apply -f - + echo "info: wait for tiller to be ready" + e2e::__wait_for_deploy kube-system tiller-deploy + else + $HELM_BIN init --service-account=tiller --wait + fi + $HELM_BIN version +} + +function e2e::image_load() { + local images=( + $TIDB_OPERATOR_IMAGE + $E2E_IMAGE + ) + if [ "$PROVIDER" == "kind" ]; then + echo "info: load images ${images[@]}" + for n in ${images[@]}; do + $KIND_BIN load docker-image --name $CLUSTER $n + done + elif [ "$PROVIDER" == "gke" ]; then + unset DOCKER_CONFIG # We don't need this and it may be read-only and fail the command to fail + gcloud auth configure-docker + GCP_TIDB_OPERATOR_IMAGE=gcr.io/$GCP_PROJECT/tidb-operator:$IMAGE_TAG + GCP_E2E_IMAGE=gcr.io/$GCP_PROJECT/tidb-operator-e2e:$IMAGE_TAG + docker tag $TIDB_OPERATOR_IMAGE $GCP_TIDB_OPERATOR_IMAGE + docker tag $E2E_IMAGE $GCP_E2E_IMAGE + echo "info: pushing $GCP_TIDB_OPERATOR_IMAGE" + docker push $GCP_TIDB_OPERATOR_IMAGE + echo "info: pushing $GCP_E2E_IMAGE" + docker push $GCP_E2E_IMAGE + TIDB_OPERATOR_IMAGE=$GCP_TIDB_OPERATOR_IMAGE + E2E_IMAGE=$GCP_E2E_IMAGE + else + echo "info: unsupported provider '$PROVIDER', skip loading images" + fi +} + +hack::ensure_kubectl +hack::ensure_helm + +if [ -z "$KUBECONTEXT" ]; then + KUBECONTEXT=$(kubectl config current-context) + echo "info: KUBECONTEXT is not set, current context $KUBECONTEXT is used" +fi + +e2e::image_load +e2e::setup_local_pvs +e2e::setup_helm_server + +echo "info: start to run e2e process" ginkgo_args=() @@ -66,7 +221,6 @@ if [[ "${GINKGO_STREAM}" == "y" ]]; then ginkgo_args+=("--stream") fi -echo "info: start to run e2e process" e2e_args=( /usr/local/bin/ginkgo ${ginkgo_args[@]:-} diff --git a/tests/e2e/e2e.go b/tests/e2e/e2e.go index 95346946a3..0e838d254c 100644 --- a/tests/e2e/e2e.go +++ b/tests/e2e/e2e.go @@ -46,6 +46,13 @@ import ( "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + + // ensure auth plugins are loaded + _ "k8s.io/client-go/plugin/pkg/client/auth" + + // ensure that cloud providers are loaded + _ "k8s.io/kubernetes/test/e2e/framework/providers/aws" + _ "k8s.io/kubernetes/test/e2e/framework/providers/gce" ) // This is modified from framework.SetupSuite().