add hack/local-up-operator.sh to run tidb-operator locally and test examples

.github/workflows/ci.yaml

@@ -17,6 +17,7 @@ jobs:
         - "check-setup check"
         - "docker e2e-docker cli"
         - "test GOFLAGS=-race"
+        - "e2e-examples"
     steps:
     - name: Set up Go 1.13
       uses: actions/setup-go@v1

Makefile

@@ -97,6 +97,9 @@ e2e-build:
 e2e:
 	./hack/e2e.sh
 
+e2e-examples:
+	./hack/e2e-examples.sh
+
 stability-test-build:
 	$(GO_BUILD) -ldflags '$(LDFLAGS)' -o tests/images/stability-test/bin/blockwriter ./tests/cmd/blockwriter
 	$(GO_BUILD) -ldflags '$(LDFLAGS)' -o tests/images/stability-test/bin/stability-test ./tests/cmd/stability

examples/basic/tidb-cluster.yaml

@@ -16,7 +16,7 @@ spec:
     baseImage: pingcap/tikv
     replicas: 3
     requests:
-      storage: "50Gi"
+      storage: "1Gi"
     config: {}
   tidb:
     baseImage: pingcap/tidb

hack/e2e-examples.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# Copyright 2020 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# E2E entrypoint script for examples.
+#
+
+ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/.. && pwd)
+cd $ROOT
+
+source "${ROOT}/hack/lib.sh"
+
+hack::ensure_kind
+
+echo "info: create a Kubernetes cluster"
+$KIND_BIN create cluster
+
+echo "info: start tidb-operator"
+hack/local-up-operator.sh
+
+echo "info: testing examples"
+export PATH=$PATH:$OUTPUT_BIN
+hack::ensure_kubectl
+for t in $(find tests/examples/ -name '*.sh'); do
+    echo "info: testing $t"
+    $t
+    if [ $? -eq 0 ]; then
+        echo "info: test $t passed"
+    else
+        echo "error: test $t failed"
+    fi
+done

hack/local-up-operator.sh

@@ -0,0 +1,178 @@
+#!/usr/bin/env bash
+
+# Copyright 2020 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This command runs tidb-operator in Kubernetes.
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/.. && pwd)
+cd $ROOT
+
+source "${ROOT}/hack/lib.sh"
+
+function usage() {
+    cat <<'EOF'
+This commands run tidb-operator in Kubernetes.
+
+Usage: hack/local-up-operator.sh [-hd]
+
+    -h      show this message and exit
+
+Environments:
+
+    PROVIDER              Kubernetes provider. Defaults: kind.
+    CLUSTER               the name of e2e cluster. Defaults to kind for kind provider.
+    KUBECONFIG            path to the kubeconfig file, defaults: ~/.kube/config
+    KUBECONTEXT           context in kubeconfig file, defaults to current context
+    NAMESPACE             Kubernetes namespace in which we run our tidb-operator.
+    DOCKER_REGISTRY       image docker registry
+    IMAGE_TAG             image tag
+    SKIP_IMAGE_BUILD      skip build and push images
+
+EOF
+}
+
+while getopts "h?" opt; do
+    case "$opt" in
+    h|\?)
+        usage
+        exit 0
+        ;;
+    esac
+done
+
+PROVIDER=${PROVIDER:-kind}
+CLUSTER=${CLUSTER:-}
+KUBECONFIG=${KUBECONFIG:-~/.kube/config}
+KUBECONTEXT=${KUBECONTEXT:-}
+NAMESPACE=${NAMESPACE:-pingcap}
+DOCKER_REGISTRY=${DOCKER_REGISTRY:-localhost:5000}
+IMAGE_TAG=${IMAGE_TAG:-latest}
+SKIP_IMAGE_BUILD=${SKIP_IMAGE_BUILD:-}
+
+hack::ensure_kubectl
+hack::ensure_helm
+
+function hack::create_namespace() {
+    local ns="$1"
+    $KUBECTL_BIN create namespace $ns
+    for ((i=0; i < 30; i++)); do
+        local phase=$(kubectl get ns $ns -ojsonpath='{.status.phase}')
+        if [ "$phase" == "Active" ]; then
+            return 0
+        fi
+        sleep 1
+    done
+    return 1
+}
+
+function hack::wait_for_deploy() {
+    local ns="$1"
+    local name="$2"
+    local retries="${3:-300}"
+    echo "info: waiting for pods of deployment $ns/$name are ready (retries: $retries, interval: 1s)"
+    for ((i = 0; i < retries; i++)) {
+        read a b <<<$($KUBECTL_BIN --context $KUBECONTEXT -n $ns get deploy/$name -ojsonpath='{.spec.replicas} {.status.readyReplicas}{"\n"}')
+        if [[ "$a" -gt 0 && "$a" -eq "$b" ]]; then
+            echo "info: all pods of deployment $ns/$name are ready (desired: $a, ready: $b)"
+            return 0
+        fi
+        echo "info: pods of deployment $ns/$name (desired: $a, ready: $b)"
+        sleep 1
+    }
+    echo "info: timed out waiting for pods of deployment $ns/$name are ready"
+    return 1
+}
+
+function hack::cluster_exists() {
+    local c="$1"
+    for n in $($KIND_BIN get clusters); do
+        if [ "$n" == "$c" ]; then
+            return 0
+        fi
+    done
+    return 1
+}
+
+echo "info: checking clusters"
+
+if [ "$PROVIDER" == "kind" ]; then
+    if [ -z "$CLUSTER" ]; then
+        CLUSTER=kind
+    fi
+    if ! hack::cluster_exists "$CLUSTER"; then
+        echo "error: kind cluster '$CLUSTER' not found, please create it or specify the right cluster name with CLUSTER environment"
+        exit 1
+    fi
+else
+    echo "erorr: only kind PROVIDER is supported"
+    exit 1
+fi
+
+if [ -z "$KUBECONTEXT" ]; then
+    KUBECONTEXT=$(kubectl config current-context)
+    echo "info: KUBECONTEXT is not set, current context $KUBECONTEXT is used"
+fi
+
+if [ -z "$SKIP_IMAGE_BUILD" ]; then
+    echo "info: building docker images"
+    DOCKER_REGISTRY=$DOCKER_REGISTRY IMAGE_TAG=$IMAGE_TAG make docker
+else
+    echo "info: skip building docker images"
+fi
+
+echo "info: loading images into cluster"
+images=(
+    $DOCKER_REGISTRY/pingcap/tidb-operator:${IMAGE_TAG}
+)
+for n in ${images[@]}; do
+    echo "info: loading image $n"
+    $KIND_BIN load docker-image --name $CLUSTER $n
+done
+
+echo "info: uninstall tidb-operator"
+$KUBECTL_BIN -n "$NAMESPACE" delete deploy -l app.kubernetes.io/name=tidb-operator
+$KUBECTL_BIN -n "$NAMESPACE" delete pods -l app.kubernetes.io/name=tidb-operator
+
+echo "info: create namespace '$NAMESPACE' if absent"
+if ! $KUBECTL_BIN get ns "$NAMESPACE" &>/dev/null; then
+    hack::create_namespace "$NAMESPACE"
+fi
+
+echo "info: installing crds"
+$KUBECTL_BIN apply -f manifests/crd.yaml
+
+echo "info: deploying tidb-operator"
+helm_args=(
+    template
+    --name tidb-operator-dev
+    --namespace "$NAMESPACE"
+    --set operatorImage=$DOCKER_REGISTRY/pingcap/tidb-operator:${IMAGE_TAG}
+)
+
+$HELM_BIN ${helm_args[@]} ./charts/tidb-operator/ | kubectl -n "$NAMESPACE" apply -f  -
+
+deploys=(
+    tidb-controller-manager 
+    tidb-scheduler
+)
+for deploy in ${deploys[@]}; do
+    echo "info: waiting for $NAMESPACE/$deploy to be ready"
+	hack::wait_for_deploy "$NAMESPACE" "$deploy"
+done

tests/examples/001-basic.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# Copyright 2020 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd)
+cd $ROOT
+
+source "${ROOT}/hack/lib.sh"
+
+function cleanup() {
+    kubectl delete -f examples/basic/tidb-cluster.yaml 
+}
+
+trap cleanup EXIT
+
+function checkReplicas() {
+    local pdDesiredReplicas="$1"
+    local tikvDesiredReplicas="$2"
+    local tidbDesiredReplicas="$3"
+    local pdReplicas=$(kubectl get tc basic -ojsonpath='{.status.pd.statefulSet.readyReplicas}')
+    if [[ "$pdReplicas" != "$pdDesiredReplicas" ]]; then
+        echo "info: got pd replicas $pdReplicas, expects $pdDesiredReplicas"
+        return 1
+    fi
+    local tikvReplicas=$(kubectl get tc basic -ojsonpath='{.status.tikv.statefulSet.readyReplicas}')
+    if [[ "$tikvReplicas" != "$tikvDesiredReplicas" ]]; then
+        echo "info: got tikv replicas $tikvReplicas, expects $tikvDesiredReplicas"
+        return 1
+    fi
+    local tidbReplicas=$(kubectl get tc basic -ojsonpath='{.status.tidb.statefulSet.readyReplicas}')
+    if [[ "$tidbReplicas" != "$tidbDesiredReplicas" ]]; then
+        echo "info: got tidb replicas $tidbReplicas, expects $tidbDesiredReplicas"
+        return 1
+    fi
+    echo "info: pd replicas $pdReplicas, tikv replicas $tikvReplicas, tidb replicas $tidbReplicas"
+    return 0
+}
+
+kubectl apply -f examples/basic/tidb-cluster.yaml 
+
+hack::wait_for_success 600 3 "checkReplicas 3 3 2"