From 7b57a0caa0ccd501dfb5bc5ad7c62da908589139 Mon Sep 17 00:00:00 2001 From: Yecheng Fu Date: Mon, 23 Mar 2020 16:00:39 +0800 Subject: [PATCH] add spec.paused field to pause the tidb cluster syncing --- docs/api-references/docs.html | 26 ++++++++ .../pingcap/v1alpha1/openapi_generated.go | 7 ++ pkg/apis/pingcap/v1alpha1/types.go | 5 ++ .../tidbcluster/tidb_cluster_control.go | 7 ++ tests/e2e/tidbcluster/tidbcluster.go | 64 +++++++++++++++++++ 5 files changed, 109 insertions(+) diff --git a/docs/api-references/docs.html b/docs/api-references/docs.html index 286be033d2c..c87ee597da2 100644 --- a/docs/api-references/docs.html +++ b/docs/api-references/docs.html @@ -739,6 +739,19 @@

TidbCluster +Paused
+ +bool + + + +(Optional) +

Indicates that the tidb cluster is paused and will not be processed by +the controller.

+ + + + version
string @@ -11631,6 +11644,19 @@

TidbClusterSpec +Paused
+ +bool + + + +(Optional) +

Indicates that the tidb cluster is paused and will not be processed by +the controller.

+ + + + version
string diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index ae26ed35e2e..b7fb8b82f56 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -6084,6 +6084,13 @@ func schema_pkg_apis_pingcap_v1alpha1_TidbClusterSpec(ref common.ReferenceCallba Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.HelperSpec"), }, }, + "Paused": { + SchemaProps: spec.SchemaProps{ + Description: "Indicates that the tidb cluster is paused and will not be processed by the controller.", + Type: []string{"boolean"}, + Format: "", + }, + }, "version": { SchemaProps: spec.SchemaProps{ Description: "TiDB cluster version", diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index 499bb65e303..0d3fb2ce3c1 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -119,6 +119,11 @@ type TidbClusterSpec struct { // +optional Helper *HelperSpec `json:"helper,omitempty"` + // Indicates that the tidb cluster is paused and will not be processed by + // the controller. + // +optional + Paused bool + // TODO: remove optional after defaulting logic introduced // TiDB cluster version // +optional diff --git a/pkg/controller/tidbcluster/tidb_cluster_control.go b/pkg/controller/tidbcluster/tidb_cluster_control.go index fa217e7d81b..bc740677e03 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_control.go +++ b/pkg/controller/tidbcluster/tidb_cluster_control.go @@ -114,6 +114,13 @@ func (tcc *defaultTidbClusterControl) validate(tc *v1alpha1.TidbCluster) bool { } func (tcc *defaultTidbClusterControl) updateTidbCluster(tc *v1alpha1.TidbCluster) error { + // Skip syncing if the tidb cluster is paused. + if tc.Spec.Paused { + // TODO when we record condition of the tidb cluster, we can emit event on state change + klog.Infof("tidb cluster %s/%s is paused", tc.GetNamespace(), tc.GetName()) + return nil + } + // syncing all PVs managed by operator's reclaim policy to Retain if err := tcc.reclaimPolicyManager.Sync(tc); err != nil { return err diff --git a/tests/e2e/tidbcluster/tidbcluster.go b/tests/e2e/tidbcluster/tidbcluster.go index a5d1437ccd3..48c43d85f3c 100644 --- a/tests/e2e/tidbcluster/tidbcluster.go +++ b/tests/e2e/tidbcluster/tidbcluster.go @@ -38,6 +38,7 @@ import ( "github.com/pingcap/tidb-operator/tests/apiserver" e2econfig "github.com/pingcap/tidb-operator/tests/e2e/config" utilimage "github.com/pingcap/tidb-operator/tests/e2e/util/image" + utilpod "github.com/pingcap/tidb-operator/tests/e2e/util/pod" "github.com/pingcap/tidb-operator/tests/e2e/util/portforward" "github.com/pingcap/tidb-operator/tests/pkg/apimachinery" "github.com/pingcap/tidb-operator/tests/pkg/blockwriter" @@ -46,8 +47,10 @@ import ( v1 "k8s.io/api/core/v1" apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" utilversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" @@ -863,6 +866,67 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() { framework.ExpectNoError(err) }) + ginkgo.It("TiDB cluster can be paused and unpaused", func() { + tcName := "paused" + tc := fixture.GetTidbCluster(ns, tcName, utilimage.TiDBV3Version) + tc.Spec.PD.Replicas = 1 + tc.Spec.TiKV.Replicas = 1 + tc.Spec.TiDB.Replicas = 1 + err := genericCli.Create(context.TODO(), tc) + framework.ExpectNoError(err) + err = oa.WaitForTidbClusterReady(tc, 30*time.Minute, 15*time.Second) + framework.ExpectNoError(err) + + podListBeforePaused, err := c.CoreV1().Pods(ns).List(metav1.ListOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("Pause the tidb cluster") + err = controller.GuaranteedUpdate(genericCli, tc, func() error { + tc.Spec.Paused = true + return nil + }) + framework.ExpectNoError(err) + ginkgo.By("Make a change") + err = controller.GuaranteedUpdate(genericCli, tc, func() error { + tc.Spec.Version = utilimage.TiDBV3UpgradeVersion + return nil + }) + framework.ExpectNoError(err) + + ginkgo.By("Check pods are not changed when the tidb cluster is paused") + err = utilpod.WaitForPodsAreChanged(c, podListBeforePaused.Items, time.Minute*5) + framework.ExpectEqual(err, wait.ErrWaitTimeout, "Pods are changed when the tidb cluster is paused") + + ginkgo.By("Unpause the tidb cluster") + err = controller.GuaranteedUpdate(genericCli, tc, func() error { + tc.Spec.Paused = false + return nil + }) + framework.ExpectNoError(err) + + ginkgo.By("Check the tidb cluster will be upgraded now") + listOptions := metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet(label.New().Instance(tcName).Component(label.TiDBLabelVal).Labels()).String(), + } + err = wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) { + podList, err := c.CoreV1().Pods(ns).List(listOptions) + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + for _, pod := range podList.Items { + for _, c := range pod.Spec.Containers { + if c.Name == v1alpha1.TiDBMemberType.String() { + if c.Image == tc.TiDBImage() { + return true, nil + } + } + } + } + return false, nil + }) + framework.ExpectNoError(err) + }) + ginkgo.It("tidb-scale: clear TiDB failureMembers when scale TiDB to zero", func() { cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "tidb-scale", "admin", "") cluster.Resources["pd.replicas"] = "3"