diff --git a/go.mod b/go.mod index ae55ec2aaf8..f97543c019f 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea // indirect github.com/davecgh/go-spew v1.1.0 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect + github.com/dnephin/govet v0.0.0-20171012192244-4a96d43e39d3 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 // indirect github.com/emicklei/go-restful v2.8.0+incompatible diff --git a/go.sum b/go.sum index 112305904e8..9f157b70842 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,7 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dnephin/govet v0.0.0-20171012192244-4a96d43e39d3/go.mod h1:pPTX0MEEoAnfbrAGFj4nSVNhl6YbugRj6eardUZdtGo= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= diff --git a/pkg/apis/pingcap.com/v1alpha1/tidbcluster.go b/pkg/apis/pingcap.com/v1alpha1/tidbcluster.go index 9f0840ef090..dc15cf2f053 100644 --- a/pkg/apis/pingcap.com/v1alpha1/tidbcluster.go +++ b/pkg/apis/pingcap.com/v1alpha1/tidbcluster.go @@ -102,3 +102,51 @@ func (tc *TidbCluster) TiDBAllMembersReady() bool { func (tc *TidbCluster) TiDBRealReplicas() int32 { return tc.Spec.TiDB.Replicas + int32(len(tc.Status.TiDB.FailureMembers)) } + +func (tc *TidbCluster) PDIsAvailable() bool { + lowerLimit := tc.Spec.PD.Replicas/2 + 1 + if int32(len(tc.Status.PD.Members)) < lowerLimit { + return false + } + + var availableNum int32 + for _, pdMember := range tc.Status.PD.Members { + if pdMember.Health { + availableNum++ + } + } + + if availableNum < lowerLimit { + return false + } + + if tc.Status.PD.StatefulSet == nil || tc.Status.PD.StatefulSet.ReadyReplicas < lowerLimit { + return false + } + + return true +} + +func (tc *TidbCluster) TiKVIsAvailable() bool { + var lowerLimit int32 = 1 + if int32(len(tc.Status.TiKV.Stores)) < lowerLimit { + return false + } + + var availableNum int32 + for _, store := range tc.Status.TiKV.Stores { + if store.State == TiKVStateUp { + availableNum++ + } + } + + if availableNum < lowerLimit { + return false + } + + if tc.Status.TiKV.StatefulSet == nil || tc.Status.TiKV.StatefulSet.ReadyReplicas < lowerLimit { + return false + } + + return true +} diff --git a/pkg/apis/pingcap.com/v1alpha1/tidbcluster_test.go b/pkg/apis/pingcap.com/v1alpha1/tidbcluster_test.go new file mode 100644 index 00000000000..901545bc2c5 --- /dev/null +++ b/pkg/apis/pingcap.com/v1alpha1/tidbcluster_test.go @@ -0,0 +1,190 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + "testing" + + . "github.com/onsi/gomega" + apps "k8s.io/api/apps/v1beta1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +func TestPDIsAvailable(t *testing.T) { + g := NewGomegaWithT(t) + + type testcase struct { + name string + update func(*TidbCluster) + expectFn func(*GomegaWithT, bool) + } + testFn := func(test *testcase, t *testing.T) { + t.Log(test.name) + + tc := newTidbCluster() + test.update(tc) + test.expectFn(g, tc.PDIsAvailable()) + } + tests := []testcase{ + { + name: "pd members count is 1", + update: func(tc *TidbCluster) { + tc.Status.PD.Members = map[string]PDMember{ + "pd-0": {Name: "pd-0", Health: true}, + } + }, + expectFn: func(g *GomegaWithT, b bool) { + g.Expect(b).To(BeFalse()) + }, + }, + { + name: "pd members count is 2, but health count is 1", + update: func(tc *TidbCluster) { + tc.Status.PD.Members = map[string]PDMember{ + "pd-0": {Name: "pd-0", Health: true}, + "pd-1": {Name: "pd-1", Health: false}, + } + }, + expectFn: func(g *GomegaWithT, b bool) { + g.Expect(b).To(BeFalse()) + }, + }, + { + name: "pd members count is 3, health count is 3, but ready replicas is 1", + update: func(tc *TidbCluster) { + tc.Status.PD.Members = map[string]PDMember{ + "pd-0": {Name: "pd-0", Health: true}, + "pd-1": {Name: "pd-1", Health: true}, + "pd-2": {Name: "pd-2", Health: true}, + } + tc.Status.PD.StatefulSet = &apps.StatefulSetStatus{ReadyReplicas: 1} + }, + expectFn: func(g *GomegaWithT, b bool) { + g.Expect(b).To(BeFalse()) + }, + }, + { + name: "pd is available", + update: func(tc *TidbCluster) { + tc.Status.PD.Members = map[string]PDMember{ + "pd-0": {Name: "pd-0", Health: true}, + "pd-1": {Name: "pd-1", Health: true}, + "pd-2": {Name: "pd-2", Health: true}, + } + tc.Status.PD.StatefulSet = &apps.StatefulSetStatus{ReadyReplicas: 3} + }, + expectFn: func(g *GomegaWithT, b bool) { + g.Expect(b).To(BeTrue()) + }, + }, + } + + for i := range tests { + testFn(&tests[i], t) + } +} + +func TestTiKVIsAvailable(t *testing.T) { + g := NewGomegaWithT(t) + + type testcase struct { + name string + update func(*TidbCluster) + expectFn func(*GomegaWithT, bool) + } + testFn := func(test *testcase, t *testing.T) { + t.Log(test.name) + + tc := newTidbCluster() + test.update(tc) + test.expectFn(g, tc.TiKVIsAvailable()) + } + tests := []testcase{ + { + name: "tikv stores count is 0", + update: func(tc *TidbCluster) { + tc.Status.TiKV.Stores = map[string]TiKVStore{} + }, + expectFn: func(g *GomegaWithT, b bool) { + g.Expect(b).To(BeFalse()) + }, + }, + { + name: "tikv stores count is 1, but available count is 0", + update: func(tc *TidbCluster) { + tc.Status.TiKV.Stores = map[string]TiKVStore{ + "tikv-0": {PodName: "tikv-0", State: TiKVStateDown}, + } + }, + expectFn: func(g *GomegaWithT, b bool) { + g.Expect(b).To(BeFalse()) + }, + }, + { + name: "tikv stores count is 1, available count is 1, ready replicas is 0", + update: func(tc *TidbCluster) { + tc.Status.TiKV.Stores = map[string]TiKVStore{ + "tikv-0": {PodName: "tikv-0", State: TiKVStateUp}, + } + tc.Status.TiKV.StatefulSet = &apps.StatefulSetStatus{ReadyReplicas: 0} + }, + expectFn: func(g *GomegaWithT, b bool) { + g.Expect(b).To(BeFalse()) + }, + }, + { + name: "tikv is available", + update: func(tc *TidbCluster) { + tc.Status.TiKV.Stores = map[string]TiKVStore{ + "tikv-0": {PodName: "tikv-0", State: TiKVStateUp}, + } + tc.Status.TiKV.StatefulSet = &apps.StatefulSetStatus{ReadyReplicas: 1} + }, + expectFn: func(g *GomegaWithT, b bool) { + g.Expect(b).To(BeTrue()) + }, + }, + } + + for i := range tests { + testFn(&tests[i], t) + } +} + +func newTidbCluster() *TidbCluster { + return &TidbCluster{ + TypeMeta: metav1.TypeMeta{ + Kind: "TidbCluster", + APIVersion: "pingcap.com/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pd", + Namespace: corev1.NamespaceDefault, + UID: types.UID("test"), + }, + Spec: TidbClusterSpec{ + PD: PDSpec{ + Replicas: 3, + }, + TiKV: TiKVSpec{ + Replicas: 3, + }, + TiDB: TiDBSpec{ + Replicas: 1, + }, + }, + } +} diff --git a/pkg/controller/tidbcluster/tidb_cluster_control.go b/pkg/controller/tidbcluster/tidb_cluster_control.go index 553e7998d90..8f8da6be684 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_control.go +++ b/pkg/controller/tidbcluster/tidb_cluster_control.go @@ -67,120 +67,76 @@ type defaultTidbClusterControl struct { // UpdateStatefulSet executes the core logic loop for a tidbcluster. func (tcc *defaultTidbClusterControl) UpdateTidbCluster(tc *v1alpha1.TidbCluster) error { - // perform the main update function and get the status - oldStatus := tc.Status.DeepCopy() var errs []error + oldStatus := tc.Status.DeepCopy() - err := tcc.updateTidbCluster(tc) - if err != nil { + if err := tcc.updateTidbCluster(tc); err != nil { errs = append(errs, err) } - - if !apiequality.Semantic.DeepEqual(&tc.Status, oldStatus) { - _, err := tcc.tcControl.UpdateTidbCluster(tc.DeepCopy(), &tc.Status, oldStatus) - if err != nil { - errs = append(errs, err) - } + if apiequality.Semantic.DeepEqual(&tc.Status, oldStatus) { + return errorutils.NewAggregate(errs) + } + if _, err := tcc.tcControl.UpdateTidbCluster(tc.DeepCopy(), &tc.Status, oldStatus); err != nil { + errs = append(errs, err) } return errorutils.NewAggregate(errs) } func (tcc *defaultTidbClusterControl) updateTidbCluster(tc *v1alpha1.TidbCluster) error { - ns := tc.GetNamespace() - tcName := tc.GetName() - - // ReclaimPolicyManager - err := tcc.reclaimPolicyManager.Sync(tc) - if err != nil { + // syncing all PVs managed by operator's reclaim policy to Retain + if err := tcc.reclaimPolicyManager.Sync(tc); err != nil { return err } - _, err = tcc.orphanPodsCleaner.Clean(tc) - if err != nil { + // cleaning all orphan pods(pd or tikv which don't have a related PVC) managed by operator + if _, err := tcc.orphanPodsCleaner.Clean(tc); err != nil { return err } - // PD - err = tcc.pdMemberManager.Sync(tc) - if err != nil { + // works that should do to making the pd cluster current state match the desired state: + // - create or update the pd service + // - create or update the pd headless service + // - create the pd statefulset + // - sync pd cluster status from pd to TidbCluster object + // - set two annotations to the first pd member: + // - label.Bootstrapping + // - label.Replicas + // - upgrade the pd cluster + // - scale out/in the pd cluster + // - failover the pd cluster + if err := tcc.pdMemberManager.Sync(tc); err != nil { return err } - if !tcc.IsPDAvailable(tc) { - return controller.RequeueErrorf("TidbCluster: [%s/%s], waiting for PD cluster running", ns, tcName) - } - - // TiKV - err = tcc.tikvMemberManager.Sync(tc) - if err != nil { + // works that should do to making the tikv cluster current state match the desired state: + // - waiting for the pd cluster available(pd cluster is in quorum) + // - create or update tikv headless service + // - create the tikv statefulset + // - sync tikv cluster status from pd to TidbCluster object + // - set scheduler labels to tikv stores + // - upgrade the tikv cluster + // - scale out/in the tikv cluster + // - failover the tikv cluster + if err := tcc.tikvMemberManager.Sync(tc); err != nil { return err } - // Wait tikv status sync - if !tcc.IsTiKVAvailable(tc) { - return controller.RequeueErrorf("TidbCluster: [%s/%s], waiting for TiKV cluster running", ns, tcName) - } - - // TiDB - err = tcc.tidbMemberManager.Sync(tc) - if err != nil { - return err - } - - // MetaManager - err = tcc.metaManager.Sync(tc) - if err != nil { + // works that should do to making the tidb cluster current state match the desired state: + // - waiting for the tikv cluster available(at least one peer works) + // - create or update tidb headless service + // - create the tidb statefulset + // - sync tidb cluster status from pd to TidbCluster object + // - upgrade the tidb cluster + // - scale out/in the tidb cluster + // - failover the tidb cluster + if err := tcc.tidbMemberManager.Sync(tc); err != nil { return err } - return nil -} - -func (tcc *defaultTidbClusterControl) IsPDAvailable(tc *v1alpha1.TidbCluster) bool { - lowerLimit := tc.Spec.PD.Replicas/2 + 1 - if int32(len(tc.Status.PD.Members)) < lowerLimit { - return false - } - - var availableNum int32 - for _, pdMember := range tc.Status.PD.Members { - if pdMember.Health { - availableNum++ - } - } - - if availableNum < lowerLimit { - return false - } - - if tc.Status.PD.StatefulSet == nil || tc.Status.PD.StatefulSet.ReadyReplicas < lowerLimit { - return false - } - - return true -} - -func (tcc *defaultTidbClusterControl) IsTiKVAvailable(tc *v1alpha1.TidbCluster) bool { - var lowerLimit int32 = 1 - if int32(len(tc.Status.TiKV.Stores)) < lowerLimit { - return false - } - - var availableNum int32 - for _, store := range tc.Status.TiKV.Stores { - if store.State == v1alpha1.TiKVStateUp { - availableNum++ - } - } - - if availableNum < lowerLimit { - return false - } - - if tc.Status.TiKV.StatefulSet == nil || tc.Status.TiKV.StatefulSet.ReadyReplicas < lowerLimit { - return false - } - - return true + // syncing the labels from Pod to PVC and PV, these labels include: + // - label.StoreIDLabelKey + // - label.MemberIDLabelKey + // - label.NamespaceLabelKey + return tcc.metaManager.Sync(tc) } diff --git a/pkg/controller/tidbcluster/tidb_cluster_control_test.go b/pkg/controller/tidbcluster/tidb_cluster_control_test.go index c5ab6e12709..3f3c4dbd849 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_control_test.go +++ b/pkg/controller/tidbcluster/tidb_cluster_control_test.go @@ -19,7 +19,6 @@ import ( "testing" . "github.com/onsi/gomega" - perrors "github.com/pingcap/errors" "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned/fake" informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions" @@ -104,64 +103,6 @@ func TestTidbClusterControlUpdateTidbCluster(t *testing.T) { g.Expect(strings.Contains(err.Error(), "pd member manager sync error")).To(Equal(true)) }, }, - { - name: "pd members count is 1", - update: func(cluster *v1alpha1.TidbCluster) { - cluster.Status.PD.Members = map[string]v1alpha1.PDMember{ - "pd-0": {Name: "pd-0", Health: true}, - } - }, - syncReclaimPolicyErr: false, - syncPDMemberManagerErr: false, - syncTiKVMemberManagerErr: false, - syncTiDBMemberManagerErr: false, - syncMetaManagerErr: false, - errExpectFn: func(g *GomegaWithT, err error) { - g.Expect(err).To(HaveOccurred()) - g.Expect(perrors.Find(err, controller.IsRequeueError)).NotTo(BeNil()) - g.Expect(strings.Contains(err.Error(), "waiting for PD cluster running")).To(Equal(true)) - }, - }, - { - name: "pd members count is 2, but health count is 1", - update: func(cluster *v1alpha1.TidbCluster) { - cluster.Status.PD.Members = map[string]v1alpha1.PDMember{ - "pd-0": {Name: "pd-0", Health: true}, - "pd-1": {Name: "pd-1", Health: false}, - } - }, - syncReclaimPolicyErr: false, - syncPDMemberManagerErr: false, - syncTiKVMemberManagerErr: false, - syncTiDBMemberManagerErr: false, - syncMetaManagerErr: false, - errExpectFn: func(g *GomegaWithT, err error) { - g.Expect(err).To(HaveOccurred()) - g.Expect(perrors.Find(err, controller.IsRequeueError)).NotTo(BeNil()) - g.Expect(strings.Contains(err.Error(), "waiting for PD cluster running")).To(Equal(true)) - }, - }, - { - name: "pd members count is 3, health count is 3, but ready replicas is 1", - update: func(cluster *v1alpha1.TidbCluster) { - cluster.Status.PD.Members = map[string]v1alpha1.PDMember{ - "pd-0": {Name: "pd-0", Health: true}, - "pd-1": {Name: "pd-1", Health: true}, - "pd-2": {Name: "pd-2", Health: true}, - } - cluster.Status.PD.StatefulSet = &apps.StatefulSetStatus{ReadyReplicas: 1} - }, - syncReclaimPolicyErr: false, - syncPDMemberManagerErr: false, - syncTiKVMemberManagerErr: false, - syncTiDBMemberManagerErr: false, - syncMetaManagerErr: false, - errExpectFn: func(g *GomegaWithT, err error) { - g.Expect(err).To(HaveOccurred()) - g.Expect(perrors.Find(err, controller.IsRequeueError)).NotTo(BeNil()) - g.Expect(strings.Contains(err.Error(), "waiting for PD cluster running")).To(Equal(true)) - }, - }, { name: "tikv member manager sync error", update: func(cluster *v1alpha1.TidbCluster) { @@ -182,77 +123,6 @@ func TestTidbClusterControlUpdateTidbCluster(t *testing.T) { g.Expect(strings.Contains(err.Error(), "tikv member manager sync error")).To(Equal(true)) }, }, - { - name: "pd is ready, tikv stores count is 0", - update: func(cluster *v1alpha1.TidbCluster) { - cluster.Status.PD.Members = map[string]v1alpha1.PDMember{ - "pd-0": {Name: "pd-0", Health: true}, - "pd-1": {Name: "pd-1", Health: true}, - "pd-2": {Name: "pd-2", Health: true}, - } - cluster.Status.PD.StatefulSet = &apps.StatefulSetStatus{ReadyReplicas: 3} - cluster.Status.TiKV.Stores = map[string]v1alpha1.TiKVStore{} - }, - syncReclaimPolicyErr: false, - syncPDMemberManagerErr: false, - syncTiKVMemberManagerErr: false, - syncTiDBMemberManagerErr: false, - syncMetaManagerErr: false, - errExpectFn: func(g *GomegaWithT, err error) { - g.Expect(err).To(HaveOccurred()) - g.Expect(perrors.Find(err, controller.IsRequeueError)).NotTo(BeNil()) - g.Expect(strings.Contains(err.Error(), "waiting for TiKV cluster running")).To(Equal(true)) - }, - }, - { - name: "pd is ready, tikv stores count is 1, but available count is 0", - update: func(cluster *v1alpha1.TidbCluster) { - cluster.Status.PD.Members = map[string]v1alpha1.PDMember{ - "pd-0": {Name: "pd-0", Health: true}, - "pd-1": {Name: "pd-1", Health: true}, - "pd-2": {Name: "pd-2", Health: true}, - } - cluster.Status.PD.StatefulSet = &apps.StatefulSetStatus{ReadyReplicas: 3} - cluster.Status.TiKV.Stores = map[string]v1alpha1.TiKVStore{ - "tikv-0": {PodName: "tikv-0", State: v1alpha1.TiKVStateDown}, - } - }, - syncReclaimPolicyErr: false, - syncPDMemberManagerErr: false, - syncTiKVMemberManagerErr: false, - syncTiDBMemberManagerErr: false, - syncMetaManagerErr: false, - errExpectFn: func(g *GomegaWithT, err error) { - g.Expect(err).To(HaveOccurred()) - g.Expect(perrors.Find(err, controller.IsRequeueError)).NotTo(BeNil()) - g.Expect(strings.Contains(err.Error(), "waiting for TiKV cluster running")).To(Equal(true)) - }, - }, - { - name: "pd is ready, tikv stores count is 1, available count is 1, ready replicas is 0", - update: func(cluster *v1alpha1.TidbCluster) { - cluster.Status.PD.Members = map[string]v1alpha1.PDMember{ - "pd-0": {Name: "pd-0", Health: true}, - "pd-1": {Name: "pd-1", Health: true}, - "pd-2": {Name: "pd-2", Health: true}, - } - cluster.Status.PD.StatefulSet = &apps.StatefulSetStatus{ReadyReplicas: 3} - cluster.Status.TiKV.Stores = map[string]v1alpha1.TiKVStore{ - "tikv-0": {PodName: "tikv-0", State: v1alpha1.TiKVStateUp}, - } - cluster.Status.TiKV.StatefulSet = &apps.StatefulSetStatus{ReadyReplicas: 0} - }, - syncReclaimPolicyErr: false, - syncPDMemberManagerErr: false, - syncTiKVMemberManagerErr: false, - syncTiDBMemberManagerErr: false, - syncMetaManagerErr: false, - errExpectFn: func(g *GomegaWithT, err error) { - g.Expect(err).To(HaveOccurred()) - g.Expect(perrors.Find(err, controller.IsRequeueError)).NotTo(BeNil()) - g.Expect(strings.Contains(err.Error(), "waiting for TiKV cluster running")).To(Equal(true)) - }, - }, { name: "tidb member manager sync error", update: func(cluster *v1alpha1.TidbCluster) { @@ -367,17 +237,6 @@ func newFakeTidbClusterControl() (ControlInterface, *meta.FakeReclaimPolicyManag return control, reclaimPolicyManager, pdMemberManager, tikvMemberManager, tidbMemberManager, metaManager } -func syncTidbClusterControl(tc *v1alpha1.TidbCluster, _ *controller.FakeStatefulSetControl, control ControlInterface) error { - for tc.Status.PD.StatefulSet == nil { - err := control.UpdateTidbCluster(tc) - if err != nil { - return err - } - } - - return nil -} - func newTidbClusterForTidbClusterControl() *v1alpha1.TidbCluster { return &v1alpha1.TidbCluster{ TypeMeta: metav1.TypeMeta{ diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index 44986419264..e67a32b98e5 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -68,6 +68,13 @@ func NewTiDBMemberManager(setControl controller.StatefulSetControlInterface, } func (tmm *tidbMemberManager) Sync(tc *v1alpha1.TidbCluster) error { + ns := tc.GetNamespace() + tcName := tc.GetName() + + if !tc.TiKVIsAvailable() { + return controller.RequeueErrorf("TidbCluster: [%s/%s], waiting for TiKV cluster running", ns, tcName) + } + // Sync TiDB Headless Service if err := tmm.syncTiDBHeadlessServiceForTidbCluster(tc); err != nil { return err diff --git a/pkg/manager/member/tidb_member_manager_test.go b/pkg/manager/member/tidb_member_manager_test.go index 5f9fbb4fcb6..38a62b96822 100644 --- a/pkg/manager/member/tidb_member_manager_test.go +++ b/pkg/manager/member/tidb_member_manager_test.go @@ -47,7 +47,14 @@ func TestTiDBMemberManagerSyncCreate(t *testing.T) { } testFn := func(test *testcase, t *testing.T) { + t.Log(test.name) + tc := newTidbClusterForTiDB() + tc.Status.TiKV.Stores = map[string]v1alpha1.TiKVStore{ + "tikv-0": {PodName: "tikv-0", State: v1alpha1.TiKVStateUp}, + } + tc.Status.TiKV.StatefulSet = &apps.StatefulSetStatus{ReadyReplicas: 1} + ns := tc.GetNamespace() tcName := tc.GetName() oldSpec := tc.Spec @@ -87,6 +94,15 @@ func TestTiDBMemberManagerSyncCreate(t *testing.T) { err: false, setCreated: true, }, + { + name: "tikv is not available", + prepare: func(tc *v1alpha1.TidbCluster) { + tc.Status.TiKV.Stores = map[string]v1alpha1.TiKVStore{} + }, + errWhenCreateStatefulSet: false, + err: true, + setCreated: false, + }, { name: "error when create statefulset", prepare: nil, @@ -113,7 +129,14 @@ func TestTiDBMemberManagerSyncUpdate(t *testing.T) { } testFn := func(test *testcase, t *testing.T) { + t.Log(test.name) + tc := newTidbClusterForTiDB() + tc.Status.TiKV.Stores = map[string]v1alpha1.TiKVStore{ + "tikv-0": {PodName: "tikv-0", State: v1alpha1.TiKVStateUp}, + } + tc.Status.TiKV.StatefulSet = &apps.StatefulSetStatus{ReadyReplicas: 1} + ns := tc.GetNamespace() tcName := tc.GetName() diff --git a/pkg/manager/member/tikv_member_manager.go b/pkg/manager/member/tikv_member_manager.go index 430e20da9ae..320343d23df 100644 --- a/pkg/manager/member/tikv_member_manager.go +++ b/pkg/manager/member/tikv_member_manager.go @@ -92,6 +92,13 @@ type SvcConfig struct { // Sync fulfills the manager.Manager interface func (tkmm *tikvMemberManager) Sync(tc *v1alpha1.TidbCluster) error { + ns := tc.GetNamespace() + tcName := tc.GetName() + + if !tc.PDIsAvailable() { + return controller.RequeueErrorf("TidbCluster: [%s/%s], waiting for PD cluster running", ns, tcName) + } + svcList := []SvcConfig{ { Name: "peer", diff --git a/pkg/manager/member/tikv_member_manager_test.go b/pkg/manager/member/tikv_member_manager_test.go index 309cecd3873..a7723c6b58c 100644 --- a/pkg/manager/member/tikv_member_manager_test.go +++ b/pkg/manager/member/tikv_member_manager_test.go @@ -53,7 +53,16 @@ func TestTiKVMemberManagerSyncCreate(t *testing.T) { } testFn := func(test *testcase, t *testing.T) { + t.Log(test.name) + tc := newTidbClusterForPD() + tc.Status.PD.Members = map[string]v1alpha1.PDMember{ + "pd-0": {Name: "pd-0", Health: true}, + "pd-1": {Name: "pd-1", Health: true}, + "pd-2": {Name: "pd-2", Health: true}, + } + tc.Status.PD.StatefulSet = &apps.StatefulSetStatus{ReadyReplicas: 3} + ns := tc.Namespace tcName := tc.Name oldSpec := tc.Spec @@ -114,9 +123,8 @@ func TestTiKVMemberManagerSyncCreate(t *testing.T) { tests := []testcase{ { - name: "normal", - prepare: nil, - + name: "normal", + prepare: nil, errWhenCreateStatefulSet: false, errWhenCreateTiKVPeerService: false, err: false, @@ -125,6 +133,19 @@ func TestTiKVMemberManagerSyncCreate(t *testing.T) { pdStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, tombstoneStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, }, + { + name: "pd is not available", + prepare: func(tc *v1alpha1.TidbCluster) { + tc.Status.PD.Members = map[string]v1alpha1.PDMember{} + }, + errWhenCreateStatefulSet: false, + errWhenCreateTiKVPeerService: false, + err: true, + tikvPeerSvcCreated: false, + setCreated: false, + pdStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, + tombstoneStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, + }, { name: "tidbcluster's storage format is wrong", prepare: func(tc *v1alpha1.TidbCluster) { @@ -185,8 +206,16 @@ func TestTiKVMemberManagerSyncUpdate(t *testing.T) { } testFn := func(test *testcase, t *testing.T) { + t.Log(test.name) tc := newTidbClusterForPD() + tc.Status.PD.Members = map[string]v1alpha1.PDMember{ + "pd-0": {Name: "pd-0", Health: true}, + "pd-1": {Name: "pd-1", Health: true}, + "pd-2": {Name: "pd-2", Health: true}, + } + tc.Status.PD.StatefulSet = &apps.StatefulSetStatus{ReadyReplicas: 3} + ns := tc.Namespace tcName := tc.Name