diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index 0bc66327037..d24ef59b910 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -499,12 +499,13 @@ type Service struct { // PDStatus is PD status type PDStatus struct { - Synced bool `json:"synced,omitempty"` - Phase MemberPhase `json:"phase,omitempty"` - StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` - Members map[string]PDMember `json:"members,omitempty"` - Leader PDMember `json:"leader,omitempty"` - FailureMembers map[string]PDFailureMember `json:"failureMembers,omitempty"` + Synced bool `json:"synced,omitempty"` + Phase MemberPhase `json:"phase,omitempty"` + StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"` + Members map[string]PDMember `json:"members,omitempty"` + Leader PDMember `json:"leader,omitempty"` + FailureMembers map[string]PDFailureMember `json:"failureMembers,omitempty"` + UnjoinedMembers map[string]UnjoinedMember `json:"unjoinedMembers,omitempty"` } // PDMember is PD member @@ -528,6 +529,13 @@ type PDFailureMember struct { CreatedAt metav1.Time `json:"createdAt,omitempty"` } +// UnjoinedMember is the pd unjoin cluster member information +type UnjoinedMember struct { + PodName string `json:"podName,omitempty"` + PVCUID types.UID `json:"pvcUID,omitempty"` + CreatedAt metav1.Time `json:"createdAt,omitempty"` +} + // TiDBStatus is TiDB status type TiDBStatus struct { Phase MemberPhase `json:"phase,omitempty"` diff --git a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go index b1976db66fb..71833be5c74 100644 --- a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go @@ -1381,6 +1381,13 @@ func (in *PDStatus) DeepCopyInto(out *PDStatus) { (*out)[key] = *val.DeepCopy() } } + if in.UnjoinedMembers != nil { + in, out := &in.UnjoinedMembers, &out.UnjoinedMembers + *out = make(map[string]UnjoinedMember, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } return } @@ -4090,3 +4097,20 @@ func (in *TxnLocalLatches) DeepCopy() *TxnLocalLatches { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UnjoinedMember) DeepCopyInto(out *UnjoinedMember) { + *out = *in + in.CreatedAt.DeepCopyInto(&out.CreatedAt) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UnjoinedMember. +func (in *UnjoinedMember) DeepCopy() *UnjoinedMember { + if in == nil { + return nil + } + out := new(UnjoinedMember) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/manager/member/pd_member_manager.go b/pkg/manager/member/pd_member_manager.go index 90c5d71e12e..7a7cb99c311 100644 --- a/pkg/manager/member/pd_member_manager.go +++ b/pkg/manager/member/pd_member_manager.go @@ -15,6 +15,7 @@ package member import ( "fmt" + "github.com/pingcap/tidb-operator/pkg/util" "strconv" "strings" @@ -389,6 +390,11 @@ func (pmm *pdMemberManager) syncTidbClusterStatus(tc *v1alpha1.TidbCluster, set tc.Status.PD.Members = pdStatus tc.Status.PD.Leader = tc.Status.PD.Members[leader.GetName()] + // k8s check + err = pmm.collectUnjoinedMembers(tc, set, pdStatus) + if err != nil { + return err + } return nil } @@ -738,6 +744,53 @@ func getPDConfigMap(tc *v1alpha1.TidbCluster) (*corev1.ConfigMap, error) { return cm, nil } +func (pmm *pdMemberManager) collectUnjoinedMembers(tc *v1alpha1.TidbCluster, set *apps.StatefulSet, pdStatus map[string]v1alpha1.PDMember) error { + podSelector, podSelectErr := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if podSelectErr != nil { + return podSelectErr + } + pods, podErr := pmm.podLister.Pods(tc.Namespace).List(podSelector) + if podErr != nil { + return podErr + } + for _, pod := range pods { + var joined = false + for podName := range pdStatus { + if strings.EqualFold(pod.Name, podName) { + joined = true + break + } + } + if !joined { + if tc.Status.PD.UnjoinedMembers == nil { + tc.Status.PD.UnjoinedMembers = map[string]v1alpha1.UnjoinedMember{} + } + ordinal, err := util.GetOrdinalFromPodName(pod.Name) + if err != nil { + return err + } + pvcName := ordinalPVCName(v1alpha1.PDMemberType, controller.PDMemberName(tc.Name), ordinal) + pvc, err := pmm.pvcLister.PersistentVolumeClaims(tc.Namespace).Get(pvcName) + if err != nil { + return err + } + tc.Status.PD.UnjoinedMembers[pod.Name] = v1alpha1.UnjoinedMember{ + PodName: pod.Name, + PVCUID: pvc.UID, + CreatedAt: metav1.Now(), + } + } else { + if tc.Status.PD.UnjoinedMembers != nil { + if _, ok := tc.Status.PD.UnjoinedMembers[pod.Name]; ok { + delete(tc.Status.PD.UnjoinedMembers, pod.Name) + } + + } + } + } + return nil +} + type FakePDMemberManager struct { err error } diff --git a/pkg/manager/member/pd_member_manager_test.go b/pkg/manager/member/pd_member_manager_test.go index b94263a983b..eeaf9acb5f0 100644 --- a/pkg/manager/member/pd_member_manager_test.go +++ b/pkg/manager/member/pd_member_manager_test.go @@ -1408,3 +1408,141 @@ func TestGetNewPdServiceForTidbCluster(t *testing.T) { }) } } + +func TestPDMemberManagerSyncPDStsWhenPdNotJoinCluster(t *testing.T) { + g := NewGomegaWithT(t) + type testcase struct { + name string + modify func(cluster *v1alpha1.TidbCluster, podIndexer cache.Indexer, pvcIndexer cache.Indexer) + pdHealth *pdapi.HealthInfo + tcStatusChange func(cluster *v1alpha1.TidbCluster) + err bool + expectTidbClusterFn func(*GomegaWithT, *v1alpha1.TidbCluster) + } + + testFn := func(test *testcase, t *testing.T) { + tc := newTidbClusterForPD() + ns := tc.Namespace + tcName := tc.Name + + pmm, _, _, fakePDControl, podIndexer, pvcIndexer, _ := newFakePDMemberManager() + pdClient := controller.NewFakePDClient(fakePDControl, tc) + + pdClient.AddReaction(pdapi.GetHealthActionType, func(action *pdapi.Action) (interface{}, error) { + return test.pdHealth, nil + }) + pdClient.AddReaction(pdapi.GetClusterActionType, func(action *pdapi.Action) (interface{}, error) { + return &metapb.Cluster{Id: uint64(1)}, nil + }) + + err := pmm.Sync(tc) + g.Expect(controller.IsRequeueError(err)).To(BeTrue()) + _, err = pmm.svcLister.Services(ns).Get(controller.PDMemberName(tcName)) + g.Expect(err).NotTo(HaveOccurred()) + _, err = pmm.svcLister.Services(ns).Get(controller.PDPeerMemberName(tcName)) + g.Expect(err).NotTo(HaveOccurred()) + _, err = pmm.setLister.StatefulSets(ns).Get(controller.PDMemberName(tcName)) + g.Expect(err).NotTo(HaveOccurred()) + if test.tcStatusChange != nil { + test.tcStatusChange(tc) + } + test.modify(tc, podIndexer, pvcIndexer) + err = pmm.syncPDStatefulSetForTidbCluster(tc) + if test.err { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + if test.expectTidbClusterFn != nil { + test.expectTidbClusterFn(g, tc) + } + } + tests := []testcase{ + { + name: "add pd unjoin cluster member info ", + modify: func(cluster *v1alpha1.TidbCluster, podIndexer cache.Indexer, pvcIndexer cache.Indexer) { + for ordinal := 0; ordinal < 3; ordinal++ { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: ordinalPodName(v1alpha1.PDMemberType, cluster.GetName(), int32(ordinal)), + Namespace: metav1.NamespaceDefault, + Annotations: map[string]string{}, + Labels: label.New().Instance(cluster.GetInstanceName()).PD().Labels(), + }, + } + podIndexer.Add(pod) + } + for ordinal := 0; ordinal < 3; ordinal++ { + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: ordinalPVCName(v1alpha1.PDMemberType, controller.PDMemberName(cluster.GetName()), int32(ordinal)), + Namespace: metav1.NamespaceDefault, + Annotations: map[string]string{}, + Labels: label.New().Instance(cluster.GetInstanceName()).PD().Labels(), + }, + } + pvcIndexer.Add(pvc) + } + + }, + pdHealth: &pdapi.HealthInfo{Healths: []pdapi.MemberHealth{ + {Name: "test-pd-0", MemberID: uint64(1), ClientUrls: []string{"http://test-pd-0:2379"}, Health: false}, + {Name: "test-pd-1", MemberID: uint64(2), ClientUrls: []string{"http://test-pd-1:2379"}, Health: false}, + }}, + err: false, + expectTidbClusterFn: func(g *GomegaWithT, tc *v1alpha1.TidbCluster) { + g.Expect(tc.Status.PD.UnjoinedMembers["test-pd-2"]).NotTo(BeNil()) + }, + }, + { + name: "clear unjoin cluster member info when the member join the cluster ", + tcStatusChange: func(cluster *v1alpha1.TidbCluster) { + cluster.Status.PD.UnjoinedMembers = map[string]v1alpha1.UnjoinedMember{ + "test-pd-0": { + PodName: "test-pd-0", + CreatedAt: metav1.Now(), + }, + } + }, + modify: func(cluster *v1alpha1.TidbCluster, podIndexer cache.Indexer, pvcIndexer cache.Indexer) { + for ordinal := 0; ordinal < 3; ordinal++ { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: ordinalPodName(v1alpha1.PDMemberType, cluster.GetName(), int32(ordinal)), + Namespace: metav1.NamespaceDefault, + Annotations: map[string]string{}, + Labels: label.New().Instance(cluster.GetInstanceName()).PD().Labels(), + }, + } + podIndexer.Add(pod) + } + for ordinal := 0; ordinal < 3; ordinal++ { + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: ordinalPVCName(v1alpha1.PDMemberType, controller.PDMemberName(cluster.GetName()), int32(ordinal)), + Namespace: metav1.NamespaceDefault, + Annotations: map[string]string{}, + Labels: label.New().Instance(cluster.GetInstanceName()).PD().Labels(), + }, + } + pvcIndexer.Add(pvc) + } + + }, + pdHealth: &pdapi.HealthInfo{Healths: []pdapi.MemberHealth{ + {Name: "test-pd-0", MemberID: uint64(1), ClientUrls: []string{"http://test-pd-0:2379"}, Health: false}, + {Name: "test-pd-1", MemberID: uint64(2), ClientUrls: []string{"http://test-pd-1:2379"}, Health: false}, + {Name: "test-pd-2", MemberID: uint64(2), ClientUrls: []string{"http://test-pd-2:2379"}, Health: false}, + }}, + err: false, + expectTidbClusterFn: func(g *GomegaWithT, tc *v1alpha1.TidbCluster) { + g.Expect(tc.Status.PD.UnjoinedMembers).To(BeEmpty()) + }, + }, + } + for i := range tests { + t.Logf("begin: %s", tests[i].name) + testFn(&tests[i], t) + t.Logf("end: %s", tests[i].name) + } +}