From 304ed5b1c9583573da413514ac1957e394dd12a0 Mon Sep 17 00:00:00 2001 From: weekface Date: Fri, 3 Jan 2020 16:41:33 +0800 Subject: [PATCH] address comment --- .../tidbcluster/tidb_cluster_controller.go | 6 +-- pkg/manager/member/pd_failover.go | 23 +++------ pkg/manager/member/pd_failover_test.go | 50 +++++++++---------- 3 files changed, 32 insertions(+), 47 deletions(-) diff --git a/pkg/controller/tidbcluster/tidb_cluster_controller.go b/pkg/controller/tidbcluster/tidb_cluster_controller.go index ebc2919481..0de7b69fbf 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_controller.go +++ b/pkg/controller/tidbcluster/tidb_cluster_controller.go @@ -76,11 +76,11 @@ func NewController( tikvFailoverPeriod time.Duration, tidbFailoverPeriod time.Duration, ) *Controller { - eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster := record.NewBroadcasterWithCorrelatorOptions(record.CorrelatorOptions{QPS: 10}) + eventBroadcaster.StartLogging(glog.V(4).Infof) eventBroadcaster.StartRecordingToSink(&eventv1.EventSinkImpl{ Interface: eventv1.New(kubeCli.CoreV1().RESTClient()).Events("")}) - recorder := eventBroadcaster.NewRecorder(v1alpha1.Scheme, corev1.EventSource{Component: "tidbcluster"}) + recorder := eventBroadcaster.NewRecorder(v1alpha1.Scheme, corev1.EventSource{Component: "tidb-controller-manager"}) tcInformer := informerFactory.Pingcap().V1alpha1().TidbClusters() setInformer := kubeInformerFactory.Apps().V1().StatefulSets() diff --git a/pkg/manager/member/pd_failover.go b/pkg/manager/member/pd_failover.go index f5d09d1d8a..449474d28a 100644 --- a/pkg/manager/member/pd_failover.go +++ b/pkg/manager/member/pd_failover.go @@ -82,13 +82,8 @@ func (pf *pdFailover) Failover(tc *v1alpha1.TidbCluster) error { if pdMember.Health { healthCount++ } else { - pod, err := pf.podLister.Pods(ns).Get(podName) - if err != nil { - return err - } - - pf.recorder.Eventf(pod, apiv1.EventTypeWarning, "PDMemberUnhealthy", - "member %s is unhealthy", pdMember.ID) + pf.recorder.Eventf(tc, apiv1.EventTypeWarning, "PDMemberUnhealthy", + "%s(%s) is unhealthy", podName, pdMember.ID) } } inQuorum := healthCount > len(tc.Status.PD.Members)/2 @@ -144,13 +139,9 @@ func (pf *pdFailover) tryToMarkAPeerAsFailure(tc *v1alpha1.TidbCluster) error { if err != nil { return err } - pod, err := pf.podLister.Pods(ns).Get(podName) - if err != nil { - return err - } - pf.recorder.Eventf(pod, apiv1.EventTypeWarning, "PDMemberMarkedAsFailure", - "member %s marked as a failure member", pdMember.ID) + pf.recorder.Eventf(tc, apiv1.EventTypeWarning, "PDMemberMarkedAsFailure", + "%s(%s) marked as a failure member", podName, pdMember.ID) tc.Status.PD.FailureMembers[podName] = v1alpha1.PDFailureMember{ PodName: podName, @@ -193,6 +184,8 @@ func (pf *pdFailover) tryToDeleteAFailureMember(tc *v1alpha1.TidbCluster) error return err } glog.Infof("pd failover: delete member: %d successfully", memberID) + pf.recorder.Eventf(tc, apiv1.EventTypeWarning, "PDMemberDeleted", + "%s(%d) deleted from cluster", failurePodName, memberID) // The order of old PVC deleting and the new Pod creating is not guaranteed by Kubernetes. // If new Pod is created before old PVC deleted, new Pod will reuse old PVC. @@ -202,10 +195,6 @@ func (pf *pdFailover) tryToDeleteAFailureMember(tc *v1alpha1.TidbCluster) error if err != nil && !errors.IsNotFound(err) { return err } - if pod != nil { - pf.recorder.Eventf(pod, apiv1.EventTypeWarning, "PDMemberDeleted", - "member %d deleted from cluster", memberID) - } ordinal, err := util.GetOrdinalFromPodName(failurePodName) if err != nil { diff --git a/pkg/manager/member/pd_failover_test.go b/pkg/manager/member/pd_failover_test.go index f6d0639fbf..cd9ca847d1 100644 --- a/pkg/manager/member/pd_failover_test.go +++ b/pkg/manager/member/pd_failover_test.go @@ -77,15 +77,11 @@ func TestPDFailoverFailover(t *testing.T) { pvcIndexer.Add(pvc) } if test.hasPod { - pod0 := newPodForPDFailover(tc, v1alpha1.PDMemberType, 0) - pod1 := newPodForPDFailover(tc, v1alpha1.PDMemberType, 1) - pod2 := newPodForPDFailover(tc, v1alpha1.PDMemberType, 2) + pod := newPodForPDFailover(tc, v1alpha1.PDMemberType, 1) if test.podWithDeletionTimestamp { - pod1.DeletionTimestamp = &metav1.Time{Time: time.Now()} + pod.DeletionTimestamp = &metav1.Time{Time: time.Now()} } - podIndexer.Add(pod0) - podIndexer.Add(pod1) - podIndexer.Add(pod2) + podIndexer.Add(pod) } if test.delPodFailed { fakePodControl.SetDeletePodError(errors.NewInternalError(fmt.Errorf("delete pod: API server failed")), 0) @@ -156,8 +152,8 @@ func TestPDFailoverFailover(t *testing.T) { events := collectEvents(recorder.Events) sort.Strings(events) g.Expect(events).To(HaveLen(2)) - g.Expect(events[0]).To(ContainSubstring("member 0 is unhealthy")) - g.Expect(events[1]).To(ContainSubstring("member 12891273174085095651 is unhealthy")) + g.Expect(events[0]).To(ContainSubstring("test-pd-0(0) is unhealthy")) + g.Expect(events[1]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy")) }, }, { @@ -179,7 +175,7 @@ func TestPDFailoverFailover(t *testing.T) { g.Expect(pd1.MemberDeleted).To(Equal(true)) events := collectEvents(recorder.Events) g.Expect(events).To(HaveLen(1)) - g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 deleted from cluster")) + g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) deleted from cluster")) }, }, { @@ -204,7 +200,7 @@ func TestPDFailoverFailover(t *testing.T) { g.Expect(len(tc.Status.PD.FailureMembers)).To(Equal(0)) events := collectEvents(recorder.Events) g.Expect(events).To(HaveLen(1)) - g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy")) + g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy")) }, }, { @@ -229,7 +225,7 @@ func TestPDFailoverFailover(t *testing.T) { g.Expect(len(tc.Status.PD.FailureMembers)).To(Equal(0)) events := collectEvents(recorder.Events) g.Expect(events).To(HaveLen(1)) - g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy")) + g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy")) }, }, { @@ -251,7 +247,7 @@ func TestPDFailoverFailover(t *testing.T) { g.Expect(len(tc.Status.PD.FailureMembers)).To(Equal(0)) events := collectEvents(recorder.Events) g.Expect(events).To(HaveLen(1)) - g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy")) + g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy")) }, }, { @@ -278,8 +274,8 @@ func TestPDFailoverFailover(t *testing.T) { g.Expect(failureMembers.MemberDeleted).To(BeFalse()) events := collectEvents(recorder.Events) g.Expect(events).To(HaveLen(2)) - g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy")) - g.Expect(events[1]).To(ContainSubstring("member 12891273174085095651 marked as a failure member")) + g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy")) + g.Expect(events[1]).To(ContainSubstring("test-pd-1(12891273174085095651) marked as a failure member")) }, }, { @@ -301,8 +297,8 @@ func TestPDFailoverFailover(t *testing.T) { g.Expect(pd1.MemberDeleted).To(Equal(true)) events := collectEvents(recorder.Events) g.Expect(events).To(HaveLen(2)) - g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy")) - g.Expect(events[1]).To(ContainSubstring("member 12891273174085095651 deleted from cluster")) + g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy")) + g.Expect(events[1]).To(ContainSubstring("test-pd-1(12891273174085095651) deleted from cluster")) }, }, { @@ -333,7 +329,7 @@ func TestPDFailoverFailover(t *testing.T) { g.Expect(pd1.MemberDeleted).To(Equal(false)) events := collectEvents(recorder.Events) g.Expect(events).To(HaveLen(1)) - g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy")) + g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy")) }, }, { @@ -358,7 +354,7 @@ func TestPDFailoverFailover(t *testing.T) { g.Expect(pd1.MemberDeleted).To(Equal(false)) events := collectEvents(recorder.Events) g.Expect(events).To(HaveLen(1)) - g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy")) + g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy")) }, }, { @@ -383,8 +379,8 @@ func TestPDFailoverFailover(t *testing.T) { g.Expect(pd1.MemberDeleted).To(Equal(false)) events := collectEvents(recorder.Events) g.Expect(events).To(HaveLen(2)) - g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy")) - g.Expect(events[1]).To(ContainSubstring("member 12891273174085095651 deleted from cluster")) + g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy")) + g.Expect(events[1]).To(ContainSubstring("test-pd-1(12891273174085095651) deleted from cluster")) }, }, { @@ -409,8 +405,8 @@ func TestPDFailoverFailover(t *testing.T) { g.Expect(pd1.MemberDeleted).To(Equal(false)) events := collectEvents(recorder.Events) g.Expect(events).To(HaveLen(2)) - g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy")) - g.Expect(events[1]).To(ContainSubstring("member 12891273174085095651 deleted from cluster")) + g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy")) + g.Expect(events[1]).To(ContainSubstring("test-pd-1(12891273174085095651) deleted from cluster")) }, }, { @@ -438,8 +434,8 @@ func TestPDFailoverFailover(t *testing.T) { g.Expect(errors.IsNotFound(err)).To(BeTrue()) events := collectEvents(recorder.Events) g.Expect(events).To(HaveLen(2)) - g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy")) - g.Expect(events[1]).To(ContainSubstring("member 12891273174085095651 deleted from cluster")) + g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy")) + g.Expect(events[1]).To(ContainSubstring("test-pd-1(12891273174085095651) deleted from cluster")) }, }, { @@ -468,8 +464,8 @@ func TestPDFailoverFailover(t *testing.T) { g.Expect(err).NotTo(HaveOccurred()) events := collectEvents(recorder.Events) g.Expect(events).To(HaveLen(2)) - g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy")) - g.Expect(events[1]).To(ContainSubstring("member 12891273174085095651 deleted from cluster")) + g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy")) + g.Expect(events[1]).To(ContainSubstring("test-pd-1(12891273174085095651) deleted from cluster")) }, }, }