Skip to content

Commit

Permalink
address comment
Browse files Browse the repository at this point in the history
  • Loading branch information
@weekface
weekface committed Jan 3, 2020
1 parent c8ed4a0 commit 304ed5b
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 47 deletions.
6 changes: 3 additions & 3 deletions pkg/controller/tidbcluster/tidb_cluster_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,11 @@ func NewController(
tikvFailoverPeriod time.Duration,
tidbFailoverPeriod time.Duration,
) *Controller {
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartLogging(glog.Infof)
eventBroadcaster := record.NewBroadcasterWithCorrelatorOptions(record.CorrelatorOptions{QPS: 10})
eventBroadcaster.StartLogging(glog.V(4).Infof)
eventBroadcaster.StartRecordingToSink(&eventv1.EventSinkImpl{
Interface: eventv1.New(kubeCli.CoreV1().RESTClient()).Events("")})
recorder := eventBroadcaster.NewRecorder(v1alpha1.Scheme, corev1.EventSource{Component: "tidbcluster"})
recorder := eventBroadcaster.NewRecorder(v1alpha1.Scheme, corev1.EventSource{Component: "tidb-controller-manager"})

tcInformer := informerFactory.Pingcap().V1alpha1().TidbClusters()
setInformer := kubeInformerFactory.Apps().V1().StatefulSets()
Expand Down
23 changes: 6 additions & 17 deletions pkg/manager/member/pd_failover.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,13 +82,8 @@ func (pf *pdFailover) Failover(tc *v1alpha1.TidbCluster) error {
if pdMember.Health {
healthCount++
} else {
pod, err := pf.podLister.Pods(ns).Get(podName)
if err != nil {
return err
}

pf.recorder.Eventf(pod, apiv1.EventTypeWarning, "PDMemberUnhealthy",
"member %s is unhealthy", pdMember.ID)
pf.recorder.Eventf(tc, apiv1.EventTypeWarning, "PDMemberUnhealthy",
"%s(%s) is unhealthy", podName, pdMember.ID)
}
}
inQuorum := healthCount > len(tc.Status.PD.Members)/2
Expand Down Expand Up @@ -144,13 +139,9 @@ func (pf *pdFailover) tryToMarkAPeerAsFailure(tc *v1alpha1.TidbCluster) error {
if err != nil {
return err
}
pod, err := pf.podLister.Pods(ns).Get(podName)
if err != nil {
return err
}

pf.recorder.Eventf(pod, apiv1.EventTypeWarning, "PDMemberMarkedAsFailure",
"member %s marked as a failure member", pdMember.ID)
pf.recorder.Eventf(tc, apiv1.EventTypeWarning, "PDMemberMarkedAsFailure",
"%s(%s) marked as a failure member", podName, pdMember.ID)

tc.Status.PD.FailureMembers[podName] = v1alpha1.PDFailureMember{
PodName: podName,
Expand Down Expand Up @@ -193,6 +184,8 @@ func (pf *pdFailover) tryToDeleteAFailureMember(tc *v1alpha1.TidbCluster) error
return err
}
glog.Infof("pd failover: delete member: %d successfully", memberID)
pf.recorder.Eventf(tc, apiv1.EventTypeWarning, "PDMemberDeleted",
"%s(%d) deleted from cluster", failurePodName, memberID)

// The order of old PVC deleting and the new Pod creating is not guaranteed by Kubernetes.
// If new Pod is created before old PVC deleted, new Pod will reuse old PVC.
Expand All @@ -202,10 +195,6 @@ func (pf *pdFailover) tryToDeleteAFailureMember(tc *v1alpha1.TidbCluster) error
if err != nil && !errors.IsNotFound(err) {
return err
}
if pod != nil {
pf.recorder.Eventf(pod, apiv1.EventTypeWarning, "PDMemberDeleted",
"member %d deleted from cluster", memberID)
}

ordinal, err := util.GetOrdinalFromPodName(failurePodName)
if err != nil {
Expand Down
50 changes: 23 additions & 27 deletions pkg/manager/member/pd_failover_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,15 +77,11 @@ func TestPDFailoverFailover(t *testing.T) {
pvcIndexer.Add(pvc)
}
if test.hasPod {
pod0 := newPodForPDFailover(tc, v1alpha1.PDMemberType, 0)
pod1 := newPodForPDFailover(tc, v1alpha1.PDMemberType, 1)
pod2 := newPodForPDFailover(tc, v1alpha1.PDMemberType, 2)
pod := newPodForPDFailover(tc, v1alpha1.PDMemberType, 1)
if test.podWithDeletionTimestamp {
pod1.DeletionTimestamp = &metav1.Time{Time: time.Now()}
pod.DeletionTimestamp = &metav1.Time{Time: time.Now()}
}
podIndexer.Add(pod0)
podIndexer.Add(pod1)
podIndexer.Add(pod2)
podIndexer.Add(pod)
}
if test.delPodFailed {
fakePodControl.SetDeletePodError(errors.NewInternalError(fmt.Errorf("delete pod: API server failed")), 0)
Expand Down Expand Up @@ -156,8 +152,8 @@ func TestPDFailoverFailover(t *testing.T) {
events := collectEvents(recorder.Events)
sort.Strings(events)
g.Expect(events).To(HaveLen(2))
g.Expect(events[0]).To(ContainSubstring("member 0 is unhealthy"))
g.Expect(events[1]).To(ContainSubstring("member 12891273174085095651 is unhealthy"))
g.Expect(events[0]).To(ContainSubstring("test-pd-0(0) is unhealthy"))
g.Expect(events[1]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy"))
},
},
{
Expand All @@ -179,7 +175,7 @@ func TestPDFailoverFailover(t *testing.T) {
g.Expect(pd1.MemberDeleted).To(Equal(true))
events := collectEvents(recorder.Events)
g.Expect(events).To(HaveLen(1))
g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 deleted from cluster"))
g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) deleted from cluster"))
},
},
{
Expand All @@ -204,7 +200,7 @@ func TestPDFailoverFailover(t *testing.T) {
g.Expect(len(tc.Status.PD.FailureMembers)).To(Equal(0))
events := collectEvents(recorder.Events)
g.Expect(events).To(HaveLen(1))
g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy"))
g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy"))
},
},
{
Expand All @@ -229,7 +225,7 @@ func TestPDFailoverFailover(t *testing.T) {
g.Expect(len(tc.Status.PD.FailureMembers)).To(Equal(0))
events := collectEvents(recorder.Events)
g.Expect(events).To(HaveLen(1))
g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy"))
g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy"))
},
},
{
Expand All @@ -251,7 +247,7 @@ func TestPDFailoverFailover(t *testing.T) {
g.Expect(len(tc.Status.PD.FailureMembers)).To(Equal(0))
events := collectEvents(recorder.Events)
g.Expect(events).To(HaveLen(1))
g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy"))
g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy"))
},
},
{
Expand All @@ -278,8 +274,8 @@ func TestPDFailoverFailover(t *testing.T) {
g.Expect(failureMembers.MemberDeleted).To(BeFalse())
events := collectEvents(recorder.Events)
g.Expect(events).To(HaveLen(2))
g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy"))
g.Expect(events[1]).To(ContainSubstring("member 12891273174085095651 marked as a failure member"))
g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy"))
g.Expect(events[1]).To(ContainSubstring("test-pd-1(12891273174085095651) marked as a failure member"))
},
},
{
Expand All @@ -301,8 +297,8 @@ func TestPDFailoverFailover(t *testing.T) {
g.Expect(pd1.MemberDeleted).To(Equal(true))
events := collectEvents(recorder.Events)
g.Expect(events).To(HaveLen(2))
g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy"))
g.Expect(events[1]).To(ContainSubstring("member 12891273174085095651 deleted from cluster"))
g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy"))
g.Expect(events[1]).To(ContainSubstring("test-pd-1(12891273174085095651) deleted from cluster"))
},
},
{
Expand Down Expand Up @@ -333,7 +329,7 @@ func TestPDFailoverFailover(t *testing.T) {
g.Expect(pd1.MemberDeleted).To(Equal(false))
events := collectEvents(recorder.Events)
g.Expect(events).To(HaveLen(1))
g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy"))
g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy"))
},
},
{
Expand All @@ -358,7 +354,7 @@ func TestPDFailoverFailover(t *testing.T) {
g.Expect(pd1.MemberDeleted).To(Equal(false))
events := collectEvents(recorder.Events)
g.Expect(events).To(HaveLen(1))
g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy"))
g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy"))
},
},
{
Expand All @@ -383,8 +379,8 @@ func TestPDFailoverFailover(t *testing.T) {
g.Expect(pd1.MemberDeleted).To(Equal(false))
events := collectEvents(recorder.Events)
g.Expect(events).To(HaveLen(2))
g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy"))
g.Expect(events[1]).To(ContainSubstring("member 12891273174085095651 deleted from cluster"))
g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy"))
g.Expect(events[1]).To(ContainSubstring("test-pd-1(12891273174085095651) deleted from cluster"))
},
},
{
Expand All @@ -409,8 +405,8 @@ func TestPDFailoverFailover(t *testing.T) {
g.Expect(pd1.MemberDeleted).To(Equal(false))
events := collectEvents(recorder.Events)
g.Expect(events).To(HaveLen(2))
g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy"))
g.Expect(events[1]).To(ContainSubstring("member 12891273174085095651 deleted from cluster"))
g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy"))
g.Expect(events[1]).To(ContainSubstring("test-pd-1(12891273174085095651) deleted from cluster"))
},
},
{
Expand Down Expand Up @@ -438,8 +434,8 @@ func TestPDFailoverFailover(t *testing.T) {
g.Expect(errors.IsNotFound(err)).To(BeTrue())
events := collectEvents(recorder.Events)
g.Expect(events).To(HaveLen(2))
g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy"))
g.Expect(events[1]).To(ContainSubstring("member 12891273174085095651 deleted from cluster"))
g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy"))
g.Expect(events[1]).To(ContainSubstring("test-pd-1(12891273174085095651) deleted from cluster"))
},
},
{
Expand Down Expand Up @@ -468,8 +464,8 @@ func TestPDFailoverFailover(t *testing.T) {
g.Expect(err).NotTo(HaveOccurred())
events := collectEvents(recorder.Events)
g.Expect(events).To(HaveLen(2))
g.Expect(events[0]).To(ContainSubstring("member 12891273174085095651 is unhealthy"))
g.Expect(events[1]).To(ContainSubstring("member 12891273174085095651 deleted from cluster"))
g.Expect(events[0]).To(ContainSubstring("test-pd-1(12891273174085095651) is unhealthy"))
g.Expect(events[1]).To(ContainSubstring("test-pd-1(12891273174085095651) deleted from cluster"))
},
},
}
Expand Down

0 comments on commit 304ed5b

Please sign in to comment.