support auto-failover when paused is enabled

pingcap · Mar 23, 2020 · 346f94e · 346f94e
1 parent 21cec44
commit 346f94e
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 31 deletions.
diff --git a/pkg/manager/member/pd_member_manager.go b/pkg/manager/member/pd_member_manager.go
@@ -208,6 +208,16 @@ func (pmm *pdMemberManager) syncPDStatefulSetForTidbCluster(tc *v1alpha1.TidbClu
 		klog.Errorf("failed to sync TidbCluster: [%s/%s]'s status, error: %v", ns, tcName, err)
 	}
 
+	if pmm.autoFailover {
+		if tc.PDAllPodsStarted() && tc.PDAllMembersReady() && tc.Status.PD.FailureMembers != nil {
+			pmm.pdFailover.Recover(tc)
+		} else if tc.PDAllPodsStarted() && !tc.PDAllMembersReady() || tc.PDAutoFailovering() {
+			if err := pmm.pdFailover.Failover(tc); err != nil {
+				return err
+			}
+		}
+	}
+
 	if tc.Spec.Paused {
 		klog.V(4).Infof("tidb cluster %s/%s is paused, skip syncing for pd statefulset", tc.GetNamespace(), tc.GetName())
 		return nil
@@ -253,16 +263,6 @@ func (pmm *pdMemberManager) syncPDStatefulSetForTidbCluster(tc *v1alpha1.TidbClu
 		return err
 	}
 
-	if pmm.autoFailover {
-		if tc.PDAllPodsStarted() && tc.PDAllMembersReady() && tc.Status.PD.FailureMembers != nil {
-			pmm.pdFailover.Recover(tc)
-		} else if tc.PDAllPodsStarted() && !tc.PDAllMembersReady() || tc.PDAutoFailovering() {
-			if err := pmm.pdFailover.Failover(tc); err != nil {
-				return err
-			}
-		}
-	}
-
 	return updateStatefulSet(pmm.setControl, tc, newPDSet, oldPDSet)
 }
 

diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go
@@ -179,6 +179,19 @@ func (tmm *tidbMemberManager) syncTiDBStatefulSetForTidbCluster(tc *v1alpha1.Tid
 		return err
 	}
 
+	if tmm.autoFailover {
+		if tc.Spec.TiDB.Replicas == int32(0) && tc.Status.TiDB.FailureMembers != nil {
+			tmm.tidbFailover.Recover(tc)
+		}
+		if tc.TiDBAllPodsStarted() && tc.TiDBAllMembersReady() && tc.Status.TiDB.FailureMembers != nil {
+			tmm.tidbFailover.Recover(tc)
+		} else if tc.TiDBAllPodsStarted() && !tc.TiDBAllMembersReady() {
+			if err := tmm.tidbFailover.Failover(tc); err != nil {
+				return err
+			}
+		}
+	}
+
 	if tc.Spec.Paused {
 		klog.V(4).Infof("tidb cluster %s/%s is paused, skip syncing for tidb statefulset", tc.GetNamespace(), tc.GetName())
 		return nil
@@ -209,19 +222,6 @@ func (tmm *tidbMemberManager) syncTiDBStatefulSetForTidbCluster(tc *v1alpha1.Tid
 		}
 	}
 
-	if tmm.autoFailover {
-		if tc.Spec.TiDB.Replicas == int32(0) && tc.Status.TiDB.FailureMembers != nil {
-			tmm.tidbFailover.Recover(tc)
-		}
-		if tc.TiDBAllPodsStarted() && tc.TiDBAllMembersReady() && tc.Status.TiDB.FailureMembers != nil {
-			tmm.tidbFailover.Recover(tc)
-		} else if tc.TiDBAllPodsStarted() && !tc.TiDBAllMembersReady() {
-			if err := tmm.tidbFailover.Failover(tc); err != nil {
-				return err
-			}
-		}
-	}
-
 	return updateStatefulSet(tmm.setControl, tc, newTiDBSet, oldTiDBSet)
 }
 

diff --git a/pkg/manager/member/tikv_member_manager.go b/pkg/manager/member/tikv_member_manager.go
@@ -190,6 +190,14 @@ func (tkmm *tikvMemberManager) syncStatefulSetForTidbCluster(tc *v1alpha1.TidbCl
 		return err
 	}
 
+	if tkmm.autoFailover {
+		if tc.TiKVAllPodsStarted() && !tc.TiKVAllStoresReady() {
+			if err := tkmm.tikvFailover.Failover(tc); err != nil {
+				return err
+			}
+		}
+	}
+
 	if tc.Spec.Paused {
 		klog.V(4).Infof("tikv cluster %s/%s is paused, skip syncing for tikv statefulset", tc.GetNamespace(), tc.GetName())
 		return nil
@@ -231,14 +239,6 @@ func (tkmm *tikvMemberManager) syncStatefulSetForTidbCluster(tc *v1alpha1.TidbCl
 		return err
 	}
 
-	if tkmm.autoFailover {
-		if tc.TiKVAllPodsStarted() && !tc.TiKVAllStoresReady() {
-			if err := tkmm.tikvFailover.Failover(tc); err != nil {
-				return err
-			}
-		}
-	}
-
 	return updateStatefulSet(tkmm.setControl, tc, newSet, oldSet)
 }