fix pd-failover: scale out directly when failover occurs (pingcap#217)

* fix pd-failover: scale out directly when failover occurs

* address comment

pkg/manager/member/pd_scaler.go

@@ -56,6 +56,11 @@ func (psd *pdScaler) ScaleOut(tc *v1alpha1.TidbCluster, oldSet *apps.StatefulSet
 		return fmt.Errorf("TidbCluster: %s/%s's pd status sync failed,can't scale out now", ns, tcName)
 	}
 
+	if len(tc.Status.PD.FailureMembers) != 0 {
+		increaseReplicas(newSet, oldSet)
+		return nil
+	}
+
 	var i int32 = 0
 	healthCount := 0
 	totalCount := *oldSet.Spec.Replicas

pkg/manager/member/pd_scaler_test.go

@@ -139,6 +139,26 @@ func TestPDScalerScaleOut(t *testing.T) {
 			err:              true,
 			changed:          false,
 		},
+		{
+			name: "failover now",
+			update: func(tc *v1alpha1.TidbCluster) {
+				normalPDMember(tc)
+				podName := ordinalPodName(v1alpha1.PDMemberType, tc.GetName(), 0)
+				tc.Status.PD.FailureMembers = map[string]v1alpha1.PDFailureMember{
+					podName: {PodName: podName},
+				}
+				pd := tc.Status.PD.Members[podName]
+				pd.Health = false
+				tc.Status.PD.Members[podName] = pd
+			},
+			pdUpgrading:      false,
+			hasPVC:           true,
+			hasDeferAnn:      true,
+			pvcDeleteErr:     false,
+			statusSyncFailed: false,
+			err:              false,
+			changed:          true,
+		},
 		{
 			name: "don't have members",
 			update: func(tc *v1alpha1.TidbCluster) {