add more unit tests and give priority to syncing reclaim policy (#169)

* add more unit tests
sync reclaim policy first

* Update pkg/scheduler/predicates/ha_test.go

Co-Authored-By: weekface <weekface@gmail.com>

* address comment

Makefile

@@ -35,8 +35,8 @@ e2e-docker-push: e2e-docker
 e2e-docker: e2e-build
 	mkdir -p images/tidb-operator-e2e/bin
 	mv tests/e2e/e2e.test images/tidb-operator-e2e/bin/
-	[[ -d images/tidb-operator-e2e/tidb-operator ]] && rm -r images/tidb-operator-e2e/tidb-operator
-	[[ -d images/tidb-operator-e2e/tidb-cluster ]] && rm -r images/tidb-operator-e2e/tidb-cluster
+	[[ -d images/tidb-operator-e2e/tidb-operator ]] && rm -r images/tidb-operator-e2e/tidb-operator || true
+	[[ -d images/tidb-operator-e2e/tidb-cluster ]] && rm -r images/tidb-operator-e2e/tidb-cluster || true
 	cp -r charts/tidb-operator images/tidb-operator-e2e/
 	cp -r charts/tidb-cluster images/tidb-operator-e2e/
 	docker build -t "${DOCKER_REGISTRY}/pingcap/tidb-operator-e2e:latest" images/tidb-operator-e2e

pkg/controller/tidbcluster/tidb_cluster_control.go

@@ -14,7 +14,6 @@
 package tidbcluster
 
 import (
-	"github.com/golang/glog"
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1"
 	"github.com/pingcap/tidb-operator/pkg/controller"
 	"github.com/pingcap/tidb-operator/pkg/manager"
@@ -24,11 +23,6 @@ import (
 	"k8s.io/client-go/tools/record"
 )
 
-const (
-// unused
-// pdConnTimeout = 2 * time.Second
-)
-
 // ControlInterface implements the control logic for updating TidbClusters and their children StatefulSets.
 // It is implemented as an interface to allow for extensions that provide different semantics.
 // Currently, there is only one implementation.
@@ -90,15 +84,23 @@ func (tcc *defaultTidbClusterControl) UpdateTidbCluster(tc *v1alpha1.TidbCluster
 }
 
 func (tcc *defaultTidbClusterControl) updateTidbCluster(tc *v1alpha1.TidbCluster) error {
+	ns := tc.GetNamespace()
+	tcName := tc.GetName()
+
+	// ReclaimPolicyManager
+	err := tcc.reclaimPolicyManager.Sync(tc)
+	if err != nil {
+		return err
+	}
+
 	// PD
-	err := tcc.pdMemberManager.Sync(tc)
+	err = tcc.pdMemberManager.Sync(tc)
 	if err != nil {
 		return err
 	}
 
 	if !tcc.IsPDAvailable(tc) {
-		glog.Infof("tidbcluster: [%s/%s]'s pd cluster is not running.", tc.GetNamespace(), tc.GetName())
-		return nil
+		return controller.RequeueErrorf("TidbCluster: [%s/%s], waiting for PD cluster running", ns, tcName)
 	}
 
 	// TiKV
@@ -109,8 +111,7 @@ func (tcc *defaultTidbClusterControl) updateTidbCluster(tc *v1alpha1.TidbCluster
 
 	// Wait tikv status sync
 	if !tcc.IsTiKVAvailable(tc) {
-		glog.Infof("tidbcluster: [%s/%s]'s tikv cluster is not running.", tc.GetNamespace(), tc.GetName())
-		return nil
+		return controller.RequeueErrorf("TidbCluster: [%s/%s], waiting for TiKV cluster running", ns, tcName)
 	}
 
 	// TiDB
@@ -119,12 +120,6 @@ func (tcc *defaultTidbClusterControl) updateTidbCluster(tc *v1alpha1.TidbCluster
 		return err
 	}
 
-	// ReclaimPolicyManager
-	err = tcc.reclaimPolicyManager.Sync(tc)
-	if err != nil {
-		return err
-	}
-
 	// MetaManager
 	err = tcc.metaManager.Sync(tc)
 	if err != nil {
@@ -151,7 +146,7 @@ func (tcc *defaultTidbClusterControl) IsPDAvailable(tc *v1alpha1.TidbCluster) bo
 		return false
 	}
 
-	if tc.Status.PD.StatefulSet.ReadyReplicas < lowerLimit {
+	if tc.Status.PD.StatefulSet == nil || tc.Status.PD.StatefulSet.ReadyReplicas < lowerLimit {
 		return false
 	}
 
@@ -175,7 +170,7 @@ func (tcc *defaultTidbClusterControl) IsTiKVAvailable(tc *v1alpha1.TidbCluster)
 		return false
 	}
 
-	if tc.Status.TiKV.StatefulSet.ReadyReplicas < lowerLimit {
+	if tc.Status.TiKV.StatefulSet == nil || tc.Status.TiKV.StatefulSet.ReadyReplicas < lowerLimit {
 		return false
 	}