scheduler: inject a failure by pod annotation (#716)

scheduler: inject a failure by pod label

This can be useful for testing scheduling failures.
You can for example add it to monitor-deployment
along with changing the scheduler to the custom scheduler.

pkg/label/label.go

@@ -45,6 +45,9 @@ const (
 	StoreIDLabelKey string = "tidb.pingcap.com/store-id"
 	// MemberIDLabelKey is member id label key
 	MemberIDLabelKey string = "tidb.pingcap.com/member-id"
+	// AnnFailTiDBScheduler is for injecting a failure into the TiDB custom scheduler
+	// A pod with this annotation will produce an error when scheduled.
+	AnnFailTiDBScheduler string = "tidb.pingcap.com/fail-scheduler"
 	// AnnPodNameKey is pod name annotation key used in PV/PVC for synchronizing tidb cluster meta info
 	AnnPodNameKey string = "tidb.pingcap.com/pod-name"
 	// AnnPVCDeferDeleting is pvc defer deletion annotation key used in PVC for defer deleting PVC

pkg/scheduler/scheduler.go

@@ -14,6 +14,7 @@
 package scheduler
 
 import (
+	"fmt"
 	"github.com/golang/glog"
 	"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
 	"github.com/pingcap/tidb-operator/pkg/features"
@@ -79,6 +80,12 @@ func (s *scheduler) Filter(args *schedulerapiv1.ExtenderArgs) (*schedulerapiv1.E
 	podName := pod.GetName()
 	kubeNodes := args.Nodes.Items
 
+	if pod.Annotations != nil {
+		if _, ok := pod.Annotations[label.AnnFailTiDBScheduler]; ok {
+			return nil, FailureError{PodName: pod.Name}
+		}
+	}
+
 	var instanceName string
 	var exist bool
 	if instanceName, exist = pod.Labels[label.InstanceLabelKey]; !exist {
@@ -118,6 +125,15 @@ func (s *scheduler) Filter(args *schedulerapiv1.ExtenderArgs) (*schedulerapiv1.E
 	}, nil
 }
 
+// FailureError is returned when the FailTiDBSchedulerLabelKey is seen
+type FailureError struct {
+	PodName string
+}
+
+func (ferr FailureError) Error() string {
+	return fmt.Sprintf("pod %s had an intentional failure injected", ferr.PodName)
+}
+
 // We don't pass `prioritizeVerb` to kubernetes scheduler extender's config file, this method will not be called.
 func (s *scheduler) Priority(args *schedulerapiv1.ExtenderArgs) (schedulerapiv1.HostPriorityList, error) {
 	result := schedulerapiv1.HostPriorityList{}