From 46c7b761dbd9fcbf1b9b60ecec4a401d4c901912 Mon Sep 17 00:00:00 2001
From: Greg Weber <greg@gregweber.info>
Date: Wed, 7 Aug 2019 14:00:00 -0700
Subject: [PATCH] scheduler: inject a failure by pod annotation (#716)

scheduler: inject a failure by pod label

This can be useful for testing scheduling failures.
You can for example add it to monitor-deployment
along with changing the scheduler to the custom scheduler.
---
 pkg/label/label.go         |  3 +++
 pkg/scheduler/scheduler.go | 16 ++++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/pkg/label/label.go b/pkg/label/label.go
index 1c1099437c..38136bbb7f 100644
--- a/pkg/label/label.go
+++ b/pkg/label/label.go
@@ -45,6 +45,9 @@ const (
 	StoreIDLabelKey string = "tidb.pingcap.com/store-id"
 	// MemberIDLabelKey is member id label key
 	MemberIDLabelKey string = "tidb.pingcap.com/member-id"
+	// AnnFailTiDBScheduler is for injecting a failure into the TiDB custom scheduler
+	// A pod with this annotation will produce an error when scheduled.
+	AnnFailTiDBScheduler string = "tidb.pingcap.com/fail-scheduler"
 	// AnnPodNameKey is pod name annotation key used in PV/PVC for synchronizing tidb cluster meta info
 	AnnPodNameKey string = "tidb.pingcap.com/pod-name"
 	// AnnPVCDeferDeleting is pvc defer deletion annotation key used in PVC for defer deleting PVC
diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go
index 0ecacbe170..3d3cf758e2 100644
--- a/pkg/scheduler/scheduler.go
+++ b/pkg/scheduler/scheduler.go
@@ -14,6 +14,7 @@
 package scheduler
 
 import (
+	"fmt"
 	"github.com/golang/glog"
 	"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
 	"github.com/pingcap/tidb-operator/pkg/features"
@@ -79,6 +80,12 @@ func (s *scheduler) Filter(args *schedulerapiv1.ExtenderArgs) (*schedulerapiv1.E
 	podName := pod.GetName()
 	kubeNodes := args.Nodes.Items
 
+	if pod.Annotations != nil {
+		if _, ok := pod.Annotations[label.AnnFailTiDBScheduler]; ok {
+			return nil, FailureError{PodName: pod.Name}
+		}
+	}
+
 	var instanceName string
 	var exist bool
 	if instanceName, exist = pod.Labels[label.InstanceLabelKey]; !exist {
@@ -118,6 +125,15 @@ func (s *scheduler) Filter(args *schedulerapiv1.ExtenderArgs) (*schedulerapiv1.E
 	}, nil
 }
 
+// FailureError is returned when the FailTiDBSchedulerLabelKey is seen
+type FailureError struct {
+	PodName string
+}
+
+func (ferr FailureError) Error() string {
+	return fmt.Sprintf("pod %s had an intentional failure injected", ferr.PodName)
+}
+
 // We don't pass `prioritizeVerb` to kubernetes scheduler extender's config file, this method will not be called.
 func (s *scheduler) Priority(args *schedulerapiv1.ExtenderArgs) (schedulerapiv1.HostPriorityList, error) {
 	result := schedulerapiv1.HostPriorityList{}