Add extended scheduler to operator (pingcap#145)

* scheduler: add extended scheduler to operator

Gopkg.toml

@@ -74,3 +74,7 @@ required = [
     name = "github.com/pkg/errors"
     version = "0.9.0"
     source = "https://github.com/pingcap/errors.git"
+
+[[constraint]]
+    name = "github.com/emicklei/go-restful"
+    version = "v2.8.0"

Makefile

@@ -21,11 +21,14 @@ docker-push: docker
 docker: build
 	docker build --tag "${DOCKER_REGISTRY}/pingcap/tidb-operator:latest" images/tidb-operator
 
-build: controller-manager
+build: controller-manager scheduler
 
 controller-manager:
 	$(GO) build -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-controller-manager cmd/controller-manager/main.go
 
+scheduler:
+	$(GO) build -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-scheduler cmd/scheduler/main.go
+
 e2e-docker-push: e2e-docker
 	docker push "${DOCKER_REGISTRY}/pingcap/tidb-operator-e2e:latest"
 

README.md

@@ -20,7 +20,7 @@ TiDB Operator manages [TiDB](https://github.com/pingcap/tidb) clusters on [Kuber
 
     Users can deploy and manage multiple TiDB clusters on a single Kubernetes cluster easily.
 
-- __Automatic failover__ (WIP)
+- __Automatic failover__
 
     TiDB Operator automatically performs failover for your TiDB cluster when node failures occur.
 

charts/tidb-cluster/templates/tidb-cluster.yaml

@@ -13,6 +13,7 @@ spec:
   timezone: {{ .Values.timezone | default "UTC" }}
   services:
 {{ toYaml .Values.services | indent 4 }}
+  schedulerName: {{ .Values.schedulerName | default "default-scheduler" }}
   pd:
     replicas: {{ .Values.pd.replicas }}
     image: {{ .Values.pd.image }}

charts/tidb-cluster/values.yaml

@@ -11,6 +11,9 @@ rbac:
 # if multiple clusters are deployed in the same namespace.
 clusterName: demo
 
+# schedulerName must be same with charts/tidb-operator/values#scheduler.schedulerName
+schedulerName: tidb-scheduler
+
 # timezone is the default system timzone for TiDB
 timezone: UTC
 
@@ -140,11 +143,11 @@ tidb:
       # cloud.google.com/load-balancer-type: Internal
 
 monitor:
+  create: true
   # Also see rbac.create
   # If you set rbac.create to false, you need to provide a value here.
   # If you set rbac.create to true, you should leave this empty.
   serviceAccount:
-  create: true
   persistent: false
   storageClassName: local-storage
   storage: 10Gi

charts/tidb-operator/files/scheduler-policy.json

@@ -0,0 +1,38 @@
+{
+  "kind" : "Policy",
+  "apiVersion" : "v1",
+  "predicates": [
+    {"name": "MatchInterPodAffinity"},
+    {"name": "CheckVolumeBinding"},
+    {"name": "CheckNodeCondition"},
+    {"name": "GeneralPredicates"},
+    {"name": "HostName"},
+    {"name": "PodFitsHostPorts"},
+    {"name": "MatchNodeSelector"},
+    {"name": "PodFitsResources"},
+    {"name": "NoDiskConflict"},
+    {"name": "PodToleratesNodeTaints"},
+    {"name": "CheckNodeMemoryPressure"},
+    {"name": "CheckNodeDiskPressure"}
+  ],
+  "priorities": [
+    {"name": "EqualPriority", "weight": 1},
+    {"name": "ImageLocalityPriority", "weight": 1},
+    {"name": "LeastRequestedPriority", "weight": 1},
+    {"name": "BalancedResourceAllocation", "weight": 1},
+    {"name": "SelectorSpreadPriority", "weight": 1},
+    {"name": "NodePreferAvoidPodsPriority", "weight": 1},
+    {"name": "NodeAffinityPriority", "weight": 1},
+    {"name": "TaintTolerationPriority", "weight": 1},
+    {"name": "MostRequestedPriority", "weight": 1}
+  ],
+  "extenders": [
+    {
+      "urlPrefix": "http://127.0.0.1:10262/scheduler",
+      "filterVerb": "filter",
+      "weight": 1,
+      "httpTimeout": 30000000000,
+      "enableHttps": false
+    }
+  ]
+}

charts/tidb-operator/templates/scheduler-deployment.yaml

@@ -0,0 +1,50 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: tidb-scheduler
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: scheduler
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+spec:
+  replicas: {{ .Values.scheduler.replicas }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ template "chart.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: scheduler
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ template "chart.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: scheduler
+    spec:
+    {{- if .Values.scheduler.serviceAccount }}
+      serviceAccount: {{ .Values.scheduler.serviceAccount }}
+    {{- end }}
+      containers:
+      - name: tidb-scheduler
+        image: {{ .Values.operatorImage }}
+        resources:
+{{ toYaml .Values.scheduler.resources | indent 12 }}
+        command:
+          - /usr/local/bin/tidb-scheduler
+          - -v={{ .Values.scheduler.logLevel }}
+          - -port=10262
+      - name: kube-scheduler
+        image: {{ .Values.scheduler.kubeSchedulerImage }}
+        resources:
+{{ toYaml .Values.scheduler.resources | indent 12 }}
+        command:
+        - kube-scheduler
+        - --port=10261
+        - --leader-elect=true
+        - --lock-object-name={{ .Values.scheduler.schedulerName }}
+        - --lock-object-namespace={{ .Release.Namespace }}
+        - --scheduler-name={{ .Values.scheduler.schedulerName }}
+        - --v={{ .Values.scheduler.logLevel }}
+        - --policy-configmap={{ .Values.scheduler.schedulerName }}-policy
+        - --policy-configmap-namespace={{ .Release.Namespace }}

charts/tidb-operator/templates/scheduler-policy-configmap.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: tidb-scheduler-policy
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: scheduler
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+data:
+  policy.cfg: |-
+{{ .Files.Get "files/scheduler-policy.json" | indent 4 }}

charts/tidb-operator/templates/scheduler-rbac.yaml

@@ -0,0 +1,125 @@
+{{- if .Values.rbac.create }}
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: {{ .Values.scheduler.serviceAccount }}
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: scheduler
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: {{ .Release.Name }}:tidb-scheduler
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: scheduler
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+rules:
+{{- if .Values.clusterScoped }}
+- apiGroups: [""]
+  resources: ["pods", "services", "configmaps", "replicationcontrollers", "persistentvolumeclaims", "endpoints"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["pods/binding"]
+  verbs: ["create"]
+- apiGroups: [""]
+  resources: ["endpoints", "events"]
+  verbs: ["get", "list", "watch", "create", "update", "patch"]
+- apiGroups: ["apps"]
+  resources: ["statefulsets"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["policy"]
+  resources: ["poddisruptionbudgets"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["extensions"]
+  resources: ["replicasets"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["storage.k8s.io"]
+  resources: ["storageclasses"]
+  verbs: ["get", "list", "watch"]
+{{- end }}
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["persistentvolumes"]
+  verbs: ["get", "list", "watch", "update"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: {{ .Release.Name }}:tidb-scheduler
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: scheduler
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+subjects:
+- kind: ServiceAccount
+  name: {{ .Values.scheduler.serviceAccount }}
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ .Release.Name }}:tidb-scheduler
+  apiGroup: rbac.authorization.k8s.io
+{{- if (not .Values.clusterScoped) }}
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: {{ .Release.Name }}:tidb-scheduler
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: scheduler
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+rules:
+- apiGroups: [""]
+  resources: ["pods", "services", "configmaps", "replicationcontrollers", "persistentvolumeclaims", "endpoints"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["pods/binding"]
+  verbs: ["create"]
+- apiGroups: [""]
+  resources: ["endpoints", "events"]
+  verbs: ["get", "list", "watch", "create", "update", "patch"]
+- apiGroups: ["apps"]
+  resources: ["statefulsets"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["policy"]
+  resources: ["poddisruptionbudgets"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["extensions"]
+  resources: ["replicasets"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["storage.k8s.io"]
+  resources: ["storageclasses"]
+  verbs: ["get", "list", "watch"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: {{ .Release.Name }}:tidb-scheduler
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: scheduler
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+subjects:
+- kind: ServiceAccount
+  name: {{ .Values.scheduler.serviceAccount }}
+roleRef:
+  kind: Role
+  name: {{ .Release.Name }}:tidb-scheduler
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}
+{{- end }}

charts/tidb-operator/values.yaml

@@ -34,3 +34,20 @@ controllerManager:
   pdFailoverPeriod: 5m
   # tidb failover period default(5m)
   tidbFailoverPeriod: 5m
+
+scheduler:
+  # With rbac.create=false, the user is responsible for creating this account
+  # With rbac.create=true, this service account will be created
+  # Also see rbac.create and clusterScoped
+  serviceAccount: tidb-scheduler
+  logLevel: 2
+  replicas: 1
+  schedulerName: tidb-scheduler
+  resources:
+    limits:
+      cpu: 250m
+      memory: 150Mi
+    requests:
+      cpu: 80m
+      memory: 50Mi
+  kubeSchedulerImage: quay.io/coreos/hyperkube:v1.10.4_coreos.0