-
Notifications
You must be signed in to change notification settings - Fork 501
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add extended scheduler to operator #145
Changes from 1 commit
d317598
9260f71
449176e
a10f684
b5daa75
3d361c2
db67aa3
dfbb8d7
2aba403
d0b17b9
728d347
29ad131
27f0a8d
1e37aea
504314b
b7e8d1e
2527a18
2675d6f
e42c89e
098e853
40fa7e2
682d8c2
af54fb5
60db86d
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
{{- $defaultHyperkubeImage := "quay.io/coreos/hyperkube:v1.10.4_coreos.0" -}} | ||
apiVersion: extensions/v1beta1 | ||
kind: Deployment | ||
metadata: | ||
name: tidb-scheduler | ||
labels: | ||
app.kubernetes.io/name: {{ template "chart.name" . }} | ||
app.kubernetes.io/managed-by: {{ .Release.Service }} | ||
app.kubernetes.io/instance: {{ .Release.Name }} | ||
app.kubernetes.io/component: scheduler | ||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} | ||
spec: | ||
replicas: {{ .Values.scheduler.replicas }} | ||
selector: | ||
matchLabels: | ||
app.kubernetes.io/name: {{ template "chart.name" . }} | ||
app.kubernetes.io/instance: {{ .Release.Name }} | ||
app.kubernetes.io/component: scheduler | ||
template: | ||
metadata: | ||
labels: | ||
app.kubernetes.io/name: {{ template "chart.name" . }} | ||
app.kubernetes.io/instance: {{ .Release.Name }} | ||
app.kubernetes.io/component: scheduler | ||
spec: | ||
{{- if .Values.scheduler.serviceAccount }} | ||
serviceAccount: {{ .Values.scheduler.serviceAccount }} | ||
{{- end }} | ||
containers: | ||
- name: tidb-scheduler | ||
image: {{ .Values.operatorImage }} | ||
resources: | ||
{{ toYaml .Values.scheduler.resources | indent 12 }} | ||
command: | ||
- /usr/local/bin/tidb-scheduler | ||
- -v={{ .Values.scheduler.logLevel }} | ||
- -port=10262 | ||
- -pd-replicas={{ .Values.scheduler.pdReplicas | default 3 }} | ||
- -tikv-replicas={{ .Values.scheduler.tikvReplicas | default 3 }} | ||
- name: kube-scheduler | ||
{{- if .Values.scheduler.kubeSchedulerImage }} | ||
image: {{ .Values.scheduler.kubeSchedulerImage }} | ||
{{- else if .Values.scheduler.hyperkubeImage }} | ||
image: {{ .Values.scheduler.hyperkubeImage }} | ||
{{- else }} | ||
image: {{ $defaultHyperkubeImage }} | ||
{{- end }} | ||
resources: | ||
{{ toYaml .Values.scheduler.resources | indent 12 }} | ||
command: | ||
{{- if .Values.scheduler.kubeSchedulerImage }} | ||
- kube-scheduler | ||
{{- else }} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. hyperkube image also has |
||
- /hyperkube | ||
- scheduler | ||
{{- end }} | ||
- --port=10261 | ||
- --leader-elect=true | ||
- --lock-object-name=tidb-scheduler | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The lock should be customizable and I think it should be the same as |
||
- --lock-object-namespace={{ .Release.Namespace }} | ||
- --scheduler-name={{ .Values.scheduler.schedulerName }} | ||
- --v={{ .Values.scheduler.logLevel }} | ||
- --policy-configmap=tidb-scheduler-policy | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. same here. |
||
- --policy-configmap-namespace={{ .Release.Namespace }} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
apiVersion: v1 | ||
kind: ConfigMap | ||
metadata: | ||
name: tidb-scheduler-policy | ||
labels: | ||
app.kubernetes.io/name: {{ template "chart.name" . }} | ||
app.kubernetes.io/managed-by: {{ .Release.Service }} | ||
app.kubernetes.io/instance: {{ .Release.Name }} | ||
app.kubernetes.io/component: scheduler | ||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} | ||
data: | ||
policy.cfg: |- | ||
{ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This JSON doesn't need to be templated, so using There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. OK, waiting for: #149 merged. |
||
"kind" : "Policy", | ||
"apiVersion" : "v1", | ||
"predicates": [ | ||
{"name": "MatchInterPodAffinity"}, | ||
{"name": "CheckVolumeBinding"}, | ||
{"name": "CheckNodeCondition"}, | ||
{"name": "GeneralPredicates"}, | ||
{"name": "HostName"}, | ||
{"name": "PodFitsHostPorts"}, | ||
{"name": "MatchNodeSelector"}, | ||
{"name": "PodFitsResources"}, | ||
{"name": "NoDiskConflict"}, | ||
{"name": "PodToleratesNodeTaints"}, | ||
{"name": "CheckNodeMemoryPressure"}, | ||
{"name": "CheckNodeDiskPressure"} | ||
], | ||
"priorities": [ | ||
{"name": "EqualPriority", "weight": 1}, | ||
{"name": "ImageLocalityPriority", "weight": 1}, | ||
{"name": "LeastRequestedPriority", "weight": 1}, | ||
{"name": "BalancedResourceAllocation", "weight": 1}, | ||
{"name": "SelectorSpreadPriority", "weight": 1}, | ||
{"name": "NodePreferAvoidPodsPriority", "weight": 1}, | ||
{"name": "NodeAffinityPriority", "weight": 1}, | ||
{"name": "TaintTolerationPriority", "weight": 1}, | ||
{"name": "MostRequestedPriority", "weight": 1} | ||
], | ||
"extenders": [ | ||
{ | ||
"urlPrefix": "http://127.0.0.1:10262/scheduler", | ||
"filterVerb": "filter", | ||
"weight": 1, | ||
"httpTimeout": 30000000000, | ||
"enableHttps": false | ||
} | ||
] | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,125 @@ | ||
{{- if .Values.rbac.create }} | ||
kind: ServiceAccount | ||
apiVersion: v1 | ||
metadata: | ||
name: {{ .Values.scheduler.serviceAccount }} | ||
labels: | ||
app.kubernetes.io/name: {{ template "chart.name" . }} | ||
app.kubernetes.io/managed-by: {{ .Release.Service }} | ||
app.kubernetes.io/instance: {{ .Release.Name }} | ||
app.kubernetes.io/component: scheduler | ||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} | ||
--- | ||
kind: ClusterRole | ||
apiVersion: rbac.authorization.k8s.io/v1beta1 | ||
metadata: | ||
name: {{ .Release.Name }}:tidb-scheduler | ||
labels: | ||
app.kubernetes.io/name: {{ template "chart.name" . }} | ||
app.kubernetes.io/managed-by: {{ .Release.Service }} | ||
app.kubernetes.io/instance: {{ .Release.Name }} | ||
app.kubernetes.io/component: scheduler | ||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} | ||
rules: | ||
{{- if .Values.clusterScoped }} | ||
- apiGroups: [""] | ||
resources: ["pods", "services", "configmaps", "replicationcontrollers", "persistentvolumeclaims", "endpoints"] | ||
verbs: ["get", "list", "watch"] | ||
- apiGroups: [""] | ||
resources: ["pods/binding"] | ||
verbs: ["create"] | ||
- apiGroups: [""] | ||
resources: ["endpoints", "events"] | ||
verbs: ["get", "list", "watch", "create", "update", "patch"] | ||
- apiGroups: ["apps"] | ||
resources: ["statefulsets"] | ||
verbs: ["get", "list", "watch"] | ||
- apiGroups: ["policy"] | ||
resources: ["poddisruptionbudgets"] | ||
verbs: ["get", "list", "watch"] | ||
- apiGroups: ["extensions"] | ||
resources: ["replicasets"] | ||
verbs: ["get", "list", "watch"] | ||
- apiGroups: ["storage.k8s.io"] | ||
resources: ["storageclasses"] | ||
verbs: ["get", "list", "watch"] | ||
{{- end }} | ||
- apiGroups: [""] | ||
resources: ["nodes"] | ||
verbs: ["get", "list", "watch"] | ||
- apiGroups: [""] | ||
resources: ["persistentvolumes"] | ||
verbs: ["get", "list", "watch", "update"] | ||
--- | ||
kind: ClusterRoleBinding | ||
apiVersion: rbac.authorization.k8s.io/v1beta1 | ||
metadata: | ||
name: {{ .Release.Name }}:tidb-scheduler | ||
labels: | ||
app.kubernetes.io/name: {{ template "chart.name" . }} | ||
app.kubernetes.io/managed-by: {{ .Release.Service }} | ||
app.kubernetes.io/instance: {{ .Release.Name }} | ||
app.kubernetes.io/component: scheduler | ||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} | ||
subjects: | ||
- kind: ServiceAccount | ||
name: {{ .Values.scheduler.serviceAccount }} | ||
namespace: {{ .Release.Namespace }} | ||
roleRef: | ||
kind: ClusterRole | ||
name: {{ .Release.Name }}:tidb-scheduler | ||
apiGroup: rbac.authorization.k8s.io | ||
{{- if (not .Values.clusterScoped) }} | ||
--- | ||
kind: Role | ||
apiVersion: rbac.authorization.k8s.io/v1beta1 | ||
metadata: | ||
name: {{ .Release.Name }}:tidb-scheduler | ||
labels: | ||
app.kubernetes.io/name: {{ template "chart.name" . }} | ||
app.kubernetes.io/managed-by: {{ .Release.Service }} | ||
app.kubernetes.io/instance: {{ .Release.Name }} | ||
app.kubernetes.io/component: scheduler | ||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} | ||
rules: | ||
- apiGroups: [""] | ||
resources: ["pods", "services", "configmaps", "replicationcontrollers", "persistentvolumeclaims", "endpoints"] | ||
verbs: ["get", "list", "watch"] | ||
- apiGroups: [""] | ||
resources: ["pods/binding"] | ||
verbs: ["create"] | ||
- apiGroups: [""] | ||
resources: ["endpoints", "events"] | ||
verbs: ["get", "list", "watch", "create", "update", "patch"] | ||
- apiGroups: ["apps"] | ||
resources: ["statefulsets"] | ||
verbs: ["get", "list", "watch"] | ||
- apiGroups: ["policy"] | ||
resources: ["poddisruptionbudgets"] | ||
verbs: ["get", "list", "watch"] | ||
- apiGroups: ["extensions"] | ||
resources: ["replicasets"] | ||
verbs: ["get", "list", "watch"] | ||
- apiGroups: ["storage.k8s.io"] | ||
resources: ["storageclasses"] | ||
verbs: ["get", "list", "watch"] | ||
--- | ||
kind: RoleBinding | ||
apiVersion: rbac.authorization.k8s.io/v1beta1 | ||
metadata: | ||
name: {{ .Release.Name }}:tidb-scheduler | ||
labels: | ||
app.kubernetes.io/name: {{ template "chart.name" . }} | ||
app.kubernetes.io/managed-by: {{ .Release.Service }} | ||
app.kubernetes.io/instance: {{ .Release.Name }} | ||
app.kubernetes.io/component: scheduler | ||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} | ||
subjects: | ||
- kind: ServiceAccount | ||
name: {{ .Values.scheduler.serviceAccount }} | ||
roleRef: | ||
kind: Role | ||
name: {{ .Release.Name }}:tidb-scheduler | ||
apiGroup: rbac.authorization.k8s.io | ||
{{- end }} | ||
{{- end }} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -34,3 +34,25 @@ controllerManager: | |
pdFailoverPeriod: 5m | ||
# tidb failover period default(5m) | ||
tidbFailoverPeriod: 5m | ||
|
||
scheduler: | ||
# With rbac.create=false, the user is responsible for creating this account | ||
# With rbac.create=true, this service account will be created | ||
# Also see rbac.create and clusterScoped | ||
serviceAccount: tidb-scheduler | ||
logLevel: 2 | ||
replicas: 1 | ||
schedulerName: tidb-scheduler | ||
resources: | ||
limits: | ||
cpu: 250m | ||
memory: 150Mi | ||
requests: | ||
cpu: 80m | ||
memory: 50Mi | ||
# pd replicas | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The replicas are cluster specific, it should not be defined in the scheduler. And since PD is single raft cluster, its replica is the same as its member count. But TiKV replica is configured in PD configuration, this can be retrieved by PD API. |
||
pdReplicas: 3 | ||
# tikv replicas | ||
tikvReplicas: 3 | ||
hyperkubeImage: quay.io/coreos/hyperkube:v1.10.4_coreos.0 | ||
# kubeSchedulerImage: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Unnecessary to use two image variable. |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
why add schedulerName on here
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This
schedulerName
used at here: https://github.com/weekface/tidb-operator/blob/449176ebcaa90b37be63156eb72a27fee5266a01/pkg/manager/member/pd_member_manager.go#L513