Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add AdvancedStatefulSet feature #1108

Merged
merged 6 commits into from
Nov 18, 2019
Merged

add AdvancedStatefulSet feature #1108

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a417b6d
add AdvancedStatefulSet feature
cofyc Nov 11, 2019
d826b8b
Merge branch 'master' into fix1094
cofyc Nov 15, 2019
111e107
fix
cofyc Nov 15, 2019
06d1a89
Merge branch 'fix1094' of github.com:cofyc/tidb-operator into fix1094
cofyc Nov 15, 2019
8adaf52
Merge branch 'master' into fix1094
cofyc Nov 18, 2019
ef0b59c
Merge branch 'master' into fix1094
tennix Nov 18, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 55 additions & 0 deletions charts/tidb-operator/templates/advanced-statefulset-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
{{- if .Values.advancedStatefulset.create }}
apiVersion: apps/v1
cofyc marked this conversation as resolved.
Show resolved Hide resolved
kind: Deployment
metadata:
name: advanced-statefulset-controller
labels:
app.kubernetes.io/name: {{ template "chart.name" . }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: advanced-statefulset-controller
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
spec:
replicas: {{ .Values.advancedStatefulset.replicas }}
selector:
matchLabels:
app.kubernetes.io/name: {{ template "chart.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: advanced-statefulset-controller
template:
metadata:
labels:
app.kubernetes.io/name: {{ template "chart.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: advanced-statefulset-controller
spec:
serviceAccountName: {{ .Values.advancedStatefulset.serviceAccount }}
containers:
- name: advanced-statefulset-controller
image: {{ .Values.advancedStatefulset.image }}
imagePullPolicy: {{ .Values.advancedStatefulset.imagePullPolicy | default "IfNotPresent" }}
args:
- --v={{ .Values.advancedStatefulset.logLevel }}
- --leader-elect
- --leader-elect-resource-name=advanced-statefulset-controller
- --leader-elect-resource-namespace=$(POD_NAMESPACE)
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
resources:
{{ toYaml .Values.advancedStatefulset.resources | indent 12 }}
{{- with .Values.advancedStatefulset.nodeSelector }}
nodeSelector:
{{ toYaml . | indent 8 }}
{{- end }}
{{- with .Values.advancedStatefulset.affinity }}
affinity:
{{ toYaml . | indent 8 }}
{{- end }}
{{- with .Values.advancedStatefulset.tolerations }}
tolerations:
{{ toYaml . | indent 8 }}
{{- end }}
{{- end }}
96 changes: 96 additions & 0 deletions charts/tidb-operator/templates/advanced-statefulset-rbac.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
{{- if .Values.advancedStatefulset.create }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ .Values.advancedStatefulset.serviceAccount }}
labels:
app.kubernetes.io/name: {{ template "chart.name" . }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: advanced-statefulset-controller
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: advanced-statefulset-controller
labels:
app.kubernetes.io/name: {{ template "chart.name" . }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: advanced-statefulset-controller
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
rules:
- apiGroups:
- apps.pingcap.com
resources:
- '*'
verbs:
- '*'
- apiGroups:
- 'apps'
resources:
- 'controllerrevisions'
verbs:
- '*'
- apiGroups:
- ''
resources:
- 'pods'
- 'persistentvolumeclaims'
- 'persistentvolumes'
verbs:
- '*'
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: advanced-statefulset-controller
labels:
app.kubernetes.io/name: {{ template "chart.name" . }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: advanced-statefulset-controller
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: advanced-statefulset-controller
subjects:
- kind: ServiceAccount
name: advanced-statefulset-controller
namespace: {{ .Release.Namespace }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: advanced-statefulset-controller
namespace: {{ .Release.Namespace }}
labels:
app.kubernetes.io/name: {{ template "chart.name" . }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: advanced-statefulset-controller
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
rules:
- apiGroups:
- ''
resources:
- 'endpoints'
verbs:
- '*'
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: advanced-statefulset-controller
namespace: {{ .Release.Namespace }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: advanced-statefulset-controller
subjects:
- kind: ServiceAccount
name: advanced-statefulset-controller
namespace: {{ .Release.Namespace }}
{{- end }}
3 changes: 3 additions & 0 deletions charts/tidb-operator/templates/controller-manager-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ spec:
{{- if .Values.testMode }}
- -test-mode={{ .Values.testMode }}
{{- end}}
{{- if .Values.features }}
- -features={{ join "," .Values.features }}
{{- end }}
env:
- name: NAMESPACE
valueFrom:
Expand Down
16 changes: 16 additions & 0 deletions charts/tidb-operator/templates/controller-manager-rbac.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,14 @@ rules:
- restores
- restores/finalizers
verbs: ["*"]
{{- if .Values.features | has "AdvancedStatefulSet=true" }}
- apiGroups:
- apps.pingcap.com
resources:
- statefulsets
verbs:
- '*'
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this be configured only when the advance sts feature is turned on, keeping the principle of least privilege?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

{{- end }}
{{- end }}
- apiGroups: [""]
resources: ["nodes"]
Expand Down Expand Up @@ -135,6 +143,14 @@ rules:
- restores
- restores/finalizers
verbs: ["*"]
{{- if .Values.features | has "AdvancedStatefulSet=true" }}
- apiGroups:
- apps.pingcap.com
resources:
- statefulsets
verbs:
- '*'
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

{{- end }}
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
Expand Down
4 changes: 2 additions & 2 deletions charts/tidb-operator/templates/scheduler-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@ spec:
- /usr/local/bin/tidb-scheduler
- -v={{ .Values.scheduler.logLevel }}
- -port=10262
{{- if .Values.scheduler.features }}
- -features={{ join "," .Values.scheduler.features }}
{{- if .Values.features }}
- -features={{ join "," .Values.features }}
{{- end }}
{{- if and (ne .Values.timezone "UTC") (ne .Values.timezone "") }}
env:
Expand Down
52 changes: 50 additions & 2 deletions charts/tidb-operator/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,23 @@ defaultStorageClassName: local-storage
# tidbBackupManagerImage: pingcap/tidb-backup-manager:latest
# defaultBackupStorageClassName: local-storage

#
# Enable or disable tidb-operator features:
#
# StableScheduling (default: true)
# Enable stable scheduling of tidb servers.
#
# AdvancedStatefulSet (default: false)
# If enabled, tidb-operator will use AdvancedStatefulSet to manage pods
# instead of Kubernetes StatefulSet.
# It's ok to turn it on if this feature is not enabled. However it's not ok
# to turn it off when the tidb-operator already uses AdvancedStatefulSet to
# manage pods. This is in alpha phase.
#
features: []
# - AdvancedStatefulSet=false
# - StableScheduling=true
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

some features involve multiple components, it's better to make it global


controllerManager:
# With rbac.create=false, the user is responsible for creating this account
# With rbac.create=true, this service account will be created
Expand Down Expand Up @@ -66,8 +83,6 @@ scheduler:
logLevel: 2
replicas: 1
schedulerName: tidb-scheduler
# features:
# - StableScheduling=true
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this feature is enabled by default and I guess no one will disable it in production, so I think we don't need to support scheduler.features for backward compatibility

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Breaking backward compatibility should be noted in the PR description.

resources:
limits:
cpu: 250m
Expand Down Expand Up @@ -127,3 +142,36 @@ apiserver:
# value: tidb-operator
# effect: "NoSchedule"

# When AdvancedStatefulSet feature is enabled, you must install
# AdvancedStatefulSet controller.
# Note that AdvancedStatefulSet CRD must be installed manually via the following
# command:
# kubectl apply -f manifests/advanced-statefulset-crd.v1beta1.yaml
advancedStatefulset:
create: false
image: pingcap/advanced-statefulset:v0.1.0
imagePullPolicy: IfNotPresent
serviceAccount: advanced-statefulset-controller
logLevel: 2
replicas: 1
resources:
limits:
cpu: 500m
memory: 300Mi
requests:
cpu: 200m
memory: 50Mi
## affinity defines pod scheduling rules,affinity default settings is empty.
## please read the affinity document before set your scheduling rule:
## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
affinity: {}
## nodeSelector ensure pods only assigning to nodes which have each of the indicated key-value pairs as labels
## ref:https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
nodeSelector: {}
## Tolerations are applied to pods, and allow pods to schedule onto nodes with matching taints.
## refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration
tolerations: []
# - key: node-role
# operator: Equal
# value: tidb-operator
# effect: "NoSchedule"
18 changes: 17 additions & 1 deletion cmd/admission-controller/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,12 @@ import (
"os/signal"
"syscall"

"github.com/pingcap/advanced-statefulset/pkg/apis/apps/v1alpha1/helper"
asclientset "github.com/pingcap/advanced-statefulset/pkg/client/clientset/versioned"
"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions"
"github.com/pingcap/tidb-operator/pkg/controller"
"github.com/pingcap/tidb-operator/pkg/features"
"github.com/pingcap/tidb-operator/pkg/version"
"github.com/pingcap/tidb-operator/pkg/webhook"
"k8s.io/apimachinery/pkg/util/wait"
Expand All @@ -44,6 +47,7 @@ func init() {
flag.BoolVar(&printVersion, "version", false, "Show version and quit")
flag.StringVar(&certFile, "tlsCertFile", "/etc/webhook/certs/cert.pem", "File containing the x509 Certificate for HTTPS.")
flag.StringVar(&keyFile, "tlsKeyFile", "/etc/webhook/certs/key.pem", "File containing the x509 private key to --tlsCertFile.")
features.DefaultFeatureGate.AddFlag(flag.CommandLine)
flag.Parse()
}

Expand All @@ -68,10 +72,22 @@ func main() {
glog.Fatalf("failed to create Clientset: %v", err)
}

kubeCli, err := kubernetes.NewForConfig(cfg)
var kubeCli kubernetes.Interface
kubeCli, err = kubernetes.NewForConfig(cfg)
if err != nil {
glog.Fatalf("failed to get kubernetes Clientset: %v", err)
}
asCli, err := asclientset.NewForConfig(cfg)
if err != nil {
glog.Fatalf("failed to get advanced-statefulset Clientset: %v", err)
}

if features.DefaultFeatureGate.Enabled(features.AdvancedStatefulSet) {
// If AdvancedStatefulSet is enabled, we hijack the Kubernetes client to use
// AdvancedStatefulSet.
kubeCli = helper.NewHijackClient(kubeCli, asCli)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://github.com/cofyc/advanced-statefulset/blob/master/pkg/apis/apps/v1alpha1/helper/hijack.go

}

informerFactory := informers.NewSharedInformerFactory(cli, controller.ResyncDuration)
kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeCli, controller.ResyncDuration)

Expand Down
17 changes: 16 additions & 1 deletion cmd/controller-manager/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,16 @@ import (
"os"
"time"

"github.com/pingcap/advanced-statefulset/pkg/apis/apps/v1alpha1/helper"
asclientset "github.com/pingcap/advanced-statefulset/pkg/client/clientset/versioned"
"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions"
"github.com/pingcap/tidb-operator/pkg/controller"
"github.com/pingcap/tidb-operator/pkg/controller/backup"
"github.com/pingcap/tidb-operator/pkg/controller/backupschedule"
"github.com/pingcap/tidb-operator/pkg/controller/restore"
"github.com/pingcap/tidb-operator/pkg/controller/tidbcluster"
"github.com/pingcap/tidb-operator/pkg/features"
"github.com/pingcap/tidb-operator/pkg/version"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
Expand Down Expand Up @@ -68,6 +71,7 @@ func init() {
flag.DurationVar(&controller.ResyncDuration, "resync-duration", time.Duration(30*time.Second), "Resync time of informer")
flag.BoolVar(&controller.TestMode, "test-mode", false, "whether tidb-operator run in test mode")
flag.StringVar(&controller.TidbBackupManagerImage, "tidb-backup-manager-image", "pingcap/tidb-backup-manager:latest", "The image of backup manager tool")
features.DefaultFeatureGate.AddFlag(flag.CommandLine)

flag.Parse()
}
Expand Down Expand Up @@ -101,10 +105,21 @@ func main() {
if err != nil {
glog.Fatalf("failed to create Clientset: %v", err)
}
kubeCli, err := kubernetes.NewForConfig(cfg)
var kubeCli kubernetes.Interface
kubeCli, err = kubernetes.NewForConfig(cfg)
if err != nil {
glog.Fatalf("failed to get kubernetes Clientset: %v", err)
}
asCli, err := asclientset.NewForConfig(cfg)
if err != nil {
glog.Fatalf("failed to get advanced-statefulset Clientset: %v", err)
}

if features.DefaultFeatureGate.Enabled(features.AdvancedStatefulSet) {
// If AdvancedStatefulSet is enabled, we hijack the Kubernetes client to use
// AdvancedStatefulSet.
kubeCli = helper.NewHijackClient(kubeCli, asCli)
}

var informerFactory informers.SharedInformerFactory
var kubeInformerFactory kubeinformers.SharedInformerFactory
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ require (
github.com/onsi/gomega v1.5.0
github.com/opentracing/opentracing-go v1.1.0 // indirect
github.com/pierrec/lz4 v2.0.5+incompatible // indirect
github.com/pingcap/advanced-statefulset v0.1.0
github.com/pingcap/errors v0.11.0
github.com/pingcap/kvproto v0.0.0-20190516013202-4cf58ad90b6c
github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd
Expand Down
4 changes: 4 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -568,6 +568,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pingcap/advanced-statefulset v0.1.0 h1:rjUx6Tc90YwlRrgIXWTvjr8D97fx0GP4E49dObmBqj0=
github.com/pingcap/advanced-statefulset v0.1.0/go.mod h1:rg2p1v6AGsKhvEZi6Sm0YNYJCmdXdZZhQ6Sviei7Ivs=
github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg=
github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ=
github.com/pingcap/errors v0.11.0 h1:DCJQB8jrHbQ1VVlMFIrbj2ApScNNotVmkSNplu2yUt4=
Expand Down Expand Up @@ -816,6 +818,8 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w=
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20170824195420-5d2fd3ccab98/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181003024731-2f84ea8ef872/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
Expand Down
Loading