add AdvancedStatefulSet feature

charts/tidb-operator/templates/advanced-statefulset-deployment.yaml

@@ -0,0 +1,24 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: advanced-statefulset-controller
+  namespace: pingcap
+  labels:
+    app: advanced-statefulset-controller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: advanced-statefulset-controller
+  template:
+    metadata:
+      labels:
+        app: advanced-statefulset-controller
+    spec:
+      containers:
+      - name: advanced-statefulset-controller
+        image: quay.io/cofyc/advanced-statefulset:latest
+        imagePullPolicy: IfNotPresent
+        args:
+        - -v=4
+      serviceAccountName: advanced-statefulset-controller

charts/tidb-operator/templates/advanced-statefulset-rbac.yaml

@@ -0,0 +1,71 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: advanced-statefulset-controller
+  namespace: pingcap
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: advanced-statefulset-controller
+rules:
+- apiGroups:
+  - apps.pingcap.com
+  resources:
+  - '*'
+  verbs:
+  - '*'
+- apiGroups:
+  - 'apps'
+  resources:
+  - 'controllerrevisions'
+  verbs:
+  - '*'
+- apiGroups:
+  - ''
+  resources:
+  - 'pods'
+  - 'persistentvolumeclaims'
+  - 'persistentvolumes'
+  verbs:
+  - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: advanced-statefulset-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: advanced-statefulset-controller
+subjects:
+- kind: ServiceAccount
+  name: advanced-statefulset-controller
+  namespace: pingcap
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: advanced-statefulset-controller
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - 'endpoints'
+  verbs:
+  - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: advanced-statefulset-controller
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: advanced-statefulset-controller
+subjects:
+- kind: ServiceAccount
+  name: advanced-statefulset-controller
+  namespace: pingcap

charts/tidb-operator/templates/controller-manager-deployment.yaml

@@ -49,6 +49,9 @@ spec:
           {{- if .Values.testMode }}
           - -test-mode={{ .Values.testMode }}
           {{- end}}
+          {{- if .Values.controllerManager.features }}
+          - -features={{ join "," .Values.controllerManager.features }}
+          {{- end }}
         env:
           - name: NAMESPACE
             valueFrom:

charts/tidb-operator/templates/crd.v1beta1.yaml

@@ -0,0 +1,51 @@
+# For Kubernetes before 1.16.
+# TODO more validations, etc.
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: statefulsets.apps.pingcap.com
+spec:
+  group: apps.pingcap.com
+  version: v1alpha1
+  scope: Namespaced
+  names:
+    plural: statefulsets
+    singular: statefulset
+    kind: StatefulSet
+    shortNames:
+    - asts
+  validation:
+    openAPIV3Schema:
+      type: object
+      properties:
+        spec:
+          type: object
+          properties:
+            replicas:
+              type: integer
+              minimum: 0
+            selector:
+              type: object
+            serviceName:
+              type: string
+            template:
+              type: object
+            revisionHistoryLimit:
+              type: integer
+              minimum: 1
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+  # subresources describes the subresources for custom resources.
+  subresources:
+    # status enables the status subresource.
+    status: {}
+    # scale enables the scale subresource.
+    scale:
+      # specReplicasPath defines the JSONPath inside of a custom resource that corresponds to Scale.Spec.Replicas.
+      specReplicasPath: .spec.replicas
+      # statusReplicasPath defines the JSONPath inside of a custom resource that corresponds to Scale.Status.Replicas.
+      statusReplicasPath: .status.replicas
+      # labelSelectorPath defines the JSONPath inside of a custom resource that corresponds to Scale.Status.Selector.
+      labelSelectorPath: .status.labelSelector

charts/tidb-operator/values.yaml

@@ -54,6 +54,8 @@ controllerManager:
   #   operator: Equal
   #   value: tidb-operator
   #   effect: "NoSchedule"
+  # features:
+  # - AdvancedStatefulSet=true
 
 scheduler:
   # With rbac.create=false, the user is responsible for creating this account

cmd/controller-manager/main.go

@@ -21,13 +21,17 @@ import (
 	"os"
 	"time"
 
+	"github.com/cofyc/advanced-statefulset/pkg/apis/apps/v1alpha1/helper"
+	asclientset "github.com/cofyc/advanced-statefulset/pkg/client/clientset/versioned"
+	asinformers "github.com/cofyc/advanced-statefulset/pkg/client/informers/externalversions"
 	"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
 	informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions"
 	"github.com/pingcap/tidb-operator/pkg/controller"
 	"github.com/pingcap/tidb-operator/pkg/controller/backup"
 	"github.com/pingcap/tidb-operator/pkg/controller/backupschedule"
 	"github.com/pingcap/tidb-operator/pkg/controller/restore"
 	"github.com/pingcap/tidb-operator/pkg/controller/tidbcluster"
+	"github.com/pingcap/tidb-operator/pkg/features"
 	"github.com/pingcap/tidb-operator/pkg/version"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -105,9 +109,12 @@ func main() {
 	if err != nil {
 		glog.Fatalf("failed to get kubernetes Clientset: %v", err)
 	}
+	asCli, err := asclientset.NewForConfig(cfg)
 
 	var informerFactory informers.SharedInformerFactory
 	var kubeInformerFactory kubeinformers.SharedInformerFactory
+	var asInformerFactory asinformers.SharedInformerFactory
+
 	if controller.ClusterScoped {
 		informerFactory = informers.NewSharedInformerFactory(cli, controller.ResyncDuration)
 		kubeInformerFactory = kubeinformers.NewSharedInformerFactory(kubeCli, controller.ResyncDuration)
@@ -135,24 +142,35 @@ func main() {
 		},
 	}
 
-	tcController := tidbcluster.NewController(kubeCli, cli, informerFactory, kubeInformerFactory, autoFailover, pdFailoverPeriod, tikvFailoverPeriod, tidbFailoverPeriod)
-	backupController := backup.NewController(kubeCli, cli, informerFactory, kubeInformerFactory)
-	restoreController := restore.NewController(kubeCli, cli, informerFactory, kubeInformerFactory)
-	bsController := backupschedule.NewController(kubeCli, cli, informerFactory, kubeInformerFactory)
+	var hijackKubeInformerFactory kubeinformers.SharedInformerFactory
+	var hijackKubeCli kubernetes.Interface
+	if features.DefaultFeatureGate.Enabled(features.AdvancedStatefulSet) {
+		// If AdvancedStatefulSet is enabled, we hijack client and informer factory to use AdvancedStatefulSet.
+		hijackKubeCli = helper.NewHijackClient(kubeCli, asCli)
+		hijackKubeInformerFactory = helper.NewHijackSharedInformerFactory(kubeInformerFactory, asInformerFactory)
+	} else {
+		hijackKubeCli = kubeCli
+		hijackKubeInformerFactory = kubeInformerFactory
+	}
+
+	tcController := tidbcluster.NewController(hijackKubeCli, cli, informerFactory, hijackKubeInformerFactory, autoFailover, pdFailoverPeriod, tikvFailoverPeriod, tidbFailoverPeriod)
+	backupController := backup.NewController(hijackKubeCli, cli, informerFactory, hijackKubeInformerFactory)
+	restoreController := restore.NewController(hijackKubeCli, cli, informerFactory, hijackKubeInformerFactory)
+	bsController := backupschedule.NewController(hijackKubeCli, cli, informerFactory, hijackKubeInformerFactory)
 	controllerCtx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
 	// Start informer factories after all controller are initialized.
 	informerFactory.Start(controllerCtx.Done())
-	kubeInformerFactory.Start(controllerCtx.Done())
+	hijackKubeInformerFactory.Start(controllerCtx.Done())
 
 	// Wait for all started informers' cache were synced.
 	for v, synced := range informerFactory.WaitForCacheSync(wait.NeverStop) {
 		if !synced {
 			glog.Fatalf("error syncing informer for %v", v)
 		}
 	}
-	for v, synced := range kubeInformerFactory.WaitForCacheSync(wait.NeverStop) {
+	for v, synced := range hijackKubeInformerFactory.WaitForCacheSync(wait.NeverStop) {
 		if !synced {
 			glog.Fatalf("error syncing informer for %v", v)
 		}

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/ant31/crd-validation v0.0.0-20180702145049-30f8a35d0ac2
 	github.com/chai2010/gettext-go v0.0.0-20170215093142-bf70f2a70fb1 // indirect
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
+	github.com/cofyc/advanced-statefulset v0.0.0-20191105120117-6c0ac9b9df68
 	github.com/coreos/go-semver v0.3.0
 	github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0
 	github.com/docker/go-connections v0.4.0 // indirect
@@ -56,7 +57,7 @@ require (
 	github.com/sirupsen/logrus v1.4.2
 	github.com/soheilhy/cmux v0.1.4 // indirect
 	github.com/spf13/cobra v0.0.5
-	github.com/spf13/pflag v1.0.3
+	github.com/spf13/pflag v1.0.5
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 // indirect
 	github.com/uber-go/atomic v0.0.0-00010101000000-000000000000 // indirect
 	github.com/uber/jaeger-client-go v2.19.0+incompatible // indirect

go.sum

@@ -81,6 +81,8 @@ github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0=
+github.com/cofyc/advanced-statefulset v0.0.0-20191105120117-6c0ac9b9df68 h1:GsnTaZMgLtt0LR1TdL2GsWsEHDXp8/SbuWgpmVd1+k4=
+github.com/cofyc/advanced-statefulset v0.0.0-20191105120117-6c0ac9b9df68/go.mod h1:4KdFa5r17cDhDS6PfRzw/S+Mwk2GwEjqE7aY6EGT0QE=
 github.com/container-storage-interface/spec v1.1.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
 github.com/containerd/console v0.0.0-20170925154832-84eeaae905fa/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
 github.com/containerd/containerd v1.0.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
@@ -648,6 +650,8 @@ github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
 github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.2.1/go.mod h1:P4AexN0a+C9tGAnUFNwDMYYZv3pjFuvmeiMyKRaNVlI=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/storageos/go-api v0.0.0-20180912212459-343b3eff91fc/go.mod h1:ZrLn+e0ZuF3Y65PNF6dIwbJPZqfmtCXxFm9ckv0agOY=
@@ -814,6 +818,8 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w=
+golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20170824195420-5d2fd3ccab98/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181003024731-2f84ea8ef872/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

pkg/features/features.go

@@ -25,7 +25,8 @@ import (
 var (
 	allFeatures     = sets.NewString(StableScheduling)
 	defaultFeatures = map[string]bool{
-		StableScheduling: true,
+		StableScheduling:    true,
+		AdvancedStatefulSet: false,
 	}
 	// DefaultFeatureGate is a shared global FeatureGate.
 	DefaultFeatureGate FeatureGate = NewFeatureGate()
@@ -34,6 +35,9 @@ var (
 const (
 	// StableScheduling controls stable scheduling of TiDB members.
 	StableScheduling string = "StableScheduling"
+
+	// AdvancedStatefulSet controls whether to use AdvancedStatefulSet to manage pods
+	AdvancedStatefulSet string = "AdvancedStatefulSet"
 )
 
 type FeatureGate interface {

tests/actions.go

@@ -222,20 +222,21 @@ type event struct {
 var _ = OperatorActions(&operatorActions{})
 
 type OperatorConfig struct {
-	Namespace          string
-	ReleaseName        string
-	Image              string
-	Tag                string
-	SchedulerImage     string
-	SchedulerTag       string
-	SchedulerFeatures  []string
-	LogLevel           string
-	WebhookServiceName string
-	WebhookSecretName  string
-	WebhookConfigName  string
-	Context            *apimachinery.CertContext
-	ImagePullPolicy    corev1.PullPolicy
-	TestMode           bool
+	Namespace                 string
+	ReleaseName               string
+	Image                     string
+	Tag                       string
+	SchedulerImage            string
+	SchedulerTag              string
+	SchedulerFeatures         []string
+	ControllerManagerFeatures []string
+	LogLevel                  string
+	WebhookServiceName        string
+	WebhookSecretName         string
+	WebhookConfigName         string
+	Context                   *apimachinery.CertContext
+	ImagePullPolicy           corev1.PullPolicy
+	TestMode                  bool
 }
 
 type TidbClusterConfig struct {
@@ -360,6 +361,9 @@ func (oi *OperatorConfig) OperatorHelmSetString(m map[string]string) string {
 	if len(oi.SchedulerFeatures) > 0 {
 		set["scheduler.features"] = fmt.Sprintf("{%s}", strings.Join(oi.SchedulerFeatures, ","))
 	}
+	if len(oi.ControllerManagerFeatures) > 0 {
+		set["controllerManager.features"] = fmt.Sprintf("{%s}", strings.Join(oi.ControllerManagerFeatures, ","))
+	}
 
 	arr := make([]string, 0, len(set))
 	for k, v := range set {

tests/cmd/e2e/main.go

@@ -238,7 +238,10 @@ func newOperatorConfig() *tests.OperatorConfig {
 		SchedulerFeatures: []string{
 			"StableScheduling=true",
 		},
-		LogLevel:           "2",
+		ControllerManagerFeatures: []string{
+			"AdvancedStatefulSet=true",
+		},
+		LogLevel:           "4",
 		WebhookServiceName: "webhook-service",
 		WebhookSecretName:  "webhook-secret",
 		WebhookConfigName:  "webhook-config",