From d6359e2dcab915ab3d4bcb52b9b001792f24f0b4 Mon Sep 17 00:00:00 2001 From: Yecheng Fu Date: Tue, 5 Nov 2019 20:45:27 +0800 Subject: [PATCH] add AdvancedStatefulSet feature --- .../advanced-statefulset-deployment.yaml | 24 +++++++ .../templates/advanced-statefulset-rbac.yaml | 71 +++++++++++++++++++ .../controller-manager-deployment.yaml | 3 + .../tidb-operator/templates/crd.v1beta1.yaml | 51 +++++++++++++ charts/tidb-operator/values.yaml | 2 + cmd/controller-manager/main.go | 31 ++++++-- go.mod | 3 +- go.sum | 6 ++ pkg/features/features.go | 6 +- tests/actions.go | 32 +++++---- tests/cmd/e2e/main.go | 5 +- 11 files changed, 211 insertions(+), 23 deletions(-) create mode 100644 charts/tidb-operator/templates/advanced-statefulset-deployment.yaml create mode 100644 charts/tidb-operator/templates/advanced-statefulset-rbac.yaml create mode 100644 charts/tidb-operator/templates/crd.v1beta1.yaml diff --git a/charts/tidb-operator/templates/advanced-statefulset-deployment.yaml b/charts/tidb-operator/templates/advanced-statefulset-deployment.yaml new file mode 100644 index 00000000000..af7a315020a --- /dev/null +++ b/charts/tidb-operator/templates/advanced-statefulset-deployment.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: advanced-statefulset-controller + namespace: pingcap + labels: + app: advanced-statefulset-controller +spec: + replicas: 1 + selector: + matchLabels: + app: advanced-statefulset-controller + template: + metadata: + labels: + app: advanced-statefulset-controller + spec: + containers: + - name: advanced-statefulset-controller + image: quay.io/cofyc/advanced-statefulset:latest + imagePullPolicy: IfNotPresent + args: + - -v=4 + serviceAccountName: advanced-statefulset-controller diff --git a/charts/tidb-operator/templates/advanced-statefulset-rbac.yaml b/charts/tidb-operator/templates/advanced-statefulset-rbac.yaml new file mode 100644 index 00000000000..925d4046f3e --- /dev/null +++ b/charts/tidb-operator/templates/advanced-statefulset-rbac.yaml @@ -0,0 +1,71 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: advanced-statefulset-controller + namespace: pingcap +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: advanced-statefulset-controller +rules: +- apiGroups: + - apps.pingcap.com + resources: + - '*' + verbs: + - '*' +- apiGroups: + - 'apps' + resources: + - 'controllerrevisions' + verbs: + - '*' +- apiGroups: + - '' + resources: + - 'pods' + - 'persistentvolumeclaims' + - 'persistentvolumes' + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: advanced-statefulset-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: advanced-statefulset-controller +subjects: +- kind: ServiceAccount + name: advanced-statefulset-controller + namespace: pingcap +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: advanced-statefulset-controller + namespace: kube-system +rules: +- apiGroups: + - '' + resources: + - 'endpoints' + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: advanced-statefulset-controller + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: advanced-statefulset-controller +subjects: +- kind: ServiceAccount + name: advanced-statefulset-controller + namespace: pingcap diff --git a/charts/tidb-operator/templates/controller-manager-deployment.yaml b/charts/tidb-operator/templates/controller-manager-deployment.yaml index d593c175165..0b36248ec0a 100644 --- a/charts/tidb-operator/templates/controller-manager-deployment.yaml +++ b/charts/tidb-operator/templates/controller-manager-deployment.yaml @@ -49,6 +49,9 @@ spec: {{- if .Values.testMode }} - -test-mode={{ .Values.testMode }} {{- end}} + {{- if .Values.controllerManager.features }} + - -features={{ join "," .Values.controllerManager.features }} + {{- end }} env: - name: NAMESPACE valueFrom: diff --git a/charts/tidb-operator/templates/crd.v1beta1.yaml b/charts/tidb-operator/templates/crd.v1beta1.yaml new file mode 100644 index 00000000000..8b0c4fca8f0 --- /dev/null +++ b/charts/tidb-operator/templates/crd.v1beta1.yaml @@ -0,0 +1,51 @@ +# For Kubernetes before 1.16. +# TODO more validations, etc. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: statefulsets.apps.pingcap.com +spec: + group: apps.pingcap.com + version: v1alpha1 + scope: Namespaced + names: + plural: statefulsets + singular: statefulset + kind: StatefulSet + shortNames: + - asts + validation: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + replicas: + type: integer + minimum: 0 + selector: + type: object + serviceName: + type: string + template: + type: object + revisionHistoryLimit: + type: integer + minimum: 1 + versions: + - name: v1alpha1 + served: true + storage: true + # subresources describes the subresources for custom resources. + subresources: + # status enables the status subresource. + status: {} + # scale enables the scale subresource. + scale: + # specReplicasPath defines the JSONPath inside of a custom resource that corresponds to Scale.Spec.Replicas. + specReplicasPath: .spec.replicas + # statusReplicasPath defines the JSONPath inside of a custom resource that corresponds to Scale.Status.Replicas. + statusReplicasPath: .status.replicas + # labelSelectorPath defines the JSONPath inside of a custom resource that corresponds to Scale.Status.Selector. + labelSelectorPath: .status.labelSelector diff --git a/charts/tidb-operator/values.yaml b/charts/tidb-operator/values.yaml index 8ca2c066af9..62760227b35 100644 --- a/charts/tidb-operator/values.yaml +++ b/charts/tidb-operator/values.yaml @@ -54,6 +54,8 @@ controllerManager: # operator: Equal # value: tidb-operator # effect: "NoSchedule" + # features: + # - AdvancedStatefulSet=true scheduler: # With rbac.create=false, the user is responsible for creating this account diff --git a/cmd/controller-manager/main.go b/cmd/controller-manager/main.go index ac745548325..63ceea1c002 100644 --- a/cmd/controller-manager/main.go +++ b/cmd/controller-manager/main.go @@ -21,6 +21,9 @@ import ( "os" "time" + "github.com/cofyc/advanced-statefulset/pkg/apis/apps/v1alpha1/helper" + asclientset "github.com/cofyc/advanced-statefulset/pkg/client/clientset/versioned" + asinformers "github.com/cofyc/advanced-statefulset/pkg/client/informers/externalversions" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions" "github.com/pingcap/tidb-operator/pkg/controller" @@ -28,6 +31,7 @@ import ( "github.com/pingcap/tidb-operator/pkg/controller/backupschedule" "github.com/pingcap/tidb-operator/pkg/controller/restore" "github.com/pingcap/tidb-operator/pkg/controller/tidbcluster" + "github.com/pingcap/tidb-operator/pkg/features" "github.com/pingcap/tidb-operator/pkg/version" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -68,6 +72,7 @@ func init() { flag.DurationVar(&controller.ResyncDuration, "resync-duration", time.Duration(30*time.Second), "Resync time of informer") flag.BoolVar(&controller.TestMode, "test-mode", false, "whether tidb-operator run in test mode") flag.StringVar(&controller.TidbBackupManagerImage, "tidb-backup-manager-image", "pingcap/tidb-backup-manager:latest", "The image of backup manager tool") + features.DefaultFeatureGate.AddFlag(flag.CommandLine) flag.Parse() } @@ -105,9 +110,12 @@ func main() { if err != nil { glog.Fatalf("failed to get kubernetes Clientset: %v", err) } + asCli, err := asclientset.NewForConfig(cfg) var informerFactory informers.SharedInformerFactory var kubeInformerFactory kubeinformers.SharedInformerFactory + var asInformerFactory asinformers.SharedInformerFactory + if controller.ClusterScoped { informerFactory = informers.NewSharedInformerFactory(cli, controller.ResyncDuration) kubeInformerFactory = kubeinformers.NewSharedInformerFactory(kubeCli, controller.ResyncDuration) @@ -135,16 +143,27 @@ func main() { }, } - tcController := tidbcluster.NewController(kubeCli, cli, informerFactory, kubeInformerFactory, autoFailover, pdFailoverPeriod, tikvFailoverPeriod, tidbFailoverPeriod) - backupController := backup.NewController(kubeCli, cli, informerFactory, kubeInformerFactory) - restoreController := restore.NewController(kubeCli, cli, informerFactory, kubeInformerFactory) - bsController := backupschedule.NewController(kubeCli, cli, informerFactory, kubeInformerFactory) + var hijackKubeInformerFactory kubeinformers.SharedInformerFactory + var hijackKubeCli kubernetes.Interface + if features.DefaultFeatureGate.Enabled(features.AdvancedStatefulSet) { + // If AdvancedStatefulSet is enabled, we hijack client and informer factory to use AdvancedStatefulSet. + hijackKubeCli = helper.NewHijackClient(kubeCli, asCli) + hijackKubeInformerFactory = helper.NewHijackSharedInformerFactory(kubeInformerFactory, asInformerFactory) + } else { + hijackKubeCli = kubeCli + hijackKubeInformerFactory = kubeInformerFactory + } + + tcController := tidbcluster.NewController(hijackKubeCli, cli, informerFactory, hijackKubeInformerFactory, autoFailover, pdFailoverPeriod, tikvFailoverPeriod, tidbFailoverPeriod) + backupController := backup.NewController(hijackKubeCli, cli, informerFactory, hijackKubeInformerFactory) + restoreController := restore.NewController(hijackKubeCli, cli, informerFactory, hijackKubeInformerFactory) + bsController := backupschedule.NewController(hijackKubeCli, cli, informerFactory, hijackKubeInformerFactory) controllerCtx, cancel := context.WithCancel(context.Background()) defer cancel() // Start informer factories after all controller are initialized. informerFactory.Start(controllerCtx.Done()) - kubeInformerFactory.Start(controllerCtx.Done()) + hijackKubeInformerFactory.Start(controllerCtx.Done()) // Wait for all started informers' cache were synced. for v, synced := range informerFactory.WaitForCacheSync(wait.NeverStop) { @@ -152,7 +171,7 @@ func main() { glog.Fatalf("error syncing informer for %v", v) } } - for v, synced := range kubeInformerFactory.WaitForCacheSync(wait.NeverStop) { + for v, synced := range hijackKubeInformerFactory.WaitForCacheSync(wait.NeverStop) { if !synced { glog.Fatalf("error syncing informer for %v", v) } diff --git a/go.mod b/go.mod index d50ec8bb411..e15229ca186 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/ant31/crd-validation v0.0.0-20180702145049-30f8a35d0ac2 github.com/chai2010/gettext-go v0.0.0-20170215093142-bf70f2a70fb1 // indirect github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect + github.com/cofyc/advanced-statefulset v0.0.0-20191105120117-6c0ac9b9df68 github.com/coreos/go-semver v0.3.0 github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0 github.com/docker/go-connections v0.4.0 // indirect @@ -56,7 +57,7 @@ require ( github.com/sirupsen/logrus v1.4.2 github.com/soheilhy/cmux v0.1.4 // indirect github.com/spf13/cobra v0.0.5 - github.com/spf13/pflag v1.0.3 + github.com/spf13/pflag v1.0.5 github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 // indirect github.com/uber-go/atomic v0.0.0-00010101000000-000000000000 // indirect github.com/uber/jaeger-client-go v2.19.0+incompatible // indirect diff --git a/go.sum b/go.sum index 4e2f8e053d5..1f08d346475 100644 --- a/go.sum +++ b/go.sum @@ -81,6 +81,8 @@ github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0= +github.com/cofyc/advanced-statefulset v0.0.0-20191105120117-6c0ac9b9df68 h1:GsnTaZMgLtt0LR1TdL2GsWsEHDXp8/SbuWgpmVd1+k4= +github.com/cofyc/advanced-statefulset v0.0.0-20191105120117-6c0ac9b9df68/go.mod h1:4KdFa5r17cDhDS6PfRzw/S+Mwk2GwEjqE7aY6EGT0QE= github.com/container-storage-interface/spec v1.1.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= github.com/containerd/console v0.0.0-20170925154832-84eeaae905fa/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.0.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= @@ -648,6 +650,8 @@ github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.2.1/go.mod h1:P4AexN0a+C9tGAnUFNwDMYYZv3pjFuvmeiMyKRaNVlI= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/storageos/go-api v0.0.0-20180912212459-343b3eff91fc/go.mod h1:ZrLn+e0ZuF3Y65PNF6dIwbJPZqfmtCXxFm9ckv0agOY= @@ -814,6 +818,8 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w= +golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20170824195420-5d2fd3ccab98/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181003024731-2f84ea8ef872/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/pkg/features/features.go b/pkg/features/features.go index 4c40bccf507..32f2276627b 100644 --- a/pkg/features/features.go +++ b/pkg/features/features.go @@ -25,7 +25,8 @@ import ( var ( allFeatures = sets.NewString(StableScheduling) defaultFeatures = map[string]bool{ - StableScheduling: true, + StableScheduling: true, + AdvancedStatefulSet: false, } // DefaultFeatureGate is a shared global FeatureGate. DefaultFeatureGate FeatureGate = NewFeatureGate() @@ -34,6 +35,9 @@ var ( const ( // StableScheduling controls stable scheduling of TiDB members. StableScheduling string = "StableScheduling" + + // AdvancedStatefulSet controls whether to use AdvancedStatefulSet to manage pods + AdvancedStatefulSet string = "AdvancedStatefulSet" ) type FeatureGate interface { diff --git a/tests/actions.go b/tests/actions.go index dba50784186..73d9ad1459e 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -222,20 +222,21 @@ type event struct { var _ = OperatorActions(&operatorActions{}) type OperatorConfig struct { - Namespace string - ReleaseName string - Image string - Tag string - SchedulerImage string - SchedulerTag string - SchedulerFeatures []string - LogLevel string - WebhookServiceName string - WebhookSecretName string - WebhookConfigName string - Context *apimachinery.CertContext - ImagePullPolicy corev1.PullPolicy - TestMode bool + Namespace string + ReleaseName string + Image string + Tag string + SchedulerImage string + SchedulerTag string + SchedulerFeatures []string + ControllerManagerFeatures []string + LogLevel string + WebhookServiceName string + WebhookSecretName string + WebhookConfigName string + Context *apimachinery.CertContext + ImagePullPolicy corev1.PullPolicy + TestMode bool } type TidbClusterConfig struct { @@ -360,6 +361,9 @@ func (oi *OperatorConfig) OperatorHelmSetString(m map[string]string) string { if len(oi.SchedulerFeatures) > 0 { set["scheduler.features"] = fmt.Sprintf("{%s}", strings.Join(oi.SchedulerFeatures, ",")) } + if len(oi.ControllerManagerFeatures) > 0 { + set["controllerManager.features"] = fmt.Sprintf("{%s}", strings.Join(oi.ControllerManagerFeatures, ",")) + } arr := make([]string, 0, len(set)) for k, v := range set { diff --git a/tests/cmd/e2e/main.go b/tests/cmd/e2e/main.go index b6d81958937..fa26752626b 100644 --- a/tests/cmd/e2e/main.go +++ b/tests/cmd/e2e/main.go @@ -238,7 +238,10 @@ func newOperatorConfig() *tests.OperatorConfig { SchedulerFeatures: []string{ "StableScheduling=true", }, - LogLevel: "2", + ControllerManagerFeatures: []string{ + "AdvancedStatefulSet=true", + }, + LogLevel: "4", WebhookServiceName: "webhook-service", WebhookSecretName: "webhook-secret", WebhookConfigName: "webhook-config",