From c3ff1be896005ed1e83b558eaa3ed61ed2c73958 Mon Sep 17 00:00:00 2001 From: weekface Date: Thu, 12 Mar 2020 11:44:02 +0800 Subject: [PATCH] tls: TLS between TiDB components (#1870) --- charts/tidb-cluster/templates/_helpers.tpl | 18 ++--- .../templates/config/_prometheus-config.tpl | 24 +++---- .../templates/config/_pump-config.tpl | 2 +- .../templates/monitor-deployment.yaml | 12 ++-- .../templates/pump-statefulset.yaml | 4 +- .../tidb-cluster/templates/tidb-cluster.yaml | 5 +- charts/tidb-cluster/values.yaml | 23 ++++-- manifests/crd.yaml | 5 +- .../v1alpha1/defaulting/tidbcluster.go | 5 +- .../pingcap/v1alpha1/openapi_generated.go | 9 ++- pkg/apis/pingcap/v1alpha1/tidbcluster.go | 6 +- pkg/apis/pingcap/v1alpha1/types.go | 29 +++++++- .../pingcap/v1alpha1/zz_generated.deepcopy.go | 22 +++++- pkg/controller/tidb_control.go | 52 ++++++++------ .../tidbcluster/tidb_cluster_controller.go | 2 +- pkg/manager/member/pd_member_manager.go | 70 +------------------ pkg/manager/member/tidb_member_manager.go | 50 +------------ .../member/tidb_member_manager_test.go | 4 +- pkg/manager/member/tikv_member_manager.go | 39 +---------- pkg/manager/member/utils.go | 4 ++ pkg/monitor/monitor/template.go | 15 ++-- pkg/monitor/monitor/util.go | 9 +-- pkg/pdapi/pdapi.go | 12 ++-- pkg/util/crypto/certs.go | 17 ----- pkg/util/util.go | 12 ++++ tests/actions.go | 2 +- tests/e2e/tidbcluster/tidbcluster.go | 7 -- 27 files changed, 180 insertions(+), 279 deletions(-) diff --git a/charts/tidb-cluster/templates/_helpers.tpl b/charts/tidb-cluster/templates/_helpers.tpl index 837138eadc..766d43789b 100644 --- a/charts/tidb-cluster/templates/_helpers.tpl +++ b/charts/tidb-cluster/templates/_helpers.tpl @@ -28,7 +28,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- end -}} {{- define "cluster.scheme" -}} -{{ if .Values.enableTLSCluster }}https{{ else }}http{{ end }} +{{ if and .Values.tlsCluster .Values.tlsCluster.enabled }}https{{ else }}http{{ end }} {{- end -}} {{/* @@ -41,9 +41,9 @@ config-file: |- {{- if .Values.pd.config }} {{ .Values.pd.config | indent 2 }} {{- end -}} - {{- if .Values.enableTLSCluster }} + {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }} [security] - cacert-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + cacert-path = "/var/lib/pd-tls/ca.crt" cert-path = "/var/lib/pd-tls/tls.crt" key-path = "/var/lib/pd-tls/tls.key" {{- end -}} @@ -64,9 +64,9 @@ config-file: |- {{- if .Values.tikv.config }} {{ .Values.tikv.config | indent 2 }} {{- end -}} - {{- if .Values.enableTLSCluster }} + {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }} [security] - ca-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + ca-path = "/var/lib/tikv-tls/ca.crt" cert-path = "/var/lib/tikv-tls/tls.crt" key-path = "/var/lib/tikv-tls/tls.key" {{- end -}} @@ -91,11 +91,11 @@ config-file: |- {{- if .Values.tidb.config }} {{ .Values.tidb.config | indent 2 }} {{- end -}} - {{- if or .Values.enableTLSCluster (and .Values.tidb.tlsClient .Values.tidb.tlsClient.enabled) }} + {{- if or (and .Values.tlsCluster .Values.tlsCluster.enabled) (and .Values.tidb.tlsClient .Values.tidb.tlsClient.enabled) }} [security] {{- end -}} - {{- if .Values.enableTLSCluster }} - cluster-ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }} + cluster-ssl-ca = "/var/lib/tidb-tls/ca.crt" cluster-ssl-cert = "/var/lib/tidb-tls/tls.crt" cluster-ssl-key = "/var/lib/tidb-tls/tls.key" {{- end -}} @@ -122,7 +122,7 @@ Encapsulate pump configmap data for consistent digest calculation pump-config: |- {{- if .Values.binlog.pump.config }} {{ .Values.binlog.pump.config | indent 2 }} - {{- if .Values.enableTLSCluster }} + {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }} [security] ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" ssl-cert = "/var/lib/pump-tls/tls.crt" diff --git a/charts/tidb-cluster/templates/config/_prometheus-config.tpl b/charts/tidb-cluster/templates/config/_prometheus-config.tpl index 34f3f0f4d0..e9e9144095 100644 --- a/charts/tidb-cluster/templates/config/_prometheus-config.tpl +++ b/charts/tidb-cluster/templates/config/_prometheus-config.tpl @@ -21,10 +21,10 @@ scrape_configs: {{- end }} tls_config: insecure_skip_verify: true - {{- if .Values.enableTLSCluster }} - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - cert_file: /var/lib/pd-client-tls/tls.crt - key_file: /var/lib/pd-client-tls/tls.key + {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }} + ca_file: /var/lib/cluster-client-tls/ca.crt + cert_file: /var/lib/cluster-client-tls/tls.crt + key_file: /var/lib/cluster-client-tls/tls.key scheme: https {{- end }} relabel_configs: @@ -73,10 +73,10 @@ scrape_configs: {{- end }} tls_config: insecure_skip_verify: true - {{- if .Values.enableTLSCluster }} - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - cert_file: /var/lib/pd-client-tls/tls.crt - key_file: /var/lib/pd-client-tls/tls.key + {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }} + ca_file: /var/lib/cluster-client-tls/ca.crt + cert_file: /var/lib/cluster-client-tls/tls.crt + key_file: /var/lib/cluster-client-tls/tls.key scheme: https {{- end }} relabel_configs: @@ -127,10 +127,10 @@ scrape_configs: insecure_skip_verify: true # TiKV doesn't support scheme https for now. # And we should fix it after TiKV fix this issue: https://github.com/tikv/tikv/issues/5340 -# {{- if .Values.enableTLSCluster }} -# ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt -# cert_file: /var/lib/pd-client-tls/tls.crt -# key_file: /var/lib/pd-client-tls/tls.key +# {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }} +# ca_file: /var/lib/cluster-client-tls/ca.crt +# cert_file: /var/lib/cluster-client-tls/tls.crt +# key_file: /var/lib/cluster-client-tls/tls.key # scheme: https # {{- end }} relabel_configs: diff --git a/charts/tidb-cluster/templates/config/_pump-config.tpl b/charts/tidb-cluster/templates/config/_pump-config.tpl index 9c20bb8020..07795a899e 100644 --- a/charts/tidb-cluster/templates/config/_pump-config.tpl +++ b/charts/tidb-cluster/templates/config/_pump-config.tpl @@ -35,7 +35,7 @@ sync-log = {{ .Values.binlog.pump.syncLog | default true }} # write-buffer = 67108864 # write-L0-pause-trigger = 24 # write-L0-slowdown-trigger = 17 -{{ if .Values.enableTLSCluster }} +{{ if and .Values.tlsCluster .Values.tlsCluster.enabled }} [security] # Path of file that contains list of trusted SSL CAs for connection with cluster components. ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" diff --git a/charts/tidb-cluster/templates/monitor-deployment.yaml b/charts/tidb-cluster/templates/monitor-deployment.yaml index 3b7c82c7f4..b297a1553c 100644 --- a/charts/tidb-cluster/templates/monitor-deployment.yaml +++ b/charts/tidb-cluster/templates/monitor-deployment.yaml @@ -134,9 +134,9 @@ spec: - name: prometheus-rules mountPath: /prometheus-rules readOnly: false - {{- if .Values.enableTLSCluster }} - - name: tls-pd-client - mountPath: /var/lib/pd-client-tls + {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }} + - name: cluster-client-tls + mountPath: /var/lib/cluster-client-tls readOnly: true {{- end }} {{- if .Values.monitor.grafana.create }} @@ -241,11 +241,11 @@ spec: name: prometheus-rules - emptyDir: {} name: grafana-dashboard - {{- if .Values.enableTLSCluster }} - - name: tls-pd-client + {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }} + - name: cluster-client-tls secret: defaultMode: 420 - secretName: {{ .Release.Name }}-pd-client + secretName: {{ .Release.Name }}-cluster-client-secret {{- end }} {{- if .Values.monitor.tolerations }} tolerations: diff --git a/charts/tidb-cluster/templates/pump-statefulset.yaml b/charts/tidb-cluster/templates/pump-statefulset.yaml index f1bd44510e..f73fa3c06d 100644 --- a/charts/tidb-cluster/templates/pump-statefulset.yaml +++ b/charts/tidb-cluster/templates/pump-statefulset.yaml @@ -55,7 +55,7 @@ spec: mountPath: /data - name: config mountPath: /etc/pump - {{- if .Values.enableTLSCluster }} + {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }} - name: pump-tls mountPath: /var/lib/pump-tls readOnly: true @@ -78,7 +78,7 @@ spec: items: - key: pump-config path: pump.toml - {{- if .Values.enableTLSCluster }} + {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }} - name: pump-tls secret: secretName: {{ include "pump.tlsSecretName" . }} diff --git a/charts/tidb-cluster/templates/tidb-cluster.yaml b/charts/tidb-cluster/templates/tidb-cluster.yaml index 3a94ea400b..1567b6d9df 100644 --- a/charts/tidb-cluster/templates/tidb-cluster.yaml +++ b/charts/tidb-cluster/templates/tidb-cluster.yaml @@ -21,7 +21,10 @@ spec: pvReclaimPolicy: {{ .Values.pvReclaimPolicy }} enablePVReclaim: {{ .Values.enablePVReclaim }} timezone: {{ .Values.timezone | default "UTC" }} - enableTLSCluster: {{ .Values.enableTLSCluster | default false }} +{{- if .Values.tlsCluster }} + tlsCluster: +{{ toYaml .Values.tlsCluster | indent 4 }} +{{- end }} services: {{ toYaml .Values.services | indent 4 }} schedulerName: {{ .Values.schedulerName | default "default-scheduler" }} diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index 82f1aa8188..ebe10a85b4 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -55,10 +55,23 @@ discovery: # if the ConfigMap was not changed. enableConfigMapRollout: true -# Whether enable TLS connections between server nodes. -# When enabled, PD/TiDB/TiKV/PUMP will use TLS encrypted connections to transfer data between each node, -# certificates will be generated automatically (if not already present). -enableTLSCluster: false +# Whether enable the TLS connection between TiDB server components +tlsCluster: + # The steps to enable this feature: + # 1. Generate TiDB server components certificates and a client-side certifiacete for them. + # There are multiple ways to generate these certificates: + # - user-provided certificates: https://pingcap.com/docs/stable/how-to/secure/generate-self-signed-certificates/ + # - use the K8s built-in certificate signing system signed certificates: https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/ + # - or use cert-manager signed certificates: https://cert-manager.io/ + # 2. Create one secret object for one component which contains the certificates created above. + # The name of this Secret must be: --cluster-secret. + # For PD: kubectl create secret generic -pd-cluster-secret --namespace= --from-file=tls.crt= --from-file=tls.key= --from-file=ca.crt= + # For TiKV: kubectl create secret generic -tikv-cluster-secret --namespace= --from-file=tls.crt= --from-file=tls.key= --from-file=ca.crt= + # For TiDB: kubectl create secret generic -tidb-cluster-secret --namespace= --from-file=tls.crt= --from-file=tls.key= --from-file=ca.crt= + # For Client: kubectl create secret generic -cluster-client-secret --namespace= --from-file=tls.crt= --from-file=tls.key= --from-file=ca.crt= + # Same for other components. + # 3. Then create the TiDB cluster with `tlsCluster.enabled` set to `true`. + enabled: false pd: # Please refer to https://github.com/pingcap/pd/blob/master/conf/config.toml for the default @@ -618,7 +631,7 @@ binlog: # pump configurations (change to the tags of your pump version), # just follow the format in the file and configure in the 'config' section # as below if you want to customize any configuration. - # [security] section will be generated automatically if enableTLSCluster is set to true so users do not need to configure it. + # [security] section will be generated automatically if tlsCluster.enabled is set to true so users do not need to configure it. # config: | # gc = 7 # heartbeat-interval = 2 diff --git a/manifests/crd.yaml b/manifests/crd.yaml index a9fd1ffbb0..5ad1b2b840 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -675,10 +675,6 @@ spec: description: 'Whether enable PVC reclaim for orphan PVC left by statefulset scale-in Optional: Defaults to false' type: boolean - enableTLSCluster: - description: 'Enable TLS connection between TiDB server components Optional: - Defaults to false' - type: boolean helper: description: HelperSpec contains details of helper component properties: @@ -6055,6 +6051,7 @@ spec: timezone: description: 'Time zone of TiDB cluster Pods Optional: Defaults to UTC' type: string + tlsCluster: {} tolerations: description: Base tolerations of TiDB cluster Pods, components may add more tolerations upon this respectively diff --git a/pkg/apis/pingcap/v1alpha1/defaulting/tidbcluster.go b/pkg/apis/pingcap/v1alpha1/defaulting/tidbcluster.go index 2c2fa69f38..3d8e1cadac 100644 --- a/pkg/apis/pingcap/v1alpha1/defaulting/tidbcluster.go +++ b/pkg/apis/pingcap/v1alpha1/defaulting/tidbcluster.go @@ -40,9 +40,8 @@ func setTidbClusterSpecDefault(tc *v1alpha1.TidbCluster) { if string(tc.Spec.ImagePullPolicy) == "" { tc.Spec.ImagePullPolicy = corev1.PullIfNotPresent } - if tc.Spec.EnableTLSCluster == nil { - d := false - tc.Spec.EnableTLSCluster = &d + if tc.Spec.TLSCluster == nil { + tc.Spec.TLSCluster = &v1alpha1.TLSCluster{Enabled: false} } if tc.Spec.EnablePVReclaim == nil { d := false diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index 7f74ad0157..d47a960349 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -6104,11 +6104,10 @@ func schema_pkg_apis_pingcap_v1alpha1_TidbClusterSpec(ref common.ReferenceCallba Format: "", }, }, - "enableTLSCluster": { + "tlsCluster": { SchemaProps: spec.SchemaProps{ - Description: "Enable TLS connection between TiDB server components Optional: Defaults to false", - Type: []string{"boolean"}, - Format: "", + Description: "Whether enable the TLS connection between TiDB server components Optional: Defaults to nil", + Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TLSCluster"), }, }, "hostNetwork": { @@ -6186,7 +6185,7 @@ func schema_pkg_apis_pingcap_v1alpha1_TidbClusterSpec(ref common.ReferenceCallba }, }, Dependencies: []string{ - "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.HelperSpec", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.PDSpec", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.PumpSpec", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiDBSpec", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVSpec", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.Toleration"}, + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.HelperSpec", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.PDSpec", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.PumpSpec", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TLSCluster", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiDBSpec", "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiKVSpec", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.Toleration"}, } } diff --git a/pkg/apis/pingcap/v1alpha1/tidbcluster.go b/pkg/apis/pingcap/v1alpha1/tidbcluster.go index ce3ce7f270..a2f7449f21 100644 --- a/pkg/apis/pingcap/v1alpha1/tidbcluster.go +++ b/pkg/apis/pingcap/v1alpha1/tidbcluster.go @@ -300,11 +300,7 @@ func (tc *TidbCluster) GetClusterID() string { } func (tc *TidbCluster) IsTLSClusterEnabled() bool { - enableTLCluster := tc.Spec.EnableTLSCluster - if enableTLCluster == nil { - return defaultEnableTLSCluster - } - return *enableTLCluster + return tc.Spec.TLSCluster != nil && tc.Spec.TLSCluster.Enabled } func (tc *TidbCluster) Scheme() string { diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index 1dfaeadb84..82363dba05 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -150,10 +150,10 @@ type TidbClusterSpec struct { // +optional EnablePVReclaim *bool `json:"enablePVReclaim,omitempty"` - // Enable TLS connection between TiDB server components - // Optional: Defaults to false + // Whether enable the TLS connection between TiDB server components + // Optional: Defaults to nil // +optional - EnableTLSCluster *bool `json:"enableTLSCluster,omitempty"` + TLSCluster *TLSCluster `json:"tlsCluster,omitempty"` // Whether Hostnetwork is enabled for TiDB cluster Pods // Optional: Defaults to false @@ -618,6 +618,29 @@ type TiDBTLSClient struct { Enabled bool `json:"enabled,omitempty"` } +// TLSCluster can enable TLS connection between TiDB server components +// https://pingcap.com/docs/stable/how-to/secure/enable-tls-between-components/ +type TLSCluster struct { + // Enable mutual TLS authentication among TiDB components + // Once enabled, the mutual authentication applies to all components, + // and it does not support applying to only part of the components. + // The steps to enable this feature: + // 1. Generate TiDB server components certificates and a client-side certifiacete for them. + // There are multiple ways to generate these certificates: + // - user-provided certificates: https://pingcap.com/docs/stable/how-to/secure/generate-self-signed-certificates/ + // - use the K8s built-in certificate signing system signed certificates: https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/ + // - or use cert-manager signed certificates: https://cert-manager.io/ + // 2. Create one secret object for one component which contains the certificates created above. + // The name of this Secret must be: --cluster-secret. + // For PD: kubectl create secret generic -pd-cluster-secret --namespace= --from-file=tls.crt= --from-file=tls.key= --from-file=ca.crt= + // For TiKV: kubectl create secret generic -tikv-cluster-secret --namespace= --from-file=tls.crt= --from-file=tls.key= --from-file=ca.crt= + // For TiDB: kubectl create secret generic -tidb-cluster-secret --namespace= --from-file=tls.crt= --from-file=tls.key= --from-file=ca.crt= + // For Client: kubectl create secret generic -cluster-client-secret --namespace= --from-file=tls.crt= --from-file=tls.key= --from-file=ca.crt= + // Same for other components. + // +optional + Enabled bool `json:"enabled,omitempty"` +} + // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go index f660699e83..e8ed1ac39c 100644 --- a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go @@ -2215,6 +2215,22 @@ func (in StringSlice) DeepCopy() StringSlice { return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSCluster) DeepCopyInto(out *TLSCluster) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSCluster. +func (in *TLSCluster) DeepCopy() *TLSCluster { + if in == nil { + return nil + } + out := new(TLSCluster) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TiDBAccessConfig) DeepCopyInto(out *TiDBAccessConfig) { *out = *in @@ -3999,9 +4015,9 @@ func (in *TidbClusterSpec) DeepCopyInto(out *TidbClusterSpec) { *out = new(bool) **out = **in } - if in.EnableTLSCluster != nil { - in, out := &in.EnableTLSCluster, &out.EnableTLSCluster - *out = new(bool) + if in.TLSCluster != nil { + in, out := &in.TLSCluster, &out.TLSCluster + *out = new(TLSCluster) **out = **in } if in.HostNetwork != nil { diff --git a/pkg/controller/tidb_control.go b/pkg/controller/tidb_control.go index 75c6ac975c..bf6282cb3a 100644 --- a/pkg/controller/tidb_control.go +++ b/pkg/controller/tidb_control.go @@ -15,16 +15,19 @@ package controller import ( "crypto/tls" + "crypto/x509" "encoding/json" "fmt" - "io/ioutil" - "net/http" - "time" - "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/httputil" - certutil "github.com/pingcap/tidb-operator/pkg/util/crypto" + "github.com/pingcap/tidb-operator/pkg/util" "github.com/pingcap/tidb/config" + "io/ioutil" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "net/http" + "time" ) const ( @@ -51,24 +54,33 @@ type TiDBControlInterface interface { // defaultTiDBControl is default implementation of TiDBControlInterface. type defaultTiDBControl struct { httpClient *http.Client + kubeCli kubernetes.Interface } // NewDefaultTiDBControl returns a defaultTiDBControl instance -func NewDefaultTiDBControl() TiDBControlInterface { - return &defaultTiDBControl{httpClient: &http.Client{Timeout: timeout}} +func NewDefaultTiDBControl(kubeCli kubernetes.Interface) TiDBControlInterface { + return &defaultTiDBControl{httpClient: &http.Client{Timeout: timeout}, kubeCli: kubeCli} } -func (tdc *defaultTiDBControl) useTLSHTTPClient(enableTLS bool) error { - if enableTLS { - rootCAs, err := certutil.ReadCACerts() - if err != nil { - return err - } - config := &tls.Config{ - RootCAs: rootCAs, - } - tdc.httpClient.Transport = &http.Transport{TLSClientConfig: config} +func (tdc *defaultTiDBControl) useTLSHTTPClient(tc *v1alpha1.TidbCluster) error { + if !tc.IsTLSClusterEnabled() { + return nil + } + + tcName := tc.Name + ns := tc.Namespace + secretName := util.ClusterClientTLSSecretName(tcName) + secret, err := tdc.kubeCli.CoreV1().Secrets(ns).Get(secretName, metav1.GetOptions{}) + if err != nil { + return err + } + + rootCAs := x509.NewCertPool() + rootCAs.AppendCertsFromPEM(secret.Data[v1.ServiceAccountRootCAKey]) + config := &tls.Config{ + RootCAs: rootCAs, } + tdc.httpClient.Transport = &http.Transport{TLSClientConfig: config} return nil } @@ -77,7 +89,7 @@ func (tdc *defaultTiDBControl) GetHealth(tc *v1alpha1.TidbCluster, ordinal int32 ns := tc.GetNamespace() scheme := tc.Scheme() - if err := tdc.useTLSHTTPClient(tc.IsTLSClusterEnabled()); err != nil { + if err := tdc.useTLSHTTPClient(tc); err != nil { return false, err } @@ -91,7 +103,7 @@ func (tdc *defaultTiDBControl) GetInfo(tc *v1alpha1.TidbCluster, ordinal int32) tcName := tc.GetName() ns := tc.GetNamespace() scheme := tc.Scheme() - if err := tdc.useTLSHTTPClient(tc.IsTLSClusterEnabled()); err != nil { + if err := tdc.useTLSHTTPClient(tc); err != nil { return nil, err } @@ -126,7 +138,7 @@ func (tdc *defaultTiDBControl) GetSettings(tc *v1alpha1.TidbCluster, ordinal int tcName := tc.GetName() ns := tc.GetNamespace() scheme := tc.Scheme() - if err := tdc.useTLSHTTPClient(tc.IsTLSClusterEnabled()); err != nil { + if err := tdc.useTLSHTTPClient(tc); err != nil { return nil, err } diff --git a/pkg/controller/tidbcluster/tidb_cluster_controller.go b/pkg/controller/tidbcluster/tidb_cluster_controller.go index 3de44bd02b..0c0963c779 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_controller.go +++ b/pkg/controller/tidbcluster/tidb_cluster_controller.go @@ -96,7 +96,7 @@ func NewController( tcControl := controller.NewRealTidbClusterControl(cli, tcInformer.Lister(), recorder) pdControl := pdapi.NewDefaultPDControl(kubeCli) - tidbControl := controller.NewDefaultTiDBControl() + tidbControl := controller.NewDefaultTiDBControl(kubeCli) cmControl := controller.NewRealConfigMapControl(kubeCli, cmInformer.Lister(), recorder) setControl := controller.NewRealStatefuSetControl(kubeCli, setInformer.Lister(), recorder) svcControl := controller.NewRealServiceControl(kubeCli, svcInformer.Lister(), recorder) diff --git a/pkg/manager/member/pd_member_manager.go b/pkg/manager/member/pd_member_manager.go index f6ad708588..0e430bffa4 100644 --- a/pkg/manager/member/pd_member_manager.go +++ b/pkg/manager/member/pd_member_manager.go @@ -206,16 +206,6 @@ func (pmm *pdMemberManager) syncPDStatefulSetForTidbCluster(tc *v1alpha1.TidbClu if err != nil { return err } - if tc.IsTLSClusterEnabled() { - err := pmm.syncPDServerCerts(tc) - if err != nil { - return err - } - err = pmm.syncPDClientCerts(tc) - if err != nil { - return err - } - } if err := pmm.setControl.CreateStatefulSet(tc, newPDSet); err != nil { return err } @@ -260,62 +250,6 @@ func (pmm *pdMemberManager) syncPDStatefulSetForTidbCluster(tc *v1alpha1.TidbClu return updateStatefulSet(pmm.setControl, tc, newPDSet, oldPDSet) } -func (pmm *pdMemberManager) syncPDClientCerts(tc *v1alpha1.TidbCluster) error { - ns := tc.GetNamespace() - tcName := tc.GetName() - commonName := fmt.Sprintf("%s-pd-client", tcName) - - hostList := []string{ - commonName, - } - - certOpts := &controller.TiDBClusterCertOptions{ - Namespace: ns, - Instance: tcName, - CommonName: commonName, - HostList: hostList, - Component: "pd", - Suffix: "pd-client", - } - - return pmm.certControl.Create(controller.GetOwnerRef(tc), certOpts) -} - -func (pmm *pdMemberManager) syncPDServerCerts(tc *v1alpha1.TidbCluster) error { - ns := tc.GetNamespace() - tcName := tc.GetName() - svcName := controller.PDMemberName(tcName) - peerName := controller.PDPeerMemberName(tcName) - - if pmm.certControl.CheckSecret(ns, svcName) { - return nil - } - - hostList := []string{ - svcName, - peerName, - fmt.Sprintf("%s.%s", svcName, ns), - fmt.Sprintf("%s.%s", peerName, ns), - fmt.Sprintf("*.%s.%s.svc", peerName, ns), - } - - ipList := []string{ - "127.0.0.1", "::1", // able to access https endpoint via loopback network - } - - certOpts := &controller.TiDBClusterCertOptions{ - Namespace: ns, - Instance: tcName, - CommonName: svcName, - HostList: hostList, - IPList: ipList, - Component: "pd", - Suffix: "pd", - } - - return pmm.certControl.Create(controller.GetOwnerRef(tc), certOpts) -} - func (pmm *pdMemberManager) syncTidbClusterStatus(tc *v1alpha1.TidbCluster, set *apps.StatefulSet) error { ns := tc.GetNamespace() tcName := tc.GetName() @@ -581,7 +515,7 @@ func getNewPDSetForTidbCluster(tc *v1alpha1.TidbCluster, cm *corev1.ConfigMap) ( vols = append(vols, corev1.Volume{ Name: "pd-tls", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: controller.PDMemberName(tcName), + SecretName: util.ClusterTLSSecretName(tc.Name, label.PDLabelVal), }, }, }) @@ -724,7 +658,7 @@ func getPDConfigMap(tc *v1alpha1.TidbCluster) (*corev1.ConfigMap, error) { if config.Security == nil { config.Security = &v1alpha1.PDSecurityConfig{} } - config.Security.CAPath = serviceAccountCAPath + config.Security.CAPath = path.Join(pdClusterCertPath, tlsSecretRootCAKey) config.Security.CertPath = path.Join(pdClusterCertPath, corev1.TLSCertKey) config.Security.KeyPath = path.Join(pdClusterCertPath, corev1.TLSPrivateKeyKey) } diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index e31c0c8ca8..74cca48f57 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -180,12 +180,6 @@ func (tmm *tidbMemberManager) syncTiDBStatefulSetForTidbCluster(tc *v1alpha1.Tid if err != nil { return err } - if tc.IsTLSClusterEnabled() { - err := tmm.syncTiDBClusterCerts(tc) - if err != nil { - return err - } - } err = tmm.setControl.CreateStatefulSet(tc, newTiDBSet) if err != nil { return err @@ -220,46 +214,6 @@ func (tmm *tidbMemberManager) syncTiDBStatefulSetForTidbCluster(tc *v1alpha1.Tid return updateStatefulSet(tmm.setControl, tc, newTiDBSet, oldTiDBSet) } -// syncTiDBClusterCerts creates the cert pair for TiDB if not exist, the cert -// pair is used to communicate with other TiDB components, like TiKVs and PDs -func (tmm *tidbMemberManager) syncTiDBClusterCerts(tc *v1alpha1.TidbCluster) error { - ns := tc.GetNamespace() - tcName := tc.GetName() - svcName := controller.TiDBMemberName(tcName) - peerName := controller.TiDBPeerMemberName(tcName) - - if tmm.certControl.CheckSecret(ns, svcName) { - return nil - } - - hostList := []string{ - svcName, - peerName, - fmt.Sprintf("%s.%s", svcName, ns), - fmt.Sprintf("%s.%s.svc", svcName, ns), - fmt.Sprintf("%s.%s", peerName, ns), - fmt.Sprintf("%s.%s.svc", peerName, ns), - fmt.Sprintf("*.%s.%s", peerName, ns), - fmt.Sprintf("*.%s.%s.svc", peerName, ns), - } - - ipList := []string{ - "127.0.0.1", "::1", // able to access https endpoint via loopback network - } - - certOpts := &controller.TiDBClusterCertOptions{ - Namespace: ns, - Instance: tcName, - CommonName: svcName, - HostList: hostList, - IPList: ipList, - Component: "tidb", - Suffix: "tidb", - } - - return tmm.certControl.Create(controller.GetOwnerRef(tc), certOpts) -} - func (tmm *tidbMemberManager) syncTiDBService(tc *v1alpha1.TidbCluster) error { newSvc := getNewTiDBServiceOrNil(tc) @@ -349,7 +303,7 @@ func getTiDBConfigMap(tc *v1alpha1.TidbCluster) (*corev1.ConfigMap, error) { if config.Security == nil { config.Security = &v1alpha1.Security{} } - config.Security.ClusterSSLCA = pointer.StringPtr(serviceAccountCAPath) + config.Security.ClusterSSLCA = pointer.StringPtr(path.Join(clusterCertPath, tlsSecretRootCAKey)) config.Security.ClusterSSLCert = pointer.StringPtr(path.Join(clusterCertPath, corev1.TLSCertKey)) config.Security.ClusterSSLKey = pointer.StringPtr(path.Join(clusterCertPath, corev1.TLSPrivateKeyKey)) } @@ -544,7 +498,7 @@ func getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbCluster, cm *corev1.ConfigMap) vols = append(vols, corev1.Volume{ Name: "tidb-tls", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: controller.TiDBMemberName(tcName), + SecretName: util.ClusterTLSSecretName(tcName, label.TiDBLabelVal), }, }, }) diff --git a/pkg/manager/member/tidb_member_manager_test.go b/pkg/manager/member/tidb_member_manager_test.go index 550dadea1b..88e46ae063 100644 --- a/pkg/manager/member/tidb_member_manager_test.go +++ b/pkg/manager/member/tidb_member_manager_test.go @@ -1628,7 +1628,7 @@ func TestGetTiDBConfigMap(t *testing.T) { Namespace: "ns", }, Spec: v1alpha1.TidbClusterSpec{ - EnableTLSCluster: pointer.BoolPtr(true), + TLSCluster: &v1alpha1.TLSCluster{Enabled: true}, TiDB: v1alpha1.TiDBSpec{ ComponentSpec: v1alpha1.ComponentSpec{ ConfigUpdateStrategy: &updateStrategy, @@ -1669,7 +1669,7 @@ func TestGetTiDBConfigMap(t *testing.T) { ssl-ca = "/var/lib/tidb-server-tls/ca.crt" ssl-cert = "/var/lib/tidb-server-tls/tls.crt" ssl-key = "/var/lib/tidb-server-tls/tls.key" - cluster-ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + cluster-ssl-ca = "/var/lib/tidb-tls/ca.crt" cluster-ssl-cert = "/var/lib/tidb-tls/tls.crt" cluster-ssl-key = "/var/lib/tidb-tls/tls.key" `, diff --git a/pkg/manager/member/tikv_member_manager.go b/pkg/manager/member/tikv_member_manager.go index 0baac4124f..29d3c71224 100644 --- a/pkg/manager/member/tikv_member_manager.go +++ b/pkg/manager/member/tikv_member_manager.go @@ -15,6 +15,7 @@ package member import ( "fmt" + "github.com/pingcap/tidb-operator/pkg/util" "path" "reflect" "regexp" @@ -193,12 +194,6 @@ func (tkmm *tikvMemberManager) syncStatefulSetForTidbCluster(tc *v1alpha1.TidbCl if err != nil { return err } - if tc.IsTLSClusterEnabled() { - err := tkmm.syncTiKVServerCerts(tc) - if err != nil { - return err - } - } err = tkmm.setControl.CreateStatefulSet(tc, newSet) if err != nil { return err @@ -236,34 +231,6 @@ func (tkmm *tikvMemberManager) syncStatefulSetForTidbCluster(tc *v1alpha1.TidbCl return updateStatefulSet(tkmm.setControl, tc, newSet, oldSet) } -func (tkmm *tikvMemberManager) syncTiKVServerCerts(tc *v1alpha1.TidbCluster) error { - ns := tc.GetNamespace() - tcName := tc.GetName() - svcName := controller.TiKVMemberName(tcName) - peerName := controller.TiKVPeerMemberName(tcName) - - if tkmm.certControl.CheckSecret(ns, svcName) { - return nil - } - - hostList := []string{ - peerName, - fmt.Sprintf("%s.%s", peerName, ns), - fmt.Sprintf("*.%s.%s.svc", peerName, ns), - } - - certOpts := &controller.TiDBClusterCertOptions{ - Namespace: ns, - Instance: tcName, - CommonName: svcName, - HostList: hostList, - Component: "tikv", - Suffix: "tikv", - } - - return tkmm.certControl.Create(controller.GetOwnerRef(tc), certOpts) -} - func (tkmm *tikvMemberManager) syncTiKVConfigMap(tc *v1alpha1.TidbCluster, set *apps.StatefulSet) (*corev1.ConfigMap, error) { // For backward compatibility, only sync tidb configmap when .tikv.config is non-nil if tc.Spec.TiKV.Config == nil { @@ -366,7 +333,7 @@ func getNewTiKVSetForTidbCluster(tc *v1alpha1.TidbCluster, cm *corev1.ConfigMap) vols = append(vols, corev1.Volume{ Name: "tikv-tls", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: controller.TiKVMemberName(tcName), + SecretName: util.ClusterTLSSecretName(tc.Name, label.TiKVLabelVal), }, }, }) @@ -538,7 +505,7 @@ func getTikVConfigMap(tc *v1alpha1.TidbCluster) (*corev1.ConfigMap, error) { if config.Security == nil { config.Security = &v1alpha1.TiKVSecurityConfig{} } - config.Security.CAPath = serviceAccountCAPath + config.Security.CAPath = path.Join(tikvClusterCertPath, tlsSecretRootCAKey) config.Security.CertPath = path.Join(tikvClusterCertPath, corev1.TLSCertKey) config.Security.KeyPath = path.Join(tikvClusterCertPath, corev1.TLSPrivateKeyKey) } diff --git a/pkg/manager/member/utils.go b/pkg/manager/member/utils.go index 7c451d26af..04b66bb88d 100644 --- a/pkg/manager/member/utils.go +++ b/pkg/manager/member/utils.go @@ -306,3 +306,7 @@ func updateStatefulSet(setCtl controller.StatefulSetControlInterface, tc *v1alph return nil } + +func clusterSecretName(tc *v1alpha1.TidbCluster, component string) string { + return fmt.Sprintf("%s-%s-cluster-secret", tc.Name, component) +} diff --git a/pkg/monitor/monitor/template.go b/pkg/monitor/monitor/template.go index 49bea64112..eff7051488 100644 --- a/pkg/monitor/monitor/template.go +++ b/pkg/monitor/monitor/template.go @@ -14,12 +14,14 @@ package monitor import ( - "time" - + "github.com/pingcap/tidb-operator/pkg/util" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/config" "gopkg.in/yaml.v2" + corev1 "k8s.io/api/core/v1" "k8s.io/klog" + "path" + "time" ) const ( @@ -32,9 +34,6 @@ const ( podNameLabel = "__meta_kubernetes_pod_name" nodeNameLabel = "__meta_kubernetes_pod_node_name" podIPLabel = "__meta_kubernetes_pod_ip" - caFilePath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - certFilePath = "/var/lib/pd-client-tls/tls.crt" - keyFilePath = "/var/lib/pd-client-tls/tls.key" ) var ( @@ -249,9 +248,9 @@ func addTlsConfig(pc *config.Config) { // And we should fix it after TiKV fix this issue: https://github.com/tikv/tikv/issues/5340 if sconfig.JobName == "pd" || sconfig.JobName == "tidb" { sconfig.HTTPClientConfig.TLSConfig = config.TLSConfig{ - CAFile: caFilePath, - CertFile: certFilePath, - KeyFile: keyFilePath, + CAFile: path.Join(util.ClusterClientTLSPath, corev1.ServiceAccountRootCAKey), + CertFile: path.Join(util.ClusterClientTLSPath, corev1.TLSCertKey), + KeyFile: path.Join(util.ClusterClientTLSPath, corev1.TLSPrivateKeyKey), } pc.ScrapeConfigs[id] = sconfig sconfig.HTTPClientConfig.XXX["scheme"] = "https" diff --git a/pkg/monitor/monitor/util.go b/pkg/monitor/monitor/util.go index e319f41df2..7aa15cd3b3 100644 --- a/pkg/monitor/monitor/util.go +++ b/pkg/monitor/monitor/util.go @@ -16,6 +16,7 @@ package monitor import ( "encoding/json" "fmt" + "github.com/pingcap/tidb-operator/pkg/util" "strconv" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" @@ -426,8 +427,8 @@ func getMonitorPrometheusContainer(monitor *v1alpha1.TidbMonitor, tc *v1alpha1.T if tc.IsTLSClusterEnabled() { c.VolumeMounts = append(c.VolumeMounts, core.VolumeMount{ - Name: "tls-pd-client", - MountPath: "/var/lib/pd-client-tls", + Name: "cluster-client-tls", + MountPath: util.ClusterClientTLSPath, ReadOnly: true, }) } @@ -635,10 +636,10 @@ func getMonitorVolumes(config *core.ConfigMap, monitor *v1alpha1.TidbMonitor, tc if tc.IsTLSClusterEnabled() { defaultMode := int32(420) tlsPDClient := core.Volume{ - Name: "tls-pd-client", + Name: "cluster-client-tls", VolumeSource: core.VolumeSource{ Secret: &core.SecretVolumeSource{ - SecretName: fmt.Sprintf("%s-pd-client", tc.Name), + SecretName: util.ClusterClientTLSSecretName(tc.Name), DefaultMode: &defaultMode, }, }, diff --git a/pkg/pdapi/pdapi.go b/pkg/pdapi/pdapi.go index 2702cc65ab..32b9371c27 100644 --- a/pkg/pdapi/pdapi.go +++ b/pkg/pdapi/pdapi.go @@ -19,6 +19,7 @@ import ( "crypto/x509" "encoding/json" "fmt" + "github.com/pingcap/tidb-operator/pkg/util" "io/ioutil" "net/http" "strings" @@ -33,7 +34,6 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/pd/pkg/typeutil" "github.com/pingcap/tidb-operator/pkg/httputil" - certutil "github.com/pingcap/tidb-operator/pkg/util/crypto" types "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) @@ -66,23 +66,19 @@ func NewDefaultPDControl(kubeCli kubernetes.Interface) PDControlInterface { // GetTLSConfig returns *tls.Config for given TiDB cluster. // It loads in-cluster root ca if caCert is empty. func GetTLSConfig(kubeCli kubernetes.Interface, namespace Namespace, tcName string, caCert []byte) (*tls.Config, error) { - secretName := fmt.Sprintf("%s-pd-client", tcName) + secretName := util.ClusterClientTLSSecretName(tcName) secret, err := kubeCli.CoreV1().Secrets(string(namespace)).Get(secretName, types.GetOptions{}) if err != nil { return nil, fmt.Errorf("unable to load certificates from secret %s/%s: %v", namespace, secretName, err) } - var rootCAs *x509.CertPool + rootCAs := x509.NewCertPool() var tlsCert tls.Certificate if len(caCert) > 0 { - rootCAs = x509.NewCertPool() rootCAs.AppendCertsFromPEM(caCert) } else { - rootCAs, err = certutil.ReadCACerts() - if err != nil { - return nil, err - } + rootCAs.AppendCertsFromPEM(secret.Data[v1.ServiceAccountRootCAKey]) } clientCert, certExists := secret.Data[v1.TLSCertKey] diff --git a/pkg/util/crypto/certs.go b/pkg/util/crypto/certs.go index 318d9b3187..c149ef5865 100644 --- a/pkg/util/crypto/certs.go +++ b/pkg/util/crypto/certs.go @@ -16,11 +16,9 @@ package crypto import ( "crypto/rand" "crypto/rsa" - "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/pem" - "fmt" "io/ioutil" "net" @@ -105,18 +103,3 @@ func ReadCACerts() (*x509.CertPool, error) { } return rootCAs, nil } - -func LoadCerts(cert []byte, key []byte) (*x509.CertPool, tls.Certificate, error) { - if cert == nil || key == nil { - return nil, tls.Certificate{}, fmt.Errorf("fail to load certs, cert and key can not be empty") - } - - rootCAs, err := ReadCACerts() - if err != nil { - return rootCAs, tls.Certificate{}, err - } - - // load client cert - tlsCert, err := tls.X509KeyPair(cert, key) - return rootCAs, tlsCert, err -} diff --git a/pkg/util/util.go b/pkg/util/util.go index 6dd83b57cb..f1b571f4ac 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -28,6 +28,10 @@ import ( "k8s.io/apimachinery/pkg/util/sets" ) +var ( + ClusterClientTLSPath = "/var/lib/cluster-client-tls" +) + func GetOrdinalFromPodName(podName string) (int32, error) { ordinalStr := podName[strings.LastIndex(podName, "-")+1:] ordinalInt, err := strconv.Atoi(ordinalStr) @@ -163,3 +167,11 @@ func Encode(obj interface{}) (string, error) { } return string(b), nil } + +func ClusterClientTLSSecretName(tcName string) string { + return fmt.Sprintf("%s-cluster-client-secret", tcName) +} + +func ClusterTLSSecretName(tcName, component string) string { + return fmt.Sprintf("%s-%s-cluster-secret", tcName, component) +} diff --git a/tests/actions.go b/tests/actions.go index d25d693402..947c0820ae 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -124,7 +124,7 @@ func NewOperatorActions(cli versioned.Interface, framework.ExpectNoError(err) oa.tidbControl = proxiedtidbclient.NewProxiedTiDBClient(fw, kubeCfg.TLSClientConfig.CAData) } else { - oa.tidbControl = controller.NewDefaultTiDBControl() + oa.tidbControl = controller.NewDefaultTiDBControl(kubeCli) } oa.clusterEvents = make(map[string]*clusterEvent) for _, c := range clusters { diff --git a/tests/e2e/tidbcluster/tidbcluster.go b/tests/e2e/tidbcluster/tidbcluster.go index b450875b73..b5d2ec7b47 100644 --- a/tests/e2e/tidbcluster/tidbcluster.go +++ b/tests/e2e/tidbcluster/tidbcluster.go @@ -142,13 +142,6 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() { "tikv.resources.limits.storage": "1G", }, }, - { - Version: utilimage.TiDBTLSVersion, - Name: "basic-v3-cluster-tls", - Values: map[string]string{ - "enableTLSCluster": "true", - }, - }, } for _, clusterCfg := range clusterCfgs {