Support user-defined tidb server/client certificate (#1714)

* support user custom certificate * refine API * fix typo * fix some bugs * create service before certificate * tiny fix * address comments * address comment * address comment Co-authored-by: Song Gao <disxiaofei@163.com>
pingcap · Feb 20, 2020 · 759dca3 · 759dca3
1 parent f61e56d
commit 759dca3
Show file tree

Hide file tree

Showing 10 changed files with 141 additions and 111 deletions.
diff --git a/charts/tidb-cluster/templates/_helpers.tpl b/charts/tidb-cluster/templates/_helpers.tpl
@@ -91,16 +91,20 @@ config-file: |-
     {{- if .Values.tidb.config }}
 {{ .Values.tidb.config | indent 2 }}
     {{- end -}}
-    {{- if or .Values.enableTLSCluster .Values.tidb.enableTLSClient }}
+    {{- if or .Values.enableTLSCluster .Values.tidb.tlsClient.enabled }}
   [security]
     {{- end -}}
     {{- if .Values.enableTLSCluster }}
   cluster-ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
   cluster-ssl-cert = "/var/lib/tidb-tls/cert"
   cluster-ssl-key = "/var/lib/tidb-tls/key"
     {{- end -}}
-    {{- if .Values.tidb.enableTLSClient }}
+    {{- if .Values.tidb.tlsClient.enabled }}
+    {{- if .Values.tidb.tlsClient.secretName }}
+  ssl-ca = "/var/lib/tidb-server-tls/ca"
+    {{- else }}
   ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+    {{- end }}
   ssl-cert = "/var/lib/tidb-server-tls/cert"
   ssl-key = "/var/lib/tidb-server-tls/key"
     {{- end -}}

diff --git a/charts/tidb-cluster/templates/tidb-cluster.yaml b/charts/tidb-cluster/templates/tidb-cluster.yaml
@@ -92,18 +92,9 @@ spec:
   {{- end }}
     maxFailoverCount: {{ .Values.tikv.maxFailoverCount | default 3 }}
   tidb:
-    enableTLSClient: {{ .Values.tidb.enableTLSClient | default false }}
-  {{- if .Values.tidb.extraSANIPList }}
-    extraSANIPList:
-    {{- range .Values.tidb.extraSANIPList }}
-    - {{ . }}
-    {{- end }}
-  {{- end }}
-  {{- if .Values.tidb.extraSANDomainList }}
-    extraSANDomainList:
-    {{- range .Values.tidb.extraSANDomainList }}
-    - {{ . }}
-    {{- end }}
+  {{- if .Values.tidb.tlsClient }}
+    tlsClient:
+{{ toYaml .Values.tidb.tlsClient | indent 6 }}
   {{- end }}
     replicas: {{ .Values.tidb.replicas }}
     image: {{ .Values.tidb.image }}

diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml
@@ -428,18 +428,26 @@ tidb:
     list: ["whitelist-1"]
 
   # Whether enable TLS connection between TiDB server and MySQL client.
-  # When enabled, TiDB will accept TLS encrypted connections from MySQL client, certificates will be generated
-  # automatically.
   # Note: TLS connection is not forced on the server side, plain connections are also accepted after enableing.
-  enableTLSClient: false
-  # # extra SAN IP list when you set tidb.enableTLSClient to true
-  # extraSANIPList:
-  #   - 1.1.1.1
-  #   - 2.2.2.2
-  # # extra SAN Domain List when you set tidb.enableTLSClient to true
-  # extraSANDomainList:
-  #   - example1.com
-  #   - example2.com
+  tlsClient:
+    # When enabled, TiDB will accept TLS encrypted connections from MySQL client
+    enabled: false
+    # # secretName is the name of the secret that stores user-defined tidb server certificate, key and ca...
+    # # If not specified but tls client is enabled, certificated signed by k8s is created automatically.
+    # # Create this secret with the following command:
+    # # kubectl create secret generic <secret-name> --namespace=<namespace> --from-file=cert=<tidb server certificate file path> --from-file=key=<tidb server key file path> --from-file=ca=<ca file path>
+    # secretName: "demo-tidb-server-secret"
+
+    # Auto-generated certificate in k8s: https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/
+    # autoGenerated:
+    #   # Extra SAN IP list
+    #   extraSANIPList:
+    #    - 1.1.1.1
+    #    - 2.2.2.2
+    #   # Extra SAN Domain list
+    #   extraSANDomainList:
+    #    - example1.com
+    #    - example2.com
 
 # mysqlClient is used to set password for TiDB
 # it must has Python MySQL client installed

diff --git a/manifests/crd.yaml b/manifests/crd.yaml
@@ -3779,21 +3779,6 @@ spec:
                     cluster-level updateStrategy if present Optional: Defaults to
                     cluster-level setting'
                   type: string
-                enableTLSClient:
-                  description: 'Whether enable the TLS connection between the SQL
-                    client and TiDB server Optional: Defaults to false'
-                  type: boolean
-                extraSANDomainList:
-                  description: extra SAN Domain list when setting EnableTLSClient
-                    to true
-                  items:
-                    type: string
-                  type: array
-                extraSANIPList:
-                  description: extra SAN IP list when setting EnableTLSClient to true
-                  items:
-                    type: string
-                  type: array
                 hostNetwork:
                   description: 'Whether Hostnetwork of the component is enabled. Override
                     the cluster-level setting if present Optional: Defaults to cluster-level
@@ -3988,6 +3973,7 @@ spec:
                         to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
                       type: object
                   type: object
+                tlsClient: {}
                 tolerations:
                   description: 'Tolerations of the component. Override the cluster-level
                     tolerations if non-empty Optional: Defaults to cluster-level setting'

diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go
diff --git a/pkg/apis/pingcap/v1alpha1/tidbcluster.go b/pkg/apis/pingcap/v1alpha1/tidbcluster.go
@@ -339,11 +339,11 @@ func (tc *TidbCluster) IsTiDBBinlogEnabled() bool {
 }
 
 func (tidb *TiDBSpec) IsTLSClientEnabled() bool {
-	enableTLSClient := tidb.EnableTLSClient
-	if enableTLSClient == nil {
-		return defaultEnableTLSClient
-	}
-	return *enableTLSClient
+	return tidb.TLSClient != nil && tidb.TLSClient.Enabled
+}
+
+func (tidb *TiDBSpec) IsUserGeneratedCertificate() bool {
+	return tidb.IsTLSClientEnabled() && tidb.TLSClient.SecretName != ""
 }
 
 func (tidb *TiDBSpec) ShouldSeparateSlowLog() bool {

diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go
@@ -308,15 +308,9 @@ type TiDBSpec struct {
 	SeparateSlowLog *bool `json:"separateSlowLog,omitempty"`
 
 	// Whether enable the TLS connection between the SQL client and TiDB server
-	// Optional: Defaults to false
+	// Optional: Defaults to nil
 	// +optional
-	EnableTLSClient *bool `json:"enableTLSClient,omitempty"`
-
-	// extra SAN IP list when setting EnableTLSClient to true
-	ExtraSANIPList []string `json:"extraSANIPList,omitempty"`
-
-	// extra SAN Domain list when setting EnableTLSClient to true
-	ExtraSANDomainList []string `json:"extraSANDomainList,omitempty"`
+	TLSClient *TiDBTLSClient `json:"tlsClient,omitempty"`
 
 	// The spec of the slow log tailer sidecar
 	// +optional
@@ -602,6 +596,32 @@ type PumpStatus struct {
 	StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"`
 }
 
+// TiDBTLSClient can enable TLS connection between TiDB server and MySQL client
+type TiDBTLSClient struct {
+	// When enabled, TiDB will accept TLS encrypted connections from MySQL client
+	// +optional
+	Enabled bool `json:"enabled,omitempty"`
+
+	// Secret name which stores user-defined TiDB Server certificate, key and ca
+	// +optional
+	SecretName string `json:"secretName,omitempty"`
+
+	// Auto-generated certificate
+	// +optional
+	AutoGenerated *TiDBAutoGeneratedCertificate `json:"autoGenerated,omitempty"`
+}
+
+// TiDBAutoGeneratedCertificate is TiDB auto-generated certificate
+type TiDBAutoGeneratedCertificate struct {
+	// Extra SAN IP list
+	// +optional
+	ExtraSANIPList []string `json:"extraSANIPList,omitempty"`
+
+	// Extra SAN Domain list
+	// +optional
+	ExtraSANDomainList []string `json:"extraSANDomainList,omitempty"`
+}
+
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 

diff --git a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go
diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go
@@ -108,12 +108,13 @@ func (tmm *tidbMemberManager) Sync(tc *v1alpha1.TidbCluster) error {
 		return err
 	}
 
-	// Sync Tidb StatefulSet
-	if err := tmm.syncTiDBStatefulSetForTidbCluster(tc); err != nil {
+	// Sync TiDB Service before syncing TiDB StatefulSet
+	if err := tmm.syncTiDBService(tc); err != nil {
 		return err
 	}
 
-	return tmm.syncTiDBService(tc)
+	// Sync TiDB StatefulSet
+	return tmm.syncTiDBStatefulSetForTidbCluster(tc)
 }
 
 func (tmm *tidbMemberManager) syncTiDBHeadlessServiceForTidbCluster(tc *v1alpha1.TidbCluster) error {
@@ -181,7 +182,7 @@ func (tmm *tidbMemberManager) syncTiDBStatefulSetForTidbCluster(tc *v1alpha1.Tid
 				return err
 			}
 		}
-		if tc.Spec.TiDB.IsTLSClientEnabled() {
+		if tc.Spec.TiDB.IsTLSClientEnabled() && !tc.Spec.TiDB.IsUserGeneratedCertificate() {
 			err := tmm.syncTiDBServerCerts(tc)
 			if err != nil {
 				return err
@@ -288,13 +289,15 @@ func (tmm *tidbMemberManager) syncTiDBServerCerts(tc *v1alpha1.TidbCluster) erro
 		fmt.Sprintf("%s.%s.svc", svcName, ns),
 		"localhost",
 	}
-	hostList = append(hostList, tc.Spec.TiDB.ExtraSANDomainList...)
-
 	ipList := []string{
 		"127.0.0.1", "::1",
 		svc.Spec.ClusterIP,
 	}
-	ipList = append(ipList, tc.Spec.TiDB.ExtraSANIPList...)
+
+	if tc.Spec.TiDB.TLSClient.AutoGenerated != nil {
+		hostList = append(hostList, tc.Spec.TiDB.TLSClient.AutoGenerated.ExtraSANDomainList...)
+		ipList = append(ipList, tc.Spec.TiDB.TLSClient.AutoGenerated.ExtraSANIPList...)
+	}
 
 	certOpts := &controller.TiDBClusterCertOptions{
 		Namespace:  ns,
@@ -434,7 +437,11 @@ func getTiDBConfigMap(tc *v1alpha1.TidbCluster) (*corev1.ConfigMap, error) {
 		if config.Security == nil {
 			config.Security = &v1alpha1.Security{}
 		}
-		config.Security.SSLCA = pointer.StringPtr(serviceAccountCAPath)
+		if tc.Spec.TiDB.IsUserGeneratedCertificate() {
+			config.Security.SSLCA = pointer.StringPtr(path.Join(serverCertPath, "ca"))
+		} else {
+			config.Security.SSLCA = pointer.StringPtr(serviceAccountCAPath)
+		}
 		config.Security.SSLCert = pointer.StringPtr(path.Join(serverCertPath, "cert"))
 		config.Security.SSLKey = pointer.StringPtr(path.Join(serverCertPath, "key"))
 	}
@@ -622,10 +629,16 @@ func getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbCluster, cm *corev1.ConfigMap)
 		})
 	}
 	if tc.Spec.TiDB.IsTLSClientEnabled() {
+		var secretName string
+		if tc.Spec.TiDB.IsUserGeneratedCertificate() {
+			secretName = tc.Spec.TiDB.TLSClient.SecretName
+		} else {
+			secretName = fmt.Sprintf("%s-%s", controller.TiDBMemberName(tcName), "server")
+		}
 		vols = append(vols, corev1.Volume{
 			Name: "tidb-server-tls", VolumeSource: corev1.VolumeSource{
 				Secret: &corev1.SecretVolumeSource{
-					SecretName: fmt.Sprintf("%s-%s", controller.TiDBMemberName(tcName), "server"),
+					SecretName: secretName,
 				},
 			},
 		})