From f31172f91eee41b9c56ea1c9b2ca8aaaeebbd8e6 Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Mon, 5 Aug 2019 15:43:33 +0800 Subject: [PATCH 01/17] discovery: add HTTPS URL support --- charts/tidb-cluster/templates/tidb-cluster.yaml | 2 ++ charts/tidb-cluster/values.yaml | 17 ++++++++++++++--- pkg/apis/pingcap.com/v1alpha1/types.go | 4 ++++ pkg/discovery/discovery.go | 7 ++++++- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/charts/tidb-cluster/templates/tidb-cluster.yaml b/charts/tidb-cluster/templates/tidb-cluster.yaml index bf8adc11dd..0e234af3a8 100644 --- a/charts/tidb-cluster/templates/tidb-cluster.yaml +++ b/charts/tidb-cluster/templates/tidb-cluster.yaml @@ -20,6 +20,8 @@ metadata: spec: pvReclaimPolicy: {{ .Values.pvReclaimPolicy }} timezone: {{ .Values.timezone | default "UTC" }} + enableTLSServer: {{ .Values.enableTLSServer | default false }} + enableTLSClient: {{ .Values.enableTLSClient | default false }} services: {{ toYaml .Values.services | indent 4 }} schedulerName: {{ .Values.schedulerName | default "default-scheduler" }} diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index dcb94bf0e4..e26c8dcf9f 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -47,9 +47,20 @@ discovery: # if the ConfigMap was not changed. enableConfigMapRollout: true +# Whether enable TLS connections between server nodes. +# When enabled, PD/TiDB/TiKV will use TLS encrypted connections to transfer data between each node, +# certificates will be generated automatically (if not already present). +enableTLSServer: false + +# Whether enable TLS connection between TiDB server and MySQL client. +# When enabled, TiDB will accept TLS encrypted connections from MySQL client, certificates will be generated +# automatically. +# Note: TLS connection is not forced on the server side, plain connections are also accepted after enableing. +enableTLSClient: false + pd: # Please refer to https://github.com/pingcap/pd/blob/master/conf/config.toml for the default - # pd configurations (change to the tags of your pd version), + # pd configurations (change to the tags of your pd version), # just follow the format in the file and configure in the 'config' section # as below if you want to customize any configuration. # Please refer to https://pingcap.com/docs-cn/v3.0/reference/configuration/pd-server/configuration-file/ @@ -59,7 +70,7 @@ pd: level = "info" [replication] location-labels = ["region", "zone", "rack", "host"] - + replicas: 3 image: pingcap/pd:v3.0.1 # storageClassName is a StorageClass provides a way for administrators to describe the "classes" of storage they offer. @@ -258,7 +269,7 @@ tidb: config: | [log] level = "info" - + replicas: 2 # The secret name of root password, you can create secret with following command: # kubectl create secret generic tidb-secret --from-literal=root= --namespace= diff --git a/pkg/apis/pingcap.com/v1alpha1/types.go b/pkg/apis/pingcap.com/v1alpha1/types.go index 1911cab98c..7a72534c00 100644 --- a/pkg/apis/pingcap.com/v1alpha1/types.go +++ b/pkg/apis/pingcap.com/v1alpha1/types.go @@ -93,6 +93,10 @@ type TidbClusterSpec struct { Services []Service `json:"services,omitempty"` PVReclaimPolicy corev1.PersistentVolumeReclaimPolicy `json:"pvReclaimPolicy,omitempty"` Timezone string `json:"timezone,omitempty"` + // Enable TLS connection between TiDB server compoments + EnableTLSServer bool `json:"enableTLSServer,omitempty"` + // Accept TLS connection from client + EnableTLSClient bool `json:"enableTLSClient,omitempty"` } // TidbClusterStatus represents the current status of a tidb cluster. diff --git a/pkg/discovery/discovery.go b/pkg/discovery/discovery.go index 7a8f432818..8bc6b27450 100644 --- a/pkg/discovery/discovery.go +++ b/pkg/discovery/discovery.go @@ -92,9 +92,14 @@ func (td *tidbDiscovery) Discover(advertisePeerUrl string) (string, error) { currentCluster = td.clusters[keyName] currentCluster.peers[podName] = struct{}{} + schema := "http" + if tc.Spec.EnableTLSServer { + schema = "https" + } + if len(currentCluster.peers) == int(replicas) { delete(currentCluster.peers, podName) - return fmt.Sprintf("--initial-cluster=%s=http://%s", podName, advertisePeerUrl), nil + return fmt.Sprintf("--initial-cluster=%s=%s://%s", podName, schema, advertisePeerUrl), nil } pdClient := td.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName()) From 5bb23845dacdd12192bf8e87e3d304821b3ca43f Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Mon, 5 Aug 2019 19:37:29 +0800 Subject: [PATCH 02/17] tls: add basic support of certis --- charts/tidb-cluster/templates/_helpers.tpl | 24 +++++++++++++++++++ .../templates/scripts/_start_pd.sh.tpl | 15 ++++++++---- pkg/manager/member/pd_member_manager.go | 15 ++++++++++++ pkg/manager/member/tidb_member_manager.go | 15 ++++++++++++ pkg/manager/member/tikv_member_manager.go | 15 ++++++++++++ 5 files changed, 80 insertions(+), 4 deletions(-) diff --git a/charts/tidb-cluster/templates/_helpers.tpl b/charts/tidb-cluster/templates/_helpers.tpl index a2f476e7f4..05683cdc75 100644 --- a/charts/tidb-cluster/templates/_helpers.tpl +++ b/charts/tidb-cluster/templates/_helpers.tpl @@ -37,6 +37,12 @@ config-file: |- {{- if .Values.pd.config }} {{ .Values.pd.config | indent 2 }} {{- end -}} + {{- if .Values.enableTLSServer }} + [security] + cacert-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + cert-path = "/var/lib/pd-tls/pd.crt" + key-path = "/var/lib/pd-tls/pd.key" + {{- end -}} {{- end -}} {{- define "pd-configmap.data-digest" -}} @@ -53,6 +59,12 @@ config-file: |- {{- if .Values.tikv.config }} {{ .Values.tikv.config | indent 2 }} {{- end -}} + {{- if .Values.enableTLSServer }} + [security] + ca-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + cert-path = "/var/lib/tikv-tls/tikv.crt" + key-path = "/var/lib/tikv-tls/tikv.key" + {{- end -}} {{- end -}} {{- define "tikv-configmap.data-digest" -}} @@ -73,6 +85,18 @@ config-file: |- {{- if .Values.tidb.config }} {{ .Values.tidb.config | indent 2 }} {{- end -}} + {{- if or .Values.enableTLSServer .Values.enableTLSClient }} + [security] + {{- end -}} + {{- if .Values.enableTLSServer }} + cacert-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + cert-path = "/var/lib/pd-tls/pd.crt" + key-path = "/var/lib/pd-tls/pd.key" + {{- end -}} + {{- if .Values.enableTLSClient }} + ssl-cert = "/var/lib/tidb-tls/tidb.crt" + ssl-key = "/var/lib/tidb-tls/tidb.key" + {{- end -}} {{- end -}} {{- define "tidb-configmap.data-digest" -}} diff --git a/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl index 36273d8693..a3bcb91526 100644 --- a/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl +++ b/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl @@ -56,12 +56,19 @@ while true; do fi done +if [[ {{ .Values.enableTLSServer }} == "true" ]] +then + SCHEMA="https" +else + SCHEMA="http" +fi + ARGS="--data-dir=/var/lib/pd \ --name=${HOSTNAME} \ ---peer-urls=http://0.0.0.0:2380 \ ---advertise-peer-urls=http://${domain}:2380 \ ---client-urls=http://0.0.0.0:2379 \ ---advertise-client-urls=http://${domain}:2379 \ +--peer-urls=$SCHEMA://0.0.0.0:2380 \ +--advertise-peer-urls=$SCHEMA://${domain}:2380 \ +--client-urls=$SCHEMA://0.0.0.0:2379 \ +--advertise-client-urls=$SCHEMA://${domain}:2379 \ --config=/etc/pd/pd.toml \ " diff --git a/pkg/manager/member/pd_member_manager.go b/pkg/manager/member/pd_member_manager.go index f85013eeea..0a127d5e77 100644 --- a/pkg/manager/member/pd_member_manager.go +++ b/pkg/manager/member/pd_member_manager.go @@ -437,6 +437,12 @@ func (pmm *pdMemberManager) getNewPDSetForTidbCluster(tc *v1alpha1.TidbCluster) {Name: "startup-script", ReadOnly: true, MountPath: "/usr/local/bin"}, {Name: v1alpha1.PDMemberType.String(), MountPath: "/var/lib/pd"}, } + if tc.Spec.EnableTLSServer { + volMounts = append(volMounts, corev1.VolumeMount{ + Name: "pd-tls", ReadOnly: true, MountPath: "/var/lib/pd-tls", + }) + } + vols := []corev1.Volume{ annVolume, {Name: "config", @@ -460,6 +466,15 @@ func (pmm *pdMemberManager) getNewPDSetForTidbCluster(tc *v1alpha1.TidbCluster) }, }, } + if tc.Spec.EnableTLSServer { + vols = append(vols, corev1.Volume{ + Name: "pd-tls", VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: fmt.Sprintf("%s-pd", tc.ClusterName), + }, + }, + }) + } var q resource.Quantity var err error diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index 9a28d0451a..a909a20c8e 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -229,6 +229,12 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust {Name: "config", ReadOnly: true, MountPath: "/etc/tidb"}, {Name: "startup-script", ReadOnly: true, MountPath: "/usr/local/bin"}, } + if tc.Spec.EnableTLSServer { + volMounts = append(volMounts, corev1.VolumeMount{ + Name: "tidb-tls", ReadOnly: true, MountPath: "/var/lib/tidb-tls", + }) + } + vols := []corev1.Volume{ annVolume, {Name: "config", VolumeSource: corev1.VolumeSource{ @@ -248,6 +254,15 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust }}, }, } + if tc.Spec.EnableTLSServer { + vols = append(vols, corev1.Volume{ + Name: "tidb-tls", VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: fmt.Sprintf("%s-tidb", tc.ClusterName), + }, + }, + }) + } var containers []corev1.Container if tc.Spec.TiDB.SeparateSlowLog { diff --git a/pkg/manager/member/tikv_member_manager.go b/pkg/manager/member/tikv_member_manager.go index 8b7c50dbe9..704853d1ab 100644 --- a/pkg/manager/member/tikv_member_manager.go +++ b/pkg/manager/member/tikv_member_manager.go @@ -279,6 +279,12 @@ func (tkmm *tikvMemberManager) getNewSetForTidbCluster(tc *v1alpha1.TidbCluster) {Name: "config", ReadOnly: true, MountPath: "/etc/tikv"}, {Name: "startup-script", ReadOnly: true, MountPath: "/usr/local/bin"}, } + if tc.Spec.EnableTLSServer { + volMounts = append(volMounts, corev1.VolumeMount{ + Name: "tikv-tls", ReadOnly: true, MountPath: "/var/lib/tikv-tls", + }) + } + vols := []corev1.Volume{ annVolume, {Name: "config", VolumeSource: corev1.VolumeSource{ @@ -298,6 +304,15 @@ func (tkmm *tikvMemberManager) getNewSetForTidbCluster(tc *v1alpha1.TidbCluster) }}, }, } + if tc.Spec.EnableTLSServer { + vols = append(vols, corev1.Volume{ + Name: "tikv-tls", VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: fmt.Sprintf("%s-tikv", tc.ClusterName), + }, + }, + }) + } var q resource.Quantity var err error From 99956b7bdb47c75aabd8ae122936efacea6aefc3 Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Tue, 6 Aug 2019 15:58:25 +0800 Subject: [PATCH 03/17] tls: support tls client for pd --- .../templates/discovery-deployment.yaml | 12 ++++- pkg/controller/pd_control.go | 6 ++- pkg/controller/pod_control.go | 8 ++- pkg/discovery/discovery.go | 2 +- pkg/manager/member/pd_member_manager.go | 2 +- pkg/manager/member/tidb_member_manager.go | 2 +- pkg/manager/member/tikv_member_manager.go | 2 +- pkg/manager/member/tikv_upgrader.go | 6 ++- pkg/pdapi/pdapi.go | 51 ++++++++++++++++--- 9 files changed, 75 insertions(+), 16 deletions(-) diff --git a/charts/tidb-cluster/templates/discovery-deployment.yaml b/charts/tidb-cluster/templates/discovery-deployment.yaml index a661efacb4..7f55ae8464 100644 --- a/charts/tidb-cluster/templates/discovery-deployment.yaml +++ b/charts/tidb-cluster/templates/discovery-deployment.yaml @@ -45,4 +45,14 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - +{{- if .Values.enableTLSServer }} + volumeMounts: + - mountPath: /var/lib/pd-tls + name: pd-tls + readOnly: true + volumes: + - name: pd-tls + secret: + defaultMode: 420 + secretName: {{ .Release.Name }}-pd +{{- end -}} diff --git a/pkg/controller/pd_control.go b/pkg/controller/pd_control.go index 597f1fa408..6b1bb2850b 100644 --- a/pkg/controller/pd_control.go +++ b/pkg/controller/pd_control.go @@ -20,7 +20,11 @@ import ( // GetPDClient gets the pd client from the TidbCluster func GetPDClient(pdControl pdapi.PDControlInterface, tc *v1alpha1.TidbCluster) pdapi.PDClient { - return pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName()) + schema := "http" + if tc.Spec.EnableTLSServer { + schema = "https" + } + return pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), schema) } // NewFakePDClient creates a fake pdclient that is set as the pd client diff --git a/pkg/controller/pod_control.go b/pkg/controller/pod_control.go index 824a1da0a2..2ee876cf3a 100644 --- a/pkg/controller/pod_control.go +++ b/pkg/controller/pod_control.go @@ -111,7 +111,13 @@ func (rpc *realPodControl) UpdateMetaInfo(tc *v1alpha1.TidbCluster, pod *corev1. clusterID := labels[label.ClusterIDLabelKey] memberID := labels[label.MemberIDLabelKey] storeID := labels[label.StoreIDLabelKey] - pdClient := rpc.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tcName) + + schema := "http" + if tc.Spec.EnableTLSServer { + schema = "https" + } + + pdClient := rpc.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tcName, schema) if labels[label.ClusterIDLabelKey] == "" { cluster, err := pdClient.GetCluster() if err != nil { diff --git a/pkg/discovery/discovery.go b/pkg/discovery/discovery.go index 8bc6b27450..c397b2518b 100644 --- a/pkg/discovery/discovery.go +++ b/pkg/discovery/discovery.go @@ -102,7 +102,7 @@ func (td *tidbDiscovery) Discover(advertisePeerUrl string) (string, error) { return fmt.Sprintf("--initial-cluster=%s=%s://%s", podName, schema, advertisePeerUrl), nil } - pdClient := td.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName()) + pdClient := td.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), schema) membersInfo, err := pdClient.GetMembers() if err != nil { return "", err diff --git a/pkg/manager/member/pd_member_manager.go b/pkg/manager/member/pd_member_manager.go index 0a127d5e77..7707e6e97b 100644 --- a/pkg/manager/member/pd_member_manager.go +++ b/pkg/manager/member/pd_member_manager.go @@ -470,7 +470,7 @@ func (pmm *pdMemberManager) getNewPDSetForTidbCluster(tc *v1alpha1.TidbCluster) vols = append(vols, corev1.Volume{ Name: "pd-tls", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: fmt.Sprintf("%s-pd", tc.ClusterName), + SecretName: fmt.Sprintf("%s-pd", tcName), }, }, }) diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index a909a20c8e..899ec86fff 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -258,7 +258,7 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust vols = append(vols, corev1.Volume{ Name: "tidb-tls", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: fmt.Sprintf("%s-tidb", tc.ClusterName), + SecretName: fmt.Sprintf("%s-tidb", tcName), }, }, }) diff --git a/pkg/manager/member/tikv_member_manager.go b/pkg/manager/member/tikv_member_manager.go index 704853d1ab..acf1b2c5a2 100644 --- a/pkg/manager/member/tikv_member_manager.go +++ b/pkg/manager/member/tikv_member_manager.go @@ -308,7 +308,7 @@ func (tkmm *tikvMemberManager) getNewSetForTidbCluster(tc *v1alpha1.TidbCluster) vols = append(vols, corev1.Volume{ Name: "tikv-tls", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: fmt.Sprintf("%s-tikv", tc.ClusterName), + SecretName: fmt.Sprintf("%s-tikv", tcName), }, }, }) diff --git a/pkg/manager/member/tikv_upgrader.go b/pkg/manager/member/tikv_upgrader.go index fc9e05ba64..b642c686dc 100644 --- a/pkg/manager/member/tikv_upgrader.go +++ b/pkg/manager/member/tikv_upgrader.go @@ -196,7 +196,11 @@ func (tku *tikvUpgrader) endEvictLeader(tc *v1alpha1.TidbCluster, ordinal int32) return err } - err = tku.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName()).EndEvictLeader(storeID) + schema := "http" + if tc.Spec.EnableTLSServer { + schema = "https" + } + err = tku.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), schema).EndEvictLeader(storeID) if err != nil { glog.Errorf("tikv upgrader: failed to end evict leader storeID: %d ordinal: %d, %v", storeID, ordinal, err) return err diff --git a/pkg/pdapi/pdapi.go b/pkg/pdapi/pdapi.go index f96ff84200..01244e822d 100644 --- a/pkg/pdapi/pdapi.go +++ b/pkg/pdapi/pdapi.go @@ -15,6 +15,8 @@ package pdapi import ( "bytes" + "crypto/tls" + "crypto/x509" "encoding/json" "fmt" "io/ioutil" @@ -33,7 +35,10 @@ import ( ) const ( - timeout = 5 * time.Second + timeout = 5 * time.Second + k8sCAFile = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + pdCert = "/var/lib/pd-tls/pd.crt" + pdKey = "/var/lib/pd-tls/pd.key" ) // Namespace is a newtype of a string @@ -42,7 +47,7 @@ type Namespace string // PDControlInterface is an interface that knows how to manage and get tidb cluster's PD client type PDControlInterface interface { // GetPDClient provides PDClient of the tidb cluster. - GetPDClient(Namespace, string) PDClient + GetPDClient(Namespace, string, string) PDClient } // defaultPDControl is the default implementation of PDControlInterface. @@ -57,12 +62,12 @@ func NewDefaultPDControl() PDControlInterface { } // GetPDClient provides a PDClient of real pd cluster,if the PDClient not existing, it will create new one. -func (pdc *defaultPDControl) GetPDClient(namespace Namespace, tcName string) PDClient { +func (pdc *defaultPDControl) GetPDClient(namespace Namespace, tcName string, schema string) PDClient { pdc.mutex.Lock() defer pdc.mutex.Unlock() key := pdClientKey(namespace, tcName) if _, ok := pdc.pdClients[key]; !ok { - pdc.pdClients[key] = NewPDClient(PdClientURL(namespace, tcName), timeout) + pdc.pdClients[key] = NewPDClient(PdClientURL(namespace, tcName, schema), timeout) } return pdc.pdClients[key] } @@ -73,8 +78,8 @@ func pdClientKey(namespace Namespace, clusterName string) string { } // pdClientUrl builds the url of pd client -func PdClientURL(namespace Namespace, clusterName string) string { - return fmt.Sprintf("http://%s-pd.%s:2379", clusterName, string(namespace)) +func PdClientURL(namespace Namespace, clusterName string, schema string) string { + return fmt.Sprintf("%s://%s-pd.%s:2379", schema, clusterName, string(namespace)) } // PDClient provides pd server's api @@ -135,9 +140,39 @@ type pdClient struct { // NewPDClient returns a new PDClient func NewPDClient(url string, timeout time.Duration) PDClient { + // load CA certs + rootCAs, _ := x509.SystemCertPool() + if rootCAs == nil { + rootCAs = x509.NewCertPool() + } + + caCert, err := ioutil.ReadFile(k8sCAFile) + if err != nil { + glog.Errorf("fail to read CA file %s, error: %v", k8sCAFile, err) + return nil + } + if ok := rootCAs.AppendCertsFromPEM(caCert); !ok { + glog.Warningf("fail to append cert to pool, using system certs only") + } + + // load client cert + cert, err := tls.LoadX509KeyPair(pdCert, pdKey) + if err != nil { + glog.Errorf("fail to load client cert") + return nil + } + + config := &tls.Config{ + RootCAs: rootCAs, + Certificates: []tls.Certificate{cert}, + } + return &pdClient{ - url: url, - httpClient: &http.Client{Timeout: timeout}, + url: url, + httpClient: &http.Client{ + Timeout: timeout, + Transport: &http.Transport{TLSClientConfig: config}, + }, } } From 84cc8bdb94666348c5bafa04253f9b3d89b05808 Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Tue, 6 Aug 2019 21:37:12 +0800 Subject: [PATCH 04/17] tls: support tls client for controller --- charts/tidb-cluster/templates/_helpers.tpl | 28 ++++---- .../templates/discovery-deployment.yaml | 8 +-- .../controller-manager-deployment.yaml | 9 +++ pkg/controller/pd_control.go | 6 +- pkg/controller/pod_control.go | 7 +- pkg/controller/tidb_control.go | 49 +++++++++++-- pkg/discovery/discovery.go | 2 +- pkg/httputil/httputil.go | 34 ++++++++++ pkg/manager/member/tikv_upgrader.go | 6 +- pkg/pdapi/pdapi.go | 68 ++++++++----------- 10 files changed, 137 insertions(+), 80 deletions(-) diff --git a/charts/tidb-cluster/templates/_helpers.tpl b/charts/tidb-cluster/templates/_helpers.tpl index 05683cdc75..263ce2532c 100644 --- a/charts/tidb-cluster/templates/_helpers.tpl +++ b/charts/tidb-cluster/templates/_helpers.tpl @@ -38,10 +38,10 @@ config-file: |- {{ .Values.pd.config | indent 2 }} {{- end -}} {{- if .Values.enableTLSServer }} - [security] - cacert-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - cert-path = "/var/lib/pd-tls/pd.crt" - key-path = "/var/lib/pd-tls/pd.key" + [security] + cacert-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + cert-path = "/var/lib/pd-tls/pd.crt" + key-path = "/var/lib/pd-tls/pd.key" {{- end -}} {{- end -}} @@ -60,10 +60,10 @@ config-file: |- {{ .Values.tikv.config | indent 2 }} {{- end -}} {{- if .Values.enableTLSServer }} - [security] - ca-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - cert-path = "/var/lib/tikv-tls/tikv.crt" - key-path = "/var/lib/tikv-tls/tikv.key" + [security] + ca-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + cert-path = "/var/lib/tikv-tls/tikv.crt" + key-path = "/var/lib/tikv-tls/tikv.key" {{- end -}} {{- end -}} @@ -86,16 +86,16 @@ config-file: |- {{ .Values.tidb.config | indent 2 }} {{- end -}} {{- if or .Values.enableTLSServer .Values.enableTLSClient }} - [security] + [security] {{- end -}} {{- if .Values.enableTLSServer }} - cacert-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - cert-path = "/var/lib/pd-tls/pd.crt" - key-path = "/var/lib/pd-tls/pd.key" + cacert-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + cert-path = "/var/lib/pd-tls/pd.crt" + key-path = "/var/lib/pd-tls/pd.key" {{- end -}} {{- if .Values.enableTLSClient }} - ssl-cert = "/var/lib/tidb-tls/tidb.crt" - ssl-key = "/var/lib/tidb-tls/tidb.key" + ssl-cert = "/var/lib/tidb-tls/tidb.crt" + ssl-key = "/var/lib/tidb-tls/tidb.key" {{- end -}} {{- end -}} diff --git a/charts/tidb-cluster/templates/discovery-deployment.yaml b/charts/tidb-cluster/templates/discovery-deployment.yaml index 7f55ae8464..da0eb7451b 100644 --- a/charts/tidb-cluster/templates/discovery-deployment.yaml +++ b/charts/tidb-cluster/templates/discovery-deployment.yaml @@ -47,12 +47,12 @@ spec: fieldPath: metadata.namespace {{- if .Values.enableTLSServer }} volumeMounts: - - mountPath: /var/lib/pd-tls - name: pd-tls + - mountPath: /var/lib/tls + name: tls readOnly: true volumes: - - name: pd-tls + - name: tls secret: defaultMode: 420 - secretName: {{ .Release.Name }}-pd + secretName: client-tls {{- end -}} diff --git a/charts/tidb-operator/templates/controller-manager-deployment.yaml b/charts/tidb-operator/templates/controller-manager-deployment.yaml index b848b89c22..dc3e2146e5 100644 --- a/charts/tidb-operator/templates/controller-manager-deployment.yaml +++ b/charts/tidb-operator/templates/controller-manager-deployment.yaml @@ -50,3 +50,12 @@ spec: fieldPath: metadata.namespace - name: TZ value: {{ .Values.timezone | default "UTC" }} + volumeMounts: + - mountPath: /var/lib/tls + name: tls + readOnly: true + volumes: + - name: tls + secret: + defaultMode: 420 + secretName: client-tls diff --git a/pkg/controller/pd_control.go b/pkg/controller/pd_control.go index 6b1bb2850b..2bd1c4cf10 100644 --- a/pkg/controller/pd_control.go +++ b/pkg/controller/pd_control.go @@ -20,11 +20,7 @@ import ( // GetPDClient gets the pd client from the TidbCluster func GetPDClient(pdControl pdapi.PDControlInterface, tc *v1alpha1.TidbCluster) pdapi.PDClient { - schema := "http" - if tc.Spec.EnableTLSServer { - schema = "https" - } - return pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), schema) + return pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSServer) } // NewFakePDClient creates a fake pdclient that is set as the pd client diff --git a/pkg/controller/pod_control.go b/pkg/controller/pod_control.go index 2ee876cf3a..07fba867bf 100644 --- a/pkg/controller/pod_control.go +++ b/pkg/controller/pod_control.go @@ -112,12 +112,7 @@ func (rpc *realPodControl) UpdateMetaInfo(tc *v1alpha1.TidbCluster, pod *corev1. memberID := labels[label.MemberIDLabelKey] storeID := labels[label.StoreIDLabelKey] - schema := "http" - if tc.Spec.EnableTLSServer { - schema = "https" - } - - pdClient := rpc.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tcName, schema) + pdClient := rpc.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tcName, tc.Spec.EnableTLSServer) if labels[label.ClusterIDLabelKey] == "" { cluster, err := pdClient.GetCluster() if err != nil { diff --git a/pkg/controller/tidb_control.go b/pkg/controller/tidb_control.go index 5cd376b689..7730ee8e71 100644 --- a/pkg/controller/tidb_control.go +++ b/pkg/controller/tidb_control.go @@ -14,6 +14,7 @@ package controller import ( + "crypto/tls" "encoding/json" "fmt" "io/ioutil" @@ -63,10 +64,20 @@ func (tdc *defaultTiDBControl) GetHealth(tc *v1alpha1.TidbCluster) map[string]bo tcName := tc.GetName() ns := tc.GetNamespace() + schema := "http" + if tc.Spec.EnableTLSServer { + schema = "https" + rootCAs, _ := httputil.ReadCACerts() + config := &tls.Config{ + RootCAs: rootCAs, + } + tdc.httpClient.Transport = &http.Transport{TLSClientConfig: config} + } + result := map[string]bool{} for i := 0; i < int(tc.TiDBRealReplicas()); i++ { hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), i) - url := fmt.Sprintf("http://%s.%s.%s:10080/status", hostName, TiDBPeerMemberName(tcName), ns) + url := fmt.Sprintf("%s://%s.%s.%s:10080/status", schema, hostName, TiDBPeerMemberName(tcName), ns) _, err := tdc.getBodyOK(url) if err != nil { result[hostName] = false @@ -81,8 +92,18 @@ func (tdc *defaultTiDBControl) ResignDDLOwner(tc *v1alpha1.TidbCluster, ordinal tcName := tc.GetName() ns := tc.GetNamespace() + schema := "http" + if tc.Spec.EnableTLSServer { + schema = "https" + rootCAs, _ := httputil.ReadCACerts() + config := &tls.Config{ + RootCAs: rootCAs, + } + tdc.httpClient.Transport = &http.Transport{TLSClientConfig: config} + } + hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal) - url := fmt.Sprintf("http://%s.%s.%s:10080/ddl/owner/resign", hostName, TiDBPeerMemberName(tcName), ns) + url := fmt.Sprintf("%s://%s.%s.%s:10080/ddl/owner/resign", schema, hostName, TiDBPeerMemberName(tcName), ns) req, err := http.NewRequest("POST", url, nil) if err != nil { return false, err @@ -106,8 +127,18 @@ func (tdc *defaultTiDBControl) GetInfo(tc *v1alpha1.TidbCluster, ordinal int32) tcName := tc.GetName() ns := tc.GetNamespace() + schema := "http" + if tc.Spec.EnableTLSServer { + schema = "https" + rootCAs, _ := httputil.ReadCACerts() + config := &tls.Config{ + RootCAs: rootCAs, + } + tdc.httpClient.Transport = &http.Transport{TLSClientConfig: config} + } + hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal) - url := fmt.Sprintf("http://%s.%s.%s:10080/info", hostName, TiDBPeerMemberName(tcName), ns) + url := fmt.Sprintf("%s://%s.%s.%s:10080/info", schema, hostName, TiDBPeerMemberName(tcName), ns) req, err := http.NewRequest("POST", url, nil) if err != nil { return nil, err @@ -137,8 +168,18 @@ func (tdc *defaultTiDBControl) GetSettings(tc *v1alpha1.TidbCluster, ordinal int tcName := tc.GetName() ns := tc.GetNamespace() + schema := "http" + if tc.Spec.EnableTLSServer { + schema = "https" + rootCAs, _ := httputil.ReadCACerts() + config := &tls.Config{ + RootCAs: rootCAs, + } + tdc.httpClient.Transport = &http.Transport{TLSClientConfig: config} + } + hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal) - url := fmt.Sprintf("http://%s.%s.%s:10080/settings", hostName, TiDBPeerMemberName(tcName), ns) + url := fmt.Sprintf("%s://%s.%s.%s:10080/settings", schema, hostName, TiDBPeerMemberName(tcName), ns) req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err diff --git a/pkg/discovery/discovery.go b/pkg/discovery/discovery.go index c397b2518b..355702ba3a 100644 --- a/pkg/discovery/discovery.go +++ b/pkg/discovery/discovery.go @@ -102,7 +102,7 @@ func (td *tidbDiscovery) Discover(advertisePeerUrl string) (string, error) { return fmt.Sprintf("--initial-cluster=%s=%s://%s", podName, schema, advertisePeerUrl), nil } - pdClient := td.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), schema) + pdClient := td.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSServer) membersInfo, err := pdClient.GetMembers() if err != nil { return "", err diff --git a/pkg/httputil/httputil.go b/pkg/httputil/httputil.go index 84b0c70780..640d3475a1 100644 --- a/pkg/httputil/httputil.go +++ b/pkg/httputil/httputil.go @@ -1,6 +1,8 @@ package httputil import ( + "crypto/tls" + "crypto/x509" "fmt" "io" "io/ioutil" @@ -9,6 +11,12 @@ import ( "github.com/golang/glog" ) +const ( + k8sCAFile = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + clientCert = "/var/lib/tls/client.crt" + clientKey = "/var/lib/tls/client.key" +) + // DeferClose captures and prints the error from closing (if an error occurs). // This is designed to be used in a defer statement. func DeferClose(c io.Closer) { @@ -44,3 +52,29 @@ func GetBodyOK(httpClient *http.Client, apiURL string) ([]byte, error) { } return body, err } + +func ReadCACerts() (*x509.CertPool, error) { + // load CA certs + rootCAs, _ := x509.SystemCertPool() + if rootCAs == nil { + rootCAs = x509.NewCertPool() + } + + caCert, err := ioutil.ReadFile(k8sCAFile) + if err != nil { + glog.Errorf("fail to read CA file %s, error: %v", k8sCAFile, err) + return nil, err + } + if ok := rootCAs.AppendCertsFromPEM(caCert); !ok { + glog.Warningf("fail to append cert to pool, using system certs only") + } + return rootCAs, err +} + +func ReadCerts() (*x509.CertPool, tls.Certificate, error) { + rootCAs, err := ReadCACerts() + + // load client cert + cert, err := tls.LoadX509KeyPair(clientCert, clientKey) + return rootCAs, cert, err +} diff --git a/pkg/manager/member/tikv_upgrader.go b/pkg/manager/member/tikv_upgrader.go index b642c686dc..b7f35453d5 100644 --- a/pkg/manager/member/tikv_upgrader.go +++ b/pkg/manager/member/tikv_upgrader.go @@ -196,11 +196,7 @@ func (tku *tikvUpgrader) endEvictLeader(tc *v1alpha1.TidbCluster, ordinal int32) return err } - schema := "http" - if tc.Spec.EnableTLSServer { - schema = "https" - } - err = tku.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), schema).EndEvictLeader(storeID) + err = tku.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSServer).EndEvictLeader(storeID) if err != nil { glog.Errorf("tikv upgrader: failed to end evict leader storeID: %d ordinal: %d, %v", storeID, ordinal, err) return err diff --git a/pkg/pdapi/pdapi.go b/pkg/pdapi/pdapi.go index 01244e822d..1cc1503eb0 100644 --- a/pkg/pdapi/pdapi.go +++ b/pkg/pdapi/pdapi.go @@ -16,7 +16,6 @@ package pdapi import ( "bytes" "crypto/tls" - "crypto/x509" "encoding/json" "fmt" "io/ioutil" @@ -35,10 +34,7 @@ import ( ) const ( - timeout = 5 * time.Second - k8sCAFile = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - pdCert = "/var/lib/pd-tls/pd.crt" - pdKey = "/var/lib/pd-tls/pd.key" + timeout = 5 * time.Second ) // Namespace is a newtype of a string @@ -47,7 +43,7 @@ type Namespace string // PDControlInterface is an interface that knows how to manage and get tidb cluster's PD client type PDControlInterface interface { // GetPDClient provides PDClient of the tidb cluster. - GetPDClient(Namespace, string, string) PDClient + GetPDClient(Namespace, string, bool) PDClient } // defaultPDControl is the default implementation of PDControlInterface. @@ -62,12 +58,12 @@ func NewDefaultPDControl() PDControlInterface { } // GetPDClient provides a PDClient of real pd cluster,if the PDClient not existing, it will create new one. -func (pdc *defaultPDControl) GetPDClient(namespace Namespace, tcName string, schema string) PDClient { +func (pdc *defaultPDControl) GetPDClient(namespace Namespace, tcName string, tlsEnabled bool) PDClient { pdc.mutex.Lock() defer pdc.mutex.Unlock() key := pdClientKey(namespace, tcName) if _, ok := pdc.pdClients[key]; !ok { - pdc.pdClients[key] = NewPDClient(PdClientURL(namespace, tcName, schema), timeout) + pdc.pdClients[key] = NewPDClient(PdClientURL(namespace, tcName, tlsEnabled), timeout, tlsEnabled) } return pdc.pdClients[key] } @@ -78,7 +74,11 @@ func pdClientKey(namespace Namespace, clusterName string) string { } // pdClientUrl builds the url of pd client -func PdClientURL(namespace Namespace, clusterName string, schema string) string { +func PdClientURL(namespace Namespace, clusterName string, tlsEnabled bool) string { + schema := "http" + if tlsEnabled { + schema = "https" + } return fmt.Sprintf("%s://%s-pd.%s:2379", schema, clusterName, string(namespace)) } @@ -139,40 +139,26 @@ type pdClient struct { } // NewPDClient returns a new PDClient -func NewPDClient(url string, timeout time.Duration) PDClient { - // load CA certs - rootCAs, _ := x509.SystemCertPool() - if rootCAs == nil { - rootCAs = x509.NewCertPool() - } - - caCert, err := ioutil.ReadFile(k8sCAFile) - if err != nil { - glog.Errorf("fail to read CA file %s, error: %v", k8sCAFile, err) - return nil - } - if ok := rootCAs.AppendCertsFromPEM(caCert); !ok { - glog.Warningf("fail to append cert to pool, using system certs only") - } - - // load client cert - cert, err := tls.LoadX509KeyPair(pdCert, pdKey) - if err != nil { - glog.Errorf("fail to load client cert") - return nil - } - - config := &tls.Config{ - RootCAs: rootCAs, - Certificates: []tls.Certificate{cert}, +func NewPDClient(url string, timeout time.Duration, tlsEnabled bool) PDClient { + httpClient := &http.Client{Timeout: timeout} + if tlsEnabled { + rootCAs, cert, err := httputil.ReadCerts() + if err != nil { + glog.Errorf("fail to load certs, fallback to plain connection, err: %s", err) + } else { + config := &tls.Config{ + RootCAs: rootCAs, + Certificates: []tls.Certificate{cert}, + } + httpClient = &http.Client{ + Timeout: timeout, + Transport: &http.Transport{TLSClientConfig: config}, + } + } } - return &pdClient{ - url: url, - httpClient: &http.Client{ - Timeout: timeout, - Transport: &http.Transport{TLSClientConfig: config}, - }, + url: url, + httpClient: httpClient, } } From 71bf08d09b7f0fb0fbb116e8023e4579d0b675e9 Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Wed, 7 Aug 2019 11:15:10 +0800 Subject: [PATCH 05/17] tls: update startup script templates --- charts/tidb-cluster/templates/scripts/_start_tidb.sh.tpl | 9 ++++++++- charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/charts/tidb-cluster/templates/scripts/_start_tidb.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_tidb.sh.tpl index ee79a7594e..2f4ca49b59 100644 --- a/charts/tidb-cluster/templates/scripts/_start_tidb.sh.tpl +++ b/charts/tidb-cluster/templates/scripts/_start_tidb.sh.tpl @@ -26,9 +26,16 @@ then tail -f /dev/null fi +if [[ {{ .Values.enableTLSServer }} == "true" ]] +then + SCHEMA="https" +else + SCHEMA="http" +fi + ARGS="--store=tikv \ --host=0.0.0.0 \ ---path=${CLUSTER_NAME}-pd:2379 \ +--path=$SCHEMA://${CLUSTER_NAME}-pd:2379 \ --config=/etc/tidb/tidb.toml " diff --git a/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl index 8345ffed6c..f0d9718ad2 100644 --- a/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl +++ b/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl @@ -28,7 +28,14 @@ then tail -f /dev/null fi -ARGS="--pd=${CLUSTER_NAME}-pd:2379 \ +if [[ {{ .Values.enableTLSServer }} == "true" ]] +then + SCHEMA="https" +else + SCHEMA="http" +fi + +ARGS="--pd=$SCHEMA://${CLUSTER_NAME}-pd:2379 \ --advertise-addr=${HOSTNAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:20160 \ --addr=0.0.0.0:20160 \ --data-dir=/var/lib/tikv \ From ecfe87e5c1c17e9b380cfa3d75e3ec5fc1fc75c3 Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Wed, 7 Aug 2019 14:08:16 +0800 Subject: [PATCH 06/17] tls: fix configs --- charts/tidb-cluster/templates/_helpers.tpl | 10 +++++++--- .../tidb-cluster/templates/scripts/_start_tidb.sh.tpl | 9 +-------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/charts/tidb-cluster/templates/_helpers.tpl b/charts/tidb-cluster/templates/_helpers.tpl index 263ce2532c..341afdb6e5 100644 --- a/charts/tidb-cluster/templates/_helpers.tpl +++ b/charts/tidb-cluster/templates/_helpers.tpl @@ -43,6 +43,7 @@ config-file: |- cert-path = "/var/lib/pd-tls/pd.crt" key-path = "/var/lib/pd-tls/pd.key" {{- end -}} + {{- end -}} {{- define "pd-configmap.data-digest" -}} @@ -65,6 +66,7 @@ config-file: |- cert-path = "/var/lib/tikv-tls/tikv.crt" key-path = "/var/lib/tikv-tls/tikv.key" {{- end -}} + {{- end -}} {{- define "tikv-configmap.data-digest" -}} @@ -89,14 +91,16 @@ config-file: |- [security] {{- end -}} {{- if .Values.enableTLSServer }} - cacert-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - cert-path = "/var/lib/pd-tls/pd.crt" - key-path = "/var/lib/pd-tls/pd.key" + cluster-ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + cluster-ssl-cert = "/var/lib/tidb-tls/tidb.crt" + cluster-ssl-key = "/var/lib/tidb-tls/tidb.key" {{- end -}} {{- if .Values.enableTLSClient }} + ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" ssl-cert = "/var/lib/tidb-tls/tidb.crt" ssl-key = "/var/lib/tidb-tls/tidb.key" {{- end -}} + {{- end -}} {{- define "tidb-configmap.data-digest" -}} diff --git a/charts/tidb-cluster/templates/scripts/_start_tidb.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_tidb.sh.tpl index 2f4ca49b59..ee79a7594e 100644 --- a/charts/tidb-cluster/templates/scripts/_start_tidb.sh.tpl +++ b/charts/tidb-cluster/templates/scripts/_start_tidb.sh.tpl @@ -26,16 +26,9 @@ then tail -f /dev/null fi -if [[ {{ .Values.enableTLSServer }} == "true" ]] -then - SCHEMA="https" -else - SCHEMA="http" -fi - ARGS="--store=tikv \ --host=0.0.0.0 \ ---path=$SCHEMA://${CLUSTER_NAME}-pd:2379 \ +--path=${CLUSTER_NAME}-pd:2379 \ --config=/etc/tidb/tidb.toml " From 926073e3b64ee7698551280b00336bef6f087556 Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Thu, 8 Aug 2019 15:16:40 +0800 Subject: [PATCH 07/17] tls: fix health check for tidb --- pkg/controller/tidb_control.go | 24 +++++++++++------------ pkg/discovery/discovery.go | 6 +++--- pkg/manager/member/tidb_member_manager.go | 9 +++++++-- pkg/pdapi/pdapi.go | 6 +++--- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/pkg/controller/tidb_control.go b/pkg/controller/tidb_control.go index 7730ee8e71..90f1357864 100644 --- a/pkg/controller/tidb_control.go +++ b/pkg/controller/tidb_control.go @@ -64,9 +64,9 @@ func (tdc *defaultTiDBControl) GetHealth(tc *v1alpha1.TidbCluster) map[string]bo tcName := tc.GetName() ns := tc.GetNamespace() - schema := "http" + scheme := "http" if tc.Spec.EnableTLSServer { - schema = "https" + scheme = "https" rootCAs, _ := httputil.ReadCACerts() config := &tls.Config{ RootCAs: rootCAs, @@ -77,7 +77,7 @@ func (tdc *defaultTiDBControl) GetHealth(tc *v1alpha1.TidbCluster) map[string]bo result := map[string]bool{} for i := 0; i < int(tc.TiDBRealReplicas()); i++ { hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), i) - url := fmt.Sprintf("%s://%s.%s.%s:10080/status", schema, hostName, TiDBPeerMemberName(tcName), ns) + url := fmt.Sprintf("%s://%s.%s.%s:10080/status", scheme, hostName, TiDBPeerMemberName(tcName), ns) _, err := tdc.getBodyOK(url) if err != nil { result[hostName] = false @@ -92,9 +92,9 @@ func (tdc *defaultTiDBControl) ResignDDLOwner(tc *v1alpha1.TidbCluster, ordinal tcName := tc.GetName() ns := tc.GetNamespace() - schema := "http" + scheme := "http" if tc.Spec.EnableTLSServer { - schema = "https" + scheme = "https" rootCAs, _ := httputil.ReadCACerts() config := &tls.Config{ RootCAs: rootCAs, @@ -103,7 +103,7 @@ func (tdc *defaultTiDBControl) ResignDDLOwner(tc *v1alpha1.TidbCluster, ordinal } hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal) - url := fmt.Sprintf("%s://%s.%s.%s:10080/ddl/owner/resign", schema, hostName, TiDBPeerMemberName(tcName), ns) + url := fmt.Sprintf("%s://%s.%s.%s:10080/ddl/owner/resign", scheme, hostName, TiDBPeerMemberName(tcName), ns) req, err := http.NewRequest("POST", url, nil) if err != nil { return false, err @@ -127,9 +127,9 @@ func (tdc *defaultTiDBControl) GetInfo(tc *v1alpha1.TidbCluster, ordinal int32) tcName := tc.GetName() ns := tc.GetNamespace() - schema := "http" + scheme := "http" if tc.Spec.EnableTLSServer { - schema = "https" + scheme = "https" rootCAs, _ := httputil.ReadCACerts() config := &tls.Config{ RootCAs: rootCAs, @@ -138,7 +138,7 @@ func (tdc *defaultTiDBControl) GetInfo(tc *v1alpha1.TidbCluster, ordinal int32) } hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal) - url := fmt.Sprintf("%s://%s.%s.%s:10080/info", schema, hostName, TiDBPeerMemberName(tcName), ns) + url := fmt.Sprintf("%s://%s.%s.%s:10080/info", scheme, hostName, TiDBPeerMemberName(tcName), ns) req, err := http.NewRequest("POST", url, nil) if err != nil { return nil, err @@ -168,9 +168,9 @@ func (tdc *defaultTiDBControl) GetSettings(tc *v1alpha1.TidbCluster, ordinal int tcName := tc.GetName() ns := tc.GetNamespace() - schema := "http" + scheme := "http" if tc.Spec.EnableTLSServer { - schema = "https" + scheme = "https" rootCAs, _ := httputil.ReadCACerts() config := &tls.Config{ RootCAs: rootCAs, @@ -179,7 +179,7 @@ func (tdc *defaultTiDBControl) GetSettings(tc *v1alpha1.TidbCluster, ordinal int } hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal) - url := fmt.Sprintf("%s://%s.%s.%s:10080/settings", schema, hostName, TiDBPeerMemberName(tcName), ns) + url := fmt.Sprintf("%s://%s.%s.%s:10080/settings", scheme, hostName, TiDBPeerMemberName(tcName), ns) req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err diff --git a/pkg/discovery/discovery.go b/pkg/discovery/discovery.go index 355702ba3a..9d4cc018e0 100644 --- a/pkg/discovery/discovery.go +++ b/pkg/discovery/discovery.go @@ -92,14 +92,14 @@ func (td *tidbDiscovery) Discover(advertisePeerUrl string) (string, error) { currentCluster = td.clusters[keyName] currentCluster.peers[podName] = struct{}{} - schema := "http" + scheme := "http" if tc.Spec.EnableTLSServer { - schema = "https" + scheme = "https" } if len(currentCluster.peers) == int(replicas) { delete(currentCluster.peers, podName) - return fmt.Sprintf("--initial-cluster=%s=%s://%s", podName, schema, advertisePeerUrl), nil + return fmt.Sprintf("--initial-cluster=%s=%s://%s", podName, scheme, advertisePeerUrl), nil } pdClient := td.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSServer) diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index 899ec86fff..c88c6b0389 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -313,6 +313,10 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust }, } + scheme := corev1.URISchemeHTTP + if tc.Spec.EnableTLSServer { + scheme = corev1.URISchemeHTTPS + } containers = append(containers, corev1.Container{ Name: v1alpha1.TiDBMemberType.String(), Image: tc.Spec.TiDB.Image, @@ -336,8 +340,9 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust ReadinessProbe: &corev1.Probe{ Handler: corev1.Handler{ HTTPGet: &corev1.HTTPGetAction{ - Path: "/status", - Port: intstr.FromInt(10080), + Path: "/status", + Port: intstr.FromInt(10080), + Scheme: scheme, }, }, InitialDelaySeconds: int32(10), diff --git a/pkg/pdapi/pdapi.go b/pkg/pdapi/pdapi.go index 1cc1503eb0..4d68928acd 100644 --- a/pkg/pdapi/pdapi.go +++ b/pkg/pdapi/pdapi.go @@ -75,11 +75,11 @@ func pdClientKey(namespace Namespace, clusterName string) string { // pdClientUrl builds the url of pd client func PdClientURL(namespace Namespace, clusterName string, tlsEnabled bool) string { - schema := "http" + scheme := "http" if tlsEnabled { - schema = "https" + scheme = "https" } - return fmt.Sprintf("%s://%s-pd.%s:2379", schema, clusterName, string(namespace)) + return fmt.Sprintf("%s://%s-pd.%s:2379", scheme, clusterName, string(namespace)) } // PDClient provides pd server's api From e4b319a518a9776ec38faff0fc27868ae491a945 Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Thu, 8 Aug 2019 16:53:15 +0800 Subject: [PATCH 08/17] tls: create new pd client when scheme changed --- pkg/pdapi/pdapi.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pkg/pdapi/pdapi.go b/pkg/pdapi/pdapi.go index 4d68928acd..b60e51379c 100644 --- a/pkg/pdapi/pdapi.go +++ b/pkg/pdapi/pdapi.go @@ -61,24 +61,25 @@ func NewDefaultPDControl() PDControlInterface { func (pdc *defaultPDControl) GetPDClient(namespace Namespace, tcName string, tlsEnabled bool) PDClient { pdc.mutex.Lock() defer pdc.mutex.Unlock() - key := pdClientKey(namespace, tcName) + + scheme := "http" + if tlsEnabled { + scheme = "https" + } + key := pdClientKey(scheme, namespace, tcName) if _, ok := pdc.pdClients[key]; !ok { - pdc.pdClients[key] = NewPDClient(PdClientURL(namespace, tcName, tlsEnabled), timeout, tlsEnabled) + pdc.pdClients[key] = NewPDClient(PdClientURL(namespace, tcName, scheme), timeout, tlsEnabled) } return pdc.pdClients[key] } // pdClientKey returns the pd client key -func pdClientKey(namespace Namespace, clusterName string) string { - return fmt.Sprintf("%s.%s", clusterName, string(namespace)) +func pdClientKey(scheme string, namespace Namespace, clusterName string) string { + return fmt.Sprintf("%s.%s.%s", scheme, clusterName, string(namespace)) } // pdClientUrl builds the url of pd client -func PdClientURL(namespace Namespace, clusterName string, tlsEnabled bool) string { - scheme := "http" - if tlsEnabled { - scheme = "https" - } +func PdClientURL(namespace Namespace, clusterName string, scheme string) string { return fmt.Sprintf("%s://%s-pd.%s:2379", scheme, clusterName, string(namespace)) } @@ -613,7 +614,7 @@ func NewFakePDControl() *FakePDControl { } func (fpc *FakePDControl) SetPDClient(namespace Namespace, tcName string, pdclient PDClient) { - fpc.defaultPDControl.pdClients[pdClientKey(namespace, tcName)] = pdclient + fpc.defaultPDControl.pdClients[pdClientKey("http", namespace, tcName)] = pdclient } type ActionType string From 4da9be38e45288f4a10815b8c186bc4f59a92791 Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Fri, 9 Aug 2019 11:42:25 +0800 Subject: [PATCH 09/17] tls: fix return value when loading CAs --- pkg/httputil/httputil.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/httputil/httputil.go b/pkg/httputil/httputil.go index 640d3475a1..f0da668fec 100644 --- a/pkg/httputil/httputil.go +++ b/pkg/httputil/httputil.go @@ -54,21 +54,22 @@ func GetBodyOK(httpClient *http.Client, apiURL string) ([]byte, error) { } func ReadCACerts() (*x509.CertPool, error) { - // load CA certs + // try to load system CA certs rootCAs, _ := x509.SystemCertPool() if rootCAs == nil { rootCAs = x509.NewCertPool() } + // load k8s CA cert caCert, err := ioutil.ReadFile(k8sCAFile) if err != nil { glog.Errorf("fail to read CA file %s, error: %v", k8sCAFile, err) return nil, err } if ok := rootCAs.AppendCertsFromPEM(caCert); !ok { - glog.Warningf("fail to append cert to pool, using system certs only") + glog.Warningf("fail to append CA file to pool, using system CAs only") } - return rootCAs, err + return rootCAs, nil } func ReadCerts() (*x509.CertPool, tls.Certificate, error) { From b6cd879cb99b095f848ac7398cbbbd8d9c1ace24 Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Fri, 9 Aug 2019 19:18:19 +0800 Subject: [PATCH 10/17] tls: fix test errors --- pkg/pdapi/pdapi_test.go | 20 ++++++++++---------- tests/pkg/webhook/pods.go | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/pdapi/pdapi_test.go b/pkg/pdapi/pdapi_test.go index 96a4157c92..c6f949a288 100644 --- a/pkg/pdapi/pdapi_test.go +++ b/pkg/pdapi/pdapi_test.go @@ -73,7 +73,7 @@ func TestHealth(t *testing.T) { }) defer svc.Close() - pdClient := NewPDClient(svc.URL, timeout) + pdClient := NewPDClient(svc.URL, timeout, false) result, err := pdClient.GetHealth() g.Expect(err).NotTo(HaveOccurred()) g.Expect(result).To(Equal(&HealthInfo{healths})) @@ -114,7 +114,7 @@ func TestGetConfig(t *testing.T) { }) defer svc.Close() - pdClient := NewPDClient(svc.URL, timeout) + pdClient := NewPDClient(svc.URL, timeout, false) result, err := pdClient.GetConfig() g.Expect(err).NotTo(HaveOccurred()) g.Expect(result).To(Equal(config)) @@ -152,7 +152,7 @@ func TestGetCluster(t *testing.T) { }) defer svc.Close() - pdClient := NewPDClient(svc.URL, timeout) + pdClient := NewPDClient(svc.URL, timeout, false) result, err := pdClient.GetCluster() g.Expect(err).NotTo(HaveOccurred()) g.Expect(result).To(Equal(cluster)) @@ -204,7 +204,7 @@ func TestGetMembers(t *testing.T) { }) defer svc.Close() - pdClient := NewPDClient(svc.URL, timeout) + pdClient := NewPDClient(svc.URL, timeout, false) result, err := pdClient.GetMembers() g.Expect(err).NotTo(HaveOccurred()) g.Expect(result).To(Equal(members)) @@ -256,7 +256,7 @@ func TestGetStores(t *testing.T) { }) defer svc.Close() - pdClient := NewPDClient(svc.URL, timeout) + pdClient := NewPDClient(svc.URL, timeout, false) result, err := pdClient.GetStores() g.Expect(err).NotTo(HaveOccurred()) g.Expect(result).To(Equal(stores)) @@ -301,7 +301,7 @@ func TestGetStore(t *testing.T) { }) defer svc.Close() - pdClient := NewPDClient(svc.URL, timeout) + pdClient := NewPDClient(svc.URL, timeout, false) result, err := pdClient.GetStore(tc.id) g.Expect(err).NotTo(HaveOccurred()) g.Expect(result).To(Equal(store)) @@ -350,7 +350,7 @@ func TestSetStoreLabels(t *testing.T) { }) defer svc.Close() - pdClient := NewPDClient(svc.URL, timeout) + pdClient := NewPDClient(svc.URL, timeout, false) result, _ := pdClient.SetStoreLabels(id, labels) g.Expect(result).To(Equal(tc.want)) } @@ -440,7 +440,7 @@ func TestDeleteMember(t *testing.T) { }) defer svc.Close() - pdClient := NewPDClient(svc.URL, timeout) + pdClient := NewPDClient(svc.URL, timeout, false) err := pdClient.DeleteMember(name) if tc.want { g.Expect(err).NotTo(HaveOccurred(), "check result") @@ -534,7 +534,7 @@ func TestDeleteMemberByID(t *testing.T) { }) defer svc.Close() - pdClient := NewPDClient(svc.URL, timeout) + pdClient := NewPDClient(svc.URL, timeout, false) err := pdClient.DeleteMemberByID(id) if tc.want { g.Expect(err).NotTo(HaveOccurred(), "check result") @@ -626,7 +626,7 @@ func TestDeleteStore(t *testing.T) { }) defer svc.Close() - pdClient := NewPDClient(svc.URL, timeout) + pdClient := NewPDClient(svc.URL, timeout, false) err := pdClient.DeleteStore(storeID) if tc.want { g.Expect(err).NotTo(HaveOccurred(), "check result") diff --git a/tests/pkg/webhook/pods.go b/tests/pkg/webhook/pods.go index 7b76692de9..5a72f00201 100644 --- a/tests/pkg/webhook/pods.go +++ b/tests/pkg/webhook/pods.go @@ -53,7 +53,7 @@ func admitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { return &reviewResponse } - pdClient := pdapi.NewDefaultPDControl().GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName()) + pdClient := pdapi.NewDefaultPDControl().GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), false) tidbController := controller.NewDefaultTiDBControl() // if pod is already deleting, return Allowed From 030a04c764ba3ad831baa9b93ded725aab067012 Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Mon, 12 Aug 2019 10:44:07 +0800 Subject: [PATCH 11/17] tls: fix typo in scripts --- .../tidb-cluster/templates/scripts/_start_pd.sh.tpl | 12 ++++++------ .../templates/scripts/_start_tikv.sh.tpl | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl index a3bcb91526..2db7a9ef5c 100644 --- a/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl +++ b/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl @@ -58,17 +58,17 @@ done if [[ {{ .Values.enableTLSServer }} == "true" ]] then - SCHEMA="https" + SCHEME="https" else - SCHEMA="http" + SCHEME="http" fi ARGS="--data-dir=/var/lib/pd \ --name=${HOSTNAME} \ ---peer-urls=$SCHEMA://0.0.0.0:2380 \ ---advertise-peer-urls=$SCHEMA://${domain}:2380 \ ---client-urls=$SCHEMA://0.0.0.0:2379 \ ---advertise-client-urls=$SCHEMA://${domain}:2379 \ +--peer-urls=$SCHEME://0.0.0.0:2380 \ +--advertise-peer-urls=$SCHEME://${domain}:2380 \ +--client-urls=$SCHEME://0.0.0.0:2379 \ +--advertise-client-urls=$SCHEME://${domain}:2379 \ --config=/etc/pd/pd.toml \ " diff --git a/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl index f0d9718ad2..943aa6e73e 100644 --- a/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl +++ b/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl @@ -30,12 +30,12 @@ fi if [[ {{ .Values.enableTLSServer }} == "true" ]] then - SCHEMA="https" + SCHEME="https" else - SCHEMA="http" + SCHEME="http" fi -ARGS="--pd=$SCHEMA://${CLUSTER_NAME}-pd:2379 \ +ARGS="--pd=$SCHEME://${CLUSTER_NAME}-pd:2379 \ --advertise-addr=${HOSTNAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:20160 \ --addr=0.0.0.0:20160 \ --data-dir=/var/lib/tikv \ From 4b27ddd3df58acb48ec4dfc8f5a4a97e0d40facb Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Sat, 17 Aug 2019 15:41:47 +0800 Subject: [PATCH 12/17] fix test errors --- tests/actions.go | 2 +- tests/failover.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/actions.go b/tests/actions.go index 2ac1119048..694fbbfb3c 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -991,7 +991,7 @@ func (oa *operatorActions) CheckUpgrade(ctx context.Context, info *TidbClusterCo glog.Errorf("failed to get tidbcluster: %s/%s, %v", ns, tcName, err) continue } - pdClient := pdapi.NewDefaultPDControl().GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName()) + pdClient := pdapi.NewDefaultPDControl().GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), false) replicas := tc.TiKVRealReplicas() for i := replicas - 1; i >= 0; i-- { diff --git a/tests/failover.go b/tests/failover.go index 6dcbb89d09..ad4598b000 100644 --- a/tests/failover.go +++ b/tests/failover.go @@ -38,7 +38,7 @@ func (oa *operatorActions) TruncateSSTFileThenCheckFailover(info *TidbClusterCon } // checkout pd config - pdCfg, err := oa.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName()).GetConfig() + pdCfg, err := oa.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), false).GetConfig() if err != nil { glog.Errorf("failed to get the pd config: tc=%s err=%s", info.ClusterName, err.Error()) return err From 57ff5971d3b8bdcba98c56ce99234c4e2a0dc0fc Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Wed, 21 Aug 2019 11:16:06 +0800 Subject: [PATCH 13/17] tls: change config name and use better layout --- charts/tidb-cluster/templates/_helpers.tpl | 4 ++-- charts/tidb-cluster/values.yaml | 14 +++++++------- pkg/apis/pingcap.com/v1alpha1/types.go | 11 +++++------ pkg/controller/pd_control.go | 2 +- pkg/controller/pod_control.go | 2 +- pkg/controller/tidb_control.go | 8 ++++---- pkg/discovery/discovery.go | 4 ++-- pkg/manager/member/pd_member_manager.go | 4 ++-- pkg/manager/member/tidb_member_manager.go | 6 +++--- pkg/manager/member/tikv_member_manager.go | 4 ++-- pkg/manager/member/tikv_upgrader.go | 2 +- 11 files changed, 30 insertions(+), 31 deletions(-) diff --git a/charts/tidb-cluster/templates/_helpers.tpl b/charts/tidb-cluster/templates/_helpers.tpl index 371a27a8e7..ccc5f969b3 100644 --- a/charts/tidb-cluster/templates/_helpers.tpl +++ b/charts/tidb-cluster/templates/_helpers.tpl @@ -90,12 +90,12 @@ config-file: |- {{- if or .Values.enableTLSServer .Values.enableTLSClient }} [security] {{- end -}} - {{- if .Values.enableTLSServer }} + {{- if .Values.enableTLSCluster }} cluster-ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" cluster-ssl-cert = "/var/lib/tidb-tls/tidb.crt" cluster-ssl-key = "/var/lib/tidb-tls/tidb.key" {{- end -}} - {{- if .Values.enableTLSClient }} + {{- if .Values.tidb.enableTLSClient }} ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" ssl-cert = "/var/lib/tidb-tls/tidb.crt" ssl-key = "/var/lib/tidb-tls/tidb.key" diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index 9c04133f7f..aff79d2082 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -50,13 +50,7 @@ enableConfigMapRollout: true # Whether enable TLS connections between server nodes. # When enabled, PD/TiDB/TiKV will use TLS encrypted connections to transfer data between each node, # certificates will be generated automatically (if not already present). -enableTLSServer: false - -# Whether enable TLS connection between TiDB server and MySQL client. -# When enabled, TiDB will accept TLS encrypted connections from MySQL client, certificates will be generated -# automatically. -# Note: TLS connection is not forced on the server side, plain connections are also accepted after enableing. -enableTLSClient: false +enableTLSCluster: false pd: # Please refer to https://github.com/pingcap/pd/blob/master/conf/config.toml for the default @@ -334,6 +328,12 @@ tidb: # the start argument to specify the plugin id (name "-" version) that needs to be loaded, e.g. 'conn_limit-1'. list: ["whitelist-1"] + # Whether enable TLS connection between TiDB server and MySQL client. + # When enabled, TiDB will accept TLS encrypted connections from MySQL client, certificates will be generated + # automatically. + # Note: TLS connection is not forced on the server side, plain connections are also accepted after enableing. + enableTLSClient: false + # mysqlClient is used to set password for TiDB # it must has Python MySQL client installed mysqlClient: diff --git a/pkg/apis/pingcap.com/v1alpha1/types.go b/pkg/apis/pingcap.com/v1alpha1/types.go index 0a88e16778..1955604371 100644 --- a/pkg/apis/pingcap.com/v1alpha1/types.go +++ b/pkg/apis/pingcap.com/v1alpha1/types.go @@ -94,9 +94,7 @@ type TidbClusterSpec struct { PVReclaimPolicy corev1.PersistentVolumeReclaimPolicy `json:"pvReclaimPolicy,omitempty"` Timezone string `json:"timezone,omitempty"` // Enable TLS connection between TiDB server compoments - EnableTLSServer bool `json:"enableTLSServer,omitempty"` - // Accept TLS connection from client - EnableTLSClient bool `json:"enableTLSClient,omitempty"` + EnableTLSCluster bool `json:"enableTLSServer,omitempty"` } // TidbClusterStatus represents the current status of a tidb cluster. @@ -107,7 +105,7 @@ type TidbClusterStatus struct { TiDB TiDBStatus `json:"tidb,omitempty"` } -// PDSpec contains details of PD member +// PDSpec contains details of PD members type PDSpec struct { ContainerSpec Replicas int32 `json:"replicas"` @@ -118,7 +116,7 @@ type PDSpec struct { Annotations map[string]string `json:"annotations,omitempty"` } -// TiDBSpec contains details of PD member +// TiDBSpec contains details of TiDB members type TiDBSpec struct { ContainerSpec Replicas int32 `json:"replicas"` @@ -131,6 +129,7 @@ type TiDBSpec struct { MaxFailoverCount int32 `json:"maxFailoverCount,omitempty"` SeparateSlowLog bool `json:"separateSlowLog,omitempty"` SlowLogTailer TiDBSlowLogTailerSpec `json:"slowLogTailer,omitempty"` + EnableTLSClient bool `json:"enableTLSClient,omitempty"` } // TiDBSlowLogTailerSpec represents an optional log tailer sidecar with TiDB @@ -138,7 +137,7 @@ type TiDBSlowLogTailerSpec struct { ContainerSpec } -// TiKVSpec contains details of PD member +// TiKVSpec contains details of TiKV members type TiKVSpec struct { ContainerSpec Privileged bool `json:"privileged,omitempty"` diff --git a/pkg/controller/pd_control.go b/pkg/controller/pd_control.go index 2bd1c4cf10..715eb3d03e 100644 --- a/pkg/controller/pd_control.go +++ b/pkg/controller/pd_control.go @@ -20,7 +20,7 @@ import ( // GetPDClient gets the pd client from the TidbCluster func GetPDClient(pdControl pdapi.PDControlInterface, tc *v1alpha1.TidbCluster) pdapi.PDClient { - return pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSServer) + return pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSCluster) } // NewFakePDClient creates a fake pdclient that is set as the pd client diff --git a/pkg/controller/pod_control.go b/pkg/controller/pod_control.go index 07fba867bf..e80f95105a 100644 --- a/pkg/controller/pod_control.go +++ b/pkg/controller/pod_control.go @@ -112,7 +112,7 @@ func (rpc *realPodControl) UpdateMetaInfo(tc *v1alpha1.TidbCluster, pod *corev1. memberID := labels[label.MemberIDLabelKey] storeID := labels[label.StoreIDLabelKey] - pdClient := rpc.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tcName, tc.Spec.EnableTLSServer) + pdClient := rpc.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tcName, tc.Spec.EnableTLSCluster) if labels[label.ClusterIDLabelKey] == "" { cluster, err := pdClient.GetCluster() if err != nil { diff --git a/pkg/controller/tidb_control.go b/pkg/controller/tidb_control.go index 90f1357864..3e5e809719 100644 --- a/pkg/controller/tidb_control.go +++ b/pkg/controller/tidb_control.go @@ -65,7 +65,7 @@ func (tdc *defaultTiDBControl) GetHealth(tc *v1alpha1.TidbCluster) map[string]bo ns := tc.GetNamespace() scheme := "http" - if tc.Spec.EnableTLSServer { + if tc.Spec.EnableTLSCluster { scheme = "https" rootCAs, _ := httputil.ReadCACerts() config := &tls.Config{ @@ -93,7 +93,7 @@ func (tdc *defaultTiDBControl) ResignDDLOwner(tc *v1alpha1.TidbCluster, ordinal ns := tc.GetNamespace() scheme := "http" - if tc.Spec.EnableTLSServer { + if tc.Spec.EnableTLSCluster { scheme = "https" rootCAs, _ := httputil.ReadCACerts() config := &tls.Config{ @@ -128,7 +128,7 @@ func (tdc *defaultTiDBControl) GetInfo(tc *v1alpha1.TidbCluster, ordinal int32) ns := tc.GetNamespace() scheme := "http" - if tc.Spec.EnableTLSServer { + if tc.Spec.EnableTLSCluster { scheme = "https" rootCAs, _ := httputil.ReadCACerts() config := &tls.Config{ @@ -169,7 +169,7 @@ func (tdc *defaultTiDBControl) GetSettings(tc *v1alpha1.TidbCluster, ordinal int ns := tc.GetNamespace() scheme := "http" - if tc.Spec.EnableTLSServer { + if tc.Spec.EnableTLSCluster { scheme = "https" rootCAs, _ := httputil.ReadCACerts() config := &tls.Config{ diff --git a/pkg/discovery/discovery.go b/pkg/discovery/discovery.go index 9d4cc018e0..3ad79bacfa 100644 --- a/pkg/discovery/discovery.go +++ b/pkg/discovery/discovery.go @@ -93,7 +93,7 @@ func (td *tidbDiscovery) Discover(advertisePeerUrl string) (string, error) { currentCluster.peers[podName] = struct{}{} scheme := "http" - if tc.Spec.EnableTLSServer { + if tc.Spec.EnableTLSCluster { scheme = "https" } @@ -102,7 +102,7 @@ func (td *tidbDiscovery) Discover(advertisePeerUrl string) (string, error) { return fmt.Sprintf("--initial-cluster=%s=%s://%s", podName, scheme, advertisePeerUrl), nil } - pdClient := td.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSServer) + pdClient := td.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSCluster) membersInfo, err := pdClient.GetMembers() if err != nil { return "", err diff --git a/pkg/manager/member/pd_member_manager.go b/pkg/manager/member/pd_member_manager.go index 7707e6e97b..898020aef4 100644 --- a/pkg/manager/member/pd_member_manager.go +++ b/pkg/manager/member/pd_member_manager.go @@ -437,7 +437,7 @@ func (pmm *pdMemberManager) getNewPDSetForTidbCluster(tc *v1alpha1.TidbCluster) {Name: "startup-script", ReadOnly: true, MountPath: "/usr/local/bin"}, {Name: v1alpha1.PDMemberType.String(), MountPath: "/var/lib/pd"}, } - if tc.Spec.EnableTLSServer { + if tc.Spec.EnableTLSCluster { volMounts = append(volMounts, corev1.VolumeMount{ Name: "pd-tls", ReadOnly: true, MountPath: "/var/lib/pd-tls", }) @@ -466,7 +466,7 @@ func (pmm *pdMemberManager) getNewPDSetForTidbCluster(tc *v1alpha1.TidbCluster) }, }, } - if tc.Spec.EnableTLSServer { + if tc.Spec.EnableTLSCluster { vols = append(vols, corev1.Volume{ Name: "pd-tls", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index c88c6b0389..e017e65f77 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -229,7 +229,7 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust {Name: "config", ReadOnly: true, MountPath: "/etc/tidb"}, {Name: "startup-script", ReadOnly: true, MountPath: "/usr/local/bin"}, } - if tc.Spec.EnableTLSServer { + if tc.Spec.EnableTLSCluster { volMounts = append(volMounts, corev1.VolumeMount{ Name: "tidb-tls", ReadOnly: true, MountPath: "/var/lib/tidb-tls", }) @@ -254,7 +254,7 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust }}, }, } - if tc.Spec.EnableTLSServer { + if tc.Spec.EnableTLSCluster { vols = append(vols, corev1.Volume{ Name: "tidb-tls", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ @@ -314,7 +314,7 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust } scheme := corev1.URISchemeHTTP - if tc.Spec.EnableTLSServer { + if tc.Spec.EnableTLSCluster { scheme = corev1.URISchemeHTTPS } containers = append(containers, corev1.Container{ diff --git a/pkg/manager/member/tikv_member_manager.go b/pkg/manager/member/tikv_member_manager.go index acf1b2c5a2..9c15854736 100644 --- a/pkg/manager/member/tikv_member_manager.go +++ b/pkg/manager/member/tikv_member_manager.go @@ -279,7 +279,7 @@ func (tkmm *tikvMemberManager) getNewSetForTidbCluster(tc *v1alpha1.TidbCluster) {Name: "config", ReadOnly: true, MountPath: "/etc/tikv"}, {Name: "startup-script", ReadOnly: true, MountPath: "/usr/local/bin"}, } - if tc.Spec.EnableTLSServer { + if tc.Spec.EnableTLSCluster { volMounts = append(volMounts, corev1.VolumeMount{ Name: "tikv-tls", ReadOnly: true, MountPath: "/var/lib/tikv-tls", }) @@ -304,7 +304,7 @@ func (tkmm *tikvMemberManager) getNewSetForTidbCluster(tc *v1alpha1.TidbCluster) }}, }, } - if tc.Spec.EnableTLSServer { + if tc.Spec.EnableTLSCluster { vols = append(vols, corev1.Volume{ Name: "tikv-tls", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ diff --git a/pkg/manager/member/tikv_upgrader.go b/pkg/manager/member/tikv_upgrader.go index b7f35453d5..c325795a13 100644 --- a/pkg/manager/member/tikv_upgrader.go +++ b/pkg/manager/member/tikv_upgrader.go @@ -196,7 +196,7 @@ func (tku *tikvUpgrader) endEvictLeader(tc *v1alpha1.TidbCluster, ordinal int32) return err } - err = tku.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSServer).EndEvictLeader(storeID) + err = tku.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSCluster).EndEvictLeader(storeID) if err != nil { glog.Errorf("tikv upgrader: failed to end evict leader storeID: %d ordinal: %d, %v", storeID, ordinal, err) return err From d8e982e4ccc94e12fcbfa692c399b1d9e8cf6b67 Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Wed, 21 Aug 2019 12:09:25 +0800 Subject: [PATCH 14/17] tls: refine scheme detection --- .../templates/scripts/_start_pd.sh.tpl | 7 +- .../templates/scripts/_start_tikv.sh.tpl | 7 +- pkg/apis/pingcap.com/v1alpha1/tidbcluster.go | 7 ++ pkg/controller/tidb_control.go | 66 ++++++++----------- pkg/discovery/discovery.go | 7 +- pkg/httputil/httputil.go | 5 +- 6 files changed, 40 insertions(+), 59 deletions(-) diff --git a/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl index 2db7a9ef5c..40610917a3 100644 --- a/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl +++ b/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl @@ -56,12 +56,7 @@ while true; do fi done -if [[ {{ .Values.enableTLSServer }} == "true" ]] -then - SCHEME="https" -else - SCHEME="http" -fi +SCHEME={{ if .Values.enableTLSCluster }}"https"{{ else }}"http"{{ end }} ARGS="--data-dir=/var/lib/pd \ --name=${HOSTNAME} \ diff --git a/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl index 943aa6e73e..14b1a7101f 100644 --- a/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl +++ b/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl @@ -28,12 +28,7 @@ then tail -f /dev/null fi -if [[ {{ .Values.enableTLSServer }} == "true" ]] -then - SCHEME="https" -else - SCHEME="http" -fi +SCHEME={{ if .Values.enableTLSCluster }}"https"{{ else }}"http"{{ end }} ARGS="--pd=$SCHEME://${CLUSTER_NAME}-pd:2379 \ --advertise-addr=${HOSTNAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:20160 \ diff --git a/pkg/apis/pingcap.com/v1alpha1/tidbcluster.go b/pkg/apis/pingcap.com/v1alpha1/tidbcluster.go index 47c45b0279..8d999cced3 100644 --- a/pkg/apis/pingcap.com/v1alpha1/tidbcluster.go +++ b/pkg/apis/pingcap.com/v1alpha1/tidbcluster.go @@ -158,3 +158,10 @@ func (tc *TidbCluster) TiKVIsAvailable() bool { func (tc *TidbCluster) GetClusterID() string { return tc.Status.ClusterID } + +func (tc *TidbCluster) Scheme() string { + if tc.Spec.EnableTLSCluster { + return "https" + } + return "http" +} diff --git a/pkg/controller/tidb_control.go b/pkg/controller/tidb_control.go index 3e5e809719..ef0aefca27 100644 --- a/pkg/controller/tidb_control.go +++ b/pkg/controller/tidb_control.go @@ -15,12 +15,14 @@ package controller import ( "crypto/tls" + "crypto/x509" "encoding/json" "fmt" "io/ioutil" "net/http" "time" + "github.com/golang/glog" "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" "github.com/pingcap/tidb-operator/pkg/httputil" "github.com/pingcap/tidb/config" @@ -52,27 +54,35 @@ type TiDBControlInterface interface { // defaultTiDBControl is default implementation of TiDBControlInterface. type defaultTiDBControl struct { httpClient *http.Client + rootCAs *x509.CertPool } // NewDefaultTiDBControl returns a defaultTiDBControl instance func NewDefaultTiDBControl() TiDBControlInterface { httpClient := &http.Client{Timeout: timeout} - return &defaultTiDBControl{httpClient: httpClient} -} -func (tdc *defaultTiDBControl) GetHealth(tc *v1alpha1.TidbCluster) map[string]bool { - tcName := tc.GetName() - ns := tc.GetNamespace() + rootCAs, err := httputil.ReadCACerts() + if err != nil { + glog.Errorf("fail to load CA certs, use plain connections only, TLS enabled clusters may not work") + } + + return &defaultTiDBControl{httpClient: httpClient, rootCAs: rootCAs} +} - scheme := "http" - if tc.Spec.EnableTLSCluster { - scheme = "https" - rootCAs, _ := httputil.ReadCACerts() +func (tdc *defaultTiDBControl) useTLSHTTPClient(enableTLS bool) { + if enableTLS { config := &tls.Config{ - RootCAs: rootCAs, + RootCAs: tdc.rootCAs, } tdc.httpClient.Transport = &http.Transport{TLSClientConfig: config} } +} + +func (tdc *defaultTiDBControl) GetHealth(tc *v1alpha1.TidbCluster) map[string]bool { + tcName := tc.GetName() + ns := tc.GetNamespace() + scheme := tc.Scheme() + tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster) result := map[string]bool{} for i := 0; i < int(tc.TiDBRealReplicas()); i++ { @@ -91,16 +101,8 @@ func (tdc *defaultTiDBControl) GetHealth(tc *v1alpha1.TidbCluster) map[string]bo func (tdc *defaultTiDBControl) ResignDDLOwner(tc *v1alpha1.TidbCluster, ordinal int32) (bool, error) { tcName := tc.GetName() ns := tc.GetNamespace() - - scheme := "http" - if tc.Spec.EnableTLSCluster { - scheme = "https" - rootCAs, _ := httputil.ReadCACerts() - config := &tls.Config{ - RootCAs: rootCAs, - } - tdc.httpClient.Transport = &http.Transport{TLSClientConfig: config} - } + scheme := tc.Scheme() + tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster) hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal) url := fmt.Sprintf("%s://%s.%s.%s:10080/ddl/owner/resign", scheme, hostName, TiDBPeerMemberName(tcName), ns) @@ -126,16 +128,8 @@ func (tdc *defaultTiDBControl) ResignDDLOwner(tc *v1alpha1.TidbCluster, ordinal func (tdc *defaultTiDBControl) GetInfo(tc *v1alpha1.TidbCluster, ordinal int32) (*dbInfo, error) { tcName := tc.GetName() ns := tc.GetNamespace() - - scheme := "http" - if tc.Spec.EnableTLSCluster { - scheme = "https" - rootCAs, _ := httputil.ReadCACerts() - config := &tls.Config{ - RootCAs: rootCAs, - } - tdc.httpClient.Transport = &http.Transport{TLSClientConfig: config} - } + scheme := tc.Scheme() + tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster) hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal) url := fmt.Sprintf("%s://%s.%s.%s:10080/info", scheme, hostName, TiDBPeerMemberName(tcName), ns) @@ -167,16 +161,8 @@ func (tdc *defaultTiDBControl) GetInfo(tc *v1alpha1.TidbCluster, ordinal int32) func (tdc *defaultTiDBControl) GetSettings(tc *v1alpha1.TidbCluster, ordinal int32) (*config.Config, error) { tcName := tc.GetName() ns := tc.GetNamespace() - - scheme := "http" - if tc.Spec.EnableTLSCluster { - scheme = "https" - rootCAs, _ := httputil.ReadCACerts() - config := &tls.Config{ - RootCAs: rootCAs, - } - tdc.httpClient.Transport = &http.Transport{TLSClientConfig: config} - } + scheme := tc.Scheme() + tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster) hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal) url := fmt.Sprintf("%s://%s.%s.%s:10080/settings", scheme, hostName, TiDBPeerMemberName(tcName), ns) diff --git a/pkg/discovery/discovery.go b/pkg/discovery/discovery.go index 3ad79bacfa..f4e45df934 100644 --- a/pkg/discovery/discovery.go +++ b/pkg/discovery/discovery.go @@ -92,14 +92,9 @@ func (td *tidbDiscovery) Discover(advertisePeerUrl string) (string, error) { currentCluster = td.clusters[keyName] currentCluster.peers[podName] = struct{}{} - scheme := "http" - if tc.Spec.EnableTLSCluster { - scheme = "https" - } - if len(currentCluster.peers) == int(replicas) { delete(currentCluster.peers, podName) - return fmt.Sprintf("--initial-cluster=%s=%s://%s", podName, scheme, advertisePeerUrl), nil + return fmt.Sprintf("--initial-cluster=%s=%s://%s", podName, tc.Scheme(), advertisePeerUrl), nil } pdClient := td.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSCluster) diff --git a/pkg/httputil/httputil.go b/pkg/httputil/httputil.go index f0da668fec..5bf6d65524 100644 --- a/pkg/httputil/httputil.go +++ b/pkg/httputil/httputil.go @@ -55,7 +55,10 @@ func GetBodyOK(httpClient *http.Client, apiURL string) ([]byte, error) { func ReadCACerts() (*x509.CertPool, error) { // try to load system CA certs - rootCAs, _ := x509.SystemCertPool() + rootCAs, err := x509.SystemCertPool() + if err != nil { + return nil, err + } if rootCAs == nil { rootCAs = x509.NewCertPool() } From 4f9bea57e2a6c58b4485d92fb6e9d2b4be70bf32 Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Wed, 21 Aug 2019 14:13:50 +0800 Subject: [PATCH 15/17] tls: fix keys in templates --- charts/tidb-cluster/templates/_helpers.tpl | 6 +++--- charts/tidb-cluster/templates/discovery-deployment.yaml | 2 +- charts/tidb-cluster/templates/tidb-cluster.yaml | 2 +- pkg/apis/pingcap.com/v1alpha1/types.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/tidb-cluster/templates/_helpers.tpl b/charts/tidb-cluster/templates/_helpers.tpl index ccc5f969b3..2fa3adac3d 100644 --- a/charts/tidb-cluster/templates/_helpers.tpl +++ b/charts/tidb-cluster/templates/_helpers.tpl @@ -37,7 +37,7 @@ config-file: |- {{- if .Values.pd.config }} {{ .Values.pd.config | indent 2 }} {{- end -}} - {{- if .Values.enableTLSServer }} + {{- if .Values.enableTLSCluster }} [security] cacert-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" cert-path = "/var/lib/pd-tls/pd.crt" @@ -60,7 +60,7 @@ config-file: |- {{- if .Values.tikv.config }} {{ .Values.tikv.config | indent 2 }} {{- end -}} - {{- if .Values.enableTLSServer }} + {{- if .Values.enableTLSCluster }} [security] ca-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" cert-path = "/var/lib/tikv-tls/tikv.crt" @@ -87,7 +87,7 @@ config-file: |- {{- if .Values.tidb.config }} {{ .Values.tidb.config | indent 2 }} {{- end -}} - {{- if or .Values.enableTLSServer .Values.enableTLSClient }} + {{- if or .Values.enableTLSCluster .Values.enableTLSClient }} [security] {{- end -}} {{- if .Values.enableTLSCluster }} diff --git a/charts/tidb-cluster/templates/discovery-deployment.yaml b/charts/tidb-cluster/templates/discovery-deployment.yaml index da0eb7451b..137d0a9abc 100644 --- a/charts/tidb-cluster/templates/discovery-deployment.yaml +++ b/charts/tidb-cluster/templates/discovery-deployment.yaml @@ -45,7 +45,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace -{{- if .Values.enableTLSServer }} +{{- if .Values.enableTLSCluster }} volumeMounts: - mountPath: /var/lib/tls name: tls diff --git a/charts/tidb-cluster/templates/tidb-cluster.yaml b/charts/tidb-cluster/templates/tidb-cluster.yaml index 0e234af3a8..329d93abb0 100644 --- a/charts/tidb-cluster/templates/tidb-cluster.yaml +++ b/charts/tidb-cluster/templates/tidb-cluster.yaml @@ -20,7 +20,7 @@ metadata: spec: pvReclaimPolicy: {{ .Values.pvReclaimPolicy }} timezone: {{ .Values.timezone | default "UTC" }} - enableTLSServer: {{ .Values.enableTLSServer | default false }} + enableTLSCluster: {{ .Values.enableTLSCluster | default false }} enableTLSClient: {{ .Values.enableTLSClient | default false }} services: {{ toYaml .Values.services | indent 4 }} diff --git a/pkg/apis/pingcap.com/v1alpha1/types.go b/pkg/apis/pingcap.com/v1alpha1/types.go index 1955604371..8c080cd8fb 100644 --- a/pkg/apis/pingcap.com/v1alpha1/types.go +++ b/pkg/apis/pingcap.com/v1alpha1/types.go @@ -94,7 +94,7 @@ type TidbClusterSpec struct { PVReclaimPolicy corev1.PersistentVolumeReclaimPolicy `json:"pvReclaimPolicy,omitempty"` Timezone string `json:"timezone,omitempty"` // Enable TLS connection between TiDB server compoments - EnableTLSCluster bool `json:"enableTLSServer,omitempty"` + EnableTLSCluster bool `json:"enableTLSCluster,omitempty"` } // TidbClusterStatus represents the current status of a tidb cluster. From 04a9b59d8d80a245de991d7f0de0396913d004bc Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Wed, 21 Aug 2019 14:40:45 +0800 Subject: [PATCH 16/17] tls: fix test cases --- pkg/httputil/httputil.go | 3 +++ tests/actions.go | 2 +- tests/failover.go | 2 +- tests/pkg/webhook/pods.go | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/httputil/httputil.go b/pkg/httputil/httputil.go index 5bf6d65524..503b4a6e3f 100644 --- a/pkg/httputil/httputil.go +++ b/pkg/httputil/httputil.go @@ -77,6 +77,9 @@ func ReadCACerts() (*x509.CertPool, error) { func ReadCerts() (*x509.CertPool, tls.Certificate, error) { rootCAs, err := ReadCACerts() + if err != nil { + return rootCAs, nil, err + } // load client cert cert, err := tls.LoadX509KeyPair(clientCert, clientKey) diff --git a/tests/actions.go b/tests/actions.go index 943484ea85..794f538091 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -994,7 +994,7 @@ func (oa *operatorActions) CheckUpgrade(ctx context.Context, info *TidbClusterCo glog.Errorf("failed to get tidbcluster: %s/%s, %v", ns, tcName, err) continue } - pdClient := pdapi.NewDefaultPDControl().GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), false) + pdClient := pdapi.NewDefaultPDControl().GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSCluster) replicas := tc.TiKVRealReplicas() for i := replicas - 1; i >= 0; i-- { diff --git a/tests/failover.go b/tests/failover.go index ad4598b000..e1d0be82a6 100644 --- a/tests/failover.go +++ b/tests/failover.go @@ -38,7 +38,7 @@ func (oa *operatorActions) TruncateSSTFileThenCheckFailover(info *TidbClusterCon } // checkout pd config - pdCfg, err := oa.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), false).GetConfig() + pdCfg, err := oa.pdControl.GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSCluster).GetConfig() if err != nil { glog.Errorf("failed to get the pd config: tc=%s err=%s", info.ClusterName, err.Error()) return err diff --git a/tests/pkg/webhook/pods.go b/tests/pkg/webhook/pods.go index 5a72f00201..be6dc20e0e 100644 --- a/tests/pkg/webhook/pods.go +++ b/tests/pkg/webhook/pods.go @@ -53,7 +53,7 @@ func admitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { return &reviewResponse } - pdClient := pdapi.NewDefaultPDControl().GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), false) + pdClient := pdapi.NewDefaultPDControl().GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSCluster) tidbController := controller.NewDefaultTiDBControl() // if pod is already deleting, return Allowed From 7d9d03b7368ac0cc8423934af9fdfcdd2a996d4b Mon Sep 17 00:00:00 2001 From: Allen Zhong Date: Tue, 27 Aug 2019 18:37:57 +0800 Subject: [PATCH 17/17] tls: update coding styles --- .../templates/scripts/_start_pd.sh.tpl | 8 ++-- .../templates/scripts/_start_tikv.sh.tpl | 2 +- pkg/controller/tidb_control.go | 39 +++++++++++-------- pkg/httputil/httputil.go | 3 +- pkg/manager/member/pd_member_manager.go | 2 +- pkg/manager/member/tidb_member_manager.go | 2 +- pkg/manager/member/tikv_member_manager.go | 2 +- 7 files changed, 31 insertions(+), 27 deletions(-) diff --git a/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl index 71b5235583..7c1747d7dc 100644 --- a/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl +++ b/charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl @@ -62,10 +62,10 @@ SCHEME={{ if .Values.enableTLSCluster }}"https"{{ else }}"http"{{ end }} ARGS="--data-dir=/var/lib/pd \ --name=${POD_NAME} \ ---peer-urls=$SCHEME://0.0.0.0:2380 \ ---advertise-peer-urls=$SCHEME://${domain}:2380 \ ---client-urls=$SCHEME://0.0.0.0:2379 \ ---advertise-client-urls=$SCHEME://${domain}:2379 \ +--peer-urls=${SCHEME}://0.0.0.0:2380 \ +--advertise-peer-urls=${SCHEME}://${domain}:2380 \ +--client-urls=${SCHEME}://0.0.0.0:2379 \ +--advertise-client-urls=${SCHEME}://${domain}:2379 \ --config=/etc/pd/pd.toml \ " diff --git a/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl index 8b97789e9a..5c598d63d5 100644 --- a/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl +++ b/charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl @@ -32,7 +32,7 @@ SCHEME={{ if .Values.enableTLSCluster }}"https"{{ else }}"http"{{ end }} # Use HOSTNAME if POD_NAME is unset for backward compatibility. POD_NAME=${POD_NAME:-$HOSTNAME} -ARGS="--pd=$SCHEME://${CLUSTER_NAME}-pd:2379 \ +ARGS="--pd=${SCHEME}://${CLUSTER_NAME}-pd:2379 \ --advertise-addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:20160 \ --addr=0.0.0.0:20160 \ --status-addr=0.0.0.0:20180 \ diff --git a/pkg/controller/tidb_control.go b/pkg/controller/tidb_control.go index ef0aefca27..f9e150b787 100644 --- a/pkg/controller/tidb_control.go +++ b/pkg/controller/tidb_control.go @@ -15,14 +15,12 @@ package controller import ( "crypto/tls" - "crypto/x509" "encoding/json" "fmt" "io/ioutil" "net/http" "time" - "github.com/golang/glog" "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" "github.com/pingcap/tidb-operator/pkg/httputil" "github.com/pingcap/tidb/config" @@ -54,37 +52,38 @@ type TiDBControlInterface interface { // defaultTiDBControl is default implementation of TiDBControlInterface. type defaultTiDBControl struct { httpClient *http.Client - rootCAs *x509.CertPool } // NewDefaultTiDBControl returns a defaultTiDBControl instance func NewDefaultTiDBControl() TiDBControlInterface { - httpClient := &http.Client{Timeout: timeout} - - rootCAs, err := httputil.ReadCACerts() - if err != nil { - glog.Errorf("fail to load CA certs, use plain connections only, TLS enabled clusters may not work") - } - - return &defaultTiDBControl{httpClient: httpClient, rootCAs: rootCAs} + return &defaultTiDBControl{httpClient: &http.Client{Timeout: timeout}} } -func (tdc *defaultTiDBControl) useTLSHTTPClient(enableTLS bool) { +func (tdc *defaultTiDBControl) useTLSHTTPClient(enableTLS bool) error { if enableTLS { + rootCAs, err := httputil.ReadCACerts() + if err != nil { + return err + } config := &tls.Config{ - RootCAs: tdc.rootCAs, + RootCAs: rootCAs, } tdc.httpClient.Transport = &http.Transport{TLSClientConfig: config} } + return nil } func (tdc *defaultTiDBControl) GetHealth(tc *v1alpha1.TidbCluster) map[string]bool { tcName := tc.GetName() ns := tc.GetNamespace() scheme := tc.Scheme() - tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster) result := map[string]bool{} + + if err := tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster); err != nil { + return result + } + for i := 0; i < int(tc.TiDBRealReplicas()); i++ { hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), i) url := fmt.Sprintf("%s://%s.%s.%s:10080/status", scheme, hostName, TiDBPeerMemberName(tcName), ns) @@ -102,7 +101,9 @@ func (tdc *defaultTiDBControl) ResignDDLOwner(tc *v1alpha1.TidbCluster, ordinal tcName := tc.GetName() ns := tc.GetNamespace() scheme := tc.Scheme() - tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster) + if err := tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster); err != nil { + return false, err + } hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal) url := fmt.Sprintf("%s://%s.%s.%s:10080/ddl/owner/resign", scheme, hostName, TiDBPeerMemberName(tcName), ns) @@ -129,7 +130,9 @@ func (tdc *defaultTiDBControl) GetInfo(tc *v1alpha1.TidbCluster, ordinal int32) tcName := tc.GetName() ns := tc.GetNamespace() scheme := tc.Scheme() - tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster) + if err := tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster); err != nil { + return nil, err + } hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal) url := fmt.Sprintf("%s://%s.%s.%s:10080/info", scheme, hostName, TiDBPeerMemberName(tcName), ns) @@ -162,7 +165,9 @@ func (tdc *defaultTiDBControl) GetSettings(tc *v1alpha1.TidbCluster, ordinal int tcName := tc.GetName() ns := tc.GetNamespace() scheme := tc.Scheme() - tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster) + if err := tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster); err != nil { + return nil, err + } hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal) url := fmt.Sprintf("%s://%s.%s.%s:10080/settings", scheme, hostName, TiDBPeerMemberName(tcName), ns) diff --git a/pkg/httputil/httputil.go b/pkg/httputil/httputil.go index 96cc989cd6..91bf94adc8 100644 --- a/pkg/httputil/httputil.go +++ b/pkg/httputil/httputil.go @@ -66,8 +66,7 @@ func ReadCACerts() (*x509.CertPool, error) { // load k8s CA cert caCert, err := ioutil.ReadFile(k8sCAFile) if err != nil { - glog.Errorf("fail to read CA file %s, error: %v", k8sCAFile, err) - return nil, err + return nil, fmt.Errorf("fail to read CA file %s, error: %v", k8sCAFile, err) } if ok := rootCAs.AppendCertsFromPEM(caCert); !ok { glog.Warningf("fail to append CA file to pool, using system CAs only") diff --git a/pkg/manager/member/pd_member_manager.go b/pkg/manager/member/pd_member_manager.go index edc54130d3..359c8b51b7 100644 --- a/pkg/manager/member/pd_member_manager.go +++ b/pkg/manager/member/pd_member_manager.go @@ -470,7 +470,7 @@ func (pmm *pdMemberManager) getNewPDSetForTidbCluster(tc *v1alpha1.TidbCluster) vols = append(vols, corev1.Volume{ Name: "pd-tls", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: fmt.Sprintf("%s-pd", tcName), + SecretName: controller.PDMemberName(tcName), }, }, }) diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index 29bed21320..c9fce8b81a 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -258,7 +258,7 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust vols = append(vols, corev1.Volume{ Name: "tidb-tls", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: fmt.Sprintf("%s-tidb", tcName), + SecretName: controller.TiDBMemberName(tcName), }, }, }) diff --git a/pkg/manager/member/tikv_member_manager.go b/pkg/manager/member/tikv_member_manager.go index 40b0b1272e..9293e924ae 100644 --- a/pkg/manager/member/tikv_member_manager.go +++ b/pkg/manager/member/tikv_member_manager.go @@ -308,7 +308,7 @@ func (tkmm *tikvMemberManager) getNewSetForTidbCluster(tc *v1alpha1.TidbCluster) vols = append(vols, corev1.Volume{ Name: "tikv-tls", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: fmt.Sprintf("%s-tikv", tcName), + SecretName: controller.TiKVMemberName(tcName), }, }, })