From a358038060ef88e623378451c87048bb4daf5606 Mon Sep 17 00:00:00 2001 From: pingcap-github-bot Date: Tue, 3 Mar 2020 20:42:38 +0800 Subject: [PATCH] binglog: fix tls error when create pump with TLS when use CRD (#1799) (#1838) --- charts/tidb-cluster/templates/_helpers.tpl | 10 ++ .../templates/config/_pump-config.tpl | 18 ++- .../templates/pump-statefulset.yaml | 10 ++ charts/tidb-cluster/values.yaml | 3 +- charts/tidb-drainer/templates/_helpers.tpl | 14 ++ .../templates/drainer-statefulset.yaml | 10 ++ .../templates/scripts/_start_drainer.sh.tpl | 2 +- charts/tidb-drainer/values.yaml | 6 + go.mod | 2 + go.sum | 13 ++ hack/create-cert.sh | 151 ++++++++++++++++++ .../tidbcluster/tidb_cluster_controller.go | 1 + pkg/manager/member/pump_member_manager.go | 104 ++++++++++-- .../member/pump_member_manager_test.go | 5 + 14 files changed, 325 insertions(+), 24 deletions(-) create mode 100755 hack/create-cert.sh diff --git a/charts/tidb-cluster/templates/_helpers.tpl b/charts/tidb-cluster/templates/_helpers.tpl index eaaced7add..2a7b3091d6 100644 --- a/charts/tidb-cluster/templates/_helpers.tpl +++ b/charts/tidb-cluster/templates/_helpers.tpl @@ -118,10 +118,20 @@ config-file: |- {{/* Encapsulate pump configmap data for consistent digest calculation */}} +{{- define "pump.tlsSecretName" -}} +{{ .Values.clusterName }}-pump +{{- end -}} + {{- define "pump-configmap.data" -}} pump-config: |- {{- if .Values.binlog.pump.config }} {{ .Values.binlog.pump.config | indent 2 }} + {{- if .Values.enableTLSCluster }} + [security] + ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + ssl-cert = "/var/lib/pump-tls/cert" + ssl-key = "/var/lib/pump-tls/key" + {{- end -}} {{- else -}} {{ tuple "config/_pump-config.tpl" . | include "helm-toolkit.utils.template" | indent 2 }} {{- end -}} diff --git a/charts/tidb-cluster/templates/config/_pump-config.tpl b/charts/tidb-cluster/templates/config/_pump-config.tpl index d0b41eddf0..5398655f72 100644 --- a/charts/tidb-cluster/templates/config/_pump-config.tpl +++ b/charts/tidb-cluster/templates/config/_pump-config.tpl @@ -19,14 +19,6 @@ heartbeat-interval = {{ .Values.binlog.pump.heartbeatInterval | default 2 }} # a comma separated list of PD endpoints pd-urls = "{{ template "cluster.scheme" . }}://{{ template "cluster.name" . }}-pd:2379" -#[security] -# Path of file that contains list of trusted SSL CAs for connection with cluster components. -# ssl-ca = "/path/to/ca.pem" -# Path of file that contains X509 certificate in PEM format for connection with cluster components. -# ssl-cert = "/path/to/drainer.pem" -# Path of file that contains X509 key in PEM format for connection with cluster components. -# ssl-key = "/path/to/drainer-key.pem" -# [storage] # Set to `true` (default) for best reliability, which prevents data loss when there is a power failure. sync-log = {{ .Values.binlog.pump.syncLog | default true }} @@ -43,3 +35,13 @@ sync-log = {{ .Values.binlog.pump.syncLog | default true }} # write-buffer = 67108864 # write-L0-pause-trigger = 24 # write-L0-slowdown-trigger = 17 +{{ if .Values.enableTLSCluster }} +[security] +# Path of file that contains list of trusted SSL CAs for connection with cluster components. +ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" +# Path of file that contains X509 certificate in PEM format for connection with cluster components. +ssl-cert = "/var/lib/pump-tls/cert" +# Path of file that contains X509 key in PEM format for connection with cluster components. +ssl-key = "/var/lib/pump-tls/key" +{{- end -}} + diff --git a/charts/tidb-cluster/templates/pump-statefulset.yaml b/charts/tidb-cluster/templates/pump-statefulset.yaml index 90a61b62c9..f1bd44510e 100644 --- a/charts/tidb-cluster/templates/pump-statefulset.yaml +++ b/charts/tidb-cluster/templates/pump-statefulset.yaml @@ -55,6 +55,11 @@ spec: mountPath: /data - name: config mountPath: /etc/pump + {{- if .Values.enableTLSCluster }} + - name: pump-tls + mountPath: /var/lib/pump-tls + readOnly: true + {{- end }} resources: {{ toYaml .Values.binlog.pump.resources | indent 10 }} {{- if and (ne .Values.timezone "UTC") (ne .Values.timezone "") }} @@ -73,6 +78,11 @@ spec: items: - key: pump-config path: pump.toml + {{- if .Values.enableTLSCluster }} + - name: pump-tls + secret: + secretName: {{ include "pump.tlsSecretName" . }} + {{- end }} volumeClaimTemplates: - metadata: name: data diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index ed23c0b637..e972d8d08f 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -56,7 +56,7 @@ discovery: enableConfigMapRollout: true # Whether enable TLS connections between server nodes. -# When enabled, PD/TiDB/TiKV will use TLS encrypted connections to transfer data between each node, +# When enabled, PD/TiDB/TiKV/PUMP will use TLS encrypted connections to transfer data between each node, # certificates will be generated automatically (if not already present). enableTLSCluster: false @@ -612,6 +612,7 @@ binlog: # pump configurations (change to the tags of your pump version), # just follow the format in the file and configure in the 'config' section # as below if you want to customize any configuration. + # [security] section will be generated automatically if enableTLSCluster is set to true so users do not need to configure it. # config: | # gc = 7 # heartbeat-interval = 2 diff --git a/charts/tidb-drainer/templates/_helpers.tpl b/charts/tidb-drainer/templates/_helpers.tpl index f725f0a626..cf08f693af 100644 --- a/charts/tidb-drainer/templates/_helpers.tpl +++ b/charts/tidb-drainer/templates/_helpers.tpl @@ -5,6 +5,10 @@ {{ .Values.clusterName }}-{{ .Release.Name }}-drainer {{- end -}} +{{- define "drainer.tlsSecretName" -}} +{{ .Values.clusterName }}-drainer +{{- end -}} + {{/* Encapsulate config data for consistent digest calculation */}} @@ -13,12 +17,22 @@ config-file: |- {{- if .Values.config }} {{ .Values.config | indent 2 }} {{- end -}} + {{- if .Values.enableTLSCluster }} + [security] + ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + ssl-cert = "/var/lib/drainer-tls/cert" + ssl-key = "/var/lib/drainer-tls/key" + {{- end -}} {{- end -}} {{- define "drainer-configmap.name" -}} {{ include "drainer.name" . }}-{{ include "drainer-configmap.data" . | sha256sum | trunc 8 }} {{- end -}} +{{- define "cluster.scheme" -}} +{{ if .Values.enableTLSCluster }}https{{ else }}http{{ end }} +{{- end -}} + {{- define "helm-toolkit.utils.template" -}} {{- $name := index . 0 -}} {{- $context := index . 1 -}} diff --git a/charts/tidb-drainer/templates/drainer-statefulset.yaml b/charts/tidb-drainer/templates/drainer-statefulset.yaml index 8d162763f5..19383b25ad 100644 --- a/charts/tidb-drainer/templates/drainer-statefulset.yaml +++ b/charts/tidb-drainer/templates/drainer-statefulset.yaml @@ -46,6 +46,11 @@ spec: mountPath: /data - name: config mountPath: /etc/drainer + {{- if .Values.enableTLSCluster }} + - name: drainer-tls + mountPath: /var/lib/drainer-tls + readOnly: true + {{- end }} {{- if and (ne .Values.timezone "UTC") (ne .Values.timezone "") }} env: - name: TZ @@ -60,6 +65,11 @@ spec: items: - key: config-file path: drainer.toml + {{- if .Values.enableTLSCluster }} + - name: drainer-tls + secret: + secretName: {{ include "drainer.tlsSecretName" . }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{ toYaml . | indent 8 }} diff --git a/charts/tidb-drainer/templates/scripts/_start_drainer.sh.tpl b/charts/tidb-drainer/templates/scripts/_start_drainer.sh.tpl index ada3d3128f..8f258ce6f3 100644 --- a/charts/tidb-drainer/templates/scripts/_start_drainer.sh.tpl +++ b/charts/tidb-drainer/templates/scripts/_start_drainer.sh.tpl @@ -26,7 +26,7 @@ done /drainer \ -L={{ .Values.logLevel | default "info" }} \ --pd-urls=http://{{ .Values.clusterName }}-pd:2379 \ +-pd-urls={{ include "cluster.scheme" . }}://{{ .Values.clusterName }}-pd:2379 \ -addr=`echo ${HOSTNAME}`.{{ include "drainer.name" . }}:8249 \ -config=/etc/drainer/drainer.toml \ -disable-detect={{ .Values.disableDetect | default false }} \ diff --git a/charts/tidb-drainer/values.yaml b/charts/tidb-drainer/values.yaml index c3aeff6178..4c35d88daf 100644 --- a/charts/tidb-drainer/values.yaml +++ b/charts/tidb-drainer/values.yaml @@ -29,7 +29,13 @@ disableDetect: false # if drainer donesn't have checkpoint, use initial commitTS to initial checkpoint initialCommitTs: 0 +# Whether enable TLS connections between server nodes. +# When enabled, DRAINER will use TLS encrypted connections to transfer data with PUMP node, +# certificates will be generated by script "hack/create-cert.sh" manually +enableTLSCluster: false + # Refer to https://github.com/pingcap/tidb-binlog/blob/master/cmd/drainer/drainer.toml +# [security] section will be generated automatically if enableTLSCluster is set to true so users do not need to configure it. config: | detect-interval = 10 compressor = "" diff --git a/go.mod b/go.mod index 6cf9613060..8153d820a0 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ module github.com/pingcap/tidb-operator go 1.13 require ( + github.com/Azure/go-autorest/autorest/mocks v0.3.0 // indirect github.com/BurntSushi/toml v0.3.1 github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e // indirect github.com/Microsoft/go-winio v0.4.12 // indirect @@ -35,6 +36,7 @@ require ( github.com/gophercloud/gophercloud v0.3.0 // indirect github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.13.0 // indirect github.com/imdario/mergo v0.3.7 // indirect github.com/juju/errors v0.0.0-20180806074554-22422dad46e1 github.com/juju/loggo v0.0.0-20180524022052-584905176618 // indirect diff --git a/go.sum b/go.sum index efcb65876d..397fda71e7 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,8 @@ github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjW github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0 h1:Ww5g4zThfD/6cLb4z6xxgeyDa7QDkizMkJKe0ysZXp0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= github.com/Azure/go-autorest/autorest/to v0.2.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc= github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8= github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= @@ -68,6 +70,7 @@ github.com/Rican7/retry v0.1.0/go.mod h1:FgOROf8P5bebcC1DS0PdOQiqGUridaZvikzUmkF github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/ant31/crd-validation v0.0.0-20180702145049-30f8a35d0ac2 h1:CDDf61yprxfS7bmBPyhH8pxaobD2VbO3d7laAxJbZos= github.com/ant31/crd-validation v0.0.0-20180702145049-30f8a35d0ac2/go.mod h1:X0noFIik9YqfhGYBLEHg8LJKEwy7QIitLQuFMpKLcPk= +github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -459,6 +462,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.2 h1:S+ef0492XaIknb8LMjcwgW2i3cNTzDYMmDrOThOJNWc= github.com/grpc-ecosystem/grpc-gateway v1.9.2/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.13.0 h1:sBDQoHXrOlfPobnKw69FIKa1wg9qsLLvvQ/Y19WtFgI= +github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -680,6 +685,7 @@ github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uY github.com/robfig/cron v1.1.0 h1:jk4/Hud3TTdcrJgUOBgsqrZBarcxl6ADIjSC2iniwLY= github.com/robfig/cron v1.1.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/rogpeppe/go-internal v1.0.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.0.1-alpha.3/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -863,6 +869,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191002035440-2ec189313ef0 h1:2mqDk8w/o6UmeUCu5Qiq2y7iMf6anbx+YA8d1JFoFrs= +golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -985,6 +993,8 @@ google.golang.org/genproto v0.0.0-20190508193815-b515fa19cec8/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190620144150-6af8c5fc6601 h1:9VBRTdmgQxbs6HE0sUnMrSWNePppAJU07NYvX5dIB04= google.golang.org/genproto v0.0.0-20190620144150-6af8c5fc6601/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c h1:hrpEMCZ2O7DR5gC1n2AJGVhrwiEjOi35+jxtIuZpTMo= +google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -992,6 +1002,8 @@ google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1026,6 +1038,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.0.0/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= diff --git a/hack/create-cert.sh b/hack/create-cert.sh new file mode 100755 index 0000000000..0854efbc9b --- /dev/null +++ b/hack/create-cert.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash + +# Copyright 2020 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +usage() { + cat <> ${tmpdir}/csr.conf +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +[req_distinguished_name] +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = @alt_names +[alt_names] +DNS.1 = ${service} +DNS.2 = ${service}.${namespace} +DNS.3 = ${service}.${namespace}.svc +DNS.4 = *.${service} +DNS.5 = *.${service}.${namespace} +DNS.5 = *.${service}.${namespace}.svc +IP.1 = 127.0.0.1 +EOF + +openssl genrsa -out ${tmpdir}/server-key.pem 2048 +openssl req -new -key ${tmpdir}/server-key.pem -subj "/CN=${service}.${namespace}.svc" -out ${tmpdir}/server.csr -config ${tmpdir}/csr.conf + + # clean-up any previously created CSR for our service. Ignore errors if not present. +kubectl delete csr ${csrName} 2>/dev/null || true + + # create server cert/key CSR and send to k8s API +cat <&2 + exit 1 +fi + + echo ${serverCert} | openssl base64 -d -A -out ${tmpdir}/server-cert.pem + + # create the secret with CA cert and server cert/key +kubectl create secret generic ${secret} \ + --from-file=key=${tmpdir}/server-key.pem \ + --from-file=cert=${tmpdir}/server-cert.pem \ + --dry-run -o yaml | + kubectl -n ${namespace} apply -f - diff --git a/pkg/controller/tidbcluster/tidb_cluster_controller.go b/pkg/controller/tidbcluster/tidb_cluster_controller.go index e4f84ee1a6..3de44bd02b 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_controller.go +++ b/pkg/controller/tidbcluster/tidb_cluster_controller.go @@ -195,6 +195,7 @@ func NewController( pvControl, ), mm.NewPumpMemberManager( + certControl, setControl, svcControl, typedControl, diff --git a/pkg/manager/member/pump_member_manager.go b/pkg/manager/member/pump_member_manager.go index 19f47ccd80..af57ea830c 100644 --- a/pkg/manager/member/pump_member_manager.go +++ b/pkg/manager/member/pump_member_manager.go @@ -15,6 +15,7 @@ package member import ( "fmt" + "path" "strings" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" @@ -34,9 +35,12 @@ import ( const ( defaultPumpLogLevel = "info" + pumpCertVolumeMount = "pump-tls" + pumpCertPath = "/var/lib/pump-tls" ) type pumpMemberManager struct { + certControl controller.CertControlInterface setControl controller.StatefulSetControlInterface svcControl controller.ServiceControlInterface typedControl controller.TypedControlInterface @@ -49,6 +53,7 @@ type pumpMemberManager struct { // NewPumpMemberManager returns a controller to reconcile pump clusters func NewPumpMemberManager( + certControl controller.CertControlInterface, setControl controller.StatefulSetControlInterface, svcControl controller.ServiceControlInterface, typedControl controller.TypedControlInterface, @@ -58,6 +63,7 @@ func NewPumpMemberManager( cmLister corelisters.ConfigMapLister, podLister corelisters.PodLister) manager.Manager { return &pumpMemberManager{ + certControl, setControl, svcControl, typedControl, @@ -81,7 +87,6 @@ func (pmm *pumpMemberManager) Sync(tc *v1alpha1.TidbCluster) error { //syncPumpStatefulSetForTidbCluster sync statefulset status of pump to tidbcluster func (pmm *pumpMemberManager) syncPumpStatefulSetForTidbCluster(tc *v1alpha1.TidbCluster) error { - oldPumpSetTemp, err := pmm.setLister.StatefulSets(tc.Namespace).Get(controller.PumpMemberName(tc.Name)) if err != nil && !errors.IsNotFound(err) { return err @@ -94,6 +99,13 @@ func (pmm *pumpMemberManager) syncPumpStatefulSetForTidbCluster(tc *v1alpha1.Tid return err } + if tc.IsTLSClusterEnabled() { + err := pmm.syncPumpStatefulsetCerts(tc) + if err != nil { + return err + } + } + newPumpSet, err := getNewPumpStatefulSet(tc, cm) if err != nil { return err @@ -242,9 +254,20 @@ func getNewPumpConfigMap(tc *v1alpha1.TidbCluster) (*corev1.ConfigMap, error) { } name := controller.PumpMemberName(tc.Name) + confTextStr := string(confText) + + if tc.IsTLSClusterEnabled() { + confTextStr = strings.Join([]string{ + confTextStr, + "[security]", + fmt.Sprintf("ssl-ca = \"%s\"", serviceAccountCAPath), + fmt.Sprintf("ssl-cert = \"%s\"", path.Join(pumpCertPath, "cert")), + fmt.Sprintf("ssl-key = \"%s\"", path.Join(pumpCertPath, "key"))}, "\n") + } data := map[string]string{ - "pump-config": string(confText), + "pump-config": confTextStr, } + if basePumpSpec.ConfigUpdateStrategy() == v1alpha1.ConfigUpdateStrategyRollingUpdate { sum, err := Sha256Sum(data) if err != nil { @@ -297,6 +320,21 @@ func getNewPumpStatefulSet(tc *v1alpha1.TidbCluster, cm *corev1.ConfigMap) (*app }, }) } + volumeMounts := []corev1.VolumeMount{ + { + Name: "data", + MountPath: "/data", + }, + { + Name: "config", + MountPath: "/etc/pump", + }, + } + if tc.IsTLSClusterEnabled() { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: pumpCertVolumeMount, ReadOnly: true, MountPath: pumpCertPath, + }) + } containers := []corev1.Container{ { Name: "pump", @@ -311,18 +349,9 @@ func getNewPumpStatefulSet(tc *v1alpha1.TidbCluster, cm *corev1.ConfigMap) (*app Name: "pump", ContainerPort: 8250, }}, - Resources: controller.ContainerResource(tc.Spec.Pump.ResourceRequirements), - Env: envs, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "data", - MountPath: "/data", - }, - { - Name: "config", - MountPath: "/etc/pump", - }, - }, + Resources: controller.ContainerResource(tc.Spec.Pump.ResourceRequirements), + Env: envs, + VolumeMounts: volumeMounts, }, } @@ -346,6 +375,16 @@ func getNewPumpStatefulSet(tc *v1alpha1.TidbCluster, cm *corev1.ConfigMap) (*app }, } + if tc.IsTLSClusterEnabled() { + volumes = append(volumes, corev1.Volume{ + Name: pumpCertVolumeMount, VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: controller.PumpMemberName(tc.Name), + }, + }, + }) + } + volumeClaims := []corev1.PersistentVolumeClaim{ { ObjectMeta: metav1.ObjectMeta{ @@ -440,6 +479,43 @@ func getPumpLogLevel(tc *v1alpha1.TidbCluster) string { return logLevel } +// syncPumpStatefulsetCerts creates the cert pair for Pump if not exist, the cert +// pair is used to communicate with other TiDB components, like TiDB and Drainer +func (pmm *pumpMemberManager) syncPumpStatefulsetCerts(tc *v1alpha1.TidbCluster) error { + ns := tc.GetNamespace() + tcName := tc.GetName() + svcName := controller.PumpMemberName(tcName) + peerName := controller.PumpPeerMemberName(tcName) + + if pmm.certControl.CheckSecret(ns, svcName) { + return nil + } + + hostList := []string{ + svcName, + peerName, + fmt.Sprintf("%s.%s", svcName, ns), + fmt.Sprintf("%s.%s", peerName, ns), + fmt.Sprintf("*.%s.%s", peerName, ns), + } + + ipList := []string{ + "127.0.0.1", "::1", // able to access https endpoint via loopback network + } + + certOpts := &controller.TiDBClusterCertOptions{ + Namespace: ns, + Instance: tcName, + CommonName: svcName, + HostList: hostList, + IPList: ipList, + Component: "pump", + Suffix: "pump", + } + + return pmm.certControl.Create(controller.GetOwnerRef(tc), certOpts) +} + func (pmm *pumpMemberManager) pumpStatefulSetIsUpgrading(set *apps.StatefulSet, tc *v1alpha1.TidbCluster) (bool, error) { if statefulSetIsUpgrading(set) { return true, nil diff --git a/pkg/manager/member/pump_member_manager_test.go b/pkg/manager/member/pump_member_manager_test.go index c92e41b0de..c43e4475d3 100644 --- a/pkg/manager/member/pump_member_manager_test.go +++ b/pkg/manager/member/pump_member_manager_test.go @@ -441,14 +441,19 @@ func newFakePumpMemberManager() (*pumpMemberManager, *pumpFakeControls, *pumpFak setInformer := kubeinformers.NewSharedInformerFactory(kubeCli, 0).Apps().V1().StatefulSets() tcInformer := informers.NewSharedInformerFactory(cli, 0).Pingcap().V1alpha1().TidbClusters() svcInformer := kubeinformers.NewSharedInformerFactory(kubeCli, 0).Core().V1().Services() + secretInformer := kubeinformers.NewSharedInformerFactory(kubeCli, 0).Core().V1().Secrets() + csrInformer := kubeinformers.NewSharedInformerFactory(kubeCli, 0).Certificates().V1beta1().CertificateSigningRequests() epsInformer := kubeinformers.NewSharedInformerFactory(kubeCli, 0).Core().V1().Endpoints() cmInformer := kubeinformers.NewSharedInformerFactory(kubeCli, 0).Core().V1().ConfigMaps() podInformer := kubeinformers.NewSharedInformerFactory(kubeCli, 0).Core().V1().Pods() setControl := controller.NewFakeStatefulSetControl(setInformer, tcInformer) + secControl := controller.NewFakeSecretControl(kubeCli, secretInformer.Lister()) + certControl := controller.NewFakeCertControl(kubeCli, csrInformer.Lister(), secControl) svcControl := controller.NewFakeServiceControl(svcInformer, epsInformer, tcInformer) cmControl := controller.NewFakeConfigMapControl(cmInformer) genericControl := controller.NewFakeGenericControl() pmm := &pumpMemberManager{ + certControl, setControl, svcControl, controller.NewTypedControl(genericControl),