[incubator/etcd] implement TLS/SSL for client and peers in etcd (#16741)

* added TLS/SSL ability #16569 Signed-off-by:Moty Fux <moty.fux@gmail.com> Signed-off-by: Moty Fux <moty.fux@gmail.com> * removed hardcoded protocol and bumped version in Chart.yaml #16569 Signed-off-by:Moty Fux <moty.fux@gmail.com> Signed-off-by: Moty Fux <moty.fux@gmail.com> * added some comments in README.md #16569 Signed-off-by:Moty Fux <moty.fux@gmail.com> Signed-off-by: Moty Fux <moty.fux@gmail.com>
helm · Sep 15, 2019 · 6133aab · 6133aab
1 parent de8f438
commit 6133aab
Show file tree

Hide file tree

Showing 5 changed files with 163 additions and 52 deletions.
diff --git a/incubator/etcd/Chart.yaml b/incubator/etcd/Chart.yaml
@@ -1,8 +1,8 @@
 apiVersion: v1
 name: etcd
 home: https://github.com/coreos/etcd
-version: 0.6.3
-appVersion: 2.2.5
+version: 0.7.3
+appVersion: 3.2.26
 description: Distributed reliable key-value store for the most critical data of a
   distributed system.
 icon: https://raw.githubusercontent.com/coreos/etcd/master/logos/etcd-horizontal-color.png

diff --git a/incubator/etcd/README.md b/incubator/etcd/README.md
@@ -7,16 +7,14 @@ Credit to https://github.com/ingvagabund. This is an implementation of that work
 ## Prerequisites Details
 * Kubernetes 1.5 (for `StatefulSets` support)
 * PV support on the underlying infrastructure
+* ETCD version >= 3.0.0
 
 ## StatefulSet Details
 * https://kubernetes.io/docs/concepts/abstractions/controllers/statefulsets/
 
 ## StatefulSet Caveats
 * https://kubernetes.io/docs/concepts/abstractions/controllers/statefulsets/#limitations
 
-## Todo
-* Implement SSL
-
 ## Chart Details
 This chart will do the following:
 
@@ -35,22 +33,27 @@ $ helm install --name my-release incubator/etcd
 
 The following table lists the configurable parameters of the etcd chart and their default values.
 
-| Parameter               | Description                          | Default                                            |
-| ----------------------- | ------------------------------------ | -------------------------------------------------- |
-| `image.repository`      | Container image repository           | `k8s.gcr.io/etcd-amd64`                            |
-| `image.tag`             | Container image tag                  | `2.2.5`                                            |
-| `image.pullPolicy`      | Container pull policy                | `IfNotPresent`                                     |
-| `replicas`              | k8s statefulset replicas             | `3`                                                |
-| `resources`             | container required resources         | `{}`                                             |                                            |
-| `clientPort`            | k8s service port                     | `2379`                                             |
-| `peerPorts`             | Container listening port             | `2380`                                             |
-| `storage`               | Persistent volume size               | `1Gi`                                              |
-| `storageClass`          | Persistent volume storage class      | `anything`                                         |
-| `affinity`              | affinity settings for pod assignment | `{}`                                               |
-| `nodeSelector`          | Node labels for pod assignment       | `{}`                                               |
-| `tolerations`           | Toleration labels for pod assignment | `[]`                                               |
-| `extraEnv`              | Optional environment variables       | `[]`                                               |
-| `memoryMode`            | Using memory as backend storage      | `false`                                            |
+| Parameter                           | Description                          | Default                                            |
+| ----------------------------------- | ------------------------------------ | -------------------------------------------------- |
+| `image.repository`                  | Container image repository           | `k8s.gcr.io/etcd-amd64`                            |
+| `image.tag`                         | Container image tag                  | `3.2.26`                                           |
+| `image.pullPolicy`                  | Container pull policy                | `IfNotPresent`                                     |
+| `replicas`                          | k8s statefulset replicas             | `3`                                                |
+| `resources`                         | container required resources         | `{}`                                               |
+| `clientPort`                        | k8s service port                     | `2379`                                             |
+| `peerPorts`                         | Container listening port             | `2380`                                             |
+| `storage`                           | Persistent volume size               | `1Gi`                                              |
+| `storageClass`                      | Persistent volume storage class      | `anything`                                         |
+| `affinity`                          | affinity settings for pod assignment | `{}`                                               |
+| `nodeSelector`                      | Node labels for pod assignment       | `{}`                                               |
+| `tolerations`                       | Toleration labels for pod assignment | `[]`                                               |
+| `extraEnv`                          | Optional environment variables       | `[]`                                               |
+| `memoryMode`                        | Using memory as backend storage      | `false`                                            |
+| `auth.client.enableAuthentication`  | Enables host authentication using TLS certificates. Existing secret is required.    | `false` |
+| `auth.client.secureTransport`       | Enables encryption of client communication using TLS certificates | `false` |
+| `auth.peer.useAutoTLS`              | Automatically create the TLS certificates | `false` |
+| `auth.peer.secureTransport`         | Enables encryption peer communication using TLS certificates **(At the moment works only with Auto TLS)** | `false` |
+| `auth.peer.enableAuthentication`     | Enables host authentication using TLS certificates. Existing secret required | `false` |
 
 Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`.
 
@@ -59,8 +62,19 @@ Alternatively, a YAML file that specifies the values for the parameters can be p
 ```bash
 $ helm install --name my-release -f values.yaml incubator/etcd
 ```
-
 > **Tip**: You can use the default [values.yaml](values.yaml)
+# To install the chart with secure transport enabled
+First you must create a secret which would contain the client certificates: cert, key and the CA which was to used to sign them.
+Create the secret using this command:
+```bash
+$ kubectl create secret generic etcd-client-certs --from-file=ca.crt=path/to/ca.crt --from-file=cert.pem=path/to/cert.pem --from-file=key.pem=path/to/key.pem
+```
+Deploy the chart with the following flags enabled:
+```bash
+$ helm install --name my-release --set auth.client.secureTransport=true --set auth.client.enableAuthentication=true --set auth.client.existingSecret=etcd-client-certs --set auth.peer.useAutoTLS=true incubator/etcd
+```
+Reference to how to generate the needed certificate:
+> Ref: https://coreos.com/os/docs/latest/generate-self-signed-certificates.html
 
 # Deep dive
 

diff --git a/incubator/etcd/templates/_helpers.tpl b/incubator/etcd/templates/_helpers.tpl
@@ -24,6 +24,43 @@ If release name contains chart name it will be used as a full name.
 {{- end -}}
 {{- end -}}
 
+{{/*
+Return the proper etcd peer protocol
+*/}}
+{{- define "etcd.peerProtocol" -}}
+{{- if .Values.auth.peer.secureTransport -}}
+{{- print "https" -}}
+{{- else -}}
+{{- print "http" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the proper etcd client protocol
+*/}}
+{{- define "etcd.clientProtocol" -}}
+{{- if .Values.auth.client.secureTransport -}}
+{{- print "https" -}}
+{{- else -}}
+{{- print "http" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the proper etcdctl authentication options
+*/}}
+{{- define "etcd.authOptions" -}}
+{{- $certsOption := " --cert=\"$ETCD_CERT_FILE\" --key=\"$ETCD_KEY_FILE\"" -}}
+{{- $caOption := " --cacert=\"$ETCD_TRUSTED_CA_FILE\"" -}}
+{{- if .Values.auth.client.secureTransport -}}
+{{- printf "%s" $certsOption -}}
+{{- end -}}
+{{- if .Values.auth.client.enableAuthentication -}}
+{{- printf "%s" $caOption -}}
+{{- end -}}
+{{- end -}}
+
+
 {{/*
 Create chart name and version as used by the chart label.
 */}}

diff --git a/incubator/etcd/templates/statefulset.yaml b/incubator/etcd/templates/statefulset.yaml
@@ -1,4 +1,7 @@
-apiVersion: apps/v1beta1
+{{- $etcdPeerProtocol := include "etcd.peerProtocol" . }}
+{{- $etcdClientProtocol := include "etcd.clientProtocol" . }}
+{{- $etcdAuthOptions := include "etcd.authOptions" . }}
+apiVersion: apps/v1beta2
 kind: StatefulSet
 metadata:
   name: {{ template "etcd.fullname" . }}
@@ -7,7 +10,13 @@ metadata:
     release: {{ .Release.Name | quote }}
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
     app: {{ template "etcd.name" . }}
+    app.kubernetes.io/name: {{ template "etcd.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
 spec:
+  selector:
+    matchLabels:
+      app: {{ template "etcd.name" . }}
+      #app.kubernetes.io/instance: {{ .Release.Name }}
   serviceName: {{ template "etcd.fullname" . }}
   replicas: {{ .Values.replicas }}
   template:
@@ -43,16 +52,39 @@ spec:
         resources:
 {{ toYaml .Values.resources | indent 10 }}          
         env:
+        - name: ETCDCTL_API
+          value: "3"
         - name: INITIAL_CLUSTER_SIZE
           value: {{ .Values.replicas | quote }}
         - name: SET_NAME
           value: {{ template "etcd.fullname" . }}
+        {{- if and .Values.auth.peer.secureTransport .Values.auth.peer.useAutoTLS }}
+        - name: ETCD_PEER_AUTO_TLS
+          value: "true"
+        {{- end }}
+        {{- if .Values.auth.client.secureTransport }}
+        - name: ETCD_CERT_FILE
+          value: "/opt/etcd/certs/client/cert.pem"
+        - name: ETCD_KEY_FILE
+          value: "/opt/etcd/certs/client/key.pem"
+        {{- if .Values.auth.client.enableAuthentication }}
+        - name: ETCD_CLIENT_CERT_AUTH
+          value: "true"
+        - name: ETCD_TRUSTED_CA_FILE
+          value: "/opt/etcd/certs/client/ca.crt"
+        {{- end }}
+        {{- end }}
 {{- if .Values.extraEnv }}
 {{ toYaml .Values.extraEnv | indent 8 }}
 {{- end }}
         volumeMounts:
         - name: datadir
           mountPath: /var/run/etcd
+        {{- if or .Values.auth.client.enableAuthentication (and .Values.auth.client.secureTransport ) }}
+        - name: etcd-client-certs
+          mountPath: /opt/etcd/certs/client/
+          readOnly: true
+        {{- end }}             
         lifecycle:
           preStop:
             exec:
@@ -62,20 +94,21 @@ spec:
                 - |
                   EPS=""
                   for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do
-                      EPS="${EPS}${EPS:+,}http://${SET_NAME}-${i}.${SET_NAME}:2379"
+                      EPS="${EPS}${EPS:+,}{{ $etcdPeerProtocol }}://${SET_NAME}-${i}.${SET_NAME}:2379"
                   done
 
                   HOSTNAME=$(hostname)
+                  AUTH_OPTIONS="{{ $etcdAuthOptions }}"                  
 
                   member_hash() {
-                      etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1
+                      etcdctl $AUTH_OPTIONS member list | grep {{ $etcdPeerProtocol }}://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1
                   }
 
                   SET_ID=${HOSTNAME##*[^0-9]}
 
                   if [ "${SET_ID}" -ge ${INITIAL_CLUSTER_SIZE} ]; then
                       echo "Removing ${HOSTNAME} from etcd cluster"
-                      ETCDCTL_ENDPOINT=${EPS} etcdctl member remove $(member_hash)
+                      ETCDCTL_ENDPOINT=${EPS} etcdctl $AUTH_OPTIONS member remove $(member_hash)
                       if [ $? -eq 0 ]; then
                           # Remove everything otherwise the cluster will no longer scale-up
                           rm -rf /var/run/etcd/*
@@ -86,24 +119,24 @@ spec:
           - "-ec"
           - |
             HOSTNAME=$(hostname)
-
+            AUTH_OPTIONS="{{ $etcdAuthOptions }}"
             # store member id into PVC for later member replacement
-            collect_member() {
-                while ! etcdctl member list &>/dev/null; do sleep 1; done
-                etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1 > /var/run/etcd/member_id
+            collect_member() {                
+                while ! etcdctl $AUTH_OPTIONS member list &>/dev/null; do sleep 1; done
+                etcdctl $AUTH_OPTIONS member list | grep {{ $etcdPeerProtocol }}://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1 > /var/run/etcd/member_id                
                 exit 0
             }
 
             eps() {
                 EPS=""
                 for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do
-                    EPS="${EPS}${EPS:+,}http://${SET_NAME}-${i}.${SET_NAME}:2379"
+                    EPS="${EPS}${EPS:+,}{{ $etcdPeerProtocol }}://${SET_NAME}-${i}.${SET_NAME}:2379"
                 done
                 echo ${EPS}
             }
 
             member_hash() {
-                etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1
+                etcdctl $AUTH_OPTIONS member list | grep {{ $etcdPeerProtocol }}://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1
             }
 
             # we should wait for other pods to be up before trying to join
@@ -113,21 +146,22 @@ spec:
                     echo "Waiting for ${SET_NAME}-${i}.${SET_NAME} to come up"
                     ping -W 1 -c 1 ${SET_NAME}-${i}.${SET_NAME} > /dev/null && break
                     sleep 1s
-                done
+                done                
             done
-
+            
             # re-joining after failure?
             if [ -e /var/run/etcd/default.etcd ]; then
                 echo "Re-joining etcd member"
                 member_id=$(cat /var/run/etcd/member_id)
 
                 # re-join member
-                ETCDCTL_ENDPOINT=$(eps) etcdctl member update ${member_id} http://${HOSTNAME}.${SET_NAME}:2380 | true
+                ETCDCTL_ENDPOINT=$(eps) etcdctl $AUTH_OPTIONS member update ${member_id} {{ $etcdPeerProtocol }}://${HOSTNAME}.${SET_NAME}:2380 | true
                 exec etcd --name ${HOSTNAME} \
-                    --listen-peer-urls http://0.0.0.0:2380 \
-                    --listen-client-urls http://0.0.0.0:2379\
-                    --advertise-client-urls http://${HOSTNAME}.${SET_NAME}:2379 \
+                    --listen-peer-urls {{ $etcdPeerProtocol }}://0.0.0.0:2380 \
+                    --listen-client-urls {{ $etcdClientProtocol }}://0.0.0.0:2379\
+                    --advertise-client-urls {{ $etcdClientProtocol }}://${HOSTNAME}.${SET_NAME}:2379 \
                     --data-dir /var/run/etcd/default.etcd
+                    
             fi
 
             # etcd-SET_ID
@@ -143,11 +177,11 @@ spec:
                     # the member hash exists but for some reason etcd failed
                     # as the datadir has not be created, we can remove the member
                     # and retrieve new hash
-                    etcdctl member remove ${MEMBER_HASH}
+                    etcdctl $AUTH_OPTIONS member remove ${MEMBER_HASH}
                 fi
 
                 echo "Adding new member"
-                etcdctl member add ${HOSTNAME} http://${HOSTNAME}.${SET_NAME}:2380 | grep "^ETCD_" > /var/run/etcd/new_member_envs
+                etcdctl $AUTH_OPTIONS member add ${HOSTNAME} {{ $etcdPeerProtocol }}://${HOSTNAME}.${SET_NAME}:2380 | grep "^ETCD_" > /var/run/etcd/new_member_envs
 
                 if [ $? -ne 0 ]; then
                     echo "Exiting"
@@ -161,32 +195,40 @@ spec:
                 collect_member &
 
                 exec etcd --name ${HOSTNAME} \
-                    --listen-peer-urls http://0.0.0.0:2380 \
-                    --listen-client-urls http://0.0.0.0:2379 \
-                    --advertise-client-urls http://${HOSTNAME}.${SET_NAME}:2379 \
+                    --listen-peer-urls {{ $etcdPeerProtocol }}://0.0.0.0:2380 \
+                    --listen-client-urls {{ $etcdClientProtocol }}://0.0.0.0:2379 \
+                    --advertise-client-urls {{ $etcdClientProtocol }}://${HOSTNAME}.${SET_NAME}:2379 \
                     --data-dir /var/run/etcd/default.etcd \
-                    --initial-advertise-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \
+                    --initial-advertise-peer-urls {{ $etcdPeerProtocol }}://${HOSTNAME}.${SET_NAME}:2380 \
                     --initial-cluster ${ETCD_INITIAL_CLUSTER} \
                     --initial-cluster-state ${ETCD_INITIAL_CLUSTER_STATE}
+                    
             fi
 
             PEERS=""
             for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do
-                PEERS="${PEERS}${PEERS:+,}${SET_NAME}-${i}=http://${SET_NAME}-${i}.${SET_NAME}:2380"
+                PEERS="${PEERS}${PEERS:+,}${SET_NAME}-${i}={{ $etcdPeerProtocol }}://${SET_NAME}-${i}.${SET_NAME}:2380"
             done
 
             collect_member &
 
             # join member
             exec etcd --name ${HOSTNAME} \
-                --initial-advertise-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \
-                --listen-peer-urls http://0.0.0.0:2380 \
-                --listen-client-urls http://0.0.0.0:2379 \
-                --advertise-client-urls http://${HOSTNAME}.${SET_NAME}:2379 \
+                --initial-advertise-peer-urls {{ $etcdPeerProtocol }}://${HOSTNAME}.${SET_NAME}:2380 \
+                --listen-peer-urls {{ $etcdPeerProtocol }}://0.0.0.0:2380 \
+                --listen-client-urls {{ $etcdClientProtocol }}://0.0.0.0:2379 \
+                --advertise-client-urls {{ $etcdClientProtocol }}://${HOSTNAME}.${SET_NAME}:2379 \
                 --initial-cluster-token etcd-cluster-1 \
                 --initial-cluster ${PEERS} \
                 --initial-cluster-state new \
                 --data-dir /var/run/etcd/default.etcd
+      volumes:
+      {{- if or .Values.auth.client.enableAuthentication (and .Values.auth.client.secureTransport ) }}
+      - name: etcd-client-certs
+        secret:
+          secretName: {{ required "A secret containinig the client certificates is required" .Values.auth.client.existingSecret }}
+          defaultMode: 256
+      {{- end }}      
   {{- if .Values.persistentVolume.enabled }}
   volumeClaimTemplates:
   - metadata:
@@ -206,12 +248,11 @@ spec:
     {{- end }}
     {{- end }}
   {{- else }}
-      volumes:
       - name: datadir
       {{- if .Values.memoryMode }}
         emptyDir:
           medium: Memory
       {{- else }}
         emptyDir: {}
       {{- end }}
-  {{- end }}
+  {{- end }}
diff --git a/incubator/etcd/values.yaml b/incubator/etcd/values.yaml
@@ -9,7 +9,7 @@ component: "etcd"
 replicas: 3
 image:
   repository: "k8s.gcr.io/etcd-amd64"
-  tag: "2.2.5"
+  tag: "3.2.26"
   pullPolicy: "IfNotPresent"
 resources: {}
 # We usually recommend not to specify default resources and to leave this as a conscious
@@ -23,6 +23,25 @@ resources: {}
 #  cpu: 100m
 #  memory: 128Mi
 
+# Authentication parameters
+##
+auth:
+  client:
+    ## Switch to encrypt client communication using TLS certificates
+    secureTransport: false
+    ## Switch to enable host authentication using TLS certificates. Requires existing secret.
+    enableAuthentication: false
+    ## Name of the existing secret containing cert files for peer communication.
+    # existingSecret:
+
+  peer:
+    ## Switch to encrypt peer communication using TLS certificates
+    secureTransport: false
+    ## Switch to automatically create the TLS certificates
+    useAutoTLS: false
+    ## Switch to enable host authentication using TLS certificates. Requires existing secret.
+    enableAuthentication: false
+
 persistentVolume:
   enabled: false
   # storage: "1Gi"