ACLs: Support external servers (hashicorp#420)

* server-acl-init-job sets server addresses if 'externalServers.enabled' is true * server-acl-init and server-acl-init-cleanup jobs and their related resources now run either when servers are enabled or when externalServers are enabled * Add new acls.bootstrapToken value for providing your own bootstrap token. * Allow custom auth method configuration * Fail if both server and externalServers are enabled
ndhanushkodi · Apr 23, 2020 · 554f936 · 554f936
1 parent f89be18
commit 554f936
Show file tree

Hide file tree

Showing 22 changed files with 722 additions and 28 deletions.
diff --git a/templates/server-acl-init-cleanup-clusterrole.yaml b/templates/server-acl-init-cleanup-clusterrole.yaml
@@ -1,4 +1,6 @@
-{{- if (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) }}
+{{- $serverEnabled := (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) -}}
+{{- if (and $serverEnabled .Values.externalServers.enabled) }}{{ fail "only one of server.enabled or externalServers.enabled can be set" }}{{ end -}}
+{{- if (or $serverEnabled .Values.externalServers.enabled) }}
 {{- if (or .Values.global.acls.manageSystemACLs .Values.global.bootstrapACLs) }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

diff --git a/templates/server-acl-init-cleanup-clusterrolebinding.yaml b/templates/server-acl-init-cleanup-clusterrolebinding.yaml
@@ -1,4 +1,6 @@
-{{- if (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) }}
+{{- $serverEnabled := (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) -}}
+{{- if (and $serverEnabled .Values.externalServers.enabled) }}{{ fail "only one of server.enabled or externalServers.enabled can be set" }}{{ end -}}
+{{- if (or $serverEnabled .Values.externalServers.enabled) }}
 {{- if (or .Values.global.acls.manageSystemACLs .Values.global.bootstrapACLs) }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

diff --git a/templates/server-acl-init-cleanup-job.yaml b/templates/server-acl-init-cleanup-job.yaml
@@ -1,4 +1,6 @@
-{{- if (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) }}
+{{- $serverEnabled := (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) -}}
+{{- if (and $serverEnabled .Values.externalServers.enabled) }}{{ fail "only one of server.enabled or externalServers.enabled can be set" }}{{ end -}}
+{{- if (or $serverEnabled .Values.externalServers.enabled) }}
 {{- if (or .Values.global.acls.manageSystemACLs .Values.global.bootstrapACLs) }}
 {{- /* See reason for this in server-acl-init-job.yaml */ -}}
 {{- if eq (int .Values.server.updatePartition) 0 }}

diff --git a/templates/server-acl-init-cleanup-podsecuritypolicy.yaml b/templates/server-acl-init-cleanup-podsecuritypolicy.yaml
@@ -1,4 +1,6 @@
-{{- if (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) }}
+{{- $serverEnabled := (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) -}}
+{{- if (or $serverEnabled .Values.externalServers.enabled) }}
+{{- if (and $serverEnabled .Values.externalServers.enabled) }}{{ fail "only one of server.enabled or externalServers.enabled can be set" }}{{ end -}}
 {{- if (or .Values.global.acls.manageSystemACLs .Values.global.bootstrapACLs) }}
 {{- if .Values.global.enablePodSecurityPolicies }}
 apiVersion: policy/v1beta1

diff --git a/templates/server-acl-init-cleanup-serviceaccount.yaml b/templates/server-acl-init-cleanup-serviceaccount.yaml
@@ -1,4 +1,6 @@
-{{- if (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) }}
+{{- $serverEnabled := (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) -}}
+{{- if (and $serverEnabled .Values.externalServers.enabled) }}{{ fail "only one of server.enabled or externalServers.enabled can be set" }}{{ end -}}
+{{- if (or $serverEnabled .Values.externalServers.enabled) }}
 {{- if (or .Values.global.acls.manageSystemACLs .Values.global.bootstrapACLs) }}
 apiVersion: v1
 kind: ServiceAccount

diff --git a/templates/server-acl-init-clusterrole.yaml b/templates/server-acl-init-clusterrole.yaml
@@ -1,4 +1,6 @@
-{{- if (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) }}
+{{- $serverEnabled := (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) -}}
+{{- if (and $serverEnabled .Values.externalServers.enabled) }}{{ fail "only one of server.enabled or externalServers.enabled can be set" }}{{ end -}}
+{{- if (or $serverEnabled .Values.externalServers.enabled) }}
 {{- if (or .Values.global.acls.manageSystemACLs .Values.global.bootstrapACLs) }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -30,11 +32,8 @@ rules:
   - apiGroups: [""]
     resources:
       - serviceaccounts
-    verbs:
-      - get
-  - apiGroups: [""]
-    resources:
-      - services
+    resourceNames:
+      - {{ template "consul.fullname" . }}-connect-injector-authmethod-svc-account
     verbs:
       - get
 {{- end }}

diff --git a/templates/server-acl-init-clusterrolebinding.yaml b/templates/server-acl-init-clusterrolebinding.yaml
@@ -1,4 +1,6 @@
-{{- if (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) }}
+{{- $serverEnabled := (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) -}}
+{{- if (and $serverEnabled .Values.externalServers.enabled) }}{{ fail "only one of server.enabled or externalServers.enabled can be set" }}{{ end -}}
+{{- if (or $serverEnabled .Values.externalServers.enabled) }}
 {{- if (or .Values.global.acls.manageSystemACLs .Values.global.bootstrapACLs) }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

diff --git a/templates/server-acl-init-job.yaml b/templates/server-acl-init-job.yaml
@@ -1,4 +1,6 @@
-{{- if (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) }}
+{{- $serverEnabled := (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) -}}
+{{- if (and $serverEnabled .Values.externalServers.enabled) }}{{ fail "only one of server.enabled or externalServers.enabled can be set" }}{{ end -}}
+{{- if (or $serverEnabled .Values.externalServers.enabled) }}
 {{- if (or .Values.global.acls.manageSystemACLs .Values.global.bootstrapACLs) }}
 {{- /* We don't render this job when server.updatePartition > 0 because that
     means a server rollout is in progress and this job won't complete unless
@@ -32,7 +34,7 @@ spec:
     spec:
       restartPolicy: Never
       serviceAccountName: {{ template "consul.fullname" . }}-server-acl-init
-      {{- if (or .Values.global.tls.enabled (and .Values.global.acls.replicationToken.secretName .Values.global.acls.replicationToken.secretKey)) }}
+      {{- if (or .Values.global.tls.enabled (and .Values.global.acls.replicationToken.secretName .Values.global.acls.replicationToken.secretKey) (and .Values.global.acls.bootstrapToken.secretName .Values.global.acls.bootstrapToken.secretKey)) }}
       volumes:
         {{- if .Values.global.tls.enabled }}
         - name: consul-ca-cert
@@ -46,7 +48,14 @@ spec:
               - key: {{ default "tls.crt" .Values.global.tls.caCert.secretKey }}
                 path: tls.crt
         {{- end }}
-        {{- if (and .Values.global.acls.replicationToken.secretName .Values.global.acls.replicationToken.secretKey) }}
+        {{- if (and .Values.global.acls.bootstrapToken.secretName .Values.global.acls.bootstrapToken.secretKey) }}
+        - name: bootstrap-token
+          secret:
+            secretName: {{ .Values.global.acls.bootstrapToken.secretName }}
+            items:
+              - key: {{ .Values.global.acls.bootstrapToken.secretKey }}
+                path: bootstrap-token
+        {{- else if (and .Values.global.acls.replicationToken.secretName .Values.global.acls.replicationToken.secretKey) }}
         - name: acl-replication-token
           secret:
             secretName: {{ .Values.global.acls.replicationToken.secretName }}
@@ -63,14 +72,18 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
-          {{- if (or .Values.global.tls.enabled (and .Values.global.acls.replicationToken.secretName .Values.global.acls.replicationToken.secretKey)) }}
+          {{- if (or .Values.global.tls.enabled (and .Values.global.acls.replicationToken.secretName .Values.global.acls.replicationToken.secretKey) (and .Values.global.acls.bootstrapToken.secretName .Values.global.acls.bootstrapToken.secretKey)) }}
           volumeMounts:
             {{- if .Values.global.tls.enabled }}
             - name: consul-ca-cert
               mountPath: /consul/tls/ca
               readOnly: true
             {{- end }}
-            {{- if (and .Values.global.acls.replicationToken.secretName .Values.global.acls.replicationToken.secretKey) }}
+            {{- if (and .Values.global.acls.bootstrapToken.secretName .Values.global.acls.bootstrapToken.secretKey) }}
+            - name: bootstrap-token
+              mountPath: /consul/acl/tokens
+              readOnly: true
+            {{- else if (and .Values.global.acls.replicationToken.secretName .Values.global.acls.replicationToken.secretKey) }}
             - name: acl-replication-token
               mountPath: /consul/acl/tokens
               readOnly: true
@@ -83,16 +96,32 @@ spec:
               CONSUL_FULLNAME="{{template "consul.fullname" . }}"
 
               consul-k8s server-acl-init \
+                {{- if .Values.externalServers.enabled }}
+                {{- if not (or .Values.externalServers.https.address .Values.client.join)}}{{ fail "either client.join or externalServers.https.address must be set if externalServers.enabled is true" }}{{ end -}}
+                {{- if .Values.externalServers.https.address }}
+                -server-address={{ .Values.externalServers.https.address }} \
+                {{- else }}
+                {{- range .Values.client.join }}
+                -server-address={{ . }} \
+                {{- end }}
+                {{- end }}
+                -server-port={{ .Values.externalServers.https.port }} \
+                {{- else }}
                 {{- range $index := until (.Values.server.replicas | int) }}
                 -server-address="${CONSUL_FULLNAME}-server-{{ $index }}.${CONSUL_FULLNAME}-server.${NAMESPACE}.svc" \
                 {{- end }}
-                -resource-prefix={{ template "consul.fullname" . }} \
+                {{- end }}
+                -resource-prefix=${CONSUL_FULLNAME} \
                 -k8s-namespace={{ .Release.Namespace }} \
                 {{- if .Values.global.tls.enabled }}
                 -use-https \
+                {{- if not (and .Values.externalServers.enabled .Values.externalServers.https.useSystemRoots) }}
                 -consul-ca-cert=/consul/tls/ca/tls.crt \
+                {{- end }}
+                {{- if not .Values.externalServers.enabled }}
                 -server-port=8501 \
                 {{- end }}
+                {{- end }}
                 {{- if .Values.syncCatalog.enabled }}
                 -create-sync-token=true \
                 {{- end }}
@@ -101,6 +130,9 @@ spec:
                 {{- end }}
                 {{- if .Values.connectInject.enabled }}
                 -create-inject-auth-method=true \
+                {{- if .Values.connectInject.overrideAuthMethodHost }}
+                -inject-auth-method-host={{ .Values.connectInject.overrideAuthMethodHost }} \
+                {{- end }}
                 {{- end }}
                 {{- if .Values.meshGateway.enabled }}
                 -create-mesh-gateway-token=true \
@@ -120,7 +152,9 @@ spec:
                 {{- if .Values.global.acls.createReplicationToken }}
                 -create-acl-replication-token=true \
                 {{- end }}
-                {{- if (and .Values.global.acls.replicationToken.secretName .Values.global.acls.replicationToken.secretKey) }}
+                {{- if (and .Values.global.acls.bootstrapToken.secretName .Values.global.acls.bootstrapToken.secretKey) }}
+                -bootstrap-token-file=/consul/acl/tokens/bootstrap-token \
+                {{- else if (and .Values.global.acls.replicationToken.secretName .Values.global.acls.replicationToken.secretKey) }}
                 -acl-replication-token-file=/consul/acl/tokens/acl-replication-token \
                 {{- end }}
                 {{- if .Values.global.enableConsulNamespaces }}

diff --git a/templates/server-acl-init-podsecuritypolicy.yaml b/templates/server-acl-init-podsecuritypolicy.yaml
@@ -1,4 +1,6 @@
-{{- if (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) }}
+{{- $serverEnabled := (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) -}}
+{{- if (and $serverEnabled .Values.externalServers.enabled) }}{{ fail "only one of server.enabled or externalServers.enabled can be set" }}{{ end -}}
+{{- if (or $serverEnabled .Values.externalServers.enabled) }}
 {{- if (or .Values.global.acls.manageSystemACLs .Values.global.bootstrapACLs) }}
 {{- if .Values.global.enablePodSecurityPolicies }}
 apiVersion: policy/v1beta1

diff --git a/templates/server-acl-init-serviceaccount.yaml b/templates/server-acl-init-serviceaccount.yaml
@@ -1,4 +1,6 @@
-{{- if (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) }}
+{{- $serverEnabled := (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) -}}
+{{- if (and $serverEnabled .Values.externalServers.enabled) }}{{ fail "only one of server.enabled or externalServers.enabled can be set" }}{{ end -}}
+{{- if (or $serverEnabled .Values.externalServers.enabled) }}
 {{- if (or .Values.global.acls.manageSystemACLs .Values.global.bootstrapACLs) }}
 apiVersion: v1
 kind: ServiceAccount

diff --git a/test/unit/helpers.bats b/test/unit/helpers.bats
@@ -138,6 +138,7 @@ load _helpers
       -x templates/tests/test-runner.yaml  \
       --set 'global.tls.enabled=true' \
       --set 'global.tls.enableAutoEncrypt=true' \
+      --set 'server.enabled=false' \
       --set 'externalServers.enabled=true' \
       --set 'client.join[0]=consul-server.com' \
       . | tee /dev/stderr |
@@ -162,6 +163,7 @@ load _helpers
       -x templates/tests/test-runner.yaml  \
       --set 'global.tls.enabled=true' \
       --set 'global.tls.enableAutoEncrypt=true' \
+      --set 'server.enabled=false' \
       --set 'externalServers.enabled=true' \
       --set 'externalServers.https.address=consul.io' \
       . | tee /dev/stderr |
@@ -197,6 +199,7 @@ load _helpers
       -x templates/tests/test-runner.yaml  \
       --set 'global.tls.enabled=true' \
       --set 'global.tls.enableAutoEncrypt=true' \
+      --set 'server.enabled=false' \
       --set 'externalServers.enabled=true' \
       --set 'externalServers.https.address=consul.io' \
       --set 'externalServers.https.port=8501' \
@@ -222,6 +225,7 @@ load _helpers
       -x templates/tests/test-runner.yaml  \
       --set 'global.tls.enabled=true' \
       --set 'global.tls.enableAutoEncrypt=true' \
+      --set 'server.enabled=false' \
       --set 'externalServers.enabled=true' \
       --set 'externalServers.https.address=consul.io' \
       --set 'externalServers.https.tlsServerName=custom-server-name' \
@@ -237,6 +241,7 @@ load _helpers
       -x templates/tests/test-runner.yaml  \
       --set 'global.tls.enabled=true' \
       --set 'global.tls.enableAutoEncrypt=true' \
+      --set 'server.enabled=false' \
       --set 'externalServers.enabled=true' \
       --set 'externalServers.https.address=consul.io' \
       --set 'externalServers.https.useSystemRoots=true' \
@@ -252,6 +257,7 @@ load _helpers
       -x templates/tests/test-runner.yaml  \
       --set 'global.tls.enabled=true' \
       --set 'global.tls.enableAutoEncrypt=true' \
+      --set 'server.enabled=false' \
       --set 'externalServers.enabled=true' \
       --set 'externalServers.https.address=consul.io' \
       --set 'externalServers.https.useSystemRoots=true' \

diff --git a/test/unit/server-acl-init-cleanup-clusterrole.bats b/test/unit/server-acl-init-cleanup-clusterrole.bats
@@ -43,6 +43,38 @@ load _helpers
   [ "${actual}" = "true" ]
 }
 
+@test "serverACLInitCleanup/ClusterRole: enabled with externalServers.enabled=true and global.acls.manageSystemACLs=true, but server.enabled set to false" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -x templates/server-acl-init-cleanup-clusterrole.yaml  \
+      --set 'server.enabled=false' \
+      --set 'global.acls.manageSystemACLs=true' \
+      --set 'externalServers.enabled=true' \
+      --set 'externalServers.https.address=foo.com' \
+      . | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "serverACLInitCleanup/ClusterRole: fails if both externalServers.enabled=true and server.enabled=true" {
+  cd `chart_dir`
+  run helm template \
+      -x templates/server-acl-init-cleanup-clusterrole.yaml  \
+      --set 'server.enabled=true' \
+      --set 'externalServers.enabled=true' .
+  [ "$status" -eq 1 ]
+  [[ "$output" =~ "only one of server.enabled or externalServers.enabled can be set" ]]
+}
+
+@test "serverACLInitCleanup/ClusterRole: fails if both externalServers.enabled=true and server.enabled not set to false" {
+  cd `chart_dir`
+  run helm template \
+      -x templates/server-acl-init-cleanup-clusterrole.yaml  \
+      --set 'externalServers.enabled=true' .
+  [ "$status" -eq 1 ]
+  [[ "$output" =~ "only one of server.enabled or externalServers.enabled can be set" ]]
+}
+
 #--------------------------------------------------------------------
 # global.enablePodSecurityPolicies
 

diff --git a/test/unit/server-acl-init-cleanup-clusterrolebinding.bats b/test/unit/server-acl-init-cleanup-clusterrolebinding.bats
@@ -42,3 +42,35 @@ load _helpers
       yq 'length > 0' | tee /dev/stderr)
   [ "${actual}" = "true" ]
 }
+
+@test "serverACLInitCleanup/ClusterRoleBinding: enabled with externalServers.enabled=true and global.acls.manageSystemACLs=true, but server.enabled set to false" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -x templates/server-acl-init-cleanup-clusterrolebinding.yaml  \
+      --set 'server.enabled=false' \
+      --set 'global.acls.manageSystemACLs=true' \
+      --set 'externalServers.enabled=true' \
+      --set 'externalServers.https.address=foo.com' \
+      . | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "serverACLInitCleanup/ClusterRoleBinding: fails if both externalServers.enabled=true and server.enabled=true" {
+  cd `chart_dir`
+  run helm template \
+      -x templates/server-acl-init-cleanup-clusterrolebinding.yaml  \
+      --set 'server.enabled=true' \
+      --set 'externalServers.enabled=true' .
+  [ "$status" -eq 1 ]
+  [[ "$output" =~ "only one of server.enabled or externalServers.enabled can be set" ]]
+}
+
+@test "serverACLInitCleanup/ClusterRoleBinding: fails if both externalServers.enabled=true and server.enabled not set to false" {
+  cd `chart_dir`
+  run helm template \
+      -x templates/server-acl-init-cleanup-clusterrolebinding.yaml  \
+      --set 'externalServers.enabled=true' .
+  [ "$status" -eq 1 ]
+  [[ "$output" =~ "only one of server.enabled or externalServers.enabled can be set" ]]
+}