Adjust API gateway controller deployment appropriately when Vault con…

…figured as secrets backend (#2083)

* Adjust mount based on whether Vault is enabled as secrets backend

* Add changelog entry

* Improve wording of changelog entry

* Use Vault serverca for CONSUL_CACERT when secrets backend enabled

* Add comment to Helm template explaining logic

* Add unit test for CONSUL_CACERT with Vault secret path

* Add unit tests for removing mounts when Vault is secrets backend

.changelog/2083.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+api-gateway: fix issue where the API Gateway controller is unable to start up successfully when Vault is configured as the secrets backend
+```

charts/consul/templates/api-gateway-controller-deployment.yaml

@@ -65,7 +65,14 @@ spec:
         {{- if or (not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots)) .Values.client.enabled }}
         {{- if .Values.global.tls.enabled }}
         - name: CONSUL_CACERT
+          {{- /* When Vault is being used as a secrets backend, auto-encrypt must be enabled. Since clients use a separate
+          root CA from servers when auto-encrypt is enabled, and our controller communicates with the agent when clients are
+          enabled, we only use the Vault server CA if clients are disabled and our controller will be communicating w/ the server. */}}
+          {{- if and (not .Values.client.enabled) .Values.global.secretsBackend.vault.enabled }}
+          value: /vault/secrets/serverca.crt
+          {{- else }}
           value: /consul/tls/ca/tls.crt
+          {{- end }}
         {{- end }}
         {{- end }}
         - name: HOST_IP
@@ -156,7 +163,7 @@ spec:
         - name: consul-bin
           mountPath: /consul-bin
         {{- end }}
-        {{- if or (not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots)) .Values.client.enabled }}
+        {{- if or (not (or (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) .Values.global.secretsBackend.vault.enabled)) .Values.client.enabled }}
         {{- if .Values.global.tls.enabled }}
         {{- if and .Values.client.enabled .Values.global.tls.enableAutoEncrypt }}
         - name: consul-auto-encrypt-ca-cert
@@ -186,7 +193,7 @@ spec:
         emptyDir: { }
       {{- end }}
       {{- if .Values.global.tls.enabled }}
-      {{- if not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) }}
+      {{- if not (or (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) .Values.global.secretsBackend.vault.enabled) }}
       - name: consul-ca-cert
         secret:
           {{- if .Values.global.tls.caCert.secretName }}
@@ -253,7 +260,7 @@ spec:
         - mountPath: /consul/login
           name: consul-data
           readOnly: false
-        {{- if not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) }}
+        {{- if not (or (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) .Values.global.secretsBackend.vault.enabled) }}
         {{- if .Values.global.tls.enabled }}
         - name: consul-ca-cert
           mountPath: /consul/tls/ca

charts/consul/test/unit/api-gateway-controller-deployment.bats

@@ -1522,6 +1522,23 @@ load _helpers
   [ "${actual}" = "true" ]
 }
 
+@test "apiGateway/Deployment: CONSUL_CACERT has correct path with Vault as secrets backend and client disabled" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/api-gateway-controller-deployment.yaml \
+      --set 'apiGateway.enabled=true' \
+      --set 'apiGateway.image=bar' \
+      --set 'global.tls.enabled=true' \
+      --set 'global.tls.caCert.secretName=foo' \
+      --set 'server.enabled=true' \
+      --set 'client.enabled=false' \
+      --set 'global.secretsBackend.vault.enabled=true' \
+      --set 'global.secretsBackend.vault.consulServerRole=foo' \
+      . | tee /dev/stderr|
+      yq '.spec.template.spec.containers[0].env[0].value == "/vault/secrets/serverca.crt"' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
 @test "apiGateway/Deployment: CONSUL_CACERT is not set when using tls and useSystemRoots" {
   cd `chart_dir`
   local actual=$(helm template \
@@ -1555,6 +1572,21 @@ load _helpers
   [ "${actual}" = "" ]
 }
 
+@test "apiGateway/Deployment: consul-ca-cert volume mount is not set when using Vault as a secrets backend" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/api-gateway-controller-deployment.yaml  \
+      --set 'apiGateway.enabled=true' \
+      --set 'apiGateway.image=bar' \
+      --set 'global.acls.manageSystemACLs=true' \
+      --set 'global.tls.enabled=true' \
+      --set 'server.enabled=true' \
+      --set 'global.secretsBackend.vault.enabled=true' \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers[0].volumeMounts[] | select(.name == "consul-ca-cert")' | tee /dev/stderr)
+  [ "${actual}" = "" ]
+}
+
 @test "apiGateway/Deployment: consul-ca-cert volume mount is not set on acl-init when using externalServers and useSystemRoots" {
   cd `chart_dir`
   local actual=$(helm template \
@@ -1572,6 +1604,21 @@ load _helpers
   [ "${actual}" = "" ]
 }
 
+@test "apiGateway/Deployment: consul-ca-cert volume mount is not set on acl-init when using Vault as secrets backend" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/api-gateway-controller-deployment.yaml  \
+      --set 'apiGateway.enabled=true' \
+      --set 'apiGateway.image=bar' \
+      --set 'global.acls.manageSystemACLs=true' \
+      --set 'global.tls.enabled=true' \
+      --set 'server.enabled=true' \
+      --set 'global.secretsBackend.vault.enabled=true' \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.initContainers[1].volumeMounts[] | select(.name == "consul-ca-cert")' | tee /dev/stderr)
+  [ "${actual}" = "" ]
+}
+
 @test "apiGateway/Deployment: consul-auto-encrypt-ca-cert volume mount is set when tls.enabled, client.enabled, externalServers, useSystemRoots, and autoencrypt" {
   cd `chart_dir`
   local actual=$(helm template \