From dad54ce3d3e522df0ccd1023d346b281e0a3ce04 Mon Sep 17 00:00:00 2001
From: Kyle Schochenmaier <kschoche@gmail.com>
Date: Thu, 28 Oct 2021 13:33:02 -0500
Subject: [PATCH 01/20] Add base bootstrapping logic and acceptance tests

---
 acceptance/framework/helpers/helpers.go       |  2 +-
 acceptance/framework/vault/vault_cluster.go   | 14 +--
 acceptance/tests/vault/vault_test.go          | 97 ++++++++++++++++++-
 charts/consul/templates/client-daemonset.yaml | 16 +++
 .../consul/templates/server-statefulset.yaml  | 16 +++
 charts/consul/values.yaml                     | 19 ++++
 6 files changed, 147 insertions(+), 17 deletions(-)

diff --git a/acceptance/framework/helpers/helpers.go b/acceptance/framework/helpers/helpers.go
index 294e395cb2..9acd5eebc4 100644
--- a/acceptance/framework/helpers/helpers.go
+++ b/acceptance/framework/helpers/helpers.go
@@ -76,7 +76,7 @@ func CheckForPriorInstallations(t *testing.T, client kubernetes.Interface, optio
 func WaitForAllPodsToBeReady(t *testing.T, client kubernetes.Interface, namespace, podLabelSelector string) {
 	t.Helper()
 
-	logger.Log(t, "Waiting for pods to be ready.")
+	logger.Logf(t, "Waiting for pods to be ready.")
 
 	// Wait up to 15m.
 	// On Azure, volume provisioning can sometimes take close to 5 min,
diff --git a/acceptance/framework/vault/vault_cluster.go b/acceptance/framework/vault/vault_cluster.go
index 36644c1abe..e51e5e3530 100644
--- a/acceptance/framework/vault/vault_cluster.go
+++ b/acceptance/framework/vault/vault_cluster.go
@@ -1,12 +1,9 @@
 package vault
 
 import (
-	"context"
 	"fmt"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"testing"
 	"time"
 
@@ -179,15 +176,8 @@ func (v *VaultCluster) Create(t *testing.T, ctx environment.TestContext) {
 
 	// Install Vault.
 	helm.Install(t, v.vaultHelmOptions, "hashicorp/vault", v.vaultReleaseName)
-	// Wait for the injector pod to become Ready, but not the server.
-	helpers.WaitForAllPodsToBeReady(t, v.kubernetesClient, v.vaultHelmOptions.KubectlOptions.Namespace, "app.kubernetes.io/name=vault-agent-injector")
-	// Wait for the server pod to be PodRunning, it will not be Ready because it has not been Init+Unseal'd yet.
-	// The vault server has health checks bound to unseal status, and Unseal is done as part of bootstrap (below).
-	retry.RunWith(&retry.Counter{Wait: 1 * time.Second, Count: 30}, t, func(r *retry.R) {
-		pod, err := v.kubernetesClient.CoreV1().Pods(v.vaultHelmOptions.KubectlOptions.Namespace).Get(context.Background(), fmt.Sprintf("%s-vault-0", v.vaultReleaseName), metav1.GetOptions{})
-		require.NoError(r, err)
-		require.Equal(r, pod.Status.Phase, corev1.PodRunning)
-	})
+	// Wait for the injector and vault server pods to become Ready.
+	helpers.WaitForAllPodsToBeReady(t, v.kubernetesClient, v.vaultHelmOptions.KubectlOptions.Namespace, fmt.Sprintf("helm.sh/chart=%s", v.vaultChartName))
 	// Now call bootstrap()
 	v.bootstrap(t, ctx)
 }
diff --git a/acceptance/tests/vault/vault_test.go b/acceptance/tests/vault/vault_test.go
index fac892bf0e..14dd1b74de 100644
--- a/acceptance/tests/vault/vault_test.go
+++ b/acceptance/tests/vault/vault_test.go
@@ -1,13 +1,17 @@
 package vault
 
 import (
-	"testing"
-	"time"
-
+	"fmt"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/vault"
 	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+const (
+	gossipKey = "3R7oLrdpkk2V0Y7yHLizyxXeS2RtaVuy07DkU15Lhws="
 )
 
 // Installs Vault, bootstraps it with the kube auth method
@@ -39,5 +43,90 @@ func TestVault_Create(t *testing.T) {
 	require.NoError(t, err)
 	logger.Log(t, "Auth List: ", authList)
 	require.NotNil(t, authList["kubernetes/"])
-	time.Sleep(time.Second * 60)
+}
+
+// Installs Vault, bootstraps it with secrets, policies, and kube auth method
+// then creates a gossip encryption secret and uses this to bootstrap Consul
+func TestVaultConsulGossipEncryptionKeyIntegration(t *testing.T) {
+	cfg := suite.Config()
+	ctx := suite.Environment().DefaultContext(t)
+
+	consulReleaseName := helpers.RandomName()
+	vaultReleaseName := helpers.RandomName()
+	consulClientServiceAccountName := fmt.Sprintf("%s-consul-client", consulReleaseName)
+	consulServerServiceAccountName := fmt.Sprintf("%s-consul-server", consulReleaseName)
+
+	vaultCluster := vault.NewVaultCluster(t, nil, ctx, cfg, vaultReleaseName)
+	vaultCluster.Create(t, ctx)
+	// Vault is now installed in the cluster.
+
+	// Now fetch the Vault client so we can create the policies and secrets.
+	vaultClient := vaultCluster.VaultClient(t)
+
+	// Create the Vault Policy for the gossip key.
+	logger.Log(t, "Creating the gossip policy")
+	rules := `
+path "consul/data/secret/gossip" {
+  capabilities = ["read"]
+}`
+	err := vaultClient.Sys().PutPolicy("consul-gossip", rules)
+	require.NoError(t, err)
+
+	// Create the Auth Roles for consul-server + consul-client.
+	logger.Log(t, "Creating the gossip auth roles")
+	params := map[string]interface{}{
+		"bound_service_account_names":      consulClientServiceAccountName,
+		"bound_service_account_namespaces": "default",
+		"policies":                         "consul-gossip",
+		"ttl":                              "24h",
+	}
+	_, err = vaultClient.Logical().Write("auth/kubernetes/role/consul-client", params)
+	require.NoError(t, err)
+
+	params = map[string]interface{}{
+		"bound_service_account_names":      consulServerServiceAccountName,
+		"bound_service_account_namespaces": "default",
+		"policies":                         "consul-gossip",
+		"ttl":                              "24h",
+	}
+	_, err = vaultClient.Logical().Write("auth/kubernetes/role/consul-server", params)
+	require.NoError(t, err)
+
+	// Create the gossip key.
+	logger.Log(t, "Creating the gossip secret")
+	params = map[string]interface{}{
+		"data": map[string]interface{}{
+			"gossip": gossipKey,
+		},
+	}
+	_, err = vaultClient.Logical().Write("consul/data/secret/gossip", params)
+	require.NoError(t, err)
+
+	consulHelmValues := map[string]string{
+		"server.enabled":  "true",
+		"server.replicas": "1",
+
+		"connectInject.enabled": "true",
+
+		"global.secretsBackend.vault.enabled":          "true",
+		"global.secretsBackend.vault.consulServerRole": "consul-server",
+		"global.secretsBackend.vault.consulclientRole": "consul-client",
+
+		"global.acls.manageSystemACLs":       "true",
+		"global.tls.enabled":                 "true",
+		"global.gossipEncryption.secretName": "consul/data/secret/gossip",
+		"global.gossipEncryption.secretKey":  ".Data.data.gossip",
+	}
+	logger.Log(t, "Installing Consul")
+	consulCluster := consul.NewHelmCluster(t, consulHelmValues, ctx, cfg, consulReleaseName)
+	consulCluster.Create(t)
+
+	// Validate that the gossip encryption key is set correctly.
+	logger.Log(t, "Validating the gossip key has been set correctly.")
+	consulClient := consulCluster.SetupConsulClient(t, true)
+	keys, err := consulClient.Operator().KeyringList(nil)
+	require.NoError(t, err)
+	// we use keys[0] because KeyringList returns a list of keyrings for each dc, in this case there is only 1 dc.
+	require.NotNil(t, keys[0].PrimaryKeys[gossipKey])
+
 }
diff --git a/charts/consul/templates/client-daemonset.yaml b/charts/consul/templates/client-daemonset.yaml
index d4e1c05107..2c7264022b 100644
--- a/charts/consul/templates/client-daemonset.yaml
+++ b/charts/consul/templates/client-daemonset.yaml
@@ -37,6 +37,16 @@ spec:
           {{- toYaml .Values.client.extraLabels | nindent 8 }}
         {{- end }}
       annotations:
+        {{- if .Values.global.secretsBackend.vault.enabled }}
+        "vault.hashicorp.com/agent-inject": "true"
+        "vault.hashicorp.com/agent-init-first": "true"
+        "vault.hashicorp.com/role": "{{ .Values.global.secretsBackend.vault.consulClientRole }}"
+        {{- with .Values.global.gossipEncryption }}
+        "vault.hashicorp.com/agent-inject-secret-gossip.txt": "{{ .secretName }}"
+        "vault.hashicorp.com/agent-inject-template-gossip.txt": '{{ "{{" }}- with secret "{{ .secretName }}" -{{ "}}" }} {{ "{{" }} {{ .secretKey }} {{ "}}" }} {{ "{{" }}- end -{{ "}}" }}'
+        "vault.hashicorp.com/template-static-secret-render-interval": "10s"
+        {{- end }}
+        {{- end }}
         "consul.hashicorp.com/connect-inject": "false"
         "consul.hashicorp.com/config-checksum": {{ include (print $.Template.BasePath "/client-config-configmap.yaml") . | sha256sum }}
         {{- if .Values.client.annotations }}
@@ -169,6 +179,7 @@ spec:
             - name: CONSUL_DISABLE_PERM_MGMT
               value: "true"
             {{- if (or .Values.global.gossipEncryption.autoGenerate (and .Values.global.gossipEncryption.secretName .Values.global.gossipEncryption.secretKey)) }}
+            {{- if not .Values.global.secretsBackend.vault.enabled }}
             - name: GOSSIP_KEY
               valueFrom:
                 secretKeyRef:
@@ -180,6 +191,7 @@ spec:
                   key: {{ .Values.global.gossipEncryption.secretKey }}
                 {{- end }}
             {{- end }}
+            {{- end }}
             {{- if (and .Values.server.enterpriseLicense.secretName .Values.server.enterpriseLicense.secretKey .Values.server.enterpriseLicense.enableLicenseAutoload (not .Values.global.acls.manageSystemACLs)) }}
             - name: CONSUL_LICENSE_PATH
               value: /consul/license/{{ .Values.server.enterpriseLicense.secretKey }}
@@ -202,6 +214,10 @@ spec:
             - |
               CONSUL_FULLNAME="{{template "consul.fullname" . }}"
 
+              {{- if .Values.global.secretsBackend.vault.enabled }}
+              GOSSIP_KEY=`cat /vault/secrets/gossip.txt`
+              {{- end }}
+
               {{ template "consul.extraconfig" }}
 
               exec /usr/local/bin/docker-entrypoint.sh consul agent \
diff --git a/charts/consul/templates/server-statefulset.yaml b/charts/consul/templates/server-statefulset.yaml
index fe2dcff468..bbf35e44b0 100644
--- a/charts/consul/templates/server-statefulset.yaml
+++ b/charts/consul/templates/server-statefulset.yaml
@@ -46,6 +46,16 @@ spec:
           {{- toYaml .Values.server.extraLabels | nindent 8 }}
         {{- end }}
       annotations:
+        {{- if .Values.global.secretsBackend.vault.enabled }}
+        "vault.hashicorp.com/agent-inject": "true"
+        "vault.hashicorp.com/agent-init-first": "true"
+        "vault.hashicorp.com/role": "{{ .Values.global.secretsBackend.vault.consulServerRole }}"
+        {{- with .Values.global.gossipEncryption }}
+        "vault.hashicorp.com/agent-inject-secret-gossip.txt": "{{ .secretName }}"
+        "vault.hashicorp.com/agent-inject-template-gossip.txt": '{{ "{{" }}- with secret "{{ .secretName }}" -{{ "}}" }} {{ "{{" }} {{ .secretKey }} {{ "}}" }} {{ "{{" }}- end -{{ "}}" }}'
+        "vault.hashicorp.com/template-static-secret-render-interval": "10s"
+        {{- end }}
+        {{- end }}
         "consul.hashicorp.com/connect-inject": "false"
         "consul.hashicorp.com/config-checksum": {{ include (print $.Template.BasePath "/server-config-configmap.yaml") . | sha256sum }}
         {{- if .Values.server.annotations }}
@@ -157,6 +167,7 @@ spec:
             - name: CONSUL_DISABLE_PERM_MGMT
               value: "true"
             {{- if (or .Values.global.gossipEncryption.autoGenerate (and .Values.global.gossipEncryption.secretName .Values.global.gossipEncryption.secretKey)) }}
+            {{- if not .Values.global.secretsBackend.vault.enabled }}
             - name: GOSSIP_KEY
               valueFrom:
                 secretKeyRef:
@@ -168,6 +179,7 @@ spec:
                   key: {{ .Values.global.gossipEncryption.secretKey }}
                 {{- end }}
             {{- end }}
+            {{- end }}
             {{- if .Values.global.tls.enabled }}
             - name: CONSUL_HTTP_ADDR
               value: https://localhost:8501
@@ -192,6 +204,10 @@ spec:
             - |
               CONSUL_FULLNAME="{{template "consul.fullname" . }}"
 
+              {{- if .Values.global.secretsBackend.vault.enabled }}
+              GOSSIP_KEY=`cat /vault/secrets/gossip.txt`
+              {{- end }}
+
               {{ template "consul.extraconfig" }}
 
               exec /usr/local/bin/docker-entrypoint.sh consul agent \
diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml
index 44b81cc327..ce67aee1f0 100644
--- a/charts/consul/values.yaml
+++ b/charts/consul/values.yaml
@@ -117,6 +117,24 @@ global:
   # created by this chart. See https://kubernetes.io/docs/concepts/policy/pod-security-policy/.
   enablePodSecurityPolicies: false
 
+  # secretsBackend is used to enable and disable the use of Vault as the secrets backend for the Consul on Kubernetes installation.
+  # In order to utilize this configuration it is required to have an accessible vault cluster which has been pre-configured
+  # by a Vault operator to have the Vault Kube Auth Method, KV2 + PKI secrets engines enabled as well as pre-created
+  # vault secrets, policies and roles.
+  secretsBackend:
+    vault:
+      # Enabling the Vault secrets backend will cause K8S secrets to no longer be used for the following Consul secrets:
+      # - gossip encryption key defined by `global.gossipEncryption.secretName/Key`
+      enabled: false
+
+      # The consulServerRole must be created and have read capabilties via a Policy for the following secrets:
+      # - gossip encryption key defined by `global.gossipEncryption.secretName/Key`.
+      consulServerRole: ""  # e.g. "consul-server"
+
+      # The consulClientRole must be created and have read capabilties via a Policy for the following secrets:
+      # - gossip encryption key defined by `global.gossipEncryption.secretName/Key`
+      consulClientRole: ""  # e.g. "consul-client"
+
   # Configures Consul's gossip encryption key, set as a Kubernetes secret
   # (see `-encrypt` (https://consul.io/docs/agent/options#_encrypt)).
   # By default, gossip encryption is not enabled. The gossip encryption key may be set automatically or manually.
@@ -129,6 +147,7 @@ global:
   # ```
   # $ kubectl create secret generic consul-gossip-encryption-key --from-literal=key=$(consul keygen)
   # ```
+
   gossipEncryption:
     # Automatically generate a gossip encryption key and save it to a Kubernetes secret.
     autoGenerate: false

From 60ea8ead91c7f246b33edbfe26c91f63a481cfea Mon Sep 17 00:00:00 2001
From: Kyle Schochenmaier <kschoche@gmail.com>
Date: Thu, 28 Oct 2021 13:42:22 -0500
Subject: [PATCH 02/20] a little bit of cleanup

---
 acceptance/framework/helpers/helpers.go     |  2 +-
 acceptance/framework/vault/vault_cluster.go |  4 ++--
 acceptance/tests/vault/vault_test.go        | 11 ++++++-----
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/acceptance/framework/helpers/helpers.go b/acceptance/framework/helpers/helpers.go
index 9acd5eebc4..294e395cb2 100644
--- a/acceptance/framework/helpers/helpers.go
+++ b/acceptance/framework/helpers/helpers.go
@@ -76,7 +76,7 @@ func CheckForPriorInstallations(t *testing.T, client kubernetes.Interface, optio
 func WaitForAllPodsToBeReady(t *testing.T, client kubernetes.Interface, namespace, podLabelSelector string) {
 	t.Helper()
 
-	logger.Logf(t, "Waiting for pods to be ready.")
+	logger.Log(t, "Waiting for pods to be ready.")
 
 	// Wait up to 15m.
 	// On Azure, volume provisioning can sometimes take close to 5 min,
diff --git a/acceptance/framework/vault/vault_cluster.go b/acceptance/framework/vault/vault_cluster.go
index e51e5e3530..c69cf696a5 100644
--- a/acceptance/framework/vault/vault_cluster.go
+++ b/acceptance/framework/vault/vault_cluster.go
@@ -2,8 +2,6 @@ package vault
 
 import (
 	"fmt"
-	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
-	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
 	"testing"
 	"time"
 
@@ -12,6 +10,8 @@ import (
 	terratestLogger "github.com/gruntwork-io/terratest/modules/logger"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/config"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/environment"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 	"github.com/hashicorp/consul/sdk/testutil/retry"
 	vapi "github.com/hashicorp/vault/api"
diff --git a/acceptance/tests/vault/vault_test.go b/acceptance/tests/vault/vault_test.go
index 14dd1b74de..2e4aadd982 100644
--- a/acceptance/tests/vault/vault_test.go
+++ b/acceptance/tests/vault/vault_test.go
@@ -2,19 +2,20 @@ package vault
 
 import (
 	"fmt"
+	"testing"
+
 	"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/vault"
 	"github.com/stretchr/testify/require"
-	"testing"
 )
 
 const (
 	gossipKey = "3R7oLrdpkk2V0Y7yHLizyxXeS2RtaVuy07DkU15Lhws="
 )
 
-// Installs Vault, bootstraps it with the kube auth method
+// Installs Vault, bootstraps it with the Kube Auth Method
 // and then validates that the KV2 secrets engine is online
 // and the Kube Auth Method is enabled.
 func TestVault_Create(t *testing.T) {
@@ -45,9 +46,9 @@ func TestVault_Create(t *testing.T) {
 	require.NotNil(t, authList["kubernetes/"])
 }
 
-// Installs Vault, bootstraps it with secrets, policies, and kube auth method
-// then creates a gossip encryption secret and uses this to bootstrap Consul
-func TestVaultConsulGossipEncryptionKeyIntegration(t *testing.T) {
+// Installs Vault, bootstraps it with secrets, policies, and Kube Auth Method
+// then creates a gossip encryption secret and uses this to bootstrap Consul.
+func TestVault_BootstrapConsulGossipEncryptionKey(t *testing.T) {
 	cfg := suite.Config()
 	ctx := suite.Environment().DefaultContext(t)
 

From 209be6d174764ef8c1c29f1ea6e4571bfe65e97a Mon Sep 17 00:00:00 2001
From: Kyle Schochenmaier <kschoche@gmail.com>
Date: Fri, 29 Oct 2021 13:38:48 -0500
Subject: [PATCH 03/20] add bats tests

---
 charts/consul/templates/client-daemonset.yaml |  2 +-
 .../consul/templates/server-statefulset.yaml  |  2 +
 charts/consul/test/unit/client-daemonset.bats | 72 ++++++++++++++++++-
 .../consul/test/unit/server-statefulset.bats  | 72 ++++++++++++++++++-
 4 files changed, 145 insertions(+), 3 deletions(-)

diff --git a/charts/consul/templates/client-daemonset.yaml b/charts/consul/templates/client-daemonset.yaml
index 2c7264022b..50bffbe59c 100644
--- a/charts/consul/templates/client-daemonset.yaml
+++ b/charts/consul/templates/client-daemonset.yaml
@@ -214,7 +214,7 @@ spec:
             - |
               CONSUL_FULLNAME="{{template "consul.fullname" . }}"
 
-              {{- if .Values.global.secretsBackend.vault.enabled }}
+              {{- if and .Values.global.secretsBackend.vault.enabled .Values.global.gossipEncryption.secretName }}
               GOSSIP_KEY=`cat /vault/secrets/gossip.txt`
               {{- end }}
 
diff --git a/charts/consul/templates/server-statefulset.yaml b/charts/consul/templates/server-statefulset.yaml
index bbf35e44b0..901e4a7fae 100644
--- a/charts/consul/templates/server-statefulset.yaml
+++ b/charts/consul/templates/server-statefulset.yaml
@@ -5,6 +5,8 @@
 {{- if .Values.server.disableFsGroupSecurityContext }}{{ fail "server.disableFsGroupSecurityContext has been removed. Please use global.openshift.enabled instead." }}{{ end }}
 {{- if .Values.server.bootstrapExpect }}{{ if lt (int .Values.server.bootstrapExpect) (int .Values.server.replicas) }}{{ fail "server.bootstrapExpect cannot be less than server.replicas" }}{{ end }}{{ end }}
 {{- if (and (and .Values.global.tls.enabled .Values.global.tls.httpsOnly) (and .Values.global.metrics.enabled .Values.global.metrics.enableAgentMetrics))}}{{ fail "global.metrics.enableAgentMetrics cannot be enabled if TLS (HTTPS only) is enabled" }}{{ end -}}
+{{- if (and .Values.global.gossipEncryption.secretName (not .Values.global.gossipEncryption.secretKey)) }}{{fail "gossipEncryption.secretKey and secretName must both be specified." }}{{ end -}}
+{{- if (and (not .Values.global.gossipEncryption.secretName) .Values.global.gossipEncryption.secretKey) }}{{fail "gossipEncryption.secretKey and secretName must both be specified." }}{{ end -}}
 # StatefulSet to run the actual Consul server cluster.
 apiVersion: apps/v1
 kind: StatefulSet
diff --git a/charts/consul/test/unit/client-daemonset.bats b/charts/consul/test/unit/client-daemonset.bats
index 64a9d0e4c0..41c56c60f8 100755
--- a/charts/consul/test/unit/client-daemonset.bats
+++ b/charts/consul/test/unit/client-daemonset.bats
@@ -602,7 +602,7 @@ load _helpers
   local actual=$(helm template \
       -s templates/client-daemonset.yaml  \
       . | tee /dev/stderr |
-      yq '.spec.template.spec.containers[] | select(.name=="consul") | .env[] | select(.name == "GOSSIP_KEY") | length > 0' | tee /dev/stderr)
+      yq '.spec.template.spec.containers[] | select(.name=="consul") | .env[] | select(.name == "GOSSIP_KEY")' | tee /dev/stderr)
   [ "${actual}" = "" ]
 }
 
@@ -1516,3 +1516,73 @@ rollingUpdate:
 
   [ "${object}" = 1 ]
 }
+
+#--------------------------------------------------------------------
+# vault integration
+
+@test "client/DaemonSet: vault annotations not attached by default" {
+  cd `chart_dir`
+  local actual=$(helm template \
+    -s templates/client-daemonset.yaml  \
+    . | tee /dev/stderr |
+     yq '.spec.template.metadata.annotations["vault.hashicorp.com/agent-inject"] | length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+}
+
+@test "client/DaemonSet: vault annotations added when vault is enabled" {
+  cd `chart_dir`
+  local object=$(helm template \
+    -s templates/client-daemonset.yaml  \
+    --set 'global.secretsBackend.vault.enabled=true' \
+    . | tee /dev/stderr |
+      yq -r '.spec.template.metadata' | tee /dev/stderr)
+
+  local actual=$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/agent-inject"] | length > 0' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+  local actual=$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/agent-init-first"] | length > 0' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+  local actual=$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/role"]' | tee /dev/stderr)
+  [ "${actual}" = "" ]
+}
+
+@test "client/DaemonSet: vault gossip annotations are correct when enabled" {
+  cd `chart_dir`
+  local object=$(helm template \
+    -s templates/client-daemonset.yaml  \
+    --set 'global.secretsBackend.vault.enabled=true' \
+    --set 'global.gossipEncryption.secretName=path/to/secret/key' \
+    --set 'global.gossipEncryption.secretKey=.Data.gossip.gossip' \
+    . | tee /dev/stderr |
+      yq -r '.spec.template.metadata' | tee /dev/stderr)
+
+  local actual=$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/agent-inject-secret-gossip.txt"]' | tee /dev/stderr)
+  [ "${actual}" = "path/to/secret/key" ]
+  local actual=$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/agent-inject-template-gossip.txt"]' | tee /dev/stderr)
+  [ "${actual}" = '{{- with secret "path/to/secret/key" -}} {{ .Data.gossip.gossip }} {{- end -}}' ]
+}
+
+@test "client/DaemonSet: vault no GOSSIP_KEY env variable and command defines GOSSIP_KEY" {
+  cd `chart_dir`
+  local object=$(helm template \
+    -s templates/client-daemonset.yaml  \
+    --set 'global.secretsBackend.vault.enabled=true' \
+    --set 'global.gossipEncryption.secretName=a/b/c/d' \
+    --set 'global.gossipEncryption.secretKey=.Data.data.gossip' \
+    . | tee /dev/stderr |
+      yq -r '.spec.template.spec' | tee /dev/stderr)
+
+
+  local actual=$(echo $object |
+    yq -r '.containers[] | select(.name=="consul") | .env[] | select(.name == "GOSSIP_KEY")' | tee /dev/stderr)
+  [ "${actual}" = "" ]
+
+  local actual=$(echo $object |
+    yq -r '.containers[] | select(.name=="consul") | .command | any(contains("GOSSIP_KEY="))' \
+      | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
diff --git a/charts/consul/test/unit/server-statefulset.bats b/charts/consul/test/unit/server-statefulset.bats
index 6418cef313..a7e7fa6073 100755
--- a/charts/consul/test/unit/server-statefulset.bats
+++ b/charts/consul/test/unit/server-statefulset.bats
@@ -838,7 +838,7 @@ load _helpers
   local actual=$(helm template \
       -s templates/server-statefulset.yaml  \
       . | tee /dev/stderr |
-      yq '.spec.template.spec.containers[] | select(.name=="consul") | .env[] | select(.name == "GOSSIP_KEY") | length > 0' | tee /dev/stderr)
+      yq '.spec.template.spec.containers[] | select(.name=="consul") | .env[] | select(.name == "GOSSIP_KEY")' | tee /dev/stderr)
   [ "${actual}" = "" ]
 }
 
@@ -1415,3 +1415,73 @@ load _helpers
 
   [ "${object}" = 1 ]
 }
+
+#--------------------------------------------------------------------
+# vault integration
+
+@test "server/StatefulSet: vault annotations not attached by default" {
+  cd `chart_dir`
+  local actual=$(helm template \
+    -s templates/server-statefulset.yaml  \
+    . | tee /dev/stderr |
+     yq '.spec.template.metadata.annotations["vault.hashicorp.com/agent-inject"] | length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+}
+
+@test "server/StatefulSet: vault annotations added when vault is enabled" {
+  cd `chart_dir`
+  local object=$(helm template \
+    -s templates/server-statefulset.yaml  \
+    --set 'global.secretsBackend.vault.enabled=true' \
+    . | tee /dev/stderr |
+      yq -r '.spec.template.metadata' | tee /dev/stderr)
+
+  local actual=$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/agent-inject"] | length > 0' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+  local actual=$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/agent-init-first"] | length > 0' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+  local actual=$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/role"]' | tee /dev/stderr)
+  [ "${actual}" = "" ]
+}
+
+@test "server/StatefulSet: vault gossip annotations are correct when enabled" {
+  cd `chart_dir`
+  local object=$(helm template \
+    -s templates/server-statefulset.yaml  \
+    --set 'global.secretsBackend.vault.enabled=true' \
+    --set 'global.gossipEncryption.secretName=path/to/secret/key' \
+    --set 'global.gossipEncryption.secretKey=.Data.gossip.gossip' \
+    . | tee /dev/stderr |
+      yq -r '.spec.template.metadata' | tee /dev/stderr)
+
+  local actual=$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/agent-inject-secret-gossip.txt"]' | tee /dev/stderr)
+  [ "${actual}" = "path/to/secret/key" ]
+  local actual=$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/agent-inject-template-gossip.txt"]' | tee /dev/stderr)
+  [ "${actual}" = '{{- with secret "path/to/secret/key" -}} {{ .Data.gossip.gossip }} {{- end -}}' ]
+}
+
+@test "server/StatefulSet: vault no GOSSIP_KEY env variable and command defines GOSSIP_KEY" {
+  cd `chart_dir`
+  local object=$(helm template \
+    -s templates/server-statefulset.yaml  \
+    --set 'global.secretsBackend.vault.enabled=true' \
+    --set 'global.gossipEncryption.secretName=a/b/c/d' \
+    --set 'global.gossipEncryption.secretKey=.Data.data.gossip' \
+    . | tee /dev/stderr |
+      yq -r '.spec.template.spec' | tee /dev/stderr)
+
+
+  local actual=$(echo $object |
+    yq -r '.containers[] | select(.name=="consul") | .env[] | select(.name == "GOSSIP_KEY")' | tee /dev/stderr)
+  [ "${actual}" = "" ]
+
+  local actual=$(echo $object |
+    yq -r '.containers[] | select(.name=="consul") | .command | any(contains("GOSSIP_KEY="))' \
+      | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}

From 547888e0479f1aa66d0ea160dd7d2ac05ac3d04a Mon Sep 17 00:00:00 2001
From: Kyle Schochenmaier <kschoche@gmail.com>
Date: Mon, 1 Nov 2021 09:17:44 -0500
Subject: [PATCH 04/20] add fail if secrename/key arent specified properly

---
 .../gossip-encryption-autogenerate-job.bats   | 23 -------------------
 .../consul/test/unit/server-statefulset.bats  | 21 +++++++----------
 2 files changed, 8 insertions(+), 36 deletions(-)

diff --git a/charts/consul/test/unit/gossip-encryption-autogenerate-job.bats b/charts/consul/test/unit/gossip-encryption-autogenerate-job.bats
index b78b9c231d..a2a0de0a95 100644
--- a/charts/consul/test/unit/gossip-encryption-autogenerate-job.bats
+++ b/charts/consul/test/unit/gossip-encryption-autogenerate-job.bats
@@ -38,26 +38,3 @@ load _helpers
   [ "$status" -eq 1 ]
   [[ "$output" =~ "If global.gossipEncryption.autoGenerate is true, global.gossipEncryption.secretName and global.gossipEncryption.secretKey must not be set." ]]
 }
-
-@test "gossipEncryptionAutogenerate/Job: fails if global.gossipEncryption.autoGenerate=true and global.gossipEncryption.secretName is set" {
-  cd `chart_dir`
-  run helm template \
-      -s templates/gossip-encryption-autogenerate-job.yaml  \
-      --set 'global.gossipEncryption.autoGenerate=true' \
-      --set 'global.gossipEncryption.secretName=name' \
-      . 
-  [ "$status" -eq 1 ]
-  [[ "$output" =~ "If global.gossipEncryption.autoGenerate is true, global.gossipEncryption.secretName and global.gossipEncryption.secretKey must not be set." ]]
-}
-
-@test "gossipEncryptionAutogenerate/Job: fails if global.gossipEncryption.autoGenerate=true and global.gossipEncryption.secretKey is set" {
-  cd `chart_dir`
-  run helm template \
-      -s templates/gossip-encryption-autogenerate-job.yaml  \
-      --set 'global.gossipEncryption.autoGenerate=true' \
-      --set 'global.gossipEncryption.secretKey=key' \
-      . 
-  [ "$status" -eq 1 ]
-  [[ "$output" =~ "If global.gossipEncryption.autoGenerate is true, global.gossipEncryption.secretName and global.gossipEncryption.secretKey must not be set." ]]
-}
-
diff --git a/charts/consul/test/unit/server-statefulset.bats b/charts/consul/test/unit/server-statefulset.bats
index a7e7fa6073..0aa6ae3335 100755
--- a/charts/consul/test/unit/server-statefulset.bats
+++ b/charts/consul/test/unit/server-statefulset.bats
@@ -863,25 +863,20 @@ load _helpers
   [ "${actual}" = "true" ]
 }
 
-
-@test "server/StatefulSet: gossip encryption disabled in server StatefulSet when secretName is missing" {
+@test "server/StatefulSet: fail if global.gossipEncyption.gossipEncryption.secretName is set but not global.gossipEncyption.secretKey" {
   cd `chart_dir`
-  local actual=$(helm template \
+  run helm template \
       -s templates/server-statefulset.yaml  \
-      --set 'global.gossipEncryption.secretKey=bar' \
-      . | tee /dev/stderr |
-      yq '.spec.template.spec.containers[] | select(.name=="consul") | .env[] | select(.name == "GOSSIP_KEY") | length > 0' | tee /dev/stderr)
-  [ "${actual}" = "" ]
+      --set 'global.gossipEncryption.secretName=bar' .
+  [[ "$output" =~ "gossipEncryption.secretKey and secretName must both be specified." ]]
 }
 
-@test "server/StatefulSet: gossip encryption disabled in server StatefulSet when secretKey is missing" {
+@test "server/StatefulSet: fail if global.gossipEncyption.gossipEncryption.secretKey is set but not global.gossipEncyption.secretName" {
   cd `chart_dir`
-  local actual=$(helm template \
+  run helm template \
       -s templates/server-statefulset.yaml  \
-      --set 'global.gossipEncryption.secretName=foo' \
-      . | tee /dev/stderr |
-      yq '.spec.template.spec.containers[] | select(.name=="consul") | .env[] | select(.name == "GOSSIP_KEY") | length > 0' | tee /dev/stderr)
-  [ "${actual}" = "" ]
+      --set 'global.gossipEncryption.secretKey=bar' .
+  [[ "$output" =~ "gossipEncryption.secretKey and secretName must both be specified." ]]
 }
 
 @test "server/StatefulSet: gossip environment variable present in server StatefulSet when all config is provided" {

From 51d012a539fad5cfdaf3564fea7119bd91c82719 Mon Sep 17 00:00:00 2001
From: Kyle Schochenmaier <kschoche@gmail.com>
Date: Mon, 1 Nov 2021 10:57:41 -0500
Subject: [PATCH 05/20] test

---
 acceptance/framework/vault/vault_cluster.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/acceptance/framework/vault/vault_cluster.go b/acceptance/framework/vault/vault_cluster.go
index c69cf696a5..dc6449f0e1 100644
--- a/acceptance/framework/vault/vault_cluster.go
+++ b/acceptance/framework/vault/vault_cluster.go
@@ -72,6 +72,7 @@ func NewVaultCluster(
 	// Ignoring the error from `helm repo update` as it could fail due to stale cache or unreachable servers and we're
 	// asserting a chart version on Install which would fail in an obvious way should this not succeed.
 	_, _ = helm.RunHelmCommandAndGetOutputE(t, &helm.Options{}, "repo", "update")
+	_, _ = helm.RunHelmCommandAndGetOutputE(t, &helm.Options{}, "pull", "hashicorp/vault", "--version", vaultChartVersion)
 
 	return &VaultCluster{
 		ctx:                ctx,

From 52d4af6e4e885aedc987d24a5c67e7b344aa6e5f Mon Sep 17 00:00:00 2001
From: Kyle Schochenmaier <kschoche@gmail.com>
Date: Mon, 1 Nov 2021 12:31:32 -0500
Subject: [PATCH 06/20] test

---
 acceptance/framework/vault/vault_cluster.go | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/acceptance/framework/vault/vault_cluster.go b/acceptance/framework/vault/vault_cluster.go
index dc6449f0e1..9a3069a344 100644
--- a/acceptance/framework/vault/vault_cluster.go
+++ b/acceptance/framework/vault/vault_cluster.go
@@ -71,8 +71,14 @@ func NewVaultCluster(
 	helm.AddRepo(t, &helm.Options{}, "hashicorp/vault", "https://helm.releases.hashicorp.com")
 	// Ignoring the error from `helm repo update` as it could fail due to stale cache or unreachable servers and we're
 	// asserting a chart version on Install which would fail in an obvious way should this not succeed.
-	_, _ = helm.RunHelmCommandAndGetOutputE(t, &helm.Options{}, "repo", "update")
-	_, _ = helm.RunHelmCommandAndGetOutputE(t, &helm.Options{}, "pull", "hashicorp/vault", "--version", vaultChartVersion)
+	errStr, err := helm.RunHelmCommandAndGetOutputE(t, &helm.Options{}, "repo", "update")
+	if err != nil {
+		logger.Logf(t, "Unable to update helm repository: %s, %s", err, errStr)
+	}
+	errStr, err = helm.RunHelmCommandAndGetOutputE(t, &helm.Options{}, "pull", "hashicorp/vault", "--version", vaultChartVersion)
+	if err != nil {
+		logger.Logf(t, "Unable to pull helm repository: %s, %s", err, errStr)
+	}
 
 	return &VaultCluster{
 		ctx:                ctx,

From e80cd6323973fc344b73879e3b98356a26d69dc8 Mon Sep 17 00:00:00 2001
From: Kyle Schochenmaier <kschoche@gmail.com>
Date: Mon, 1 Nov 2021 14:06:41 -0500
Subject: [PATCH 07/20] PR comments

---
 acceptance/tests/vault/vault_test.go | 3 +--
 charts/consul/values.yaml            | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/acceptance/tests/vault/vault_test.go b/acceptance/tests/vault/vault_test.go
index 2e4aadd982..8720232a5b 100644
--- a/acceptance/tests/vault/vault_test.go
+++ b/acceptance/tests/vault/vault_test.go
@@ -128,6 +128,5 @@ path "consul/data/secret/gossip" {
 	keys, err := consulClient.Operator().KeyringList(nil)
 	require.NoError(t, err)
 	// we use keys[0] because KeyringList returns a list of keyrings for each dc, in this case there is only 1 dc.
-	require.NotNil(t, keys[0].PrimaryKeys[gossipKey])
-
+	require.Equal(t, 1, keys[0].PrimaryKeys[gossipKey])
 }
diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml
index ce67aee1f0..604f36359e 100644
--- a/charts/consul/values.yaml
+++ b/charts/consul/values.yaml
@@ -123,7 +123,7 @@ global:
   # vault secrets, policies and roles.
   secretsBackend:
     vault:
-      # Enabling the Vault secrets backend will cause K8S secrets to no longer be used for the following Consul secrets:
+      # Enabling the Vault secrets backend will cause Kubernetes secrets to no longer be used for the following Consul secrets:
       # - gossip encryption key defined by `global.gossipEncryption.secretName/Key`
       enabled: false
 

From 2d214c0fe9569ff6471f4d211684c85d3ad5915a Mon Sep 17 00:00:00 2001
From: Kyle Schochenmaier <kschoche@gmail.com>
Date: Tue, 2 Nov 2021 11:31:46 -0500
Subject: [PATCH 08/20] update gossip stanza

---
 charts/consul/values.yaml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml
index 604f36359e..073ca740d5 100644
--- a/charts/consul/values.yaml
+++ b/charts/consul/values.yaml
@@ -135,14 +135,14 @@ global:
       # - gossip encryption key defined by `global.gossipEncryption.secretName/Key`
       consulClientRole: ""  # e.g. "consul-client"
 
-  # Configures Consul's gossip encryption key, set as a Kubernetes secret
+  # Configures Consul's gossip encryption key.
   # (see `-encrypt` (https://consul.io/docs/agent/options#_encrypt)).
   # By default, gossip encryption is not enabled. The gossip encryption key may be set automatically or manually.
   # The recommended method is to automatically generate the key.
   # To automatically generate and set a gossip encryption key, set autoGenerate to true.
   # Values for secretName and secretKey should not be set if autoGenerate is true.
   # To manually generate a gossip encryption key, set secretName and secretKey and use Consul to generate
-  # a Kubernetes secret referencing these values.
+  # a Kubernetes secret or Vault secret path and key, if `global.secretsBackend.vault.enabled=true`..
   #
   # ```
   # $ kubectl create secret generic consul-gossip-encryption-key --from-literal=key=$(consul keygen)
@@ -151,10 +151,10 @@ global:
   gossipEncryption:
     # Automatically generate a gossip encryption key and save it to a Kubernetes secret.
     autoGenerate: false
-    # secretName is the name of the Kubernetes secret that holds the gossip
-    # encryption key. The secret must be in the same namespace that Consul is installed into.
+    # secretName is the name of the Kubernetes secret or Vault secret path that holds the gossip
+    # encryption key. The Kubernetes secret must be in the same namespace that Consul is installed into.
     secretName: ""
-    # secretKey is the key within the Kubernetes secret that holds the gossip
+    # secretKey is the key within the Kubernetes secret or Vault secret that holds the gossip
     # encryption key.
     secretKey: ""
 

From 4bbe42b86e58c9ab21bd0360bc49684823f8ea1a Mon Sep 17 00:00:00 2001
From: Kyle Schochenmaier <kschoche@gmail.com>
Date: Fri, 5 Nov 2021 12:59:24 -0500
Subject: [PATCH 09/20] some cleanup

---
 acceptance/framework/consul/consul_cluster.go |  2 +-
 acceptance/framework/helpers/helpers.go       |  4 +--
 acceptance/framework/vault/vault_cluster.go   | 18 +++--------
 acceptance/tests/vault/vault_test.go          | 31 -------------------
 charts/consul/values.yaml                     | 14 ++++++++-
 5 files changed, 21 insertions(+), 48 deletions(-)

diff --git a/acceptance/framework/consul/consul_cluster.go b/acceptance/framework/consul/consul_cluster.go
index 15cd39a28c..90b5dc392b 100644
--- a/acceptance/framework/consul/consul_cluster.go
+++ b/acceptance/framework/consul/consul_cluster.go
@@ -114,7 +114,7 @@ func (h *HelmCluster) Create(t *testing.T) {
 	})
 
 	// Fail if there are any existing installations of the Helm chart.
-	helpers.CheckForPriorInstallations(t, h.kubernetesClient, h.helmOptions, "consul-helm")
+	helpers.CheckForPriorInstallations(t, h.kubernetesClient, h.helmOptions, "consul-helm", "chart=consul-helm")
 
 	helm.Install(t, h.helmOptions, config.HelmChartPath, h.releaseName)
 
diff --git a/acceptance/framework/helpers/helpers.go b/acceptance/framework/helpers/helpers.go
index 294e395cb2..16e8a30fbb 100644
--- a/acceptance/framework/helpers/helpers.go
+++ b/acceptance/framework/helpers/helpers.go
@@ -29,7 +29,7 @@ func RandomName() string {
 
 // CheckForPriorInstallations checks if there is an existing Helm release
 // for this Helm chart already installed. If there is, it fails the tests.
-func CheckForPriorInstallations(t *testing.T, client kubernetes.Interface, options *helm.Options, chartName string) {
+func CheckForPriorInstallations(t *testing.T, client kubernetes.Interface, options *helm.Options, chartName, label string) {
 	t.Helper()
 
 	var helmListOutput string
@@ -57,7 +57,7 @@ func CheckForPriorInstallations(t *testing.T, client kubernetes.Interface, optio
 	// Wait for all pods in the "default" namespace to exit. A previous
 	// release may not be listed by Helm but its pods may still be terminating.
 	retry.RunWith(&retry.Counter{Wait: 1 * time.Second, Count: 60}, t, func(r *retry.R) {
-		pods, err := client.CoreV1().Pods(options.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: fmt.Sprintf("chart=%s", chartName)})
+		pods, err := client.CoreV1().Pods(options.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: label})
 		require.NoError(r, err)
 		if len(pods.Items) > 0 {
 			var podNames []string
diff --git a/acceptance/framework/vault/vault_cluster.go b/acceptance/framework/vault/vault_cluster.go
index 9a3069a344..1801c892c0 100644
--- a/acceptance/framework/vault/vault_cluster.go
+++ b/acceptance/framework/vault/vault_cluster.go
@@ -20,9 +20,9 @@ import (
 )
 
 const (
-	vaultNS           = "default"
-	vaultChartVersion = "0.17.0"
-	vaultRootToken    = "abcd1234"
+	vaultNS        = "default"
+	vaultPodLabel  = "app.kubernetes.io/instance="
+	vaultRootToken = "abcd1234"
 )
 
 // VaultCluster
@@ -32,7 +32,6 @@ type VaultCluster struct {
 
 	vaultHelmOptions *helm.Options
 	vaultReleaseName string
-	vaultChartName   string
 	vaultClient      *vapi.Client
 
 	kubectlOptions *terratestk8s.KubectlOptions
@@ -65,9 +64,7 @@ func NewVaultCluster(
 		SetValues:      defaultVaultValues(),
 		KubectlOptions: kopts,
 		Logger:         logger,
-		Version:        vaultChartVersion,
 	}
-	// Add the vault helm repo in case it is missing, and do an update so we can utilise `vaultChartVersion` to install.
 	helm.AddRepo(t, &helm.Options{}, "hashicorp/vault", "https://helm.releases.hashicorp.com")
 	// Ignoring the error from `helm repo update` as it could fail due to stale cache or unreachable servers and we're
 	// asserting a chart version on Install which would fail in an obvious way should this not succeed.
@@ -75,10 +72,6 @@ func NewVaultCluster(
 	if err != nil {
 		logger.Logf(t, "Unable to update helm repository: %s, %s", err, errStr)
 	}
-	errStr, err = helm.RunHelmCommandAndGetOutputE(t, &helm.Options{}, "pull", "hashicorp/vault", "--version", vaultChartVersion)
-	if err != nil {
-		logger.Logf(t, "Unable to pull helm repository: %s, %s", err, errStr)
-	}
 
 	return &VaultCluster{
 		ctx:                ctx,
@@ -93,7 +86,6 @@ func NewVaultCluster(
 		debugDirectory:     cfg.DebugDirectory,
 		logger:             logger,
 		vaultReleaseName:   releaseName,
-		vaultChartName:     fmt.Sprintf("vault-%s", vaultChartVersion),
 	}
 }
 
@@ -179,12 +171,12 @@ func (v *VaultCluster) Create(t *testing.T, ctx environment.TestContext) {
 	})
 
 	// Fail if there are any existing installations of the Helm chart.
-	helpers.CheckForPriorInstallations(t, v.kubernetesClient, v.vaultHelmOptions, v.vaultChartName)
+	helpers.CheckForPriorInstallations(t, v.kubernetesClient, v.vaultHelmOptions, "", fmt.Sprintf("%s=%s", vaultPodLabel, v.vaultReleaseName))
 
 	// Install Vault.
 	helm.Install(t, v.vaultHelmOptions, "hashicorp/vault", v.vaultReleaseName)
 	// Wait for the injector and vault server pods to become Ready.
-	helpers.WaitForAllPodsToBeReady(t, v.kubernetesClient, v.vaultHelmOptions.KubectlOptions.Namespace, fmt.Sprintf("helm.sh/chart=%s", v.vaultChartName))
+	helpers.WaitForAllPodsToBeReady(t, v.kubernetesClient, v.vaultHelmOptions.KubectlOptions.Namespace, fmt.Sprintf("%s=%s", vaultPodLabel, v.vaultReleaseName))
 	// Now call bootstrap()
 	v.bootstrap(t, ctx)
 }
diff --git a/acceptance/tests/vault/vault_test.go b/acceptance/tests/vault/vault_test.go
index 8720232a5b..a77da6d463 100644
--- a/acceptance/tests/vault/vault_test.go
+++ b/acceptance/tests/vault/vault_test.go
@@ -15,37 +15,6 @@ const (
 	gossipKey = "3R7oLrdpkk2V0Y7yHLizyxXeS2RtaVuy07DkU15Lhws="
 )
 
-// Installs Vault, bootstraps it with the Kube Auth Method
-// and then validates that the KV2 secrets engine is online
-// and the Kube Auth Method is enabled.
-func TestVault_Create(t *testing.T) {
-	cfg := suite.Config()
-	ctx := suite.Environment().DefaultContext(t)
-
-	vaultReleaseName := helpers.RandomName()
-	vaultCluster := vault.NewVaultCluster(t, nil, ctx, cfg, vaultReleaseName)
-	vaultCluster.Create(t, ctx)
-	logger.Log(t, "Finished Installing and Bootstrapping")
-
-	vaultClient := vaultCluster.VaultClient(t)
-
-	// Write to the KV2 engine succeeds.
-	logger.Log(t, "Creating a KV2 Secret")
-	params := map[string]interface{}{
-		"data": map[string]interface{}{
-			"foo": "bar",
-		},
-	}
-	_, err := vaultClient.Logical().Write("consul/data/secret/test", params)
-	require.NoError(t, err)
-
-	// Validate that the Auth Method exists.
-	authList, err := vaultClient.Sys().ListAuth()
-	require.NoError(t, err)
-	logger.Log(t, "Auth List: ", authList)
-	require.NotNil(t, authList["kubernetes/"])
-}
-
 // Installs Vault, bootstraps it with secrets, policies, and Kube Auth Method
 // then creates a gossip encryption secret and uses this to bootstrap Consul.
 func TestVault_BootstrapConsulGossipEncryptionKey(t *testing.T) {
diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml
index 073ca740d5..4a0ce78b16 100644
--- a/charts/consul/values.yaml
+++ b/charts/consul/values.yaml
@@ -121,6 +121,9 @@ global:
   # In order to utilize this configuration it is required to have an accessible vault cluster which has been pre-configured
   # by a Vault operator to have the Vault Kube Auth Method, KV2 + PKI secrets engines enabled as well as pre-created
   # vault secrets, policies and roles.
+  # Note: When using Vault KV2 secrets engines the "data" field is implicitly required for Vault API calls,
+  # secretName should be in the form of  "vault-kv2-mount-path/data/secret-name".
+  # secretKey should be in the form of "key".
   secretsBackend:
     vault:
       # Enabling the Vault secrets backend will cause Kubernetes secrets to no longer be used for the following Consul secrets:
@@ -142,11 +145,20 @@ global:
   # To automatically generate and set a gossip encryption key, set autoGenerate to true.
   # Values for secretName and secretKey should not be set if autoGenerate is true.
   # To manually generate a gossip encryption key, set secretName and secretKey and use Consul to generate
-  # a Kubernetes secret or Vault secret path and key, if `global.secretsBackend.vault.enabled=true`..
+  # a key, saving this as a Kubernetes secret or Vault secret path and key.
+  # If `global.secretsBackend.vault.enabled=true` be sure to add the "data" field to the secretName as required by
+  # the Vault KV-2 secrets engine [see example].
   #
   # ```
   # $ kubectl create secret generic consul-gossip-encryption-key --from-literal=key=$(consul keygen)
   # ```
+  #
+  # Vault CLI Example:
+  # ```
+  # $ vault kv put consul/secrets/gossip key=$(consul keygen)
+  # ```
+  # `gossipEncryption.secretName="consul/data/secrets/gossip"`
+  # `gossipEncryption.secretKey="key"`
 
   gossipEncryption:
     # Automatically generate a gossip encryption key and save it to a Kubernetes secret.

From 2c8960e304d5d82b67bb4a4e286c307eb771d952 Mon Sep 17 00:00:00 2001
From: Kyle Schochenmaier <kschoche@gmail.com>
Date: Fri, 5 Nov 2021 15:32:21 -0500
Subject: [PATCH 10/20] update templating and fix some tests

---
 acceptance/tests/vault/vault_test.go          | 30 ++++++++++++++++---
 charts/consul/templates/_helpers.tpl          |  6 ++++
 charts/consul/templates/client-daemonset.yaml |  4 +--
 .../consul/templates/server-statefulset.yaml  |  4 +--
 charts/consul/values.yaml                     | 13 ++++----
 5 files changed, 40 insertions(+), 17 deletions(-)

diff --git a/acceptance/tests/vault/vault_test.go b/acceptance/tests/vault/vault_test.go
index a77da6d463..3d0f63eea8 100644
--- a/acceptance/tests/vault/vault_test.go
+++ b/acceptance/tests/vault/vault_test.go
@@ -1,8 +1,11 @@
 package vault
 
 import (
+	"crypto/rand"
+	"encoding/base64"
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
@@ -11,9 +14,22 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const (
-	gossipKey = "3R7oLrdpkk2V0Y7yHLizyxXeS2RtaVuy07DkU15Lhws="
-)
+// generateGossipSecret generates a random 32 byte secret returned as a base64 encoded string.
+func generateGossipSecret() (string, error) {
+	// This code was copied from Consul's Keygen command:
+	// https://github.com/hashicorp/consul/blob/d652cc86e3d0322102c2b5e9026c6a60f36c17a5/command/keygen/keygen.go
+
+	key := make([]byte, 32)
+	n, err := rand.Reader.Read(key)
+	if err != nil {
+		return "", fmt.Errorf("error reading random data: %s", err)
+	}
+	if n != 32 {
+		return "", fmt.Errorf("couldn't read enough entropy")
+	}
+
+	return base64.StdEncoding.EncodeToString(key), nil
+}
 
 // Installs Vault, bootstraps it with secrets, policies, and Kube Auth Method
 // then creates a gossip encryption secret and uses this to bootstrap Consul.
@@ -30,6 +46,9 @@ func TestVault_BootstrapConsulGossipEncryptionKey(t *testing.T) {
 	vaultCluster.Create(t, ctx)
 	// Vault is now installed in the cluster.
 
+	// FIXME: There is a *slight* delay between when the vault pods are Ready and when the vaultClient connect attempt
+	// will work, on occassion the client connection will time out before it's ready. Fix later.
+	time.Sleep(1 * time.Second)
 	// Now fetch the Vault client so we can create the policies and secrets.
 	vaultClient := vaultCluster.VaultClient(t)
 
@@ -62,7 +81,10 @@ path "consul/data/secret/gossip" {
 	_, err = vaultClient.Logical().Write("auth/kubernetes/role/consul-server", params)
 	require.NoError(t, err)
 
-	// Create the gossip key.
+	gossipKey, err := generateGossipSecret()
+	require.NoError(t, err)
+
+	// Create the gossip secret.
 	logger.Log(t, "Creating the gossip secret")
 	params = map[string]interface{}{
 		"data": map[string]interface{}{
diff --git a/charts/consul/templates/_helpers.tpl b/charts/consul/templates/_helpers.tpl
index 173eb68cc9..d040777099 100644
--- a/charts/consul/templates/_helpers.tpl
+++ b/charts/consul/templates/_helpers.tpl
@@ -15,6 +15,12 @@ as well as the global.name setting.
 {{- end -}}
 {{- end -}}
 
+{{- define "consul.vaultGossipTemplate" -}}
+ |
+            {{ "{{" }}- with secret "{{ .secretName }}" -{{ "}}" }}
+            {{ "{{" }}- {{ .secretKey }} -{{ "}}" }}
+            {{ "{{" }}- end -{{ "}}" }}
+{{- end -}}
 {{/*
 Sets up the extra-from-values config file passed to consul and then uses sed to do any necessary
 substitution for HOST_IP/POD_IP/HOSTNAME. Useful for dogstats telemetry. The output file
diff --git a/charts/consul/templates/client-daemonset.yaml b/charts/consul/templates/client-daemonset.yaml
index 50bffbe59c..0d3a8f9d40 100644
--- a/charts/consul/templates/client-daemonset.yaml
+++ b/charts/consul/templates/client-daemonset.yaml
@@ -39,12 +39,10 @@ spec:
       annotations:
         {{- if .Values.global.secretsBackend.vault.enabled }}
         "vault.hashicorp.com/agent-inject": "true"
-        "vault.hashicorp.com/agent-init-first": "true"
         "vault.hashicorp.com/role": "{{ .Values.global.secretsBackend.vault.consulClientRole }}"
         {{- with .Values.global.gossipEncryption }}
         "vault.hashicorp.com/agent-inject-secret-gossip.txt": "{{ .secretName }}"
-        "vault.hashicorp.com/agent-inject-template-gossip.txt": '{{ "{{" }}- with secret "{{ .secretName }}" -{{ "}}" }} {{ "{{" }} {{ .secretKey }} {{ "}}" }} {{ "{{" }}- end -{{ "}}" }}'
-        "vault.hashicorp.com/template-static-secret-render-interval": "10s"
+        "vault.hashicorp.com/agent-inject-template-gossip.txt": {{ template "consul.vaultGossipTemplate" . }}
         {{- end }}
         {{- end }}
         "consul.hashicorp.com/connect-inject": "false"
diff --git a/charts/consul/templates/server-statefulset.yaml b/charts/consul/templates/server-statefulset.yaml
index 901e4a7fae..5713490b71 100644
--- a/charts/consul/templates/server-statefulset.yaml
+++ b/charts/consul/templates/server-statefulset.yaml
@@ -50,12 +50,10 @@ spec:
       annotations:
         {{- if .Values.global.secretsBackend.vault.enabled }}
         "vault.hashicorp.com/agent-inject": "true"
-        "vault.hashicorp.com/agent-init-first": "true"
         "vault.hashicorp.com/role": "{{ .Values.global.secretsBackend.vault.consulServerRole }}"
         {{- with .Values.global.gossipEncryption }}
         "vault.hashicorp.com/agent-inject-secret-gossip.txt": "{{ .secretName }}"
-        "vault.hashicorp.com/agent-inject-template-gossip.txt": '{{ "{{" }}- with secret "{{ .secretName }}" -{{ "}}" }} {{ "{{" }} {{ .secretKey }} {{ "}}" }} {{ "{{" }}- end -{{ "}}" }}'
-        "vault.hashicorp.com/template-static-secret-render-interval": "10s"
+        "vault.hashicorp.com/agent-inject-template-gossip.txt": {{ template "consul.vaultGossipTemplate" . }}
         {{- end }}
         {{- end }}
         "consul.hashicorp.com/connect-inject": "false"
diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml
index 4a0ce78b16..54501360cc 100644
--- a/charts/consul/values.yaml
+++ b/charts/consul/values.yaml
@@ -126,17 +126,16 @@ global:
   # secretKey should be in the form of "key".
   secretsBackend:
     vault:
-      # Enabling the Vault secrets backend will cause Kubernetes secrets to no longer be used for the following Consul secrets:
-      # - gossip encryption key defined by `global.gossipEncryption.secretName/Key`
+      # Enabling the Vault secrets backend will replace Kubernetes secrets with referenced Vault secrets.
       enabled: false
 
       # The consulServerRole must be created and have read capabilties via a Policy for the following secrets:
-      # - gossip encryption key defined by `global.gossipEncryption.secretName/Key`.
-      consulServerRole: ""  # e.g. "consul-server"
+      # - gossip encryption key defined by `global.gossipEncryption.secretName`.
+      consulServerRole: ""
 
       # The consulClientRole must be created and have read capabilties via a Policy for the following secrets:
-      # - gossip encryption key defined by `global.gossipEncryption.secretName/Key`
-      consulClientRole: ""  # e.g. "consul-client"
+      # - gossip encryption key defined by `global.gossipEncryption.secretName`
+      consulClientRole: ""
 
   # Configures Consul's gossip encryption key.
   # (see `-encrypt` (https://consul.io/docs/agent/options#_encrypt)).
@@ -164,7 +163,7 @@ global:
     # Automatically generate a gossip encryption key and save it to a Kubernetes secret.
     autoGenerate: false
     # secretName is the name of the Kubernetes secret or Vault secret path that holds the gossip
-    # encryption key. The Kubernetes secret must be in the same namespace that Consul is installed into.
+    # encryption key. A Kubernetes secret must be in the same namespace that Consul is installed into.
     secretName: ""
     # secretKey is the key within the Kubernetes secret or Vault secret that holds the gossip
     # encryption key.

From 843be9307b57f615a8dde19723b70598bdfb0c8e Mon Sep 17 00:00:00 2001
From: Kyle Schochenmaier <kschoche@gmail.com>
Date: Mon, 8 Nov 2021 10:22:56 -0600
Subject: [PATCH 11/20] Apply suggestions from code review

Co-authored-by: Iryna Shustava <ishustava@users.noreply.github.com>
---
 charts/consul/test/unit/client-daemonset.bats |  8 ++++----
 charts/consul/values.yaml                     | 11 ++++++-----
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/charts/consul/test/unit/client-daemonset.bats b/charts/consul/test/unit/client-daemonset.bats
index 41c56c60f8..bd63c61fa7 100755
--- a/charts/consul/test/unit/client-daemonset.bats
+++ b/charts/consul/test/unit/client-daemonset.bats
@@ -1520,7 +1520,7 @@ rollingUpdate:
 #--------------------------------------------------------------------
 # vault integration
 
-@test "client/DaemonSet: vault annotations not attached by default" {
+@test "client/DaemonSet: vault annotations not set by default" {
   cd `chart_dir`
   local actual=$(helm template \
     -s templates/client-daemonset.yaml  \
@@ -1548,13 +1548,13 @@ rollingUpdate:
   [ "${actual}" = "" ]
 }
 
-@test "client/DaemonSet: vault gossip annotations are correct when enabled" {
+@test "client/DaemonSet: vault gossip annotations are set when gossip encryption enabled" {
   cd `chart_dir`
   local object=$(helm template \
     -s templates/client-daemonset.yaml  \
     --set 'global.secretsBackend.vault.enabled=true' \
     --set 'global.gossipEncryption.secretName=path/to/secret/key' \
-    --set 'global.gossipEncryption.secretKey=.Data.gossip.gossip' \
+    --set 'global.gossipEncryption.secretKey=.Data.data.gossip' \
     . | tee /dev/stderr |
       yq -r '.spec.template.metadata' | tee /dev/stderr)
 
@@ -1566,7 +1566,7 @@ rollingUpdate:
   [ "${actual}" = '{{- with secret "path/to/secret/key" -}} {{ .Data.gossip.gossip }} {{- end -}}' ]
 }
 
-@test "client/DaemonSet: vault no GOSSIP_KEY env variable and command defines GOSSIP_KEY" {
+@test "client/DaemonSet: GOSSIP_KEY env variable is not set and command defines GOSSIP_KEY when vault is enabled" {
   cd `chart_dir`
   local object=$(helm template \
     -s templates/client-daemonset.yaml  \
diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml
index 54501360cc..d21ab38cc5 100644
--- a/charts/consul/values.yaml
+++ b/charts/consul/values.yaml
@@ -117,10 +117,11 @@ global:
   # created by this chart. See https://kubernetes.io/docs/concepts/policy/pod-security-policy/.
   enablePodSecurityPolicies: false
 
-  # secretsBackend is used to enable and disable the use of Vault as the secrets backend for the Consul on Kubernetes installation.
-  # In order to utilize this configuration it is required to have an accessible vault cluster which has been pre-configured
-  # by a Vault operator to have the Vault Kube Auth Method, KV2 + PKI secrets engines enabled as well as pre-created
-  # vault secrets, policies and roles.
+  # secretsBackend is used to configure Vault as the secrets backend for the Consul on Kubernetes installation.
+  # The Vault cluster needs to have the Kubernetes Auth Method, 
+  # KV2 and PKI secrets engines enabled and have necessary secrets, 
+  # policies and roles created prior to installing Consul.
+  # The Vault cluster should not have Consul as its storage backend.
   # Note: When using Vault KV2 secrets engines the "data" field is implicitly required for Vault API calls,
   # secretName should be in the form of  "vault-kv2-mount-path/data/secret-name".
   # secretKey should be in the form of "key".
@@ -165,7 +166,7 @@ global:
     # secretName is the name of the Kubernetes secret or Vault secret path that holds the gossip
     # encryption key. A Kubernetes secret must be in the same namespace that Consul is installed into.
     secretName: ""
-    # secretKey is the key within the Kubernetes secret or Vault secret that holds the gossip
+    # secretKey is the key within the Kubernetes secret or Vault secret key that holds the gossip
     # encryption key.
     secretKey: ""
 

From 39570a3aa49fe1a43466f7f2e208ae6791ca5246 Mon Sep 17 00:00:00 2001
From: Kyle Schochenmaier <kschoche@gmail.com>
Date: Mon, 8 Nov 2021 14:42:24 -0600
Subject: [PATCH 12/20] changes

---
 charts/consul/templates/client-daemonset.yaml |  3 ++
 .../consul/templates/server-statefulset.yaml  |  3 ++
 charts/consul/test/unit/client-daemonset.bats | 45 ++++++++++++++-----
 .../consul/test/unit/server-statefulset.bats  |  7 ++-
 4 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/charts/consul/templates/client-daemonset.yaml b/charts/consul/templates/client-daemonset.yaml
index 0d3a8f9d40..af213f9e4e 100644
--- a/charts/consul/templates/client-daemonset.yaml
+++ b/charts/consul/templates/client-daemonset.yaml
@@ -2,6 +2,7 @@
 {{- if (and (and .Values.global.tls.enabled .Values.global.tls.httpsOnly) (and .Values.global.metrics.enabled .Values.global.metrics.enableAgentMetrics))}}{{ fail "global.metrics.enableAgentMetrics cannot be enabled if TLS (HTTPS only) is enabled" }}{{ end -}}
 {{- $serverEnabled := (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) -}}
 {{- if (and .Values.global.adminPartitions.enabled $serverEnabled (ne .Values.global.adminPartitions.name "default"))}}{{ fail "global.adminPartitions.name has to be \"default\" in the server cluster" }}{{ end -}}
+{{- if (and (not .Values.global.secretsBackend.vault.consulClientRole) .Values.global.secretsBackend.vault.enabled) }}{{ fail "global.secretsBackend.vault.consulClientRole must be provided if global.secretsBackend.vault.enabled=true." }}{{ end -}}
 # DaemonSet to run the Consul clients on every node.
 apiVersion: apps/v1
 kind: DaemonSet
@@ -40,11 +41,13 @@ spec:
         {{- if .Values.global.secretsBackend.vault.enabled }}
         "vault.hashicorp.com/agent-inject": "true"
         "vault.hashicorp.com/role": "{{ .Values.global.secretsBackend.vault.consulClientRole }}"
+        {{- if .Values.global.gossipEncryption.secretName }}
         {{- with .Values.global.gossipEncryption }}
         "vault.hashicorp.com/agent-inject-secret-gossip.txt": "{{ .secretName }}"
         "vault.hashicorp.com/agent-inject-template-gossip.txt": {{ template "consul.vaultGossipTemplate" . }}
         {{- end }}
         {{- end }}
+        {{- end }}
         "consul.hashicorp.com/connect-inject": "false"
         "consul.hashicorp.com/config-checksum": {{ include (print $.Template.BasePath "/client-config-configmap.yaml") . | sha256sum }}
         {{- if .Values.client.annotations }}
diff --git a/charts/consul/templates/server-statefulset.yaml b/charts/consul/templates/server-statefulset.yaml
index 5713490b71..fae06441a0 100644
--- a/charts/consul/templates/server-statefulset.yaml
+++ b/charts/consul/templates/server-statefulset.yaml
@@ -7,6 +7,7 @@
 {{- if (and (and .Values.global.tls.enabled .Values.global.tls.httpsOnly) (and .Values.global.metrics.enabled .Values.global.metrics.enableAgentMetrics))}}{{ fail "global.metrics.enableAgentMetrics cannot be enabled if TLS (HTTPS only) is enabled" }}{{ end -}}
 {{- if (and .Values.global.gossipEncryption.secretName (not .Values.global.gossipEncryption.secretKey)) }}{{fail "gossipEncryption.secretKey and secretName must both be specified." }}{{ end -}}
 {{- if (and (not .Values.global.gossipEncryption.secretName) .Values.global.gossipEncryption.secretKey) }}{{fail "gossipEncryption.secretKey and secretName must both be specified." }}{{ end -}}
+{{- if (and (not .Values.global.secretsBackend.vault.consulServerRole) .Values.global.secretsBackend.vault.enabled) }}{{ fail "global.secretsBackend.vault.consulServerRole must be provided if global.secretsBackend.vault.enabled=true." }}{{ end -}}
 # StatefulSet to run the actual Consul server cluster.
 apiVersion: apps/v1
 kind: StatefulSet
@@ -51,11 +52,13 @@ spec:
         {{- if .Values.global.secretsBackend.vault.enabled }}
         "vault.hashicorp.com/agent-inject": "true"
         "vault.hashicorp.com/role": "{{ .Values.global.secretsBackend.vault.consulServerRole }}"
+        {{- if .Values.global.gossipEncryption.secretName }}
         {{- with .Values.global.gossipEncryption }}
         "vault.hashicorp.com/agent-inject-secret-gossip.txt": "{{ .secretName }}"
         "vault.hashicorp.com/agent-inject-template-gossip.txt": {{ template "consul.vaultGossipTemplate" . }}
         {{- end }}
         {{- end }}
+        {{- end }}
         "consul.hashicorp.com/connect-inject": "false"
         "consul.hashicorp.com/config-checksum": {{ include (print $.Template.BasePath "/server-config-configmap.yaml") . | sha256sum }}
         {{- if .Values.server.annotations }}
diff --git a/charts/consul/test/unit/client-daemonset.bats b/charts/consul/test/unit/client-daemonset.bats
index bd63c61fa7..b9ab0bf31b 100755
--- a/charts/consul/test/unit/client-daemonset.bats
+++ b/charts/consul/test/unit/client-daemonset.bats
@@ -1520,12 +1520,29 @@ rollingUpdate:
 #--------------------------------------------------------------------
 # vault integration
 
+@test "client/DaemonSet: fail when vault is enabled but the consulClientRole is not provided" {
+  cd `chart_dir`
+  run helm template \
+      -s templates/client-daemonset.yaml  \
+      --set 'global.secretsBackend.vault.enabled=true'  \
+      --set 'global.secretsBackend.vault.consulServerRole=test' \
+      .
+  [ "$status" -eq 1 ]
+  [[ "$output" =~ "secretsBackend.vault.consulClientRole must be provided if secretsBackend.vault.enabled=true." ]]
+}
+
 @test "client/DaemonSet: vault annotations not set by default" {
   cd `chart_dir`
-  local actual=$(helm template \
+  local object=$(helm template \
     -s templates/client-daemonset.yaml  \
     . | tee /dev/stderr |
-     yq '.spec.template.metadata.annotations["vault.hashicorp.com/agent-inject"] | length > 0' | tee /dev/stderr)
+      yq -r '.spec.template.metadata' | tee /dev/stderr)
+
+  local actual=$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/agent-inject"] | length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+  local actual=$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/role"] | length > 0 ' | tee /dev/stderr)
   [ "${actual}" = "false" ]
 }
 
@@ -1534,18 +1551,17 @@ rollingUpdate:
   local object=$(helm template \
     -s templates/client-daemonset.yaml  \
     --set 'global.secretsBackend.vault.enabled=true' \
+    --set 'global.secretsBackend.vault.consulClientRole=foo' \
+    --set 'global.secretsBackend.vault.consulServerRole=test' \
     . | tee /dev/stderr |
       yq -r '.spec.template.metadata' | tee /dev/stderr)
 
   local actual=$(echo $object |
-      yq -r '.annotations["vault.hashicorp.com/agent-inject"] | length > 0' | tee /dev/stderr)
-  [ "${actual}" = "true" ]
-  local actual=$(echo $object |
-      yq -r '.annotations["vault.hashicorp.com/agent-init-first"] | length > 0' | tee /dev/stderr)
+      yq -r '.annotations["vault.hashicorp.com/agent-inject"]' | tee /dev/stderr)
   [ "${actual}" = "true" ]
   local actual=$(echo $object |
       yq -r '.annotations["vault.hashicorp.com/role"]' | tee /dev/stderr)
-  [ "${actual}" = "" ]
+  [ "${actual}" = "foo" ]
 }
 
 @test "client/DaemonSet: vault gossip annotations are set when gossip encryption enabled" {
@@ -1553,17 +1569,24 @@ rollingUpdate:
   local object=$(helm template \
     -s templates/client-daemonset.yaml  \
     --set 'global.secretsBackend.vault.enabled=true' \
-    --set 'global.gossipEncryption.secretName=path/to/secret/key' \
+    --set 'global.secretsBackend.vault.consulClientRole=test' \
+    --set 'global.secretsBackend.vault.consulServerRole=foo' \
+    --set 'global.gossipEncryption.secretName=path/to/secret' \
     --set 'global.gossipEncryption.secretKey=.Data.data.gossip' \
     . | tee /dev/stderr |
       yq -r '.spec.template.metadata' | tee /dev/stderr)
 
   local actual=$(echo $object |
       yq -r '.annotations["vault.hashicorp.com/agent-inject-secret-gossip.txt"]' | tee /dev/stderr)
-  [ "${actual}" = "path/to/secret/key" ]
+  [ "${actual}" = "path/to/secret" ]
   local actual=$(echo $object |
       yq -r '.annotations["vault.hashicorp.com/agent-inject-template-gossip.txt"]' | tee /dev/stderr)
-  [ "${actual}" = '{{- with secret "path/to/secret/key" -}} {{ .Data.gossip.gossip }} {{- end -}}' ]
+  echo "Actual:"
+  echo $actual
+  local expected='{{- with secret "path/to/secret" -}} {{- .Data.data.gossip -}} {{- end -}}'
+  echo "Expected:"
+  echo $expected
+  [ '${actual}' = '${expected}' ]
 }
 
 @test "client/DaemonSet: GOSSIP_KEY env variable is not set and command defines GOSSIP_KEY when vault is enabled" {
@@ -1571,6 +1594,8 @@ rollingUpdate:
   local object=$(helm template \
     -s templates/client-daemonset.yaml  \
     --set 'global.secretsBackend.vault.enabled=true' \
+    --set 'global.secretsBackend.vault.consulClientRole=test' \
+    --set 'global.secretsBackend.vault.consulServerRole=foo' \
     --set 'global.gossipEncryption.secretName=a/b/c/d' \
     --set 'global.gossipEncryption.secretKey=.Data.data.gossip' \
     . | tee /dev/stderr |
diff --git a/charts/consul/test/unit/server-statefulset.bats b/charts/consul/test/unit/server-statefulset.bats
index 0aa6ae3335..4f8782b2a6 100755
--- a/charts/consul/test/unit/server-statefulset.bats
+++ b/charts/consul/test/unit/server-statefulset.bats
@@ -1428,18 +1428,16 @@ load _helpers
   local object=$(helm template \
     -s templates/server-statefulset.yaml  \
     --set 'global.secretsBackend.vault.enabled=true' \
+    --set 'global.secretsBackend.vault.consulServerRole=test' \
     . | tee /dev/stderr |
       yq -r '.spec.template.metadata' | tee /dev/stderr)
 
   local actual=$(echo $object |
       yq -r '.annotations["vault.hashicorp.com/agent-inject"] | length > 0' | tee /dev/stderr)
   [ "${actual}" = "true" ]
-  local actual=$(echo $object |
-      yq -r '.annotations["vault.hashicorp.com/agent-init-first"] | length > 0' | tee /dev/stderr)
-  [ "${actual}" = "true" ]
   local actual=$(echo $object |
       yq -r '.annotations["vault.hashicorp.com/role"]' | tee /dev/stderr)
-  [ "${actual}" = "" ]
+  [ "${actual}" = "test" ]
 }
 
 @test "server/StatefulSet: vault gossip annotations are correct when enabled" {
@@ -1447,6 +1445,7 @@ load _helpers
   local object=$(helm template \
     -s templates/server-statefulset.yaml  \
     --set 'global.secretsBackend.vault.enabled=true' \
+    --set 'global.secretsBackend.vault.consulServerRole=test' \
     --set 'global.gossipEncryption.secretName=path/to/secret/key' \
     --set 'global.gossipEncryption.secretKey=.Data.gossip.gossip' \
     . | tee /dev/stderr |

From 2e24a038184ae8ea02653c5fce67907a238328ba Mon Sep 17 00:00:00 2001
From: Kyle Schochenmaier <kschoche@gmail.com>
Date: Tue, 9 Nov 2021 13:09:32 -0600
Subject: [PATCH 13/20] update bats tests

---
 .../consul/templates/server-statefulset.yaml  |  2 +-
 charts/consul/test/unit/client-daemonset.bats | 14 +++----
 .../consul/test/unit/server-statefulset.bats  | 37 +++++++++++++++----
 3 files changed, 36 insertions(+), 17 deletions(-)

diff --git a/charts/consul/templates/server-statefulset.yaml b/charts/consul/templates/server-statefulset.yaml
index fae06441a0..77fa3c68de 100644
--- a/charts/consul/templates/server-statefulset.yaml
+++ b/charts/consul/templates/server-statefulset.yaml
@@ -7,7 +7,7 @@
 {{- if (and (and .Values.global.tls.enabled .Values.global.tls.httpsOnly) (and .Values.global.metrics.enabled .Values.global.metrics.enableAgentMetrics))}}{{ fail "global.metrics.enableAgentMetrics cannot be enabled if TLS (HTTPS only) is enabled" }}{{ end -}}
 {{- if (and .Values.global.gossipEncryption.secretName (not .Values.global.gossipEncryption.secretKey)) }}{{fail "gossipEncryption.secretKey and secretName must both be specified." }}{{ end -}}
 {{- if (and (not .Values.global.gossipEncryption.secretName) .Values.global.gossipEncryption.secretKey) }}{{fail "gossipEncryption.secretKey and secretName must both be specified." }}{{ end -}}
-{{- if (and (not .Values.global.secretsBackend.vault.consulServerRole) .Values.global.secretsBackend.vault.enabled) }}{{ fail "global.secretsBackend.vault.consulServerRole must be provided if global.secretsBackend.vault.enabled=true." }}{{ end -}}
+{{- if (and .Values.global.secretsBackend.vault.enabled (not .Values.global.secretsBackend.vault.consulServerRole)) }}{{ fail "global.secretsBackend.vault.consulServerRole must be provided if global.secretsBackend.vault.enabled=true." }}{{ end -}}
 # StatefulSet to run the actual Consul server cluster.
 apiVersion: apps/v1
 kind: StatefulSet
diff --git a/charts/consul/test/unit/client-daemonset.bats b/charts/consul/test/unit/client-daemonset.bats
index b9ab0bf31b..4119158a61 100755
--- a/charts/consul/test/unit/client-daemonset.bats
+++ b/charts/consul/test/unit/client-daemonset.bats
@@ -1528,7 +1528,7 @@ rollingUpdate:
       --set 'global.secretsBackend.vault.consulServerRole=test' \
       .
   [ "$status" -eq 1 ]
-  [[ "$output" =~ "secretsBackend.vault.consulClientRole must be provided if secretsBackend.vault.enabled=true." ]]
+  [[ "$output" =~ "global.secretsBackend.vault.consulClientRole must be provided if global.secretsBackend.vault.enabled=true" ]]
 }
 
 @test "client/DaemonSet: vault annotations not set by default" {
@@ -1579,14 +1579,10 @@ rollingUpdate:
   local actual=$(echo $object |
       yq -r '.annotations["vault.hashicorp.com/agent-inject-secret-gossip.txt"]' | tee /dev/stderr)
   [ "${actual}" = "path/to/secret" ]
-  local actual=$(echo $object |
-      yq -r '.annotations["vault.hashicorp.com/agent-inject-template-gossip.txt"]' | tee /dev/stderr)
-  echo "Actual:"
-  echo $actual
-  local expected='{{- with secret "path/to/secret" -}} {{- .Data.data.gossip -}} {{- end -}}'
-  echo "Expected:"
-  echo $expected
-  [ '${actual}' = '${expected}' ]
+  local actual="$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/agent-inject-template-gossip.txt"]' | tee /dev/stderr)"
+  local expected=$'{{- with secret \"path/to/secret\" -}}\n{{- .Data.data.gossip -}}\n{{- end -}}'
+  [ "${actual}" = "${expected}" ]
 }
 
 @test "client/DaemonSet: GOSSIP_KEY env variable is not set and command defines GOSSIP_KEY when vault is enabled" {
diff --git a/charts/consul/test/unit/server-statefulset.bats b/charts/consul/test/unit/server-statefulset.bats
index 4f8782b2a6..8a9eb20674 100755
--- a/charts/consul/test/unit/server-statefulset.bats
+++ b/charts/consul/test/unit/server-statefulset.bats
@@ -1414,12 +1414,28 @@ load _helpers
 #--------------------------------------------------------------------
 # vault integration
 
-@test "server/StatefulSet: vault annotations not attached by default" {
+@test "server/StatefulSet: fail when vault is enabled but the consulServerRole is not provided" {
   cd `chart_dir`
-  local actual=$(helm template \
+  run helm template \
+      -s templates/server-statefulset.yaml  \
+      --set 'global.secretsBackend.vault.enabled=true'  \
+      --set 'global.secretsBackend.vault.consulClientRole=test' .
+  [ "$status" -eq 1 ]
+  [[ "$output" =~ "global.secretsBackend.vault.consulServerRole must be provided if global.secretsBackend.vault.enabled=true" ]]
+}
+
+@test "server/StatefulSet: vault annotations not set by default" {
+  cd `chart_dir`
+  local object=$(helm template \
     -s templates/server-statefulset.yaml  \
     . | tee /dev/stderr |
-     yq '.spec.template.metadata.annotations["vault.hashicorp.com/agent-inject"] | length > 0' | tee /dev/stderr)
+      yq -r '.spec.template.metadata' | tee /dev/stderr)
+
+  local actual=$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/agent-inject"] | length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+  local actual=$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/role"] | length > 0 ' | tee /dev/stderr)
   [ "${actual}" = "false" ]
 }
 
@@ -1428,6 +1444,7 @@ load _helpers
   local object=$(helm template \
     -s templates/server-statefulset.yaml  \
     --set 'global.secretsBackend.vault.enabled=true' \
+    --set 'global.secretsBackend.vault.consulClientRole=foo' \
     --set 'global.secretsBackend.vault.consulServerRole=test' \
     . | tee /dev/stderr |
       yq -r '.spec.template.metadata' | tee /dev/stderr)
@@ -1445,18 +1462,22 @@ load _helpers
   local object=$(helm template \
     -s templates/server-statefulset.yaml  \
     --set 'global.secretsBackend.vault.enabled=true' \
+    --set 'global.secretsBackend.vault.consulClientRole=foo' \
     --set 'global.secretsBackend.vault.consulServerRole=test' \
-    --set 'global.gossipEncryption.secretName=path/to/secret/key' \
-    --set 'global.gossipEncryption.secretKey=.Data.gossip.gossip' \
+    --set 'global.gossipEncryption.secretName=path/to/secret' \
+    --set 'global.gossipEncryption.secretKey=.Data.data.gossip' \
     . | tee /dev/stderr |
       yq -r '.spec.template.metadata' | tee /dev/stderr)
 
   local actual=$(echo $object |
       yq -r '.annotations["vault.hashicorp.com/agent-inject-secret-gossip.txt"]' | tee /dev/stderr)
-  [ "${actual}" = "path/to/secret/key" ]
+  [ "${actual}" = "path/to/secret" ]
   local actual=$(echo $object |
       yq -r '.annotations["vault.hashicorp.com/agent-inject-template-gossip.txt"]' | tee /dev/stderr)
-  [ "${actual}" = '{{- with secret "path/to/secret/key" -}} {{ .Data.gossip.gossip }} {{- end -}}' ]
+  local actual="$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/agent-inject-template-gossip.txt"]' | tee /dev/stderr)"
+  local expected=$'{{- with secret \"path/to/secret\" -}}\n{{- .Data.data.gossip -}}\n{{- end -}}'
+  [ "${actual}" = "${expected}" ]
 }
 
 @test "server/StatefulSet: vault no GOSSIP_KEY env variable and command defines GOSSIP_KEY" {
@@ -1464,6 +1485,8 @@ load _helpers
   local object=$(helm template \
     -s templates/server-statefulset.yaml  \
     --set 'global.secretsBackend.vault.enabled=true' \
+    --set 'global.secretsBackend.vault.consulClientRole=foo' \
+    --set 'global.secretsBackend.vault.consulServerRole=test' \
     --set 'global.gossipEncryption.secretName=a/b/c/d' \
     --set 'global.gossipEncryption.secretKey=.Data.data.gossip' \
     . | tee /dev/stderr |

From 9081bc3cf2fd6c4a7ec6ecafb23837a9c265d288 Mon Sep 17 00:00:00 2001
From: Kyle Schochenmaier <kschoche@gmail.com>
Date: Tue, 9 Nov 2021 13:25:25 -0600
Subject: [PATCH 14/20] remove .data.data

---
 acceptance/tests/vault/vault_test.go            | 2 +-
 charts/consul/templates/_helpers.tpl            | 2 +-
 charts/consul/test/unit/client-daemonset.bats   | 4 ++--
 charts/consul/test/unit/server-statefulset.bats | 4 ++--
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/acceptance/tests/vault/vault_test.go b/acceptance/tests/vault/vault_test.go
index 3d0f63eea8..69e14ce32a 100644
--- a/acceptance/tests/vault/vault_test.go
+++ b/acceptance/tests/vault/vault_test.go
@@ -107,7 +107,7 @@ path "consul/data/secret/gossip" {
 		"global.acls.manageSystemACLs":       "true",
 		"global.tls.enabled":                 "true",
 		"global.gossipEncryption.secretName": "consul/data/secret/gossip",
-		"global.gossipEncryption.secretKey":  ".Data.data.gossip",
+		"global.gossipEncryption.secretKey":  "gossip",
 	}
 	logger.Log(t, "Installing Consul")
 	consulCluster := consul.NewHelmCluster(t, consulHelmValues, ctx, cfg, consulReleaseName)
diff --git a/charts/consul/templates/_helpers.tpl b/charts/consul/templates/_helpers.tpl
index d040777099..07ced2d778 100644
--- a/charts/consul/templates/_helpers.tpl
+++ b/charts/consul/templates/_helpers.tpl
@@ -18,7 +18,7 @@ as well as the global.name setting.
 {{- define "consul.vaultGossipTemplate" -}}
  |
             {{ "{{" }}- with secret "{{ .secretName }}" -{{ "}}" }}
-            {{ "{{" }}- {{ .secretKey }} -{{ "}}" }}
+            {{ "{{" }}- {{ printf ".Data.data.%s" .secretKey }} -{{ "}}" }}
             {{ "{{" }}- end -{{ "}}" }}
 {{- end -}}
 {{/*
diff --git a/charts/consul/test/unit/client-daemonset.bats b/charts/consul/test/unit/client-daemonset.bats
index 4119158a61..f83c802e32 100755
--- a/charts/consul/test/unit/client-daemonset.bats
+++ b/charts/consul/test/unit/client-daemonset.bats
@@ -1572,7 +1572,7 @@ rollingUpdate:
     --set 'global.secretsBackend.vault.consulClientRole=test' \
     --set 'global.secretsBackend.vault.consulServerRole=foo' \
     --set 'global.gossipEncryption.secretName=path/to/secret' \
-    --set 'global.gossipEncryption.secretKey=.Data.data.gossip' \
+    --set 'global.gossipEncryption.secretKey=gossip' \
     . | tee /dev/stderr |
       yq -r '.spec.template.metadata' | tee /dev/stderr)
 
@@ -1593,7 +1593,7 @@ rollingUpdate:
     --set 'global.secretsBackend.vault.consulClientRole=test' \
     --set 'global.secretsBackend.vault.consulServerRole=foo' \
     --set 'global.gossipEncryption.secretName=a/b/c/d' \
-    --set 'global.gossipEncryption.secretKey=.Data.data.gossip' \
+    --set 'global.gossipEncryption.secretKey=gossip' \
     . | tee /dev/stderr |
       yq -r '.spec.template.spec' | tee /dev/stderr)
 
diff --git a/charts/consul/test/unit/server-statefulset.bats b/charts/consul/test/unit/server-statefulset.bats
index 8a9eb20674..909fb647b2 100755
--- a/charts/consul/test/unit/server-statefulset.bats
+++ b/charts/consul/test/unit/server-statefulset.bats
@@ -1465,7 +1465,7 @@ load _helpers
     --set 'global.secretsBackend.vault.consulClientRole=foo' \
     --set 'global.secretsBackend.vault.consulServerRole=test' \
     --set 'global.gossipEncryption.secretName=path/to/secret' \
-    --set 'global.gossipEncryption.secretKey=.Data.data.gossip' \
+    --set 'global.gossipEncryption.secretKey=gossip' \
     . | tee /dev/stderr |
       yq -r '.spec.template.metadata' | tee /dev/stderr)
 
@@ -1488,7 +1488,7 @@ load _helpers
     --set 'global.secretsBackend.vault.consulClientRole=foo' \
     --set 'global.secretsBackend.vault.consulServerRole=test' \
     --set 'global.gossipEncryption.secretName=a/b/c/d' \
-    --set 'global.gossipEncryption.secretKey=.Data.data.gossip' \
+    --set 'global.gossipEncryption.secretKey=gossip' \
     . | tee /dev/stderr |
       yq -r '.spec.template.spec' | tee /dev/stderr)
 

From 2cb9e65b2527c4c02678a54b61226fc48ff39cb5 Mon Sep 17 00:00:00 2001
From: Kyle Schochenmaier <kschoche@gmail.com>
Date: Tue, 9 Nov 2021 15:11:02 -0600
Subject: [PATCH 15/20] bug fixes in test

---
 acceptance/framework/vault/vault_cluster.go | 2 +-
 acceptance/tests/vault/vault_test.go        | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/acceptance/framework/vault/vault_cluster.go b/acceptance/framework/vault/vault_cluster.go
index 1801c892c0..51850b5b8e 100644
--- a/acceptance/framework/vault/vault_cluster.go
+++ b/acceptance/framework/vault/vault_cluster.go
@@ -65,7 +65,7 @@ func NewVaultCluster(
 		KubectlOptions: kopts,
 		Logger:         logger,
 	}
-	helm.AddRepo(t, &helm.Options{}, "hashicorp/vault", "https://helm.releases.hashicorp.com")
+	helm.AddRepo(t, &helm.Options{}, "hashicorp", "https://helm.releases.hashicorp.com")
 	// Ignoring the error from `helm repo update` as it could fail due to stale cache or unreachable servers and we're
 	// asserting a chart version on Install which would fail in an obvious way should this not succeed.
 	errStr, err := helm.RunHelmCommandAndGetOutputE(t, &helm.Options{}, "repo", "update")
diff --git a/acceptance/tests/vault/vault_test.go b/acceptance/tests/vault/vault_test.go
index 69e14ce32a..1736d2788f 100644
--- a/acceptance/tests/vault/vault_test.go
+++ b/acceptance/tests/vault/vault_test.go
@@ -48,7 +48,7 @@ func TestVault_BootstrapConsulGossipEncryptionKey(t *testing.T) {
 
 	// FIXME: There is a *slight* delay between when the vault pods are Ready and when the vaultClient connect attempt
 	// will work, on occassion the client connection will time out before it's ready. Fix later.
-	time.Sleep(1 * time.Second)
+	time.Sleep(5 * time.Second)
 	// Now fetch the Vault client so we can create the policies and secrets.
 	vaultClient := vaultCluster.VaultClient(t)
 
@@ -102,7 +102,7 @@ path "consul/data/secret/gossip" {
 
 		"global.secretsBackend.vault.enabled":          "true",
 		"global.secretsBackend.vault.consulServerRole": "consul-server",
-		"global.secretsBackend.vault.consulclientRole": "consul-client",
+		"global.secretsBackend.vault.consulClientRole": "consul-client",
 
 		"global.acls.manageSystemACLs":       "true",
 		"global.tls.enabled":                 "true",

From b3a4d9fffcab0e7112b6d4d80e6a6eea9f829e94 Mon Sep 17 00:00:00 2001
From: Kyle Schochenmaier <kschoche@gmail.com>
Date: Wed, 10 Nov 2021 14:54:41 -0600
Subject: [PATCH 16/20] update test framework to use k8s svc as issuer to work
 around vault bug

---
 acceptance/framework/vault/vault_cluster.go | 4 ++--
 acceptance/tests/vault/vault_test.go        | 7 +------
 2 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/acceptance/framework/vault/vault_cluster.go b/acceptance/framework/vault/vault_cluster.go
index 51850b5b8e..14c2b35e93 100644
--- a/acceptance/framework/vault/vault_cluster.go
+++ b/acceptance/framework/vault/vault_cluster.go
@@ -111,7 +111,7 @@ func (v *VaultCluster) SetupVaultClient(t *testing.T) *vapi.Client {
 		v.logger)
 
 	// Retry creating the port forward since it can fail occasionally.
-	retry.RunWith(&retry.Counter{Wait: 1 * time.Second, Count: 3}, t, func(r *retry.R) {
+	retry.RunWith(&retry.Counter{Wait: 1 * time.Second, Count: 10}, t, func(r *retry.R) {
 		// NOTE: It's okay to pass in `t` to ForwardPortE despite being in a retry
 		// because we're using ForwardPortE (not ForwardPort) so the `t` won't
 		// get used to fail the test, just for logging.
@@ -154,7 +154,7 @@ func (v *VaultCluster) bootstrap(t *testing.T, ctx environment.TestContext) {
 	}
 	// We need to kubectl exec this one on the vault server:
 	// This is taken from https://learn.hashicorp.com/tutorials/vault/kubernetes-google-cloud-gke?in=vault/kubernetes#configure-kubernetes-authentication
-	cmdString := fmt.Sprintf("VAULT_TOKEN=%s vault write auth/kubernetes/config token_reviewer_jwt=\"$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\" kubernetes_host=\"https://${KUBERNETES_PORT_443_TCP_ADDR}:443\" kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", vaultRootToken)
+	cmdString := fmt.Sprintf("VAULT_TOKEN=%s vault write auth/kubernetes/config issuer=\"https://kubernetes.default.svc.cluster.local\" token_reviewer_jwt=\"$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\" kubernetes_host=\"https://${KUBERNETES_PORT_443_TCP_ADDR}:443\" kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", vaultRootToken)
 
 	v.logger.Logf(t, "updating vault kube auth config")
 	k8s.RunKubectl(t, ctx.KubectlOptions(t), "exec", "-i", fmt.Sprintf("%s-vault-0", v.vaultReleaseName), "--", "sh", "-c", cmdString)
diff --git a/acceptance/tests/vault/vault_test.go b/acceptance/tests/vault/vault_test.go
index 1736d2788f..d44f10ea1c 100644
--- a/acceptance/tests/vault/vault_test.go
+++ b/acceptance/tests/vault/vault_test.go
@@ -4,14 +4,12 @@ import (
 	"crypto/rand"
 	"encoding/base64"
 	"fmt"
-	"testing"
-	"time"
-
 	"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/vault"
 	"github.com/stretchr/testify/require"
+	"testing"
 )
 
 // generateGossipSecret generates a random 32 byte secret returned as a base64 encoded string.
@@ -46,9 +44,6 @@ func TestVault_BootstrapConsulGossipEncryptionKey(t *testing.T) {
 	vaultCluster.Create(t, ctx)
 	// Vault is now installed in the cluster.
 
-	// FIXME: There is a *slight* delay between when the vault pods are Ready and when the vaultClient connect attempt
-	// will work, on occassion the client connection will time out before it's ready. Fix later.
-	time.Sleep(5 * time.Second)
 	// Now fetch the Vault client so we can create the policies and secrets.
 	vaultClient := vaultCluster.VaultClient(t)
 

From f7f3705cdd297106c0335688e8e894526fbb1159 Mon Sep 17 00:00:00 2001
From: Kyle Schochenmaier <kschoche@gmail.com>
Date: Thu, 11 Nov 2021 10:07:20 -0600
Subject: [PATCH 17/20] update autogen test since secretkey and name together
 are required now

---
 .../unit/gossip-encryption-autogenerate-job.bats    | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/charts/consul/test/unit/gossip-encryption-autogenerate-job.bats b/charts/consul/test/unit/gossip-encryption-autogenerate-job.bats
index a2a0de0a95..4b5938ab91 100644
--- a/charts/consul/test/unit/gossip-encryption-autogenerate-job.bats
+++ b/charts/consul/test/unit/gossip-encryption-autogenerate-job.bats
@@ -38,3 +38,16 @@ load _helpers
   [ "$status" -eq 1 ]
   [[ "$output" =~ "If global.gossipEncryption.autoGenerate is true, global.gossipEncryption.secretName and global.gossipEncryption.secretKey must not be set." ]]
 }
+
+@test "gossipEncryptionAutogenerate/Job: fails if global.gossipEncryption.autoGenerate=true and global.gossipEncryption.secretName+key are set" {
+  cd `chart_dir`
+  run helm template \
+      -s templates/gossip-encryption-autogenerate-job.yaml  \
+      --set 'global.gossipEncryption.autoGenerate=true' \
+      --set 'global.gossipEncryption.secretName=name' \
+      --set 'global.gossipEncryption.secretKey=name' \
+      .
+  [ "$status" -eq 1 ]
+  [[ "$output" =~ "If global.gossipEncryption.autoGenerate is true, global.gossipEncryption.secretName and global.gossipEncryption.secretKey must not be set." ]]
+}
+

From bb266b33089ec1d91617e6e4b8698407996a944c Mon Sep 17 00:00:00 2001
From: Kyle Schochenmaier <kschoche@gmail.com>
Date: Fri, 12 Nov 2021 08:59:35 -0600
Subject: [PATCH 18/20] Apply suggestions from code review

Co-authored-by: Iryna Shustava <ishustava@users.noreply.github.com>
---
 acceptance/tests/vault/vault_test.go |  4 ++--
 charts/consul/values.yaml            | 11 +++++++++--
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/acceptance/tests/vault/vault_test.go b/acceptance/tests/vault/vault_test.go
index d44f10ea1c..67b81f1884 100644
--- a/acceptance/tests/vault/vault_test.go
+++ b/acceptance/tests/vault/vault_test.go
@@ -57,7 +57,7 @@ path "consul/data/secret/gossip" {
 	require.NoError(t, err)
 
 	// Create the Auth Roles for consul-server + consul-client.
-	logger.Log(t, "Creating the gossip auth roles")
+	logger.Log(t, "Creating the consul-server and consul-client-roles")
 	params := map[string]interface{}{
 		"bound_service_account_names":      consulClientServiceAccountName,
 		"bound_service_account_namespaces": "default",
@@ -113,6 +113,6 @@ path "consul/data/secret/gossip" {
 	consulClient := consulCluster.SetupConsulClient(t, true)
 	keys, err := consulClient.Operator().KeyringList(nil)
 	require.NoError(t, err)
-	// we use keys[0] because KeyringList returns a list of keyrings for each dc, in this case there is only 1 dc.
+	// We use keys[0] because KeyringList returns a list of keyrings for each dc, in this case there is only 1 dc.
 	require.Equal(t, 1, keys[0].PrimaryKeys[gossipKey])
 }
diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml
index d21ab38cc5..9a9a7fe2c8 100644
--- a/charts/consul/values.yaml
+++ b/charts/consul/values.yaml
@@ -130,8 +130,15 @@ global:
       # Enabling the Vault secrets backend will replace Kubernetes secrets with referenced Vault secrets.
       enabled: false
 
-      # The consulServerRole must be created and have read capabilties via a Policy for the following secrets:
+      # The Vault role for the Consul server.
+      # The role must be connected to the Consul server's service account and
+      # have have a policy with read capabilities for the following secrets:
       # - gossip encryption key defined by `global.gossipEncryption.secretName`.
+      # To discover the service account name of the Consul server, run
+      #     ```
+      #     helm template -s templates/server-serviceaccount.yaml <release-name> charts/consul
+      #     ```
+      # and check the name of `metadata.name`.
       consulServerRole: ""
 
       # The consulClientRole must be created and have read capabilties via a Policy for the following secrets:
@@ -146,7 +153,7 @@ global:
   # Values for secretName and secretKey should not be set if autoGenerate is true.
   # To manually generate a gossip encryption key, set secretName and secretKey and use Consul to generate
   # a key, saving this as a Kubernetes secret or Vault secret path and key.
-  # If `global.secretsBackend.vault.enabled=true` be sure to add the "data" field to the secretName as required by
+  # If `global.secretsBackend.vault.enabled=true`, be sure to add the "data" field to the secretName as required by
   # the Vault KV-2 secrets engine [see example].
   #
   # ```

From 1f065986a3413c70ff07dfe856dc22a626c22ab5 Mon Sep 17 00:00:00 2001
From: Kyle Schochenmaier <kschoche@gmail.com>
Date: Fri, 12 Nov 2021 11:44:49 -0600
Subject: [PATCH 19/20] review comments

---
 acceptance/framework/helpers/helpers.go     |  4 ++--
 acceptance/framework/vault/vault_cluster.go |  8 ++++----
 charts/consul/values.yaml                   | 13 ++++++++++---
 3 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/acceptance/framework/helpers/helpers.go b/acceptance/framework/helpers/helpers.go
index 16e8a30fbb..f173aeea6f 100644
--- a/acceptance/framework/helpers/helpers.go
+++ b/acceptance/framework/helpers/helpers.go
@@ -29,7 +29,7 @@ func RandomName() string {
 
 // CheckForPriorInstallations checks if there is an existing Helm release
 // for this Helm chart already installed. If there is, it fails the tests.
-func CheckForPriorInstallations(t *testing.T, client kubernetes.Interface, options *helm.Options, chartName, label string) {
+func CheckForPriorInstallations(t *testing.T, client kubernetes.Interface, options *helm.Options, chartName, labelSelector string) {
 	t.Helper()
 
 	var helmListOutput string
@@ -57,7 +57,7 @@ func CheckForPriorInstallations(t *testing.T, client kubernetes.Interface, optio
 	// Wait for all pods in the "default" namespace to exit. A previous
 	// release may not be listed by Helm but its pods may still be terminating.
 	retry.RunWith(&retry.Counter{Wait: 1 * time.Second, Count: 60}, t, func(r *retry.R) {
-		pods, err := client.CoreV1().Pods(options.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: label})
+		pods, err := client.CoreV1().Pods(options.KubectlOptions.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: labelSelector})
 		require.NoError(r, err)
 		if len(pods.Items) > 0 {
 			var podNames []string
diff --git a/acceptance/framework/vault/vault_cluster.go b/acceptance/framework/vault/vault_cluster.go
index 14c2b35e93..e9e06baa41 100644
--- a/acceptance/framework/vault/vault_cluster.go
+++ b/acceptance/framework/vault/vault_cluster.go
@@ -65,12 +65,12 @@ func NewVaultCluster(
 		KubectlOptions: kopts,
 		Logger:         logger,
 	}
-	helm.AddRepo(t, &helm.Options{}, "hashicorp", "https://helm.releases.hashicorp.com")
+	helm.AddRepo(t, vaultHelmOpts, "hashicorp", "https://helm.releases.hashicorp.com")
 	// Ignoring the error from `helm repo update` as it could fail due to stale cache or unreachable servers and we're
 	// asserting a chart version on Install which would fail in an obvious way should this not succeed.
-	errStr, err := helm.RunHelmCommandAndGetOutputE(t, &helm.Options{}, "repo", "update")
+	_, err := helm.RunHelmCommandAndGetOutputE(t, &helm.Options{}, "repo", "update")
 	if err != nil {
-		logger.Logf(t, "Unable to update helm repository: %s, %s", err, errStr)
+		logger.Logf(t, "Unable to update helm repository, proceeding anyway: %s.", err)
 	}
 
 	return &VaultCluster{
@@ -154,7 +154,7 @@ func (v *VaultCluster) bootstrap(t *testing.T, ctx environment.TestContext) {
 	}
 	// We need to kubectl exec this one on the vault server:
 	// This is taken from https://learn.hashicorp.com/tutorials/vault/kubernetes-google-cloud-gke?in=vault/kubernetes#configure-kubernetes-authentication
-	cmdString := fmt.Sprintf("VAULT_TOKEN=%s vault write auth/kubernetes/config issuer=\"https://kubernetes.default.svc.cluster.local\" token_reviewer_jwt=\"$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\" kubernetes_host=\"https://${KUBERNETES_PORT_443_TCP_ADDR}:443\" kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", vaultRootToken)
+	cmdString := fmt.Sprintf("VAULT_TOKEN=%s vault write auth/kubernetes/config disable_iss_validation=\"true\" token_reviewer_jwt=\"$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\" kubernetes_host=\"https://${KUBERNETES_PORT_443_TCP_ADDR}:443\" kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", vaultRootToken)
 
 	v.logger.Logf(t, "updating vault kube auth config")
 	k8s.RunKubectl(t, ctx.KubectlOptions(t), "exec", "-i", fmt.Sprintf("%s-vault-0", v.vaultReleaseName), "--", "sh", "-c", cmdString)
diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml
index 9a9a7fe2c8..2a04d7416c 100644
--- a/charts/consul/values.yaml
+++ b/charts/consul/values.yaml
@@ -132,7 +132,7 @@ global:
 
       # The Vault role for the Consul server.
       # The role must be connected to the Consul server's service account and
-      # have have a policy with read capabilities for the following secrets:
+      # have a policy with read capabilities for the following secrets:
       # - gossip encryption key defined by `global.gossipEncryption.secretName`.
       # To discover the service account name of the Consul server, run
       #     ```
@@ -141,8 +141,15 @@ global:
       # and check the name of `metadata.name`.
       consulServerRole: ""
 
-      # The consulClientRole must be created and have read capabilties via a Policy for the following secrets:
-      # - gossip encryption key defined by `global.gossipEncryption.secretName`
+      # The Vault role for the Consul client.
+      # The role must be connected to the Consul client's service account and
+      # have a policy with read capabilities for the following secrets:
+      # - gossip encryption key defined by `global.gossipEncryption.secretName`.
+      # To discover the service account name of the Consul server, run
+      #     ```
+      #     helm template -s templates/client-daemonset.yaml <release-name> charts/consul
+      #     ```
+      # and check the name of `metadata.name`.
       consulClientRole: ""
 
   # Configures Consul's gossip encryption key.

From d99e8456a938d788258d467628c9905235af6443 Mon Sep 17 00:00:00 2001
From: Kyle Schochenmaier <kschoche@gmail.com>
Date: Fri, 12 Nov 2021 12:27:48 -0600
Subject: [PATCH 20/20] update retries

---
 acceptance/framework/vault/vault_cluster.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/acceptance/framework/vault/vault_cluster.go b/acceptance/framework/vault/vault_cluster.go
index e9e06baa41..01e3141e1f 100644
--- a/acceptance/framework/vault/vault_cluster.go
+++ b/acceptance/framework/vault/vault_cluster.go
@@ -111,7 +111,7 @@ func (v *VaultCluster) SetupVaultClient(t *testing.T) *vapi.Client {
 		v.logger)
 
 	// Retry creating the port forward since it can fail occasionally.
-	retry.RunWith(&retry.Counter{Wait: 1 * time.Second, Count: 10}, t, func(r *retry.R) {
+	retry.RunWith(&retry.Counter{Wait: 1 * time.Second, Count: 60}, t, func(r *retry.R) {
 		// NOTE: It's okay to pass in `t` to ForwardPortE despite being in a retry
 		// because we're using ForwardPortE (not ForwardPort) so the `t` won't
 		// get used to fail the test, just for logging.