Bootstrap gossip encryption with Vault

acceptance/framework/vault/vault_cluster.go

@@ -1,12 +1,7 @@
 package vault
 
 import (
-	"context"
 	"fmt"
-	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
-	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"testing"
 	"time"
 
@@ -15,6 +10,8 @@ import (
 	terratestLogger "github.com/gruntwork-io/terratest/modules/logger"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/config"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/environment"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 	"github.com/hashicorp/consul/sdk/testutil/retry"
 	vapi "github.com/hashicorp/vault/api"
@@ -74,7 +71,14 @@ func NewVaultCluster(
 	helm.AddRepo(t, &helm.Options{}, "hashicorp/vault", "https://helm.releases.hashicorp.com")
 	// Ignoring the error from `helm repo update` as it could fail due to stale cache or unreachable servers and we're
 	// asserting a chart version on Install which would fail in an obvious way should this not succeed.
-	_, _ = helm.RunHelmCommandAndGetOutputE(t, &helm.Options{}, "repo", "update")
+	errStr, err := helm.RunHelmCommandAndGetOutputE(t, &helm.Options{}, "repo", "update")
+	if err != nil {
+		logger.Logf(t, "Unable to update helm repository: %s, %s", err, errStr)
+	}
+	errStr, err = helm.RunHelmCommandAndGetOutputE(t, &helm.Options{}, "pull", "hashicorp/vault", "--version", vaultChartVersion)
+	if err != nil {
+		logger.Logf(t, "Unable to pull helm repository: %s, %s", err, errStr)
+	}
 
 	return &VaultCluster{
 		ctx:                ctx,
@@ -179,15 +183,8 @@ func (v *VaultCluster) Create(t *testing.T, ctx environment.TestContext) {
 
 	// Install Vault.
 	helm.Install(t, v.vaultHelmOptions, "hashicorp/vault", v.vaultReleaseName)
-	// Wait for the injector pod to become Ready, but not the server.
-	helpers.WaitForAllPodsToBeReady(t, v.kubernetesClient, v.vaultHelmOptions.KubectlOptions.Namespace, "app.kubernetes.io/name=vault-agent-injector")
-	// Wait for the server pod to be PodRunning, it will not be Ready because it has not been Init+Unseal'd yet.
-	// The vault server has health checks bound to unseal status, and Unseal is done as part of bootstrap (below).
-	retry.RunWith(&retry.Counter{Wait: 1 * time.Second, Count: 30}, t, func(r *retry.R) {
-		pod, err := v.kubernetesClient.CoreV1().Pods(v.vaultHelmOptions.KubectlOptions.Namespace).Get(context.Background(), fmt.Sprintf("%s-vault-0", v.vaultReleaseName), metav1.GetOptions{})
-		require.NoError(r, err)
-		require.Equal(r, pod.Status.Phase, corev1.PodRunning)
-	})
+	// Wait for the injector and vault server pods to become Ready.
+	helpers.WaitForAllPodsToBeReady(t, v.kubernetesClient, v.vaultHelmOptions.KubectlOptions.Namespace, fmt.Sprintf("helm.sh/chart=%s", v.vaultChartName))
 	// Now call bootstrap()
 	v.bootstrap(t, ctx)
 }

acceptance/tests/vault/vault_test.go

@@ -1,16 +1,21 @@
 package vault
 
 import (
+	"fmt"
 	"testing"
-	"time"
 
+	"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/vault"
 	"github.com/stretchr/testify/require"
 )
 
-// Installs Vault, bootstraps it with the kube auth method
+const (
+	gossipKey = "3R7oLrdpkk2V0Y7yHLizyxXeS2RtaVuy07DkU15Lhws="
+)
+
+// Installs Vault, bootstraps it with the Kube Auth Method
 // and then validates that the KV2 secrets engine is online
 // and the Kube Auth Method is enabled.
 func TestVault_Create(t *testing.T) {
@@ -39,5 +44,89 @@ func TestVault_Create(t *testing.T) {
 	require.NoError(t, err)
 	logger.Log(t, "Auth List: ", authList)
 	require.NotNil(t, authList["kubernetes/"])
-	time.Sleep(time.Second * 60)
+}
+
+// Installs Vault, bootstraps it with secrets, policies, and Kube Auth Method
+// then creates a gossip encryption secret and uses this to bootstrap Consul.
+func TestVault_BootstrapConsulGossipEncryptionKey(t *testing.T) {
+	cfg := suite.Config()
+	ctx := suite.Environment().DefaultContext(t)
+
+	consulReleaseName := helpers.RandomName()
+	vaultReleaseName := helpers.RandomName()
+	consulClientServiceAccountName := fmt.Sprintf("%s-consul-client", consulReleaseName)
+	consulServerServiceAccountName := fmt.Sprintf("%s-consul-server", consulReleaseName)
+
+	vaultCluster := vault.NewVaultCluster(t, nil, ctx, cfg, vaultReleaseName)
+	vaultCluster.Create(t, ctx)
+	// Vault is now installed in the cluster.
+
+	// Now fetch the Vault client so we can create the policies and secrets.
+	vaultClient := vaultCluster.VaultClient(t)
+
+	// Create the Vault Policy for the gossip key.
+	logger.Log(t, "Creating the gossip policy")
+	rules := `
+path "consul/data/secret/gossip" {
+  capabilities = ["read"]
+}`
+	err := vaultClient.Sys().PutPolicy("consul-gossip", rules)
+	require.NoError(t, err)
+
+	// Create the Auth Roles for consul-server + consul-client.
+	logger.Log(t, "Creating the gossip auth roles")
+	params := map[string]interface{}{
+		"bound_service_account_names":      consulClientServiceAccountName,
+		"bound_service_account_namespaces": "default",
+		"policies":                         "consul-gossip",
+		"ttl":                              "24h",
+	}
+	_, err = vaultClient.Logical().Write("auth/kubernetes/role/consul-client", params)
+	require.NoError(t, err)
+
+	params = map[string]interface{}{
+		"bound_service_account_names":      consulServerServiceAccountName,
+		"bound_service_account_namespaces": "default",
+		"policies":                         "consul-gossip",
+		"ttl":                              "24h",
+	}
+	_, err = vaultClient.Logical().Write("auth/kubernetes/role/consul-server", params)
+	require.NoError(t, err)
+
+	// Create the gossip key.
+	logger.Log(t, "Creating the gossip secret")
+	params = map[string]interface{}{
+		"data": map[string]interface{}{
+			"gossip": gossipKey,
+		},
+	}
+	_, err = vaultClient.Logical().Write("consul/data/secret/gossip", params)
+	require.NoError(t, err)
+
+	consulHelmValues := map[string]string{
+		"server.enabled":  "true",
+		"server.replicas": "1",
+
+		"connectInject.enabled": "true",
+
+		"global.secretsBackend.vault.enabled":          "true",
+		"global.secretsBackend.vault.consulServerRole": "consul-server",
+		"global.secretsBackend.vault.consulclientRole": "consul-client",
+
+		"global.acls.manageSystemACLs":       "true",
+		"global.tls.enabled":                 "true",
+		"global.gossipEncryption.secretName": "consul/data/secret/gossip",
+		"global.gossipEncryption.secretKey":  ".Data.data.gossip",
+	}
+	logger.Log(t, "Installing Consul")
+	consulCluster := consul.NewHelmCluster(t, consulHelmValues, ctx, cfg, consulReleaseName)
+	consulCluster.Create(t)
+
+	// Validate that the gossip encryption key is set correctly.
+	logger.Log(t, "Validating the gossip key has been set correctly.")
+	consulClient := consulCluster.SetupConsulClient(t, true)
+	keys, err := consulClient.Operator().KeyringList(nil)
+	require.NoError(t, err)
+	// we use keys[0] because KeyringList returns a list of keyrings for each dc, in this case there is only 1 dc.
+	require.Equal(t, 1, keys[0].PrimaryKeys[gossipKey])
 }

charts/consul/templates/client-daemonset.yaml

@@ -37,6 +37,16 @@ spec:
           {{- toYaml .Values.client.extraLabels | nindent 8 }}
         {{- end }}
       annotations:
+        {{- if .Values.global.secretsBackend.vault.enabled }}
+        "vault.hashicorp.com/agent-inject": "true"
+        "vault.hashicorp.com/agent-init-first": "true"
+        "vault.hashicorp.com/role": "{{ .Values.global.secretsBackend.vault.consulClientRole }}"
+        {{- with .Values.global.gossipEncryption }}
+        "vault.hashicorp.com/agent-inject-secret-gossip.txt": "{{ .secretName }}"
+        "vault.hashicorp.com/agent-inject-template-gossip.txt": '{{ "{{" }}- with secret "{{ .secretName }}" -{{ "}}" }} {{ "{{" }} {{ .secretKey }} {{ "}}" }} {{ "{{" }}- end -{{ "}}" }}'
+        "vault.hashicorp.com/template-static-secret-render-interval": "10s"
+        {{- end }}
+        {{- end }}
         "consul.hashicorp.com/connect-inject": "false"
         "consul.hashicorp.com/config-checksum": {{ include (print $.Template.BasePath "/client-config-configmap.yaml") . | sha256sum }}
         {{- if .Values.client.annotations }}
@@ -169,6 +179,7 @@ spec:
             - name: CONSUL_DISABLE_PERM_MGMT
               value: "true"
             {{- if (or .Values.global.gossipEncryption.autoGenerate (and .Values.global.gossipEncryption.secretName .Values.global.gossipEncryption.secretKey)) }}
+            {{- if not .Values.global.secretsBackend.vault.enabled }}
             - name: GOSSIP_KEY
               valueFrom:
                 secretKeyRef:
@@ -180,6 +191,7 @@ spec:
                   key: {{ .Values.global.gossipEncryption.secretKey }}
                 {{- end }}
             {{- end }}
+            {{- end }}
             {{- if (and .Values.server.enterpriseLicense.secretName .Values.server.enterpriseLicense.secretKey .Values.server.enterpriseLicense.enableLicenseAutoload (not .Values.global.acls.manageSystemACLs)) }}
             - name: CONSUL_LICENSE_PATH
               value: /consul/license/{{ .Values.server.enterpriseLicense.secretKey }}
@@ -202,6 +214,10 @@ spec:
             - |
               CONSUL_FULLNAME="{{template "consul.fullname" . }}"
 
+              {{- if and .Values.global.secretsBackend.vault.enabled .Values.global.gossipEncryption.secretName }}
+              GOSSIP_KEY=`cat /vault/secrets/gossip.txt`
+              {{- end }}
+
               {{ template "consul.extraconfig" }}
 
               exec /usr/local/bin/docker-entrypoint.sh consul agent \

charts/consul/templates/server-statefulset.yaml

@@ -5,6 +5,8 @@
 {{- if .Values.server.disableFsGroupSecurityContext }}{{ fail "server.disableFsGroupSecurityContext has been removed. Please use global.openshift.enabled instead." }}{{ end }}
 {{- if .Values.server.bootstrapExpect }}{{ if lt (int .Values.server.bootstrapExpect) (int .Values.server.replicas) }}{{ fail "server.bootstrapExpect cannot be less than server.replicas" }}{{ end }}{{ end }}
 {{- if (and (and .Values.global.tls.enabled .Values.global.tls.httpsOnly) (and .Values.global.metrics.enabled .Values.global.metrics.enableAgentMetrics))}}{{ fail "global.metrics.enableAgentMetrics cannot be enabled if TLS (HTTPS only) is enabled" }}{{ end -}}
+{{- if (and .Values.global.gossipEncryption.secretName (not .Values.global.gossipEncryption.secretKey)) }}{{fail "gossipEncryption.secretKey and secretName must both be specified." }}{{ end -}}
+{{- if (and (not .Values.global.gossipEncryption.secretName) .Values.global.gossipEncryption.secretKey) }}{{fail "gossipEncryption.secretKey and secretName must both be specified." }}{{ end -}}
 # StatefulSet to run the actual Consul server cluster.
 apiVersion: apps/v1
 kind: StatefulSet
@@ -46,6 +48,16 @@ spec:
           {{- toYaml .Values.server.extraLabels | nindent 8 }}
         {{- end }}
       annotations:
+        {{- if .Values.global.secretsBackend.vault.enabled }}
+        "vault.hashicorp.com/agent-inject": "true"
+        "vault.hashicorp.com/agent-init-first": "true"
+        "vault.hashicorp.com/role": "{{ .Values.global.secretsBackend.vault.consulServerRole }}"
+        {{- with .Values.global.gossipEncryption }}
+        "vault.hashicorp.com/agent-inject-secret-gossip.txt": "{{ .secretName }}"
+        "vault.hashicorp.com/agent-inject-template-gossip.txt": '{{ "{{" }}- with secret "{{ .secretName }}" -{{ "}}" }} {{ "{{" }} {{ .secretKey }} {{ "}}" }} {{ "{{" }}- end -{{ "}}" }}'
+        "vault.hashicorp.com/template-static-secret-render-interval": "10s"
+        {{- end }}
+        {{- end }}
         "consul.hashicorp.com/connect-inject": "false"
         "consul.hashicorp.com/config-checksum": {{ include (print $.Template.BasePath "/server-config-configmap.yaml") . | sha256sum }}
         {{- if .Values.server.annotations }}
@@ -157,6 +169,7 @@ spec:
             - name: CONSUL_DISABLE_PERM_MGMT
               value: "true"
             {{- if (or .Values.global.gossipEncryption.autoGenerate (and .Values.global.gossipEncryption.secretName .Values.global.gossipEncryption.secretKey)) }}
+            {{- if not .Values.global.secretsBackend.vault.enabled }}
             - name: GOSSIP_KEY
               valueFrom:
                 secretKeyRef:
@@ -168,6 +181,7 @@ spec:
                   key: {{ .Values.global.gossipEncryption.secretKey }}
                 {{- end }}
             {{- end }}
+            {{- end }}
             {{- if .Values.global.tls.enabled }}
             - name: CONSUL_HTTP_ADDR
               value: https://localhost:8501
@@ -192,6 +206,10 @@ spec:
             - |
               CONSUL_FULLNAME="{{template "consul.fullname" . }}"
 
+              {{- if .Values.global.secretsBackend.vault.enabled }}
+              GOSSIP_KEY=`cat /vault/secrets/gossip.txt`
+              {{- end }}
+
               {{ template "consul.extraconfig" }}
 
               exec /usr/local/bin/docker-entrypoint.sh consul agent \

charts/consul/test/unit/client-daemonset.bats

@@ -602,7 +602,7 @@ load _helpers
   local actual=$(helm template \
       -s templates/client-daemonset.yaml  \
       . | tee /dev/stderr |
-      yq '.spec.template.spec.containers[] | select(.name=="consul") | .env[] | select(.name == "GOSSIP_KEY") | length > 0' | tee /dev/stderr)
+      yq '.spec.template.spec.containers[] | select(.name=="consul") | .env[] | select(.name == "GOSSIP_KEY")' | tee /dev/stderr)
   [ "${actual}" = "" ]
 }
 
@@ -1516,3 +1516,73 @@ rollingUpdate:
 
   [ "${object}" = 1 ]
 }
+
+#--------------------------------------------------------------------
+# vault integration
+
+@test "client/DaemonSet: vault annotations not attached by default" {
+  cd `chart_dir`
+  local actual=$(helm template \
+    -s templates/client-daemonset.yaml  \
+    . | tee /dev/stderr |
+     yq '.spec.template.metadata.annotations["vault.hashicorp.com/agent-inject"] | length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+}
+
+@test "client/DaemonSet: vault annotations added when vault is enabled" {
+  cd `chart_dir`
+  local object=$(helm template \
+    -s templates/client-daemonset.yaml  \
+    --set 'global.secretsBackend.vault.enabled=true' \
+    . | tee /dev/stderr |
+      yq -r '.spec.template.metadata' | tee /dev/stderr)
+
+  local actual=$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/agent-inject"] | length > 0' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+  local actual=$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/agent-init-first"] | length > 0' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+  local actual=$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/role"]' | tee /dev/stderr)
+  [ "${actual}" = "" ]
+}
+
+@test "client/DaemonSet: vault gossip annotations are correct when enabled" {
+  cd `chart_dir`
+  local object=$(helm template \
+    -s templates/client-daemonset.yaml  \
+    --set 'global.secretsBackend.vault.enabled=true' \
+    --set 'global.gossipEncryption.secretName=path/to/secret/key' \
+    --set 'global.gossipEncryption.secretKey=.Data.gossip.gossip' \
+    . | tee /dev/stderr |
+      yq -r '.spec.template.metadata' | tee /dev/stderr)
+
+  local actual=$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/agent-inject-secret-gossip.txt"]' | tee /dev/stderr)
+  [ "${actual}" = "path/to/secret/key" ]
+  local actual=$(echo $object |
+      yq -r '.annotations["vault.hashicorp.com/agent-inject-template-gossip.txt"]' | tee /dev/stderr)
+  [ "${actual}" = '{{- with secret "path/to/secret/key" -}} {{ .Data.gossip.gossip }} {{- end -}}' ]
+}
+
+@test "client/DaemonSet: vault no GOSSIP_KEY env variable and command defines GOSSIP_KEY" {
+  cd `chart_dir`
+  local object=$(helm template \
+    -s templates/client-daemonset.yaml  \
+    --set 'global.secretsBackend.vault.enabled=true' \
+    --set 'global.gossipEncryption.secretName=a/b/c/d' \
+    --set 'global.gossipEncryption.secretKey=.Data.data.gossip' \
+    . | tee /dev/stderr |
+      yq -r '.spec.template.spec' | tee /dev/stderr)
+
+
+  local actual=$(echo $object |
+    yq -r '.containers[] | select(.name=="consul") | .env[] | select(.name == "GOSSIP_KEY")' | tee /dev/stderr)
+  [ "${actual}" = "" ]
+
+  local actual=$(echo $object |
+    yq -r '.containers[] | select(.name=="consul") | .command | any(contains("GOSSIP_KEY="))' \
+      | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}

charts/consul/test/unit/gossip-encryption-autogenerate-job.bats

@@ -38,26 +38,3 @@ load _helpers
   [ "$status" -eq 1 ]
   [[ "$output" =~ "If global.gossipEncryption.autoGenerate is true, global.gossipEncryption.secretName and global.gossipEncryption.secretKey must not be set." ]]
 }
-
-@test "gossipEncryptionAutogenerate/Job: fails if global.gossipEncryption.autoGenerate=true and global.gossipEncryption.secretName is set" {
-  cd `chart_dir`
-  run helm template \
-      -s templates/gossip-encryption-autogenerate-job.yaml  \
-      --set 'global.gossipEncryption.autoGenerate=true' \
-      --set 'global.gossipEncryption.secretName=name' \
-      . 
-  [ "$status" -eq 1 ]
-  [[ "$output" =~ "If global.gossipEncryption.autoGenerate is true, global.gossipEncryption.secretName and global.gossipEncryption.secretKey must not be set." ]]
-}
-
-@test "gossipEncryptionAutogenerate/Job: fails if global.gossipEncryption.autoGenerate=true and global.gossipEncryption.secretKey is set" {
-  cd `chart_dir`
-  run helm template \
-      -s templates/gossip-encryption-autogenerate-job.yaml  \
-      --set 'global.gossipEncryption.autoGenerate=true' \
-      --set 'global.gossipEncryption.secretKey=key' \
-      . 
-  [ "$status" -eq 1 ]
-  [[ "$output" =~ "If global.gossipEncryption.autoGenerate is true, global.gossipEncryption.secretName and global.gossipEncryption.secretKey must not be set." ]]
-}
-