NET-1721: Automatic ACL bootstrap with Vault secrets backend

.changelog/1920.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+helm: When the `global.acls.bootstrapToken` field is set and the content of the secret is empty, the bootstrap ACL token is written to that secret after bootstrapping ACLs. This applies to both the Vault and Consul secrets backends.
+```

acceptance/framework/vault/helpers.go

@@ -167,6 +167,20 @@ func (config *KV2Secret) SaveSecretAndAddReadPolicy(t *testing.T, vaultClient *v
 	path "%s" {
 	  capabilities = ["read"]
 	}`, config.Path)
+	config.saveSecretAndAddPolicy(t, vaultClient, policy)
+}
+
+// SaveSecretAndAddUpdatePolicy will create an update policy for the PolicyName
+// on the KV2Secret and then will save the secret in the KV2 store.
+func (config *KV2Secret) SaveSecretAndAddUpdatePolicy(t *testing.T, vaultClient *vapi.Client) {
+	policy := fmt.Sprintf(`
+	path "%s" {
+	  capabilities = ["read", "update"]
+	}`, config.Path)
+	config.saveSecretAndAddPolicy(t, vaultClient, policy)
+}
+
+func (config *KV2Secret) saveSecretAndAddPolicy(t *testing.T, vaultClient *vapi.Client, policy string) {
 	// Create the Vault Policy for the secret.
 	logger.Log(t, "Creating policy")
 	err := vaultClient.Sys().PutPolicy(config.PolicyName, policy)

acceptance/tests/vault/vault_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/portforward"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/vault"
+	"github.com/hashicorp/consul/sdk/testutil/retry"
 	"github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/go-version"
 	"github.com/stretchr/testify/require"
@@ -31,6 +32,29 @@ const (
 // TestVault installs Vault, bootstraps it with secrets, policies, and Kube Auth Method.
 // It then configures Consul to use vault as the backend and checks that it works.
 func TestVault(t *testing.T) {
+	cases := map[string]struct {
+		autoBootstrap bool
+	}{
+		"manual ACL bootstrap": {},
+		"automatic ACL bootstrap": {
+			autoBootstrap: true,
+		},
+	}
+	for name, c := range cases {
+		c := c
+		t.Run(name, func(t *testing.T) {
+			testVault(t, c.autoBootstrap)
+		})
+	}
+}
+
+// testVault is the implementation for TestVault:
+//
+//   - testAutoBootstrap = false. Test when ACL bootstrapping has already occurred.
+//     The test pre-populates a Vault secret with the bootstrap token.
+//   - testAutoBootstrap = true. Test that server-acl-init automatically ACL bootstraps
+//     consul and writes the bootstrap token to Vault.
+func testVault(t *testing.T, testAutoBootstrap bool) {
 	cfg := suite.Config()
 	ctx := suite.Environment().DefaultContext(t)
 	kubectlOptions := ctx.KubectlOptions(t)
@@ -123,16 +147,22 @@ func TestVault(t *testing.T) {
 		licenseSecret.SaveSecretAndAddReadPolicy(t, vaultClient)
 	}
 
-	// Bootstrap Token
-	bootstrapToken, err := uuid.GenerateUUID()
-	require.NoError(t, err)
 	bootstrapTokenSecret := &vault.KV2Secret{
 		Path:       "consul/data/secret/bootstrap",
 		Key:        "token",
-		Value:      bootstrapToken,
+		Value:      "",
 		PolicyName: "bootstrap",
 	}
-	bootstrapTokenSecret.SaveSecretAndAddReadPolicy(t, vaultClient)
+	if testAutoBootstrap {
+		bootstrapTokenSecret.SaveSecretAndAddUpdatePolicy(t, vaultClient)
+	} else {
+		id, err := uuid.GenerateUUID()
+		require.NoError(t, err)
+		bootstrapTokenSecret.Value = id
+		bootstrapTokenSecret.SaveSecretAndAddReadPolicy(t, vaultClient)
+	}
+
+	bootstrapToken := bootstrapTokenSecret.Value
 
 	// -------------------------
 	// Additional Auth Roles
@@ -265,6 +295,26 @@ func TestVault(t *testing.T) {
 	logger.Logf(t, "Wait %d seconds for certificates to rotate....", expirationInSeconds)
 	time.Sleep(time.Duration(expirationInSeconds) * time.Second)
 
+	if testAutoBootstrap {
+		logger.Logf(t, "Validating the ACL bootstrap token was stored in Vault.")
+		timer := &retry.Timer{Timeout: 10 * time.Second, Wait: 1 * time.Second}
+		retry.RunWith(timer, t, func(r *retry.R) {
+			secret, err := vaultClient.Logical().Read("consul/data/secret/bootstrap")
+			require.NoError(r, err)
+
+			data, ok := secret.Data["data"].(map[string]interface{})
+			require.True(r, ok)
+			require.NotNil(r, data)
+
+			tok, ok := data["token"].(string)
+			require.True(r, ok)
+			require.NotEmpty(r, tok)
+
+			// Set bootstrapToken for subsequent validations.
+			bootstrapToken = tok
+		})
+	}
+
 	// Validate that the gossip encryption key is set correctly.
 	logger.Log(t, "Validating the gossip key has been set correctly.")
 	consulCluster.ACLToken = bootstrapToken

charts/consul/templates/server-acl-init-job.yaml

@@ -47,14 +47,23 @@ spec:
       annotations:
         "consul.hashicorp.com/connect-inject": "false"
         {{- if .Values.global.secretsBackend.vault.enabled }}
-        "vault.hashicorp.com/agent-pre-populate-only": "true"
+
+        {{- /* Run the Vault agent as both an init container and sidecar.
+        The Vault agent sidecar is needed when server-acl-init bootstraps ACLs
+        and writes the bootstrap token back to Vault.
+          * agent-pre-populate: true - Run the Vault agent init container.
+          * agent-pre-populate-only: false - Also, run the Vault agent sidecar.
+          * agent-cache-enable: true - Enable the Agent cache listener.
+          * agent-cache-listener-port: 8200 - (optional) Listen on 127.0.0.1:8200.
+          * agent-enable-quit: true - Enable a "quit" endpoint. server-acl-init
+            tells the Vault agent to stop (without this the Job will not complete).
+        */}}
+        "vault.hashicorp.com/agent-pre-populate": "true"
+        "vault.hashicorp.com/agent-pre-populate-only": "false"
+        "vault.hashicorp.com/agent-cache-enable": "true"
+        "vault.hashicorp.com/agent-cache-listener-port": "8200"
+        "vault.hashicorp.com/agent-enable-quit": "true"
         "vault.hashicorp.com/agent-inject": "true"
-        {{- if .Values.global.acls.bootstrapToken.secretName }}
-        {{- with .Values.global.acls.bootstrapToken }}
-        "vault.hashicorp.com/agent-inject-secret-bootstrap-token": "{{ .secretName }}"
-        "vault.hashicorp.com/agent-inject-template-bootstrap-token": {{ template "consul.vaultSecretTemplate" . }}
-        {{- end }}
-        {{- end }}
         {{- if .Values.global.acls.partitionToken.secretName }}
         {{- with .Values.global.acls.partitionToken }}
         "vault.hashicorp.com/agent-inject-secret-partition-token": "{{ .secretName }}"
@@ -101,14 +110,7 @@ spec:
             path: tls.crt
       {{- end }}
       {{- end }}
-      {{- if (and .Values.global.acls.bootstrapToken.secretName (not .Values.global.secretsBackend.vault.enabled)) }}
-      - name: bootstrap-token
-        secret:
-          secretName: {{ .Values.global.acls.bootstrapToken.secretName }}
-          items:
-          - key: {{ .Values.global.acls.bootstrapToken.secretKey }}
-            path: bootstrap-token
-      {{- else if and .Values.global.acls.replicationToken.secretName (not .Values.global.secretsBackend.vault.enabled) }}
+      {{- if and .Values.global.acls.replicationToken.secretName (not .Values.global.secretsBackend.vault.enabled) }}
       - name: acl-replication-token
         secret:
           secretName: {{ .Values.global.acls.replicationToken.secretName }}
@@ -129,6 +131,13 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
+        # Extract the Vault namespace from the Vault agent annotations.
+        {{- if .Values.global.secretsBackend.vault.enabled }}
+        {{- if .Values.global.secretsBackend.vault.agentAnnotations }}
+        - name: VAULT_NAMESPACE
+          value: {{ get (tpl .Values.global.secretsBackend.vault.agentAnnotations . | fromYaml) "vault.hashicorp.com/namespace" }}
+        {{- end }}
+        {{- end }}
         {{- include "consul.consulK8sConsulServerEnvVars" . | nindent 8 }}
         {{- if (or .Values.global.tls.enabled .Values.global.acls.replicationToken.secretName .Values.global.acls.bootstrapToken.secretName) }}
         volumeMounts:
@@ -139,11 +148,7 @@ spec:
           readOnly: true
         {{- end }}
         {{- end }}
-        {{- if (and .Values.global.acls.bootstrapToken.secretName (not .Values.global.secretsBackend.vault.enabled)) }}
-        - name: bootstrap-token
-          mountPath: /consul/acl/tokens
-          readOnly: true
-        {{- else if and .Values.global.acls.replicationToken.secretName (not .Values.global.secretsBackend.vault.enabled) }}
+        {{- if and .Values.global.acls.replicationToken.secretName (not .Values.global.secretsBackend.vault.enabled) }}
         - name: acl-replication-token
           mountPath: /consul/acl/tokens
           readOnly: true
@@ -161,13 +166,15 @@ spec:
             -resource-prefix=${CONSUL_FULLNAME} \
             -k8s-namespace={{ .Release.Namespace }} \
             -set-server-tokens={{ $serverEnabled }} \
-
-            {{- if .Values.global.acls.bootstrapToken.secretName }}
             {{- if .Values.global.secretsBackend.vault.enabled }}
-            -bootstrap-token-file=/vault/secrets/bootstrap-token \
+            -secrets-backend=vault \
             {{- else }}
-            -bootstrap-token-file=/consul/acl/tokens/bootstrap-token \
+            -secrets-backend=kubernetes \
             {{- end }}
+
+            {{- if .Values.global.acls.bootstrapToken.secretName }}
+            -bootstrap-token-secret-name={{ .Values.global.acls.bootstrapToken.secretName }} \
+            -bootstrap-token-secret-key={{ .Values.global.acls.bootstrapToken.secretKey }} \
             {{- end }}
 
             {{- if .Values.syncCatalog.enabled }}