Support Vault server running with TLS (#874)

* Change vault cluster in acceptance tests to only run with TLS. All tests will run against vault with TLS because that is the use case we think will be the most valuable for users to test
* Support adding Vault CA as a secret to pods that will be using vault agent. We need to add two annotations to pods:
      * vault.hashicorp.com/agent-extra-secret with the value of the vault CA secret name. The secret will be mounted to vault agent at /vault/custom path. See docs here
      * vault.hashicorp.com/ca-cert - with the path of the ca file inside the vault agent container. This should be /vault/custom/<secret key>
* Most pods will only need those annotations. The server pods also need the Vault CA secret to be mounted as a volume because it needs the CA to be on the file system for the vault connect CA provider.

acceptance/framework/helpers/helpers.go

@@ -77,7 +77,7 @@ func CheckForPriorInstallations(t *testing.T, client kubernetes.Interface, optio
 func WaitForAllPodsToBeReady(t *testing.T, client kubernetes.Interface, namespace, podLabelSelector string) {
 	t.Helper()
 
-	logger.Log(t, "Waiting for pods to be ready.")
+	logger.Logf(t, "Waiting for pods with label %q to be ready.", podLabelSelector)
 
 	// Wait up to 15m.
 	// On Azure, volume provisioning can sometimes take close to 5 min,
@@ -101,7 +101,7 @@ func WaitForAllPodsToBeReady(t *testing.T, client kubernetes.Interface, namespac
 	logger.Log(t, "Finished waiting for pods to be ready.")
 }
 
-// Sets up a goroutine that will wait for interrupt signals
+// SetupInterruptHandler sets up a goroutine that will wait for interrupt signals
 // and call cleanup function when it catches it.
 func SetupInterruptHandler(cleanup func()) {
 	c := make(chan os.Signal, 1)

acceptance/framework/vault/vault_cluster.go

@@ -1,6 +1,7 @@
 package vault
 
 import (
+	"context"
 	"fmt"
 	"testing"
 	"time"
@@ -13,55 +14,45 @@ import (
 	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
+	"github.com/hashicorp/consul-k8s/control-plane/helper/cert"
 	"github.com/hashicorp/consul/sdk/testutil/retry"
 	vapi "github.com/hashicorp/vault/api"
 	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 )
 
 const (
-	vaultNS        = "default"
-	vaultPodLabel  = "app.kubernetes.io/instance="
-	vaultRootToken = "abcd1234"
+	releaseLabel = "app.kubernetes.io/instance="
 )
 
-// VaultCluster
+// VaultCluster represents a vault installation.
 type VaultCluster struct {
-	ctx       environment.TestContext
-	namespace string
+	ctx environment.TestContext
 
-	vaultHelmOptions *helm.Options
-	vaultReleaseName string
-	vaultClient      *vapi.Client
+	helmOptions *helm.Options
+	releaseName string
+	vaultClient *vapi.Client
 
 	kubectlOptions *terratestk8s.KubectlOptions
-	values         map[string]string
 
 	kubernetesClient kubernetes.Interface
-	kubeConfig       string
-	kubeContext      string
 
 	noCleanupOnFailure bool
 	debugDirectory     string
 	logger             terratestLogger.TestLogger
 }
 
 // NewVaultCluster creates a VaultCluster which will be used to install Vault using Helm.
-func NewVaultCluster(
-	t *testing.T,
-	helmValues map[string]string,
-	ctx environment.TestContext,
-	cfg *config.TestConfig,
-	releaseName string,
-) *VaultCluster {
+func NewVaultCluster(t *testing.T, ctx environment.TestContext, cfg *config.TestConfig, releaseName string) *VaultCluster {
 
 	logger := terratestLogger.New(logger.TestLogger{})
 
 	kopts := ctx.KubectlOptions(t)
-	kopts.Namespace = vaultNS
 
 	vaultHelmOpts := &helm.Options{
-		SetValues:      defaultVaultValues(),
+		SetValues:      defaultHelmValues(releaseName),
 		KubectlOptions: kopts,
 		Logger:         logger,
 	}
@@ -75,35 +66,34 @@ func NewVaultCluster(
 
 	return &VaultCluster{
 		ctx:                ctx,
-		vaultHelmOptions:   vaultHelmOpts,
+		helmOptions:        vaultHelmOpts,
 		kubectlOptions:     kopts,
-		namespace:          cfg.KubeNamespace,
-		values:             helmValues,
 		kubernetesClient:   ctx.KubernetesClient(t),
-		kubeConfig:         cfg.Kubeconfig,
-		kubeContext:        cfg.KubeContext,
 		noCleanupOnFailure: cfg.NoCleanupOnFailure,
 		debugDirectory:     cfg.DebugDirectory,
 		logger:             logger,
-		vaultReleaseName:   releaseName,
+		releaseName:        releaseName,
 	}
 }
 
 // VaultClient returns the vault client.
 func (v *VaultCluster) VaultClient(t *testing.T) *vapi.Client { return v.vaultClient }
 
-// Setup Vault Client returns a Vault Client.
-// TODO: We need to support Vault with TLS enabled.
+// SetupVaultClient sets up and returns a Vault Client.
 func (v *VaultCluster) SetupVaultClient(t *testing.T) *vapi.Client {
 	t.Helper()
 
+	if v.vaultClient != nil {
+		return v.vaultClient
+	}
+
 	config := vapi.DefaultConfig()
 	localPort := terratestk8s.GetAvailablePort(t)
 	remotePort := 8200 // use non-secure by default
 
-	serverPod := fmt.Sprintf("%s-vault-0", v.vaultReleaseName)
+	serverPod := fmt.Sprintf("%s-vault-0", v.releaseName)
 	tunnel := terratestk8s.NewTunnelWithLogger(
-		v.vaultHelmOptions.KubectlOptions,
+		v.helmOptions.KubectlOptions,
 		terratestk8s.ResourceTypePod,
 		serverPod,
 		localPort,
@@ -122,7 +112,10 @@ func (v *VaultCluster) SetupVaultClient(t *testing.T) *vapi.Client {
 		tunnel.Close()
 	})
 
-	config.Address = fmt.Sprintf("http://127.0.0.1:%d", localPort)
+	config.Address = fmt.Sprintf("https://127.0.0.1:%d", localPort)
+	// We don't need to verify TLS for localhost traffic.
+	err := config.ConfigureTLS(&vapi.TLSConfig{Insecure: true})
+	require.NoError(t, err)
 	vaultClient, err := vapi.NewClient(config)
 	require.NoError(t, err)
 	return vaultClient
@@ -132,7 +125,6 @@ func (v *VaultCluster) SetupVaultClient(t *testing.T) *vapi.Client {
 func (v *VaultCluster) bootstrap(t *testing.T, ctx environment.TestContext) {
 
 	v.vaultClient = v.SetupVaultClient(t)
-	v.vaultClient.SetToken(vaultRootToken)
 
 	// Enable the KV-V2 Secrets engine.
 	err := v.vaultClient.Sys().Mount("consul", &vapi.MountInput{
@@ -163,18 +155,28 @@ func (v *VaultCluster) bootstrap(t *testing.T, ctx environment.TestContext) {
 
 	// Enable Kube Auth.
 	err = v.vaultClient.Sys().EnableAuthWithOptions("kubernetes", &vapi.EnableAuthOptions{
-		Type:   "kubernetes",
-		Config: vapi.MountConfigInput{},
+		Type: "kubernetes",
 	})
 	if err != nil {
 		t.Fatal("unable to enable kube auth", "err", err)
 	}
-	// We need to kubectl exec this one on the vault server:
-	// This is taken from https://learn.hashicorp.com/tutorials/vault/kubernetes-google-cloud-gke?in=vault/kubernetes#configure-kubernetes-authentication
-	cmdString := fmt.Sprintf("VAULT_TOKEN=%s vault write auth/kubernetes/config disable_iss_validation=\"true\" token_reviewer_jwt=\"$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\" kubernetes_host=\"https://${KUBERNETES_PORT_443_TCP_ADDR}:443\" kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", vaultRootToken)
 
 	v.logger.Logf(t, "updating vault kube auth config")
-	k8s.RunKubectl(t, ctx.KubectlOptions(t), "exec", "-i", fmt.Sprintf("%s-vault-0", v.vaultReleaseName), "--", "sh", "-c", cmdString)
+
+	// To configure the auth method, we need to read the token and the ca cert from the Vault's server
+	// service account token.
+	namespace := v.helmOptions.KubectlOptions.Namespace
+	sa, err := v.kubernetesClient.CoreV1().ServiceAccounts(namespace).Get(context.Background(), fmt.Sprintf("%s-vault", v.releaseName), metav1.GetOptions{})
+	require.NoError(t, err)
+	require.Len(t, sa.Secrets, 1)
+	tokenSecret, err := v.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), sa.Secrets[0].Name, metav1.GetOptions{})
+	require.NoError(t, err)
+	_, err = v.vaultClient.Logical().Write("auth/kubernetes/config", map[string]interface{}{
+		"token_reviewer_jwt": tokenSecret.StringData["token"],
+		"kubernetes_ca_cert": tokenSecret.StringData["ca.crt"],
+		"kubernetes_host":    "https://kubernetes.default.svc",
+	})
+	require.NoError(t, err)
 }
 
 // Create installs Vault via Helm and then calls bootstrap to initialize it.
@@ -188,35 +190,190 @@ func (v *VaultCluster) Create(t *testing.T, ctx environment.TestContext) {
 	})
 
 	// Fail if there are any existing installations of the Helm chart.
-	helpers.CheckForPriorInstallations(t, v.kubernetesClient, v.vaultHelmOptions, "", fmt.Sprintf("%s=%s", vaultPodLabel, v.vaultReleaseName))
+	helpers.CheckForPriorInstallations(t, v.kubernetesClient, v.helmOptions, "", v.releaseLabelSelector())
+
+	v.createTLSCerts(t)
 
 	// Install Vault.
-	helm.Install(t, v.vaultHelmOptions, "hashicorp/vault", v.vaultReleaseName)
+	helm.Install(t, v.helmOptions, "hashicorp/vault", v.releaseName)
+
+	v.initAndUnseal(t)
+
 	// Wait for the injector and vault server pods to become Ready.
-	helpers.WaitForAllPodsToBeReady(t, v.kubernetesClient, v.vaultHelmOptions.KubectlOptions.Namespace, fmt.Sprintf("%s=%s", vaultPodLabel, v.vaultReleaseName))
-	// Now call bootstrap()
+	helpers.WaitForAllPodsToBeReady(t, v.kubernetesClient, v.helmOptions.KubectlOptions.Namespace, v.releaseLabelSelector())
+
+	// Now call bootstrap().
 	v.bootstrap(t, ctx)
 }
 
 // Destroy issues a helm delete and deletes the PVC + any helm secrets related to the release that are leftover.
 func (v *VaultCluster) Destroy(t *testing.T) {
 	t.Helper()
 
-	k8s.WritePodsDebugInfoIfFailed(t, v.kubectlOptions, v.debugDirectory, "release="+v.vaultReleaseName)
+	k8s.WritePodsDebugInfoIfFailed(t, v.kubectlOptions, v.debugDirectory, v.releaseLabelSelector())
 	// Ignore the error returned by the helm delete here so that we can
 	// always idempotently clean up resources in the cluster.
-	_ = helm.DeleteE(t, v.vaultHelmOptions, v.vaultReleaseName, true)
-	// We do not need to do any PVC deletion in vault dev mode.
+	_ = helm.DeleteE(t, v.helmOptions, v.releaseName, true)
+
+	err := v.kubernetesClient.CoreV1().PersistentVolumeClaims(v.helmOptions.KubectlOptions.Namespace).DeleteCollection(context.Background(),
+		metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: v.releaseLabelSelector()})
+	require.NoError(t, err)
 }
 
-func defaultVaultValues() map[string]string {
+func defaultHelmValues(releaseName string) map[string]string {
+	certSecret := certSecretName(releaseName)
+	caSecret := CASecretName(releaseName)
+
+	serverConfig := fmt.Sprintf(`
+      listener "tcp" {
+        address = "[::]:8200"
+        cluster_address = "[::]:8201"
+        tls_cert_file = "/vault/userconfig/%s/tls.crt"
+        tls_key_file  = "/vault/userconfig/%s/tls.key"
+        tls_client_ca_file = "/vault/userconfig/%s/tls.crt"
+      }
+
+      storage "file" {
+        path = "/vault/data"
+      }`, certSecret, certSecret, caSecret)
+
 	return map[string]string{
-		"server.replicas":         "1",
-		"server.dev.enabled":      "true",
-		"server.dev.devRootToken": vaultRootToken,
-		"ui.enabled":              "true",
-		"server.bootstrapExpect":  "1",
-		"injector.enabled":        "true",
-		"global.enabled":          "true",
+		"global.tlsDisable":                        "false",
+		"server.extraEnvironmentVars.VAULT_CACERT": fmt.Sprintf("/vault/userconfig/%s/tls.crt", caSecret),
+		"server.extraVolumes[0].name":              caSecret,
+		"server.extraVolumes[0].type":              "secret",
+		"server.extraVolumes[1].name":              certSecret,
+		"server.extraVolumes[1].type":              "secret",
+		"server.standalone.enabled":                "true",
+		"server.standalone.config":                 serverConfig,
+		"injector.enabled":                         "true",
 	}
 }
+
+// certSecretName returns the Kubernetes secret name of the certificate and key
+// for the Vault server.
+func certSecretName(releaseName string) string {
+	return fmt.Sprintf("%s-vault-server-tls", releaseName)
+}
+
+// CASecretName returns the Kubernetes secret name of the CA for the Vault server.
+func CASecretName(releaseName string) string {
+	return fmt.Sprintf("%s-vault-ca", releaseName)
+}
+
+// Address is the in-cluster API address of the Vault server.
+func (v *VaultCluster) Address() string {
+	return fmt.Sprintf("https://%s-vault:8200", v.releaseName)
+}
+
+// releaseLabelSelector returns label selector that selects all pods
+// from a Vault installation.
+func (v *VaultCluster) releaseLabelSelector() string {
+	return fmt.Sprintf("%s=%s", releaseLabel, v.releaseName)
+}
+
+// createTLSCerts generates a self-signed CA and uses it to generate
+// certificate and key  for the Vault server. It then saves those as
+// Kubernetes secrets.
+func (v *VaultCluster) createTLSCerts(t *testing.T) {
+	v.logger.Logf(t, "generating Vault TLS certificates")
+
+	namespace := v.helmOptions.KubectlOptions.Namespace
+
+	// Generate CA and cert and create secrets for them.
+	signer, _, caPem, caCertTmpl, err := cert.GenerateCA("Vault CA")
+	require.NoError(t, err)
+	vaultService := fmt.Sprintf("%s-vault", v.releaseName)
+	certSANs := []string{
+		vaultService,
+		fmt.Sprintf("%s.default", vaultService),
+		fmt.Sprintf("%s.default.svc", vaultService),
+	}
+	certPem, keyPem, err := cert.GenerateCert("Vault server", 24*time.Hour, caCertTmpl, signer, certSANs)
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		if !v.noCleanupOnFailure {
+			// We're ignoring error here because secret deletion is best-effort.
+			_ = v.kubernetesClient.CoreV1().Secrets(namespace).Delete(context.Background(), certSecretName(v.releaseName), metav1.DeleteOptions{})
+			_ = v.kubernetesClient.CoreV1().Secrets(namespace).Delete(context.Background(), CASecretName(v.releaseName), metav1.DeleteOptions{})
+		}
+	})
+
+	_, err = v.kubernetesClient.CoreV1().Secrets(namespace).Create(context.Background(), &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespace,
+			Name:      certSecretName(v.releaseName),
+		},
+		Data: map[string][]byte{
+			corev1.TLSCertKey:       []byte(certPem),
+			corev1.TLSPrivateKeyKey: []byte(keyPem),
+		},
+		Type: corev1.SecretTypeTLS,
+	}, metav1.CreateOptions{})
+	require.NoError(t, err)
+
+	_, err = v.kubernetesClient.CoreV1().Secrets(namespace).Create(context.Background(), &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      CASecretName(v.releaseName),
+			Namespace: namespace,
+		},
+		Data: map[string][]byte{
+			corev1.TLSCertKey: []byte(caPem),
+		},
+		Type: corev1.SecretTypeOpaque,
+	}, metav1.CreateOptions{})
+	require.NoError(t, err)
+}
+
+// initAndUnseal initializes and unseals Vault.
+// Once initialized, it saves the Vault root token into a Kubernetes secret.
+func (v *VaultCluster) initAndUnseal(t *testing.T) {
+	v.logger.Logf(t, "initializing and unsealing Vault")
+
+	namespace := v.helmOptions.KubectlOptions.Namespace
+	retrier := &retry.Timer{Timeout: 2 * time.Minute, Wait: 1 * time.Second}
+	retry.RunWith(retrier, t, func(r *retry.R) {
+		// Wait for vault server pod to be running so that we can create Vault client without errors.
+		serverPod, err := v.kubernetesClient.CoreV1().Pods(namespace).Get(context.Background(), fmt.Sprintf("%s-vault-0", v.releaseName), metav1.GetOptions{})
+		require.NoError(r, err)
+		require.Equal(r, serverPod.Status.Phase, corev1.PodRunning)
+
+		// Set up the client so that we can make API calls to initialize and unseal.
+		v.vaultClient = v.SetupVaultClient(t)
+
+		// Initialize Vault with 1 secret share. We don't need to
+		// more key shares for this test installation.
+		initResp, err := v.vaultClient.Sys().Init(&vapi.InitRequest{
+			SecretShares:    1,
+			SecretThreshold: 1,
+		})
+		require.NoError(r, err)
+		v.vaultClient.SetToken(initResp.RootToken)
+
+		// Unseal Vault with the unseal key we got when initialized it.
+		// There should be one unseal key since we're only using one secret share.
+		_, err = v.vaultClient.Sys().Unseal(initResp.KeysB64[0])
+		require.NoError(r, err)
+	})
+
+	v.logger.Logf(t, "successfully initialized and unsealed Vault")
+
+	rootTokenSecret := fmt.Sprintf("%s-vault-root-token", v.releaseName)
+	v.logger.Logf(t, "saving Vault root token to %q Kubernetes secret", rootTokenSecret)
+
+	helpers.Cleanup(t, v.noCleanupOnFailure, func() {
+		_ = v.kubernetesClient.CoreV1().Secrets(namespace).Delete(context.Background(), rootTokenSecret, metav1.DeleteOptions{})
+	})
+	_, err := v.kubernetesClient.CoreV1().Secrets(namespace).Create(context.Background(), &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      rootTokenSecret,
+			Namespace: namespace,
+		},
+		Data: map[string][]byte{
+			"token": []byte(v.vaultClient.Token()),
+		},
+		Type: corev1.SecretTypeOpaque,
+	}, metav1.CreateOptions{})
+	require.NoError(t, err)
+}