From 854ae71d6d2dfc718327abb42c8cc2d17190fe0e Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Tue, 1 Aug 2023 22:07:27 +0100 Subject: [PATCH 01/12] Support setting VAULT_CACERT_BYTES for sidecars --- Makefile | 16 +++++++----- agent-inject/agent/agent.go | 7 ++++- agent-inject/agent/container_env.go | 16 ++++++++++++ agent-inject/handler.go | 2 ++ subcommand/injector/command.go | 2 ++ subcommand/injector/flags.go | 9 +++++++ subcommand/injector/flags_test.go | 1 + test/cert-manager/ca.yaml | 31 ++++++++++++++++++++++ test/cert-manager/vault-cert.yaml | 20 ++++++++++++++ test/vault/dev.values.yaml | 10 +++++++ test/vault/vault-tls-dev.values.yaml | 39 ++++++++++++++++++++++++++++ 11 files changed, 146 insertions(+), 7 deletions(-) create mode 100644 test/cert-manager/ca.yaml create mode 100644 test/cert-manager/vault-cert.yaml create mode 100644 test/vault/dev.values.yaml create mode 100644 test/vault/vault-tls-dev.values.yaml diff --git a/Makefile b/Makefile index b2bf19b8..f6ec9745 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ TESTARGS ?= '-test.v' HELM_CHART_VERSION ?= 0.25.0 -.PHONY: all test build image clean version deploy exercise teardown +.PHONY: all test build image clean version deploy exercise teardown install-cert-manager all: build version: @@ -37,12 +37,10 @@ deploy: image kind load docker-image hashicorp/vault-k8s:$(VERSION) helm upgrade --install vault vault --repo https://helm.releases.hashicorp.com --version=$(HELM_CHART_VERSION) \ --wait --timeout=5m \ - --set 'server.dev.enabled=true' \ - --set 'server.logLevel=debug' \ + --values=test/vault/dev.values.yaml \ + --values=test/vault/vault-tls-dev.values.yaml \ --set 'injector.image.tag=$(VERSION)' \ - --set 'injector.image.pullPolicy=Never' \ - --set 'injector.affinity=null' \ - --set 'injector.annotations.deployed=unix-$(shell date +%s)' + --set "injector.extraEnvironmentVars.AGENT_INJECT_VAULT_CACERT_BYTES=$(shell kubectl get secret vault-cert -o=jsonpath="{.data.ca\.crt}")" # Populates the Vault dev server with a secret, configures kubernetes auth, and # deploys an nginx pod with annotations to have the secret injected. @@ -66,6 +64,12 @@ exercise: kubectl wait --for=condition=Ready --timeout=5m pod nginx kubectl exec nginx -c nginx -- cat /vault/secrets/secret.txt +install-cert-manager: + helm upgrade --install cert-manager cert-manager --repo https://charts.jetstack.io \ + --set installCRDs=true \ + --wait=true --timeout=5m + kubectl apply -f 'test/cert-manager/*' + # Teardown any resources created in deploy and exercise targets. teardown: helm uninstall vault || true diff --git a/agent-inject/agent/agent.go b/agent-inject/agent/agent.go index 99ffb04a..128e8fd9 100644 --- a/agent-inject/agent/agent.go +++ b/agent-inject/agent/agent.go @@ -246,9 +246,14 @@ type Vault struct { AuthConfig map[string]interface{} // CACert is the name of the Certificate Authority certificate - // to use when validating Vault's server certificates. + // to use when validating Vault's server certificates. It takes + // precedence over CACertBytes. CACert string + // CACertBytes is the contents of the CA certificate to trust + // for TLS with Vault as a PEM-encoded certificate or bundle. + CACertBytes string + // CAKey is the name of the Certificate Authority key // to use when validating Vault's server certificates. CAKey string diff --git a/agent-inject/agent/container_env.go b/agent-inject/agent/container_env.go index 7498bec2..7317b3a0 100644 --- a/agent-inject/agent/container_env.go +++ b/agent-inject/agent/container_env.go @@ -139,6 +139,13 @@ func (a *Agent) ContainerEnvVars(init bool) ([]corev1.EnvVar, error) { } } + if a.Vault.CACertBytes != "" { + envs = append(envs, corev1.EnvVar{ + Name: "VAULT_CACERT_BYTES", + Value: decodeIfBase64(a.Vault.CACertBytes), + }) + } + // Add IRSA AWS Env variables for vault containers if a.Vault.AuthType == "aws" { envMap := a.getAwsEnvsFromContainer(a.Pod) @@ -160,3 +167,12 @@ func (a *Agent) ContainerEnvVars(init bool) ([]corev1.EnvVar, error) { return envs, nil } + +func decodeIfBase64(s string) string { + decoded, err := base64.StdEncoding.DecodeString(s) + if err == nil { + return string(decoded) + } + + return s +} diff --git a/agent-inject/handler.go b/agent-inject/handler.go index dbc7d35b..12dde6b3 100644 --- a/agent-inject/handler.go +++ b/agent-inject/handler.go @@ -49,6 +49,7 @@ type Handler struct { // If this is false, injection is default. RequireAnnotation bool VaultAddress string + VaultCACertBytes string VaultAuthType string VaultAuthPath string VaultNamespace string @@ -226,6 +227,7 @@ func (h *Handler) Mutate(req *admissionv1.AdmissionRequest) *admissionv1.Admissi err := fmt.Errorf("error creating new agent sidecar: %s", err) return admissionError(req.UID, err) } + agentSidecar.Vault.CACertBytes = h.VaultCACertBytes h.Log.Debug("validating agent configuration..") err = agentSidecar.Validate() diff --git a/subcommand/injector/command.go b/subcommand/injector/command.go index 9f577a01..e5a019fd 100644 --- a/subcommand/injector/command.go +++ b/subcommand/injector/command.go @@ -54,6 +54,7 @@ type Command struct { flagAutoName string // MutatingWebhookConfiguration for updating flagAutoHosts string // SANs for the auto-generated TLS cert. flagVaultService string // Name of the Vault service + flagVaultCACertBytes string // CA Cert to trust for TLS with Vault. flagProxyAddress string // HTTP proxy address used to talk to the Vault service flagVaultImage string // Name of the Vault Image to use flagVaultAuthType string // Type of Vault Auth Method to use @@ -194,6 +195,7 @@ func (c *Command) Run(args []string) int { // Build the HTTP handler and server injector := agentInject.Handler{ VaultAddress: c.flagVaultService, + VaultCACertBytes: c.flagVaultCACertBytes, VaultAuthType: c.flagVaultAuthType, VaultAuthPath: c.flagVaultAuthPath, VaultNamespace: c.flagVaultNamespace, diff --git a/subcommand/injector/flags.go b/subcommand/injector/flags.go index 225449da..52b86090 100644 --- a/subcommand/injector/flags.go +++ b/subcommand/injector/flags.go @@ -60,6 +60,10 @@ type Specification struct { // VaultAddr is the AGENT_INJECT_VAULT_ADDR environment variable. VaultAddr string `split_words:"true"` + // VaultCACertBytes is the AGENT_INJECT_VAULT_CACERT_BYTES environment variable. + // Specifies the CA cert to trust for TLS with Vault. + VaultCACertBytes string `envconfig:"AGENT_INJECT_VAULT_CACERT_BYTES"` + // ProxyAddr is the AGENT_INJECT_PROXY_ADDR environment variable. ProxyAddr string `split_words:"true"` @@ -159,6 +163,8 @@ func (c *Command) init() { fmt.Sprintf("Docker image for Vault. Defaults to %q.", agent.DefaultVaultImage)) c.flagSet.StringVar(&c.flagVaultService, "vault-address", "", "Address of the Vault server.") + c.flagSet.StringVar(&c.flagVaultCACertBytes, "vault-cacert-bytes", "", + "CA certificate to trust for TLS with Vault, specified as a PEM-encoded certificate or bundle.") c.flagSet.StringVar(&c.flagProxyAddress, "proxy-address", "", "HTTP proxy address used to talk to the Vault service.") c.flagSet.StringVar(&c.flagVaultAuthType, "vault-auth-type", agent.DefaultVaultAuthType, @@ -295,6 +301,9 @@ func (c *Command) parseEnvs() error { if envs.VaultAddr != "" { c.flagVaultService = envs.VaultAddr } + if envs.VaultCACertBytes != "" { + c.flagVaultCACertBytes = envs.VaultCACertBytes + } if envs.ProxyAddr != "" { c.flagProxyAddress = envs.ProxyAddr diff --git a/subcommand/injector/flags_test.go b/subcommand/injector/flags_test.go index e5d155b1..7de8b04c 100644 --- a/subcommand/injector/flags_test.go +++ b/subcommand/injector/flags_test.go @@ -116,6 +116,7 @@ func TestCommandEnvs(t *testing.T) { }{ {env: "AGENT_INJECT_LISTEN", value: ":8080", cmdPtr: &cmd.flagListen}, {env: "AGENT_INJECT_VAULT_ADDR", value: "http://vault:8200", cmdPtr: &cmd.flagVaultService}, + {env: "AGENT_INJECT_VAULT_CACERT_BYTES", value: "foo", cmdPtr: &cmd.flagVaultCACertBytes}, {env: "AGENT_INJECT_PROXY_ADDR", value: "http://proxy:3128", cmdPtr: &cmd.flagProxyAddress}, {env: "AGENT_INJECT_VAULT_AUTH_PATH", value: "auth-path-test", cmdPtr: &cmd.flagVaultAuthPath}, {env: "AGENT_INJECT_VAULT_IMAGE", value: "hashicorp/vault:1.13.3", cmdPtr: &cmd.flagVaultImage}, diff --git a/test/cert-manager/ca.yaml b/test/cert-manager/ca.yaml new file mode 100644 index 00000000..4900a151 --- /dev/null +++ b/test/cert-manager/ca.yaml @@ -0,0 +1,31 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: vault-ca +spec: + isCA: true + commonName: Vault CA + secretName: vault-ca + duration: 87660h # 10 years + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: selfsigned + kind: Issuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: vault-ca-issuer +spec: + ca: + secretName: vault-ca \ No newline at end of file diff --git a/test/cert-manager/vault-cert.yaml b/test/cert-manager/vault-cert.yaml new file mode 100644 index 00000000..d116de27 --- /dev/null +++ b/test/cert-manager/vault-cert.yaml @@ -0,0 +1,20 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: vault-certificate +spec: + secretName: vault-cert + duration: 24h + renewBefore: 144m # roughly 10% of 24h + dnsNames: + - vault + - vault.default + - vault.default.svc + - vault-internal + - vault-internal.default + - vault-internal.default.svc + ipAddresses: + - "127.0.0.1" + issuerRef: + name: vault-ca-issuer + commonName: Vault Server \ No newline at end of file diff --git a/test/vault/dev.values.yaml b/test/vault/dev.values.yaml new file mode 100644 index 00000000..460300f0 --- /dev/null +++ b/test/vault/dev.values.yaml @@ -0,0 +1,10 @@ +injector: + image: + pullPolicy: Never + affinity: null + annotations: + deployed: unix-{{ now | unix-epoch }} +server: + dev: + enabled: true + logLevel: debug diff --git a/test/vault/vault-tls-dev.values.yaml b/test/vault/vault-tls-dev.values.yaml new file mode 100644 index 00000000..cfb282bb --- /dev/null +++ b/test/vault/vault-tls-dev.values.yaml @@ -0,0 +1,39 @@ +global: + tlsDisable: false +server: + # Move the default TLS-disabled dev listener out of the way so we can add our + # own listener on 8200 that does use TLS. + extraArgs: "-dev-listen-address=127.0.0.1:8202 -config=/etc/config/listener.hcl" + extraEnvironmentVars: + VAULT_CACERT: /etc/tls/ca.crt + volumeMounts: + - name: cert + mountPath: /etc/tls + readOnly: true + - name: config + mountPath: /etc/config + readOnly: true + volumes: + - name: cert + secret: + secretName: vault-cert + - name: config + emptyDir: {} + extraInitContainers: + - name: write-config + image: "alpine" + command: [sh, -c] + args: + - | + cat < /etc/config/listener.hcl + listener "tcp" { + address = "[::]:8200" + tls_cert_file = "/etc/tls/tls.crt" + tls_key_file = "/etc/tls/tls.key" + proxy_protocol_behavior = "allow_authorized" + proxy_protocol_authorized_addrs = "[::]:8200" + } + EOF + volumeMounts: + - name: config + mountPath: /etc/config From 0584c89dff360866ea62e7ddef15c5f493cf2b52 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Wed, 2 Aug 2023 13:45:16 +0100 Subject: [PATCH 02/12] Workaround for consul-template not supporting VAULT_CACERT_BYTES --- Makefile | 28 ++++++++++++++------ agent-inject/agent/container_env.go | 8 ++++++ agent-inject/agent/container_init_sidecar.go | 8 +++++- agent-inject/agent/container_sidecar.go | 10 +++++++ agent-inject/agent/container_volume.go | 1 + test/vault/dev.values.yaml | 2 -- 6 files changed, 46 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index f6ec9745..5ec46c57 100644 --- a/Makefile +++ b/Makefile @@ -14,9 +14,19 @@ PKG=github.com/hashicorp/vault-k8s/version LDFLAGS?="-X '$(PKG).Version=v$(VERSION)'" TESTARGS ?= '-test.v' -HELM_CHART_VERSION ?= 0.25.0 +VAULT_TLS?=false +VAULT_HELM_CHART_VERSION ?= 0.25.0 +VAULT_HELM_FLAGS?=--repo https://helm.releases.hashicorp.com --version=$(VAULT_HELM_CHART_VERSION) \ + --wait --timeout=5m \ + --values=test/vault/dev.values.yaml \ + --set 'injector.image.tag=$(VERSION)' -.PHONY: all test build image clean version deploy exercise teardown install-cert-manager +ifeq ($(VAULT_TLS), true) + VAULT_HELM_FLAGS += --values=test/vault/vault-tls-dev.values.yaml \ + --set "injector.extraEnvironmentVars.AGENT_INJECT_VAULT_CACERT_BYTES=$(shell kubectl get secret vault-cert -o=jsonpath="{.data.ca\.crt}")" +endif + +.PHONY: all test build image clean version deploy deploy-tls exercise teardown install-cert-manager all: build version: @@ -35,12 +45,12 @@ image: build # Run multiple times to deploy new builds of the injector. deploy: image kind load docker-image hashicorp/vault-k8s:$(VERSION) - helm upgrade --install vault vault --repo https://helm.releases.hashicorp.com --version=$(HELM_CHART_VERSION) \ - --wait --timeout=5m \ - --values=test/vault/dev.values.yaml \ - --values=test/vault/vault-tls-dev.values.yaml \ - --set 'injector.image.tag=$(VERSION)' \ - --set "injector.extraEnvironmentVars.AGENT_INJECT_VAULT_CACERT_BYTES=$(shell kubectl get secret vault-cert -o=jsonpath="{.data.ca\.crt}")" + helm upgrade --install vault vault $(VAULT_HELM_FLAGS) + kubectl delete pod -l "app.kubernetes.io/instance=vault" + kubectl wait --for=condition=Ready --timeout=5m pod -l "app.kubernetes.io/instance=vault" + +deploy-tls: install-cert-manager + VAULT_TLS=true make deploy # Populates the Vault dev server with a secret, configures kubernetes auth, and # deploys an nginx pod with annotations to have the secret injected. @@ -69,10 +79,12 @@ install-cert-manager: --set installCRDs=true \ --wait=true --timeout=5m kubectl apply -f 'test/cert-manager/*' + kubectl wait --for=condition=Ready --timeout=5m certificate vault-certificate # Teardown any resources created in deploy and exercise targets. teardown: helm uninstall vault || true + helm uninstall cert-manager || true kubectl delete --ignore-not-found serviceaccount test-app-sa kubectl delete --ignore-not-found pod nginx diff --git a/agent-inject/agent/container_env.go b/agent-inject/agent/container_env.go index 7317b3a0..fb147512 100644 --- a/agent-inject/agent/container_env.go +++ b/agent-inject/agent/container_env.go @@ -5,6 +5,7 @@ package agent import ( "encoding/base64" + "path" "strconv" corev1 "k8s.io/api/core/v1" @@ -144,6 +145,13 @@ func (a *Agent) ContainerEnvVars(init bool) ([]corev1.EnvVar, error) { Name: "VAULT_CACERT_BYTES", Value: decodeIfBase64(a.Vault.CACertBytes), }) + // TODO(tomhjp): Remove when consul-template supports VAULT_CACERT_BYTES + if a.Vault.CACert == "" { + envs = append(envs, corev1.EnvVar{ + Name: "VAULT_CACERT", + Value: path.Join(tokenVolumePath, caFileName), + }) + } } // Add IRSA AWS Env variables for vault containers diff --git a/agent-inject/agent/container_init_sidecar.go b/agent-inject/agent/container_init_sidecar.go index 82f1a2c3..33c95ffe 100644 --- a/agent-inject/agent/container_init_sidecar.go +++ b/agent-inject/agent/container_init_sidecar.go @@ -7,7 +7,7 @@ import ( "encoding/json" "fmt" - "github.com/evanphx/json-patch" + jsonpatch "github.com/evanphx/json-patch" corev1 "k8s.io/api/core/v1" ) @@ -67,6 +67,12 @@ func (a *Agent) ContainerInitSidecar() (corev1.Container, error) { MountPath: tlsSecretVolumePath, ReadOnly: true, }) + } else if a.Vault.CACert == "" && a.Vault.CACertBytes != "" { + // TODO(tomhjp): Remove when consul-template supports VAULT_CACERT_BYTES. + // consul-template does not yet support VAULT_CACERT_BYTES, so we write + // it out to a file and set VAULT_CACERT as well to ensure templating + // picks up the CA. + arg = prependWriteCAToFile(arg) } if a.VaultAgentCache.Persist { diff --git a/agent-inject/agent/container_sidecar.go b/agent-inject/agent/container_sidecar.go index c89645e0..6fd745bb 100644 --- a/agent-inject/agent/container_sidecar.go +++ b/agent-inject/agent/container_sidecar.go @@ -79,6 +79,12 @@ func (a *Agent) ContainerSidecar() (corev1.Container, error) { MountPath: tlsSecretVolumePath, ReadOnly: true, }) + } else if a.Vault.CACert == "" && a.Vault.CACertBytes != "" { + // TODO(tomhjp): Remove when consul-template supports VAULT_CACERT_BYTES. + // consul-template does not yet support VAULT_CACERT_BYTES, so we write + // it out to a file and set VAULT_CACERT as well to ensure templating + // picks up the CA. + arg = prependWriteCAToFile(arg) } if a.VaultAgentCache.Persist { @@ -136,6 +142,10 @@ func (a *Agent) ContainerSidecar() (corev1.Container, error) { return newContainer, nil } +func prependWriteCAToFile(arg string) string { + return fmt.Sprintf(`printf "%%s" "${VAULT_CACERT_BYTES}" > %s/%s && %s`, tokenVolumePath, caFileName, arg) +} + // Valid resource notations: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu func (a *Agent) parseResources() (corev1.ResourceRequirements, error) { resources := corev1.ResourceRequirements{} diff --git a/agent-inject/agent/container_volume.go b/agent-inject/agent/container_volume.go index 59a0afbf..404166ad 100644 --- a/agent-inject/agent/container_volume.go +++ b/agent-inject/agent/container_volume.go @@ -14,6 +14,7 @@ const ( tokenVolumeNameInit = "home-init" tokenVolumeNameSidecar = "home-sidecar" tokenVolumePath = "/home/vault" + caFileName = "ca.crt" configVolumeName = "vault-config" configVolumePath = "/vault/configs" secretVolumeName = "vault-secrets" diff --git a/test/vault/dev.values.yaml b/test/vault/dev.values.yaml index 460300f0..e4598989 100644 --- a/test/vault/dev.values.yaml +++ b/test/vault/dev.values.yaml @@ -2,8 +2,6 @@ injector: image: pullPolicy: Never affinity: null - annotations: - deployed: unix-{{ now | unix-epoch }} server: dev: enabled: true From d50e787e0e4cb65f43ef44bb8498d69585a2ea95 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Tue, 15 Aug 2023 13:23:35 +0100 Subject: [PATCH 03/12] Add comments about optional base64 encoding --- subcommand/injector/flags.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/subcommand/injector/flags.go b/subcommand/injector/flags.go index 52b86090..5b809f50 100644 --- a/subcommand/injector/flags.go +++ b/subcommand/injector/flags.go @@ -61,7 +61,9 @@ type Specification struct { VaultAddr string `split_words:"true"` // VaultCACertBytes is the AGENT_INJECT_VAULT_CACERT_BYTES environment variable. - // Specifies the CA cert to trust for TLS with Vault. + // Specifies the CA cert to trust for TLS with Vault as a PEM-encoded + // certificate or bundle. The multi-line PEM contents may optionally be base64 + // encoded to avoid line breaks. VaultCACertBytes string `envconfig:"AGENT_INJECT_VAULT_CACERT_BYTES"` // ProxyAddr is the AGENT_INJECT_PROXY_ADDR environment variable. @@ -164,7 +166,8 @@ func (c *Command) init() { c.flagSet.StringVar(&c.flagVaultService, "vault-address", "", "Address of the Vault server.") c.flagSet.StringVar(&c.flagVaultCACertBytes, "vault-cacert-bytes", "", - "CA certificate to trust for TLS with Vault, specified as a PEM-encoded certificate or bundle.") + "CA certificate to trust for TLS with Vault, specified as a PEM-encoded certificate or bundle. "+ + "The multi-line PEM contents may optionally be base64 encoded to avoid line breaks.") c.flagSet.StringVar(&c.flagProxyAddress, "proxy-address", "", "HTTP proxy address used to talk to the Vault service.") c.flagSet.StringVar(&c.flagVaultAuthType, "vault-auth-type", agent.DefaultVaultAuthType, From c66100d8bdac573d3d444c29c70d74aa88eba809 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Tue, 15 Aug 2023 13:24:43 +0100 Subject: [PATCH 04/12] One more comment --- agent-inject/agent/agent.go | 1 + 1 file changed, 1 insertion(+) diff --git a/agent-inject/agent/agent.go b/agent-inject/agent/agent.go index 128e8fd9..a293e770 100644 --- a/agent-inject/agent/agent.go +++ b/agent-inject/agent/agent.go @@ -252,6 +252,7 @@ type Vault struct { // CACertBytes is the contents of the CA certificate to trust // for TLS with Vault as a PEM-encoded certificate or bundle. + // Can also be base64 encoded PEM contents. CACertBytes string // CAKey is the name of the Certificate Authority key From b28d518c7314fa9d6e939394572cdcd9d8af20a2 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Tue, 15 Aug 2023 12:59:50 +0100 Subject: [PATCH 05/12] Use Vault's -dev-tls flag instead --- .gitignore | 5 +++- Makefile | 36 ++++++++++++------------- test/cert-manager/ca.yaml | 31 ---------------------- test/cert-manager/vault-cert.yaml | 20 -------------- test/vault/dev.values.yaml | 5 ++++ test/vault/vault-tls-dev.values.yaml | 39 ---------------------------- 6 files changed, 26 insertions(+), 110 deletions(-) delete mode 100644 test/cert-manager/ca.yaml delete mode 100644 test/cert-manager/vault-cert.yaml delete mode 100644 test/vault/vault-tls-dev.values.yaml diff --git a/.gitignore b/.gitignore index 1e0d9f9b..986d1c81 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,7 @@ # Output directory for binaries built in CircleCI /pkg -/dist/ \ No newline at end of file +/dist/ + +# make deploy output +ca.crt \ No newline at end of file diff --git a/Makefile b/Makefile index 5ec46c57..1b2652ae 100644 --- a/Makefile +++ b/Makefile @@ -21,12 +21,7 @@ VAULT_HELM_FLAGS?=--repo https://helm.releases.hashicorp.com --version=$(VAULT_H --values=test/vault/dev.values.yaml \ --set 'injector.image.tag=$(VERSION)' -ifeq ($(VAULT_TLS), true) - VAULT_HELM_FLAGS += --values=test/vault/vault-tls-dev.values.yaml \ - --set "injector.extraEnvironmentVars.AGENT_INJECT_VAULT_CACERT_BYTES=$(shell kubectl get secret vault-cert -o=jsonpath="{.data.ca\.crt}")" -endif - -.PHONY: all test build image clean version deploy deploy-tls exercise teardown install-cert-manager +.PHONY: all test build image clean version deploy exercise teardown all: build version: @@ -41,16 +36,26 @@ build: image: build docker build --build-arg VERSION=$(VERSION) --no-cache -t $(IMAGE_TAG) . +.PHONY: secret +secret: + kubectl exec vault-0 -- cat /tmp/vault-ca.pem + kubectl get secret vault-ca -o=jsonpath="{.data.ca\.crt}" | base64 -d + kubectl get pod -l "app.kubernetes.io/name=vault-agent-injector" -o=jsonpath='{.items[0].spec.containers[0].env[?(@.name == "AGENT_INJECT_VAULT_CACERT_BYTES")].value}' | base64 -d + # Deploys Vault dev server and a locally built Agent Injector. # Run multiple times to deploy new builds of the injector. -deploy: image +deploy: kind load docker-image hashicorp/vault-k8s:$(VERSION) - helm upgrade --install vault vault $(VAULT_HELM_FLAGS) + helm upgrade --install vault vault $(VAULT_HELM_FLAGS) \ + --set "injector.enabled=false" kubectl delete pod -l "app.kubernetes.io/instance=vault" kubectl wait --for=condition=Ready --timeout=5m pod -l "app.kubernetes.io/instance=vault" - -deploy-tls: install-cert-manager - VAULT_TLS=true make deploy + kubectl delete secret --ignore-not-found vault-ca + kubectl exec vault-0 -- cat /tmp/vault-ca.pem > test/vault/ca.crt + kubectl create secret generic vault-ca --from-file=test/vault/ca.crt + helm upgrade --install vault vault $(VAULT_HELM_FLAGS) \ + --set "injector.enabled=true" \ + --set "injector.extraEnvironmentVars.AGENT_INJECT_VAULT_CACERT_BYTES=$$(kubectl get secret vault-ca -o=jsonpath="{.data.ca\.crt}")" # Populates the Vault dev server with a secret, configures kubernetes auth, and # deploys an nginx pod with annotations to have the secret injected. @@ -74,19 +79,12 @@ exercise: kubectl wait --for=condition=Ready --timeout=5m pod nginx kubectl exec nginx -c nginx -- cat /vault/secrets/secret.txt -install-cert-manager: - helm upgrade --install cert-manager cert-manager --repo https://charts.jetstack.io \ - --set installCRDs=true \ - --wait=true --timeout=5m - kubectl apply -f 'test/cert-manager/*' - kubectl wait --for=condition=Ready --timeout=5m certificate vault-certificate - # Teardown any resources created in deploy and exercise targets. teardown: helm uninstall vault || true - helm uninstall cert-manager || true kubectl delete --ignore-not-found serviceaccount test-app-sa kubectl delete --ignore-not-found pod nginx + kubectl delete --ignore-not-found secret vault-ca clean: -rm -rf $(BUILD_DIR) diff --git a/test/cert-manager/ca.yaml b/test/cert-manager/ca.yaml deleted file mode 100644 index 4900a151..00000000 --- a/test/cert-manager/ca.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: selfsigned -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: vault-ca -spec: - isCA: true - commonName: Vault CA - secretName: vault-ca - duration: 87660h # 10 years - privateKey: - algorithm: ECDSA - size: 256 - issuerRef: - name: selfsigned - kind: Issuer - group: cert-manager.io ---- -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: vault-ca-issuer -spec: - ca: - secretName: vault-ca \ No newline at end of file diff --git a/test/cert-manager/vault-cert.yaml b/test/cert-manager/vault-cert.yaml deleted file mode 100644 index d116de27..00000000 --- a/test/cert-manager/vault-cert.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: vault-certificate -spec: - secretName: vault-cert - duration: 24h - renewBefore: 144m # roughly 10% of 24h - dnsNames: - - vault - - vault.default - - vault.default.svc - - vault-internal - - vault-internal.default - - vault-internal.default.svc - ipAddresses: - - "127.0.0.1" - issuerRef: - name: vault-ca-issuer - commonName: Vault Server \ No newline at end of file diff --git a/test/vault/dev.values.yaml b/test/vault/dev.values.yaml index e4598989..6679deda 100644 --- a/test/vault/dev.values.yaml +++ b/test/vault/dev.values.yaml @@ -1,3 +1,5 @@ +global: + tlsDisable: false injector: image: pullPolicy: Never @@ -6,3 +8,6 @@ server: dev: enabled: true logLevel: debug + extraArgs: "-dev-tls -dev-tls-cert-dir=/tmp" + extraEnvironmentVars: + VAULT_CACERT: /tmp/vault-ca.pem diff --git a/test/vault/vault-tls-dev.values.yaml b/test/vault/vault-tls-dev.values.yaml deleted file mode 100644 index cfb282bb..00000000 --- a/test/vault/vault-tls-dev.values.yaml +++ /dev/null @@ -1,39 +0,0 @@ -global: - tlsDisable: false -server: - # Move the default TLS-disabled dev listener out of the way so we can add our - # own listener on 8200 that does use TLS. - extraArgs: "-dev-listen-address=127.0.0.1:8202 -config=/etc/config/listener.hcl" - extraEnvironmentVars: - VAULT_CACERT: /etc/tls/ca.crt - volumeMounts: - - name: cert - mountPath: /etc/tls - readOnly: true - - name: config - mountPath: /etc/config - readOnly: true - volumes: - - name: cert - secret: - secretName: vault-cert - - name: config - emptyDir: {} - extraInitContainers: - - name: write-config - image: "alpine" - command: [sh, -c] - args: - - | - cat < /etc/config/listener.hcl - listener "tcp" { - address = "[::]:8200" - tls_cert_file = "/etc/tls/tls.crt" - tls_key_file = "/etc/tls/tls.key" - proxy_protocol_behavior = "allow_authorized" - proxy_protocol_authorized_addrs = "[::]:8200" - } - EOF - volumeMounts: - - name: config - mountPath: /etc/config From 89a252ba4c81e22ffbcf42faea536b6131250001 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Mon, 23 Oct 2023 15:15:05 +0100 Subject: [PATCH 06/12] Backout hacks and use -dev-tls-san option from Vault 1.15.0 --- agent-inject/agent/container_env.go | 8 -------- agent-inject/agent/container_init_sidecar.go | 6 ------ agent-inject/agent/container_sidecar.go | 10 ---------- subcommand/injector/command.go | 1 - test/vault/dev.values.yaml | 9 ++++++++- 5 files changed, 8 insertions(+), 26 deletions(-) diff --git a/agent-inject/agent/container_env.go b/agent-inject/agent/container_env.go index fb147512..7317b3a0 100644 --- a/agent-inject/agent/container_env.go +++ b/agent-inject/agent/container_env.go @@ -5,7 +5,6 @@ package agent import ( "encoding/base64" - "path" "strconv" corev1 "k8s.io/api/core/v1" @@ -145,13 +144,6 @@ func (a *Agent) ContainerEnvVars(init bool) ([]corev1.EnvVar, error) { Name: "VAULT_CACERT_BYTES", Value: decodeIfBase64(a.Vault.CACertBytes), }) - // TODO(tomhjp): Remove when consul-template supports VAULT_CACERT_BYTES - if a.Vault.CACert == "" { - envs = append(envs, corev1.EnvVar{ - Name: "VAULT_CACERT", - Value: path.Join(tokenVolumePath, caFileName), - }) - } } // Add IRSA AWS Env variables for vault containers diff --git a/agent-inject/agent/container_init_sidecar.go b/agent-inject/agent/container_init_sidecar.go index 33c95ffe..2e36d229 100644 --- a/agent-inject/agent/container_init_sidecar.go +++ b/agent-inject/agent/container_init_sidecar.go @@ -67,12 +67,6 @@ func (a *Agent) ContainerInitSidecar() (corev1.Container, error) { MountPath: tlsSecretVolumePath, ReadOnly: true, }) - } else if a.Vault.CACert == "" && a.Vault.CACertBytes != "" { - // TODO(tomhjp): Remove when consul-template supports VAULT_CACERT_BYTES. - // consul-template does not yet support VAULT_CACERT_BYTES, so we write - // it out to a file and set VAULT_CACERT as well to ensure templating - // picks up the CA. - arg = prependWriteCAToFile(arg) } if a.VaultAgentCache.Persist { diff --git a/agent-inject/agent/container_sidecar.go b/agent-inject/agent/container_sidecar.go index 6fd745bb..c89645e0 100644 --- a/agent-inject/agent/container_sidecar.go +++ b/agent-inject/agent/container_sidecar.go @@ -79,12 +79,6 @@ func (a *Agent) ContainerSidecar() (corev1.Container, error) { MountPath: tlsSecretVolumePath, ReadOnly: true, }) - } else if a.Vault.CACert == "" && a.Vault.CACertBytes != "" { - // TODO(tomhjp): Remove when consul-template supports VAULT_CACERT_BYTES. - // consul-template does not yet support VAULT_CACERT_BYTES, so we write - // it out to a file and set VAULT_CACERT as well to ensure templating - // picks up the CA. - arg = prependWriteCAToFile(arg) } if a.VaultAgentCache.Persist { @@ -142,10 +136,6 @@ func (a *Agent) ContainerSidecar() (corev1.Container, error) { return newContainer, nil } -func prependWriteCAToFile(arg string) string { - return fmt.Sprintf(`printf "%%s" "${VAULT_CACERT_BYTES}" > %s/%s && %s`, tokenVolumePath, caFileName, arg) -} - // Valid resource notations: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu func (a *Agent) parseResources() (corev1.ResourceRequirements, error) { resources := corev1.ResourceRequirements{} diff --git a/subcommand/injector/command.go b/subcommand/injector/command.go index e5a019fd..1529b618 100644 --- a/subcommand/injector/command.go +++ b/subcommand/injector/command.go @@ -88,7 +88,6 @@ type Command struct { cert atomic.Value } -// TODO Add flag for Vault TLS func (c *Command) Run(args []string) int { ctx, cancelFunc := context.WithCancel(context.Background()) defer cancelFunc() diff --git a/test/vault/dev.values.yaml b/test/vault/dev.values.yaml index 6679deda..770b7208 100644 --- a/test/vault/dev.values.yaml +++ b/test/vault/dev.values.yaml @@ -8,6 +8,13 @@ server: dev: enabled: true logLevel: debug - extraArgs: "-dev-tls -dev-tls-cert-dir=/tmp" + # >- to convert to a single line with no line breaks. + extraArgs: >- + -dev-tls + -dev-tls-cert-dir=/tmp + -dev-tls-san=vault.default.svc.cluster.local + -dev-tls-san=vault.default.svc + -dev-tls-san=vault.default + -dev-tls-san=vault extraEnvironmentVars: VAULT_CACERT: /tmp/vault-ca.pem From 8287054887c8c6461c78ebc22da129345e490067 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Mon, 23 Oct 2023 15:25:30 +0100 Subject: [PATCH 07/12] backout debug Makefile changes --- .gitignore | 2 +- Makefile | 9 +-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 986d1c81..3d41da85 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,4 @@ /dist/ # make deploy output -ca.crt \ No newline at end of file +test/vault/ca.crt diff --git a/Makefile b/Makefile index 1b2652ae..0a516af2 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,6 @@ PKG=github.com/hashicorp/vault-k8s/version LDFLAGS?="-X '$(PKG).Version=v$(VERSION)'" TESTARGS ?= '-test.v' -VAULT_TLS?=false VAULT_HELM_CHART_VERSION ?= 0.25.0 VAULT_HELM_FLAGS?=--repo https://helm.releases.hashicorp.com --version=$(VAULT_HELM_CHART_VERSION) \ --wait --timeout=5m \ @@ -36,15 +35,9 @@ build: image: build docker build --build-arg VERSION=$(VERSION) --no-cache -t $(IMAGE_TAG) . -.PHONY: secret -secret: - kubectl exec vault-0 -- cat /tmp/vault-ca.pem - kubectl get secret vault-ca -o=jsonpath="{.data.ca\.crt}" | base64 -d - kubectl get pod -l "app.kubernetes.io/name=vault-agent-injector" -o=jsonpath='{.items[0].spec.containers[0].env[?(@.name == "AGENT_INJECT_VAULT_CACERT_BYTES")].value}' | base64 -d - # Deploys Vault dev server and a locally built Agent Injector. # Run multiple times to deploy new builds of the injector. -deploy: +deploy: image kind load docker-image hashicorp/vault-k8s:$(VERSION) helm upgrade --install vault vault $(VAULT_HELM_FLAGS) \ --set "injector.enabled=false" From 45c63a021dfd2df2af0c152adeb6f36a92c0915e Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Mon, 23 Oct 2023 15:55:39 +0100 Subject: [PATCH 08/12] Simplify CA sharing and pin to 1.15.0 until next release --- .gitignore | 3 --- Makefile | 7 +------ test/vault/dev.values.yaml | 4 ++++ 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 3d41da85..da8dc13d 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,3 @@ # Output directory for binaries built in CircleCI /pkg /dist/ - -# make deploy output -test/vault/ca.crt diff --git a/Makefile b/Makefile index 0a516af2..ec76ffb0 100644 --- a/Makefile +++ b/Makefile @@ -43,12 +43,8 @@ deploy: image --set "injector.enabled=false" kubectl delete pod -l "app.kubernetes.io/instance=vault" kubectl wait --for=condition=Ready --timeout=5m pod -l "app.kubernetes.io/instance=vault" - kubectl delete secret --ignore-not-found vault-ca - kubectl exec vault-0 -- cat /tmp/vault-ca.pem > test/vault/ca.crt - kubectl create secret generic vault-ca --from-file=test/vault/ca.crt helm upgrade --install vault vault $(VAULT_HELM_FLAGS) \ - --set "injector.enabled=true" \ - --set "injector.extraEnvironmentVars.AGENT_INJECT_VAULT_CACERT_BYTES=$$(kubectl get secret vault-ca -o=jsonpath="{.data.ca\.crt}")" + --set "injector.extraEnvironmentVars.AGENT_INJECT_VAULT_CACERT_BYTES=$$(kubectl exec vault-0 -- cat /tmp/vault-ca.pem | base64)" # Populates the Vault dev server with a secret, configures kubernetes auth, and # deploys an nginx pod with annotations to have the secret injected. @@ -77,7 +73,6 @@ teardown: helm uninstall vault || true kubectl delete --ignore-not-found serviceaccount test-app-sa kubectl delete --ignore-not-found pod nginx - kubectl delete --ignore-not-found secret vault-ca clean: -rm -rf $(BUILD_DIR) diff --git a/test/vault/dev.values.yaml b/test/vault/dev.values.yaml index 770b7208..35c449cd 100644 --- a/test/vault/dev.values.yaml +++ b/test/vault/dev.values.yaml @@ -4,7 +4,11 @@ injector: image: pullPolicy: Never affinity: null + agentImage: + tag: 1.15.0 server: + image: + tag: 1.15.0 dev: enabled: true logLevel: debug From da6722bf559c6989a820cf1f589111ead3e43844 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Mon, 23 Oct 2023 16:00:10 +0100 Subject: [PATCH 09/12] Delete unused variable --- agent-inject/agent/container_volume.go | 1 - 1 file changed, 1 deletion(-) diff --git a/agent-inject/agent/container_volume.go b/agent-inject/agent/container_volume.go index 404166ad..59a0afbf 100644 --- a/agent-inject/agent/container_volume.go +++ b/agent-inject/agent/container_volume.go @@ -14,7 +14,6 @@ const ( tokenVolumeNameInit = "home-init" tokenVolumeNameSidecar = "home-sidecar" tokenVolumePath = "/home/vault" - caFileName = "ca.crt" configVolumeName = "vault-config" configVolumePath = "/vault/configs" secretVolumeName = "vault-secrets" From f2b80fa1bf39a474bf3d317124dad2841afbf487 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Mon, 23 Oct 2023 16:07:42 +0100 Subject: [PATCH 10/12] Use 'make deploy exercise teardown' in CI to ensure it continues to work --- .github/workflows/tests.yaml | 10 ++++++++++ Makefile | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 620d95a3..1da5147e 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -51,6 +51,11 @@ jobs: - run: pip install yq shell: bash + # Checkout this repo. + - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + path: "vault-k8s" + # Checkout vault-helm for acceptance test code. - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 with: @@ -75,7 +80,12 @@ jobs: docker image load --input vault-k8s-image.docker.tar kind load docker-image hashicorp/vault-k8s:0.0.0-dev + - name: Makefile tests + working-directory: vault-k8s + run: make deploy exercise teardown + - name: bats tests + working-directory: vault-helm run: | yq --in-place --yaml-roundtrip '.injector.image.tag |= "0.0.0-dev"' ./vault-helm/values.yaml bats ./vault-helm/test/acceptance -t --filter injector diff --git a/Makefile b/Makefile index ec76ffb0..e1a65ac9 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ image: build # Deploys Vault dev server and a locally built Agent Injector. # Run multiple times to deploy new builds of the injector. -deploy: image +deploy: kind load docker-image hashicorp/vault-k8s:$(VERSION) helm upgrade --install vault vault $(VAULT_HELM_FLAGS) \ --set "injector.enabled=false" From ca52e6f61ee3482f56d88f7140f7c6d9e4568754 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Tue, 24 Oct 2023 10:41:32 +0100 Subject: [PATCH 11/12] Add changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac64ec74..45cbbf6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,9 @@ Changes: * `k8s.io/client-go` v0.27.4 => v0.28.2 * `github.com/hashicorp/vault/sdk` v0.9.2 => v0.10.2 +Improvements: +* Injector can set CA certificate for injected pods via `AGENT_INJECT_VAULT_CACERT_BYTES` env var or `-vault-cacert-bytes` flag [GH-507](https://github.com/hashicorp/vault-k8s/pull/507) + ## 1.3.0 (August 16, 2023) Improvements: From 93ed4d9eab792b1190035cfc1e2ce6e2f400df21 Mon Sep 17 00:00:00 2001 From: Tom Proctor Date: Tue, 24 Oct 2023 10:54:06 +0100 Subject: [PATCH 12/12] Fix relative directories in CI --- .github/workflows/tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 1da5147e..78206fee 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -87,5 +87,5 @@ jobs: - name: bats tests working-directory: vault-helm run: | - yq --in-place --yaml-roundtrip '.injector.image.tag |= "0.0.0-dev"' ./vault-helm/values.yaml - bats ./vault-helm/test/acceptance -t --filter injector + yq --in-place --yaml-roundtrip '.injector.image.tag |= "0.0.0-dev"' ./values.yaml + bats ./test/acceptance -t --filter injector