diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a1dbf082..823fb5640 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Features: * server: New `extraPorts` option for adding ports to the Vault server statefulset [GH-841](https://github.com/hashicorp/vault-helm/pull/841) * server: Add configurable Port Number in readinessProbe and livenessProbe for the server-statefulset [GH-831](https://github.com/hashicorp/vault-helm/pull/831) * injector: Make livenessProbe and readinessProbe configurable and add configurable startupProbe [GH-852](https://github.com/hashicorp/vault-helm/pull/852) +* csi: Add an Agent sidecar to Vault CSI Provider pods to provide lease caching and renewals [GH-749](https://github.com/hashicorp/vault-helm/pull/749) ## 0.23.0 (November 28th, 2022) diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl index dcfcbb8b8..4b6baf10e 100644 --- a/templates/_helpers.tpl +++ b/templates/_helpers.tpl @@ -778,6 +778,16 @@ Sets the container resources if the user has set any. {{ end }} {{- end -}} +{{/* +Sets the container resources for CSI's Agent sidecar if the user has set any. +*/}} +{{- define "csi.agent.resources" -}} + {{- if .Values.csi.agent.resources -}} + resources: +{{ toYaml .Values.csi.agent.resources | indent 12}} + {{ end }} +{{- end -}} + {{/* Sets extra CSI daemonset annotations */}} diff --git a/templates/csi-agent-configmap.yaml b/templates/csi-agent-configmap.yaml new file mode 100644 index 000000000..cb373f833 --- /dev/null +++ b/templates/csi-agent-configmap.yaml @@ -0,0 +1,29 @@ +{{- template "vault.csiEnabled" . -}} +{{- if and (.csiEnabled) (eq (.Values.csi.agent.enabled | toString) "true") -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "vault.fullname" . }}-csi-provider-agent-config + namespace: {{ .Release.Namespace }} + labels: + helm.sh/chart: {{ include "vault.chart" . }} + app.kubernetes.io/name: {{ include "vault.name" . }}-csi-provider + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +data: + config.hcl: | + vault { + {{- if .Values.global.externalVaultAddr }} + "address" = "{{ .Values.global.externalVaultAddr }}" + {{- else }} + "address" = "{{ include "vault.scheme" . }}://{{ template "vault.fullname" . }}.{{ .Release.Namespace }}.svc:{{ .Values.server.service.port }}" + {{- end }} + } + + cache {} + + listener "unix" { + address = "/var/run/vault/agent.sock" + tls_disable = true + } +{{- end }} diff --git a/templates/csi-daemonset.yaml b/templates/csi-daemonset.yaml index e38cc47d5..0285a0cbb 100644 --- a/templates/csi-daemonset.yaml +++ b/templates/csi-daemonset.yaml @@ -55,11 +55,13 @@ spec: - --endpoint=/provider/vault.sock - --debug={{ .Values.csi.debug }} {{- if .Values.csi.extraArgs }} - {{- toYaml .Values.csi.extraArgs | nindent 12 }} + {{- toYaml .Values.csi.extraArgs | nindent 12 }} {{- end }} env: - name: VAULT_ADDR - {{- if .Values.global.externalVaultAddr }} + {{- if eq (.Values.csi.agent.enabled | toString) "true" }} + value: "unix:///var/run/vault/agent.sock" + {{- else if .Values.global.externalVaultAddr }} value: "{{ .Values.global.externalVaultAddr }}" {{- else }} value: {{ include "vault.scheme" . }}://{{ template "vault.fullname" . }}.{{ .Release.Namespace }}.svc:{{ .Values.server.service.port }} @@ -67,9 +69,10 @@ spec: volumeMounts: - name: providervol mountPath: "/provider" - - name: mountpoint-dir - mountPath: {{ .Values.csi.daemonSet.kubeletRootDir }}/pods - mountPropagation: HostToContainer + {{- if eq (.Values.csi.agent.enabled | toString) "true" }} + - name: agent-unix-socket + mountPath: /var/run/vault + {{- end }} {{- if .Values.csi.volumeMounts }} {{- toYaml .Values.csi.volumeMounts | nindent 12}} {{- end }} @@ -91,15 +94,57 @@ spec: periodSeconds: {{ .Values.csi.readinessProbe.periodSeconds }} successThreshold: {{ .Values.csi.readinessProbe.successThreshold }} timeoutSeconds: {{ .Values.csi.readinessProbe.timeoutSeconds }} + {{- if eq (.Values.csi.agent.enabled | toString) "true" }} + - name: {{ include "vault.name" . }}-agent + image: "{{ .Values.csi.agent.image.repository }}:{{ .Values.csi.agent.image.tag }}" + imagePullPolicy: {{ .Values.csi.agent.image.pullPolicy }} + {{ template "csi.agent.resources" . }} + command: + - vault + args: + - agent + - -config=/etc/vault/config.hcl + {{- if .Values.csi.agent.extraArgs }} + {{- toYaml .Values.csi.agent.extraArgs | nindent 12 }} + {{- end }} + ports: + - containerPort: 8200 + env: + - name: VAULT_LOG_LEVEL + value: "{{ .Values.csi.agent.logLevel }}" + - name: VAULT_LOG_FORMAT + value: "{{ .Values.csi.agent.logFormat }}" + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsUser: 100 + runAsGroup: 1000 + volumeMounts: + - name: agent-config + mountPath: /etc/vault/config.hcl + subPath: config.hcl + readOnly: true + - name: agent-unix-socket + mountPath: /var/run/vault + {{- if .Values.csi.volumeMounts }} + {{- toYaml .Values.csi.volumeMounts | nindent 12 }} + {{- end }} + {{- end }} volumes: - name: providervol hostPath: path: {{ .Values.csi.daemonSet.providersDir }} - - name: mountpoint-dir - hostPath: - path: {{ .Values.csi.daemonSet.kubeletRootDir }}/pods - {{- if .Values.csi.volumes }} - {{- toYaml .Values.csi.volumes | nindent 8}} - {{- end }} + {{- if eq (.Values.csi.agent.enabled | toString) "true" }} + - name: agent-config + configMap: + name: {{ template "vault.fullname" . }}-csi-provider-agent-config + - name: agent-unix-socket + emptyDir: + medium: Memory + {{- end }} + {{- if .Values.csi.volumes }} + {{- toYaml .Values.csi.volumes | nindent 8}} + {{- end }} {{- include "imagePullSecrets" . | nindent 6 }} {{- end }} diff --git a/test/acceptance/csi-test/vault-kv-secretproviderclass.yaml b/test/acceptance/csi-test/vault-kv-secretproviderclass.yaml index b9470fe29..d52fab1bc 100644 --- a/test/acceptance/csi-test/vault-kv-secretproviderclass.yaml +++ b/test/acceptance/csi-test/vault-kv-secretproviderclass.yaml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: MPL-2.0 # The "Hello World" Vault SecretProviderClass -apiVersion: secrets-store.csi.x-k8s.io/v1alpha1 +apiVersion: secrets-store.csi.x-k8s.io/v1 kind: SecretProviderClass metadata: name: vault-kv @@ -10,7 +10,6 @@ spec: provider: vault parameters: roleName: "kv-role" - vaultAddress: http://vault:8200 objects: | - objectName: "bar" secretPath: "secret/data/kv1" diff --git a/test/acceptance/csi.bats b/test/acceptance/csi.bats index ea164f7dd..2d7ba8de9 100644 --- a/test/acceptance/csi.bats +++ b/test/acceptance/csi.bats @@ -9,19 +9,28 @@ load _helpers kubectl create namespace acceptance # Install Secrets Store CSI driver - CSI_DRIVER_VERSION=1.0.0 - helm install secrets-store-csi-driver https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts/secrets-store-csi-driver-${CSI_DRIVER_VERSION}.tgz?raw=true \ + # Configure it to pass in a JWT for the provider to use, and rotate secrets rapidly + # so we can see Agent's cache working. + CSI_DRIVER_VERSION=1.3.2 + helm install secrets-store-csi-driver secrets-store-csi-driver \ + --repo https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts \ + --version=$(CSI_DRIVER_VERSION) \ --wait --timeout=5m \ --namespace=acceptance \ --set linux.image.pullPolicy="IfNotPresent" \ - --set syncSecret.enabled=true + --set tokenRequests[0].audience="vault" \ + --set enableSecretRotation=true \ + --set rotationPollInterval=5s # Install Vault and Vault provider helm install vault \ --wait --timeout=5m \ --namespace=acceptance \ --set="server.dev.enabled=true" \ --set="csi.enabled=true" \ - --set="injector.enabled=false" . + --set="csi.debug=true" \ + --set="csi.agent.logLevel=debug" \ + --set="injector.enabled=false" \ + . kubectl --namespace=acceptance wait --for=condition=Ready --timeout=5m pod -l app.kubernetes.io/name=vault kubectl --namespace=acceptance wait --for=condition=Ready --timeout=5m pod -l app.kubernetes.io/name=vault-csi-provider @@ -29,10 +38,7 @@ load _helpers cat ./test/acceptance/csi-test/vault-policy.hcl | kubectl --namespace=acceptance exec -i vault-0 -- vault policy write kv-policy - kubectl --namespace=acceptance exec vault-0 -- vault auth enable kubernetes kubectl --namespace=acceptance exec vault-0 -- sh -c 'vault write auth/kubernetes/config \ - token_reviewer_jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \ - kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443" \ - kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt \ - disable_iss_validation=true' + kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443"' kubectl --namespace=acceptance exec vault-0 -- vault write auth/kubernetes/role/kv-role \ bound_service_account_names=nginx \ bound_service_account_namespaces=acceptance \ @@ -46,6 +52,22 @@ load _helpers result=$(kubectl --namespace=acceptance exec nginx -- cat /mnt/secrets-store/bar) [[ "$result" == "hello1" ]] + + for i in $(seq 10); do + sleep 2 + if [ "$(kubectl --namespace=acceptance logs --tail=-1 -l "app.kubernetes.io/name=vault-csi-provider" -c vault-agent | grep "returning cached response: path=/v1/auth/kubernetes/login")" ]; then + echo "Agent returned a cached login response" + return + fi + + echo "Waiting for a cached response from Agent..." + done + + # Print the logs and fail the test + echo "Failed to find a log for a cached Agent response" + kubectl --namespace=acceptance logs --tail=-1 -l "app.kubernetes.io/name=vault-csi-provider" -c vault-agent + kubectl --namespace=acceptance logs --tail=-1 -l "app.kubernetes.io/name=vault-csi-provider" -c vault-csi-provider + exit 1 } # Clean up diff --git a/test/unit/csi-agent-configmap.bats b/test/unit/csi-agent-configmap.bats new file mode 100644 index 000000000..4ae4a30b8 --- /dev/null +++ b/test/unit/csi-agent-configmap.bats @@ -0,0 +1,45 @@ +#!/usr/bin/env bats + +load _helpers + +@test "csi/Agent-ConfigMap: disabled by default" { + cd `chart_dir` + local actual=$( (helm template \ + --show-only templates/csi-agent-configmap.yaml \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] +} + +@test "csi/Agent-ConfigMap: name" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/csi-agent-configmap.yaml \ + --set "csi.enabled=true" \ + . | tee /dev/stderr | + yq -r '.metadata.name' | tee /dev/stderr) + [ "${actual}" = "release-name-vault-csi-provider-agent-config" ] +} + +@test "csi/Agent-ConfigMap: Vault addr not affected by injector setting" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/csi-agent-configmap.yaml \ + --set "csi.enabled=true" \ + --release-name not-external-test \ + --set 'injector.externalVaultAddr=http://vault-outside' \ + . | tee /dev/stderr | + yq -r '.data["config.hcl"]' | tee /dev/stderr) + echo "${actual}" | grep "http://not-external-test-vault.default.svc:8200" +} + +@test "csi/Agent-ConfigMap: Vault addr correctly set for externalVaultAddr" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/csi-agent-configmap.yaml \ + --set "csi.enabled=true" \ + --set 'global.externalVaultAddr=http://vault-outside' \ + . | tee /dev/stderr | + yq -r '.data["config.hcl"]' | tee /dev/stderr) + echo "${actual}" | grep "http://vault-outside" +} \ No newline at end of file diff --git a/test/unit/csi-daemonset.bats b/test/unit/csi-daemonset.bats index 0da308b67..76c74b37f 100644 --- a/test/unit/csi-daemonset.bats +++ b/test/unit/csi-daemonset.bats @@ -65,24 +65,32 @@ load _helpers } # Image -@test "csi/daemonset: image is configurable" { +@test "csi/daemonset: images are configurable" { cd `chart_dir` - local actual=$(helm template \ + local object=$(helm template \ --show-only templates/csi-daemonset.yaml \ --set "csi.enabled=true" \ - --set "csi.image.repository=SomeOtherImage" \ + --set "csi.image.repository=Image1" \ --set "csi.image.tag=0.0.1" \ + --set "csi.image.pullPolicy=PullPolicy1" \ + --set "csi.agent.image.repository=Image2" \ + --set "csi.agent.image.tag=0.0.2" \ + --set "csi.agent.image.pullPolicy=PullPolicy2" \ . | tee /dev/stderr | - yq -r '.spec.template.spec.containers[0].image' | tee /dev/stderr) - [ "${actual}" = "SomeOtherImage:0.0.1" ] + yq -r '.spec.template.spec.containers' | tee /dev/stderr) - local actual=$(helm template \ - --show-only templates/csi-daemonset.yaml \ - --set "csi.enabled=true" \ - --set "csi.image.pullPolicy=SomePullPolicy" \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.containers[0].imagePullPolicy' | tee /dev/stderr) - [ "${actual}" = "SomePullPolicy" ] + local actual=$(echo $object | + yq -r '.[0].image' | tee /dev/stderr) + [ "${actual}" = "Image1:0.0.1" ] + local actual=$(echo $object | + yq -r '.[0].imagePullPolicy' | tee /dev/stderr) + [ "${actual}" = "PullPolicy1" ] + local actual=$(echo $object | + yq -r '.[1].image' | tee /dev/stderr) + [ "${actual}" = "Image2:0.0.2" ] + local actual=$(echo $object | + yq -r '.[1].imagePullPolicy' | tee /dev/stderr) + [ "${actual}" = "PullPolicy2" ] } @test "csi/daemonset: Custom imagePullSecrets" { @@ -379,21 +387,6 @@ load _helpers [ "${actual}" = "/etc/kubernetes/secrets-store-csi-providers" ] } -@test "csi/daemonset: csi kubeletRootDir default" { - cd `chart_dir` - - # Test that it defines it - local object=$(helm template \ - --show-only templates/csi-daemonset.yaml \ - --set 'csi.enabled=true' \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.volumes[] | select(.name == "mountpoint-dir")' | tee /dev/stderr) - - local actual=$(echo $object | - yq -r '.hostPath.path' | tee /dev/stderr) - [ "${actual}" = "/var/lib/kubelet/pods" ] -} - @test "csi/daemonset: csi providersDir override " { cd `chart_dir` @@ -410,22 +403,6 @@ load _helpers [ "${actual}" = "/alt/csi-prov-dir" ] } -@test "csi/daemonset: csi kubeletRootDir override" { - cd `chart_dir` - - # Test that it defines it - local object=$(helm template \ - --show-only templates/csi-daemonset.yaml \ - --set 'csi.enabled=true' \ - --set 'csi.daemonSet.kubeletRootDir=/alt/kubelet-root' \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.volumes[] | select(.name == "mountpoint-dir")' | tee /dev/stderr) - - local actual=$(echo $object | - yq -r '.hostPath.path' | tee /dev/stderr) - [ "${actual}" = "/alt/kubelet-root/pods" ] -} - #-------------------------------------------------------------------- # volumeMounts @@ -564,11 +541,39 @@ load _helpers [ "${actual}" = "14" ] } +@test "csi/daemonset: VAULT_ADDR defaults to Agent unix socket" { + cd `chart_dir` + local object=$(helm template \ + --show-only templates/csi-daemonset.yaml \ + --set 'csi.enabled=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr) + + local value=$(echo $object | + yq -r 'map(select(.name=="VAULT_ADDR")) | .[] .value' | tee /dev/stderr) + [ "${value}" = "unix:///var/run/vault/agent.sock" ] +} + +@test "csi/daemonset: VAULT_ADDR remains pointed to Agent unix socket if external Vault" { + cd `chart_dir` + local object=$(helm template \ + --show-only templates/csi-daemonset.yaml \ + --set 'csi.enabled=true' \ + --set 'global.externalVaultAddr=http://vault-outside' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr) + + local value=$(echo $object | + yq -r 'map(select(.name=="VAULT_ADDR")) | .[] .value' | tee /dev/stderr) + [ "${value}" = "unix:///var/run/vault/agent.sock" ] +} + @test "csi/daemonset: with only injector.externalVaultAddr" { cd `chart_dir` local object=$(helm template \ --show-only templates/csi-daemonset.yaml \ --set 'csi.enabled=true' \ + --set 'csi.agent.enabled=false' \ --release-name not-external-test \ --set 'injector.externalVaultAddr=http://vault-outside' \ . | tee /dev/stderr | @@ -584,6 +589,7 @@ load _helpers local object=$(helm template \ --show-only templates/csi-daemonset.yaml \ --set 'csi.enabled=true' \ + --set 'csi.agent.enabled=false' \ --set 'global.externalVaultAddr=http://vault-outside' \ . | tee /dev/stderr | yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr) @@ -648,3 +654,93 @@ load _helpers yq -r '.spec.template.spec.containers[0].securityContext.foo' | tee /dev/stderr) [ "${actual}" = "bar" ] } + +#-------------------------------------------------------------------- +# Agent sidecar configurables + +@test "csi/daemonset: Agent sidecar enabled by default" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/csi-daemonset.yaml \ + --set 'csi.enabled=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers | length' | tee /dev/stderr) + [ "${actual}" = "2" ] +} + +@test "csi/daemonset: Agent sidecar can pass extra args" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/csi-daemonset.yaml \ + --set 'csi.enabled=true' \ + --set 'csi.agent.extraArgs[0]=-config=extra-config.hcl' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[1].args[2]' | tee /dev/stderr) + [ "${actual}" = "-config=extra-config.hcl" ] +} + +@test "csi/daemonset: Agent log level settable" { + cd `chart_dir` + local object=$(helm template \ + --show-only templates/csi-daemonset.yaml \ + --set 'csi.enabled=true' \ + --set 'csi.agent.logLevel=error' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[1].env' | tee /dev/stderr) + + local value=$(echo $object | + yq -r 'map(select(.name=="VAULT_LOG_LEVEL")) | .[] .value' | tee /dev/stderr) + [ "${value}" = "error" ] +} + +@test "csi/daemonset: Agent log format settable" { + cd `chart_dir` + local object=$(helm template \ + --show-only templates/csi-daemonset.yaml \ + --set 'csi.enabled=true' \ + --set 'csi.agent.logFormat=json' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[1].env' | tee /dev/stderr) + + local value=$(echo $object | + yq -r 'map(select(.name=="VAULT_LOG_FORMAT")) | .[] .value' | tee /dev/stderr) + [ "${value}" = "json" ] +} + +@test "csi/daemonset: Agent default resources" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/csi-daemonset.yaml \ + --set 'csi.enabled=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[1].resources' | tee /dev/stderr) + [ "${actual}" = "null" ] +} + +@test "csi/daemonset: Agent custom resources" { + cd `chart_dir` + local object=$(helm template \ + --show-only templates/csi-daemonset.yaml \ + --set 'csi.enabled=true' \ + --set 'csi.agent.resources.requests.memory=256Mi' \ + --set 'csi.agent.resources.requests.cpu=250m' \ + --set 'csi.agent.resources.limits.memory=512Mi' \ + --set 'csi.agent.resources.limits.cpu=500m' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[1].resources' | tee /dev/stderr) + local value=$(echo $object | + yq -r '.requests.memory' | tee /dev/stderr) + [ "${value}" = "256Mi" ] + + local value=$(echo $object | + yq -r '.requests.cpu' | tee /dev/stderr) + [ "${value}" = "250m" ] + + local value=$(echo $object | + yq -r '.limits.memory' | tee /dev/stderr) + [ "${value}" = "512Mi" ] + + local value=$(echo $object | + yq -r '.limits.cpu' | tee /dev/stderr) + [ "${value}" = "500m" ] +} \ No newline at end of file diff --git a/values.schema.json b/values.schema.json index c52c20088..44980e169 100644 --- a/values.schema.json +++ b/values.schema.json @@ -5,6 +5,40 @@ "csi": { "type": "object", "properties": { + "agent": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "extraArgs": { + "type": "array" + }, + "image": { + "type": "object", + "properties": { + "pullPolicy": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + }, + "logFormat": { + "type": "string" + }, + "logLevel": { + "type": "string" + }, + "resources": { + "type": "object" + } + } + }, "daemonSet": { "type": "object", "properties": { diff --git a/values.yaml b/values.yaml index e59992719..75da40b28 100644 --- a/values.yaml +++ b/values.yaml @@ -997,7 +997,7 @@ csi: image: repository: "hashicorp/vault-csi-provider" - tag: "1.2.1" + tag: "1.3.0" pullPolicy: IfNotPresent # volumes is a list of volumes made available to all containers. These are rendered @@ -1061,7 +1061,26 @@ csi: # This should be a YAML map of the labels to apply to the csi provider pod extraLabels: {} - + agent: + enabled: true + extraArgs: [] + + image: + repository: "hashicorp/vault" + tag: "1.13.1" + pullPolicy: IfNotPresent + + logFormat: standard + logLevel: info + + resources: {} + # resources: + # requests: + # memory: 256Mi + # cpu: 250m + # limits: + # memory: 256Mi + # cpu: 250m # Priority class for csi pods priorityClassName: ""