feat(kuma-cp) sidecar env vars override (#1562)

Signed-off-by: Jakub Dyszkiewicz <jakub.dyszkiewicz@gmail.com>

pkg/api-server/config_ws_test.go

@@ -159,6 +159,7 @@ var _ = Describe("Config WS", func() {
                   "image": "kuma/kuma-init:latest"
                 },
                 "sidecarContainer": {
+                  "envVars": {},
                   "adminPort": 9901,
                   "drainTime": "30s",
                   "gid": 5678,

pkg/config/app/kuma-cp/kuma-cp.defaults.yaml

@@ -231,6 +231,8 @@ runtime:
             cpu: 1000m # ENV: KUMA_INJECTOR_SIDECAR_CONTAINER_RESOURCES_LIMITS_CPU
             # Memory, in bytes. (500Gi = 500GiB = 500 * 1024 * 1024 * 1024)
             memory: 512Mi # ENV: KUMA_INJECTOR_SIDECAR_CONTAINER_RESOURCES_LIMITS_MEMORY
+        # Additional environment variables that can be placed on Kuma DP sidecar
+        envVars: # ENV: KUMA_RUNTIME_KUBERNETES_INJECTOR_SIDECAR_CONTAINER_ENV_VARS
       # InitContainer defines configuration of the Kuma init container
       initContainer:
         # Image name.

pkg/config/loader_test.go

@@ -149,6 +149,7 @@ var _ = Describe("Config loader", func() {
 			Expect(cfg.Runtime.Kubernetes.Injector.VirtualProbesPort).To(Equal(uint32(1111)))
 			Expect(cfg.Runtime.Kubernetes.Injector.CNIEnabled).To(BeTrue())
 			Expect(cfg.Runtime.Kubernetes.Injector.InitContainer.Image).To(Equal("test-image:test"))
+			Expect(cfg.Runtime.Kubernetes.Injector.SidecarContainer.EnvVars).To(Equal(map[string]string{"a": "b", "c": "d"}))
 			Expect(cfg.Runtime.Kubernetes.Injector.SidecarContainer.RedirectPortInbound).To(Equal(uint32(2020)))
 			Expect(cfg.Runtime.Kubernetes.Injector.SidecarContainer.RedirectPortOutbound).To(Equal(uint32(1010)))
 			Expect(cfg.Runtime.Kubernetes.Injector.SidecarContainer.UID).To(Equal(int64(100)))
@@ -336,6 +337,9 @@ runtime:
           periodSeconds: 18
           failureThreshold: 22
           timeoutSeconds: 24
+        envVars:
+          a: b
+          c: d
       sidecarTraffic:
         excludeInboundPorts:
         - 1234
@@ -469,6 +473,7 @@ sdsServer:
 				"KUMA_RUNTIME_KUBERNETES_INJECTOR_SIDECAR_CONTAINER_REDIRECT_PORT_INBOUND":                 "2020",
 				"KUMA_RUNTIME_KUBERNETES_INJECTOR_SIDECAR_CONTAINER_REDIRECT_PORT_OUTBOUND":                "1010",
 				"KUMA_RUNTIME_KUBERNETES_INJECTOR_CNI_ENABLED":                                             "true",
+				"KUMA_RUNTIME_KUBERNETES_INJECTOR_SIDECAR_CONTAINER_ENV_VARS":                              "a:b,c:d",
 				"KUMA_RUNTIME_KUBERNETES_INJECTOR_SIDECAR_CONTAINER_UID":                                   "100",
 				"KUMA_RUNTIME_KUBERNETES_INJECTOR_SIDECAR_CONTAINER_ADMIN_PORT":                            "1099",
 				"KUMA_RUNTIME_KUBERNETES_INJECTOR_SIDECAR_CONTAINER_DRAIN_TIME":                            "33s",

pkg/config/plugins/runtime/k8s/config.go

@@ -157,6 +157,8 @@ type SidecarContainer struct {
 	LivenessProbe SidecarLivenessProbe `yaml:"livenessProbe,omitempty"`
 	// Compute resource requirements.
 	Resources SidecarResources `yaml:"resources,omitempty"`
+	// EnvVars are additional environment variables that can be placed on Kuma DP sidecar
+	EnvVars map[string]string `yaml:"envVars" envconfig:"kuma_runtime_kubernetes_injector_sidecar_container_env_vars"`
 }
 
 // SidecarReadinessProbe defines periodic probe of container service readiness.

pkg/config/plugins/runtime/k8s/testdata/default-config.golden.yaml

@@ -29,6 +29,7 @@ injector:
       limits:
         cpu: 1000m
         memory: 512Mi
+    envVars: {}
   initContainer:
     image: kuma/kuma-init:latest
   cniEnabled: false

pkg/plugins/runtime/k8s/metadata/annotations.go

@@ -2,6 +2,7 @@ package metadata
 
 import (
 	"strconv"
+	"strings"
 
 	"github.com/pkg/errors"
 )
@@ -44,6 +45,10 @@ const (
 	// KumaVirtualProbesPortAnnotation is an insecure port for listening virtual probes
 	KumaVirtualProbesPortAnnotation = "kuma.io/virtual-probes-port"
 
+	// KumaSidecarEnvVarsAnnotation is a ; separated list of env vars that will be applied on Kuma Sidecar
+	// Example value: TEST1=1;TEST2=2
+	KumaSidecarEnvVarsAnnotation = "kuma.io/sidecar-env-vars"
+
 	// KumaMetricsPrometheusPort allows to override `Mesh`-wide default port
 	KumaMetricsPrometheusPort = "prometheus.metrics.kuma.io/port"
 
@@ -117,3 +122,22 @@ func (a Annotations) GetBool(key string) (bool, bool, error) {
 	}
 	return b, true, nil
 }
+
+// GetMap returns map from annotation. Example: "kuma.io/sidecar-env-vars: TEST1=1;TEST2=2"
+func (a Annotations) GetMap(key string) (map[string]string, error) {
+	value, ok := a[key]
+	if !ok {
+		return nil, nil
+	}
+	result := map[string]string{}
+
+	pairs := strings.Split(value, ";")
+	for _, pair := range pairs {
+		kvSplit := strings.Split(pair, "=")
+		if len(kvSplit) != 2 {
+			return nil, errors.Errorf("invalid format. Map in %q has to be provided in the following format: key1=value1;key2=value2", key)
+		}
+		result[kvSplit[0]] = kvSplit[1]
+	}
+	return result, nil
+}

pkg/plugins/runtime/k8s/metadata/annotations_test.go

@@ -37,4 +37,33 @@ var _ = Describe("Kubernetes Annotations", func() {
 			Expect(exist).To(BeTrue())
 		})
 	})
+
+	Context("GetMap()", func() {
+		It("should parse value to map", func() {
+			// given
+			annotations := map[string]string{
+				"key1": "TEST1=1;TEST2=2",
+			}
+
+			// when
+			m, err := metadata.Annotations(annotations).GetMap("key1")
+
+			// then
+			Expect(err).ToNot(HaveOccurred())
+			Expect(m).To(Equal(map[string]string{"TEST1": "1", "TEST2": "2"}))
+		})
+
+		It("should return error if value has wrong format", func() {
+			// given
+			annotations := map[string]string{
+				"key1": "TESTTEST",
+			}
+
+			// when
+			_, err := metadata.Annotations(annotations).GetMap("key1")
+
+			// then
+			Expect(err).To(MatchError(`invalid format. Map in "key1" has to be provided in the following format: key1=value1;key2=value2`))
+		})
+	})
 })

pkg/plugins/runtime/k8s/webhooks/injector/injector.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	"sort"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -80,7 +81,11 @@ func (i *KumaInjector) InjectKuma(pod *kube_core.Pod) error {
 	if pod.Spec.Containers == nil {
 		pod.Spec.Containers = []kube_core.Container{}
 	}
-	pod.Spec.Containers = append(pod.Spec.Containers, i.NewSidecarContainer(pod, ns))
+	container, err := i.NewSidecarContainer(pod, ns)
+	if err != nil {
+		return err
+	}
+	pod.Spec.Containers = append(pod.Spec.Containers, container)
 
 	mesh, err := i.meshFor(pod, ns)
 	if err != nil {
@@ -193,8 +198,12 @@ func (i *KumaInjector) namespaceFor(pod *kube_core.Pod) (*kube_core.Namespace, e
 	return ns, nil
 }
 
-func (i *KumaInjector) NewSidecarContainer(pod *kube_core.Pod, ns *kube_core.Namespace) kube_core.Container {
+func (i *KumaInjector) NewSidecarContainer(pod *kube_core.Pod, ns *kube_core.Namespace) (kube_core.Container, error) {
 	mesh := meshName(pod, ns)
+	env, err := i.sidecarEnvVars(mesh, pod.GetAnnotations())
+	if err != nil {
+		return kube_core.Container{}, err
+	}
 	return kube_core.Container{
 		Name:            util.KumaSidecarContainerName,
 		Image:           i.cfg.SidecarContainer.Image,
@@ -203,65 +212,7 @@ func (i *KumaInjector) NewSidecarContainer(pod *kube_core.Pod, ns *kube_core.Nam
 			"run",
 			"--log-level=info",
 		},
-		Env: []kube_core.EnvVar{
-			{
-				Name: "POD_NAME",
-				ValueFrom: &kube_core.EnvVarSource{
-					FieldRef: &kube_core.ObjectFieldSelector{
-						APIVersion: "v1",
-						FieldPath:  "metadata.name",
-					},
-				},
-			},
-			{
-				Name: "POD_NAMESPACE",
-				ValueFrom: &kube_core.EnvVarSource{
-					FieldRef: &kube_core.ObjectFieldSelector{
-						APIVersion: "v1",
-						FieldPath:  "metadata.namespace",
-					},
-				},
-			},
-			{
-				Name: "INSTANCE_IP",
-				ValueFrom: &kube_core.EnvVarSource{
-					FieldRef: &kube_core.ObjectFieldSelector{
-						APIVersion: "v1",
-						FieldPath:  "status.podIP",
-					},
-				},
-			},
-			{
-				Name:  "KUMA_CONTROL_PLANE_URL",
-				Value: i.controlPlaneUrl,
-			},
-			{
-				Name:  "KUMA_DATAPLANE_MESH",
-				Value: mesh,
-			},
-			{
-				Name: "KUMA_DATAPLANE_NAME",
-				// notice that Pod name might not be available at this time (in case of Deployment, ReplicaSet, etc)
-				// that is why we have to use a runtime reference to POD_NAME instead
-				Value: "$(POD_NAME).$(POD_NAMESPACE)", // variable references get expanded by Kubernetes
-			},
-			{
-				Name:  "KUMA_DATAPLANE_ADMIN_PORT",
-				Value: fmt.Sprintf("%d", i.cfg.SidecarContainer.AdminPort),
-			},
-			{
-				Name:  "KUMA_DATAPLANE_DRAIN_TIME",
-				Value: i.cfg.SidecarContainer.DrainTime.String(),
-			},
-			{
-				Name:  "KUMA_DATAPLANE_RUNTIME_TOKEN_PATH",
-				Value: "/var/run/secrets/kubernetes.io/serviceaccount/token",
-			},
-			{
-				Name:  "KUMA_CONTROL_PLANE_CA_CERT",
-				Value: i.caCert,
-			},
-		},
+		Env: env,
 		SecurityContext: &kube_core.SecurityContext{
 			RunAsUser:  &i.cfg.SidecarContainer.UID,
 			RunAsGroup: &i.cfg.SidecarContainer.GID,
@@ -311,7 +262,101 @@ func (i *KumaInjector) NewSidecarContainer(pod *kube_core.Pod, ns *kube_core.Nam
 		// That's why it is a responsibility of every mutating web hook to copy
 		// ServiceAccount volume mount into containers it creates.
 		VolumeMounts: i.NewVolumeMounts(pod),
+	}, nil
+}
+
+func (i *KumaInjector) sidecarEnvVars(mesh string, podAnnotations map[string]string) ([]kube_core.EnvVar, error) {
+	envVars := map[string]kube_core.EnvVar{
+		"KUMA_CONTROL_PLANE_URL": {
+			Name:  "KUMA_CONTROL_PLANE_URL",
+			Value: i.controlPlaneUrl,
+		},
+		"KUMA_DATAPLANE_MESH": {
+			Name:  "KUMA_DATAPLANE_MESH",
+			Value: mesh,
+		},
+		"KUMA_DATAPLANE_NAME": {
+			Name: "KUMA_DATAPLANE_NAME",
+			// notice that Pod name might not be available at this time (in case of Deployment, ReplicaSet, etc)
+			// that is why we have to use a runtime reference to POD_NAME instead
+			Value: "$(POD_NAME).$(POD_NAMESPACE)", // variable references get expanded by Kubernetes
+		},
+		"KUMA_DATAPLANE_ADMIN_PORT": {
+			Name:  "KUMA_DATAPLANE_ADMIN_PORT",
+			Value: fmt.Sprintf("%d", i.cfg.SidecarContainer.AdminPort),
+		},
+		"KUMA_DATAPLANE_DRAIN_TIME": {
+			Name:  "KUMA_DATAPLANE_DRAIN_TIME",
+			Value: i.cfg.SidecarContainer.DrainTime.String(),
+		},
+		"KUMA_DATAPLANE_RUNTIME_TOKEN_PATH": {
+			Name:  "KUMA_DATAPLANE_RUNTIME_TOKEN_PATH",
+			Value: "/var/run/secrets/kubernetes.io/serviceaccount/token",
+		},
+		"KUMA_CONTROL_PLANE_CA_CERT": {
+			Name:  "KUMA_CONTROL_PLANE_CA_CERT",
+			Value: i.caCert,
+		},
+	}
+
+	// override defaults with cfg env vars
+	for envName, envVal := range i.cfg.SidecarContainer.EnvVars {
+		envVars[envName] = kube_core.EnvVar{
+			Name:  envName,
+			Value: envVal,
+		}
+	}
+
+	// override defaults and cfg env vars with annotations
+	annotationEnvVars, err := metadata.Annotations(podAnnotations).GetMap(metadata.KumaSidecarEnvVarsAnnotation)
+	if err != nil {
+		return nil, err
+	}
+	for envName, envVal := range annotationEnvVars {
+		envVars[envName] = kube_core.EnvVar{
+			Name:  envName,
+			Value: envVal,
+		}
+	}
+
+	var result []kube_core.EnvVar
+	for _, v := range envVars {
+		result = append(result, v)
 	}
+	sort.Stable(EnvVarsByName(result))
+
+	// those values needs to be added before other vars, otherwise expressions like "$(POD_NAME).$(POD_NAMESPACE)" won't be evaluated
+	result = append([]kube_core.EnvVar{
+		{
+			Name: "POD_NAME",
+			ValueFrom: &kube_core.EnvVarSource{
+				FieldRef: &kube_core.ObjectFieldSelector{
+					APIVersion: "v1",
+					FieldPath:  "metadata.name",
+				},
+			},
+		},
+		{
+			Name: "POD_NAMESPACE",
+			ValueFrom: &kube_core.EnvVarSource{
+				FieldRef: &kube_core.ObjectFieldSelector{
+					APIVersion: "v1",
+					FieldPath:  "metadata.namespace",
+				},
+			},
+		},
+		{
+			Name: "INSTANCE_IP",
+			ValueFrom: &kube_core.EnvVarSource{
+				FieldRef: &kube_core.ObjectFieldSelector{
+					APIVersion: "v1",
+					FieldPath:  "status.podIP",
+				},
+			},
+		},
+	}, result...)
+
+	return result, nil
 }
 
 func (i *KumaInjector) NewVolumeMounts(pod *kube_core.Pod) []kube_core.VolumeMount {
@@ -432,3 +477,11 @@ func portsToAnnotationValue(ports []uint32) string {
 	}
 	return strings.Join(stringPorts, ",")
 }
+
+type EnvVarsByName []kube_core.EnvVar
+
+func (a EnvVarsByName) Len() int      { return len(a) }
+func (a EnvVarsByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a EnvVarsByName) Less(i, j int) bool {
+	return a[i].Name < a[j].Name
+}

pkg/plugins/runtime/k8s/webhooks/injector/injector_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/kumahq/kuma/pkg/plugins/resources/k8s"
 	"github.com/kumahq/kuma/pkg/plugins/resources/k8s/native/api/v1alpha1"
 	inject "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/webhooks/injector"
+	"github.com/kumahq/kuma/pkg/test/matchers"
 
 	kube_core "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
@@ -85,12 +86,7 @@ var _ = Describe("Injector", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			By("comparing actual against golden")
-			// when
-			expected, err := ioutil.ReadFile(goldenFile)
-			// then
-			Expect(err).ToNot(HaveOccurred())
-			// and
-			Expect(actual).To(MatchYAML(expected))
+			Expect(actual).To(matchers.MatchGoldenYAML(goldenFile))
 		},
 		Entry("01. Pod without init containers and annotations", testCase{
 			num: "01",
@@ -491,5 +487,21 @@ var _ = Describe("Injector", func() {
                   kuma.io/sidecar-injection: enabled`,
 			cfgFile: "inject.config.yaml",
 		}),
+		Entry("24. sidecar env var config overrides", testCase{
+			num: "24",
+			mesh: `
+              apiVersion: kuma.io/v1alpha1
+              kind: Mesh
+              metadata:
+                name: default`,
+			namespace: `
+              apiVersion: v1
+              kind: Namespace
+              metadata:
+                name: default
+                annotations:
+                  kuma.io/sidecar-injection: enabled`,
+			cfgFile: "inject.env-vars.config.yaml",
+		}),
 	)
 })