From b4826d412436b7a8d03b642576ad20fd2664fb67 Mon Sep 17 00:00:00 2001 From: Iryna Shustava Date: Tue, 27 Apr 2021 16:04:50 -0600 Subject: [PATCH] Allow to exclude inbound and outbound traffic when using tproxy via annotations We allow exclusion of the following: * Exclude inbound ports * Exclude outbound ports * Exclude outbound CIDRs * Exclude UIDs --- connect-inject/annotations.go | 12 +++ connect-inject/container_init.go | 57 +++++++++++-- connect-inject/container_init_test.go | 117 +++++++++++++++++++++----- connect-inject/handler_test.go | 1 - 4 files changed, 158 insertions(+), 29 deletions(-) diff --git a/connect-inject/annotations.go b/connect-inject/annotations.go index c81f36a3f8..9be8bbd0c1 100644 --- a/connect-inject/annotations.go +++ b/connect-inject/annotations.go @@ -88,6 +88,18 @@ const ( // This annotation takes a boolean value (true/false). annotationTransparentProxy = "consul.hashicorp.com/transparent-proxy" + // annotationTProxyExcludeInboundPorts is a comma-separated list of inbound ports to exclude from traffic redirection. + annotationTProxyExcludeInboundPorts = "consul.hashicorp.com/transparent-proxy-exclude-inbound-ports" + + // annotationTProxyExcludeOutboundPorts is a comma-separated list of outbound ports to exclude from traffic redirection. + annotationTProxyExcludeOutboundPorts = "consul.hashicorp.com/transparent-proxy-exclude-outbound-ports" + + // annotationTProxyExcludeOutboundCIDRs is a comma-separated list of outbound CIDRs to exclude from traffic redirection. + annotationTProxyExcludeOutboundCIDRs = "consul.hashicorp.com/transparent-proxy-exclude-outbound-cidrs" + + // annotationTProxyExcludeUIDs is a comma-separated list of additional user IDs to exclude from traffic redirection. + annotationTProxyExcludeUIDs = "consul.hashicorp.com/transparent-proxy-exclude-uids" + // injected is used as the annotation value for annotationInjected. injected = "injected" ) diff --git a/connect-inject/container_init.go b/connect-inject/container_init.go index 9204b2a145..21bcd72089 100644 --- a/connect-inject/container_init.go +++ b/connect-inject/container_init.go @@ -46,6 +46,22 @@ type initContainerCommandData struct { // i.e. run consul connect redirect-traffic command and add the required privileges to the // container to do that. EnableTransparentProxy bool + + // TProxyExcludeInboundPorts is a list of inbound ports to exclude from traffic redirection via + // the consul connect redirect-traffic command. + TProxyExcludeInboundPorts []string + + // TProxyExcludeOutboundPorts is a list of outbound ports to exclude from traffic redirection via + // the consul connect redirect-traffic command. + TProxyExcludeOutboundPorts []string + + // TProxyExcludeOutboundCIDRs is a list of outbound CIDRs to exclude from traffic redirection via + // the consul connect redirect-traffic command. + TProxyExcludeOutboundCIDRs []string + + // TProxyExcludeUIDs is a list of additional user IDs to exclude from traffic redirection via + // the consul connect redirect-traffic command. + TProxyExcludeUIDs []string } // containerInitCopyContainer returns the init container spec for the copy container which places @@ -84,12 +100,16 @@ func (h *Handler) containerInit(pod corev1.Pod, k8sNamespace string) (corev1.Con } data := initContainerCommandData{ - AuthMethod: h.AuthMethod, - ConsulNamespace: h.consulNamespace(k8sNamespace), - NamespaceMirroringEnabled: h.EnableK8SNSMirroring, - ConsulCACert: h.ConsulCACert, - EnableTransparentProxy: tproxyEnabled, - EnvoyUID: envoyUserAndGroupID, + AuthMethod: h.AuthMethod, + ConsulNamespace: h.consulNamespace(k8sNamespace), + NamespaceMirroringEnabled: h.EnableK8SNSMirroring, + ConsulCACert: h.ConsulCACert, + EnableTransparentProxy: tproxyEnabled, + TProxyExcludeInboundPorts: splitCommaSeparatedItemsFromAnnotation(annotationTProxyExcludeInboundPorts, pod), + TProxyExcludeOutboundPorts: splitCommaSeparatedItemsFromAnnotation(annotationTProxyExcludeOutboundPorts, pod), + TProxyExcludeOutboundCIDRs: splitCommaSeparatedItemsFromAnnotation(annotationTProxyExcludeOutboundCIDRs, pod), + TProxyExcludeUIDs: splitCommaSeparatedItemsFromAnnotation(annotationTProxyExcludeUIDs, pod), + EnvoyUID: envoyUserAndGroupID, } if data.AuthMethod != "" { @@ -213,6 +233,19 @@ func pointerToBool(b bool) *bool { return &b } +// splitCommaSeparatedItemsFromAnnotation takes an annotation and a pod +// and returns the comma-separated value of the annotation as a list of strings. +func splitCommaSeparatedItemsFromAnnotation(annotation string, pod corev1.Pod) []string { + var items []string + if raw, ok := pod.Annotations[annotation]; ok { + for _, item := range strings.Split(raw, ",") { + items = append(items, item) + } + } + + return items +} + // initContainerCommandTpl is the template for the command executed by // the init container. const initContainerCommandTpl = ` @@ -272,6 +305,18 @@ consul-k8s connect-init -pod-name=${POD_NAME} -pod-namespace=${POD_NAMESPACE} \ {{- if .ConsulNamespace }} -namespace="{{ .ConsulNamespace }}" \ {{- end }} + {{- range .TProxyExcludeInboundPorts }} + -exclude-inbound-port="{{ . }}" \ + {{- end }} + {{- range .TProxyExcludeOutboundPorts }} + -exclude-outbound-port="{{ . }}" \ + {{- end }} + {{- range .TProxyExcludeOutboundCIDRs }} + -exclude-outbound-cidr="{{ . }}" \ + {{- end }} + {{- range .TProxyExcludeUIDs }} + -exclude-uid="{{ . }}" \ + {{- end }} -proxy-id="$(cat /consul/connect-inject/proxyid)" \ -proxy-uid={{ .EnvoyUID }} {{- end }} diff --git a/connect-inject/container_init_test.go b/connect-inject/container_init_test.go index 6299801fca..da3b0dd908 100644 --- a/connect-inject/container_init_test.go +++ b/connect-inject/container_init_test.go @@ -1,7 +1,6 @@ package connectinject import ( - "strconv" "strings" "testing" @@ -143,48 +142,125 @@ consul-k8s connect-init -pod-name=${POD_NAME} -pod-namespace=${POD_NAMESPACE} \ func TestHandlerContainerInit_transparentProxy(t *testing.T) { cases := map[string]struct { - globalEnabled bool - annotationEnabled *bool - expectEnabled bool + globalEnabled bool + annotations map[string]string + expectedContainsCmd string + expectedNotContainsCmd string }{ "enabled globally, annotation not provided": { true, nil, - true, + `/consul/connect-inject/consul connect redirect-traffic \ + -proxy-id="$(cat /consul/connect-inject/proxyid)" \ + -proxy-uid=5995`, + "", }, "enabled globally, annotation is false": { true, - pointerToBool(false), - false, + map[string]string{ + annotationTransparentProxy: "false", + }, + "", + `/consul/connect-inject/consul connect redirect-traffic \ + -proxy-id="$(cat /consul/connect-inject/proxyid)" \ + -proxy-uid=5995`, }, "enabled globally, annotation is true": { true, - pointerToBool(true), - true, + map[string]string{ + annotationTransparentProxy: "true", + }, + `/consul/connect-inject/consul connect redirect-traffic \ + -proxy-id="$(cat /consul/connect-inject/proxyid)" \ + -proxy-uid=5995`, + "", }, "disabled globally, annotation not provided": { false, nil, - false, + "", + `/consul/connect-inject/consul connect redirect-traffic \ + -proxy-id="$(cat /consul/connect-inject/proxyid)" \ + -proxy-uid=5995`, }, "disabled globally, annotation is false": { false, - pointerToBool(false), - false, + map[string]string{ + annotationTransparentProxy: "false", + }, + "", + `/consul/connect-inject/consul connect redirect-traffic \ + -proxy-id="$(cat /consul/connect-inject/proxyid)" \ + -proxy-uid=5995`, }, "disabled globally, annotation is true": { false, - pointerToBool(true), + map[string]string{ + annotationTransparentProxy: "true", + }, + `/consul/connect-inject/consul connect redirect-traffic \ + -proxy-id="$(cat /consul/connect-inject/proxyid)" \ + -proxy-uid=5995`, + "", + }, + "exclude-inbound-ports annotation is provided": { + true, + map[string]string{ + annotationTransparentProxy: "true", + annotationTProxyExcludeInboundPorts: "9090,9091", + }, + `/consul/connect-inject/consul connect redirect-traffic \ + -exclude-inbound-port="9090" \ + -exclude-inbound-port="9091" \ + -proxy-id="$(cat /consul/connect-inject/proxyid)" \ + -proxy-uid=5995`, + "", + }, + "exclude-outbound-ports annotation is provided": { + true, + map[string]string{ + annotationTransparentProxy: "true", + annotationTProxyExcludeOutboundPorts: "9090,9091", + }, + `/consul/connect-inject/consul connect redirect-traffic \ + -exclude-outbound-port="9090" \ + -exclude-outbound-port="9091" \ + -proxy-id="$(cat /consul/connect-inject/proxyid)" \ + -proxy-uid=5995`, + "", + }, + "exclude-outbound-cidrs annotation is provided": { true, + map[string]string{ + annotationTransparentProxy: "true", + annotationTProxyExcludeOutboundCIDRs: "1.1.1.1,2.2.2.2/24", + }, + `/consul/connect-inject/consul connect redirect-traffic \ + -exclude-outbound-cidr="1.1.1.1" \ + -exclude-outbound-cidr="2.2.2.2/24" \ + -proxy-id="$(cat /consul/connect-inject/proxyid)" \ + -proxy-uid=5995`, + "", + }, + "exclude-uids annotation is provided": { + true, + map[string]string{ + annotationTransparentProxy: "true", + annotationTProxyExcludeUIDs: "6000,7000", + }, + `/consul/connect-inject/consul connect redirect-traffic \ + -exclude-uid="6000" \ + -exclude-uid="7000" \ + -proxy-id="$(cat /consul/connect-inject/proxyid)" \ + -proxy-uid=5995`, + "", }, } for name, c := range cases { t.Run(name, func(t *testing.T) { h := Handler{EnableTransparentProxy: c.globalEnabled} pod := minimal() - if c.annotationEnabled != nil { - pod.Annotations[annotationTransparentProxy] = strconv.FormatBool(*c.annotationEnabled) - } + pod.Annotations = c.annotations expectedSecurityContext := &corev1.SecurityContext{ RunAsUser: pointerToInt64(0), @@ -194,19 +270,16 @@ func TestHandlerContainerInit_transparentProxy(t *testing.T) { }, RunAsNonRoot: pointerToBool(false), } - expectedCmd := `/consul/connect-inject/consul connect redirect-traffic \ - -proxy-id="$(cat /consul/connect-inject/proxyid)" \ - -proxy-uid=5995` container, err := h.containerInit(*pod, k8sNamespace) require.NoError(t, err) actualCmd := strings.Join(container.Command, " ") - if c.expectEnabled { + if c.expectedContainsCmd != "" { require.Equal(t, expectedSecurityContext, container.SecurityContext) - require.Contains(t, actualCmd, expectedCmd) + require.Contains(t, actualCmd, c.expectedContainsCmd) } else { require.Nil(t, container.SecurityContext) - require.NotContains(t, actualCmd, expectedCmd) + require.NotContains(t, actualCmd, c.expectedNotContainsCmd) } }) } diff --git a/connect-inject/handler_test.go b/connect-inject/handler_test.go index 69657a36e8..1d82b2d876 100644 --- a/connect-inject/handler_test.go +++ b/connect-inject/handler_test.go @@ -126,7 +126,6 @@ func TestHandlerHandle(t *testing.T) { }, }, - // todo: why is upstreams different then basic { "pod with upstreams specified", Handler{