Automate DNS redirection to Consul DNS (#833)

* template kube dns service IP using api request
* when dns is enabled, dns queries are directed to consul.
* Update control-plane/connect-inject/container_init.go
* Add CHANGELOG

Co-authored-by: Ashwin Venkatesh <ashwin@hashicorp.com>
Co-authored-by: Luke Kysow <1034429+lkysow@users.noreply.github.com>

CHANGELOG.md

@@ -1,5 +1,12 @@
 ## UNRELEASED
 
+FEATURES:
+* Helm Chart
+  * Add support for Consul services to utilize Consul DNS for service discovery. Set `dns.enableRedirection` to allow services to
+    use Consul DNS via the Consul DNS Service. [[GH-833](https://github.com/hashicorp/consul-k8s/pull/833)]
+* Control Plane
+  * Connect: Allow services using Connect to utilize Consul DNS to perform service discovery. [[GH-833](https://github.com/hashicorp/consul-k8s/pull/833)]
+
 BREAKING CHANGES:
 * Previously [UI metrics](https://www.consul.io/docs/connect/observability/ui-visualization) would be enabled when
   `global.metrics=false` and `ui.metrics.enabled=-`. If you are no longer seeing UI metrics,

charts/consul/templates/_helpers.tpl

@@ -28,6 +28,18 @@ is passed to consul as a -config-file param on command line.
               [ -n "${HOSTNAME}" ] && sed -Ei "s|HOSTNAME|${HOSTNAME?}|g" /consul/extra-config/extra-from-values.json
 {{- end -}}
 
+{{/*
+Sets up a list of recusor flags for Consul agents by iterating over the IPs of every nameserver
+in /etc/resolv.conf and concatenating them into a string of arguments that can be passed directly
+to the consul agent command.
+*/}}
+{{- define "consul.recursors" -}}
+              recursor_flags=""
+              for ip in $(cat /etc/resolv.conf | grep nameserver | cut -d' ' -f2)
+              do
+                 recursor_flags="$recursor_flags -recursor=$ip"
+              done
+{{- end -}}
 
 {{/*
 Create chart name and version as used by the chart label.

charts/consul/templates/client-daemonset.yaml

@@ -203,6 +203,10 @@ spec:
             - |
               CONSUL_FULLNAME="{{template "consul.fullname" . }}"
 
+              {{- if (and .Values.dns.enabled .Values.dns.enableRedirection) }}
+              {{ template "consul.recursors" }}
+              {{- end }}
+
               {{ template "consul.extraconfig" }}
 
               exec /usr/local/bin/docker-entrypoint.sh consul agent \
@@ -276,6 +280,9 @@ spec:
                 {{- range $value := .Values.global.recursors }}
                 -recursor={{ quote $value }} \
                 {{- end }}
+                {{- if (and .Values.dns.enabled .Values.dns.enableRedirection) }}
+                $recursor_flags \
+                {{- end }}
                 -config-file=/consul/extra-config/extra-from-values.json \
                 -domain={{ .Values.global.domain }}
           volumeMounts:

charts/consul/templates/connect-inject-deployment.yaml

@@ -83,8 +83,6 @@ spec:
             - "/bin/sh"
             - "-ec"
             - |
-              CONSUL_FULLNAME="{{template "consul.fullname" . }}"
-
               consul-k8s-control-plane inject-connect \
                 -log-level={{ default .Values.global.logLevel .Values.connectInject.logLevel }} \
                 -log-json={{ .Values.global.logJSON }} \
@@ -108,6 +106,10 @@ spec:
                 {{- else }}
                 -transparent-proxy-default-overwrite-probes=false \
                 {{- end }}
+                -resource-prefix={{ template "consul.fullname" . }} \
+                {{- if (and .Values.dns.enabled .Values.dns.enableRedirection) }}
+                -enable-consul-dns=true \
+                {{- end }}
                 {{- if .Values.global.openshift.enabled }}
                 -enable-openshift \
                 {{- end }}

charts/consul/templates/server-statefulset.yaml

@@ -192,6 +192,10 @@ spec:
             - |
               CONSUL_FULLNAME="{{template "consul.fullname" . }}"
 
+              {{- if (and .Values.dns.enabled .Values.dns.enableRedirection) }}
+              {{ template "consul.recursors" }}
+              {{- end }}
+
               {{ template "consul.extraconfig" }}
 
               exec /usr/local/bin/docker-entrypoint.sh consul agent \
@@ -254,6 +258,9 @@ spec:
                 {{- range $value := .Values.global.recursors }}
                 -recursor={{ quote $value }} \
                 {{- end }}
+                {{- if (and .Values.dns.enabled .Values.dns.enableRedirection) }}
+                $recursor_flags \
+                {{- end }}
                 -config-file=/consul/extra-config/extra-from-values.json \
                 -server
           volumeMounts:

charts/consul/test/unit/client-daemonset.bats

@@ -1170,6 +1170,28 @@ load _helpers
   [ "${actual}" = "true" ]
 }
 
+#--------------------------------------------------------------------
+# DNS
+
+@test "client/DaemonSet: recursor flags is not set by default" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/client-daemonset.yaml \
+      . | tee /dev/stderr |
+      yq -c -r '.spec.template.spec.containers[0].command | join(" ") | contains("$recursor_flags")' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+}
+
+@test "client/DaemonSet: add recursor flags if dns.enableRedirection is true" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/client-daemonset.yaml \
+      --set 'dns.enableRedirection=true' \
+      . | tee /dev/stderr |
+      yq -c -r '.spec.template.spec.containers[0].command | join(" ") | contains("$recursor_flags")' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
 #--------------------------------------------------------------------
 # hostNetwork
 

charts/consul/test/unit/connect-inject-deployment.bats

@@ -519,6 +519,40 @@ EOF
   [ "${actual}" = "true" ]
 }
 
+#--------------------------------------------------------------------
+# DNS
+
+@test "connectInject/Deployment: -enable-consul-dns unset by default" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/connect-inject-deployment.yaml \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq -c -r '.spec.template.spec.containers[0].command | join(" ") | contains("-enable-consul-dns=true")' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+}
+
+@test "connectInject/Deployment: -enable-consul-dns is true if dns.enabled=true and dns.enableRedirection=true" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/connect-inject-deployment.yaml \
+      --set 'connectInject.enabled=true' \
+      --set 'dns.enableRedirection=true' \
+      . | tee /dev/stderr |
+      yq -c -r '.spec.template.spec.containers[0].command | join(" ") | contains("-enable-consul-dns=true")' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "connectInject/Deployment: -resource-prefix always set" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/connect-inject-deployment.yaml \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq -c -r '.spec.template.spec.containers[0].command | join(" ") | contains("-resource-prefix=RELEASE-NAME-consul")' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
 #--------------------------------------------------------------------
 # global.tls.enabled
 

charts/consul/test/unit/server-statefulset.bats

@@ -580,6 +580,28 @@ load _helpers
   [ "${actualBaz}" = "qux" ]
 }
 
+#--------------------------------------------------------------------
+# DNS
+
+@test "server/StatefulSet: recursor flags unset by default" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/server-statefulset.yaml \
+      . | tee /dev/stderr |
+      yq -c -r '.spec.template.spec.containers[0].command | join(" ") | contains("$recursor_flags")' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+}
+
+@test "server/StatefulSet: add recursor flags if dns.enableRedirection is true" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/server-statefulset.yaml \
+      --set 'dns.enableRedirection=true' \
+      . | tee /dev/stderr |
+      yq -c -r '.spec.template.spec.containers[0].command | join(" ") | contains("$recursor_flags")' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
 #--------------------------------------------------------------------
 # annotations
 

charts/consul/values.yaml

@@ -1122,6 +1122,12 @@ dns:
   # @type: boolean
   enabled: "-"
 
+  # If true, services using Consul Connect will use Consul DNS
+  # for default DNS resolution. The DNS lookups fall back to the nameserver IPs
+  # listed in /etc/resolv.conf if not found in Consul.
+  # @type: boolean
+  enableRedirection: false
+
   # Used to control the type of service created. For
   # example, setting this to "LoadBalancer" will create an external load
   # balancer (for supported K8S installations)

control-plane/connect-inject/annotations.go

@@ -90,6 +90,12 @@ const (
 	// annotationConsulNamespace is the Consul namespace the service is registered into.
 	annotationConsulNamespace = "consul.hashicorp.com/consul-namespace"
 
+	// keyConsulDNS enables or disables Consul DNS for a given pod. It can also be set as a label
+	// on a namespace to define the default behaviour for connect-injected pods which do not otherwise override this setting
+	// with their own annotation.
+	// This annotation/label takes a boolean value (true/false).
+	keyConsulDNS = "consul.hashicorp.com/consul-dns"
+
 	// keyTransparentProxy enables or disables transparent proxy for a given pod. It can also be set as a label
 	// on a namespace to define the default behaviour for connect-injected pods which do not otherwise override this setting
 	// with their own annotation.

control-plane/connect-inject/container_init.go

@@ -2,6 +2,8 @@ package connectinject
 
 import (
 	"bytes"
+	"fmt"
+	"os"
 	"strconv"
 	"strings"
 	"text/template"
@@ -16,6 +18,7 @@ const (
 	envoyUserAndGroupID         = 5995
 	copyContainerUserAndGroupID = 5996
 	netAdminCapability          = "NET_ADMIN"
+	dnsServiceHostEnvSuffix     = "DNS_SERVICE_HOST"
 )
 
 type initContainerCommandData struct {
@@ -66,6 +69,9 @@ type initContainerCommandData struct {
 	// TProxyExcludeUIDs is a list of additional user IDs to exclude from traffic redirection via
 	// the consul connect redirect-traffic command.
 	TProxyExcludeUIDs []string
+
+	// ConsulDNSClusterIP is the IP of the Consul DNS Service.
+	ConsulDNSClusterIP string
 }
 
 // initCopyContainer returns the init container spec for the copy container which places
@@ -107,6 +113,22 @@ func (h *Handler) containerInit(namespace corev1.Namespace, pod corev1.Pod) (cor
 		return corev1.Container{}, err
 	}
 
+	dnsEnabled, err := consulDNSEnabled(namespace, pod, h.EnableConsulDNS)
+	if err != nil {
+		return corev1.Container{}, err
+	}
+
+	var consulDNSClusterIP string
+	if dnsEnabled {
+		// If Consul DNS is enabled, we find the environment variable that has the value
+		// of the ClusterIP of the Consul DNS Service. constructDNSServiceHostName returns
+		// the name of the env variable whose value is the ClusterIP of the Consul DNS Service.
+		consulDNSClusterIP = os.Getenv(h.constructDNSServiceHostName())
+		if consulDNSClusterIP == "" {
+			return corev1.Container{}, fmt.Errorf("environment variable %s is not found", h.constructDNSServiceHostName())
+		}
+	}
+
 	data := initContainerCommandData{
 		AuthMethod:                 h.AuthMethod,
 		ConsulPartition:            h.ConsulPartition,
@@ -118,6 +140,7 @@ func (h *Handler) containerInit(namespace corev1.Namespace, pod corev1.Pod) (cor
 		TProxyExcludeOutboundPorts: splitCommaSeparatedItemsFromAnnotation(annotationTProxyExcludeOutboundPorts, pod),
 		TProxyExcludeOutboundCIDRs: splitCommaSeparatedItemsFromAnnotation(annotationTProxyExcludeOutboundCIDRs, pod),
 		TProxyExcludeUIDs:          splitCommaSeparatedItemsFromAnnotation(annotationTProxyExcludeUIDs, pod),
+		ConsulDNSClusterIP:         consulDNSClusterIP,
 		EnvoyUID:                   envoyUserAndGroupID,
 	}
 
@@ -223,6 +246,15 @@ func (h *Handler) containerInit(namespace corev1.Namespace, pod corev1.Pod) (cor
 	return container, nil
 }
 
+// constructDNSServiceHostName use the resource prefix and the DNS Service hostname suffix to construct the
+// key of the env variable whose value is the cluster IP of the Consul DNS Service.
+// It translates "resource-prefix" into "RESOURCE_PREFIX_DNS_SERVICE_HOST".
+func (h *Handler) constructDNSServiceHostName() string {
+	upcaseResourcePrefix := strings.ToUpper(h.ResourcePrefix)
+	upcaseResourcePrefixWithUnderscores := strings.ReplaceAll(upcaseResourcePrefix, "-", "_")
+	return strings.Join([]string{upcaseResourcePrefixWithUnderscores, dnsServiceHostEnvSuffix}, "_")
+}
+
 // transparentProxyEnabled returns true if transparent proxy should be enabled for this pod.
 // It returns an error when the annotation value cannot be parsed by strconv.ParseBool or if we are unable
 // to read the pod's namespace label when it exists.
@@ -239,6 +271,22 @@ func transparentProxyEnabled(namespace corev1.Namespace, pod corev1.Pod, globalE
 	return globalEnabled, nil
 }
 
+// consulDNSEnabled returns true if Consul DNS should be enabled for this pod.
+// It returns an error when the annotation value cannot be parsed by strconv.ParseBool or if we are unable
+// to read the pod's namespace label when it exists.
+func consulDNSEnabled(namespace corev1.Namespace, pod corev1.Pod, globalEnabled bool) (bool, error) {
+	// First check to see if the pod annotation exists to override the namespace or global settings.
+	if raw, ok := pod.Annotations[keyConsulDNS]; ok {
+		return strconv.ParseBool(raw)
+	}
+	// Next see if the namespace has been defaulted.
+	if raw, ok := namespace.Labels[keyConsulDNS]; ok {
+		return strconv.ParseBool(raw)
+	}
+	// Else fall back to the global default.
+	return globalEnabled, nil
+}
+
 // pointerToInt64 takes an int64 and returns a pointer to it.
 func pointerToInt64(i int64) *int64 {
 	return &i
@@ -331,6 +379,9 @@ consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD
   {{- if .ConsulNamespace }}
   -namespace="{{ .ConsulNamespace }}" \
   {{- end }}
+  {{- if .ConsulDNSClusterIP }}
+  -consul-dns-ip="{{ .ConsulDNSClusterIP }}" \
+  {{- end }}
   {{- range .TProxyExcludeInboundPorts }}
   -exclude-inbound-port="{{ . }}" \
   {{- end }}