feat(kuma-cp) permissive mTLS mode (#2579)

Signed-off-by: Ilya Lobkov <lobkovilya@yandex.ru>
kumahq · Aug 23, 2021 · 46d6a19 · 46d6a19
1 parent fd4bdd2
commit 46d6a19
Show file tree

Hide file tree

Showing 33 changed files with 2,073 additions and 139 deletions.
diff --git a/api/mesh/v1alpha1/mesh.pb.go b/api/mesh/v1alpha1/mesh.pb.go
diff --git a/api/mesh/v1alpha1/mesh.proto b/api/mesh/v1alpha1/mesh.proto
@@ -83,6 +83,21 @@ message CertificateAuthorityBackend {
 
   // Configuration of the backend
   google.protobuf.Struct conf = 4;
+
+  enum Mode {
+    // A STRICT mode implies that the server validates the connection and
+    // accepts only encrypted TLS traffic
+    STRICT = 0;
+    // A PERMISSIVE mode implies that the outbounds encrypt traffic the same way
+    // it happens in strict mode, but inbounds accept both TLS and plaintext
+    // traffic. This allows applications residing in the mesh to accept requests
+    // from outside of the mesh.
+    PERMISSIVE = 1;
+  }
+
+  // Mode defines the behaviour of inbound listeners with regard to traffic
+  // encryption
+  Mode mode = 5;
 }
 
 // Networking defines the networking configuration of the mesh

diff --git a/pkg/xds/envoy/clusters/v3/client_side_mtls_configurer_test.go b/pkg/xds/envoy/clusters/v3/client_side_mtls_configurer_test.go
@@ -80,6 +80,8 @@ var _ = Describe("EdsClusterConfigurer", func() {
               typedConfig:
                 '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
                 commonTlsContext:
+                  alpnProtocols:
+                  - kuma
                   combinedValidationContext:
                     defaultValidationContext:
                       matchSubjectAltNames:
@@ -145,6 +147,8 @@ var _ = Describe("EdsClusterConfigurer", func() {
                 typedConfig:
                   '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
                   commonTlsContext:
+                    alpnProtocols:
+                    - kuma
                     combinedValidationContext:
                       defaultValidationContext:
                         matchSubjectAltNames:
@@ -168,6 +172,8 @@ var _ = Describe("EdsClusterConfigurer", func() {
                 typedConfig:
                   '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
                   commonTlsContext:
+                    alpnProtocols:
+                    - kuma
                     combinedValidationContext:
                       defaultValidationContext:
                         matchSubjectAltNames:
@@ -229,6 +235,8 @@ var _ = Describe("EdsClusterConfigurer", func() {
               typedConfig:
                 '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
                 commonTlsContext:
+                  alpnProtocols:
+                  - kuma
                   combinedValidationContext:
                     defaultValidationContext:
                       matchSubjectAltNames:

diff --git a/pkg/xds/envoy/listeners/filter_chain_configurers.go b/pkg/xds/envoy/listeners/filter_chain_configurers.go
@@ -59,10 +59,11 @@ func HttpConnectionManager(statsName string, forwardClientCertDetails bool) Filt
 	})
 }
 
-func FilterChainMatch(transport string, serverNames ...string) FilterChainBuilderOpt {
+func FilterChainMatch(transport string, serverNames, applicationProtocols []string) FilterChainBuilderOpt {
 	return AddFilterChainConfigurer(&v3.FilterChainMatchConfigurer{
-		ServerNames:       serverNames,
-		TransportProtocol: transport,
+		ServerNames:          serverNames,
+		TransportProtocol:    transport,
+		ApplicationProtocols: applicationProtocols,
 	})
 }
 

diff --git a/pkg/xds/envoy/listeners/v3/filter_chain_match_configurer.go b/pkg/xds/envoy/listeners/v3/filter_chain_match_configurer.go
@@ -3,8 +3,9 @@ package v3
 import envoy_listener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 
 type FilterChainMatchConfigurer struct {
-	ServerNames       []string
-	TransportProtocol string
+	ServerNames          []string
+	TransportProtocol    string
+	ApplicationProtocols []string
 }
 
 func (f *FilterChainMatchConfigurer) Configure(filterChain *envoy_listener.FilterChain) error {
@@ -14,5 +15,8 @@ func (f *FilterChainMatchConfigurer) Configure(filterChain *envoy_listener.Filte
 	if f.TransportProtocol != "" {
 		filterChain.FilterChainMatch.TransportProtocol = f.TransportProtocol
 	}
+	if len(f.ApplicationProtocols) != 0 {
+		filterChain.FilterChainMatch.ApplicationProtocols = f.ApplicationProtocols
+	}
 	return nil
 }
diff --git a/pkg/xds/envoy/tls/tls.go b/pkg/xds/envoy/tls/tls.go
@@ -9,6 +9,11 @@ const (
 	IdentityCertResource = "identity_cert"
 )
 
+// KumaALPNProtocols are set for UpstreamTlsContext to show that mTLS is created by mesh.
+// On the inbound side we have to distinguish Kuma mTLS and application TLS to properly
+// support PERMISSIVE mode
+var KumaALPNProtocols = []string{"kuma"}
+
 func MeshSpiffeIDPrefix(mesh string) string {
 	return fmt.Sprintf("spiffe://%s/", mesh)
 }

diff --git a/pkg/xds/envoy/tls/v3/tls.go b/pkg/xds/envoy/tls/v3/tls.go
@@ -50,6 +50,7 @@ func CreateUpstreamTlsContext(ctx xds_context.Context, upstreamService string, s
 	if err != nil {
 		return nil, err
 	}
+	commonTlsContext.AlpnProtocols = xds_tls.KumaALPNProtocols
 	return &envoy_tls.UpstreamTlsContext{
 		CommonTlsContext: commonTlsContext,
 		Sni:              sni,

diff --git a/pkg/xds/envoy/tls/v3/tls_test.go b/pkg/xds/envoy/tls/v3/tls_test.go
@@ -165,6 +165,8 @@ var _ = Describe("CreateUpstreamTlsContext()", func() {
 				upstreamService: "backend",
 				expected: `
                 commonTlsContext:
+                  alpnProtocols:
+                  - kuma
                   combinedValidationContext:
                     defaultValidationContext:
                       matchSubjectAltNames:

diff --git a/pkg/xds/generator/admin_proxy_generator.go b/pkg/xds/generator/admin_proxy_generator.go
@@ -82,7 +82,7 @@ func (g AdminProxyGenerator) Generate(ctx xds_context.Context, proxy *core_xds.P
 				se.HeaderExactMatch = fmt.Sprintf("Bearer %s", token)
 			}
 			filterChains = append(filterChains, envoy_listeners.FilterChain(envoy_listeners.NewFilterChainBuilder(proxy.APIVersion).
-				Configure(envoy_listeners.FilterChainMatch("tls")).
+				Configure(envoy_listeners.FilterChainMatch("tls", nil, nil)).
 				Configure(envoy_listeners.StaticTlsEndpoints(envoy_names.GetAdminListenerName(), ctx.ControlPlane.AdminProxyKeyPair, staticTlsEndpointPaths)),
 			))
 		}

diff --git a/pkg/xds/generator/inbound_proxy_generator.go b/pkg/xds/generator/inbound_proxy_generator.go
@@ -14,6 +14,7 @@ import (
 	envoy_names "github.com/kumahq/kuma/pkg/xds/envoy/names"
 	v3 "github.com/kumahq/kuma/pkg/xds/envoy/routes/v3"
 	"github.com/kumahq/kuma/pkg/xds/envoy/tags"
+	xds_tls "github.com/kumahq/kuma/pkg/xds/envoy/tls"
 )
 
 // OriginInbound is a marker to indicate by which ProxyGenerator resources were generated.
@@ -65,7 +66,7 @@ func (g InboundProxyGenerator) Generate(ctx xds_context.Context, proxy *model.Pr
 		// generate LDS resource
 		service := iface.GetService()
 		inboundListenerName := envoy_names.GetInboundListenerName(endpoint.DataplaneIP, endpoint.DataplanePort)
-		filterChainBuilder := func() *envoy_listeners.FilterChainBuilder {
+		filterChainBuilder := func(serverSideMTLS bool) *envoy_listeners.FilterChainBuilder {
 			filterChainBuilder := envoy_listeners.NewFilterChainBuilder(proxy.APIVersion)
 			switch protocol {
 			// configuration for HTTP case
@@ -94,15 +95,36 @@ func (g InboundProxyGenerator) Generate(ctx xds_context.Context, proxy *model.Pr
 				// configuration for non-HTTP cases
 				filterChainBuilder.Configure(envoy_listeners.TcpProxy(localClusterName, envoy_common.NewCluster(envoy_common.WithService(localClusterName))))
 			}
+			if serverSideMTLS {
+				filterChainBuilder.
+					Configure(envoy_listeners.ServerSideMTLS(ctx))
+			}
 			return filterChainBuilder.
-				Configure(envoy_listeners.ServerSideMTLS(ctx)).
 				Configure(envoy_listeners.NetworkRBAC(inboundListenerName, ctx.Mesh.Resource.MTLSEnabled(), proxy.Policies.TrafficPermissions[endpoint]))
-		}()
-		inboundListener, err := envoy_listeners.NewListenerBuilder(proxy.APIVersion).
+		}
+
+		listenerBuilder := envoy_listeners.NewListenerBuilder(proxy.APIVersion).
 			Configure(envoy_listeners.InboundListener(inboundListenerName, endpoint.DataplaneIP, endpoint.DataplanePort, model.SocketAddressProtocolTCP)).
-			Configure(envoy_listeners.FilterChain(filterChainBuilder)).
-			Configure(envoy_listeners.TransparentProxying(proxy.Dataplane.Spec.Networking.GetTransparentProxying())).
-			Build()
+			Configure(envoy_listeners.TLSInspector()).
+			Configure(envoy_listeners.TransparentProxying(proxy.Dataplane.Spec.Networking.GetTransparentProxying()))
+
+		switch ctx.Mesh.Resource.GetEnabledCertificateAuthorityBackend().GetMode() {
+		case mesh_proto.CertificateAuthorityBackend_STRICT:
+			listenerBuilder.
+				Configure(envoy_listeners.FilterChain(filterChainBuilder(true)))
+		case mesh_proto.CertificateAuthorityBackend_PERMISSIVE:
+			listenerBuilder.
+				Configure(envoy_listeners.FilterChain(filterChainBuilder(false).
+					Configure(envoy_listeners.FilterChainMatch("raw_buffer", nil, nil)))).
+				Configure(envoy_listeners.FilterChain(filterChainBuilder(false).
+					Configure(envoy_listeners.FilterChainMatch("tls", nil, nil)))).
+				Configure(envoy_listeners.FilterChain(filterChainBuilder(true).
+					Configure(envoy_listeners.FilterChainMatch("tls", nil, xds_tls.KumaALPNProtocols))))
+		default:
+			return nil, errors.New("unknown mode for CA backend")
+		}
+
+		inboundListener, err := listenerBuilder.Build()
 		if err != nil {
 			return nil, errors.Wrapf(err, "%s: could not generate listener %s", validators.RootedAt("dataplane").Field("networking").Field("inbound").Index(i), inboundListenerName)
 		}

diff --git a/pkg/xds/generator/inbound_proxy_generator_test.go b/pkg/xds/generator/inbound_proxy_generator_test.go
@@ -26,6 +26,7 @@ var _ = Describe("InboundProxyGenerator", func() {
 	type testCase struct {
 		dataplaneFile string
 		expected      string
+		mode          mesh_proto.CertificateAuthorityBackend_Mode
 	}
 
 	DescribeTable("Generate Envoy xDS resources",
@@ -48,6 +49,7 @@ var _ = Describe("InboundProxyGenerator", func() {
 									{
 										Name: "builtin",
 										Type: "builtin",
+										Mode: given.mode,
 									},
 								},
 							},
@@ -235,5 +237,15 @@ var _ = Describe("InboundProxyGenerator", func() {
 			dataplaneFile: "4-dataplane.input.yaml",
 			expected:      "4-envoy-config.golden.yaml",
 		}),
+		Entry("05. transparent_proxying=false, ip_addresses=2, ports=2, mode=permissive", testCase{
+			dataplaneFile: "5-dataplane.input.yaml",
+			expected:      "5-envoy-config.golden.yaml",
+			mode:          mesh_proto.CertificateAuthorityBackend_PERMISSIVE,
+		}),
+		Entry("06. transparent_proxying=true, ip_addresses=2, ports=2, mode=permissive", testCase{
+			dataplaneFile: "6-dataplane.input.yaml",
+			expected:      "6-envoy-config.golden.yaml",
+			mode:          mesh_proto.CertificateAuthorityBackend_PERMISSIVE,
+		}),
 	)
 })
diff --git a/pkg/xds/generator/ingress_generator.go b/pkg/xds/generator/ingress_generator.go
@@ -97,7 +97,7 @@ func (i IngressGenerator) generateLDS(
 			sniUsed[sni] = true
 			inboundListenerBuilder = inboundListenerBuilder.
 				Configure(envoy_listeners.FilterChain(envoy_listeners.NewFilterChainBuilder(apiVersion).
-					Configure(envoy_listeners.FilterChainMatch("tls", sni)).
+					Configure(envoy_listeners.FilterChainMatch("tls", []string{sni}, nil)).
 					Configure(envoy_listeners.TcpProxyWithMetadata(service, envoy_common.NewCluster(
 						envoy_common.WithService(service),
 						envoy_common.WithTags(meshDestination.WithoutTags(mesh_proto.ServiceTag)),

diff --git a/pkg/xds/generator/testdata/inbound-proxy/3-envoy-config.golden.yaml b/pkg/xds/generator/testdata/inbound-proxy/3-envoy-config.golden.yaml
@@ -70,6 +70,10 @@ resources:
                 ads: {}
                 resourceApiVersion: V3
           requireClientCertificate: true
+    listenerFilters:
+    - name: envoy.filters.listener.tls_inspector
+      typedConfig:
+        '@type': type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector
     name: inbound:192.168.0.1:443
     trafficDirection: INBOUND
 - name: inbound:192.168.0.1:80
@@ -218,6 +222,10 @@ resources:
                 ads: {}
                 resourceApiVersion: V3
           requireClientCertificate: true
+    listenerFilters:
+    - name: envoy.filters.listener.tls_inspector
+      typedConfig:
+        '@type': type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector
     name: inbound:192.168.0.1:80
     trafficDirection: INBOUND
 - name: inbound:192.168.0.2:443
@@ -259,6 +267,10 @@ resources:
                 ads: {}
                 resourceApiVersion: V3
           requireClientCertificate: true
+    listenerFilters:
+    - name: envoy.filters.listener.tls_inspector
+      typedConfig:
+        '@type': type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector
     name: inbound:192.168.0.2:443
     trafficDirection: INBOUND
 - name: inbound:192.168.0.2:80
@@ -319,5 +331,9 @@ resources:
                 ads: {}
                 resourceApiVersion: V3
           requireClientCertificate: true
+    listenerFilters:
+    - name: envoy.filters.listener.tls_inspector
+      typedConfig:
+        '@type': type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector
     name: inbound:192.168.0.2:80
     trafficDirection: INBOUND
diff --git a/pkg/xds/generator/testdata/inbound-proxy/4-envoy-config.golden.yaml b/pkg/xds/generator/testdata/inbound-proxy/4-envoy-config.golden.yaml
@@ -71,6 +71,10 @@ resources:
                 ads: {}
                 resourceApiVersion: V3
           requireClientCertificate: true
+    listenerFilters:
+    - name: envoy.filters.listener.tls_inspector
+      typedConfig:
+        '@type': type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector
     name: inbound:192.168.0.1:443
     trafficDirection: INBOUND
 - name: inbound:192.168.0.1:80
@@ -220,6 +224,10 @@ resources:
                 ads: {}
                 resourceApiVersion: V3
           requireClientCertificate: true
+    listenerFilters:
+    - name: envoy.filters.listener.tls_inspector
+      typedConfig:
+        '@type': type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector
     name: inbound:192.168.0.1:80
     trafficDirection: INBOUND
 - name: inbound:192.168.0.2:443
@@ -262,6 +270,10 @@ resources:
                 ads: {}
                 resourceApiVersion: V3
           requireClientCertificate: true
+    listenerFilters:
+    - name: envoy.filters.listener.tls_inspector
+      typedConfig:
+        '@type': type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector
     name: inbound:192.168.0.2:443
     trafficDirection: INBOUND
 - name: inbound:192.168.0.2:80
@@ -323,5 +335,9 @@ resources:
                 ads: {}
                 resourceApiVersion: V3
           requireClientCertificate: true
+    listenerFilters:
+    - name: envoy.filters.listener.tls_inspector
+      typedConfig:
+        '@type': type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector
     name: inbound:192.168.0.2:80
     trafficDirection: INBOUND
diff --git a/pkg/xds/generator/testdata/inbound-proxy/5-dataplane.input.yaml b/pkg/xds/generator/testdata/inbound-proxy/5-dataplane.input.yaml
@@ -0,0 +1,23 @@
+networking:
+  address: 192.168.0.1
+  inbound:
+    - port: 80
+      servicePort: 8080
+      tags:
+        kuma.io/service: backend1
+        kuma.io/protocol: http
+    - port: 443
+      servicePort: 8443
+      tags:
+        kuma.io/service: backend2
+    - address: 192.168.0.2
+      port: 80
+      servicePort: 8080
+      tags:
+        kuma.io/service: backend3
+        kuma.io/protocol: http
+    - address: 192.168.0.2
+      port: 443
+      servicePort: 8443
+      tags:
+        kuma.io/service: backend4