feat(kuma-cp) permissive mTLS mode

api/mesh/v1alpha1/mesh.proto

@@ -85,11 +85,11 @@ message CertificateAuthorityBackend {
   google.protobuf.Struct conf = 4;
 
   enum Mode {
-    // A STRICT mode implies server validates the connection and accepts only
-    // encrypted TLS traffic
+    // A STRICT mode implies that the server validates the connection and
+    // accepts only encrypted TLS traffic
     STRICT = 0;
-    // A PERMISSIVE mode implies outbounds encrypt traffic the same way it
-    // happens in strict mode, but inbounds accept both TLS and plaintext
+    // A PERMISSIVE mode implies that the outbounds encrypt traffic the same way
+    // it happens in strict mode, but inbounds accept both TLS and plaintext
     // traffic. This allows applications residing in the mesh to accept requests
     // from outside of the mesh.
     PERMISSIVE = 1;

pkg/xds/envoy/clusters/v3/client_side_mtls_configurer_test.go

@@ -81,6 +81,8 @@ var _ = Describe("EdsClusterConfigurer", func() {
               typedConfig:
                 '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
                 commonTlsContext:
+                  alpnProtocols:
+                  - kuma
                   combinedValidationContext:
                     defaultValidationContext:
                       matchSubjectAltNames:
@@ -146,6 +148,8 @@ var _ = Describe("EdsClusterConfigurer", func() {
                 typedConfig:
                   '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
                   commonTlsContext:
+                    alpnProtocols:
+                    - kuma
                     combinedValidationContext:
                       defaultValidationContext:
                         matchSubjectAltNames:
@@ -169,6 +173,8 @@ var _ = Describe("EdsClusterConfigurer", func() {
                 typedConfig:
                   '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
                   commonTlsContext:
+                    alpnProtocols:
+                    - kuma
                     combinedValidationContext:
                       defaultValidationContext:
                         matchSubjectAltNames:
@@ -230,6 +236,8 @@ var _ = Describe("EdsClusterConfigurer", func() {
               typedConfig:
                 '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
                 commonTlsContext:
+                  alpnProtocols:
+                  - kuma
                   combinedValidationContext:
                     defaultValidationContext:
                       matchSubjectAltNames:

pkg/xds/envoy/listeners/configurers.go

@@ -67,10 +67,11 @@ func HttpConnectionManager(statsName string, forwardClientCertDetails bool) Filt
 	})
 }
 
-func FilterChainMatch(transport string, serverNames ...string) FilterChainBuilderOpt {
+func FilterChainMatch(transport string, serverNames, applicationProtocols []string) FilterChainBuilderOpt {
 	return AddFilterChainConfigurer(&v3.FilterChainMatchConfigurer{
-		ServerNames:       serverNames,
-		TransportProtocol: transport,
+		ServerNames:          serverNames,
+		TransportProtocol:    transport,
+		ApplicationProtocols: applicationProtocols,
 	})
 }
 

pkg/xds/envoy/listeners/v3/filter_chain_match_configurer.go

@@ -3,8 +3,9 @@ package v3
 import envoy_listener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 
 type FilterChainMatchConfigurer struct {
-	ServerNames       []string
-	TransportProtocol string
+	ServerNames          []string
+	TransportProtocol    string
+	ApplicationProtocols []string
 }
 
 func (f *FilterChainMatchConfigurer) Configure(filterChain *envoy_listener.FilterChain) error {
@@ -14,5 +15,8 @@ func (f *FilterChainMatchConfigurer) Configure(filterChain *envoy_listener.Filte
 	if f.TransportProtocol != "" {
 		filterChain.FilterChainMatch.TransportProtocol = f.TransportProtocol
 	}
+	if len(f.ApplicationProtocols) != 0 {
+		filterChain.FilterChainMatch.ApplicationProtocols = f.ApplicationProtocols
+	}
 	return nil
 }

pkg/xds/envoy/tls/v3/tls.go

@@ -11,6 +11,11 @@ import (
 	xds_tls "github.com/kumahq/kuma/pkg/xds/envoy/tls"
 )
 
+// KumaALPNProtocols are set for UpstreamTlsContext to show that mTLS is created by mesh.
+// On the inbound side we have to distinguish Kuma mTLS and application TLS to properly
+// support PERMISSIVE mode
+var KumaALPNProtocols = []string{"kuma"}
+
 // CreateDownstreamTlsContext creates DownstreamTlsContext for incoming connections
 // It verifies that incoming connection has TLS certificate signed by Mesh CA with URI SAN of prefix spiffe://{mesh_name}/
 // It secures inbound listener with certificate of "identity_cert" that will be received from the SDS (it contains URI SANs of all inbounds).
@@ -50,6 +55,7 @@ func CreateUpstreamTlsContext(ctx xds_context.Context, upstreamService string, s
 	if err != nil {
 		return nil, err
 	}
+	commonTlsContext.AlpnProtocols = KumaALPNProtocols
 	return &envoy_tls.UpstreamTlsContext{
 		CommonTlsContext: commonTlsContext,
 		Sni:              sni,

pkg/xds/envoy/tls/v3/tls_test.go

@@ -165,6 +165,8 @@ var _ = Describe("CreateUpstreamTlsContext()", func() {
 				upstreamService: "backend",
 				expected: `
                 commonTlsContext:
+                  alpnProtocols:
+                  - kuma
                   combinedValidationContext:
                     defaultValidationContext:
                       matchSubjectAltNames:

pkg/xds/generator/admin_proxy_generator.go

@@ -82,7 +82,7 @@ func (g AdminProxyGenerator) Generate(ctx xds_context.Context, proxy *core_xds.P
 				se.HeaderExactMatch = fmt.Sprintf("Bearer %s", token)
 			}
 			filterChains = append(filterChains, envoy_listeners.FilterChain(envoy_listeners.NewFilterChainBuilder(proxy.APIVersion).
-				Configure(envoy_listeners.FilterChainMatch("tls")).
+				Configure(envoy_listeners.FilterChainMatch("tls", nil, nil)).
 				Configure(envoy_listeners.StaticTlsEndpoints(envoy_names.GetAdminListenerName(), ctx.ControlPlane.AdminProxyKeyPair, staticTlsEndpointPaths)),
 			))
 		}

pkg/xds/generator/inbound_proxy_generator.go

@@ -14,6 +14,7 @@ import (
 	envoy_names "github.com/kumahq/kuma/pkg/xds/envoy/names"
 	v3 "github.com/kumahq/kuma/pkg/xds/envoy/routes/v3"
 	"github.com/kumahq/kuma/pkg/xds/envoy/tags"
+	xds_tls "github.com/kumahq/kuma/pkg/xds/envoy/tls/v3"
 )
 
 // OriginInbound is a marker to indicate by which ProxyGenerator resources were generated.
@@ -114,9 +115,13 @@ func (g InboundProxyGenerator) Generate(ctx xds_context.Context, proxy *model.Pr
 		case mesh_proto.CertificateAuthorityBackend_PERMISSIVE:
 			listenerBuilder.
 				Configure(envoy_listeners.FilterChain(filterChainBuilder(false).
-					Configure(envoy_listeners.FilterChainMatch("raw_buffer")))).
+					Configure(envoy_listeners.FilterChainMatch("raw_buffer", nil, nil)))).
+				Configure(envoy_listeners.FilterChain(filterChainBuilder(false).
+					Configure(envoy_listeners.FilterChainMatch("tls", nil, nil)))).
 				Configure(envoy_listeners.FilterChain(filterChainBuilder(true).
-					Configure(envoy_listeners.FilterChainMatch("tls"))))
+					Configure(envoy_listeners.FilterChainMatch("tls", nil, xds_tls.KumaALPNProtocols))))
+		default:
+			return nil, errors.New("unknown mode for CA backend")
 		}
 
 		inboundListener, err := listenerBuilder.Build()

pkg/xds/generator/ingress_generator.go

@@ -97,7 +97,7 @@ func (i IngressGenerator) generateLDS(
 			sniUsed[sni] = true
 			inboundListenerBuilder = inboundListenerBuilder.
 				Configure(envoy_listeners.FilterChain(envoy_listeners.NewFilterChainBuilder(apiVersion).
-					Configure(envoy_listeners.FilterChainMatch("tls", sni)).
+					Configure(envoy_listeners.FilterChainMatch("tls", []string{sni}, nil)).
 					Configure(envoy_listeners.TcpProxyWithMetadata(service, envoy_common.NewCluster(
 						envoy_common.WithService(service),
 						envoy_common.WithTags(meshDestination.WithoutTags(mesh_proto.ServiceTag)),

pkg/xds/generator/testdata/inbound-proxy/5-envoy-config.golden.yaml

@@ -65,6 +65,21 @@ resources:
           '@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
           cluster: localhost:8443
           statPrefix: localhost_8443
+    - filterChainMatch:
+        applicationProtocols:
+        - kuma
+        transportProtocol: tls
+      filters:
+      - name: envoy.filters.network.rbac
+        typedConfig:
+          '@type': type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC
+          rules: {}
+          statPrefix: inbound_192_168_0_1_443.
+      - name: envoy.filters.network.tcp_proxy
+        typedConfig:
+          '@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
+          cluster: localhost:8443
+          statPrefix: localhost_8443
       transportSocket:
         name: envoy.transport_sockets.tls
         typedConfig:
@@ -339,6 +354,128 @@ resources:
           setCurrentClientCertDetails:
             uri: true
           statPrefix: localhost_8080
+    - filterChainMatch:
+        applicationProtocols:
+        - kuma
+        transportProtocol: tls
+      filters:
+      - name: envoy.filters.network.rbac
+        typedConfig:
+          '@type': type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC
+          rules:
+            policies:
+              tp-1:
+                permissions:
+                - any: true
+                principals:
+                - andIds:
+                    ids:
+                    - authenticated:
+                        principalName:
+                          exact: kuma://version/1.0
+                    - authenticated:
+                        principalName:
+                          exact: spiffe://default/web1
+          statPrefix: inbound_192_168_0_1_80.
+      - name: envoy.filters.network.http_connection_manager
+        typedConfig:
+          '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          forwardClientCertDetails: SANITIZE_SET
+          httpFilters:
+          - name: envoy.filters.http.local_ratelimit
+            typedConfig:
+              '@type': type.googleapis.com/envoy.extensions.filters.http.local_ratelimit.v3.LocalRateLimit
+              statPrefix: rate_limit
+          - name: envoy.filters.http.fault
+            typedConfig:
+              '@type': type.googleapis.com/envoy.extensions.filters.http.fault.v3.HTTPFault
+              delay:
+                fixedDelay: 5s
+                percentage:
+                  numerator: 50
+              headers:
+              - name: x-kuma-tags
+                safeRegexMatch:
+                  googleRe2: {}
+                  regex: .*&kuma.io/service=[^&]*frontend[,&].*
+          - name: envoy.filters.http.router
+          routeConfig:
+            name: inbound:backend1
+            requestHeadersToRemove:
+            - x-kuma-tags
+            validateClusters: false
+            virtualHosts:
+            - domains:
+              - '*'
+              name: backend1
+              routes:
+              - match:
+                  headers:
+                  - name: x-kuma-tags
+                    safeRegexMatch:
+                      googleRe2: {}
+                      regex: .*&kuma.io/service=[^&]*frontend[,&].*
+                  prefix: /
+                route:
+                  cluster: localhost:8080
+                  timeout: 0s
+                typedPerFilterConfig:
+                  envoy.filters.http.local_ratelimit:
+                    '@type': type.googleapis.com/envoy.extensions.filters.http.local_ratelimit.v3.LocalRateLimit
+                    filterEnabled:
+                      defaultValue:
+                        numerator: 100
+                      runtimeKey: local_rate_limit_enabled
+                    filterEnforced:
+                      defaultValue:
+                        numerator: 100
+                      runtimeKey: local_rate_limit_enforced
+                    statPrefix: rate_limit
+                    tokenBucket:
+                      fillInterval: 10s
+                      maxTokens: 200
+                      tokensPerFill: 200
+              - match:
+                  headers:
+                  - name: x-kuma-tags
+                    safeRegexMatch:
+                      googleRe2: {}
+                      regex: .*&kuma.io/service=.*
+                  prefix: /
+                route:
+                  cluster: localhost:8080
+                  timeout: 0s
+                typedPerFilterConfig:
+                  envoy.filters.http.local_ratelimit:
+                    '@type': type.googleapis.com/envoy.extensions.filters.http.local_ratelimit.v3.LocalRateLimit
+                    filterEnabled:
+                      defaultValue:
+                        numerator: 100
+                      runtimeKey: local_rate_limit_enabled
+                    filterEnforced:
+                      defaultValue:
+                        numerator: 100
+                      runtimeKey: local_rate_limit_enforced
+                    responseHeadersToAdd:
+                    - append: false
+                      header:
+                        key: x-rate-limited
+                        value: "true"
+                    statPrefix: rate_limit
+                    status:
+                      code: NotFound
+                    tokenBucket:
+                      fillInterval: 2s
+                      maxTokens: 100
+                      tokensPerFill: 100
+              - match:
+                  prefix: /
+                route:
+                  cluster: localhost:8080
+                  timeout: 0s
+          setCurrentClientCertDetails:
+            uri: true
+          statPrefix: localhost_8080
       transportSocket:
         name: envoy.transport_sockets.tls
         typedConfig:
@@ -399,6 +536,21 @@ resources:
           '@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
           cluster: localhost:8443
           statPrefix: localhost_8443
+    - filterChainMatch:
+        applicationProtocols:
+        - kuma
+        transportProtocol: tls
+      filters:
+      - name: envoy.filters.network.rbac
+        typedConfig:
+          '@type': type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC
+          rules: {}
+          statPrefix: inbound_192_168_0_2_443.
+      - name: envoy.filters.network.tcp_proxy
+        typedConfig:
+          '@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
+          cluster: localhost:8443
+          statPrefix: localhost_8443
       transportSocket:
         name: envoy.transport_sockets.tls
         typedConfig:
@@ -497,6 +649,40 @@ resources:
           setCurrentClientCertDetails:
             uri: true
           statPrefix: localhost_8080
+    - filterChainMatch:
+        applicationProtocols:
+        - kuma
+        transportProtocol: tls
+      filters:
+      - name: envoy.filters.network.rbac
+        typedConfig:
+          '@type': type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC
+          rules: {}
+          statPrefix: inbound_192_168_0_2_80.
+      - name: envoy.filters.network.http_connection_manager
+        typedConfig:
+          '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          forwardClientCertDetails: SANITIZE_SET
+          httpFilters:
+          - name: envoy.filters.http.router
+          routeConfig:
+            name: inbound:backend3
+            requestHeadersToRemove:
+            - x-kuma-tags
+            validateClusters: false
+            virtualHosts:
+            - domains:
+              - '*'
+              name: backend3
+              routes:
+              - match:
+                  prefix: /
+                route:
+                  cluster: localhost:8080
+                  timeout: 0s
+          setCurrentClientCertDetails:
+            uri: true
+          statPrefix: localhost_8080
       transportSocket:
         name: envoy.transport_sockets.tls
         typedConfig: