From 9675229423284bde81ce8127df2a7a857165b3fb Mon Sep 17 00:00:00 2001 From: Yaroslav Skopets Date: Sat, 15 Feb 2020 13:43:02 +0100 Subject: [PATCH] refactoring: introduce FilterChainBuilder in addition to ListenerBuilder --- .../envoy/listeners/filter_chain_builder.go | 67 +++++++ .../listeners/filter_chain_configurer.go | 26 +++ .../listeners/filter_chain_configurer_test.go | 54 +++++ .../http_connection_manager_configurer.go | 22 +-- ...http_connection_manager_configurer_test.go | 3 +- .../http_inbound_route_configurer.go | 13 +- .../http_inbound_route_configurer_test.go | 10 +- .../listeners/inbound_listener_configurer.go | 5 +- .../inbound_listener_configurer_test.go | 2 - .../{builder.go => listener_builder.go} | 2 +- .../network_access_log_configurer.go | 10 +- .../network_access_log_configurer_test.go | 14 +- .../listeners/network_rbac_configurer.go | 45 ++--- .../listeners/network_rbac_configurer_test.go | 12 +- .../original_dst_forwarder_configurer_test.go | 8 +- .../listeners/outbound_listener_configurer.go | 5 +- .../outbound_listener_configurer_test.go | 2 - .../prometheus_endpoint_configurer.go | 25 ++- .../prometheus_endpoint_configurer_test.go | 3 +- .../envoy/listeners/server_mtls_configurer.go | 19 +- .../listeners/server_mtls_configurer_test.go | 12 +- .../envoy/listeners/tcp_proxy_configurer.go | 31 ++- .../listeners/tcp_proxy_configurer_test.go | 10 +- .../transparent_proxying_configurer_test.go | 4 - pkg/xds/envoy/listeners/util.go | 48 +++-- pkg/xds/envoy/listeners/util_test.go | 185 +++++------------- pkg/xds/envoy/types.go | 7 + .../prometheus_endpoint_generator.go | 3 +- pkg/xds/generator/proxy_template.go | 28 +-- 29 files changed, 365 insertions(+), 310 deletions(-) create mode 100644 pkg/xds/envoy/listeners/filter_chain_builder.go create mode 100644 pkg/xds/envoy/listeners/filter_chain_configurer.go create mode 100644 pkg/xds/envoy/listeners/filter_chain_configurer_test.go rename pkg/xds/envoy/listeners/{builder.go => listener_builder.go} (97%) create mode 100644 pkg/xds/envoy/types.go diff --git a/pkg/xds/envoy/listeners/filter_chain_builder.go b/pkg/xds/envoy/listeners/filter_chain_builder.go new file mode 100644 index 000000000000..ebef1cab3e7b --- /dev/null +++ b/pkg/xds/envoy/listeners/filter_chain_builder.go @@ -0,0 +1,67 @@ +package listeners + +import ( + envoy_listener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener" +) + +// FilterChainConfigurer is responsible for configuring a single aspect of the entire Envoy filter chain, +// such as TcpProxy filter, RBAC filter, access log, etc. +type FilterChainConfigurer interface { + // Configure configures a single aspect on a given Envoy filter chain. + Configure(filterChain *envoy_listener.FilterChain) error +} + +// FilterChainBuilderOpt is a configuration option for FilterChainBuilder. +// +// The goal of FilterChainBuilderOpt is to facilitate fluent FilterChainBuilder API. +type FilterChainBuilderOpt interface { + // ApplyTo adds FilterChainConfigurer(s) to the FilterChainBuilder. + ApplyTo(config *FilterChainBuilderConfig) +} + +func NewFilterChainBuilder() *FilterChainBuilder { + return &FilterChainBuilder{} +} + +// FilterChainBuilder is responsible for generating an Envoy filter chain +// by applying a series of FilterChainConfigurers. +type FilterChainBuilder struct { + config FilterChainBuilderConfig +} + +// Configure configures FilterChainBuilder by adding individual FilterChainConfigurers. +func (b *FilterChainBuilder) Configure(opts ...FilterChainBuilderOpt) *FilterChainBuilder { + for _, opt := range opts { + opt.ApplyTo(&b.config) + } + return b +} + +// Build generates an Envoy filter chain by applying a series of FilterChainConfigurers. +func (b *FilterChainBuilder) Build() (*envoy_listener.FilterChain, error) { + filterChain := envoy_listener.FilterChain{} + for _, configurer := range b.config.Configurers { + if err := configurer.Configure(&filterChain); err != nil { + return nil, err + } + } + return &filterChain, nil +} + +// FilterChainBuilderConfig holds configuration of a FilterChainBuilder. +type FilterChainBuilderConfig struct { + // A series of FilterChainConfigurers to apply to Envoy filter chain. + Configurers []FilterChainConfigurer +} + +// Add appends a given FilterChainConfigurer to the end of the chain. +func (c *FilterChainBuilderConfig) Add(configurer FilterChainConfigurer) { + c.Configurers = append(c.Configurers, configurer) +} + +// FilterChainBuilderOptFunc is a convenience type adapter. +type FilterChainBuilderOptFunc func(config *FilterChainBuilderConfig) + +func (f FilterChainBuilderOptFunc) ApplyTo(config *FilterChainBuilderConfig) { + f(config) +} diff --git a/pkg/xds/envoy/listeners/filter_chain_configurer.go b/pkg/xds/envoy/listeners/filter_chain_configurer.go new file mode 100644 index 000000000000..4db8149833d6 --- /dev/null +++ b/pkg/xds/envoy/listeners/filter_chain_configurer.go @@ -0,0 +1,26 @@ +package listeners + +import ( + v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2" +) + +func FilterChain(builder *FilterChainBuilder) ListenerBuilderOpt { + return ListenerBuilderOptFunc(func(config *ListenerBuilderConfig) { + config.Add(&ListenerFilterChainConfigurer{ + builder: builder, + }) + }) +} + +type ListenerFilterChainConfigurer struct { + builder *FilterChainBuilder +} + +func (c ListenerFilterChainConfigurer) Configure(listener *v2.Listener) error { + filterChain, err := c.builder.Build() + if err != nil { + return err + } + listener.FilterChains = append(listener.FilterChains, filterChain) + return nil +} diff --git a/pkg/xds/envoy/listeners/filter_chain_configurer_test.go b/pkg/xds/envoy/listeners/filter_chain_configurer_test.go new file mode 100644 index 000000000000..c483d66f1d19 --- /dev/null +++ b/pkg/xds/envoy/listeners/filter_chain_configurer_test.go @@ -0,0 +1,54 @@ +package listeners_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + + . "github.com/Kong/kuma/pkg/xds/envoy/listeners" + + util_proto "github.com/Kong/kuma/pkg/util/proto" +) + +var _ = Describe("ListenerFilterChainConfigurer", func() { + + type testCase struct { + listenerName string + listenerAddress string + listenerPort uint32 + expected string + } + + DescribeTable("should generate proper Envoy config", + func(given testCase) { + // when + listener, err := NewListenerBuilder(). + Configure(InboundListener(given.listenerName, given.listenerAddress, given.listenerPort)). + Configure(FilterChain(NewFilterChainBuilder())). + Build() + // then + Expect(err).ToNot(HaveOccurred()) + + // when + actual, err := util_proto.ToYAML(listener) + // then + Expect(err).ToNot(HaveOccurred()) + // and + Expect(actual).To(MatchYAML(given.expected)) + }, + Entry("basic listener with an empty filter chain", testCase{ + listenerName: "inbound:192.168.0.1:8080", + listenerAddress: "192.168.0.1", + listenerPort: 8080, + expected: ` + name: inbound:192.168.0.1:8080 + address: + socketAddress: + address: 192.168.0.1 + portValue: 8080 + filterChains: + - {} +`, + }), + ) +}) diff --git a/pkg/xds/envoy/listeners/http_connection_manager_configurer.go b/pkg/xds/envoy/listeners/http_connection_manager_configurer.go index 0689e3ac53fd..ae0d99e789a0 100644 --- a/pkg/xds/envoy/listeners/http_connection_manager_configurer.go +++ b/pkg/xds/envoy/listeners/http_connection_manager_configurer.go @@ -3,7 +3,6 @@ package listeners import ( "github.com/golang/protobuf/ptypes" - v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2" envoy_listener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener" envoy_hcm "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2" envoy_wellknown "github.com/envoyproxy/go-control-plane/pkg/wellknown" @@ -11,8 +10,8 @@ import ( util_xds "github.com/Kong/kuma/pkg/util/xds" ) -func HttpConnectionManager(statsName string) ListenerBuilderOpt { - return ListenerBuilderOptFunc(func(config *ListenerBuilderConfig) { +func HttpConnectionManager(statsName string) FilterChainBuilderOpt { + return FilterChainBuilderOptFunc(func(config *FilterChainBuilderConfig) { config.Add(&HttpConnectionManagerConfigurer{ statsName: statsName, }) @@ -23,7 +22,7 @@ type HttpConnectionManagerConfigurer struct { statsName string } -func (c *HttpConnectionManagerConfigurer) Configure(l *v2.Listener) error { +func (c *HttpConnectionManagerConfigurer) Configure(filterChain *envoy_listener.FilterChain) error { config := &envoy_hcm.HttpConnectionManager{ StatPrefix: util_xds.SanitizeMetric(c.statsName), CodecType: envoy_hcm.HttpConnectionManager_AUTO, @@ -38,14 +37,11 @@ func (c *HttpConnectionManagerConfigurer) Configure(l *v2.Listener) error { return err } - for i := range l.FilterChains { - l.FilterChains[i].Filters = append(l.FilterChains[i].Filters, &envoy_listener.Filter{ - Name: envoy_wellknown.HTTPConnectionManager, - ConfigType: &envoy_listener.Filter_TypedConfig{ - TypedConfig: pbst, - }, - }) - } - + filterChain.Filters = append(filterChain.Filters, &envoy_listener.Filter{ + Name: envoy_wellknown.HTTPConnectionManager, + ConfigType: &envoy_listener.Filter_TypedConfig{ + TypedConfig: pbst, + }, + }) return nil } diff --git a/pkg/xds/envoy/listeners/http_connection_manager_configurer_test.go b/pkg/xds/envoy/listeners/http_connection_manager_configurer_test.go index ff960be23e11..8da6636a3e13 100644 --- a/pkg/xds/envoy/listeners/http_connection_manager_configurer_test.go +++ b/pkg/xds/envoy/listeners/http_connection_manager_configurer_test.go @@ -25,7 +25,8 @@ var _ = Describe("HttpConnectionManagerConfigurer", func() { // when listener, err := NewListenerBuilder(). Configure(InboundListener(given.listenerName, given.listenerAddress, given.listenerPort)). - Configure(HttpConnectionManager(given.statsName)). + Configure(FilterChain(NewFilterChainBuilder(). + Configure(HttpConnectionManager(given.statsName)))). Build() // then Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/xds/envoy/listeners/http_inbound_route_configurer.go b/pkg/xds/envoy/listeners/http_inbound_route_configurer.go index dc0636eda1c7..ec6a72b9285d 100644 --- a/pkg/xds/envoy/listeners/http_inbound_route_configurer.go +++ b/pkg/xds/envoy/listeners/http_inbound_route_configurer.go @@ -7,13 +7,16 @@ import ( wrappers "github.com/golang/protobuf/ptypes/wrappers" v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2" + envoy_listener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener" envoy_route "github.com/envoyproxy/go-control-plane/envoy/api/v2/route" envoy_hcm "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2" envoy_wellknown "github.com/envoyproxy/go-control-plane/pkg/wellknown" + + envoy_common "github.com/Kong/kuma/pkg/xds/envoy" ) -func HttpInboundRoute(service string, cluster ClusterInfo) ListenerBuilderOpt { - return ListenerBuilderOptFunc(func(config *ListenerBuilderConfig) { +func HttpInboundRoute(service string, cluster envoy_common.ClusterInfo) FilterChainBuilderOpt { + return FilterChainBuilderOptFunc(func(config *FilterChainBuilderConfig) { config.Add(&HttpInboundRouteConfigurer{ service: service, cluster: cluster, @@ -24,13 +27,13 @@ func HttpInboundRoute(service string, cluster ClusterInfo) ListenerBuilderOpt { type HttpInboundRouteConfigurer struct { service string // Cluster to forward traffic to. - cluster ClusterInfo + cluster envoy_common.ClusterInfo } -func (c *HttpInboundRouteConfigurer) Configure(l *v2.Listener) error { +func (c *HttpInboundRouteConfigurer) Configure(filterChain *envoy_listener.FilterChain) error { routeConfig := c.routeConfiguration() - return UpdateFilterConfig(l, envoy_wellknown.HTTPConnectionManager, func(filterConfig proto.Message) error { + return UpdateFilterConfig(filterChain, envoy_wellknown.HTTPConnectionManager, func(filterConfig proto.Message) error { hcm, ok := filterConfig.(*envoy_hcm.HttpConnectionManager) if !ok { return NewUnexpectedFilterConfigTypeError(filterConfig, &envoy_hcm.HttpConnectionManager{}) diff --git a/pkg/xds/envoy/listeners/http_inbound_route_configurer_test.go b/pkg/xds/envoy/listeners/http_inbound_route_configurer_test.go index fd1992b078e4..569395696f92 100644 --- a/pkg/xds/envoy/listeners/http_inbound_route_configurer_test.go +++ b/pkg/xds/envoy/listeners/http_inbound_route_configurer_test.go @@ -8,6 +8,7 @@ import ( . "github.com/Kong/kuma/pkg/xds/envoy/listeners" util_proto "github.com/Kong/kuma/pkg/util/proto" + envoy_common "github.com/Kong/kuma/pkg/xds/envoy" ) var _ = Describe("HttpInboundRouteConfigurer", func() { @@ -18,7 +19,7 @@ var _ = Describe("HttpInboundRouteConfigurer", func() { listenerPort uint32 statsName string service string - cluster ClusterInfo + cluster envoy_common.ClusterInfo expected string } @@ -27,8 +28,9 @@ var _ = Describe("HttpInboundRouteConfigurer", func() { // when listener, err := NewListenerBuilder(). Configure(InboundListener(given.listenerName, given.listenerAddress, given.listenerPort)). - Configure(HttpConnectionManager(given.statsName)). - Configure(HttpInboundRoute(given.service, given.cluster)). + Configure(FilterChain(NewFilterChainBuilder(). + Configure(HttpConnectionManager(given.statsName)). + Configure(HttpInboundRoute(given.service, given.cluster)))). Build() // then Expect(err).ToNot(HaveOccurred()) @@ -45,7 +47,7 @@ var _ = Describe("HttpInboundRouteConfigurer", func() { listenerPort: 8080, statsName: "localhost:8080", service: "backend", - cluster: ClusterInfo{Name: "localhost:8080", Weight: 200}, + cluster: envoy_common.ClusterInfo{Name: "localhost:8080", Weight: 200}, expected: ` name: inbound:192.168.0.1:8080 address: diff --git a/pkg/xds/envoy/listeners/inbound_listener_configurer.go b/pkg/xds/envoy/listeners/inbound_listener_configurer.go index e2332de31c7b..9a4f8ccf9e2f 100644 --- a/pkg/xds/envoy/listeners/inbound_listener_configurer.go +++ b/pkg/xds/envoy/listeners/inbound_listener_configurer.go @@ -3,7 +3,6 @@ package listeners import ( v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2" envoy_core "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - envoy_listener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener" ) func InboundListener(listenerName string, address string, port uint32) ListenerBuilderOpt { @@ -35,9 +34,7 @@ func (c *InboundListenerConfigurer) Configure(l *v2.Listener) error { }, }, } - l.FilterChains = []*envoy_listener.FilterChain{ - {}, // 1 filter chain that will be configured later on - } + // notice that filter chain configuration is left up to other configurers return nil } diff --git a/pkg/xds/envoy/listeners/inbound_listener_configurer_test.go b/pkg/xds/envoy/listeners/inbound_listener_configurer_test.go index a965ba891c4e..601b95e74d80 100644 --- a/pkg/xds/envoy/listeners/inbound_listener_configurer_test.go +++ b/pkg/xds/envoy/listeners/inbound_listener_configurer_test.go @@ -44,8 +44,6 @@ var _ = Describe("InboundListenerConfigurer", func() { socketAddress: address: 192.168.0.1 portValue: 8080 - filterChains: - - {} `, }), ) diff --git a/pkg/xds/envoy/listeners/builder.go b/pkg/xds/envoy/listeners/listener_builder.go similarity index 97% rename from pkg/xds/envoy/listeners/builder.go rename to pkg/xds/envoy/listeners/listener_builder.go index 2e88dbaac724..c7f7ea3bcf1a 100644 --- a/pkg/xds/envoy/listeners/builder.go +++ b/pkg/xds/envoy/listeners/listener_builder.go @@ -5,7 +5,7 @@ import ( ) // ListenerConfigurer is responsible for configuring a single aspect of the entire Envoy listener, -// such as TcpProxy filter, RBAC filter, access log, etc. +// such as filter chain, transparent proxying, etc. type ListenerConfigurer interface { // Configure configures a single aspect on a given Envoy listener. Configure(listener *v2.Listener) error diff --git a/pkg/xds/envoy/listeners/network_access_log_configurer.go b/pkg/xds/envoy/listeners/network_access_log_configurer.go index b336c3e1a858..9ac68f6c4080 100644 --- a/pkg/xds/envoy/listeners/network_access_log_configurer.go +++ b/pkg/xds/envoy/listeners/network_access_log_configurer.go @@ -9,8 +9,8 @@ import ( "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes" - v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2" envoy_core "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" + envoy_listener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener" accesslog "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v2" filter_accesslog "github.com/envoyproxy/go-control-plane/envoy/config/filter/accesslog/v2" envoy_tcp "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/tcp_proxy/v2" @@ -20,8 +20,8 @@ import ( core_xds "github.com/Kong/kuma/pkg/core/xds" ) -func NetworkAccessLog(sourceService string, destinationService string, backend *v1alpha1.LoggingBackend, proxy *core_xds.Proxy) ListenerBuilderOpt { - return ListenerBuilderOptFunc(func(config *ListenerBuilderConfig) { +func NetworkAccessLog(sourceService string, destinationService string, backend *v1alpha1.LoggingBackend, proxy *core_xds.Proxy) FilterChainBuilderOpt { + return FilterChainBuilderOptFunc(func(config *FilterChainBuilderConfig) { if backend != nil { config.Add(&NetworkAccessLogConfigurer{ sourceService: sourceService, @@ -40,13 +40,13 @@ type NetworkAccessLogConfigurer struct { proxy *core_xds.Proxy } -func (c *NetworkAccessLogConfigurer) Configure(l *v2.Listener) error { +func (c *NetworkAccessLogConfigurer) Configure(filterChain *envoy_listener.FilterChain) error { accessLog, err := convertLoggingBackend(c.sourceService, c.destinationService, c.backend, c.proxy) if err != nil { return err } - return UpdateFilterConfig(l, envoy_wellknown.TCPProxy, func(filterConfig proto.Message) error { + return UpdateFilterConfig(filterChain, envoy_wellknown.TCPProxy, func(filterConfig proto.Message) error { proxy, ok := filterConfig.(*envoy_tcp.TcpProxy) if !ok { return NewUnexpectedFilterConfigTypeError(filterConfig, &envoy_tcp.TcpProxy{}) diff --git a/pkg/xds/envoy/listeners/network_access_log_configurer_test.go b/pkg/xds/envoy/listeners/network_access_log_configurer_test.go index 25429d6a7d8e..8e38b6954a29 100644 --- a/pkg/xds/envoy/listeners/network_access_log_configurer_test.go +++ b/pkg/xds/envoy/listeners/network_access_log_configurer_test.go @@ -12,6 +12,7 @@ import ( "github.com/Kong/kuma/pkg/core/xds" core_xds "github.com/Kong/kuma/pkg/core/xds" util_proto "github.com/Kong/kuma/pkg/util/proto" + envoy_common "github.com/Kong/kuma/pkg/xds/envoy" ) var _ = Describe("NetworkAccessLogConfigurer", func() { @@ -21,7 +22,7 @@ var _ = Describe("NetworkAccessLogConfigurer", func() { listenerAddress string listenerPort uint32 statsName string - clusters []ClusterInfo + clusters []envoy_common.ClusterInfo backend *mesh_proto.LoggingBackend expected string } @@ -57,8 +58,9 @@ var _ = Describe("NetworkAccessLogConfigurer", func() { // when listener, err := NewListenerBuilder(). Configure(OutboundListener(given.listenerName, given.listenerAddress, given.listenerPort)). - Configure(TcpProxy(given.statsName, given.clusters...)). - Configure(NetworkAccessLog(sourceService, destinationService, given.backend, proxy)). + Configure(FilterChain(NewFilterChainBuilder(). + Configure(TcpProxy(given.statsName, given.clusters...)). + Configure(NetworkAccessLog(sourceService, destinationService, given.backend, proxy)))). Build() // then Expect(err).ToNot(HaveOccurred()) @@ -74,7 +76,7 @@ var _ = Describe("NetworkAccessLogConfigurer", func() { listenerAddress: "127.0.0.1", listenerPort: 5432, statsName: "db", - clusters: []ClusterInfo{{Name: "db", Weight: 200}}, + clusters: []envoy_common.ClusterInfo{{Name: "db", Weight: 200}}, backend: nil, expected: ` name: outbound:127.0.0.1:5432 @@ -96,7 +98,7 @@ var _ = Describe("NetworkAccessLogConfigurer", func() { listenerAddress: "127.0.0.1", listenerPort: 5432, statsName: "db", - clusters: []ClusterInfo{{Name: "db", Weight: 200}}, + clusters: []envoy_common.ClusterInfo{{Name: "db", Weight: 200}}, backend: &mesh_proto.LoggingBackend{ Name: "file", Type: &mesh_proto.LoggingBackend_File_{ @@ -132,7 +134,7 @@ var _ = Describe("NetworkAccessLogConfigurer", func() { listenerAddress: "127.0.0.1", listenerPort: 5432, statsName: "db", - clusters: []ClusterInfo{{Name: "db", Weight: 200}}, + clusters: []envoy_common.ClusterInfo{{Name: "db", Weight: 200}}, backend: &mesh_proto.LoggingBackend{ Name: "tcp", Format: "custom format", diff --git a/pkg/xds/envoy/listeners/network_rbac_configurer.go b/pkg/xds/envoy/listeners/network_rbac_configurer.go index 0ef08037e246..db20f6e95cea 100644 --- a/pkg/xds/envoy/listeners/network_rbac_configurer.go +++ b/pkg/xds/envoy/listeners/network_rbac_configurer.go @@ -3,48 +3,49 @@ package listeners import ( "fmt" - "github.com/Kong/kuma/api/mesh/v1alpha1" - mesh_core "github.com/Kong/kuma/pkg/core/resources/apis/mesh" - util_xds "github.com/Kong/kuma/pkg/util/xds" - v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2" + "github.com/golang/protobuf/ptypes" + envoy_listener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener" rbac "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/rbac/v2" rbac_config "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v2" + envoy_matcher "github.com/envoyproxy/go-control-plane/envoy/type/matcher" envoy_wellknown "github.com/envoyproxy/go-control-plane/pkg/wellknown" - "github.com/golang/protobuf/ptypes" - envoy_matcher "github.com/envoyproxy/go-control-plane/envoy/type/matcher" + "github.com/Kong/kuma/api/mesh/v1alpha1" + mesh_core "github.com/Kong/kuma/pkg/core/resources/apis/mesh" + util_xds "github.com/Kong/kuma/pkg/util/xds" ) -func NetworkRBAC(rbacEnabled bool, permissions *mesh_core.TrafficPermissionResourceList) ListenerBuilderOpt { - return ListenerBuilderOptFunc(func(config *ListenerBuilderConfig) { +func NetworkRBAC(statsName string, rbacEnabled bool, permissions *mesh_core.TrafficPermissionResourceList) FilterChainBuilderOpt { + return FilterChainBuilderOptFunc(func(config *FilterChainBuilderConfig) { if rbacEnabled { - config.Add(&NetworkRBACConfigurer{permissions}) + config.Add(&NetworkRBACConfigurer{ + statsName: statsName, + permissions: permissions, + }) } }) } type NetworkRBACConfigurer struct { + statsName string // Traffic Permissions to enforce. permissions *mesh_core.TrafficPermissionResourceList } -func (c *NetworkRBACConfigurer) Configure(l *v2.Listener) error { - for i := range l.FilterChains { - filter, err := createRbacFilter(l.Name, c.permissions) - if err != nil { - return err - } - - // RBAC filter should be the first in the chain - l.FilterChains[i].Filters = append([]*envoy_listener.Filter{filter}, l.FilterChains[i].Filters...) +func (c *NetworkRBACConfigurer) Configure(filterChain *envoy_listener.FilterChain) error { + filter, err := createRbacFilter(c.statsName, c.permissions) + if err != nil { + return err } + // RBAC filter should be the first in the chain + filterChain.Filters = append([]*envoy_listener.Filter{filter}, filterChain.Filters...) return nil } -func createRbacFilter(listenerName string, permissions *mesh_core.TrafficPermissionResourceList) (*envoy_listener.Filter, error) { - rbacRule := createRbacRule(listenerName, permissions) +func createRbacFilter(statsName string, permissions *mesh_core.TrafficPermissionResourceList) (*envoy_listener.Filter, error) { + rbacRule := createRbacRule(statsName, permissions) rbacMarshalled, err := ptypes.MarshalAny(rbacRule) if err != nil { return nil, err @@ -57,7 +58,7 @@ func createRbacFilter(listenerName string, permissions *mesh_core.TrafficPermiss }, nil } -func createRbacRule(listenerName string, permissions *mesh_core.TrafficPermissionResourceList) *rbac.RBAC { +func createRbacRule(statsName string, permissions *mesh_core.TrafficPermissionResourceList) *rbac.RBAC { policies := make(map[string]*rbac_config.Policy, len(permissions.Items)) for _, permission := range permissions.Items { policyName := permission.Meta.GetName() @@ -69,7 +70,7 @@ func createRbacRule(listenerName string, permissions *mesh_core.TrafficPermissio Action: rbac_config.RBAC_ALLOW, Policies: policies, }, - StatPrefix: fmt.Sprintf("%s.", util_xds.SanitizeMetric(listenerName)), // we include dot to change "inbound:127.0.0.1:21011rbac.allowed" metric to "inbound:127.0.0.1:21011.rbac.allowed" + StatPrefix: fmt.Sprintf("%s.", util_xds.SanitizeMetric(statsName)), // we include dot to change "inbound:127.0.0.1:21011rbac.allowed" metric to "inbound:127.0.0.1:21011.rbac.allowed" } } diff --git a/pkg/xds/envoy/listeners/network_rbac_configurer_test.go b/pkg/xds/envoy/listeners/network_rbac_configurer_test.go index faf0726bab94..41278f1c8219 100644 --- a/pkg/xds/envoy/listeners/network_rbac_configurer_test.go +++ b/pkg/xds/envoy/listeners/network_rbac_configurer_test.go @@ -13,6 +13,7 @@ import ( test_model "github.com/Kong/kuma/pkg/test/resources/model" util_proto "github.com/Kong/kuma/pkg/util/proto" + envoy_common "github.com/Kong/kuma/pkg/xds/envoy" ) var _ = Describe("NetworkRbacConfigurer", func() { @@ -22,7 +23,7 @@ var _ = Describe("NetworkRbacConfigurer", func() { listenerAddress string listenerPort uint32 statsName string - clusters []ClusterInfo + clusters []envoy_common.ClusterInfo rbacEnabled bool permissions *mesh_core.TrafficPermissionResourceList expected string @@ -33,8 +34,9 @@ var _ = Describe("NetworkRbacConfigurer", func() { // when listener, err := NewListenerBuilder(). Configure(InboundListener(given.listenerName, given.listenerAddress, given.listenerPort)). - Configure(TcpProxy(given.statsName, given.clusters...)). - Configure(NetworkRBAC(given.rbacEnabled, given.permissions)). + Configure(FilterChain(NewFilterChainBuilder(). + Configure(TcpProxy(given.statsName, given.clusters...)). + Configure(NetworkRBAC(given.listenerName, given.rbacEnabled, given.permissions)))). Build() // then Expect(err).ToNot(HaveOccurred()) @@ -50,7 +52,7 @@ var _ = Describe("NetworkRbacConfigurer", func() { listenerAddress: "192.168.0.1", listenerPort: 8080, statsName: "localhost:8080", - clusters: []ClusterInfo{{Name: "localhost:8080", Weight: 200}}, + clusters: []envoy_common.ClusterInfo{{Name: "localhost:8080", Weight: 200}}, rbacEnabled: true, permissions: &mesh_core.TrafficPermissionResourceList{ Items: []*mesh_core.TrafficPermissionResource{ @@ -113,7 +115,7 @@ var _ = Describe("NetworkRbacConfigurer", func() { listenerAddress: "192.168.0.1", listenerPort: 8080, statsName: "localhost:8080", - clusters: []ClusterInfo{{Name: "localhost:8080", Weight: 200}}, + clusters: []envoy_common.ClusterInfo{{Name: "localhost:8080", Weight: 200}}, rbacEnabled: false, permissions: &mesh_core.TrafficPermissionResourceList{ Items: []*mesh_core.TrafficPermissionResource{ diff --git a/pkg/xds/envoy/listeners/original_dst_forwarder_configurer_test.go b/pkg/xds/envoy/listeners/original_dst_forwarder_configurer_test.go index 9dbd00577982..cba748925c47 100644 --- a/pkg/xds/envoy/listeners/original_dst_forwarder_configurer_test.go +++ b/pkg/xds/envoy/listeners/original_dst_forwarder_configurer_test.go @@ -8,6 +8,7 @@ import ( . "github.com/Kong/kuma/pkg/xds/envoy/listeners" util_proto "github.com/Kong/kuma/pkg/util/proto" + envoy_common "github.com/Kong/kuma/pkg/xds/envoy" ) var _ = Describe("OriginalDstForwarderConfigurer", func() { @@ -17,7 +18,7 @@ var _ = Describe("OriginalDstForwarderConfigurer", func() { listenerAddress string listenerPort uint32 statsName string - clusters []ClusterInfo + clusters []envoy_common.ClusterInfo expected string } @@ -26,7 +27,8 @@ var _ = Describe("OriginalDstForwarderConfigurer", func() { // when listener, err := NewListenerBuilder(). Configure(OutboundListener(given.listenerName, given.listenerAddress, given.listenerPort)). - Configure(TcpProxy(given.statsName, given.clusters...)). + Configure(FilterChain(NewFilterChainBuilder(). + Configure(TcpProxy(given.statsName, given.clusters...)))). Configure(OriginalDstForwarder()). Build() // then @@ -43,7 +45,7 @@ var _ = Describe("OriginalDstForwarderConfigurer", func() { listenerAddress: "0.0.0.0", listenerPort: 12345, statsName: "pass_through", - clusters: []ClusterInfo{{Name: "pass_through", Weight: 200}}, + clusters: []envoy_common.ClusterInfo{{Name: "pass_through", Weight: 200}}, expected: ` name: catch_all address: diff --git a/pkg/xds/envoy/listeners/outbound_listener_configurer.go b/pkg/xds/envoy/listeners/outbound_listener_configurer.go index 548dd5ba7853..6d989fd03aab 100644 --- a/pkg/xds/envoy/listeners/outbound_listener_configurer.go +++ b/pkg/xds/envoy/listeners/outbound_listener_configurer.go @@ -3,7 +3,6 @@ package listeners import ( v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2" envoy_core "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - envoy_listener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener" ) func OutboundListener(listenerName string, address string, port uint32) ListenerBuilderOpt { @@ -35,9 +34,7 @@ func (c *OutboundListenerConfigurer) Configure(l *v2.Listener) error { }, }, } - l.FilterChains = []*envoy_listener.FilterChain{ - {}, // 1 filter chain that will be configured later on - } + // notice that filter chain configuration is left up to other configurers return nil } diff --git a/pkg/xds/envoy/listeners/outbound_listener_configurer_test.go b/pkg/xds/envoy/listeners/outbound_listener_configurer_test.go index 1e10035616c8..add84515e3b1 100644 --- a/pkg/xds/envoy/listeners/outbound_listener_configurer_test.go +++ b/pkg/xds/envoy/listeners/outbound_listener_configurer_test.go @@ -44,8 +44,6 @@ var _ = Describe("OutboundListenerConfigurer", func() { socketAddress: address: 192.168.0.1 portValue: 8080 - filterChains: - - {} `, }), ) diff --git a/pkg/xds/envoy/listeners/prometheus_endpoint_configurer.go b/pkg/xds/envoy/listeners/prometheus_endpoint_configurer.go index e2de1ff1b418..30231a64135f 100644 --- a/pkg/xds/envoy/listeners/prometheus_endpoint_configurer.go +++ b/pkg/xds/envoy/listeners/prometheus_endpoint_configurer.go @@ -12,9 +12,10 @@ import ( util_xds "github.com/Kong/kuma/pkg/util/xds" ) -func PrometheusEndpoint(path string, clusterName string) ListenerBuilderOpt { - return ListenerBuilderOptFunc(func(config *ListenerBuilderConfig) { +func PrometheusEndpoint(statsName string, path string, clusterName string) FilterChainBuilderOpt { + return FilterChainBuilderOptFunc(func(config *FilterChainBuilderConfig) { config.Add(&PrometheusEndpointConfigurer{ + statsName: statsName, path: path, clusterName: clusterName, }) @@ -22,13 +23,14 @@ func PrometheusEndpoint(path string, clusterName string) ListenerBuilderOpt { } type PrometheusEndpointConfigurer struct { + statsName string path string clusterName string } -func (c *PrometheusEndpointConfigurer) Configure(l *v2.Listener) error { +func (c *PrometheusEndpointConfigurer) Configure(filterChain *envoy_listener.FilterChain) error { config := &envoy_hcm.HttpConnectionManager{ - StatPrefix: util_xds.SanitizeMetric(l.Name), + StatPrefix: util_xds.SanitizeMetric(c.statsName), CodecType: envoy_hcm.HttpConnectionManager_AUTO, HttpFilters: []*envoy_hcm.HttpFilter{{ Name: envoy_wellknown.Router, @@ -62,14 +64,11 @@ func (c *PrometheusEndpointConfigurer) Configure(l *v2.Listener) error { return err } - for i := range l.FilterChains { - l.FilterChains[i].Filters = append(l.FilterChains[i].Filters, &envoy_listener.Filter{ - Name: envoy_wellknown.HTTPConnectionManager, - ConfigType: &envoy_listener.Filter_TypedConfig{ - TypedConfig: pbst, - }, - }) - } - + filterChain.Filters = append(filterChain.Filters, &envoy_listener.Filter{ + Name: envoy_wellknown.HTTPConnectionManager, + ConfigType: &envoy_listener.Filter_TypedConfig{ + TypedConfig: pbst, + }, + }) return nil } diff --git a/pkg/xds/envoy/listeners/prometheus_endpoint_configurer_test.go b/pkg/xds/envoy/listeners/prometheus_endpoint_configurer_test.go index a7a4d04b83eb..0f000fab33ef 100644 --- a/pkg/xds/envoy/listeners/prometheus_endpoint_configurer_test.go +++ b/pkg/xds/envoy/listeners/prometheus_endpoint_configurer_test.go @@ -26,7 +26,8 @@ var _ = Describe("PrometheusEndpointConfigurer", func() { // when listener, err := NewListenerBuilder(). Configure(InboundListener(given.listenerName, given.listenerAddress, given.listenerPort)). - Configure(PrometheusEndpoint(given.path, given.clusterName)). + Configure(FilterChain(NewFilterChainBuilder(). + Configure(PrometheusEndpoint(given.listenerName, given.path, given.clusterName)))). Build() // then Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/xds/envoy/listeners/server_mtls_configurer.go b/pkg/xds/envoy/listeners/server_mtls_configurer.go index 2539202fc014..be207949c77e 100644 --- a/pkg/xds/envoy/listeners/server_mtls_configurer.go +++ b/pkg/xds/envoy/listeners/server_mtls_configurer.go @@ -1,15 +1,15 @@ package listeners import ( - v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2" + envoy_listener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener" core_xds "github.com/Kong/kuma/pkg/core/xds" xds_context "github.com/Kong/kuma/pkg/xds/context" "github.com/Kong/kuma/pkg/xds/envoy" ) -func ServerSideMTLS(ctx xds_context.Context, metadata *core_xds.DataplaneMetadata) ListenerBuilderOpt { - return ListenerBuilderOptFunc(func(config *ListenerBuilderConfig) { +func ServerSideMTLS(ctx xds_context.Context, metadata *core_xds.DataplaneMetadata) FilterChainBuilderOpt { + return FilterChainBuilderOptFunc(func(config *FilterChainBuilderConfig) { config.Add(&ServerSideMTLSConfigurer{ ctx: ctx, metadata: metadata, @@ -22,14 +22,11 @@ type ServerSideMTLSConfigurer struct { metadata *core_xds.DataplaneMetadata } -func (c *ServerSideMTLSConfigurer) Configure(l *v2.Listener) error { - for i := range l.FilterChains { - tlsContext, err := envoy.CreateDownstreamTlsContext(c.ctx, c.metadata) - if err != nil { - return err - } - l.FilterChains[i].TlsContext = tlsContext +func (c *ServerSideMTLSConfigurer) Configure(filterChain *envoy_listener.FilterChain) error { + tlsContext, err := envoy.CreateDownstreamTlsContext(c.ctx, c.metadata) + if err != nil { + return err } - + filterChain.TlsContext = tlsContext return nil } diff --git a/pkg/xds/envoy/listeners/server_mtls_configurer_test.go b/pkg/xds/envoy/listeners/server_mtls_configurer_test.go index ceff9b75883f..ff2877ed805e 100644 --- a/pkg/xds/envoy/listeners/server_mtls_configurer_test.go +++ b/pkg/xds/envoy/listeners/server_mtls_configurer_test.go @@ -13,6 +13,7 @@ import ( xds_context "github.com/Kong/kuma/pkg/xds/context" util_proto "github.com/Kong/kuma/pkg/util/proto" + envoy_common "github.com/Kong/kuma/pkg/xds/envoy" ) var _ = Describe("ServerMtlsConfigurer", func() { @@ -22,7 +23,7 @@ var _ = Describe("ServerMtlsConfigurer", func() { listenerAddress string listenerPort uint32 statsName string - clusters []ClusterInfo + clusters []envoy_common.ClusterInfo ctx xds_context.Context metadata core_xds.DataplaneMetadata expected string @@ -33,8 +34,9 @@ var _ = Describe("ServerMtlsConfigurer", func() { // when listener, err := NewListenerBuilder(). Configure(InboundListener(given.listenerName, given.listenerAddress, given.listenerPort)). - Configure(ServerSideMTLS(given.ctx, &given.metadata)). - Configure(TcpProxy(given.statsName, given.clusters...)). + Configure(FilterChain(NewFilterChainBuilder(). + Configure(ServerSideMTLS(given.ctx, &given.metadata)). + Configure(TcpProxy(given.statsName, given.clusters...)))). Build() // then Expect(err).ToNot(HaveOccurred()) @@ -50,7 +52,7 @@ var _ = Describe("ServerMtlsConfigurer", func() { listenerAddress: "192.168.0.1", listenerPort: 8080, statsName: "localhost:8080", - clusters: []ClusterInfo{{Name: "localhost:8080", Weight: 200}}, + clusters: []envoy_common.ClusterInfo{{Name: "localhost:8080", Weight: 200}}, ctx: xds_context.Context{ ControlPlane: &xds_context.ControlPlaneContext{ SdsLocation: "kuma-control-plane:5677", @@ -115,7 +117,7 @@ var _ = Describe("ServerMtlsConfigurer", func() { listenerAddress: "192.168.0.1", listenerPort: 8080, statsName: "localhost:8080", - clusters: []ClusterInfo{{Name: "localhost:8080", Weight: 200}}, + clusters: []envoy_common.ClusterInfo{{Name: "localhost:8080", Weight: 200}}, ctx: xds_context.Context{ ControlPlane: &xds_context.ControlPlaneContext{ SdsLocation: "kuma-control-plane:5677", diff --git a/pkg/xds/envoy/listeners/tcp_proxy_configurer.go b/pkg/xds/envoy/listeners/tcp_proxy_configurer.go index 5d554d277bc7..fee6d1a9e5b1 100644 --- a/pkg/xds/envoy/listeners/tcp_proxy_configurer.go +++ b/pkg/xds/envoy/listeners/tcp_proxy_configurer.go @@ -3,22 +3,16 @@ package listeners import ( "github.com/golang/protobuf/ptypes" - v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2" envoy_listener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener" envoy_tcp "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/tcp_proxy/v2" envoy_wellknown "github.com/envoyproxy/go-control-plane/pkg/wellknown" util_xds "github.com/Kong/kuma/pkg/util/xds" + envoy_common "github.com/Kong/kuma/pkg/xds/envoy" ) -type ClusterInfo struct { - Name string - Weight uint32 - Tags map[string]string -} - -func TcpProxy(statsName string, clusters ...ClusterInfo) ListenerBuilderOpt { - return ListenerBuilderOptFunc(func(config *ListenerBuilderConfig) { +func TcpProxy(statsName string, clusters ...envoy_common.ClusterInfo) FilterChainBuilderOpt { + return FilterChainBuilderOptFunc(func(config *FilterChainBuilderConfig) { config.Add(&TcpProxyConfigurer{ statsName: statsName, clusters: clusters, @@ -29,10 +23,10 @@ func TcpProxy(statsName string, clusters ...ClusterInfo) ListenerBuilderOpt { type TcpProxyConfigurer struct { statsName string // Clusters to forward traffic to. - clusters []ClusterInfo + clusters []envoy_common.ClusterInfo } -func (c *TcpProxyConfigurer) Configure(l *v2.Listener) error { +func (c *TcpProxyConfigurer) Configure(filterChain *envoy_listener.FilterChain) error { tcpProxy := c.tcpProxy() pbst, err := ptypes.MarshalAny(tcpProxy) @@ -40,15 +34,12 @@ func (c *TcpProxyConfigurer) Configure(l *v2.Listener) error { return err } - for i := range l.FilterChains { - l.FilterChains[i].Filters = append(l.FilterChains[i].Filters, &envoy_listener.Filter{ - Name: envoy_wellknown.TCPProxy, - ConfigType: &envoy_listener.Filter_TypedConfig{ - TypedConfig: pbst, - }, - }) - } - + filterChain.Filters = append(filterChain.Filters, &envoy_listener.Filter{ + Name: envoy_wellknown.TCPProxy, + ConfigType: &envoy_listener.Filter_TypedConfig{ + TypedConfig: pbst, + }, + }) return nil } diff --git a/pkg/xds/envoy/listeners/tcp_proxy_configurer_test.go b/pkg/xds/envoy/listeners/tcp_proxy_configurer_test.go index 861208e9287f..ea1a242dff80 100644 --- a/pkg/xds/envoy/listeners/tcp_proxy_configurer_test.go +++ b/pkg/xds/envoy/listeners/tcp_proxy_configurer_test.go @@ -8,6 +8,7 @@ import ( . "github.com/Kong/kuma/pkg/xds/envoy/listeners" util_proto "github.com/Kong/kuma/pkg/util/proto" + envoy_common "github.com/Kong/kuma/pkg/xds/envoy" ) var _ = Describe("TcpProxyConfigurer", func() { @@ -17,7 +18,7 @@ var _ = Describe("TcpProxyConfigurer", func() { listenerAddress string listenerPort uint32 statsName string - clusters []ClusterInfo + clusters []envoy_common.ClusterInfo expected string } @@ -26,7 +27,8 @@ var _ = Describe("TcpProxyConfigurer", func() { // when listener, err := NewListenerBuilder(). Configure(InboundListener(given.listenerName, given.listenerAddress, given.listenerPort)). - Configure(TcpProxy(given.statsName, given.clusters...)). + Configure(FilterChain(NewFilterChainBuilder(). + Configure(TcpProxy(given.statsName, given.clusters...)))). Build() // then Expect(err).ToNot(HaveOccurred()) @@ -42,7 +44,7 @@ var _ = Describe("TcpProxyConfigurer", func() { listenerAddress: "192.168.0.1", listenerPort: 8080, statsName: "localhost:8080", - clusters: []ClusterInfo{ + clusters: []envoy_common.ClusterInfo{ {Name: "localhost:8080", Weight: 200}, }, expected: ` @@ -65,7 +67,7 @@ var _ = Describe("TcpProxyConfigurer", func() { listenerAddress: "127.0.0.1", listenerPort: 5432, statsName: "db", - clusters: []ClusterInfo{{ + clusters: []envoy_common.ClusterInfo{{ Name: "db{version=v1}", Weight: 10, Tags: map[string]string{"service": "db", "version": "v1"}, diff --git a/pkg/xds/envoy/listeners/transparent_proxying_configurer_test.go b/pkg/xds/envoy/listeners/transparent_proxying_configurer_test.go index bff059a7782c..1644b8dff3a6 100644 --- a/pkg/xds/envoy/listeners/transparent_proxying_configurer_test.go +++ b/pkg/xds/envoy/listeners/transparent_proxying_configurer_test.go @@ -53,8 +53,6 @@ var _ = Describe("TransparentProxyingConfigurer", func() { portValue: 8080 deprecatedV1: bindToPort: false - filterChains: - - {} `, }), Entry("basic listener without transparent proxying", testCase{ @@ -68,8 +66,6 @@ var _ = Describe("TransparentProxyingConfigurer", func() { socketAddress: address: 192.168.0.1 portValue: 8080 - filterChains: - - {} `, }), ) diff --git a/pkg/xds/envoy/listeners/util.go b/pkg/xds/envoy/listeners/util.go index 96c24032fbab..6689ae07a25a 100644 --- a/pkg/xds/envoy/listeners/util.go +++ b/pkg/xds/envoy/listeners/util.go @@ -6,38 +6,34 @@ import ( "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes" - v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2" envoy_listener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener" ) -func UpdateFilterConfig(l *v2.Listener, filterName string, updateFunc func(proto.Message) error) error { - for i := range l.FilterChains { - for j, filter := range l.FilterChains[i].Filters { - if filter.Name == filterName { - if filter.GetTypedConfig() == nil { - return errors.Errorf("filter_chains[%d].filters[%d]: config cannot be 'nil'", i, j) - } - - var dany ptypes.DynamicAny - if err := ptypes.UnmarshalAny(filter.GetTypedConfig(), &dany); err != nil { - return err - } - if err := updateFunc(dany.Message); err != nil { - return err - } - - pbst, err := ptypes.MarshalAny(dany.Message) - if err != nil { - return err - } - - filter.ConfigType = &envoy_listener.Filter_TypedConfig{ - TypedConfig: pbst, - } +func UpdateFilterConfig(filterChain *envoy_listener.FilterChain, filterName string, updateFunc func(proto.Message) error) error { + for i, filter := range filterChain.Filters { + if filter.Name == filterName { + if filter.GetTypedConfig() == nil { + return errors.Errorf("filters[%d]: config cannot be 'nil'", i) + } + + var dany ptypes.DynamicAny + if err := ptypes.UnmarshalAny(filter.GetTypedConfig(), &dany); err != nil { + return err + } + if err := updateFunc(dany.Message); err != nil { + return err + } + + pbst, err := ptypes.MarshalAny(dany.Message) + if err != nil { + return err + } + + filter.ConfigType = &envoy_listener.Filter_TypedConfig{ + TypedConfig: pbst, } } } - return nil } diff --git a/pkg/xds/envoy/listeners/util_test.go b/pkg/xds/envoy/listeners/util_test.go index 0576eff468a7..9a93b5890533 100644 --- a/pkg/xds/envoy/listeners/util_test.go +++ b/pkg/xds/envoy/listeners/util_test.go @@ -12,7 +12,6 @@ import ( "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes" - v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2" envoy_listener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener" envoy_hcm "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2" envoy_tcp "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/tcp_proxy/v2" @@ -26,91 +25,41 @@ var _ = Describe("UpdateFilterConfig()", func() { Context("happy path", func() { type testCase struct { - listener *v2.Listener - filterName string - updateFunc func(proto.Message) error - expected string + filterChain *envoy_listener.FilterChain + filterName string + updateFunc func(proto.Message) error + expected string } DescribeTable("should update filter config", func(given testCase) { // when - err := UpdateFilterConfig(given.listener, given.filterName, given.updateFunc) + err := UpdateFilterConfig(given.filterChain, given.filterName, given.updateFunc) // then Expect(err).ToNot(HaveOccurred()) // when - actual, err := util_proto.ToYAML(given.listener) + actual, err := util_proto.ToYAML(given.filterChain) // then Expect(err).ToNot(HaveOccurred()) // and Expect(actual).To(MatchYAML(given.expected)) }, - Entry("0 chains", testCase{ - listener: &v2.Listener{}, - filterName: envoy_wellknown.TCPProxy, - updateFunc: func(proto.Message) error { return errors.New("should never happen") }, - expected: `{}`, - }), - Entry("1 chain, 0 filters", testCase{ - listener: &v2.Listener{ - FilterChains: []*envoy_listener.FilterChain{{}}, - }, - filterName: envoy_wellknown.TCPProxy, - updateFunc: func(proto.Message) error { return errors.New("should never happen") }, - expected: ` - filterChains: - - {} -`, + Entry("0 filters", testCase{ + filterChain: &envoy_listener.FilterChain{}, + filterName: envoy_wellknown.TCPProxy, + updateFunc: func(proto.Message) error { return errors.New("should never happen") }, + expected: `{}`, }), - Entry("1 chain, 1 filter", func() testCase { - pbst, err := ptypes.MarshalAny(&envoy_tcp.TcpProxy{}) - util_error.MustNot(err) - return testCase{ - listener: &v2.Listener{ - FilterChains: []*envoy_listener.FilterChain{{ - Filters: []*envoy_listener.Filter{{ - Name: envoy_wellknown.TCPProxy, - ConfigType: &envoy_listener.Filter_TypedConfig{ - TypedConfig: pbst, - }, - }}, - }}, - }, - filterName: envoy_wellknown.TCPProxy, - updateFunc: func(filterConfig proto.Message) error { - proxy := filterConfig.(*envoy_tcp.TcpProxy) - proxy.ClusterSpecifier = &envoy_tcp.TcpProxy_Cluster{ - Cluster: "backend", - } - return nil - }, - expected: ` - filterChains: - - filters: - - name: envoy.tcp_proxy - typedConfig: - '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy - cluster: backend -`, - } - }()), - Entry("1 chain, 2 filters", func() testCase { + Entry("1 filter", func() testCase { pbst, err := ptypes.MarshalAny(&envoy_tcp.TcpProxy{}) util_error.MustNot(err) return testCase{ - listener: &v2.Listener{ - FilterChains: []*envoy_listener.FilterChain{{ - Filters: []*envoy_listener.Filter{ - { - Name: envoy_wellknown.RoleBasedAccessControl, - }, - { - Name: envoy_wellknown.TCPProxy, - ConfigType: &envoy_listener.Filter_TypedConfig{ - TypedConfig: pbst, - }, - }, + filterChain: &envoy_listener.FilterChain{ + Filters: []*envoy_listener.Filter{{ + Name: envoy_wellknown.TCPProxy, + ConfigType: &envoy_listener.Filter_TypedConfig{ + TypedConfig: pbst, }, }}, }, @@ -123,49 +72,27 @@ var _ = Describe("UpdateFilterConfig()", func() { return nil }, expected: ` - filterChains: - - filters: - - name: envoy.filters.network.rbac - - name: envoy.tcp_proxy - typedConfig: - '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy - cluster: backend + filters: + - name: envoy.tcp_proxy + typedConfig: + '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy + cluster: backend `, } }()), - Entry("2 chain, 2 filters", func() testCase { + Entry("2 filters", func() testCase { pbst, err := ptypes.MarshalAny(&envoy_tcp.TcpProxy{}) util_error.MustNot(err) return testCase{ - listener: &v2.Listener{ - FilterChains: []*envoy_listener.FilterChain{ + filterChain: &envoy_listener.FilterChain{ + Filters: []*envoy_listener.Filter{ { - Filters: []*envoy_listener.Filter{ - { - Name: envoy_wellknown.RoleBasedAccessControl, - }, - { - Name: envoy_wellknown.TCPProxy, - ConfigType: &envoy_listener.Filter_TypedConfig{ - TypedConfig: pbst, - }, - }, - }, + Name: envoy_wellknown.RoleBasedAccessControl, }, { - Filters: []*envoy_listener.Filter{ - { - Name: envoy_wellknown.TCPProxy, - ConfigType: &envoy_listener.Filter_TypedConfig{ - TypedConfig: pbst, - }, - }, - { - Name: envoy_wellknown.TCPProxy, - ConfigType: &envoy_listener.Filter_TypedConfig{ - TypedConfig: pbst, - }, - }, + Name: envoy_wellknown.TCPProxy, + ConfigType: &envoy_listener.Filter_TypedConfig{ + TypedConfig: pbst, }, }, }, @@ -179,22 +106,12 @@ var _ = Describe("UpdateFilterConfig()", func() { return nil }, expected: ` - filterChains: - - filters: - - name: envoy.filters.network.rbac - - name: envoy.tcp_proxy - typedConfig: - '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy - cluster: backend - - filters: - - name: envoy.tcp_proxy - typedConfig: - '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy - cluster: backend - - name: envoy.tcp_proxy - typedConfig: - '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy - cluster: backend + filters: + - name: envoy.filters.network.rbac + - name: envoy.tcp_proxy + typedConfig: + '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy + cluster: backend `, } }()), @@ -204,7 +121,7 @@ var _ = Describe("UpdateFilterConfig()", func() { Context("error path", func() { type testCase struct { - listener *v2.Listener + filterChain *envoy_listener.FilterChain filterName string updateFunc func(proto.Message) error expectedErr string @@ -213,36 +130,32 @@ var _ = Describe("UpdateFilterConfig()", func() { DescribeTable("should return an error", func(given testCase) { // when - err := UpdateFilterConfig(given.listener, given.filterName, given.updateFunc) + err := UpdateFilterConfig(given.filterChain, given.filterName, given.updateFunc) // then Expect(err).To(HaveOccurred()) // and Expect(err.Error()).To(Equal(given.expectedErr)) }, - Entry("1 chain, 1 filter without config", testCase{ - listener: &v2.Listener{ - FilterChains: []*envoy_listener.FilterChain{{ - Filters: []*envoy_listener.Filter{{ - Name: envoy_wellknown.TCPProxy, - }}, + Entry("1 filter without config", testCase{ + filterChain: &envoy_listener.FilterChain{ + Filters: []*envoy_listener.Filter{{ + Name: envoy_wellknown.TCPProxy, }}, }, filterName: envoy_wellknown.TCPProxy, updateFunc: func(proto.Message) error { return errors.New("should never happen") }, - expectedErr: `filter_chains[0].filters[0]: config cannot be 'nil'`, + expectedErr: `filters[0]: config cannot be 'nil'`, }), - Entry("1 chain, 1 filter with a wrong config type", func() testCase { + Entry("1 filter with a wrong config type", func() testCase { pbst, err := ptypes.MarshalAny(&envoy_hcm.HttpConnectionManager{}) util_error.MustNot(err) return testCase{ - listener: &v2.Listener{ - FilterChains: []*envoy_listener.FilterChain{{ - Filters: []*envoy_listener.Filter{{ - Name: envoy_wellknown.TCPProxy, - ConfigType: &envoy_listener.Filter_TypedConfig{ - TypedConfig: pbst, - }, - }}, + filterChain: &envoy_listener.FilterChain{ + Filters: []*envoy_listener.Filter{{ + Name: envoy_wellknown.TCPProxy, + ConfigType: &envoy_listener.Filter_TypedConfig{ + TypedConfig: pbst, + }, }}, }, filterName: envoy_wellknown.TCPProxy, diff --git a/pkg/xds/envoy/types.go b/pkg/xds/envoy/types.go new file mode 100644 index 000000000000..efff425446cf --- /dev/null +++ b/pkg/xds/envoy/types.go @@ -0,0 +1,7 @@ +package envoy + +type ClusterInfo struct { + Name string + Weight uint32 + Tags map[string]string +} diff --git a/pkg/xds/generator/prometheus_endpoint_generator.go b/pkg/xds/generator/prometheus_endpoint_generator.go index bd25faa2a73d..e6ccf56cfde6 100644 --- a/pkg/xds/generator/prometheus_endpoint_generator.go +++ b/pkg/xds/generator/prometheus_endpoint_generator.go @@ -62,7 +62,8 @@ func (g PrometheusEndpointGenerator) Generate(ctx xds_context.Context, proxy *co listener, err := envoy_listeners.NewListenerBuilder(). Configure(envoy_listeners.InboundListener(prometheusListenerName, prometheusEndpointAddress, prometheusEndpoint.Port)). - Configure(envoy_listeners.PrometheusEndpoint(prometheusEndpoint.Path, envoyAdminClusterName)). + Configure(envoy_listeners.FilterChain(envoy_listeners.NewFilterChainBuilder(). + Configure(envoy_listeners.PrometheusEndpoint(prometheusListenerName, prometheusEndpoint.Path, envoyAdminClusterName)))). Configure(envoy_listeners.TransparentProxying(proxy.Dataplane.Spec.Networking.GetTransparentProxying())). Build() if err != nil { diff --git a/pkg/xds/generator/proxy_template.go b/pkg/xds/generator/proxy_template.go index deb990c020a9..9405722a7a3d 100644 --- a/pkg/xds/generator/proxy_template.go +++ b/pkg/xds/generator/proxy_template.go @@ -14,6 +14,7 @@ import ( util_envoy "github.com/Kong/kuma/pkg/util/envoy" xds_context "github.com/Kong/kuma/pkg/xds/context" + envoy_common "github.com/Kong/kuma/pkg/xds/envoy" envoy_clusters "github.com/Kong/kuma/pkg/xds/envoy/clusters" envoy_endpoints "github.com/Kong/kuma/pkg/xds/envoy/endpoints" envoy_listeners "github.com/Kong/kuma/pkg/xds/envoy/listeners" @@ -113,9 +114,10 @@ func (g InboundProxyGenerator) Generate(ctx xds_context.Context, proxy *model.Pr inboundListenerName := localListenerName(endpoint.DataplaneIP, endpoint.DataplanePort) inboundListener, err := envoy_listeners.NewListenerBuilder(). Configure(envoy_listeners.InboundListener(inboundListenerName, endpoint.DataplaneIP, endpoint.DataplanePort)). - Configure(envoy_listeners.ServerSideMTLS(ctx, proxy.Metadata)). - Configure(g.protocolSpecificOpts(service, protocol, envoy_listeners.ClusterInfo{Name: localClusterName})...). - Configure(envoy_listeners.NetworkRBAC(ctx.Mesh.Resource.Spec.GetMtls().GetEnabled(), proxy.TrafficPermissions.Get(endpoint.String()))). + Configure(envoy_listeners.FilterChain(envoy_listeners.NewFilterChainBuilder(). + Configure(g.protocolSpecificOpts(service, protocol, envoy_common.ClusterInfo{Name: localClusterName})...). + Configure(envoy_listeners.ServerSideMTLS(ctx, proxy.Metadata)). + Configure(envoy_listeners.NetworkRBAC(inboundListenerName, ctx.Mesh.Resource.Spec.GetMtls().GetEnabled(), proxy.TrafficPermissions.Get(endpoint.String()))))). Configure(envoy_listeners.TransparentProxying(proxy.Dataplane.Spec.Networking.GetTransparentProxying())). Build() if err != nil { @@ -130,17 +132,17 @@ func (g InboundProxyGenerator) Generate(ctx xds_context.Context, proxy *model.Pr return resources.List(), nil } -func (_ InboundProxyGenerator) protocolSpecificOpts(service string, protocol mesh_core.Protocol, localCluster envoy_listeners.ClusterInfo) []envoy_listeners.ListenerBuilderOpt { +func (_ InboundProxyGenerator) protocolSpecificOpts(service string, protocol mesh_core.Protocol, localCluster envoy_common.ClusterInfo) []envoy_listeners.FilterChainBuilderOpt { switch protocol { case mesh_core.ProtocolHTTP: - return []envoy_listeners.ListenerBuilderOpt{ + return []envoy_listeners.FilterChainBuilderOpt{ envoy_listeners.HttpConnectionManager(localCluster.Name), envoy_listeners.HttpInboundRoute(service, localCluster), } case mesh_core.ProtocolTCP: fallthrough default: - return []envoy_listeners.ListenerBuilderOpt{ + return []envoy_listeners.FilterChainBuilderOpt{ envoy_listeners.TcpProxy(localCluster.Name, localCluster), } } @@ -187,8 +189,9 @@ func (g OutboundProxyGenerator) Generate(ctx xds_context.Context, proxy *model.P listener, err := envoy_listeners.NewListenerBuilder(). Configure(envoy_listeners.OutboundListener(outboundListenerName, endpoint.DataplaneIP, endpoint.DataplanePort)). - Configure(envoy_listeners.TcpProxy(oface.Service, clusters...)). - Configure(envoy_listeners.NetworkAccessLog(sourceService, destinationService, proxy.Logs[oface.Service], proxy)). + Configure(envoy_listeners.FilterChain(envoy_listeners.NewFilterChainBuilder(). + Configure(envoy_listeners.TcpProxy(oface.Service, clusters...)). + Configure(envoy_listeners.NetworkAccessLog(sourceService, destinationService, proxy.Logs[oface.Service], proxy)))). Configure(envoy_listeners.TransparentProxying(proxy.Dataplane.Spec.Networking.GetTransparentProxying())). Build() if err != nil { @@ -202,7 +205,7 @@ func (g OutboundProxyGenerator) Generate(ctx xds_context.Context, proxy *model.P return resources.List(), nil } -func (_ OutboundProxyGenerator) determineClusters(ctx xds_context.Context, proxy *model.Proxy, route *mesh_core.TrafficRouteResource) (clusters []envoy_listeners.ClusterInfo, err error) { +func (_ OutboundProxyGenerator) determineClusters(ctx xds_context.Context, proxy *model.Proxy, route *mesh_core.TrafficRouteResource) (clusters []envoy_common.ClusterInfo, err error) { for j, destination := range route.Spec.Conf { service, ok := destination.Destination[kuma_mesh.ServiceTag] if !ok { @@ -212,7 +215,7 @@ func (_ OutboundProxyGenerator) determineClusters(ctx xds_context.Context, proxy // Envoy doesn't support 0 weight continue } - clusters = append(clusters, envoy_listeners.ClusterInfo{ + clusters = append(clusters, envoy_common.ClusterInfo{ Name: destinationClusterName(service, destination.Destination), Weight: destination.Weight, Tags: destination.Destination, @@ -221,7 +224,7 @@ func (_ OutboundProxyGenerator) determineClusters(ctx xds_context.Context, proxy return } -func (_ OutboundProxyGenerator) generateEds(ctx xds_context.Context, proxy *model.Proxy, clusters []envoy_listeners.ClusterInfo) (resources []*model.Resource, _ error) { +func (_ OutboundProxyGenerator) generateEds(ctx xds_context.Context, proxy *model.Proxy, clusters []envoy_common.ClusterInfo) (resources []*model.Resource, _ error) { for _, cluster := range clusters { serviceName := cluster.Tags[kuma_mesh.ServiceTag] healthCheck := proxy.HealthChecks[serviceName] @@ -252,7 +255,8 @@ func (_ TransparentProxyGenerator) Generate(ctx xds_context.Context, proxy *mode } listener, err := envoy_listeners.NewListenerBuilder(). Configure(envoy_listeners.OutboundListener("catch_all", "0.0.0.0", redirectPort)). - Configure(envoy_listeners.TcpProxy("pass_through", envoy_listeners.ClusterInfo{Name: "pass_through"})). + Configure(envoy_listeners.FilterChain(envoy_listeners.NewFilterChainBuilder(). + Configure(envoy_listeners.TcpProxy("pass_through", envoy_common.ClusterInfo{Name: "pass_through"})))). Configure(envoy_listeners.OriginalDstForwarder()). Build() if err != nil {