diff --git a/pkg/xds/envoy/listeners/access_log_configurer.go b/pkg/xds/envoy/listeners/access_log_configurer.go new file mode 100644 index 000000000000..6f68bfd2750f --- /dev/null +++ b/pkg/xds/envoy/listeners/access_log_configurer.go @@ -0,0 +1,109 @@ +package listeners + +import ( + "fmt" + "net" + + "github.com/pkg/errors" + + "github.com/golang/protobuf/ptypes" + + envoy_core "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" + envoy_accesslog "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v2" + filter_accesslog "github.com/envoyproxy/go-control-plane/envoy/config/filter/accesslog/v2" + envoy_wellknown "github.com/envoyproxy/go-control-plane/pkg/wellknown" + + mesh_proto "github.com/Kong/kuma/api/mesh/v1alpha1" + core_xds "github.com/Kong/kuma/pkg/core/xds" + "github.com/Kong/kuma/pkg/envoy/accesslog" +) + +const accessLogSink = "access_log_sink" + +type AccessLogConfigurer struct { + sourceService string + destinationService string + backend *mesh_proto.LoggingBackend + proxy *core_xds.Proxy +} + +func convertLoggingBackend(sourceService string, destinationService string, backend *mesh_proto.LoggingBackend, proxy *core_xds.Proxy, defaultFormat string) (*filter_accesslog.AccessLog, error) { + if backend == nil { + return nil, nil + } + formatString := defaultFormat + if backend.Format != "" { + formatString = backend.Format + } + format, err := accesslog.ParseFormat(formatString) + if err != nil { + return nil, errors.Wrapf(err, "invalid access log format string: %s", formatString) + } + + variables := accesslog.InterpolationVariables{ + accesslog.CMD_KUMA_SOURCE_ADDRESS: net.JoinHostPort(proxy.Dataplane.GetIP(), "0"), // deprecated variable + accesslog.CMD_KUMA_SOURCE_ADDRESS_WITHOUT_PORT: proxy.Dataplane.GetIP(), // replacement variable + accesslog.CMD_KUMA_SOURCE_SERVICE: sourceService, + accesslog.CMD_KUMA_DESTINATION_SERVICE: destinationService, + } + + format, err = format.Interpolate(variables) + if err != nil { + return nil, errors.Wrapf(err, "failed to interpolate access log format string with Kuma-specific variables: %s", formatString) + } + + if file, ok := backend.GetType().(*mesh_proto.LoggingBackend_File_); ok { + return fileAccessLog(format, file) + } else if tcp, ok := backend.GetType().(*mesh_proto.LoggingBackend_Tcp_); ok { + return tcpAccessLog(format, tcp) + } else { + return nil, errors.Errorf("could not convert LoggingBackend of type %T to AccessLog", backend.GetType()) + } +} + +func tcpAccessLog(format *accesslog.AccessLogFormat, tcp *mesh_proto.LoggingBackend_Tcp_) (*filter_accesslog.AccessLog, error) { + httpGrpcAccessLog := &envoy_accesslog.HttpGrpcAccessLogConfig{ + CommonConfig: &envoy_accesslog.CommonGrpcAccessLogConfig{ + LogName: fmt.Sprintf("%s;%s", tcp.Tcp.Address, format.String()), + GrpcService: &envoy_core.GrpcService{ + TargetSpecifier: &envoy_core.GrpcService_EnvoyGrpc_{ + EnvoyGrpc: &envoy_core.GrpcService_EnvoyGrpc{ + ClusterName: accessLogSink, + }, + }, + }, + }, + } + if err := format.ConfigureHttpLog(httpGrpcAccessLog); err != nil { + return nil, errors.Wrapf(err, "failed to configure %T according to the format string: %s", httpGrpcAccessLog, format) + } + marshalled, err := ptypes.MarshalAny(httpGrpcAccessLog) + if err != nil { + return nil, errors.Wrapf(err, "could not marshall %T", httpGrpcAccessLog) + } + return &filter_accesslog.AccessLog{ + Name: envoy_wellknown.HTTPGRPCAccessLog, + ConfigType: &filter_accesslog.AccessLog_TypedConfig{ + TypedConfig: marshalled, + }, + }, nil +} + +func fileAccessLog(format *accesslog.AccessLogFormat, file *mesh_proto.LoggingBackend_File_) (*filter_accesslog.AccessLog, error) { + fileAccessLog := &envoy_accesslog.FileAccessLog{ + AccessLogFormat: &envoy_accesslog.FileAccessLog_Format{ + Format: format.String(), + }, + Path: file.File.Path, + } + marshalled, err := ptypes.MarshalAny(fileAccessLog) + if err != nil { + return nil, errors.Wrapf(err, "could not marshall %T", fileAccessLog) + } + return &filter_accesslog.AccessLog{ + Name: envoy_wellknown.FileAccessLog, + ConfigType: &filter_accesslog.AccessLog_TypedConfig{ + TypedConfig: marshalled, + }, + }, nil +} diff --git a/pkg/xds/envoy/listeners/http_access_log_configurer.go b/pkg/xds/envoy/listeners/http_access_log_configurer.go new file mode 100644 index 000000000000..ab76c9f005cc --- /dev/null +++ b/pkg/xds/envoy/listeners/http_access_log_configurer.go @@ -0,0 +1,43 @@ +package listeners + +import ( + envoy_listener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener" + envoy_hcm "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2" + + mesh_proto "github.com/Kong/kuma/api/mesh/v1alpha1" + core_xds "github.com/Kong/kuma/pkg/core/xds" +) + +const defaultHttpAccessLogFormat = `[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%KUMA_SOURCE_SERVICE%" "%KUMA_DESTINATION_SERVICE%" "%KUMA_SOURCE_ADDRESS_WITHOUT_PORT%" "%UPSTREAM_HOST%" +` // intentional newline at the end + +func HttpAccessLog(sourceService string, destinationService string, backend *mesh_proto.LoggingBackend, proxy *core_xds.Proxy) FilterChainBuilderOpt { + return FilterChainBuilderOptFunc(func(config *FilterChainBuilderConfig) { + if backend != nil { + config.Add(&HttpAccessLogConfigurer{ + AccessLogConfigurer: AccessLogConfigurer{ + sourceService: sourceService, + destinationService: destinationService, + backend: backend, + proxy: proxy, + }, + }) + } + }) +} + +type HttpAccessLogConfigurer struct { + AccessLogConfigurer +} + +func (c *HttpAccessLogConfigurer) Configure(filterChain *envoy_listener.FilterChain) error { + accessLog, err := convertLoggingBackend(c.AccessLogConfigurer.sourceService, c.AccessLogConfigurer.destinationService, c.AccessLogConfigurer.backend, c.AccessLogConfigurer.proxy, defaultHttpAccessLogFormat) + if err != nil { + return err + } + + return UpdateHttpConnectionManagerConfig(filterChain, func(hcm *envoy_hcm.HttpConnectionManager) error { + hcm.AccessLog = append(hcm.AccessLog, accessLog) + return nil + }) +} diff --git a/pkg/xds/envoy/listeners/http_access_log_configurer_test.go b/pkg/xds/envoy/listeners/http_access_log_configurer_test.go new file mode 100644 index 000000000000..ed639b0a1f3f --- /dev/null +++ b/pkg/xds/envoy/listeners/http_access_log_configurer_test.go @@ -0,0 +1,200 @@ +package listeners_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + + . "github.com/Kong/kuma/pkg/xds/envoy/listeners" + + mesh_proto "github.com/Kong/kuma/api/mesh/v1alpha1" + mesh_core "github.com/Kong/kuma/pkg/core/resources/apis/mesh" + "github.com/Kong/kuma/pkg/core/xds" + core_xds "github.com/Kong/kuma/pkg/core/xds" + util_proto "github.com/Kong/kuma/pkg/util/proto" +) + +var _ = Describe("HttpAccessLogConfigurer", func() { + + type testCase struct { + listenerName string + listenerAddress string + listenerPort uint32 + statsName string + routeName string + backend *mesh_proto.LoggingBackend + expected string + } + + DescribeTable("should generate proper Envoy config", + func(given testCase) { + // given + sourceService := "web" + destinationService := "backend" + proxy := &core_xds.Proxy{ + Id: xds.ProxyId{ + Name: "web", + Mesh: "example", + }, + Dataplane: &mesh_core.DataplaneResource{ + Spec: mesh_proto.Dataplane{ + Networking: &mesh_proto.Dataplane_Networking{ + Inbound: []*mesh_proto.Dataplane_Networking_Inbound{{ + Interface: "192.168.0.1:80:8080", + Tags: map[string]string{ + "service": "web", + }, + }}, + Outbound: []*mesh_proto.Dataplane_Networking_Outbound{{ + Interface: ":27070", + Service: "backend", + }}, + }, + }, + }, + } + + // when + listener, err := NewListenerBuilder(). + Configure(OutboundListener(given.listenerName, given.listenerAddress, given.listenerPort)). + Configure(FilterChain(NewFilterChainBuilder(). + Configure(HttpConnectionManager(given.statsName)). + Configure(HttpOutboundRoute(given.routeName)). + Configure(HttpAccessLog(sourceService, destinationService, given.backend, proxy)))). + Build() + // then + Expect(err).ToNot(HaveOccurred()) + + // when + actual, err := util_proto.ToYAML(listener) + Expect(err).ToNot(HaveOccurred()) + // and + Expect(actual).To(MatchYAML(given.expected)) + }, + Entry("basic http_connection_manager without access log", testCase{ + listenerName: "outbound:127.0.0.1:27070", + listenerAddress: "127.0.0.1", + listenerPort: 27070, + statsName: "backend", + routeName: "outbound:backend", + backend: nil, + expected: ` + name: outbound:127.0.0.1:27070 + address: + socketAddress: + address: 127.0.0.1 + portValue: 27070 + filterChains: + - filters: + - name: envoy.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager + httpFilters: + - name: envoy.router + rds: + configSource: + ads: {} + routeConfigName: outbound:backend + statPrefix: backend +`, + }), + Entry("basic http_connection_manager with file access log", testCase{ + listenerName: "outbound:127.0.0.1:27070", + listenerAddress: "127.0.0.1", + listenerPort: 27070, + statsName: "backend", + routeName: "outbound:backend", + backend: &mesh_proto.LoggingBackend{ + Name: "file", + Type: &mesh_proto.LoggingBackend_File_{ + File: &mesh_proto.LoggingBackend_File{ + Path: "/tmp/log", + }, + }, + }, + expected: ` + name: outbound:127.0.0.1:27070 + address: + socketAddress: + address: 127.0.0.1 + portValue: 27070 + filterChains: + - filters: + - name: envoy.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager + accessLog: + - name: envoy.file_access_log + typedConfig: + '@type': type.googleapis.com/envoy.config.accesslog.v2.FileAccessLog + format: | + [%START_TIME%] "%REQ(:method)% %REQ(x-envoy-original-path?:path)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(x-envoy-upstream-service-time)% "%REQ(x-forwarded-for)%" "%REQ(user-agent)%" "%REQ(x-request-id)%" "%REQ(:authority)%" "web" "backend" "192.168.0.1" "%UPSTREAM_HOST%" + path: /tmp/log + httpFilters: + - name: envoy.router + rds: + configSource: + ads: {} + routeConfigName: outbound:backend + statPrefix: backend +`, + }), + Entry("basic http_connection_manager with tcp access log", testCase{ + listenerName: "outbound:127.0.0.1:27070", + listenerAddress: "127.0.0.1", + listenerPort: 27070, + statsName: "backend", + routeName: "outbound:backend", + backend: &mesh_proto.LoggingBackend{ + Name: "tcp", + Format: `[%START_TIME%] "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%REQ(ORIGIN)%" "%REQ(CONTENT-TYPE)%" "%KUMA_SOURCE_SERVICE%" "%KUMA_DESTINATION_SERVICE%" "%KUMA_SOURCE_ADDRESS%" "%KUMA_SOURCE_ADDRESS_WITHOUT_PORT%" "%UPSTREAM_HOST%" + +"%RESP(SERVER):5%" "%TRAILER(GRPC-MESSAGE):7%" "DYNAMIC_METADATA(namespace:object:key):9" "FILTER_STATE(filter.state.key):12" +`, // intentional newline at the end + Type: &mesh_proto.LoggingBackend_Tcp_{ + Tcp: &mesh_proto.LoggingBackend_Tcp{ + Address: "127.0.0.1:1234", + }, + }, + }, + expected: ` + name: outbound:127.0.0.1:27070 + address: + socketAddress: + address: 127.0.0.1 + portValue: 27070 + filterChains: + - filters: + - name: envoy.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager + accessLog: + - name: envoy.http_grpc_access_log + typedConfig: + '@type': type.googleapis.com/envoy.config.accesslog.v2.HttpGrpcAccessLogConfig + additionalRequestHeadersToLog: + - origin + - content-type + additionalResponseHeadersToLog: + - server + additionalResponseTrailersToLog: + - grpc-message + commonConfig: + grpcService: + envoyGrpc: + clusterName: access_log_sink + logName: | + 127.0.0.1:1234;[%START_TIME%] "%REQ(x-request-id)%" "%REQ(:authority)%" "%REQ(origin)%" "%REQ(content-type)%" "web" "backend" "192.168.0.1:0" "192.168.0.1" "%UPSTREAM_HOST%" + + "%RESP(server):5%" "%TRAILER(grpc-message):7%" "DYNAMIC_METADATA(namespace:object:key):9" "FILTER_STATE(filter.state.key):12" + httpFilters: + - name: envoy.router + rds: + configSource: + ads: {} + routeConfigName: outbound:backend + statPrefix: backend +`, + }), + ) +}) diff --git a/pkg/xds/envoy/listeners/network_access_log_configurer.go b/pkg/xds/envoy/listeners/network_access_log_configurer.go index 9ac68f6c4080..fc54611b5122 100644 --- a/pkg/xds/envoy/listeners/network_access_log_configurer.go +++ b/pkg/xds/envoy/listeners/network_access_log_configurer.go @@ -1,131 +1,43 @@ package listeners import ( - "fmt" - "strings" - - "github.com/pkg/errors" - - "github.com/golang/protobuf/proto" - "github.com/golang/protobuf/ptypes" - - envoy_core "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" envoy_listener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener" - accesslog "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v2" - filter_accesslog "github.com/envoyproxy/go-control-plane/envoy/config/filter/accesslog/v2" envoy_tcp "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/tcp_proxy/v2" - envoy_wellknown "github.com/envoyproxy/go-control-plane/pkg/wellknown" "github.com/Kong/kuma/api/mesh/v1alpha1" core_xds "github.com/Kong/kuma/pkg/core/xds" ) +const defaultNetworkAccessLogFormat = `[%START_TIME%] %KUMA_SOURCE_ADDRESS_WITHOUT_PORT%(%KUMA_SOURCE_SERVICE%)->%UPSTREAM_HOST%(%KUMA_DESTINATION_SERVICE%) took %DURATION%ms, sent %BYTES_SENT% bytes, received: %BYTES_RECEIVED% bytes +` // intentional newline at the end + func NetworkAccessLog(sourceService string, destinationService string, backend *v1alpha1.LoggingBackend, proxy *core_xds.Proxy) FilterChainBuilderOpt { return FilterChainBuilderOptFunc(func(config *FilterChainBuilderConfig) { if backend != nil { config.Add(&NetworkAccessLogConfigurer{ - sourceService: sourceService, - destinationService: destinationService, - backend: backend, - proxy: proxy, + AccessLogConfigurer: AccessLogConfigurer{ + sourceService: sourceService, + destinationService: destinationService, + backend: backend, + proxy: proxy, + }, }) } }) } type NetworkAccessLogConfigurer struct { - sourceService string - destinationService string - backend *v1alpha1.LoggingBackend - proxy *core_xds.Proxy + AccessLogConfigurer } func (c *NetworkAccessLogConfigurer) Configure(filterChain *envoy_listener.FilterChain) error { - accessLog, err := convertLoggingBackend(c.sourceService, c.destinationService, c.backend, c.proxy) + accessLog, err := convertLoggingBackend(c.AccessLogConfigurer.sourceService, c.AccessLogConfigurer.destinationService, c.AccessLogConfigurer.backend, c.AccessLogConfigurer.proxy, defaultNetworkAccessLogFormat) if err != nil { return err } - return UpdateFilterConfig(filterChain, envoy_wellknown.TCPProxy, func(filterConfig proto.Message) error { - proxy, ok := filterConfig.(*envoy_tcp.TcpProxy) - if !ok { - return NewUnexpectedFilterConfigTypeError(filterConfig, &envoy_tcp.TcpProxy{}) - } - proxy.AccessLog = append(proxy.AccessLog, accessLog) + return UpdateTcpProxyConfig(filterChain, func(tcpProxy *envoy_tcp.TcpProxy) error { + tcpProxy.AccessLog = append(tcpProxy.AccessLog, accessLog) return nil }) } - -const AccessLogDefaultFormat = "[%START_TIME%] %KUMA_SOURCE_ADDRESS%(%KUMA_SOURCE_SERVICE%)->%UPSTREAM_HOST%(%KUMA_DESTINATION_SERVICE%) took %DURATION%ms, sent %BYTES_SENT% bytes, received: %BYTES_RECEIVED% bytes\n" - -const AccessLogSink = "access_log_sink" - -func convertLoggingBackend(sourceService string, destinationService string, backend *v1alpha1.LoggingBackend, proxy *core_xds.Proxy) (*filter_accesslog.AccessLog, error) { - if backend == nil { - return nil, nil - } - format := AccessLogDefaultFormat - if backend.Format != "" { - format = backend.Format - } - iface, _ := proxy.Dataplane.Spec.Networking.GetInboundInterface(sourceService) - sourceAddress := "" - if iface != nil { - sourceAddress = iface.DataplaneIP - } - format = strings.ReplaceAll(format, "%KUMA_SOURCE_ADDRESS%", fmt.Sprintf("%s:0", sourceAddress)) - format = strings.ReplaceAll(format, "%KUMA_SOURCE_SERVICE%", sourceService) - format = strings.ReplaceAll(format, "%KUMA_DESTINATION_SERVICE%", destinationService) - - if file, ok := backend.GetType().(*v1alpha1.LoggingBackend_File_); ok { - return fileAccessLog(format, file) - } else if tcp, ok := backend.GetType().(*v1alpha1.LoggingBackend_Tcp_); ok { - return tcpAccessLog(format, tcp) - } else { - return nil, errors.Errorf("could not convert LoggingBackend of type %T to AccessLog", backend.GetType()) - } -} - -func tcpAccessLog(format string, tcp *v1alpha1.LoggingBackend_Tcp_) (*filter_accesslog.AccessLog, error) { - fileAccessLog := &accesslog.HttpGrpcAccessLogConfig{ - CommonConfig: &accesslog.CommonGrpcAccessLogConfig{ - LogName: fmt.Sprintf("%s;%s", tcp.Tcp.Address, format), - GrpcService: &envoy_core.GrpcService{ - TargetSpecifier: &envoy_core.GrpcService_EnvoyGrpc_{ - EnvoyGrpc: &envoy_core.GrpcService_EnvoyGrpc{ - ClusterName: AccessLogSink, - }, - }, - }, - }, - } - marshalled, err := ptypes.MarshalAny(fileAccessLog) - if err != nil { - return nil, errors.Wrap(err, "could not marshall FileAccessLog") - } - return &filter_accesslog.AccessLog{ - Name: envoy_wellknown.HTTPGRPCAccessLog, - ConfigType: &filter_accesslog.AccessLog_TypedConfig{ - TypedConfig: marshalled, - }, - }, nil -} - -func fileAccessLog(format string, file *v1alpha1.LoggingBackend_File_) (*filter_accesslog.AccessLog, error) { - fileAccessLog := &accesslog.FileAccessLog{ - AccessLogFormat: &accesslog.FileAccessLog_Format{ - Format: format, - }, - Path: file.File.Path, - } - marshalled, err := ptypes.MarshalAny(fileAccessLog) - if err != nil { - return nil, errors.Wrap(err, "could not marshall FileAccessLog") - } - return &filter_accesslog.AccessLog{ - Name: envoy_wellknown.FileAccessLog, - ConfigType: &filter_accesslog.AccessLog_TypedConfig{ - TypedConfig: marshalled, - }, - }, nil -} diff --git a/pkg/xds/envoy/listeners/network_access_log_configurer_test.go b/pkg/xds/envoy/listeners/network_access_log_configurer_test.go index e44dac76ae83..6bb3291e5de4 100644 --- a/pkg/xds/envoy/listeners/network_access_log_configurer_test.go +++ b/pkg/xds/envoy/listeners/network_access_log_configurer_test.go @@ -125,7 +125,7 @@ var _ = Describe("NetworkAccessLogConfigurer", func() { typedConfig: '@type': type.googleapis.com/envoy.config.accesslog.v2.FileAccessLog format: | - [%START_TIME%] 192.168.0.1:0(backend)->%UPSTREAM_HOST%(db) took %DURATION%ms, sent %BYTES_SENT% bytes, received: %BYTES_RECEIVED% bytes + [%START_TIME%] 192.168.0.1(backend)->%UPSTREAM_HOST%(db) took %DURATION%ms, sent %BYTES_SENT% bytes, received: %BYTES_RECEIVED% bytes path: /tmp/log cluster: db statPrefix: db @@ -138,8 +138,11 @@ var _ = Describe("NetworkAccessLogConfigurer", func() { statsName: "db", clusters: []envoy_common.ClusterInfo{{Name: "db", Weight: 200}}, backend: &mesh_proto.LoggingBackend{ - Name: "tcp", - Format: "custom format", + Name: "tcp", + Format: `[%START_TIME%] "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%REQ(ORIGIN)%" "%REQ(CONTENT-TYPE)%" "%KUMA_SOURCE_SERVICE%" "%KUMA_DESTINATION_SERVICE%" "%KUMA_SOURCE_ADDRESS%" "%KUMA_SOURCE_ADDRESS_WITHOUT_PORT%" "%UPSTREAM_HOST% + +"%RESP(SERVER):5%" "%TRAILER(GRPC-MESSAGE):7%" "DYNAMIC_METADATA(namespace:object:key):9" "FILTER_STATE(filter.state.key):12" +`, // intentional newline at the end Type: &mesh_proto.LoggingBackend_Tcp_{ Tcp: &mesh_proto.LoggingBackend_Tcp{ Address: "127.0.0.1:1234", @@ -161,11 +164,21 @@ var _ = Describe("NetworkAccessLogConfigurer", func() { - name: envoy.http_grpc_access_log typedConfig: '@type': type.googleapis.com/envoy.config.accesslog.v2.HttpGrpcAccessLogConfig + additionalRequestHeadersToLog: + - origin + - content-type + additionalResponseHeadersToLog: + - server + additionalResponseTrailersToLog: + - grpc-message commonConfig: grpcService: envoyGrpc: clusterName: access_log_sink - logName: 127.0.0.1:1234;custom format + logName: | + 127.0.0.1:1234;[%START_TIME%] "%REQ(x-request-id)%" "%REQ(:authority)%" "%REQ(origin)%" "%REQ(content-type)%" "backend" "db" "192.168.0.1:0" "192.168.0.1" "%UPSTREAM_HOST% + + "%RESP(server):5%" "%TRAILER(grpc-message):7%" "DYNAMIC_METADATA(namespace:object:key):9" "FILTER_STATE(filter.state.key):12" cluster: db statPrefix: db `, diff --git a/pkg/xds/envoy/listeners/util.go b/pkg/xds/envoy/listeners/util.go index 6689ae07a25a..512cbf48dc66 100644 --- a/pkg/xds/envoy/listeners/util.go +++ b/pkg/xds/envoy/listeners/util.go @@ -7,8 +7,31 @@ import ( "github.com/golang/protobuf/ptypes" envoy_listener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener" + envoy_hcm "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2" + envoy_tcp "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/tcp_proxy/v2" + envoy_wellknown "github.com/envoyproxy/go-control-plane/pkg/wellknown" ) +func UpdateTcpProxyConfig(filterChain *envoy_listener.FilterChain, updateFunc func(*envoy_tcp.TcpProxy) error) error { + return UpdateFilterConfig(filterChain, envoy_wellknown.TCPProxy, func(filterConfig proto.Message) error { + tcpProxy, ok := filterConfig.(*envoy_tcp.TcpProxy) + if !ok { + return NewUnexpectedFilterConfigTypeError(filterConfig, (*envoy_tcp.TcpProxy)(nil)) + } + return updateFunc(tcpProxy) + }) +} + +func UpdateHttpConnectionManagerConfig(filterChain *envoy_listener.FilterChain, updateFunc func(*envoy_hcm.HttpConnectionManager) error) error { + return UpdateFilterConfig(filterChain, envoy_wellknown.HTTPConnectionManager, func(filterConfig proto.Message) error { + hcm, ok := filterConfig.(*envoy_hcm.HttpConnectionManager) + if !ok { + return NewUnexpectedFilterConfigTypeError(filterConfig, (*envoy_hcm.HttpConnectionManager)(nil)) + } + return updateFunc(hcm) + }) +} + func UpdateFilterConfig(filterChain *envoy_listener.FilterChain, filterName string, updateFunc func(proto.Message) error) error { for i, filter := range filterChain.Filters { if filter.Name == filterName { diff --git a/pkg/xds/generator/outbound_proxy_generator_test.go b/pkg/xds/generator/outbound_proxy_generator_test.go index 304617bccdfa..319d990946ec 100644 --- a/pkg/xds/generator/outbound_proxy_generator_test.go +++ b/pkg/xds/generator/outbound_proxy_generator_test.go @@ -139,6 +139,24 @@ var _ = Describe("OutboundProxyGenerator", func() { {Target: "192.168.0.3", Port: 5432, Tags: map[string]string{"service": "db", "role": "master"}}, }, }, + Logs: model.LogMap{ + "api-http": &mesh_proto.LoggingBackend{ + Name: "file", + Type: &mesh_proto.LoggingBackend_File_{ + File: &mesh_proto.LoggingBackend_File{ + Path: "/var/log", + }, + }, + }, + "api-tcp": &mesh_proto.LoggingBackend{ + Name: "elk", + Type: &mesh_proto.LoggingBackend_Tcp_{ + Tcp: &mesh_proto.LoggingBackend_Tcp{ + Address: "logstash:1234", + }, + }, + }, + }, Metadata: &model.DataplaneMetadata{}, } @@ -179,6 +197,7 @@ var _ = Describe("OutboundProxyGenerator", func() { ctx: plainCtx, dataplane: ` networking: + address: 10.0.0.1 outbound: - port: 18080 service: backend @@ -189,6 +208,7 @@ var _ = Describe("OutboundProxyGenerator", func() { ctx: mtlsCtx, dataplane: ` networking: + address: 10.0.0.1 outbound: - port: 18080 service: backend @@ -201,6 +221,7 @@ var _ = Describe("OutboundProxyGenerator", func() { ctx: plainCtx, dataplane: ` networking: + address: 10.0.0.1 outbound: - port: 18080 service: backend @@ -211,6 +232,7 @@ var _ = Describe("OutboundProxyGenerator", func() { ctx: mtlsCtx, dataplane: ` networking: + address: 10.0.0.1 outbound: - port: 18080 service: backend @@ -223,6 +245,7 @@ var _ = Describe("OutboundProxyGenerator", func() { ctx: plainCtx, dataplane: ` networking: + address: 10.0.0.1 outbound: - port: 18080 service: backend @@ -235,6 +258,7 @@ var _ = Describe("OutboundProxyGenerator", func() { ctx: mtlsCtx, dataplane: ` networking: + address: 10.0.0.1 outbound: - port: 18080 service: backend @@ -249,6 +273,10 @@ var _ = Describe("OutboundProxyGenerator", func() { ctx: plainCtx, dataplane: ` networking: + address: 10.0.0.1 + gateway: + tags: + service: gateway outbound: - port: 18080 service: backend @@ -265,6 +293,11 @@ var _ = Describe("OutboundProxyGenerator", func() { ctx: mtlsCtx, dataplane: ` networking: + address: 10.0.0.1 + inbound: + - port: 8080 + tags: + service: web outbound: - port: 18080 service: backend diff --git a/pkg/xds/generator/proxy_template.go b/pkg/xds/generator/proxy_template.go index 7c6e1a2d3e72..66ed4188e4a8 100644 --- a/pkg/xds/generator/proxy_template.go +++ b/pkg/xds/generator/proxy_template.go @@ -195,15 +195,17 @@ func (g OutboundProxyGenerator) Generate(ctx xds_context.Context, proxy *model.P // configuration for HTTP case filterChainBuilder. Configure(envoy_listeners.HttpConnectionManager(outbound.Service)). - Configure(envoy_listeners.HttpOutboundRoute(outboundRouteName)) + Configure(envoy_listeners.HttpOutboundRoute(outboundRouteName)). + Configure(envoy_listeners.HttpAccessLog(sourceService, destinationService, proxy.Logs[outbound.Service], proxy)) case mesh_core.ProtocolTCP: fallthrough default: // configuration for non-HTTP cases - filterChainBuilder.Configure(envoy_listeners.TcpProxy(outbound.Service, clusters...)) + filterChainBuilder. + Configure(envoy_listeners.TcpProxy(outbound.Service, clusters...)). + Configure(envoy_listeners.NetworkAccessLog(sourceService, destinationService, proxy.Logs[outbound.Service], proxy)) } - // TODO(yskopets): configure NetworkAccessLog only for non-HTTP cases - return filterChainBuilder.Configure(envoy_listeners.NetworkAccessLog(sourceService, destinationService, proxy.Logs[outbound.Service], proxy)) + return filterChainBuilder }() listener, err := envoy_listeners.NewListenerBuilder(). Configure(envoy_listeners.OutboundListener(outboundListenerName, ofaces[i].DataplaneIP, ofaces[i].DataplanePort)). diff --git a/pkg/xds/generator/testdata/outbound-proxy/09.envoy.golden.yaml b/pkg/xds/generator/testdata/outbound-proxy/09.envoy.golden.yaml index 437d680d622a..755155bdd0e4 100644 --- a/pkg/xds/generator/testdata/outbound-proxy/09.envoy.golden.yaml +++ b/pkg/xds/generator/testdata/outbound-proxy/09.envoy.golden.yaml @@ -150,6 +150,13 @@ resources: - name: envoy.http_connection_manager typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager + accessLog: + - name: envoy.file_access_log + typedConfig: + '@type': type.googleapis.com/envoy.config.accesslog.v2.FileAccessLog + format: | + [%START_TIME%] "%REQ(:method)% %REQ(x-envoy-original-path?:path)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(x-envoy-upstream-service-time)% "%REQ(x-forwarded-for)%" "%REQ(user-agent)%" "%REQ(x-request-id)%" "%REQ(:authority)%" "gateway" "api-http" "10.0.0.1" "%UPSTREAM_HOST%" + path: /var/log httpFilters: - name: envoy.router rds: @@ -220,6 +227,16 @@ resources: - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy + accessLog: + - name: envoy.http_grpc_access_log + typedConfig: + '@type': type.googleapis.com/envoy.config.accesslog.v2.HttpGrpcAccessLogConfig + commonConfig: + grpcService: + envoyGrpc: + clusterName: access_log_sink + logName: | + logstash:1234;[%START_TIME%] 10.0.0.1(gateway)->%UPSTREAM_HOST%(api-tcp) took %DURATION%ms, sent %BYTES_SENT% bytes, received: %BYTES_RECEIVED% bytes cluster: api-tcp statPrefix: api-tcp name: outbound:127.0.0.1:40002 diff --git a/pkg/xds/generator/testdata/outbound-proxy/10.envoy.golden.yaml b/pkg/xds/generator/testdata/outbound-proxy/10.envoy.golden.yaml index b540b316a991..0d533a19601e 100644 --- a/pkg/xds/generator/testdata/outbound-proxy/10.envoy.golden.yaml +++ b/pkg/xds/generator/testdata/outbound-proxy/10.envoy.golden.yaml @@ -268,6 +268,13 @@ resources: - name: envoy.http_connection_manager typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager + accessLog: + - name: envoy.file_access_log + typedConfig: + '@type': type.googleapis.com/envoy.config.accesslog.v2.FileAccessLog + format: | + [%START_TIME%] "%REQ(:method)% %REQ(x-envoy-original-path?:path)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(x-envoy-upstream-service-time)% "%REQ(x-forwarded-for)%" "%REQ(user-agent)%" "%REQ(x-request-id)%" "%REQ(:authority)%" "web" "api-http" "10.0.0.1" "%UPSTREAM_HOST%" + path: /var/log httpFilters: - name: envoy.router rds: @@ -368,6 +375,16 @@ resources: - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy + accessLog: + - name: envoy.http_grpc_access_log + typedConfig: + '@type': type.googleapis.com/envoy.config.accesslog.v2.HttpGrpcAccessLogConfig + commonConfig: + grpcService: + envoyGrpc: + clusterName: access_log_sink + logName: | + logstash:1234;[%START_TIME%] 10.0.0.1(web)->%UPSTREAM_HOST%(api-tcp) took %DURATION%ms, sent %BYTES_SENT% bytes, received: %BYTES_RECEIVED% bytes cluster: api-tcp statPrefix: api-tcp name: outbound:127.0.0.1:40002