kuma-cp: generate HTTP-specific configuration of access log

pkg/xds/envoy/listeners/access_log_configurer.go

@@ -0,0 +1,109 @@
+package listeners
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/pkg/errors"
+
+	"github.com/golang/protobuf/ptypes"
+
+	envoy_core "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	envoy_accesslog "github.com/envoyproxy/go-control-plane/envoy/config/accesslog/v2"
+	filter_accesslog "github.com/envoyproxy/go-control-plane/envoy/config/filter/accesslog/v2"
+	envoy_wellknown "github.com/envoyproxy/go-control-plane/pkg/wellknown"
+
+	mesh_proto "github.com/Kong/kuma/api/mesh/v1alpha1"
+	core_xds "github.com/Kong/kuma/pkg/core/xds"
+	"github.com/Kong/kuma/pkg/envoy/accesslog"
+)
+
+const accessLogSink = "access_log_sink"
+
+type AccessLogConfigurer struct {
+	sourceService      string
+	destinationService string
+	backend            *mesh_proto.LoggingBackend
+	proxy              *core_xds.Proxy
+}
+
+func convertLoggingBackend(sourceService string, destinationService string, backend *mesh_proto.LoggingBackend, proxy *core_xds.Proxy, defaultFormat string) (*filter_accesslog.AccessLog, error) {
+	if backend == nil {
+		return nil, nil
+	}
+	formatString := defaultFormat
+	if backend.Format != "" {
+		formatString = backend.Format
+	}
+	format, err := accesslog.ParseFormat(formatString)
+	if err != nil {
+		return nil, errors.Wrapf(err, "invalid access log format string: %s", formatString)
+	}
+
+	variables := accesslog.InterpolationVariables{
+		accesslog.CMD_KUMA_SOURCE_ADDRESS:              net.JoinHostPort(proxy.Dataplane.GetIP(), "0"), // deprecated variable
+		accesslog.CMD_KUMA_SOURCE_ADDRESS_WITHOUT_PORT: proxy.Dataplane.GetIP(),                        // replacement variable
+		accesslog.CMD_KUMA_SOURCE_SERVICE:              sourceService,
+		accesslog.CMD_KUMA_DESTINATION_SERVICE:         destinationService,
+	}
+
+	format, err = format.Interpolate(variables)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to interpolate access log format string with Kuma-specific variables: %s", formatString)
+	}
+
+	if file, ok := backend.GetType().(*mesh_proto.LoggingBackend_File_); ok {
+		return fileAccessLog(format, file)
+	} else if tcp, ok := backend.GetType().(*mesh_proto.LoggingBackend_Tcp_); ok {
+		return tcpAccessLog(format, tcp)
+	} else {
+		return nil, errors.Errorf("could not convert LoggingBackend of type %T to AccessLog", backend.GetType())
+	}
+}
+
+func tcpAccessLog(format *accesslog.AccessLogFormat, tcp *mesh_proto.LoggingBackend_Tcp_) (*filter_accesslog.AccessLog, error) {
+	httpGrpcAccessLog := &envoy_accesslog.HttpGrpcAccessLogConfig{
+		CommonConfig: &envoy_accesslog.CommonGrpcAccessLogConfig{
+			LogName: fmt.Sprintf("%s;%s", tcp.Tcp.Address, format.String()),
+			GrpcService: &envoy_core.GrpcService{
+				TargetSpecifier: &envoy_core.GrpcService_EnvoyGrpc_{
+					EnvoyGrpc: &envoy_core.GrpcService_EnvoyGrpc{
+						ClusterName: accessLogSink,
+					},
+				},
+			},
+		},
+	}
+	if err := format.ConfigureHttpLog(httpGrpcAccessLog); err != nil {
+		return nil, errors.Wrapf(err, "failed to configure %T according to the format string: %s", httpGrpcAccessLog, format)
+	}
+	marshalled, err := ptypes.MarshalAny(httpGrpcAccessLog)
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not marshall %T", httpGrpcAccessLog)
+	}
+	return &filter_accesslog.AccessLog{
+		Name: envoy_wellknown.HTTPGRPCAccessLog,
+		ConfigType: &filter_accesslog.AccessLog_TypedConfig{
+			TypedConfig: marshalled,
+		},
+	}, nil
+}
+
+func fileAccessLog(format *accesslog.AccessLogFormat, file *mesh_proto.LoggingBackend_File_) (*filter_accesslog.AccessLog, error) {
+	fileAccessLog := &envoy_accesslog.FileAccessLog{
+		AccessLogFormat: &envoy_accesslog.FileAccessLog_Format{
+			Format: format.String(),
+		},
+		Path: file.File.Path,
+	}
+	marshalled, err := ptypes.MarshalAny(fileAccessLog)
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not marshall %T", fileAccessLog)
+	}
+	return &filter_accesslog.AccessLog{
+		Name: envoy_wellknown.FileAccessLog,
+		ConfigType: &filter_accesslog.AccessLog_TypedConfig{
+			TypedConfig: marshalled,
+		},
+	}, nil
+}

pkg/xds/envoy/listeners/http_access_log_configurer.go

@@ -0,0 +1,43 @@
+package listeners
+
+import (
+	envoy_listener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener"
+	envoy_hcm "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2"
+
+	mesh_proto "github.com/Kong/kuma/api/mesh/v1alpha1"
+	core_xds "github.com/Kong/kuma/pkg/core/xds"
+)
+
+const defaultHttpAccessLogFormat = `[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%KUMA_SOURCE_SERVICE%" "%KUMA_DESTINATION_SERVICE%" "%KUMA_SOURCE_ADDRESS_WITHOUT_PORT%" "%UPSTREAM_HOST%"
+` // intentional newline at the end
+
+func HttpAccessLog(sourceService string, destinationService string, backend *mesh_proto.LoggingBackend, proxy *core_xds.Proxy) FilterChainBuilderOpt {
+	return FilterChainBuilderOptFunc(func(config *FilterChainBuilderConfig) {
+		if backend != nil {
+			config.Add(&HttpAccessLogConfigurer{
+				AccessLogConfigurer: AccessLogConfigurer{
+					sourceService:      sourceService,
+					destinationService: destinationService,
+					backend:            backend,
+					proxy:              proxy,
+				},
+			})
+		}
+	})
+}
+
+type HttpAccessLogConfigurer struct {
+	AccessLogConfigurer
+}
+
+func (c *HttpAccessLogConfigurer) Configure(filterChain *envoy_listener.FilterChain) error {
+	accessLog, err := convertLoggingBackend(c.AccessLogConfigurer.sourceService, c.AccessLogConfigurer.destinationService, c.AccessLogConfigurer.backend, c.AccessLogConfigurer.proxy, defaultHttpAccessLogFormat)
+	if err != nil {
+		return err
+	}
+
+	return UpdateHttpConnectionManagerConfig(filterChain, func(hcm *envoy_hcm.HttpConnectionManager) error {
+		hcm.AccessLog = append(hcm.AccessLog, accessLog)
+		return nil
+	})
+}

pkg/xds/envoy/listeners/http_access_log_configurer_test.go

@@ -0,0 +1,200 @@
+package listeners_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/extensions/table"
+	. "github.com/onsi/gomega"
+
+	. "github.com/Kong/kuma/pkg/xds/envoy/listeners"
+
+	mesh_proto "github.com/Kong/kuma/api/mesh/v1alpha1"
+	mesh_core "github.com/Kong/kuma/pkg/core/resources/apis/mesh"
+	"github.com/Kong/kuma/pkg/core/xds"
+	core_xds "github.com/Kong/kuma/pkg/core/xds"
+	util_proto "github.com/Kong/kuma/pkg/util/proto"
+)
+
+var _ = Describe("HttpAccessLogConfigurer", func() {
+
+	type testCase struct {
+		listenerName    string
+		listenerAddress string
+		listenerPort    uint32
+		statsName       string
+		routeName       string
+		backend         *mesh_proto.LoggingBackend
+		expected        string
+	}
+
+	DescribeTable("should generate proper Envoy config",
+		func(given testCase) {
+			// given
+			sourceService := "web"
+			destinationService := "backend"
+			proxy := &core_xds.Proxy{
+				Id: xds.ProxyId{
+					Name: "web",
+					Mesh: "example",
+				},
+				Dataplane: &mesh_core.DataplaneResource{
+					Spec: mesh_proto.Dataplane{
+						Networking: &mesh_proto.Dataplane_Networking{
+							Inbound: []*mesh_proto.Dataplane_Networking_Inbound{{
+								Interface: "192.168.0.1:80:8080",
+								Tags: map[string]string{
+									"service": "web",
+								},
+							}},
+							Outbound: []*mesh_proto.Dataplane_Networking_Outbound{{
+								Interface: ":27070",
+								Service:   "backend",
+							}},
+						},
+					},
+				},
+			}
+
+			// when
+			listener, err := NewListenerBuilder().
+				Configure(OutboundListener(given.listenerName, given.listenerAddress, given.listenerPort)).
+				Configure(FilterChain(NewFilterChainBuilder().
+					Configure(HttpConnectionManager(given.statsName)).
+					Configure(HttpOutboundRoute(given.routeName)).
+					Configure(HttpAccessLog(sourceService, destinationService, given.backend, proxy)))).
+				Build()
+			// then
+			Expect(err).ToNot(HaveOccurred())
+
+			// when
+			actual, err := util_proto.ToYAML(listener)
+			Expect(err).ToNot(HaveOccurred())
+			// and
+			Expect(actual).To(MatchYAML(given.expected))
+		},
+		Entry("basic http_connection_manager without access log", testCase{
+			listenerName:    "outbound:127.0.0.1:27070",
+			listenerAddress: "127.0.0.1",
+			listenerPort:    27070,
+			statsName:       "backend",
+			routeName:       "outbound:backend",
+			backend:         nil,
+			expected: `
+            name: outbound:127.0.0.1:27070
+            address:
+              socketAddress:
+                address: 127.0.0.1
+                portValue: 27070
+            filterChains:
+            - filters:
+              - name: envoy.http_connection_manager
+                typedConfig:
+                  '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager
+                  httpFilters:
+                  - name: envoy.router
+                  rds:
+                    configSource:
+                      ads: {}
+                    routeConfigName: outbound:backend
+                  statPrefix: backend
+`,
+		}),
+		Entry("basic http_connection_manager with file access log", testCase{
+			listenerName:    "outbound:127.0.0.1:27070",
+			listenerAddress: "127.0.0.1",
+			listenerPort:    27070,
+			statsName:       "backend",
+			routeName:       "outbound:backend",
+			backend: &mesh_proto.LoggingBackend{
+				Name: "file",
+				Type: &mesh_proto.LoggingBackend_File_{
+					File: &mesh_proto.LoggingBackend_File{
+						Path: "/tmp/log",
+					},
+				},
+			},
+			expected: `
+            name: outbound:127.0.0.1:27070
+            address:
+              socketAddress:
+                address: 127.0.0.1
+                portValue: 27070
+            filterChains:
+            - filters:
+              - name: envoy.http_connection_manager
+                typedConfig:
+                  '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager
+                  accessLog:
+                  - name: envoy.file_access_log
+                    typedConfig:
+                      '@type': type.googleapis.com/envoy.config.accesslog.v2.FileAccessLog
+                      format: |
+                        [%START_TIME%] "%REQ(:method)% %REQ(x-envoy-original-path?:path)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(x-envoy-upstream-service-time)% "%REQ(x-forwarded-for)%" "%REQ(user-agent)%" "%REQ(x-request-id)%" "%REQ(:authority)%" "web" "backend" "192.168.0.1" "%UPSTREAM_HOST%"
+                      path: /tmp/log
+                  httpFilters:
+                  - name: envoy.router
+                  rds:
+                    configSource:
+                      ads: {}
+                    routeConfigName: outbound:backend
+                  statPrefix: backend
+`,
+		}),
+		Entry("basic http_connection_manager with tcp access log", testCase{
+			listenerName:    "outbound:127.0.0.1:27070",
+			listenerAddress: "127.0.0.1",
+			listenerPort:    27070,
+			statsName:       "backend",
+			routeName:       "outbound:backend",
+			backend: &mesh_proto.LoggingBackend{
+				Name: "tcp",
+				Format: `[%START_TIME%] "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%REQ(ORIGIN)%" "%REQ(CONTENT-TYPE)%" "%KUMA_SOURCE_SERVICE%" "%KUMA_DESTINATION_SERVICE%" "%KUMA_SOURCE_ADDRESS%" "%KUMA_SOURCE_ADDRESS_WITHOUT_PORT%" "%UPSTREAM_HOST%"
+
+"%RESP(SERVER):5%" "%TRAILER(GRPC-MESSAGE):7%" "DYNAMIC_METADATA(namespace:object:key):9" "FILTER_STATE(filter.state.key):12"
+`, // intentional newline at the end
+				Type: &mesh_proto.LoggingBackend_Tcp_{
+					Tcp: &mesh_proto.LoggingBackend_Tcp{
+						Address: "127.0.0.1:1234",
+					},
+				},
+			},
+			expected: `
+            name: outbound:127.0.0.1:27070
+            address:
+              socketAddress:
+                address: 127.0.0.1
+                portValue: 27070
+            filterChains:
+            - filters:
+              - name: envoy.http_connection_manager
+                typedConfig:
+                  '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager
+                  accessLog:
+                  - name: envoy.http_grpc_access_log
+                    typedConfig:
+                      '@type': type.googleapis.com/envoy.config.accesslog.v2.HttpGrpcAccessLogConfig
+                      additionalRequestHeadersToLog:
+                      - origin
+                      - content-type
+                      additionalResponseHeadersToLog:
+                      - server
+                      additionalResponseTrailersToLog:
+                      - grpc-message
+                      commonConfig:
+                        grpcService:
+                          envoyGrpc:
+                            clusterName: access_log_sink
+                        logName: |
+                          127.0.0.1:1234;[%START_TIME%] "%REQ(x-request-id)%" "%REQ(:authority)%" "%REQ(origin)%" "%REQ(content-type)%" "web" "backend" "192.168.0.1:0" "192.168.0.1" "%UPSTREAM_HOST%"
+
+                          "%RESP(server):5%" "%TRAILER(grpc-message):7%" "DYNAMIC_METADATA(namespace:object:key):9" "FILTER_STATE(filter.state.key):12"
+                  httpFilters:
+                  - name: envoy.router
+                  rds:
+                    configSource:
+                      ads: {}
+                    routeConfigName: outbound:backend
+                  statPrefix: backend
+`,
+		}),
+	)
+})