From 11c72230e0ad00875022c43ce3079f1d866ebe78 Mon Sep 17 00:00:00 2001 From: Jakub Dyszkiewicz Date: Mon, 27 Jan 2020 17:31:33 +0100 Subject: [PATCH 1/4] feat(kuma-cp) sanitize stats --- Makefile.e2e.mk | 16 ++-- pkg/util/xds/metric_sanitizer.go | 18 +++++ pkg/util/xds/metric_sanitizer_test.go | 20 +++++ pkg/xds/envoy/envoy.go | 11 +-- pkg/xds/envoy/envoy_test.go | 12 +-- pkg/xds/envoy/rbac.go | 3 +- .../outbound_proxy_generator_test.go | 75 +++++++++++++++++++ .../prometheus_endpoint_generator_test.go | 4 +- pkg/xds/generator/proxy_template.go | 7 +- .../inbound-proxy/3-envoy-config.golden.yaml | 4 +- .../inbound-proxy/4-envoy-config.golden.yaml | 4 +- .../inbound-proxy/5-envoy-config.golden.yaml | 8 +- .../inbound-proxy/6-envoy-config.golden.yaml | 8 +- .../inbound-proxy/7-envoy-config.golden.yaml | 16 ++-- .../inbound-proxy/8-envoy-config.golden.yaml | 16 ++-- .../cluster-dots.envoy.golden.yaml | 61 +++++++++++++++ .../profile-source/1-envoy-config.golden.yaml | 4 +- .../profile-source/2-envoy-config.golden.yaml | 4 +- .../profile-source/3-envoy-config.golden.yaml | 6 +- .../profile-source/4-envoy-config.golden.yaml | 6 +- .../template-proxy/1-envoy-config.golden.yaml | 4 +- .../server/testdata/envoy-config.golden.yaml | 16 ++-- 22 files changed, 251 insertions(+), 72 deletions(-) create mode 100644 pkg/util/xds/metric_sanitizer.go create mode 100644 pkg/util/xds/metric_sanitizer_test.go create mode 100644 pkg/xds/generator/testdata/outbound-proxy/cluster-dots.envoy.golden.yaml diff --git a/Makefile.e2e.mk b/Makefile.e2e.mk index 11d86ee19428..cc991b21152a 100644 --- a/Makefile.e2e.mk +++ b/Makefile.e2e.mk @@ -332,7 +332,7 @@ wait/example/minikube: ## Minikube: Wait for demo setup to get ready wait/example/minikube/mtls: ## Minikube: Wait until incoming Listener and outgoing Cluster have been configured for mTLS $(call kubectl_exec,kuma-demo,demo-client,demo-client) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_mtls_listeners_count,inbound,3000) ) -eq 1 ]]; then echo "listener has been configured for mTLS "; exit 0; fi; sleep 1; done; echo -e "\nError: listener has not been configured for mTLS" ; exit 1' - $(call kubectl_exec,kuma-demo,demo-client,demo-client) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_mtls_clusters_count,demo-app.kuma-demo.svc:8000) ) -eq 1 ]]; then echo "cluster has been configured for mTLS "; exit 0; fi; sleep 1; done; echo -e "\nError: cluster has not been configured for mTLS" ; exit 1' + $(call kubectl_exec,kuma-demo,demo-client,demo-client) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_mtls_clusters_count,demo-app_kuma-demo_svc_8000) ) -eq 1 ]]; then echo "cluster has been configured for mTLS "; exit 0; fi; sleep 1; done; echo -e "\nError: cluster has not been configured for mTLS" ; exit 1' curl/example/minikube: ## Minikube: Make sample requests to demo setup $(call kubectl_exec,kuma-demo,demo-client,demo-client) $(call curl_example_client) @@ -352,7 +352,7 @@ verify/example/minikube/mtls: verify/example/minikube/mtls/outbound ## Minikube: verify/example/minikube/mtls/outbound: @echo "Checking number of Outbound mTLS requests via Envoy ..." - test $$( $(call kubectl_exec,kuma-demo,demo-client,kuma-sidecar) wget -qO- http://localhost:9901/stats/prometheus | grep 'envoy_cluster_kuma_demo_svc_8000_ssl_handshake{envoy_cluster_name="demo-app"}' | awk '{print $$2}' | tr -d [:space:] ) -ge 5 + test $$( $(call kubectl_exec,kuma-demo,demo-client,kuma-sidecar) wget -qO- http://localhost:9901/stats/prometheus | grep 'envoy_cluster_ssl_handshake{envoy_cluster_name="demo-app_kuma-demo_svc_8000"}' | awk '{print $$2}' | tr -d [:space:] ) -ge 5 @echo "Check passed!" kumactl/example/minikube: @@ -404,7 +404,7 @@ wait/traffic-routing/minikube/mtls: ## Minikube: Wait until incoming Listener an @echo "Waiting until incoming Listener and outgoing Cluster have been configured for mTLS ..." @echo $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_mtls_listeners_count,inbound,6060) ) -eq 1 ]]; then echo "listener has been configured for mTLS "; exit 0; fi; sleep 1; done; echo -e "\nError: listener has not been configured for mTLS" ; exit 1' - $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_mtls_clusters_count,kuma-example-backend.kuma-example.svc:7070) ) -eq 1 ]]; then echo "cluster has been configured for mTLS "; exit 0; fi; sleep 1; done; echo -e "\nError: cluster has not been configured for mTLS" ; exit 1' + $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_mtls_clusters_count,kuma-example-backend_kuma-example_svc_7070) ) -eq 1 ]]; then echo "cluster has been configured for mTLS "; exit 0; fi; sleep 1; done; echo -e "\nError: cluster has not been configured for mTLS" ; exit 1' apply/traffic-routing/minikube/no-mtls: ## Minikube: disable mTLS @echo @@ -417,7 +417,7 @@ wait/traffic-routing/minikube/no-mtls: ## Minikube: Wait until mTLS has been dis @echo "Waiting until mTLS has been disabled on incoming Listener and outgoing Cluster ..." @echo $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_mtls_listeners_count,inbound,6060) ) -eq 0 ]]; then echo "listener is no longer configured for mTLS "; exit 0; fi; sleep 1; done; echo -e "\nError: listener is still configured for mTLS" ; exit 1' - $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_mtls_clusters_count,kuma-example-backend.kuma-example.svc:7070) ) -eq 0 ]]; then echo "cluster is no longer configured for mTLS "; exit 0; fi; sleep 1; done; echo -e "\nError: cluster is still configured for mTLS" ; exit 1' + $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_mtls_clusters_count,kuma-example-backend_kuma-example_svc_7070) ) -eq 0 ]]; then echo "cluster is no longer configured for mTLS "; exit 0; fi; sleep 1; done; echo -e "\nError: cluster is still configured for mTLS" ; exit 1' wait/traffic-routing/minikube: ## Minikube: Wait for example setup for TrafficRoute to get ready @echo @@ -442,8 +442,8 @@ wait/traffic-routing/minikube/web-to-backend-route: ## Minikube: Wait until cust @echo @echo "Waiting until custom 'web-to-backend' TrafficRoute is applied ..." @echo - $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_routing_listeners_count,outbound,7070,kuma-example-backend.kuma-example.svc:7070{version=v2}) ) -eq 1 ]]; then echo "listener is now configured for subset routing "; exit 0; fi; sleep 1; done; echo -e "\nError: listener has not been configured for subset routing" ; exit 1' - $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_routing_clusters_count,kuma-example-backend.kuma-example.svc:7070{version=v2}) ) -eq 1 ]]; then echo "cluster is now configured for subset routing "; exit 0; fi; sleep 1; done; echo -e "\nError: cluster has not been configured for subset routing" ; exit 1' + $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_routing_listeners_count,outbound,7070,kuma-example-backend_kuma-example_svc_7070{version=v2}) ) -eq 1 ]]; then echo "listener is now configured for subset routing "; exit 0; fi; sleep 1; done; echo -e "\nError: listener has not been configured for subset routing" ; exit 1' + $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_routing_clusters_count,kuma-example-backend_kuma-example_svc_7070{version=v2}) ) -eq 1 ]]; then echo "cluster is now configured for subset routing "; exit 0; fi; sleep 1; done; echo -e "\nError: cluster has not been configured for subset routing" ; exit 1' verify/traffic-routing/minikube/web-to-backend-route: ## Minikube: Make sample requests to example setup for TrafficRoute @echo @@ -462,8 +462,8 @@ wait/traffic-routing/minikube/no-web-to-backend-route: ## Minikube: Wait until c @echo @echo "Waiting until custom 'web-to-backend' TrafficRoute is removed ..." @echo - $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_routing_listeners_count,outbound,7070,kuma-example-backend.kuma-example.svc:7070{version=v2}) ) -eq 0 ]]; then echo "listener is no longer configured for subset routing "; exit 0; fi; sleep 1; done; echo -e "\nError: listener is still configured for subset routing" ; exit 1' - $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_routing_clusters_count,kuma-example-backend.kuma-example.svc:7070{version=v2}) ) -eq 0 ]]; then echo "cluster is no longer configured for subset routing "; exit 0; fi; sleep 1; done; echo -e "\nError: cluster is still configured for subset routing" ; exit 1' + $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_routing_listeners_count,outbound,7070,kuma-example-backend_kuma-example_svc_7070{version=v2}) ) -eq 0 ]]; then echo "listener is no longer configured for subset routing "; exit 0; fi; sleep 1; done; echo -e "\nError: listener is still configured for subset routing" ; exit 1' + $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_routing_clusters_count,kuma-example-backend_kuma-example_svc_7070{version=v2}) ) -eq 0 ]]; then echo "cluster is no longer configured for subset routing "; exit 0; fi; sleep 1; done; echo -e "\nError: cluster is still configured for subset routing" ; exit 1' undeploy/traffic-routing/minikube: ## Minikube: Undeploy example setup for TrafficRoute @echo diff --git a/pkg/util/xds/metric_sanitizer.go b/pkg/util/xds/metric_sanitizer.go new file mode 100644 index 000000000000..1d6aa773a941 --- /dev/null +++ b/pkg/util/xds/metric_sanitizer.go @@ -0,0 +1,18 @@ +package xds + +import ( + "regexp" + "strings" +) + +var ( + whitespaces = regexp.MustCompile(`\s+`) + illegalChars = regexp.MustCompile(`[^a-zA-Z_\-0-9{}=]`) +) + +func SanitizeMetric(metric string) string { + result := whitespaces.ReplaceAllString(metric, "_") + result = strings.ReplaceAll(result, "/", "_") + result = illegalChars.ReplaceAllString(result, "_") + return result +} diff --git a/pkg/util/xds/metric_sanitizer_test.go b/pkg/util/xds/metric_sanitizer_test.go new file mode 100644 index 000000000000..f7771e482f57 --- /dev/null +++ b/pkg/util/xds/metric_sanitizer_test.go @@ -0,0 +1,20 @@ +package xds_test + +import ( + "github.com/Kong/kuma/pkg/util/xds" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Metric sanitizer", func() { + It("should sanitize metrics", func() { + // given + metric := "some metric with chars :/_-0123{}" + + // when + sanitized := xds.SanitizeMetric(metric) + + // then + Expect(sanitized).To(Equal("some_metric_with_chars____-0123{}")) + }) +}) diff --git a/pkg/xds/envoy/envoy.go b/pkg/xds/envoy/envoy.go index 1ec349a78e5f..9fa528d9806d 100644 --- a/pkg/xds/envoy/envoy.go +++ b/pkg/xds/envoy/envoy.go @@ -15,6 +15,7 @@ import ( mesh_core "github.com/Kong/kuma/pkg/core/resources/apis/mesh" core_xds "github.com/Kong/kuma/pkg/core/xds" util_error "github.com/Kong/kuma/pkg/util/error" + util_xds "github.com/Kong/kuma/pkg/util/xds" xds_context "github.com/Kong/kuma/pkg/xds/context" v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2" envoy_cluster "github.com/envoyproxy/go-control-plane/envoy/api/v2/cluster" @@ -185,7 +186,7 @@ func CreateOutboundListener(ctx xds_context.Context, listenerName string, addres } config := &envoy_tcp.TcpProxy{ - StatPrefix: statsName, + StatPrefix: util_xds.SanitizeMetric(statsName), AccessLog: accessLogs, } if len(clusters) == 1 { @@ -241,7 +242,7 @@ func CreateOutboundListener(ctx xds_context.Context, listenerName string, addres func CreateInboundListener(ctx xds_context.Context, listenerName string, address string, port uint32, clusterName string, virtual bool, permissions *mesh_core.TrafficPermissionResourceList, metadata *core_xds.DataplaneMetadata) *v2.Listener { config := &envoy_tcp.TcpProxy{ - StatPrefix: clusterName, + StatPrefix: util_xds.SanitizeMetric(clusterName), ClusterSpecifier: &envoy_tcp.TcpProxy_Cluster{ Cluster: clusterName, }, @@ -289,7 +290,7 @@ func CreateInboundListener(ctx xds_context.Context, listenerName string, address func CreatePrometheusListener(ctx xds_context.Context, listenerName string, address string, port uint32, path string, clusterName string, virtual bool, metadata *core_xds.DataplaneMetadata) *v2.Listener { config := &envoy_hcm.HttpConnectionManager{ - StatPrefix: listenerName, + StatPrefix: util_xds.SanitizeMetric(listenerName), CodecType: envoy_hcm.HttpConnectionManager_AUTO, HttpFilters: []*envoy_hcm.HttpFilter{{ Name: wellknown.Router, @@ -423,7 +424,7 @@ func sdsSecretConfig(context xds_context.Context, name string, metadata *core_xd TargetSpecifier: &envoy_core.GrpcService_GoogleGrpc_{ GoogleGrpc: withCallCredentials(&envoy_core.GrpcService_GoogleGrpc{ TargetUri: context.ControlPlane.SdsLocation, - StatPrefix: "sds_" + name, + StatPrefix: util_xds.SanitizeMetric("sds_" + name), ChannelCredentials: &envoy_core.GrpcService_GoogleGrpc_ChannelCredentials{ CredentialSpecifier: &envoy_core.GrpcService_GoogleGrpc_ChannelCredentials_SslCredentials{ SslCredentials: &envoy_core.GrpcService_GoogleGrpc_SslCredentials{ @@ -447,7 +448,7 @@ func sdsSecretConfig(context xds_context.Context, name string, metadata *core_xd func CreateCatchAllListener(ctx xds_context.Context, listenerName string, address string, port uint32, clusterName string) *v2.Listener { config := &envoy_tcp.TcpProxy{ - StatPrefix: clusterName, + StatPrefix: util_xds.SanitizeMetric(clusterName), ClusterSpecifier: &envoy_tcp.TcpProxy_Cluster{ Cluster: clusterName, }, diff --git a/pkg/xds/envoy/envoy_test.go b/pkg/xds/envoy/envoy_test.go index a7e6ad4a11df..04a5c576f7bd 100644 --- a/pkg/xds/envoy/envoy_test.go +++ b/pkg/xds/envoy/envoy_test.go @@ -578,7 +578,7 @@ var _ = Describe("Envoy", func() { typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8080 - statPrefix: localhost:8080 + statPrefix: localhost_8080 `, }), Entry("with transparent proxying", testCase{ @@ -601,7 +601,7 @@ var _ = Describe("Envoy", func() { typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8080 - statPrefix: localhost:8080 + statPrefix: localhost_8080 deprecatedV1: bindToPort: false `, @@ -642,12 +642,12 @@ var _ = Describe("Envoy", func() { - authenticated: principalName: exact: spiffe://default/web1 - statPrefix: inbound:192.168.0.1:8080. + statPrefix: inbound_192_168_0_1_8080. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8080 - statPrefix: localhost:8080 + statPrefix: localhost_8080 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: @@ -719,12 +719,12 @@ var _ = Describe("Envoy", func() { - authenticated: principalName: exact: spiffe://default/web1 - statPrefix: inbound:192.168.0.1:8080. + statPrefix: inbound_192_168_0_1_8080. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8080 - statPrefix: localhost:8080 + statPrefix: localhost_8080 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: diff --git a/pkg/xds/envoy/rbac.go b/pkg/xds/envoy/rbac.go index de42070d3653..2f51fb236a78 100644 --- a/pkg/xds/envoy/rbac.go +++ b/pkg/xds/envoy/rbac.go @@ -5,6 +5,7 @@ import ( "github.com/Kong/kuma/api/mesh/v1alpha1" mesh_core "github.com/Kong/kuma/pkg/core/resources/apis/mesh" util_error "github.com/Kong/kuma/pkg/util/error" + util_xds "github.com/Kong/kuma/pkg/util/xds" envoy_listener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener" rbac "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/rbac/v2" rbac_config "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v2" @@ -38,7 +39,7 @@ func createRbacRule(listenerName string, permissions *mesh_core.TrafficPermissio Action: rbac_config.RBAC_ALLOW, Policies: policies, }, - StatPrefix: fmt.Sprintf("%s.", listenerName), // we include dot to change "inbound:127.0.0.1:21011rbac.allowed" metric to "inbound:127.0.0.1:21011.rbac.allowed" + StatPrefix: fmt.Sprintf("%s.", util_xds.SanitizeMetric(listenerName)), // we include dot to change "inbound:127.0.0.1:21011rbac.allowed" metric to "inbound:127.0.0.1:21011.rbac.allowed" } } diff --git a/pkg/xds/generator/outbound_proxy_generator_test.go b/pkg/xds/generator/outbound_proxy_generator_test.go index 90bd62b0a055..3d26d44193fd 100644 --- a/pkg/xds/generator/outbound_proxy_generator_test.go +++ b/pkg/xds/generator/outbound_proxy_generator_test.go @@ -213,6 +213,81 @@ var _ = Describe("OutboundProxyGenerator", func() { }), ) + It("Sanitize cluster names", func() { + // setup + gen := &generator.OutboundProxyGenerator{} + dp := ` + networking: + outbound: + - interface: :18080 + service: backend.kuma-system + - interface: :54321 + service: db.kuma-system` + + dataplane := mesh_proto.Dataplane{} + Expect(util_proto.FromYAML([]byte(dp), &dataplane)).To(Succeed()) + + proxy := &model.Proxy{ + Id: model.ProxyId{Name: "side-car", Mesh: "default"}, + Dataplane: &mesh_core.DataplaneResource{ + Meta: &test_model.ResourceMeta{ + Version: "1", + }, + Spec: dataplane, + }, + TrafficRoutes: model.RouteMap{ + "backend.kuma-system": &mesh_core.TrafficRouteResource{ + Spec: mesh_proto.TrafficRoute{ + Conf: []*mesh_proto.TrafficRoute_WeightedDestination{{ + Weight: 100, + Destination: mesh_proto.MatchService("backend.kuma-system"), + }}, + }, + }, + "db.kuma-system": &mesh_core.TrafficRouteResource{ + Spec: mesh_proto.TrafficRoute{ + Conf: []*mesh_proto.TrafficRoute_WeightedDestination{{ + Weight: 100, + Destination: mesh_proto.TagSelector{"service": "db", "version": "3.2.0"}, + }, + }}, + }, + }, + OutboundSelectors: model.DestinationMap{ + "backend.kuma-system": model.TagSelectorSet{ + {"service": "backend.kuma-system"}, + }, + "db.kuma-system": model.TagSelectorSet{ + {"service": "db", "version": "3.2.0"}, + }, + }, + OutboundTargets: model.EndpointMap{ + "backend.kuma-system": []model.Endpoint{ + {Target: "192.168.0.1", Port: 8082}, + }, + "db.kuma-system": []model.Endpoint{ + {Target: "192.168.0.2", Port: 5432, Tags: map[string]string{"service": "db", "role": "master"}}, + }, + }, + Metadata: &model.DataplaneMetadata{}, + } + + // when + rs, err := gen.Generate(plainCtx, proxy) + + // then + Expect(err).ToNot(HaveOccurred()) + + // then + resp := model.ResourceList(rs).ToDeltaDiscoveryResponse() + actual, err := util_proto.ToYAML(resp) + Expect(err).ToNot(HaveOccurred()) + + expected, err := ioutil.ReadFile(filepath.Join("testdata", "outbound-proxy", "cluster-dots.envoy.golden.yaml")) + Expect(err).ToNot(HaveOccurred()) + Expect(actual).To(MatchYAML(expected)) + }) + Describe("fail when a user-defined configuration (Dataplane, TrafficRoute, etc) is not valid", func() { type testCase struct { diff --git a/pkg/xds/generator/prometheus_endpoint_generator_test.go b/pkg/xds/generator/prometheus_endpoint_generator_test.go index 62035d79940a..69f8ef4b62d2 100644 --- a/pkg/xds/generator/prometheus_endpoint_generator_test.go +++ b/pkg/xds/generator/prometheus_endpoint_generator_test.go @@ -251,7 +251,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { route: cluster: kuma:envoy:admin prefixRewrite: /stats/prometheus - statPrefix: kuma:metrics:prometheus + statPrefix: kuma_metrics_prometheus name: kuma:metrics:prometheus `, }), @@ -335,7 +335,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { route: cluster: kuma:envoy:admin prefixRewrite: /stats/prometheus - statPrefix: kuma:metrics:prometheus + statPrefix: kuma_metrics_prometheus name: kuma:metrics:prometheus `, }), diff --git a/pkg/xds/generator/proxy_template.go b/pkg/xds/generator/proxy_template.go index c03a1b847553..9696bcb993f7 100644 --- a/pkg/xds/generator/proxy_template.go +++ b/pkg/xds/generator/proxy_template.go @@ -12,6 +12,7 @@ import ( "github.com/Kong/kuma/pkg/core/validators" model "github.com/Kong/kuma/pkg/core/xds" util_envoy "github.com/Kong/kuma/pkg/util/envoy" + util_xds "github.com/Kong/kuma/pkg/util/xds" xds_context "github.com/Kong/kuma/pkg/xds/context" "github.com/Kong/kuma/pkg/xds/envoy" ) @@ -229,8 +230,10 @@ func destinationClusterName(service string, selector map[string]string) string { pairs = append(pairs, fmt.Sprintf("%s=%s", key, value)) } if len(pairs) == 0 { - return service + // we need to replace . in cluster name to _ to have consistent metrics across all the environments. + // Otherwise on K8S we would have backend.kuma-system.svc_1234 which breaks prometheus metrics converter. + return util_xds.SanitizeMetric(service) } sort.Strings(pairs) - return fmt.Sprintf("%s{%s}", service, strings.Join(pairs, ",")) + return util_xds.SanitizeMetric(fmt.Sprintf("%s{%s}", service, strings.Join(pairs, ","))) } diff --git a/pkg/xds/generator/testdata/inbound-proxy/3-envoy-config.golden.yaml b/pkg/xds/generator/testdata/inbound-proxy/3-envoy-config.golden.yaml index df1db3c8fb9e..42de8f0a58a3 100644 --- a/pkg/xds/generator/testdata/inbound-proxy/3-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/inbound-proxy/3-envoy-config.golden.yaml @@ -36,12 +36,12 @@ resources: - authenticated: principalName: exact: spiffe://default/web1 - statPrefix: inbound:192.168.0.1:80. + statPrefix: inbound_192_168_0_1_80. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8080 - statPrefix: localhost:8080 + statPrefix: localhost_8080 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: diff --git a/pkg/xds/generator/testdata/inbound-proxy/4-envoy-config.golden.yaml b/pkg/xds/generator/testdata/inbound-proxy/4-envoy-config.golden.yaml index c07f4ee26611..5250ec43e1d8 100644 --- a/pkg/xds/generator/testdata/inbound-proxy/4-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/inbound-proxy/4-envoy-config.golden.yaml @@ -38,12 +38,12 @@ resources: - authenticated: principalName: exact: spiffe://default/web1 - statPrefix: inbound:192.168.0.1:80. + statPrefix: inbound_192_168_0_1_80. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8080 - statPrefix: localhost:8080 + statPrefix: localhost_8080 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: diff --git a/pkg/xds/generator/testdata/inbound-proxy/5-envoy-config.golden.yaml b/pkg/xds/generator/testdata/inbound-proxy/5-envoy-config.golden.yaml index 21c1b4126a8d..9b5cb5c2609b 100644 --- a/pkg/xds/generator/testdata/inbound-proxy/5-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/inbound-proxy/5-envoy-config.golden.yaml @@ -35,12 +35,12 @@ resources: - authenticated: principalName: exact: spiffe://default/web1 - statPrefix: inbound:192.168.0.1:80. + statPrefix: inbound_192_168_0_1_80. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8080 - statPrefix: localhost:8080 + statPrefix: localhost_8080 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: @@ -99,12 +99,12 @@ resources: typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.rbac.v2.RBAC rules: {} - statPrefix: inbound:192.168.0.1:443. + statPrefix: inbound_192_168_0_1_443. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8443 - statPrefix: localhost:8443 + statPrefix: localhost_8443 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: diff --git a/pkg/xds/generator/testdata/inbound-proxy/6-envoy-config.golden.yaml b/pkg/xds/generator/testdata/inbound-proxy/6-envoy-config.golden.yaml index 662861344d99..5308a196f7d9 100644 --- a/pkg/xds/generator/testdata/inbound-proxy/6-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/inbound-proxy/6-envoy-config.golden.yaml @@ -37,12 +37,12 @@ resources: - authenticated: principalName: exact: spiffe://default/web1 - statPrefix: inbound:192.168.0.1:80. + statPrefix: inbound_192_168_0_1_80. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8080 - statPrefix: localhost:8080 + statPrefix: localhost_8080 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: @@ -103,12 +103,12 @@ resources: typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.rbac.v2.RBAC rules: {} - statPrefix: inbound:192.168.0.1:443. + statPrefix: inbound_192_168_0_1_443. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8443 - statPrefix: localhost:8443 + statPrefix: localhost_8443 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: diff --git a/pkg/xds/generator/testdata/inbound-proxy/7-envoy-config.golden.yaml b/pkg/xds/generator/testdata/inbound-proxy/7-envoy-config.golden.yaml index 2b64e2ae6845..4ad71fb1532b 100644 --- a/pkg/xds/generator/testdata/inbound-proxy/7-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/inbound-proxy/7-envoy-config.golden.yaml @@ -35,12 +35,12 @@ resources: - authenticated: principalName: exact: spiffe://default/web1 - statPrefix: inbound:192.168.0.1:80. + statPrefix: inbound_192_168_0_1_80. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8080 - statPrefix: localhost:8080 + statPrefix: localhost_8080 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: @@ -99,12 +99,12 @@ resources: typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.rbac.v2.RBAC rules: {} - statPrefix: inbound:192.168.0.1:443. + statPrefix: inbound_192_168_0_1_443. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8443 - statPrefix: localhost:8443 + statPrefix: localhost_8443 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: @@ -148,12 +148,12 @@ resources: typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.rbac.v2.RBAC rules: {} - statPrefix: inbound:192.168.0.2:80. + statPrefix: inbound_192_168_0_2_80. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8080 - statPrefix: localhost:8080 + statPrefix: localhost_8080 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: @@ -197,12 +197,12 @@ resources: typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.rbac.v2.RBAC rules: {} - statPrefix: inbound:192.168.0.2:443. + statPrefix: inbound_192_168_0_2_443. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8443 - statPrefix: localhost:8443 + statPrefix: localhost_8443 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: diff --git a/pkg/xds/generator/testdata/inbound-proxy/8-envoy-config.golden.yaml b/pkg/xds/generator/testdata/inbound-proxy/8-envoy-config.golden.yaml index 25a5bef12a51..f1efd0b8d353 100644 --- a/pkg/xds/generator/testdata/inbound-proxy/8-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/inbound-proxy/8-envoy-config.golden.yaml @@ -37,12 +37,12 @@ resources: - authenticated: principalName: exact: spiffe://default/web1 - statPrefix: inbound:192.168.0.1:80. + statPrefix: inbound_192_168_0_1_80. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8080 - statPrefix: localhost:8080 + statPrefix: localhost_8080 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: @@ -103,12 +103,12 @@ resources: typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.rbac.v2.RBAC rules: {} - statPrefix: inbound:192.168.0.1:443. + statPrefix: inbound_192_168_0_1_443. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8443 - statPrefix: localhost:8443 + statPrefix: localhost_8443 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: @@ -154,12 +154,12 @@ resources: typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.rbac.v2.RBAC rules: {} - statPrefix: inbound:192.168.0.2:80. + statPrefix: inbound_192_168_0_2_80. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8080 - statPrefix: localhost:8080 + statPrefix: localhost_8080 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: @@ -205,12 +205,12 @@ resources: typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.rbac.v2.RBAC rules: {} - statPrefix: inbound:192.168.0.2:443. + statPrefix: inbound_192_168_0_2_443. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8443 - statPrefix: localhost:8443 + statPrefix: localhost_8443 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: diff --git a/pkg/xds/generator/testdata/outbound-proxy/cluster-dots.envoy.golden.yaml b/pkg/xds/generator/testdata/outbound-proxy/cluster-dots.envoy.golden.yaml new file mode 100644 index 000000000000..a509a3400ec1 --- /dev/null +++ b/pkg/xds/generator/testdata/outbound-proxy/cluster-dots.envoy.golden.yaml @@ -0,0 +1,61 @@ +resources: + - name: backend_kuma-system + resource: + '@type': type.googleapis.com/envoy.api.v2.Cluster + connectTimeout: 5s + edsClusterConfig: + edsConfig: + ads: {} + name: backend_kuma-system + type: EDS + - name: backend_kuma-system + resource: + '@type': type.googleapis.com/envoy.api.v2.ClusterLoadAssignment + clusterName: backend_kuma-system + endpoints: + - {} + - name: outbound:127.0.0.1:18080 + resource: + '@type': type.googleapis.com/envoy.api.v2.Listener + address: + socketAddress: + address: 127.0.0.1 + portValue: 18080 + filterChains: + - filters: + - name: envoy.tcp_proxy + typedConfig: + '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy + cluster: backend_kuma-system + statPrefix: backend_kuma-system + name: outbound:127.0.0.1:18080 + - name: db{version=3_2_0} + resource: + '@type': type.googleapis.com/envoy.api.v2.Cluster + connectTimeout: 5s + edsClusterConfig: + edsConfig: + ads: {} + name: db{version=3_2_0} + type: EDS + - name: db{version=3_2_0} + resource: + '@type': type.googleapis.com/envoy.api.v2.ClusterLoadAssignment + clusterName: db{version=3_2_0} + endpoints: + - {} + - name: outbound:127.0.0.1:54321 + resource: + '@type': type.googleapis.com/envoy.api.v2.Listener + address: + socketAddress: + address: 127.0.0.1 + portValue: 54321 + filterChains: + - filters: + - name: envoy.tcp_proxy + typedConfig: + '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy + cluster: db{version=3_2_0} + statPrefix: db_kuma-system + name: outbound:127.0.0.1:54321 \ No newline at end of file diff --git a/pkg/xds/generator/testdata/profile-source/1-envoy-config.golden.yaml b/pkg/xds/generator/testdata/profile-source/1-envoy-config.golden.yaml index 104551d3f8c8..5387f85e4110 100644 --- a/pkg/xds/generator/testdata/profile-source/1-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/profile-source/1-envoy-config.golden.yaml @@ -27,12 +27,12 @@ resources: typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.rbac.v2.RBAC rules: {} - statPrefix: inbound:192.168.0.1:80. + statPrefix: inbound_192_168_0_1_80. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8080 - statPrefix: localhost:8080 + statPrefix: localhost_8080 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: diff --git a/pkg/xds/generator/testdata/profile-source/2-envoy-config.golden.yaml b/pkg/xds/generator/testdata/profile-source/2-envoy-config.golden.yaml index 748e8017a9bc..7e1f1ab89e87 100644 --- a/pkg/xds/generator/testdata/profile-source/2-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/profile-source/2-envoy-config.golden.yaml @@ -54,12 +54,12 @@ resources: typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.rbac.v2.RBAC rules: {} - statPrefix: inbound:192.168.0.1:80. + statPrefix: inbound_192_168_0_1_80. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8080 - statPrefix: localhost:8080 + statPrefix: localhost_8080 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: diff --git a/pkg/xds/generator/testdata/profile-source/3-envoy-config.golden.yaml b/pkg/xds/generator/testdata/profile-source/3-envoy-config.golden.yaml index cd45a17fbbd3..7585ac596c5e 100644 --- a/pkg/xds/generator/testdata/profile-source/3-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/profile-source/3-envoy-config.golden.yaml @@ -39,7 +39,7 @@ resources: route: cluster: kuma:envoy:admin prefixRewrite: /stats/prometheus - statPrefix: kuma:metrics:prometheus + statPrefix: kuma_metrics_prometheus name: kuma:metrics:prometheus - name: localhost:8080 resource: @@ -69,12 +69,12 @@ resources: typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.rbac.v2.RBAC rules: {} - statPrefix: inbound:192.168.0.1:80. + statPrefix: inbound_192_168_0_1_80. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8080 - statPrefix: localhost:8080 + statPrefix: localhost_8080 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: diff --git a/pkg/xds/generator/testdata/profile-source/4-envoy-config.golden.yaml b/pkg/xds/generator/testdata/profile-source/4-envoy-config.golden.yaml index db68f367c23e..44bf1ad15bc7 100644 --- a/pkg/xds/generator/testdata/profile-source/4-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/profile-source/4-envoy-config.golden.yaml @@ -41,7 +41,7 @@ resources: route: cluster: kuma:envoy:admin prefixRewrite: /stats/prometheus - statPrefix: kuma:metrics:prometheus + statPrefix: kuma_metrics_prometheus name: kuma:metrics:prometheus - name: catch_all resource: @@ -98,12 +98,12 @@ resources: typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.rbac.v2.RBAC rules: {} - statPrefix: inbound:192.168.0.1:80. + statPrefix: inbound_192_168_0_1_80. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8080 - statPrefix: localhost:8080 + statPrefix: localhost_8080 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: diff --git a/pkg/xds/generator/testdata/template-proxy/1-envoy-config.golden.yaml b/pkg/xds/generator/testdata/template-proxy/1-envoy-config.golden.yaml index 9fcc526d9890..6ea297d12042 100644 --- a/pkg/xds/generator/testdata/template-proxy/1-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/template-proxy/1-envoy-config.golden.yaml @@ -54,12 +54,12 @@ resources: typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.rbac.v2.RBAC rules: {} - statPrefix: inbound:192.168.0.1:80. + statPrefix: inbound_192_168_0_1_80. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8080 - statPrefix: localhost:8080 + statPrefix: localhost_8080 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: diff --git a/pkg/xds/server/testdata/envoy-config.golden.yaml b/pkg/xds/server/testdata/envoy-config.golden.yaml index ce744bc8a6ef..6683383517aa 100644 --- a/pkg/xds/server/testdata/envoy-config.golden.yaml +++ b/pkg/xds/server/testdata/envoy-config.golden.yaml @@ -67,12 +67,12 @@ resources: typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.rbac.v2.RBAC rules: {} - statPrefix: inbound:192.168.0.1:443. + statPrefix: inbound_192_168_0_1_443. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8443 - statPrefix: localhost:8443 + statPrefix: localhost_8443 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: @@ -126,12 +126,12 @@ resources: - authenticated: principalName: exact: spiffe://default/web1 - statPrefix: inbound:192.168.0.1:80. + statPrefix: inbound_192_168_0_1_80. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8080 - statPrefix: localhost:8080 + statPrefix: localhost_8080 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: @@ -177,12 +177,12 @@ resources: typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.rbac.v2.RBAC rules: {} - statPrefix: inbound:192.168.0.2:443. + statPrefix: inbound_192_168_0_2_443. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8443 - statPrefix: localhost:8443 + statPrefix: localhost_8443 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: @@ -228,12 +228,12 @@ resources: typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.rbac.v2.RBAC rules: {} - statPrefix: inbound:192.168.0.2:80. + statPrefix: inbound_192_168_0_2_80. - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy cluster: localhost:8080 - statPrefix: localhost:8080 + statPrefix: localhost_8080 tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: From dcfd3bca56e0fc10c3ed0480a20f9ee070736242 Mon Sep 17 00:00:00 2001 From: Jakub Dyszkiewicz Date: Tue, 28 Jan 2020 15:15:24 +0100 Subject: [PATCH 2/4] feat(kuma-cp) use alt_stats_name for clusters --- Makefile.e2e.mk | 14 ++++++------ pkg/util/xds/metric_sanitizer.go | 2 +- pkg/util/xds/metric_sanitizer_test.go | 4 ++-- pkg/xds/envoy/envoy.go | 18 ++++++++++----- pkg/xds/envoy/envoy_test.go | 4 ++++ .../outbound_proxy_generator_test.go | 2 +- .../prometheus_endpoint_generator_test.go | 2 ++ pkg/xds/generator/proxy_template.go | 7 ++---- .../inbound-proxy/3-envoy-config.golden.yaml | 1 + .../inbound-proxy/4-envoy-config.golden.yaml | 1 + .../inbound-proxy/5-envoy-config.golden.yaml | 2 ++ .../inbound-proxy/6-envoy-config.golden.yaml | 2 ++ .../inbound-proxy/7-envoy-config.golden.yaml | 2 ++ .../inbound-proxy/8-envoy-config.golden.yaml | 2 ++ .../outbound-proxy/07.envoy.golden.yaml | 2 ++ .../outbound-proxy/08.envoy.golden.yaml | 2 ++ .../cluster-dots.envoy.golden.yaml | 22 ++++++++++--------- .../profile-source/1-envoy-config.golden.yaml | 1 + .../profile-source/2-envoy-config.golden.yaml | 1 + .../profile-source/3-envoy-config.golden.yaml | 2 ++ .../profile-source/4-envoy-config.golden.yaml | 2 ++ .../template-proxy/1-envoy-config.golden.yaml | 2 ++ .../1-proxy-template.input.yaml | 1 + .../server/testdata/envoy-config.golden.yaml | 2 ++ 24 files changed, 69 insertions(+), 31 deletions(-) diff --git a/Makefile.e2e.mk b/Makefile.e2e.mk index cc991b21152a..dd63e489b444 100644 --- a/Makefile.e2e.mk +++ b/Makefile.e2e.mk @@ -332,7 +332,7 @@ wait/example/minikube: ## Minikube: Wait for demo setup to get ready wait/example/minikube/mtls: ## Minikube: Wait until incoming Listener and outgoing Cluster have been configured for mTLS $(call kubectl_exec,kuma-demo,demo-client,demo-client) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_mtls_listeners_count,inbound,3000) ) -eq 1 ]]; then echo "listener has been configured for mTLS "; exit 0; fi; sleep 1; done; echo -e "\nError: listener has not been configured for mTLS" ; exit 1' - $(call kubectl_exec,kuma-demo,demo-client,demo-client) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_mtls_clusters_count,demo-app_kuma-demo_svc_8000) ) -eq 1 ]]; then echo "cluster has been configured for mTLS "; exit 0; fi; sleep 1; done; echo -e "\nError: cluster has not been configured for mTLS" ; exit 1' + $(call kubectl_exec,kuma-demo,demo-client,demo-client) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_mtls_clusters_count,demo-app.kuma-demo.svc:8000) ) -eq 1 ]]; then echo "cluster has been configured for mTLS "; exit 0; fi; sleep 1; done; echo -e "\nError: cluster has not been configured for mTLS" ; exit 1' curl/example/minikube: ## Minikube: Make sample requests to demo setup $(call kubectl_exec,kuma-demo,demo-client,demo-client) $(call curl_example_client) @@ -404,7 +404,7 @@ wait/traffic-routing/minikube/mtls: ## Minikube: Wait until incoming Listener an @echo "Waiting until incoming Listener and outgoing Cluster have been configured for mTLS ..." @echo $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_mtls_listeners_count,inbound,6060) ) -eq 1 ]]; then echo "listener has been configured for mTLS "; exit 0; fi; sleep 1; done; echo -e "\nError: listener has not been configured for mTLS" ; exit 1' - $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_mtls_clusters_count,kuma-example-backend_kuma-example_svc_7070) ) -eq 1 ]]; then echo "cluster has been configured for mTLS "; exit 0; fi; sleep 1; done; echo -e "\nError: cluster has not been configured for mTLS" ; exit 1' + $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_mtls_clusters_count,kuma-example-backend.kuma-example.svc:7070) ) -eq 1 ]]; then echo "cluster has been configured for mTLS "; exit 0; fi; sleep 1; done; echo -e "\nError: cluster has not been configured for mTLS" ; exit 1' apply/traffic-routing/minikube/no-mtls: ## Minikube: disable mTLS @echo @@ -417,7 +417,7 @@ wait/traffic-routing/minikube/no-mtls: ## Minikube: Wait until mTLS has been dis @echo "Waiting until mTLS has been disabled on incoming Listener and outgoing Cluster ..." @echo $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_mtls_listeners_count,inbound,6060) ) -eq 0 ]]; then echo "listener is no longer configured for mTLS "; exit 0; fi; sleep 1; done; echo -e "\nError: listener is still configured for mTLS" ; exit 1' - $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_mtls_clusters_count,kuma-example-backend_kuma-example_svc_7070) ) -eq 0 ]]; then echo "cluster is no longer configured for mTLS "; exit 0; fi; sleep 1; done; echo -e "\nError: cluster is still configured for mTLS" ; exit 1' + $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_mtls_clusters_count,kuma-example-backend.kuma-example.svc:7070) ) -eq 0 ]]; then echo "cluster is no longer configured for mTLS "; exit 0; fi; sleep 1; done; echo -e "\nError: cluster is still configured for mTLS" ; exit 1' wait/traffic-routing/minikube: ## Minikube: Wait for example setup for TrafficRoute to get ready @echo @@ -442,8 +442,8 @@ wait/traffic-routing/minikube/web-to-backend-route: ## Minikube: Wait until cust @echo @echo "Waiting until custom 'web-to-backend' TrafficRoute is applied ..." @echo - $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_routing_listeners_count,outbound,7070,kuma-example-backend_kuma-example_svc_7070{version=v2}) ) -eq 1 ]]; then echo "listener is now configured for subset routing "; exit 0; fi; sleep 1; done; echo -e "\nError: listener has not been configured for subset routing" ; exit 1' - $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_routing_clusters_count,kuma-example-backend_kuma-example_svc_7070{version=v2}) ) -eq 1 ]]; then echo "cluster is now configured for subset routing "; exit 0; fi; sleep 1; done; echo -e "\nError: cluster has not been configured for subset routing" ; exit 1' + $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_routing_listeners_count,outbound,7070,kuma-example-backend.kuma-example.svc:7070{version=v2}) ) -eq 1 ]]; then echo "listener is now configured for subset routing "; exit 0; fi; sleep 1; done; echo -e "\nError: listener has not been configured for subset routing" ; exit 1' + $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_routing_clusters_count,kuma-example-backend.kuma-example.svc:7070{version=v2}) ) -eq 1 ]]; then echo "cluster is now configured for subset routing "; exit 0; fi; sleep 1; done; echo -e "\nError: cluster has not been configured for subset routing" ; exit 1' verify/traffic-routing/minikube/web-to-backend-route: ## Minikube: Make sample requests to example setup for TrafficRoute @echo @@ -462,8 +462,8 @@ wait/traffic-routing/minikube/no-web-to-backend-route: ## Minikube: Wait until c @echo @echo "Waiting until custom 'web-to-backend' TrafficRoute is removed ..." @echo - $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_routing_listeners_count,outbound,7070,kuma-example-backend_kuma-example_svc_7070{version=v2}) ) -eq 0 ]]; then echo "listener is no longer configured for subset routing "; exit 0; fi; sleep 1; done; echo -e "\nError: listener is still configured for subset routing" ; exit 1' - $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_routing_clusters_count,kuma-example-backend_kuma-example_svc_7070{version=v2}) ) -eq 0 ]]; then echo "cluster is no longer configured for subset routing "; exit 0; fi; sleep 1; done; echo -e "\nError: cluster is still configured for subset routing" ; exit 1' + $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_routing_listeners_count,outbound,7070,kuma-example-backend.kuma-example.svc:7070{version=v2}) ) -eq 0 ]]; then echo "listener is no longer configured for subset routing "; exit 0; fi; sleep 1; done; echo -e "\nError: listener is still configured for subset routing" ; exit 1' + $(call kubectl_exec,kuma-example,kuma-example-web,kuma-example-web) sh -c 'for i in `seq 1 10`; do echo -n "try #$$i: " ; if [[ $$( $(call envoy_active_routing_clusters_count,kuma-example-backend.kuma-example.svc:7070{version=v2}) ) -eq 0 ]]; then echo "cluster is no longer configured for subset routing "; exit 0; fi; sleep 1; done; echo -e "\nError: cluster is still configured for subset routing" ; exit 1' undeploy/traffic-routing/minikube: ## Minikube: Undeploy example setup for TrafficRoute @echo diff --git a/pkg/util/xds/metric_sanitizer.go b/pkg/util/xds/metric_sanitizer.go index 1d6aa773a941..5cccb427698f 100644 --- a/pkg/util/xds/metric_sanitizer.go +++ b/pkg/util/xds/metric_sanitizer.go @@ -7,7 +7,7 @@ import ( var ( whitespaces = regexp.MustCompile(`\s+`) - illegalChars = regexp.MustCompile(`[^a-zA-Z_\-0-9{}=]`) + illegalChars = regexp.MustCompile(`[^a-zA-Z_\-0-9]`) ) func SanitizeMetric(metric string) string { diff --git a/pkg/util/xds/metric_sanitizer_test.go b/pkg/util/xds/metric_sanitizer_test.go index f7771e482f57..94af05ab9df6 100644 --- a/pkg/util/xds/metric_sanitizer_test.go +++ b/pkg/util/xds/metric_sanitizer_test.go @@ -9,12 +9,12 @@ import ( var _ = Describe("Metric sanitizer", func() { It("should sanitize metrics", func() { // given - metric := "some metric with chars :/_-0123{}" + metric := "some metric with chars :/_-0123{version=3.0}" // when sanitized := xds.SanitizeMetric(metric) // then - Expect(sanitized).To(Equal("some_metric_with_chars____-0123{}")) + Expect(sanitized).To(Equal("some_metric_with_chars____-0123_version_3_0_")) }) }) diff --git a/pkg/xds/envoy/envoy.go b/pkg/xds/envoy/envoy.go index 9fa528d9806d..eee5b435a571 100644 --- a/pkg/xds/envoy/envoy.go +++ b/pkg/xds/envoy/envoy.go @@ -116,16 +116,16 @@ func CreateClusterLoadAssignment(clusterName string, endpoints []core_xds.Endpoi } func CreateLocalCluster(clusterName string, address string, port uint32) *v2.Cluster { - return &v2.Cluster{ + return clusterWithAltStatName(&v2.Cluster{ Name: clusterName, ConnectTimeout: ptypes.DurationProto(defaultConnectTimeout), ClusterDiscoveryType: &v2.Cluster_Type{Type: v2.Cluster_STATIC}, LoadAssignment: CreateStaticEndpoint(clusterName, address, port), - } + }) } func CreateEdsCluster(ctx xds_context.Context, clusterName string, metadata *core_xds.DataplaneMetadata) *v2.Cluster { - return &v2.Cluster{ + return clusterWithAltStatName(&v2.Cluster{ Name: clusterName, ConnectTimeout: ptypes.DurationProto(defaultConnectTimeout), ClusterDiscoveryType: &v2.Cluster_Type{Type: v2.Cluster_EDS}, @@ -137,7 +137,15 @@ func CreateEdsCluster(ctx xds_context.Context, clusterName string, metadata *cor }, }, TlsContext: CreateUpstreamTlsContext(ctx, metadata), + }) +} + +func clusterWithAltStatName(cluster *v2.Cluster) *v2.Cluster { + sanitizedName := util_xds.SanitizeMetric(cluster.Name) + if sanitizedName != cluster.Name { + cluster.AltStatName = sanitizedName } + return cluster } func ClusterWithHealthChecks(cluster *v2.Cluster, healthCheck *mesh_core.HealthCheckResource) *v2.Cluster { @@ -167,12 +175,12 @@ func ClusterWithHealthChecks(cluster *v2.Cluster, healthCheck *mesh_core.HealthC } func CreatePassThroughCluster(clusterName string) *v2.Cluster { - return &v2.Cluster{ + return clusterWithAltStatName(&v2.Cluster{ Name: clusterName, ConnectTimeout: ptypes.DurationProto(defaultConnectTimeout), ClusterDiscoveryType: &v2.Cluster_Type{Type: v2.Cluster_ORIGINAL_DST}, LbPolicy: v2.Cluster_ORIGINAL_DST_LB, - } + }) } func CreateOutboundListener(ctx xds_context.Context, listenerName string, address string, port uint32, statsName string, clusters []ClusterInfo, virtual bool, sourceService string, destinationService string, backend *v1alpha1.LoggingBackend, proxy *core_xds.Proxy) (*v2.Listener, error) { diff --git a/pkg/xds/envoy/envoy_test.go b/pkg/xds/envoy/envoy_test.go index 04a5c576f7bd..a559b763e60f 100644 --- a/pkg/xds/envoy/envoy_test.go +++ b/pkg/xds/envoy/envoy_test.go @@ -48,6 +48,7 @@ var _ = Describe("Envoy", func() { // given expected := ` name: localhost:8080 + altStatName: localhost_8080 type: STATIC connectTimeout: 5s loadAssignment: @@ -178,6 +179,7 @@ var _ = Describe("Envoy", func() { edsConfig: ads: {} name: 192.168.0.1:8080 + altStatName: "192_168_0_1_8080" type: EDS `, }), @@ -204,6 +206,7 @@ var _ = Describe("Envoy", func() { edsConfig: ads: {} name: 192.168.0.1:8080 + altStatName: "192_168_0_1_8080" tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: @@ -260,6 +263,7 @@ var _ = Describe("Envoy", func() { edsConfig: ads: {} name: 192.168.0.1:8080 + altStatName: "192_168_0_1_8080" tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: diff --git a/pkg/xds/generator/outbound_proxy_generator_test.go b/pkg/xds/generator/outbound_proxy_generator_test.go index 3d26d44193fd..2f4d946c5a9d 100644 --- a/pkg/xds/generator/outbound_proxy_generator_test.go +++ b/pkg/xds/generator/outbound_proxy_generator_test.go @@ -213,7 +213,7 @@ var _ = Describe("OutboundProxyGenerator", func() { }), ) - It("Sanitize cluster names", func() { + It("Add sanitized alternative cluster name for stats", func() { // setup gen := &generator.OutboundProxyGenerator{} dp := ` diff --git a/pkg/xds/generator/prometheus_endpoint_generator_test.go b/pkg/xds/generator/prometheus_endpoint_generator_test.go index 69f8ef4b62d2..94c0fc51e9c6 100644 --- a/pkg/xds/generator/prometheus_endpoint_generator_test.go +++ b/pkg/xds/generator/prometheus_endpoint_generator_test.go @@ -225,6 +225,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { address: 127.0.0.1 portValue: 9902 name: kuma:envoy:admin + altStatName: kuma_envoy_admin type: STATIC - name: kuma:metrics:prometheus resource: @@ -309,6 +310,7 @@ var _ = Describe("PrometheusEndpointGenerator", func() { address: 127.0.0.1 portValue: 9902 name: kuma:envoy:admin + altStatName: kuma_envoy_admin type: STATIC - name: kuma:metrics:prometheus resource: diff --git a/pkg/xds/generator/proxy_template.go b/pkg/xds/generator/proxy_template.go index 9696bcb993f7..c03a1b847553 100644 --- a/pkg/xds/generator/proxy_template.go +++ b/pkg/xds/generator/proxy_template.go @@ -12,7 +12,6 @@ import ( "github.com/Kong/kuma/pkg/core/validators" model "github.com/Kong/kuma/pkg/core/xds" util_envoy "github.com/Kong/kuma/pkg/util/envoy" - util_xds "github.com/Kong/kuma/pkg/util/xds" xds_context "github.com/Kong/kuma/pkg/xds/context" "github.com/Kong/kuma/pkg/xds/envoy" ) @@ -230,10 +229,8 @@ func destinationClusterName(service string, selector map[string]string) string { pairs = append(pairs, fmt.Sprintf("%s=%s", key, value)) } if len(pairs) == 0 { - // we need to replace . in cluster name to _ to have consistent metrics across all the environments. - // Otherwise on K8S we would have backend.kuma-system.svc_1234 which breaks prometheus metrics converter. - return util_xds.SanitizeMetric(service) + return service } sort.Strings(pairs) - return util_xds.SanitizeMetric(fmt.Sprintf("%s{%s}", service, strings.Join(pairs, ","))) + return fmt.Sprintf("%s{%s}", service, strings.Join(pairs, ",")) } diff --git a/pkg/xds/generator/testdata/inbound-proxy/3-envoy-config.golden.yaml b/pkg/xds/generator/testdata/inbound-proxy/3-envoy-config.golden.yaml index 42de8f0a58a3..3e69458b9f35 100644 --- a/pkg/xds/generator/testdata/inbound-proxy/3-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/inbound-proxy/3-envoy-config.golden.yaml @@ -13,6 +13,7 @@ resources: address: 127.0.0.1 portValue: 8080 name: localhost:8080 + altStatName: localhost_8080 type: STATIC - name: inbound:192.168.0.1:80 resource: diff --git a/pkg/xds/generator/testdata/inbound-proxy/4-envoy-config.golden.yaml b/pkg/xds/generator/testdata/inbound-proxy/4-envoy-config.golden.yaml index 5250ec43e1d8..cbbb60cafd6d 100644 --- a/pkg/xds/generator/testdata/inbound-proxy/4-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/inbound-proxy/4-envoy-config.golden.yaml @@ -13,6 +13,7 @@ resources: address: 127.0.0.1 portValue: 8080 name: localhost:8080 + altStatName: localhost_8080 type: STATIC - name: inbound:192.168.0.1:80 resource: diff --git a/pkg/xds/generator/testdata/inbound-proxy/5-envoy-config.golden.yaml b/pkg/xds/generator/testdata/inbound-proxy/5-envoy-config.golden.yaml index 9b5cb5c2609b..a05538df4b41 100644 --- a/pkg/xds/generator/testdata/inbound-proxy/5-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/inbound-proxy/5-envoy-config.golden.yaml @@ -13,6 +13,7 @@ resources: address: 127.0.0.1 portValue: 8080 name: localhost:8080 + altStatName: localhost_8080 type: STATIC - name: inbound:192.168.0.1:80 resource: @@ -85,6 +86,7 @@ resources: address: 127.0.0.1 portValue: 8443 name: localhost:8443 + altStatName: localhost_8443 type: STATIC - name: inbound:192.168.0.1:443 resource: diff --git a/pkg/xds/generator/testdata/inbound-proxy/6-envoy-config.golden.yaml b/pkg/xds/generator/testdata/inbound-proxy/6-envoy-config.golden.yaml index 5308a196f7d9..ade16d46b415 100644 --- a/pkg/xds/generator/testdata/inbound-proxy/6-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/inbound-proxy/6-envoy-config.golden.yaml @@ -13,6 +13,7 @@ resources: address: 127.0.0.1 portValue: 8080 name: localhost:8080 + altStatName: localhost_8080 type: STATIC - name: inbound:192.168.0.1:80 resource: @@ -87,6 +88,7 @@ resources: address: 127.0.0.1 portValue: 8443 name: localhost:8443 + altStatName: localhost_8443 type: STATIC - name: inbound:192.168.0.1:443 resource: diff --git a/pkg/xds/generator/testdata/inbound-proxy/7-envoy-config.golden.yaml b/pkg/xds/generator/testdata/inbound-proxy/7-envoy-config.golden.yaml index 4ad71fb1532b..a21e6791373f 100644 --- a/pkg/xds/generator/testdata/inbound-proxy/7-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/inbound-proxy/7-envoy-config.golden.yaml @@ -13,6 +13,7 @@ resources: address: 127.0.0.1 portValue: 8080 name: localhost:8080 + altStatName: localhost_8080 type: STATIC - name: inbound:192.168.0.1:80 resource: @@ -85,6 +86,7 @@ resources: address: 127.0.0.1 portValue: 8443 name: localhost:8443 + altStatName: localhost_8443 type: STATIC - name: inbound:192.168.0.1:443 resource: diff --git a/pkg/xds/generator/testdata/inbound-proxy/8-envoy-config.golden.yaml b/pkg/xds/generator/testdata/inbound-proxy/8-envoy-config.golden.yaml index f1efd0b8d353..2d574d417aa9 100644 --- a/pkg/xds/generator/testdata/inbound-proxy/8-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/inbound-proxy/8-envoy-config.golden.yaml @@ -13,6 +13,7 @@ resources: address: 127.0.0.1 portValue: 8080 name: localhost:8080 + altStatName: localhost_8080 type: STATIC - name: inbound:192.168.0.1:80 resource: @@ -87,6 +88,7 @@ resources: address: 127.0.0.1 portValue: 8443 name: localhost:8443 + altStatName: localhost_8443 type: STATIC - name: inbound:192.168.0.1:443 resource: diff --git a/pkg/xds/generator/testdata/outbound-proxy/07.envoy.golden.yaml b/pkg/xds/generator/testdata/outbound-proxy/07.envoy.golden.yaml index 87674c82b137..317f6b5f2a6a 100644 --- a/pkg/xds/generator/testdata/outbound-proxy/07.envoy.golden.yaml +++ b/pkg/xds/generator/testdata/outbound-proxy/07.envoy.golden.yaml @@ -47,6 +47,7 @@ resources: edsConfig: ads: {} name: db{role=master} + altStatName: db_role_master_ type: EDS - name: db{role=master} resource: @@ -72,6 +73,7 @@ resources: edsConfig: ads: {} name: db{role=replica} + altStatName: db_role_replica_ type: EDS - name: db{role=replica} resource: diff --git a/pkg/xds/generator/testdata/outbound-proxy/08.envoy.golden.yaml b/pkg/xds/generator/testdata/outbound-proxy/08.envoy.golden.yaml index 6edcbee8fa52..8783fea873fa 100644 --- a/pkg/xds/generator/testdata/outbound-proxy/08.envoy.golden.yaml +++ b/pkg/xds/generator/testdata/outbound-proxy/08.envoy.golden.yaml @@ -77,6 +77,7 @@ resources: edsConfig: ads: {} name: db{role=master} + altStatName: db_role_master_ tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: @@ -130,6 +131,7 @@ resources: edsConfig: ads: {} name: db{role=replica} + altStatName: db_role_replica_ tlsContext: commonTlsContext: tlsCertificateSdsSecretConfigs: diff --git a/pkg/xds/generator/testdata/outbound-proxy/cluster-dots.envoy.golden.yaml b/pkg/xds/generator/testdata/outbound-proxy/cluster-dots.envoy.golden.yaml index a509a3400ec1..ebe8dfcbcd1e 100644 --- a/pkg/xds/generator/testdata/outbound-proxy/cluster-dots.envoy.golden.yaml +++ b/pkg/xds/generator/testdata/outbound-proxy/cluster-dots.envoy.golden.yaml @@ -1,17 +1,18 @@ resources: - - name: backend_kuma-system + - name: backend.kuma-system resource: '@type': type.googleapis.com/envoy.api.v2.Cluster connectTimeout: 5s edsClusterConfig: edsConfig: ads: {} - name: backend_kuma-system + name: backend.kuma-system + altStatName: backend_kuma-system type: EDS - - name: backend_kuma-system + - name: backend.kuma-system resource: '@type': type.googleapis.com/envoy.api.v2.ClusterLoadAssignment - clusterName: backend_kuma-system + clusterName: backend.kuma-system endpoints: - {} - name: outbound:127.0.0.1:18080 @@ -26,22 +27,23 @@ resources: - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy - cluster: backend_kuma-system + cluster: backend.kuma-system statPrefix: backend_kuma-system name: outbound:127.0.0.1:18080 - - name: db{version=3_2_0} + - name: db{version=3.2.0} resource: '@type': type.googleapis.com/envoy.api.v2.Cluster connectTimeout: 5s edsClusterConfig: edsConfig: ads: {} - name: db{version=3_2_0} + name: db{version=3.2.0} + altStatName: db_version_3_2_0_ type: EDS - - name: db{version=3_2_0} + - name: db{version=3.2.0} resource: '@type': type.googleapis.com/envoy.api.v2.ClusterLoadAssignment - clusterName: db{version=3_2_0} + clusterName: db{version=3.2.0} endpoints: - {} - name: outbound:127.0.0.1:54321 @@ -56,6 +58,6 @@ resources: - name: envoy.tcp_proxy typedConfig: '@type': type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy - cluster: db{version=3_2_0} + cluster: db{version=3.2.0} statPrefix: db_kuma-system name: outbound:127.0.0.1:54321 \ No newline at end of file diff --git a/pkg/xds/generator/testdata/profile-source/1-envoy-config.golden.yaml b/pkg/xds/generator/testdata/profile-source/1-envoy-config.golden.yaml index 5387f85e4110..756aa428e2d7 100644 --- a/pkg/xds/generator/testdata/profile-source/1-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/profile-source/1-envoy-config.golden.yaml @@ -13,6 +13,7 @@ resources: address: 127.0.0.1 portValue: 8080 name: localhost:8080 + altStatName: localhost_8080 type: STATIC - name: inbound:192.168.0.1:80 resource: diff --git a/pkg/xds/generator/testdata/profile-source/2-envoy-config.golden.yaml b/pkg/xds/generator/testdata/profile-source/2-envoy-config.golden.yaml index 7e1f1ab89e87..3bbea82d31aa 100644 --- a/pkg/xds/generator/testdata/profile-source/2-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/profile-source/2-envoy-config.golden.yaml @@ -38,6 +38,7 @@ resources: address: 127.0.0.1 portValue: 8080 name: localhost:8080 + altStatName: localhost_8080 type: STATIC - name: inbound:192.168.0.1:80 resource: diff --git a/pkg/xds/generator/testdata/profile-source/3-envoy-config.golden.yaml b/pkg/xds/generator/testdata/profile-source/3-envoy-config.golden.yaml index 7585ac596c5e..139b96eb15cd 100644 --- a/pkg/xds/generator/testdata/profile-source/3-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/profile-source/3-envoy-config.golden.yaml @@ -13,6 +13,7 @@ resources: address: 127.0.0.1 portValue: 9902 name: kuma:envoy:admin + altStatName: kuma_envoy_admin type: STATIC - name: kuma:metrics:prometheus resource: @@ -55,6 +56,7 @@ resources: address: 127.0.0.1 portValue: 8080 name: localhost:8080 + altStatName: localhost_8080 type: STATIC - name: inbound:192.168.0.1:80 resource: diff --git a/pkg/xds/generator/testdata/profile-source/4-envoy-config.golden.yaml b/pkg/xds/generator/testdata/profile-source/4-envoy-config.golden.yaml index 44bf1ad15bc7..7c61ae1820b1 100644 --- a/pkg/xds/generator/testdata/profile-source/4-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/profile-source/4-envoy-config.golden.yaml @@ -13,6 +13,7 @@ resources: address: 127.0.0.1 portValue: 9902 name: kuma:envoy:admin + altStatName: kuma_envoy_admin type: STATIC - name: kuma:metrics:prometheus resource: @@ -82,6 +83,7 @@ resources: address: 127.0.0.1 portValue: 8080 name: localhost:8080 + altStatName: localhost_8080 type: STATIC - name: inbound:192.168.0.1:80 resource: diff --git a/pkg/xds/generator/testdata/template-proxy/1-envoy-config.golden.yaml b/pkg/xds/generator/testdata/template-proxy/1-envoy-config.golden.yaml index 6ea297d12042..ac16bcae979a 100644 --- a/pkg/xds/generator/testdata/template-proxy/1-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/template-proxy/1-envoy-config.golden.yaml @@ -38,6 +38,7 @@ resources: address: 127.0.0.1 portValue: 8080 name: localhost:8080 + altStatName: localhost_8080 type: STATIC - name: inbound:192.168.0.1:80 resource: @@ -104,5 +105,6 @@ resources: address: 127.0.0.1 portValue: 8443 name: localhost:8443 + altStatName: localhost_8443 type: STATIC version: raw-version diff --git a/pkg/xds/generator/testdata/template-proxy/1-proxy-template.input.yaml b/pkg/xds/generator/testdata/template-proxy/1-proxy-template.input.yaml index 59bb53e69628..20e9ac41f0c1 100644 --- a/pkg/xds/generator/testdata/template-proxy/1-proxy-template.input.yaml +++ b/pkg/xds/generator/testdata/template-proxy/1-proxy-template.input.yaml @@ -17,4 +17,5 @@ conf: address: 127.0.0.1 portValue: 8443 name: localhost:8443 + altStatName: localhost_8443 type: STATIC diff --git a/pkg/xds/server/testdata/envoy-config.golden.yaml b/pkg/xds/server/testdata/envoy-config.golden.yaml index 6683383517aa..8c91fb044adc 100644 --- a/pkg/xds/server/testdata/envoy-config.golden.yaml +++ b/pkg/xds/server/testdata/envoy-config.golden.yaml @@ -13,6 +13,7 @@ resources: address: 127.0.0.1 portValue: 8080 name: localhost:8080 + altStatName: localhost_8080 type: STATIC - name: localhost:8443 resource: @@ -28,6 +29,7 @@ resources: address: 127.0.0.1 portValue: 8443 name: localhost:8443 + altStatName: localhost_8443 type: STATIC - name: pass_through resource: From 527c6f41348166d3852b8f85a711911a87ceef96 Mon Sep 17 00:00:00 2001 From: Jakub Dyszkiewicz Date: Wed, 29 Jan 2020 11:35:35 +0100 Subject: [PATCH 3/4] feat(kuma-cp) simplify sanitizer and add comment --- pkg/util/xds/metric_sanitizer.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/util/xds/metric_sanitizer.go b/pkg/util/xds/metric_sanitizer.go index 5cccb427698f..4f3d7a58aeba 100644 --- a/pkg/util/xds/metric_sanitizer.go +++ b/pkg/util/xds/metric_sanitizer.go @@ -2,17 +2,15 @@ package xds import ( "regexp" - "strings" ) var ( - whitespaces = regexp.MustCompile(`\s+`) illegalChars = regexp.MustCompile(`[^a-zA-Z_\-0-9]`) ) +// We need to sanitize metrics in order to not break statsd and prometheus format. +// StatsD only allow [a-zA-Z_\-0-9.] characters, everything else is removed +// Extra dots breaks many regexes that converts statsd metric to prometheus one with tags func SanitizeMetric(metric string) string { - result := whitespaces.ReplaceAllString(metric, "_") - result = strings.ReplaceAll(result, "/", "_") - result = illegalChars.ReplaceAllString(result, "_") - return result + return illegalChars.ReplaceAllString(metric, "_") } From 6b80282e8189677e80c84ed9ddae39ed2268286b Mon Sep 17 00:00:00 2001 From: Jakub Dyszkiewicz Date: Wed, 29 Jan 2020 11:46:45 +0100 Subject: [PATCH 4/4] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c6ee60b242e..a9ff5e86f9ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Changes: +* feature: sanitize metrics for StatsD and Prometheus + [#562](https://github.com/Kong/kuma/pull/562) * feature: reformat some Envoy metrics available in Prometheus [#558](https://github.com/Kong/kuma/pull/558) * feature: make maximum number of open connections to Postgres configurable