Unify Gatekeeper and controller-runtime metrics into a single endpoint (

#1482)

* Unify Gatekeeper and controller-runtime metrics into a single endpoint

Fixes #1420

Signed-off-by: Oren Shomron <shomron@gmail.com>

* Suppress rest_client_request_latency_* metrics due to cardinality concerns

Signed-off-by: Oren Shomron <shomron@gmail.com>

* Resolve lint warning

Signed-off-by: Oren Shomron <shomron@gmail.com>

Co-authored-by: Rita Zhang <rita.z.zhang@gmail.com>

go.mod

@@ -3,13 +3,14 @@ module github.com/open-policy-agent/gatekeeper
 go 1.16
 
 require (
-	contrib.go.opencensus.io/exporter/prometheus v0.1.0
+	contrib.go.opencensus.io/exporter/prometheus v0.3.0
 	github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
 	github.com/davecgh/go-spew v1.1.1
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-logr/logr v0.4.0
 	github.com/go-logr/zapr v0.2.0
 	github.com/go-openapi/spec v0.20.3 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-cmp v0.5.6
 	github.com/google/uuid v1.1.2
 	github.com/mitchellh/mapstructure v1.4.1 // indirect
@@ -19,13 +20,18 @@ require (
 	github.com/open-policy-agent/frameworks/constraint v0.0.0-20210701194838-1dbe2618668d
 	github.com/open-policy-agent/opa v0.29.4
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.7.1
+	github.com/prometheus/client_golang v1.11.0
+	github.com/prometheus/common v0.30.0 // indirect
+	github.com/prometheus/procfs v0.7.1 // indirect
+	github.com/prometheus/statsd_exporter v0.21.0 // indirect
 	github.com/spf13/cobra v1.1.3
-	go.opencensus.io v0.22.4
+	go.opencensus.io v0.23.0
 	go.uber.org/zap v1.15.0
-	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
+	golang.org/x/net v0.0.0-20210525063256-abc453219eb5
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
+	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	k8s.io/api v0.20.2

main.go

@@ -156,6 +156,10 @@ func main() {
 
 	webhooks = webhook.AppendMutationWebhookIfEnabled(webhooks)
 
+	// Disable high-cardinality REST client metrics (rest_client_request_latency).
+	// Must be called before ctrl.NewManager!
+	metrics.DisableRESTClientMetrics()
+
 	mgr, err := ctrl.NewManager(config, ctrl.Options{
 		NewCache:               dynamiccache.New,
 		Scheme:                 scheme,

pkg/metrics/client_metrics.go

@@ -0,0 +1,34 @@
+/*
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"net/url"
+	"time"
+
+	clientmetrics "k8s.io/client-go/tools/metrics"
+	_ "sigs.k8s.io/controller-runtime/pkg/metrics" // Needed for init() side effect
+)
+
+// DisableRESTClientMetrics disables the rest client latency histograms configured by
+// controller-runtime in sigs.k8s.io/controller-runtime/pkg/metrics/client_go_adapter.go#registerClientMetrics.
+func DisableRESTClientMetrics() {
+	clientmetrics.RequestLatency = noopLatency{}
+}
+
+type noopLatency struct{}
+
+func (noopLatency) Observe(string, url.URL, time.Duration) {}

pkg/metrics/prometheus_exporter.go

@@ -7,6 +7,7 @@ import (
 	"contrib.go.opencensus.io/exporter/prometheus"
 	"go.opencensus.io/stats/view"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	ctlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
 var curPromSrv *http.Server
@@ -16,7 +17,11 @@ var log = logf.Log.WithName("metrics")
 const namespace = "gatekeeper"
 
 func newPrometheusExporter() (view.Exporter, error) {
-	e, err := prometheus.NewExporter(prometheus.Options{Namespace: namespace})
+	e, err := prometheus.NewExporter(prometheus.Options{
+		Namespace:  namespace,
+		Registerer: ctlmetrics.Registry,
+		Gatherer:   ctlmetrics.Registry,
+	})
 	if err != nil {
 		log.Error(err, "Failed to create the Prometheus exporter.")
 		return nil, err