metrics: configure Prometheus operator

Deploy the needed configuration to make the prometheus
operator to find and scrape the sriov-network-metrics-exporter
endpoints, including the ServiceMonitor, Role and RoleBinding.

Resources are installed only if the Prometheus operator is installed.

When useing `ServiceMonitors`, Prometheus Operator needs permissions
to read Services,Endpoint and Pods in the monitored namespace (i.e. the SRIOV
operator ns).

Make the ServiceAccount subject configurable via environment variables.

Signed-off-by: Andrea Panattoni <apanatto@redhat.com>

bindata/manifests/metrics-exporter/metrics-prometheus.yaml

@@ -0,0 +1,56 @@
+{{ if .IsPrometheusOperatorInstalled }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: sriov-network-metrics-exporter
+  namespace: {{.Namespace}}
+spec:
+  endpoints:
+    - interval: 30s
+      port: sriov-network-metrics
+      bearerTokenFile: "/var/run/secrets/kubernetes.io/serviceaccount/token"
+      scheme: "https"
+      honorLabels: true
+      tlsConfig:
+        serverName: sriov-network-metrics-exporter-service.{{.Namespace}}.svc
+        caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+        insecureSkipVerify: false
+  namespaceSelector:
+    matchNames:
+      - {{.Namespace}}
+  selector:
+    matchLabels:
+      name: sriov-network-metrics-exporter-service
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: prometheus-k8s
+  namespace: {{.Namespace}}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: prometheus-k8s
+subjects:
+- kind: ServiceAccount
+  name: {{.PrometheusOperatorServiceAccount}}
+  namespace: {{.PrometheusOperatorNamespace}}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: prometheus-k8s
+  namespace: {{.Namespace}}
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - services
+  - endpoints
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+{{ end }}

bindata/manifests/metrics-exporter/metrics-service.yaml

@@ -5,7 +5,7 @@ metadata:
   namespace: {{.Namespace}}
   annotations:
     prometheus.io/target: "true"
-    {{- if eq .ClusterType "openshift" }}
+    {{ if .IsOpenshift }}
     service.beta.openshift.io/serving-cert-secret-name: {{ .MetricsExporterSecretName }}
     {{- end }}
   labels:

controllers/sriovoperatorconfig_controller.go

@@ -236,7 +236,11 @@ func (r *SriovOperatorConfigReconciler) syncMetricsExporter(ctx context.Context,
 	data.Data["MetricsExporterSecretName"] = os.Getenv("METRICS_EXPORTER_SECRET_NAME")
 	data.Data["MetricsExporterPort"] = os.Getenv("METRICS_EXPORTER_PORT")
 	data.Data["MetricsExporterKubeRbacProxyImage"] = os.Getenv("METRICS_EXPORTER_KUBE_RBAC_PROXY_IMAGE")
-	data.Data["ClusterType"] = vars.ClusterType
+	data.Data["IsOpenshift"] = r.PlatformHelper.IsOpenshiftCluster()
+	prometheusOperatorNamespace := os.Getenv("METRICS_EXPORTER_PROMETHEUS_OPERATOR_NAMESPACE")
+	data.Data["IsPrometheusOperatorInstalled"] = prometheusOperatorNamespace != ""
+	data.Data["PrometheusOperatorServiceAccount"] = os.Getenv("METRICS_EXPORTER_PROMETHEUS_OPERATOR_SERVICE_ACCOUNT")
+	data.Data["PrometheusOperatorNamespace"] = prometheusOperatorNamespace
 	data.Data["NodeSelectorField"] = GetDefaultNodeSelector()
 	if dc.Spec.ConfigDaemonNodeSelector != nil {
 		data.Data["NodeSelectorField"] = dc.Spec.ConfigDaemonNodeSelector
@@ -248,23 +252,21 @@ func (r *SriovOperatorConfigReconciler) syncMetricsExporter(ctx context.Context,
 		return err
 	}
 
-	deployMetricsExporter, ok := dc.Spec.FeatureGates[consts.MetricsExporterFeatureGate]
-	if ok && deployMetricsExporter {
+	if r.FeatureGate.IsEnabled(consts.MetricsExporterFeatureGate) {
 		for _, obj := range objs {
 			err = r.syncK8sResource(ctx, dc, obj)
 			if err != nil {
 				logger.Error(err, "Couldn't sync metrics exporter objects")
 				return err
 			}
 		}
+
 		return nil
 	}
 
-	for _, obj := range objs {
-		err = r.deleteK8sResource(ctx, obj)
-		if err != nil {
-			return err
-		}
+	err = r.deleteK8sResources(ctx, objs)
+	if err != nil {
+		return err
 	}
 
 	return nil
@@ -360,6 +362,16 @@ func (r *SriovOperatorConfigReconciler) deleteK8sResource(ctx context.Context, i
 	return nil
 }
 
+func (r *SriovOperatorConfigReconciler) deleteK8sResources(ctx context.Context, objs []*uns.Unstructured) error {
+	for _, obj := range objs {
+		err := r.deleteK8sResource(ctx, obj)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (r *SriovOperatorConfigReconciler) syncK8sResource(ctx context.Context, cr *sriovnetworkv1.SriovOperatorConfig, in *uns.Unstructured) error {
 	switch in.GetKind() {
 	case clusterRoleResourceName, clusterRoleBindingResourceName, mutatingWebhookConfigurationCRDName, validatingWebhookConfigurationCRDName, machineConfigCRDName:

controllers/sriovoperatorconfig_controller_test.go

@@ -12,8 +12,11 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/golang/mock/gomock"
 	. "github.com/onsi/ginkgo/v2"
@@ -105,7 +108,7 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() {
 	})
 
 	Context("When is up", func() {
-		JustBeforeEach(func() {
+		BeforeEach(func() {
 			config := &sriovnetworkv1.SriovOperatorConfig{}
 			err := util.WaitForNamespacedObject(config, k8sClient, testNamespace, "default", util.RetryInterval, util.APITimeout)
 			Expect(err).NotTo(HaveOccurred())
@@ -333,41 +336,61 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() {
 			Expect(err).ToNot(HaveOccurred())
 		})
 
-		It("should deploy the metrics-exporter when the feature gate is enabled", func() {
-			config := &sriovnetworkv1.SriovOperatorConfig{}
-			Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: "default"}, config)).NotTo(HaveOccurred())
-
-			daemonSet := &appsv1.DaemonSet{}
-			err := k8sClient.Get(ctx, types.NamespacedName{Name: "sriov-metrics-exporter", Namespace: testNamespace}, daemonSet)
-			Expect(err).To(HaveOccurred())
-			Expect(errors.IsNotFound(err)).To(BeTrue())
-
-			By("Turn `metricsExporter` feature gate on")
-			config.Spec.FeatureGates = map[string]bool{constants.MetricsExporterFeatureGate: true}
-			err = k8sClient.Update(ctx, config)
-			Expect(err).NotTo(HaveOccurred())
-
-			DeferCleanup(func() {
-				config.Spec.FeatureGates = map[string]bool{}
-				err = k8sClient.Update(ctx, config)
-				Expect(err).NotTo(HaveOccurred())
+		Context("metricsExporter feature gate", func() {
+			When("is disabled", func() {
+				It("should not deploy the daemonset", func() {
+					daemonSet := &appsv1.DaemonSet{}
+					err := k8sClient.Get(ctx, types.NamespacedName{Name: "sriov-metrics-exporter", Namespace: testNamespace}, daemonSet)
+					Expect(err).To(HaveOccurred())
+					Expect(errors.IsNotFound(err)).To(BeTrue())
+				})
 			})
 
-			err = util.WaitForNamespacedObject(&appsv1.DaemonSet{}, k8sClient, testNamespace, "sriov-network-metrics-exporter", util.RetryInterval, util.APITimeout)
-			Expect(err).NotTo(HaveOccurred())
-
-			err = util.WaitForNamespacedObject(&corev1.Service{}, k8sClient, testNamespace, "sriov-network-metrics-exporter-service", util.RetryInterval, util.APITimeout)
-			Expect(err).ToNot(HaveOccurred())
-
-			By("Turn `metricsExporter` feature gate off")
-			config.Spec.FeatureGates = map[string]bool{}
-			err = k8sClient.Update(ctx, config)
-
-			err = util.WaitForNamespacedObjectDeleted(&appsv1.DaemonSet{}, k8sClient, testNamespace, "sriov-network-metrics-exporter", util.RetryInterval, util.APITimeout)
-			Expect(err).NotTo(HaveOccurred())
-
-			err = util.WaitForNamespacedObjectDeleted(&corev1.Service{}, k8sClient, testNamespace, "sriov-network-metrics-exporter-service", util.RetryInterval, util.APITimeout)
-			Expect(err).ToNot(HaveOccurred())
+			When("is enabled", func() {
+				BeforeEach(func() {
+					config := &sriovnetworkv1.SriovOperatorConfig{}
+					Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: "default"}, config)).NotTo(HaveOccurred())
+
+					By("Turn `metricsExporter` feature gate on")
+					config.Spec.FeatureGates = map[string]bool{constants.MetricsExporterFeatureGate: true}
+					err := k8sClient.Update(ctx, config)
+					Expect(err).NotTo(HaveOccurred())
+				})
+
+				It("should deploy the sriov-network-metrics-exporter DaemonSet", func() {
+					err := util.WaitForNamespacedObject(&appsv1.DaemonSet{}, k8sClient, testNamespace, "sriov-network-metrics-exporter", util.RetryInterval, util.APITimeout)
+					Expect(err).NotTo(HaveOccurred())
+
+					err = util.WaitForNamespacedObject(&corev1.Service{}, k8sClient, testNamespace, "sriov-network-metrics-exporter-service", util.RetryInterval, util.APITimeout)
+					Expect(err).ToNot(HaveOccurred())
+				})
+
+				It("should deploy extra configuration when the Prometheus operator is installed", func() {
+					assertResourceExists(
+						schema.GroupVersionKind{
+							Group:   "monitoring.coreos.com",
+							Kind:    "ServiceMonitor",
+							Version: "v1",
+						},
+						client.ObjectKey{Namespace: testNamespace, Name: "sriov-network-metrics-exporter"})
+
+					assertResourceExists(
+						schema.GroupVersionKind{
+							Group:   "rbac.authorization.k8s.io",
+							Kind:    "Role",
+							Version: "v1",
+						},
+						client.ObjectKey{Namespace: testNamespace, Name: "prometheus-k8s"})
+
+					assertResourceExists(
+						schema.GroupVersionKind{
+							Group:   "rbac.authorization.k8s.io",
+							Kind:    "RoleBinding",
+							Version: "v1",
+						},
+						client.ObjectKey{Namespace: testNamespace, Name: "prometheus-k8s"})
+				})
+			})
 		})
 
 		// This test verifies that the CABundle field in the webhook configuration  added by third party components is not
@@ -429,6 +452,7 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() {
 				g.Expect(injectorCfg.Webhooks[0].ClientConfig.CABundle).To(Equal([]byte("ca-bundle-2\n")))
 			}, "1s").Should(Succeed())
 		})
+
 		It("should reconcile to a converging state when multiple node policies are set", func() {
 			By("Creating a consistent number of node policies")
 			for i := 0; i < 30; i++ {
@@ -477,3 +501,10 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() {
 		})
 	})
 })
+
+func assertResourceExists(gvk schema.GroupVersionKind, key client.ObjectKey) {
+	u := &unstructured.Unstructured{}
+	u.SetGroupVersionKind(gvk)
+	err := k8sClient.Get(context.Background(), key, u)
+	Expect(err).NotTo(HaveOccurred())
+}

controllers/suite_test.go

@@ -41,6 +41,7 @@ import (
 	netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	openshiftconfigv1 "github.com/openshift/api/config/v1"
 	mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
+	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 
 	//+kubebuilder:scaffold:imports
 	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
@@ -135,6 +136,10 @@ var _ = BeforeSuite(func() {
 	Expect(err).NotTo(HaveOccurred())
 	err = os.Setenv("METRICS_EXPORTER_KUBE_RBAC_PROXY_IMAGE", "mock-image")
 	Expect(err).NotTo(HaveOccurred())
+	err = os.Setenv("METRICS_EXPORTER_PROMETHEUS_OPERATOR_SERVICE_ACCOUNT", "k8s-prometheus")
+	Expect(err).NotTo(HaveOccurred())
+	err = os.Setenv("METRICS_EXPORTER_PROMETHEUS_OPERATOR_NAMESPACE", "default")
+	Expect(err).NotTo(HaveOccurred())
 
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
@@ -157,6 +162,8 @@ var _ = BeforeSuite(func() {
 	Expect(err).NotTo(HaveOccurred())
 	err = openshiftconfigv1.AddToScheme(scheme.Scheme)
 	Expect(err).NotTo(HaveOccurred())
+	err = monitoringv1.AddToScheme(scheme.Scheme)
+	Expect(err).NotTo(HaveOccurred())
 
 	vars.Config = cfg
 	vars.Scheme = scheme.Scheme

deploy/operator.yaml

@@ -72,6 +72,10 @@ spec:
               value: $METRICS_EXPORTER_IMAGE
             - name: METRICS_EXPORTER_KUBE_RBAC_PROXY_IMAGE
               value: $METRICS_EXPORTER_KUBE_RBAC_PROXY_IMAGE
+            - name: METRICS_EXPORTER_PROMETHEUS_OPERATOR_SERVICE_ACCOUNT
+              value: $METRICS_EXPORTER_PROMETHEUS_OPERATOR_SERVICE_ACCOUNT
+            - name: METRICS_EXPORTER_PROMETHEUS_OPERATOR_NAMESPACE
+              value: $METRICS_EXPORTER_PROMETHEUS_OPERATOR_NAMESPACE
             - name: RESOURCE_PREFIX
               value: $RESOURCE_PREFIX
             - name: DEV_MODE

deployment/sriov-network-operator-chart/README.md

@@ -86,6 +86,9 @@ We have introduced the following Chart parameters.
 | `operator.clustertype` | string | `kubernetes` | Cluster environment type |
 | `operator.metricsExporter.port` | string | `9110` | Port where the Network Metrics Exporter listen |
 | `operator.metricsExporter.certificates.secretName` | string | `metrics-exporter-cert` | Secret name to serve metrics via TLS. The secret must have the same fields as `operator.admissionControllers.certificates.secretNames` |
+| `operator.metricsExporter.prometheusOperator.serviceAccount` | string | `prometheus-k8s` | The service account used by the Prometheus Operator. This is used to give Prometheus the permission to list resource in the SR-IOV operator namespace |
+| `operator.metricsExporter.prometheusOperator.namespace` | string | `` | The namespace where the Prometheus Operator is installed. Setting this variable makes the operator deploy `monitoring.coreos.com` resources. |
+| `operator.metricsExporter.port` | string | `9110` | Port where the Network Metrics Exporter listen |
 
 #### Admission Controllers parameters
 

deployment/sriov-network-operator-chart/templates/operator.yaml

@@ -76,6 +76,10 @@ spec:
               value: "{{ .Values.operator.metricsExporter.port }}"
             - name: METRICS_EXPORTER_SECRET_NAME
               value: {{ .Values.operator.metricsExporter.certificates.secretName }}
+            - name: METRICS_EXPORTER_PROMETHEUS_OPERATOR_SERVICE_ACCOUNT
+              value: {{ .Values.operator.metricsExporter.prometheusOperator.serviceAccount }}
+            - name: METRICS_EXPORTER_PROMETHEUS_OPERATOR_NAMESPACE
+              value: {{ .Values.operator.metricsExporter.prometheusOperator.namespace }}
             - name: METRICS_EXPORTER_KUBE_RBAC_PROXY_IMAGE
               value: {{ .Values.images.metricsExporterKubeRbacProxy }}
             - name: RESOURCE_PREFIX

deployment/sriov-network-operator-chart/values.yaml

@@ -31,6 +31,9 @@ operator:
     port: "9110"
     certificates:
       secretName: "metrics-exporter-cert"
+    prometheusOperator:
+      serviceAccount: "prometheus-k8s"
+      namespace: ""
   admissionControllers:
     enabled: false
     certificates:

hack/run-e2e-conformance-virtual-ocp.sh

@@ -190,6 +190,8 @@ export CLUSTER_TYPE=openshift
 export DEV_MODE=TRUE
 export CLUSTER_HAS_EMULATED_PF=TRUE
 export OPERATOR_LEADER_ELECTION_ENABLE=true
+export METRICS_EXPORTER_PROMETHEUS_OPERATOR_SERVICE_ACCOUNT=${METRICS_EXPORTER_PROMETHEUS_OPERATOR_SERVICE_ACCOUNT:-"prometheus-k8s"}
+export METRICS_EXPORTER_PROMETHEUS_OPERATOR_NAMESPACE=${METRICS_EXPORTER_PROMETHEUS_OPERATOR_NAMESPACE:-"openshfit-monitoring"}
 
 export SRIOV_NETWORK_OPERATOR_IMAGE="$registry/$NAMESPACE/sriov-network-operator:latest"
 export SRIOV_NETWORK_CONFIG_DAEMON_IMAGE="$registry/$NAMESPACE/sriov-network-config-daemon:latest"

test/conformance/tests/test_sriov_operator.go

@@ -308,7 +308,7 @@ var _ = Describe("[sriov] operator", func() {
 		})
 
 		Context("SriovNetworkMetricsExporter", func() {
-			It("should be deployed if the feature gate is enabled", func() {
+			BeforeEach(func() {
 				if discovery.Enabled() {
 					Skip("Test unsuitable to be run in discovery mode")
 				}
@@ -321,12 +321,26 @@ var _ = Describe("[sriov] operator", func() {
 
 				By("Enabling `metricsExporter` feature flag")
 				setFeatureFlag("metricsExporter", true)
+			})
 
+			It("should be deployed if the feature gate is enabled", func() {
 				By("Checking that a daemon is scheduled on selected node")
 				Eventually(func() bool {
 					return isDaemonsetScheduledOnNodes("node-role.kubernetes.io/worker", "app=sriov-network-metrics-exporter")
 				}, 1*time.Minute, 1*time.Second).Should(Equal(true))
+			})
+
+			It("should deploy ServiceMonitor if the Promethueus operator is installed", func() {
+				_, err := clients.ServiceMonitors(operatorNamespace).List(context.Background(), metav1.ListOptions{})
+				if k8serrors.IsNotFound(err) {
+					Skip("Prometheus operator not available in the cluster")
+				}
 
+				By("Checking ServiceMonitor is deployed if needed")
+				Eventually(func(g Gomega) {
+					_, err := clients.ServiceMonitors(operatorNamespace).Get(context.Background(), "sriov-network-metrics-exporter", metav1.GetOptions{})
+					g.Expect(err).ToNot(HaveOccurred())
+				}).Should(Succeed())
 			})
 		})
 	})

test/util/client/clients.go

@@ -18,6 +18,8 @@ import (
 	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
+	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/typed/monitoring/v1"
+
 	sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
 	clientsriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned/typed/sriovnetwork/v1"
 	snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log"
@@ -39,6 +41,7 @@ type ClientSet struct {
 	Config *rest.Config
 	runtimeclient.Client
 	coordinationv1.CoordinationV1Interface
+	monitoringv1.MonitoringV1Interface
 }
 
 // New returns a *ClientBuilder with the given kubeconfig.
@@ -70,6 +73,7 @@ func New(kubeconfig string) *ClientSet {
 	clientSet.DiscoveryInterface = discovery.NewDiscoveryClientForConfigOrDie(config)
 	clientSet.SriovnetworkV1Interface = clientsriovv1.NewForConfigOrDie(config)
 	clientSet.CoordinationV1Interface = coordinationv1.NewForConfigOrDie(config)
+	clientSet.MonitoringV1Interface = monitoringv1.NewForConfigOrDie(config)
 	clientSet.Config = config
 
 	crScheme := runtime.NewScheme()