Skip to content

Commit

Permalink
metrics-exporter: configure Prometheus operator
Browse files Browse the repository at this point in the history
Deploy the needed configuration to make the prometheus
operator to find and scrape the sriov-network-metrics-exporter
endpoints, including the ServiceMonitor, Role and RoleBinding

Signed-off-by: Andrea Panattoni <apanatto@redhat.com>
  • Loading branch information
zeeke committed May 17, 2024
1 parent 41fb159 commit 7999f7e
Show file tree
Hide file tree
Showing 5 changed files with 1,077 additions and 23 deletions.
4 changes: 3 additions & 1 deletion bindata/manifests/metrics-exporter/metrics-service.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ metadata:
namespace: {{.Namespace}}
annotations:
prometheus.io/target: "true"
{{- if eq .ClusterType "openshift" }}
{{ if .IsOpenshift }}
service.beta.openshift.io/serving-cert-secret-name: {{ .MetricsExporterSecretName }}
{{- end }}
labels:
Expand All @@ -18,6 +18,7 @@ spec:
name: sriov-network-metrics
port: {{ .MetricsExporterPort }}
targetPort: {{ .MetricsExporterPort }}
{{ if .IsOpenshift }}
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
Expand Down Expand Up @@ -72,3 +73,4 @@ rules:
- get
- list
- watch
{{ end }}
28 changes: 22 additions & 6 deletions controllers/sriovoperatorconfig_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import (
snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log"
"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms"
render "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/render"
"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
)

Expand Down Expand Up @@ -230,7 +231,7 @@ func (r *SriovOperatorConfigReconciler) syncMetricsExporter(ctx context.Context,
data.Data["MetricsExporterSecretName"] = os.Getenv("METRICS_EXPORTER_SECRET_NAME")
data.Data["MetricsExporterPort"] = os.Getenv("METRICS_EXPORTER_PORT")
data.Data["MetricsExporterKubeRbacProxyImage"] = os.Getenv("METRICS_EXPORTER_KUBE_RBAC_PROXY_IMAGE")
data.Data["ClusterType"] = vars.ClusterType
data.Data["IsOpenshift"] = r.PlatformHelper.IsOpenshiftCluster()
data.Data["NodeSelectorField"] = GetDefaultNodeSelector()
if dc.Spec.ConfigDaemonNodeSelector != nil {
data.Data["NodeSelectorField"] = dc.Spec.ConfigDaemonNodeSelector
Expand All @@ -251,14 +252,19 @@ func (r *SriovOperatorConfigReconciler) syncMetricsExporter(ctx context.Context,
return err
}
}

if r.PlatformHelper.IsOpenshiftCluster() {
err = utils.AddLabelToNamespace(ctx, vars.Namespace, "openshift.io/cluster-monitoring", "true", r.Client)
if err != nil {
return err
}
}
return nil
}

for _, obj := range objs {
err = r.deleteK8sResource(ctx, obj)
if err != nil {
return err
}
err = r.deleteK8sResources(ctx, objs)
if err != nil {
return err
}

return nil
Expand Down Expand Up @@ -354,6 +360,16 @@ func (r *SriovOperatorConfigReconciler) deleteK8sResource(ctx context.Context, i
return nil
}

func (r *SriovOperatorConfigReconciler) deleteK8sResources(ctx context.Context, objs []*uns.Unstructured) error {
for _, obj := range objs {
err := r.deleteK8sResource(ctx, obj)
if err != nil {
return err
}
}
return nil
}

func (r *SriovOperatorConfigReconciler) syncK8sResource(ctx context.Context, cr *sriovnetworkv1.SriovOperatorConfig, in *uns.Unstructured) error {
switch in.GetKind() {
case clusterRoleResourceName, clusterRoleBindingResourceName, mutatingWebhookConfigurationCRDName, validatingWebhookConfigurationCRDName, machineConfigCRDName:
Expand Down
114 changes: 98 additions & 16 deletions controllers/sriovoperatorconfig_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,11 @@ import (
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/golang/mock/gomock"
. "github.com/onsi/ginkgo/v2"
Expand All @@ -28,6 +31,7 @@ import (
var _ = Describe("SriovOperatorConfig controller", Ordered, func() {
var cancel context.CancelFunc
var ctx context.Context
var reconciler *SriovOperatorConfigReconciler

BeforeAll(func() {
By("Create SriovOperatorConfig controller k8s objs")
Expand Down Expand Up @@ -73,12 +77,13 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() {
platformHelper.EXPECT().IsOpenshiftCluster().Return(false).AnyTimes()
platformHelper.EXPECT().IsHypershift().Return(false).AnyTimes()

err = (&SriovOperatorConfigReconciler{
reconciler = &SriovOperatorConfigReconciler{
Client: k8sManager.GetClient(),
Scheme: k8sManager.GetScheme(),
PlatformHelper: platformHelper,
FeatureGate: featuregate.New(),
}).SetupWithManager(k8sManager)
}
err = reconciler.SetupWithManager(k8sManager)
Expect(err).ToNot(HaveOccurred())

ctx, cancel = context.WithCancel(context.Background())
Expand All @@ -100,21 +105,21 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() {
})
})

Context("When is up", func() {
JustBeforeEach(func() {
config := &sriovnetworkv1.SriovOperatorConfig{}
err := util.WaitForNamespacedObject(config, k8sClient, testNamespace, "default", util.RetryInterval, util.APITimeout)
Expect(err).NotTo(HaveOccurred())
config.Spec = sriovnetworkv1.SriovOperatorConfigSpec{
EnableInjector: true,
EnableOperatorWebhook: true,
LogLevel: 2,
FeatureGates: map[string]bool{},
}
err = k8sClient.Update(ctx, config)
Expect(err).NotTo(HaveOccurred())
})
BeforeEach(func() {
config := &sriovnetworkv1.SriovOperatorConfig{}
err := util.WaitForNamespacedObject(config, k8sClient, testNamespace, "default", util.RetryInterval, util.APITimeout)
Expect(err).NotTo(HaveOccurred())
config.Spec = sriovnetworkv1.SriovOperatorConfigSpec{
EnableInjector: true,
EnableOperatorWebhook: true,
LogLevel: 2,
FeatureGates: map[string]bool{},
}
err = k8sClient.Update(ctx, config)
Expect(err).NotTo(HaveOccurred())
})

Context("When is up", func() {
It("should have webhook enable", func() {
mutateCfg := &admv1.MutatingWebhookConfiguration{}
err := util.WaitForNamespacedObject(mutateCfg, k8sClient, testNamespace, "sriov-operator-webhook-config", util.RetryInterval, util.APITimeout*3)
Expand Down Expand Up @@ -371,6 +376,83 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() {
err = util.WaitForNamespacedObjectDeleted(&v1.Service{}, k8sClient, testNamespace, "sriov-network-metrics-exporter-service", util.RetryInterval, util.APITimeout)
Expect(err).ToNot(HaveOccurred())
})
})

Context("on OpenShift clusters", func() {

BeforeAll(func() {
t := GinkgoT()
mockCtrl := gomock.NewController(t)
openshiftPlatformHelper := mock_platforms.NewMockInterface(mockCtrl)
openshiftPlatformHelper.EXPECT().GetFlavor().Return(openshift.OpenshiftFlavorDefault).AnyTimes()
openshiftPlatformHelper.EXPECT().IsHypershift().Return(false).AnyTimes()

openshiftPlatformHelper.EXPECT().IsOpenshiftCluster().Return(true).AnyTimes()

previousPlatformHelper := reconciler.PlatformHelper
DeferCleanup(func() {
reconciler.PlatformHelper = previousPlatformHelper
})

reconciler.PlatformHelper = openshiftPlatformHelper
})

It("should deploy specific featureswhen the metrics-exporter feature is enabled", func() {
config := &sriovnetworkv1.SriovOperatorConfig{}
Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: "default"}, config)).NotTo(HaveOccurred())

config.Spec.FeatureGates = map[string]bool{constants.MetricsExporterFeatureGate: true}
err := k8sClient.Update(ctx, config)
Expect(err).NotTo(HaveOccurred())

DeferCleanup(func() {
Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: "default"}, config)).NotTo(HaveOccurred())
config.Spec.FeatureGates = map[string]bool{}
err = k8sClient.Update(ctx, config)
Expect(err).NotTo(HaveOccurred())
})

By("check the `openshift.io/cluster-monitoring` label has been addedd to the operator's namespace")

Eventually(func(g Gomega) {
ns := &v1.Namespace{}
err := k8sClient.Get(ctx, types.NamespacedName{Name: testNamespace}, ns)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(ns.Labels).To(HaveKeyWithValue("openshift.io/cluster-monitoring", "true"))
}).Should(Succeed())

By("check prometheus related configuration has been deployed")
assertCRIsPresent(
schema.GroupVersionKind{
Group: "monitoring.coreos.com",
Kind: "ServiceMonitor",
Version: "v1",
},
client.ObjectKey{Namespace: testNamespace, Name: "sriov-network-metrics-exporter"})

assertCRIsPresent(
schema.GroupVersionKind{
Group: "rbac.authorization.k8s.io",
Kind: "Role",
Version: "v1",
},
client.ObjectKey{Namespace: testNamespace, Name: "prometheus-k8s"})

assertCRIsPresent(
schema.GroupVersionKind{
Group: "rbac.authorization.k8s.io",
Kind: "RoleBinding",
Version: "v1",
},
client.ObjectKey{Namespace: testNamespace, Name: "prometheus-k8s"})

})
})
})

func assertCRIsPresent(gvk schema.GroupVersionKind, key client.ObjectKey) {
u := &unstructured.Unstructured{}
u.SetGroupVersionKind(gvk)
err := k8sClient.Get(context.Background(), key, u)
Expect(err).NotTo(HaveOccurred())
}
26 changes: 26 additions & 0 deletions pkg/utils/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -161,3 +161,29 @@ func AnnotateNode(ctx context.Context, nodeName string, key, value string, c cli

return AnnotateObject(ctx, node, key, value, c)
}


Check failure on line 165 in pkg/utils/cluster.go

View workflow job for this annotation

GitHub Actions / Golangci-lint

File is not `gofmt`-ed with `-s` (gofmt)
func AddLabelToNamespace(ctx context.Context, namespaceName, key, value string, c client.Client) error {
ns := &corev1.Namespace{}
err := c.Get(ctx, client.ObjectKey{Name: namespaceName}, ns)
if err != nil {
return fmt.Errorf("failed to get namespace [%s]: %v", namespaceName, err)
}

if ns.Labels == nil {
ns.Labels = make(map[string]string)
}

if ns.Labels[key] == value {
return nil
}

ns.Labels[key] = value

err = c.Update(ctx, ns)
if err != nil {
return fmt.Errorf("failed to update namespace [%s] with label [%s: %s]: %v", namespaceName, key, value, err)
}

return nil
}
Loading

0 comments on commit 7999f7e

Please sign in to comment.