From ff10675bcbeb510d3cc2335410414427bc352cbf Mon Sep 17 00:00:00 2001
From: gnmahanth <mahanth@deepfence.io>
Date: Fri, 8 Mar 2024 14:26:43 +0000
Subject: [PATCH] add metrics to track notifications sent

---
 deepfence_worker/cronjobs/notification.go | 26 +++++++++++++++++++++++
 deepfence_worker/metrics.go               |  3 ++-
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/deepfence_worker/cronjobs/notification.go b/deepfence_worker/cronjobs/notification.go
index e8676a9512..6301925a29 100644
--- a/deepfence_worker/cronjobs/notification.go
+++ b/deepfence_worker/cronjobs/notification.go
@@ -11,6 +11,7 @@ import (
 	"time"
 
 	reporters_search "github.com/deepfence/ThreatMapper/deepfence_server/reporters/search"
+	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/deepfence/ThreatMapper/deepfence_server/model"
 	"github.com/deepfence/ThreatMapper/deepfence_server/pkg/integration"
@@ -25,6 +26,13 @@ import (
 
 const NOTIFICATION_INTERVAL = 60000 //in milliseconds
 
+var (
+	NotificationRecordsCounts = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "notification_records_total",
+		Help: "Total number of records sent by notification types",
+	}, []string{"resource", "type", "status", "namespace"})
+)
+
 var fieldsMap = map[string]map[string]string{
 	utils.ScanTypeDetectedNode[utils.NEO4JVulnerabilityScan]: {
 		"cve_severity":          "Severity",
@@ -434,8 +442,26 @@ func processIntegration[T any](ctx context.Context, task *asynq.Task, integratio
 		err = integrationModel.SendNotification(ctx, string(messageByte), extras)
 		totalSendTime = totalSendTime + time.Since(profileStart).Milliseconds()
 		if err != nil {
+			NotificationRecordsCounts.WithLabelValues(
+				integrationRow.Resource,
+				integrationRow.IntegrationType,
+				"error",
+				func() string {
+					ns, _ := directory.ExtractNamespace(ctx)
+					return string(ns)
+				}(),
+			).Add(float64(len(updatedResults)))
 			return err
 		}
+		NotificationRecordsCounts.WithLabelValues(
+			integrationRow.Resource,
+			integrationRow.IntegrationType,
+			"success",
+			func() string {
+				ns, _ := directory.ExtractNamespace(ctx)
+				return string(ns)
+			}(),
+		).Add(float64(len(updatedResults)))
 		log.Info().Msgf("Notification sent %s scan %d messages using %s id %d, time taken:%d",
 			integrationRow.Resource, len(results), integrationRow.IntegrationType,
 			integrationRow.ID, time.Since(profileStart).Milliseconds())
diff --git a/deepfence_worker/metrics.go b/deepfence_worker/metrics.go
index bd137a8e02..86fbec8e80 100644
--- a/deepfence_worker/metrics.go
+++ b/deepfence_worker/metrics.go
@@ -3,6 +3,7 @@ package main
 import (
 	"github.com/deepfence/ThreatMapper/deepfence_utils/directory"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/log"
+	"github.com/deepfence/ThreatMapper/deepfence_worker/cronjobs"
 	"github.com/deepfence/ThreatMapper/deepfence_worker/processors"
 	"github.com/hibiken/asynq"
 	"github.com/prometheus/client_golang/prometheus"
@@ -22,7 +23,7 @@ func NewMetrics(mode string) *prometheus.Registry {
 	case "ingester":
 		registry.MustRegister(processors.CommitNeo4jRecordsCounts, processors.KafkaTopicsLag)
 	case "worker":
-		registry.MustRegister(newWorkerCollector())
+		registry.MustRegister(newWorkerCollector(), cronjobs.NotificationRecordsCounts)
 	}
 
 	return registry