signal 0 when no metric unmarshaled

receiver/splunkenterprisereceiver/documentation.md

@@ -416,7 +416,7 @@ Backup and restore status of the KV store.
 
 | Name | Description | Values |
 | ---- | ----------- | ------ |
-| splunk.kvstore.status | The status returned when reporting on KV store using the introspection endpoint | Any Str |
+| splunk.kvstore.status.value | The string value of the status returned when reporting on KV store using the introspection endpoint | Any Str |
 
 ### splunk.kvstore.replication.status
 
@@ -430,7 +430,7 @@ Replication status of the KV store.
 
 | Name | Description | Values |
 | ---- | ----------- | ------ |
-| splunk.kvstore.status | The status returned when reporting on KV store using the introspection endpoint | Any Str |
+| splunk.kvstore.status.value | The string value of the status returned when reporting on KV store using the introspection endpoint | Any Str |
 
 ### splunk.kvstore.status
 
@@ -446,7 +446,7 @@ This is the overall status of the kvstore for the given deployment.
 | ---- | ----------- | ------ |
 | splunk.kvstore.storage.engine | The backend storage used by the KV store. | Any Str |
 | splunk.kvstore.external | Value denoting if the KV store is using an external service. | Any Str |
-| splunk.kvstore.status | The status returned when reporting on KV store using the introspection endpoint | Any Str |
+| splunk.kvstore.status.value | The string value of the status returned when reporting on KV store using the introspection endpoint | Any Str |
 
 ### splunk.server.introspection.queues.current
 

receiver/splunkenterprisereceiver/metadata.yaml

@@ -27,8 +27,8 @@ attributes:
   splunk.queue.name:
     description: The name of the queue reporting a specific KPI
     type: string
-  splunk.kvstore.status:
-    description: The status returned when reporting on KV store using the introspection endpoint
+  splunk.kvstore.status.value:
+    description: The string value of the status returned when reporting on KV store using the introspection endpoint
     type: string
   splunk.kvstore.external:
     description: Value denoting if the KV store is using an external service.
@@ -254,21 +254,21 @@ metrics:
     unit: '{status}'
     gauge:
       value_type: int
-    attributes: [splunk.kvstore.storage.engine, splunk.kvstore.external, splunk.kvstore.status]
+    attributes: [splunk.kvstore.storage.engine, splunk.kvstore.external, splunk.kvstore.status.value]
   splunk.kvstore.replication.status:
     enabled: false
     description: Replication status of the KV store.
     unit: '{status}'
     gauge:
       value_type: int
-    attributes: [splunk.kvstore.status]
+    attributes: [splunk.kvstore.status.value]
   splunk.kvstore.backup.status:
     enabled: false
     description: Backup and restore status of the KV store.
     unit: '{status}'
     gauge:
       value_type: int
-    attributes: [splunk.kvstore.status]
+    attributes: [splunk.kvstore.status.value]
 
 tests:
   config:

receiver/splunkenterprisereceiver/scraper.go

@@ -1589,14 +1589,7 @@ func (s *splunkScraper) scrapeKVStoreStatus(ctx context.Context, now pcommon.Tim
 	}
 	defer res.Body.Close()
 
-	body, err := io.ReadAll(res.Body)
-	if err != nil {
-		errs <- err
-		return
-	}
-
-	err = json.Unmarshal(body, &kvs)
-	if err != nil {
+	if err := json.NewDecoder(res.Body).Decode(&kvs); err != nil {
 		errs <- err
 		return
 	}
@@ -1609,8 +1602,28 @@ func (s *splunkScraper) scrapeKVStoreStatus(ctx context.Context, now pcommon.Tim
 		se = kv.Content.Current.StorageEngine
 		ext = kv.Content.KVService.Status
 
-		s.mb.RecordSplunkKvstoreStatusDataPoint(now, 1, se, ext, st)
-		s.mb.RecordSplunkKvstoreBackupStatusDataPoint(now, 1, brs)
-		s.mb.RecordSplunkKvstoreReplicationStatusDataPoint(now, 1, rs)
+		// a 0 gauge value means that the metric was not reported in the api call
+		// to the introspection endpoint.
+		if st == "" {
+			st = KVStatusUnknown
+			// set to 0 to indicate no status being reported
+			s.mb.RecordSplunkKvstoreStatusDataPoint(now, 0, se, ext, st)
+		} else {
+			s.mb.RecordSplunkKvstoreStatusDataPoint(now, 1, se, ext, st)
+		}
+
+		if rs == "" {
+			rs = KVRestoreStatusUnknown
+			s.mb.RecordSplunkKvstoreReplicationStatusDataPoint(now, 0, rs)
+		} else {
+			s.mb.RecordSplunkKvstoreReplicationStatusDataPoint(now, 1, rs)
+		}
+
+		if brs == "" {
+			brs = KVBackupStatusFailed
+			s.mb.RecordSplunkKvstoreBackupStatusDataPoint(now, 0, brs)
+		} else {
+			s.mb.RecordSplunkKvstoreBackupStatusDataPoint(now, 1, brs)
+		}
 	}
 }

receiver/splunkenterprisereceiver/search_result.go

@@ -104,6 +104,13 @@ type IdxQContent struct {
 }
 
 // '/services/kvstore/status'
+const (
+	// unknown/failed values
+	KVStatusUnknown        = "unknown"
+	KVRestoreStatusUnknown = "Unknown status"
+	KVBackupStatusFailed   = "Failed"
+)
+
 type KVStoreStatus struct {
 	Entries []KVEntry `json:"entry"`
 }