From 2cb5e212c4ce25bd513caa48b8b325cf328d472b Mon Sep 17 00:00:00 2001 From: shalper2 <99686388+shalper2@users.noreply.github.com> Date: Wed, 23 Oct 2024 09:37:45 -0500 Subject: [PATCH] [receiver/splunkenterprise] 35445 add kvstore telemetry (#35657) #### Description Add telemetry around KV Store status to the Splunk Enterprise Receiver. #### Link to tracking issue Fixes [35445](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/35445) #### Testing Unit tests were updated and receiver component was deployed to test environments. #### Documentation Documentation was updated to reflect additional metrics --- .chloggen/35445-add-kvstore-telemetry.yaml | 27 +++ .../splunkenterprisereceiver/documentation.md | 44 +++++ .../internal/metadata/generated_config.go | 12 ++ .../metadata/generated_config_test.go | 6 + .../internal/metadata/generated_metrics.go | 179 ++++++++++++++++++ .../metadata/generated_metrics_test.go | 60 ++++++ .../internal/metadata/testdata/config.yaml | 12 ++ .../splunkenterprisereceiver/metadata.yaml | 33 +++- receiver/splunkenterprisereceiver/scraper.go | 67 +++++++ .../splunkenterprisereceiver/search_result.go | 33 ++++ 10 files changed, 472 insertions(+), 1 deletion(-) create mode 100644 .chloggen/35445-add-kvstore-telemetry.yaml diff --git a/.chloggen/35445-add-kvstore-telemetry.yaml b/.chloggen/35445-add-kvstore-telemetry.yaml new file mode 100644 index 000000000000..6e6292d009d3 --- /dev/null +++ b/.chloggen/35445-add-kvstore-telemetry.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: 'enhancement' + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: splunkenterprisereceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add telemetry around the Splunk Enterprise kv-store. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [35445] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/receiver/splunkenterprisereceiver/documentation.md b/receiver/splunkenterprisereceiver/documentation.md index a9a58472ec41..18671d969bc2 100644 --- a/receiver/splunkenterprisereceiver/documentation.md +++ b/receiver/splunkenterprisereceiver/documentation.md @@ -404,6 +404,50 @@ Gauge tracking average bytes per second throughput of indexer. *Note:** Must be | ---- | ----------- | ------ | | splunk.indexer.status | The status message reported for a specific object | Any Str | +### splunk.kvstore.backup.status + +Backup and restore status of the KV store. + +| Unit | Metric Type | Value Type | +| ---- | ----------- | ---------- | +| {status} | Gauge | Int | + +#### Attributes + +| Name | Description | Values | +| ---- | ----------- | ------ | +| splunk.kvstore.status.value | The string value of the status returned when reporting on KV store using the introspection endpoint | Any Str | + +### splunk.kvstore.replication.status + +Replication status of the KV store. + +| Unit | Metric Type | Value Type | +| ---- | ----------- | ---------- | +| {status} | Gauge | Int | + +#### Attributes + +| Name | Description | Values | +| ---- | ----------- | ------ | +| splunk.kvstore.status.value | The string value of the status returned when reporting on KV store using the introspection endpoint | Any Str | + +### splunk.kvstore.status + +This is the overall status of the kvstore for the given deployment. + +| Unit | Metric Type | Value Type | +| ---- | ----------- | ---------- | +| {status} | Gauge | Int | + +#### Attributes + +| Name | Description | Values | +| ---- | ----------- | ------ | +| splunk.kvstore.storage.engine | The backend storage used by the KV store. | Any Str | +| splunk.kvstore.external | Value denoting if the KV store is using an external service. | Any Str | +| splunk.kvstore.status.value | The string value of the status returned when reporting on KV store using the introspection endpoint | Any Str | + ### splunk.server.introspection.queues.current Gauge tracking current length of queue. *Note:** Must be pointed at specific indexer `endpoint` and gathers metrics from only that indexer. diff --git a/receiver/splunkenterprisereceiver/internal/metadata/generated_config.go b/receiver/splunkenterprisereceiver/internal/metadata/generated_config.go index 797f6b806c35..4887e11ff0b0 100644 --- a/receiver/splunkenterprisereceiver/internal/metadata/generated_config.go +++ b/receiver/splunkenterprisereceiver/internal/metadata/generated_config.go @@ -47,6 +47,9 @@ type MetricsConfig struct { SplunkIndexesMedianDataAge MetricConfig `mapstructure:"splunk.indexes.median.data.age"` SplunkIndexesSize MetricConfig `mapstructure:"splunk.indexes.size"` SplunkIoAvgIops MetricConfig `mapstructure:"splunk.io.avg.iops"` + SplunkKvstoreBackupStatus MetricConfig `mapstructure:"splunk.kvstore.backup.status"` + SplunkKvstoreReplicationStatus MetricConfig `mapstructure:"splunk.kvstore.replication.status"` + SplunkKvstoreStatus MetricConfig `mapstructure:"splunk.kvstore.status"` SplunkLicenseIndexUsage MetricConfig `mapstructure:"splunk.license.index.usage"` SplunkParseQueueRatio MetricConfig `mapstructure:"splunk.parse.queue.ratio"` SplunkPipelineSetCount MetricConfig `mapstructure:"splunk.pipeline.set.count"` @@ -120,6 +123,15 @@ func DefaultMetricsConfig() MetricsConfig { SplunkIoAvgIops: MetricConfig{ Enabled: true, }, + SplunkKvstoreBackupStatus: MetricConfig{ + Enabled: false, + }, + SplunkKvstoreReplicationStatus: MetricConfig{ + Enabled: false, + }, + SplunkKvstoreStatus: MetricConfig{ + Enabled: false, + }, SplunkLicenseIndexUsage: MetricConfig{ Enabled: true, }, diff --git a/receiver/splunkenterprisereceiver/internal/metadata/generated_config_test.go b/receiver/splunkenterprisereceiver/internal/metadata/generated_config_test.go index 358c3d143451..d904af8f5963 100644 --- a/receiver/splunkenterprisereceiver/internal/metadata/generated_config_test.go +++ b/receiver/splunkenterprisereceiver/internal/metadata/generated_config_test.go @@ -45,6 +45,9 @@ func TestMetricsBuilderConfig(t *testing.T) { SplunkIndexesMedianDataAge: MetricConfig{Enabled: true}, SplunkIndexesSize: MetricConfig{Enabled: true}, SplunkIoAvgIops: MetricConfig{Enabled: true}, + SplunkKvstoreBackupStatus: MetricConfig{Enabled: true}, + SplunkKvstoreReplicationStatus: MetricConfig{Enabled: true}, + SplunkKvstoreStatus: MetricConfig{Enabled: true}, SplunkLicenseIndexUsage: MetricConfig{Enabled: true}, SplunkParseQueueRatio: MetricConfig{Enabled: true}, SplunkPipelineSetCount: MetricConfig{Enabled: true}, @@ -81,6 +84,9 @@ func TestMetricsBuilderConfig(t *testing.T) { SplunkIndexesMedianDataAge: MetricConfig{Enabled: false}, SplunkIndexesSize: MetricConfig{Enabled: false}, SplunkIoAvgIops: MetricConfig{Enabled: false}, + SplunkKvstoreBackupStatus: MetricConfig{Enabled: false}, + SplunkKvstoreReplicationStatus: MetricConfig{Enabled: false}, + SplunkKvstoreStatus: MetricConfig{Enabled: false}, SplunkLicenseIndexUsage: MetricConfig{Enabled: false}, SplunkParseQueueRatio: MetricConfig{Enabled: false}, SplunkPipelineSetCount: MetricConfig{Enabled: false}, diff --git a/receiver/splunkenterprisereceiver/internal/metadata/generated_metrics.go b/receiver/splunkenterprisereceiver/internal/metadata/generated_metrics.go index e613558a3306..21a081c6031c 100644 --- a/receiver/splunkenterprisereceiver/internal/metadata/generated_metrics.go +++ b/receiver/splunkenterprisereceiver/internal/metadata/generated_metrics.go @@ -1035,6 +1035,161 @@ func newMetricSplunkIoAvgIops(cfg MetricConfig) metricSplunkIoAvgIops { return m } +type metricSplunkKvstoreBackupStatus struct { + data pmetric.Metric // data buffer for generated metric. + config MetricConfig // metric config provided by user. + capacity int // max observed number of data points added to the metric. +} + +// init fills splunk.kvstore.backup.status metric with initial data. +func (m *metricSplunkKvstoreBackupStatus) init() { + m.data.SetName("splunk.kvstore.backup.status") + m.data.SetDescription("Backup and restore status of the KV store.") + m.data.SetUnit("{status}") + m.data.SetEmptyGauge() + m.data.Gauge().DataPoints().EnsureCapacity(m.capacity) +} + +func (m *metricSplunkKvstoreBackupStatus) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val int64, splunkKvstoreStatusValueAttributeValue string) { + if !m.config.Enabled { + return + } + dp := m.data.Gauge().DataPoints().AppendEmpty() + dp.SetStartTimestamp(start) + dp.SetTimestamp(ts) + dp.SetIntValue(val) + dp.Attributes().PutStr("splunk.kvstore.status.value", splunkKvstoreStatusValueAttributeValue) +} + +// updateCapacity saves max length of data point slices that will be used for the slice capacity. +func (m *metricSplunkKvstoreBackupStatus) updateCapacity() { + if m.data.Gauge().DataPoints().Len() > m.capacity { + m.capacity = m.data.Gauge().DataPoints().Len() + } +} + +// emit appends recorded metric data to a metrics slice and prepares it for recording another set of data points. +func (m *metricSplunkKvstoreBackupStatus) emit(metrics pmetric.MetricSlice) { + if m.config.Enabled && m.data.Gauge().DataPoints().Len() > 0 { + m.updateCapacity() + m.data.MoveTo(metrics.AppendEmpty()) + m.init() + } +} + +func newMetricSplunkKvstoreBackupStatus(cfg MetricConfig) metricSplunkKvstoreBackupStatus { + m := metricSplunkKvstoreBackupStatus{config: cfg} + if cfg.Enabled { + m.data = pmetric.NewMetric() + m.init() + } + return m +} + +type metricSplunkKvstoreReplicationStatus struct { + data pmetric.Metric // data buffer for generated metric. + config MetricConfig // metric config provided by user. + capacity int // max observed number of data points added to the metric. +} + +// init fills splunk.kvstore.replication.status metric with initial data. +func (m *metricSplunkKvstoreReplicationStatus) init() { + m.data.SetName("splunk.kvstore.replication.status") + m.data.SetDescription("Replication status of the KV store.") + m.data.SetUnit("{status}") + m.data.SetEmptyGauge() + m.data.Gauge().DataPoints().EnsureCapacity(m.capacity) +} + +func (m *metricSplunkKvstoreReplicationStatus) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val int64, splunkKvstoreStatusValueAttributeValue string) { + if !m.config.Enabled { + return + } + dp := m.data.Gauge().DataPoints().AppendEmpty() + dp.SetStartTimestamp(start) + dp.SetTimestamp(ts) + dp.SetIntValue(val) + dp.Attributes().PutStr("splunk.kvstore.status.value", splunkKvstoreStatusValueAttributeValue) +} + +// updateCapacity saves max length of data point slices that will be used for the slice capacity. +func (m *metricSplunkKvstoreReplicationStatus) updateCapacity() { + if m.data.Gauge().DataPoints().Len() > m.capacity { + m.capacity = m.data.Gauge().DataPoints().Len() + } +} + +// emit appends recorded metric data to a metrics slice and prepares it for recording another set of data points. +func (m *metricSplunkKvstoreReplicationStatus) emit(metrics pmetric.MetricSlice) { + if m.config.Enabled && m.data.Gauge().DataPoints().Len() > 0 { + m.updateCapacity() + m.data.MoveTo(metrics.AppendEmpty()) + m.init() + } +} + +func newMetricSplunkKvstoreReplicationStatus(cfg MetricConfig) metricSplunkKvstoreReplicationStatus { + m := metricSplunkKvstoreReplicationStatus{config: cfg} + if cfg.Enabled { + m.data = pmetric.NewMetric() + m.init() + } + return m +} + +type metricSplunkKvstoreStatus struct { + data pmetric.Metric // data buffer for generated metric. + config MetricConfig // metric config provided by user. + capacity int // max observed number of data points added to the metric. +} + +// init fills splunk.kvstore.status metric with initial data. +func (m *metricSplunkKvstoreStatus) init() { + m.data.SetName("splunk.kvstore.status") + m.data.SetDescription("This is the overall status of the kvstore for the given deployment.") + m.data.SetUnit("{status}") + m.data.SetEmptyGauge() + m.data.Gauge().DataPoints().EnsureCapacity(m.capacity) +} + +func (m *metricSplunkKvstoreStatus) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val int64, splunkKvstoreStorageEngineAttributeValue string, splunkKvstoreExternalAttributeValue string, splunkKvstoreStatusValueAttributeValue string) { + if !m.config.Enabled { + return + } + dp := m.data.Gauge().DataPoints().AppendEmpty() + dp.SetStartTimestamp(start) + dp.SetTimestamp(ts) + dp.SetIntValue(val) + dp.Attributes().PutStr("splunk.kvstore.storage.engine", splunkKvstoreStorageEngineAttributeValue) + dp.Attributes().PutStr("splunk.kvstore.external", splunkKvstoreExternalAttributeValue) + dp.Attributes().PutStr("splunk.kvstore.status.value", splunkKvstoreStatusValueAttributeValue) +} + +// updateCapacity saves max length of data point slices that will be used for the slice capacity. +func (m *metricSplunkKvstoreStatus) updateCapacity() { + if m.data.Gauge().DataPoints().Len() > m.capacity { + m.capacity = m.data.Gauge().DataPoints().Len() + } +} + +// emit appends recorded metric data to a metrics slice and prepares it for recording another set of data points. +func (m *metricSplunkKvstoreStatus) emit(metrics pmetric.MetricSlice) { + if m.config.Enabled && m.data.Gauge().DataPoints().Len() > 0 { + m.updateCapacity() + m.data.MoveTo(metrics.AppendEmpty()) + m.init() + } +} + +func newMetricSplunkKvstoreStatus(cfg MetricConfig) metricSplunkKvstoreStatus { + m := metricSplunkKvstoreStatus{config: cfg} + if cfg.Enabled { + m.data = pmetric.NewMetric() + m.init() + } + return m +} + type metricSplunkLicenseIndexUsage struct { data pmetric.Metric // data buffer for generated metric. config MetricConfig // metric config provided by user. @@ -1522,6 +1677,9 @@ type MetricsBuilder struct { metricSplunkIndexesMedianDataAge metricSplunkIndexesMedianDataAge metricSplunkIndexesSize metricSplunkIndexesSize metricSplunkIoAvgIops metricSplunkIoAvgIops + metricSplunkKvstoreBackupStatus metricSplunkKvstoreBackupStatus + metricSplunkKvstoreReplicationStatus metricSplunkKvstoreReplicationStatus + metricSplunkKvstoreStatus metricSplunkKvstoreStatus metricSplunkLicenseIndexUsage metricSplunkLicenseIndexUsage metricSplunkParseQueueRatio metricSplunkParseQueueRatio metricSplunkPipelineSetCount metricSplunkPipelineSetCount @@ -1577,6 +1735,9 @@ func NewMetricsBuilder(mbc MetricsBuilderConfig, settings receiver.Settings, opt metricSplunkIndexesMedianDataAge: newMetricSplunkIndexesMedianDataAge(mbc.Metrics.SplunkIndexesMedianDataAge), metricSplunkIndexesSize: newMetricSplunkIndexesSize(mbc.Metrics.SplunkIndexesSize), metricSplunkIoAvgIops: newMetricSplunkIoAvgIops(mbc.Metrics.SplunkIoAvgIops), + metricSplunkKvstoreBackupStatus: newMetricSplunkKvstoreBackupStatus(mbc.Metrics.SplunkKvstoreBackupStatus), + metricSplunkKvstoreReplicationStatus: newMetricSplunkKvstoreReplicationStatus(mbc.Metrics.SplunkKvstoreReplicationStatus), + metricSplunkKvstoreStatus: newMetricSplunkKvstoreStatus(mbc.Metrics.SplunkKvstoreStatus), metricSplunkLicenseIndexUsage: newMetricSplunkLicenseIndexUsage(mbc.Metrics.SplunkLicenseIndexUsage), metricSplunkParseQueueRatio: newMetricSplunkParseQueueRatio(mbc.Metrics.SplunkParseQueueRatio), metricSplunkPipelineSetCount: newMetricSplunkPipelineSetCount(mbc.Metrics.SplunkPipelineSetCount), @@ -1671,6 +1832,9 @@ func (mb *MetricsBuilder) EmitForResource(options ...ResourceMetricsOption) { mb.metricSplunkIndexesMedianDataAge.emit(ils.Metrics()) mb.metricSplunkIndexesSize.emit(ils.Metrics()) mb.metricSplunkIoAvgIops.emit(ils.Metrics()) + mb.metricSplunkKvstoreBackupStatus.emit(ils.Metrics()) + mb.metricSplunkKvstoreReplicationStatus.emit(ils.Metrics()) + mb.metricSplunkKvstoreStatus.emit(ils.Metrics()) mb.metricSplunkLicenseIndexUsage.emit(ils.Metrics()) mb.metricSplunkParseQueueRatio.emit(ils.Metrics()) mb.metricSplunkPipelineSetCount.emit(ils.Metrics()) @@ -1801,6 +1965,21 @@ func (mb *MetricsBuilder) RecordSplunkIoAvgIopsDataPoint(ts pcommon.Timestamp, v mb.metricSplunkIoAvgIops.recordDataPoint(mb.startTime, ts, val, splunkHostAttributeValue) } +// RecordSplunkKvstoreBackupStatusDataPoint adds a data point to splunk.kvstore.backup.status metric. +func (mb *MetricsBuilder) RecordSplunkKvstoreBackupStatusDataPoint(ts pcommon.Timestamp, val int64, splunkKvstoreStatusValueAttributeValue string) { + mb.metricSplunkKvstoreBackupStatus.recordDataPoint(mb.startTime, ts, val, splunkKvstoreStatusValueAttributeValue) +} + +// RecordSplunkKvstoreReplicationStatusDataPoint adds a data point to splunk.kvstore.replication.status metric. +func (mb *MetricsBuilder) RecordSplunkKvstoreReplicationStatusDataPoint(ts pcommon.Timestamp, val int64, splunkKvstoreStatusValueAttributeValue string) { + mb.metricSplunkKvstoreReplicationStatus.recordDataPoint(mb.startTime, ts, val, splunkKvstoreStatusValueAttributeValue) +} + +// RecordSplunkKvstoreStatusDataPoint adds a data point to splunk.kvstore.status metric. +func (mb *MetricsBuilder) RecordSplunkKvstoreStatusDataPoint(ts pcommon.Timestamp, val int64, splunkKvstoreStorageEngineAttributeValue string, splunkKvstoreExternalAttributeValue string, splunkKvstoreStatusValueAttributeValue string) { + mb.metricSplunkKvstoreStatus.recordDataPoint(mb.startTime, ts, val, splunkKvstoreStorageEngineAttributeValue, splunkKvstoreExternalAttributeValue, splunkKvstoreStatusValueAttributeValue) +} + // RecordSplunkLicenseIndexUsageDataPoint adds a data point to splunk.license.index.usage metric. func (mb *MetricsBuilder) RecordSplunkLicenseIndexUsageDataPoint(ts pcommon.Timestamp, val int64, splunkIndexNameAttributeValue string) { mb.metricSplunkLicenseIndexUsage.recordDataPoint(mb.startTime, ts, val, splunkIndexNameAttributeValue) diff --git a/receiver/splunkenterprisereceiver/internal/metadata/generated_metrics_test.go b/receiver/splunkenterprisereceiver/internal/metadata/generated_metrics_test.go index 5ac5fc90930f..1e225a521ac8 100644 --- a/receiver/splunkenterprisereceiver/internal/metadata/generated_metrics_test.go +++ b/receiver/splunkenterprisereceiver/internal/metadata/generated_metrics_test.go @@ -131,6 +131,15 @@ func TestMetricsBuilder(t *testing.T) { allMetricsCount++ mb.RecordSplunkIoAvgIopsDataPoint(ts, 1, "splunk.host-val") + allMetricsCount++ + mb.RecordSplunkKvstoreBackupStatusDataPoint(ts, 1, "splunk.kvstore.status.value-val") + + allMetricsCount++ + mb.RecordSplunkKvstoreReplicationStatusDataPoint(ts, 1, "splunk.kvstore.status.value-val") + + allMetricsCount++ + mb.RecordSplunkKvstoreStatusDataPoint(ts, 1, "splunk.kvstore.storage.engine-val", "splunk.kvstore.external-val", "splunk.kvstore.status.value-val") + defaultMetricsCount++ allMetricsCount++ mb.RecordSplunkLicenseIndexUsageDataPoint(ts, 1, "splunk.index.name-val") @@ -499,6 +508,57 @@ func TestMetricsBuilder(t *testing.T) { attrVal, ok := dp.Attributes().Get("splunk.host") assert.True(t, ok) assert.EqualValues(t, "splunk.host-val", attrVal.Str()) + case "splunk.kvstore.backup.status": + assert.False(t, validatedMetrics["splunk.kvstore.backup.status"], "Found a duplicate in the metrics slice: splunk.kvstore.backup.status") + validatedMetrics["splunk.kvstore.backup.status"] = true + assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) + assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) + assert.Equal(t, "Backup and restore status of the KV store.", ms.At(i).Description()) + assert.Equal(t, "{status}", ms.At(i).Unit()) + dp := ms.At(i).Gauge().DataPoints().At(0) + assert.Equal(t, start, dp.StartTimestamp()) + assert.Equal(t, ts, dp.Timestamp()) + assert.Equal(t, pmetric.NumberDataPointValueTypeInt, dp.ValueType()) + assert.Equal(t, int64(1), dp.IntValue()) + attrVal, ok := dp.Attributes().Get("splunk.kvstore.status.value") + assert.True(t, ok) + assert.EqualValues(t, "splunk.kvstore.status.value-val", attrVal.Str()) + case "splunk.kvstore.replication.status": + assert.False(t, validatedMetrics["splunk.kvstore.replication.status"], "Found a duplicate in the metrics slice: splunk.kvstore.replication.status") + validatedMetrics["splunk.kvstore.replication.status"] = true + assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) + assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) + assert.Equal(t, "Replication status of the KV store.", ms.At(i).Description()) + assert.Equal(t, "{status}", ms.At(i).Unit()) + dp := ms.At(i).Gauge().DataPoints().At(0) + assert.Equal(t, start, dp.StartTimestamp()) + assert.Equal(t, ts, dp.Timestamp()) + assert.Equal(t, pmetric.NumberDataPointValueTypeInt, dp.ValueType()) + assert.Equal(t, int64(1), dp.IntValue()) + attrVal, ok := dp.Attributes().Get("splunk.kvstore.status.value") + assert.True(t, ok) + assert.EqualValues(t, "splunk.kvstore.status.value-val", attrVal.Str()) + case "splunk.kvstore.status": + assert.False(t, validatedMetrics["splunk.kvstore.status"], "Found a duplicate in the metrics slice: splunk.kvstore.status") + validatedMetrics["splunk.kvstore.status"] = true + assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) + assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) + assert.Equal(t, "This is the overall status of the kvstore for the given deployment.", ms.At(i).Description()) + assert.Equal(t, "{status}", ms.At(i).Unit()) + dp := ms.At(i).Gauge().DataPoints().At(0) + assert.Equal(t, start, dp.StartTimestamp()) + assert.Equal(t, ts, dp.Timestamp()) + assert.Equal(t, pmetric.NumberDataPointValueTypeInt, dp.ValueType()) + assert.Equal(t, int64(1), dp.IntValue()) + attrVal, ok := dp.Attributes().Get("splunk.kvstore.storage.engine") + assert.True(t, ok) + assert.EqualValues(t, "splunk.kvstore.storage.engine-val", attrVal.Str()) + attrVal, ok = dp.Attributes().Get("splunk.kvstore.external") + assert.True(t, ok) + assert.EqualValues(t, "splunk.kvstore.external-val", attrVal.Str()) + attrVal, ok = dp.Attributes().Get("splunk.kvstore.status.value") + assert.True(t, ok) + assert.EqualValues(t, "splunk.kvstore.status.value-val", attrVal.Str()) case "splunk.license.index.usage": assert.False(t, validatedMetrics["splunk.license.index.usage"], "Found a duplicate in the metrics slice: splunk.license.index.usage") validatedMetrics["splunk.license.index.usage"] = true diff --git a/receiver/splunkenterprisereceiver/internal/metadata/testdata/config.yaml b/receiver/splunkenterprisereceiver/internal/metadata/testdata/config.yaml index 44ad32cb2a7e..c3e150597d16 100644 --- a/receiver/splunkenterprisereceiver/internal/metadata/testdata/config.yaml +++ b/receiver/splunkenterprisereceiver/internal/metadata/testdata/config.yaml @@ -41,6 +41,12 @@ all_set: enabled: true splunk.io.avg.iops: enabled: true + splunk.kvstore.backup.status: + enabled: true + splunk.kvstore.replication.status: + enabled: true + splunk.kvstore.status: + enabled: true splunk.license.index.usage: enabled: true splunk.parse.queue.ratio: @@ -101,6 +107,12 @@ none_set: enabled: false splunk.io.avg.iops: enabled: false + splunk.kvstore.backup.status: + enabled: false + splunk.kvstore.replication.status: + enabled: false + splunk.kvstore.status: + enabled: false splunk.license.index.usage: enabled: false splunk.parse.queue.ratio: diff --git a/receiver/splunkenterprisereceiver/metadata.yaml b/receiver/splunkenterprisereceiver/metadata.yaml index 1daed654a74e..4c4c23b2c8e9 100644 --- a/receiver/splunkenterprisereceiver/metadata.yaml +++ b/receiver/splunkenterprisereceiver/metadata.yaml @@ -26,7 +26,16 @@ attributes: type: string splunk.queue.name: description: The name of the queue reporting a specific KPI - type: string + type: string + splunk.kvstore.status.value: + description: The string value of the status returned when reporting on KV store using the introspection endpoint + type: string + splunk.kvstore.external: + description: Value denoting if the KV store is using an external service. + type: string + splunk.kvstore.storage.engine: + description: The backend storage used by the KV store. + type: string metrics: splunk.license.index.usage: @@ -238,6 +247,28 @@ metrics: gauge: value_type: int attributes: [splunk.queue.name] +#'services/kvstore/status' + splunk.kvstore.status: + enabled: false + description: This is the overall status of the kvstore for the given deployment. + unit: '{status}' + gauge: + value_type: int + attributes: [splunk.kvstore.storage.engine, splunk.kvstore.external, splunk.kvstore.status.value] + splunk.kvstore.replication.status: + enabled: false + description: Replication status of the KV store. + unit: '{status}' + gauge: + value_type: int + attributes: [splunk.kvstore.status.value] + splunk.kvstore.backup.status: + enabled: false + description: Backup and restore status of the KV store. + unit: '{status}' + gauge: + value_type: int + attributes: [splunk.kvstore.status.value] tests: config: diff --git a/receiver/splunkenterprisereceiver/scraper.go b/receiver/splunkenterprisereceiver/scraper.go index 27ffab77d290..ed731080acad 100644 --- a/receiver/splunkenterprisereceiver/scraper.go +++ b/receiver/splunkenterprisereceiver/scraper.go @@ -101,6 +101,7 @@ func (s *splunkScraper) scrape(ctx context.Context) (pmetric.Metrics, error) { s.scrapeAvgIopsByHost, s.scrapeSchedulerRunTimeByHost, s.scrapeIndexerAvgRate, + s.scrapeKVStoreStatus, } errChan := make(chan error, len(metricScrapes)) @@ -1560,3 +1561,69 @@ func (s *splunkScraper) scrapeIntrospectionQueuesBytes(ctx context.Context, now s.mb.RecordSplunkServerIntrospectionQueuesCurrentBytesDataPoint(now, currentQueueSizeBytes, name) } } + +// Scrape introspection kv store status +func (s *splunkScraper) scrapeKVStoreStatus(ctx context.Context, now pcommon.Timestamp, errs chan error) { + if !s.conf.MetricsBuilderConfig.Metrics.SplunkKvstoreStatus.Enabled || + !s.conf.MetricsBuilderConfig.Metrics.SplunkKvstoreReplicationStatus.Enabled || + !s.conf.MetricsBuilderConfig.Metrics.SplunkKvstoreBackupStatus.Enabled || + !s.splunkClient.isConfigured(typeCm) { + return + } + + ctx = context.WithValue(ctx, endpointType("type"), typeCm) + var kvs KVStoreStatus + + ept := apiDict[`SplunkKVStoreStatus`] + + req, err := s.splunkClient.createAPIRequest(ctx, ept) + if err != nil { + errs <- err + return + } + + res, err := s.splunkClient.makeRequest(req) + if err != nil { + errs <- err + return + } + defer res.Body.Close() + + if err := json.NewDecoder(res.Body).Decode(&kvs); err != nil { + errs <- err + return + } + + var st, brs, rs, se, ext string + for _, kv := range kvs.Entries { + st = kv.Content.Current.Status // overall status + brs = kv.Content.Current.BackupRestoreStatus + rs = kv.Content.Current.ReplicationStatus + se = kv.Content.Current.StorageEngine + ext = kv.Content.KVService.Status + + // a 0 gauge value means that the metric was not reported in the api call + // to the introspection endpoint. + if st == "" { + st = KVStatusUnknown + // set to 0 to indicate no status being reported + s.mb.RecordSplunkKvstoreStatusDataPoint(now, 0, se, ext, st) + } else { + s.mb.RecordSplunkKvstoreStatusDataPoint(now, 1, se, ext, st) + } + + if rs == "" { + rs = KVRestoreStatusUnknown + s.mb.RecordSplunkKvstoreReplicationStatusDataPoint(now, 0, rs) + } else { + s.mb.RecordSplunkKvstoreReplicationStatusDataPoint(now, 1, rs) + } + + if brs == "" { + brs = KVBackupStatusFailed + s.mb.RecordSplunkKvstoreBackupStatusDataPoint(now, 0, brs) + } else { + s.mb.RecordSplunkKvstoreBackupStatusDataPoint(now, 1, brs) + } + } +} diff --git a/receiver/splunkenterprisereceiver/search_result.go b/receiver/splunkenterprisereceiver/search_result.go index 03f5d2a3f45a..2a9146411ef4 100644 --- a/receiver/splunkenterprisereceiver/search_result.go +++ b/receiver/splunkenterprisereceiver/search_result.go @@ -23,6 +23,7 @@ var apiDict = map[string]string{ `SplunkIndexerThroughput`: `/services/server/introspection/indexer?output_mode=json`, `SplunkDataIndexesExtended`: `/services/data/indexes-extended?output_mode=json&count=-1`, `SplunkIntrospectionQueues`: `/services/server/introspection/queues?output_mode=json&count=-1`, + `SplunkKVStoreStatus`: `/services/kvstore/status?output_mode=json`, } type searchResponse struct { @@ -101,3 +102,35 @@ type IdxQContent struct { LargestSize int `json:"largest_size"` MaxSizeBytes int `json:"max_size_bytes"` } + +// '/services/kvstore/status' +const ( + // unknown/failed values + KVStatusUnknown = "unknown" + KVRestoreStatusUnknown = "Unknown status" + KVBackupStatusFailed = "Failed" +) + +type KVStoreStatus struct { + Entries []KVEntry `json:"entry"` +} + +type KVEntry struct { + Content KVStatus `json:"content"` +} + +type KVStatus struct { + Current KVStoreCurrent `json:"current"` + KVService KVService `json:"externalKVStore,omitempty"` +} + +type KVService struct { + Status string `json:"status"` +} + +type KVStoreCurrent struct { + Status string `json:"status"` + BackupRestoreStatus string `json:"backupRestoreStatus"` + ReplicationStatus string `json:"replicationStatus"` + StorageEngine string `json:"storageEngine"` +}