From b8f87dd72fdfbaf6d2d229fdce08c68b5f7fcb43 Mon Sep 17 00:00:00 2001 From: Alex Kats <56042997+akats7@users.noreply.github.com> Date: Wed, 21 Jun 2023 14:12:13 -0400 Subject: [PATCH] [receiver/zookeeper] Add support for ruok command (#22726) **Description:** Add zookeeper ruok health check metric based on response from ruok 4lw command **NOTE**: preliminary PR not ready to be merged **Link to tracking Issue:** resolves #21481 **Testing:**: Modified existing unit tests to account for additional metric from additional command. Made minor change to test structure to account for separate text responses from separate commands. Added unit test to handle following cases, valid "imok" return from server, invalid response from server, and empty response from server. --- .chloggen/support-ruok-4lw-cmd.yaml | 20 +++ receiver/zookeeperreceiver/documentation.md | 8 + .../zookeeperreceiver/integration_test.go | 2 +- .../internal/metadata/generated_config.go | 4 + .../metadata/generated_config_test.go | 2 + .../internal/metadata/generated_metrics.go | 57 +++++++ .../metadata/generated_metrics_test.go | 16 ++ .../internal/metadata/testdata/config.yaml | 4 + receiver/zookeeperreceiver/metadata.yaml | 6 + receiver/zookeeperreceiver/metrics.go | 4 + receiver/zookeeperreceiver/scraper.go | 43 ++++- receiver/zookeeperreceiver/scraper_test.go | 149 +++++++++++++----- .../testdata/integration/expected-3.4.13.yaml | 8 + .../expected-3.5.10-standalone.yaml | 8 + .../testdata/integration/expected-3.5.10.yaml | 8 + .../zookeeperreceiver/testdata/ruok-invalid | 1 + receiver/zookeeperreceiver/testdata/ruok-null | 0 .../zookeeperreceiver/testdata/ruok-valid | 1 + .../testdata/scraper/correctness-ruok.yaml | 11 ++ .../testdata/scraper/correctness-v3.4.14.yaml | 8 + .../testdata/scraper/correctness-v3.5.5.yaml | 8 + .../testdata/scraper/disable-watches.yaml | 8 + .../scraper/error-closing-connection.yaml | 8 + .../error-setting-connection-deadline.yaml | 8 + .../testdata/scraper/invalid-ruok.yaml | 140 ++++++++++++++++ .../testdata/scraper/null-ruok.yaml | 148 +++++++++++++++++ 26 files changed, 636 insertions(+), 44 deletions(-) create mode 100644 .chloggen/support-ruok-4lw-cmd.yaml create mode 100644 receiver/zookeeperreceiver/testdata/ruok-invalid create mode 100644 receiver/zookeeperreceiver/testdata/ruok-null create mode 100644 receiver/zookeeperreceiver/testdata/ruok-valid create mode 100644 receiver/zookeeperreceiver/testdata/scraper/correctness-ruok.yaml create mode 100644 receiver/zookeeperreceiver/testdata/scraper/invalid-ruok.yaml create mode 100644 receiver/zookeeperreceiver/testdata/scraper/null-ruok.yaml diff --git a/.chloggen/support-ruok-4lw-cmd.yaml b/.chloggen/support-ruok-4lw-cmd.yaml new file mode 100644 index 000000000000..f3fda595d966 --- /dev/null +++ b/.chloggen/support-ruok-4lw-cmd.yaml @@ -0,0 +1,20 @@ +# Use this changelog template to create an entry for release notes. +# If your change doesn't affect end users, such as a test fix or a tooling change, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: zookeeperreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Adds an additional health check metric based off of the response from the zookeeper ruok 4lw command. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [21481] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: \ No newline at end of file diff --git a/receiver/zookeeperreceiver/documentation.md b/receiver/zookeeperreceiver/documentation.md index 824a4da9cdea..6574de703e1a 100644 --- a/receiver/zookeeperreceiver/documentation.md +++ b/receiver/zookeeperreceiver/documentation.md @@ -120,6 +120,14 @@ Number of currently executing requests. | ---- | ----------- | ---------- | ----------------------- | --------- | | {requests} | Sum | Int | Cumulative | false | +### zookeeper.ruok + +Response from zookeeper ruok command + +| Unit | Metric Type | Value Type | +| ---- | ----------- | ---------- | +| 1 | Gauge | Int | + ### zookeeper.sync.pending The number of pending syncs from the followers. Only exposed by the leader. diff --git a/receiver/zookeeperreceiver/integration_test.go b/receiver/zookeeperreceiver/integration_test.go index 6170ae90448f..c368a52a0acd 100644 --- a/receiver/zookeeperreceiver/integration_test.go +++ b/receiver/zookeeperreceiver/integration_test.go @@ -34,7 +34,7 @@ func integrationTest(name string, image string, standalone bool) func(*testing.T testcontainers.ContainerRequest{ Image: image, Env: map[string]string{ - "ZOO_4LW_COMMANDS_WHITELIST": "srvr,mntr", + "ZOO_4LW_COMMANDS_WHITELIST": "srvr,mntr,ruok", "ZOO_STANDALONE_ENABLED": fmt.Sprintf("%t", standalone), }, ExposedPorts: []string{zookeeperPort}, diff --git a/receiver/zookeeperreceiver/internal/metadata/generated_config.go b/receiver/zookeeperreceiver/internal/metadata/generated_config.go index 8bcfefdd1e28..8af5825235d0 100644 --- a/receiver/zookeeperreceiver/internal/metadata/generated_config.go +++ b/receiver/zookeeperreceiver/internal/metadata/generated_config.go @@ -37,6 +37,7 @@ type MetricsConfig struct { ZookeeperLatencyMin MetricConfig `mapstructure:"zookeeper.latency.min"` ZookeeperPacketCount MetricConfig `mapstructure:"zookeeper.packet.count"` ZookeeperRequestActive MetricConfig `mapstructure:"zookeeper.request.active"` + ZookeeperRuok MetricConfig `mapstructure:"zookeeper.ruok"` ZookeeperSyncPending MetricConfig `mapstructure:"zookeeper.sync.pending"` ZookeeperWatchCount MetricConfig `mapstructure:"zookeeper.watch.count"` ZookeeperZnodeCount MetricConfig `mapstructure:"zookeeper.znode.count"` @@ -80,6 +81,9 @@ func DefaultMetricsConfig() MetricsConfig { ZookeeperRequestActive: MetricConfig{ Enabled: true, }, + ZookeeperRuok: MetricConfig{ + Enabled: true, + }, ZookeeperSyncPending: MetricConfig{ Enabled: true, }, diff --git a/receiver/zookeeperreceiver/internal/metadata/generated_config_test.go b/receiver/zookeeperreceiver/internal/metadata/generated_config_test.go index 8eb2bb86d24b..aa2feb6357b5 100644 --- a/receiver/zookeeperreceiver/internal/metadata/generated_config_test.go +++ b/receiver/zookeeperreceiver/internal/metadata/generated_config_test.go @@ -38,6 +38,7 @@ func TestMetricsBuilderConfig(t *testing.T) { ZookeeperLatencyMin: MetricConfig{Enabled: true}, ZookeeperPacketCount: MetricConfig{Enabled: true}, ZookeeperRequestActive: MetricConfig{Enabled: true}, + ZookeeperRuok: MetricConfig{Enabled: true}, ZookeeperSyncPending: MetricConfig{Enabled: true}, ZookeeperWatchCount: MetricConfig{Enabled: true}, ZookeeperZnodeCount: MetricConfig{Enabled: true}, @@ -64,6 +65,7 @@ func TestMetricsBuilderConfig(t *testing.T) { ZookeeperLatencyMin: MetricConfig{Enabled: false}, ZookeeperPacketCount: MetricConfig{Enabled: false}, ZookeeperRequestActive: MetricConfig{Enabled: false}, + ZookeeperRuok: MetricConfig{Enabled: false}, ZookeeperSyncPending: MetricConfig{Enabled: false}, ZookeeperWatchCount: MetricConfig{Enabled: false}, ZookeeperZnodeCount: MetricConfig{Enabled: false}, diff --git a/receiver/zookeeperreceiver/internal/metadata/generated_metrics.go b/receiver/zookeeperreceiver/internal/metadata/generated_metrics.go index 9c92d805dad4..521e75f6e61a 100644 --- a/receiver/zookeeperreceiver/internal/metadata/generated_metrics.go +++ b/receiver/zookeeperreceiver/internal/metadata/generated_metrics.go @@ -671,6 +671,55 @@ func newMetricZookeeperRequestActive(cfg MetricConfig) metricZookeeperRequestAct return m } +type metricZookeeperRuok struct { + data pmetric.Metric // data buffer for generated metric. + config MetricConfig // metric config provided by user. + capacity int // max observed number of data points added to the metric. +} + +// init fills zookeeper.ruok metric with initial data. +func (m *metricZookeeperRuok) init() { + m.data.SetName("zookeeper.ruok") + m.data.SetDescription("Response from zookeeper ruok command") + m.data.SetUnit("1") + m.data.SetEmptyGauge() +} + +func (m *metricZookeeperRuok) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val int64) { + if !m.config.Enabled { + return + } + dp := m.data.Gauge().DataPoints().AppendEmpty() + dp.SetStartTimestamp(start) + dp.SetTimestamp(ts) + dp.SetIntValue(val) +} + +// updateCapacity saves max length of data point slices that will be used for the slice capacity. +func (m *metricZookeeperRuok) updateCapacity() { + if m.data.Gauge().DataPoints().Len() > m.capacity { + m.capacity = m.data.Gauge().DataPoints().Len() + } +} + +// emit appends recorded metric data to a metrics slice and prepares it for recording another set of data points. +func (m *metricZookeeperRuok) emit(metrics pmetric.MetricSlice) { + if m.config.Enabled && m.data.Gauge().DataPoints().Len() > 0 { + m.updateCapacity() + m.data.MoveTo(metrics.AppendEmpty()) + m.init() + } +} + +func newMetricZookeeperRuok(cfg MetricConfig) metricZookeeperRuok { + m := metricZookeeperRuok{config: cfg} + if cfg.Enabled { + m.data = pmetric.NewMetric() + m.init() + } + return m +} + type metricZookeeperSyncPending struct { data pmetric.Metric // data buffer for generated metric. config MetricConfig // metric config provided by user. @@ -845,6 +894,7 @@ type MetricsBuilder struct { metricZookeeperLatencyMin metricZookeeperLatencyMin metricZookeeperPacketCount metricZookeeperPacketCount metricZookeeperRequestActive metricZookeeperRequestActive + metricZookeeperRuok metricZookeeperRuok metricZookeeperSyncPending metricZookeeperSyncPending metricZookeeperWatchCount metricZookeeperWatchCount metricZookeeperZnodeCount metricZookeeperZnodeCount @@ -878,6 +928,7 @@ func NewMetricsBuilder(mbc MetricsBuilderConfig, settings receiver.CreateSetting metricZookeeperLatencyMin: newMetricZookeeperLatencyMin(mbc.Metrics.ZookeeperLatencyMin), metricZookeeperPacketCount: newMetricZookeeperPacketCount(mbc.Metrics.ZookeeperPacketCount), metricZookeeperRequestActive: newMetricZookeeperRequestActive(mbc.Metrics.ZookeeperRequestActive), + metricZookeeperRuok: newMetricZookeeperRuok(mbc.Metrics.ZookeeperRuok), metricZookeeperSyncPending: newMetricZookeeperSyncPending(mbc.Metrics.ZookeeperSyncPending), metricZookeeperWatchCount: newMetricZookeeperWatchCount(mbc.Metrics.ZookeeperWatchCount), metricZookeeperZnodeCount: newMetricZookeeperZnodeCount(mbc.Metrics.ZookeeperZnodeCount), @@ -963,6 +1014,7 @@ func (mb *MetricsBuilder) EmitForResource(rmo ...ResourceMetricsOption) { mb.metricZookeeperLatencyMin.emit(ils.Metrics()) mb.metricZookeeperPacketCount.emit(ils.Metrics()) mb.metricZookeeperRequestActive.emit(ils.Metrics()) + mb.metricZookeeperRuok.emit(ils.Metrics()) mb.metricZookeeperSyncPending.emit(ils.Metrics()) mb.metricZookeeperWatchCount.emit(ils.Metrics()) mb.metricZookeeperZnodeCount.emit(ils.Metrics()) @@ -1046,6 +1098,11 @@ func (mb *MetricsBuilder) RecordZookeeperRequestActiveDataPoint(ts pcommon.Times mb.metricZookeeperRequestActive.recordDataPoint(mb.startTime, ts, val) } +// RecordZookeeperRuokDataPoint adds a data point to zookeeper.ruok metric. +func (mb *MetricsBuilder) RecordZookeeperRuokDataPoint(ts pcommon.Timestamp, val int64) { + mb.metricZookeeperRuok.recordDataPoint(mb.startTime, ts, val) +} + // RecordZookeeperSyncPendingDataPoint adds a data point to zookeeper.sync.pending metric. func (mb *MetricsBuilder) RecordZookeeperSyncPendingDataPoint(ts pcommon.Timestamp, val int64) { mb.metricZookeeperSyncPending.recordDataPoint(mb.startTime, ts, val) diff --git a/receiver/zookeeperreceiver/internal/metadata/generated_metrics_test.go b/receiver/zookeeperreceiver/internal/metadata/generated_metrics_test.go index ee39040f65e0..e7de0ac3aa71 100644 --- a/receiver/zookeeperreceiver/internal/metadata/generated_metrics_test.go +++ b/receiver/zookeeperreceiver/internal/metadata/generated_metrics_test.go @@ -102,6 +102,10 @@ func TestMetricsBuilder(t *testing.T) { allMetricsCount++ mb.RecordZookeeperRequestActiveDataPoint(ts, 1) + defaultMetricsCount++ + allMetricsCount++ + mb.RecordZookeeperRuokDataPoint(ts, 1) + defaultMetricsCount++ allMetricsCount++ mb.RecordZookeeperSyncPendingDataPoint(ts, 1) @@ -319,6 +323,18 @@ func TestMetricsBuilder(t *testing.T) { assert.Equal(t, ts, dp.Timestamp()) assert.Equal(t, pmetric.NumberDataPointValueTypeInt, dp.ValueType()) assert.Equal(t, int64(1), dp.IntValue()) + case "zookeeper.ruok": + assert.False(t, validatedMetrics["zookeeper.ruok"], "Found a duplicate in the metrics slice: zookeeper.ruok") + validatedMetrics["zookeeper.ruok"] = true + assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) + assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) + assert.Equal(t, "Response from zookeeper ruok command", ms.At(i).Description()) + assert.Equal(t, "1", ms.At(i).Unit()) + dp := ms.At(i).Gauge().DataPoints().At(0) + assert.Equal(t, start, dp.StartTimestamp()) + assert.Equal(t, ts, dp.Timestamp()) + assert.Equal(t, pmetric.NumberDataPointValueTypeInt, dp.ValueType()) + assert.Equal(t, int64(1), dp.IntValue()) case "zookeeper.sync.pending": assert.False(t, validatedMetrics["zookeeper.sync.pending"], "Found a duplicate in the metrics slice: zookeeper.sync.pending") validatedMetrics["zookeeper.sync.pending"] = true diff --git a/receiver/zookeeperreceiver/internal/metadata/testdata/config.yaml b/receiver/zookeeperreceiver/internal/metadata/testdata/config.yaml index e44569003730..feccb696418e 100644 --- a/receiver/zookeeperreceiver/internal/metadata/testdata/config.yaml +++ b/receiver/zookeeperreceiver/internal/metadata/testdata/config.yaml @@ -25,6 +25,8 @@ all_set: enabled: true zookeeper.request.active: enabled: true + zookeeper.ruok: + enabled: true zookeeper.sync.pending: enabled: true zookeeper.watch.count: @@ -62,6 +64,8 @@ none_set: enabled: false zookeeper.request.active: enabled: false + zookeeper.ruok: + enabled: false zookeeper.sync.pending: enabled: false zookeeper.watch.count: diff --git a/receiver/zookeeperreceiver/metadata.yaml b/receiver/zookeeperreceiver/metadata.yaml index 3df61cd34ef7..81e89fd86fd6 100644 --- a/receiver/zookeeperreceiver/metadata.yaml +++ b/receiver/zookeeperreceiver/metadata.yaml @@ -145,3 +145,9 @@ metrics: value_type: int monotonic: true aggregation: cumulative + zookeeper.ruok: + enabled: true + description: Response from zookeeper ruok command + unit: 1 + gauge: + value_type: int diff --git a/receiver/zookeeperreceiver/metrics.go b/receiver/zookeeperreceiver/metrics.go index ef39fd23f8c0..d3e22a6eb10c 100644 --- a/receiver/zookeeperreceiver/metrics.go +++ b/receiver/zookeeperreceiver/metrics.go @@ -36,6 +36,8 @@ const ( serverStateKey = "zk_server_state" zkVersionKey = "zk_version" + + ruokKey = "ruok" ) // metricCreator handles generation of metric and metric recording @@ -89,6 +91,8 @@ func (m *metricCreator) recordDataPointsFunc(metric string) func(ts pcommon.Time return m.mb.RecordZookeeperFileDescriptorLimitDataPoint case fSyncThresholdExceedCountMetricKey: return m.mb.RecordZookeeperFsyncExceededThresholdCountDataPoint + case ruokKey: + return m.mb.RecordZookeeperRuokDataPoint case packetsReceivedMetricKey: return func(ts pcommon.Timestamp, val int64) { m.mb.RecordZookeeperPacketCountDataPoint(ts, val, metadata.AttributeDirectionReceived) diff --git a/receiver/zookeeperreceiver/scraper.go b/receiver/zookeeperreceiver/scraper.go index 06c886f0fbdd..276d35b8448b 100644 --- a/receiver/zookeeperreceiver/scraper.go +++ b/receiver/zookeeperreceiver/scraper.go @@ -25,6 +25,7 @@ var zookeeperFormatRE = regexp.MustCompile(`(^zk_\w+)\s+([\w\.\-]+)`) const ( mntrCommand = "mntr" + ruokCommand = "ruok" ) type zookeeperMetricsScraper struct { @@ -77,16 +78,27 @@ func (z *zookeeperMetricsScraper) scrape(ctx context.Context) (pmetric.Metrics, var ctxWithTimeout context.Context ctxWithTimeout, z.cancel = context.WithTimeout(ctx, z.config.Timeout) - response, err := z.runCommand(ctxWithTimeout, "mntr") + responseMntr, err := z.runCommand(ctxWithTimeout, "mntr") if err != nil { return pmetric.NewMetrics(), err } - return z.processMntr(response) + responseRuok, err := z.runCommand(ctxWithTimeout, "ruok") + if err != nil { + return pmetric.NewMetrics(), err + } + + resourceOpts := make([]metadata.ResourceMetricsOption, 0, 2) + + resourceOpts = z.processMntr(responseMntr, resourceOpts) + z.processRuok(responseRuok) + + return z.mb.Emit(resourceOpts...), nil } func (z *zookeeperMetricsScraper) runCommand(ctx context.Context, command string) ([]string, error) { conn, err := z.config.Dial() + if err != nil { z.logger.Error("failed to establish connection", zap.String("endpoint", z.config.Endpoint), @@ -123,10 +135,9 @@ func (z *zookeeperMetricsScraper) runCommand(ctx context.Context, command string return response, nil } -func (z *zookeeperMetricsScraper) processMntr(response []string) (pmetric.Metrics, error) { +func (z *zookeeperMetricsScraper) processMntr(response []string, resourceOpts []metadata.ResourceMetricsOption) []metadata.ResourceMetricsOption { creator := newMetricCreator(z.mb) now := pcommon.NewTimestampFromTime(time.Now()) - resourceOpts := make([]metadata.ResourceMetricsOption, 0, 2) for _, line := range response { parts := zookeeperFormatRE.FindStringSubmatch(line) if len(parts) != 3 { @@ -167,7 +178,29 @@ func (z *zookeeperMetricsScraper) processMntr(response []string) (pmetric.Metric // Generate computed metrics creator.generateComputedMetrics(z.logger, now) - return z.mb.Emit(resourceOpts...), nil + return resourceOpts +} + +func (z *zookeeperMetricsScraper) processRuok(response []string) { + creator := newMetricCreator(z.mb) + now := pcommon.NewTimestampFromTime(time.Now()) + + metricKey := "ruok" + metricValue := int64(0) + + if len(response) > 0 { + if response[0] == "imok" { + metricValue = int64(1) + } else { + z.logger.Error("invalid response from ruok", + zap.String("command", ruokCommand), + ) + return + } + } + + recordDataPoints := creator.recordDataPointsFunc(metricKey) + recordDataPoints(now, metricValue) } func closeConnection(conn net.Conn) error { diff --git a/receiver/zookeeperreceiver/scraper_test.go b/receiver/zookeeperreceiver/scraper_test.go index d7b699dd8b63..794c418fe542 100644 --- a/receiver/zookeeperreceiver/scraper_test.go +++ b/receiver/zookeeperreceiver/scraper_test.go @@ -39,23 +39,26 @@ func TestZookeeperMetricsScraperScrape(t *testing.T) { } tests := []struct { - name string - expectedMetricsFilename string - expectedResourceAttributes map[string]string - metricsConfig func() metadata.MetricsConfig - mockedZKOutputSourceFilename string - mockZKConnectionErr bool - expectedLogs []logMsg - expectedNumResourceMetrics int - setConnectionDeadline func(net.Conn, time.Time) error - closeConnection func(net.Conn) error - sendCmd func(net.Conn, string) (*bufio.Scanner, error) - wantErr bool + name string + expectedMetricsFilename string + expectedResourceAttributes map[string]string + metricsConfig func() metadata.MetricsConfig + mockedZKCmdToOutputFilename map[string]string + mockZKConnectionErr bool + expectedLogs []logMsg + expectedNumResourceMetrics int + setConnectionDeadline func(net.Conn, time.Time) error + closeConnection func(net.Conn) error + sendCmd func(net.Conn, string) (*bufio.Scanner, error) + wantErr bool }{ { - name: "Test correctness with v3.4.14", - mockedZKOutputSourceFilename: "mntr-3.4.14", - expectedMetricsFilename: "correctness-v3.4.14", + name: "Test correctness with v3.4.14", + mockedZKCmdToOutputFilename: map[string]string{ + "mntr": "mntr-3.4.14", + "ruok": "ruok-valid", + }, + expectedMetricsFilename: "correctness-v3.4.14", expectedResourceAttributes: map[string]string{ "server.state": "standalone", "zk.version": "3.4.14-4c25d480e66aadd371de8bd2fd8da255ac140bcf", @@ -69,9 +72,12 @@ func TestZookeeperMetricsScraperScrape(t *testing.T) { expectedNumResourceMetrics: 1, }, { - name: "Test correctness with v3.5.5", - mockedZKOutputSourceFilename: "mntr-3.5.5", - expectedMetricsFilename: "correctness-v3.5.5", + name: "Test correctness with v3.5.5", + mockedZKCmdToOutputFilename: map[string]string{ + "mntr": "mntr-3.5.5", + "ruok": "ruok-valid", + }, + expectedMetricsFilename: "correctness-v3.5.5", expectedResourceAttributes: map[string]string{ "server.state": "leader", "zk.version": "3.5.5-390fe37ea45dee01bf87dc1c042b5e3dcce88653", @@ -90,8 +96,11 @@ func TestZookeeperMetricsScraperScrape(t *testing.T) { wantErr: true, }, { - name: "Unexpected line format in mntr", - mockedZKOutputSourceFilename: "mntr-unexpected_line_format", + name: "Unexpected line format in mntr", + mockedZKCmdToOutputFilename: map[string]string{ + "mntr": "mntr-unexpected_line_format", + "ruok": "ruok-valid", + }, expectedLogs: []logMsg{ { msg: "unexpected line in response", @@ -105,8 +114,11 @@ func TestZookeeperMetricsScraperScrape(t *testing.T) { expectedNumResourceMetrics: 0, }, { - name: "Unexpected value type in mntr", - mockedZKOutputSourceFilename: "mntr-unexpected_value_type", + name: "Unexpected value type in mntr", + mockedZKCmdToOutputFilename: map[string]string{ + "mntr": "mntr-unexpected_value_type", + "ruok": "ruok-valid", + }, expectedLogs: []logMsg{ { msg: "non-integer value from mntr", @@ -120,9 +132,50 @@ func TestZookeeperMetricsScraperScrape(t *testing.T) { expectedNumResourceMetrics: 0, }, { - name: "Error setting connection deadline", - mockedZKOutputSourceFilename: "mntr-3.4.14", + name: "Empty response from ruok", + mockedZKCmdToOutputFilename: map[string]string{ + "mntr": "mntr-3.4.14", + "ruok": "ruok-null", + }, + expectedMetricsFilename: "null-ruok", + expectedLogs: []logMsg{ + { + msg: "metric computation failed", + level: zapcore.DebugLevel, + }, + }, + expectedNumResourceMetrics: 2, + }, + { + name: "Invalid response from ruok", + mockedZKCmdToOutputFilename: map[string]string{ + "mntr": "mntr-3.4.14", + "ruok": "ruok-invalid", + }, + expectedMetricsFilename: "invalid-ruok", + expectedLogs: []logMsg{ + { + msg: "metric computation failed", + level: zapcore.DebugLevel, + }, + { + msg: "invalid response from ruok", + level: zapcore.ErrorLevel, + }, + }, + expectedNumResourceMetrics: 2, + }, + { + name: "Error setting connection deadline", + mockedZKCmdToOutputFilename: map[string]string{ + "mntr": "mntr-3.4.14", + "ruok": "ruok-valid", + }, expectedLogs: []logMsg{ + { + msg: "failed to set deadline on connection", + level: zapcore.WarnLevel, + }, { msg: "failed to set deadline on connection", level: zapcore.WarnLevel, @@ -143,9 +196,16 @@ func TestZookeeperMetricsScraperScrape(t *testing.T) { }, }, { - name: "Error closing connection", - mockedZKOutputSourceFilename: "mntr-3.4.14", + name: "Error closing connection", + mockedZKCmdToOutputFilename: map[string]string{ + "mntr": "mntr-3.4.14", + "ruok": "ruok-valid", + }, expectedLogs: []logMsg{ + { + msg: "failed to shutdown connection", + level: zapcore.WarnLevel, + }, { msg: "failed to shutdown connection", level: zapcore.WarnLevel, @@ -166,8 +226,11 @@ func TestZookeeperMetricsScraperScrape(t *testing.T) { }, }, { - name: "Failed to send command", - mockedZKOutputSourceFilename: "mntr-3.4.14", + name: "Failed to send command", + mockedZKCmdToOutputFilename: map[string]string{ + "mntr": "mntr-3.4.14", + "ruok": "ruok-valid", + }, expectedLogs: []logMsg{ { msg: "failed to send command", @@ -185,8 +248,11 @@ func TestZookeeperMetricsScraperScrape(t *testing.T) { ms.ZookeeperWatchCount.Enabled = false return ms }, - mockedZKOutputSourceFilename: "mntr-3.4.14", - expectedMetricsFilename: "disable-watches", + mockedZKCmdToOutputFilename: map[string]string{ + "mntr": "mntr-3.4.14", + "ruok": "ruok-valid", + }, + expectedMetricsFilename: "disable-watches", expectedResourceAttributes: map[string]string{ "server.state": "standalone", "zk.version": "3.4.14-4c25d480e66aadd371de8bd2fd8da255ac140bcf", @@ -205,7 +271,7 @@ func TestZookeeperMetricsScraperScrape(t *testing.T) { localAddr := testutil.GetAvailableLocalAddress(t) if !tt.mockZKConnectionErr { ms := mockedServer{ready: make(chan bool, 1)} - go ms.mockZKServer(t, localAddr, tt.mockedZKOutputSourceFilename) + go ms.mockZKServer(t, localAddr, tt.mockedZKCmdToOutputFilename) <-ms.ready } @@ -250,7 +316,6 @@ func TestZookeeperMetricsScraperScrape(t *testing.T) { require.Error(t, err) require.Equal(t, pmetric.NewMetrics(), actualMetrics) } - require.NoError(t, z.shutdown(ctx)) return } @@ -276,17 +341,25 @@ type mockedServer struct { ready chan bool } -func (ms *mockedServer) mockZKServer(t *testing.T, endpoint string, filename string) { +func (ms *mockedServer) mockZKServer(t *testing.T, endpoint string, cmdToFileMap map[string]string) { + var cmd string listener, err := net.Listen("tcp", endpoint) require.NoError(t, err) defer listener.Close() - ms.ready <- true - conn, err := listener.Accept() - require.NoError(t, err) - for { + conn, err := listener.Accept() + require.NoError(t, err) + reader := bufio.NewReader(conn) + scanner := bufio.NewScanner(reader) + scanner.Scan() + if cmd = scanner.Text(); cmd == "" { + continue + } + + require.NoError(t, err) + filename := cmdToFileMap[cmd] out, err := os.ReadFile(filepath.Join("testdata", filename)) require.NoError(t, err) @@ -294,6 +367,6 @@ func (ms *mockedServer) mockZKServer(t *testing.T, endpoint string, filename str require.NoError(t, err) conn.Close() - return + } } diff --git a/receiver/zookeeperreceiver/testdata/integration/expected-3.4.13.yaml b/receiver/zookeeperreceiver/testdata/integration/expected-3.4.13.yaml index 98f18098dd5d..4cce07acba52 100644 --- a/receiver/zookeeperreceiver/testdata/integration/expected-3.4.13.yaml +++ b/receiver/zookeeperreceiver/testdata/integration/expected-3.4.13.yaml @@ -117,6 +117,14 @@ resourceMetrics: startTimeUnixNano: "1684779353305084000" timeUnixNano: "1684779363308697000" unit: '{requests}' + - description: Response from zookeeper ruok command + name: zookeeper.ruok + gauge: + dataPoints: + - asInt: "1" + startTimeUnixNano: "1642685966447704000" + timeUnixNano: "1642685966447860000" + unit: '1' - description: Number of watches placed on Z-Nodes on a ZooKeeper server. name: zookeeper.watch.count sum: diff --git a/receiver/zookeeperreceiver/testdata/integration/expected-3.5.10-standalone.yaml b/receiver/zookeeperreceiver/testdata/integration/expected-3.5.10-standalone.yaml index 0484056fe48b..e607237b5743 100644 --- a/receiver/zookeeperreceiver/testdata/integration/expected-3.5.10-standalone.yaml +++ b/receiver/zookeeperreceiver/testdata/integration/expected-3.5.10-standalone.yaml @@ -107,6 +107,14 @@ resourceMetrics: startTimeUnixNano: "1684780465963493000" timeUnixNano: "1684780475969536000" unit: '{requests}' + - description: Response from zookeeper ruok command + name: zookeeper.ruok + gauge: + dataPoints: + - asInt: "1" + startTimeUnixNano: "1642685966447704000" + timeUnixNano: "1642685966447860000" + unit: '1' - description: Number of watches placed on Z-Nodes on a ZooKeeper server. name: zookeeper.watch.count sum: diff --git a/receiver/zookeeperreceiver/testdata/integration/expected-3.5.10.yaml b/receiver/zookeeperreceiver/testdata/integration/expected-3.5.10.yaml index 8088ab368286..2888189536cc 100644 --- a/receiver/zookeeperreceiver/testdata/integration/expected-3.5.10.yaml +++ b/receiver/zookeeperreceiver/testdata/integration/expected-3.5.10.yaml @@ -127,6 +127,14 @@ resourceMetrics: startTimeUnixNano: "1684779382573670000" timeUnixNano: "1684779392577522000" unit: '{requests}' + - description: Response from zookeeper ruok command + name: zookeeper.ruok + gauge: + dataPoints: + - asInt: "1" + startTimeUnixNano: "1642685966447704000" + timeUnixNano: "1642685966447860000" + unit: '1' - description: The number of pending syncs from the followers. Only exposed by the leader. name: zookeeper.sync.pending sum: diff --git a/receiver/zookeeperreceiver/testdata/ruok-invalid b/receiver/zookeeperreceiver/testdata/ruok-invalid new file mode 100644 index 000000000000..3907c4e53515 --- /dev/null +++ b/receiver/zookeeperreceiver/testdata/ruok-invalid @@ -0,0 +1 @@ +invalid ruok \ No newline at end of file diff --git a/receiver/zookeeperreceiver/testdata/ruok-null b/receiver/zookeeperreceiver/testdata/ruok-null new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/receiver/zookeeperreceiver/testdata/ruok-valid b/receiver/zookeeperreceiver/testdata/ruok-valid new file mode 100644 index 000000000000..cccf84e14345 --- /dev/null +++ b/receiver/zookeeperreceiver/testdata/ruok-valid @@ -0,0 +1 @@ +imok \ No newline at end of file diff --git a/receiver/zookeeperreceiver/testdata/scraper/correctness-ruok.yaml b/receiver/zookeeperreceiver/testdata/scraper/correctness-ruok.yaml new file mode 100644 index 000000000000..e612c1dc075b --- /dev/null +++ b/receiver/zookeeperreceiver/testdata/scraper/correctness-ruok.yaml @@ -0,0 +1,11 @@ +resourceMetrics: + scopeMetrics: + - metrics: + - description: Response from zookeeper ruok command + name: zookeeper.ruok + gauge: + dataPoints: + - asInt: "1" + startTimeUnixNano: "1642685966447704000" + timeUnixNano: "1642685966447860000" + unit: 1 diff --git a/receiver/zookeeperreceiver/testdata/scraper/correctness-v3.4.14.yaml b/receiver/zookeeperreceiver/testdata/scraper/correctness-v3.4.14.yaml index eccff3029dda..03d616cfc8b2 100644 --- a/receiver/zookeeperreceiver/testdata/scraper/correctness-v3.4.14.yaml +++ b/receiver/zookeeperreceiver/testdata/scraper/correctness-v3.4.14.yaml @@ -117,6 +117,14 @@ resourceMetrics: startTimeUnixNano: "1642685966444471000" timeUnixNano: "1642685966444739000" unit: '{requests}' + - description: Response from zookeeper ruok command + name: zookeeper.ruok + gauge: + dataPoints: + - asInt: "1" + startTimeUnixNano: "1642685966447704000" + timeUnixNano: "1642685966447860000" + unit: '1' - description: Number of watches placed on Z-Nodes on a ZooKeeper server. name: zookeeper.watch.count sum: diff --git a/receiver/zookeeperreceiver/testdata/scraper/correctness-v3.5.5.yaml b/receiver/zookeeperreceiver/testdata/scraper/correctness-v3.5.5.yaml index c6a23a28b554..c75ff35f8ef2 100644 --- a/receiver/zookeeperreceiver/testdata/scraper/correctness-v3.5.5.yaml +++ b/receiver/zookeeperreceiver/testdata/scraper/correctness-v3.5.5.yaml @@ -127,6 +127,14 @@ resourceMetrics: startTimeUnixNano: "1642685966447704000" timeUnixNano: "1642685966447860000" unit: '{requests}' + - description: Response from zookeeper ruok command + name: zookeeper.ruok + gauge: + dataPoints: + - asInt: "1" + startTimeUnixNano: "1642685966447704000" + timeUnixNano: "1642685966447860000" + unit: '1' - description: The number of pending syncs from the followers. Only exposed by the leader. name: zookeeper.sync.pending sum: diff --git a/receiver/zookeeperreceiver/testdata/scraper/disable-watches.yaml b/receiver/zookeeperreceiver/testdata/scraper/disable-watches.yaml index 20d946783366..c2fa8271b4b8 100644 --- a/receiver/zookeeperreceiver/testdata/scraper/disable-watches.yaml +++ b/receiver/zookeeperreceiver/testdata/scraper/disable-watches.yaml @@ -117,6 +117,14 @@ resourceMetrics: startTimeUnixNano: "1642685966462139000" timeUnixNano: "1642685966462251000" unit: '{requests}' + - description: Response from zookeeper ruok command + name: zookeeper.ruok + gauge: + dataPoints: + - asInt: "1" + startTimeUnixNano: "1642685966447704000" + timeUnixNano: "1642685966447860000" + unit: '1' - description: Number of z-nodes that a ZooKeeper server has in its data tree. name: zookeeper.znode.count sum: diff --git a/receiver/zookeeperreceiver/testdata/scraper/error-closing-connection.yaml b/receiver/zookeeperreceiver/testdata/scraper/error-closing-connection.yaml index 35ff76994fdc..7581414ff1b4 100644 --- a/receiver/zookeeperreceiver/testdata/scraper/error-closing-connection.yaml +++ b/receiver/zookeeperreceiver/testdata/scraper/error-closing-connection.yaml @@ -117,6 +117,14 @@ resourceMetrics: startTimeUnixNano: "1642685966458919000" timeUnixNano: "1642685966459091000" unit: '{requests}' + - description: Response from zookeeper ruok command + name: zookeeper.ruok + gauge: + dataPoints: + - asInt: "1" + startTimeUnixNano: "1642685966447704000" + timeUnixNano: "1642685966447860000" + unit: '1' - description: Number of watches placed on Z-Nodes on a ZooKeeper server. name: zookeeper.watch.count sum: diff --git a/receiver/zookeeperreceiver/testdata/scraper/error-setting-connection-deadline.yaml b/receiver/zookeeperreceiver/testdata/scraper/error-setting-connection-deadline.yaml index 3289b2b5e6e1..233e9d7c10f5 100644 --- a/receiver/zookeeperreceiver/testdata/scraper/error-setting-connection-deadline.yaml +++ b/receiver/zookeeperreceiver/testdata/scraper/error-setting-connection-deadline.yaml @@ -117,6 +117,14 @@ resourceMetrics: startTimeUnixNano: "1642685966456266000" timeUnixNano: "1642685966456364000" unit: '{requests}' + - description: Response from zookeeper ruok command + name: zookeeper.ruok + gauge: + dataPoints: + - asInt: "1" + startTimeUnixNano: "1642685966447704000" + timeUnixNano: "1642685966447860000" + unit: '1' - description: Number of watches placed on Z-Nodes on a ZooKeeper server. name: zookeeper.watch.count sum: diff --git a/receiver/zookeeperreceiver/testdata/scraper/invalid-ruok.yaml b/receiver/zookeeperreceiver/testdata/scraper/invalid-ruok.yaml new file mode 100644 index 000000000000..eccff3029dda --- /dev/null +++ b/receiver/zookeeperreceiver/testdata/scraper/invalid-ruok.yaml @@ -0,0 +1,140 @@ +resourceMetrics: + - resource: + attributes: + - key: server.state + value: + stringValue: standalone + - key: zk.version + value: + stringValue: 3.4.14-4c25d480e66aadd371de8bd2fd8da255ac140bcf + scopeMetrics: + - metrics: + - description: Number of active clients connected to a ZooKeeper server. + name: zookeeper.connection.active + sum: + aggregationTemporality: 2 + dataPoints: + - asInt: "1" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + unit: '{connections}' + - description: Number of ephemeral nodes that a ZooKeeper server has in its data tree. + name: zookeeper.data_tree.ephemeral_node.count + sum: + aggregationTemporality: 2 + dataPoints: + - asInt: "0" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + unit: '{nodes}' + - description: Size of data in bytes that a ZooKeeper server has in its data tree. + name: zookeeper.data_tree.size + sum: + aggregationTemporality: 2 + dataPoints: + - asInt: "27" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + unit: By + - description: Maximum number of file descriptors that a ZooKeeper server can open. + gauge: + dataPoints: + - asInt: "1048576" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + name: zookeeper.file_descriptor.limit + unit: '{file_descriptors}' + - description: Number of file descriptors that a ZooKeeper server has open. + name: zookeeper.file_descriptor.open + sum: + aggregationTemporality: 2 + dataPoints: + - asInt: "26" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + unit: '{file_descriptors}' + - description: Number of times fsync duration has exceeded warning threshold. + name: zookeeper.fsync.exceeded_threshold.count + sum: + aggregationTemporality: 2 + dataPoints: + - asInt: "0" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + isMonotonic: true + unit: '{events}' + - description: Average time in milliseconds for requests to be processed. + gauge: + dataPoints: + - asInt: "0" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + name: zookeeper.latency.avg + unit: ms + - description: Maximum time in milliseconds for requests to be processed. + gauge: + dataPoints: + - asInt: "0" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + name: zookeeper.latency.max + unit: ms + - description: Minimum time in milliseconds for requests to be processed. + gauge: + dataPoints: + - asInt: "0" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + name: zookeeper.latency.min + unit: ms + - description: The number of ZooKeeper packets received or sent by a server. + name: zookeeper.packet.count + sum: + aggregationTemporality: 2 + dataPoints: + - asInt: "1" + attributes: + - key: direction + value: + stringValue: received + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + - asInt: "0" + attributes: + - key: direction + value: + stringValue: sent + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + isMonotonic: true + unit: '{packets}' + - description: Number of currently executing requests. + name: zookeeper.request.active + sum: + aggregationTemporality: 2 + dataPoints: + - asInt: "0" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + unit: '{requests}' + - description: Number of watches placed on Z-Nodes on a ZooKeeper server. + name: zookeeper.watch.count + sum: + aggregationTemporality: 2 + dataPoints: + - asInt: "0" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + unit: '{watches}' + - description: Number of z-nodes that a ZooKeeper server has in its data tree. + name: zookeeper.znode.count + sum: + aggregationTemporality: 2 + dataPoints: + - asInt: "4" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + unit: '{znodes}' + scope: + name: otelcol/zookeeperreceiver + version: latest diff --git a/receiver/zookeeperreceiver/testdata/scraper/null-ruok.yaml b/receiver/zookeeperreceiver/testdata/scraper/null-ruok.yaml new file mode 100644 index 000000000000..a0ec60404fa0 --- /dev/null +++ b/receiver/zookeeperreceiver/testdata/scraper/null-ruok.yaml @@ -0,0 +1,148 @@ +resourceMetrics: + - resource: + attributes: + - key: server.state + value: + stringValue: standalone + - key: zk.version + value: + stringValue: 3.4.14-4c25d480e66aadd371de8bd2fd8da255ac140bcf + scopeMetrics: + - metrics: + - description: Number of active clients connected to a ZooKeeper server. + name: zookeeper.connection.active + sum: + aggregationTemporality: 2 + dataPoints: + - asInt: "1" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + unit: '{connections}' + - description: Number of ephemeral nodes that a ZooKeeper server has in its data tree. + name: zookeeper.data_tree.ephemeral_node.count + sum: + aggregationTemporality: 2 + dataPoints: + - asInt: "0" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + unit: '{nodes}' + - description: Size of data in bytes that a ZooKeeper server has in its data tree. + name: zookeeper.data_tree.size + sum: + aggregationTemporality: 2 + dataPoints: + - asInt: "27" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + unit: By + - description: Maximum number of file descriptors that a ZooKeeper server can open. + gauge: + dataPoints: + - asInt: "1048576" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + name: zookeeper.file_descriptor.limit + unit: '{file_descriptors}' + - description: Number of file descriptors that a ZooKeeper server has open. + name: zookeeper.file_descriptor.open + sum: + aggregationTemporality: 2 + dataPoints: + - asInt: "26" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + unit: '{file_descriptors}' + - description: Number of times fsync duration has exceeded warning threshold. + name: zookeeper.fsync.exceeded_threshold.count + sum: + aggregationTemporality: 2 + dataPoints: + - asInt: "0" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + isMonotonic: true + unit: '{events}' + - description: Average time in milliseconds for requests to be processed. + gauge: + dataPoints: + - asInt: "0" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + name: zookeeper.latency.avg + unit: ms + - description: Maximum time in milliseconds for requests to be processed. + gauge: + dataPoints: + - asInt: "0" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + name: zookeeper.latency.max + unit: ms + - description: Minimum time in milliseconds for requests to be processed. + gauge: + dataPoints: + - asInt: "0" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + name: zookeeper.latency.min + unit: ms + - description: The number of ZooKeeper packets received or sent by a server. + name: zookeeper.packet.count + sum: + aggregationTemporality: 2 + dataPoints: + - asInt: "1" + attributes: + - key: direction + value: + stringValue: received + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + - asInt: "0" + attributes: + - key: direction + value: + stringValue: sent + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + isMonotonic: true + unit: '{packets}' + - description: Number of currently executing requests. + name: zookeeper.request.active + sum: + aggregationTemporality: 2 + dataPoints: + - asInt: "0" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + unit: '{requests}' + - description: Response from zookeeper ruok command + name: zookeeper.ruok + gauge: + dataPoints: + - asInt: "0" + startTimeUnixNano: "1642685966447704000" + timeUnixNano: "1642685966447860000" + unit: '1' + - description: Number of watches placed on Z-Nodes on a ZooKeeper server. + name: zookeeper.watch.count + sum: + aggregationTemporality: 2 + dataPoints: + - asInt: "0" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + unit: '{watches}' + - description: Number of z-nodes that a ZooKeeper server has in its data tree. + name: zookeeper.znode.count + sum: + aggregationTemporality: 2 + dataPoints: + - asInt: "4" + startTimeUnixNano: "1642685966444471000" + timeUnixNano: "1642685966444739000" + unit: '{znodes}' + scope: + name: otelcol/zookeeperreceiver + version: latest