[receiver/zookeeper] Add support for ruok command (open-telemetry#22726)

**Description:** <Describe what has changed.>
Add zookeeper ruok health check metric based on response from ruok 4lw
command
**NOTE**: preliminary PR not ready to be merged

**Link to tracking Issue:** resolves open-telemetry#21481 

**Testing:**: Modified existing unit tests to account for additional
metric from additional command. Made minor change to test structure to
account for separate text responses from separate commands. Added unit
test to handle following cases, valid "imok" return from server, invalid
response from server, and empty response from server.

.chloggen/support-ruok-4lw-cmd.yaml

@@ -0,0 +1,20 @@
+# Use this changelog template to create an entry for release notes.
+# If your change doesn't affect end users, such as a test fix or a tooling change,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: zookeeperreceiver
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Adds an additional health check metric based off of the response from the zookeeper ruok 4lw command.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [21481]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

receiver/zookeeperreceiver/documentation.md

@@ -120,6 +120,14 @@ Number of currently executing requests.
 | ---- | ----------- | ---------- | ----------------------- | --------- |
 | {requests} | Sum | Int | Cumulative | false |
 
+### zookeeper.ruok
+
+Response from zookeeper ruok command
+
+| Unit | Metric Type | Value Type |
+| ---- | ----------- | ---------- |
+| 1 | Gauge | Int |
+
 ### zookeeper.sync.pending
 
 The number of pending syncs from the followers. Only exposed by the leader.

receiver/zookeeperreceiver/integration_test.go

@@ -34,7 +34,7 @@ func integrationTest(name string, image string, standalone bool) func(*testing.T
 			testcontainers.ContainerRequest{
 				Image: image,
 				Env: map[string]string{
-					"ZOO_4LW_COMMANDS_WHITELIST": "srvr,mntr",
+					"ZOO_4LW_COMMANDS_WHITELIST": "srvr,mntr,ruok",
 					"ZOO_STANDALONE_ENABLED":     fmt.Sprintf("%t", standalone),
 				},
 				ExposedPorts: []string{zookeeperPort},

receiver/zookeeperreceiver/internal/metadata/testdata/config.yaml

@@ -25,6 +25,8 @@ all_set:
       enabled: true
     zookeeper.request.active:
       enabled: true
+    zookeeper.ruok:
+      enabled: true
     zookeeper.sync.pending:
       enabled: true
     zookeeper.watch.count:
@@ -62,6 +64,8 @@ none_set:
       enabled: false
     zookeeper.request.active:
       enabled: false
+    zookeeper.ruok:
+      enabled: false
     zookeeper.sync.pending:
       enabled: false
     zookeeper.watch.count:

receiver/zookeeperreceiver/metadata.yaml

@@ -145,3 +145,9 @@ metrics:
       value_type: int
       monotonic: true
       aggregation: cumulative
+  zookeeper.ruok:
+    enabled: true
+    description: Response from zookeeper ruok command
+    unit: 1
+    gauge:
+      value_type: int

receiver/zookeeperreceiver/metrics.go

@@ -36,6 +36,8 @@ const (
 
 	serverStateKey = "zk_server_state"
 	zkVersionKey   = "zk_version"
+
+	ruokKey = "ruok"
 )
 
 // metricCreator handles generation of metric and metric recording
@@ -89,6 +91,8 @@ func (m *metricCreator) recordDataPointsFunc(metric string) func(ts pcommon.Time
 		return m.mb.RecordZookeeperFileDescriptorLimitDataPoint
 	case fSyncThresholdExceedCountMetricKey:
 		return m.mb.RecordZookeeperFsyncExceededThresholdCountDataPoint
+	case ruokKey:
+		return m.mb.RecordZookeeperRuokDataPoint
 	case packetsReceivedMetricKey:
 		return func(ts pcommon.Timestamp, val int64) {
 			m.mb.RecordZookeeperPacketCountDataPoint(ts, val, metadata.AttributeDirectionReceived)

receiver/zookeeperreceiver/scraper.go

@@ -25,6 +25,7 @@ var zookeeperFormatRE = regexp.MustCompile(`(^zk_\w+)\s+([\w\.\-]+)`)
 
 const (
 	mntrCommand = "mntr"
+	ruokCommand = "ruok"
 )
 
 type zookeeperMetricsScraper struct {
@@ -77,16 +78,27 @@ func (z *zookeeperMetricsScraper) scrape(ctx context.Context) (pmetric.Metrics,
 	var ctxWithTimeout context.Context
 	ctxWithTimeout, z.cancel = context.WithTimeout(ctx, z.config.Timeout)
 
-	response, err := z.runCommand(ctxWithTimeout, "mntr")
+	responseMntr, err := z.runCommand(ctxWithTimeout, "mntr")
 	if err != nil {
 		return pmetric.NewMetrics(), err
 	}
 
-	return z.processMntr(response)
+	responseRuok, err := z.runCommand(ctxWithTimeout, "ruok")
+	if err != nil {
+		return pmetric.NewMetrics(), err
+	}
+
+	resourceOpts := make([]metadata.ResourceMetricsOption, 0, 2)
+
+	resourceOpts = z.processMntr(responseMntr, resourceOpts)
+	z.processRuok(responseRuok)
+
+	return z.mb.Emit(resourceOpts...), nil
 }
 
 func (z *zookeeperMetricsScraper) runCommand(ctx context.Context, command string) ([]string, error) {
 	conn, err := z.config.Dial()
+
 	if err != nil {
 		z.logger.Error("failed to establish connection",
 			zap.String("endpoint", z.config.Endpoint),
@@ -123,10 +135,9 @@ func (z *zookeeperMetricsScraper) runCommand(ctx context.Context, command string
 	return response, nil
 }
 
-func (z *zookeeperMetricsScraper) processMntr(response []string) (pmetric.Metrics, error) {
+func (z *zookeeperMetricsScraper) processMntr(response []string, resourceOpts []metadata.ResourceMetricsOption) []metadata.ResourceMetricsOption {
 	creator := newMetricCreator(z.mb)
 	now := pcommon.NewTimestampFromTime(time.Now())
-	resourceOpts := make([]metadata.ResourceMetricsOption, 0, 2)
 	for _, line := range response {
 		parts := zookeeperFormatRE.FindStringSubmatch(line)
 		if len(parts) != 3 {
@@ -167,7 +178,29 @@ func (z *zookeeperMetricsScraper) processMntr(response []string) (pmetric.Metric
 
 	// Generate computed metrics
 	creator.generateComputedMetrics(z.logger, now)
-	return z.mb.Emit(resourceOpts...), nil
+	return resourceOpts
+}
+
+func (z *zookeeperMetricsScraper) processRuok(response []string) {
+	creator := newMetricCreator(z.mb)
+	now := pcommon.NewTimestampFromTime(time.Now())
+
+	metricKey := "ruok"
+	metricValue := int64(0)
+
+	if len(response) > 0 {
+		if response[0] == "imok" {
+			metricValue = int64(1)
+		} else {
+			z.logger.Error("invalid response from ruok",
+				zap.String("command", ruokCommand),
+			)
+			return
+		}
+	}
+
+	recordDataPoints := creator.recordDataPointsFunc(metricKey)
+	recordDataPoints(now, metricValue)
 }
 
 func closeConnection(conn net.Conn) error {