Collect OneMineRate metrics in Kafka module (#22733)

MeanRate is rate an average rate since Kafka startup, it doesn't mean anything and doesn't shown Kafka current state. OneMinuteRate is a better representation of current state.
elastic · Dec 7, 2020 · 08a7586 · 08a7586
1 parent 02cf564
commit 08a7586
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 35 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -449,6 +449,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix failiures caused by custom beat names with more than 15 characters {pull}22550[22550]
 - Stop generating NaN values from Cloud Foundry module to avoid errors in outputs. {pull}22634[22634]
 - Update NATS dashboards to leverage connection and route metricsets {pull}22646[22646]
+- Fix rate metrics in Kafka broker metricset by using last minute rate instead of mean rate. {pull}22733[22733]
 - Fix `logstash` module when `xpack.enabled: true` is set from emitting redundant events. {pull}22808[22808]
 - Fix SQL module mapping NULL values as string {pull}18955[18955] {issue}18898[18898
 - Modify doc for app_insights metricset to contain example of config. {pull}20185[20185]

diff --git a/metricbeat/module/kafka/broker/manifest.yml b/metricbeat/module/kafka/broker/manifest.yml
@@ -11,81 +11,81 @@ input:
         attributes:
           - attr: Value
             field: request.channel.queue.size
-      - mbean: 'kafka.server:name=FailedProduceRequestsPerSec,type=BrokerTopicMetrics'
+      - mbean: 'kafka.server:type=BrokerTopicMetrics,name=FailedProduceRequestsPerSec'
         attributes:
-          - attr: MeanRate
+          - attr: OneMinuteRate
             field: request.produce.failed_per_second
-      - mbean: 'kafka.server:name=FailedFetchRequestsPerSec,type=BrokerTopicMetrics'
+      - mbean: 'kafka.server:type=BrokerTopicMetrics,name=FailedFetchRequestsPerSec'
         attributes:
-          - attr: MeanRate
+          - attr: OneMinuteRate
             field: request.fetch.failed_per_second
-      - mbean: 'kafka.server:name=FailedProduceRequestsPerSec,type=BrokerTopicMetrics'
+      - mbean: 'kafka.server:type=BrokerTopicMetrics,name=FailedProduceRequestsPerSec'
         attributes:
-          - attr: MeanRate
+          - attr: OneMinuteRate
             field: request.produce.failed
-      - mbean: 'kafka.server:name=FailedFetchRequestsPerSec,type=BrokerTopicMetrics'
+      - mbean: 'kafka.server:type=BrokerTopicMetrics,name=FailedFetchRequestsPerSec'
         attributes:
-          - attr: MeanRate
+          - attr: OneMinuteRate
             field: request.fetch.failed
-      - mbean: 'kafka.controller:name=LeaderElectionRateAndTimeMs,type=ControllerStats'
+      - mbean: 'kafka.controller:type=ControllerStats,name=LeaderElectionRateAndTimeMs'
         attributes:
-          - attr: MeanRate
+          - attr: OneMinuteRate
             field: replication.leader_elections
       - mbean: 'kafka.controller:type=ControllerStats,name=UncleanLeaderElectionsPerSec'
         attributes:
-          - attr: MeanRate
+          - attr: OneMinuteRate
             field: replication.unclean_leader_elections
-      - mbean: 'kafka.server:name=ZooKeeperDisconnectsPerSec,type=SessionExpireListener'
+      - mbean: 'kafka.server:type=SessionExpireListener,name=ZooKeeperDisconnectsPerSec'
         attributes:
-          - attr: MeanRate
+          - attr: OneMinuteRate
             field: session.zookeeper.disconnect
-      - mbean: 'kafka.server:name=ZooKeeperExpiresPerSec,type=SessionExpireListener'
+      - mbean: 'kafka.server:type=SessionExpireListener,name=ZooKeeperExpiresPerSec'
         attributes:
-          - attr: MeanRate
+          - attr: OneMinuteRate
             field: session.zookeeper.expire
-      - mbean: 'kafka.server:name=ZooKeeperReadOnlyConnectsPerSec,type=SessionExpireListener'
+      - mbean: 'kafka.server:type=SessionExpireListener,name=ZooKeeperReadOnlyConnectsPerSec'
         attributes:
-          - attr: MeanRate
+          - attr: OneMinuteRate
             field: session.zookeeper.readonly
-      - mbean: 'kafka.server:name=ZooKeeperSyncConnectsPerSec,type=SessionExpireListener'
+      - mbean: 'kafka.server:type=SessionExpireListener,name=ZooKeeperSyncConnectsPerSec'
         attributes:
-          - attr: MeanRate
+          - attr: OneMinuteRate
             field: session.zookeeper.sync
       - mbean: 'kafka.log:type=LogFlushStats,name=LogFlushRateAndTimeMs'
         attributes:
-          - attr: MeanRate
+          - attr: OneMinuteRate
             field: log.flush_rate
-      - mbean: 'kafka.server:name=BytesRejectedPerSec,topic=*,type=BrokerTopicMetrics'
+      - mbean: 'kafka.server:type=BrokerTopicMetrics,name=BytesRejectedPerSec,topic=*'
         attributes:
-          - attr: MeanRate
+          - attr: OneMinuteRate
             field: topic.net.rejected.bytes_per_sec
-      - mbean: 'kafka.server:name=BytesInPerSec,topic=*,type=BrokerTopicMetrics,topic=*'
+      - mbean: 'kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec,topic=*'
         attributes:
-          - attr: MeanRate
+          - attr: OneMinuteRate
             field: topic.net.in.bytes_per_sec
-      - mbean: 'kafka.server:name=BytesOutPerSec,topic=*,type=BrokerTopicMetrics,topic=*'
+      - mbean: 'kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec,topic=*'
         attributes:
-          - attr: MeanRate
+          - attr: OneMinuteRate
             field: topic.net.out.bytes_per_sec
-      - mbean: 'kafka.server:type=BrokerTopicMetrics,topic=*,name=MessagesInPerSec,topic=*'
+      - mbean: 'kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec,topic=*'
         attributes:
-          - attr: MeanRate
+          - attr: OneMinuteRate
             field: topic.messages_in
-      - mbean: 'kafka.server:name=BytesRejectedPerSec,type=BrokerTopicMetrics'
+      - mbean: 'kafka.server:type=BrokerTopicMetrics,name=BytesRejectedPerSec'
         attributes:
-          - attr: MeanRate
+          - attr: OneMinuteRate
             field: net.rejected.bytes_per_sec
-      - mbean: 'kafka.server:name=BytesInPerSec,type=BrokerTopicMetrics'
+      - mbean: 'kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec'
         attributes:
-          - attr: MeanRate
+          - attr: OneMinuteRate
             field: net.in.bytes_per_sec
-      - mbean: 'kafka.server:name=BytesOutPerSec,type=BrokerTopicMetrics'
+      - mbean: 'kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec'
         attributes:
-          - attr: MeanRate
+          - attr: OneMinuteRate
             field: net.out.bytes_per_sec
       - mbean: 'kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec'
         attributes:
-          - attr: MeanRate
+          - attr: OneMinuteRate
             field: messages_in