From 5a0a8b87ba82a25ef9ce2f051b5fa70b6d907eac Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 1 Oct 2018 12:26:50 +0200 Subject: [PATCH 1/5] Add kafka fields needed for dashboards Kafka dashboards calculate cardinalities of composed fields, this is expensive in query time, but can be optimized if the composed fields are precalculated. Continues with #7767, needed for #8457. --- CHANGELOG.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 18 ++++++++++++++++++ metricbeat/module/kafka/_meta/fields.yml | 10 ++++++++++ .../module/kafka/consumergroup/_meta/data.json | 6 ++++-- .../kafka/consumergroup/consumergroup.go | 9 ++++++++- metricbeat/module/kafka/fields.go | 2 +- .../module/kafka/partition/_meta/data.json | 9 ++++++--- metricbeat/module/kafka/partition/partition.go | 13 ++++++++++--- 8 files changed, 58 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 63de9a4bfa25..dc9a02888e09 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -141,6 +141,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff] - Support for Kafka 2.0.0 {pull}8399[8399] - Add container image for docker metricsets. {issue}8214[8214] {pull}8438[8438] - Add support for `full` status page output for php-fpm module as a separate metricset called `process`. {pull}8394[8394] +- Precalculate composed id fields for kafka dashboards {pull}8504[8504] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 7e88c7622868..7ddcb6758b7a 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -7118,6 +7118,24 @@ type: long Partition id. +-- + +*`kafka.partition.topic_id`*:: ++ +-- +type: keyworkd + +Unique id of the partition in the topic. + +-- + +*`kafka.partition.topic_broker_id`*:: ++ +-- +type: keyworkd + +Unique id of the partition in the topic and the broker. + -- [float] diff --git a/metricbeat/module/kafka/_meta/fields.yml b/metricbeat/module/kafka/_meta/fields.yml index 99e93eafd22a..69753db8241e 100644 --- a/metricbeat/module/kafka/_meta/fields.yml +++ b/metricbeat/module/kafka/_meta/fields.yml @@ -39,3 +39,13 @@ type: long description: > Partition id. + + - name: partition.topic_id + type: keyworkd + description: + Unique id of the partition in the topic. + + - name: partition.topic_broker_id + type: keyworkd + description: + Unique id of the partition in the topic and the broker. diff --git a/metricbeat/module/kafka/consumergroup/_meta/data.json b/metricbeat/module/kafka/consumergroup/_meta/data.json index b163e6f7475c..a63a640af593 100644 --- a/metricbeat/module/kafka/consumergroup/_meta/data.json +++ b/metricbeat/module/kafka/consumergroup/_meta/data.json @@ -17,7 +17,7 @@ "client": { "host": "172.18.0.1", "id": "sarama", - "member_id": "sarama-714cfb8b-39e5-4128-9109-e5056c5e8f56" + "member_id": "sarama-fcb5a5db-0474-4f3a-a5af-29e2f14549c5" }, "error": { "code": 0 @@ -29,7 +29,9 @@ "topic": "metricbeat-test" }, "partition": { - "id": 0 + "id": 0, + "topic_broker_id": "0-metricbeat-test-0", + "topic_id": "0-metricbeat-test" }, "topic": { "name": "metricbeat-test" diff --git a/metricbeat/module/kafka/consumergroup/consumergroup.go b/metricbeat/module/kafka/consumergroup/consumergroup.go index 713e27e5df95..ed73db6bcdde 100644 --- a/metricbeat/module/kafka/consumergroup/consumergroup.go +++ b/metricbeat/module/kafka/consumergroup/consumergroup.go @@ -19,6 +19,7 @@ package consumergroup import ( "crypto/tls" + "fmt" "github.com/pkg/errors" @@ -111,6 +112,10 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { } emitEvent := func(event common.MapStr) { + // Helpful IDs for dashboards + partitionTopicID := fmt.Sprintf("%d-%s", event["partition"], event["topic"]) + partitionTopicBrokerID := fmt.Sprintf("%s-%d", partitionTopicID, m.broker.ID()) + // TODO (deprecation): Remove fields from MetricSetFields moved to ModuleFields event["broker"] = brokerInfo r.Event(mb.Event{ @@ -120,7 +125,9 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { "name": event["topic"], }, "partition": common.MapStr{ - "id": event["partition"], + "id": event["partition"], + "topic_id": partitionTopicID, + "topic_broker_id": partitionTopicBrokerID, }, }, MetricSetFields: event, diff --git a/metricbeat/module/kafka/fields.go b/metricbeat/module/kafka/fields.go index d303ae9568c7..eb3f71fec8f8 100644 --- a/metricbeat/module/kafka/fields.go +++ b/metricbeat/module/kafka/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJy8WE2P2joU3fMrrmbFLCZv9d4iiye1M1WFWrXVtHtk4huwcGxkG6b8+8oOIYkTOx8wzar1kHPO/b7OE+zxnMKe5HuyADDMcEzh4Yv9/8MCgKLOFDsYJkUK/y8AANzfoJD0yHEBoHdSmXUmRc62KeSEa3uqkCPRmMIGjQXOGXKqUwfwBIIUWJPax5wPmMJWyePhctLD3IZpQm2U3KO6HvfhBTHL56NDgGcp9LFABZ/tq7ASuVQFsS/AjpwQNogCFBIKuZIFLC+v7YignIltC9LsELIKz0l5TBo/8G1p2sNo67iyh0uPImpSwyxGF708hFKFWveS7fH8JpUvZBwfoSdUhmmkV4pOzIw8sCyx/+7ErUsdof1lcRxmiAOVkirJJO0yeR4dpHFQYKGSLtuBKMPsu0krflOZflQwwGjSIakyys/tyRkfAvJLt3pi+dqpv7CiUtVBYUYM0hT+S/6dlWL3rFQYrtaQByBWtRCr3BGmQquCq5Mgf381w0BFT9Phc0xpXyM6ih/VlziRq+9JXKMTr9FWohKuRT++Y4/WUHeCIUfIPNdoZg2Na9KXGMCEkbVVsEEmtq6Q4gqKdreo+Q3+9oW1+Y/ayKKWYYGAEkNAG9Ws1F7a3t4+3vhurrut5Krmn9oRzdYflZRxhqI/Fv3NMKrng9ZsK5BeYF0kbETwhMLosrmFcvCGhjWyVzyXolYvsCwdp9EYK69UmzD6ONy3dlL77porpAUVJCyw2KBa324+EwaVINybHBeCZtXG2sXk0d0HMmdsR7rGnEw9EcbJhuMFV4PM3VjdshOKxnY0MUsFvmEkQeaP1m8OuOp7F7FBmQ23cfo+gr474GFBMwbRjHjW08d247+xDMUG4wjBEFyegyo5EtpZXEcoHdTx1QEDo7Asl+PONtlwlQ4rIEqR80wJzOWShQdGdZhf4YGzzJ/e9/DCa4nc74awP4Q+i2w9JGsjJUfip/tIZStBmc01DSyvHABMAxMZP1KkwISrPiaerJjqJwbthIPl6ufrKEv0eiDD3sUIK9zjDUoMLlBwh/h/uu5M5aKSo8l2dj0Y29QiF/iovptumKZz1Q9L875fwP0uIF1dY68kZaUlUz4i3X4f99g8LXM+NN2uKXhZXfwJAAD//8X6Jzc=" + return "eJy8WE1v4zYQvftXDHJyDlFP7UGHAm1SFEaL3UV292zQ4sgmTJFaknbW/35ByrIlSqQ+7ESnmJHeexwO3wz5BHs8pbAn+Z4sAAwzHFN4+M/+flgAUNSZYqVhUqTw5wIAwP0PCkkPHBcAeieVWWdS5GybQk64tqMKORKNKWzQWOCcIac6dQBPIEiBV1L7mFOJKWyVPJTnkR7mNkwTaqPkHtVluA8viFk9fzsEeJZCHwpU8K/9FFYil6og9gPYkSPCBlGAQkIhV7KA5fmzHRGUM7FtQZodQlbjOSmPSeMFfy7N+TDaGq7nw6VHEZ1SY1qMLnp5CKUKte4l2+PpTSpfyDg+Qo+oDNNILxSdNTOyZFli/+6sW5c6QvvN4jjMEAcqJVWSSdpl8iI6SOOgwEIlXbaSKMPst0lr/aYyfalhgNEoi5vduoerit8+FMAW23fBfhwQGAWZu5Qtr/TCDVRBHBZSbcIP0gNEUPerYk066uqd53vAZGcIAfkWVz+xfd3xqbCiSlWpMCMGaQp/JL/P2or3dDQYdrVQBCDmbhBzuBFThZbT1SNB/n7XgwHnm6bD55hi8yOc11/VlziR2zKTuEYnXsN+oxIu23h8ZRut4eqYQ4GQea7RzCqul6SvMIAJIxvmtEEmtm4jxRUUbbe48hv86Qtr8x+0kcVVhgUCSgwBbVRzp/bS9tbA8ZPv5rrr3i5qfrsGolkio5IyzlD0r0W/GUb1/KU12wqkZ1i3EnZF8IjC6MrcQjl4g2GN9IrnStTqBZZV4DQaY+VVahNGH4d9aye1H665QlpQQcICi41f0WexMmFQCcK9ynEmaO7amF1MLt19IHPKdsQ15mTqkTBONhzPuLrudLbsiKLRVk3MUoFvGEmQ+aX1kwOufc9vy3yZjbBx+j6CPjvgYUEzCtGM9bxWH+vGH9EMxQrjCMHQOWTUo0GVHAntNK4jlA7q+N8B23Z/WTXHnW6yESodVkCUIqeZEpjLJQsPjOowv8KSs8yv3veIwmuF3B+GcDyEPolsPSRrIyVH4qf7SGUrQZnNNQ0srwMATAMTGT9QpPWhjIknK6Z+xaCtcLBcfX0dNRO9Hsiwd5mEFe7xBiUGGyi4w/r/c+mZqkYlR5PtbHsw1tQiFx1RfTedME3nSiQszbvngfsdQLq6xh5JzlcHUy7bbj+Pe2yeljkXcrdrCh5WF78CAAD//5z2ftQ=" } diff --git a/metricbeat/module/kafka/partition/_meta/data.json b/metricbeat/module/kafka/partition/_meta/data.json index 519feb1bf75c..f8a2bbf50854 100644 --- a/metricbeat/module/kafka/partition/_meta/data.json +++ b/metricbeat/module/kafka/partition/_meta/data.json @@ -14,6 +14,7 @@ "address": "172.18.0.2:9092", "id": 0 }, + "id": 0, "offset": { "newest": 0, "oldest": 0 @@ -26,11 +27,13 @@ "replica": 0 }, "topic": { - "name": "foo-1532945633-185372965" - } + "name": "foo-1538389014-739473801" + }, + "topic_broker_id": "0-foo-1538389014-739473801-0", + "topic_id": "0-foo-1538389014-739473801" }, "topic": { - "name": "foo-1532945633-185372965" + "name": "foo-1538389014-739473801" } }, "metricset": { diff --git a/metricbeat/module/kafka/partition/partition.go b/metricbeat/module/kafka/partition/partition.go index e9cbbf9154ec..57ec7b5f1f61 100644 --- a/metricbeat/module/kafka/partition/partition.go +++ b/metricbeat/module/kafka/partition/partition.go @@ -20,6 +20,7 @@ package partition import ( "crypto/tls" "errors" + "fmt" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/cfgwarn" @@ -167,8 +168,17 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { } } + // Helpful IDs for dashboards + partitionTopicID := fmt.Sprintf("%d-%s", partition.ID, topic.Name) + partitionTopicBrokerID := fmt.Sprintf("%s-%d", partitionTopicID, b.ID()) + // create event event := common.MapStr{ + // Common `kafka.partition` fields + "id": partition.ID, + "topic_id": partitionTopicID, + "topic_broker_id": partitionTopicBrokerID, + "topic": evtTopic, "broker": evtBroker, "partition": partitionEvent, @@ -183,9 +193,6 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { ModuleFields: common.MapStr{ "broker": evtBroker, "topic": evtTopic, - "partition": common.MapStr{ - "id": partition.ID, - }, }, MetricSetFields: event, }) From c8c484542f8ee9413578f0d9d428e2ec4ab64bd6 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 1 Oct 2018 13:13:39 +0200 Subject: [PATCH 2/5] Changes in comments --- CHANGELOG.asciidoc | 2 +- metricbeat/module/kafka/consumergroup/consumergroup.go | 2 +- metricbeat/module/kafka/partition/partition.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index dc9a02888e09..5b4754c65266 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -141,7 +141,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff] - Support for Kafka 2.0.0 {pull}8399[8399] - Add container image for docker metricsets. {issue}8214[8214] {pull}8438[8438] - Add support for `full` status page output for php-fpm module as a separate metricset called `process`. {pull}8394[8394] -- Precalculate composed id fields for kafka dashboards {pull}8504[8504] +- Precalculate composed id fields for kafka dashboards. {pull}8504[8504] *Packetbeat* diff --git a/metricbeat/module/kafka/consumergroup/consumergroup.go b/metricbeat/module/kafka/consumergroup/consumergroup.go index ed73db6bcdde..a11967e9bae4 100644 --- a/metricbeat/module/kafka/consumergroup/consumergroup.go +++ b/metricbeat/module/kafka/consumergroup/consumergroup.go @@ -112,7 +112,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { } emitEvent := func(event common.MapStr) { - // Helpful IDs for dashboards + // Helpful IDs to avoid scripts on queries partitionTopicID := fmt.Sprintf("%d-%s", event["partition"], event["topic"]) partitionTopicBrokerID := fmt.Sprintf("%s-%d", partitionTopicID, m.broker.ID()) diff --git a/metricbeat/module/kafka/partition/partition.go b/metricbeat/module/kafka/partition/partition.go index 57ec7b5f1f61..16e3632fc4ff 100644 --- a/metricbeat/module/kafka/partition/partition.go +++ b/metricbeat/module/kafka/partition/partition.go @@ -168,7 +168,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { } } - // Helpful IDs for dashboards + // Helpful IDs to avoid scripts on queries partitionTopicID := fmt.Sprintf("%d-%s", partition.ID, topic.Name) partitionTopicBrokerID := fmt.Sprintf("%s-%d", partitionTopicID, b.ID()) From b266c2a226c464f4f339381b9566c197f6cd9450 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 16 Oct 2018 17:00:02 +0200 Subject: [PATCH 3/5] Typo on fields type --- metricbeat/docs/fields.asciidoc | 4 ++-- metricbeat/module/kafka/_meta/fields.yml | 4 ++-- metricbeat/module/kafka/fields.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 7e27d950ec42..3def7e6282af 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -8793,7 +8793,7 @@ Partition id. *`kafka.partition.topic_id`*:: + -- -type: keyworkd +type: keyword Unique id of the partition in the topic. @@ -8802,7 +8802,7 @@ Unique id of the partition in the topic. *`kafka.partition.topic_broker_id`*:: + -- -type: keyworkd +type: keyword Unique id of the partition in the topic and the broker. diff --git a/metricbeat/module/kafka/_meta/fields.yml b/metricbeat/module/kafka/_meta/fields.yml index 69753db8241e..e6b507c1449f 100644 --- a/metricbeat/module/kafka/_meta/fields.yml +++ b/metricbeat/module/kafka/_meta/fields.yml @@ -41,11 +41,11 @@ Partition id. - name: partition.topic_id - type: keyworkd + type: keyword description: Unique id of the partition in the topic. - name: partition.topic_broker_id - type: keyworkd + type: keyword description: Unique id of the partition in the topic and the broker. diff --git a/metricbeat/module/kafka/fields.go b/metricbeat/module/kafka/fields.go index eb3f71fec8f8..80e88dd2b5b1 100644 --- a/metricbeat/module/kafka/fields.go +++ b/metricbeat/module/kafka/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJy8WE1v4zYQvftXDHJyDlFP7UGHAm1SFEaL3UV292zQ4sgmTJFaknbW/35ByrIlSqQ+7ESnmJHeexwO3wz5BHs8pbAn+Z4sAAwzHFN4+M/+flgAUNSZYqVhUqTw5wIAwP0PCkkPHBcAeieVWWdS5GybQk64tqMKORKNKWzQWOCcIac6dQBPIEiBV1L7mFOJKWyVPJTnkR7mNkwTaqPkHtVluA8viFk9fzsEeJZCHwpU8K/9FFYil6og9gPYkSPCBlGAQkIhV7KA5fmzHRGUM7FtQZodQlbjOSmPSeMFfy7N+TDaGq7nw6VHEZ1SY1qMLnp5CKUKte4l2+PpTSpfyDg+Qo+oDNNILxSdNTOyZFli/+6sW5c6QvvN4jjMEAcqJVWSSdpl8iI6SOOgwEIlXbaSKMPst0lr/aYyfalhgNEoi5vduoerit8+FMAW23fBfhwQGAWZu5Qtr/TCDVRBHBZSbcIP0gNEUPerYk066uqd53vAZGcIAfkWVz+xfd3xqbCiSlWpMCMGaQp/JL/P2or3dDQYdrVQBCDmbhBzuBFThZbT1SNB/n7XgwHnm6bD55hi8yOc11/VlziR2zKTuEYnXsN+oxIu23h8ZRut4eqYQ4GQea7RzCqul6SvMIAJIxvmtEEmtm4jxRUUbbe48hv86Qtr8x+0kcVVhgUCSgwBbVRzp/bS9tbA8ZPv5rrr3i5qfrsGolkio5IyzlD0r0W/GUb1/KU12wqkZ1i3EnZF8IjC6MrcQjl4g2GN9IrnStTqBZZV4DQaY+VVahNGH4d9aye1H665QlpQQcICi41f0WexMmFQCcK9ynEmaO7amF1MLt19IHPKdsQ15mTqkTBONhzPuLrudLbsiKLRVk3MUoFvGEmQ+aX1kwOufc9vy3yZjbBx+j6CPjvgYUEzCtGM9bxWH+vGH9EMxQrjCMHQOWTUo0GVHAntNK4jlA7q+N8B23Z/WTXHnW6yESodVkCUIqeZEpjLJQsPjOowv8KSs8yv3veIwmuF3B+GcDyEPolsPSRrIyVH4qf7SGUrQZnNNQ0srwMATAMTGT9QpPWhjIknK6Z+xaCtcLBcfX0dNRO9Hsiwd5mEFe7xBiUGGyi4w/r/c+mZqkYlR5PtbHsw1tQiFx1RfTedME3nSiQszbvngfsdQLq6xh5JzlcHUy7bbj+Pe2yeljkXcrdrCh5WF78CAAD//5z2ftQ=" + return "eJy8WE1v4zYQvftXDHJyDlFP7UGHAm1SFEaL3UV292zQ4sgmTJFaknbW/35ByrIlSqQ+7ESnmJHeexwO3wz5BHs8pbAn+Z4sAAwzHFN4+M/+flgAUNSZYqVhUqTw5wIAwP0PCkkPHBcAeieVWWdS5GybQk64tqMKORKNKWzQWOCcIac6dQBPIEiBV1L7mFOJKWyVPJTnkR7mNkwTaqPkHtVluA8viFk9fzsEeJZCHwpU8K/9FFYil6og9gPYkSPCBlGAQkIhV7KA5fmzHRGUM7FtQZodQlbjOSmPSeMFfy7N+TDaGq7nw6VHEZ1SY1qMLnp5CKUKte4l2+PpTSpfyDg+Qo+oDNNILxSdNTOyZFli/+6sW5c6QvvN4jjMEAcqJVWSSdpl8iI6SOOgwEIlXbaSKMPst0lr/aYyfalhgNEoi5vduocrHr8W2XfBfhwQGAWZu4wtr+zCDVQxHNZR7cGPkQNEUPerIk064up95zvAZF8IAfkGVz+xXd1xqbCiSlWpMCMGaQp/JL/P2oj39DMY9rRQBCDmbRDztxFThZbP1SNB/n7PgwHfm6bD55hi8iN811/VlziR2zKTuEYnXsN8oxIu23h8XRut4eqXQ4GQea7RzCqtl6SvMIAJIxvmtEEmtm4jxRUUbbe48hv86Qtr8x+0kcVVhgUCSgwBbVRzp/bS9lbA8ZPv5rrr3S5qfrsGolkgo5IyzlD0r0W/GUb1/KU12wqkZ1i3EnZF8IjC6MrcQjl4g2GN9IrnStTqBZZV4DQaY+VVahNGH4d9aye1H665QlpQQcICi41f0GexMmFQCcK9ynEmaO7amF1MLt19IHPKdsQ15mTqkTBONhzPuLrudLbsiKLRVU3MUoFvGEmQ+aX1kwOufc9vy3yZjbBx+j6CPjvgYUEzCtGM9bxWH+vGH9EMxQrjCMHQOWLUo0GVHAntNK4jlA7q+N8B23Z/WTXHnW6yESodVkCUIqeZEpjLJQsPjOowv8KSs8yv3veIwmuF3B+GcDyEPolsPSRrIyVH4qf7SGUrQZnNNQ0srwMATAMTGT9QpPWhjIknK6Z+xaCtcLBcfX0dNRO9Hsiwd5mEFe7xBiUGGyi4w/r/c+mZqkYlR5PtbHsw1tQi1xxRfTedME3nQiQszbvlgfsdQLq6xh5JzlcHU67abj+Pe2yeljnXcbdrCh5WF78CAAD//1a2ff4=" } From 41c151f50db60a9ee120d3425bab84af5721c079 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 17 Oct 2018 11:16:41 +0200 Subject: [PATCH 4/5] Use replica broker id for composed partition broker id --- metricbeat/module/kafka/partition/partition.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/kafka/partition/partition.go b/metricbeat/module/kafka/partition/partition.go index 16e3632fc4ff..bd68c1554158 100644 --- a/metricbeat/module/kafka/partition/partition.go +++ b/metricbeat/module/kafka/partition/partition.go @@ -170,7 +170,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { // Helpful IDs to avoid scripts on queries partitionTopicID := fmt.Sprintf("%d-%s", partition.ID, topic.Name) - partitionTopicBrokerID := fmt.Sprintf("%s-%d", partitionTopicID, b.ID()) + partitionTopicBrokerID := fmt.Sprintf("%s-%d", partitionTopicID, id) // create event event := common.MapStr{ From 935a103132b73629348969684f00496e15939262 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 17 Oct 2018 18:21:34 +0200 Subject: [PATCH 5/5] Don't publish the partition broker id on the consumergroups --- metricbeat/module/kafka/consumergroup/_meta/data.json | 3 +-- metricbeat/module/kafka/consumergroup/consumergroup.go | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/metricbeat/module/kafka/consumergroup/_meta/data.json b/metricbeat/module/kafka/consumergroup/_meta/data.json index a63a640af593..55cc9cefda84 100644 --- a/metricbeat/module/kafka/consumergroup/_meta/data.json +++ b/metricbeat/module/kafka/consumergroup/_meta/data.json @@ -30,7 +30,6 @@ }, "partition": { "id": 0, - "topic_broker_id": "0-metricbeat-test-0", "topic_id": "0-metricbeat-test" }, "topic": { @@ -43,4 +42,4 @@ "name": "consumergroup", "rtt": 115 } -} \ No newline at end of file +} diff --git a/metricbeat/module/kafka/consumergroup/consumergroup.go b/metricbeat/module/kafka/consumergroup/consumergroup.go index a11967e9bae4..c79ad06d1ad5 100644 --- a/metricbeat/module/kafka/consumergroup/consumergroup.go +++ b/metricbeat/module/kafka/consumergroup/consumergroup.go @@ -114,7 +114,6 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { emitEvent := func(event common.MapStr) { // Helpful IDs to avoid scripts on queries partitionTopicID := fmt.Sprintf("%d-%s", event["partition"], event["topic"]) - partitionTopicBrokerID := fmt.Sprintf("%s-%d", partitionTopicID, m.broker.ID()) // TODO (deprecation): Remove fields from MetricSetFields moved to ModuleFields event["broker"] = brokerInfo @@ -125,9 +124,8 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { "name": event["topic"], }, "partition": common.MapStr{ - "id": event["partition"], - "topic_id": partitionTopicID, - "topic_broker_id": partitionTopicBrokerID, + "id": event["partition"], + "topic_id": partitionTopicID, }, }, MetricSetFields: event,