diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 04a61474c6de..8a5b7d1c6e24 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -238,6 +238,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add a system/entropy metricset {pull}12450[12450] - Allow redis URL format in redis hosts config. {pull}12408[12408] - Add tags into ec2 metricset. {issue}[12263]12263 {pull}12372[12372] +- Add `beat` module. {pull}12181[12181] {pull}12615[12615] - Collect tags for cloudwatch metricset in aws module. {issue}[12263]12263 {pull}12480[12480] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index ab516b8662e3..c16b0676b307 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -1600,6 +1600,53 @@ type: keyword Beat type. +type: keyword + +-- + +[float] +=== state + +Beat state + + + +*`beat.state.management.enabled`*:: ++ +-- +Is central management enabled? + + +type: boolean + +-- + +*`beat.state.module.count`*:: ++ +-- +Number of modules enabled + + +type: integer + +-- + +*`beat.state.output.name`*:: ++ +-- +Name of output used by Beat + + +type: keyword + +-- + +*`beat.state.queue.name`*:: ++ +-- +Name of queue being used by Beat + + type: keyword -- diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 84e7f21c6dab..fd4f5000807c 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -31,6 +31,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/apache" _ "github.com/elastic/beats/metricbeat/module/apache/status" _ "github.com/elastic/beats/metricbeat/module/beat" + _ "github.com/elastic/beats/metricbeat/module/beat/state" _ "github.com/elastic/beats/metricbeat/module/beat/stats" _ "github.com/elastic/beats/metricbeat/module/ceph" _ "github.com/elastic/beats/metricbeat/module/ceph/cluster_disk" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 4514890d3b34..03808e3d9272 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -160,6 +160,7 @@ metricbeat.modules: - module: beat metricsets: - stats + - state period: 10s hosts: ["http://localhost:5066"] #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] diff --git a/metricbeat/module/beat/_meta/config-xpack.yml b/metricbeat/module/beat/_meta/config-xpack.yml index 011e01b04ada..0f72a4e9d032 100644 --- a/metricbeat/module/beat/_meta/config-xpack.yml +++ b/metricbeat/module/beat/_meta/config-xpack.yml @@ -1,6 +1,7 @@ - module: beat metricsets: - stats + - state period: 10s hosts: ["http://localhost:5066"] #username: "user" diff --git a/metricbeat/module/beat/_meta/config.reference.yml b/metricbeat/module/beat/_meta/config.reference.yml index a82db61430a5..5ba5959b0701 100644 --- a/metricbeat/module/beat/_meta/config.reference.yml +++ b/metricbeat/module/beat/_meta/config.reference.yml @@ -1,6 +1,7 @@ - module: beat metricsets: - stats + - state period: 10s hosts: ["http://localhost:5066"] #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] diff --git a/metricbeat/module/beat/_meta/config.yml b/metricbeat/module/beat/_meta/config.yml index 1b7457a89898..4e5586517a08 100644 --- a/metricbeat/module/beat/_meta/config.yml +++ b/metricbeat/module/beat/_meta/config.yml @@ -1,6 +1,7 @@ - module: beat metricsets: - stats + - state period: 10s hosts: ["http://localhost:5066"] diff --git a/metricbeat/module/beat/beat_integration_test.go b/metricbeat/module/beat/beat_integration_test.go index ec91196fdd68..e894fa9a8243 100644 --- a/metricbeat/module/beat/beat_integration_test.go +++ b/metricbeat/module/beat/beat_integration_test.go @@ -27,11 +27,13 @@ import ( "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/beats/metricbeat/module/beat" + _ "github.com/elastic/beats/metricbeat/module/beat/state" _ "github.com/elastic/beats/metricbeat/module/beat/stats" ) var metricSets = []string{ "stats", + "state", } func TestFetch(t *testing.T) { diff --git a/metricbeat/module/beat/fields.go b/metricbeat/module/beat/fields.go index ec8d2423860e..53102ecb26ce 100644 --- a/metricbeat/module/beat/fields.go +++ b/metricbeat/module/beat/fields.go @@ -32,5 +32,5 @@ func init() { // AssetBeat returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/beat. func AssetBeat() string { - return "eJzcls+K2zAQxu95isHn7j6ADz2UttBLC2Whh1KKYk8cEVkjpNEGv32RUjuO/8hJya7L6hQk/H0/fePx5AEO2OSwRcEbAJasMIfsAwrONgAlusJKw5J0Du83AADhCGoqvcINgEWFwmEOldgAOGSWunI5/MycU9k7yPbMJvsVzvZk+XdBeierHHZCufD8TqIqXR6VH0CLGjuWsLgxQduSN393JoguVfpKsuy2Wq0DNkey/f1JxdOKd/3y8XEkHLTuIB2eG4s7FuxG6v0QrtEeqlxWql3D5Poc3rCs8bF2F6ctjyJdDQ4SSB3WSXTSz3odDSuy5FlqvJfxV19v0QLt4Cwd3LTUFUgdwSaJlNxuh2dz9biC43MMGwqqa9LABEKpaD686FRV+lzk2fghVhrtCrywvkXl0cuzBAbzvTHGG/fJjZBhPTUGQ0VnozgD4TPqydvAYmI3AH0KLlCQ14x2zi2VXx9ZFAecCwhSnfAP4HDRIaewAoCmo8KySnCcaVk+z9X8ZXFP1ukS9yaL4GI/+qy8DmlEXCRoSUtLxqz7CiwhdKjeKFkIXjNXd6ZYBt4JqdaNdoGg+5IS1UI3q5AyEQTza1uLiYV6RdKn4Ad6kGxyDlgUc5nfbQp8R1HOTk+4YQJsm7UaKjqnwupNVmtpdta9LGXgS/m3iEcrOf135A5V/xFM3kTZQ1yM+n+ufKzoECD1408AAAD//2l5TJo=" + return "eJzcl0+L2zwQxu/5FIPP7+YD+PAWSlvYSwtloYdSimxPvGJljSuNdvG3L5LXiWPJdtJmE6hOiRU/z0/zR1Lu4Am7HAoUvAFgyQpzyN6j4GwDUKEtjWxZks7h/w0AgJ+ChiqncANgUKGwmEMtNgAWmaWubQ7fM2tV9h9kj8xt9sPPPZLhnyXpnaxz2All/fs7iaqyeVC+Ay0a3LP4wV3rtQ259vVJguhYZawkq/2jQesJuxcy4+dJxX6Etd5/2EbCXusC0v69WNyy4Fh9HIRTtKcqx5kaxjRyY45GaFFjg5q3qEWhsDr62QBWECkUejK3gOfHvYUSNRuhRjbwavMujRNqbluS05wEkZqxRnMeyGfXFGiAdq/6FlJrHRjIcet4678kEeISOAVBNOgBenFwFisoupDFJMQvhw7fiCFoQ4FS12mQcZHaixSp/asidS3LBreNTcZCka7PC0TA6kWTfsbpYFiTIcdS46WMD4V4kPZu2qdC6vlyULIopnNz+TiB41MINpTUNKSBCYRSwXy60FRWIOqVaHoJ7QQ8P770jTItnjUwmN/AY7x0F50B6cdD1446exEIn1EnVwOrETsD6KN3gbCFoplzW4rfGFmUT9GREFMnOuEPwOGoQ/pgeQBNLwqreoHjQMvyeS7nb4vbWy+neHT9EVw+RtvKdUgD4irBQFoZatvblsAawh7VtUqWgm8ZV3ugWAfeCRlfuK4Ku0Kw30mJGqG7m5Aykb8+dqe2FhMLdUXSB+8HehLZxXPAoJiL+cVOga8oqtnTE844AYruVg0VnJeCNTpZjaHZs+5tKT3fkv+A+GIkL19HLpD1b97kn0i7Dxfj9B9nzHnDzIeMTgGWPvwOAAD//2j1+Zk=" } diff --git a/metricbeat/module/beat/state/_meta/data.json b/metricbeat/module/beat/state/_meta/data.json new file mode 100644 index 000000000000..6cdbfaaf350e --- /dev/null +++ b/metricbeat/module/beat/state/_meta/data.json @@ -0,0 +1,33 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "beat": { + "state": { + "management": { + "enabled": false + }, + "module": { + "count": 3 + }, + "output": { + "name": "elasticsearch" + }, + "queue": { + "name": "mem" + } + } + }, + "event": { + "dataset": "beat.state", + "duration": 115000, + "module": "beat" + }, + "metricset": { + "name": "state" + }, + "service": { + "address": "127.0.0.1:5066", + "id": "1f0c187b-f2ef-4950-b9cc-dd6864b9191a", + "name": "Shaunaks-MacBook-Pro-2.local", + "type": "metricbeat" + } +} \ No newline at end of file diff --git a/metricbeat/module/beat/state/_meta/fields.yml b/metricbeat/module/beat/state/_meta/fields.yml new file mode 100644 index 000000000000..0debc24a2d70 --- /dev/null +++ b/metricbeat/module/beat/state/_meta/fields.yml @@ -0,0 +1,22 @@ +- name: state + type: group + description: > + Beat state + release: ga + fields: + - name: management.enabled + type: boolean + description: > + Is central management enabled? + - name: module.count + type: integer + description: > + Number of modules enabled + - name: output.name + type: keyword + description: > + Name of output used by Beat + - name: queue.name + type: keyword + description: > + Name of queue being used by Beat diff --git a/metricbeat/module/beat/state/_meta/test/state.800.json b/metricbeat/module/beat/state/_meta/test/state.800.json new file mode 100644 index 000000000000..990cecc21d71 --- /dev/null +++ b/metricbeat/module/beat/state/_meta/test/state.800.json @@ -0,0 +1,43 @@ +{ + "beat": { + "name": "Shaunaks-MBP-2" + }, + "host": { + "architecture": "x86_64", + "hostname": "Shaunaks-MBP-2", + "id": "EF6274EA-462F-5316-A14A-850E7BFD8126", + "os": { + "build": "18F132", + "family": "darwin", + "kernel": "18.6.0", + "name": "Mac OS X", + "platform": "darwin", + "version": "10.14.5" + } + }, + "management": { + "enabled": false + }, + "module": { + "count": 3, + "names": [ + "system" + ] + }, + "output": { + "name": "elasticsearch" + }, + "outputs": { + "elasticsearch": { + "cluster_uuid": "c8sjZYNjRcOiCTGzQZoyqw" + } + }, + "queue": { + "name": "mem" + }, + "service": { + "id": "1f0c187b-f2ef-4950-b9cc-dd6864b9191a", + "name": "metricbeat", + "version": "8.0.0" + } +} diff --git a/metricbeat/module/beat/state/data.go b/metricbeat/module/beat/state/data.go new file mode 100644 index 000000000000..15a258389590 --- /dev/null +++ b/metricbeat/module/beat/state/data.go @@ -0,0 +1,72 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package state + +import ( + "encoding/json" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + s "github.com/elastic/beats/libbeat/common/schema" + c "github.com/elastic/beats/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/module/beat" +) + +var ( + schema = s.Schema{ + "management": c.Dict("management", s.Schema{ + "enabled": c.Bool("enabled"), + }), + "module": c.Dict("module", s.Schema{ + "count": c.Int("count"), + }), + "output": c.Dict("output", s.Schema{ + "name": c.Str("name"), + }), + "queue": c.Dict("queue", s.Schema{ + "name": c.Str("name"), + }), + } +) + +func eventMapping(r mb.ReporterV2, info beat.Info, content []byte) error { + var event mb.Event + event.RootFields = common.MapStr{} + event.RootFields.Put("service", common.MapStr{ + "id": info.UUID, + "name": info.Name, + }) + + event.Service = info.Beat + + var data map[string]interface{} + err := json.Unmarshal(content, &data) + if err != nil { + return errors.Wrap(err, "failure parsing Beat's State API response") + } + + event.MetricSetFields, err = schema.Apply(data) + if err != nil { + return errors.Wrap(err, "failure to apply state schema") + } + + r.Event(event) + return nil +} diff --git a/metricbeat/module/beat/state/data_test.go b/metricbeat/module/beat/state/data_test.go new file mode 100644 index 000000000000..b1150592ca23 --- /dev/null +++ b/metricbeat/module/beat/state/data_test.go @@ -0,0 +1,55 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !integration + +package state + +import ( + "io/ioutil" + "path/filepath" + "testing" + + "github.com/elastic/beats/metricbeat/module/beat" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +func TestEventMapping(t *testing.T) { + + files, err := filepath.Glob("./_meta/test/state.*.json") + assert.NoError(t, err) + + info := beat.Info{ + UUID: "1234", + Beat: "helloworld", + } + + for _, f := range files { + input, err := ioutil.ReadFile(f) + assert.NoError(t, err) + + reporter := &mbtest.CapturingReporterV2{} + err = eventMapping(reporter, info, input) + + assert.NoError(t, err, f) + assert.True(t, len(reporter.GetEvents()) >= 1, f) + assert.Equal(t, 0, len(reporter.GetErrors()), f) + } +} diff --git a/metricbeat/module/beat/state/data_xpack.go b/metricbeat/module/beat/state/data_xpack.go new file mode 100644 index 000000000000..67fed65d981d --- /dev/null +++ b/metricbeat/module/beat/state/data_xpack.go @@ -0,0 +1,110 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package state + +import ( + "encoding/json" + "time" + + "github.com/pkg/errors" + + "github.com/elastic/beats/metricbeat/helper/elastic" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/module/beat" +) + +func eventMappingXPack(r mb.ReporterV2, m *MetricSet, info beat.Info, content []byte) error { + now := time.Now() + + // Massage info into beat + beat := common.MapStr{ + "name": info.Name, + "host": info.Hostname, + "type": info.Beat, + "uuid": info.UUID, + "version": info.Version, + } + + var state map[string]interface{} + err := json.Unmarshal(content, &state) + if err != nil { + return errors.Wrap(err, "failure parsing Beat's State API response") + } + + fields := common.MapStr{ + "state": state, + "beat": beat, + "timestamp": now, + } + + clusterUUID := getClusterUUID(state) + + var event mb.Event + event.RootFields = common.MapStr{ + "cluster_uuid": clusterUUID, + "timestamp": now, + "interval_ms": m.calculateIntervalMs(), + "type": "beats_state", + "beats_state": fields, + } + + event.Index = elastic.MakeXPackMonitoringIndexName(elastic.Beats) + + r.Event(event) + return nil +} + +func (m *MetricSet) calculateIntervalMs() int64 { + return m.Module().Config().Period.Nanoseconds() / 1000 / 1000 +} + +func getClusterUUID(state map[string]interface{}) string { + o, exists := state["outputs"] + if !exists { + return "" + } + + outputs, ok := o.(map[string]interface{}) + if !ok { + return "" + } + + e, exists := outputs["elasticsearch"] + if !exists { + return "" + } + + elasticsearch, ok := e.(map[string]interface{}) + if !ok { + return "" + } + + c, exists := elasticsearch["cluster_uuid"] + if !exists { + return "" + } + + clusterUUID, ok := c.(string) + if !ok { + return "" + } + + return clusterUUID +} diff --git a/metricbeat/module/beat/state/state.go b/metricbeat/module/beat/state/state.go new file mode 100644 index 000000000000..218686abce39 --- /dev/null +++ b/metricbeat/module/beat/state/state.go @@ -0,0 +1,83 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package state + +import ( + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/beat" +) + +func init() { + mb.Registry.MustAddMetricSet(beat.ModuleName, "state", New, + mb.WithHostParser(hostParser), + ) +} + +const ( + statePath = "state" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: "http", + DefaultPath: statePath, + }.Build() +) + +// MetricSet defines all fields of the MetricSet +type MetricSet struct { + *beat.MetricSet +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := beat.NewMetricSet(base) + if err != nil { + return nil, err + } + return &MetricSet{MetricSet: ms}, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +func (m *MetricSet) Fetch(r mb.ReporterV2) error { + content, err := m.HTTP.FetchContent() + if err != nil { + return err + } + + info, err := beat.GetInfo(m.MetricSet) + if err != nil { + return err + } + + if m.MetricSet.XPackEnabled { + err = eventMappingXPack(r, m, *info, content) + if err != nil { + // Since this is an x-pack code path, we log the error but don't + // return it. Otherwise it would get reported into `metricbeat-*` + // indices. + m.Logger().Error(err) + return nil + } + } else { + return eventMapping(r, *info, content) + } + + return nil +} diff --git a/metricbeat/modules.d/beat-xpack.yml.disabled b/metricbeat/modules.d/beat-xpack.yml.disabled index e941a5257876..ed7ecc00df6c 100644 --- a/metricbeat/modules.d/beat-xpack.yml.disabled +++ b/metricbeat/modules.d/beat-xpack.yml.disabled @@ -4,6 +4,7 @@ - module: beat metricsets: - stats + - state period: 10s hosts: ["http://localhost:5066"] #username: "user" diff --git a/metricbeat/modules.d/beat.yml.disabled b/metricbeat/modules.d/beat.yml.disabled index ed31130210a4..af2907f77b44 100644 --- a/metricbeat/modules.d/beat.yml.disabled +++ b/metricbeat/modules.d/beat.yml.disabled @@ -4,6 +4,7 @@ - module: beat metricsets: - stats + - state period: 10s hosts: ["http://localhost:5066"] diff --git a/metricbeat/tests/system/test_beat.py b/metricbeat/tests/system/test_beat.py index 42ccb9dd8e04..4fb16336c6db 100644 --- a/metricbeat/tests/system/test_beat.py +++ b/metricbeat/tests/system/test_beat.py @@ -2,31 +2,21 @@ import metricbeat import unittest import time +from parameterized import parameterized class Test(metricbeat.BaseTest): COMPOSE_SERVICES = ['metricbeat'] + FIELDS = ['beat'] + @parameterized.expand([ + "stats", + "state" + ]) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - def test_stats(self): + def test_metricsets(self, metricset): """ - beat stats metricset test + beat metricset tests """ - self.render_config_template(modules=[{ - "name": "beat", - "metricsets": ["stats"], - "hosts": self.get_hosts(), - "period": "1s", - }]) - proc = self.start_beat() - self.wait_until(lambda: self.output_lines() > 0, max_timeout=20) - proc.check_kill_and_wait() - self.assert_no_logged_warnings() - - output = self.read_output_json() - self.assertTrue(len(output) >= 1) - evt = output[0] - print(evt) - - self.assert_fields_are_documented(evt) + self.check_metricset("beat", metricset, self.get_hosts(), self.FIELDS + ["service"]) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 68f52ef85d02..306c1338e4f2 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -211,6 +211,7 @@ metricbeat.modules: - module: beat metricsets: - stats + - state period: 10s hosts: ["http://localhost:5066"] #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]