From 227eaecbfab25f2b7b76cd45ed210e1a33281fd1 Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Mon, 5 Aug 2024 11:06:12 +0530 Subject: [PATCH 01/33] [vSphere][host] Data collection and Fields mappings --- metricbeat/docs/fields.asciidoc | 210 ++++++++++++++++++ metricbeat/module/vsphere/fields.go | 2 +- .../module/vsphere/host/_meta/data.json | 82 ++++++- .../module/vsphere/host/_meta/fields.yml | 84 +++++++ metricbeat/module/vsphere/host/data.go | 109 ++++++++- metricbeat/module/vsphere/host/data_test.go | 3 +- metricbeat/module/vsphere/host/host.go | 207 ++++++++++++----- 7 files changed, 627 insertions(+), 70 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index df93b4a6d8c7..840ef22dbe43 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -66946,6 +66946,26 @@ type: keyword -- +*`vsphere.host.status`*:: ++ +-- +The overall health status of a host in the vSphere environment. + + +type: keyword + +-- + +*`vsphere.host.uptime`*:: ++ +-- +The total uptime of a host in the vSphere environment. + + +type: double + +-- + *`vsphere.host.cpu.used.mhz`*:: + -- @@ -66976,6 +66996,56 @@ type: long -- +*`vsphere.host.disk.throughput.bytes`*:: ++ +-- +The average disk throughput usage over a specified interval. + + +type: double + +-- + +*`vsphere.host.disk.device.latency.ms`*:: ++ +-- +Average amount of time it takes to complete an SCSI command from physical device. + + +type: double + +-- + +*`vsphere.host.disk.latency.total.ms`*:: ++ +-- +Highest latency value across all disks used by the host. + + +type: double + +-- + +*`vsphere.host.disk.total.bytes`*:: ++ +-- +Aggregated disk I/O rate. + + +type: double + +-- + +*`vsphere.host.disk.read.bytes`*:: ++ +-- +Average number of kilobytes read from the disk each second. + + +type: double + +-- + *`vsphere.host.memory.used.bytes`*:: + -- @@ -67022,6 +67092,146 @@ type: keyword -- +*`vsphere.host.network.bandwidth.transmitted.bytes`*:: ++ +-- +Average rate at which data was transmitted during the interval. This represents the bandwidth of the network. + + +type: double + +-- + +*`vsphere.host.network.bandwidth.received.bytes`*:: ++ +-- +Average rate at which data was received during the interval. This represents the bandwidth of the network. + + +type: double + +-- + +*`vsphere.host.network.bandwidth.total.bytes`*:: ++ +-- +Network utilization (combined transmit- and receive-rates). + + +type: double + +-- + +*`vsphere.host.network.packets.transmitted.count`*:: ++ +-- +Number of packets transmitted. + + +type: double + +-- + +*`vsphere.host.network.packets.received.count`*:: ++ +-- +Number of packets received. + + +type: double + +-- + +*`vsphere.host.network.packets.errors.transmitted.count`*:: ++ +-- +Number of packets with errors transmitted. + + +type: double + +-- + +*`vsphere.host.network.packets.errors.received.count`*:: ++ +-- +Number of packets with errors received. + + +type: double + +-- + +*`vsphere.host.network.packets.errors.total.count`*:: ++ +-- +Total number of packets with errors. + + +type: double + +-- + +*`vsphere.host.network.packets.multicast.transmitted.count`*:: ++ +-- +Number of multicast packets transmitted. + + +type: double + +-- + +*`vsphere.host.network.packets.multicast.received.count`*:: ++ +-- +Number of multicast packets received. + + +type: double + +-- + +*`vsphere.host.network.packets.multicast.total.count`*:: ++ +-- +Total number of multicast packets. + + +type: double + +-- + +*`vsphere.host.network.packets.dropped.transmitted.count`*:: ++ +-- +Number of transmitted packets dropped. + + +type: double + +-- + +*`vsphere.host.network.packets.dropped.received.count`*:: ++ +-- +Number of received packets dropped. + + +type: double + +-- + +*`vsphere.host.network.packets.dropped.total.count`*:: ++ +-- +Total number of packets dropped. + + +type: double + +-- + [float] === virtualmachine diff --git a/metricbeat/module/vsphere/fields.go b/metricbeat/module/vsphere/fields.go index fae0b2cc2647..d41b4d83853f 100644 --- a/metricbeat/module/vsphere/fields.go +++ b/metricbeat/module/vsphere/fields.go @@ -32,5 +32,5 @@ func init() { // AssetVsphere returns asset data. // This is the base64 encoded zlib format compressed contents of module/vsphere. func AssetVsphere() string { - return "eJzsl8+OmzAQxu95itHedx+AQ6Vqq20vaSttt9fIgSG4MQyyh6zo01e2IeKP002KaS/1ASnYfN8PzHxD7uGIbQInUxeocQPAkhUmcHd6dmfuNgAZmlTLmiVVCbzbAAB0s1BS1ih7mUaFwmACB7EByCWqzCRu6T1UosShhR3c1naxpqbuzgRcxkJDsUywMExnubDkRdluKiAyvo9+TDGGKPY4muhJjti+ks4mc7/hseNDzzTX7Q1zY/XjWT5JhaY1jCXMhHvPVNQildw+MLFQD/uW0QQJFFWH2+y/WUVwikA5cIHBjbEjJ10KTmBuP+PMNWJUzCeNGJ2yMZhFpXwxmK1DWaccZDSpUJjtckViuuAa1hp1ihVfS9stP8/OgqEgw0syYXL9v46DT2T4chKkdeM3pyx+xnyBHr++gKxgO1Ed2vociOfrU+AKY1fY8XxdWb9hW2JJeq1i3Tpxax+SfrtKO7i1cjkS3jpxvBSuQn4lfdzZX2GwPyraz14WxrLn7yCpuRGqFGkhq0XfLxeVbk8tG3sPcnqfS4NrIjgys4eVsrJvJpMsXyujv/ttgK3fh8txTRHfsanplxq1YFkd4Nl/yv1vGus1jfcnIZXYq5s6x6FBw6v1D8rhozVYnNSO1VXoiqg+HaK0vPiPddT4oj1X9wrGhx32weWsaWOYyp3vFEFC2v/A2T8Bf3K3IM4enTEEjP9qr/4VAAD//9YITTs=" + return "eJzsmU1v4zYTx+/5FIM9Pc8h7t2HAosU2+wh3QLJ9hqMybHEmiIFcijD++kLUi8ry7LjRJJ7qQ4GLMrz/3E4nKHG97CjwxoqX+bk6A6AFWtaw6fqOd35dAcgyQunSlbWrOHXOwCAZhQKK4OOP3OkCT2tIcM7gK0iLf06PXoPBgvqS8SLD2V82NlQNndGVI4N9Y1JZPRsO3PjJs+abYZGjBzPo72GGH2U+Hk00JLs6LC3Tg7GLvDE67eW6dRuK7j10f58kl+UJn/wTAWcGG41BZYoFB9WbBn1anNg8qME2prsffIv0SIki2C3wDmNLky8ttYVyGs4lT/h3DqiWTG/OKLZKYMnOSvld09yGcpS8CijF6hJvm61xeED17CW5AQZvpa2ebwbPUkMufU8JScMfv9vp4NH6/l8JvCMHMZD50NqLzmBrcih1pATas4bibg8mHwDyqSFagsAmUo5awoyvBplDCWrMx6RNmz0cOgKxJSDGsMTyEQZ6tAu8h9zbr+HP79HlqeB1b5snUXn061z6BXCKS3Op5uS4huyUvndinNnQ5aXgS+ku4/GA8aQzSgpwU8lCD7ejQENCL4kobaKJCjD5CrU41GRcCVVStBKI5MRh1UxH+/nhhULG5q0F6NYMTDuyANbELYoNTEBGnh+eP4abxRoJGydLaDMD14J1NAwnp9ES98E23xzeFRZTp6hEYAKdSBA4az3EHNHlPcQ6lqUNmXcoBdQ3zpVfMjTWeYoQyZZB8bXX76BQ77kMUd4qRxPWW8Tig25uN47pW1doaNcvaip9EVIQpGDJ2GNHOcsqLBuqYPDUzIeN/OY6bdPDA3cUmfEmfCWORpOhTPEe+t2r/HbjCX9j9osnJodCK82aOReSc5X7ND4QjFfjLEpuyHuQ0CGfa5Eng59sEcPPWGQwSmTpZ3RJWx4yVXcNqUjT4Z9Gu242zNkO6ErZ+tIkKpuPdVW9XbzXCLHttEVWGn1A+Nj8D9hi40yJLvlvIdYvZoZ30eH+P9fpi5R7Ij9USSKWDDnI+/ycaPVD77r4LrAWZqsE7oKi5yz7rau2yvOodZ9vxsb3pt5sw/7Mc+mjTQvZl3gzCXY6xiLoFkJ9HyTAOjUPr6LfgIvHgGntO9b/55vbxACJ7TXQUpny5LkTZa/X69bl7b674JdfOm7cjsJ85Y7v9WEsS5XpRwH1AWKXJlJPfCzlt7f+UrveGp4Pp3a/BoYPBKLHwv129pD1qAfuFSf7696GeCpXofzLT8747vBUPRbSQ45nkaf678DzlL81zqbrPu5QqVxo9/sn/Xf+LNA/lIHbdp7v93C71Fg8ht2Yk07dEHURzsDaR1N87v1qGExm19TCM4P2+9fTGcVwbMtXutKMUpoN3/Tyb9J9c3XCensIQnDiPBNeyz/BAAA//9CQWDd" } diff --git a/metricbeat/module/vsphere/host/_meta/data.json b/metricbeat/module/vsphere/host/_meta/data.json index b2df68bd4f6e..a5bb8bcfb6cb 100644 --- a/metricbeat/module/vsphere/host/_meta/data.json +++ b/metricbeat/module/vsphere/host/_meta/data.json @@ -26,6 +26,30 @@ "mhz": 4521 } }, + "disk":{ + "throughput": { + "bytes": 0 + }, + "device":{ + "latency": { + "ms": 0 + } + }, + "latency":{ + "total": { + "ms": 0 + } + }, + "total":{ + "bytes": 0 + }, + "read":{ + "bytes": 0 + }, + "write":{ + "bytes": 0 + } + }, "memory": { "free": { "bytes": 2822230016 @@ -37,10 +61,66 @@ "bytes": 1472200704 } }, + "network": { + "bandwidth": { + "transmitted": { + "bytes": 0 + }, + "received": { + "bytes": 0 + }, + "total": { + "bytes": 0 + } + }, + "packets":{ + "transmitted": { + "count": 0 + }, + "received": { + "count": 0 + }, + "errors": { + "transmitted": { + "count": 0 + }, + "received": { + "count": 0 + }, + "total": { + "count": 0 + } + }, + "multicast": { + "transmitted": { + "count": 0 + }, + "received": { + "count": 0 + }, + "total": { + "count": 0 + } + }, + "dropped": { + "transmitted": { + "count": 0 + }, + "received": { + "count": 0 + }, + "total": { + "count": 0 + } + } + } + }, "network_names": [ "VM Network" ], - "name": "DC0_H0" + "name": "DC0_H0", + "status": "green", + "uptime": 0 } } } \ No newline at end of file diff --git a/metricbeat/module/vsphere/host/_meta/fields.yml b/metricbeat/module/vsphere/host/_meta/fields.yml index ef6331082eb9..09487fe1a2e2 100644 --- a/metricbeat/module/vsphere/host/_meta/fields.yml +++ b/metricbeat/module/vsphere/host/_meta/fields.yml @@ -8,6 +8,14 @@ type: keyword description: > Host name + - name: status + type: keyword + description: > + The overall health status of a host in the vSphere environment. + - name: uptime + type: double + description: > + The total uptime of a host in the vSphere environment. - name: cpu.used.mhz type: long description: > @@ -20,6 +28,26 @@ type: long description: > Free CPU in Mhz + - name: disk.throughput.bytes + type: double + description: > + The average disk throughput usage over a specified interval. + - name: disk.device.latency.ms + type: double + description: > + Average amount of time it takes to complete an SCSI command from physical device. + - name: disk.latency.total.ms + type: double + description: > + Highest latency value across all disks used by the host. + - name: disk.total.bytes + type: double + description: > + Aggregated disk I/O rate. + - name: disk.read.bytes + type: double + description: > + Average number of kilobytes read from the disk each second. - name: memory.used.bytes type: long description: > @@ -39,3 +67,59 @@ type: keyword description: > Network names + - name: network.bandwidth.transmitted.bytes + type: double + description: > + Average rate at which data was transmitted during the interval. This represents the bandwidth of the network. + - name: network.bandwidth.received.bytes + type: double + description: > + Average rate at which data was received during the interval. This represents the bandwidth of the network. + - name: network.bandwidth.total.bytes + type: double + description: > + Network utilization (combined transmit- and receive-rates). + - name: network.packets.transmitted.count + type: double + description: > + Number of packets transmitted. + - name: network.packets.received.count + type: double + description: > + Number of packets received. + - name: network.packets.errors.transmitted.count + type: double + description: > + Number of packets with errors transmitted. + - name: network.packets.errors.received.count + type: double + description: > + Number of packets with errors received. + - name: network.packets.errors.total.count + type: double + description: > + Total number of packets with errors. + - name: network.packets.multicast.transmitted.count + type: double + description: > + Number of multicast packets transmitted. + - name: network.packets.multicast.received.count + type: double + description: > + Number of multicast packets received. + - name: network.packets.multicast.total.count + type: double + description: > + Total number of multicast packets. + - name: network.packets.dropped.transmitted.count + type: double + description: > + Number of transmitted packets dropped. + - name: network.packets.dropped.received.count + type: double + description: > + Number of received packets dropped. + - name: network.packets.dropped.total.count + type: double + description: > + Total number of packets dropped. diff --git a/metricbeat/module/vsphere/host/data.go b/metricbeat/module/vsphere/host/data.go index 4be828fd4482..802ee775a680 100644 --- a/metricbeat/module/vsphere/host/data.go +++ b/metricbeat/module/vsphere/host/data.go @@ -23,14 +23,30 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) -func eventMapping(hs mo.HostSystem) mapstr.M { - totalCPU := int64(hs.Summary.Hardware.CpuMhz) * int64(hs.Summary.Hardware.NumCpuCores) - freeCPU := int64(totalCPU) - int64(hs.Summary.QuickStats.OverallCpuUsage) - usedMemory := int64(hs.Summary.QuickStats.OverallMemoryUsage) * 1024 * 1024 - freeMemory := int64(hs.Summary.Hardware.MemorySize) - usedMemory +func (m *MetricSet) eventMapping(hs mo.HostSystem, perfMertics *PerformanceMetrics, networkNames []string) mapstr.M { + totalErrorPacketsCount := perfMertics.NetErrorsTransmitted + perfMertics.NetErrorsReceived + totalMulticastPacketsCount := perfMertics.NetMulticastTransmitted + perfMertics.NetMulticastReceived + totalDroppedPacketsCount := perfMertics.NetDroppedTransmitted + perfMertics.NetDroppedReceived + totalCPU := int64(0) + freeCPU := int64(0) + freeMemory := int64(0) + totalMemory := int64(0) + usedMemory := int64(0) + + if hs.Summary.Hardware != nil { + totalCPU = int64(hs.Summary.Hardware.CpuMhz) * int64(hs.Summary.Hardware.NumCpuCores) + freeCPU = int64(totalCPU) - int64(hs.Summary.QuickStats.OverallCpuUsage) + usedMemory = int64(hs.Summary.QuickStats.OverallMemoryUsage) * 1024 * 1024 + freeMemory = int64(hs.Summary.Hardware.MemorySize) - usedMemory + totalMemory = hs.Summary.Hardware.MemorySize + } else { + m.Logger().Debug("'Hardware' or 'Summary' data not found. This is either a parsing error from vsphere library, an error trying to reach host/guest or incomplete information returned from host/guest") + } event := mapstr.M{ - "name": hs.Summary.Config.Name, + "name": hs.Summary.Config.Name, + "status": hs.Summary.OverallStatus, + "uptime": hs.Summary.QuickStats.Uptime, "cpu": mapstr.M{ "used": mapstr.M{ "mhz": hs.Summary.QuickStats.OverallCpuUsage, @@ -42,17 +58,96 @@ func eventMapping(hs mo.HostSystem) mapstr.M { "mhz": freeCPU, }, }, + "disk": mapstr.M{ + "device": mapstr.M{ + "latency": mapstr.M{ + "ms": perfMertics.DiskDeviceLatency, + }, + }, + "latency": mapstr.M{ + "total": mapstr.M{ + "ms": perfMertics.DiskMaxTotalLatency, + }, + }, + "total": mapstr.M{ + "bytes": perfMertics.DiskUsage, + }, + "read": mapstr.M{ + "bytes": perfMertics.DiskRead, + }, + "write": mapstr.M{ + "bytes": perfMertics.DiskWrite, + }, + }, "memory": mapstr.M{ "used": mapstr.M{ "bytes": usedMemory, }, "total": mapstr.M{ - "bytes": hs.Summary.Hardware.MemorySize, + "bytes": totalMemory, }, "free": mapstr.M{ "bytes": freeMemory, }, }, + "network": mapstr.M{ + "bandwidth": mapstr.M{ + "transmitted": mapstr.M{ + "bytes": perfMertics.NetTransmitted, + }, + "received": mapstr.M{ + "bytes": perfMertics.NetReceived, + }, + "total": mapstr.M{ + "bytes": perfMertics.NetUsage, + }, + }, + "packets": mapstr.M{ + "transmitted": mapstr.M{ + "count": perfMertics.NetPacketTransmitted, + }, + "received": mapstr.M{ + "count": perfMertics.NetPacketReceived, + }, + "errors": mapstr.M{ + "transmitted": mapstr.M{ + "count": perfMertics.NetErrorsTransmitted, + }, + "received": mapstr.M{ + "count": perfMertics.NetErrorsReceived, + }, + "total": mapstr.M{ + "count": totalErrorPacketsCount, + }, + }, + "multicast": mapstr.M{ + "transmitted": mapstr.M{ + "count": perfMertics.NetMulticastTransmitted, + }, + "received": mapstr.M{ + "count": perfMertics.NetMulticastReceived, + }, + "total": mapstr.M{ + "count": totalMulticastPacketsCount, + }, + }, + "dropped": mapstr.M{ + "transmitted": mapstr.M{ + "count": perfMertics.NetDroppedTransmitted, + }, + "received": mapstr.M{ + "count": perfMertics.NetDroppedReceived, + }, + "total": mapstr.M{ + "count": totalDroppedPacketsCount, + }, + }, + }, + }, + } + + if len(networkNames) > 0 { + event.Put("network_names", networkNames) } return event diff --git a/metricbeat/module/vsphere/host/data_test.go b/metricbeat/module/vsphere/host/data_test.go index 2a991a7c265c..91d86c4acc7d 100644 --- a/metricbeat/module/vsphere/host/data_test.go +++ b/metricbeat/module/vsphere/host/data_test.go @@ -27,6 +27,7 @@ import ( ) func TestEventMapping(t *testing.T) { + var m *MetricSet var HostSystemTest = mo.HostSystem{ Summary: types.HostListSummary{ Host: &types.ManagedObjectReference{Type: "HostSystem", Value: "ha-host"}, @@ -45,7 +46,7 @@ func TestEventMapping(t *testing.T) { }, } - event := eventMapping(HostSystemTest) + event := m.eventMapping(HostSystemTest, &PerformanceMetrics{}, []string{}) cpuUsed, _ := event.GetValue("cpu.used.mhz") assert.EqualValues(t, 67, cpuUsed) diff --git a/metricbeat/module/vsphere/host/host.go b/metricbeat/module/vsphere/host/host.go index 457d4a654525..98c0a6db65b9 100644 --- a/metricbeat/module/vsphere/host/host.go +++ b/metricbeat/module/vsphere/host/host.go @@ -19,18 +19,16 @@ package host import ( "context" - "errors" "fmt" "strings" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/vsphere" - "github.com/elastic/elastic-agent-libs/mapstr" "github.com/vmware/govmomi" + "github.com/vmware/govmomi/performance" "github.com/vmware/govmomi/property" "github.com/vmware/govmomi/view" - "github.com/vmware/govmomi/vim25" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" ) @@ -56,6 +54,26 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ms}, nil } +// Structure to hold performance metrics values +type PerformanceMetrics struct { + NetUsage int64 + NetDroppedTransmitted int64 + NetDroppedReceived int64 + NetMulticastTransmitted int64 + NetMulticastReceived int64 + NetErrorsTransmitted int64 + NetErrorsReceived int64 + NetPacketTransmitted int64 + NetPacketReceived int64 + NetReceived int64 + NetTransmitted int64 + DiskWrite int64 + DiskRead int64 + DiskUsage int64 + DiskMaxTotalLatency int64 + DiskDeviceLatency int64 +} + // Fetch methods implements the data gathering and data conversion to the right // format. It publishes the event which is then forwarded to the output. In case // of an error set the Error field of mb.Event or simply call report.Error(). @@ -92,83 +110,152 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { // Retrieve summary property for all hosts. var hst []mo.HostSystem - err = v.Retrieve(ctx, []string{"HostSystem"}, []string{"summary"}, &hst) + err = v.Retrieve(ctx, []string{"HostSystem"}, []string{"summary", "network"}, &hst) if err != nil { return fmt.Errorf("error in Retrieve: %w", err) } - for _, hs := range hst { + // Create a performance manager + perfManager := performance.NewManager(c) - event := mapstr.M{} + // Retrieve metric IDs for the specified metric names + metrics, err := perfManager.CounterInfoByName(ctx) + if err != nil { + return fmt.Errorf("failed to retrieve metrics: %w", err) + } - event["name"] = hs.Summary.Config.Name - event.Put("cpu.used.mhz", hs.Summary.QuickStats.OverallCpuUsage) - event.Put("memory.used.bytes", int64(hs.Summary.QuickStats.OverallMemoryUsage)*1024*1024) + // Define metrics to be collected + metricNames := []string{ + "disk.deviceLatency.average", + "disk.maxTotalLatency.latest", + "disk.usage.average", + "disk.read.average", + "disk.write.average", + "net.transmitted.average", + "net.received.average", + "net.usage.average", + "net.packetsTx.summation", + "net.packetsRx.summation", + "net.errorsTx.summation", + "net.errorsRx.summation", + "net.multicastTx.summation", + "net.multicastRx.summation", + "net.droppedTx.summation", + "net.droppedRx.summation", + } - if hs.Summary.Hardware != nil { - totalCPU := int64(hs.Summary.Hardware.CpuMhz) * int64(hs.Summary.Hardware.NumCpuCores) - event.Put("cpu.total.mhz", totalCPU) - event.Put("cpu.free.mhz", int64(totalCPU)-int64(hs.Summary.QuickStats.OverallCpuUsage)) - event.Put("memory.free.bytes", int64(hs.Summary.Hardware.MemorySize)-(int64(hs.Summary.QuickStats.OverallMemoryUsage)*1024*1024)) - event.Put("memory.total.bytes", hs.Summary.Hardware.MemorySize) - } else { - m.Logger().Debug("'Hardware' or 'Summary' data not found. This is either a parsing error from vsphere library, an error trying to reach host/guest or incomplete information returned from host/guest") - } + // Define refrence of structure + var metricsVar PerformanceMetrics - if hs.Summary.Host != nil { - networkNames, err := getNetworkNames(ctx, c, hs.Summary.Host.Reference()) - if err != nil { - m.Logger().Debugf("error trying to get network names: %s", err.Error()) - } else { - if len(networkNames) > 0 { - event["network_names"] = networkNames - } - } - } - reporter.Event(mb.Event{ - MetricSetFields: event, - }) + // Map metric names to struture fields + metricMap := map[string]*int64{ + "disk.deviceLatency.average": &metricsVar.DiskDeviceLatency, + "disk.maxTotalLatency.latest": &metricsVar.DiskMaxTotalLatency, + "net.usage.average": &metricsVar.NetUsage, + "disk.usage.average": &metricsVar.DiskUsage, + "disk.read.average": &metricsVar.DiskRead, + "disk.write.average": &metricsVar.DiskWrite, + "net.transmitted.average": &metricsVar.NetTransmitted, + "net.received.average": &metricsVar.NetReceived, + "net.packetsTx.summation": &metricsVar.NetPacketTransmitted, + "net.packetsRx.summation": &metricsVar.NetPacketReceived, + "net.errorsTx.summation": &metricsVar.NetErrorsTransmitted, + "net.errorsRx.summation": &metricsVar.NetErrorsReceived, + "net.multicastTx.summation": &metricsVar.NetMulticastTransmitted, + "net.multicastRx.summation": &metricsVar.NetMulticastReceived, + "net.droppedTx.summation": &metricsVar.NetDroppedTransmitted, + "net.droppedRx.summation": &metricsVar.NetDroppedReceived, } - return nil -} + // Map metric IDs to metric names + metricNamesById := make(map[int32]string) + for name, metric := range metrics { + metricNamesById[metric.Key] = name + } -func getNetworkNames(ctx context.Context, c *vim25.Client, ref types.ManagedObjectReference) ([]string, error) { - var outputNetworkNames []string + var spec types.PerfQuerySpec + var metricIDs []types.PerfMetricId - pc := property.DefaultCollector(c) + for _, metricName := range metricNames { + metric, exists := metrics[metricName] + if !exists { + m.Logger().Debug("Metric %v not found", metricName) + continue + } - var hs mo.HostSystem - err := pc.RetrieveOne(ctx, ref, []string{"network"}, &hs) - if err != nil { - return nil, fmt.Errorf("error retrieving host information: %v", err) + metricIDs = append(metricIDs, types.PerfMetricId{ + CounterId: metric.Key, + }) } - if len(hs.Network) == 0 { - return nil, errors.New("no networks found") - } + pc := property.DefaultCollector(c) + for _, hs := range hst { + var networkRefs []types.ManagedObjectReference + for _, obj := range hs.Network { + if obj.Type == "Network" { + networkRefs = append(networkRefs, obj) + } + } - var networkRefs []types.ManagedObjectReference - for _, obj := range hs.Network { - if obj.Type == "Network" { - networkRefs = append(networkRefs, obj) + var nets []mo.Network + if len(networkRefs) > 0 { + err = pc.Retrieve(ctx, networkRefs, []string{"name"}, &nets) + if err != nil { + m.Logger().Errorf("error retrieving network from host: %v", err) + } } - } - if len(networkRefs) == 0 { - return nil, errors.New("no networks found") - } + var outputNetworkNames []string + for _, net := range nets { + name := strings.Replace(net.Name, ".", "_", -1) + outputNetworkNames = append(outputNetworkNames, name) + } - var nets []mo.Network - err = pc.Retrieve(ctx, networkRefs, []string{"name"}, &nets) - if err != nil { - return nil, fmt.Errorf("error retrieving network from host: %v", err) - } + spec = types.PerfQuerySpec{ + Entity: hs.Reference(), + MetricId: metricIDs, + MaxSample: 1, + IntervalId: 20, // right now we are only grabbing real time metrics from the performance manager + } + + // Query performance data + samples, err := perfManager.Query(ctx, []types.PerfQuerySpec{spec}) + if err != nil { + m.Logger().Debug("Failed to query performance data: %v", err) + continue + } - for _, net := range nets { - name := strings.Replace(net.Name, ".", "_", -1) - outputNetworkNames = append(outputNetworkNames, name) + if len(samples) > 0 { + entityMetrics, ok := samples[0].(*types.PerfEntityMetric) + if !ok { + m.Logger().Debug("Unexpected metric type") + continue + } + + for _, value := range entityMetrics.Value { + metricSeries, ok := value.(*types.PerfMetricIntSeries) + if !ok { + m.Logger().Debug("Unexpected metric series type") + continue + } + + if len(metricSeries.Value) > 0 { + metricName := metricNamesById[metricSeries.Id.CounterId] + if assignValue, exists := metricMap[metricName]; exists { + *assignValue = metricSeries.Value[0] // Assign the metric value to the variable + } + } else { + m.Logger().Debug("Metric %v: No result found\n", metricNamesById[metricSeries.Id.CounterId]) + } + } + } else { + m.Logger().Debug("No samples returned from performance manager") + } + + reporter.Event(mb.Event{ + MetricSetFields: m.eventMapping(hs, &metricsVar, outputNetworkNames), + }) } - return outputNetworkNames, nil + return nil } From b505cec9cfdcf2b31add6577a05fb17918ece141 Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Mon, 5 Aug 2024 11:16:22 +0530 Subject: [PATCH 02/33] add changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2e568740aa5e..d0dc12020ec7 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -63,6 +63,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Add support of Graphite series 1.1.0+ tagging extension for statsd module. {pull}39619[39619] - Remove fallback to the node limit for the `kubernetes.pod.cpu.usage.limit.pct` and `kubernetes.pod.memory.usage.limit.pct` metrics calculation - Add support for Kibana status metricset in v8 format {pull}40275[40275] +- Add new metrics for the Vsphere Host metricset. {pull}40429[40429] *Osquerybeat* From 8c61c2ec9c4c34ac771fe73584829a5f6b1c55b8 Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Mon, 5 Aug 2024 12:19:28 +0530 Subject: [PATCH 03/33] update github.com/vmware/govmomi version --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 24c3f6e05d65..e3b12847effe 100644 --- a/go.mod +++ b/go.mod @@ -134,7 +134,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b github.com/ugorji/go/codec v1.1.8 - github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41 + github.com/vmware/govmomi v0.39.0 go.elastic.co/ecszap v1.0.2 go.elastic.co/go-licence-detector v0.6.1 go.etcd.io/bbolt v1.3.10 diff --git a/go.sum b/go.sum index ea4ec0dee0d2..8be4c07a4a09 100644 --- a/go.sum +++ b/go.sum @@ -1651,6 +1651,8 @@ github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvV github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41 h1:NeNpIvfvaFOh0BH7nMEljE5Rk/VJlxhm58M41SeOD20= github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= +github.com/vmware/govmomi v0.39.0 h1:soLZ08Q2zvjRSinNup8xVlw0KDDCJPPA1rIDmBhi7As= +github.com/vmware/govmomi v0.39.0/go.mod h1:oHzAQ1r6152zYDGcUqeK+EO8LhKo5wjtvWZBGHws2Hc= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= From b54e54535fb71e2644d6e944dbf71c479d51d488 Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Mon, 5 Aug 2024 15:48:34 +0530 Subject: [PATCH 04/33] update go.sum --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 8be4c07a4a09..263b89f9b0bc 100644 --- a/go.sum +++ b/go.sum @@ -1649,8 +1649,6 @@ github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhg github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA= github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41 h1:NeNpIvfvaFOh0BH7nMEljE5Rk/VJlxhm58M41SeOD20= -github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/vmware/govmomi v0.39.0 h1:soLZ08Q2zvjRSinNup8xVlw0KDDCJPPA1rIDmBhi7As= github.com/vmware/govmomi v0.39.0/go.mod h1:oHzAQ1r6152zYDGcUqeK+EO8LhKo5wjtvWZBGHws2Hc= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= From ef8dc8c9feb9f638b92307a208eb4bcbf46cc5c0 Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Mon, 5 Aug 2024 16:06:17 +0530 Subject: [PATCH 05/33] make check for beats --- NOTICE.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 67076b74e924..f6c61f7ea796 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -22673,11 +22673,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/vmware/govmomi -Version: v0.0.0-20170802214208-2cad15190b41 +Version: v0.39.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/vmware/govmomi@v0.0.0-20170802214208-2cad15190b41/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/vmware/govmomi@v0.39.0/LICENSE.txt: Apache License From 2aa91c35af930071069223a81bea8b1ea54c5a39 Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Mon, 5 Aug 2024 16:22:00 +0530 Subject: [PATCH 06/33] golint-ci fix --- metricbeat/module/vsphere/host/data.go | 4 ++-- metricbeat/module/vsphere/host/host.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/metricbeat/module/vsphere/host/data.go b/metricbeat/module/vsphere/host/data.go index 802ee775a680..acd939cd535c 100644 --- a/metricbeat/module/vsphere/host/data.go +++ b/metricbeat/module/vsphere/host/data.go @@ -35,9 +35,9 @@ func (m *MetricSet) eventMapping(hs mo.HostSystem, perfMertics *PerformanceMetri if hs.Summary.Hardware != nil { totalCPU = int64(hs.Summary.Hardware.CpuMhz) * int64(hs.Summary.Hardware.NumCpuCores) - freeCPU = int64(totalCPU) - int64(hs.Summary.QuickStats.OverallCpuUsage) + freeCPU = totalCPU - int64(hs.Summary.QuickStats.OverallCpuUsage) usedMemory = int64(hs.Summary.QuickStats.OverallMemoryUsage) * 1024 * 1024 - freeMemory = int64(hs.Summary.Hardware.MemorySize) - usedMemory + freeMemory = hs.Summary.Hardware.MemorySize - usedMemory totalMemory = hs.Summary.Hardware.MemorySize } else { m.Logger().Debug("'Hardware' or 'Summary' data not found. This is either a parsing error from vsphere library, an error trying to reach host/guest or incomplete information returned from host/guest") diff --git a/metricbeat/module/vsphere/host/host.go b/metricbeat/module/vsphere/host/host.go index 98c0a6db65b9..ababa3aba406 100644 --- a/metricbeat/module/vsphere/host/host.go +++ b/metricbeat/module/vsphere/host/host.go @@ -144,7 +144,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { "net.droppedRx.summation", } - // Define refrence of structure + // Define reference of structure var metricsVar PerformanceMetrics // Map metric names to struture fields @@ -174,7 +174,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { } var spec types.PerfQuerySpec - var metricIDs []types.PerfMetricId + metricIDs := make([]types.PerfMetricId, 0, len(metricNames)) for _, metricName := range metricNames { metric, exists := metrics[metricName] From 492ec96c5b15299c18741ee8d77b4b13f3202f5b Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Mon, 5 Aug 2024 16:28:56 +0530 Subject: [PATCH 07/33] remove extra fields --- metricbeat/module/vsphere/host/_meta/data.json | 3 --- metricbeat/module/vsphere/host/_meta/fields.yml | 4 ---- 2 files changed, 7 deletions(-) diff --git a/metricbeat/module/vsphere/host/_meta/data.json b/metricbeat/module/vsphere/host/_meta/data.json index a5bb8bcfb6cb..e7c203a37ef5 100644 --- a/metricbeat/module/vsphere/host/_meta/data.json +++ b/metricbeat/module/vsphere/host/_meta/data.json @@ -27,9 +27,6 @@ } }, "disk":{ - "throughput": { - "bytes": 0 - }, "device":{ "latency": { "ms": 0 diff --git a/metricbeat/module/vsphere/host/_meta/fields.yml b/metricbeat/module/vsphere/host/_meta/fields.yml index 09487fe1a2e2..1f8c3c53b569 100644 --- a/metricbeat/module/vsphere/host/_meta/fields.yml +++ b/metricbeat/module/vsphere/host/_meta/fields.yml @@ -28,10 +28,6 @@ type: long description: > Free CPU in Mhz - - name: disk.throughput.bytes - type: double - description: > - The average disk throughput usage over a specified interval. - name: disk.device.latency.ms type: double description: > From dd6e96ec8e5f3e7cd36c3dfe57934e7a58dafbdb Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Wed, 7 Aug 2024 11:20:59 +0530 Subject: [PATCH 08/33] update the field mappings --- .../module/vsphere/host/_meta/data.json | 223 +++++++++--------- .../module/vsphere/host/_meta/fields.yml | 89 +++---- metricbeat/module/vsphere/host/data.go | 21 +- metricbeat/module/vsphere/host/host.go | 35 +-- 4 files changed, 188 insertions(+), 180 deletions(-) diff --git a/metricbeat/module/vsphere/host/_meta/data.json b/metricbeat/module/vsphere/host/_meta/data.json index e7c203a37ef5..0ca79096c234 100644 --- a/metricbeat/module/vsphere/host/_meta/data.json +++ b/metricbeat/module/vsphere/host/_meta/data.json @@ -1,123 +1,128 @@ { "@timestamp": "2022-09-06T06:41:22.128Z", "metricset": { - "name": "host", - "period": 10000 + "name": "host", + "period": 10000 }, "service": { - "address": "https://localhost:8989/sdk", - "type": "vsphere" + "address": "https://localhost:8989/sdk", + "type": "vsphere" }, "event": { - "module": "vsphere", - "duration": 23519250, - "dataset": "vsphere.host" + "module": "vsphere", + "duration": 23519250, + "dataset": "vsphere.host" }, "vsphere": { - "host": { - "cpu": { - "used": { - "mhz": 67 - }, - "total": { - "mhz": 4588 - }, - "free": { - "mhz": 4521 - } - }, - "disk":{ - "device":{ - "latency": { - "ms": 0 - } + "host": { + "cpu": { + "used": { + "mhz": 67 + }, + "total": { + "mhz": 4588 + }, + "free": { + "mhz": 4521 + } }, - "latency":{ - "total": { - "ms": 0 - } + "disk": { + "capacity": { + "usage": { + "bytes": 0 + } + }, + "devicelatency": { + "average": { + "ms": 0 + } + }, + "latency": { + "total": { + "ms": 18 + } + }, + "total": { + "bytes": 262000 + }, + "read": { + "bytes": 13000 + }, + "write": { + "bytes": 248000 + } }, - "total":{ - "bytes": 0 + "memory": { + "free": { + "bytes": 2822230016 + }, + "total": { + "bytes": 4294430720 + }, + "used": { + "bytes": 1472200704 + } }, - "read":{ - "bytes": 0 + "network": { + "bandwidth": { + "total": { + "bytes": 372000 + }, + "transmitted": { + "bytes": 0 + }, + "received": { + "bytes": 371000 + } + }, + "packets": { + "received": { + "count": 9463 + }, + "errors": { + "transmitted": { + "count": 0 + }, + "received": { + "count": 0 + }, + "total": { + "count": 0 + } + }, + "multicast": { + "total": { + "count": 6679 + }, + "transmitted": { + "count": 0 + }, + "received": { + "count": 6679 + } + }, + "dropped": { + "received": { + "count": 0 + }, + "total": { + "count": 0 + }, + "transmitted": { + "count": 0 + } + }, + "transmitted": { + "count": 54 + } + } }, - "write":{ - "bytes": 0 - } - }, - "memory": { - "free": { - "bytes": 2822230016 - }, - "total": { - "bytes": 4294430720 - }, - "used": { - "bytes": 1472200704 - } - }, - "network": { - "bandwidth": { - "transmitted": { - "bytes": 0 - }, - "received": { - "bytes": 0 - }, - "total": { - "bytes": 0 - } - }, - "packets":{ - "transmitted": { - "count": 0 - }, - "received": { - "count": 0 - }, - "errors": { - "transmitted": { - "count": 0 - }, - "received": { - "count": 0 - }, - "total": { - "count": 0 - } - }, - "multicast": { - "transmitted": { - "count": 0 - }, - "received": { - "count": 0 - }, - "total": { - "count": 0 - } - }, - "dropped": { - "transmitted": { - "count": 0 - }, - "received": { - "count": 0 - }, - "total": { - "count": 0 - } - } - } - }, - "network_names": [ - "VM Network" - ], - "name": "DC0_H0", - "status": "green", - "uptime": 0 - } + "network_names": [ + "VM Network" + ], + "name": "DC0_H0", + "status": "green", + "uptime": 1728865 + } } -} \ No newline at end of file +} diff --git a/metricbeat/module/vsphere/host/_meta/fields.yml b/metricbeat/module/vsphere/host/_meta/fields.yml index 1f8c3c53b569..2a83188d2c7a 100644 --- a/metricbeat/module/vsphere/host/_meta/fields.yml +++ b/metricbeat/module/vsphere/host/_meta/fields.yml @@ -4,18 +4,6 @@ host release: ga fields: - - name: name - type: keyword - description: > - Host name - - name: status - type: keyword - description: > - The overall health status of a host in the vSphere environment. - - name: uptime - type: double - description: > - The total uptime of a host in the vSphere environment. - name: cpu.used.mhz type: long description: > @@ -28,94 +16,115 @@ type: long description: > Free CPU in Mhz - - name: disk.device.latency.ms - type: double + - name: disk.capacity.usage.bytes + type: long + description: > + The amount of storage capacity currently being consumed by or on the entity. + - name: disk.devicelatency.average.ms + type: long description: > Average amount of time it takes to complete an SCSI command from physical device. - name: disk.latency.total.ms - type: double + type: long description: > Highest latency value across all disks used by the host. + - name: disk.read.bytes + type: long + description: > + Average number of bytes read from the disk each second. + format: bytes - name: disk.total.bytes - type: double + type: long description: > Aggregated disk I/O rate. - - name: disk.read.bytes - type: double - description: > - Average number of kilobytes read from the disk each second. - - name: memory.used.bytes + format: bytes + - name: memory.free.bytes type: long description: > - Used Memory in bytes + Free Memory in bytes format: bytes - name: memory.total.bytes type: long description: > Total Memory in bytes format: bytes - - name: memory.free.bytes + - name: memory.used.bytes type: long description: > - Free Memory in bytes + Used Memory in bytes format: bytes + - name: name + type: keyword + description: > + Host name - name: network_names type: keyword description: > Network names - name: network.bandwidth.transmitted.bytes - type: double + type: long description: > Average rate at which data was transmitted during the interval. This represents the bandwidth of the network. + format: bytes - name: network.bandwidth.received.bytes - type: double + type: long description: > Average rate at which data was received during the interval. This represents the bandwidth of the network. + format: bytes - name: network.bandwidth.total.bytes - type: double + type: long description: > - Network utilization (combined transmit- and receive-rates). + Network utilization (combined transmit-rates and receive-rates). + format: bytes - name: network.packets.transmitted.count - type: double + type: long description: > Number of packets transmitted. - name: network.packets.received.count - type: double + type: long description: > Number of packets received. - name: network.packets.errors.transmitted.count - type: double + type: long description: > Number of packets with errors transmitted. - name: network.packets.errors.received.count - type: double + type: long description: > Number of packets with errors received. - name: network.packets.errors.total.count - type: double + type: long description: > Total number of packets with errors. - name: network.packets.multicast.transmitted.count - type: double + type: long description: > Number of multicast packets transmitted. - name: network.packets.multicast.received.count - type: double + type: long description: > Number of multicast packets received. - name: network.packets.multicast.total.count - type: double + type: long description: > Total number of multicast packets. - name: network.packets.dropped.transmitted.count - type: double + type: long description: > Number of transmitted packets dropped. - name: network.packets.dropped.received.count - type: double + type: long description: > Number of received packets dropped. - name: network.packets.dropped.total.count - type: double + type: long + description: > + Total number of packets dropped. + - name: status + type: keyword + description: > + The overall health status of a host in the vSphere environment. + - name: uptime + type: long description: > - Total number of packets dropped. + The total uptime of a host in seconds within the vSphere environment. diff --git a/metricbeat/module/vsphere/host/data.go b/metricbeat/module/vsphere/host/data.go index acd939cd535c..b60d2356dee4 100644 --- a/metricbeat/module/vsphere/host/data.go +++ b/metricbeat/module/vsphere/host/data.go @@ -59,8 +59,13 @@ func (m *MetricSet) eventMapping(hs mo.HostSystem, perfMertics *PerformanceMetri }, }, "disk": mapstr.M{ - "device": mapstr.M{ - "latency": mapstr.M{ + "capacity": mapstr.M{ + "usage": mapstr.M{ + "bytes": int64(perfMertics.DiskUsage) * 1000, + }, + }, + "devicelatency": mapstr.M{ + "average": mapstr.M{ "ms": perfMertics.DiskDeviceLatency, }, }, @@ -70,13 +75,13 @@ func (m *MetricSet) eventMapping(hs mo.HostSystem, perfMertics *PerformanceMetri }, }, "total": mapstr.M{ - "bytes": perfMertics.DiskUsage, + "bytes": int64(perfMertics.DiskUsage) * 1000, }, "read": mapstr.M{ - "bytes": perfMertics.DiskRead, + "bytes": int64(perfMertics.DiskRead) * 1000, }, "write": mapstr.M{ - "bytes": perfMertics.DiskWrite, + "bytes": int64(perfMertics.DiskWrite) * 1000, }, }, "memory": mapstr.M{ @@ -93,13 +98,13 @@ func (m *MetricSet) eventMapping(hs mo.HostSystem, perfMertics *PerformanceMetri "network": mapstr.M{ "bandwidth": mapstr.M{ "transmitted": mapstr.M{ - "bytes": perfMertics.NetTransmitted, + "bytes": int64(perfMertics.NetTransmitted) * 1000, }, "received": mapstr.M{ - "bytes": perfMertics.NetReceived, + "bytes": int64(perfMertics.NetReceived) * 1000, }, "total": mapstr.M{ - "bytes": perfMertics.NetUsage, + "bytes": int64(perfMertics.NetUsage) * 1000, }, }, "packets": mapstr.M{ diff --git a/metricbeat/module/vsphere/host/host.go b/metricbeat/module/vsphere/host/host.go index ababa3aba406..4190e34cd597 100644 --- a/metricbeat/module/vsphere/host/host.go +++ b/metricbeat/module/vsphere/host/host.go @@ -72,6 +72,7 @@ type PerformanceMetrics struct { DiskUsage int64 DiskMaxTotalLatency int64 DiskDeviceLatency int64 + DiskCapacityUsage int64 } // Fetch methods implements the data gathering and data conversion to the right @@ -126,6 +127,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { // Define metrics to be collected metricNames := []string{ + "disk.capacity.usage.average", "disk.deviceLatency.average", "disk.maxTotalLatency.latest", "disk.usage.average", @@ -149,6 +151,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { // Map metric names to struture fields metricMap := map[string]*int64{ + "disk.capacity.usage.average": &metricsVar.DiskCapacityUsage, "disk.deviceLatency.average": &metricsVar.DiskDeviceLatency, "disk.maxTotalLatency.latest": &metricsVar.DiskMaxTotalLatency, "net.usage.average": &metricsVar.NetUsage, @@ -167,19 +170,13 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { "net.droppedRx.summation": &metricsVar.NetDroppedReceived, } - // Map metric IDs to metric names - metricNamesById := make(map[int32]string) - for name, metric := range metrics { - metricNamesById[metric.Key] = name - } - var spec types.PerfQuerySpec metricIDs := make([]types.PerfMetricId, 0, len(metricNames)) for _, metricName := range metricNames { metric, exists := metrics[metricName] if !exists { - m.Logger().Debug("Metric %v not found", metricName) + m.Logger().Debug("Metric ", metricName, " not found") continue } @@ -226,26 +223,18 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { } if len(samples) > 0 { - entityMetrics, ok := samples[0].(*types.PerfEntityMetric) - if !ok { - m.Logger().Debug("Unexpected metric type") - continue + results, err := perfManager.ToMetricSeries(ctx, samples) + if err != nil { + m.Logger().Debug("Failed to query performance data: %v", err) } - for _, value := range entityMetrics.Value { - metricSeries, ok := value.(*types.PerfMetricIntSeries) - if !ok { - m.Logger().Debug("Unexpected metric series type") - continue - } - - if len(metricSeries.Value) > 0 { - metricName := metricNamesById[metricSeries.Id.CounterId] - if assignValue, exists := metricMap[metricName]; exists { - *assignValue = metricSeries.Value[0] // Assign the metric value to the variable + for _, result := range results[0].Value { + if len(result.Value) > 0 { + if assignValue, exists := metricMap[result.Name]; exists { + *assignValue = result.Value[0] // Assign the metric value to the variable } } else { - m.Logger().Debug("Metric %v: No result found\n", metricNamesById[metricSeries.Id.CounterId]) + m.Logger().Debug("Metric ", result.Name, ": No result found") } } } else { From 8af84b055e369a41c98cbbf81f8a72f45fef177f Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Wed, 7 Aug 2024 11:22:31 +0530 Subject: [PATCH 09/33] mage update --- metricbeat/docs/fields.asciidoc | 134 +++++++++++++++------------- metricbeat/module/vsphere/fields.go | 2 +- 2 files changed, 73 insertions(+), 63 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 840ef22dbe43..8916497a1b00 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -66936,36 +66936,6 @@ host -*`vsphere.host.name`*:: -+ --- -Host name - - -type: keyword - --- - -*`vsphere.host.status`*:: -+ --- -The overall health status of a host in the vSphere environment. - - -type: keyword - --- - -*`vsphere.host.uptime`*:: -+ --- -The total uptime of a host in the vSphere environment. - - -type: double - --- - *`vsphere.host.cpu.used.mhz`*:: + -- @@ -66996,23 +66966,23 @@ type: long -- -*`vsphere.host.disk.throughput.bytes`*:: +*`vsphere.host.disk.capacity.usage.bytes`*:: + -- -The average disk throughput usage over a specified interval. +The amount of storage capacity currently being consumed by or on the entity. -type: double +type: long -- -*`vsphere.host.disk.device.latency.ms`*:: +*`vsphere.host.disk.devicelatency.average.ms`*:: + -- Average amount of time it takes to complete an SCSI command from physical device. -type: double +type: long -- @@ -67022,34 +66992,38 @@ type: double Highest latency value across all disks used by the host. -type: double +type: long -- -*`vsphere.host.disk.total.bytes`*:: +*`vsphere.host.disk.read.bytes`*:: + -- -Aggregated disk I/O rate. +Average number of bytes read from the disk each second. -type: double +type: long + +format: bytes -- -*`vsphere.host.disk.read.bytes`*:: +*`vsphere.host.disk.total.bytes`*:: + -- -Average number of kilobytes read from the disk each second. +Aggregated disk I/O rate. -type: double +type: long + +format: bytes -- -*`vsphere.host.memory.used.bytes`*:: +*`vsphere.host.memory.free.bytes`*:: + -- -Used Memory in bytes +Free Memory in bytes type: long @@ -67070,10 +67044,10 @@ format: bytes -- -*`vsphere.host.memory.free.bytes`*:: +*`vsphere.host.memory.used.bytes`*:: + -- -Free Memory in bytes +Used Memory in bytes type: long @@ -67082,6 +67056,16 @@ format: bytes -- +*`vsphere.host.name`*:: ++ +-- +Host name + + +type: keyword + +-- + *`vsphere.host.network_names`*:: + -- @@ -67098,7 +67082,9 @@ type: keyword Average rate at which data was transmitted during the interval. This represents the bandwidth of the network. -type: double +type: long + +format: bytes -- @@ -67108,17 +67094,21 @@ type: double Average rate at which data was received during the interval. This represents the bandwidth of the network. -type: double +type: long + +format: bytes -- *`vsphere.host.network.bandwidth.total.bytes`*:: + -- -Network utilization (combined transmit- and receive-rates). +Network utilization (combined transmit-rates and receive-rates). -type: double +type: long + +format: bytes -- @@ -67128,7 +67118,7 @@ type: double Number of packets transmitted. -type: double +type: long -- @@ -67138,7 +67128,7 @@ type: double Number of packets received. -type: double +type: long -- @@ -67148,7 +67138,7 @@ type: double Number of packets with errors transmitted. -type: double +type: long -- @@ -67158,7 +67148,7 @@ type: double Number of packets with errors received. -type: double +type: long -- @@ -67168,7 +67158,7 @@ type: double Total number of packets with errors. -type: double +type: long -- @@ -67178,7 +67168,7 @@ type: double Number of multicast packets transmitted. -type: double +type: long -- @@ -67188,7 +67178,7 @@ type: double Number of multicast packets received. -type: double +type: long -- @@ -67198,7 +67188,7 @@ type: double Total number of multicast packets. -type: double +type: long -- @@ -67208,7 +67198,7 @@ type: double Number of transmitted packets dropped. -type: double +type: long -- @@ -67218,17 +67208,37 @@ type: double Number of received packets dropped. -type: double +type: long -- *`vsphere.host.network.packets.dropped.total.count`*:: + -- -Total number of packets dropped. +Total number of packets dropped. -type: double +type: long + +-- + +*`vsphere.host.status`*:: ++ +-- +The overall health status of a host in the vSphere environment. + + +type: keyword + +-- + +*`vsphere.host.uptime`*:: ++ +-- +The total uptime of a host in seconds within the vSphere environment. + + +type: long -- diff --git a/metricbeat/module/vsphere/fields.go b/metricbeat/module/vsphere/fields.go index d41b4d83853f..c0ef64258009 100644 --- a/metricbeat/module/vsphere/fields.go +++ b/metricbeat/module/vsphere/fields.go @@ -32,5 +32,5 @@ func init() { // AssetVsphere returns asset data. // This is the base64 encoded zlib format compressed contents of module/vsphere. func AssetVsphere() string { - return "eJzsmU1v4zYTx+/5FIM9Pc8h7t2HAosU2+wh3QLJ9hqMybHEmiIFcijD++kLUi8ry7LjRJJ7qQ4GLMrz/3E4nKHG97CjwxoqX+bk6A6AFWtaw6fqOd35dAcgyQunSlbWrOHXOwCAZhQKK4OOP3OkCT2tIcM7gK0iLf06PXoPBgvqS8SLD2V82NlQNndGVI4N9Y1JZPRsO3PjJs+abYZGjBzPo72GGH2U+Hk00JLs6LC3Tg7GLvDE67eW6dRuK7j10f58kl+UJn/wTAWcGG41BZYoFB9WbBn1anNg8qME2prsffIv0SIki2C3wDmNLky8ttYVyGs4lT/h3DqiWTG/OKLZKYMnOSvld09yGcpS8CijF6hJvm61xeED17CW5AQZvpa2ebwbPUkMufU8JScMfv9vp4NH6/l8JvCMHMZD50NqLzmBrcih1pATas4bibg8mHwDyqSFagsAmUo5awoyvBplDCWrMx6RNmz0cOgKxJSDGsMTyEQZ6tAu8h9zbr+HP79HlqeB1b5snUXn061z6BXCKS3Op5uS4huyUvndinNnQ5aXgS+ku4/GA8aQzSgpwU8lCD7ejQENCL4kobaKJCjD5CrU41GRcCVVStBKI5MRh1UxH+/nhhULG5q0F6NYMTDuyANbELYoNTEBGnh+eP4abxRoJGydLaDMD14J1NAwnp9ES98E23xzeFRZTp6hEYAKdSBA4az3EHNHlPcQ6lqUNmXcoBdQ3zpVfMjTWeYoQyZZB8bXX76BQ77kMUd4qRxPWW8Tig25uN47pW1doaNcvaip9EVIQpGDJ2GNHOcsqLBuqYPDUzIeN/OY6bdPDA3cUmfEmfCWORpOhTPEe+t2r/HbjCX9j9osnJodCK82aOReSc5X7ND4QjFfjLEpuyHuQ0CGfa5Eng59sEcPPWGQwSmTpZ3RJWx4yVXcNqUjT4Z9Gu242zNkO6ErZ+tIkKpuPdVW9XbzXCLHttEVWGn1A+Nj8D9hi40yJLvlvIdYvZoZ30eH+P9fpi5R7Ij9USSKWDDnI+/ycaPVD77r4LrAWZqsE7oKi5yz7rau2yvOodZ9vxsb3pt5sw/7Mc+mjTQvZl3gzCXY6xiLoFkJ9HyTAOjUPr6LfgIvHgGntO9b/55vbxACJ7TXQUpny5LkTZa/X69bl7b674JdfOm7cjsJ85Y7v9WEsS5XpRwH1AWKXJlJPfCzlt7f+UrveGp4Pp3a/BoYPBKLHwv129pD1qAfuFSf7696GeCpXofzLT8747vBUPRbSQ45nkaf678DzlL81zqbrPu5QqVxo9/sn/Xf+LNA/lIHbdp7v93C71Fg8ht2Yk07dEHURzsDaR1N87v1qGExm19TCM4P2+9fTGcVwbMtXutKMUpoN3/Tyb9J9c3XCensIQnDiPBNeyz/BAAA//9CQWDd" + return "eJzsmc1u4zYQx+95isGe2kPcuw8FghTb7CHdAsn2GoypscWaHwI5suF9+oKkpMqSnLVjyuihOhiwJM//xyFnOBzfw5YOS9j5qiRHdwAsWdESPu1e4p1PdwAFeeFkxdKaJfx6BwDQPAVti1qFnzlShJ6WsME7gLUkVfhlfPUeDGrqS4SLD1V42dm6au5MqBwb6hsrkNGz7cxNmzxptnk0YeR4HO01xOijhM+jBy3Jlg5764rBs3d4wvVbyzS22wqufbCfT/KzVOQPnknDyHCrKbBCIfmwYMuoFqsDk58kUNZsLpN/DRYhWgS7Bi5pcmLCtbZOIy9hLD/iXDuirJifHVF2ytpTkZXym6diHspK8CSjF6ioeFsri8MXzmGtyAkyfC5t83r3dJQYSuv5mpww+P3l6UBUdXKYLr/nnNTHP7+BNPA8sNqXTbGZTzdF5hnCMdjy6cZQ+4FsIf120VuhuMkb7q8lAWpbp6UZliRuqAsJELVzZFgdYEXSbEBY42sdYw+sA2viaibDge70AAraSUEKmYw4LHBHQWWhcw3iIRnsDYSlJpAMjFvywBaE1ZUiJkADL48vX8INjaaAtbMaqvLgpUAFCfSdkbRjaNZgrhE8yU1JnqExDztUNQEKZ70HVCqKe6hT2otODxH8DqgjzJtyWyebWq/IBSen/BuEkhdjYpN+C4SiBE/CmmJxcTqO9Pm334fNxtEGmYrE+OWXr+CQ6XJATdq6uXbe52g8ZIQp02fDzVW+ZMKbpx64Fi5vhftkPZ8ubg3x3rrtW/g27YQPif6RzMLY7EB4sUJT7GXB5YIdGq8lc+Y5aRNGiDJAhn0pRRkrH9ijh54sFLULu0tIINIwuR2qBbyWMmSXypEnwz4+7ajbQqodzuWTPfKDI0Fyd1sntJr/DQ/kzxrteqxZKvkdw2vwk7B6JQ0V3RK4D97xELbjxiHpzs8fH1SFYkvsjxa3CNVBroF1u2Cj1F/P09vyEK1bb/NydTJnQZFz1t3SbXvJJSTVy13Y0N7Ik33Uj3k1xldOyLQrm/dQzyPUtWIp0PMNpr7T+njs/Is789yPWS+b+Z5fZ5/8Eet5iIWzVUXFDSa+v+W37mzVL0KdedK7XfkqyNtFe6vYbxT1CT0j1xmrzNeSwIbCRikoCRWXjUSAwngyDVV4qE7aBjqZnXTWaDInDq11Fc7sGbsa0f2N2WOudDJNSfIczK6zLx3XqDSKUpqrOvInLV3eh4ttADmcwGvPLQODR2LhY6ajUlvTDrqTcx3R/krTAM9pHk6f1mzG4BmKfq3IIYfi/yX9OXGS4v+W69W6DzuUClfqh33XfqNiU5Pn2doVdg2/B4E8TZUYoTOiPtkMpGk15XfrUY8qm1/jEswP22/3Xc8qas9Wv6WdYpLQrv6m0X9b6ebbFensMQrDhPBNW13/BAAA//+YDoMU" } From 4520b6324eb63f0e1cd7a637a053596f16146b28 Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Wed, 7 Aug 2024 11:55:25 +0530 Subject: [PATCH 10/33] golanlint-ci fix --- metricbeat/module/vsphere/host/data.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/vsphere/host/data.go b/metricbeat/module/vsphere/host/data.go index b60d2356dee4..1bf9ea8e7a11 100644 --- a/metricbeat/module/vsphere/host/data.go +++ b/metricbeat/module/vsphere/host/data.go @@ -75,13 +75,13 @@ func (m *MetricSet) eventMapping(hs mo.HostSystem, perfMertics *PerformanceMetri }, }, "total": mapstr.M{ - "bytes": int64(perfMertics.DiskUsage) * 1000, + "bytes": perfMertics.DiskUsage * 1000, }, "read": mapstr.M{ - "bytes": int64(perfMertics.DiskRead) * 1000, + "bytes": perfMertics.DiskRead * 1000, }, "write": mapstr.M{ - "bytes": int64(perfMertics.DiskWrite) * 1000, + "bytes": perfMertics.DiskWrite * 1000, }, }, "memory": mapstr.M{ From 86a91517d1ac8ae600eea78392bc1e7432d85f01 Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Wed, 7 Aug 2024 12:28:52 +0530 Subject: [PATCH 11/33] fix linting issues --- metricbeat/module/vsphere/host/data.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metricbeat/module/vsphere/host/data.go b/metricbeat/module/vsphere/host/data.go index 1bf9ea8e7a11..70e61458e470 100644 --- a/metricbeat/module/vsphere/host/data.go +++ b/metricbeat/module/vsphere/host/data.go @@ -61,7 +61,7 @@ func (m *MetricSet) eventMapping(hs mo.HostSystem, perfMertics *PerformanceMetri "disk": mapstr.M{ "capacity": mapstr.M{ "usage": mapstr.M{ - "bytes": int64(perfMertics.DiskUsage) * 1000, + "bytes": perfMertics.DiskUsage * 1000, }, }, "devicelatency": mapstr.M{ @@ -98,13 +98,13 @@ func (m *MetricSet) eventMapping(hs mo.HostSystem, perfMertics *PerformanceMetri "network": mapstr.M{ "bandwidth": mapstr.M{ "transmitted": mapstr.M{ - "bytes": int64(perfMertics.NetTransmitted) * 1000, + "bytes": perfMertics.NetTransmitted * 1000, }, "received": mapstr.M{ - "bytes": int64(perfMertics.NetReceived) * 1000, + "bytes": perfMertics.NetReceived * 1000, }, "total": mapstr.M{ - "bytes": int64(perfMertics.NetUsage) * 1000, + "bytes": perfMertics.NetUsage * 1000, }, }, "packets": mapstr.M{ From 2d8997fc1fca1a43ccce74fba7bcb12f7d9b5905 Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Wed, 7 Aug 2024 14:24:09 +0530 Subject: [PATCH 12/33] add more UTs --- .../module/vsphere/host/_meta/fields.yml | 5 ++ metricbeat/module/vsphere/host/host_test.go | 69 ++++++++++++++++++- 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/metricbeat/module/vsphere/host/_meta/fields.yml b/metricbeat/module/vsphere/host/_meta/fields.yml index 2a83188d2c7a..923344cbd02a 100644 --- a/metricbeat/module/vsphere/host/_meta/fields.yml +++ b/metricbeat/module/vsphere/host/_meta/fields.yml @@ -33,6 +33,11 @@ description: > Average number of bytes read from the disk each second. format: bytes + - name: disk.write.bytes + type: long + description: > + Average number of kilobytes written to the disk each second. + format: bytes - name: disk.total.bytes type: long description: > diff --git a/metricbeat/module/vsphere/host/host_test.go b/metricbeat/module/vsphere/host/host_test.go index 17bd2a09dd19..fa54955c6d74 100644 --- a/metricbeat/module/vsphere/host/host_test.go +++ b/metricbeat/module/vsphere/host/host_test.go @@ -50,6 +50,10 @@ func TestFetchEventContents(t *testing.T) { assert.EqualValues(t, "localhost.localdomain", event["name"]) + assert.NotNil(t, event["status"]) + + assert.GreaterOrEqual(t, event["uptime"], int32(0)) + cpu := event["cpu"].(mapstr.M) cpuUsed := cpu["used"].(mapstr.M) @@ -61,16 +65,79 @@ func TestFetchEventContents(t *testing.T) { cpuFree := cpu["free"].(mapstr.M) assert.EqualValues(t, 4521, cpuFree["mhz"]) + disk := event["disk"].(mapstr.M) + + diskCapacity := disk["capacity"].(mapstr.M) + diskCapacityUsage := diskCapacity["usage"].(mapstr.M) + assert.GreaterOrEqual(t, diskCapacityUsage["bytes"], int64(0)) + + diskDevielatency := disk["devicelatency"].(mapstr.M) + diskDevielatencyAverage := diskDevielatency["average"].(mapstr.M) + assert.GreaterOrEqual(t, diskDevielatencyAverage["ms"], int64(0)) + + diskLatency := disk["latency"].(mapstr.M) + diskLatencyTotal := diskLatency["total"].(mapstr.M) + assert.GreaterOrEqual(t, diskLatencyTotal["ms"], int64(0)) + + diskTotal := disk["total"].(mapstr.M) + assert.GreaterOrEqual(t, diskTotal["bytes"], int64(0)) + + diskRead := disk["read"].(mapstr.M) + assert.GreaterOrEqual(t, diskRead["bytes"], int64(0)) + + diskWrite := disk["write"].(mapstr.M) + assert.GreaterOrEqual(t, diskWrite["bytes"], int64(0)) + memory := event["memory"].(mapstr.M) memoryUsed := memory["used"].(mapstr.M) - assert.EqualValues(t, uint64(1472200704), memoryUsed["bytes"]) + assert.EqualValues(t, 1472200704, memoryUsed["bytes"]) memoryTotal := memory["total"].(mapstr.M) assert.EqualValues(t, uint64(4294430720), memoryTotal["bytes"]) memoryFree := memory["free"].(mapstr.M) assert.EqualValues(t, uint64(2822230016), memoryFree["bytes"]) + + network := event["network"].(mapstr.M) + + networkBandwidth := network["bandwidth"].(mapstr.M) + networkBandwidthTransmitted := networkBandwidth["transmitted"].(mapstr.M) + networkBandwidthReceived := networkBandwidth["received"].(mapstr.M) + networkBandwidthTotal := networkBandwidth["total"].(mapstr.M) + assert.GreaterOrEqual(t, networkBandwidthTransmitted["bytes"], int64(0)) + assert.GreaterOrEqual(t, networkBandwidthReceived["bytes"], int64(0)) + assert.GreaterOrEqual(t, networkBandwidthTotal["bytes"], int64(0)) + + networkPackets := network["packets"].(mapstr.M) + networkPacketsTransmitted := networkPackets["transmitted"].(mapstr.M) + networkPacketsReceived := networkPackets["received"].(mapstr.M) + assert.GreaterOrEqual(t, networkPacketsTransmitted["count"], int64(0)) + assert.GreaterOrEqual(t, networkPacketsReceived["count"], int64(0)) + + networkErrors := networkPackets["errors"].(mapstr.M) + networkErrorsTransmitted := networkErrors["transmitted"].(mapstr.M) + networkErrorsReceived := networkErrors["received"].(mapstr.M) + networkErrorsTotal := networkErrors["total"].(mapstr.M) + assert.GreaterOrEqual(t, networkErrorsTransmitted["count"], int64(0)) + assert.GreaterOrEqual(t, networkErrorsReceived["count"], int64(0)) + assert.GreaterOrEqual(t, networkErrorsTotal["count"], int64(0)) + + networkMulticast := networkPackets["multicast"].(mapstr.M) + networkMulticastTransmitted := networkMulticast["transmitted"].(mapstr.M) + networkMulticastReceived := networkMulticast["received"].(mapstr.M) + networkMulticastTotal := networkMulticast["total"].(mapstr.M) + assert.GreaterOrEqual(t, networkMulticastTransmitted["count"], int64(0)) + assert.GreaterOrEqual(t, networkMulticastReceived["count"], int64(0)) + assert.GreaterOrEqual(t, networkMulticastTotal["count"], int64(0)) + + networkDropped := networkPackets["dropped"].(mapstr.M) + networkDroppedTransmitted := networkDropped["transmitted"].(mapstr.M) + networkDroppedReceived := networkDropped["received"].(mapstr.M) + networkDroppedTotal := networkDropped["total"].(mapstr.M) + assert.GreaterOrEqual(t, networkDroppedTransmitted["count"], int64(0)) + assert.GreaterOrEqual(t, networkDroppedReceived["count"], int64(0)) + assert.GreaterOrEqual(t, networkDroppedTotal["count"], int64(0)) } func TestData(t *testing.T) { From 5c2061f9c6bcaad0897db0aa355a940666b24d6c Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Thu, 8 Aug 2024 11:35:19 +0530 Subject: [PATCH 13/33] update changelog entry --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ccc2ff66dd3c..ed9a2b328c9d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -64,7 +64,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Allow metricsets to report their status via control v2 protocol. {pull}40025[40025] - Remove fallback to the node limit for the `kubernetes.pod.cpu.usage.limit.pct` and `kubernetes.pod.memory.usage.limit.pct` metrics calculation - Add support for Kibana status metricset in v8 format {pull}40275[40275] -- Add new metrics for the Vsphere Host metricset. {pull}40429[40429] +- Update metrics for the Vsphere Host metricset. {pull}40429[40429] *Osquerybeat* From d6f3b5ad385ab92d1661f5ce1d59126584c0f609 Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Thu, 8 Aug 2024 11:56:31 +0530 Subject: [PATCH 14/33] metricbeat: mage update --- metricbeat/docs/fields.asciidoc | 12 ++++++++++++ metricbeat/module/vsphere/fields.go | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 8916497a1b00..f1b75e1888ce 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -67002,6 +67002,18 @@ type: long Average number of bytes read from the disk each second. +type: long + +format: bytes + +-- + +*`vsphere.host.disk.write.bytes`*:: ++ +-- +Average number of kilobytes written to the disk each second. + + type: long format: bytes diff --git a/metricbeat/module/vsphere/fields.go b/metricbeat/module/vsphere/fields.go index c0ef64258009..d539e4eb99d1 100644 --- a/metricbeat/module/vsphere/fields.go +++ b/metricbeat/module/vsphere/fields.go @@ -32,5 +32,5 @@ func init() { // AssetVsphere returns asset data. // This is the base64 encoded zlib format compressed contents of module/vsphere. func AssetVsphere() string { - return "eJzsmc1u4zYQx+95isGe2kPcuw8FghTb7CHdAsn2GoypscWaHwI5suF9+oKkpMqSnLVjyuihOhiwJM//xyFnOBzfw5YOS9j5qiRHdwAsWdESPu1e4p1PdwAFeeFkxdKaJfx6BwDQPAVti1qFnzlShJ6WsME7gLUkVfhlfPUeDGrqS4SLD1V42dm6au5MqBwb6hsrkNGz7cxNmzxptnk0YeR4HO01xOijhM+jBy3Jlg5764rBs3d4wvVbyzS22wqufbCfT/KzVOQPnknDyHCrKbBCIfmwYMuoFqsDk58kUNZsLpN/DRYhWgS7Bi5pcmLCtbZOIy9hLD/iXDuirJifHVF2ytpTkZXym6diHspK8CSjF6ioeFsri8MXzmGtyAkyfC5t83r3dJQYSuv5mpww+P3l6UBUdXKYLr/nnNTHP7+BNPA8sNqXTbGZTzdF5hnCMdjy6cZQ+4FsIf120VuhuMkb7q8lAWpbp6UZliRuqAsJELVzZFgdYEXSbEBY42sdYw+sA2viaibDge70AAraSUEKmYw4LHBHQWWhcw3iIRnsDYSlJpAMjFvywBaE1ZUiJkADL48vX8INjaaAtbMaqvLgpUAFCfSdkbRjaNZgrhE8yU1JnqExDztUNQEKZ70HVCqKe6hT2otODxH8DqgjzJtyWyebWq/IBSen/BuEkhdjYpN+C4SiBE/CmmJxcTqO9Pm334fNxtEGmYrE+OWXr+CQ6XJATdq6uXbe52g8ZIQp02fDzVW+ZMKbpx64Fi5vhftkPZ8ubg3x3rrtW/g27YQPif6RzMLY7EB4sUJT7GXB5YIdGq8lc+Y5aRNGiDJAhn0pRRkrH9ijh54sFLULu0tIINIwuR2qBbyWMmSXypEnwz4+7ajbQqodzuWTPfKDI0Fyd1sntJr/DQ/kzxrteqxZKvkdw2vwk7B6JQ0V3RK4D97xELbjxiHpzs8fH1SFYkvsjxa3CNVBroF1u2Cj1F/P09vyEK1bb/NydTJnQZFz1t3SbXvJJSTVy13Y0N7Ik33Uj3k1xldOyLQrm/dQzyPUtWIp0PMNpr7T+njs/Is789yPWS+b+Z5fZ5/8Eet5iIWzVUXFDSa+v+W37mzVL0KdedK7XfkqyNtFe6vYbxT1CT0j1xmrzNeSwIbCRikoCRWXjUSAwngyDVV4qE7aBjqZnXTWaDInDq11Fc7sGbsa0f2N2WOudDJNSfIczK6zLx3XqDSKUpqrOvInLV3eh4ttADmcwGvPLQODR2LhY6ajUlvTDrqTcx3R/krTAM9pHk6f1mzG4BmKfq3IIYfi/yX9OXGS4v+W69W6DzuUClfqh33XfqNiU5Pn2doVdg2/B4E8TZUYoTOiPtkMpGk15XfrUY8qm1/jEswP22/3Xc8qas9Wv6WdYpLQrv6m0X9b6ebbFensMQrDhPBNW13/BAAA//+YDoMU" + return "eJzsmc1u4zYQx+95isGe2kPcuw8FghTb7CHdAsn2GoypscWaHwI5suF9+oKkpMqSnLVjyuihOgSIJM//x+HMaDS6hy0dlrDzVUmO7gBYsqIlfNq9xDOf7gAK8sLJiqU1S/j1DgCguQraFrUKP3OkCD0tYYN3AGtJqvDLeOs9GNTUlwgHH6pws7N11ZyZUDk21DdWIKNn25mbNnnSbHNpwsjxOtpjiNFHCX+PLrQkWzrsrSsG197hCcdvLdPYbiu49sF+PsnPUpE/eCYNI8OtpsAKheTDgi2jWqwOTH6SQFmzuUz+NViEaBHsGrikyY0Jx9o6jbyEsfyIc+2IsmJ+dkTZKWtPRVbKb56KeSgrwZOMXqCi4m2tLA5vOIe1IifI8Lm0ze3d1VFhKK3na2rC4PeXlwNR1clhuvyec1Mf//wG0sDzwGpfNuVmPt2UmWcIx2TLpxtT7QeyhfTbRS9CcZM33V9LAtS2TqEZQhI31KUEiNo5MqwOsCJpNiCs8bWOuQfWgTUxmslwoDu9gIJ2UpBCJiMOC9xRUFnoXIt4SAZ7C2GpCSQD45Y8sAVhdaWICdDAy+PLl3BCoylg7ayGqjx4KVBBAn1nJe0amhjMtYInuSnJMzTmYYeqJkDhrPeASkVxD3Uqe9HpIYPfAXWEeUtu62RT6xW54ORUf4NQ8mIsbNJvgVCU4ElYUywuLseRfu8k5w30Mf5WKpuWENSYTIiTjGvI30I8bDaONshUJMYvv3wFh0yXA2rS1s3VPTxH46GqTZk+G26uFiwT3jw9zbVwebv0J+v5dINuiPfWbd/Cf9NO+JDoH8ksjM0OhBcrNMVeFlwu2KHxOmTxPEUvZBkgw76UoozdG+zRQ08WitqFJ2QoINIwuR2qBbyWMlTIypEnwz5e7ajbZrBdzuWbPfKDI0Fyd1sntJr/DQ/krxptPNYslfyO4Tb4SVi9koaKLgTug3c8hJaicUg68/PHF1Wh2BL7o+AWocPJtbDuUdgo9eN5urUYonXxNi9XJ3MWFDln3S3dtpdcQlK93IUN7Y082Uf9mFdjfuWETE9l8x7qeYS6ViwFer7B1ndaH8+df3Fn3vsx62U73/Pr7Js/Yj0PsXC2qqi4wcb3H/mtO1v1i1Bn3vTuqXwV5O2yvVXsD7v6hJ6R64xd5mtJYENjoxSUhIrLRiJAYXy7Dl146E7ajwBkdtJZo8mcePGuK5Yn2u+PTWai+xuzx1zpzTQVyXMwu68T0nGNSqMopbnqq8JJS5fPEuMoQw438Nr3loHBI7HwZ6ZXpbanHUxY53pF+yttAzynfTj9tmYzJs9Q9GtFDjk0/y/pA8tJiv/HxlfrPuxQKlypH86O+4OKTU2eZxtX2DX8HgTyDFVihs6I+mQzkKZoyu/WoxlVNr/GEMwP2x/3Xc8qas9Wv6UnxSShXf1No+9z6eTbFeXsMQrDhPBNR13/BAAA///vbrjf" } From bd628497a56597986bc036c20c09cc8fa90cc44c Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Thu, 8 Aug 2024 13:56:47 +0530 Subject: [PATCH 15/33] Resolve review comments --- .../module/vsphere/host/_meta/fields.yml | 2 +- metricbeat/module/vsphere/host/data.go | 49 ++++++------------- metricbeat/module/vsphere/host/host_test.go | 2 +- 3 files changed, 18 insertions(+), 35 deletions(-) diff --git a/metricbeat/module/vsphere/host/_meta/fields.yml b/metricbeat/module/vsphere/host/_meta/fields.yml index 923344cbd02a..c7ce93d96fad 100644 --- a/metricbeat/module/vsphere/host/_meta/fields.yml +++ b/metricbeat/module/vsphere/host/_meta/fields.yml @@ -36,7 +36,7 @@ - name: disk.write.bytes type: long description: > - Average number of kilobytes written to the disk each second. + Average number of bytes written to the disk each second. format: bytes - name: disk.total.bytes type: long diff --git a/metricbeat/module/vsphere/host/data.go b/metricbeat/module/vsphere/host/data.go index 70e61458e470..5b4760739517 100644 --- a/metricbeat/module/vsphere/host/data.go +++ b/metricbeat/module/vsphere/host/data.go @@ -27,21 +27,6 @@ func (m *MetricSet) eventMapping(hs mo.HostSystem, perfMertics *PerformanceMetri totalErrorPacketsCount := perfMertics.NetErrorsTransmitted + perfMertics.NetErrorsReceived totalMulticastPacketsCount := perfMertics.NetMulticastTransmitted + perfMertics.NetMulticastReceived totalDroppedPacketsCount := perfMertics.NetDroppedTransmitted + perfMertics.NetDroppedReceived - totalCPU := int64(0) - freeCPU := int64(0) - freeMemory := int64(0) - totalMemory := int64(0) - usedMemory := int64(0) - - if hs.Summary.Hardware != nil { - totalCPU = int64(hs.Summary.Hardware.CpuMhz) * int64(hs.Summary.Hardware.NumCpuCores) - freeCPU = totalCPU - int64(hs.Summary.QuickStats.OverallCpuUsage) - usedMemory = int64(hs.Summary.QuickStats.OverallMemoryUsage) * 1024 * 1024 - freeMemory = hs.Summary.Hardware.MemorySize - usedMemory - totalMemory = hs.Summary.Hardware.MemorySize - } else { - m.Logger().Debug("'Hardware' or 'Summary' data not found. This is either a parsing error from vsphere library, an error trying to reach host/guest or incomplete information returned from host/guest") - } event := mapstr.M{ "name": hs.Summary.Config.Name, @@ -51,17 +36,11 @@ func (m *MetricSet) eventMapping(hs mo.HostSystem, perfMertics *PerformanceMetri "used": mapstr.M{ "mhz": hs.Summary.QuickStats.OverallCpuUsage, }, - "total": mapstr.M{ - "mhz": totalCPU, - }, - "free": mapstr.M{ - "mhz": freeCPU, - }, }, "disk": mapstr.M{ "capacity": mapstr.M{ "usage": mapstr.M{ - "bytes": perfMertics.DiskUsage * 1000, + "bytes": perfMertics.DiskCapacityUsage * 1000, }, }, "devicelatency": mapstr.M{ @@ -84,17 +63,6 @@ func (m *MetricSet) eventMapping(hs mo.HostSystem, perfMertics *PerformanceMetri "bytes": perfMertics.DiskWrite * 1000, }, }, - "memory": mapstr.M{ - "used": mapstr.M{ - "bytes": usedMemory, - }, - "total": mapstr.M{ - "bytes": totalMemory, - }, - "free": mapstr.M{ - "bytes": freeMemory, - }, - }, "network": mapstr.M{ "bandwidth": mapstr.M{ "transmitted": mapstr.M{ @@ -150,6 +118,21 @@ func (m *MetricSet) eventMapping(hs mo.HostSystem, perfMertics *PerformanceMetri }, }, } + if hs.Summary.Hardware != nil { + totalCPU := int64(hs.Summary.Hardware.CpuMhz) * int64(hs.Summary.Hardware.NumCpuCores) + freeCPU := totalCPU - int64(hs.Summary.QuickStats.OverallCpuUsage) + usedMemory := int64(hs.Summary.QuickStats.OverallMemoryUsage) * 1024 * 1024 + freeMemory := hs.Summary.Hardware.MemorySize - usedMemory + totalMemory := hs.Summary.Hardware.MemorySize + + event.Put("cpu.total.mhz", totalCPU) + event.Put("cpu.free.mhz", freeCPU) + event.Put("memory.used.bytes", usedMemory) + event.Put("memory.free.bytes", freeMemory) + event.Put("memory.total.bytes", totalMemory) + } else { + m.Logger().Debug("'Hardware' or 'Summary' data not found. This is either a parsing error from vsphere library, an error trying to reach host/guest or incomplete information returned from host/guest") + } if len(networkNames) > 0 { event.Put("network_names", networkNames) diff --git a/metricbeat/module/vsphere/host/host_test.go b/metricbeat/module/vsphere/host/host_test.go index fa54955c6d74..6bac0e1e2505 100644 --- a/metricbeat/module/vsphere/host/host_test.go +++ b/metricbeat/module/vsphere/host/host_test.go @@ -91,7 +91,7 @@ func TestFetchEventContents(t *testing.T) { memory := event["memory"].(mapstr.M) memoryUsed := memory["used"].(mapstr.M) - assert.EqualValues(t, 1472200704, memoryUsed["bytes"]) + assert.EqualValues(t, uint64(1472200704), memoryUsed["bytes"]) memoryTotal := memory["total"].(mapstr.M) assert.EqualValues(t, uint64(4294430720), memoryTotal["bytes"]) From 4f9224828dac54856ddd504db3e560d33a528e95 Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Mon, 12 Aug 2024 10:38:44 +0530 Subject: [PATCH 16/33] Minor changes - Update interval - Mage update --- metricbeat/docs/fields.asciidoc | 18 +++++++++--------- metricbeat/docs/modules/vsphere.asciidoc | 3 ++- metricbeat/metricbeat.reference.yml | 3 ++- .../module/vsphere/_meta/config.reference.yml | 3 ++- metricbeat/module/vsphere/_meta/config.yml | 3 ++- metricbeat/module/vsphere/fields.go | 2 +- .../module/vsphere/host/_meta/fields.yml | 16 ++++++++-------- metricbeat/modules.d/vsphere.yml.disabled | 3 ++- 8 files changed, 28 insertions(+), 23 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index f1b75e1888ce..3a805e164ff9 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -66939,7 +66939,7 @@ host *`vsphere.host.cpu.used.mhz`*:: + -- -Used CPU in Mhz +Used CPU in Mhz. type: long @@ -66949,7 +66949,7 @@ type: long *`vsphere.host.cpu.total.mhz`*:: + -- -Total CPU in Mhz +Total CPU in Mhz. type: long @@ -66959,7 +66959,7 @@ type: long *`vsphere.host.cpu.free.mhz`*:: + -- -Free CPU in Mhz +Free CPU in Mhz. type: long @@ -67011,7 +67011,7 @@ format: bytes *`vsphere.host.disk.write.bytes`*:: + -- -Average number of kilobytes written to the disk each second. +Average number of bytes written to the disk each second. type: long @@ -67035,7 +67035,7 @@ format: bytes *`vsphere.host.memory.free.bytes`*:: + -- -Free Memory in bytes +Free Memory in bytes. type: long @@ -67047,7 +67047,7 @@ format: bytes *`vsphere.host.memory.total.bytes`*:: + -- -Total Memory in bytes +Total Memory in bytes. type: long @@ -67059,7 +67059,7 @@ format: bytes *`vsphere.host.memory.used.bytes`*:: + -- -Used Memory in bytes +Used Memory in bytes. type: long @@ -67071,7 +67071,7 @@ format: bytes *`vsphere.host.name`*:: + -- -Host name +Host name. type: keyword @@ -67081,7 +67081,7 @@ type: keyword *`vsphere.host.network_names`*:: + -- -Network names +Network names. type: keyword diff --git a/metricbeat/docs/modules/vsphere.asciidoc b/metricbeat/docs/modules/vsphere.asciidoc index daab01568eb8..738bc1153d24 100644 --- a/metricbeat/docs/modules/vsphere.asciidoc +++ b/metricbeat/docs/modules/vsphere.asciidoc @@ -36,7 +36,8 @@ metricbeat.modules: - module: vsphere enabled: true metricsets: ["datastore", "host", "virtualmachine"] - period: 10s + # Real-time data collection – An ESXi Server collects data for each performance counter every 20 seconds. + period: 20s hosts: ["https://localhost/sdk"] username: "user" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 985e4795910c..4c72f9dad04d 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1008,7 +1008,8 @@ metricbeat.modules: - module: vsphere enabled: true metricsets: ["datastore", "host", "virtualmachine"] - period: 10s + # Real-time data collection – An ESXi Server collects data for each performance counter every 20 seconds. + period: 20s hosts: ["https://localhost/sdk"] username: "user" diff --git a/metricbeat/module/vsphere/_meta/config.reference.yml b/metricbeat/module/vsphere/_meta/config.reference.yml index d4803d0c7753..18da5eb6b91e 100644 --- a/metricbeat/module/vsphere/_meta/config.reference.yml +++ b/metricbeat/module/vsphere/_meta/config.reference.yml @@ -1,7 +1,8 @@ - module: vsphere enabled: true metricsets: ["datastore", "host", "virtualmachine"] - period: 10s + # Real-time data collection – An ESXi Server collects data for each performance counter every 20 seconds. + period: 20s hosts: ["https://localhost/sdk"] username: "user" diff --git a/metricbeat/module/vsphere/_meta/config.yml b/metricbeat/module/vsphere/_meta/config.yml index 6862012209b1..8b76d29f9074 100644 --- a/metricbeat/module/vsphere/_meta/config.yml +++ b/metricbeat/module/vsphere/_meta/config.yml @@ -3,7 +3,8 @@ # - datastore # - host # - virtualmachine - period: 10s + # Real-time data collection – An ESXi Server collects data for each performance counter every 20 seconds. + period: 20s hosts: ["https://localhost/sdk"] username: "user" diff --git a/metricbeat/module/vsphere/fields.go b/metricbeat/module/vsphere/fields.go index d539e4eb99d1..c9b88cb38d93 100644 --- a/metricbeat/module/vsphere/fields.go +++ b/metricbeat/module/vsphere/fields.go @@ -32,5 +32,5 @@ func init() { // AssetVsphere returns asset data. // This is the base64 encoded zlib format compressed contents of module/vsphere. func AssetVsphere() string { - return "eJzsmc1u4zYQx+95isGe2kPcuw8FghTb7CHdAsn2GoypscWaHwI5suF9+oKkpMqSnLVjyuihOgSIJM//x+HMaDS6hy0dlrDzVUmO7gBYsqIlfNq9xDOf7gAK8sLJiqU1S/j1DgCguQraFrUKP3OkCD0tYYN3AGtJqvDLeOs9GNTUlwgHH6pws7N11ZyZUDk21DdWIKNn25mbNnnSbHNpwsjxOtpjiNFHCX+PLrQkWzrsrSsG197hCcdvLdPYbiu49sF+PsnPUpE/eCYNI8OtpsAKheTDgi2jWqwOTH6SQFmzuUz+NViEaBHsGrikyY0Jx9o6jbyEsfyIc+2IsmJ+dkTZKWtPRVbKb56KeSgrwZOMXqCi4m2tLA5vOIe1IifI8Lm0ze3d1VFhKK3na2rC4PeXlwNR1clhuvyec1Mf//wG0sDzwGpfNuVmPt2UmWcIx2TLpxtT7QeyhfTbRS9CcZM33V9LAtS2TqEZQhI31KUEiNo5MqwOsCJpNiCs8bWOuQfWgTUxmslwoDu9gIJ2UpBCJiMOC9xRUFnoXIt4SAZ7C2GpCSQD45Y8sAVhdaWICdDAy+PLl3BCoylg7ayGqjx4KVBBAn1nJe0amhjMtYInuSnJMzTmYYeqJkDhrPeASkVxD3Uqe9HpIYPfAXWEeUtu62RT6xW54ORUf4NQ8mIsbNJvgVCU4ElYUywuLseRfu8k5w30Mf5WKpuWENSYTIiTjGvI30I8bDaONshUJMYvv3wFh0yXA2rS1s3VPTxH46GqTZk+G26uFiwT3jw9zbVwebv0J+v5dINuiPfWbd/Cf9NO+JDoH8ksjM0OhBcrNMVeFlwu2KHxOmTxPEUvZBkgw76UoozdG+zRQ08WitqFJ2QoINIwuR2qBbyWMlTIypEnwz5e7ajbZrBdzuWbPfKDI0Fyd1sntJr/DQ/krxptPNYslfyO4Tb4SVi9koaKLgTug3c8hJaicUg68/PHF1Wh2BL7o+AWocPJtbDuUdgo9eN5urUYonXxNi9XJ3MWFDln3S3dtpdcQlK93IUN7Y082Uf9mFdjfuWETE9l8x7qeYS6ViwFer7B1ndaH8+df3Fn3vsx62U73/Pr7Js/Yj0PsXC2qqi4wcb3H/mtO1v1i1Bn3vTuqXwV5O2yvVXsD7v6hJ6R64xd5mtJYENjoxSUhIrLRiJAYXy7Dl146E7ajwBkdtJZo8mcePGuK5Yn2u+PTWai+xuzx1zpzTQVyXMwu68T0nGNSqMopbnqq8JJS5fPEuMoQw438Nr3loHBI7HwZ6ZXpbanHUxY53pF+yttAzynfTj9tmYzJs9Q9GtFDjk0/y/pA8tJiv/HxlfrPuxQKlypH86O+4OKTU2eZxtX2DX8HgTyDFVihs6I+mQzkKZoyu/WoxlVNr/GEMwP2x/3Xc8qas9Wv6UnxSShXf1No+9z6eTbFeXsMQrDhPBNR13/BAAA///vbrjf" + return "eJzMmU1v4zYTx+/5FIM9Pc8h7t2HAkGKbfaQboFkew3G1NhizReBHNlwPn1BUlJlW/L6hTKqQ4BI8vx/HM6MRqNHWNNuDhtfleToAYAlK5rDl81bPPPlAaAgL5ysWFozh18fAACaq6BtUavwM0eK0NMcVvgAsJSkCj+Ptz6CQU19iXDwrgo3O1tXzZkBlX1DfWMFMnq2nblhk6Nmm0sDRvbX0R6HGH2U8HfvQkuypt3WuuLg2gmecPzWMh3bbQWXPtjPJ/lVKvI7z6ThyHCrKbBCIXk3Y8uoZosdkx8kUNasLpN/DxYhWgS7BC5pcGPCsbROI8/hWP6Ic+mIsmJ+dUTZKWtPRVbKH56KaSgrwYOMXqCi4mOpLB7ecA5rRU6Q4XNpm9u7q0eFobSeb6kJB7+/vByIqk4O0+Vnzk19/vMHSAOv5edsVDclZz7hlJrnKMd0yycck+1nuoX061kvSHGVN+PfSwLUtk7RGaISV9RlBYjaOTKsdrAgaVYgrPG1jukH1oE1MaDJcKAbX0BBGylIIZMRuxluKKjMdK5FPCWDvYWw1ASSgXFNHtiCsLpSxARo4O357Vs4odEUsHRWQ1XuvBSoIIGeWEm7hiYKc63gRa5K8gyNedigqglQOOs9oFJR3EOdKl90ekjiE6COMG/VbZ1sar0gF5ycSnAQSl6MtU36NRCKEjwJa4rZxRU50m+d5LyBPoYflJhMiJGM/Pk7iKfVytEKmYrE+O2X7+CQ6XJATdq6qZqH12g8lLRo+mq6qVqwXHzTNDU30+Xt01+s52hyuM4Y4q1164/w37AbrlL9I5mNIv6k8myBptjKgssZOzReh1SepuqFVANk2JZSlLGDgy166MlCUbvwiAxVRBomt0E1g/dShhJZOfJk2MerHXXbELbLuXy7j/zgSJDc3NcJreZ/wwP5K0cbkDVLJT8x3Ab/E1YvpKGiC4HH4B0PoadoHJLO/P/6RVUo1sR+L7hFaHFyLax7FjZK/Xg+nXktWhdv03J1MmdBkXPW3dNtW8klJNXLXdjQ3smTfdTrvBrzKydkejKbU6jnEepasRTo+Q5b32ldnzv/4k6898esl+18z6+Tb/4R63mIhbNVRcUdNr7/yG/d2apfhDrxpndP5Zsg75ftrWJ/4NUn9IxcZ2wz30sCGxobpaAkVFw2EgEK4+t1aMRDd9J+CCCzkc4aTWbkzbuuWI404NeNZqL7G7P7XOn1NBXJczC7LxTScY1KoyiluenLwqily+eJcZYhDzfw1jeXA4N7YuHPRC9LbU97MGWd6iXtr7QN8Jr2YfyTis2YPIei3ytyyKH5f0sfWUYpph8dj8pOPTkeFc48OH7aoFS4UPQz7f6sYlWT58kmFnYJvweBbnRx01wlZuiEqC82A2mKpvxu3ZtTZfNrDMH8sP2Z3+2sovZs9Ud6UgwS2sXfdPSNLp38uKGcPUdhGBC+56zr4Z8AAAD//9gpuKA=" } diff --git a/metricbeat/module/vsphere/host/_meta/fields.yml b/metricbeat/module/vsphere/host/_meta/fields.yml index c7ce93d96fad..2d2a05c3c053 100644 --- a/metricbeat/module/vsphere/host/_meta/fields.yml +++ b/metricbeat/module/vsphere/host/_meta/fields.yml @@ -7,15 +7,15 @@ - name: cpu.used.mhz type: long description: > - Used CPU in Mhz + Used CPU in Mhz. - name: cpu.total.mhz type: long description: > - Total CPU in Mhz + Total CPU in Mhz. - name: cpu.free.mhz type: long description: > - Free CPU in Mhz + Free CPU in Mhz. - name: disk.capacity.usage.bytes type: long description: > @@ -46,26 +46,26 @@ - name: memory.free.bytes type: long description: > - Free Memory in bytes + Free Memory in bytes. format: bytes - name: memory.total.bytes type: long description: > - Total Memory in bytes + Total Memory in bytes. format: bytes - name: memory.used.bytes type: long description: > - Used Memory in bytes + Used Memory in bytes. format: bytes - name: name type: keyword description: > - Host name + Host name. - name: network_names type: keyword description: > - Network names + Network names. - name: network.bandwidth.transmitted.bytes type: long description: > diff --git a/metricbeat/modules.d/vsphere.yml.disabled b/metricbeat/modules.d/vsphere.yml.disabled index 874b3b5b2e89..6659aa39bbc9 100644 --- a/metricbeat/modules.d/vsphere.yml.disabled +++ b/metricbeat/modules.d/vsphere.yml.disabled @@ -6,7 +6,8 @@ # - datastore # - host # - virtualmachine - period: 10s + # Real-time data collection – An ESXi Server collects data for each performance counter every 20 seconds. + period: 20s hosts: ["https://localhost/sdk"] username: "user" From 5c3cf69de5401b7a49100dfa5bc8276ae52189db Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Mon, 12 Aug 2024 14:38:10 +0530 Subject: [PATCH 17/33] resolved review comments --- metricbeat/docs/fields.asciidoc | 2 +- metricbeat/module/vsphere/fields.go | 2 +- .../module/vsphere/host/_meta/fields.yml | 2 +- metricbeat/module/vsphere/host/host.go | 61 ++++++++++--------- x-pack/metricbeat/metricbeat.reference.yml | 3 +- 5 files changed, 38 insertions(+), 32 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 3a805e164ff9..7b91e6524954 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -67227,7 +67227,7 @@ type: long *`vsphere.host.network.packets.dropped.total.count`*:: + -- -Total number of packets dropped. +Total number of packets dropped. type: long diff --git a/metricbeat/module/vsphere/fields.go b/metricbeat/module/vsphere/fields.go index c9b88cb38d93..0c3513620f33 100644 --- a/metricbeat/module/vsphere/fields.go +++ b/metricbeat/module/vsphere/fields.go @@ -32,5 +32,5 @@ func init() { // AssetVsphere returns asset data. // This is the base64 encoded zlib format compressed contents of module/vsphere. func AssetVsphere() string { - return "eJzMmU1v4zYTx+/5FIM9Pc8h7t2HAkGKbfaQboFkew3G1NhizReBHNlwPn1BUlJlW/L6hTKqQ4BI8vx/HM6MRqNHWNNuDhtfleToAYAlK5rDl81bPPPlAaAgL5ysWFozh18fAACaq6BtUavwM0eK0NMcVvgAsJSkCj+Ptz6CQU19iXDwrgo3O1tXzZkBlX1DfWMFMnq2nblhk6Nmm0sDRvbX0R6HGH2U8HfvQkuypt3WuuLg2gmecPzWMh3bbQWXPtjPJ/lVKvI7z6ThyHCrKbBCIXk3Y8uoZosdkx8kUNasLpN/DxYhWgS7BC5pcGPCsbROI8/hWP6Ic+mIsmJ+dUTZKWtPRVbKH56KaSgrwYOMXqCi4mOpLB7ecA5rRU6Q4XNpm9u7q0eFobSeb6kJB7+/vByIqk4O0+Vnzk19/vMHSAOv5edsVDclZz7hlJrnKMd0yycck+1nuoX061kvSHGVN+PfSwLUtk7RGaISV9RlBYjaOTKsdrAgaVYgrPG1jukH1oE1MaDJcKAbX0BBGylIIZMRuxluKKjMdK5FPCWDvYWw1ASSgXFNHtiCsLpSxARo4O357Vs4odEUsHRWQ1XuvBSoIIGeWEm7hiYKc63gRa5K8gyNedigqglQOOs9oFJR3EOdKl90ekjiE6COMG/VbZ1sar0gF5ycSnAQSl6MtU36NRCKEjwJa4rZxRU50m+d5LyBPoYflJhMiJGM/Pk7iKfVytEKmYrE+O2X7+CQ6XJATdq6qZqH12g8lLRo+mq6qVqwXHzTNDU30+Xt01+s52hyuM4Y4q1164/w37AbrlL9I5mNIv6k8myBptjKgssZOzReh1SepuqFVANk2JZSlLGDgy166MlCUbvwiAxVRBomt0E1g/dShhJZOfJk2MerHXXbELbLuXy7j/zgSJDc3NcJreZ/wwP5K0cbkDVLJT8x3Ab/E1YvpKGiC4HH4B0PoadoHJLO/P/6RVUo1sR+L7hFaHFyLax7FjZK/Xg+nXktWhdv03J1MmdBkXPW3dNtW8klJNXLXdjQ3smTfdTrvBrzKydkejKbU6jnEepasRTo+Q5b32ldnzv/4k6898esl+18z6+Tb/4R63mIhbNVRcUdNr7/yG/d2apfhDrxpndP5Zsg75ftrWJ/4NUn9IxcZ2wz30sCGxobpaAkVFw2EgEK4+t1aMRDd9J+CCCzkc4aTWbkzbuuWI404NeNZqL7G7P7XOn1NBXJczC7LxTScY1KoyiluenLwqily+eJcZYhDzfw1jeXA4N7YuHPRC9LbU97MGWd6iXtr7QN8Jr2YfyTis2YPIei3ytyyKH5f0sfWUYpph8dj8pOPTkeFc48OH7aoFS4UPQz7f6sYlWT58kmFnYJvweBbnRx01wlZuiEqC82A2mKpvxu3ZtTZfNrDMH8sP2Z3+2sovZs9Ud6UgwS2sXfdPSNLp38uKGcPUdhGBC+56zr4Z8AAAD//9gpuKA=" + return "eJzMmU1v4zYTx+/5FIM9Pc8h7t2HAkGKbfaQboFkew3G1NhizReBHNlwPn1BUlJlW/L6hTKqQ4BI8vx/HM6MRqNHWNNuDhtfleToAYAlK5rDl81bPPPlAaAgL5ysWFozh18fAACaq6BtUavwM0eK0NMcVvgAsJSkCj+Ptz6CQU19iXDwrgo3O1tXzZkBlX1DfWMFMnq2nblhk6Nmm0sDRvbX0R6HGH2U8HfvQkuypt3WuuLg2gmecPzWMh3bbQWXPtjPJ/lVKvI7z6ThyHCrKbBCIXk3Y8uoZosdkx8kUNasLpN/DxYhWgS7BC5pcGPCsbROI8/hWP6Ic+mIsmJ+dUTZKWtPRVbKH56KaSgrwYOMXqCi4mOpLB7ecA5rRU6Q4XNpm9u7q0eFobSeb6kJB7+/vByIqk4O0+Vnzk19/vMHSAOv5edsVDclZz7hlJrnKMd0yycck+1nuoX061kvSHGVN+PfSwLUtk7RGaISV9RlBYjaOTKsdrAgaVYgrPG1jukH1oE1MaDJcKAbX0BBGylIIZMRuxluKKjMdK5FPCWDvYWw1ASSgXFNHtiCsLpSxARo4O357Vs4odEUsHRWQ1XuvBSoIIGeWEm7hiYKc63gRa5K8gyNedigqglQOOs9oFJR3EOdKl90ekjiE6COMG/VbZ1sar0gF5ycSnAQSl6MtU36NRCKEjwJa4rZxRU50m+d5LyBPoYflJhMiJGM/Pk7iKfVytEKmYrE+O2X7+CQ6XJATdq6qZqH12g8lLRo+mq6qVqwXHzTNDU30+Xt01+s52hyuM4Y4q1164/w37AbrlL9I5mNIv6k8myBptjKgssZOzReh1SepuqFVANk2JZSlLGDgy166MlCUbvwiAxVRBomt0E1g/dShhJZOfJk2MerHXXbELbLuXy7j/zgSJDc3NcJreZ/wwP5K0cbkDVLJT8x3Ab/E1YvpKGiC4HH4B0PoadoHJLO/P/6RVUo1sR+L7hFaHFyLax7FjZK/Xg+nXktWhdv03J1MmdBkXPW3dNtW8klJNXLXdjQ3smTfdTrvBrzKydkejKbU6jnEepasRTo+Q5b32ldnzv/4k6898esl+18z6+Tb/4R63mIhbNVRcUdNr7/yG/d2apfhDrxpndP5Zsg75ftJ/E8I9cZe8z3ksCGrkYpKAkVl41EIML4bh268NCatF8ByGyks0aTGXntriuWI933dXOZ6PvG7D5XejdNFfIczO7zhHRco9IoSmlu+qwwaunyYWIcZMjDDbz1teXA4J5Y+DPRm1Lb0B6MWKd6Q/srbQO8pn0Y/55iMybPoej3ihxy6Pzf0heWUYrp58ajslOPjUeFM0+NnzYoFS4U/Uy7P6hY1eR5snGFXcLvQaCbW9w0VIkZOiHqi81AmqIpv1v3hlTZ/BpDMD9sf+B3O6uoPVv9kZ4Ug4R28TcdfaBLJz9uKGfPURgGhO856Hr4JwAA//81fLhA" } diff --git a/metricbeat/module/vsphere/host/_meta/fields.yml b/metricbeat/module/vsphere/host/_meta/fields.yml index 2d2a05c3c053..366fa50e2d33 100644 --- a/metricbeat/module/vsphere/host/_meta/fields.yml +++ b/metricbeat/module/vsphere/host/_meta/fields.yml @@ -124,7 +124,7 @@ - name: network.packets.dropped.total.count type: long description: > - Total number of packets dropped. + Total number of packets dropped. - name: status type: keyword description: > diff --git a/metricbeat/module/vsphere/host/host.go b/metricbeat/module/vsphere/host/host.go index 4190e34cd597..f99e23a5d48c 100644 --- a/metricbeat/module/vsphere/host/host.go +++ b/metricbeat/module/vsphere/host/host.go @@ -149,7 +149,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { // Define reference of structure var metricsVar PerformanceMetrics - // Map metric names to struture fields + // Map metric names to structure fields metricMap := map[string]*int64{ "disk.capacity.usage.average": &metricsVar.DiskCapacityUsage, "disk.deviceLatency.average": &metricsVar.DiskDeviceLatency, @@ -170,16 +170,20 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { "net.droppedRx.summation": &metricsVar.NetDroppedReceived, } - var spec types.PerfQuerySpec - metricIDs := make([]types.PerfMetricId, 0, len(metricNames)) + // Retrieve only the required metrics + requiredMetrics := make(map[string]*types.PerfCounterInfo) - for _, metricName := range metricNames { - metric, exists := metrics[metricName] + for _, name := range metricNames { + metric, exists := metrics[name] if !exists { - m.Logger().Debug("Metric ", metricName, " not found") + m.Logger().Warnf("Metric %s not found", name) continue } + requiredMetrics[name] = metric + } + metricIDs := make([]types.PerfMetricId, 0, len(requiredMetrics)) + for _, metric := range requiredMetrics { metricIDs = append(metricIDs, types.PerfMetricId{ CounterId: metric.Key, }) @@ -187,7 +191,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { pc := property.DefaultCollector(c) for _, hs := range hst { - var networkRefs []types.ManagedObjectReference + networkRefs := make([]types.ManagedObjectReference, 0, len(hs.Network)) for _, obj := range hs.Network { if obj.Type == "Network" { networkRefs = append(networkRefs, obj) @@ -196,19 +200,19 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { var nets []mo.Network if len(networkRefs) > 0 { - err = pc.Retrieve(ctx, networkRefs, []string{"name"}, &nets) - if err != nil { - m.Logger().Errorf("error retrieving network from host: %v", err) + if err := pc.Retrieve(ctx, networkRefs, []string{"name"}, &nets); err != nil { + m.Logger().Errorf("Failed to retrieve network from host: %v", err) + continue } } - var outputNetworkNames []string + outputNetworkNames := make([]string, 0, len(nets)) for _, net := range nets { - name := strings.Replace(net.Name, ".", "_", -1) + name := strings.ReplaceAll(net.Name, ".", "_") outputNetworkNames = append(outputNetworkNames, name) } - spec = types.PerfQuerySpec{ + spec := types.PerfQuerySpec{ Entity: hs.Reference(), MetricId: metricIDs, MaxSample: 1, @@ -218,27 +222,28 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { // Query performance data samples, err := perfManager.Query(ctx, []types.PerfQuerySpec{spec}) if err != nil { - m.Logger().Debug("Failed to query performance data: %v", err) + m.Logger().Errorf("Failed to query performance data: %v", err) continue } - if len(samples) > 0 { - results, err := perfManager.ToMetricSeries(ctx, samples) - if err != nil { - m.Logger().Debug("Failed to query performance data: %v", err) - } + if len(samples) == 0 { + m.Logger().Debug("No samples returned from performance manager") + continue + } - for _, result := range results[0].Value { - if len(result.Value) > 0 { - if assignValue, exists := metricMap[result.Name]; exists { - *assignValue = result.Value[0] // Assign the metric value to the variable - } - } else { - m.Logger().Debug("Metric ", result.Name, ": No result found") + results, err := perfManager.ToMetricSeries(ctx, samples) + if err != nil { + m.Logger().Errorf("Failed to convert performance data: %v", err) + } + + for _, result := range results[0].Value { + if len(result.Value) > 0 { + if assignValue, exists := metricMap[result.Name]; exists { + *assignValue = result.Value[0] // Assign the metric value to the variable } + } else { + m.Logger().Debug("Metric ", result.Name, ": No result found") } - } else { - m.Logger().Debug("No samples returned from performance manager") } reporter.Event(mb.Event{ diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 1d2957306db9..026c1c6f9935 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1569,7 +1569,8 @@ metricbeat.modules: - module: vsphere enabled: true metricsets: ["datastore", "host", "virtualmachine"] - period: 10s + # Real-time data collection – An ESXi Server collects data for each performance counter every 20 seconds. + period: 20s hosts: ["https://localhost/sdk"] username: "user" From 674641b133edf076da35c49dd6bb33df17e6e7bd Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Mon, 12 Aug 2024 14:56:18 +0530 Subject: [PATCH 18/33] resolved review comments --- metricbeat/module/vsphere/host/data.go | 112 ++++++------------------- 1 file changed, 27 insertions(+), 85 deletions(-) diff --git a/metricbeat/module/vsphere/host/data.go b/metricbeat/module/vsphere/host/data.go index 5b4760739517..4036aecfe4ce 100644 --- a/metricbeat/module/vsphere/host/data.go +++ b/metricbeat/module/vsphere/host/data.go @@ -23,113 +23,55 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) -func (m *MetricSet) eventMapping(hs mo.HostSystem, perfMertics *PerformanceMetrics, networkNames []string) mapstr.M { - totalErrorPacketsCount := perfMertics.NetErrorsTransmitted + perfMertics.NetErrorsReceived - totalMulticastPacketsCount := perfMertics.NetMulticastTransmitted + perfMertics.NetMulticastReceived - totalDroppedPacketsCount := perfMertics.NetDroppedTransmitted + perfMertics.NetDroppedReceived - +func (m *MetricSet) eventMapping(hs mo.HostSystem, perfMetrics *PerformanceMetrics, networkNames []string) mapstr.M { event := mapstr.M{ "name": hs.Summary.Config.Name, "status": hs.Summary.OverallStatus, "uptime": hs.Summary.QuickStats.Uptime, - "cpu": mapstr.M{ - "used": mapstr.M{ - "mhz": hs.Summary.QuickStats.OverallCpuUsage, - }, - }, + "cpu": mapstr.M{"used": mapstr.M{"mhz": hs.Summary.QuickStats.OverallCpuUsage}}, "disk": mapstr.M{ - "capacity": mapstr.M{ - "usage": mapstr.M{ - "bytes": perfMertics.DiskCapacityUsage * 1000, - }, - }, - "devicelatency": mapstr.M{ - "average": mapstr.M{ - "ms": perfMertics.DiskDeviceLatency, - }, - }, - "latency": mapstr.M{ - "total": mapstr.M{ - "ms": perfMertics.DiskMaxTotalLatency, - }, - }, - "total": mapstr.M{ - "bytes": perfMertics.DiskUsage * 1000, - }, - "read": mapstr.M{ - "bytes": perfMertics.DiskRead * 1000, - }, - "write": mapstr.M{ - "bytes": perfMertics.DiskWrite * 1000, - }, + "capacity": mapstr.M{"usage": mapstr.M{"bytes": perfMetrics.DiskCapacityUsage * 1000}}, + "devicelatency": mapstr.M{"average": mapstr.M{"ms": perfMetrics.DiskDeviceLatency}}, + "latency": mapstr.M{"total": mapstr.M{"ms": perfMetrics.DiskMaxTotalLatency}}, + "total": mapstr.M{"bytes": perfMetrics.DiskUsage * 1000}, + "read": mapstr.M{"bytes": perfMetrics.DiskRead * 1000}, + "write": mapstr.M{"bytes": perfMetrics.DiskWrite * 1000}, }, "network": mapstr.M{ "bandwidth": mapstr.M{ - "transmitted": mapstr.M{ - "bytes": perfMertics.NetTransmitted * 1000, - }, - "received": mapstr.M{ - "bytes": perfMertics.NetReceived * 1000, - }, - "total": mapstr.M{ - "bytes": perfMertics.NetUsage * 1000, - }, + "transmitted": mapstr.M{"bytes": perfMetrics.NetTransmitted * 1000}, + "received": mapstr.M{"bytes": perfMetrics.NetReceived * 1000}, + "total": mapstr.M{"bytes": perfMetrics.NetUsage * 1000}, }, "packets": mapstr.M{ - "transmitted": mapstr.M{ - "count": perfMertics.NetPacketTransmitted, - }, - "received": mapstr.M{ - "count": perfMertics.NetPacketReceived, - }, + "transmitted": mapstr.M{"count": perfMetrics.NetPacketTransmitted}, + "received": mapstr.M{"count": perfMetrics.NetPacketReceived}, "errors": mapstr.M{ - "transmitted": mapstr.M{ - "count": perfMertics.NetErrorsTransmitted, - }, - "received": mapstr.M{ - "count": perfMertics.NetErrorsReceived, - }, - "total": mapstr.M{ - "count": totalErrorPacketsCount, - }, + "transmitted": mapstr.M{"count": perfMetrics.NetErrorsTransmitted}, + "received": mapstr.M{"count": perfMetrics.NetErrorsReceived}, + "total": mapstr.M{"count": perfMetrics.NetErrorsTransmitted + perfMetrics.NetErrorsReceived}, }, "multicast": mapstr.M{ - "transmitted": mapstr.M{ - "count": perfMertics.NetMulticastTransmitted, - }, - "received": mapstr.M{ - "count": perfMertics.NetMulticastReceived, - }, - "total": mapstr.M{ - "count": totalMulticastPacketsCount, - }, + "transmitted": mapstr.M{"count": perfMetrics.NetMulticastTransmitted}, + "received": mapstr.M{"count": perfMetrics.NetMulticastReceived}, + "total": mapstr.M{"count": perfMetrics.NetMulticastTransmitted + perfMetrics.NetMulticastReceived}, }, "dropped": mapstr.M{ - "transmitted": mapstr.M{ - "count": perfMertics.NetDroppedTransmitted, - }, - "received": mapstr.M{ - "count": perfMertics.NetDroppedReceived, - }, - "total": mapstr.M{ - "count": totalDroppedPacketsCount, - }, + "transmitted": mapstr.M{"count": perfMetrics.NetDroppedTransmitted}, + "received": mapstr.M{"count": perfMetrics.NetDroppedReceived}, + "total": mapstr.M{"count": perfMetrics.NetDroppedTransmitted + perfMetrics.NetDroppedReceived}, }, }, }, } - if hs.Summary.Hardware != nil { - totalCPU := int64(hs.Summary.Hardware.CpuMhz) * int64(hs.Summary.Hardware.NumCpuCores) - freeCPU := totalCPU - int64(hs.Summary.QuickStats.OverallCpuUsage) + if hw := hs.Summary.Hardware; hw != nil { + totalCPU := int64(hw.CpuMhz) * int64(hw.NumCpuCores) usedMemory := int64(hs.Summary.QuickStats.OverallMemoryUsage) * 1024 * 1024 - freeMemory := hs.Summary.Hardware.MemorySize - usedMemory - totalMemory := hs.Summary.Hardware.MemorySize - event.Put("cpu.total.mhz", totalCPU) - event.Put("cpu.free.mhz", freeCPU) + event.Put("cpu.free.mhz", totalCPU-int64(hs.Summary.QuickStats.OverallCpuUsage)) event.Put("memory.used.bytes", usedMemory) - event.Put("memory.free.bytes", freeMemory) - event.Put("memory.total.bytes", totalMemory) + event.Put("memory.free.bytes", hw.MemorySize-usedMemory) + event.Put("memory.total.bytes", hw.MemorySize) } else { m.Logger().Debug("'Hardware' or 'Summary' data not found. This is either a parsing error from vsphere library, an error trying to reach host/guest or incomplete information returned from host/guest") } From 3613bd13a53c8eda9d910fad755880d34e94eb8c Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Mon, 12 Aug 2024 15:56:05 +0530 Subject: [PATCH 19/33] update UTs --- metricbeat/module/vsphere/host/data_test.go | 68 ++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/metricbeat/module/vsphere/host/data_test.go b/metricbeat/module/vsphere/host/data_test.go index 91d86c4acc7d..64973910d61c 100644 --- a/metricbeat/module/vsphere/host/data_test.go +++ b/metricbeat/module/vsphere/host/data_test.go @@ -46,7 +46,27 @@ func TestEventMapping(t *testing.T) { }, } - event := m.eventMapping(HostSystemTest, &PerformanceMetrics{}, []string{}) + var PerformanceMetricsTest = PerformanceMetrics{ + NetUsage: 100, + NetDroppedTransmitted: 5, + NetDroppedReceived: 3, + NetMulticastTransmitted: 10, + NetMulticastReceived: 8, + NetErrorsTransmitted: 1, + NetErrorsReceived: 2, + NetPacketTransmitted: 1500, + NetPacketReceived: 1450, + NetReceived: 2000, + NetTransmitted: 1800, + DiskWrite: 500, + DiskRead: 600, + DiskUsage: 700, + DiskMaxTotalLatency: 100, + DiskDeviceLatency: 50, + DiskCapacityUsage: 80, + } + + event := m.eventMapping(HostSystemTest, &PerformanceMetricsTest, []string{"network-1", "network-2"}) cpuUsed, _ := event.GetValue("cpu.used.mhz") assert.EqualValues(t, 67, cpuUsed) @@ -65,4 +85,50 @@ func TestEventMapping(t *testing.T) { memoryFree, _ := event.GetValue("memory.free.bytes") assert.EqualValues(t, 0, memoryFree) + + // New asserts for PerformanceMetricsTest + diskCapacityUsage, _ := event.GetValue("disk.capacity.usage.bytes") + assert.EqualValues(t, PerformanceMetricsTest.DiskCapacityUsage*1000, diskCapacityUsage) + diskDevicelatency, _ := event.GetValue("disk.devicelatency.average.ms") + assert.EqualValues(t, PerformanceMetricsTest.DiskDeviceLatency, diskDevicelatency) + diskLatency, _ := event.GetValue("disk.latency.total.ms") + assert.EqualValues(t, PerformanceMetricsTest.DiskMaxTotalLatency, diskLatency) + diskTotal, _ := event.GetValue("disk.total.bytes") + assert.EqualValues(t, PerformanceMetricsTest.DiskUsage*1000, diskTotal) + diskRead, _ := event.GetValue("disk.read.bytes") + assert.EqualValues(t, PerformanceMetricsTest.DiskRead*1000, diskRead) + diskWrite, _ := event.GetValue("disk.write.bytes") + assert.EqualValues(t, PerformanceMetricsTest.DiskWrite*1000, diskWrite) + + networkBandwidthTransmitted, _ := event.GetValue("network.bandwidth.transmitted.bytes") + assert.EqualValues(t, PerformanceMetricsTest.NetTransmitted*1000, networkBandwidthTransmitted) + networkBandwidthReceived, _ := event.GetValue("network.bandwidth.received.bytes") + assert.EqualValues(t, PerformanceMetricsTest.NetReceived*1000, networkBandwidthReceived) + networkBandwidthTotal, _ := event.GetValue("network.bandwidth.total.bytes") + assert.EqualValues(t, PerformanceMetricsTest.NetUsage*1000, networkBandwidthTotal) + networkPacketsTransmitted, _ := event.GetValue("network.packets.transmitted.count") + assert.EqualValues(t, PerformanceMetricsTest.NetPacketTransmitted, networkPacketsTransmitted) + networkPacketsReceived, _ := event.GetValue("network.packets.received.count") + assert.EqualValues(t, PerformanceMetricsTest.NetPacketReceived, networkPacketsReceived) + networkPacketsErrorsTransmitted, _ := event.GetValue("network.packets.errors.transmitted.count") + assert.EqualValues(t, PerformanceMetricsTest.NetErrorsTransmitted, networkPacketsErrorsTransmitted) + networkPacketsErrorsReceived, _ := event.GetValue("network.packets.errors.received.count") + assert.EqualValues(t, PerformanceMetricsTest.NetErrorsReceived, networkPacketsErrorsReceived) + networkPacketsErrorsTotal, _ := event.GetValue("network.packets.errors.total.count") + assert.EqualValues(t, PerformanceMetricsTest.NetErrorsTransmitted+PerformanceMetricsTest.NetErrorsReceived, networkPacketsErrorsTotal) + networkPacketsMulticastTransmitted, _ := event.GetValue("network.packets.multicast.transmitted.count") + assert.EqualValues(t, PerformanceMetricsTest.NetMulticastTransmitted, networkPacketsMulticastTransmitted) + networkPacketsMulticastReceived, _ := event.GetValue("network.packets.multicast.received.count") + assert.EqualValues(t, PerformanceMetricsTest.NetMulticastReceived, networkPacketsMulticastReceived) + networkPacketsMulticastTotal, _ := event.GetValue("network.packets.multicast.total.count") + assert.EqualValues(t, PerformanceMetricsTest.NetMulticastTransmitted+PerformanceMetricsTest.NetMulticastReceived, networkPacketsMulticastTotal) + networkPacketsDroppedTransmitted, _ := event.GetValue("network.packets.dropped.transmitted.count") + assert.EqualValues(t, PerformanceMetricsTest.NetDroppedTransmitted, networkPacketsDroppedTransmitted) + networkPacketsDroppedReceived, _ := event.GetValue("network.packets.dropped.received.count") + assert.EqualValues(t, PerformanceMetricsTest.NetDroppedReceived, networkPacketsDroppedReceived) + networkPacketsDroppedTotal, _ := event.GetValue("network.packets.dropped.total.count") + assert.EqualValues(t, PerformanceMetricsTest.NetDroppedTransmitted+PerformanceMetricsTest.NetDroppedReceived, networkPacketsDroppedTotal) + + networkNames, _ := event.GetValue("network_names") + assert.EqualValues(t, []string{"network-1", "network-2"}, networkNames) } From cbc08f19616db644fdbfcaea3a21b39c93991e65 Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Mon, 12 Aug 2024 17:42:05 +0530 Subject: [PATCH 20/33] add vm and datastore names with count per host --- .../module/vsphere/host/_meta/data.json | 16 ++++++- .../module/vsphere/host/_meta/fields.yml | 20 ++++++++ metricbeat/module/vsphere/host/data.go | 13 +++++- metricbeat/module/vsphere/host/data_test.go | 8 +++- metricbeat/module/vsphere/host/host.go | 46 ++++++++++++++++++- metricbeat/module/vsphere/host/host_test.go | 11 +++++ 6 files changed, 109 insertions(+), 5 deletions(-) diff --git a/metricbeat/module/vsphere/host/_meta/data.json b/metricbeat/module/vsphere/host/_meta/data.json index 0ca79096c234..cb8d7896ce52 100644 --- a/metricbeat/module/vsphere/host/_meta/data.json +++ b/metricbeat/module/vsphere/host/_meta/data.json @@ -117,8 +117,22 @@ } } }, + "vm": { + "count": 2, + "names": [ + "DC0_H0_VM0", + "DC0_H0_VM1" + ] + }, + "datastore": { + "count": 1, + "names": [ + "LocalDS_0" + ] + }, + "network_count": 1, "network_names": [ - "VM Network" + "VM Network" ], "name": "DC0_H0", "status": "green", diff --git a/metricbeat/module/vsphere/host/_meta/fields.yml b/metricbeat/module/vsphere/host/_meta/fields.yml index 366fa50e2d33..4e6598b4d7b3 100644 --- a/metricbeat/module/vsphere/host/_meta/fields.yml +++ b/metricbeat/module/vsphere/host/_meta/fields.yml @@ -16,6 +16,14 @@ type: long description: > Free CPU in Mhz. + - name: datastore.names + type: keyword + description: > + Names of the datastores on the host. + - name: datastore.count + type: long + description: > + Number of datastores on the host. - name: disk.capacity.usage.bytes type: long description: > @@ -66,6 +74,10 @@ type: keyword description: > Network names. + - name: network_count + type: long + description: > + Number of networks on the host. - name: network.bandwidth.transmitted.bytes type: long description: > @@ -133,3 +145,11 @@ type: long description: > The total uptime of a host in seconds within the vSphere environment. + - name: vm.names + type: keyword + description: > + Names of the virtual machines on the host. + - name: vm.count + type: long + description: > + Number of virtual machines on the host. diff --git a/metricbeat/module/vsphere/host/data.go b/metricbeat/module/vsphere/host/data.go index 4036aecfe4ce..30cdd89b3145 100644 --- a/metricbeat/module/vsphere/host/data.go +++ b/metricbeat/module/vsphere/host/data.go @@ -23,7 +23,7 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) -func (m *MetricSet) eventMapping(hs mo.HostSystem, perfMetrics *PerformanceMetrics, networkNames []string) mapstr.M { +func (m *MetricSet) eventMapping(hs mo.HostSystem, perfMetrics *PerformanceMetrics, networkNames, datastoreNames, virtualmachine []string) mapstr.M { event := mapstr.M{ "name": hs.Summary.Config.Name, "status": hs.Summary.OverallStatus, @@ -76,8 +76,19 @@ func (m *MetricSet) eventMapping(hs mo.HostSystem, perfMetrics *PerformanceMetri m.Logger().Debug("'Hardware' or 'Summary' data not found. This is either a parsing error from vsphere library, an error trying to reach host/guest or incomplete information returned from host/guest") } + if len(virtualmachine) > 0 { + event.Put("vm.names", virtualmachine) + event.Put("vm.count", len(virtualmachine)) + } + + if len(datastoreNames) > 0 { + event.Put("datastore.names", datastoreNames) + event.Put("datastore.count", len(datastoreNames)) + } + if len(networkNames) > 0 { event.Put("network_names", networkNames) + event.Put("network_count", len(networkNames)) } return event diff --git a/metricbeat/module/vsphere/host/data_test.go b/metricbeat/module/vsphere/host/data_test.go index 64973910d61c..e0163d4e45d7 100644 --- a/metricbeat/module/vsphere/host/data_test.go +++ b/metricbeat/module/vsphere/host/data_test.go @@ -66,7 +66,7 @@ func TestEventMapping(t *testing.T) { DiskCapacityUsage: 80, } - event := m.eventMapping(HostSystemTest, &PerformanceMetricsTest, []string{"network-1", "network-2"}) + event := m.eventMapping(HostSystemTest, &PerformanceMetricsTest, []string{"network-1", "network-2"}, []string{"datastore-1", "datastore-2"}, []string{"vm-1", "vm-2"}) cpuUsed, _ := event.GetValue("cpu.used.mhz") assert.EqualValues(t, 67, cpuUsed) @@ -131,4 +131,10 @@ func TestEventMapping(t *testing.T) { networkNames, _ := event.GetValue("network_names") assert.EqualValues(t, []string{"network-1", "network-2"}, networkNames) + + vmNames, _ := event.GetValue("vm.names") + assert.EqualValues(t, []string{"vm-1", "vm-2"}, vmNames) + + datastoreNames, _ := event.GetValue("datastore.names") + assert.EqualValues(t, []string{"datastore-1", "datastore-2"}, datastoreNames) } diff --git a/metricbeat/module/vsphere/host/host.go b/metricbeat/module/vsphere/host/host.go index f99e23a5d48c..00469d16b2c7 100644 --- a/metricbeat/module/vsphere/host/host.go +++ b/metricbeat/module/vsphere/host/host.go @@ -111,7 +111,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { // Retrieve summary property for all hosts. var hst []mo.HostSystem - err = v.Retrieve(ctx, []string{"HostSystem"}, []string{"summary", "network"}, &hst) + err = v.Retrieve(ctx, []string{"HostSystem"}, []string{"summary", "network", "name", "vm", "datastore"}, &hst) if err != nil { return fmt.Errorf("error in Retrieve: %w", err) } @@ -191,6 +191,48 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { pc := property.DefaultCollector(c) for _, hs := range hst { + datastoreRefs := make([]types.ManagedObjectReference, 0, len(hs.Datastore)) + for _, obj := range hs.Datastore { + if obj.Type == "Datastore" { + datastoreRefs = append(datastoreRefs, obj) + } + } + + var datastores []mo.Datastore + if len(datastoreRefs) > 0 { + if err := pc.Retrieve(ctx, datastoreRefs, []string{"name"}, &datastores); err != nil { + m.Logger().Errorf("Failed to retrieve datastore from host: %v", err) + continue + } + } + + outputDsNames := make([]string, 0, len(datastores)) + for _, ds := range datastores { + name := strings.ReplaceAll(ds.Name, ".", "_") + outputDsNames = append(outputDsNames, name) + } + + virtualMachineRefs := make([]types.ManagedObjectReference, 0, len(hs.Vm)) + for _, obj := range hs.Vm { + if obj.Type == "VirtualMachine" { + virtualMachineRefs = append(virtualMachineRefs, obj) + } + } + + var vms []mo.VirtualMachine + if len(virtualMachineRefs) > 0 { + if err := pc.Retrieve(ctx, virtualMachineRefs, []string{"name"}, &vms); err != nil { + m.Logger().Errorf("Failed to retrieve virtual machine from host: %v", err) + continue + } + } + + outputVmNames := make([]string, 0, len(vms)) + for _, vm := range vms { + name := strings.ReplaceAll(vm.Name, ".", "_") + outputVmNames = append(outputVmNames, name) + } + networkRefs := make([]types.ManagedObjectReference, 0, len(hs.Network)) for _, obj := range hs.Network { if obj.Type == "Network" { @@ -247,7 +289,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { } reporter.Event(mb.Event{ - MetricSetFields: m.eventMapping(hs, &metricsVar, outputNetworkNames), + MetricSetFields: m.eventMapping(hs, &metricsVar, outputNetworkNames, outputDsNames, outputVmNames), }) } diff --git a/metricbeat/module/vsphere/host/host_test.go b/metricbeat/module/vsphere/host/host_test.go index 6bac0e1e2505..90fcc68c5fdb 100644 --- a/metricbeat/module/vsphere/host/host_test.go +++ b/metricbeat/module/vsphere/host/host_test.go @@ -50,6 +50,17 @@ func TestFetchEventContents(t *testing.T) { assert.EqualValues(t, "localhost.localdomain", event["name"]) + vm := event["vm"].(mapstr.M) + assert.NotNil(t, vm["names"]) + assert.GreaterOrEqual(t, vm["count"], 0) + + dataStore := event["datastore"].(mapstr.M) + assert.NotNil(t, dataStore["names"]) + assert.GreaterOrEqual(t, dataStore["count"], 0) + + assert.NotNil(t, event["network_names"]) + assert.GreaterOrEqual(t, event["network_count"], 0) + assert.NotNil(t, event["status"]) assert.GreaterOrEqual(t, event["uptime"], int32(0)) From 7c9173cf1b200067f663be1bed2a06671089cfc5 Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Mon, 12 Aug 2024 17:44:48 +0530 Subject: [PATCH 21/33] mage update --- metricbeat/docs/fields.asciidoc | 50 +++++++++++++++++++++++++++++ metricbeat/module/vsphere/fields.go | 2 +- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 7b91e6524954..c2388ac2d5aa 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -66962,6 +66962,26 @@ type: long Free CPU in Mhz. +type: long + +-- + +*`vsphere.host.datastore.names`*:: ++ +-- +Names of the datastores on the host. + + +type: keyword + +-- + +*`vsphere.host.datastore.count`*:: ++ +-- +Number of datastores on the host. + + type: long -- @@ -67088,6 +67108,16 @@ type: keyword -- +*`vsphere.host.network_count`*:: ++ +-- +Number of networks on the host. + + +type: long + +-- + *`vsphere.host.network.bandwidth.transmitted.bytes`*:: + -- @@ -67250,6 +67280,26 @@ type: keyword The total uptime of a host in seconds within the vSphere environment. +type: long + +-- + +*`vsphere.host.vm.names`*:: ++ +-- +Names of the virtual machines on the host. + + +type: keyword + +-- + +*`vsphere.host.vm.count`*:: ++ +-- +Number of virtual machines on the host. + + type: long -- diff --git a/metricbeat/module/vsphere/fields.go b/metricbeat/module/vsphere/fields.go index 0c3513620f33..ef1a28bac603 100644 --- a/metricbeat/module/vsphere/fields.go +++ b/metricbeat/module/vsphere/fields.go @@ -32,5 +32,5 @@ func init() { // AssetVsphere returns asset data. // This is the base64 encoded zlib format compressed contents of module/vsphere. func AssetVsphere() string { - return "eJzMmU1v4zYTx+/5FIM9Pc8h7t2HAkGKbfaQboFkew3G1NhizReBHNlwPn1BUlJlW/L6hTKqQ4BI8vx/HM6MRqNHWNNuDhtfleToAYAlK5rDl81bPPPlAaAgL5ysWFozh18fAACaq6BtUavwM0eK0NMcVvgAsJSkCj+Ptz6CQU19iXDwrgo3O1tXzZkBlX1DfWMFMnq2nblhk6Nmm0sDRvbX0R6HGH2U8HfvQkuypt3WuuLg2gmecPzWMh3bbQWXPtjPJ/lVKvI7z6ThyHCrKbBCIXk3Y8uoZosdkx8kUNasLpN/DxYhWgS7BC5pcGPCsbROI8/hWP6Ic+mIsmJ+dUTZKWtPRVbKH56KaSgrwYOMXqCi4mOpLB7ecA5rRU6Q4XNpm9u7q0eFobSeb6kJB7+/vByIqk4O0+Vnzk19/vMHSAOv5edsVDclZz7hlJrnKMd0yycck+1nuoX061kvSHGVN+PfSwLUtk7RGaISV9RlBYjaOTKsdrAgaVYgrPG1jukH1oE1MaDJcKAbX0BBGylIIZMRuxluKKjMdK5FPCWDvYWw1ASSgXFNHtiCsLpSxARo4O357Vs4odEUsHRWQ1XuvBSoIIGeWEm7hiYKc63gRa5K8gyNedigqglQOOs9oFJR3EOdKl90ekjiE6COMG/VbZ1sar0gF5ycSnAQSl6MtU36NRCKEjwJa4rZxRU50m+d5LyBPoYflJhMiJGM/Pk7iKfVytEKmYrE+O2X7+CQ6XJATdq6qZqH12g8lLRo+mq6qVqwXHzTNDU30+Xt01+s52hyuM4Y4q1164/w37AbrlL9I5mNIv6k8myBptjKgssZOzReh1SepuqFVANk2JZSlLGDgy166MlCUbvwiAxVRBomt0E1g/dShhJZOfJk2MerHXXbELbLuXy7j/zgSJDc3NcJreZ/wwP5K0cbkDVLJT8x3Ab/E1YvpKGiC4HH4B0PoadoHJLO/P/6RVUo1sR+L7hFaHFyLax7FjZK/Xg+nXktWhdv03J1MmdBkXPW3dNtW8klJNXLXdjQ3smTfdTrvBrzKydkejKbU6jnEepasRTo+Q5b32ldnzv/4k6898esl+18z6+Tb/4R63mIhbNVRcUdNr7/yG/d2apfhDrxpndP5Zsg75ftJ/E8I9cZe8z3ksCGrkYpKAkVl41EIML4bh268NCatF8ByGyks0aTGXntriuWI933dXOZ6PvG7D5XejdNFfIczO7zhHRco9IoSmlu+qwwaunyYWIcZMjDDbz1teXA4J5Y+DPRm1Lb0B6MWKd6Q/srbQO8pn0Y/55iMybPoej3ihxy6Pzf0heWUYrp58ajslOPjUeFM0+NnzYoFS4U/Uy7P6hY1eR5snGFXcLvQaCbW9w0VIkZOiHqi81AmqIpv1v3hlTZ/BpDMD9sf+B3O6uoPVv9kZ4Ug4R28TcdfaBLJz9uKGfPURgGhO856Hr4JwAA//81fLhA" + return "eJzMmUtv4zYQx+/+FIM9tYe4dx8KBCm22UN2CyTbazCmxhZrPgRyJMP59AWpR+WHHMumjOoQIBI9/x+HM8PXA2xot4DKFzk5mgGwZEUL+FK9xjdfZgAZeeFkwdKaBfw+AwBovoK2WanCzxwpQk8LWOMMYCVJZX4Rmz6AQU19ifDwrgiNnS2L5s0JlX1DfWMZMnq2nbnTJgfNNp9OGNnvR/scYvRRwt+9Dy3JhnZb67KDb2d4wvNHy3RstxVc+WA/neRXqcjvPJOGI8OtpsACheTdnC2jmi93TP4kgbJmPU7+LViEaBHsCjinkwMTnpV1GnkBx/JHnCtHlBTzqyNKTll6ypJS/vSUTUNZCD7J6AUqyt5XyuJhg0tYC3KCDF9K2zTvvh4Vhtx6vqUmHPx+fDkQRVk7TOcfKQf16a+fIA285B/zQd06OdMJ16l5iXJMt3TCMdk+0+1iZR7+P51BVxXE78HcUUR6sCa+CSHyGZGwpTmdL+Od8b3US3KBZxSL9Jt5L4VxnbYevuUEqEM3A1mgwjV1NQNE6RwZVjtYkjRrENb4UsfiBNa1+GQ40A13IKNKClLIZMRujhUFlblO1YnH2mCvIyw1gWRg3JAHtiCsLhQxARp4fXr9Fl5oNBmsnNVQ5DsvBSqoQc/0pO1Dk6OpevAs1zl5hsY8VKhKAhTOeg+oVBT3UNbzwiUx4wjTzkmtk00Xx/UEFYRqL8Y8k34DhCIHT8KabD56vor0Wyc5baAP4QclJhNiJCF/+vXV43rtaI1MWc347bcf4JBpPKAmbd1US6uXaDwU/Gj6arqpFqip+KZZ8t1Ml3YX82w9R5On64wh3lq3eU89b9dmo4g/rzzN/NxYv2B2blrOl2iyrcw4n7ND43UoKdNU35DygAzbXIo8riRgix56spCVLkzVgVwaJlehmsNbLkOpLhx5Muzj1466XSS13Rkfdkd+cCRIVvd1Qqv5//BA+grWJkbJUskPDM3gF2H1UhrKuhB4CN7xENY2jUPqN79e36kCxYbY7wX3NKnXKPXj+XzmtWhdvE3L1clcBEXOWXdPt20l51CrjndhQ3snT/ZRr/NqzK+UkPUKwZxDvYxQl4qlQM93GPpO6/rc+Q934rE/Zh038j2/Tj74R6yXIWbOFgVldxj4/pTfurNVH4U68aB3s/JNkPfL9rN4npHLhGvdt5zAhlWNUpATKs4biUCEceUZdgNhadLe1ZCppLNGkxlYlJYFy4FdwHXnQ9H3jdl9rnqPXFfIsZiVnvTAr5KOS1SgUeTSXHLUVumJMuAylA6jbt00nh2CjDgEH7Q0/jg8osrD8bh1a3lgcE8s/JloN9uGyMElwVS76L+b4X+px2H4RtAmzIVD0R8FOeSwK3qt7wgHKaa/+RiUnfriY1A48b3HY4VS4VLRZ9r9w6R1SZ4nO1KyK/gzCHRnS6M3g33WmKEToj7bBKR1NKV3695BYjK/xhBMD9s/lL2dVZSerX6vZ4qThHb5Dx1dMdcv328oZ09RGE4I3/MwcvZvAAAA//9HHnGY" } From 204933bff9d743c8594a21eed69e3efb92df9c5b Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Tue, 13 Aug 2024 17:47:40 +0530 Subject: [PATCH 22/33] fix and update mapping methods --- metricbeat/docs/fields.asciidoc | 12 +- metricbeat/module/vsphere/fields.go | 2 +- .../module/vsphere/host/_meta/fields.yml | 6 +- metricbeat/module/vsphere/host/data.go | 135 ++++++++---- metricbeat/module/vsphere/host/data_test.go | 91 ++++---- metricbeat/module/vsphere/host/host.go | 207 +++++++----------- metricbeat/module/vsphere/host/host_test.go | 82 ++++--- 7 files changed, 282 insertions(+), 253 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index c2388ac2d5aa..3a4a2131d2d6 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -67108,7 +67108,17 @@ type: keyword -- -*`vsphere.host.network_count`*:: +*`vsphere.host.network.names`*:: ++ +-- +Network names. + + +type: keyword + +-- + +*`vsphere.host.network.count`*:: + -- Number of networks on the host. diff --git a/metricbeat/module/vsphere/fields.go b/metricbeat/module/vsphere/fields.go index ef1a28bac603..8c3856b8a874 100644 --- a/metricbeat/module/vsphere/fields.go +++ b/metricbeat/module/vsphere/fields.go @@ -32,5 +32,5 @@ func init() { // AssetVsphere returns asset data. // This is the base64 encoded zlib format compressed contents of module/vsphere. func AssetVsphere() string { - return "eJzMmUtv4zYQx+/+FIM9tYe4dx8KBCm22UN2CyTbazCmxhZrPgRyJMP59AWpR+WHHMumjOoQIBI9/x+HM8PXA2xot4DKFzk5mgGwZEUL+FK9xjdfZgAZeeFkwdKaBfw+AwBovoK2WanCzxwpQk8LWOMMYCVJZX4Rmz6AQU19ifDwrgiNnS2L5s0JlX1DfWMZMnq2nbnTJgfNNp9OGNnvR/scYvRRwt+9Dy3JhnZb67KDb2d4wvNHy3RstxVc+WA/neRXqcjvPJOGI8OtpsACheTdnC2jmi93TP4kgbJmPU7+LViEaBHsCjinkwMTnpV1GnkBx/JHnCtHlBTzqyNKTll6ypJS/vSUTUNZCD7J6AUqyt5XyuJhg0tYC3KCDF9K2zTvvh4Vhtx6vqUmHPx+fDkQRVk7TOcfKQf16a+fIA285B/zQd06OdMJ16l5iXJMt3TCMdk+0+1iZR7+P51BVxXE78HcUUR6sCa+CSHyGZGwpTmdL+Od8b3US3KBZxSL9Jt5L4VxnbYevuUEqEM3A1mgwjV1NQNE6RwZVjtYkjRrENb4UsfiBNa1+GQ40A13IKNKClLIZMRujhUFlblO1YnH2mCvIyw1gWRg3JAHtiCsLhQxARp4fXr9Fl5oNBmsnNVQ5DsvBSqoQc/0pO1Dk6OpevAs1zl5hsY8VKhKAhTOeg+oVBT3UNbzwiUx4wjTzkmtk00Xx/UEFYRqL8Y8k34DhCIHT8KabD56vor0Wyc5baAP4QclJhNiJCF/+vXV43rtaI1MWc347bcf4JBpPKAmbd1US6uXaDwU/Gj6arqpFqip+KZZ8t1Ml3YX82w9R5On64wh3lq3eU89b9dmo4g/rzzN/NxYv2B2blrOl2iyrcw4n7ND43UoKdNU35DygAzbXIo8riRgix56spCVLkzVgVwaJlehmsNbLkOpLhx5Muzj1466XSS13Rkfdkd+cCRIVvd1Qqv5//BA+grWJkbJUskPDM3gF2H1UhrKuhB4CN7xENY2jUPqN79e36kCxYbY7wX3NKnXKPXj+XzmtWhdvE3L1clcBEXOWXdPt20l51CrjndhQ3snT/ZRr/NqzK+UkPUKwZxDvYxQl4qlQM93GPpO6/rc+Q934rE/Zh038j2/Tj74R6yXIWbOFgVldxj4/pTfurNVH4U68aB3s/JNkPfL9rN4npHLhGvdt5zAhlWNUpATKs4biUCEceUZdgNhadLe1ZCppLNGkxlYlJYFy4FdwHXnQ9H3jdl9rnqPXFfIsZiVnvTAr5KOS1SgUeTSXHLUVumJMuAylA6jbt00nh2CjDgEH7Q0/jg8osrD8bh1a3lgcE8s/JloN9uGyMElwVS76L+b4X+px2H4RtAmzIVD0R8FOeSwK3qt7wgHKaa/+RiUnfriY1A48b3HY4VS4VLRZ9r9w6R1SZ4nO1KyK/gzCHRnS6M3g33WmKEToj7bBKR1NKV3695BYjK/xhBMD9s/lL2dVZSerX6vZ4qThHb5Dx1dMdcv328oZ09RGE4I3/MwcvZvAAAA//9HHnGY" + return "eJzMmUtv4zYQx+/+FIM9tYe4dx8KBCm22UN2CyTbazCmxhZrPgRyJMP59AWpR+VnJJsyqkOASPL8fxzODIfUA2xot4DKFzk5mgGwZEUL+FK9xjtfZgAZeeFkwdKaBfw+AwBonoK2WanCzxwpQk8LWOMMYCVJZX4RX30Ag5r6EuHiXRFedrYsmjsnVPYN9Y1lyOjZduZOmzxrtnl0wsj+ONrrEKOPEv7uPWhJNrTbWpcdPLvAE64/WqZju63gygf76SS/SkV+55k0HBluNQUWKCTv5mwZ1Xy5Y/InCZQ163Hyb8EiRItgV8A5nZyYcK2s08gLOJY/4lw5oqSYXx1RcsrSU5aU8qenbBrKQvBJRi9QUfa+UhYPXxjCWpATZHgobfN69/SoMOTW8y014eD348uBKMraYTr/SDmpT3/9BGngJf+Yn9WtkzOdcJ2aQ5RjuqUTjsn2mW4XK/Pw/+kMuqogfg/mjiLSgzXxTgiRz4iELc3pfBnvjO+lXpILPKNYpN/MeymM67T18C0nQB2GGcgCFa6pqxkgSufIsNrBkqRZg7DGlzoWJ7CuxSfDge78ADKqpCCFTEbs5lhRUJnrVIN4rA32BsJSE0gGxg15YAvC6kIRE6CB16fXb+GGRpPBylkNRb7zUqCCGvTCSNoxNDmaagTPcp2TZ2jMQ4WqJEDhrPeASkVxD2W9LgyJGUeYdk1qnWy6OK4XqCBUezHmmfQbIBQ5eBLWZPPR61Wk3zrJaQP9HH5QYjIhRhLyp++vHtdrR2tkymrGb7/9AIdM4wE1aeumaq1eovFQ8KPpq+mmalBT8U3T8t1Ml3YX82w9R5On64wh3lq3eU+9btdmo4i/qJy8YxiuPE1n0Fgf0Be0HEs02VZmnM/ZofE6FLNp6n4oNoAM21yKPPYwsEUPPVnISheahEAuDZOrUM3hLZdhkSgceTLs49OOum3P2uGMD/gjPzgSJKv7OqHV/H94IH3tbBOjZKnkB4bX4Bdh9VIayroQeAje8RC6qsYh9Z1frx9UgWJD7PeCe5rUa5T68Xw581q0Lt6m5epkBkGRc9bd021byTnUquNd2NDeyZN91Ou8GvMrJWTdm5hLqMMIdalYCvR8h6nvtK7Pnf9wJ577Y9ZxM9/z6+STf8Q6DDFztigou8PE95f81p2t+ijUiSe9W5Vvgrxftl/E84xcJux133ICG7oapSAnVJw3EoEIY+cZ9iGhNWm/EpGppLNGkznTlJYFyzP7j+tOpqLvG7P7XPXuvK6QYzErPelRYyUdl6hAo8ilGXLIV+mJMmAYSodRv928PDsEGXH8ftbS+IP4iCoP5+PWTe2BwT2x8GeifXQbIgefJ6bav//dTP9LPQ/nv0XahLlwKPqjIIccdkWv9dfJsxTTf3M5Kzv1J5ezwom/uDxWKBUuFX2m3T/GWpfkebLDLLuCP4NAd6o1ejPYZ40ZOiHqs01AWkdTerfuHWEm82sMwfSw/ePg21lF6dnq93qlOElol//Q0cft+ub7DeXsKQrDCeF7HoPO/g0AAP//PPaPzA==" } diff --git a/metricbeat/module/vsphere/host/_meta/fields.yml b/metricbeat/module/vsphere/host/_meta/fields.yml index 4e6598b4d7b3..6d10ff4d422e 100644 --- a/metricbeat/module/vsphere/host/_meta/fields.yml +++ b/metricbeat/module/vsphere/host/_meta/fields.yml @@ -74,7 +74,11 @@ type: keyword description: > Network names. - - name: network_count + - name: network.names + type: keyword + description: > + Network names. + - name: network.count type: long description: > Number of networks on the host. diff --git a/metricbeat/module/vsphere/host/data.go b/metricbeat/module/vsphere/host/data.go index 30cdd89b3145..16c164d12a75 100644 --- a/metricbeat/module/vsphere/host/data.go +++ b/metricbeat/module/vsphere/host/data.go @@ -18,52 +18,23 @@ package host import ( + "fmt" + "github.com/vmware/govmomi/vim25/mo" "github.com/elastic/elastic-agent-libs/mapstr" ) -func (m *MetricSet) eventMapping(hs mo.HostSystem, perfMetrics *PerformanceMetrics, networkNames, datastoreNames, virtualmachine []string) mapstr.M { +func (m *MetricSet) eventMapping(hs mo.HostSystem, data *metricData) mapstr.M { event := mapstr.M{ "name": hs.Summary.Config.Name, "status": hs.Summary.OverallStatus, "uptime": hs.Summary.QuickStats.Uptime, "cpu": mapstr.M{"used": mapstr.M{"mhz": hs.Summary.QuickStats.OverallCpuUsage}}, - "disk": mapstr.M{ - "capacity": mapstr.M{"usage": mapstr.M{"bytes": perfMetrics.DiskCapacityUsage * 1000}}, - "devicelatency": mapstr.M{"average": mapstr.M{"ms": perfMetrics.DiskDeviceLatency}}, - "latency": mapstr.M{"total": mapstr.M{"ms": perfMetrics.DiskMaxTotalLatency}}, - "total": mapstr.M{"bytes": perfMetrics.DiskUsage * 1000}, - "read": mapstr.M{"bytes": perfMetrics.DiskRead * 1000}, - "write": mapstr.M{"bytes": perfMetrics.DiskWrite * 1000}, - }, - "network": mapstr.M{ - "bandwidth": mapstr.M{ - "transmitted": mapstr.M{"bytes": perfMetrics.NetTransmitted * 1000}, - "received": mapstr.M{"bytes": perfMetrics.NetReceived * 1000}, - "total": mapstr.M{"bytes": perfMetrics.NetUsage * 1000}, - }, - "packets": mapstr.M{ - "transmitted": mapstr.M{"count": perfMetrics.NetPacketTransmitted}, - "received": mapstr.M{"count": perfMetrics.NetPacketReceived}, - "errors": mapstr.M{ - "transmitted": mapstr.M{"count": perfMetrics.NetErrorsTransmitted}, - "received": mapstr.M{"count": perfMetrics.NetErrorsReceived}, - "total": mapstr.M{"count": perfMetrics.NetErrorsTransmitted + perfMetrics.NetErrorsReceived}, - }, - "multicast": mapstr.M{ - "transmitted": mapstr.M{"count": perfMetrics.NetMulticastTransmitted}, - "received": mapstr.M{"count": perfMetrics.NetMulticastReceived}, - "total": mapstr.M{"count": perfMetrics.NetMulticastTransmitted + perfMetrics.NetMulticastReceived}, - }, - "dropped": mapstr.M{ - "transmitted": mapstr.M{"count": perfMetrics.NetDroppedTransmitted}, - "received": mapstr.M{"count": perfMetrics.NetDroppedReceived}, - "total": mapstr.M{"count": perfMetrics.NetDroppedTransmitted + perfMetrics.NetDroppedReceived}, - }, - }, - }, } + + mapPerfMetricToEvent(event, data.perfMetrics) + if hw := hs.Summary.Hardware; hw != nil { totalCPU := int64(hw.CpuMhz) * int64(hw.NumCpuCores) usedMemory := int64(hs.Summary.QuickStats.OverallMemoryUsage) * 1024 * 1024 @@ -76,20 +47,96 @@ func (m *MetricSet) eventMapping(hs mo.HostSystem, perfMetrics *PerformanceMetri m.Logger().Debug("'Hardware' or 'Summary' data not found. This is either a parsing error from vsphere library, an error trying to reach host/guest or incomplete information returned from host/guest") } - if len(virtualmachine) > 0 { - event.Put("vm.names", virtualmachine) - event.Put("vm.count", len(virtualmachine)) + if len(data.assetsName.outputVmNames) > 0 { + event.Put("vm.names", data.assetsName.outputVmNames) + event.Put("vm.count", len(data.assetsName.outputVmNames)) } - if len(datastoreNames) > 0 { - event.Put("datastore.names", datastoreNames) - event.Put("datastore.count", len(datastoreNames)) + if len(data.assetsName.outputDsNames) > 0 { + event.Put("datastore.names", data.assetsName.outputDsNames) + event.Put("datastore.count", len(data.assetsName.outputDsNames)) } - if len(networkNames) > 0 { - event.Put("network_names", networkNames) - event.Put("network_count", len(networkNames)) + if len(data.assetsName.outputNetworkNames) > 0 { + event.Put("network_names", data.assetsName.outputNetworkNames) + event.Put("network.names", data.assetsName.outputNetworkNames) + event.Put("network.count", len(data.assetsName.outputNetworkNames)) } return event } + +func mapPerfMetricToEvent(event mapstr.M, perfMetricMap map[string]interface{}) { + if val, exist := perfMetricMap["disk.capacity.usage.average"]; exist { + fmt.Printf("val: %v\n", val) + event.Put("disk.capacity.usage.bytes", val.(int64)*1000) + } + if val, exist := perfMetricMap["disk.deviceLatency.average"]; exist { + event.Put("disk.devicelatency.average.ms", val) + } + if val, exist := perfMetricMap["disk.maxTotalLatency.latest"]; exist { + event.Put("disk.latency.total.ms", val) + } + if val, exist := perfMetricMap["disk.usage.average"]; exist { + event.Put("disk.total.bytes", val.(int64)*1000) + } + if val, exist := perfMetricMap["disk.read.average"]; exist { + event.Put("disk.read.bytes", val.(int64)*1000) + } + if val, exist := perfMetricMap["disk.write.average"]; exist { + event.Put("disk.write.bytes", val.(int64)*1000) + } + + if val, exist := perfMetricMap["net.transmitted.average"]; exist { + event.Put("network.bandwidth.transmitted.bytes", val.(int64)*1000) + } + if val, exist := perfMetricMap["net.received.average"]; exist { + event.Put("network.bandwidth.received.bytes", val.(int64)*1000) + } + if val, exist := perfMetricMap["net.usage.average"]; exist { + event.Put("network.bandwidth.total.bytes", val.(int64)*1000) + } + + if val, exist := perfMetricMap["net.packetsTx.summation"]; exist { + event.Put("network.packets.transmitted.count", val) + } + if val, exist := perfMetricMap["net.packetsRx.summation"]; exist { + event.Put("network.packets.received.count", val) + } + + netErrorsTransmitted, netErrorsTransmittedExist := perfMetricMap["net.errorsTx.summation"] + if netErrorsTransmittedExist { + event.Put("network.packets.errors.transmitted.count", netErrorsTransmitted) + } + netErrorsReceived, netErrorsReceivedExist := perfMetricMap["net.errorsRx.summation"] + if netErrorsReceivedExist { + event.Put("network.packets.errors.received.count", netErrorsReceived) + } + if netErrorsTransmittedExist && netErrorsReceivedExist { + event.Put("network.packets.errors.total.count", netErrorsTransmitted.(int64)+netErrorsReceived.(int64)) + } + + netMulticastTransmitted, netMulticastTransmittedExist := perfMetricMap["net.multicastTx.summation"] + if netMulticastTransmittedExist { + event.Put("network.packets.multicast.transmitted.count", netMulticastTransmitted) + } + netMulticastReceived, netMulticastReceivedExist := perfMetricMap["net.multicastRx.summation"] + if netMulticastReceivedExist { + event.Put("network.packets.multicast.received.count", netMulticastReceived) + } + if netMulticastTransmittedExist && netMulticastReceivedExist { + event.Put("network.packets.multicast.total.count", netMulticastTransmitted.(int64)+netMulticastReceived.(int64)) + } + + netDroppedTransmitted, netDroppedTransmittedExist := perfMetricMap["net.droppedTx.summation"] + if netDroppedTransmittedExist { + event.Put("network.packets.dropped.transmitted.count", netDroppedTransmitted) + } + netDroppedReceived, netDroppedReceivedExist := perfMetricMap["net.droppedRx.summation"] + if netDroppedReceivedExist { + event.Put("network.packets.dropped.received.count", netDroppedReceived) + } + if netDroppedTransmittedExist && netDroppedReceivedExist { + event.Put("network.packets.dropped.total.count", netDroppedTransmitted.(int64)+netDroppedReceived.(int64)) + } +} diff --git a/metricbeat/module/vsphere/host/data_test.go b/metricbeat/module/vsphere/host/data_test.go index e0163d4e45d7..85dc46183f76 100644 --- a/metricbeat/module/vsphere/host/data_test.go +++ b/metricbeat/module/vsphere/host/data_test.go @@ -46,27 +46,34 @@ func TestEventMapping(t *testing.T) { }, } - var PerformanceMetricsTest = PerformanceMetrics{ - NetUsage: 100, - NetDroppedTransmitted: 5, - NetDroppedReceived: 3, - NetMulticastTransmitted: 10, - NetMulticastReceived: 8, - NetErrorsTransmitted: 1, - NetErrorsReceived: 2, - NetPacketTransmitted: 1500, - NetPacketReceived: 1450, - NetReceived: 2000, - NetTransmitted: 1800, - DiskWrite: 500, - DiskRead: 600, - DiskUsage: 700, - DiskMaxTotalLatency: 100, - DiskDeviceLatency: 50, - DiskCapacityUsage: 80, + var metricDataTest = metricData{ + perfMetrics: map[string]interface{}{ + "disk.capacity.usage.average": int64(100), + "disk.deviceLatency.average": int64(5), + "disk.maxTotalLatency.latest": int64(3), + "disk.usage.average": int64(10), + "disk.read.average": int64(8), + "disk.write.average": int64(1), + "net.transmitted.average": int64(2), + "net.received.average": int64(1500), + "net.usage.average": int64(1450), + "net.packetsTx.summation": int64(2000), + "net.packetsRx.summation": int64(1800), + "net.errorsTx.summation": int64(500), + "net.errorsRx.summation": int64(600), + "net.multicastTx.summation": int64(700), + "net.multicastRx.summation": int64(100), + "net.droppedTx.summation": int64(50), + "net.droppedRx.summation": int64(80), + }, + assetsName: assetNames{ + outputNetworkNames: []string{"network-1", "network-2"}, + outputDsNames: []string{"datastore-1", "datastore-2"}, + outputVmNames: []string{"vm-1", "vm-2"}, + }, } - event := m.eventMapping(HostSystemTest, &PerformanceMetricsTest, []string{"network-1", "network-2"}, []string{"datastore-1", "datastore-2"}, []string{"vm-1", "vm-2"}) + event := m.eventMapping(HostSystemTest, &metricDataTest) cpuUsed, _ := event.GetValue("cpu.used.mhz") assert.EqualValues(t, 67, cpuUsed) @@ -88,53 +95,53 @@ func TestEventMapping(t *testing.T) { // New asserts for PerformanceMetricsTest diskCapacityUsage, _ := event.GetValue("disk.capacity.usage.bytes") - assert.EqualValues(t, PerformanceMetricsTest.DiskCapacityUsage*1000, diskCapacityUsage) + assert.EqualValues(t, metricDataTest.perfMetrics["disk.capacity.usage.average"].(int64)*1000, diskCapacityUsage) diskDevicelatency, _ := event.GetValue("disk.devicelatency.average.ms") - assert.EqualValues(t, PerformanceMetricsTest.DiskDeviceLatency, diskDevicelatency) + assert.EqualValues(t, metricDataTest.perfMetrics["disk.deviceLatency.average"], diskDevicelatency) diskLatency, _ := event.GetValue("disk.latency.total.ms") - assert.EqualValues(t, PerformanceMetricsTest.DiskMaxTotalLatency, diskLatency) + assert.EqualValues(t, metricDataTest.perfMetrics["disk.maxTotalLatency.latest"], diskLatency) diskTotal, _ := event.GetValue("disk.total.bytes") - assert.EqualValues(t, PerformanceMetricsTest.DiskUsage*1000, diskTotal) + assert.EqualValues(t, metricDataTest.perfMetrics["disk.usage.average"].(int64)*1000, diskTotal) diskRead, _ := event.GetValue("disk.read.bytes") - assert.EqualValues(t, PerformanceMetricsTest.DiskRead*1000, diskRead) + assert.EqualValues(t, metricDataTest.perfMetrics["disk.read.average"].(int64)*1000, diskRead) diskWrite, _ := event.GetValue("disk.write.bytes") - assert.EqualValues(t, PerformanceMetricsTest.DiskWrite*1000, diskWrite) + assert.EqualValues(t, metricDataTest.perfMetrics["disk.write.average"].(int64)*1000, diskWrite) networkBandwidthTransmitted, _ := event.GetValue("network.bandwidth.transmitted.bytes") - assert.EqualValues(t, PerformanceMetricsTest.NetTransmitted*1000, networkBandwidthTransmitted) + assert.EqualValues(t, metricDataTest.perfMetrics["net.transmitted.average"].(int64)*1000, networkBandwidthTransmitted) networkBandwidthReceived, _ := event.GetValue("network.bandwidth.received.bytes") - assert.EqualValues(t, PerformanceMetricsTest.NetReceived*1000, networkBandwidthReceived) + assert.EqualValues(t, metricDataTest.perfMetrics["net.received.average"].(int64)*1000, networkBandwidthReceived) networkBandwidthTotal, _ := event.GetValue("network.bandwidth.total.bytes") - assert.EqualValues(t, PerformanceMetricsTest.NetUsage*1000, networkBandwidthTotal) + assert.EqualValues(t, metricDataTest.perfMetrics["net.usage.average"].(int64)*1000, networkBandwidthTotal) networkPacketsTransmitted, _ := event.GetValue("network.packets.transmitted.count") - assert.EqualValues(t, PerformanceMetricsTest.NetPacketTransmitted, networkPacketsTransmitted) + assert.EqualValues(t, metricDataTest.perfMetrics["net.packetsTx.summation"], networkPacketsTransmitted) networkPacketsReceived, _ := event.GetValue("network.packets.received.count") - assert.EqualValues(t, PerformanceMetricsTest.NetPacketReceived, networkPacketsReceived) + assert.EqualValues(t, metricDataTest.perfMetrics["net.packetsRx.summation"], networkPacketsReceived) networkPacketsErrorsTransmitted, _ := event.GetValue("network.packets.errors.transmitted.count") - assert.EqualValues(t, PerformanceMetricsTest.NetErrorsTransmitted, networkPacketsErrorsTransmitted) + assert.EqualValues(t, metricDataTest.perfMetrics["net.errorsTx.summation"], networkPacketsErrorsTransmitted) networkPacketsErrorsReceived, _ := event.GetValue("network.packets.errors.received.count") - assert.EqualValues(t, PerformanceMetricsTest.NetErrorsReceived, networkPacketsErrorsReceived) + assert.EqualValues(t, metricDataTest.perfMetrics["net.errorsRx.summation"], networkPacketsErrorsReceived) networkPacketsErrorsTotal, _ := event.GetValue("network.packets.errors.total.count") - assert.EqualValues(t, PerformanceMetricsTest.NetErrorsTransmitted+PerformanceMetricsTest.NetErrorsReceived, networkPacketsErrorsTotal) + assert.EqualValues(t, metricDataTest.perfMetrics["net.errorsTx.summation"].(int64)+metricDataTest.perfMetrics["net.errorsRx.summation"].(int64), networkPacketsErrorsTotal) networkPacketsMulticastTransmitted, _ := event.GetValue("network.packets.multicast.transmitted.count") - assert.EqualValues(t, PerformanceMetricsTest.NetMulticastTransmitted, networkPacketsMulticastTransmitted) + assert.EqualValues(t, metricDataTest.perfMetrics["net.multicastTx.summation"], networkPacketsMulticastTransmitted) networkPacketsMulticastReceived, _ := event.GetValue("network.packets.multicast.received.count") - assert.EqualValues(t, PerformanceMetricsTest.NetMulticastReceived, networkPacketsMulticastReceived) + assert.EqualValues(t, metricDataTest.perfMetrics["net.multicastRx.summation"], networkPacketsMulticastReceived) networkPacketsMulticastTotal, _ := event.GetValue("network.packets.multicast.total.count") - assert.EqualValues(t, PerformanceMetricsTest.NetMulticastTransmitted+PerformanceMetricsTest.NetMulticastReceived, networkPacketsMulticastTotal) + assert.EqualValues(t, metricDataTest.perfMetrics["net.multicastTx.summation"].(int64)+metricDataTest.perfMetrics["net.multicastRx.summation"].(int64), networkPacketsMulticastTotal) networkPacketsDroppedTransmitted, _ := event.GetValue("network.packets.dropped.transmitted.count") - assert.EqualValues(t, PerformanceMetricsTest.NetDroppedTransmitted, networkPacketsDroppedTransmitted) + assert.EqualValues(t, metricDataTest.perfMetrics["net.droppedTx.summation"], networkPacketsDroppedTransmitted) networkPacketsDroppedReceived, _ := event.GetValue("network.packets.dropped.received.count") - assert.EqualValues(t, PerformanceMetricsTest.NetDroppedReceived, networkPacketsDroppedReceived) + assert.EqualValues(t, metricDataTest.perfMetrics["net.droppedRx.summation"], networkPacketsDroppedReceived) networkPacketsDroppedTotal, _ := event.GetValue("network.packets.dropped.total.count") - assert.EqualValues(t, PerformanceMetricsTest.NetDroppedTransmitted+PerformanceMetricsTest.NetDroppedReceived, networkPacketsDroppedTotal) + assert.EqualValues(t, metricDataTest.perfMetrics["net.droppedTx.summation"].(int64)+metricDataTest.perfMetrics["net.droppedRx.summation"].(int64), networkPacketsDroppedTotal) networkNames, _ := event.GetValue("network_names") - assert.EqualValues(t, []string{"network-1", "network-2"}, networkNames) + assert.EqualValues(t, metricDataTest.assetsName.outputNetworkNames, networkNames) vmNames, _ := event.GetValue("vm.names") - assert.EqualValues(t, []string{"vm-1", "vm-2"}, vmNames) + assert.EqualValues(t, metricDataTest.assetsName.outputVmNames, vmNames) datastoreNames, _ := event.GetValue("datastore.names") - assert.EqualValues(t, []string{"datastore-1", "datastore-2"}, datastoreNames) + assert.EqualValues(t, metricDataTest.assetsName.outputDsNames, datastoreNames) } diff --git a/metricbeat/module/vsphere/host/host.go b/metricbeat/module/vsphere/host/host.go index 00469d16b2c7..faee32519fcb 100644 --- a/metricbeat/module/vsphere/host/host.go +++ b/metricbeat/module/vsphere/host/host.go @@ -54,25 +54,36 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ms}, nil } -// Structure to hold performance metrics values -type PerformanceMetrics struct { - NetUsage int64 - NetDroppedTransmitted int64 - NetDroppedReceived int64 - NetMulticastTransmitted int64 - NetMulticastReceived int64 - NetErrorsTransmitted int64 - NetErrorsReceived int64 - NetPacketTransmitted int64 - NetPacketReceived int64 - NetReceived int64 - NetTransmitted int64 - DiskWrite int64 - DiskRead int64 - DiskUsage int64 - DiskMaxTotalLatency int64 - DiskDeviceLatency int64 - DiskCapacityUsage int64 +type metricData struct { + perfMetrics map[string]interface{} + assetsName assetNames +} + +type assetNames struct { + outputNetworkNames []string + outputDsNames []string + outputVmNames []string +} + +// Define metrics to be collected +var metricNames = []string{ + "disk.capacity.usage.average", + "disk.deviceLatency.average", + "disk.maxTotalLatency.latest", + "disk.usage.average", + "disk.read.average", + "disk.write.average", + "net.transmitted.average", + "net.received.average", + "net.usage.average", + "net.packetsTx.summation", + "net.packetsRx.summation", + "net.errorsTx.summation", + "net.errorsRx.summation", + "net.multicastTx.summation", + "net.multicastRx.summation", + "net.droppedTx.summation", + "net.droppedRx.summation", } // Fetch methods implements the data gathering and data conversion to the right @@ -125,51 +136,6 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { return fmt.Errorf("failed to retrieve metrics: %w", err) } - // Define metrics to be collected - metricNames := []string{ - "disk.capacity.usage.average", - "disk.deviceLatency.average", - "disk.maxTotalLatency.latest", - "disk.usage.average", - "disk.read.average", - "disk.write.average", - "net.transmitted.average", - "net.received.average", - "net.usage.average", - "net.packetsTx.summation", - "net.packetsRx.summation", - "net.errorsTx.summation", - "net.errorsRx.summation", - "net.multicastTx.summation", - "net.multicastRx.summation", - "net.droppedTx.summation", - "net.droppedRx.summation", - } - - // Define reference of structure - var metricsVar PerformanceMetrics - - // Map metric names to structure fields - metricMap := map[string]*int64{ - "disk.capacity.usage.average": &metricsVar.DiskCapacityUsage, - "disk.deviceLatency.average": &metricsVar.DiskDeviceLatency, - "disk.maxTotalLatency.latest": &metricsVar.DiskMaxTotalLatency, - "net.usage.average": &metricsVar.NetUsage, - "disk.usage.average": &metricsVar.DiskUsage, - "disk.read.average": &metricsVar.DiskRead, - "disk.write.average": &metricsVar.DiskWrite, - "net.transmitted.average": &metricsVar.NetTransmitted, - "net.received.average": &metricsVar.NetReceived, - "net.packetsTx.summation": &metricsVar.NetPacketTransmitted, - "net.packetsRx.summation": &metricsVar.NetPacketReceived, - "net.errorsTx.summation": &metricsVar.NetErrorsTransmitted, - "net.errorsRx.summation": &metricsVar.NetErrorsReceived, - "net.multicastTx.summation": &metricsVar.NetMulticastTransmitted, - "net.multicastRx.summation": &metricsVar.NetMulticastReceived, - "net.droppedTx.summation": &metricsVar.NetDroppedTransmitted, - "net.droppedRx.summation": &metricsVar.NetDroppedReceived, - } - // Retrieve only the required metrics requiredMetrics := make(map[string]*types.PerfCounterInfo) @@ -188,70 +154,13 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { CounterId: metric.Key, }) } + metricMap := map[string]interface{}{} pc := property.DefaultCollector(c) for _, hs := range hst { - datastoreRefs := make([]types.ManagedObjectReference, 0, len(hs.Datastore)) - for _, obj := range hs.Datastore { - if obj.Type == "Datastore" { - datastoreRefs = append(datastoreRefs, obj) - } - } - - var datastores []mo.Datastore - if len(datastoreRefs) > 0 { - if err := pc.Retrieve(ctx, datastoreRefs, []string{"name"}, &datastores); err != nil { - m.Logger().Errorf("Failed to retrieve datastore from host: %v", err) - continue - } - } - - outputDsNames := make([]string, 0, len(datastores)) - for _, ds := range datastores { - name := strings.ReplaceAll(ds.Name, ".", "_") - outputDsNames = append(outputDsNames, name) - } - - virtualMachineRefs := make([]types.ManagedObjectReference, 0, len(hs.Vm)) - for _, obj := range hs.Vm { - if obj.Type == "VirtualMachine" { - virtualMachineRefs = append(virtualMachineRefs, obj) - } - } - - var vms []mo.VirtualMachine - if len(virtualMachineRefs) > 0 { - if err := pc.Retrieve(ctx, virtualMachineRefs, []string{"name"}, &vms); err != nil { - m.Logger().Errorf("Failed to retrieve virtual machine from host: %v", err) - continue - } - } - - outputVmNames := make([]string, 0, len(vms)) - for _, vm := range vms { - name := strings.ReplaceAll(vm.Name, ".", "_") - outputVmNames = append(outputVmNames, name) - } - - networkRefs := make([]types.ManagedObjectReference, 0, len(hs.Network)) - for _, obj := range hs.Network { - if obj.Type == "Network" { - networkRefs = append(networkRefs, obj) - } - } - - var nets []mo.Network - if len(networkRefs) > 0 { - if err := pc.Retrieve(ctx, networkRefs, []string{"name"}, &nets); err != nil { - m.Logger().Errorf("Failed to retrieve network from host: %v", err) - continue - } - } - - outputNetworkNames := make([]string, 0, len(nets)) - for _, net := range nets { - name := strings.ReplaceAll(net.Name, ".", "_") - outputNetworkNames = append(outputNetworkNames, name) + assetNames, err := getAssetNames(ctx, pc, &hs) + if err != nil { + m.Logger().Errorf("Failed to retrieve object from host: %w", err) } spec := types.PerfQuerySpec{ @@ -280,18 +189,54 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { for _, result := range results[0].Value { if len(result.Value) > 0 { - if assignValue, exists := metricMap[result.Name]; exists { - *assignValue = result.Value[0] // Assign the metric value to the variable - } - } else { - m.Logger().Debug("Metric ", result.Name, ": No result found") + metricMap[result.Name] = result.Value[0] + continue } + m.Logger().Debugf("Metric %v: No result found", result.Name) } reporter.Event(mb.Event{ - MetricSetFields: m.eventMapping(hs, &metricsVar, outputNetworkNames, outputDsNames, outputVmNames), + MetricSetFields: m.eventMapping(hs, &metricData{ + perfMetrics: metricMap, + assetsName: *assetNames, + }), }) } return nil } + +func getAssetNames(ctx context.Context, pc *property.Collector, hs *mo.HostSystem) (*assetNames, error) { + referenceList := make([]types.ManagedObjectReference, 0, len(hs.Datastore)+len(hs.Vm)+len(hs.Network)) + referenceList = append(referenceList, hs.Datastore...) + referenceList = append(referenceList, hs.Vm...) + referenceList = append(referenceList, hs.Network...) + + var objects []mo.ManagedEntity + if len(referenceList) > 0 { + if err := pc.Retrieve(ctx, referenceList, []string{"name"}, &objects); err != nil { + return nil, err + } + } + + outputDsNames := make([]string, 0, len(hs.Datastore)) + outputVmNames := make([]string, 0, len(hs.Vm)) + outputNetworkNames := make([]string, 0, len(hs.Network)) + for _, ob := range objects { + name := strings.ReplaceAll(ob.Name, ".", "_") + switch ob.Reference().Type { + case "Datastore": + outputDsNames = append(outputDsNames, name) + case "VirtualMachine": + outputVmNames = append(outputVmNames, name) + case "Network": + outputNetworkNames = append(outputNetworkNames, name) + } + } + + return &assetNames{ + outputNetworkNames: outputNetworkNames, + outputDsNames: outputDsNames, + outputVmNames: outputVmNames, + }, nil +} diff --git a/metricbeat/module/vsphere/host/host_test.go b/metricbeat/module/vsphere/host/host_test.go index 90fcc68c5fdb..2889004a965b 100644 --- a/metricbeat/module/vsphere/host/host_test.go +++ b/metricbeat/module/vsphere/host/host_test.go @@ -59,7 +59,9 @@ func TestFetchEventContents(t *testing.T) { assert.GreaterOrEqual(t, dataStore["count"], 0) assert.NotNil(t, event["network_names"]) - assert.GreaterOrEqual(t, event["network_count"], 0) + network := event["network"].(mapstr.M) + assert.NotNil(t, network["names"]) + assert.GreaterOrEqual(t, network["count"], 0) assert.NotNil(t, event["status"]) @@ -78,17 +80,22 @@ func TestFetchEventContents(t *testing.T) { disk := event["disk"].(mapstr.M) - diskCapacity := disk["capacity"].(mapstr.M) - diskCapacityUsage := diskCapacity["usage"].(mapstr.M) - assert.GreaterOrEqual(t, diskCapacityUsage["bytes"], int64(0)) - - diskDevielatency := disk["devicelatency"].(mapstr.M) - diskDevielatencyAverage := diskDevielatency["average"].(mapstr.M) - assert.GreaterOrEqual(t, diskDevielatencyAverage["ms"], int64(0)) + diskCapacity, ok := disk["capacity"].(mapstr.M) + if ok { + diskCapacityUsage := diskCapacity["usage"].(mapstr.M) + assert.GreaterOrEqual(t, diskCapacityUsage["bytes"], int64(0)) + } - diskLatency := disk["latency"].(mapstr.M) - diskLatencyTotal := diskLatency["total"].(mapstr.M) - assert.GreaterOrEqual(t, diskLatencyTotal["ms"], int64(0)) + diskDevielatency, ok := disk["devicelatency"].(mapstr.M) + if ok { + diskDevielatencyAverage := diskDevielatency["average"].(mapstr.M) + assert.GreaterOrEqual(t, diskDevielatencyAverage["ms"], int64(0)) + } + diskLatency, ok := disk["latency"].(mapstr.M) + if ok { + diskLatencyTotal := diskLatency["total"].(mapstr.M) + assert.GreaterOrEqual(t, diskLatencyTotal["ms"], int64(0)) + } diskTotal := disk["total"].(mapstr.M) assert.GreaterOrEqual(t, diskTotal["bytes"], int64(0)) @@ -110,7 +117,7 @@ func TestFetchEventContents(t *testing.T) { memoryFree := memory["free"].(mapstr.M) assert.EqualValues(t, uint64(2822230016), memoryFree["bytes"]) - network := event["network"].(mapstr.M) + network = event["network"].(mapstr.M) networkBandwidth := network["bandwidth"].(mapstr.M) networkBandwidthTransmitted := networkBandwidth["transmitted"].(mapstr.M) @@ -126,29 +133,38 @@ func TestFetchEventContents(t *testing.T) { assert.GreaterOrEqual(t, networkPacketsTransmitted["count"], int64(0)) assert.GreaterOrEqual(t, networkPacketsReceived["count"], int64(0)) - networkErrors := networkPackets["errors"].(mapstr.M) - networkErrorsTransmitted := networkErrors["transmitted"].(mapstr.M) - networkErrorsReceived := networkErrors["received"].(mapstr.M) - networkErrorsTotal := networkErrors["total"].(mapstr.M) - assert.GreaterOrEqual(t, networkErrorsTransmitted["count"], int64(0)) - assert.GreaterOrEqual(t, networkErrorsReceived["count"], int64(0)) - assert.GreaterOrEqual(t, networkErrorsTotal["count"], int64(0)) + networkErrors, ok := networkPackets["errors"].(mapstr.M) + if ok { + networkErrorsTransmitted := networkErrors["transmitted"].(mapstr.M) + networkErrorsReceived := networkErrors["received"].(mapstr.M) + networkErrorsTotal := networkErrors["total"].(mapstr.M) + assert.GreaterOrEqual(t, networkErrorsTransmitted["count"], int64(0)) + assert.GreaterOrEqual(t, networkErrorsReceived["count"], int64(0)) + assert.GreaterOrEqual(t, networkErrorsTotal["count"], int64(0)) + } networkMulticast := networkPackets["multicast"].(mapstr.M) - networkMulticastTransmitted := networkMulticast["transmitted"].(mapstr.M) - networkMulticastReceived := networkMulticast["received"].(mapstr.M) - networkMulticastTotal := networkMulticast["total"].(mapstr.M) - assert.GreaterOrEqual(t, networkMulticastTransmitted["count"], int64(0)) - assert.GreaterOrEqual(t, networkMulticastReceived["count"], int64(0)) - assert.GreaterOrEqual(t, networkMulticastTotal["count"], int64(0)) - - networkDropped := networkPackets["dropped"].(mapstr.M) - networkDroppedTransmitted := networkDropped["transmitted"].(mapstr.M) - networkDroppedReceived := networkDropped["received"].(mapstr.M) - networkDroppedTotal := networkDropped["total"].(mapstr.M) - assert.GreaterOrEqual(t, networkDroppedTransmitted["count"], int64(0)) - assert.GreaterOrEqual(t, networkDroppedReceived["count"], int64(0)) - assert.GreaterOrEqual(t, networkDroppedTotal["count"], int64(0)) + if networkMulticastTransmitted, ok := networkMulticast["transmitted"].(mapstr.M); ok { + assert.GreaterOrEqual(t, networkMulticastTransmitted["count"], int64(0)) + } + if networkMulticastReceived, ok := networkMulticast["received"].(mapstr.M); ok { + assert.GreaterOrEqual(t, networkMulticastReceived["count"], int64(0)) + } + if networkMulticastTotal, ok := networkMulticast["total"].(mapstr.M); ok { + assert.GreaterOrEqual(t, networkMulticastTotal["count"], int64(0)) + } + + if networkDropped, ok := networkPackets["dropped"].(mapstr.M); ok { + if networkDroppedTransmitted, ok := networkDropped["transmitted"].(mapstr.M); ok { + assert.GreaterOrEqual(t, networkDroppedTransmitted["count"], int64(0)) + } + if networkDroppedReceived, ok := networkDropped["received"].(mapstr.M); ok { + assert.GreaterOrEqual(t, networkDroppedReceived["count"], int64(0)) + } + if networkDroppedTotal, ok := networkDropped["total"].(mapstr.M); ok { + assert.GreaterOrEqual(t, networkDroppedTotal["count"], int64(0)) + } + } } func TestData(t *testing.T) { From 7da85f401c8188b36603d95868bb686ba923dbc5 Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Wed, 14 Aug 2024 11:47:39 +0530 Subject: [PATCH 23/33] minor changes --- metricbeat/module/vsphere/host/data.go | 3 --- metricbeat/module/vsphere/host/host.go | 23 +++++++++++++++++------ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/metricbeat/module/vsphere/host/data.go b/metricbeat/module/vsphere/host/data.go index 16c164d12a75..7e984074953a 100644 --- a/metricbeat/module/vsphere/host/data.go +++ b/metricbeat/module/vsphere/host/data.go @@ -18,8 +18,6 @@ package host import ( - "fmt" - "github.com/vmware/govmomi/vim25/mo" "github.com/elastic/elastic-agent-libs/mapstr" @@ -68,7 +66,6 @@ func (m *MetricSet) eventMapping(hs mo.HostSystem, data *metricData) mapstr.M { func mapPerfMetricToEvent(event mapstr.M, perfMetricMap map[string]interface{}) { if val, exist := perfMetricMap["disk.capacity.usage.average"]; exist { - fmt.Printf("val: %v\n", val) event.Put("disk.capacity.usage.bytes", val.(int64)*1000) } if val, exist := perfMetricMap["disk.deviceLatency.average"]; exist { diff --git a/metricbeat/module/vsphere/host/host.go b/metricbeat/module/vsphere/host/host.go index faee32519fcb..66ec588b412c 100644 --- a/metricbeat/module/vsphere/host/host.go +++ b/metricbeat/module/vsphere/host/host.go @@ -154,7 +154,6 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { CounterId: metric.Key, }) } - metricMap := map[string]interface{}{} pc := property.DefaultCollector(c) for _, hs := range hst { @@ -187,6 +186,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { m.Logger().Errorf("Failed to convert performance data: %v", err) } + metricMap := make(map[string]interface{}) for _, result := range results[0].Value { if len(result.Value) > 0 { metricMap[result.Name] = result.Value[0] @@ -207,10 +207,9 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { } func getAssetNames(ctx context.Context, pc *property.Collector, hs *mo.HostSystem) (*assetNames, error) { - referenceList := make([]types.ManagedObjectReference, 0, len(hs.Datastore)+len(hs.Vm)+len(hs.Network)) + referenceList := make([]types.ManagedObjectReference, 0, len(hs.Datastore)+len(hs.Vm)) referenceList = append(referenceList, hs.Datastore...) referenceList = append(referenceList, hs.Vm...) - referenceList = append(referenceList, hs.Network...) var objects []mo.ManagedEntity if len(referenceList) > 0 { @@ -221,7 +220,6 @@ func getAssetNames(ctx context.Context, pc *property.Collector, hs *mo.HostSyste outputDsNames := make([]string, 0, len(hs.Datastore)) outputVmNames := make([]string, 0, len(hs.Vm)) - outputNetworkNames := make([]string, 0, len(hs.Network)) for _, ob := range objects { name := strings.ReplaceAll(ob.Name, ".", "_") switch ob.Reference().Type { @@ -229,11 +227,24 @@ func getAssetNames(ctx context.Context, pc *property.Collector, hs *mo.HostSyste outputDsNames = append(outputDsNames, name) case "VirtualMachine": outputVmNames = append(outputVmNames, name) - case "Network": - outputNetworkNames = append(outputNetworkNames, name) } } + // calling network explicitly because of mo.Network's ManagedEntityObject.Name does not store Network name + // instead mo.Network.Name contains correct value of Network name + var netObjects []mo.Network + if len(referenceList) > 0 { + if err := pc.Retrieve(ctx, hs.Network, []string{"name"}, &netObjects); err != nil { + return nil, err + } + } + + outputNetworkNames := make([]string, 0, len(hs.Network)) + for _, ob := range objects { + name := strings.ReplaceAll(ob.Name, ".", "_") + outputNetworkNames = append(outputNetworkNames, name) + } + return &assetNames{ outputNetworkNames: outputNetworkNames, outputDsNames: outputDsNames, From bb0398fc8ea8a05d5a590c6f2f6351bcccbd27dd Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Wed, 14 Aug 2024 11:51:34 +0530 Subject: [PATCH 24/33] remove fmt.Errorf --- metricbeat/module/vsphere/host/host.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/vsphere/host/host.go b/metricbeat/module/vsphere/host/host.go index 66ec588b412c..51a59373ffef 100644 --- a/metricbeat/module/vsphere/host/host.go +++ b/metricbeat/module/vsphere/host/host.go @@ -100,7 +100,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { defer func() { if err := client.Logout(ctx); err != nil { - m.Logger().Debug(fmt.Errorf("error trying to logout from vshphere: %w", err)) + m.Logger().Errorf("error trying to logout from vshphere: %w", err) } }() @@ -116,7 +116,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { defer func() { if err := v.Destroy(ctx); err != nil { - m.Logger().Debug(fmt.Errorf("error trying to destroy view from vshphere: %w", err)) + m.Logger().Errorf("error trying to destroy view from vshphere: %w", err) } }() From 7fade8f38ad6018929462f3ef685f1a7f7bee61a Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Wed, 14 Aug 2024 14:21:31 +0530 Subject: [PATCH 25/33] fix loop --- metricbeat/module/vsphere/host/host.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/vsphere/host/host.go b/metricbeat/module/vsphere/host/host.go index 51a59373ffef..a6613b8533d7 100644 --- a/metricbeat/module/vsphere/host/host.go +++ b/metricbeat/module/vsphere/host/host.go @@ -233,14 +233,14 @@ func getAssetNames(ctx context.Context, pc *property.Collector, hs *mo.HostSyste // calling network explicitly because of mo.Network's ManagedEntityObject.Name does not store Network name // instead mo.Network.Name contains correct value of Network name var netObjects []mo.Network - if len(referenceList) > 0 { + if len(hs.Network) > 0 { if err := pc.Retrieve(ctx, hs.Network, []string{"name"}, &netObjects); err != nil { return nil, err } } outputNetworkNames := make([]string, 0, len(hs.Network)) - for _, ob := range objects { + for _, ob := range netObjects { name := strings.ReplaceAll(ob.Name, ".", "_") outputNetworkNames = append(outputNetworkNames, name) } From d32c480e7b4234ba1b77614733cb464b4713eb86 Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Wed, 14 Aug 2024 14:35:19 +0530 Subject: [PATCH 26/33] golangci-lint fix --- metricbeat/module/vsphere/host/host.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metricbeat/module/vsphere/host/host.go b/metricbeat/module/vsphere/host/host.go index a6613b8533d7..13cb98843899 100644 --- a/metricbeat/module/vsphere/host/host.go +++ b/metricbeat/module/vsphere/host/host.go @@ -156,14 +156,14 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { } pc := property.DefaultCollector(c) - for _, hs := range hst { - assetNames, err := getAssetNames(ctx, pc, &hs) + for i := range hst { + assetNames, err := getAssetNames(ctx, pc, &hst[i]) if err != nil { m.Logger().Errorf("Failed to retrieve object from host: %w", err) } spec := types.PerfQuerySpec{ - Entity: hs.Reference(), + Entity: hst[i].Reference(), MetricId: metricIDs, MaxSample: 1, IntervalId: 20, // right now we are only grabbing real time metrics from the performance manager @@ -196,7 +196,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { } reporter.Event(mb.Event{ - MetricSetFields: m.eventMapping(hs, &metricData{ + MetricSetFields: m.eventMapping(hst[i], &metricData{ perfMetrics: metricMap, assetsName: *assetNames, }), From 95577d7e83e18cc770618c6767509c6772da54a2 Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Tue, 20 Aug 2024 10:05:45 +0530 Subject: [PATCH 27/33] resolved comments --- metricbeat/module/vsphere/host/host.go | 38 ++++++++++++-------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/metricbeat/module/vsphere/host/host.go b/metricbeat/module/vsphere/host/host.go index 13cb98843899..22e4756661c9 100644 --- a/metricbeat/module/vsphere/host/host.go +++ b/metricbeat/module/vsphere/host/host.go @@ -100,7 +100,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { defer func() { if err := client.Logout(ctx); err != nil { - m.Logger().Errorf("error trying to logout from vshphere: %w", err) + m.Logger().Errorf("error trying to log out from vSphere: %w", err) } }() @@ -116,7 +116,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { defer func() { if err := v.Destroy(ctx); err != nil { - m.Logger().Errorf("error trying to destroy view from vshphere: %w", err) + m.Logger().Errorf("error trying to destroy view from vSphere: %w", err) } }() @@ -159,7 +159,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { for i := range hst { assetNames, err := getAssetNames(ctx, pc, &hst[i]) if err != nil { - m.Logger().Errorf("Failed to retrieve object from host: %w", err) + m.Logger().Errorf("Failed to retrieve object from host %s: %w", hst[i].Name, err) } spec := types.PerfQuerySpec{ @@ -172,7 +172,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { // Query performance data samples, err := perfManager.Query(ctx, []types.PerfQuerySpec{spec}) if err != nil { - m.Logger().Errorf("Failed to query performance data: %v", err) + m.Logger().Errorf("Failed to query performance data from host %s: %v", hst[i].Name, err) continue } @@ -183,7 +183,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { results, err := perfManager.ToMetricSeries(ctx, samples) if err != nil { - m.Logger().Errorf("Failed to convert performance data: %v", err) + m.Logger().Errorf("Failed to convert performance data to metric series for host %s: %v", hst[i].Name, err) } metricMap := make(map[string]interface{}) @@ -192,13 +192,13 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { metricMap[result.Name] = result.Value[0] continue } - m.Logger().Debugf("Metric %v: No result found", result.Name) + m.Logger().Debugf("For host %s,Metric %v: No result found", hst[i].Name, result.Name) } reporter.Event(mb.Event{ MetricSetFields: m.eventMapping(hst[i], &metricData{ perfMetrics: metricMap, - assetsName: *assetNames, + assetsName: assetNames, }), }) } @@ -206,15 +206,13 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { return nil } -func getAssetNames(ctx context.Context, pc *property.Collector, hs *mo.HostSystem) (*assetNames, error) { - referenceList := make([]types.ManagedObjectReference, 0, len(hs.Datastore)+len(hs.Vm)) - referenceList = append(referenceList, hs.Datastore...) - referenceList = append(referenceList, hs.Vm...) +func getAssetNames(ctx context.Context, pc *property.Collector, hs *mo.HostSystem) (assetNames, error) { + referenceList := append(hs.Datastore, hs.Vm...) var objects []mo.ManagedEntity if len(referenceList) > 0 { if err := pc.Retrieve(ctx, referenceList, []string{"name"}, &objects); err != nil { - return nil, err + return assetNames{}, err } } @@ -232,20 +230,18 @@ func getAssetNames(ctx context.Context, pc *property.Collector, hs *mo.HostSyste // calling network explicitly because of mo.Network's ManagedEntityObject.Name does not store Network name // instead mo.Network.Name contains correct value of Network name - var netObjects []mo.Network + outputNetworkNames := make([]string, 0, len(hs.Network)) if len(hs.Network) > 0 { + var netObjects []mo.Network if err := pc.Retrieve(ctx, hs.Network, []string{"name"}, &netObjects); err != nil { - return nil, err + return assetNames{}, err + } + for _, ob := range netObjects { + outputNetworkNames = append(outputNetworkNames, strings.ReplaceAll(ob.Name, ".", "_")) } } - outputNetworkNames := make([]string, 0, len(hs.Network)) - for _, ob := range netObjects { - name := strings.ReplaceAll(ob.Name, ".", "_") - outputNetworkNames = append(outputNetworkNames, name) - } - - return &assetNames{ + return assetNames{ outputNetworkNames: outputNetworkNames, outputDsNames: outputDsNames, outputVmNames: outputVmNames, From 516221ee591ccae4fa1cda9e29bceeb7850ea0ed Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Tue, 20 Aug 2024 12:01:37 +0530 Subject: [PATCH 28/33] fix datastore tests --- metricbeat/module/vsphere/datastore/datastore_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/vsphere/datastore/datastore_test.go b/metricbeat/module/vsphere/datastore/datastore_test.go index 88f5b772b5af..1e3387adef86 100644 --- a/metricbeat/module/vsphere/datastore/datastore_test.go +++ b/metricbeat/module/vsphere/datastore/datastore_test.go @@ -48,7 +48,7 @@ func TestFetchEventContents(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint()) assert.EqualValues(t, "LocalDS_0", event["name"]) - assert.EqualValues(t, "local", event["fstype"]) + assert.EqualValues(t, "OTHER", event["fstype"]) // Values are based on the result 'df -k'. fields := []string{"capacity.total.bytes", "capacity.free.bytes", From af9b424b5d575aab28fdfff8f835145d16fbbb8f Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Wed, 21 Aug 2024 10:11:46 +0530 Subject: [PATCH 29/33] resolved review comments --- metricbeat/module/vsphere/host/host.go | 140 ++++++++++++------------- 1 file changed, 68 insertions(+), 72 deletions(-) diff --git a/metricbeat/module/vsphere/host/host.go b/metricbeat/module/vsphere/host/host.go index 22e4756661c9..1a2e3a1a7c84 100644 --- a/metricbeat/module/vsphere/host/host.go +++ b/metricbeat/module/vsphere/host/host.go @@ -66,24 +66,24 @@ type assetNames struct { } // Define metrics to be collected -var metricNames = []string{ - "disk.capacity.usage.average", - "disk.deviceLatency.average", - "disk.maxTotalLatency.latest", - "disk.usage.average", - "disk.read.average", - "disk.write.average", - "net.transmitted.average", - "net.received.average", - "net.usage.average", - "net.packetsTx.summation", - "net.packetsRx.summation", - "net.errorsTx.summation", - "net.errorsRx.summation", - "net.multicastTx.summation", - "net.multicastRx.summation", - "net.droppedTx.summation", - "net.droppedRx.summation", +var metricSet = map[string]struct{}{ + "disk.capacity.usage.average": {}, + "disk.deviceLatency.average": {}, + "disk.maxTotalLatency.latest": {}, + "disk.usage.average": {}, + "disk.read.average": {}, + "disk.write.average": {}, + "net.transmitted.average": {}, + "net.received.average": {}, + "net.usage.average": {}, + "net.packetsTx.summation": {}, + "net.packetsRx.summation": {}, + "net.errorsTx.summation": {}, + "net.errorsRx.summation": {}, + "net.multicastTx.summation": {}, + "net.multicastRx.summation": {}, + "net.droppedTx.summation": {}, + "net.droppedRx.summation": {}, } // Fetch methods implements the data gathering and data conversion to the right @@ -130,77 +130,73 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { // Create a performance manager perfManager := performance.NewManager(c) - // Retrieve metric IDs for the specified metric names + // Retrieve all available metrics metrics, err := perfManager.CounterInfoByName(ctx) if err != nil { return fmt.Errorf("failed to retrieve metrics: %w", err) } - // Retrieve only the required metrics - requiredMetrics := make(map[string]*types.PerfCounterInfo) - - for _, name := range metricNames { - metric, exists := metrics[name] - if !exists { - m.Logger().Warnf("Metric %s not found", name) - continue + // Filter for required metrics + var metricIds []types.PerfMetricId + for metricName := range metricSet { + if metric, ok := metrics[metricName]; ok { + metricIds = append(metricIds, types.PerfMetricId{CounterId: metric.Key}) + } else { + m.Logger().Warnf("Metric %s not found", metricName) } - requiredMetrics[name] = metric - } - - metricIDs := make([]types.PerfMetricId, 0, len(requiredMetrics)) - for _, metric := range requiredMetrics { - metricIDs = append(metricIDs, types.PerfMetricId{ - CounterId: metric.Key, - }) } pc := property.DefaultCollector(c) for i := range hst { - assetNames, err := getAssetNames(ctx, pc, &hst[i]) - if err != nil { - m.Logger().Errorf("Failed to retrieve object from host %s: %w", hst[i].Name, err) - } + select { + case <-ctx.Done(): + return ctx.Err() + default: + assetNames, err := getAssetNames(ctx, pc, &hst[i]) + if err != nil { + m.Logger().Errorf("Failed to retrieve object from host %s: %w", hst[i].Name, err) + } - spec := types.PerfQuerySpec{ - Entity: hst[i].Reference(), - MetricId: metricIDs, - MaxSample: 1, - IntervalId: 20, // right now we are only grabbing real time metrics from the performance manager - } + spec := types.PerfQuerySpec{ + Entity: hst[i].Reference(), + MetricId: metricIds, + MaxSample: 1, + IntervalId: 20, // right now we are only grabbing real time metrics from the performance manager + } - // Query performance data - samples, err := perfManager.Query(ctx, []types.PerfQuerySpec{spec}) - if err != nil { - m.Logger().Errorf("Failed to query performance data from host %s: %v", hst[i].Name, err) - continue - } + // Query performance data + samples, err := perfManager.Query(ctx, []types.PerfQuerySpec{spec}) + if err != nil { + m.Logger().Errorf("Failed to query performance data from host %s: %v", hst[i].Name, err) + continue + } - if len(samples) == 0 { - m.Logger().Debug("No samples returned from performance manager") - continue - } + if len(samples) == 0 { + m.Logger().Debug("No samples returned from performance manager") + continue + } - results, err := perfManager.ToMetricSeries(ctx, samples) - if err != nil { - m.Logger().Errorf("Failed to convert performance data to metric series for host %s: %v", hst[i].Name, err) - } + results, err := perfManager.ToMetricSeries(ctx, samples) + if err != nil { + m.Logger().Errorf("Failed to convert performance data to metric series for host %s: %v", hst[i].Name, err) + } - metricMap := make(map[string]interface{}) - for _, result := range results[0].Value { - if len(result.Value) > 0 { - metricMap[result.Name] = result.Value[0] - continue + metricMap := make(map[string]interface{}) + for _, result := range results[0].Value { + if len(result.Value) > 0 { + metricMap[result.Name] = result.Value[0] + continue + } + m.Logger().Debugf("For host %s,Metric %v: No result found", hst[i].Name, result.Name) } - m.Logger().Debugf("For host %s,Metric %v: No result found", hst[i].Name, result.Name) - } - reporter.Event(mb.Event{ - MetricSetFields: m.eventMapping(hst[i], &metricData{ - perfMetrics: metricMap, - assetsName: assetNames, - }), - }) + reporter.Event(mb.Event{ + MetricSetFields: m.eventMapping(hst[i], &metricData{ + perfMetrics: metricMap, + assetsName: assetNames, + }), + }) + } } return nil From 6d45870befa230ba770da0ca587365bc077a5729 Mon Sep 17 00:00:00 2001 From: Kush Rana <89848966+kush-elastic@users.noreply.github.com> Date: Wed, 21 Aug 2024 14:19:36 +0530 Subject: [PATCH 30/33] Update CHANGELOG.next.asciidoc Co-authored-by: niraj-elastic <124254029+niraj-elastic@users.noreply.github.com> --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index cdae94819c0c..1e3a1ec9eb19 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -54,7 +54,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Allow metricsets to report their status via control v2 protocol. {pull}40025[40025] - Remove fallback to the node limit for the `kubernetes.pod.cpu.usage.limit.pct` and `kubernetes.pod.memory.usage.limit.pct` metrics calculation - Add support for Kibana status metricset in v8 format {pull}40275[40275] -- Update metrics for the Vsphere Host metricset. {pull}40429[40429] +- Update metrics for the vSphere Host metricset. {pull}40429[40429] *Osquerybeat* From 432dc602bb59f3a08e9bc9252412571c7a43c57a Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Wed, 21 Aug 2024 14:29:18 +0530 Subject: [PATCH 31/33] update field description --- metricbeat/docs/fields.asciidoc | 10 +++++----- metricbeat/module/vsphere/fields.go | 2 +- metricbeat/module/vsphere/host/_meta/fields.yml | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 3a4a2131d2d6..f9518fcb8ca2 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -66969,7 +66969,7 @@ type: long *`vsphere.host.datastore.names`*:: + -- -Names of the datastores on the host. +List of all the datastore names. type: keyword @@ -66999,7 +66999,7 @@ type: long *`vsphere.host.disk.devicelatency.average.ms`*:: + -- -Average amount of time it takes to complete an SCSI command from physical device. +Average amount of time it takes to complete an SCSI command from physical device in milliseconds. type: long @@ -67009,7 +67009,7 @@ type: long *`vsphere.host.disk.latency.total.ms`*:: + -- -Highest latency value across all disks used by the host. +Highest latency value across all disks used by the host in milliseconds. type: long @@ -67111,7 +67111,7 @@ type: keyword *`vsphere.host.network.names`*:: + -- -Network names. +List of all the network names. type: keyword @@ -67297,7 +67297,7 @@ type: long *`vsphere.host.vm.names`*:: + -- -Names of the virtual machines on the host. +List of all the VM names. type: keyword diff --git a/metricbeat/module/vsphere/fields.go b/metricbeat/module/vsphere/fields.go index 8c3856b8a874..cca923c73dae 100644 --- a/metricbeat/module/vsphere/fields.go +++ b/metricbeat/module/vsphere/fields.go @@ -32,5 +32,5 @@ func init() { // AssetVsphere returns asset data. // This is the base64 encoded zlib format compressed contents of module/vsphere. func AssetVsphere() string { - return "eJzMmUtv4zYQx+/+FIM9tYe4dx8KBCm22UN2CyTbazCmxhZrPgRyJMP59AWpR+VnJJsyqkOASPL8fxzODIfUA2xot4DKFzk5mgGwZEUL+FK9xjtfZgAZeeFkwdKaBfw+AwBonoK2WanCzxwpQk8LWOMMYCVJZX4RX30Ag5r6EuHiXRFedrYsmjsnVPYN9Y1lyOjZduZOmzxrtnl0wsj+ONrrEKOPEv7uPWhJNrTbWpcdPLvAE64/WqZju63gygf76SS/SkV+55k0HBluNQUWKCTv5mwZ1Xy5Y/InCZQ163Hyb8EiRItgV8A5nZyYcK2s08gLOJY/4lw5oqSYXx1RcsrSU5aU8qenbBrKQvBJRi9QUfa+UhYPXxjCWpATZHgobfN69/SoMOTW8y014eD348uBKMraYTr/SDmpT3/9BGngJf+Yn9WtkzOdcJ2aQ5RjuqUTjsn2mW4XK/Pw/+kMuqogfg/mjiLSgzXxTgiRz4iELc3pfBnvjO+lXpILPKNYpN/MeymM67T18C0nQB2GGcgCFa6pqxkgSufIsNrBkqRZg7DGlzoWJ7CuxSfDge78ADKqpCCFTEbs5lhRUJnrVIN4rA32BsJSE0gGxg15YAvC6kIRE6CB16fXb+GGRpPBylkNRb7zUqCCGvTCSNoxNDmaagTPcp2TZ2jMQ4WqJEDhrPeASkVxD2W9LgyJGUeYdk1qnWy6OK4XqCBUezHmmfQbIBQ5eBLWZPPR61Wk3zrJaQP9HH5QYjIhRhLyp++vHtdrR2tkymrGb7/9AIdM4wE1aeumaq1eovFQ8KPpq+mmalBT8U3T8t1Ml3YX82w9R5On64wh3lq3eU+9btdmo4i/qJy8YxiuPE1n0Fgf0Be0HEs02VZmnM/ZofE6FLNp6n4oNoAM21yKPPYwsEUPPVnISheahEAuDZOrUM3hLZdhkSgceTLs49OOum3P2uGMD/gjPzgSJKv7OqHV/H94IH3tbBOjZKnkB4bX4Bdh9VIayroQeAje8RC6qsYh9Z1frx9UgWJD7PeCe5rUa5T68Xw581q0Lt6m5epkBkGRc9bd021byTnUquNd2NDeyZN91Ou8GvMrJWTdm5hLqMMIdalYCvR8h6nvtK7Pnf9wJ577Y9ZxM9/z6+STf8Q6DDFztigou8PE95f81p2t+ijUiSe9W5Vvgrxftl/E84xcJux133ICG7oapSAnVJw3EoEIY+cZ9iGhNWm/EpGppLNGkznTlJYFyzP7j+tOpqLvG7P7XPXuvK6QYzErPelRYyUdl6hAo8ilGXLIV+mJMmAYSodRv928PDsEGXH8ftbS+IP4iCoP5+PWTe2BwT2x8GeifXQbIgefJ6bav//dTP9LPQ/nv0XahLlwKPqjIIccdkWv9dfJsxTTf3M5Kzv1J5ezwom/uDxWKBUuFX2m3T/GWpfkebLDLLuCP4NAd6o1ejPYZ40ZOiHqs01AWkdTerfuHWEm82sMwfSw/ePg21lF6dnq93qlOElol//Q0cft+ub7DeXsKQrDCeF7HoPO/g0AAP//PPaPzA==" + return "eJzMmUtv4zYQx+/+FIM9tYd17z4UCFJss0CzWyDZvQZjamyx5kMgRzKcT1+QelR+yLFsyqgOASLR8/9xOEMOyc+wod0CKl/k5GgGwJIVLeBT9RLffJoBZOSFkwVLaxbw+wwAoPkK2malCj9zpAg9LWCNM4CVJJX5RWz6GQxq6kuEh3dFaOxsWTRvTqjsG+oby5DRs+3MnTY5aLb5dMLIfj/a5xCjjxL+7n1oSTa021qXHXw7wxOeP1qmY7ut4MoH++kkv0hFfueZNBwZbjUFFigk7+ZsGdV8uWPyJwmUNetx8q/BIkSLYFfAOZ0cmPCsrNPICziWP+JcOaKkmF8cUXLK0lOWlPKHp2waykLwSUYvUFH2tlIWDxtcwlqQE2T4Utqmeff1aGLIredb5oSD34+fDkRR1g7T+XvKQX38+wdIA8/5+3xQt07OdMJ1al6iHNMtnXBMto90u1iZh/9PZ9BVE+Jf0seARKX2gzIK+49ohC3N6VwZ74hvpV6SCzCdfQ/WRKwQqgMs0m/mvfTFddq58DUnQB26GcgCFa6pmy9AlM6RYbWDJUmzBmGNL3WcmMC6Fp8MB7rhDmRUSUEKmYzYzbGioDLXqTrxUBvsdYSlJpAMjBvywBaE1YUiJkADL48vX8MLjSaDlbMainznpUAFNWiIVS2Vkp6ENdlQmISetX1q8jVVj57kOifP0JiHClVJgMJZ72MwB3EPZb1GdDE0AtwRpl2v2kEwXZzXi1cQqr0cE1D6DRCKHGrE+ei1LNJvneS0iTCEH5SYTIihhPzpa6+H9drRGpmymvHrb9/BIdN4QE3auqnKrudoPMRpNH013VTFayq+acrBm+nS7nCewpQTTJ6eZwzx1rrNW+I1/Vtt9twK3ihPXU2Yy0mmqSQa6xfUES3HEk22lRnnc3ZovA6T2zTrQJh8ABm2uRR5rHlgix56spCVLhQVgVwaJlehmsNrLsOiUTjyZNjHrx11u7louzM+AY784EiQrO7rhFbz/+GB9HNpm6IlSyXfMTSDX4TVS2ko60Lgc/COh1CFNQ6p3/x6facKFBtivxfc06Reo9SP5/OZ16J18TYtVydzERQ5Z9093baVnEOtOt6FDe2dPNlHvc6rMb9SQta1ijmHehmhLhVLgZ7vMPSd1vW58x/uxGN/zDpu5Ht+nXzwj1gvQ8ycLQrK7jDw/SW/dWerPgp14kHvVuWbIO+X7WfxPCOXCWvf15zAhqpGKcgJFeeNRCyIu9OHUJq0N0pkKums0WQGitKyYDmwH7nuJCv6vjG7z9UciMQZcixmpafeSPx8PreHqPREAV9JxyUq0ChyaYaOIzuMunXTeHYIMuJkftDS+DP6iCoP3X/rnvbA4J5Y+DPRNrqt7Q9uLqbavv9shv+5Hofha0qbMPQPRb8X5JDDJuilvrgcpJj+OmZQdurbmEHhxJcxDxVKhUtFH2n3T7HWJXme7CzLruDPINAdao3e+/VZY4ZOiPpkE5DW0ZTerXsnmMn8GkMwPWz/NPh2VlF6tvqtXilOEtrlP3R0712/fLthOnuMwnBC+J6noLN/AwAA//8HG5lE" } diff --git a/metricbeat/module/vsphere/host/_meta/fields.yml b/metricbeat/module/vsphere/host/_meta/fields.yml index 6d10ff4d422e..4b33ac1ddfba 100644 --- a/metricbeat/module/vsphere/host/_meta/fields.yml +++ b/metricbeat/module/vsphere/host/_meta/fields.yml @@ -19,7 +19,7 @@ - name: datastore.names type: keyword description: > - Names of the datastores on the host. + List of all the datastore names. - name: datastore.count type: long description: > @@ -31,11 +31,11 @@ - name: disk.devicelatency.average.ms type: long description: > - Average amount of time it takes to complete an SCSI command from physical device. + Average amount of time it takes to complete an SCSI command from physical device in milliseconds. - name: disk.latency.total.ms type: long description: > - Highest latency value across all disks used by the host. + Highest latency value across all disks used by the host in milliseconds. - name: disk.read.bytes type: long description: > @@ -77,7 +77,7 @@ - name: network.names type: keyword description: > - Network names. + List of all the network names. - name: network.count type: long description: > @@ -152,7 +152,7 @@ - name: vm.names type: keyword description: > - Names of the virtual machines on the host. + List of all the VM names. - name: vm.count type: long description: > From 29558f0dc64cbffc6652e5e660d82128e9098061 Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Wed, 21 Aug 2024 17:05:16 +0530 Subject: [PATCH 32/33] update field mappings --- metricbeat/docs/fields.asciidoc | 4 +++- metricbeat/module/vsphere/fields.go | 2 +- metricbeat/module/vsphere/host/_meta/fields.yml | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index f9518fcb8ca2..7254eb1c902f 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -66994,6 +66994,8 @@ The amount of storage capacity currently being consumed by or on the entity. type: long +format: bytes + -- *`vsphere.host.disk.devicelatency.average.ms`*:: @@ -67043,7 +67045,7 @@ format: bytes *`vsphere.host.disk.total.bytes`*:: + -- -Aggregated disk I/O rate. +Aggregated disk Input/Output rate. type: long diff --git a/metricbeat/module/vsphere/fields.go b/metricbeat/module/vsphere/fields.go index cca923c73dae..8c2b6d0b4e56 100644 --- a/metricbeat/module/vsphere/fields.go +++ b/metricbeat/module/vsphere/fields.go @@ -32,5 +32,5 @@ func init() { // AssetVsphere returns asset data. // This is the base64 encoded zlib format compressed contents of module/vsphere. func AssetVsphere() string { - return "eJzMmUtv4zYQx+/+FIM9tYd17z4UCFJss0CzWyDZvQZjamyx5kMgRzKcT1+QelR+yLFsyqgOASLR8/9xOEMOyc+wod0CKl/k5GgGwJIVLeBT9RLffJoBZOSFkwVLaxbw+wwAoPkK2malCj9zpAg9LWCNM4CVJJX5RWz6GQxq6kuEh3dFaOxsWTRvTqjsG+oby5DRs+3MnTY5aLb5dMLIfj/a5xCjjxL+7n1oSTa021qXHXw7wxOeP1qmY7ut4MoH++kkv0hFfueZNBwZbjUFFigk7+ZsGdV8uWPyJwmUNetx8q/BIkSLYFfAOZ0cmPCsrNPICziWP+JcOaKkmF8cUXLK0lOWlPKHp2waykLwSUYvUFH2tlIWDxtcwlqQE2T4Utqmeff1aGLIredb5oSD34+fDkRR1g7T+XvKQX38+wdIA8/5+3xQt07OdMJ1al6iHNMtnXBMto90u1iZh/9PZ9BVE+Jf0seARKX2gzIK+49ohC3N6VwZ74hvpV6SCzCdfQ/WRKwQqgMs0m/mvfTFddq58DUnQB26GcgCFa6pmy9AlM6RYbWDJUmzBmGNL3WcmMC6Fp8MB7rhDmRUSUEKmYzYzbGioDLXqTrxUBvsdYSlJpAMjBvywBaE1YUiJkADL48vX8MLjSaDlbMainznpUAFNWiIVS2Vkp6ENdlQmISetX1q8jVVj57kOifP0JiHClVJgMJZ72MwB3EPZb1GdDE0AtwRpl2v2kEwXZzXi1cQqr0cE1D6DRCKHGrE+ei1LNJvneS0iTCEH5SYTIihhPzpa6+H9drRGpmymvHrb9/BIdN4QE3auqnKrudoPMRpNH013VTFayq+acrBm+nS7nCewpQTTJ6eZwzx1rrNW+I1/Vtt9twK3ihPXU2Yy0mmqSQa6xfUES3HEk22lRnnc3ZovA6T2zTrQJh8ABm2uRR5rHlgix56spCVLhQVgVwaJlehmsNrLsOiUTjyZNjHrx11u7louzM+AY784EiQrO7rhFbz/+GB9HNpm6IlSyXfMTSDX4TVS2ko60Lgc/COh1CFNQ6p3/x6facKFBtivxfc06Reo9SP5/OZ16J18TYtVydzERQ5Z9093baVnEOtOt6FDe2dPNlHvc6rMb9SQta1ijmHehmhLhVLgZ7vMPSd1vW58x/uxGN/zDpu5Ht+nXzwj1gvQ8ycLQrK7jDw/SW/dWerPgp14kHvVuWbIO+X7WfxPCOXCWvf15zAhqpGKcgJFeeNRCyIu9OHUJq0N0pkKums0WQGitKyYDmwH7nuJCv6vjG7z9UciMQZcixmpafeSPx8PreHqPREAV9JxyUq0ChyaYaOIzuMunXTeHYIMuJkftDS+DP6iCoP3X/rnvbA4J5Y+DPRNrqt7Q9uLqbavv9shv+5Hofha0qbMPQPRb8X5JDDJuilvrgcpJj+OmZQdurbmEHhxJcxDxVKhUtFH2n3T7HWJXme7CzLruDPINAdao3e+/VZY4ZOiPpkE5DW0ZTerXsnmMn8GkMwPWz/NPh2VlF6tvqtXilOEtrlP3R0712/fLthOnuMwnBC+J6noLN/AwAA//8HG5lE" + return "eJzMmUtv4zYQx+/+FIM9tYd17z4UCFJss0CzWyDZvQZjamyx5kMgRzKcT1+QelR+yLFsyqgOASLJ8/9xODMcUp9hQ7sFVL7IydEMgCUrWsCn6iXe+TQDyMgLJwuW1izg9xkAQPMUtM1KFX7mSBF6WsAaZwArSSrzi/jqZzCoqS8RLt4V4WVny6K5c0Jl31DfWIaMnm1n7rTJQbPNoxNG9sfRXocYfZTwd+9BS7Kh3da67ODZGZ5w/dEyHdttBVc+2E8n+UUq8jvPpOHIcKspsEAheTdny6jmyx2TP0mgrFmPk38NFiFaBLsCzunkxIRrZZ1GXsCx/BHnyhElxfziiJJTlp6ypJQ/PGXTUBaCTzJ6gYqyt5WyePjCJawFOUGGL6VtXu+eHhWG3Hq+pSYc/H58ORBFWTtM5+8pJ/Xx7x8gDTzn7/NB3To50wnXqXmJcky3dMIx2T7S7WJlHv4/nUFXFcS/pI8BiUrtB2UU9h/RCFua07ky3hHfSr0kF2A6+x6siVghVAdYpN/Me+mL67S18DUnQB2GGcgCFa6pqxcgSufIsNrBkqRZg7DGlzoWJrCuxSfDgW50WYqDy6iSghQyGbGbY0WBYK5TDfChNtgbJEtNIBkYN+SBLQirC0VMgAZeHl++hhsaTQYrZzUU+c5LgQpq0BDHWiolPQlrsqEQCiNrx9TkcqoRPcl1Tp6hMQ8VqpIAhbPex0AP4h7Kev3o4msEuCNMu5a1k2C6HKgXtiBUezkmp/QbIBQ51IhXBtTWSU6bJEP4QYnJhBhKyJ++L3tYrx2tkSmrGb+aouTfvpdclAwOmcaTatLWTdWbPUfjIWCj6avppupwU/FN0zPeTJd2G/QUak8webrgGOKtdZu3xAv/t9rsuWW+UZ665TCXk0zTbjTWL2g2Wo4lmmwrM87n7NB4HarcNAtCKD6ADNtcijw2RrBFDz1ZyEoXOo9ALg2Tq1DN4TWXYfUoHHky7OPTjrrdgbTDGZ8AR35wJEhW93VCq/n/8ED6WtqmaMlSyXcMr8EvwuqlNJR1IfA5eMdDaMcah9R3fr1+UAWKDbHfC+5pUq9R6sfz+cxr0bp4m5ark7kIipyz7p5u20rOoVYd78KG9k6e7KNe59WYXykh617FnEO9jFCXiqVAz3eY+k7r+tz5D3fiuT9mHTfzPb9OPvlHrJchZs4WBWV3mPj+kt+6s1UfhTrxpHer8k2Q98v2s3iekcuEve9rTmBDV6MU5ISK80YiNsTdMURoTdrPTmQq6azRZAaa0rJgObAfue64K/q+MbvP1ZyMxAo5FrPSU28kfj6f20NUeqKAr6TjEhVoFLk0Q2eWHUb9dvPy7BBkxPH9oKXxB/kRVR66/9Y97YHBPbHwZ6JtdNvbH3zemGr7/rOZ/ud6Hoa/ZdqEoX8o+r0ghxw2QS/1181Bium/2QzKTv3JZlA48RebhwqlwqWij7T7p1jrkjxPdpZlV/BnEOgOtUbv/fqsMUMnRH2yCUjraErv1r0TzGR+jSGYHrZ/Gnw7qyg9W/1WrxQnCe3yHzr6OF7ffLuhnD1GYTghfM9T0Nm/AQAA///8CKQh" } diff --git a/metricbeat/module/vsphere/host/_meta/fields.yml b/metricbeat/module/vsphere/host/_meta/fields.yml index 4b33ac1ddfba..6ca8380c21d7 100644 --- a/metricbeat/module/vsphere/host/_meta/fields.yml +++ b/metricbeat/module/vsphere/host/_meta/fields.yml @@ -28,6 +28,7 @@ type: long description: > The amount of storage capacity currently being consumed by or on the entity. + format: bytes - name: disk.devicelatency.average.ms type: long description: > @@ -49,7 +50,7 @@ - name: disk.total.bytes type: long description: > - Aggregated disk I/O rate. + Aggregated disk Input/Output rate. format: bytes - name: memory.free.bytes type: long From ddcbd3524cacbda662111b15ba4bac29d939dfbb Mon Sep 17 00:00:00 2001 From: Kush Rana Date: Thu, 22 Aug 2024 11:05:26 +0530 Subject: [PATCH 33/33] resolved review comments --- metricbeat/docs/fields.asciidoc | 12 ++++++------ metricbeat/module/vsphere/fields.go | 2 +- metricbeat/module/vsphere/host/_meta/fields.yml | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 7254eb1c902f..2ebe8a227834 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -66932,14 +66932,14 @@ format: percent [float] === host -host +Host information from vSphere environment. *`vsphere.host.cpu.used.mhz`*:: + -- -Used CPU in Mhz. +Used CPU in MHz. type: long @@ -66949,7 +66949,7 @@ type: long *`vsphere.host.cpu.total.mhz`*:: + -- -Total CPU in Mhz. +Total CPU in MHz. type: long @@ -66959,7 +66959,7 @@ type: long *`vsphere.host.cpu.free.mhz`*:: + -- -Free CPU in Mhz. +Free CPU in MHz. type: long @@ -67045,7 +67045,7 @@ format: bytes *`vsphere.host.disk.total.bytes`*:: + -- -Aggregated disk Input/Output rate. +Sum of disk read and write rates each second in bytes. type: long @@ -67157,7 +67157,7 @@ format: bytes *`vsphere.host.network.bandwidth.total.bytes`*:: + -- -Network utilization (combined transmit-rates and receive-rates). +Sum of network transmitted and received rates in bytes during the interval. type: long diff --git a/metricbeat/module/vsphere/fields.go b/metricbeat/module/vsphere/fields.go index 8c2b6d0b4e56..26d9707a8196 100644 --- a/metricbeat/module/vsphere/fields.go +++ b/metricbeat/module/vsphere/fields.go @@ -32,5 +32,5 @@ func init() { // AssetVsphere returns asset data. // This is the base64 encoded zlib format compressed contents of module/vsphere. func AssetVsphere() string { - return "eJzMmUtv4zYQx+/+FIM9tYd17z4UCFJss0CzWyDZvQZjamyx5kMgRzKcT1+QelR+yLFsyqgOASLJ8/9xODMcUp9hQ7sFVL7IydEMgCUrWsCn6iXe+TQDyMgLJwuW1izg9xkAQPMUtM1KFX7mSBF6WsAaZwArSSrzi/jqZzCoqS8RLt4V4WVny6K5c0Jl31DfWIaMnm1n7rTJQbPNoxNG9sfRXocYfZTwd+9BS7Kh3da67ODZGZ5w/dEyHdttBVc+2E8n+UUq8jvPpOHIcKspsEAheTdny6jmyx2TP0mgrFmPk38NFiFaBLsCzunkxIRrZZ1GXsCx/BHnyhElxfziiJJTlp6ypJQ/PGXTUBaCTzJ6gYqyt5WyePjCJawFOUGGL6VtXu+eHhWG3Hq+pSYc/H58ORBFWTtM5+8pJ/Xx7x8gDTzn7/NB3To50wnXqXmJcky3dMIx2T7S7WJlHv4/nUFXFcS/pI8BiUrtB2UU9h/RCFua07ky3hHfSr0kF2A6+x6siVghVAdYpN/Me+mL67S18DUnQB2GGcgCFa6pqxcgSufIsNrBkqRZg7DGlzoWJrCuxSfDgW50WYqDy6iSghQyGbGbY0WBYK5TDfChNtgbJEtNIBkYN+SBLQirC0VMgAZeHl++hhsaTQYrZzUU+c5LgQpq0BDHWiolPQlrsqEQCiNrx9TkcqoRPcl1Tp6hMQ8VqpIAhbPex0AP4h7Kev3o4msEuCNMu5a1k2C6HKgXtiBUezkmp/QbIBQ51IhXBtTWSU6bJEP4QYnJhBhKyJ++L3tYrx2tkSmrGb+aouTfvpdclAwOmcaTatLWTdWbPUfjIWCj6avppupwU/FN0zPeTJd2G/QUak8webrgGOKtdZu3xAv/t9rsuWW+UZ665TCXk0zTbjTWL2g2Wo4lmmwrM87n7NB4HarcNAtCKD6ADNtcijw2RrBFDz1ZyEoXOo9ALg2Tq1DN4TWXYfUoHHky7OPTjrrdgbTDGZ8AR35wJEhW93VCq/n/8ED6WtqmaMlSyXcMr8EvwuqlNJR1IfA5eMdDaMcah9R3fr1+UAWKDbHfC+5pUq9R6sfz+cxr0bp4m5ark7kIipyz7p5u20rOoVYd78KG9k6e7KNe59WYXykh617FnEO9jFCXiqVAz3eY+k7r+tz5D3fiuT9mHTfzPb9OPvlHrJchZs4WBWV3mPj+kt+6s1UfhTrxpHer8k2Q98v2s3iekcuEve9rTmBDV6MU5ISK80YiNsTdMURoTdrPTmQq6azRZAaa0rJgObAfue64K/q+MbvP1ZyMxAo5FrPSU28kfj6f20NUeqKAr6TjEhVoFLk0Q2eWHUb9dvPy7BBkxPH9oKXxB/kRVR66/9Y97YHBPbHwZ6JtdNvbH3zemGr7/rOZ/ud6Hoa/ZdqEoX8o+r0ghxw2QS/1181Bium/2QzKTv3JZlA48RebhwqlwqWij7T7p1jrkjxPdpZlV/BnEOgOtUbv/fqsMUMnRH2yCUjraErv1r0TzGR+jSGYHrZ/Gnw7qyg9W/1WrxQnCe3yHzr6OF7ffLuhnD1GYTghfM9T0Nm/AQAA///8CKQh" + return "eJzMmc2O2zYQx+9+ikHu8QP4UCDYIt0C3bTAbnJd0NTYYs0PgRzJcJ6+ICkpsi15LZs06sMCkZT5/zicGQ7Jz7DDwwoaV5VocQFAgiSu4FPzGp58WgAU6LgVFQmjV/DbAgCgfQvKFLX0/82iROZwBVu2ANgIlIVbhU8/g2YKhxL+R4fKf2xNXbVPRlSODQ2NFYyYI9ObGzc5abZ9NWLkeBzd7xRjiOL/Hr3oSHZ42BtbnLy7wON/v3dM53Y7wY3z9tNJfhUS3cERKjgz3GlyVjEu6LAkQ0wu1wdCN0ogjd7Ok3/zFiFYBLMBKnF0YvxvY6xitIJz+TPOjUVMivnVIianrB0WSSm/OyzyUFacRhkdZxKL94007PSDa1grtBw1XUvbft6/PSsMpXF0T014No5A6KgnjIaNNaovd6gbYY1WqGl5V9HgVR3dqsqfKaf+6Z/vIDS8PP9cTurGFE4nHBP4GuWQlOmEQ0p+pNtH1NL/ezzPbiqbfwkXwpZJeRy6Qdh9RMNNrcczar4jvtVqjdbD9PYdGB2wfEJMsAi3Ww6SnG3TVsy3EoEpP0xP5qnYFvuqAry2FjXJA6xR6C1wo12tQvkCYzt81OTpZhevMLgCG8FRMkLND0vWoCdYqlQD/BINDgZJQiEIAmI7dEAGuFGVREJgGl6fXv/0DxTTRSwrVXlwgjMJEdTHsRJSCofc6GIqhPzIujG1uZxqRM9iW6IjaM1Dw2SNwLg1zoVA9+IO6rjK9PE1A9wiS7vidZOg+xyIy58Xil4OySncDpDxEiLijQG1t4LSJskUvlci1D6GEvKn795eaxXqjucLLvfBHdwElvmBDKB9mATt+fQKlbG5urqXYPx+uly9cSq+PN3m3XRpN1ChgfMmx4uQRtobu3tP3Ax8i2YvLf2tcu42RF9PkqcFaa1f0YB0HGumi70oqFySZdopX/nyLBK+JAEj2JeCl6FZgj1zMJCFora+G/HkQhPahsklvJXCryiVRYeaXHjbU3d7l2448xPgzA8WOYrmsU7oNP8fHsi2UnX5MZxzv2b1w4/LVlfORr1x8/gqxndI7ijO82RhqzQc5uUk7ND60MvL1ctcBYXWGvtIt+0FlRBV57uwpX2QJ4eot3k1pFpKyNi26Euo1xGqWpLgzNEDpr7Xuj13fuFmnvtz1nkzP/Br9sk/Y70OsbCmqrB4wMQPV4LOnZ36LNTMk96vUHdBPi7bL+I5YlQnbIPfSgTjGxwpoUQmqWwlQm/cn1L4ZfyDw9xfjHVFYmJrcttpWPB9a/aYqz04CRVyLmajcu8pfrxc2k40KlPAN8JSzSQoxkuhp440e4z4dfvx4hRkxh3ApKX55/wBVZy6/97t7YnBIzH/J9OOumvzT+5Ycu3kf7TT/xLnYfpC1CQM/VPRvyu0jPwO4DVekU5S5L7SObH6uBudC8KJL3S+NExItpb4kfbwQGtbo6Nsx1pmA394gX5DOHvvN2QNGZoRtb29vI80RlN6tx4dZibzawjB9LDDg+H7WXntyKj3uFKMEpr1v3h2wx4fvt9Rzp6CMIwIP/JAdPFfAAAA//+hUbzT" } diff --git a/metricbeat/module/vsphere/host/_meta/fields.yml b/metricbeat/module/vsphere/host/_meta/fields.yml index 6ca8380c21d7..b45061b863e0 100644 --- a/metricbeat/module/vsphere/host/_meta/fields.yml +++ b/metricbeat/module/vsphere/host/_meta/fields.yml @@ -1,21 +1,21 @@ - name: host type: group description: > - host + Host information from vSphere environment. release: ga fields: - name: cpu.used.mhz type: long description: > - Used CPU in Mhz. + Used CPU in MHz. - name: cpu.total.mhz type: long description: > - Total CPU in Mhz. + Total CPU in MHz. - name: cpu.free.mhz type: long description: > - Free CPU in Mhz. + Free CPU in MHz. - name: datastore.names type: keyword description: > @@ -50,7 +50,7 @@ - name: disk.total.bytes type: long description: > - Aggregated disk Input/Output rate. + Sum of disk read and write rates each second in bytes. format: bytes - name: memory.free.bytes type: long @@ -96,7 +96,7 @@ - name: network.bandwidth.total.bytes type: long description: > - Network utilization (combined transmit-rates and receive-rates). + Sum of network transmitted and received rates in bytes during the interval. format: bytes - name: network.packets.transmitted.count type: long