From bf5b0de792a67ba4e204b35d919b82b7cc6875e9 Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 3 Apr 2019 18:31:37 +0200 Subject: [PATCH 01/16] Replace using WMI query to get the system/diskio metrics for Windows (SELECT * FROM Win32_PerfFormattedData_PerfDisk_LogicalDisk) with DeviceIOControl Win32 API method using IOCTL_DISK_PERFORMANCE control code. Fixes: #3798 and #2842 --- metricbeat/module/system/diskio/diskio.go | 3 +- .../module/system/diskio/diskio_windows.go | 146 ++++++++++++++++++ 2 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 metricbeat/module/system/diskio/diskio_windows.go diff --git a/metricbeat/module/system/diskio/diskio.go b/metricbeat/module/system/diskio/diskio.go index 42d4af50f20..905a0b341b7 100644 --- a/metricbeat/module/system/diskio/diskio.go +++ b/metricbeat/module/system/diskio/diskio.go @@ -25,7 +25,6 @@ import ( "github.com/elastic/beats/metricbeat/mb/parse" "github.com/pkg/errors" - "github.com/shirou/gopsutil/disk" ) func init() { @@ -60,7 +59,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch fetches disk IO metrics from the OS. func (m *MetricSet) Fetch(r mb.ReporterV2) { - stats, err := disk.IOCounters(m.includeDevices...) + stats, err := GetIOCounters() if err != nil { r.Error(errors.Wrap(err, "disk io counters")) return diff --git a/metricbeat/module/system/diskio/diskio_windows.go b/metricbeat/module/system/diskio/diskio_windows.go new file mode 100644 index 00000000000..59891296765 --- /dev/null +++ b/metricbeat/module/system/diskio/diskio_windows.go @@ -0,0 +1,146 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build darwin,cgo freebsd linux windows + +package diskio + +import ( + "github.com/shirou/gopsutil/disk" + "syscall" + "unsafe" +) + +const ( + FILE_DEVICE_DISK = 0x00000007 + METHOD_BUFFERED = 0 + FILE_ANY_ACCESS = 0x0000 + ERROR_SUCCESS syscall.Errno = 0 +) + +var ( + IOCTL_DISK_PERFORMANCE = CTL_CODE(FILE_DEVICE_DISK, 0x0008, METHOD_BUFFERED, FILE_ANY_ACCESS) + modkernel32 = syscall.NewLazyDLL("kernel32.dll") + procGetLogicalDriveStringsW = modkernel32.NewProc("GetLogicalDriveStringsW") +) + +type LogicalDrive struct { + Name string + UNCPath string +} +type DiskPerformance struct { + BytesRead int64 + BytesWritten int64 + ReadTime int64 + WriteTime int64 + IdleTime int64 + ReadCount uint32 + WriteCount uint32 + QueueDepth uint32 + SplitCount uint32 + QueryTime int64 + StorageDeviceNumber uint32 + StorageManagerName [8]uint16 +} + +func CTL_CODE(deviceType uint32, function uint32, method uint32, access uint32) uint32 { + return (deviceType << 16) | (access << 14) | (function << 2) | method +} + +func GetIOCounters() (map[string]disk.IOCountersStat, error) { + ret := make(map[string]disk.IOCountersStat, 0) + logicalDisks, err := GetLogicalDriveStrings() + if err != nil || len(logicalDisks) == 0 { + return nil, err + } + for _, drive := range logicalDisks { + var counter, err = IOCounter(drive.UNCPath) + if err != nil { + return nil, err + } + ret[drive.Name] = disk.IOCountersStat{ + Name: drive.Name, + ReadCount: uint64(counter.ReadCount), + WriteCount: uint64(counter.WriteCount), + ReadBytes: uint64(counter.BytesRead), + WriteBytes: uint64(counter.BytesWritten), + ReadTime: uint64(counter.ReadTime), + WriteTime: uint64(counter.WriteTime), + } + } + return ret, nil +} + +func IOCounter(path string) (DiskPerformance, error) { + var diskPerformance DiskPerformance + var diskPerformanceSize uint32 + utfPath, err := syscall.UTF16PtrFromString(path) + if err != nil { + return diskPerformance, err + } + hFile, err := syscall.CreateFile(utfPath, + syscall.GENERIC_READ|syscall.GENERIC_WRITE, + syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE, + nil, + syscall.OPEN_EXISTING, + syscall.FILE_FLAG_BACKUP_SEMANTICS, + 0) + + if err != nil { + return diskPerformance, err + } + defer syscall.CloseHandle(hFile) + + err = syscall.DeviceIoControl(hFile, + IOCTL_DISK_PERFORMANCE, + nil, + 0, + (*byte)(unsafe.Pointer(&diskPerformance)), + uint32(unsafe.Sizeof(diskPerformance)), + &diskPerformanceSize, + nil) + if err != nil { + return diskPerformance, err + } + return diskPerformance, nil +} + +func GetLogicalDriveStrings() ([]LogicalDrive, error) { + lpBuffer := make([]byte, 254) + logicalDrives := make([]LogicalDrive, 0) + r1, _, e1 := syscall.Syscall(procGetLogicalDriveStringsW.Addr(), 2, uintptr(len(lpBuffer)), uintptr(unsafe.Pointer(&lpBuffer[0])), 0) + if r1 == 0 { + if e1 != ERROR_SUCCESS { + return nil, e1 + } else { + return nil, syscall.EINVAL + } + } + + for _, v := range lpBuffer { + if v >= 65 && v <= 90 { + path := string(v) + ":" + if path == "A:" || path == "B:" { + continue + } + + drive := LogicalDrive{path, "\\\\.\\" + path} + logicalDrives = append(logicalDrives, drive) + } + } + return logicalDrives, nil +} From cc02292618f58ae2196142dffb3b257b833e72fc Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 4 Apr 2019 11:07:31 +0200 Subject: [PATCH 02/16] Fixed cross platform build, added tests and include_devices filter Added: - include_devices filter to the IOCounters - test file to assert get stats on C: returns data - addressed houndci-bot style violations --- metricbeat/module/system/diskio/diskio.go | 3 +- .../module/system/diskio/diskstat_linux.go | 6 +++ ...{diskio_windows.go => diskstat_windows.go} | 40 ++++++++++---- .../system/diskio/diskstat_windows_test.go | 53 +++++++++++++++++++ 4 files changed, 91 insertions(+), 11 deletions(-) rename metricbeat/module/system/diskio/{diskio_windows.go => diskstat_windows.go} (72%) create mode 100644 metricbeat/module/system/diskio/diskstat_windows_test.go diff --git a/metricbeat/module/system/diskio/diskio.go b/metricbeat/module/system/diskio/diskio.go index 905a0b341b7..38fe217d426 100644 --- a/metricbeat/module/system/diskio/diskio.go +++ b/metricbeat/module/system/diskio/diskio.go @@ -23,7 +23,6 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" - "github.com/pkg/errors" ) @@ -59,7 +58,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch fetches disk IO metrics from the OS. func (m *MetricSet) Fetch(r mb.ReporterV2) { - stats, err := GetIOCounters() + stats, err := IOCounters(m.includeDevices...) if err != nil { r.Error(errors.Wrap(err, "disk io counters")) return diff --git a/metricbeat/module/system/diskio/diskstat_linux.go b/metricbeat/module/system/diskio/diskstat_linux.go index 357c001b705..9ff45f5a338 100644 --- a/metricbeat/module/system/diskio/diskstat_linux.go +++ b/metricbeat/module/system/diskio/diskstat_linux.go @@ -32,6 +32,12 @@ func Get_CLK_TCK() uint32 { return uint32(100) } +//created IOCounters to map functionality to dick package for linux os +func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { + stats, err := disk.IOCounters(names...) + return stats, err +} + func NewDiskIOStat() *DiskIOStat { d := &DiskIOStat{} d.lastDiskIOCounters = make(map[string]disk.IOCountersStat) diff --git a/metricbeat/module/system/diskio/diskio_windows.go b/metricbeat/module/system/diskio/diskstat_windows.go similarity index 72% rename from metricbeat/module/system/diskio/diskio_windows.go rename to metricbeat/module/system/diskio/diskstat_windows.go index 59891296765..fa7aea328c8 100644 --- a/metricbeat/module/system/diskio/diskio_windows.go +++ b/metricbeat/module/system/diskio/diskstat_windows.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin,cgo freebsd linux windows +// +build windows package diskio @@ -33,7 +33,8 @@ const ( ) var ( - IOCTL_DISK_PERFORMANCE = CTL_CODE(FILE_DEVICE_DISK, 0x0008, METHOD_BUFFERED, FILE_ANY_ACCESS) + // control code - https://docs.microsoft.com/en-us/windows/desktop/api/winioctl/ni-winioctl-ioctl_disk_performance + IoctlDiskPerformance = GetCtlCode(FILE_DEVICE_DISK, 0x0008, METHOD_BUFFERED, FILE_ANY_ACCESS) modkernel32 = syscall.NewLazyDLL("kernel32.dll") procGetLogicalDriveStringsW = modkernel32.NewProc("GetLogicalDriveStringsW") ) @@ -57,18 +58,28 @@ type DiskPerformance struct { StorageManagerName [8]uint16 } -func CTL_CODE(deviceType uint32, function uint32, method uint32, access uint32) uint32 { +// used to get the specific IOCTL_DISK_PERFORMANCE control code +func GetCtlCode(deviceType uint32, function uint32, method uint32, access uint32) uint32 { return (deviceType << 16) | (access << 14) | (function << 2) | method } -func GetIOCounters() (map[string]disk.IOCountersStat, error) { +// the function gets the diskio counters and maps them to the list of IOCountersStat objects +func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { ret := make(map[string]disk.IOCountersStat, 0) - logicalDisks, err := GetLogicalDriveStrings() + logicalDisks, err := getLogicalDriveStrings() if err != nil || len(logicalDisks) == 0 { return nil, err } for _, drive := range logicalDisks { - var counter, err = IOCounter(drive.UNCPath) + if len(drive.Name) > 3 { // not get _Total or Harddrive + continue + } + + if len(names) > 0 && !containsDrive(names, drive.Name) { //filter by included devices + continue + } + + var counter, err = iOCounter(drive.UNCPath) if err != nil { return nil, err } @@ -85,7 +96,8 @@ func GetIOCounters() (map[string]disk.IOCountersStat, error) { return ret, nil } -func IOCounter(path string) (DiskPerformance, error) { +// the method calls syscal func CreateFile to generate a handler then executes the DeviceIoControl func in order to retrieve the metrics +func iOCounter(path string) (DiskPerformance, error) { var diskPerformance DiskPerformance var diskPerformanceSize uint32 utfPath, err := syscall.UTF16PtrFromString(path) @@ -106,7 +118,7 @@ func IOCounter(path string) (DiskPerformance, error) { defer syscall.CloseHandle(hFile) err = syscall.DeviceIoControl(hFile, - IOCTL_DISK_PERFORMANCE, + IoctlDiskPerformance, nil, 0, (*byte)(unsafe.Pointer(&diskPerformance)), @@ -119,7 +131,8 @@ func IOCounter(path string) (DiskPerformance, error) { return diskPerformance, nil } -func GetLogicalDriveStrings() ([]LogicalDrive, error) { +// method calls the syscall GetLogicalDriveStrings in order to get the list of logical drives +func getLogicalDriveStrings() ([]LogicalDrive, error) { lpBuffer := make([]byte, 254) logicalDrives := make([]LogicalDrive, 0) r1, _, e1 := syscall.Syscall(procGetLogicalDriveStringsW.Addr(), 2, uintptr(len(lpBuffer)), uintptr(unsafe.Pointer(&lpBuffer[0])), 0) @@ -144,3 +157,12 @@ func GetLogicalDriveStrings() ([]LogicalDrive, error) { } return logicalDrives, nil } + +func containsDrive(sl []string, v string) bool { + for _, vv := range sl { + if vv == v { + return true + } + } + return false +} diff --git a/metricbeat/module/system/diskio/diskstat_windows_test.go b/metricbeat/module/system/diskio/diskstat_windows_test.go new file mode 100644 index 00000000000..983df8caab9 --- /dev/null +++ b/metricbeat/module/system/diskio/diskstat_windows_test.go @@ -0,0 +1,53 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !integration +// +build windows + +package diskio + +import ( + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestCDriveFilterOnWindowsTestEnv(t *testing.T) { + conf := map[string]interface{}{ + "module": "system", + "metricsets": []string{"diskio"}, + "diskio.include_devices": []string{"C:"}, + } + + f := mbtest.NewReportingMetricSetV2(t, conf) + data, errs := mbtest.ReportingFetchV2(f) + assert.Empty(t, errs) + assert.Equal(t, 1, len(data)) +} + +func TestAllDrivesOnWindowsTestEnv(t *testing.T) { + conf := map[string]interface{}{ + "module": "system", + "metricsets": []string{"diskio"}, + } + + f := mbtest.NewReportingMetricSetV2(t, conf) + data, errs := mbtest.ReportingFetchV2(f) + assert.Empty(t, errs) + assert.True(t, len(data)>= 1) +} + From 347d61fff15797a52bd442dc436c9f08f9714e27 Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 4 Apr 2019 11:27:09 +0200 Subject: [PATCH 03/16] Fix goimports style --- metricbeat/module/system/diskio/diskstat_windows_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metricbeat/module/system/diskio/diskstat_windows_test.go b/metricbeat/module/system/diskio/diskstat_windows_test.go index 983df8caab9..50f1b46f9a3 100644 --- a/metricbeat/module/system/diskio/diskstat_windows_test.go +++ b/metricbeat/module/system/diskio/diskstat_windows_test.go @@ -48,6 +48,5 @@ func TestAllDrivesOnWindowsTestEnv(t *testing.T) { f := mbtest.NewReportingMetricSetV2(t, conf) data, errs := mbtest.ReportingFetchV2(f) assert.Empty(t, errs) - assert.True(t, len(data)>= 1) + assert.True(t, len(data) >= 1) } - From dd148be6871790205724be687c51b763ee5b1c01 Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 4 Apr 2019 12:17:12 +0200 Subject: [PATCH 04/16] Fix goimports styling --- .../module/system/diskio/diskstat_linux.go | 2 +- .../module/system/diskio/diskstat_windows.go | 22 ++++++++++--------- .../system/diskio/diskstat_windows_test.go | 3 ++- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/metricbeat/module/system/diskio/diskstat_linux.go b/metricbeat/module/system/diskio/diskstat_linux.go index 9ff45f5a338..6d9d323b325 100644 --- a/metricbeat/module/system/diskio/diskstat_linux.go +++ b/metricbeat/module/system/diskio/diskstat_linux.go @@ -32,7 +32,7 @@ func Get_CLK_TCK() uint32 { return uint32(100) } -//created IOCounters to map functionality to dick package for linux os +// function should map functionality to disk package for linux os func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { stats, err := disk.IOCounters(names...) return stats, err diff --git a/metricbeat/module/system/diskio/diskstat_windows.go b/metricbeat/module/system/diskio/diskstat_windows.go index fa7aea328c8..93e4112fa13 100644 --- a/metricbeat/module/system/diskio/diskstat_windows.go +++ b/metricbeat/module/system/diskio/diskstat_windows.go @@ -20,9 +20,10 @@ package diskio import ( - "github.com/shirou/gopsutil/disk" "syscall" "unsafe" + + "github.com/shirou/gopsutil/disk" ) const ( @@ -34,16 +35,17 @@ const ( var ( // control code - https://docs.microsoft.com/en-us/windows/desktop/api/winioctl/ni-winioctl-ioctl_disk_performance - IoctlDiskPerformance = GetCtlCode(FILE_DEVICE_DISK, 0x0008, METHOD_BUFFERED, FILE_ANY_ACCESS) + IoctlDiskPerformance = getCtlCode(FILE_DEVICE_DISK, 0x0008, METHOD_BUFFERED, FILE_ANY_ACCESS) modkernel32 = syscall.NewLazyDLL("kernel32.dll") procGetLogicalDriveStringsW = modkernel32.NewProc("GetLogicalDriveStringsW") ) -type LogicalDrive struct { +type logicalDrive struct { Name string UNCPath string } -type DiskPerformance struct { + +type diskPerformance struct { BytesRead int64 BytesWritten int64 ReadTime int64 @@ -59,11 +61,11 @@ type DiskPerformance struct { } // used to get the specific IOCTL_DISK_PERFORMANCE control code -func GetCtlCode(deviceType uint32, function uint32, method uint32, access uint32) uint32 { +func getCtlCode(deviceType uint32, function uint32, method uint32, access uint32) uint32 { return (deviceType << 16) | (access << 14) | (function << 2) | method } -// the function gets the diskio counters and maps them to the list of IOCountersStat objects +// the function gets the diskio counters and maps them to the list of counterstat objects func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { ret := make(map[string]disk.IOCountersStat, 0) logicalDisks, err := getLogicalDriveStrings() @@ -97,8 +99,8 @@ func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { } // the method calls syscal func CreateFile to generate a handler then executes the DeviceIoControl func in order to retrieve the metrics -func iOCounter(path string) (DiskPerformance, error) { - var diskPerformance DiskPerformance +func iOCounter(path string) (diskPerformance, error) { + var diskPerformance diskPerformance var diskPerformanceSize uint32 utfPath, err := syscall.UTF16PtrFromString(path) if err != nil { @@ -132,9 +134,9 @@ func iOCounter(path string) (DiskPerformance, error) { } // method calls the syscall GetLogicalDriveStrings in order to get the list of logical drives -func getLogicalDriveStrings() ([]LogicalDrive, error) { +func getLogicalDriveStrings() ([]logicalDrive, error) { lpBuffer := make([]byte, 254) - logicalDrives := make([]LogicalDrive, 0) + logicalDrives := make([]logicalDrive, 0) r1, _, e1 := syscall.Syscall(procGetLogicalDriveStringsW.Addr(), 2, uintptr(len(lpBuffer)), uintptr(unsafe.Pointer(&lpBuffer[0])), 0) if r1 == 0 { if e1 != ERROR_SUCCESS { diff --git a/metricbeat/module/system/diskio/diskstat_windows_test.go b/metricbeat/module/system/diskio/diskstat_windows_test.go index 50f1b46f9a3..b6c3368f418 100644 --- a/metricbeat/module/system/diskio/diskstat_windows_test.go +++ b/metricbeat/module/system/diskio/diskstat_windows_test.go @@ -21,9 +21,10 @@ package diskio import ( + "testing" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/stretchr/testify/assert" - "testing" ) func TestCDriveFilterOnWindowsTestEnv(t *testing.T) { From 6d65aa2797be2192115c86dfd0cdd0350bad48b1 Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 4 Apr 2019 13:37:09 +0200 Subject: [PATCH 05/16] Fix goimports and houndci-bot requests --- metricbeat/module/system/diskio/diskio.go | 1 + .../module/system/diskio/diskstat_windows.go | 23 +++++++++---------- .../system/diskio/diskstat_windows_test.go | 3 ++- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/metricbeat/module/system/diskio/diskio.go b/metricbeat/module/system/diskio/diskio.go index 38fe217d426..ec52b49d6e7 100644 --- a/metricbeat/module/system/diskio/diskio.go +++ b/metricbeat/module/system/diskio/diskio.go @@ -23,6 +23,7 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/pkg/errors" ) diff --git a/metricbeat/module/system/diskio/diskstat_windows.go b/metricbeat/module/system/diskio/diskstat_windows.go index 93e4112fa13..efe4963bfa1 100644 --- a/metricbeat/module/system/diskio/diskstat_windows.go +++ b/metricbeat/module/system/diskio/diskstat_windows.go @@ -27,15 +27,14 @@ import ( ) const ( - FILE_DEVICE_DISK = 0x00000007 - METHOD_BUFFERED = 0 - FILE_ANY_ACCESS = 0x0000 - ERROR_SUCCESS syscall.Errno = 0 + fileDeviceDisk = 0x00000007 + fileAnyAccess = 0x0000 + errorSuccess syscall.Errno = 0 ) var ( - // control code - https://docs.microsoft.com/en-us/windows/desktop/api/winioctl/ni-winioctl-ioctl_disk_performance - IoctlDiskPerformance = getCtlCode(FILE_DEVICE_DISK, 0x0008, METHOD_BUFFERED, FILE_ANY_ACCESS) + // ioctlDiskPerformance :control code - https://docs.microsoft.com/en-us/windows/desktop/api/winioctl/ni-winioctl-ioctl_disk_performance + ioctlDiskPerformance = getCtlCode(fileDeviceDisk, 0x0008, 0, fileAnyAccess) modkernel32 = syscall.NewLazyDLL("kernel32.dll") procGetLogicalDriveStringsW = modkernel32.NewProc("GetLogicalDriveStringsW") ) @@ -65,7 +64,7 @@ func getCtlCode(deviceType uint32, function uint32, method uint32, access uint32 return (deviceType << 16) | (access << 14) | (function << 2) | method } -// the function gets the diskio counters and maps them to the list of counterstat objects +// IOCounters gets the diskio counters and maps them to the list of counterstat objects func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { ret := make(map[string]disk.IOCountersStat, 0) logicalDisks, err := getLogicalDriveStrings() @@ -98,7 +97,7 @@ func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { return ret, nil } -// the method calls syscal func CreateFile to generate a handler then executes the DeviceIoControl func in order to retrieve the metrics +// iOCounter calls syscal func CreateFile to generate a handler then executes the DeviceIoControl func in order to retrieve the metrics func iOCounter(path string) (diskPerformance, error) { var diskPerformance diskPerformance var diskPerformanceSize uint32 @@ -120,7 +119,7 @@ func iOCounter(path string) (diskPerformance, error) { defer syscall.CloseHandle(hFile) err = syscall.DeviceIoControl(hFile, - IoctlDiskPerformance, + ioctlDiskPerformance, nil, 0, (*byte)(unsafe.Pointer(&diskPerformance)), @@ -133,13 +132,13 @@ func iOCounter(path string) (diskPerformance, error) { return diskPerformance, nil } -// method calls the syscall GetLogicalDriveStrings in order to get the list of logical drives +// getLogicalDriveStrings calls the syscall GetLogicalDriveStrings in order to get the list of logical drives func getLogicalDriveStrings() ([]logicalDrive, error) { lpBuffer := make([]byte, 254) logicalDrives := make([]logicalDrive, 0) r1, _, e1 := syscall.Syscall(procGetLogicalDriveStringsW.Addr(), 2, uintptr(len(lpBuffer)), uintptr(unsafe.Pointer(&lpBuffer[0])), 0) if r1 == 0 { - if e1 != ERROR_SUCCESS { + if e1 != errorSuccess { return nil, e1 } else { return nil, syscall.EINVAL @@ -153,7 +152,7 @@ func getLogicalDriveStrings() ([]logicalDrive, error) { continue } - drive := LogicalDrive{path, "\\\\.\\" + path} + drive := logicalDrive{path, "\\\\.\\" + path} logicalDrives = append(logicalDrives, drive) } } diff --git a/metricbeat/module/system/diskio/diskstat_windows_test.go b/metricbeat/module/system/diskio/diskstat_windows_test.go index b6c3368f418..33bb8316f28 100644 --- a/metricbeat/module/system/diskio/diskstat_windows_test.go +++ b/metricbeat/module/system/diskio/diskstat_windows_test.go @@ -23,8 +23,9 @@ package diskio import ( "testing" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) func TestCDriveFilterOnWindowsTestEnv(t *testing.T) { From 3d0d1ab9a09ebba510fcb84a1bde9e304c40c25f Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 4 Apr 2019 14:33:25 +0200 Subject: [PATCH 06/16] Fix support for osx --- .../module/system/diskio/diskstat_other.go | 8 +- .../module/system/diskio/diskstat_windows.go | 151 ++-------------- .../system/diskio/diskstat_windows_helper.go | 169 ++++++++++++++++++ 3 files changed, 192 insertions(+), 136 deletions(-) create mode 100644 metricbeat/module/system/diskio/diskstat_windows_helper.go diff --git a/metricbeat/module/system/diskio/diskstat_other.go b/metricbeat/module/system/diskio/diskstat_other.go index a32f2c66588..4087750b1e5 100644 --- a/metricbeat/module/system/diskio/diskstat_other.go +++ b/metricbeat/module/system/diskio/diskstat_other.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin,cgo freebsd windows +// +build darwin,cgo freebsd package diskio @@ -42,3 +42,9 @@ func (stat *DiskIOStat) CalIOStatistics(counter disk.IOCountersStat) (DiskIOMetr func (stat *DiskIOStat) CloseSampling() { return } + +// IOCounters should map functionality to disk package for linux os +func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { + stats, err := disk.IOCounters(names...) + return stats, err +} diff --git a/metricbeat/module/system/diskio/diskstat_windows.go b/metricbeat/module/system/diskio/diskstat_windows.go index efe4963bfa1..e70204ec034 100644 --- a/metricbeat/module/system/diskio/diskstat_windows.go +++ b/metricbeat/module/system/diskio/diskstat_windows.go @@ -20,150 +20,31 @@ package diskio import ( - "syscall" - "unsafe" - + "github.com/pkg/errors" "github.com/shirou/gopsutil/disk" ) -const ( - fileDeviceDisk = 0x00000007 - fileAnyAccess = 0x0000 - errorSuccess syscall.Errno = 0 -) - -var ( - // ioctlDiskPerformance :control code - https://docs.microsoft.com/en-us/windows/desktop/api/winioctl/ni-winioctl-ioctl_disk_performance - ioctlDiskPerformance = getCtlCode(fileDeviceDisk, 0x0008, 0, fileAnyAccess) - modkernel32 = syscall.NewLazyDLL("kernel32.dll") - procGetLogicalDriveStringsW = modkernel32.NewProc("GetLogicalDriveStringsW") -) - -type logicalDrive struct { - Name string - UNCPath string +func NewDiskIOStat() *DiskIOStat { + d := &DiskIOStat{} + d.lastDiskIOCounters = make(map[string]disk.IOCountersStat) + return d } -type diskPerformance struct { - BytesRead int64 - BytesWritten int64 - ReadTime int64 - WriteTime int64 - IdleTime int64 - ReadCount uint32 - WriteCount uint32 - QueueDepth uint32 - SplitCount uint32 - QueryTime int64 - StorageDeviceNumber uint32 - StorageManagerName [8]uint16 +func (stat *DiskIOStat) OpenSampling() error { + return nil } -// used to get the specific IOCTL_DISK_PERFORMANCE control code -func getCtlCode(deviceType uint32, function uint32, method uint32, access uint32) uint32 { - return (deviceType << 16) | (access << 14) | (function << 2) | method +func (stat *DiskIOStat) CalIOStatistics(counter disk.IOCountersStat) (DiskIOMetric, error) { + var result DiskIOMetric + return result, errors.New("Not implemented out of linux") } -// IOCounters gets the diskio counters and maps them to the list of counterstat objects -func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { - ret := make(map[string]disk.IOCountersStat, 0) - logicalDisks, err := getLogicalDriveStrings() - if err != nil || len(logicalDisks) == 0 { - return nil, err - } - for _, drive := range logicalDisks { - if len(drive.Name) > 3 { // not get _Total or Harddrive - continue - } - - if len(names) > 0 && !containsDrive(names, drive.Name) { //filter by included devices - continue - } - - var counter, err = iOCounter(drive.UNCPath) - if err != nil { - return nil, err - } - ret[drive.Name] = disk.IOCountersStat{ - Name: drive.Name, - ReadCount: uint64(counter.ReadCount), - WriteCount: uint64(counter.WriteCount), - ReadBytes: uint64(counter.BytesRead), - WriteBytes: uint64(counter.BytesWritten), - ReadTime: uint64(counter.ReadTime), - WriteTime: uint64(counter.WriteTime), - } - } - return ret, nil +func (stat *DiskIOStat) CloseSampling() { + return } -// iOCounter calls syscal func CreateFile to generate a handler then executes the DeviceIoControl func in order to retrieve the metrics -func iOCounter(path string) (diskPerformance, error) { - var diskPerformance diskPerformance - var diskPerformanceSize uint32 - utfPath, err := syscall.UTF16PtrFromString(path) - if err != nil { - return diskPerformance, err - } - hFile, err := syscall.CreateFile(utfPath, - syscall.GENERIC_READ|syscall.GENERIC_WRITE, - syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE, - nil, - syscall.OPEN_EXISTING, - syscall.FILE_FLAG_BACKUP_SEMANTICS, - 0) - - if err != nil { - return diskPerformance, err - } - defer syscall.CloseHandle(hFile) - - err = syscall.DeviceIoControl(hFile, - ioctlDiskPerformance, - nil, - 0, - (*byte)(unsafe.Pointer(&diskPerformance)), - uint32(unsafe.Sizeof(diskPerformance)), - &diskPerformanceSize, - nil) - if err != nil { - return diskPerformance, err - } - return diskPerformance, nil -} - -// getLogicalDriveStrings calls the syscall GetLogicalDriveStrings in order to get the list of logical drives -func getLogicalDriveStrings() ([]logicalDrive, error) { - lpBuffer := make([]byte, 254) - logicalDrives := make([]logicalDrive, 0) - r1, _, e1 := syscall.Syscall(procGetLogicalDriveStringsW.Addr(), 2, uintptr(len(lpBuffer)), uintptr(unsafe.Pointer(&lpBuffer[0])), 0) - if r1 == 0 { - if e1 != errorSuccess { - return nil, e1 - } else { - return nil, syscall.EINVAL - } - } - - for _, v := range lpBuffer { - if v >= 65 && v <= 90 { - path := string(v) + ":" - if path == "A:" || path == "B:" { - continue - } - - drive := logicalDrive{path, "\\\\.\\" + path} - logicalDrives = append(logicalDrives, drive) - } - } - return logicalDrives, nil -} - -func containsDrive(sl []string, v string) bool { - for _, vv := range sl { - if vv == v { - return true - } - } - return false +// function should map functionality to disk package for linux os +func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { + stats, err := iOCounters(names...) + return stats, err } diff --git a/metricbeat/module/system/diskio/diskstat_windows_helper.go b/metricbeat/module/system/diskio/diskstat_windows_helper.go new file mode 100644 index 00000000000..1656963cac0 --- /dev/null +++ b/metricbeat/module/system/diskio/diskstat_windows_helper.go @@ -0,0 +1,169 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package diskio + +import ( + "syscall" + "unsafe" + + "github.com/shirou/gopsutil/disk" +) + +const ( + fileDeviceDisk = 0x00000007 + fileAnyAccess = 0x0000 + errorSuccess syscall.Errno = 0 +) + +var ( + // ioctlDiskPerformance :control code - https://docs.microsoft.com/en-us/windows/desktop/api/winioctl/ni-winioctl-ioctl_disk_performance + ioctlDiskPerformance = getCtlCode(fileDeviceDisk, 0x0008, 0, fileAnyAccess) + modkernel32 = syscall.NewLazyDLL("kernel32.dll") + procGetLogicalDriveStringsW = modkernel32.NewProc("GetLogicalDriveStringsW") +) + +type logicalDrive struct { + Name string + UNCPath string +} + +type diskPerformance struct { + BytesRead int64 + BytesWritten int64 + ReadTime int64 + WriteTime int64 + IdleTime int64 + ReadCount uint32 + WriteCount uint32 + QueueDepth uint32 + SplitCount uint32 + QueryTime int64 + StorageDeviceNumber uint32 + StorageManagerName [8]uint16 +} + +// used to get the specific IOCTL_DISK_PERFORMANCE control code +func getCtlCode(deviceType uint32, function uint32, method uint32, access uint32) uint32 { + return (deviceType << 16) | (access << 14) | (function << 2) | method +} + +// IOCounters gets the diskio counters and maps them to the list of counterstat objects +func iOCounters(names ...string) (map[string]disk.IOCountersStat, error) { + ret := make(map[string]disk.IOCountersStat, 0) + logicalDisks, err := getLogicalDriveStrings() + if err != nil || len(logicalDisks) == 0 { + return nil, err + } + for _, drive := range logicalDisks { + if len(drive.Name) > 3 { // not get _Total or Harddrive + continue + } + + if len(names) > 0 && !containsDrive(names, drive.Name) { //filter by included devices + continue + } + + var counter, err = iOCounter(drive.UNCPath) + if err != nil { + return nil, err + } + ret[drive.Name] = disk.IOCountersStat{ + Name: drive.Name, + ReadCount: uint64(counter.ReadCount), + WriteCount: uint64(counter.WriteCount), + ReadBytes: uint64(counter.BytesRead), + WriteBytes: uint64(counter.BytesWritten), + ReadTime: uint64(counter.ReadTime), + WriteTime: uint64(counter.WriteTime), + } + } + return ret, nil +} + +// iOCounter calls syscal func CreateFile to generate a handler then executes the DeviceIoControl func in order to retrieve the metrics +func iOCounter(path string) (diskPerformance, error) { + var diskPerformance diskPerformance + var diskPerformanceSize uint32 + utfPath, err := syscall.UTF16PtrFromString(path) + if err != nil { + return diskPerformance, err + } + hFile, err := syscall.CreateFile(utfPath, + syscall.GENERIC_READ|syscall.GENERIC_WRITE, + syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE, + nil, + syscall.OPEN_EXISTING, + syscall.FILE_FLAG_BACKUP_SEMANTICS, + 0) + + if err != nil { + return diskPerformance, err + } + defer syscall.CloseHandle(hFile) + + err = syscall.DeviceIoControl(hFile, + ioctlDiskPerformance, + nil, + 0, + (*byte)(unsafe.Pointer(&diskPerformance)), + uint32(unsafe.Sizeof(diskPerformance)), + &diskPerformanceSize, + nil) + if err != nil { + return diskPerformance, err + } + return diskPerformance, nil +} + +// getLogicalDriveStrings calls the syscall GetLogicalDriveStrings in order to get the list of logical drives +func getLogicalDriveStrings() ([]logicalDrive, error) { + lpBuffer := make([]byte, 254) + logicalDrives := make([]logicalDrive, 0) + r1, _, e1 := syscall.Syscall(procGetLogicalDriveStringsW.Addr(), 2, uintptr(len(lpBuffer)), uintptr(unsafe.Pointer(&lpBuffer[0])), 0) + if r1 == 0 { + if e1 != errorSuccess { + return nil, e1 + } else { + return nil, syscall.EINVAL + } + } + + for _, v := range lpBuffer { + if v >= 65 && v <= 90 { + path := string(v) + ":" + if path == "A:" || path == "B:" { + continue + } + + drive := logicalDrive{path, "\\\\.\\" + path} + logicalDrives = append(logicalDrives, drive) + } + } + return logicalDrives, nil +} + +func containsDrive(devices []string, disk string) bool { + for _, vv := range devices { + if vv == disk { + return true + } + } + return false +} From 158682ac2d98def6245b220104213c931e409025 Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 4 Apr 2019 15:12:57 +0200 Subject: [PATCH 07/16] Re-run build, address houndci-bot messages --- metricbeat/module/system/diskio/diskstat_windows.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/metricbeat/module/system/diskio/diskstat_windows.go b/metricbeat/module/system/diskio/diskstat_windows.go index e70204ec034..4df4c9a8e41 100644 --- a/metricbeat/module/system/diskio/diskstat_windows.go +++ b/metricbeat/module/system/diskio/diskstat_windows.go @@ -24,26 +24,30 @@ import ( "github.com/shirou/gopsutil/disk" ) +// NewDiskIOStat :init DiskIOStat object func NewDiskIOStat() *DiskIOStat { d := &DiskIOStat{} d.lastDiskIOCounters = make(map[string]disk.IOCountersStat) return d } +// OpenSampling stub for linux implementation func (stat *DiskIOStat) OpenSampling() error { return nil } +// CalIOStatistics stub for linux implementation func (stat *DiskIOStat) CalIOStatistics(counter disk.IOCountersStat) (DiskIOMetric, error) { var result DiskIOMetric return result, errors.New("Not implemented out of linux") } +// CloseSampling stub for linux implementation func (stat *DiskIOStat) CloseSampling() { return } -// function should map functionality to disk package for linux os +// IOCounters should map functionality to disk package for linux os func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { stats, err := iOCounters(names...) return stats, err From 22594d59cc2166bd227937e311562dc78ce453e2 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 5 Apr 2019 12:33:34 +0200 Subject: [PATCH 08/16] Update changelog file --- CHANGELOG.next.asciidoc | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d01d7d3b62b..73ef29611b2 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -91,6 +91,26 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add _bucket to histogram metrics in Prometheus Collector {pull}11578[11578] - Prevent the docker/memory metricset from processing invalid events before container start {pull}11676[11676] - Change `add_cloud_metadata` processor to not overwrite `cloud` field when it already exist in the event. {pull}11612[11612] {issue}11305[11305] +- Fix panics in vsphere module when certain values where not returned by the API. {pull}9784[9784] +- Fix pod UID metadata enrichment in Kubernetes module. {pull}10081[10081] +- Fix issue that would prevent collection of processes without command line on Windows. {pull}10196[10196] +- Fixed data type for tags field in `docker/container` metricset {pull}10307[10307] +- Fixed data type for tags field in `docker/image` metricset {pull}10307[10307] +- Fixed data type for isr field in `kafka/partition` metricset {pull}10307[10307] +- Fixed data types for various hosts fields in `mongodb/replstatus` metricset {pull}10307[10307] +- Added function to close sql database connection. {pull}10355[10355] +- Fix issue with `elasticsearch/node_stats` metricset (x-pack) not indexing `source_node` field. {pull}10639[10639] +- Migrate docker autodiscover to ECS. {issue}10757[10757] {pull}10862[10862] +- Fix issue in kubernetes module preventing usage percentages to be properly calculated. {pull}10946[10946] +- Fix for not reusable http client leading to connection leaks in Jolokia module {pull}11014[11014] +- Fix parsing error using GET in Jolokia module. {pull}11075[11075] {issue}11071[11071] +- Collect metrics when EC2 instances are not in running state. {issue}11008[11008] {pull}11023[11023] +- Change ECS field cloud.provider to aws. {pull}11023[11023] +- Add documentation about jolokia autodiscover fields. {issue}10925[10925] {pull}10979[10979] +- Add missing aws.ec2.instance.state.name into fields.yml. {issue}11219[11219] {pull}11221[11221] +- Fix ec2 metricset to collect metrics from Cloudwatch with the same timestamp. {pull}11142[11142] +- Fix potential memory leak in stopped docker metricsets {pull}11294[11294] +- Change diskio metrics retrieval method (only for Windows) from wmi query to DeviceIOControl function using the IOCTL_DISK_PERFORMANCE control code {pull}11635[11635] *Packetbeat* From 73394b9034def3c00d512c49225db69b08a52e10 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 5 Apr 2019 16:20:16 +0200 Subject: [PATCH 09/16] Addressed PR comments --- metricbeat/module/system/diskio/diskio.go | 4 +-- .../module/system/diskio/diskstat_linux.go | 12 ++++----- .../system/diskio/diskstat_linux_test.go | 4 +-- .../module/system/diskio/diskstat_other.go | 8 +++--- .../module/system/diskio/diskstat_windows.go | 5 ++-- .../system/diskio/diskstat_windows_helper.go | 26 +++++++++---------- 6 files changed, 27 insertions(+), 32 deletions(-) diff --git a/metricbeat/module/system/diskio/diskio.go b/metricbeat/module/system/diskio/diskio.go index ec52b49d6e7..c9dcbc75d70 100644 --- a/metricbeat/module/system/diskio/diskio.go +++ b/metricbeat/module/system/diskio/diskio.go @@ -88,8 +88,8 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { "time": counters.IoTime, }, } - - extraMetrics, err := m.statistics.CalIOStatistics(counters) + var extraMetrics DiskIOMetric + err := m.statistics.CalIOStatistics(&extraMetrics, counters) if err == nil { event["iostat"] = common.MapStr{ "read": common.MapStr{ diff --git a/metricbeat/module/system/diskio/diskstat_linux.go b/metricbeat/module/system/diskio/diskstat_linux.go index 6d9d323b325..3ba4ff2763a 100644 --- a/metricbeat/module/system/diskio/diskstat_linux.go +++ b/metricbeat/module/system/diskio/diskstat_linux.go @@ -34,8 +34,7 @@ func Get_CLK_TCK() uint32 { // function should map functionality to disk package for linux os func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { - stats, err := disk.IOCounters(names...) - return stats, err + return disk.IOCounters(names...) } func NewDiskIOStat() *DiskIOStat { @@ -50,21 +49,20 @@ func (stat *DiskIOStat) OpenSampling() error { return stat.curCpu.Get() } -func (stat *DiskIOStat) CalIOStatistics(counter disk.IOCountersStat) (DiskIOMetric, error) { +func (stat *DiskIOStat) CalIOStatistics(result *DiskIOMetric, counter disk.IOCountersStat) error { var last disk.IOCountersStat var ok bool - var result DiskIOMetric // if last counter not found, create one and return all 0 if last, ok = stat.lastDiskIOCounters[counter.Name]; !ok { stat.lastDiskIOCounters[counter.Name] = counter - return result, nil + return nil } // calculate the delta ms between the CloseSampling and OpenSampling deltams := 1000.0 * float64(stat.curCpu.Total()-stat.lastCpu.Total()) / float64(cpu.NumCores) / float64(Get_CLK_TCK()) if deltams <= 0 { - return result, errors.New("The delta cpu time between close sampling and open sampling is less or equal to 0") + return errors.New("The delta cpu time between close sampling and open sampling is less or equal to 0") } rd_ios := counter.ReadCount - last.ReadCount @@ -117,7 +115,7 @@ func (stat *DiskIOStat) CalIOStatistics(counter disk.IOCountersStat) (DiskIOMetr } stat.lastDiskIOCounters[counter.Name] = counter - return result, nil + return nil } diff --git a/metricbeat/module/system/diskio/diskstat_linux_test.go b/metricbeat/module/system/diskio/diskstat_linux_test.go index 44511de0a76..22218fc411b 100644 --- a/metricbeat/module/system/diskio/diskstat_linux_test.go +++ b/metricbeat/module/system/diskio/diskstat_linux_test.go @@ -104,8 +104,8 @@ func TestDiskIOStat_CalIOStatistics(t *testing.T) { AvgReadAwaitTime: 1.2, AvgWriteAwaitTime: 1, } - - got, err := stat.CalIOStatistics(counter) + var got DiskIOMetric + err := stat.CalIOStatistics(&got, counter) if err != nil { t.Fatal(err) } diff --git a/metricbeat/module/system/diskio/diskstat_other.go b/metricbeat/module/system/diskio/diskstat_other.go index 4087750b1e5..ea46a68a7b6 100644 --- a/metricbeat/module/system/diskio/diskstat_other.go +++ b/metricbeat/module/system/diskio/diskstat_other.go @@ -34,9 +34,8 @@ func (stat *DiskIOStat) OpenSampling() error { return nil } -func (stat *DiskIOStat) CalIOStatistics(counter disk.IOCountersStat) (DiskIOMetric, error) { - var result DiskIOMetric - return result, errors.New("Not implemented out of linux") +func (stat *DiskIOStat) CalIOStatistics(result *DiskIOMetric, counter disk.IOCountersStat) error { + return errors.New("Not implemented out of linux") } func (stat *DiskIOStat) CloseSampling() { @@ -45,6 +44,5 @@ func (stat *DiskIOStat) CloseSampling() { // IOCounters should map functionality to disk package for linux os func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { - stats, err := disk.IOCounters(names...) - return stats, err + return disk.IOCounters(names...) } diff --git a/metricbeat/module/system/diskio/diskstat_windows.go b/metricbeat/module/system/diskio/diskstat_windows.go index 4df4c9a8e41..e952ac9cb05 100644 --- a/metricbeat/module/system/diskio/diskstat_windows.go +++ b/metricbeat/module/system/diskio/diskstat_windows.go @@ -37,9 +37,8 @@ func (stat *DiskIOStat) OpenSampling() error { } // CalIOStatistics stub for linux implementation -func (stat *DiskIOStat) CalIOStatistics(counter disk.IOCountersStat) (DiskIOMetric, error) { - var result DiskIOMetric - return result, errors.New("Not implemented out of linux") +func (stat *DiskIOStat) CalIOStatistics(result *DiskIOMetric, counter disk.IOCountersStat) error { + return errors.New("Not implemented out of linux") } // CloseSampling stub for linux implementation diff --git a/metricbeat/module/system/diskio/diskstat_windows_helper.go b/metricbeat/module/system/diskio/diskstat_windows_helper.go index 1656963cac0..0f1157b40f6 100644 --- a/metricbeat/module/system/diskio/diskstat_windows_helper.go +++ b/metricbeat/module/system/diskio/diskstat_windows_helper.go @@ -66,17 +66,18 @@ func getCtlCode(deviceType uint32, function uint32, method uint32, access uint32 // IOCounters gets the diskio counters and maps them to the list of counterstat objects func iOCounters(names ...string) (map[string]disk.IOCountersStat, error) { - ret := make(map[string]disk.IOCountersStat, 0) logicalDisks, err := getLogicalDriveStrings() - if err != nil || len(logicalDisks) == 0 { + if err != nil || logicalDisks == nil { return nil, err } + ret := make(map[string]disk.IOCountersStat, 0) for _, drive := range logicalDisks { - if len(drive.Name) > 3 { // not get _Total or Harddrive + // not get _Total or Harddrive + if len(drive.Name) > 3 { continue } - - if len(names) > 0 && !containsDrive(names, drive.Name) { //filter by included devices + //filter by included devices + if len(names) > 0 && !containsDrive(names, drive.Name) { continue } @@ -135,23 +136,22 @@ func iOCounter(path string) (diskPerformance, error) { // getLogicalDriveStrings calls the syscall GetLogicalDriveStrings in order to get the list of logical drives func getLogicalDriveStrings() ([]logicalDrive, error) { lpBuffer := make([]byte, 254) - logicalDrives := make([]logicalDrive, 0) r1, _, e1 := syscall.Syscall(procGetLogicalDriveStringsW.Addr(), 2, uintptr(len(lpBuffer)), uintptr(unsafe.Pointer(&lpBuffer[0])), 0) if r1 == 0 { + err := e1 if e1 != errorSuccess { - return nil, e1 - } else { - return nil, syscall.EINVAL + err = syscall.EINVAL } + return nil, err } - + var logicalDrives []logicalDrive for _, v := range lpBuffer { if v >= 65 && v <= 90 { - path := string(v) + ":" - if path == "A:" || path == "B:" { + s := string(v) + if s == "A" || s == "B" { continue } - + path := s + ":" drive := logicalDrive{path, "\\\\.\\" + path} logicalDrives = append(logicalDrives, drive) } From 96e00d7901dc74dc5e97900ffb5a9934c9701e6b Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 9 Apr 2019 17:37:08 +0200 Subject: [PATCH 10/16] Addressing review notes --- .../module/system/diskio/diskstat_linux.go | 18 ++++++----- .../module/system/diskio/diskstat_other.go | 14 +++++---- .../module/system/diskio/diskstat_windows.go | 21 +++++++------ .../system/diskio/diskstat_windows_helper.go | 30 +++++++------------ 4 files changed, 40 insertions(+), 43 deletions(-) diff --git a/metricbeat/module/system/diskio/diskstat_linux.go b/metricbeat/module/system/diskio/diskstat_linux.go index 3ba4ff2763a..edff3df61a5 100644 --- a/metricbeat/module/system/diskio/diskstat_linux.go +++ b/metricbeat/module/system/diskio/diskstat_linux.go @@ -27,28 +27,30 @@ import ( ) func Get_CLK_TCK() uint32 { - //return uint32(C.sysconf(C._SC_CLK_TCK)) - //NOTE: _SC_CLK_TCK should be fetched from sysconf using cgo + // return uint32(C.sysconf(C._SC_CLK_TCK)) + // NOTE: _SC_CLK_TCK should be fetched from sysconf using cgo return uint32(100) } -// function should map functionality to disk package for linux os +// IOCounters should map functionality to disk package for linux os. func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { return disk.IOCounters(names...) } +// NewDiskIOStat :init DiskIOStat object. func NewDiskIOStat() *DiskIOStat { - d := &DiskIOStat{} - d.lastDiskIOCounters = make(map[string]disk.IOCountersStat) - return d + return &DiskIOStat{ + lastDiskIOCounters: map[string]disk.IOCountersStat{}, + } } -// create current cpu sampling -// need call as soon as get IOCounters +// OpenSampling creates current cpu sampling +// need call as soon as get IOCounters. func (stat *DiskIOStat) OpenSampling() error { return stat.curCpu.Get() } +// CalIOStatistics calculates IO statistics. func (stat *DiskIOStat) CalIOStatistics(result *DiskIOMetric, counter disk.IOCountersStat) error { var last disk.IOCountersStat var ok bool diff --git a/metricbeat/module/system/diskio/diskstat_other.go b/metricbeat/module/system/diskio/diskstat_other.go index ea46a68a7b6..8f3c7df74a1 100644 --- a/metricbeat/module/system/diskio/diskstat_other.go +++ b/metricbeat/module/system/diskio/diskstat_other.go @@ -24,25 +24,29 @@ import ( "github.com/shirou/gopsutil/disk" ) +// NewDiskIOStat :init DiskIOStat object. func NewDiskIOStat() *DiskIOStat { - d := &DiskIOStat{} - d.lastDiskIOCounters = make(map[string]disk.IOCountersStat) - return d + return &DiskIOStat{ + lastDiskIOCounters: map[string]disk.IOCountersStat{}, + } } +// OpenSampling stub for linux implementation. func (stat *DiskIOStat) OpenSampling() error { return nil } +// CalIOStatistics stub for linux implementation. func (stat *DiskIOStat) CalIOStatistics(result *DiskIOMetric, counter disk.IOCountersStat) error { - return errors.New("Not implemented out of linux") + return errors.New("not implemented out of linux") } +// CloseSampling stub for linux implementation. func (stat *DiskIOStat) CloseSampling() { return } -// IOCounters should map functionality to disk package for linux os +// IOCounters should map functionality to disk package for linux os. func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { return disk.IOCounters(names...) } diff --git a/metricbeat/module/system/diskio/diskstat_windows.go b/metricbeat/module/system/diskio/diskstat_windows.go index e952ac9cb05..9308cd7d49c 100644 --- a/metricbeat/module/system/diskio/diskstat_windows.go +++ b/metricbeat/module/system/diskio/diskstat_windows.go @@ -24,30 +24,29 @@ import ( "github.com/shirou/gopsutil/disk" ) -// NewDiskIOStat :init DiskIOStat object +// NewDiskIOStat :init DiskIOStat object. func NewDiskIOStat() *DiskIOStat { - d := &DiskIOStat{} - d.lastDiskIOCounters = make(map[string]disk.IOCountersStat) - return d + return &DiskIOStat{ + lastDiskIOCounters: map[string]disk.IOCountersStat{}, + } } -// OpenSampling stub for linux implementation +// OpenSampling stub for linux implementation. func (stat *DiskIOStat) OpenSampling() error { return nil } -// CalIOStatistics stub for linux implementation +// CalIOStatistics stub for linux implementation. func (stat *DiskIOStat) CalIOStatistics(result *DiskIOMetric, counter disk.IOCountersStat) error { - return errors.New("Not implemented out of linux") + return errors.New("iostat is not implement for Windows") } -// CloseSampling stub for linux implementation +// CloseSampling stub for linux implementation. func (stat *DiskIOStat) CloseSampling() { return } -// IOCounters should map functionality to disk package for linux os +// IOCounters should map functionality to disk package for linux os. func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { - stats, err := iOCounters(names...) - return stats, err + return ioCounters(names...) } diff --git a/metricbeat/module/system/diskio/diskstat_windows_helper.go b/metricbeat/module/system/diskio/diskstat_windows_helper.go index 0f1157b40f6..2a8e24d20fb 100644 --- a/metricbeat/module/system/diskio/diskstat_windows_helper.go +++ b/metricbeat/module/system/diskio/diskstat_windows_helper.go @@ -27,14 +27,11 @@ import ( ) const ( - fileDeviceDisk = 0x00000007 - fileAnyAccess = 0x0000 - errorSuccess syscall.Errno = 0 + errorSuccess syscall.Errno = 0 + ioctlDiskPerformance = 0x70020 ) var ( - // ioctlDiskPerformance :control code - https://docs.microsoft.com/en-us/windows/desktop/api/winioctl/ni-winioctl-ioctl_disk_performance - ioctlDiskPerformance = getCtlCode(fileDeviceDisk, 0x0008, 0, fileAnyAccess) modkernel32 = syscall.NewLazyDLL("kernel32.dll") procGetLogicalDriveStringsW = modkernel32.NewProc("GetLogicalDriveStringsW") ) @@ -59,29 +56,24 @@ type diskPerformance struct { StorageManagerName [8]uint16 } -// used to get the specific IOCTL_DISK_PERFORMANCE control code -func getCtlCode(deviceType uint32, function uint32, method uint32, access uint32) uint32 { - return (deviceType << 16) | (access << 14) | (function << 2) | method -} - -// IOCounters gets the diskio counters and maps them to the list of counterstat objects -func iOCounters(names ...string) (map[string]disk.IOCountersStat, error) { +// ioCounters gets the diskio counters and maps them to the list of counterstat objects. +func ioCounters(names ...string) (map[string]disk.IOCountersStat, error) { logicalDisks, err := getLogicalDriveStrings() - if err != nil || logicalDisks == nil { + if err != nil || len(logicalDisks) == 0 { return nil, err } - ret := make(map[string]disk.IOCountersStat, 0) + ret := make(map[string]disk.IOCountersStat) for _, drive := range logicalDisks { // not get _Total or Harddrive if len(drive.Name) > 3 { continue } - //filter by included devices + // filter by included devices if len(names) > 0 && !containsDrive(names, drive.Name) { continue } - var counter, err = iOCounter(drive.UNCPath) + counter, err := ioCounter(drive.UNCPath) if err != nil { return nil, err } @@ -98,8 +90,8 @@ func iOCounters(names ...string) (map[string]disk.IOCountersStat, error) { return ret, nil } -// iOCounter calls syscal func CreateFile to generate a handler then executes the DeviceIoControl func in order to retrieve the metrics -func iOCounter(path string) (diskPerformance, error) { +// i0Counter calls syscal func CreateFile to generate a handler then executes the DeviceIoControl func in order to retrieve the metrics. +func ioCounter(path string) (diskPerformance, error) { var diskPerformance diskPerformance var diskPerformanceSize uint32 utfPath, err := syscall.UTF16PtrFromString(path) @@ -152,7 +144,7 @@ func getLogicalDriveStrings() ([]logicalDrive, error) { continue } path := s + ":" - drive := logicalDrive{path, "\\\\.\\" + path} + drive := logicalDrive{path, `\\.\` + path} logicalDrives = append(logicalDrives, drive) } } From 1119a28d07a67e8511761940c43c71b3ec02194d Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 17 Apr 2019 19:28:03 +0200 Subject: [PATCH 11/16] Enrich test and implement separate function to enable the performance counters --- .../system/diskio/diskstat_windows_helper.go | 38 ++++++++++++++++++- .../system/diskio/diskstat_windows_test.go | 21 ++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/metricbeat/module/system/diskio/diskstat_windows_helper.go b/metricbeat/module/system/diskio/diskstat_windows_helper.go index 2a8e24d20fb..712fc81a2d2 100644 --- a/metricbeat/module/system/diskio/diskstat_windows_helper.go +++ b/metricbeat/module/system/diskio/diskstat_windows_helper.go @@ -23,6 +23,9 @@ import ( "syscall" "unsafe" + "github.com/pkg/errors" + "golang.org/x/sys/windows/registry" + "github.com/shirou/gopsutil/disk" ) @@ -34,6 +37,7 @@ const ( var ( modkernel32 = syscall.NewLazyDLL("kernel32.dll") procGetLogicalDriveStringsW = modkernel32.NewProc("GetLogicalDriveStringsW") + procGetDriveTypeW = modkernel32.NewProc("GetDriveTypeW") ) type logicalDrive struct { @@ -58,6 +62,9 @@ type diskPerformance struct { // ioCounters gets the diskio counters and maps them to the list of counterstat objects. func ioCounters(names ...string) (map[string]disk.IOCountersStat, error) { + if err := enablePerformanceCounters(); err != nil { + return nil, err + } logicalDisks, err := getLogicalDriveStrings() if err != nil || len(logicalDisks) == 0 { return nil, err @@ -125,6 +132,18 @@ func ioCounter(path string) (diskPerformance, error) { return diskPerformance, nil } +// enablePerformanceCounters will enable performance counters by adding the EnableCounterForIoctl registry key +func enablePerformanceCounters() error { + key, err := registry.OpenKey(registry.LOCAL_MACHINE, "SYSTEM\\CurrentControlSet\\Services\\partmgr", registry.READ|registry.WRITE) + if err != nil { + return errors.Errorf("cannot open new key in the registry in order to enable the performance counters: %s", err) + } + if err = key.SetDWordValue("EnableCounterForIoctl", 1); err != nil { + return errors.Errorf("cannot create EnableCounterForIoctl key in the registry in order to enable the performance counters: %s", err) + } + return nil +} + // getLogicalDriveStrings calls the syscall GetLogicalDriveStrings in order to get the list of logical drives func getLogicalDriveStrings() ([]logicalDrive, error) { lpBuffer := make([]byte, 254) @@ -145,7 +164,9 @@ func getLogicalDriveStrings() ([]logicalDrive, error) { } path := s + ":" drive := logicalDrive{path, `\\.\` + path} - logicalDrives = append(logicalDrives, drive) + if isValidLogicalDrive(path) { + logicalDrives = append(logicalDrives, drive) + } } } return logicalDrives, nil @@ -159,3 +180,18 @@ func containsDrive(devices []string, disk string) bool { } return false } + +// filterLogicalDrives should filter CD-ROM type drives based on https://docs.microsoft.com/en-us/windows/desktop/api/fileapi/nf-fileapi-getdrivetypew +func isValidLogicalDrive(path string) bool { + utfPath, err := syscall.UTF16PtrFromString(path + `\`) + if err != nil { + return false + } + ret, _, err := syscall.Syscall(procGetDriveTypeW.Addr(), 1, uintptr(unsafe.Pointer(utfPath)), 0, 0) + + //DRIVE_NO_ROOT_DIR = 1 DRIVE_CDROM = 5 + if ret == 1 || ret == 5 || err != errorSuccess { + return false + } + return true +} diff --git a/metricbeat/module/system/diskio/diskstat_windows_test.go b/metricbeat/module/system/diskio/diskstat_windows_test.go index 33bb8316f28..3453ea3eac3 100644 --- a/metricbeat/module/system/diskio/diskstat_windows_test.go +++ b/metricbeat/module/system/diskio/diskstat_windows_test.go @@ -23,6 +23,8 @@ package diskio import ( "testing" + "github.com/elastic/beats/libbeat/common" + "github.com/stretchr/testify/assert" mbtest "github.com/elastic/beats/metricbeat/mb/testing" @@ -39,6 +41,25 @@ func TestCDriveFilterOnWindowsTestEnv(t *testing.T) { data, errs := mbtest.ReportingFetchV2(f) assert.Empty(t, errs) assert.Equal(t, 1, len(data)) + assert.Equal(t, data[0].MetricSetFields["name"], "C:") + reads := data[0].MetricSetFields["read"].(common.MapStr) + writes := data[0].MetricSetFields["write"].(common.MapStr) + // Check values + readCount := reads["count"].(uint64) + readBytes := reads["bytes"].(uint64) + readTime := reads["time"].(uint64) + writeCount := writes["count"].(uint64) + writeBytes := writes["bytes"].(uint64) + writeTime := writes["time"].(uint64) + + assert.True(t, readCount > 0) + assert.True(t, readBytes > 0) + assert.True(t, readTime > 0) + + assert.True(t, writeCount > 0) + assert.True(t, writeBytes > 0) + assert.True(t, writeTime > 0) + } func TestAllDrivesOnWindowsTestEnv(t *testing.T) { From 82acba32871bff28c85b2669b8ad1ccceee9ddf5 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 19 Apr 2019 14:52:44 +0200 Subject: [PATCH 12/16] Add disable performance counters functionality for testing --- .../system/diskio/diskstat_windows_helper.go | 39 ++++++++++++++++++- .../system/diskio/diskstat_windows_test.go | 9 ++++- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/system/diskio/diskstat_windows_helper.go b/metricbeat/module/system/diskio/diskstat_windows_helper.go index 712fc81a2d2..f909b71928f 100644 --- a/metricbeat/module/system/diskio/diskstat_windows_helper.go +++ b/metricbeat/module/system/diskio/diskstat_windows_helper.go @@ -30,8 +30,9 @@ import ( ) const ( - errorSuccess syscall.Errno = 0 - ioctlDiskPerformance = 0x70020 + errorSuccess syscall.Errno = 0 + ioctlDiskPerformance = 0x70020 + ioctlDiskPerformanceOff = 0x70060 ) var ( @@ -144,6 +145,40 @@ func enablePerformanceCounters() error { return nil } +// disablePerformanceCounters will disable performance counters using the IOCTL_DISK_PERFORMANCE_OFF IOCTL control code +func disablePerformanceCounters(path string) error { + utfPath, err := syscall.UTF16PtrFromString(path) + if err != nil { + return err + } + hFile, err := syscall.CreateFile(utfPath, + syscall.GENERIC_READ|syscall.GENERIC_WRITE, + syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE, + nil, + syscall.OPEN_EXISTING, + syscall.FILE_FLAG_BACKUP_SEMANTICS, + 0) + + if err != nil { + return err + } + defer syscall.CloseHandle(hFile) + var diskPerformanceSize uint32 + err = syscall.DeviceIoControl(hFile, + ioctlDiskPerformanceOff, + nil, + 0, + nil, + 0, + &diskPerformanceSize, + nil) + if err != nil { + return err + } + return nil + +} + // getLogicalDriveStrings calls the syscall GetLogicalDriveStrings in order to get the list of logical drives func getLogicalDriveStrings() ([]logicalDrive, error) { lpBuffer := make([]byte, 254) diff --git a/metricbeat/module/system/diskio/diskstat_windows_test.go b/metricbeat/module/system/diskio/diskstat_windows_test.go index 3453ea3eac3..b833d5ccebf 100644 --- a/metricbeat/module/system/diskio/diskstat_windows_test.go +++ b/metricbeat/module/system/diskio/diskstat_windows_test.go @@ -59,7 +59,8 @@ func TestCDriveFilterOnWindowsTestEnv(t *testing.T) { assert.True(t, writeCount > 0) assert.True(t, writeBytes > 0) assert.True(t, writeTime > 0) - + err := disablePerformanceCounters(`\\.\C:`) + assert.NoError(t, err) } func TestAllDrivesOnWindowsTestEnv(t *testing.T) { @@ -72,4 +73,10 @@ func TestAllDrivesOnWindowsTestEnv(t *testing.T) { data, errs := mbtest.ReportingFetchV2(f) assert.Empty(t, errs) assert.True(t, len(data) >= 1) + drives, err := getLogicalDriveStrings() + assert.NoError(t, err) + for _, drive := range drives { + err := disablePerformanceCounters(drive.UNCPath) + assert.NoError(t, err) + } } From 648d50f672ae039fba123a6636ff2427973e4465 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 23 Apr 2019 11:17:20 +0200 Subject: [PATCH 13/16] Log meesage when the EnableCounterForIoctl is added/updated in the registry --- CHANGELOG.next.asciidoc | 19 ------------------- .../system/diskio/diskstat_windows_helper.go | 6 +++++- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 73ef29611b2..27989426910 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -91,25 +91,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add _bucket to histogram metrics in Prometheus Collector {pull}11578[11578] - Prevent the docker/memory metricset from processing invalid events before container start {pull}11676[11676] - Change `add_cloud_metadata` processor to not overwrite `cloud` field when it already exist in the event. {pull}11612[11612] {issue}11305[11305] -- Fix panics in vsphere module when certain values where not returned by the API. {pull}9784[9784] -- Fix pod UID metadata enrichment in Kubernetes module. {pull}10081[10081] -- Fix issue that would prevent collection of processes without command line on Windows. {pull}10196[10196] -- Fixed data type for tags field in `docker/container` metricset {pull}10307[10307] -- Fixed data type for tags field in `docker/image` metricset {pull}10307[10307] -- Fixed data type for isr field in `kafka/partition` metricset {pull}10307[10307] -- Fixed data types for various hosts fields in `mongodb/replstatus` metricset {pull}10307[10307] -- Added function to close sql database connection. {pull}10355[10355] -- Fix issue with `elasticsearch/node_stats` metricset (x-pack) not indexing `source_node` field. {pull}10639[10639] -- Migrate docker autodiscover to ECS. {issue}10757[10757] {pull}10862[10862] -- Fix issue in kubernetes module preventing usage percentages to be properly calculated. {pull}10946[10946] -- Fix for not reusable http client leading to connection leaks in Jolokia module {pull}11014[11014] -- Fix parsing error using GET in Jolokia module. {pull}11075[11075] {issue}11071[11071] -- Collect metrics when EC2 instances are not in running state. {issue}11008[11008] {pull}11023[11023] -- Change ECS field cloud.provider to aws. {pull}11023[11023] -- Add documentation about jolokia autodiscover fields. {issue}10925[10925] {pull}10979[10979] -- Add missing aws.ec2.instance.state.name into fields.yml. {issue}11219[11219] {pull}11221[11221] -- Fix ec2 metricset to collect metrics from Cloudwatch with the same timestamp. {pull}11142[11142] -- Fix potential memory leak in stopped docker metricsets {pull}11294[11294] - Change diskio metrics retrieval method (only for Windows) from wmi query to DeviceIOControl function using the IOCTL_DISK_PERFORMANCE control code {pull}11635[11635] *Packetbeat* diff --git a/metricbeat/module/system/diskio/diskstat_windows_helper.go b/metricbeat/module/system/diskio/diskstat_windows_helper.go index f909b71928f..b638e714217 100644 --- a/metricbeat/module/system/diskio/diskstat_windows_helper.go +++ b/metricbeat/module/system/diskio/diskstat_windows_helper.go @@ -23,6 +23,8 @@ import ( "syscall" "unsafe" + "github.com/elastic/beats/libbeat/logp" + "github.com/pkg/errors" "golang.org/x/sys/windows/registry" @@ -39,6 +41,7 @@ var ( modkernel32 = syscall.NewLazyDLL("kernel32.dll") procGetLogicalDriveStringsW = modkernel32.NewProc("GetLogicalDriveStringsW") procGetDriveTypeW = modkernel32.NewProc("GetDriveTypeW") + logger = logp.NewLogger("diskio") ) type logicalDrive struct { @@ -98,7 +101,7 @@ func ioCounters(names ...string) (map[string]disk.IOCountersStat, error) { return ret, nil } -// i0Counter calls syscal func CreateFile to generate a handler then executes the DeviceIoControl func in order to retrieve the metrics. +// ioCounter calls syscal func CreateFile to generate a handler then executes the DeviceIoControl func in order to retrieve the metrics. func ioCounter(path string) (diskPerformance, error) { var diskPerformance diskPerformance var diskPerformanceSize uint32 @@ -142,6 +145,7 @@ func enablePerformanceCounters() error { if err = key.SetDWordValue("EnableCounterForIoctl", 1); err != nil { return errors.Errorf("cannot create EnableCounterForIoctl key in the registry in order to enable the performance counters: %s", err) } + logger.Info("The key with the name EnableCounterForIoctl has been added in the windows registry in order to enable the performance counters") return nil } From bd04c355b622f65d1f99aac85a012b9e4b68bf27 Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 24 Apr 2019 15:23:55 +0200 Subject: [PATCH 14/16] Check for registry key value before updating it --- .../module/system/diskio/diskstat_windows_helper.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/system/diskio/diskstat_windows_helper.go b/metricbeat/module/system/diskio/diskstat_windows_helper.go index b638e714217..6093107d4ce 100644 --- a/metricbeat/module/system/diskio/diskstat_windows_helper.go +++ b/metricbeat/module/system/diskio/diskstat_windows_helper.go @@ -142,10 +142,13 @@ func enablePerformanceCounters() error { if err != nil { return errors.Errorf("cannot open new key in the registry in order to enable the performance counters: %s", err) } - if err = key.SetDWordValue("EnableCounterForIoctl", 1); err != nil { - return errors.Errorf("cannot create EnableCounterForIoctl key in the registry in order to enable the performance counters: %s", err) + val, _, err := key.GetIntegerValue("EnableCounterForIoctl") + if val != 1 || err != nil { + if err = key.SetDWordValue("EnableCounterForIoctl", 1); err != nil { + return errors.Errorf("cannot create HKLM:SYSTEM\\CurrentControlSet\\Services\\Partmgr\\EnableCounterForIoctl key in the registry in order to enable the performance counters: %s", err) + } + logger.Info("The registry key EnableCounterForIoctl at HKLM:SYSTEM\\CurrentControlSet\\Services\\Partmgr has been created in order to enable the performance counters") } - logger.Info("The key with the name EnableCounterForIoctl has been added in the windows registry in order to enable the performance counters") return nil } From eb0bf3acf71fc07054978fb520413323edeec7ac Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 25 Apr 2019 16:40:43 +0200 Subject: [PATCH 15/16] Address new code reviews --- metricbeat/module/system/diskio/diskstat_other.go | 4 +--- metricbeat/module/system/diskio/diskstat_windows.go | 4 +--- metricbeat/module/system/diskio/diskstat_windows_helper.go | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/metricbeat/module/system/diskio/diskstat_other.go b/metricbeat/module/system/diskio/diskstat_other.go index 8f3c7df74a1..d2669117d9e 100644 --- a/metricbeat/module/system/diskio/diskstat_other.go +++ b/metricbeat/module/system/diskio/diskstat_other.go @@ -42,9 +42,7 @@ func (stat *DiskIOStat) CalIOStatistics(result *DiskIOMetric, counter disk.IOCou } // CloseSampling stub for linux implementation. -func (stat *DiskIOStat) CloseSampling() { - return -} +func (stat *DiskIOStat) CloseSampling() {} // IOCounters should map functionality to disk package for linux os. func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { diff --git a/metricbeat/module/system/diskio/diskstat_windows.go b/metricbeat/module/system/diskio/diskstat_windows.go index 9308cd7d49c..1e2b363074b 100644 --- a/metricbeat/module/system/diskio/diskstat_windows.go +++ b/metricbeat/module/system/diskio/diskstat_windows.go @@ -42,9 +42,7 @@ func (stat *DiskIOStat) CalIOStatistics(result *DiskIOMetric, counter disk.IOCou } // CloseSampling stub for linux implementation. -func (stat *DiskIOStat) CloseSampling() { - return -} +func (stat *DiskIOStat) CloseSampling() {} // IOCounters should map functionality to disk package for linux os. func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { diff --git a/metricbeat/module/system/diskio/diskstat_windows_helper.go b/metricbeat/module/system/diskio/diskstat_windows_helper.go index 6093107d4ce..c9b5b407a27 100644 --- a/metricbeat/module/system/diskio/diskstat_windows_helper.go +++ b/metricbeat/module/system/diskio/diskstat_windows_helper.go @@ -223,7 +223,7 @@ func containsDrive(devices []string, disk string) bool { return false } -// filterLogicalDrives should filter CD-ROM type drives based on https://docs.microsoft.com/en-us/windows/desktop/api/fileapi/nf-fileapi-getdrivetypew +// isValidLogicalDrive should filter CD-ROM type drives based on https://docs.microsoft.com/en-us/windows/desktop/api/fileapi/nf-fileapi-getdrivetypew func isValidLogicalDrive(path string) bool { utfPath, err := syscall.UTF16PtrFromString(path + `\`) if err != nil { From f4dae1389ff29299bde6b02f5b9632158308d924 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 26 Apr 2019 13:29:09 +0200 Subject: [PATCH 16/16] small refactoring of deviceiocontrol functions --- .../system/diskio/diskstat_windows_helper.go | 33 +++++++------------ 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/metricbeat/module/system/diskio/diskstat_windows_helper.go b/metricbeat/module/system/diskio/diskstat_windows_helper.go index c9b5b407a27..a43ca07582e 100644 --- a/metricbeat/module/system/diskio/diskstat_windows_helper.go +++ b/metricbeat/module/system/diskio/diskstat_windows_helper.go @@ -83,8 +83,8 @@ func ioCounters(names ...string) (map[string]disk.IOCountersStat, error) { if len(names) > 0 && !containsDrive(names, drive.Name) { continue } - - counter, err := ioCounter(drive.UNCPath) + var counter diskPerformance + err = ioCounter(drive.UNCPath, &counter) if err != nil { return nil, err } @@ -101,13 +101,11 @@ func ioCounters(names ...string) (map[string]disk.IOCountersStat, error) { return ret, nil } -// ioCounter calls syscal func CreateFile to generate a handler then executes the DeviceIoControl func in order to retrieve the metrics. -func ioCounter(path string) (diskPerformance, error) { - var diskPerformance diskPerformance - var diskPerformanceSize uint32 +// ioCounter calls syscall func CreateFile to generate a handler then executes the DeviceIoControl func in order to retrieve the metrics. +func ioCounter(path string, diskPerformance *diskPerformance) error { utfPath, err := syscall.UTF16PtrFromString(path) if err != nil { - return diskPerformance, err + return err } hFile, err := syscall.CreateFile(utfPath, syscall.GENERIC_READ|syscall.GENERIC_WRITE, @@ -118,22 +116,18 @@ func ioCounter(path string) (diskPerformance, error) { 0) if err != nil { - return diskPerformance, err + return err } defer syscall.CloseHandle(hFile) - - err = syscall.DeviceIoControl(hFile, + var diskPerformanceSize uint32 + return syscall.DeviceIoControl(hFile, ioctlDiskPerformance, nil, 0, - (*byte)(unsafe.Pointer(&diskPerformance)), - uint32(unsafe.Sizeof(diskPerformance)), + (*byte)(unsafe.Pointer(diskPerformance)), + uint32(unsafe.Sizeof(*diskPerformance)), &diskPerformanceSize, nil) - if err != nil { - return diskPerformance, err - } - return diskPerformance, nil } // enablePerformanceCounters will enable performance counters by adding the EnableCounterForIoctl registry key @@ -171,7 +165,7 @@ func disablePerformanceCounters(path string) error { } defer syscall.CloseHandle(hFile) var diskPerformanceSize uint32 - err = syscall.DeviceIoControl(hFile, + return syscall.DeviceIoControl(hFile, ioctlDiskPerformanceOff, nil, 0, @@ -179,11 +173,6 @@ func disablePerformanceCounters(path string) error { 0, &diskPerformanceSize, nil) - if err != nil { - return err - } - return nil - } // getLogicalDriveStrings calls the syscall GetLogicalDriveStrings in order to get the list of logical drives