From 6a83db2517e7738f84ab2ef79fd1b04de3257bd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 9 Jun 2022 14:09:23 +0200 Subject: [PATCH 01/22] Remove `metricbeat/internal/metrics/{cpu,memory}` package and use it from `elastic-agent-system-metrics` (#31822) --- NOTICE.txt | 428 +++--- go.mod | 4 +- go.sum | 4 +- metricbeat/internal/metrics/cpu/doc.go | 18 - metricbeat/internal/metrics/cpu/metrics.go | 245 ---- .../internal/metrics/cpu/metrics_aix.go | 129 -- .../internal/metrics/cpu/metrics_darwin.go | 58 - .../internal/metrics/cpu/metrics_freebsd.go | 60 - .../internal/metrics/cpu/metrics_linux.go | 69 - .../metrics/cpu/metrics_linux_test.go | 97 -- .../internal/metrics/cpu/metrics_openbsd.go | 136 -- .../metrics/cpu/metrics_procfs_common.go | 157 -- .../internal/metrics/cpu/metrics_test.go | 162 --- .../internal/metrics/cpu/metrics_windows.go | 69 - .../internal/metrics/cpu/testdata/cpuinfo | 448 ------ .../metrics/cpu/testdata/cpuinfo-armv7 | 17 - .../cpu/testdata/cpuinfo-armv7.expected.json | 23 - .../metrics/cpu/testdata/cpuinfo-quad-socket | 1296 ----------------- .../cpuinfo-quad-socket.expected.json | 338 ----- .../cpu/testdata/cpuinfo.expected.json | 114 -- metricbeat/internal/metrics/memory/memory.go | 103 -- .../internal/metrics/memory/memory_aix.go | 76 - .../internal/metrics/memory/memory_darwin.go | 134 -- .../internal/metrics/memory/memory_freebsd.go | 91 -- .../internal/metrics/memory/memory_helpers.go | 80 - .../internal/metrics/memory/memory_linux.go | 88 -- .../internal/metrics/memory/memory_openbsd.go | 237 --- .../internal/metrics/memory/memory_test.go | 125 -- .../internal/metrics/memory/memory_windows.go | 52 - .../metrics/memory/oldkern/proc/meminfo | 48 - metricbeat/module/linux/memory/data.go | 12 +- metricbeat/module/system/core/config.go | 2 +- metricbeat/module/system/core/core.go | 2 +- metricbeat/module/system/cpu/config.go | 2 +- metricbeat/module/system/cpu/cpu.go | 2 +- metricbeat/module/system/memory/memory.go | 4 +- 36 files changed, 230 insertions(+), 4700 deletions(-) delete mode 100644 metricbeat/internal/metrics/cpu/doc.go delete mode 100644 metricbeat/internal/metrics/cpu/metrics.go delete mode 100644 metricbeat/internal/metrics/cpu/metrics_aix.go delete mode 100644 metricbeat/internal/metrics/cpu/metrics_darwin.go delete mode 100644 metricbeat/internal/metrics/cpu/metrics_freebsd.go delete mode 100644 metricbeat/internal/metrics/cpu/metrics_linux.go delete mode 100644 metricbeat/internal/metrics/cpu/metrics_linux_test.go delete mode 100644 metricbeat/internal/metrics/cpu/metrics_openbsd.go delete mode 100644 metricbeat/internal/metrics/cpu/metrics_procfs_common.go delete mode 100644 metricbeat/internal/metrics/cpu/metrics_test.go delete mode 100644 metricbeat/internal/metrics/cpu/metrics_windows.go delete mode 100644 metricbeat/internal/metrics/cpu/testdata/cpuinfo delete mode 100644 metricbeat/internal/metrics/cpu/testdata/cpuinfo-armv7 delete mode 100644 metricbeat/internal/metrics/cpu/testdata/cpuinfo-armv7.expected.json delete mode 100644 metricbeat/internal/metrics/cpu/testdata/cpuinfo-quad-socket delete mode 100644 metricbeat/internal/metrics/cpu/testdata/cpuinfo-quad-socket.expected.json delete mode 100644 metricbeat/internal/metrics/cpu/testdata/cpuinfo.expected.json delete mode 100644 metricbeat/internal/metrics/memory/memory.go delete mode 100644 metricbeat/internal/metrics/memory/memory_aix.go delete mode 100644 metricbeat/internal/metrics/memory/memory_darwin.go delete mode 100644 metricbeat/internal/metrics/memory/memory_freebsd.go delete mode 100644 metricbeat/internal/metrics/memory/memory_helpers.go delete mode 100644 metricbeat/internal/metrics/memory/memory_linux.go delete mode 100644 metricbeat/internal/metrics/memory/memory_openbsd.go delete mode 100644 metricbeat/internal/metrics/memory/memory_test.go delete mode 100644 metricbeat/internal/metrics/memory/memory_windows.go delete mode 100644 metricbeat/internal/metrics/memory/oldkern/proc/meminfo diff --git a/NOTICE.txt b/NOTICE.txt index c3cdb78e7d0..c820be6e050 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -6492,11 +6492,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-l -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-system-metrics -Version: v0.3.1 +Version: v0.4.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.3.1/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.4.1/LICENSE.txt: Apache License Version 2.0, January 2004 @@ -8453,218 +8453,6 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-ucfg@v0.8.5/ limitations under the License. --------------------------------------------------------------------------------- -Dependency : github.com/elastic/go-windows -Version: v1.0.1 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-windows@v1.0.1/LICENSE.txt: - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -------------------------------------------------------------------------------- Dependency : github.com/elastic/gosigar Version: v0.14.2 @@ -25226,6 +25014,218 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/elastic/go-windows +Version: v1.0.1 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-windows@v1.0.1/LICENSE.txt: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/elazarl/goproxy Version: v0.0.0-20180725130230-947c36da3153 diff --git a/go.mod b/go.mod index 38c79f2ad87..ca89756336d 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,7 @@ require ( github.com/elastic/go-structform v0.0.9 github.com/elastic/go-sysinfo v1.8.0 github.com/elastic/go-ucfg v0.8.5 - github.com/elastic/go-windows v1.0.1 + github.com/elastic/go-windows v1.0.1 // indirect github.com/elastic/gosigar v0.14.2 github.com/fatih/color v1.13.0 github.com/fearful-symmetry/gorapl v0.0.4 @@ -162,7 +162,7 @@ require ( github.com/elastic/bayeux v1.0.5 github.com/elastic/elastic-agent-autodiscover v0.1.1 github.com/elastic/elastic-agent-libs v0.2.5 - github.com/elastic/elastic-agent-system-metrics v0.3.1 + github.com/elastic/elastic-agent-system-metrics v0.4.1 github.com/shirou/gopsutil/v3 v3.21.12 go.elastic.co/apm/module/apmelasticsearch/v2 v2.0.0 go.elastic.co/apm/module/apmhttp/v2 v2.0.0 diff --git a/go.sum b/go.sum index e8ac417dccc..6f03d22b429 100644 --- a/go.sum +++ b/go.sum @@ -542,8 +542,8 @@ github.com/elastic/elastic-agent-libs v0.0.0-20220303160015-5b4e674da3dd/go.mod github.com/elastic/elastic-agent-libs v0.2.2/go.mod h1:1xDLBhIqBIjhJ7lr2s+xRFFkQHpitSp8q2zzv1Dqg+s= github.com/elastic/elastic-agent-libs v0.2.5 h1:8+sYCW/kkWQe5KegGLMYYT3ELXUwibMcjMh+IMr2LRE= github.com/elastic/elastic-agent-libs v0.2.5/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= -github.com/elastic/elastic-agent-system-metrics v0.3.1 h1:WXdDyIaBr9zIJoyvFNzgZbcFiEnmtaKfazHPJR4DJOM= -github.com/elastic/elastic-agent-system-metrics v0.3.1/go.mod h1:RIYhJOS7mUeyIthfOSqmmbEILYSzaDWLi5zQ70bQo+o= +github.com/elastic/elastic-agent-system-metrics v0.4.1 h1:1bKgU0Y2F4PBLSCX2LmJbRd4wWoq5DOvXc9ysuXBVpI= +github.com/elastic/elastic-agent-system-metrics v0.4.1/go.mod h1:tF/f9Off38nfzTZHIVQ++FkXrDm9keFhFpJ+3pQ00iI= github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4= github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270/go.mod h1:Msl1pdboCbArMF/nSCDUXgQuWTeoMmE/z8607X+k7ng= github.com/elastic/go-concert v0.2.0 h1:GAQrhRVXprnNjtvTP9pWJ1d4ToEA4cU5ci7TwTa20xg= diff --git a/metricbeat/internal/metrics/cpu/doc.go b/metricbeat/internal/metrics/cpu/doc.go deleted file mode 100644 index ce8f92d5dc6..00000000000 --- a/metricbeat/internal/metrics/cpu/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package cpu diff --git a/metricbeat/internal/metrics/cpu/metrics.go b/metricbeat/internal/metrics/cpu/metrics.go deleted file mode 100644 index f18a0e0eb1f..00000000000 --- a/metricbeat/internal/metrics/cpu/metrics.go +++ /dev/null @@ -1,245 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package cpu - -import ( - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/opt" - "github.com/elastic/elastic-agent-libs/mapstr" - "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" -) - -// CPU manages the CPU metrics from /proc/stat -// If a given metric isn't available on a given platform, -// The value will be null. All methods that use these fields -// should assume that any value can be null. -// The values are in "ticks", which translates to milliseconds of CPU time -type CPU struct { - User opt.Uint `struct:"user,omitempty"` - Sys opt.Uint `struct:"system,omitempty"` - Idle opt.Uint `struct:"idle,omitempty"` - Nice opt.Uint `struct:"nice,omitempty"` // Linux, Darwin, BSD - Irq opt.Uint `struct:"irq,omitempty"` // Linux and openbsd - Wait opt.Uint `struct:"iowait,omitempty"` // Linux and AIX - SoftIrq opt.Uint `struct:"softirq,omitempty"` // Linux only - Stolen opt.Uint `struct:"steal,omitempty"` // Linux only -} - -// MetricOpts defines the fields that are passed along to the formatted output -type MetricOpts struct { - Ticks bool - Percentages bool - NormalizedPercentages bool -} - -// CPUInfo manages the CPU information from /proc/cpuinfo -// If a given value isn't available on a given platformn -// the value will be the type's zero-value -type CPUInfo struct { - ModelName string - ModelNumber string - Mhz float64 - PhysicalID int - CoreID int -} - -// CPUMetrics carries global and per-core CPU metrics -type CPUMetrics struct { - totals CPU - - // list carries the same data, broken down by CPU - list []CPU - - // CPUInfo carries some data from /proc/cpuinfo - CPUInfo []CPUInfo -} - -// Total returns the total CPU time in ticks as scraped by the API -func (cpu CPU) Total() uint64 { - // it's generally safe to blindly sum these up, - // As we're just trying to get a total of all CPU time. - return opt.SumOptUint(cpu.User, cpu.Nice, cpu.Sys, cpu.Idle, cpu.Wait, cpu.Irq, cpu.SoftIrq, cpu.Stolen) -} - -/* -The below code implements a "metrics tracker" that gives us the ability to -calculate CPU percentages, as we average usage across a time period. -*/ - -// Monitor is used to monitor the overall CPU usage of the system over time. -type Monitor struct { - lastSample CPUMetrics - Hostfs resolve.Resolver -} - -// New returns a new CPU metrics monitor -// Hostfs is only relevant on linux and freebsd. -func New(hostfs resolve.Resolver) *Monitor { - return &Monitor{Hostfs: hostfs} -} - -// Fetch collects a new sample of the CPU usage metrics. -// This will overwrite the currently stored samples. -func (m *Monitor) Fetch() (Metrics, error) { - metric, err := Get(m.Hostfs) - if err != nil { - return Metrics{}, errors.Wrap(err, "Error fetching CPU metrics") - } - - oldLastSample := m.lastSample - m.lastSample = metric - - return Metrics{previousSample: oldLastSample.totals, currentSample: metric.totals, count: len(metric.list), isTotals: true}, nil -} - -// FetchCores collects a new sample of CPU usage metrics per-core -// This will overwrite the currently stored samples. -func (m *Monitor) FetchCores() ([]Metrics, error) { - metric, err := Get(m.Hostfs) - if err != nil { - return nil, errors.Wrap(err, "Error fetching CPU metrics") - } - - coreMetrics := make([]Metrics, len(metric.list)) - for i := 0; i < len(metric.list); i++ { - lastMetric := CPU{} - // Count of CPUs can change - if len(m.lastSample.list) > i { - lastMetric = m.lastSample.list[i] - } - - coreMetrics[i] = Metrics{ - currentSample: metric.list[i], - previousSample: lastMetric, - isTotals: false, - } - - // Only add CPUInfo metric if it's available - // TODO: Remove this if statement once CPUInfo is supported - // by all systems - if len(metric.CPUInfo) != 0 { - coreMetrics[i].cpuInfo = metric.CPUInfo[i] - } - } - m.lastSample = metric - return coreMetrics, nil -} - -// Metrics stores the current and the last sample collected by a Beat. -type Metrics struct { - previousSample CPU - currentSample CPU - count int - cpuInfo CPUInfo - isTotals bool -} - -// Format returns the final MapStr data object for the metrics. -func (metric Metrics) Format(opts MetricOpts) (mapstr.M, error) { - - timeDelta := metric.currentSample.Total() - metric.previousSample.Total() - if timeDelta <= 0 { - return nil, errors.New("Previous sample is newer than current sample") - } - normCPU := metric.count - if !metric.isTotals { - normCPU = 1 - } - - formattedMetrics := mapstr.M{} - - reportOptMetric := func(name string, current, previous opt.Uint, norm int) { - if !current.IsZero() { - formattedMetrics[name] = fillMetric(opts, current, previous, timeDelta, norm) - } - } - - if opts.Percentages { - formattedMetrics.Put("total.pct", createTotal(metric.previousSample, metric.currentSample, timeDelta, normCPU)) - } - if opts.NormalizedPercentages { - formattedMetrics.Put("total.norm.pct", createTotal(metric.previousSample, metric.currentSample, timeDelta, 1)) - } - - // /proc/stat metrics - reportOptMetric("user", metric.currentSample.User, metric.previousSample.User, normCPU) - reportOptMetric("system", metric.currentSample.Sys, metric.previousSample.Sys, normCPU) - reportOptMetric("idle", metric.currentSample.Idle, metric.previousSample.Idle, normCPU) - reportOptMetric("nice", metric.currentSample.Nice, metric.previousSample.Nice, normCPU) - reportOptMetric("irq", metric.currentSample.Irq, metric.previousSample.Irq, normCPU) - reportOptMetric("iowait", metric.currentSample.Wait, metric.previousSample.Wait, normCPU) - reportOptMetric("softirq", metric.currentSample.SoftIrq, metric.previousSample.SoftIrq, normCPU) - reportOptMetric("steal", metric.currentSample.Stolen, metric.previousSample.Stolen, normCPU) - - // Only add CPU info metrics if we're returning information by core - // (isTotals is false) - if !metric.isTotals { - // Some platforms do not report those metrics, so metric.cpuInfo - // is empty, if that happens we do not add the empty metrics to the - // final event. - if metric.cpuInfo != (CPUInfo{}) { - // /proc/cpuinfo metrics - formattedMetrics["model_number"] = metric.cpuInfo.ModelNumber - formattedMetrics["model_name"] = metric.cpuInfo.ModelName - formattedMetrics["mhz"] = metric.cpuInfo.Mhz - formattedMetrics["core_id"] = metric.cpuInfo.CoreID - formattedMetrics["physical_id"] = metric.cpuInfo.PhysicalID - } - } - - return formattedMetrics, nil -} - -func createTotal(prev, cur CPU, timeDelta uint64, numCPU int) float64 { - idleTime := cpuMetricTimeDelta(prev.Idle, cur.Idle, timeDelta, numCPU) - // Subtract wait time from total - // Wait time is not counted from the total as per #7627. - if !cur.Wait.IsZero() { - idleTime = idleTime + cpuMetricTimeDelta(prev.Wait, cur.Wait, timeDelta, numCPU) - } - return common.Round(float64(numCPU)-idleTime, common.DefaultDecimalPlacesCount) -} - -func fillMetric(opts MetricOpts, cur, prev opt.Uint, timeDelta uint64, numCPU int) mapstr.M { - event := mapstr.M{} - if opts.Ticks { - event.Put("ticks", cur.ValueOr(0)) - } - if opts.Percentages { - event.Put("pct", cpuMetricTimeDelta(prev, cur, timeDelta, numCPU)) - } - if opts.NormalizedPercentages { - event.Put("norm.pct", cpuMetricTimeDelta(prev, cur, timeDelta, 1)) - } - - return event -} - -// CPUCount returns the count of CPUs. When available, use this instead of runtime.NumCPU() -func (m *Metrics) CPUCount() int { - return m.count -} - -// cpuMetricTimeDelta is a helper used by fillTicks to calculate the delta between two CPU tick values -func cpuMetricTimeDelta(prev, current opt.Uint, timeDelta uint64, numCPU int) float64 { - cpuDelta := int64(current.ValueOr(0) - prev.ValueOr(0)) - pct := float64(cpuDelta) / float64(timeDelta) - return common.Round(pct*float64(numCPU), common.DefaultDecimalPlacesCount) -} diff --git a/metricbeat/internal/metrics/cpu/metrics_aix.go b/metricbeat/internal/metrics/cpu/metrics_aix.go deleted file mode 100644 index 628691cc449..00000000000 --- a/metricbeat/internal/metrics/cpu/metrics_aix.go +++ /dev/null @@ -1,129 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package cpu - -/* -#cgo LDFLAGS: -L/usr/lib -lperfstat - -#include -#include -#include -#include -#include -#include -#include -#include - -*/ -import "C" - -import ( - "fmt" - "os" - - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/opt" - "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" -) - -func init() { - // sysconf(_SC_CLK_TCK) returns the number of ticks by second. - system.ticks = uint64(C.sysconf(C._SC_CLK_TCK)) - system.pagesize = uint64(os.Getpagesize()) -} - -var system struct { - ticks uint64 - btime uint64 - pagesize uint64 -} - -func tick2msec(val uint64) uint64 { - ticks := val * 1000 / system.ticks - return ticks -} - -// Get returns a metrics object for CPU data -func Get(_ resolve.Resolver) (CPUMetrics, error) { - - totals, err := getCPUTotals() - if err != nil { - return CPUMetrics{}, errors.Wrap(err, "error getting CPU totals") - } - - list, err := getPerCPUMetrics() - if err != nil { - return CPUMetrics{}, errors.Wrap(err, "error getting per-cpu metrics") - } - - return CPUMetrics{totals: totals, list: list}, nil - -} - -// getCPUTotals gets the global CPU stats -func getCPUTotals() (CPU, error) { - cpudata := C.perfstat_cpu_total_t{} - - if _, err := C.perfstat_cpu_total(nil, &cpudata, C.sizeof_perfstat_cpu_total_t, 1); err != nil { - return CPU{}, fmt.Errorf("perfstat_cpu_total: %s", err) - } - - totals := CPU{} - totals.User = opt.UintWith((uint64(cpudata.user))) - totals.Sys = opt.UintWith(tick2msec(uint64(cpudata.sys))) - totals.Idle = opt.UintWith(tick2msec(uint64(cpudata.idle))) - totals.Wait = opt.UintWith(tick2msec(uint64(cpudata.wait))) - - return totals, nil -} - -// getPerCPUMetrics gets per-CPU metrics -func getPerCPUMetrics() ([]CPU, error) { - cpudata := C.perfstat_cpu_t{} - id := C.perfstat_id_t{} - id.name[0] = 0 - - // Retrieve the number of cpu using perfstat_cpu - capacity, err := C.perfstat_cpu(nil, nil, C.sizeof_perfstat_cpu_t, 0) - if err != nil { - return nil, fmt.Errorf("error while retrieving CPU number: %s", err) - } - list := make([]CPU, 0, capacity) - - for { - if _, err := C.perfstat_cpu(&id, &cpudata, C.sizeof_perfstat_cpu_t, 1); err != nil { - return nil, fmt.Errorf("perfstat_cpu: %s", err) - } - - cpu := CPU{} - cpu.User = opt.UintWith(tick2msec(uint64(cpudata.user))) - cpu.Sys = opt.UintWith(tick2msec(uint64(cpudata.sys))) - cpu.Idle = opt.UintWith(tick2msec(uint64(cpudata.idle))) - cpu.Wait = opt.UintWith(tick2msec(uint64(cpudata.wait))) - - list = append(list, cpu) - - if id.name[0] == 0 { - break - } - } - - return list, nil - -} diff --git a/metricbeat/internal/metrics/cpu/metrics_darwin.go b/metricbeat/internal/metrics/cpu/metrics_darwin.go deleted file mode 100644 index b09d06b11cf..00000000000 --- a/metricbeat/internal/metrics/cpu/metrics_darwin.go +++ /dev/null @@ -1,58 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package cpu - -import ( - "github.com/pkg/errors" - "github.com/shirou/gopsutil/v3/cpu" - - "github.com/elastic/beats/v7/libbeat/opt" - "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" -) - -// Get is the Darwin implementation of Get -func Get(_ resolve.Resolver) (CPUMetrics, error) { - // We're using the gopsutil library here. - // The code used by both gosigar and go-sysinfo appears to be - // the same code as gopsutil, including copy-pasted comments. - // For the sake of just reducing complexity, - sum, err := cpu.Times(false) - if err != nil { - return CPUMetrics{}, errors.Wrap(err, "error fetching CPU summary data") - } - perCPU, err := cpu.Times(true) - if err != nil { - return CPUMetrics{}, errors.Wrap(err, "error fetching per-CPU data") - } - - cpulist := []CPU{} - for _, cpu := range perCPU { - cpulist = append(cpulist, fillCPU(cpu)) - } - return CPUMetrics{totals: fillCPU(sum[0]), list: cpulist}, nil -} - -func fillCPU(raw cpu.TimesStat) CPU { - totalCPU := CPU{ - Sys: opt.UintWith(uint64(raw.System)), - User: opt.UintWith(uint64(raw.User)), - Idle: opt.UintWith(uint64(raw.Idle)), - Nice: opt.UintWith(uint64(raw.Nice)), - } - return totalCPU -} diff --git a/metricbeat/internal/metrics/cpu/metrics_freebsd.go b/metricbeat/internal/metrics/cpu/metrics_freebsd.go deleted file mode 100644 index e0a84c47fc7..00000000000 --- a/metricbeat/internal/metrics/cpu/metrics_freebsd.go +++ /dev/null @@ -1,60 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package cpu - -import ( - "bufio" - "fmt" - "strings" - - "github.com/joeshaw/multierror" - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/opt" -) - -func scanStatFile(scanner *bufio.Scanner) (CPUMetrics, error) { - cpuData, err := statScanner(scanner, parseCPULine) - if err != nil { - return CPUMetrics{}, errors.Wrap(err, "error scanning stat file") - } - return cpuData, nil -} - -func parseCPULine(line string) (CPU, error) { - cpuData := CPU{} - fields := strings.Fields(line) - var errs multierror.Errors - - tryParseUint := func(name, field string) (v opt.Uint) { - u, err := touint(field) - if err != nil { - errs = append(errs, fmt.Errorf("failed to parse %v: %s", name, field)) - } else { - v = opt.UintWith(u) - } - return v - } - - cpuData.User = tryParseUint("user", fields[1]) - cpuData.Nice = tryParseUint("nice", fields[2]) - cpuData.Sys = tryParseUint("sys", fields[3]) - cpuData.Idle = tryParseUint("idle", fields[4]) - - return cpuData, errs.Err() -} diff --git a/metricbeat/internal/metrics/cpu/metrics_linux.go b/metricbeat/internal/metrics/cpu/metrics_linux.go deleted file mode 100644 index bbf4da9cbcc..00000000000 --- a/metricbeat/internal/metrics/cpu/metrics_linux.go +++ /dev/null @@ -1,69 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package cpu - -import ( - "bufio" - "fmt" - "strings" - - "github.com/joeshaw/multierror" - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/opt" -) - -func scanStatFile(scanner *bufio.Scanner) (CPUMetrics, error) { - cpuData, err := statScanner(scanner, parseCPULine) - if err != nil { - return CPUMetrics{}, errors.Wrap(err, "error scanning stat file") - } - return cpuData, nil -} - -func parseCPULine(line string) (CPU, error) { - - var errs multierror.Errors - tryParseUint := func(name, field string) (v opt.Uint) { - u, err := touint(field) - if err != nil { - errs = append(errs, fmt.Errorf("failed to parse %v: %s", name, field)) - } else { - v = opt.UintWith(u) - } - return v - } - - cpuData := CPU{} - fields := strings.Fields(line) - - cpuData.User = tryParseUint("user", fields[1]) - cpuData.Nice = tryParseUint("nice", fields[2]) - cpuData.Sys = tryParseUint("sys", fields[3]) - cpuData.Idle = tryParseUint("idle", fields[4]) - cpuData.Wait = tryParseUint("wait", fields[5]) - cpuData.Irq = tryParseUint("irq", fields[6]) - cpuData.SoftIrq = tryParseUint("softirq", fields[7]) - cpuData.Stolen = tryParseUint("stolen", fields[8]) - - return cpuData, errs.Err() -} - -func scanCPUInfoFile(scanner *bufio.Scanner) ([]CPUInfo, error) { - return cpuinfoScanner(scanner) -} diff --git a/metricbeat/internal/metrics/cpu/metrics_linux_test.go b/metricbeat/internal/metrics/cpu/metrics_linux_test.go deleted file mode 100644 index ce049a63129..00000000000 --- a/metricbeat/internal/metrics/cpu/metrics_linux_test.go +++ /dev/null @@ -1,97 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package cpu - -import ( - "bufio" - "encoding/json" - "os" - "path/filepath" - "strconv" - "testing" - - "github.com/stretchr/testify/assert" -) - -// TestScanCPUInfoFile tests the parsing of `/proc/cpuinfo` for different -// system/CPU configurations. The lscpu GitHub contains a nice set of -// test files: https://github.com/util-linux/util-linux/tree/master/tests/ts/lscpu/dumps -func TestScanCPUInfoFile(t *testing.T) { - testCases := []string{ - "cpuinfo", - "cpuinfo-quad-socket", - // Source: https://github.com/util-linux/util-linux/blob/master/tests/ts/lscpu/dumps/armv7.tar.gz - "cpuinfo-armv7", - } - for _, tc := range testCases { - t.Run(tc, func(t *testing.T) { - sourceFd, err := os.Open(filepath.Join("testdata", tc)) - if err != nil { - t.Fatalf("cannot open test file: %s", err) - } - defer sourceFd.Close() - - scanner := bufio.NewScanner(sourceFd) - cpuInfo, err := scanCPUInfoFile(scanner) - if err != nil { - t.Fatalf("scanCPUInfoFile error: %s", err) - } - - // Ignoring the error, because if there is any parsing error, generateGoldenFile - // will be false, making the test to run as expected - if generateGoldenFile, _ := strconv.ParseBool(os.Getenv("GENERATE")); generateGoldenFile { - t.Logf("generating golden files for test: %s", t.Name()) - scanCPUInfoFileGenGoldenFile(t, cpuInfo, tc) - return - } - - expectedFd, err := os.Open(filepath.Join("testdata", tc+".expected.json")) - if err != nil { - t.Fatalf("cannot open test expectation file: %s", err) - } - defer expectedFd.Close() - - expected := []CPUInfo{} - if err := json.NewDecoder(expectedFd).Decode(&expected); err != nil { - t.Fatalf("cannot decode goldenfile data: %s", err) - } - - assert.Equal(t, expected, cpuInfo) - }) - } -} - -func scanCPUInfoFileGenGoldenFile(t *testing.T, data []CPUInfo, name string) { - jsonData, err := json.MarshalIndent(data, "", " ") - if err != nil { - t.Fatalf("cannot marshal data into JSON: %s", err) - } - - // Add a line break at the end - jsonData = append(jsonData, '\n') - - expectedFd, err := os.Create(filepath.Join("testdata", name+".expected.json")) - if err != nil { - t.Fatalf("cannot open/create test expectation file: %s", err) - } - defer expectedFd.Close() - - if _, err := expectedFd.Write(jsonData); err != nil { - t.Fatalf("cannot write data to goldenfile: %s", err) - } -} diff --git a/metricbeat/internal/metrics/cpu/metrics_openbsd.go b/metricbeat/internal/metrics/cpu/metrics_openbsd.go deleted file mode 100644 index 7111937c9f9..00000000000 --- a/metricbeat/internal/metrics/cpu/metrics_openbsd.go +++ /dev/null @@ -1,136 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//go:build openbsd -// +build openbsd - -package cpu - -/* -#include -#include -#include -#include -#include -#include -#include -#include -*/ -import "C" - -import ( - "syscall" - "unsafe" - - "github.com/elastic/beats/v7/libbeat/opt" - "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" -) - -// Get is the OpenBSD implementation of get -func Get(_ resolve.Resolver) (CPUMetrics, error) { - - // see man 2 sysctl - loadGlobal := [C.CPUSTATES]C.long{ - C.CP_USER, - C.CP_NICE, - C.CP_SYS, - C.CP_INTR, - C.CP_IDLE, - } - - // first, fetch global CPU data - err := sysctlGetCPUTimes(0, 0, &loadGlobal) - if err != nil { - return CPUMetrics{}, err - } - self := CPU{} - self.User = opt.UintWith(loadGlobal[0]) - self.Nice = opt.UintWith(loadGlobal[1]) - self.Sys = opt.UintWith(loadGlobal[2]) - self.Irq = opt.UintWith(loadGlobal[3]) - self.Idle = opt.UintWith(loadGlobal[4]) - // Get count of available CPUs - ncpuMIB := [2]int32{C.CTL_HW, C.HW_NCPU} - callSize := uintptr(0) - var ncpu int - // The first call nulls out the retun pointer, so we instead fetch the expected size of expected return value. - _, _, errno := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&ncpuMIB[0])), 2, 0, uintptr(unsafe.Pointer(&callSize)), 0, 0) - - if errno != 0 || callSize == 0 { - return CPUMetrics{}, errno - } - - // Populate the cpu count - _, _, errno = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&ncpuMIB[0])), 2, uintptr(unsafe.Pointer(&ncpu)), uintptr(unsafe.Pointer(&callSize)), 0, 0) - - if errno != 0 || callSize == 0 { - return CPUMetrics{}, errno - } - - loadPerCPU := [C.CPUSTATES]C.long{ - C.CP_USER, - C.CP_NICE, - C.CP_SYS, - C.CP_INTR, - C.CP_IDLE, - } - - perCPU := make([]CPU, ncpu) - - // iterate over metrics for each CPU - for i := 0; i < ncpu; i++ { - sysctlGetCPUTimes(ncpu, i, &loadPerCPU) - perCPU[i].User = opt.UintWith(loadGlobal[0]) - perCPU[i].Nice = opt.UintWith(loadGlobal[1]) - perCPU[i].Sys = opt.UintWith(loadGlobal[2]) - perCPU[i].Irq = opt.UintWith(loadGlobal[3]) - perCPU[i].Idle = opt.UintWith(loadGlobal[4]) - } - - metrics := CPUMetrics{totals: self, list: perCPU} - - return metrics, nil -} - -// sysctlGetCPUTimes runs the CTL_KERN::KERN_CPTIME sysctl command against the host. -func sysctlGetCPUTimes(ncpu int, curcpu int, load *[C.CPUSTATES]C.long) error { - var mib []int32 - - // Use the correct mib based on the number of CPUs and fill out the - // current CPU number in case of SMP. (0 indexed cf. self.List) - if ncpu == 0 { - mib = []int32{C.CTL_KERN, C.KERN_CPTIME} - } else { - mib = []int32{C.CTL_KERN, C.KERN_CPTIME2, int32(curcpu)} - } - - len := len(mib) - - n := uintptr(0) - // First we determine how much memory we'll need to pass later on (via `n`) - _, _, errno := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), uintptr(len), 0, uintptr(unsafe.Pointer(&n)), 0, 0) - if errno != 0 || n == 0 { - return errno - } - - _, _, errno = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), uintptr(len), uintptr(unsafe.Pointer(load)), uintptr(unsafe.Pointer(&n)), 0, 0) - if errno != 0 || n == 0 { - return errno - } - - return nil -} diff --git a/metricbeat/internal/metrics/cpu/metrics_procfs_common.go b/metricbeat/internal/metrics/cpu/metrics_procfs_common.go deleted file mode 100644 index a97fe9cc6bc..00000000000 --- a/metricbeat/internal/metrics/cpu/metrics_procfs_common.go +++ /dev/null @@ -1,157 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//go:build freebsd || linux -// +build freebsd linux - -package cpu - -import ( - "bufio" - "fmt" - "os" - "strconv" - "strings" - - "github.com/pkg/errors" - - "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" -) - -// Get returns a metrics object for CPU data -func Get(procfs resolve.Resolver) (CPUMetrics, error) { - path := procfs.ResolveHostFS("/proc/stat") - fd, err := os.Open(path) - defer fd.Close() - if err != nil { - return CPUMetrics{}, errors.Wrapf(err, "error opening file %s", path) - } - - metrics, err := scanStatFile(bufio.NewScanner(fd)) - if err != nil { - return CPUMetrics{}, fmt.Errorf("scanning stat file: %w", err) - } - - cpuInfoPath := procfs.ResolveHostFS("/proc/cpuinfo") - cpuInfoFd, err := os.Open(cpuInfoPath) - if err != nil { - return CPUMetrics{}, fmt.Errorf("opening '%s': %w", cpuInfoPath, err) - } - defer cpuInfoFd.Close() - - cpuInfo, err := scanCPUInfoFile(bufio.NewScanner(cpuInfoFd)) - metrics.CPUInfo = cpuInfo - - return metrics, err -} - -func cpuinfoScanner(scanner *bufio.Scanner) ([]CPUInfo, error) { - cpuInfos := []CPUInfo{} - current := CPUInfo{} - // On my tests the order the cores appear on /proc/cpuinfo - // is the same as on /proc/stats, this means it matches our - // current 'system.core.id' metric. This information - // is also the same as the 'processor' line on /proc/cpuinfo. - coreID := 0 - for scanner.Scan() { - line := scanner.Text() - split := strings.Split(line, ":") - if len(split) != 2 { - // A blank line its a separation between CPUs - // even the last CPU contains one blank line at the end - cpuInfos = append(cpuInfos, current) - current = CPUInfo{} - coreID++ - - continue - } - - k, v := split[0], split[1] - k = strings.TrimSpace(k) - v = strings.TrimSpace(v) - switch k { - case "model": - current.ModelNumber = v - case "model name": - current.ModelName = v - case "physical id": - id, err := strconv.Atoi(v) - if err != nil { - return []CPUInfo{}, fmt.Errorf("parsing physical ID: %w", err) - } - current.PhysicalID = id - case "core id": - id, err := strconv.Atoi(v) - if err != nil { - return []CPUInfo{}, fmt.Errorf("parsing core ID: %w", err) - } - current.CoreID = id - case "cpu MHz": - mhz, err := strconv.ParseFloat(v, 64) - if err != nil { - return []CPUInfo{}, fmt.Errorf("parsing CPU %d Mhz: %w", coreID, err) - } - current.Mhz = mhz - } - } - - return cpuInfos, nil -} - -// statScanner iterates through a /proc/stat entry, reading both the global lines and per-CPU lines, each time calling lineReader, which implements the OS-specific code for parsing individual lines -func statScanner(scanner *bufio.Scanner, lineReader func(string) (CPU, error)) (CPUMetrics, error) { - cpuData := CPUMetrics{} - var err error - - for scanner.Scan() { - text := scanner.Text() - // Check to see if this is the global CPU line - if isCPUGlobalLine(text) { - cpuData.totals, err = lineReader(text) - if err != nil { - return CPUMetrics{}, errors.Wrap(err, "error parsing global CPU line") - } - } - if isCPULine(text) { - perCPU, err := lineReader(text) - if err != nil { - return CPUMetrics{}, errors.Wrap(err, "error parsing CPU line") - } - cpuData.list = append(cpuData.list, perCPU) - - } - } - return cpuData, nil -} - -func isCPUGlobalLine(line string) bool { - if len(line) > 4 && line[0:4] == "cpu " { - return true - } - return false -} - -func isCPULine(line string) bool { - if len(line) > 3 && line[0:3] == "cpu" && line[3] != ' ' { - return true - } - return false -} - -func touint(val string) (uint64, error) { - return strconv.ParseUint(val, 10, 64) -} diff --git a/metricbeat/internal/metrics/cpu/metrics_test.go b/metricbeat/internal/metrics/cpu/metrics_test.go deleted file mode 100644 index 5658bedd989..00000000000 --- a/metricbeat/internal/metrics/cpu/metrics_test.go +++ /dev/null @@ -1,162 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package cpu - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/elastic/beats/v7/libbeat/opt" - "github.com/elastic/elastic-agent-libs/mapstr" - "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" -) - -func TestMonitorSample(t *testing.T) { - cpu := &Monitor{lastSample: CPUMetrics{}, Hostfs: resolve.NewTestResolver("")} - s, err := cpu.Fetch() - if err != nil { - t.Fatal(err) - } - metricOpts := MetricOpts{Percentages: true, NormalizedPercentages: true, Ticks: true} - evt, err := s.Format(metricOpts) - assert.NoError(t, err, "error in Format") - testPopulatedEvent(evt, t, true) -} - -func TestCoresMonitorSample(t *testing.T) { - - cpuMetrics, err := Get(resolve.NewTestResolver("")) - assert.NoError(t, err, "error in Get()") - - cores := &Monitor{lastSample: CPUMetrics{list: make([]CPU, len(cpuMetrics.list))}, Hostfs: resolve.NewTestResolver("")} - sample, err := cores.FetchCores() - if err != nil { - t.Fatal(err) - } - - for _, s := range sample { - evt := mapstr.M{} - metricOpts := MetricOpts{Percentages: true, Ticks: true} - evt, err := s.Format(metricOpts) - require.NoError(t, err, "error in Format") - testPopulatedEvent(evt, t, false) - } -} - -func testPopulatedEvent(evt mapstr.M, t *testing.T, norm bool) { - user, err := evt.GetValue("user.pct") - assert.NoError(t, err, "error getting user.pct") - system, err := evt.GetValue("system.pct") - assert.NoError(t, err, "error getting system.pct") - assert.True(t, user.(float64) > 0) - assert.True(t, system.(float64) > 0) - - if norm { - normUser, err := evt.GetValue("user.norm.pct") - assert.NoError(t, err, "error getting user.norm.pct") - assert.True(t, normUser.(float64) > 0) - normSystem, err := evt.GetValue("system.norm.pct") - assert.NoError(t, err, "error getting system.norm.pct") - assert.True(t, normSystem.(float64) > 0) - assert.True(t, normUser.(float64) <= 100) - assert.True(t, normSystem.(float64) <= 100) - - assert.True(t, user.(float64) > normUser.(float64)) - assert.True(t, system.(float64) > normSystem.(float64)) - } - - userTicks, err := evt.GetValue("user.ticks") - assert.NoError(t, err, "error getting user.ticks") - assert.True(t, userTicks.(uint64) > 0) - systemTicks, err := evt.GetValue("system.ticks") - assert.NoError(t, err, "error getting system.ticks") - assert.True(t, systemTicks.(uint64) > 0) -} - -// TestMetricsRounding tests that the returned percentages are rounded to -// four decimal places. -func TestMetricsRounding(t *testing.T) { - - sample := Metrics{ - previousSample: CPU{ - User: opt.UintWith(10855311), - Sys: opt.UintWith(2021040), - Idle: opt.UintWith(17657874), - }, - currentSample: CPU{ - User: opt.UintWith(10855693), - Sys: opt.UintWith(2021058), - Idle: opt.UintWith(17657876), - }, - } - - evt, err := sample.Format(MetricOpts{NormalizedPercentages: true}) - normUser, err := evt.GetValue("user.norm.pct") - assert.NoError(t, err, "error getting user.norm.pct") - normSystem, err := evt.GetValue("system.norm.pct") - assert.NoError(t, err, "error getting system.norm.pct") - - assert.Equal(t, normUser.(float64), 0.9502) - assert.Equal(t, normSystem.(float64), 0.0448) -} - -// TestMetricsPercentages tests that Metrics returns the correct -// percentages and normalized percentages. -func TestMetricsPercentages(t *testing.T) { - numCores := 10 - // This test simulates 30% user and 70% system (normalized), or 3% and 7% - // respectively when there are 10 CPUs. - const userTest, systemTest = 30., 70. - - s0 := CPU{ - User: opt.UintWith(10000000), - Sys: opt.UintWith(10000000), - Idle: opt.UintWith(20000000), - Nice: opt.UintWith(0), - } - s1 := CPU{ - User: opt.UintWith(s0.User.ValueOr(0) + uint64(userTest)), - Sys: opt.UintWith(s0.Sys.ValueOr(0) + uint64(systemTest)), - Idle: s0.Idle, - Nice: opt.UintWith(0), - } - sample := Metrics{ - count: numCores, - isTotals: true, - previousSample: s0, - currentSample: s1, - } - - evt, err := sample.Format(MetricOpts{NormalizedPercentages: true, Percentages: true}) - assert.NoError(t, err, "error in Format") - - user, err := evt.GetValue("user.norm.pct") - assert.NoError(t, err, "error getting user.norm.pct") - system, err := evt.GetValue("system.norm.pct") - assert.NoError(t, err, "error getting system.norm.pct") - idle, err := evt.GetValue("idle.norm.pct") - assert.NoError(t, err, "error getting idle.norm.pct") - total, err := evt.GetValue("total.norm.pct") - assert.NoError(t, err, "error getting total.norm.pct") - assert.EqualValues(t, .3, user.(float64)) - assert.EqualValues(t, .7, system.(float64)) - assert.EqualValues(t, .0, idle.(float64)) - assert.EqualValues(t, 1., total.(float64)) -} diff --git a/metricbeat/internal/metrics/cpu/metrics_windows.go b/metricbeat/internal/metrics/cpu/metrics_windows.go deleted file mode 100644 index acc90b097fd..00000000000 --- a/metricbeat/internal/metrics/cpu/metrics_windows.go +++ /dev/null @@ -1,69 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -/* -For testing via the win2012 vagrant box: -vagrant winrm -s cmd -e -c "cd C:\\Gopath\src\\github.com\\elastic\\beats\\metricbeat\\module\\system\\cpu; go test -v -tags=integration -run TestFetch" win2012 -*/ - -package cpu - -import ( - "time" - - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/opt" - "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" - "github.com/elastic/gosigar/sys/windows" -) - -// Get fetches Windows CPU system times -func Get(_ resolve.Resolver) (CPUMetrics, error) { - idle, kernel, user, err := windows.GetSystemTimes() - if err != nil { - return CPUMetrics{}, errors.Wrap(err, "GetSystemTimes failed") - } - - globalMetrics := CPUMetrics{} - //convert from duration to ticks - idleMetric := uint64(idle / time.Millisecond) - sysMetric := uint64(kernel / time.Millisecond) - userMetrics := uint64(user / time.Millisecond) - globalMetrics.totals.Idle = opt.UintWith(idleMetric) - globalMetrics.totals.Sys = opt.UintWith(sysMetric) - globalMetrics.totals.User = opt.UintWith(userMetrics) - - // get per-cpu data - cpus, err := windows.NtQuerySystemProcessorPerformanceInformation() - if err != nil { - return CPUMetrics{}, errors.Wrap(err, "NtQuerySystemProcessorPerformanceInformation failed") - } - globalMetrics.list = make([]CPU, 0, len(cpus)) - for _, cpu := range cpus { - idleMetric := uint64(cpu.IdleTime / time.Millisecond) - sysMetric := uint64(cpu.KernelTime / time.Millisecond) - userMetrics := uint64(cpu.UserTime / time.Millisecond) - globalMetrics.list = append(globalMetrics.list, CPU{ - Idle: opt.UintWith(idleMetric), - Sys: opt.UintWith(sysMetric), - User: opt.UintWith(userMetrics), - }) - } - - return globalMetrics, nil -} diff --git a/metricbeat/internal/metrics/cpu/testdata/cpuinfo b/metricbeat/internal/metrics/cpu/testdata/cpuinfo deleted file mode 100644 index 25fc171e9f8..00000000000 --- a/metricbeat/internal/metrics/cpu/testdata/cpuinfo +++ /dev/null @@ -1,448 +0,0 @@ -processor : 0 -vendor_id : GenuineIntel -cpu family : 6 -model : 165 -model name : Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz -stepping : 2 -microcode : 0xf0 -cpu MHz : 2400.000 -cache size : 16384 KB -physical id : 0 -siblings : 16 -core id : 0 -cpu cores : 8 -apicid : 0 -initial apicid : 0 -fpu : yes -fpu_exception : yes -cpuid level : 22 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust sgx bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp pku ospke sgx_lc md_clear flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple shadow_vmcs pml ept_mode_based_exec -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs itlb_multihit -bogomips : 4801.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 1 -vendor_id : GenuineIntel -cpu family : 6 -model : 165 -model name : Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz -stepping : 2 -microcode : 0xf0 -cpu MHz : 2400.000 -cache size : 16384 KB -physical id : 0 -siblings : 16 -core id : 1 -cpu cores : 8 -apicid : 2 -initial apicid : 2 -fpu : yes -fpu_exception : yes -cpuid level : 22 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust sgx bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp pku ospke sgx_lc md_clear flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple shadow_vmcs pml ept_mode_based_exec -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs itlb_multihit -bogomips : 4801.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 2 -vendor_id : GenuineIntel -cpu family : 6 -model : 165 -model name : Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz -stepping : 2 -microcode : 0xf0 -cpu MHz : 2400.000 -cache size : 16384 KB -physical id : 0 -siblings : 16 -core id : 2 -cpu cores : 8 -apicid : 4 -initial apicid : 4 -fpu : yes -fpu_exception : yes -cpuid level : 22 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust sgx bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp pku ospke sgx_lc md_clear flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple shadow_vmcs pml ept_mode_based_exec -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs itlb_multihit -bogomips : 4801.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 3 -vendor_id : GenuineIntel -cpu family : 6 -model : 165 -model name : Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz -stepping : 2 -microcode : 0xf0 -cpu MHz : 2400.000 -cache size : 16384 KB -physical id : 0 -siblings : 16 -core id : 3 -cpu cores : 8 -apicid : 6 -initial apicid : 6 -fpu : yes -fpu_exception : yes -cpuid level : 22 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust sgx bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp pku ospke sgx_lc md_clear flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple shadow_vmcs pml ept_mode_based_exec -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs itlb_multihit -bogomips : 4801.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 4 -vendor_id : GenuineIntel -cpu family : 6 -model : 165 -model name : Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz -stepping : 2 -microcode : 0xf0 -cpu MHz : 2400.000 -cache size : 16384 KB -physical id : 0 -siblings : 16 -core id : 4 -cpu cores : 8 -apicid : 8 -initial apicid : 8 -fpu : yes -fpu_exception : yes -cpuid level : 22 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust sgx bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp pku ospke sgx_lc md_clear flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple shadow_vmcs pml ept_mode_based_exec -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs itlb_multihit -bogomips : 4801.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 5 -vendor_id : GenuineIntel -cpu family : 6 -model : 165 -model name : Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz -stepping : 2 -microcode : 0xf0 -cpu MHz : 2400.000 -cache size : 16384 KB -physical id : 0 -siblings : 16 -core id : 5 -cpu cores : 8 -apicid : 10 -initial apicid : 10 -fpu : yes -fpu_exception : yes -cpuid level : 22 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust sgx bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp pku ospke sgx_lc md_clear flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple shadow_vmcs pml ept_mode_based_exec -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs itlb_multihit -bogomips : 4801.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 6 -vendor_id : GenuineIntel -cpu family : 6 -model : 165 -model name : Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz -stepping : 2 -microcode : 0xf0 -cpu MHz : 2400.000 -cache size : 16384 KB -physical id : 0 -siblings : 16 -core id : 6 -cpu cores : 8 -apicid : 12 -initial apicid : 12 -fpu : yes -fpu_exception : yes -cpuid level : 22 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust sgx bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp pku ospke sgx_lc md_clear flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple shadow_vmcs pml ept_mode_based_exec -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs itlb_multihit -bogomips : 4801.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 7 -vendor_id : GenuineIntel -cpu family : 6 -model : 165 -model name : Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz -stepping : 2 -microcode : 0xf0 -cpu MHz : 2400.000 -cache size : 16384 KB -physical id : 0 -siblings : 16 -core id : 7 -cpu cores : 8 -apicid : 14 -initial apicid : 14 -fpu : yes -fpu_exception : yes -cpuid level : 22 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust sgx bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp pku ospke sgx_lc md_clear flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple shadow_vmcs pml ept_mode_based_exec -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs itlb_multihit -bogomips : 4801.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 8 -vendor_id : GenuineIntel -cpu family : 6 -model : 165 -model name : Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz -stepping : 2 -microcode : 0xf0 -cpu MHz : 2534.854 -cache size : 16384 KB -physical id : 0 -siblings : 16 -core id : 0 -cpu cores : 8 -apicid : 1 -initial apicid : 1 -fpu : yes -fpu_exception : yes -cpuid level : 22 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust sgx bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp pku ospke sgx_lc md_clear flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple shadow_vmcs pml ept_mode_based_exec -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs itlb_multihit -bogomips : 4801.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 9 -vendor_id : GenuineIntel -cpu family : 6 -model : 165 -model name : Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz -stepping : 2 -microcode : 0xf0 -cpu MHz : 2400.000 -cache size : 16384 KB -physical id : 0 -siblings : 16 -core id : 1 -cpu cores : 8 -apicid : 3 -initial apicid : 3 -fpu : yes -fpu_exception : yes -cpuid level : 22 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust sgx bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp pku ospke sgx_lc md_clear flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple shadow_vmcs pml ept_mode_based_exec -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs itlb_multihit -bogomips : 4801.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 10 -vendor_id : GenuineIntel -cpu family : 6 -model : 165 -model name : Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz -stepping : 2 -microcode : 0xf0 -cpu MHz : 2400.000 -cache size : 16384 KB -physical id : 0 -siblings : 16 -core id : 2 -cpu cores : 8 -apicid : 5 -initial apicid : 5 -fpu : yes -fpu_exception : yes -cpuid level : 22 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust sgx bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp pku ospke sgx_lc md_clear flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple shadow_vmcs pml ept_mode_based_exec -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs itlb_multihit -bogomips : 4801.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 11 -vendor_id : GenuineIntel -cpu family : 6 -model : 165 -model name : Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz -stepping : 2 -microcode : 0xf0 -cpu MHz : 2400.000 -cache size : 16384 KB -physical id : 0 -siblings : 16 -core id : 3 -cpu cores : 8 -apicid : 7 -initial apicid : 7 -fpu : yes -fpu_exception : yes -cpuid level : 22 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust sgx bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp pku ospke sgx_lc md_clear flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple shadow_vmcs pml ept_mode_based_exec -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs itlb_multihit -bogomips : 4801.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 12 -vendor_id : GenuineIntel -cpu family : 6 -model : 165 -model name : Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz -stepping : 2 -microcode : 0xf0 -cpu MHz : 2673.889 -cache size : 16384 KB -physical id : 0 -siblings : 16 -core id : 4 -cpu cores : 8 -apicid : 9 -initial apicid : 9 -fpu : yes -fpu_exception : yes -cpuid level : 22 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust sgx bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp pku ospke sgx_lc md_clear flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple shadow_vmcs pml ept_mode_based_exec -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs itlb_multihit -bogomips : 4801.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 13 -vendor_id : GenuineIntel -cpu family : 6 -model : 165 -model name : Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz -stepping : 2 -microcode : 0xf0 -cpu MHz : 2400.000 -cache size : 16384 KB -physical id : 0 -siblings : 16 -core id : 5 -cpu cores : 8 -apicid : 11 -initial apicid : 11 -fpu : yes -fpu_exception : yes -cpuid level : 22 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust sgx bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp pku ospke sgx_lc md_clear flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple shadow_vmcs pml ept_mode_based_exec -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs itlb_multihit -bogomips : 4801.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 14 -vendor_id : GenuineIntel -cpu family : 6 -model : 165 -model name : Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz -stepping : 2 -microcode : 0xf0 -cpu MHz : 2400.000 -cache size : 16384 KB -physical id : 0 -siblings : 16 -core id : 6 -cpu cores : 8 -apicid : 13 -initial apicid : 13 -fpu : yes -fpu_exception : yes -cpuid level : 22 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust sgx bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp pku ospke sgx_lc md_clear flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple shadow_vmcs pml ept_mode_based_exec -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs itlb_multihit -bogomips : 4801.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 15 -vendor_id : GenuineIntel -cpu family : 6 -model : 165 -model name : Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz -stepping : 2 -microcode : 0xf0 -cpu MHz : 2400.000 -cache size : 16384 KB -physical id : 0 -siblings : 16 -core id : 7 -cpu cores : 8 -apicid : 15 -initial apicid : 15 -fpu : yes -fpu_exception : yes -cpuid level : 22 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust sgx bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm ida arat pln pts hwp hwp_notify hwp_act_window hwp_epp pku ospke sgx_lc md_clear flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple shadow_vmcs pml ept_mode_based_exec -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs itlb_multihit -bogomips : 4801.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - diff --git a/metricbeat/internal/metrics/cpu/testdata/cpuinfo-armv7 b/metricbeat/internal/metrics/cpu/testdata/cpuinfo-armv7 deleted file mode 100644 index 6ef23caa60e..00000000000 --- a/metricbeat/internal/metrics/cpu/testdata/cpuinfo-armv7 +++ /dev/null @@ -1,17 +0,0 @@ -Processor : ARMv7 Processor rev 4 (v7l) -processor : 0 -BogoMIPS : 1694.10 - -processor : 1 -BogoMIPS : 1694.10 - -Features : swp half thumb fastmult vfp edsp thumbee neon vfpv3 tls vfpv4 idiva idivt -CPU implementer : 0x41 -CPU architecture: 7 -CPU variant : 0x0 -CPU part : 0xc0f -CPU revision : 4 - -Hardware : SAMSUNG EXYNOS5 (Flattened Device Tree) -Revision : 0000 -Serial : 0000000000000000 diff --git a/metricbeat/internal/metrics/cpu/testdata/cpuinfo-armv7.expected.json b/metricbeat/internal/metrics/cpu/testdata/cpuinfo-armv7.expected.json deleted file mode 100644 index ad93e83562d..00000000000 --- a/metricbeat/internal/metrics/cpu/testdata/cpuinfo-armv7.expected.json +++ /dev/null @@ -1,23 +0,0 @@ -[ - { - "ModelName": "", - "ModelNumber": "", - "Mhz": 0, - "PhysicalID": 0, - "CoreID": 0 - }, - { - "ModelName": "", - "ModelNumber": "", - "Mhz": 0, - "PhysicalID": 0, - "CoreID": 0 - }, - { - "ModelName": "", - "ModelNumber": "", - "Mhz": 0, - "PhysicalID": 0, - "CoreID": 0 - } -] diff --git a/metricbeat/internal/metrics/cpu/testdata/cpuinfo-quad-socket b/metricbeat/internal/metrics/cpu/testdata/cpuinfo-quad-socket deleted file mode 100644 index a1b35f97c12..00000000000 --- a/metricbeat/internal/metrics/cpu/testdata/cpuinfo-quad-socket +++ /dev/null @@ -1,1296 +0,0 @@ -processor : 0 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.033 -cache size : 12288 KB -physical id : 0 -siblings : 12 -core id : 0 -cpu cores : 6 -apicid : 0 -initial apicid : 0 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4400.07 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 1 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.027 -cache size : 12288 KB -physical id : 0 -siblings : 12 -core id : 1 -cpu cores : 6 -apicid : 2 -initial apicid : 2 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4400.07 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 2 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.038 -cache size : 12288 KB -physical id : 0 -siblings : 12 -core id : 2 -cpu cores : 6 -apicid : 4 -initial apicid : 4 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4400.07 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 3 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.516 -cache size : 12288 KB -physical id : 0 -siblings : 12 -core id : 3 -cpu cores : 6 -apicid : 6 -initial apicid : 6 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4400.07 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 4 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.040 -cache size : 12288 KB -physical id : 0 -siblings : 12 -core id : 4 -cpu cores : 6 -apicid : 8 -initial apicid : 8 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4400.07 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 5 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.048 -cache size : 12288 KB -physical id : 0 -siblings : 12 -core id : 5 -cpu cores : 6 -apicid : 10 -initial apicid : 10 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4400.07 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 6 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2199.974 -cache size : 12288 KB -physical id : 1 -siblings : 12 -core id : 0 -cpu cores : 6 -apicid : 32 -initial apicid : 32 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.38 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 7 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.037 -cache size : 12288 KB -physical id : 1 -siblings : 12 -core id : 1 -cpu cores : 6 -apicid : 34 -initial apicid : 34 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.38 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 8 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.025 -cache size : 12288 KB -physical id : 1 -siblings : 12 -core id : 2 -cpu cores : 6 -apicid : 36 -initial apicid : 36 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.38 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 9 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.010 -cache size : 12288 KB -physical id : 1 -siblings : 12 -core id : 3 -cpu cores : 6 -apicid : 38 -initial apicid : 38 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.38 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 10 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.038 -cache size : 12288 KB -physical id : 1 -siblings : 12 -core id : 4 -cpu cores : 6 -apicid : 40 -initial apicid : 40 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.38 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 11 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.042 -cache size : 12288 KB -physical id : 1 -siblings : 12 -core id : 5 -cpu cores : 6 -apicid : 42 -initial apicid : 42 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.38 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 12 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 1911.118 -cache size : 12288 KB -physical id : 2 -siblings : 12 -core id : 0 -cpu cores : 6 -apicid : 64 -initial apicid : 64 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.70 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 13 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 1479.972 -cache size : 12288 KB -physical id : 2 -siblings : 12 -core id : 1 -cpu cores : 6 -apicid : 66 -initial apicid : 66 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.70 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 14 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 1970.264 -cache size : 12288 KB -physical id : 2 -siblings : 12 -core id : 2 -cpu cores : 6 -apicid : 68 -initial apicid : 68 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.70 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 15 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 1996.097 -cache size : 12288 KB -physical id : 2 -siblings : 12 -core id : 3 -cpu cores : 6 -apicid : 70 -initial apicid : 70 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.70 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 16 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 1788.272 -cache size : 12288 KB -physical id : 2 -siblings : 12 -core id : 4 -cpu cores : 6 -apicid : 72 -initial apicid : 72 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.70 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 17 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2005.072 -cache size : 12288 KB -physical id : 2 -siblings : 12 -core id : 5 -cpu cores : 6 -apicid : 74 -initial apicid : 74 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.70 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 18 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 1658.085 -cache size : 12288 KB -physical id : 3 -siblings : 12 -core id : 0 -cpu cores : 6 -apicid : 96 -initial apicid : 96 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4404.05 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 19 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 1855.222 -cache size : 12288 KB -physical id : 3 -siblings : 12 -core id : 1 -cpu cores : 6 -apicid : 98 -initial apicid : 98 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4404.05 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 20 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2068.635 -cache size : 12288 KB -physical id : 3 -siblings : 12 -core id : 2 -cpu cores : 6 -apicid : 100 -initial apicid : 100 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4404.05 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 21 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 1806.426 -cache size : 12288 KB -physical id : 3 -siblings : 12 -core id : 3 -cpu cores : 6 -apicid : 102 -initial apicid : 102 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4404.05 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 22 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2100.647 -cache size : 12288 KB -physical id : 3 -siblings : 12 -core id : 4 -cpu cores : 6 -apicid : 104 -initial apicid : 104 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4404.05 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 23 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.108 -cache size : 12288 KB -physical id : 3 -siblings : 12 -core id : 5 -cpu cores : 6 -apicid : 106 -initial apicid : 106 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4404.05 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 24 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.440 -cache size : 12288 KB -physical id : 0 -siblings : 12 -core id : 0 -cpu cores : 6 -apicid : 1 -initial apicid : 1 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4400.07 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 25 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.029 -cache size : 12288 KB -physical id : 0 -siblings : 12 -core id : 1 -cpu cores : 6 -apicid : 3 -initial apicid : 3 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4400.07 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 26 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2201.393 -cache size : 12288 KB -physical id : 0 -siblings : 12 -core id : 2 -cpu cores : 6 -apicid : 5 -initial apicid : 5 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4400.07 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 27 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.010 -cache size : 12288 KB -physical id : 0 -siblings : 12 -core id : 3 -cpu cores : 6 -apicid : 7 -initial apicid : 7 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4400.07 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 28 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.038 -cache size : 12288 KB -physical id : 0 -siblings : 12 -core id : 4 -cpu cores : 6 -apicid : 9 -initial apicid : 9 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4400.07 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 29 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.079 -cache size : 12288 KB -physical id : 0 -siblings : 12 -core id : 5 -cpu cores : 6 -apicid : 11 -initial apicid : 11 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4400.07 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 30 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.040 -cache size : 12288 KB -physical id : 1 -siblings : 12 -core id : 0 -cpu cores : 6 -apicid : 33 -initial apicid : 33 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.38 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 31 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.098 -cache size : 12288 KB -physical id : 1 -siblings : 12 -core id : 1 -cpu cores : 6 -apicid : 35 -initial apicid : 35 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.38 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 32 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.070 -cache size : 12288 KB -physical id : 1 -siblings : 12 -core id : 2 -cpu cores : 6 -apicid : 37 -initial apicid : 37 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.38 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 33 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.038 -cache size : 12288 KB -physical id : 1 -siblings : 12 -core id : 3 -cpu cores : 6 -apicid : 39 -initial apicid : 39 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.38 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 34 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.581 -cache size : 12288 KB -physical id : 1 -siblings : 12 -core id : 4 -cpu cores : 6 -apicid : 41 -initial apicid : 41 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.38 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 35 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.017 -cache size : 12288 KB -physical id : 1 -siblings : 12 -core id : 5 -cpu cores : 6 -apicid : 43 -initial apicid : 43 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.38 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 36 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 1591.751 -cache size : 12288 KB -physical id : 2 -siblings : 12 -core id : 0 -cpu cores : 6 -apicid : 65 -initial apicid : 65 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.70 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 37 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 1956.531 -cache size : 12288 KB -physical id : 2 -siblings : 12 -core id : 1 -cpu cores : 6 -apicid : 67 -initial apicid : 67 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.70 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 38 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 1853.526 -cache size : 12288 KB -physical id : 2 -siblings : 12 -core id : 2 -cpu cores : 6 -apicid : 69 -initial apicid : 69 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.70 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 39 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2125.556 -cache size : 12288 KB -physical id : 2 -siblings : 12 -core id : 3 -cpu cores : 6 -apicid : 71 -initial apicid : 71 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.70 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 40 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 1545.681 -cache size : 12288 KB -physical id : 2 -siblings : 12 -core id : 4 -cpu cores : 6 -apicid : 73 -initial apicid : 73 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.70 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 41 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2050.233 -cache size : 12288 KB -physical id : 2 -siblings : 12 -core id : 5 -cpu cores : 6 -apicid : 75 -initial apicid : 75 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4403.70 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 42 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2160.310 -cache size : 12288 KB -physical id : 3 -siblings : 12 -core id : 0 -cpu cores : 6 -apicid : 97 -initial apicid : 97 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4404.05 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 43 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2089.592 -cache size : 12288 KB -physical id : 3 -siblings : 12 -core id : 1 -cpu cores : 6 -apicid : 99 -initial apicid : 99 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4404.05 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 44 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2200.121 -cache size : 12288 KB -physical id : 3 -siblings : 12 -core id : 2 -cpu cores : 6 -apicid : 101 -initial apicid : 101 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4404.05 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 45 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 2105.117 -cache size : 12288 KB -physical id : 3 -siblings : 12 -core id : 3 -cpu cores : 6 -apicid : 103 -initial apicid : 103 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4404.05 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 46 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 1984.419 -cache size : 12288 KB -physical id : 3 -siblings : 12 -core id : 4 -cpu cores : 6 -apicid : 105 -initial apicid : 105 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4404.05 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - -processor : 47 -vendor_id : GenuineIntel -cpu family : 6 -model : 45 -model name : Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz -stepping : 7 -microcode : 0x710 -cpu MHz : 1924.644 -cache size : 12288 KB -physical id : 3 -siblings : 12 -core id : 5 -cpu cores : 6 -apicid : 107 -initial apicid : 107 -fpu : yes -fpu_exception : yes -cpuid level : 13 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx lahf_lm epb pti tpr_shadow vnmi flexpriority ept vpid xsaveopt dtherm arat pln pts -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs itlb_multihit -bogomips : 4404.05 -clflush size : 64 -cache_alignment : 64 -address sizes : 46 bits physical, 48 bits virtual -power management: - diff --git a/metricbeat/internal/metrics/cpu/testdata/cpuinfo-quad-socket.expected.json b/metricbeat/internal/metrics/cpu/testdata/cpuinfo-quad-socket.expected.json deleted file mode 100644 index bfbe23418a1..00000000000 --- a/metricbeat/internal/metrics/cpu/testdata/cpuinfo-quad-socket.expected.json +++ /dev/null @@ -1,338 +0,0 @@ -[ - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.033, - "PhysicalID": 0, - "CoreID": 0 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.027, - "PhysicalID": 0, - "CoreID": 1 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.038, - "PhysicalID": 0, - "CoreID": 2 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.516, - "PhysicalID": 0, - "CoreID": 3 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.04, - "PhysicalID": 0, - "CoreID": 4 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.048, - "PhysicalID": 0, - "CoreID": 5 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2199.974, - "PhysicalID": 1, - "CoreID": 0 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.037, - "PhysicalID": 1, - "CoreID": 1 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.025, - "PhysicalID": 1, - "CoreID": 2 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.01, - "PhysicalID": 1, - "CoreID": 3 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.038, - "PhysicalID": 1, - "CoreID": 4 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.042, - "PhysicalID": 1, - "CoreID": 5 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 1911.118, - "PhysicalID": 2, - "CoreID": 0 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 1479.972, - "PhysicalID": 2, - "CoreID": 1 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 1970.264, - "PhysicalID": 2, - "CoreID": 2 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 1996.097, - "PhysicalID": 2, - "CoreID": 3 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 1788.272, - "PhysicalID": 2, - "CoreID": 4 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2005.072, - "PhysicalID": 2, - "CoreID": 5 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 1658.085, - "PhysicalID": 3, - "CoreID": 0 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 1855.222, - "PhysicalID": 3, - "CoreID": 1 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2068.635, - "PhysicalID": 3, - "CoreID": 2 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 1806.426, - "PhysicalID": 3, - "CoreID": 3 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2100.647, - "PhysicalID": 3, - "CoreID": 4 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.108, - "PhysicalID": 3, - "CoreID": 5 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.44, - "PhysicalID": 0, - "CoreID": 0 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.029, - "PhysicalID": 0, - "CoreID": 1 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2201.393, - "PhysicalID": 0, - "CoreID": 2 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.01, - "PhysicalID": 0, - "CoreID": 3 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.038, - "PhysicalID": 0, - "CoreID": 4 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.079, - "PhysicalID": 0, - "CoreID": 5 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.04, - "PhysicalID": 1, - "CoreID": 0 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.098, - "PhysicalID": 1, - "CoreID": 1 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.07, - "PhysicalID": 1, - "CoreID": 2 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.038, - "PhysicalID": 1, - "CoreID": 3 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.581, - "PhysicalID": 1, - "CoreID": 4 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.017, - "PhysicalID": 1, - "CoreID": 5 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 1591.751, - "PhysicalID": 2, - "CoreID": 0 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 1956.531, - "PhysicalID": 2, - "CoreID": 1 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 1853.526, - "PhysicalID": 2, - "CoreID": 2 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2125.556, - "PhysicalID": 2, - "CoreID": 3 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 1545.681, - "PhysicalID": 2, - "CoreID": 4 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2050.233, - "PhysicalID": 2, - "CoreID": 5 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2160.31, - "PhysicalID": 3, - "CoreID": 0 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2089.592, - "PhysicalID": 3, - "CoreID": 1 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2200.121, - "PhysicalID": 3, - "CoreID": 2 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 2105.117, - "PhysicalID": 3, - "CoreID": 3 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 1984.419, - "PhysicalID": 3, - "CoreID": 4 - }, - { - "ModelName": "Intel(R) Xeon(R) CPU E5-4607 0 @ 2.20GHz", - "ModelNumber": "45", - "Mhz": 1924.644, - "PhysicalID": 3, - "CoreID": 5 - } -] diff --git a/metricbeat/internal/metrics/cpu/testdata/cpuinfo.expected.json b/metricbeat/internal/metrics/cpu/testdata/cpuinfo.expected.json deleted file mode 100644 index b37c3182abb..00000000000 --- a/metricbeat/internal/metrics/cpu/testdata/cpuinfo.expected.json +++ /dev/null @@ -1,114 +0,0 @@ -[ - { - "ModelName": "Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz", - "ModelNumber": "165", - "Mhz": 2400, - "PhysicalID": 0, - "CoreID": 0 - }, - { - "ModelName": "Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz", - "ModelNumber": "165", - "Mhz": 2400, - "PhysicalID": 0, - "CoreID": 1 - }, - { - "ModelName": "Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz", - "ModelNumber": "165", - "Mhz": 2400, - "PhysicalID": 0, - "CoreID": 2 - }, - { - "ModelName": "Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz", - "ModelNumber": "165", - "Mhz": 2400, - "PhysicalID": 0, - "CoreID": 3 - }, - { - "ModelName": "Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz", - "ModelNumber": "165", - "Mhz": 2400, - "PhysicalID": 0, - "CoreID": 4 - }, - { - "ModelName": "Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz", - "ModelNumber": "165", - "Mhz": 2400, - "PhysicalID": 0, - "CoreID": 5 - }, - { - "ModelName": "Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz", - "ModelNumber": "165", - "Mhz": 2400, - "PhysicalID": 0, - "CoreID": 6 - }, - { - "ModelName": "Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz", - "ModelNumber": "165", - "Mhz": 2400, - "PhysicalID": 0, - "CoreID": 7 - }, - { - "ModelName": "Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz", - "ModelNumber": "165", - "Mhz": 2534.854, - "PhysicalID": 0, - "CoreID": 0 - }, - { - "ModelName": "Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz", - "ModelNumber": "165", - "Mhz": 2400, - "PhysicalID": 0, - "CoreID": 1 - }, - { - "ModelName": "Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz", - "ModelNumber": "165", - "Mhz": 2400, - "PhysicalID": 0, - "CoreID": 2 - }, - { - "ModelName": "Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz", - "ModelNumber": "165", - "Mhz": 2400, - "PhysicalID": 0, - "CoreID": 3 - }, - { - "ModelName": "Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz", - "ModelNumber": "165", - "Mhz": 2673.889, - "PhysicalID": 0, - "CoreID": 4 - }, - { - "ModelName": "Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz", - "ModelNumber": "165", - "Mhz": 2400, - "PhysicalID": 0, - "CoreID": 5 - }, - { - "ModelName": "Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz", - "ModelNumber": "165", - "Mhz": 2400, - "PhysicalID": 0, - "CoreID": 6 - }, - { - "ModelName": "Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz", - "ModelNumber": "165", - "Mhz": 2400, - "PhysicalID": 0, - "CoreID": 7 - } -] diff --git a/metricbeat/internal/metrics/memory/memory.go b/metricbeat/internal/metrics/memory/memory.go deleted file mode 100644 index c3047e91371..00000000000 --- a/metricbeat/internal/metrics/memory/memory.go +++ /dev/null @@ -1,103 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package memory - -import ( - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/opt" - "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" -) - -// Memory holds os-specifc memory usage data -// The vast majority of these values are cross-platform -// However, we're wrapping all them for the sake of safety, and for the more variable swap metrics -type Memory struct { - Total opt.Uint `struct:"total,omitempty"` - Used UsedMemStats `struct:"used,omitempty"` - - Free opt.Uint `struct:"free,omitempty"` - Cached opt.Uint `struct:"cached,omitempty"` - // "Actual" values are, technically, a linux-only concept - // For better or worse we've expanded it to include "derived" - // Memory values on other platforms, which we should - // probably keep for the sake of backwards compatibility - // However, because the derived value varies from platform to platform, - // We may want to more precisely document what these mean. - Actual ActualMemoryMetrics `struct:"actual,omitempty"` - - // Swap metrics - Swap SwapMetrics `struct:"swap,omitempty"` -} - -// UsedMemStats wraps used.* memory metrics -type UsedMemStats struct { - Pct opt.Float `struct:"pct,omitempty"` - Bytes opt.Uint `struct:"bytes,omitempty"` -} - -// ActualMemoryMetrics wraps the actual.* memory metrics -type ActualMemoryMetrics struct { - Free opt.Uint `struct:"free,omitempty"` - Used UsedMemStats `struct:"used,omitempty"` -} - -// SwapMetrics wraps swap.* memory metrics -type SwapMetrics struct { - Total opt.Uint `struct:"total,omitempty"` - Used UsedMemStats `struct:"used,omitempty"` - Free opt.Uint `struct:"free,omitempty"` -} - -// Get returns platform-independent memory metrics. -func Get(procfs resolve.Resolver) (Memory, error) { - base, err := get(procfs) - if err != nil { - return Memory{}, errors.Wrap(err, "error getting system memory info") - } - base.fillPercentages() - return base, nil -} - -// IsZero implements the zeroer interface for structform's folders -func (used UsedMemStats) IsZero() bool { - return used.Pct.IsZero() && used.Bytes.IsZero() -} - -// IsZero implements the zeroer interface for structform's folders -func (swap SwapMetrics) IsZero() bool { - return swap.Free.IsZero() && swap.Used.IsZero() && swap.Total.IsZero() -} - -func (base *Memory) fillPercentages() { - // Add percentages - // In theory, `Used` and `Total` are available everywhere, so assume values are good. - if base.Total.Exists() && base.Total.ValueOr(0) != 0 { - percUsed := float64(base.Used.Bytes.ValueOr(0)) / float64(base.Total.ValueOr(1)) - base.Used.Pct = opt.FloatWith(common.Round(percUsed, common.DefaultDecimalPlacesCount)) - - actualPercUsed := float64(base.Actual.Used.Bytes.ValueOr(0)) / float64(base.Total.ValueOr(0)) - base.Actual.Used.Pct = opt.FloatWith(common.Round(actualPercUsed, common.DefaultDecimalPlacesCount)) - } - - if base.Swap.Total.ValueOr(0) != 0 && base.Swap.Used.Bytes.Exists() { - perc := float64(base.Swap.Used.Bytes.ValueOr(0)) / float64(base.Swap.Total.ValueOr(0)) - base.Swap.Used.Pct = opt.FloatWith(common.Round(perc, common.DefaultDecimalPlacesCount)) - } -} diff --git a/metricbeat/internal/metrics/memory/memory_aix.go b/metricbeat/internal/metrics/memory/memory_aix.go deleted file mode 100644 index df19642488e..00000000000 --- a/metricbeat/internal/metrics/memory/memory_aix.go +++ /dev/null @@ -1,76 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package memory - -/* -#cgo LDFLAGS: -L/usr/lib -lperfstat - -#include -#include -#include -#include -#include -#include -#include -#include - -*/ -import "C" - -import ( - "fmt" - "os" - - "github.com/elastic/beats/v7/libbeat/opt" - "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" -) - -var system struct { - ticks uint64 - btime uint64 - pagesize uint64 -} - -func init() { - // sysconf(_SC_CLK_TCK) returns the number of ticks by second. - system.ticks = uint64(C.sysconf(C._SC_CLK_TCK)) - system.pagesize = uint64(os.Getpagesize()) -} - -func get(_ resolve.Resolver) (Memory, error) { - memData := Memory{} - meminfo := C.perfstat_memory_total_t{} - _, err := C.perfstat_memory_total(nil, &meminfo, C.sizeof_perfstat_memory_total_t, 1) - if err != nil { - return memData, fmt.Errorf("perfstat_memory_total: %s", err) - } - - totalMem := uint64(meminfo.real_total) * system.pagesize - freeMem := uint64(meminfo.real_free) * system.pagesize - - memData.Total = opt.UintWith(totalMem) - memData.Free = opt.UintWith(freeMem) - - kern := uint64(meminfo.numperm) * system.pagesize // number of pages in file cache - - memData.Used.Bytes = opt.UintWith(totalMem - freeMem) - memData.Actual.Free = opt.UintWith(freeMem + kern) - memData.Actual.Used.Bytes = opt.UintWith(memData.Used.Bytes.ValueOr(0) - kern) - - return memData, nil -} diff --git a/metricbeat/internal/metrics/memory/memory_darwin.go b/metricbeat/internal/metrics/memory/memory_darwin.go deleted file mode 100644 index 1a4689c1621..00000000000 --- a/metricbeat/internal/metrics/memory/memory_darwin.go +++ /dev/null @@ -1,134 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package memory - -/* -#include -#include -#include -#include -#include -#include -#include -#include -#include -*/ -import "C" - -import ( - "bytes" - "encoding/binary" - "fmt" - "syscall" - "unsafe" - - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/opt" - "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" -) - -type xswUsage struct { - Total, Avail, Used uint64 -} - -// get is the darwin implementation for fetching Memory data -func get(_ resolve.Resolver) (Memory, error) { - var vmstat C.vm_statistics_data_t - - mem := Memory{} - - var total uint64 - - if err := sysctlbyname("hw.memsize", &total); err != nil { - return Memory{}, errors.Wrap(err, "error getting memsize") - } - mem.Total = opt.UintWith(total) - - if err := vmInfo(&vmstat); err != nil { - return Memory{}, errors.Wrap(err, "error getting VM info") - } - - kern := uint64(vmstat.inactive_count) << 12 - free := uint64(vmstat.free_count) << 12 - - mem.Free = opt.UintWith(free) - mem.Used.Bytes = opt.UintWith(total - free) - - mem.Actual.Free = opt.UintWith(free + kern) - mem.Actual.Used.Bytes = opt.UintWith((total - free) - kern) - - var err error - mem.Swap, err = getSwap() - if err != nil { - return mem, errors.Wrap(err, "error getting swap memory") - } - - return mem, nil -} - -// Get fetches swap data -func getSwap() (SwapMetrics, error) { - swUsage := xswUsage{} - - swap := SwapMetrics{} - if err := sysctlbyname("vm.swapusage", &swUsage); err != nil { - return swap, errors.Wrap(err, "error getting swap usage") - } - - swap.Total = opt.UintWith(swUsage.Total) - swap.Used.Bytes = opt.UintWith(swUsage.Used) - swap.Free = opt.UintWith(swUsage.Avail) - - return swap, nil -} - -// generic Sysctl buffer unmarshalling -func sysctlbyname(name string, data interface{}) (err error) { - val, err := syscall.Sysctl(name) - if err != nil { - return err - } - - buf := []byte(val) - - switch v := data.(type) { - case *uint64: - *v = *(*uint64)(unsafe.Pointer(&buf[0])) - return - } - - bbuf := bytes.NewBuffer([]byte(val)) - return binary.Read(bbuf, binary.LittleEndian, data) -} - -func vmInfo(vmstat *C.vm_statistics_data_t) error { - var count C.mach_msg_type_number_t = C.HOST_VM_INFO_COUNT - - status := C.host_statistics( - C.host_t(C.mach_host_self()), - C.HOST_VM_INFO, - C.host_info_t(unsafe.Pointer(vmstat)), - &count) - - if status != C.KERN_SUCCESS { - return fmt.Errorf("host_statistics=%d", status) - } - - return nil -} diff --git a/metricbeat/internal/metrics/memory/memory_freebsd.go b/metricbeat/internal/metrics/memory/memory_freebsd.go deleted file mode 100644 index 380acd13733..00000000000 --- a/metricbeat/internal/metrics/memory/memory_freebsd.go +++ /dev/null @@ -1,91 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package memory - -import ( - "unsafe" - - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/opt" - "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" -) - -/* -#include -#include -#include -#include -#include -#include -#include -#include -#include -*/ -import "C" - -func get(_ resolve.Resolver) (Memory, error) { - val := C.uint32_t(0) - sc := C.size_t(4) - - memData := Memory{} - - name := C.CString("vm.stats.vm.v_page_count") - _, err := C.sysctlbyname(name, unsafe.Pointer(&val), &sc, nil, 0) - C.free(unsafe.Pointer(name)) - if err != nil { - return memData, errors.Errorf("error in vm.stats.vm.v_page_count") - } - pagecount := uint64(val) - - name = C.CString("vm.stats.vm.v_page_size") - _, err = C.sysctlbyname(name, unsafe.Pointer(&val), &sc, nil, 0) - C.free(unsafe.Pointer(name)) - if err != nil { - return memData, errors.Errorf("error in vm.stats.vm.v_page_size") - } - pagesize := uint64(val) - - name = C.CString("vm.stats.vm.v_free_count") - _, err = C.sysctlbyname(name, unsafe.Pointer(&val), &sc, nil, 0) - C.free(unsafe.Pointer(name)) - if err != nil { - return memData, errors.Errorf("error in vm.stats.vm.v_free_count") - } - - memFree := uint64(val) * pagesize - memData.Free = opt.UintWith(memFree) - - name = C.CString("vm.stats.vm.v_inactive_count") - _, err = C.sysctlbyname(name, unsafe.Pointer(&val), &sc, nil, 0) - C.free(unsafe.Pointer(name)) - if err != nil { - return memData, errors.Errorf("error in vm.stats.vm.v_inactive_count") - } - kern := uint64(val) - - memTotal := uint64(pagecount * pagesize) - - memData.Total = opt.UintWith(memTotal) - - memData.Used.Bytes = opt.UintWith(memTotal - memFree) - memData.Actual.Free = opt.UintWith(memFree + (kern * pagesize)) - memData.Actual.Used.Bytes = opt.UintWith((memTotal - memFree) - (kern * pagesize)) - - return memData, nil -} diff --git a/metricbeat/internal/metrics/memory/memory_helpers.go b/metricbeat/internal/metrics/memory/memory_helpers.go deleted file mode 100644 index 9e9c7e60bea..00000000000 --- a/metricbeat/internal/metrics/memory/memory_helpers.go +++ /dev/null @@ -1,80 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package memory - -import ( - "bufio" - "bytes" - "io" - "io/ioutil" - "strconv" - "strings" - - "github.com/pkg/errors" - - "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" -) - -// ParseMeminfo parses the contents of /proc/meminfo into a hashmap -func ParseMeminfo(rootfs resolve.Resolver) (map[string]uint64, error) { - table := map[string]uint64{} - - meminfoPath := rootfs.ResolveHostFS("/proc/meminfo") - err := readFile(meminfoPath, func(line string) bool { - fields := strings.Split(line, ":") - - if len(fields) != 2 { - return true // skip on errors - } - - valueUnit := strings.Fields(fields[1]) - value, err := strconv.ParseUint(valueUnit[0], 10, 64) - if err != nil { - return true // skip on errors - } - - if len(valueUnit) > 1 && valueUnit[1] == "kB" { - value *= 1024 - } - table[fields[0]] = value - - return true - }) - return table, err -} - -func readFile(file string, handler func(string) bool) error { - contents, err := ioutil.ReadFile(file) - if err != nil { - return errors.Wrapf(err, "error reading file %s", file) - } - - reader := bufio.NewReader(bytes.NewBuffer(contents)) - - for { - line, _, err := reader.ReadLine() - if err == io.EOF { - break - } - if !handler(string(line)) { - break - } - } - - return nil -} diff --git a/metricbeat/internal/metrics/memory/memory_linux.go b/metricbeat/internal/metrics/memory/memory_linux.go deleted file mode 100644 index 0b55a36129e..00000000000 --- a/metricbeat/internal/metrics/memory/memory_linux.go +++ /dev/null @@ -1,88 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package memory - -import ( - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/opt" - "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" -) - -// get is the linux implementation for fetching Memory data -func get(rootfs resolve.Resolver) (Memory, error) { - table, err := ParseMeminfo(rootfs) - if err != nil { - return Memory{}, errors.Wrap(err, "error fetching meminfo") - } - - memData := Memory{} - - var free, cached uint64 - var ok bool - if total, ok := table["MemTotal"]; ok { - memData.Total = opt.UintWith(total) - } - if free, ok = table["MemFree"]; ok { - memData.Free = opt.UintWith(free) - } - if cached, ok = table["Cached"]; ok { - memData.Cached = opt.UintWith(cached) - } - - // overlook parsing issues here - // On the very small chance some of these don't exist, - // It's not the end of the world - buffers, _ := table["Buffers"] - - if memAvail, ok := table["MemAvailable"]; ok { - // MemAvailable is in /proc/meminfo (kernel 3.14+) - memData.Actual.Free = opt.UintWith(memAvail) - } else { - // in the future we may want to find another way to do this. - // "MemAvailable" and other more derivied metrics - // Are very relative, and can be unhelpful in cerntain workloads - // We may want to find a way to more clearly express to users - // where a certain value is coming from and what it represents - - // The use of `cached` here is particularly concerning, - // as under certain intense DB server workloads, the cached memory can be quite large - // and give the impression that we've passed memory usage watermark - memData.Actual.Free = opt.UintWith(free + buffers + cached) - } - - memData.Used.Bytes = opt.UintWith(memData.Total.ValueOr(0) - memData.Free.ValueOr(0)) - memData.Actual.Used.Bytes = opt.UintWith(memData.Total.ValueOr(0) - memData.Actual.Free.ValueOr(0)) - - // Populate swap data - swapTotal, okST := table["SwapTotal"] - if okST { - memData.Swap.Total = opt.UintWith(swapTotal) - } - swapFree, okSF := table["SwapFree"] - if okSF { - memData.Swap.Free = opt.UintWith(swapFree) - } - - if okSF && okST { - memData.Swap.Used.Bytes = opt.UintWith(swapTotal - swapFree) - } - - return memData, nil - -} diff --git a/metricbeat/internal/metrics/memory/memory_openbsd.go b/metricbeat/internal/metrics/memory/memory_openbsd.go deleted file mode 100644 index fe713fd3ac4..00000000000 --- a/metricbeat/internal/metrics/memory/memory_openbsd.go +++ /dev/null @@ -1,237 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package memory - -/* -#include -#include -#include -#include -#include -#include -#include -#include -*/ -import "C" - -import ( - "syscall" - "unsafe" - - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/opt" - "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" -) - -// Uvmexp wraps memory data from sysctl -type Uvmexp struct { - pagesize uint32 - pagemask uint32 - pageshift uint32 - npages uint32 - free uint32 - active uint32 - inactive uint32 - paging uint32 - wired uint32 - zeropages uint32 - reserve_pagedaemon uint32 - reserve_kernel uint32 - anonpages uint32 - vnodepages uint32 - vtextpages uint32 - freemin uint32 - freetarg uint32 - inactarg uint32 - wiredmax uint32 - anonmin uint32 - vtextmin uint32 - vnodemin uint32 - anonminpct uint32 - vtextmi uint32 - npct uint32 - vnodeminpct uint32 - nswapdev uint32 - swpages uint32 - swpginuse uint32 - swpgonly uint32 - nswget uint32 - nanon uint32 - nanonneeded uint32 - nfreeanon uint32 - faults uint32 - traps uint32 - intrs uint32 - swtch uint32 - softs uint32 - syscalls uint32 - pageins uint32 - obsolete_swapins uint32 - obsolete_swapouts uint32 - pgswapin uint32 - pgswapout uint32 - forks uint32 - forks_ppwait uint32 - forks_sharevm uint32 - pga_zerohit uint32 - pga_zeromiss uint32 - zeroaborts uint32 - fltnoram uint32 - fltnoanon uint32 - fltpgwait uint32 - fltpgrele uint32 - fltrelck uint32 - fltrelckok uint32 - fltanget uint32 - fltanretry uint32 - fltamcopy uint32 - fltnamap uint32 - fltnomap uint32 - fltlget uint32 - fltget uint32 - flt_anon uint32 - flt_acow uint32 - flt_obj uint32 - flt_prcopy uint32 - flt_przero uint32 - pdwoke uint32 - pdrevs uint32 - pdswout uint32 - pdfreed uint32 - pdscans uint32 - pdanscan uint32 - pdobscan uint32 - pdreact uint32 - pdbusy uint32 - pdpageouts uint32 - pdpending uint32 - pddeact uint32 - pdreanon uint32 - pdrevnode uint32 - pdrevtext uint32 - fpswtch uint32 - kmapent uint32 -} - -// Bcachestats reports cache stats from sysctl -type Bcachestats struct { - numbufs uint64 - numbufpages uint64 - numdirtypages uint64 - numcleanpages uint64 - pendingwrites uint64 - pendingreads uint64 - numwrites uint64 - numreads uint64 - cachehits uint64 - busymapped uint64 - dmapages uint64 - highpages uint64 - delwribufs uint64 - kvaslots uint64 - kvaslots_avail uint64 -} - -// Swapent reports swap metrics from sysctl -type Swapent struct { - se_dev C.dev_t - se_flags int32 - se_nblks int32 - se_inuse int32 - se_priority int32 - sw_path []byte -} - -func get(_ resolve.Resolver) (Memory, error) { - - memData := Memory{} - - n := uintptr(0) - var uvmexp Uvmexp - mib := [2]int32{C.CTL_VM, C.VM_UVMEXP} - n = uintptr(0) - // First we determine how much memory we'll need to pass later on (via `n`) - _, _, errno := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 2, 0, uintptr(unsafe.Pointer(&n)), 0, 0) - if errno != 0 || n == 0 { - return memData, errors.Errorf("Error in size VM_UVMEXP sysctl call, errno %d", errno) - } - - _, _, errno = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 2, uintptr(unsafe.Pointer(&uvmexp)), uintptr(unsafe.Pointer(&n)), 0, 0) - if errno != 0 || n == 0 { - return memData, errors.Errorf("Error in VM_UVMEXP sysctl call, errno %d", errno) - } - - var bcachestats Bcachestats - mib3 := [3]int32{C.CTL_VFS, C.VFS_GENERIC, C.VFS_BCACHESTAT} - n = uintptr(0) - _, _, errno = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib3[0])), 3, 0, uintptr(unsafe.Pointer(&n)), 0, 0) - if errno != 0 || n == 0 { - return memData, errors.Errorf("Error in size VFS_BCACHESTAT sysctl call, errno %d", errno) - } - _, _, errno = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib3[0])), 3, uintptr(unsafe.Pointer(&bcachestats)), uintptr(unsafe.Pointer(&n)), 0, 0) - if errno != 0 || n == 0 { - return memData, errors.Errorf("Error in VFS_BCACHESTAT sysctl call, errno %d", errno) - } - - memFree := uint64(uvmexp.free) << uvmexp.pageshift - memUsed := uint64(uvmexp.npages-uvmexp.free) << uvmexp.pageshift - - memData.Total = opt.UintWith(uint64(uvmexp.npages) << uvmexp.pageshift) - memData.Used.Bytes = opt.UintWith(memUsed) - memData.Free = opt.UintWith(memFree) - - memData.Actual.Free = opt.UintWith(memFree + (uint64(bcachestats.numbufpages) << uvmexp.pageshift)) - memData.Actual.Used.Bytes = opt.UintWith(memUsed - (uint64(bcachestats.numbufpages) << uvmexp.pageshift)) - - var err error - memData.Swap, err = getSwap() - if err != nil { - return memData, errors.Wrap(err, "error getting swap data") - } - - return memData, nil -} - -func getSwap() (SwapMetrics, error) { - swapData := SwapMetrics{} - nswap := C.swapctl(C.SWAP_NSWAP, unsafe.Pointer(uintptr(0)), 0) - - // If there are no swap devices, nothing to do here. - if nswap == 0 { - return swapData, nil - } - - swdev := make([]Swapent, nswap) - - rnswap := C.swapctl(C.SWAP_STATS, unsafe.Pointer(&swdev[0]), nswap) - if rnswap == 0 { - return swapData, errors.Errorf("error in SWAP_STATS sysctl, swapctl returned %d", rnswap) - } - - for i := 0; i < int(nswap); i++ { - if swdev[i].se_flags&C.SWF_ENABLE == 2 { - swapData.Used.Bytes = opt.UintWith(swapData.Used.Bytes.ValueOr(0) + uint64(swdev[i].se_inuse/(1024/C.DEV_BSIZE))) - swapData.Total = opt.UintWith(swapData.Total.ValueOr(0) + uint64(swdev[i].se_nblks/(1024/C.DEV_BSIZE))) - } - } - - swapData.Free = opt.UintWith(swapData.Total.ValueOr(0) - swapData.Used.Bytes.ValueOr(0)) - - return swapData, nil -} diff --git a/metricbeat/internal/metrics/memory/memory_test.go b/metricbeat/internal/metrics/memory/memory_test.go deleted file mode 100644 index c1ebc8b7146..00000000000 --- a/metricbeat/internal/metrics/memory/memory_test.go +++ /dev/null @@ -1,125 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//go:build !integration && (darwin || freebsd || linux || openbsd || windows) -// +build !integration -// +build darwin freebsd linux openbsd windows - -package memory - -import ( - "runtime" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/v7/libbeat/opt" - "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" -) - -func TestGetMemory(t *testing.T) { - mem, err := Get(resolve.NewTestResolver("")) - - assert.NotNil(t, mem) - assert.NoError(t, err) - - assert.True(t, mem.Total.Exists()) - assert.True(t, (mem.Total.ValueOr(0) > 0)) - - assert.True(t, mem.Used.Bytes.Exists()) - assert.True(t, (mem.Used.Bytes.ValueOr(0) > 0)) - - assert.True(t, mem.Free.Exists()) - assert.True(t, (mem.Free.ValueOr(0) >= 0)) - - assert.True(t, mem.Actual.Free.Exists()) - assert.True(t, (mem.Actual.Free.ValueOr(0) >= 0)) - - assert.True(t, mem.Actual.Used.Bytes.Exists()) - assert.True(t, (mem.Actual.Used.Bytes.ValueOr(0) > 0)) -} - -func TestGetSwap(t *testing.T) { - if runtime.GOOS == "freebsd" { - return //no load data on freebsd - } - - mem, err := Get(resolve.NewTestResolver("")) - - assert.NotNil(t, mem) - assert.NoError(t, err) - - assert.True(t, mem.Swap.Total.Exists()) - assert.True(t, (mem.Swap.Total.ValueOr(0) >= 0)) - - assert.True(t, mem.Swap.Used.Bytes.Exists()) - assert.True(t, (mem.Swap.Used.Bytes.ValueOr(0) >= 0)) - - assert.True(t, mem.Swap.Free.Exists()) - assert.True(t, (mem.Swap.Free.ValueOr(0) >= 0)) -} - -func TestMemPercentage(t *testing.T) { - m := Memory{ - Total: opt.UintWith(7), - Used: UsedMemStats{Bytes: opt.UintWith(5)}, - Free: opt.UintWith(2), - } - m.fillPercentages() - assert.Equal(t, m.Used.Pct.ValueOr(0), 0.7143) - - m = Memory{ - Total: opt.UintWith(0), - } - m.fillPercentages() - assert.Equal(t, m.Used.Pct.ValueOr(0), 0.0) -} - -func TestActualMemPercentage(t *testing.T) { - m := Memory{ - Total: opt.UintWith(7), - Actual: ActualMemoryMetrics{ - Used: UsedMemStats{Bytes: opt.UintWith(5)}, - Free: opt.UintWith(2), - }, - } - - m.fillPercentages() - assert.Equal(t, m.Actual.Used.Pct.ValueOr(0), 0.7143) - -} - -func TestMeminfoParse(t *testing.T) { - // Make sure we're manually calculating Actual correctly on linux - if runtime.GOOS == "linux" { - mem, err := Get(resolve.NewTestResolver("./oldkern")) - assert.NoError(t, err) - - assert.Equal(t, uint64(27307106304), mem.Cached.ValueOr(0)) - assert.Equal(t, uint64(52983070720), mem.Actual.Free.ValueOr(0)) - assert.Equal(t, uint64(10137726976), mem.Actual.Used.Bytes.ValueOr(0)) - } -} - -func TestMeminfoPct(t *testing.T) { - if runtime.GOOS == "linux" { - memRaw, err := Get(resolve.NewTestResolver("./oldkern")) - assert.NoError(t, err) - assert.Equal(t, float64(0.1606), memRaw.Actual.Used.Pct.ValueOr(0)) - assert.Equal(t, float64(0.5933), memRaw.Used.Pct.ValueOr(0)) - } -} diff --git a/metricbeat/internal/metrics/memory/memory_windows.go b/metricbeat/internal/metrics/memory/memory_windows.go deleted file mode 100644 index ae1d8601c19..00000000000 --- a/metricbeat/internal/metrics/memory/memory_windows.go +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package memory - -import ( - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/opt" - "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" - "github.com/elastic/go-windows" -) - -// get is the windows implementation of get for memory metrics -func get(_ resolve.Resolver) (Memory, error) { - - memData := Memory{} - - memoryStatusEx, err := windows.GlobalMemoryStatusEx() - if err != nil { - return memData, errors.Wrap(err, "Error fetching global memory status") - } - memData.Total = opt.UintWith(memoryStatusEx.TotalPhys) - memData.Free = opt.UintWith(memoryStatusEx.AvailPhys) - - memData.Used.Bytes = opt.UintWith(memoryStatusEx.TotalPhys - memoryStatusEx.AvailPhys) - - // We shouldn't really be doing this, but we also don't want to make breaking changes right now, - // and memory.actual is used by quite a few visualizations - memData.Actual.Free = memData.Free - memData.Actual.Used.Bytes = memData.Used.Bytes - - memData.Swap.Free = opt.UintWith(memoryStatusEx.AvailPageFile) - memData.Swap.Total = opt.UintWith(memoryStatusEx.TotalPageFile) - memData.Swap.Used.Bytes = opt.UintWith(memoryStatusEx.TotalPageFile - memoryStatusEx.AvailPageFile) - - return memData, nil -} diff --git a/metricbeat/internal/metrics/memory/oldkern/proc/meminfo b/metricbeat/internal/metrics/memory/oldkern/proc/meminfo deleted file mode 100644 index 504a1e4b804..00000000000 --- a/metricbeat/internal/metrics/memory/oldkern/proc/meminfo +++ /dev/null @@ -1,48 +0,0 @@ -MemTotal: 61641404 kB -MemFree: 25071456 kB -Buffers: 2728 kB -Cached: 26667096 kB -SwapCached: 0 kB -Active: 7766692 kB -Inactive: 25447972 kB -Active(anon): 3332 kB -Inactive(anon): 6526672 kB -Active(file): 7763360 kB -Inactive(file): 18921300 kB -Unevictable: 3072 kB -Mlocked: 0 kB -SwapTotal: 8388604 kB -SwapFree: 8386812 kB -Dirty: 4576 kB -Writeback: 0 kB -AnonPages: 6396352 kB -Mapped: 863432 kB -Shmem: 9504 kB -KReclaimable: 1539312 kB -Slab: 2387096 kB -SReclaimable: 1539312 kB -SUnreclaim: 847784 kB -KernelStack: 14880 kB -PageTables: 59592 kB -NFS_Unstable: 0 kB -Bounce: 0 kB -WritebackTmp: 0 kB -CommitLimit: 39209304 kB -Committed_AS: 11220776 kB -VmallocTotal: 34359738367 kB -VmallocUsed: 87056 kB -VmallocChunk: 0 kB -Percpu: 70272 kB -HardwareCorrupted: 0 kB -AnonHugePages: 235520 kB -ShmemHugePages: 0 kB -ShmemPmdMapped: 0 kB -FileHugePages: 0 kB -FilePmdMapped: 0 kB -CmaTotal: 0 kB -CmaFree: 0 kB -Hugepagesize: 2048 kB -Hugetlb: 0 kB -DirectMap4k: 2092268 kB -DirectMap2M: 51320832 kB -DirectMap1G: 9437184 kB \ No newline at end of file diff --git a/metricbeat/module/linux/memory/data.go b/metricbeat/module/linux/memory/data.go index 7c3b2fc72c5..de41ed5df00 100644 --- a/metricbeat/module/linux/memory/data.go +++ b/metricbeat/module/linux/memory/data.go @@ -25,11 +25,11 @@ import ( "github.com/pkg/errors" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/transform/typeconv" - "github.com/elastic/beats/v7/metricbeat/internal/metrics/memory" - metrics "github.com/elastic/beats/v7/metricbeat/internal/metrics/memory" "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/transform/typeconv" + util "github.com/elastic/elastic-agent-system-metrics/metric" + "github.com/elastic/elastic-agent-system-metrics/metric/memory" + metrics "github.com/elastic/elastic-agent-system-metrics/metric/memory" "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" ) @@ -112,7 +112,7 @@ func computeEfficiency(scanName string, stealName string, fieldName string, raw stealVal, stealOk := raw[stealName] if scanVal != 0 && stealOk { inMap[fieldName] = mapstr.M{ - "pct": common.Round(float64(stealVal)/float64(scanVal), common.DefaultDecimalPlacesCount), + "pct": util.Round(float64(stealVal) / float64(scanVal)), } } @@ -144,7 +144,7 @@ func getHugePages(hostfs resolve.Resolver) (mapstr.M, error) { if total > 0 { perc = float64(total-free+reserved) / float64(total) } - thp.Put("used.pct", common.Round(perc, common.DefaultDecimalPlacesCount)) + thp.Put("used.pct", util.Round(perc)) if !okTotalSize && okDefaultSize { thp.Put("used.bytes", (total-free+reserved)*defaultSize) diff --git a/metricbeat/module/system/core/config.go b/metricbeat/module/system/core/config.go index ae1b01e32de..09b1b25f7ed 100644 --- a/metricbeat/module/system/core/config.go +++ b/metricbeat/module/system/core/config.go @@ -23,7 +23,7 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - metrics "github.com/elastic/beats/v7/metricbeat/internal/metrics/cpu" + metrics "github.com/elastic/elastic-agent-system-metrics/metric/cpu" ) // Core metric types. diff --git a/metricbeat/module/system/core/core.go b/metricbeat/module/system/core/core.go index 0373ced0bd4..efed48e20a7 100644 --- a/metricbeat/module/system/core/core.go +++ b/metricbeat/module/system/core/core.go @@ -23,9 +23,9 @@ package core import ( "github.com/pkg/errors" - metrics "github.com/elastic/beats/v7/metricbeat/internal/metrics/cpu" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" + metrics "github.com/elastic/elastic-agent-system-metrics/metric/cpu" "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" ) diff --git a/metricbeat/module/system/cpu/config.go b/metricbeat/module/system/cpu/config.go index a9f7025b3e2..9533bf7b296 100644 --- a/metricbeat/module/system/cpu/config.go +++ b/metricbeat/module/system/cpu/config.go @@ -23,7 +23,7 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - metrics "github.com/elastic/beats/v7/metricbeat/internal/metrics/cpu" + metrics "github.com/elastic/elastic-agent-system-metrics/metric/cpu" ) // CPU metric types. diff --git a/metricbeat/module/system/cpu/cpu.go b/metricbeat/module/system/cpu/cpu.go index a55968de1c6..a061d86056c 100644 --- a/metricbeat/module/system/cpu/cpu.go +++ b/metricbeat/module/system/cpu/cpu.go @@ -23,10 +23,10 @@ package cpu import ( "github.com/pkg/errors" - metrics "github.com/elastic/beats/v7/metricbeat/internal/metrics/cpu" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" "github.com/elastic/elastic-agent-libs/mapstr" + metrics "github.com/elastic/elastic-agent-system-metrics/metric/cpu" "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" ) diff --git a/metricbeat/module/system/memory/memory.go b/metricbeat/module/system/memory/memory.go index cc4ec230b99..c52e27f181c 100644 --- a/metricbeat/module/system/memory/memory.go +++ b/metricbeat/module/system/memory/memory.go @@ -23,11 +23,11 @@ package memory import ( "github.com/pkg/errors" - "github.com/elastic/beats/v7/libbeat/common/transform/typeconv" - metrics "github.com/elastic/beats/v7/metricbeat/internal/metrics/memory" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/transform/typeconv" + metrics "github.com/elastic/elastic-agent-system-metrics/metric/memory" "github.com/elastic/elastic-agent-system-metrics/metric/system/resolve" ) From c35f70e65a80913c44baf6ce98a475b2b7e47a43 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Thu, 9 Jun 2022 23:55:36 +0800 Subject: [PATCH 02/22] dev-tools/mage: add BuildArgs.BuildMode (#31881) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- dev-tools/mage/build.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dev-tools/mage/build.go b/dev-tools/mage/build.go index e93547d51c0..0432b5b20d1 100644 --- a/dev-tools/mage/build.go +++ b/dev-tools/mage/build.go @@ -37,6 +37,7 @@ type BuildArgs struct { InputFiles []string OutputDir string CGO bool + BuildMode string // Controls `go build -buildmode` Static bool Env map[string]string LDFlags []string @@ -61,7 +62,7 @@ func DefaultBuildArgs() BuildArgs { } if positionIndependentCodeSupported() { - args.ExtraFlags = append(args.ExtraFlags, "-buildmode", "pie") + args.BuildMode = "pie" } if DevBuild { @@ -161,6 +162,9 @@ func Build(params BuildArgs) error { "-o", filepath.Join(params.OutputDir, binaryName), } + if params.BuildMode != "" { + args = append(args, "-buildmode", params.BuildMode) + } args = append(args, params.ExtraFlags...) // ldflags From 01b2984fa6c7f812384f190cd9ca2b9877d13e32 Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Thu, 9 Jun 2022 09:09:16 -0700 Subject: [PATCH 03/22] Add float tags to fix broken topic structure (#31880) --- metricbeat/module/kibana/settings/_meta/docs.asciidoc | 1 + metricbeat/module/kibana/stats/_meta/docs.asciidoc | 1 + 2 files changed, 2 insertions(+) diff --git a/metricbeat/module/kibana/settings/_meta/docs.asciidoc b/metricbeat/module/kibana/settings/_meta/docs.asciidoc index c909358ffc0..93ba8cf495a 100644 --- a/metricbeat/module/kibana/settings/_meta/docs.asciidoc +++ b/metricbeat/module/kibana/settings/_meta/docs.asciidoc @@ -2,6 +2,7 @@ This is the `settings` metricset of the Kibana module. This stats endpoint is av The intention of the Kibana module is to have a minimal data set that works across Kibana versions. +[float] === Module-specific configuration notes If the Kibana instance is using a basepath in its URL, you must set the `basepath` setting for this module with the same value. diff --git a/metricbeat/module/kibana/stats/_meta/docs.asciidoc b/metricbeat/module/kibana/stats/_meta/docs.asciidoc index 316a68e6f57..6bbdf88b1ec 100644 --- a/metricbeat/module/kibana/stats/_meta/docs.asciidoc +++ b/metricbeat/module/kibana/stats/_meta/docs.asciidoc @@ -2,6 +2,7 @@ This is the `stats` metricset of the Kibana module. This stats endpoint is avail The intention of the Kibana module is to have a minimal data set that works across Kibana versions. +[float] === Module-specific configuration notes If the Kibana instance is using a basepath in its URL, you must set the `basepath` setting for this module with the same value. From 636cb35e335253d1613d04087742566381138833 Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Thu, 9 Jun 2022 09:09:52 -0700 Subject: [PATCH 04/22] Update docs.asciidoc (#31769) (#31879) Some minor spelling mistakes Co-authored-by: Damian Pfister --- x-pack/metricbeat/module/istio/proxy/_meta/docs.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/metricbeat/module/istio/proxy/_meta/docs.asciidoc b/x-pack/metricbeat/module/istio/proxy/_meta/docs.asciidoc index 87a15d72a94..b3c5f3a8807 100644 --- a/x-pack/metricbeat/module/istio/proxy/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/istio/proxy/_meta/docs.asciidoc @@ -8,11 +8,11 @@ Tested with Istio 1.7 Istio-proxy is a sidecar container that is being injected into every Pod that is being deployed on a Kubernetes cluster which's traffic is managed by Istio. -Because of this reason, in oder to collect metrics from this sidecars we need +Because of this reason, in order to collect metrics from this sidecars we need to automatically identify these sidecar containers and start monitoring them using their IP and the predifined port (15090). This can be achieved easily by -defining the proper autodiscover provider that will automatically identifies -all these sidecar containers and will start `proxy` metricset for each one of +defining the proper autodiscover provider that will automatically identify +all these sidecar containers and will start the `proxy` metricset for each one of them. Here is an example configuration that can be used for that purpose: From 1d1270991bd9451912f32b647c41087968375b72 Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Thu, 9 Jun 2022 09:10:30 -0700 Subject: [PATCH 05/22] Capitalize Beat name by using correct attribute (#31878) --- libbeat/outputs/kafka/docs/kafka.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/outputs/kafka/docs/kafka.asciidoc b/libbeat/outputs/kafka/docs/kafka.asciidoc index cd9e9fa65c9..2c0a2b1c292 100644 --- a/libbeat/outputs/kafka/docs/kafka.asciidoc +++ b/libbeat/outputs/kafka/docs/kafka.asciidoc @@ -62,11 +62,11 @@ The cluster metadata contain the actual Kafka brokers events are published to. ===== `version` -Kafka protocol version that {beatname_lc} will request when connecting. Defaults to 1.0.0. +Kafka protocol version that {beatname_uc} will request when connecting. Defaults to 1.0.0. Valid values are all kafka releases in between `0.8.2.0` and `2.6.0`. -The protocol version controls the Kafka client features available to {beatname_lc}, it does not prevent {beatname_lc} from connecting to Kafka versions newer than the protocol version. +The protocol version controls the Kafka client features available to {beatname_uc}; it does not prevent {beatname_uc} from connecting to Kafka versions newer than the protocol version. See <> for information on supported versions. From 795195ea473ee0aeaa1d2d1f1102b8b5216da392 Mon Sep 17 00:00:00 2001 From: liuwenping Date: Fri, 10 Jun 2022 01:12:27 +0800 Subject: [PATCH 06/22] [Filebeat] keep track of bytes read when max_bytes exceeded in the last line (#31882) --- libbeat/reader/readfile/line.go | 5 ++++- libbeat/reader/readfile/line_test.go | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/libbeat/reader/readfile/line.go b/libbeat/reader/readfile/line.go index e13b2a8fb00..c36832e514e 100644 --- a/libbeat/reader/readfile/line.go +++ b/libbeat/reader/readfile/line.go @@ -88,7 +88,10 @@ func (r *LineReader) Next() (b []byte, n int, err error) { // read next 'potential' line from input buffer/reader err := r.advance() if err != nil { - return nil, 0, err + // return and reset consumed bytes count + sz := r.byteCount + r.byteCount = 0 + return nil, sz, err } // Check last decoded byte really being newline also unencoded diff --git a/libbeat/reader/readfile/line_test.go b/libbeat/reader/readfile/line_test.go index 68f257bf45d..85323ea32c2 100644 --- a/libbeat/reader/readfile/line_test.go +++ b/libbeat/reader/readfile/line_test.go @@ -373,6 +373,7 @@ func TestMaxBytesLimit(t *testing.T) { b, n, err := reader.Next() if err != nil { if err == io.EOF { + readLen += n break } else { t.Fatal("unexpected error:", err) From 96c586c37c58cad67be9ffb9ad51a144180ad18c Mon Sep 17 00:00:00 2001 From: Andrew Cholakian Date: Thu, 9 Jun 2022 22:19:04 -0500 Subject: [PATCH 07/22] [Heartbeat] Fix origin, Cleanup stdfields (#31877) Fixes #31870 and refactors the wrapper code to be cleaner and more consistent. Prior to this PR we would only set the monitor.origin field if the monitor.origin config setting was set on non-browser (e.g. HTTP) monitors. The reason the original patch only worked on lightweight monitors was that there has historically been a separate codepath for adding simple core metadata fields to lightweight monitors than browser based monitors. This PR removes that codepath, funneling the writing of most browser monitor.* fields through addMonitorMeta in wrappers.go. This means that we'll have more consistent behavior between browser and regular monitors going forward. This entailed some untangling of some funky code to get working, apologies for the larger diff, but it leaves the codebase more consistent than we found it. One weird wrinkle is the monitor.project field, which we don't currently use, but customers might. This contains the literal id and name from the monitors YAML when running zip_url or local monitors. This does keep the 8.3 change moving this from suite.{id,name} to monitor.project.{id,name} but does it in a cleaner way, labeling as many relevant variables, tests, and functions as legacy. One day we can remove this when / if we remove support for zip_url and local browser sources. We should probably do this before the beta is over. This PR also tries to more consistently refer to instances of stdfields.StandardFields as sFields when used as a var. --- heartbeat/monitors/stdfields/stdfields.go | 6 +- heartbeat/monitors/wrappers/wrappers.go | 42 ++++++--- heartbeat/monitors/wrappers/wrappers_test.go | 90 +++++++++++++++---- x-pack/heartbeat/monitors/browser/project.go | 30 +++---- .../monitors/browser/project_test.go | 6 +- .../monitors/browser/synthexec/enrich.go | 60 ++++--------- .../monitors/browser/synthexec/enrich_test.go | 81 ++++++----------- .../browser/synthexec/execmultiplexer_test.go | 2 +- .../monitors/browser/synthexec/synthexec.go | 24 ++--- .../browser/synthexec/synthexec_test.go | 2 +- .../monitors/browser/synthexec/synthtypes.go | 6 +- 11 files changed, 186 insertions(+), 163 deletions(-) diff --git a/heartbeat/monitors/stdfields/stdfields.go b/heartbeat/monitors/stdfields/stdfields.go index 2be2a89b27b..b77e8129f64 100644 --- a/heartbeat/monitors/stdfields/stdfields.go +++ b/heartbeat/monitors/stdfields/stdfields.go @@ -39,7 +39,11 @@ type StdMonitorFields struct { Service ServiceFields `config:"service"` Origin string `config:"origin"` LegacyServiceName string `config:"service_name"` - Enabled bool `config:"enabled"` + // Used by zip_url and local monitors + // kibana originating monitors only run one journey at a time + // and just use the `fields` syntax / manually set monitor IDs + IsLegacyBrowserSource bool + Enabled bool `config:"enabled"` } func ConfigToStdMonitorFields(config *conf.C) (StdMonitorFields, error) { diff --git a/heartbeat/monitors/wrappers/wrappers.go b/heartbeat/monitors/wrappers/wrappers.go index 7b9d8bca433..341dc6827ee 100644 --- a/heartbeat/monitors/wrappers/wrappers.go +++ b/heartbeat/monitors/wrappers/wrappers.go @@ -69,38 +69,60 @@ func WrapBrowser(js []jobs.Job, stdMonFields stdfields.StdMonitorFields) []jobs. js, addMonitorTimespan(stdMonFields), addServiceName(stdMonFields), + addMonitorMeta(stdMonFields, false), addMonitorStatus(true), ) } // addMonitorMeta adds the id, name, and type fields to the monitor. -func addMonitorMeta(sf stdfields.StdMonitorFields, isMulti bool) jobs.JobWrapper { +func addMonitorMeta(sFields stdfields.StdMonitorFields, hashURLIntoID bool) jobs.JobWrapper { return func(job jobs.Job) jobs.Job { return func(event *beat.Event) ([]jobs.Job, error) { cont, err := job(event) - id := sf.ID - name := sf.Name + id := sFields.ID + name := sFields.Name // If multiple jobs are listed for this monitor, we can't have a single ID, so we hash the // unique URLs to create unique suffixes for the monitor. - if isMulti { + if hashURLIntoID { url, err := event.GetValue("url.full") if err != nil { logp.Error(fmt.Errorf("mandatory url.full key missing: %w", err)) url = "n/a" } urlHash, _ := hashstructure.Hash(url, nil) - id = fmt.Sprintf("%s-%x", sf.ID, urlHash) + id = fmt.Sprintf("%s-%x", sFields.ID, urlHash) } fields := mapstr.M{ - "id": id, - "name": name, - "type": sf.Type, + "type": sFields.Type, } - if sf.Origin != "" { - fields["origin"] = sf.Origin + if !sFields.IsLegacyBrowserSource { + fields["id"] = id + fields["name"] = name + } else { + // Id and name differs for local + // - We use the monitor id and name for inline journeys ignoring the + // autogenerated `inline`journey id and name. + // - Monitor id/name is concatenated with the journey id/name for + // project monitors + journeyId, err := event.GetValue("monitor.id") + if err == nil { + fields["id"] = fmt.Sprintf("%s-%s", sFields.ID, journeyId) + } + journeyName, err := event.GetValue("monitor.name") + if err == nil { + fields["name"] = fmt.Sprintf("%s - %s", sFields.Name, journeyName) + } + fields["project"] = mapstr.M{ + "id": sFields.ID, + "name": sFields.Name, + } + } + + if sFields.Origin != "" { + fields["origin"] = sFields.Origin } eventext.MergeEventFields(event, mapstr.M{"monitor": fields}) diff --git a/heartbeat/monitors/wrappers/wrappers_test.go b/heartbeat/monitors/wrappers/wrappers_test.go index 4c2dbac2147..08aef1a9401 100644 --- a/heartbeat/monitors/wrappers/wrappers_test.go +++ b/heartbeat/monitors/wrappers/wrappers_test.go @@ -47,7 +47,7 @@ import ( type testDef struct { name string - stdFields stdfields.StdMonitorFields + sFields stdfields.StdMonitorFields jobs []jobs.Job want []validator.Validator metaWant []validator.Validator @@ -70,7 +70,7 @@ var testBrowserMonFields = stdfields.StdMonitorFields{ func testCommonWrap(t *testing.T, tt testDef) { t.Run(tt.name, func(t *testing.T) { - wrapped := WrapCommon(tt.jobs, tt.stdFields) + wrapped := WrapCommon(tt.jobs, tt.sFields) core, observedLogs := observer.New(zapcore.InfoLevel) logger.SetLogger(logp.NewLogger("t", zap.WrapCore(func(in zapcore.Core) zapcore.Core { @@ -466,6 +466,10 @@ type BrowserMonitor struct { id string name string checkGroup string + // Used for testing legacy zip_url and local monitors + // where the top-level id/name are used to populate monitor.project + legacyProjectId string + legacyProjectName string } var inlineMonitorValues = BrowserMonitor{ @@ -482,8 +486,6 @@ func makeInlineBrowserJob(t *testing.T, u string) jobs.Job { "url": URLFields(parsed), "monitor": mapstr.M{ "type": "browser", - "id": inlineMonitorValues.id, - "name": inlineMonitorValues.name, "check_group": inlineMonitorValues.checkGroup, }, }) @@ -494,10 +496,12 @@ func makeInlineBrowserJob(t *testing.T, u string) jobs.Job { // Browser inline jobs monitor information should not be altered // by the wrappers as they are handled separately in synth enricher func TestInlineBrowserJob(t *testing.T) { - fields := testBrowserMonFields + sFields := testBrowserMonFields + sFields.ID = inlineMonitorValues.id + sFields.Name = inlineMonitorValues.name testCommonWrap(t, testDef{ "simple", - fields, + sFields, []jobs.Job{makeInlineBrowserJob(t, "http://foo.com")}, []validator.Validator{ lookslike.Strict( @@ -526,7 +530,16 @@ var projectMonitorValues = BrowserMonitor{ checkGroup: "journey-1-check-group", } -func makeProjectBrowserJob(t *testing.T, u string, summary bool, projectErr error) jobs.Job { +// Used for testing legacy zip_url / local monitorss +var legacyProjectMonitorValues = BrowserMonitor{ + id: "journey-1", + name: "Journey 1", + checkGroup: "acheckgroup", + legacyProjectId: "my-project", + legacyProjectName: "My Project", +} + +func makeProjectBrowserJob(t *testing.T, u string, summary bool, projectErr error, bm BrowserMonitor) jobs.Job { parsed, err := url.Parse(u) require.NoError(t, err) return func(event *beat.Event) (i []jobs.Job, e error) { @@ -534,9 +547,9 @@ func makeProjectBrowserJob(t *testing.T, u string, summary bool, projectErr erro "url": URLFields(parsed), "monitor": mapstr.M{ "type": "browser", - "id": projectMonitorValues.id, - "name": projectMonitorValues.name, - "check_group": projectMonitorValues.checkGroup, + "id": bm.id, + "name": bm.name, + "check_group": bm.checkGroup, }, }) if summary { @@ -555,14 +568,19 @@ func makeProjectBrowserJob(t *testing.T, u string, summary bool, projectErr erro } func TestProjectBrowserJob(t *testing.T) { - fields := testBrowserMonFields + sFields := testBrowserMonFields + sFields.ID = projectMonitorValues.id + sFields.Name = projectMonitorValues.name + sFields.Origin = "my-origin" urlStr := "http://foo.com" urlU, _ := url.Parse(urlStr) + expectedMonFields := lookslike.MustCompile(map[string]interface{}{ "monitor": map[string]interface{}{ "type": "browser", "id": projectMonitorValues.id, "name": projectMonitorValues.name, + "origin": "my-origin", "check_group": projectMonitorValues.checkGroup, "timespan": mapstr.M{ "gte": hbtestllext.IsTime, @@ -571,10 +589,11 @@ func TestProjectBrowserJob(t *testing.T) { }, "url": URLFields(urlU), }) + testCommonWrap(t, testDef{ "simple", // has no summary fields! - fields, - []jobs.Job{makeProjectBrowserJob(t, urlStr, false, nil)}, + sFields, + []jobs.Job{makeProjectBrowserJob(t, urlStr, false, nil, projectMonitorValues)}, []validator.Validator{ lookslike.Strict( lookslike.Compose( @@ -586,8 +605,8 @@ func TestProjectBrowserJob(t *testing.T) { }) testCommonWrap(t, testDef{ "with up summary", - fields, - []jobs.Job{makeProjectBrowserJob(t, urlStr, true, nil)}, + sFields, + []jobs.Job{makeProjectBrowserJob(t, urlStr, true, nil, projectMonitorValues)}, []validator.Validator{ lookslike.Strict( lookslike.Compose( @@ -603,8 +622,8 @@ func TestProjectBrowserJob(t *testing.T) { }) testCommonWrap(t, testDef{ "with down summary", - fields, - []jobs.Job{makeProjectBrowserJob(t, urlStr, true, fmt.Errorf("testerr"))}, + sFields, + []jobs.Job{makeProjectBrowserJob(t, urlStr, true, fmt.Errorf("testerr"), projectMonitorValues)}, []validator.Validator{ lookslike.Strict( lookslike.Compose( @@ -622,4 +641,41 @@ func TestProjectBrowserJob(t *testing.T) { nil, nil, }) + + legacySFields := testBrowserMonFields + legacySFields.ID = legacyProjectMonitorValues.legacyProjectId + legacySFields.Name = legacyProjectMonitorValues.legacyProjectName + legacySFields.IsLegacyBrowserSource = true + + expectedLegacyMonFields := lookslike.MustCompile(map[string]interface{}{ + "monitor": map[string]interface{}{ + "type": "browser", + "id": fmt.Sprintf("%s-%s", legacyProjectMonitorValues.legacyProjectId, legacyProjectMonitorValues.id), + "name": fmt.Sprintf("%s - %s", legacyProjectMonitorValues.legacyProjectName, legacyProjectMonitorValues.name), + "project": mapstr.M{ + "id": legacyProjectMonitorValues.legacyProjectId, + "name": legacyProjectMonitorValues.legacyProjectName, + }, + "check_group": legacyProjectMonitorValues.checkGroup, + "timespan": mapstr.M{ + "gte": hbtestllext.IsTime, + "lt": hbtestllext.IsTime, + }, + }, + "url": URLFields(urlU), + }) + + testCommonWrap(t, testDef{ + "legacy", // has no summary fields! + legacySFields, + []jobs.Job{makeProjectBrowserJob(t, urlStr, false, nil, legacyProjectMonitorValues)}, + []validator.Validator{ + lookslike.Strict( + lookslike.Compose( + urlValidator(t, urlStr), + expectedLegacyMonFields, + ))}, + nil, + nil, + }) } diff --git a/x-pack/heartbeat/monitors/browser/project.go b/x-pack/heartbeat/monitors/browser/project.go index abc4ccc6a4b..34d022f2d8d 100644 --- a/x-pack/heartbeat/monitors/browser/project.go +++ b/x-pack/heartbeat/monitors/browser/project.go @@ -11,6 +11,7 @@ import ( "github.com/elastic/beats/v7/heartbeat/monitors/jobs" "github.com/elastic/beats/v7/heartbeat/monitors/plugin" + "github.com/elastic/beats/v7/heartbeat/monitors/stdfields" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser/synthexec" "github.com/elastic/elastic-agent-libs/config" @@ -54,13 +55,6 @@ func (p *Project) Workdir() string { return p.projectCfg.Source.Active().Workdir() } -func (p *Project) InlineSource() (string, bool) { - if p.projectCfg.Source.Inline != nil { - return p.projectCfg.Source.Inline.Script, true - } - return "", false -} - func (p *Project) Params() map[string]interface{} { return p.projectCfg.Params } @@ -69,14 +63,18 @@ func (p *Project) FilterJourneys() synthexec.FilterJourneyConfig { return p.projectCfg.FilterJourneys } -func (p *Project) Fields() synthexec.StdProjectFields { - _, isInline := p.InlineSource() - return synthexec.StdProjectFields{ - Name: p.projectCfg.Name, - Id: p.projectCfg.Id, - IsInline: isInline, - Type: "browser", +func (p *Project) Fields() stdfields.StdMonitorFields { + sFields := stdfields.StdMonitorFields{ + Name: p.projectCfg.Name, + ID: p.projectCfg.Id, + Type: "browser", } + + if p.projectCfg.Source.Local != nil || p.projectCfg.Source.ZipUrl != nil { + sFields.IsLegacyBrowserSource = true + } + + return sFields } func (p *Project) Close() error { @@ -130,7 +128,9 @@ func (p *Project) extraArgs() []string { func (p *Project) jobs() []jobs.Job { var j jobs.Job - if src, ok := p.InlineSource(); ok { + isScript := p.projectCfg.Source.Inline != nil + if isScript { + src := p.projectCfg.Source.Inline.Script j = synthexec.InlineJourneyJob(context.TODO(), src, p.Params(), p.Fields(), p.extraArgs()...) } else { j = func(event *beat.Event) ([]jobs.Job, error) { diff --git a/x-pack/heartbeat/monitors/browser/project_test.go b/x-pack/heartbeat/monitors/browser/project_test.go index 9213f89e389..d3a03c13280 100644 --- a/x-pack/heartbeat/monitors/browser/project_test.go +++ b/x-pack/heartbeat/monitors/browser/project_test.go @@ -44,8 +44,6 @@ func TestValidLocal(t *testing.T) { s, e := NewProject(cfg) require.NoError(t, e) require.NotNil(t, s) - _, ok := s.InlineSource() - require.False(t, ok) source.GoOffline() defer source.GoOnline() @@ -77,9 +75,7 @@ func TestValidInline(t *testing.T) { s, e := NewProject(cfg) require.NoError(t, e) require.NotNil(t, s) - sSrc, ok := s.InlineSource() - require.True(t, ok) - require.Equal(t, script, sSrc) + require.Equal(t, script, s.projectCfg.Source.Inline.Script) require.Equal(t, "", s.Workdir()) require.Equal(t, testParams, s.Params()) diff --git a/x-pack/heartbeat/monitors/browser/synthexec/enrich.go b/x-pack/heartbeat/monitors/browser/synthexec/enrich.go index ac3b43955cd..d36acb43ca8 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/enrich.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/enrich.go @@ -16,21 +16,23 @@ import ( "github.com/elastic/beats/v7/heartbeat/eventext" "github.com/elastic/beats/v7/heartbeat/monitors/logger" + "github.com/elastic/beats/v7/heartbeat/monitors/stdfields" "github.com/elastic/beats/v7/libbeat/beat" ) -type enricher func(event *beat.Event, se *SynthEvent, fields StdProjectFields) error +type enricher func(event *beat.Event, se *SynthEvent) error type streamEnricher struct { - je *journeyEnricher + je *journeyEnricher + sFields stdfields.StdMonitorFields } -func (e *streamEnricher) enrich(event *beat.Event, se *SynthEvent, fields StdProjectFields) error { - if e.je == nil || (se != nil && se.Type == JourneyStart) { - e.je = newJourneyEnricher() +func (senr *streamEnricher) enrich(event *beat.Event, se *SynthEvent) error { + if senr.je == nil || (se != nil && se.Type == JourneyStart) { + senr.je = newJourneyEnricher(senr) } - return e.je.enrich(event, se, fields) + return senr.je.enrich(event, se) } // journeyEnricher holds state across received SynthEvents retaining fields @@ -44,14 +46,16 @@ type journeyEnricher struct { stepCount int // The first URL we visit is the URL for this journey, which is set on the summary event. // We store the URL fields here for use on the summary event. - urlFields mapstr.M - start time.Time - end time.Time + urlFields mapstr.M + start time.Time + end time.Time + streamEnricher *streamEnricher } -func newJourneyEnricher() *journeyEnricher { +func newJourneyEnricher(senr *streamEnricher) *journeyEnricher { return &journeyEnricher{ - checkGroup: makeUuid(), + checkGroup: makeUuid(), + streamEnricher: senr, } } @@ -63,7 +67,7 @@ func makeUuid() string { return u.String() } -func (je *journeyEnricher) enrich(event *beat.Event, se *SynthEvent, fields StdProjectFields) error { +func (je *journeyEnricher) enrich(event *beat.Event, se *SynthEvent) error { if se == nil { return nil } @@ -84,40 +88,10 @@ func (je *journeyEnricher) enrich(event *beat.Event, se *SynthEvent, fields StdP event.Timestamp = time.Now() } - // Id and name differs for inline and project monitors - // - We use the monitor id and name for inline journeys ignoring the - // autogenerated `inline`journey id and name. - // - Monitor id/name is concatenated with the journey id/name for - // project monitors - id := fields.Id - name := fields.Name - if !fields.IsInline && je.journey != nil { - id = fmt.Sprintf("%s-%s", id, je.journey.Id) - name = fmt.Sprintf("%s - %s", name, je.journey.Name) - } eventext.MergeEventFields(event, mapstr.M{ - "event": mapstr.M{ - "type": se.Type, - }, - "monitor": mapstr.M{ - "check_group": je.checkGroup, - "id": id, - "name": name, - "type": fields.Type, - }, + "event.type": se.Type, }) - // Write project level fields for project monitors - if !fields.IsInline { - eventext.MergeEventFields(event, mapstr.M{ - "monitor": mapstr.M{ - "project": mapstr.M{ - "id": fields.Id, - "name": fields.Name, - }, - }, - }) - } return je.enrichSynthEvent(event, se) } diff --git a/x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go b/x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go index 24bdeb1324d..20022cb425a 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go @@ -16,6 +16,7 @@ import ( "go.uber.org/zap/zaptest/observer" "github.com/elastic/beats/v7/heartbeat/monitors/logger" + "github.com/elastic/beats/v7/heartbeat/monitors/stdfields" "github.com/elastic/beats/v7/heartbeat/monitors/wrappers" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/beat/events" @@ -40,15 +41,14 @@ func makeStepEvent(typ string, ts float64, name string, index int, status string } func TestJourneyEnricher(t *testing.T) { - var stdFields = StdProjectFields{ - Id: "myproject", - Name: "myproject", - Type: "browser", - IsInline: false, + var sFields = stdfields.StdMonitorFields{ + ID: "myproject", + Name: "myproject", + Type: "browser", } journey := &Journey{ Name: "A Journey Name", - Id: "my-journey-id", + ID: "my-journey-id", } syntherr := &SynthError{ Message: "my-errmsg", @@ -89,23 +89,7 @@ func TestJourneyEnricher(t *testing.T) { journeyEnd, } - projectValidator := func() validator.Validator { - return lookslike.MustCompile(mapstr.M{ - "monitor.project.id": stdFields.Id, - "monitor.project.name": stdFields.Name, - "monitor.id": fmt.Sprintf("%s-%s", stdFields.Id, journey.Id), - "monitor.name": fmt.Sprintf("%s - %s", stdFields.Name, journey.Name), - "monitor.type": stdFields.Type, - }) - } - inlineValidator := func() validator.Validator { - return lookslike.MustCompile(mapstr.M{ - "monitor.id": stdFields.Id, - "monitor.name": stdFields.Name, - "monitor.type": stdFields.Type, - }) - } - commonValidator := func(se *SynthEvent) validator.Validator { + valid := func(se *SynthEvent) validator.Validator { var v []validator.Validator // We need an expectation for each input plus a final @@ -126,46 +110,40 @@ func TestJourneyEnricher(t *testing.T) { return lookslike.Compose(v...) } - je := &journeyEnricher{} - check := func(t *testing.T, se *SynthEvent, ssf StdProjectFields) { + check := func(t *testing.T, se *SynthEvent, je *journeyEnricher) { e := &beat.Event{} t.Run(fmt.Sprintf("event: %s", se.Type), func(t *testing.T) { - enrichErr := je.enrich(e, se, ssf) + enrichErr := je.enrich(e, se) if se.Error != nil { require.Equal(t, stepError(se.Error), enrichErr) } - if ssf.IsInline { - sv, _ := e.Fields.GetValue("monitor.project") - require.Nil(t, sv) - testslike.Test(t, inlineValidator(), e.Fields) - } else { - testslike.Test(t, projectValidator(), e.Fields) - } - testslike.Test(t, commonValidator(se), e.Fields) + + testslike.Test(t, valid(se), e.Fields) require.Equal(t, se.Timestamp().Unix(), e.Timestamp.Unix()) }) } tests := []struct { - name string - isInline bool + name string + IsLegacyBrowserSource bool }{ { - name: "project monitor", - isInline: false, + name: "legacy project monitor", + IsLegacyBrowserSource: true, }, { - name: "inline monitor", - isInline: true, + name: "modern monitor", + IsLegacyBrowserSource: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - stdFields.IsInline = tt.isInline + sFields.IsLegacyBrowserSource = tt.IsLegacyBrowserSource + je := &journeyEnricher{streamEnricher: &streamEnricher{sFields: sFields}} for _, se := range synthEvents { - check(t, se, stdFields) + check(t, se, je) } }) } @@ -362,7 +340,7 @@ func TestEnrichSynthEvent(t *testing.T) { func TestNoSummaryOnAfterHook(t *testing.T) { journey := &Journey{ Name: "A journey that fails after completing", - Id: "my-bad-after-all-hook", + ID: "my-bad-after-all-hook", } journeyStart := &SynthEvent{ Type: JourneyStart, @@ -398,13 +376,13 @@ func TestNoSummaryOnAfterHook(t *testing.T) { cmdStatus, } - je := &journeyEnricher{} - + stdFields := stdfields.StdMonitorFields{} + je := &journeyEnricher{streamEnricher: &streamEnricher{sFields: stdFields}} for idx, se := range synthEvents { e := &beat.Event{} - stdFields := StdProjectFields{IsInline: false} + t.Run(fmt.Sprintf("event %d", idx), func(t *testing.T) { - enrichErr := je.enrich(e, se, stdFields) + enrichErr := je.enrich(e, se) if se != nil && se.Type == CmdStatus { t.Run("no summary in cmd/status", func(t *testing.T) { @@ -435,7 +413,7 @@ func TestNoSummaryOnAfterHook(t *testing.T) { func TestSummaryWithoutJourneyEnd(t *testing.T) { journey := &Journey{ Name: "A journey that never emits journey/end but exits successfully", - Id: "no-journey-end-but-success", + ID: "no-journey-end-but-success", } journeyStart := &SynthEvent{ Type: "journey/start", @@ -458,15 +436,14 @@ func TestSummaryWithoutJourneyEnd(t *testing.T) { cmdStatus, } - je := &journeyEnricher{} - hasCmdStatus := false + stdFields := stdfields.StdMonitorFields{} + je := &journeyEnricher{streamEnricher: &streamEnricher{sFields: stdFields}} for idx, se := range synthEvents { e := &beat.Event{} - stdFields := StdProjectFields{IsInline: false} t.Run(fmt.Sprintf("event %d", idx), func(t *testing.T) { - enrichErr := je.enrich(e, se, stdFields) + enrichErr := je.enrich(e, se) if se != nil && se.Type == CmdStatus { hasCmdStatus = true diff --git a/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer_test.go b/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer_test.go index ff2641efd1c..2980a6da8ed 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer_test.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer_test.go @@ -21,7 +21,7 @@ func TestExecMultiplexer(t *testing.T) { time++ // fake time to make events seem spaced out journey := &Journey{ Name: fmt.Sprintf("J%d", jIdx), - Id: fmt.Sprintf("j-%d", jIdx), + ID: fmt.Sprintf("j-%d", jIdx), } testEvents = append(testEvents, &SynthEvent{ Journey: journey, diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go b/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go index 839105a8334..8fa198a497b 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go @@ -19,6 +19,7 @@ import ( "time" "github.com/elastic/beats/v7/heartbeat/monitors/jobs" + "github.com/elastic/beats/v7/heartbeat/monitors/stdfields" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -26,20 +27,13 @@ import ( const debugSelector = "synthexec" -type StdProjectFields struct { - Name string - Id string - Type string - IsInline bool -} - type FilterJourneyConfig struct { Tags []string `config:"tags"` Match string `config:"match"` } // ProjectJob will run a single journey by name from the given project. -func ProjectJob(ctx context.Context, projectPath string, params mapstr.M, filterJourneys FilterJourneyConfig, fields StdProjectFields, extraArgs ...string) (jobs.Job, error) { +func ProjectJob(ctx context.Context, projectPath string, params mapstr.M, filterJourneys FilterJourneyConfig, fields stdfields.StdMonitorFields, extraArgs ...string) (jobs.Job, error) { // Run the command in the given projectPath, use '.' as the first arg since the command runs // in the correct dir cmdFactory, err := projectCommandFactory(projectPath, extraArgs...) @@ -71,7 +65,7 @@ func projectCommandFactory(projectPath string, args ...string) (func() *exec.Cmd } // InlineJourneyJob returns a job that runs the given source as a single journey. -func InlineJourneyJob(ctx context.Context, script string, params mapstr.M, fields StdProjectFields, extraArgs ...string) jobs.Job { +func InlineJourneyJob(ctx context.Context, script string, params mapstr.M, fields stdfields.StdMonitorFields, extraArgs ...string) jobs.Job { newCmd := func() *exec.Cmd { return exec.Command("elastic-synthetics", append(extraArgs, "--inline")...) //nolint:gosec // we are safely building a command here, users can add args at their own risk } @@ -82,25 +76,25 @@ func InlineJourneyJob(ctx context.Context, script string, params mapstr.M, field // startCmdJob adapts commands into a heartbeat job. This is a little awkward given that the command's output is // available via a sequence of events in the multiplexer, while heartbeat jobs are tail recursive continuations. // Here, we adapt one to the other, where each recursive job pulls another item off the chan until none are left. -func startCmdJob(ctx context.Context, newCmd func() *exec.Cmd, stdinStr *string, params mapstr.M, filterJourneys FilterJourneyConfig, fields StdProjectFields) jobs.Job { +func startCmdJob(ctx context.Context, newCmd func() *exec.Cmd, stdinStr *string, params mapstr.M, filterJourneys FilterJourneyConfig, sFields stdfields.StdMonitorFields) jobs.Job { return func(event *beat.Event) ([]jobs.Job, error) { mpx, err := runCmd(ctx, newCmd(), stdinStr, params, filterJourneys) if err != nil { return nil, err } - senr := streamEnricher{} - return []jobs.Job{readResultsJob(ctx, mpx.SynthEvents(), senr.enrich, fields)}, nil + senr := streamEnricher{sFields: sFields} + return []jobs.Job{readResultsJob(ctx, mpx.SynthEvents(), senr.enrich)}, nil } } // readResultsJob adapts the output of an ExecMultiplexer into a Job, that uses continuations // to read all output. -func readResultsJob(ctx context.Context, synthEvents <-chan *SynthEvent, enrich enricher, fields StdProjectFields) jobs.Job { +func readResultsJob(ctx context.Context, synthEvents <-chan *SynthEvent, enrich enricher) jobs.Job { return func(event *beat.Event) (conts []jobs.Job, err error) { se := <-synthEvents - err = enrich(event, se, fields) + err = enrich(event, se) if se != nil { - return []jobs.Job{readResultsJob(ctx, synthEvents, enrich, fields)}, err + return []jobs.Job{readResultsJob(ctx, synthEvents, enrich)}, err } else { return nil, err } diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthexec_test.go b/x-pack/heartbeat/monitors/browser/synthexec/synthexec_test.go index 4ce9fac5578..87c69488c3d 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthexec_test.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthexec_test.go @@ -61,7 +61,7 @@ func TestJsonToSynthEvent(t *testing.T) { Type: "step/end", Journey: &Journey{ Name: "inline", - Id: "inline", + ID: "inline", }, Step: &Step{ Name: "Go to home page", diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go index ad0b87de1ca..9777918fc55 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go @@ -151,7 +151,7 @@ func (s *Step) ToMap() mapstr.M { type Journey struct { Name string `json:"name"` - Id string `json:"id"` + ID string `json:"id"` Tags []string `json:"tags"` } @@ -159,12 +159,12 @@ func (j Journey) ToMap() mapstr.M { if len(j.Tags) > 0 { return mapstr.M{ "name": j.Name, - "id": j.Id, + "id": j.ID, "tags": j.Tags, } } return mapstr.M{ "name": j.Name, - "id": j.Id, + "id": j.ID, } } From 10b6c202a2f86a76a3311336d67b31be962a1955 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Mon, 13 Jun 2022 11:16:56 -0400 Subject: [PATCH 08/22] rename queue.Batch.ACK -> queue.Batch.Done (#31903) --- CHANGELOG-developer.next.asciidoc | 1 + libbeat/publisher/pipeline/ttl_batch.go | 8 +++---- .../queue/diskqueue/benchmark_test.go | 2 +- libbeat/publisher/queue/diskqueue/consumer.go | 2 +- libbeat/publisher/queue/memqueue/ackloop.go | 2 +- libbeat/publisher/queue/memqueue/broker.go | 24 +++++++++---------- libbeat/publisher/queue/memqueue/eventloop.go | 4 ++-- .../publisher/queue/memqueue/internal_api.go | 4 ++-- .../publisher/queue/memqueue/queue_test.go | 2 +- libbeat/publisher/queue/queue.go | 6 ++--- .../queue/queuetest/producer_cancel.go | 2 +- .../publisher/queue/queuetest/queuetest.go | 2 +- 12 files changed, 30 insertions(+), 29 deletions(-) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 2c04e5965d3..8852863ee98 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -63,6 +63,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only. - Remove `common.MapStr` and use `mapstr.M` from `github.com/elastic/elastic-agent-libs` instead. {pull}31420[31420] - Remove `queue.Consumer`. Queues can now be read via a `Get` call directly on the queue object. {pull}31502[31502] - The `queue.Batch` API now provides access to individual events instead of an array. {pull}31699[31699] +- Rename `queue.Batch.ACK()` to `queue.Batch.Done()`. {pull}31903[31903] ==== Bugfixes diff --git a/libbeat/publisher/pipeline/ttl_batch.go b/libbeat/publisher/pipeline/ttl_batch.go index 897c0756719..39155442ba0 100644 --- a/libbeat/publisher/pipeline/ttl_batch.go +++ b/libbeat/publisher/pipeline/ttl_batch.go @@ -29,7 +29,7 @@ type retryer interface { type ttlBatch struct { // The callback to inform the queue (and possibly the producer) // that this batch has been acknowledged. - ack func() + done func() // The internal hook back to the eventConsumer, used to implement the // publisher.Batch retry interface. @@ -61,7 +61,7 @@ func newBatch(retryer retryer, original queue.Batch, ttl int) *ttlBatch { } b := &ttlBatch{ - ack: original.ACK, + done: original.Done, retryer: retryer, ttl: ttl, events: events, @@ -74,11 +74,11 @@ func (b *ttlBatch) Events() []publisher.Event { } func (b *ttlBatch) ACK() { - b.ack() + b.done() } func (b *ttlBatch) Drop() { - b.ack() + b.done() } func (b *ttlBatch) Retry() { diff --git a/libbeat/publisher/queue/diskqueue/benchmark_test.go b/libbeat/publisher/queue/diskqueue/benchmark_test.go index 62d791f53b0..719237d512f 100644 --- a/libbeat/publisher/queue/diskqueue/benchmark_test.go +++ b/libbeat/publisher/queue/diskqueue/benchmark_test.go @@ -101,7 +101,7 @@ func produceAndConsume(p queue.Producer, q *diskQueue, num_events int, batch_siz if err != nil { return err } - batch.ACK() + batch.Done() received = received + batch.Count() if received == num_events { break diff --git a/libbeat/publisher/queue/diskqueue/consumer.go b/libbeat/publisher/queue/diskqueue/consumer.go index 2380fd93717..1e7d6361383 100644 --- a/libbeat/publisher/queue/diskqueue/consumer.go +++ b/libbeat/publisher/queue/diskqueue/consumer.go @@ -90,6 +90,6 @@ func (batch *diskQueueBatch) Event(i int) interface{} { return batch.frames[i].event } -func (batch *diskQueueBatch) ACK() { +func (batch *diskQueueBatch) Done() { batch.queue.acks.addFrames(batch.frames) } diff --git a/libbeat/publisher/queue/memqueue/ackloop.go b/libbeat/publisher/queue/memqueue/ackloop.go index 1b1979dc834..6d467e287a9 100644 --- a/libbeat/publisher/queue/memqueue/ackloop.go +++ b/libbeat/publisher/queue/memqueue/ackloop.go @@ -109,7 +109,7 @@ func (l *ackLoop) collectAcked() chanList { for !l.ackChans.empty() && !done { acks := l.ackChans.front() select { - case <-acks.ackChan: + case <-acks.doneChan: lst.append(l.ackChans.pop()) default: diff --git a/libbeat/publisher/queue/memqueue/broker.go b/libbeat/publisher/queue/memqueue/broker.go index cf043b02714..ed4c91565bb 100644 --- a/libbeat/publisher/queue/memqueue/broker.go +++ b/libbeat/publisher/queue/memqueue/broker.go @@ -96,9 +96,9 @@ type Settings struct { } type batch struct { - queue *broker - entries []queueEntry - ackChan chan batchAckMsg + queue *broker + entries []queueEntry + doneChan chan batchDoneMsg } // batchACKState stores the metadata associated with a batch of events sent to @@ -106,7 +106,7 @@ type batch struct { // ackChan and received by type batchACKState struct { next *batchACKState - ackChan chan batchAckMsg + doneChan chan batchDoneMsg start, count int // number of events waiting for ACK entries []queueEntry } @@ -250,9 +250,9 @@ func (b *broker) Get(count int) (queue.Batch, error) { // if request has been sent, we have to wait for a response resp := <-responseChan return &batch{ - queue: b, - entries: resp.entries, - ackChan: resp.ackChan, + queue: b, + entries: resp.entries, + doneChan: resp.ackChan, }, nil } @@ -277,7 +277,7 @@ func (b *broker) Metrics() (queue.Metrics, error) { var ackChanPool = sync.Pool{ New: func() interface{} { return &batchACKState{ - ackChan: make(chan batchAckMsg, 1), + doneChan: make(chan batchDoneMsg, 1), } }, } @@ -335,11 +335,11 @@ func (l *chanList) front() *batchACKState { return l.head } -func (l *chanList) nextBatchChannel() chan batchAckMsg { +func (l *chanList) nextBatchChannel() chan batchDoneMsg { if l.head == nil { return nil } - return l.head.ackChan + return l.head.doneChan } func (l *chanList) pop() *batchACKState { @@ -384,6 +384,6 @@ func (b *batch) Event(i int) interface{} { return b.entries[i].event } -func (b *batch) ACK() { - b.ackChan <- batchAckMsg{} +func (b *batch) Done() { + b.doneChan <- batchDoneMsg{} } diff --git a/libbeat/publisher/queue/memqueue/eventloop.go b/libbeat/publisher/queue/memqueue/eventloop.go index c7e8ec07f12..b6dc4af470d 100644 --- a/libbeat/publisher/queue/memqueue/eventloop.go +++ b/libbeat/publisher/queue/memqueue/eventloop.go @@ -185,7 +185,7 @@ func (l *directEventLoop) handleGetRequest(req *getRequest) { ackCH := newBatchACKState(start, count, l.buf.entries) - req.responseChan <- getResponse{ackCH.ackChan, buf} + req.responseChan <- getResponse{ackCH.doneChan, buf} l.pendingACKs.append(ackCH) } @@ -422,7 +422,7 @@ func (l *bufferingEventLoop) handleGetRequest(req *getRequest) { entries := buf.entries[:count] acker := newBatchACKState(0, count, entries) - req.responseChan <- getResponse{acker.ackChan, entries} + req.responseChan <- getResponse{acker.doneChan, entries} l.pendingACKs.append(acker) l.unackedEventCount += len(entries) diff --git a/libbeat/publisher/queue/memqueue/internal_api.go b/libbeat/publisher/queue/memqueue/internal_api.go index 4059b71aadd..c924afc785c 100644 --- a/libbeat/publisher/queue/memqueue/internal_api.go +++ b/libbeat/publisher/queue/memqueue/internal_api.go @@ -42,11 +42,11 @@ type getRequest struct { } type getResponse struct { - ackChan chan batchAckMsg + ackChan chan batchDoneMsg entries []queueEntry } -type batchAckMsg struct{} +type batchDoneMsg struct{} // Metrics API diff --git a/libbeat/publisher/queue/memqueue/queue_test.go b/libbeat/publisher/queue/memqueue/queue_test.go index a6c448a032c..5c810ac18ab 100644 --- a/libbeat/publisher/queue/memqueue/queue_test.go +++ b/libbeat/publisher/queue/memqueue/queue_test.go @@ -120,7 +120,7 @@ func queueTestWithSettings(t *testing.T, settings Settings, eventsToTest int, te queueMetricsAreValid(t, testQueue, 5, settings.Events, 5, fmt.Sprintf("%s - Producer Getting events, no ACK", testName)) // Test metrics after ack - batch.ACK() + batch.Done() queueMetricsAreValid(t, testQueue, 0, settings.Events, 0, fmt.Sprintf("%s - Producer Getting events, no ACK", testName)) diff --git a/libbeat/publisher/queue/queue.go b/libbeat/publisher/queue/queue.go index 11ec4aec818..1ca5fda6e14 100644 --- a/libbeat/publisher/queue/queue.go +++ b/libbeat/publisher/queue/queue.go @@ -131,10 +131,10 @@ type Producer interface { Cancel() int } -// Batch of events to be returned to Consumers. The `ACK` method will send the -// ACK signal to the queue. +// Batch of events to be returned to Consumers. The `Done` method will tell the +// queue that the batch has been consumed and its events can be discarded. type Batch interface { Count() int Event(i int) interface{} - ACK() + Done() } diff --git a/libbeat/publisher/queue/queuetest/producer_cancel.go b/libbeat/publisher/queue/queuetest/producer_cancel.go index 2b44f538854..9b5671e1fee 100644 --- a/libbeat/publisher/queue/queuetest/producer_cancel.go +++ b/libbeat/publisher/queue/queuetest/producer_cancel.go @@ -84,7 +84,7 @@ func TestProducerCancelRemovesEvents(t *testing.T, factory QueueFactory) { for i := 0; i < batch.Count(); i++ { events = append(events, batch.Event(i)) } - batch.ACK() + batch.Done() } // verify diff --git a/libbeat/publisher/queue/queuetest/queuetest.go b/libbeat/publisher/queue/queuetest/queuetest.go index 0fb95b1dfbe..e71843d2de5 100644 --- a/libbeat/publisher/queue/queuetest/queuetest.go +++ b/libbeat/publisher/queue/queuetest/queuetest.go @@ -327,7 +327,7 @@ func multiConsumer(numConsumers, maxEvents, batchSize int) workerFactory { for j := 0; j < batch.Count(); j++ { events.Done() } - batch.ACK() + batch.Done() } }() } From 3cdf704f414f37760a8307c6ad4703cd53c4daef Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Mon, 13 Jun 2022 17:15:51 -0400 Subject: [PATCH 09/22] [Automation] Update elastic stack version to 8.4.0-3bedeef4 for testing (#31913) Co-authored-by: apmmachine Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 820890b6ad1..ce64e2e0c78 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-b8fce3f8-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-3bedeef4-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.4.0-b8fce3f8-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.4.0-3bedeef4-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-b8fce3f8-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-3bedeef4-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From a58094dfd3db6b7df29836f91c05e73ad7845a95 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Tue, 14 Jun 2022 10:23:53 -0400 Subject: [PATCH 10/22] [Automation] Update elastic stack version to 8.4.0-7e67f5d9 for testing (#31923) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index ce64e2e0c78..d70fdab5912 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-3bedeef4-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-7e67f5d9-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.4.0-3bedeef4-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.4.0-7e67f5d9-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-3bedeef4-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-7e67f5d9-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 2b44bb8da62b9a38abd81d805d17745f4725dbfe Mon Sep 17 00:00:00 2001 From: Kerry Gallagher Date: Tue, 14 Jun 2022 17:47:57 +0100 Subject: [PATCH 11/22] [Metricbeat] Discard partial Logstash pipeline documents (#31868) * Discard partial Logstash pipeline documents --- .../node_stats/_meta/test/node_stats.710.json | 363 +++++++++--------- .../node_stats/_meta/test/node_stats.840.json | 185 +++++++++ .../_meta/test/node_stats.840_partial.json | 156 ++++++++ metricbeat/module/logstash/node_stats/data.go | 20 +- .../module/logstash/node_stats/data_test.go | 36 +- .../module/logstash/node_stats/node_stats.go | 2 +- 6 files changed, 570 insertions(+), 192 deletions(-) create mode 100644 metricbeat/module/logstash/node_stats/_meta/test/node_stats.840.json create mode 100644 metricbeat/module/logstash/node_stats/_meta/test/node_stats.840_partial.json diff --git a/metricbeat/module/logstash/node_stats/_meta/test/node_stats.710.json b/metricbeat/module/logstash/node_stats/_meta/test/node_stats.710.json index 6b8afa8022b..c01fc05f6ad 100644 --- a/metricbeat/module/logstash/node_stats/_meta/test/node_stats.710.json +++ b/metricbeat/module/logstash/node_stats/_meta/test/node_stats.710.json @@ -1,192 +1,197 @@ { - "host": "anonymous", - "version": "7.8.1", - "http_address": "127.0.0.1:9600", - "id": "2fc9524f-a36b-4611-82e4-5246d3ac4714", - "name": "anonymous", - "ephemeral_id": "10311adf-a5ba-4920-ada2-8dadc54618fe", - "status": "green", - "snapshot": false, - "pipeline": { - "workers": 12, - "batch_size": 125, - "batch_delay": 50 + "host" : "logstash", + "version" : "7.10.0", + "http_address" : "0.0.0.0:9600", + "id" : "84539cd1-193c-4439-8432-5a160911f135", + "name" : "logstash", + "ephemeral_id" : "9c6737de-dde2-4926-bee3-9ccc0ed1d2a6", + "status" : "green", + "snapshot" : false, + "pipeline" : { + "workers" : 8, + "batch_size" : 125, + "batch_delay" : 50 }, - "jvm": { - "threads": { - "count": 45, - "peak_count": 45 - }, - "mem": { - "heap_used_percent": 35, - "heap_committed_in_bytes": 1037959168, - "heap_max_in_bytes": 1037959168, - "heap_used_in_bytes": 371211952, - "non_heap_used_in_bytes": 148921496, - "non_heap_committed_in_bytes": 169435136, - "pools": { - "young": { - "committed_in_bytes": 286326784, - "max_in_bytes": 286326784, - "used_in_bytes": 236814520, - "peak_used_in_bytes": 286326784, - "peak_max_in_bytes": 286326784 - }, - "old": { - "committed_in_bytes": 715849728, - "max_in_bytes": 715849728, - "used_in_bytes": 102245112, - "peak_used_in_bytes": 107311296, - "peak_max_in_bytes": 715849728 - }, - "survivor": { - "committed_in_bytes": 35782656, - "max_in_bytes": 35782656, - "used_in_bytes": 32152320, - "peak_used_in_bytes": 35782656, - "peak_max_in_bytes": 35782656 - } - } - }, - "gc": { - "collectors": { - "young": { - "collection_time_in_millis": 231, - "collection_count": 7 - }, - "old": { - "collection_time_in_millis": 449, - "collection_count": 3 - } - } - }, - "uptime_in_millis": 33546 + "jvm" : { + "threads" : { + "count" : 35, + "peak_count" : 35 + }, + "mem" : { + "heap_used_percent" : 13, + "heap_committed_in_bytes" : 1037959168, + "heap_max_in_bytes" : 1037959168, + "heap_used_in_bytes" : 144995280, + "non_heap_used_in_bytes" : 148041864, + "non_heap_committed_in_bytes" : 172560384, + "pools" : { + "survivor" : { + "committed_in_bytes" : 35782656, + "used_in_bytes" : 23405920, + "peak_max_in_bytes" : 35782656, + "peak_used_in_bytes" : 35782656, + "max_in_bytes" : 35782656 + }, + "young" : { + "committed_in_bytes" : 286326784, + "used_in_bytes" : 16565808, + "peak_max_in_bytes" : 286326784, + "peak_used_in_bytes" : 286326784, + "max_in_bytes" : 286326784 + }, + "old" : { + "committed_in_bytes" : 715849728, + "used_in_bytes" : 105023552, + "peak_max_in_bytes" : 715849728, + "peak_used_in_bytes" : 105023552, + "max_in_bytes" : 715849728 + } + } + }, + "gc" : { + "collectors" : { + "young" : { + "collection_time_in_millis" : 335, + "collection_count" : 7 + }, + "old" : { + "collection_time_in_millis" : 374, + "collection_count" : 2 + } + } + }, + "uptime_in_millis" : 17006 }, - "process": { - "open_file_descriptors": 103, - "peak_open_file_descriptors": 103, - "max_file_descriptors": 524288, - "mem": { - "total_virtual_in_bytes": 7189766144 - }, - "cpu": { - "total_in_millis": 68000, - "percent": 1, - "load_average": { - "1m": 2.52, - "5m": 1.8, - "15m": 1.7 - } + "process" : { + "open_file_descriptors" : 140, + "peak_open_file_descriptors" : 140, + "max_file_descriptors" : 1048576, + "mem" : { + "total_virtual_in_bytes" : 6392164352 + }, + "cpu" : { + "total_in_millis" : 58710, + "percent" : 23, + "load_average" : { + "1m" : 0.87, + "5m" : 0.22, + "15m" : 0.13 } + } }, - "events": { - "in": 0, - "filtered": 0, - "out": 0, - "duration_in_millis": 0, - "queue_push_duration_in_millis": 0 + "events" : { + "in" : 0, + "filtered" : 0, + "out" : 0, + "duration_in_millis" : 0, + "queue_push_duration_in_millis" : 0 }, - "pipelines": { - "main": { - "events": { - "filtered": 0, - "out": 0, - "queue_push_duration_in_millis": 0, - "in": 0, - "duration_in_millis": 0 - }, - "plugins": { - "inputs": [ - { - "id": "adc9aa83962ecb7f9e68a9b25d9ef5473a197dbb8f6cd7d6025742b2e2bcadcf", - "events": { - "out": 0, - "queue_push_duration_in_millis": 0 - }, - "name": "file" - } - ], - "codecs": [ - { - "id": "plain_6013fcda-72f3-4a64-8a6c-2cbab83a1cd0", - "decode": { - "out": 0, - "duration_in_millis": 0, - "writes_in": 0 - }, - "encode": { - "duration_in_millis": 0, - "writes_in": 0 - }, - "name": "plain" - }, - { - "id": "rubydebug_e8db9697-3c57-4b27-be5a-6ecbdf34defb", - "decode": { - "out": 0, - "duration_in_millis": 0, - "writes_in": 0 - }, - "encode": { - "duration_in_millis": 5, - "writes_in": 0 - }, - "name": "rubydebug" - } - ], - "filters": [], - "outputs": [ - { - "id": "488688673ecd34b96c74c843d265da79aa11629216d2cb76ca008548316e310d", - "events": { - "out": 0, - "in": 0, - "duration_in_millis": 17 - }, - "name": "stdout" - } - ] - }, - "reloads": { - "last_error": null, - "failures": 0, - "last_success_timestamp": null, - "last_failure_timestamp": null, - "successes": 0 + "pipelines" : { + "main" : { + "events" : { + "filtered" : 0, + "duration_in_millis" : 0, + "out" : 0, + "queue_push_duration_in_millis" : 0, + "in" : 0 + }, + "plugins" : { + "inputs" : [ { + "id" : "0710cad67e8f47667bc7612580d5b91f691dd8262a4187d9eca8cf87229d04aa", + "name" : "beats", + "events" : { + "out" : 0, + "queue_push_duration_in_millis" : 0 + } + } ], + "codecs" : [ { + "id" : "plain_be1a2b95-edff-4dc2-94b5-b902f11bfb07", + "encode" : { + "duration_in_millis" : 0, + "writes_in" : 0 }, - "queue": { - "type": "memory", - "events_count": 0, - "queue_size_in_bytes": 0, - "max_queue_size_in_bytes": 0 + "name" : "plain", + "decode" : { + "duration_in_millis" : 0, + "writes_in" : 0, + "out" : 0 + } + }, { + "id" : "rubydebug_93993219-1a2d-493c-a63c-341a0a5c86ac", + "encode" : { + "duration_in_millis" : 9, + "writes_in" : 0 }, - "hash": "6984380a58d40b7ebe6ba7ba8fc3134b95ffadfcef9ff328f0864ae002943792", - "ephemeral_id": "eb977feb-0052-4651-a995-39e545213723" - } + "name" : "rubydebug", + "decode" : { + "duration_in_millis" : 0, + "writes_in" : 0, + "out" : 0 + } + } ], + "filters" : [ ], + "outputs" : [ { + "id" : "f4944472678ac54e7343c1a49748c402b0bafd76ebab7fe2f3930269e0e5097b", + "name" : "stdout", + "events" : { + "duration_in_millis" : 20, + "out" : 0, + "in" : 0 + } + } ] + }, + "reloads" : { + "last_success_timestamp" : null, + "last_failure_timestamp" : null, + "last_error" : null, + "successes" : 0, + "failures" : 0 + }, + "queue" : { + "type" : "memory", + "events_count" : 0, + "queue_size_in_bytes" : 0, + "max_queue_size_in_bytes" : 0 + }, + "hash" : "1e3da6bfb66bd25bbb98769a5dcdc16c06f98a333675d3b71cccad117e103b99", + "ephemeral_id" : "793497fe-de31-4b3a-bfbd-0b77eb7a2d9b", + "vertices" : [ { + "id" : "0710cad67e8f47667bc7612580d5b91f691dd8262a4187d9eca8cf87229d04aa", + "pipeline_ephemeral_id" : "793497fe-de31-4b3a-bfbd-0b77eb7a2d9b", + "events_out" : 0, + "queue_push_duration_in_millis" : 0 + }, { + "id" : "f4944472678ac54e7343c1a49748c402b0bafd76ebab7fe2f3930269e0e5097b", + "pipeline_ephemeral_id" : "793497fe-de31-4b3a-bfbd-0b77eb7a2d9b", + "duration_in_millis" : 20, + "events_out" : 0, + "events_in" : 0 + } ] + } }, - "reloads": { - "failures": 0, - "successes": 0 + "reloads" : { + "successes" : 0, + "failures" : 0 }, - "os": { - "cgroup": { - "cpu": { - "cfs_quota_micros": -1, - "cfs_period_micros": 100000, - "control_group": "/user.slice", - "stat": { - "time_throttled_nanos": 0, - "number_of_times_throttled": 0, - "number_of_elapsed_periods": 0 - } - }, - "cpuacct": { - "usage_nanos": 24041075965919, - "control_group": "/user.slice" - } + "os" : { + "cgroup" : { + "cpuacct" : { + "control_group" : "/", + "usage_nanos" : 58485023221 + }, + "cpu" : { + "cfs_quota_micros" : -1, + "cfs_period_micros" : 100000, + "control_group" : "/", + "stat" : { + "number_of_elapsed_periods" : 0, + "number_of_times_throttled" : 0, + "time_throttled_nanos" : 0 + } } + } }, - "queue": { - "events_count": 0 + "queue" : { + "events_count" : 0 } -} + } \ No newline at end of file diff --git a/metricbeat/module/logstash/node_stats/_meta/test/node_stats.840.json b/metricbeat/module/logstash/node_stats/_meta/test/node_stats.840.json new file mode 100644 index 00000000000..8d9e9757c5f --- /dev/null +++ b/metricbeat/module/logstash/node_stats/_meta/test/node_stats.840.json @@ -0,0 +1,185 @@ +{ + "host" : "logstash", + "version" : "8.4.0", + "http_address" : "0.0.0.0:9600", + "id" : "7e40c8b9-e586-4c2c-909f-4036b23134f5", + "name" : "logstash", + "ephemeral_id" : "43fe1e91-e998-42d8-8e7b-4b342ec44304", + "status" : "green", + "snapshot" : true, + "pipeline" : { + "workers" : 8, + "batch_size" : 125, + "batch_delay" : 50 + }, + "jvm" : { + "threads" : { + "count" : 37, + "peak_count" : 39 + }, + "mem" : { + "heap_used_percent" : 15, + "heap_committed_in_bytes" : 1037959168, + "heap_max_in_bytes" : 1037959168, + "heap_used_in_bytes" : 164771936, + "non_heap_used_in_bytes" : 159436056, + "non_heap_committed_in_bytes" : 181542912, + "pools" : { + "survivor" : { + "used_in_bytes" : 19639928, + "peak_max_in_bytes" : 35782656, + "committed_in_bytes" : 35782656, + "max_in_bytes" : 35782656, + "peak_used_in_bytes" : 35782656 + }, + "old" : { + "used_in_bytes" : 134908088, + "peak_max_in_bytes" : 715849728, + "committed_in_bytes" : 715849728, + "max_in_bytes" : 715849728, + "peak_used_in_bytes" : 134908088 + }, + "young" : { + "used_in_bytes" : 10223920, + "peak_max_in_bytes" : 286326784, + "committed_in_bytes" : 286326784, + "max_in_bytes" : 286326784, + "peak_used_in_bytes" : 286326784 + } + } + }, + "gc" : { + "collectors" : { + "old" : { + "collection_count" : 2, + "collection_time_in_millis" : 132 + }, + "young" : { + "collection_count" : 7, + "collection_time_in_millis" : 469 + } + } + }, + "uptime_in_millis" : 78193 + }, + "process" : { + "open_file_descriptors" : 85, + "peak_open_file_descriptors" : 85, + "max_file_descriptors" : 1048576, + "mem" : { + "total_virtual_in_bytes" : 6588469248 + }, + "cpu" : { + "total_in_millis" : 58580, + "percent" : 0, + "load_average" : { + "1m" : 0.39, + "5m" : 0.3, + "15m" : 0.17 + } + } + }, + "events" : { + "in" : 65, + "filtered" : 65, + "out" : 65, + "duration_in_millis" : 106, + "queue_push_duration_in_millis" : 20 + }, + "pipelines" : { + "main" : { + "events" : { + "duration_in_millis" : 106, + "filtered" : 65, + "queue_push_duration_in_millis" : 20, + "out" : 65, + "in" : 65 + }, + "plugins" : { + "inputs" : [ { + "id" : "0a94a2ce1283b96e0a916019722411e62435f584371347d36e10f72edd02a578", + "events" : { + "queue_push_duration_in_millis" : 20, + "out" : 65 + }, + "name" : "java_generator" + } ], + "codecs" : [ { + "id" : "json_lines_d0464beb-2d6f-4a05-909a-395b8e11b89e", + "decode" : { + "duration_in_millis" : 0, + "writes_in" : 0, + "out" : 0 + }, + "name" : "json_lines", + "encode" : { + "duration_in_millis" : 27, + "writes_in" : 65 + } + } ], + "filters" : [ ], + "outputs" : [ { + "id" : "c17c6139a0855d3d54a5d7ff4d3684a4f53253dd8e71d04f58d3fccd58127329", + "events" : { + "duration_in_millis" : 121, + "out" : 65, + "in" : 65 + }, + "name" : "file" + } ] + }, + "reloads" : { + "successes" : 0, + "last_error" : null, + "failures" : 0, + "last_failure_timestamp" : null, + "last_success_timestamp" : null + }, + "queue" : { + "type" : "memory", + "events_count" : 0, + "queue_size_in_bytes" : 0, + "max_queue_size_in_bytes" : 0 + }, + "hash" : "18318658a1566ee135077867f8bb532b6ed2ea20ea49bd6b5d616a03e2afe5e7", + "ephemeral_id" : "8747741b-9b8e-4f75-80dc-a5cbfb065959", + "vertices" : [ { + "id" : "0a94a2ce1283b96e0a916019722411e62435f584371347d36e10f72edd02a578", + "pipeline_ephemeral_id" : "8747741b-9b8e-4f75-80dc-a5cbfb065959", + "queue_push_duration_in_millis" : 20, + "events_out" : 65 + }, { + "id" : "c17c6139a0855d3d54a5d7ff4d3684a4f53253dd8e71d04f58d3fccd58127329", + "pipeline_ephemeral_id" : "8747741b-9b8e-4f75-80dc-a5cbfb065959", + "duration_in_millis" : 121, + "events_out" : 65, + "events_in" : 65 + } ] + } + }, + "reloads" : { + "successes" : 0, + "failures" : 0 + }, + "os" : { + "cgroup" : { + "cpuacct" : { + "usage_nanos" : 58765824206, + "control_group" : "/" + }, + "cpu" : { + "cfs_quota_micros" : -1, + "stat" : { + "time_throttled_nanos" : 0, + "number_of_times_throttled" : 0, + "number_of_elapsed_periods" : 0 + }, + "control_group" : "/", + "cfs_period_micros" : 100000 + } + } + }, + "queue" : { + "events_count" : 0 + } + } \ No newline at end of file diff --git a/metricbeat/module/logstash/node_stats/_meta/test/node_stats.840_partial.json b/metricbeat/module/logstash/node_stats/_meta/test/node_stats.840_partial.json new file mode 100644 index 00000000000..3bb101e2d47 --- /dev/null +++ b/metricbeat/module/logstash/node_stats/_meta/test/node_stats.840_partial.json @@ -0,0 +1,156 @@ +{ + "host" : "logstash", + "version" : "8.4.0", + "http_address" : "0.0.0.0:9600", + "id" : "7e40c8b9-e586-4c2c-909f-4036b23134f5", + "name" : "logstash", + "ephemeral_id" : "43fe1e91-e998-42d8-8e7b-4b342ec44304", + "status" : "green", + "snapshot" : true, + "pipeline" : { + "workers" : 8, + "batch_size" : 125, + "batch_delay" : 50 + }, + "jvm" : { + "threads" : { + "count" : 7, + "peak_count" : 7 + }, + "mem" : { + "heap_used_percent" : 17, + "heap_committed_in_bytes" : 1037959168, + "heap_max_in_bytes" : 1037959168, + "heap_used_in_bytes" : 179874416, + "non_heap_used_in_bytes" : 134730416, + "non_heap_committed_in_bytes" : 151126016, + "pools" : { + "survivor" : { + "used_in_bytes" : 35782656, + "peak_max_in_bytes" : 35782656, + "committed_in_bytes" : 35782656, + "max_in_bytes" : 35782656, + "peak_used_in_bytes" : 35782656 + }, + "old" : { + "used_in_bytes" : 96185736, + "peak_max_in_bytes" : 715849728, + "committed_in_bytes" : 715849728, + "max_in_bytes" : 715849728, + "peak_used_in_bytes" : 96185736 + }, + "young" : { + "used_in_bytes" : 47906024, + "peak_max_in_bytes" : 286326784, + "committed_in_bytes" : 286326784, + "max_in_bytes" : 286326784, + "peak_used_in_bytes" : 286326784 + } + } + }, + "gc" : { + "collectors" : { + "old" : { + "collection_count" : 1, + "collection_time_in_millis" : 62 + }, + "young" : { + "collection_count" : 5, + "collection_time_in_millis" : 398 + } + } + }, + "uptime_in_millis" : 12972 + }, + "process" : { + "open_file_descriptors" : 67, + "peak_open_file_descriptors" : 67, + "max_file_descriptors" : 1048576, + "mem" : { + "total_virtual_in_bytes" : 4452380672 + }, + "cpu" : { + "total_in_millis" : 42500, + "percent" : 0, + "load_average" : { + "1m" : 1.17, + "5m" : 0.38, + "15m" : 0.19 + } + } + }, + "events" : { + "in" : null, + "filtered" : 0, + "out" : 0, + "duration_in_millis" : 0, + "queue_push_duration_in_millis" : null + }, + "pipelines" : { + "main" : { + "events" : { + "duration_in_millis" : 0, + "filtered" : 0, + "out" : 0 + }, + "plugins" : { + "inputs" : [ { + "id" : "0a94a2ce1283b96e0a916019722411e62435f584371347d36e10f72edd02a578", + "name" : "java_generator" + } ], + "codecs" : [ { + "id" : "json_lines_d0464beb-2d6f-4a05-909a-395b8e11b89e", + "decode" : { + "duration_in_millis" : 0, + "writes_in" : 0, + "out" : 0 + }, + "name" : "json_lines", + "encode" : { + "duration_in_millis" : 0, + "writes_in" : 0 + } + } ], + "filters" : [ ], + "outputs" : [ { + "id" : "c17c6139a0855d3d54a5d7ff4d3684a4f53253dd8e71d04f58d3fccd58127329", + "events" : { + "duration_in_millis" : 0, + "out" : 0, + "in" : 0 + }, + "name" : "file" + } ] + }, + "reloads" : null, + "queue" : { + "type" : "memory" + } + } + }, + "reloads" : { + "successes" : 0, + "failures" : 0 + }, + "os" : { + "cgroup" : { + "cpuacct" : { + "usage_nanos" : 42095159619, + "control_group" : "/" + }, + "cpu" : { + "cfs_quota_micros" : -1, + "stat" : { + "time_throttled_nanos" : 0, + "number_of_times_throttled" : 0, + "number_of_elapsed_periods" : 0 + }, + "control_group" : "/", + "cfs_period_micros" : 100000 + } + } + }, + "queue" : { + "events_count" : 0 + } + } \ No newline at end of file diff --git a/metricbeat/module/logstash/node_stats/data.go b/metricbeat/module/logstash/node_stats/data.go index 5eb89db9de7..f6b8b487f86 100644 --- a/metricbeat/module/logstash/node_stats/data.go +++ b/metricbeat/module/logstash/node_stats/data.go @@ -19,10 +19,13 @@ package node_stats import ( "encoding/json" + "fmt" "time" "github.com/elastic/beats/v7/metricbeat/helper/elastic" + "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/version" "github.com/elastic/beats/v7/metricbeat/module/logstash" @@ -145,7 +148,7 @@ type PipelineStats struct { Vertices []map[string]interface{} `json:"vertices"` } -func eventMapping(r mb.ReporterV2, content []byte, isXpack bool) error { +func eventMapping(r mb.ReporterV2, content []byte, isXpack bool, logger *logp.Logger) error { var nodeStats NodeStats err := json.Unmarshal(content, &nodeStats) if err != nil { @@ -174,9 +177,20 @@ func eventMapping(r mb.ReporterV2, content []byte, isXpack bool) error { } var pipelines []PipelineStats + + // The version from which we expect the node stats API to return a hash with pipeline documents. This is really just a formality so that + // unit tests with much older fixture versions still work. + var PipelineDocumentsContainHashVersion = version.MustNew("7.3.0") + var StatsVersion = version.MustNew(nodeStats.Version) + pipelineDocumentsShouldContainHash := !StatsVersion.LessThan(PipelineDocumentsContainHashVersion) + for pipelineID, pipeline := range nodeStats.Pipelines { - pipeline.ID = pipelineID - pipelines = append(pipelines, pipeline) + if pipelineDocumentsShouldContainHash && (pipeline.Hash == "" || pipeline.Vertices == nil) { + logger.Warn(fmt.Sprintf("Pipeline document was discarded due to missing properties. This can happen when the Logstash node stats API is polled before the pipeline setup has completed. Pipeline ID: %s", pipelineID)) + } else { + pipeline.ID = pipelineID + pipelines = append(pipelines, pipeline) + } } pipelines = getUserDefinedPipelines(pipelines) diff --git a/metricbeat/module/logstash/node_stats/data_test.go b/metricbeat/module/logstash/node_stats/data_test.go index ab133c4d900..d4c5fd7316b 100644 --- a/metricbeat/module/logstash/node_stats/data_test.go +++ b/metricbeat/module/logstash/node_stats/data_test.go @@ -21,13 +21,14 @@ package node_stats import ( + "fmt" "io/ioutil" "net/http" "net/http/httptest" - "path/filepath" "testing" "github.com/elastic/beats/v7/metricbeat/module/logstash" + "github.com/elastic/elastic-agent-libs/logp" "github.com/stretchr/testify/require" @@ -35,19 +36,36 @@ import ( ) func TestEventMapping(t *testing.T) { - files, err := filepath.Glob("./_meta/test/node_stats.*.json") - require.NoError(t, err) + // Contain pipeline hash + containVersions := []string{} + containVersions = append(containVersions, "710") + containVersions = append(containVersions, "840") + EventMappingForFiles(t, containVersions, 1, 0) + // Don't contain pipeline hash + dontContainVersions := []string{} + dontContainVersions = append(dontContainVersions, "641") + dontContainVersions = append(dontContainVersions, "650") + dontContainVersions = append(dontContainVersions, "700") + EventMappingForFiles(t, dontContainVersions, 1, 0) + // Don't contain pipeline hash but should (partial) + partialVersions := []string{} + partialVersions = append(partialVersions, "840_partial") + EventMappingForFiles(t, partialVersions, 0, 0) +} + +func EventMappingForFiles(t *testing.T, fixtureVersions []string, expectedEvents int, expectedErrors int) { + logger := logp.NewLogger("logstash.node_stats") - for _, f := range files { - input, err := ioutil.ReadFile(f) + for _, f := range fixtureVersions { + path := fmt.Sprintf("./_meta/test/node_stats.%s.json", f) + input, err := ioutil.ReadFile(path) require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventMapping(reporter, input, true) - + err = eventMapping(reporter, input, true, logger) require.NoError(t, err, f) - require.True(t, len(reporter.GetEvents()) >= 1, f) - require.Equal(t, 0, len(reporter.GetErrors()), f) + require.True(t, len(reporter.GetEvents()) >= expectedEvents, f) + require.Equal(t, expectedErrors, len(reporter.GetErrors()), f) } } diff --git a/metricbeat/module/logstash/node_stats/node_stats.go b/metricbeat/module/logstash/node_stats/node_stats.go index f7500348400..6f935ace955 100644 --- a/metricbeat/module/logstash/node_stats/node_stats.go +++ b/metricbeat/module/logstash/node_stats/node_stats.go @@ -77,7 +77,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return err } - if err = eventMapping(r, content, m.XPackEnabled); err != nil { + if err = eventMapping(r, content, m.XPackEnabled, m.Logger()); err != nil { return err } From 5479958e072cae4758dae4392268e336650b9b96 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Tue, 14 Jun 2022 21:33:17 -0400 Subject: [PATCH 12/22] Osquerybeat: Exit osquerybeat process if orphaned (#31931) --- x-pack/osquerybeat/beater/osquerybeat.go | 19 ++++- x-pack/osquerybeat/beater/watcher.go | 79 +++++++++++++++++++ .../internal/osqd/osqueryd_unix.go | 5 +- 3 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 x-pack/osquerybeat/beater/watcher.go diff --git a/x-pack/osquerybeat/beater/osquerybeat.go b/x-pack/osquerybeat/beater/osquerybeat.go index 402b0bf727e..1c0f1d71e9b 100644 --- a/x-pack/osquerybeat/beater/osquerybeat.go +++ b/x-pack/osquerybeat/beater/osquerybeat.go @@ -66,6 +66,9 @@ type osquerybeat struct { // Beat lifecycle context, cancelled on Stop cancel context.CancelFunc mx sync.Mutex + + // parent process watcher + watcher *Watcher } // New creates an instance of osquerybeat. @@ -87,7 +90,7 @@ func New(b *beat.Beat, cfg *conf.C) (beat.Beater, error) { return bt, nil } -func (bt *osquerybeat) initContext() (context.Context, error) { +func (bt *osquerybeat) init() (context.Context, error) { bt.mx.Lock() defer bt.mx.Unlock() if bt.cancel != nil { @@ -95,6 +98,11 @@ func (bt *osquerybeat) initContext() (context.Context, error) { } var ctx context.Context ctx, bt.cancel = context.WithCancel(context.Background()) + + if bt.watcher != nil { + bt.watcher.Close() + } + bt.watcher = NewWatcher(bt.log) return ctx, nil } @@ -108,11 +116,18 @@ func (bt *osquerybeat) close() { bt.cancel() bt.cancel = nil } + + // Start watching the parent process. + // The beat exits if the process gets orphaned. + if bt.watcher != nil { + go bt.watcher.Run() + bt.watcher = nil + } } // Run starts osquerybeat. func (bt *osquerybeat) Run(b *beat.Beat) error { - ctx, err := bt.initContext() + ctx, err := bt.init() if err != nil { return err } diff --git a/x-pack/osquerybeat/beater/watcher.go b/x-pack/osquerybeat/beater/watcher.go new file mode 100644 index 00000000000..35e34e9608e --- /dev/null +++ b/x-pack/osquerybeat/beater/watcher.go @@ -0,0 +1,79 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package beater + +import ( + "context" + "os" + "sync" + "time" + + "github.com/elastic/elastic-agent-libs/logp" +) + +const watchFrequency = 10 * time.Second + +type Watcher struct { + log *logp.Logger + ppid int + + mx sync.Mutex + cancel context.CancelFunc +} + +func NewWatcher(log *logp.Logger) *Watcher { + w := &Watcher{ + log: log, + ppid: os.Getppid(), + } + return w +} + +func (w *Watcher) Start() { + go w.Run() +} + +func (w *Watcher) Run() { + w.mx.Lock() + defer w.mx.Unlock() + + if w.cancel != nil { + w.log.Debug("watcher is already running") + return + } + + var ctx context.Context + ctx, w.cancel = context.WithCancel(context.Background()) + + ticker := time.NewTicker(watchFrequency) + defer ticker.Stop() + + f := func() { + ppid := os.Getppid() + if ppid != w.ppid { + w.log.Errorf("orphaned osquerybeat, expected ppid: %v, found ppid: %v, quitting", w.ppid, ppid) + os.Exit(1) + } + } + + for { + select { + case <-ticker.C: + f() + case <-ctx.Done(): + w.log.Info("exit watcher on context done") + } + } +} + +func (w *Watcher) Close() { + w.mx.Lock() + defer w.mx.Unlock() + + if w.cancel != nil { + w.cancel() + w.cancel = nil + } +} diff --git a/x-pack/osquerybeat/internal/osqd/osqueryd_unix.go b/x-pack/osquerybeat/internal/osqd/osqueryd_unix.go index 29a09daa506..a2fbd0347bf 100644 --- a/x-pack/osquerybeat/internal/osqd/osqueryd_unix.go +++ b/x-pack/osquerybeat/internal/osqd/osqueryd_unix.go @@ -58,5 +58,8 @@ func setpgid() *syscall.SysProcAttr { // For clean process tree kill func killProcessGroup(cmd *exec.Cmd) error { err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) - return fmt.Errorf("kill process group %d, %w", cmd.Process.Pid, err) + if err != nil { + return fmt.Errorf("kill process group %d, %w", cmd.Process.Pid, err) + } + return nil } From 9248dfdf1d093f3e37078a19a0ade2896d5a61cf Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Wed, 15 Jun 2022 02:44:22 -0400 Subject: [PATCH 13/22] [Automation] Update elastic stack version to 8.4.0-40cff009 for testing (#31939) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index d70fdab5912..cfcc8908a96 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-7e67f5d9-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-40cff009-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.4.0-7e67f5d9-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.4.0-40cff009-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-7e67f5d9-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-40cff009-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 9d0f1713102bddb3fa8913d6b5ceec136cabbb3a Mon Sep 17 00:00:00 2001 From: Yug Rajani <96041884+yug-elastic@users.noreply.github.com> Date: Wed, 15 Jun 2022 19:15:27 +0530 Subject: [PATCH 14/22] [Oracle Module] New sysmetric metricset (#31462) * Add sysmetric metricset to Oracle Metricbeat Module * Format code and try to resolve lint errors * Resolve lint issue * Run make check * Update dashboard and visualizations * Run make update * Update dashboards and visualizations * Update unit tests for sysmetric_metric * Resolve review comments and update documentation * Sanitize e.patterns to avoid SQL injection * Update unit test and resolve linting failures * Update integration tests for sysmetric metricset * Run make update * Update date parsing * Update test case to handle new timestamp format * Updating dynamically naming the fields to follow Elastic naming conventions * Update unit test case to handle the naming convention of the field * Update data.json with update in naming convention * Update implementation to have multiple metrics per document * Update test case based on the document update * Update data collection to collect long-duration queried metrics only * Update unit test case as per updated query * Update visualizations as per dynamically mapped field names * Update docs and resolve nitpicks * Update fields as per the visualizations * Resolve review comments * Update unit test case as per dropped key * Remove extra CHANGELOG.next.asciidoc entry * Resolve review comments --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 157 ++++++++ metricbeat/docs/modules/oracle.asciidoc | 55 ++- .../docs/modules/oracle/sysmetric.asciidoc | 29 ++ metricbeat/docs/modules_list.asciidoc | 3 +- x-pack/metricbeat/include/list.go | 1 + x-pack/metricbeat/metricbeat.reference.yml | 38 +- .../metricbeat/module/oracle/_meta/config.yml | 38 +- .../module/oracle/_meta/docs.asciidoc | 13 +- .../a3f4d080-c7d3-11ec-891a-5f50c7522a3a.json | 375 ++++++++++++++++++ .../00b53be0-c87c-11ec-891a-5f50c7522a3a.json | 89 +++++ .../09929750-c87a-11ec-891a-5f50c7522a3a.json | 89 +++++ .../21744290-c87c-11ec-891a-5f50c7522a3a.json | 89 +++++ .../2ee49e30-c87b-11ec-891a-5f50c7522a3a.json | 89 +++++ .../3af08cd0-c843-11ec-891a-5f50c7522a3a.json | 90 +++++ .../400c71a0-c87c-11ec-891a-5f50c7522a3a.json | 89 +++++ .../4ae42710-c9e6-11ec-891a-5f50c7522a3a.json | 109 +++++ .../53dee3a0-c9e6-11ec-891a-5f50c7522a3a.json | 128 ++++++ .../5e35d950-c87c-11ec-891a-5f50c7522a3a.json | 89 +++++ .../61d15a20-c87d-11ec-891a-5f50c7522a3a.json | 89 +++++ .../81c1f9a0-c843-11ec-891a-5f50c7522a3a.json | 91 +++++ .../84ef9e80-c9e6-11ec-891a-5f50c7522a3a.json | 89 +++++ .../8fb638f0-c90b-11ec-891a-5f50c7522a3a.json | 99 +++++ .../93361970-c922-11ec-891a-5f50c7522a3a.json | 100 +++++ .../a65d8eb0-c879-11ec-891a-5f50c7522a3a.json | 95 +++++ .../c2178960-c83f-11ec-891a-5f50c7522a3a.json | 86 ++++ .../c777ba10-c83f-11ec-891a-5f50c7522a3a.json | 86 ++++ .../d5e99370-c87b-11ec-891a-5f50c7522a3a.json | 97 +++++ .../ffd22450-c87a-11ec-891a-5f50c7522a3a.json | 89 +++++ x-pack/metricbeat/module/oracle/connection.go | 5 +- x-pack/metricbeat/module/oracle/fields.go | 2 +- .../module/oracle/sysmetric/_meta/data.json | 24 ++ .../oracle/sysmetric/_meta/docs.asciidoc | 44 ++ .../module/oracle/sysmetric/_meta/fields.yml | 66 +++ .../module/oracle/sysmetric/collector.go | 27 ++ .../module/oracle/sysmetric/data.go | 43 ++ .../module/oracle/sysmetric/data_test.go | 36 ++ .../module/oracle/sysmetric/metricset.go | 77 ++++ .../module/oracle/sysmetric/metricset_test.go | 61 +++ .../oracle/sysmetric/sysmetric_metric.go | 93 +++++ .../oracle/sysmetric/sysmetric_metric_test.go | 47 +++ .../metricbeat/modules.d/oracle.yml.disabled | 38 +- 42 files changed, 2985 insertions(+), 70 deletions(-) create mode 100644 metricbeat/docs/modules/oracle/sysmetric.asciidoc create mode 100644 x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/a3f4d080-c7d3-11ec-891a-5f50c7522a3a.json create mode 100644 x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/00b53be0-c87c-11ec-891a-5f50c7522a3a.json create mode 100644 x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/09929750-c87a-11ec-891a-5f50c7522a3a.json create mode 100644 x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/21744290-c87c-11ec-891a-5f50c7522a3a.json create mode 100644 x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/2ee49e30-c87b-11ec-891a-5f50c7522a3a.json create mode 100644 x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/3af08cd0-c843-11ec-891a-5f50c7522a3a.json create mode 100644 x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/400c71a0-c87c-11ec-891a-5f50c7522a3a.json create mode 100644 x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/4ae42710-c9e6-11ec-891a-5f50c7522a3a.json create mode 100644 x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/53dee3a0-c9e6-11ec-891a-5f50c7522a3a.json create mode 100644 x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/5e35d950-c87c-11ec-891a-5f50c7522a3a.json create mode 100644 x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/61d15a20-c87d-11ec-891a-5f50c7522a3a.json create mode 100644 x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/81c1f9a0-c843-11ec-891a-5f50c7522a3a.json create mode 100644 x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/84ef9e80-c9e6-11ec-891a-5f50c7522a3a.json create mode 100644 x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/8fb638f0-c90b-11ec-891a-5f50c7522a3a.json create mode 100644 x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/93361970-c922-11ec-891a-5f50c7522a3a.json create mode 100644 x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/a65d8eb0-c879-11ec-891a-5f50c7522a3a.json create mode 100644 x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/c2178960-c83f-11ec-891a-5f50c7522a3a.json create mode 100644 x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/c777ba10-c83f-11ec-891a-5f50c7522a3a.json create mode 100644 x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/d5e99370-c87b-11ec-891a-5f50c7522a3a.json create mode 100644 x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/ffd22450-c87a-11ec-891a-5f50c7522a3a.json create mode 100644 x-pack/metricbeat/module/oracle/sysmetric/_meta/data.json create mode 100644 x-pack/metricbeat/module/oracle/sysmetric/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/oracle/sysmetric/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/oracle/sysmetric/collector.go create mode 100644 x-pack/metricbeat/module/oracle/sysmetric/data.go create mode 100644 x-pack/metricbeat/module/oracle/sysmetric/data_test.go create mode 100644 x-pack/metricbeat/module/oracle/sysmetric/metricset.go create mode 100644 x-pack/metricbeat/module/oracle/sysmetric/metricset_test.go create mode 100644 x-pack/metricbeat/module/oracle/sysmetric/sysmetric_metric.go create mode 100644 x-pack/metricbeat/module/oracle/sysmetric/sysmetric_metric_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 3e379fb5be1..1cb6ccf4ef2 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -128,6 +128,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Add new Kubernetes module dashboards {pull}31591[31591] - system/core: add cpuinfo information for Linux hosts {pull}31643[31643] - Enhance Oracle Module: Refactor module to properly use host parsers instead of doing its own parsing of hosts {issue}31611[31611] {pull}31692[#31692] +- Enhance Oracle Module: New sysmetric metricset {issue}30946[30946] {pull}31462[#31462] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index a7e3c976efb..1fe95993d7d 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -54550,6 +54550,163 @@ type: long -- Ratio of session cursor cache hits from total number of cursors +type: double + +-- + +[float] +=== sysmetric + +Sysmetric related metrics. + + + +*`oracle.sysmetric.session_count`*:: ++ +-- +Session Count. + + +type: long + +-- + +*`oracle.sysmetric.average_active_sessions`*:: ++ +-- +Average Active Sessions. + + +type: double + +-- + +*`oracle.sysmetric.current_os_load`*:: ++ +-- +Current OS Load. + + +type: double + +-- + +*`oracle.sysmetric.physical_reads_per_sec`*:: ++ +-- +Physical Reads Per Second. + + +type: double + +-- + +*`oracle.sysmetric.user_transaction_per_sec`*:: ++ +-- +User Transaction Per Second. + + +type: double + +-- + +*`oracle.sysmetric.total_table_scans_per_txn`*:: ++ +-- +Total Table Scans Per Transaction. + + +type: double + +-- + +*`oracle.sysmetric.physical_writes_per_sec`*:: ++ +-- +Physical Writes Per Second. + + +type: double + +-- + +*`oracle.sysmetric.total_index_scans_per_txn`*:: ++ +-- +Total Index Scans Per Transaction. + + +type: double + +-- + +*`oracle.sysmetric.host_cpu_utilization_pct`*:: ++ +-- +Host CPU Utilization (%). + + +type: double + +-- + +*`oracle.sysmetric.network_traffic_volume_per_sec`*:: ++ +-- +Network Traffic Volume Per Second. + + +type: double + +-- + +*`oracle.sysmetric.user_rollbacks_per_sec`*:: ++ +-- +User Rollbacks Per Second. + + +type: long + +-- + +*`oracle.sysmetric.cpu_usage_per_sec`*:: ++ +-- +CPU Usage Per Second. + + +type: double + +-- + +*`oracle.sysmetric.db_block_changes_per_sec`*:: ++ +-- +DB Block Changes Per Second. + + +type: double + +-- + +*`oracle.sysmetric.physical_read_total_bytes_per_sec`*:: ++ +-- +Physical Read Total Bytes Per Second. + + +type: double + +-- + +*`oracle.sysmetric.response_time_per_txn`*:: ++ +-- +Response Time Per Transaction. + + type: double -- diff --git a/metricbeat/docs/modules/oracle.asciidoc b/metricbeat/docs/modules/oracle.asciidoc index d31c3cc12b7..f09621a9722 100644 --- a/metricbeat/docs/modules/oracle.asciidoc +++ b/metricbeat/docs/modules/oracle.asciidoc @@ -61,13 +61,24 @@ Then, Metricbeat can be launched. [float] == Metricsets -The following Metricset is included in the module: +The following Metricsets are included in the module: + +[float] +=== `performance` + +Includes performance related events which contains mainly cursor and cache based data. [float] === `tablespaces` Includes information about data files and temp files, grouped by Tablespace with free space available, used space, status of the data files, status of the Tablespace, etc. +[float] + +=== `sysmetric` + +Includes the system metric values captured for the most current time interval from Oracle system metrics. + :edit_url: @@ -80,22 +91,28 @@ in <>. Here is an example configuration: [source,yaml] ---- metricbeat.modules: -- module: oracle - period: 10m - metricsets: - - tablespace - enabled: true - hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"] -- module: oracle - period: 10s - metricsets: - - performance - enabled: true - hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"] - - # username: "" - # password: "" - +- module: oracle + period: 10m + metricsets: + - tablespace + enabled: true + hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"] +- module: oracle + period: 10s + metricsets: + - performance + enabled: true + hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"] +- module: oracle + period: 60s + metricsets: + - sysmetric + enabled: true + hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"] + # patterns: ["foo%","%bar","%foobar%"] + + # username: "" + # password: "" ---- [float] @@ -105,10 +122,14 @@ The following metricsets are available: * <> +* <> + * <> include::oracle/performance.asciidoc[] +include::oracle/sysmetric.asciidoc[] + include::oracle/tablespace.asciidoc[] :edit_url!: diff --git a/metricbeat/docs/modules/oracle/sysmetric.asciidoc b/metricbeat/docs/modules/oracle/sysmetric.asciidoc new file mode 100644 index 00000000000..c2b614bf3e1 --- /dev/null +++ b/metricbeat/docs/modules/oracle/sysmetric.asciidoc @@ -0,0 +1,29 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// +:edit_url: https://github.com/elastic/beats/edit/main/x-pack/metricbeat/module/oracle/sysmetric/_meta/docs.asciidoc + + +[[metricbeat-metricset-oracle-sysmetric]] +[role="xpack"] +=== Oracle sysmetric metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/oracle/sysmetric/_meta/docs.asciidoc[] + + +:edit_url: + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/oracle/sysmetric/_meta/data.json[] +---- +:edit_url!: \ No newline at end of file diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index f22a1ffea38..f8f2669268f 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -238,7 +238,8 @@ This file is generated! See scripts/mage/docs_collector.go |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> beta[] |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.2+| .2+| |<> +.3+| .3+| |<> +|<> beta[] |<> |<> |image:./images/icon-no.png[No prebuilt dashboards] | .2+| .2+| |<> diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 3fdd6556b58..bbd45ada6d3 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -51,6 +51,7 @@ import ( _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/mssql/transaction_log" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/oracle" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/oracle/performance" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/oracle/sysmetric" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/oracle/tablespace" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus/collector" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 08f289e71de..6cc20eece77 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1185,22 +1185,28 @@ metricbeat.modules: exclude: [] #-------------------------------- Oracle Module -------------------------------- -- module: oracle - period: 10m - metricsets: - - tablespace - enabled: true - hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"] -- module: oracle - period: 10s - metricsets: - - performance - enabled: true - hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"] - - # username: "" - # password: "" - +- module: oracle + period: 10m + metricsets: + - tablespace + enabled: true + hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"] +- module: oracle + period: 10s + metricsets: + - performance + enabled: true + hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"] +- module: oracle + period: 60s + metricsets: + - sysmetric + enabled: true + hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"] + # patterns: ["foo%","%bar","%foobar%"] + + # username: "" + # password: "" #------------------------------- PHP_FPM Module ------------------------------- - module: php_fpm diff --git a/x-pack/metricbeat/module/oracle/_meta/config.yml b/x-pack/metricbeat/module/oracle/_meta/config.yml index fae897db648..adc714a9c94 100644 --- a/x-pack/metricbeat/module/oracle/_meta/config.yml +++ b/x-pack/metricbeat/module/oracle/_meta/config.yml @@ -1,16 +1,22 @@ -- module: oracle - period: 10m - metricsets: - - tablespace - enabled: true - hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"] -- module: oracle - period: 10s - metricsets: - - performance - enabled: true - hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"] - - # username: "" - # password: "" - +- module: oracle + period: 10m + metricsets: + - tablespace + enabled: true + hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"] +- module: oracle + period: 10s + metricsets: + - performance + enabled: true + hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"] +- module: oracle + period: 60s + metricsets: + - sysmetric + enabled: true + hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"] + # patterns: ["foo%","%bar","%foobar%"] + + # username: "" + # password: "" diff --git a/x-pack/metricbeat/module/oracle/_meta/docs.asciidoc b/x-pack/metricbeat/module/oracle/_meta/docs.asciidoc index ce845cdc51f..4e4e05e94e3 100644 --- a/x-pack/metricbeat/module/oracle/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/oracle/_meta/docs.asciidoc @@ -49,9 +49,20 @@ Then, Metricbeat can be launched. [float] == Metricsets -The following Metricset is included in the module: +The following Metricsets are included in the module: + +[float] +=== `performance` + +Includes performance related events which contains mainly cursor and cache based data. [float] === `tablespaces` Includes information about data files and temp files, grouped by Tablespace with free space available, used space, status of the data files, status of the Tablespace, etc. + +[float] + +=== `sysmetric` + +Includes the system metric values captured for the most current time interval from Oracle system metrics. diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/a3f4d080-c7d3-11ec-891a-5f50c7522a3a.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/a3f4d080-c7d3-11ec-891a-5f50c7522a3a.json new file mode 100644 index 00000000000..bdd0674e41d --- /dev/null +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/dashboard/a3f4d080-c7d3-11ec-891a-5f50c7522a3a.json @@ -0,0 +1,375 @@ +{ + "attributes": { + "description": "An overview of key metrics from Sysmetric Metricsets in the Oracle database Metricbeat module", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 7, + "i": "c1235d00-17bf-46e3-be1a-f9e04689ba4f", + "w": 14, + "x": 0, + "y": 0 + }, + "panelIndex": "c1235d00-17bf-46e3-be1a-f9e04689ba4f", + "panelRefName": "panel_c1235d00-17bf-46e3-be1a-f9e04689ba4f", + "type": "visualization", + "version": "7.14.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 10, + "i": "25e28f7d-5aa8-4f7b-a549-fda815fd37d6", + "w": 24, + "x": 0, + "y": 7 + }, + "panelIndex": "25e28f7d-5aa8-4f7b-a549-fda815fd37d6", + "panelRefName": "panel_25e28f7d-5aa8-4f7b-a549-fda815fd37d6", + "type": "visualization", + "version": "7.14.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 10, + "i": "a8222df9-8ae7-4fb3-b2b5-0c26928869c2", + "w": 24, + "x": 24, + "y": 7 + }, + "panelIndex": "a8222df9-8ae7-4fb3-b2b5-0c26928869c2", + "panelRefName": "panel_a8222df9-8ae7-4fb3-b2b5-0c26928869c2", + "type": "visualization", + "version": "7.14.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 11, + "i": "411deb47-0f86-4bf1-a384-7977a03c11f6", + "w": 24, + "x": 0, + "y": 17 + }, + "panelIndex": "411deb47-0f86-4bf1-a384-7977a03c11f6", + "panelRefName": "panel_411deb47-0f86-4bf1-a384-7977a03c11f6", + "type": "visualization", + "version": "7.14.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 11, + "i": "114ba664-4260-47b8-86a5-ca34f7c0e2d2", + "w": 24, + "x": 24, + "y": 17 + }, + "panelIndex": "114ba664-4260-47b8-86a5-ca34f7c0e2d2", + "panelRefName": "panel_114ba664-4260-47b8-86a5-ca34f7c0e2d2", + "type": "visualization", + "version": "7.14.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 11, + "i": "fc5c5920-5103-4a35-a26d-5d33c7263c88", + "w": 24, + "x": 0, + "y": 28 + }, + "panelIndex": "fc5c5920-5103-4a35-a26d-5d33c7263c88", + "panelRefName": "panel_fc5c5920-5103-4a35-a26d-5d33c7263c88", + "type": "visualization", + "version": "7.14.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 11, + "i": "94880c13-d232-426d-892e-095ddb9e9534", + "w": 24, + "x": 24, + "y": 28 + }, + "panelIndex": "94880c13-d232-426d-892e-095ddb9e9534", + "panelRefName": "panel_94880c13-d232-426d-892e-095ddb9e9534", + "type": "visualization", + "version": "7.14.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 11, + "i": "2b006213-1ca0-4392-a0c3-91ad24d50355", + "w": 24, + "x": 0, + "y": 39 + }, + "panelIndex": "2b006213-1ca0-4392-a0c3-91ad24d50355", + "panelRefName": "panel_2b006213-1ca0-4392-a0c3-91ad24d50355", + "type": "visualization", + "version": "7.14.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 11, + "i": "b009d3ec-23cc-4838-8528-f9f928d29d6f", + "w": 24, + "x": 24, + "y": 39 + }, + "panelIndex": "b009d3ec-23cc-4838-8528-f9f928d29d6f", + "panelRefName": "panel_b009d3ec-23cc-4838-8528-f9f928d29d6f", + "type": "visualization", + "version": "7.14.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 11, + "i": "9e163181-aade-48e1-a853-5f32782f7b44", + "w": 24, + "x": 0, + "y": 50 + }, + "panelIndex": "9e163181-aade-48e1-a853-5f32782f7b44", + "panelRefName": "panel_9e163181-aade-48e1-a853-5f32782f7b44", + "type": "visualization", + "version": "7.14.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 11, + "i": "a35d8006-6c05-4c0d-87b0-af47e27e364d", + "w": 24, + "x": 24, + "y": 50 + }, + "panelIndex": "a35d8006-6c05-4c0d-87b0-af47e27e364d", + "panelRefName": "panel_a35d8006-6c05-4c0d-87b0-af47e27e364d", + "type": "visualization", + "version": "7.14.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 11, + "i": "c620c47e-dfc8-4ce9-b499-41b6c87ad9e3", + "w": 24, + "x": 0, + "y": 61 + }, + "panelIndex": "c620c47e-dfc8-4ce9-b499-41b6c87ad9e3", + "panelRefName": "panel_c620c47e-dfc8-4ce9-b499-41b6c87ad9e3", + "type": "visualization", + "version": "7.14.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 11, + "i": "79ed566c-3361-48cf-8454-8151458bcbd1", + "w": 24, + "x": 24, + "y": 61 + }, + "panelIndex": "79ed566c-3361-48cf-8454-8151458bcbd1", + "panelRefName": "panel_79ed566c-3361-48cf-8454-8151458bcbd1", + "type": "visualization", + "version": "7.14.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 7, + "i": "751cb1e0-d96c-423b-8c42-a3f6e5125742", + "w": 17, + "x": 14, + "y": 0 + }, + "panelIndex": "751cb1e0-d96c-423b-8c42-a3f6e5125742", + "panelRefName": "panel_751cb1e0-d96c-423b-8c42-a3f6e5125742", + "title": "Top 10 Session counts by Host Name [Metricbeat Oracle]", + "type": "visualization", + "version": "7.14.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 7, + "i": "4b77ac12-2696-48dc-a364-6f0686e58ad2", + "w": 17, + "x": 31, + "y": 0 + }, + "panelIndex": "4b77ac12-2696-48dc-a364-6f0686e58ad2", + "panelRefName": "panel_4b77ac12-2696-48dc-a364-6f0686e58ad2", + "title": "Top 10 Average Active Session counts by Host Name [Metricbeat Oracle]", + "type": "visualization", + "version": "7.14.0" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 11, + "i": "e0b578f9-2373-4234-a761-df7fe39579eb", + "w": 48, + "x": 0, + "y": 72 + }, + "panelIndex": "e0b578f9-2373-4234-a761-df7fe39579eb", + "panelRefName": "panel_e0b578f9-2373-4234-a761-df7fe39579eb", + "type": "visualization", + "version": "7.14.0" + } + ], + "timeRestore": false, + "title": "[Metricbeat Oracle] Sysmetric", + "version": 1 + }, + "coreMigrationVersion": "7.14.0", + "id": "a3f4d080-c7d3-11ec-891a-5f50c7522a3a", + "migrationVersion": { + "dashboard": "7.14.0" + }, + "references": [ + { + "id": "c2178960-c83f-11ec-891a-5f50c7522a3a", + "name": "c1235d00-17bf-46e3-be1a-f9e04689ba4f:panel_c1235d00-17bf-46e3-be1a-f9e04689ba4f", + "type": "visualization" + }, + { + "id": "3af08cd0-c843-11ec-891a-5f50c7522a3a", + "name": "25e28f7d-5aa8-4f7b-a549-fda815fd37d6:panel_25e28f7d-5aa8-4f7b-a549-fda815fd37d6", + "type": "visualization" + }, + { + "id": "81c1f9a0-c843-11ec-891a-5f50c7522a3a", + "name": "a8222df9-8ae7-4fb3-b2b5-0c26928869c2:panel_a8222df9-8ae7-4fb3-b2b5-0c26928869c2", + "type": "visualization" + }, + { + "id": "61d15a20-c87d-11ec-891a-5f50c7522a3a", + "name": "411deb47-0f86-4bf1-a384-7977a03c11f6:panel_411deb47-0f86-4bf1-a384-7977a03c11f6", + "type": "visualization" + }, + { + "id": "a65d8eb0-c879-11ec-891a-5f50c7522a3a", + "name": "114ba664-4260-47b8-86a5-ca34f7c0e2d2:panel_114ba664-4260-47b8-86a5-ca34f7c0e2d2", + "type": "visualization" + }, + { + "id": "09929750-c87a-11ec-891a-5f50c7522a3a", + "name": "fc5c5920-5103-4a35-a26d-5d33c7263c88:panel_fc5c5920-5103-4a35-a26d-5d33c7263c88", + "type": "visualization" + }, + { + "id": "2ee49e30-c87b-11ec-891a-5f50c7522a3a", + "name": "94880c13-d232-426d-892e-095ddb9e9534:panel_94880c13-d232-426d-892e-095ddb9e9534", + "type": "visualization" + }, + { + "id": "ffd22450-c87a-11ec-891a-5f50c7522a3a", + "name": "2b006213-1ca0-4392-a0c3-91ad24d50355:panel_2b006213-1ca0-4392-a0c3-91ad24d50355", + "type": "visualization" + }, + { + "id": "00b53be0-c87c-11ec-891a-5f50c7522a3a", + "name": "b009d3ec-23cc-4838-8528-f9f928d29d6f:panel_b009d3ec-23cc-4838-8528-f9f928d29d6f", + "type": "visualization" + }, + { + "id": "d5e99370-c87b-11ec-891a-5f50c7522a3a", + "name": "9e163181-aade-48e1-a853-5f32782f7b44:panel_9e163181-aade-48e1-a853-5f32782f7b44", + "type": "visualization" + }, + { + "id": "400c71a0-c87c-11ec-891a-5f50c7522a3a", + "name": "a35d8006-6c05-4c0d-87b0-af47e27e364d:panel_a35d8006-6c05-4c0d-87b0-af47e27e364d", + "type": "visualization" + }, + { + "id": "21744290-c87c-11ec-891a-5f50c7522a3a", + "name": "c620c47e-dfc8-4ce9-b499-41b6c87ad9e3:panel_c620c47e-dfc8-4ce9-b499-41b6c87ad9e3", + "type": "visualization" + }, + { + "id": "5e35d950-c87c-11ec-891a-5f50c7522a3a", + "name": "79ed566c-3361-48cf-8454-8151458bcbd1:panel_79ed566c-3361-48cf-8454-8151458bcbd1", + "type": "visualization" + }, + { + "id": "4ae42710-c9e6-11ec-891a-5f50c7522a3a", + "name": "751cb1e0-d96c-423b-8c42-a3f6e5125742:panel_751cb1e0-d96c-423b-8c42-a3f6e5125742", + "type": "visualization" + }, + { + "id": "53dee3a0-c9e6-11ec-891a-5f50c7522a3a", + "name": "4b77ac12-2696-48dc-a364-6f0686e58ad2:panel_4b77ac12-2696-48dc-a364-6f0686e58ad2", + "type": "visualization" + }, + { + "id": "84ef9e80-c9e6-11ec-891a-5f50c7522a3a", + "name": "e0b578f9-2373-4234-a761-df7fe39579eb:panel_e0b578f9-2373-4234-a761-df7fe39579eb", + "type": "visualization" + } + ], + "type": "dashboard", + "updated_at": "2022-05-27T05:42:52.581Z", + "version": "WzQ3MDEsMV0=" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/00b53be0-c87c-11ec-891a-5f50c7522a3a.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/00b53be0-c87c-11ec-891a-5f50c7522a3a.json new file mode 100644 index 00000000000..2709f5e79e0 --- /dev/null +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/00b53be0-c87c-11ec-891a-5f50c7522a3a.json @@ -0,0 +1,89 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Network Traffic Volume per second [Metricbeat Oracle]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "drop_last_bucket": 0, + "id": "748bd318-1ee4-4533-945b-bb6853a32ac2", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "default", + "id": "feb0d746-e5b9-4ca6-ac2e-a8e12bd25fca", + "label": "Network Traffic Volume per second", + "line_width": 1, + "metrics": [ + { + "agg_with": "avg", + "field": "oracle.sysmetric.network_traffic_volume_per_sec", + "id": "d2752c2a-af50-4cc9-a8a4-212399bb67e5", + "order": "desc", + "size": 1, + "type": "top_hit" + } + ], + "override_index_pattern": 0, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_drop_last_bucket": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "service.address", + "time_range_mode": "entire_time_range" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "timeseries", + "use_kibana_indexes": true + }, + "title": "Network Traffic Volume per second [Metricbeat Oracle]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.14.0", + "id": "00b53be0-c87c-11ec-891a-5f50c7522a3a", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2022-05-27T05:56:01.206Z", + "version": "WzYwMjIsMV0=" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/09929750-c87a-11ec-891a-5f50c7522a3a.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/09929750-c87a-11ec-891a-5f50c7522a3a.json new file mode 100644 index 00000000000..b1262bb8c99 --- /dev/null +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/09929750-c87a-11ec-891a-5f50c7522a3a.json @@ -0,0 +1,89 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Physical Writes per second [Metricbeat Oracle]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "drop_last_bucket": 0, + "id": "1aecb7d4-67f6-432c-a1c9-d7f92db7b5ea", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "default", + "id": "fbc600ac-5913-4aab-bc54-46ed3ee5f010", + "label": "Physical Writes per second", + "line_width": 1, + "metrics": [ + { + "agg_with": "avg", + "field": "oracle.sysmetric.physical_writes_per_sec", + "id": "1f50728e-6c16-4543-b103-f35a4e37a415", + "order": "desc", + "size": 1, + "type": "top_hit" + } + ], + "override_index_pattern": 0, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_drop_last_bucket": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "service.address", + "time_range_mode": "entire_time_range" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "timeseries", + "use_kibana_indexes": true + }, + "title": "Physical Writes per second [Metricbeat Oracle]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.14.0", + "id": "09929750-c87a-11ec-891a-5f50c7522a3a", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2022-05-27T05:52:44.264Z", + "version": "WzU3NTcsMV0=" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/21744290-c87c-11ec-891a-5f50c7522a3a.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/21744290-c87c-11ec-891a-5f50c7522a3a.json new file mode 100644 index 00000000000..cadb5c9176a --- /dev/null +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/21744290-c87c-11ec-891a-5f50c7522a3a.json @@ -0,0 +1,89 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "DB Block Changes per second [Metricbeat Oracle]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "drop_last_bucket": 0, + "id": "3109b107-a1a0-437c-b719-7614f6963831", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "default", + "id": "ea5f6c6d-9837-4b97-9816-a9d5810e9565", + "label": "DB Block Changes per second", + "line_width": 1, + "metrics": [ + { + "agg_with": "avg", + "field": "oracle.sysmetric.db_block_changes_per_sec", + "id": "c3f9567a-8092-4b7e-8913-5f9c94d10698", + "order": "desc", + "size": 1, + "type": "top_hit" + } + ], + "override_index_pattern": 0, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_drop_last_bucket": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "service.address", + "time_range_mode": "entire_time_range" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "timeseries", + "use_kibana_indexes": true + }, + "title": "DB Block Changes per second [Metricbeat Oracle]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.14.0", + "id": "21744290-c87c-11ec-891a-5f50c7522a3a", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2022-05-27T05:58:48.437Z", + "version": "WzYyMzYsMV0=" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/2ee49e30-c87b-11ec-891a-5f50c7522a3a.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/2ee49e30-c87b-11ec-891a-5f50c7522a3a.json new file mode 100644 index 00000000000..7781a21e6d9 --- /dev/null +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/2ee49e30-c87b-11ec-891a-5f50c7522a3a.json @@ -0,0 +1,89 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Total Index Scans per transaction [Metricbeat Oracle]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "drop_last_bucket": 0, + "id": "437d5319-0c8c-4f97-860d-094d0a62f3e9", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "default", + "id": "64209a25-471f-4c43-9053-cb4937a535e0", + "label": "Total Index Scans per transaction", + "line_width": 1, + "metrics": [ + { + "agg_with": "avg", + "field": "oracle.sysmetric.total_index_scans_per_txn", + "id": "de615d10-f4fa-42d7-aff3-1e74a43a52d7", + "order": "desc", + "size": 1, + "type": "top_hit" + } + ], + "override_index_pattern": 0, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_drop_last_bucket": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "service.address", + "time_range_mode": "entire_time_range" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "timeseries", + "use_kibana_indexes": true + }, + "title": "Total Index Scans per transaction [Metricbeat Oracle]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.14.0", + "id": "2ee49e30-c87b-11ec-891a-5f50c7522a3a", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2022-05-27T05:54:33.172Z", + "version": "WzU4OTAsMV0=" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/3af08cd0-c843-11ec-891a-5f50c7522a3a.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/3af08cd0-c843-11ec-891a-5f50c7522a3a.json new file mode 100644 index 00000000000..356defa4039 --- /dev/null +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/3af08cd0-c843-11ec-891a-5f50c7522a3a.json @@ -0,0 +1,90 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Current OS Load [Metricbeat Oracle]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "drop_last_bucket": 0, + "id": "d1ccd565-8204-4134-8b4f-7e5af26d2712", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "isModelInvalid": false, + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "default", + "id": "23a79e09-a8d8-4cad-af8e-de72117d0c5a", + "label": "Current OS Load", + "line_width": 1, + "metrics": [ + { + "agg_with": "avg", + "field": "oracle.sysmetric.current_os_load", + "id": "9f054aac-6be0-4e51-a21b-48a078761230", + "order": "desc", + "size": 1, + "type": "top_hit" + } + ], + "override_index_pattern": 0, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_drop_last_bucket": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "service.address", + "time_range_mode": "entire_time_range" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "timeseries", + "use_kibana_indexes": true + }, + "title": "Current OS Load [Metricbeat Oracle]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.14.0", + "id": "3af08cd0-c843-11ec-891a-5f50c7522a3a", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2022-05-27T05:49:18.979Z", + "version": "WzU0MTgsMV0=" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/400c71a0-c87c-11ec-891a-5f50c7522a3a.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/400c71a0-c87c-11ec-891a-5f50c7522a3a.json new file mode 100644 index 00000000000..469217261e6 --- /dev/null +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/400c71a0-c87c-11ec-891a-5f50c7522a3a.json @@ -0,0 +1,89 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "CPU Usage per second [Metricbeat Oracle]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "drop_last_bucket": 0, + "id": "2803c7f6-db0f-4d98-9e51-ebd0d5c888a3", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "default", + "id": "a61d65bc-f531-43e1-9434-f6be86e14425", + "label": "CPU Usage per second", + "line_width": 1, + "metrics": [ + { + "agg_with": "avg", + "field": "oracle.sysmetric.cpu_usage_per_sec", + "id": "05f6a0dc-0d50-4ab8-81b2-f5e736e7b4be", + "order": "desc", + "size": 1, + "type": "top_hit" + } + ], + "override_index_pattern": 0, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_drop_last_bucket": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "service.address", + "time_range_mode": "entire_time_range" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "timeseries", + "use_kibana_indexes": true + }, + "title": "CPU Usage per second [Metricbeat Oracle]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.14.0", + "id": "400c71a0-c87c-11ec-891a-5f50c7522a3a", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2022-05-27T05:57:58.481Z", + "version": "WzYxNjksMV0=" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/4ae42710-c9e6-11ec-891a-5f50c7522a3a.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/4ae42710-c9e6-11ec-891a-5f50c7522a3a.json new file mode 100644 index 00000000000..b6df7ba1ad2 --- /dev/null +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/4ae42710-c9e6-11ec-891a-5f50c7522a3a.json @@ -0,0 +1,109 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Top 10 session counts by host [Metricbeat Oracle]", + "uiStateJSON": { + "table": { + "sort": { + "column": "_default_", + "order": "asc" + } + } + }, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "bar_color_rules": [ + { + "id": "33392060-c9dd-11ec-bab5-652476a9e8f4" + } + ], + "drop_last_bucket": 0, + "id": "c206d808-ee97-447f-831c-2d968d81528e", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "max_lines_legend": 1, + "pivot_id": "service.address", + "pivot_label": "Host Name", + "pivot_rows": "10", + "pivot_type": "string", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "color_rules": [ + { + "id": "765e58a0-c9de-11ec-bab5-652476a9e8f4" + } + ], + "fill": 0.5, + "formatter": "default", + "id": "34cada37-a728-4938-a05d-4fc4b81bbe73", + "label": "Session Count", + "line_width": 1, + "metrics": [ + { + "agg_with": "avg", + "field": "oracle.sysmetric.session_count", + "id": "13104878-3145-41ff-9282-6e66fd77a9a1", + "order": "desc", + "size": "1", + "type": "top_hit" + } + ], + "override_index_pattern": 0, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_drop_last_bucket": 0, + "split_mode": "everything", + "stacked": "none", + "time_range_mode": "entire_time_range" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "table", + "use_kibana_indexes": true + }, + "title": "Top 10 session counts by host [Metricbeat Oracle]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.14.0", + "id": "4ae42710-c9e6-11ec-891a-5f50c7522a3a", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2022-05-27T06:05:07.435Z", + "version": "WzY0NjQsMV0=" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/53dee3a0-c9e6-11ec-891a-5f50c7522a3a.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/53dee3a0-c9e6-11ec-891a-5f50c7522a3a.json new file mode 100644 index 00000000000..29889984b8b --- /dev/null +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/53dee3a0-c9e6-11ec-891a-5f50c7522a3a.json @@ -0,0 +1,128 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "_id", + "negate": false, + "type": "exists", + "value": "exists" + }, + "query": { + "exists": { + "field": "_id" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Top 10 Average Active Session counts by host [Metricbeat Oracle]", + "uiStateJSON": { + "table": { + "sort": { + "column": "_default_", + "order": "asc" + } + } + }, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "bar_color_rules": [ + { + "id": "7c823200-c9df-11ec-bab5-652476a9e8f4" + } + ], + "drop_last_bucket": 0, + "id": "37de0557-33eb-44de-a7e5-50e03a0d5207", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "max_lines_legend": 1, + "pivot_id": "service.address", + "pivot_label": "Host Name", + "pivot_type": "string", + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "default", + "id": "ae0aa359-2745-4f69-9c18-cab79b2d7692", + "label": "Average Active Sessions", + "line_width": 1, + "metrics": [ + { + "agg_with": "avg", + "field": "oracle.sysmetric.average_active_sessions", + "id": "63e5475d-5d50-4796-a13a-1e53ebca3a5c", + "order": "desc", + "size": 1, + "type": "top_hit" + } + ], + "override_index_pattern": 0, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_drop_last_bucket": 0, + "split_mode": "everything", + "stacked": "none", + "time_range_mode": "entire_time_range" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "table", + "use_kibana_indexes": true + }, + "title": "Top 10 Average Active Session counts by host [Metricbeat Oracle]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.14.0", + "id": "53dee3a0-c9e6-11ec-891a-5f50c7522a3a", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2022-05-27T06:05:31.886Z", + "version": "WzY0OTgsMV0=" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/5e35d950-c87c-11ec-891a-5f50c7522a3a.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/5e35d950-c87c-11ec-891a-5f50c7522a3a.json new file mode 100644 index 00000000000..447c6251927 --- /dev/null +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/5e35d950-c87c-11ec-891a-5f50c7522a3a.json @@ -0,0 +1,89 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Physical Read Total Bytes per second [Metricbeat Oracle]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "drop_last_bucket": 0, + "id": "df95ef37-41af-4228-a8f3-cc5ba7217eb1", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "default", + "id": "81a640b4-1cff-4ae3-8b9e-2e38c6c40991", + "label": "Physical Read Total Bytes per second", + "line_width": 1, + "metrics": [ + { + "agg_with": "avg", + "field": "oracle.sysmetric.physical_read_total_bytes_per_sec", + "id": "391f7ff8-685d-43d4-b215-936a2018614f", + "order": "desc", + "size": 1, + "type": "top_hit" + } + ], + "override_index_pattern": 0, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_drop_last_bucket": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "service.address", + "time_range_mode": "entire_time_range" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "timeseries", + "use_kibana_indexes": true + }, + "title": "Physical Read Total Bytes per second [Metricbeat Oracle]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.14.0", + "id": "5e35d950-c87c-11ec-891a-5f50c7522a3a", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2022-05-27T05:59:39.135Z", + "version": "WzYzMDcsMV0=" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/61d15a20-c87d-11ec-891a-5f50c7522a3a.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/61d15a20-c87d-11ec-891a-5f50c7522a3a.json new file mode 100644 index 00000000000..c2a1a1327e8 --- /dev/null +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/61d15a20-c87d-11ec-891a-5f50c7522a3a.json @@ -0,0 +1,89 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "User Transactions per second [Metricbeat Oracle]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "drop_last_bucket": 0, + "id": "896af1a1-badb-4109-a3cd-43c91c8e2aa2", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "default", + "id": "e9a6077f-9afb-47c8-8f27-8b797c04e40c", + "label": "User Transactions per second", + "line_width": 1, + "metrics": [ + { + "agg_with": "avg", + "field": "oracle.sysmetric.user_transaction_per_sec", + "id": "313f52b8-c418-42c3-872f-578a17b28827", + "order": "desc", + "size": 1, + "type": "top_hit" + } + ], + "override_index_pattern": 0, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_drop_last_bucket": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "service.address", + "time_range_mode": "entire_time_range" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "timeseries", + "use_kibana_indexes": true + }, + "title": "User Transactions per second [Metricbeat Oracle]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.14.0", + "id": "61d15a20-c87d-11ec-891a-5f50c7522a3a", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2022-05-27T05:51:29.124Z", + "version": "WzU2NDEsMV0=" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/81c1f9a0-c843-11ec-891a-5f50c7522a3a.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/81c1f9a0-c843-11ec-891a-5f50c7522a3a.json new file mode 100644 index 00000000000..a91a894eb85 --- /dev/null +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/81c1f9a0-c843-11ec-891a-5f50c7522a3a.json @@ -0,0 +1,91 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Physical Reads per second [Metricbeat Oracle]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "drop_last_bucket": 0, + "id": "bf98b445-17ae-48cd-80d2-309d0b4fc3f1", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "isModelInvalid": false, + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "default", + "id": "f94cdd87-3e42-482d-a944-fe5551cd45ff", + "label": "Physical Reads per second", + "line_width": 1, + "metrics": [ + { + "agg_with": "avg", + "field": "oracle.sysmetric.physical_reads_per_sec", + "id": "a19b2652-8832-40a8-9f62-e2a09a78a292", + "order": "desc", + "size": 1, + "type": "top_hit" + } + ], + "override_index_pattern": 0, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_drop_last_bucket": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "service.address", + "terms_size": "10", + "time_range_mode": "entire_time_range" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "timeseries", + "use_kibana_indexes": true + }, + "title": "Physical Reads per second [Metricbeat Oracle]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.14.0", + "id": "81c1f9a0-c843-11ec-891a-5f50c7522a3a", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2022-05-27T05:50:29.715Z", + "version": "WzU1MzIsMV0=" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/84ef9e80-c9e6-11ec-891a-5f50c7522a3a.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/84ef9e80-c9e6-11ec-891a-5f50c7522a3a.json new file mode 100644 index 00000000000..5bef64e331c --- /dev/null +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/84ef9e80-c9e6-11ec-891a-5f50c7522a3a.json @@ -0,0 +1,89 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Response Time (Centi-Second) per transaction [Metricbeat Oracle]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "drop_last_bucket": 0, + "id": "8536b4c8-d395-4f9b-92b0-2032340b058d", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "default", + "id": "b7dfe6dd-e4b1-4e13-85fd-afd0fceea0b8", + "label": "", + "line_width": 1, + "metrics": [ + { + "agg_with": "avg", + "field": "oracle.sysmetric.response_time_per_txn", + "id": "6fff9fe5-29f2-4c80-9a3b-c1a4ab2ffd27", + "order": "desc", + "size": 1, + "type": "top_hit" + } + ], + "override_index_pattern": 0, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_drop_last_bucket": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "service.address", + "time_range_mode": "entire_time_range" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "timeseries", + "use_kibana_indexes": true + }, + "title": "Response Time (Centi-Second) per transaction [Metricbeat Oracle]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.14.0", + "id": "84ef9e80-c9e6-11ec-891a-5f50c7522a3a", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2022-05-27T06:20:36.923Z", + "version": "WzczNjksMV0=" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/8fb638f0-c90b-11ec-891a-5f50c7522a3a.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/8fb638f0-c90b-11ec-891a-5f50c7522a3a.json new file mode 100644 index 00000000000..79615d52748 --- /dev/null +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/8fb638f0-c90b-11ec-891a-5f50c7522a3a.json @@ -0,0 +1,99 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Session Count [Metricbeat Oracle]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "b0845e80-c7d1-11ec-92b7-c5ca08f4288e" + } + ], + "drop_last_bucket": 0, + "id": "4d7989fe-1b37-49c0-9226-7ff0945b5b0a", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "default", + "id": "5312c701-ad2c-4a4e-aaa7-725379cf1da9", + "label": "Session Count", + "line_width": 1, + "metrics": [ + { + "agg_with": "avg", + "field": "oracle.sysmetric.session_count", + "id": "ceb12d6e-4e52-4fe4-88cb-79eba943ed47", + "order": "desc", + "size": 1, + "type": "top_hit" + } + ], + "override_index_pattern": 0, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_drop_last_bucket": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "service.address", + "time_range_mode": "entire_time_range" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "metric", + "use_kibana_indexes": true + }, + "title": "Session Count [Metricbeat Oracle]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.14.0", + "id": "8fb638f0-c90b-11ec-891a-5f50c7522a3a", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2022-05-01T05:00:03.462Z", + "version": "WzI4MTkwLDFd" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/93361970-c922-11ec-891a-5f50c7522a3a.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/93361970-c922-11ec-891a-5f50c7522a3a.json new file mode 100644 index 00000000000..6993dda27b3 --- /dev/null +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/93361970-c922-11ec-891a-5f50c7522a3a.json @@ -0,0 +1,100 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Response Time (Centi-Second) per transaction [Metricbeat Oracle]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "75beaee0-c7bd-11ec-92b7-c5ca08f4288e" + } + ], + "drop_last_bucket": 0, + "id": "42d37eeb-105d-4c32-845e-347cdd0efdff", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "isModelInvalid": false, + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "default", + "id": "60ea5eb0-0b9f-40b1-bbe4-ddc2fc795155", + "label": "Response Time (Centi-Second) per txn", + "line_width": 1, + "metrics": [ + { + "agg_with": "avg", + "field": "oracle.sysmetric.response_time_per_txn", + "id": "c92b4dba-3592-4d2e-9f60-bcab33fa9072", + "order": "desc", + "size": 1, + "type": "top_hit" + } + ], + "override_index_pattern": 0, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_drop_last_bucket": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "service.address", + "time_range_mode": "entire_time_range" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "metric", + "use_kibana_indexes": true + }, + "title": "Response Time (Centi-Second) per transaction [Metricbeat Oracle]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.14.0", + "id": "93361970-c922-11ec-891a-5f50c7522a3a", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2022-05-01T07:44:47.786Z", + "version": "WzMxODEyLDFd" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/a65d8eb0-c879-11ec-891a-5f50c7522a3a.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/a65d8eb0-c879-11ec-891a-5f50c7522a3a.json new file mode 100644 index 00000000000..029c4065152 --- /dev/null +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/a65d8eb0-c879-11ec-891a-5f50c7522a3a.json @@ -0,0 +1,95 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Total Table Scans per transaction [Metricbeat Oracle]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "4fed3540-c7b0-11ec-92b7-c5ca08f4288e" + } + ], + "drop_last_bucket": 0, + "id": "0a642abd-0af3-4be0-8b32-e64e2435e4cd", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "isModelInvalid": false, + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "default", + "id": "dcb1b4af-bd64-4dbe-b27f-fc6652098b4d", + "label": "Total Table Scans per transaction", + "line_width": 1, + "metrics": [ + { + "agg_with": "avg", + "field": "oracle.sysmetric.total_table_scans_per_txn", + "id": "e7116e83-4a2a-4500-98e4-121db45772e6", + "order": "desc", + "size": 1, + "type": "top_hit" + } + ], + "override_index_pattern": 0, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_drop_last_bucket": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "service.address", + "time_range_mode": "entire_time_range" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "timeseries", + "use_kibana_indexes": true + }, + "title": "Total Table Scans per transaction [Metricbeat Oracle]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.14.0", + "id": "a65d8eb0-c879-11ec-891a-5f50c7522a3a", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2022-05-27T05:52:09.097Z", + "version": "WzU3MDAsMV0=" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/c2178960-c83f-11ec-891a-5f50c7522a3a.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/c2178960-c83f-11ec-891a-5f50c7522a3a.json new file mode 100644 index 00000000000..62dc4b806fd --- /dev/null +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/c2178960-c83f-11ec-891a-5f50c7522a3a.json @@ -0,0 +1,86 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "event.dataset", + "negate": false, + "params": { + "query": "oracle.sysmetric" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "event.dataset": "oracle.sysmetric" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Oracle Host Control [Metricbeat Oracle]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "controls": [ + { + "fieldName": "service.address", + "id": "1651233625783", + "indexPatternRefName": "control_0_index_pattern", + "label": "Oracle Host Control", + "options": { + "dynamicOptions": false, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + } + ], + "pinFilters": false, + "updateFiltersOnChange": true, + "useTimeFilter": false + }, + "title": "Oracle Host Control [Metricbeat Oracle]", + "type": "input_control_vis" + } + }, + "coreMigrationVersion": "7.14.0", + "id": "c2178960-c83f-11ec-891a-5f50c7522a3a", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "control_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2022-05-27T05:42:52.581Z", + "version": "WzQ2ODUsMV0=" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/c777ba10-c83f-11ec-891a-5f50c7522a3a.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/c777ba10-c83f-11ec-891a-5f50c7522a3a.json new file mode 100644 index 00000000000..6202eb0bcae --- /dev/null +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/c777ba10-c83f-11ec-891a-5f50c7522a3a.json @@ -0,0 +1,86 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Average Active Sessions [Metricbeat Oracle]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "aggregate": "concat", + "customLabel": "Average Active Sessions", + "field": "oracle.sysmetric.average_active_sessions", + "size": 1, + "sortField": "@timestamp", + "sortOrder": "desc" + }, + "schema": "metric", + "type": "top_hits" + } + ], + "params": { + "addLegend": false, + "addTooltip": true, + "metric": { + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 10000 + } + ], + "invertColors": false, + "labels": { + "show": true + }, + "metricColorMode": "None", + "percentageMode": false, + "style": { + "bgColor": false, + "bgFill": "#000", + "fontSize": 60, + "labelColor": false, + "subText": "" + }, + "useRanges": false + }, + "type": "metric" + }, + "title": "Average Active Sessions [Metricbeat Oracle]", + "type": "metric" + } + }, + "coreMigrationVersion": "7.14.0", + "id": "c777ba10-c83f-11ec-891a-5f50c7522a3a", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2022-04-30T04:41:19.675Z", + "version": "WzI0NzQwLDFd" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/d5e99370-c87b-11ec-891a-5f50c7522a3a.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/d5e99370-c87b-11ec-891a-5f50c7522a3a.json new file mode 100644 index 00000000000..2e5f19a591a --- /dev/null +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/d5e99370-c87b-11ec-891a-5f50c7522a3a.json @@ -0,0 +1,97 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "User Rollbacks per second [Metricbeat Oracle]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "drop_last_bucket": 0, + "gauge_color_rules": [ + { + "id": "7cf215c0-c7b9-11ec-92b7-c5ca08f4288e" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "9c78a97b-16f5-43d0-8b6b-007f0944d37f", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "default", + "id": "06d50f65-9ee9-43e9-b27a-561f598ae8c5", + "label": "User Rollbacks per second", + "line_width": 1, + "metrics": [ + { + "agg_with": "avg", + "field": "oracle.sysmetric.user_rollbacks_per_sec", + "id": "312e4114-c5a7-4518-894b-dd53c59309d4", + "order": "desc", + "size": 1, + "type": "top_hit" + } + ], + "override_index_pattern": 0, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_drop_last_bucket": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "service.address", + "time_range_mode": "entire_time_range" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "timeseries", + "use_kibana_indexes": true + }, + "title": "User Rollbacks per second [Metricbeat Oracle]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.14.0", + "id": "d5e99370-c87b-11ec-891a-5f50c7522a3a", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2022-05-27T05:56:53.899Z", + "version": "WzYwODcsMV0=" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/ffd22450-c87a-11ec-891a-5f50c7522a3a.json b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/ffd22450-c87a-11ec-891a-5f50c7522a3a.json new file mode 100644 index 00000000000..e25b562a83b --- /dev/null +++ b/x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/ffd22450-c87a-11ec-891a-5f50c7522a3a.json @@ -0,0 +1,89 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Host CPU Utilization (%) [Metricbeat Oracle]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "drop_last_bucket": 0, + "id": "0590e530-cf66-4e39-a8ac-47015d91def9", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "default", + "id": "e55e9af3-e602-4fa6-a99b-ddac067b027f", + "label": "Host CPU Utilization (%)", + "line_width": 1, + "metrics": [ + { + "agg_with": "avg", + "field": "oracle.sysmetric.host_cpu_utilization_pct", + "id": "3d751ffd-899e-41f4-b5f8-a673534be12f", + "order": "desc", + "size": 1, + "type": "top_hit" + } + ], + "override_index_pattern": 0, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_drop_last_bucket": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "service.address", + "time_range_mode": "entire_time_range" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "timeseries", + "use_kibana_indexes": true + }, + "title": "Host CPU Utilization (%) [Metricbeat Oracle]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.14.0", + "id": "ffd22450-c87a-11ec-891a-5f50c7522a3a", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2022-05-27T05:55:17.171Z", + "version": "WzU5NTAsMV0=" +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/connection.go b/x-pack/metricbeat/module/oracle/connection.go index a2b49679dcb..05f36b8eca6 100644 --- a/x-pack/metricbeat/module/oracle/connection.go +++ b/x-pack/metricbeat/module/oracle/connection.go @@ -16,8 +16,9 @@ import ( // ConnectionDetails contains all possible data that can be used to create a connection with // an Oracle db type ConnectionDetails struct { - Username string `config:"username"` - Password string `config:"password"` + Username string `config:"username"` + Password string `config:"password"` + Patterns []interface{} `config:"patterns"` } // HostParser parses host and extracts connection information and returns it to HostData diff --git a/x-pack/metricbeat/module/oracle/fields.go b/x-pack/metricbeat/module/oracle/fields.go index d259a00f9f7..cbacdb7ccbc 100644 --- a/x-pack/metricbeat/module/oracle/fields.go +++ b/x-pack/metricbeat/module/oracle/fields.go @@ -19,5 +19,5 @@ func init() { // AssetOracle returns asset data. // This is the base64 encoded zlib format compressed contents of module/oracle. func AssetOracle() string { - return "eJzEWE1v2zgTvudXDHJxC7ju3YcXSNvk3QBpnE2CAnsyxuLI4oYiVZKK4/76xVAfli3KsesgzaWuJM4888w3P8ETradgLCaKzgC89IqmcD4LD87PAAS5xMrCS6OnUD0GgR4X6AhyI8pwzmXG+nlidCqXU0hROX5qSRE6msISzwBSSUq46RkAwCfQmFNHMf/5dcHfWlMW9ZOY8lYn/3VlduUWZFNjc9QJte9iCnpK7jYHGT56EpCTtzJxYDQgOKmXXQqkdn5HzbbZzd8u1C7cHJNMatp618B9ovXKWLHzbpuZgix6qZfg1s5T3sgL0qMKF2Wakp0XxqjfVXqLOYFJwWdUiwMWB1KHRxFeNupLR7aH7RiDq1CIiml0SDO3pAwKF9UiTLlQuwC2lNxXp+Ez3EntgBk2E7ion4N0gHoNd9e3zAJqMIt/KfHgM/T8UhsfiEildT58VgclCQ6ihMLb+lCGWiiCFTpILHHUjQG1gFUmkwws/SylJQesmN3cOZlak4OQ7mlSwURbCdZlviAbHCRzcoABAitgaeQ4sFNjazGu8iTDxpxcgQOOUyZ5mtcC4rwqo5f7WL14JovLNnICq7AgvyLSMFqSz6R3o2A8/8+NYEFs8uj/zauodQ39I9flMjWlFhyROeXGrrtS4xyxfceQNImyVEi9n6TXg69DEyrFEtn4z0Ubia7h5U7qv4Z5UaphOiePXLWgkJSQax4rubBo1004rchSnDZWPUTbW4RWgkkWLwi7BbtH1oNHL53nGo0LU/pgdqcmOcBnlAoXigKwwfoUr9LQK5yTTPpJkfjeZ3sd3MP9mFFlNmTS16lQu8UVlMhUkujaMRmEVWRrJxNUc0v9ggf7krMH6q4WBXFRjcYlDVsfc1hPz5dOz+CcHIFr3Rg5O+SYLqbEaCedJx2D9ioLPYRfW3Fsrdvg2wtCLOYLLiMxI47H8K2ZMyqZsDTek44nUGmdsfGC82oKfa0Og9Rh/OGHR2YGPscMOiYdmppXGwKmIM0JsG4bfahE8WGpO0y9nAjkO76cCsIbj7uj1bEwHllGA6DBcyyQ6vhpqTrbhrAvDw5K09Lat8rRiqNNR6plB7MHMmIbzJCj3gRKN4I2M1/TephI6yPOaas6WheLkiNcd8ciOi7rJng1qpokKS2JZm5vCHTkXL8IwIEutvRGpJ7f0xapRWXPBrklMW3A1pRveqqDT9UJSEypPXwI3v54/ociooaCSjn4kKEVIYOdSf1HXiz4R/0NLxeQZJQ8ha2zXS5QcVdeN85yGbLruIeOwRt4JivTdcUOvy/I5jJQE7ZXflRqQVateXRsd49nCttKkqFekhieMmqaJ4HfOfN70qRx2xITXNXYFPEls8MjUk6o3ca8h79vQmRTzhErpAhmBHs8LxVgKdC5x6TWlDcY5+6bAW44HMO25uOFoneJ4Xlg3R2YD7jDiJ47/lLilAX9sYWwLSaqiLeSeSp7BB84urQjEos4ZYKRw23ykHDu2Fxq+bMkkIK05yneDuqMkAyvEN1TfCUV6c5tTNjyInx2Eln+OrGvPMhftNVLqsWruvCIpsohbSPHl8li7Wlodq7UTWHfN8eV6O/4IvMyD6ADLVyGhsW3a+C7ggxk184NQA/CmFqid2bzfwMfQVh13a4Z21s5j7Uhdifha0x8iap/KJe6dNWFwjOqMvTK0pHgku+8sfVnncvbcO2xp6t59OVwJzswCeOmj65CXAUNU7j4cXF9c/Hl5hKMhevbHxc319/gQ/Njp7kFK+omUV8nSs2WjgNb9IJ5oWgM2IREFAB2ukEle4UOhDVFQeLjaHht0Epqmr8NNzfoPDxps9K13JqRXsGawEwHbz/88zC7uhrzv4+X38cwu7q6ub69HMPslv9l/u4vv85+XN5P9veW/n3TwX2lU8/rqu54O/39/rI3IV9Lxt/qQ9WkwXprE9qMG7c1ZDgxOKveF2/I4/DzEHzBuj9BKNekLsD/AgAA//+LnocY" + return "eJzEWd1u2zoSvu9TDAos0gKpz70vDpCkyZ4AaZKN0yz2ShhLI4sbidQhKTvu0x8MRcmyRfkvbuubppI48803P5whv8ArLcegNMY5fQCwwuY0ho8P7sHHDwAJmViL0golx1A/hgQtTtEQFCqp3DqTKW2jWMlUzMaQYm74qaac0NAYZvgBIBWUJ2b8AQDgC0gsqKOYf3ZZ8rdaVaV/ElLe6uRfV2ZXbkk6VbpAGVP7LqSgp+RxtZDho6UECrJaxAaUBAQj5KxLgZDGbqhZN7v5bULtwi0wzoSktXcN3FdaLpRONt6tM1OSRivkDMzSWCoaeU56UOG0SlPSUalUfqzSeywIVAo2Iy8OWBwI6R4FeFmprwzpHrZDDK5DISim0SFUpClXmJiglkRV03wTwJqSp3o1/AGPQhpghtUILvxzEAZQLuHx9p5ZQAlq+n+KLdgMLb+UyjoiUqGNdZ/5oKSEgygm99YvylAmOcECDcSaOOrOAWUCi0zEGWj6uxKaDLBidnNnZapVAYkwr6MaJupasKyKKWnnIFGQAXQQWAFLI8OBnSrtxZjakwwbCzIlDjguV/Fr5AWEec2VnG1j9WJOGmdt5DhWYUp2QSThbEY2E9acOeP5f+YMpsQmn/27eRW0rqH/zHS5TFUlE47Iggqll12pYY7YvkNIGgVZKoXcTtLu4OvQhHnOEtn4P8o2Ek3Dy6OQfw3zkucN0wVZ5KoFpaCYTPM4F1ONetmE04I0hWlj1UO0nSK0YoyzcEHYLNg9siYWrTCWazROVWWd2Z2aZADnKHKc5uSADdancJWGXuEcZcKOytj2Ptvq4B7u54xqsyET1qeCd4spKRapoKRrx2gQVpktjYgxjzT1Cx5sS84eqEcvCsKiGo0zGrY+5LCensvOnsE5eQamdWNg7ZBjuphiJY0wlmQI2k4WegivWnFsrVnh2woimUZTLiMhIw7H8LXpM2qZMFPWkgwnUKWN0uGCszOFrurFIKRrf/jhgZmB85BBh6RDU/O8IaBKkpwAy3ajd5Uo3Cx1m6m3dwL5hm/vBWGVxc3W6lAYzyyjAdDgORRIvfx9qfqwDmFbHuyVppXWp8rRmqPVjuRlO7MHMmIdzJCjTgKlG0Grnq/ZephIbQPOaas6ahOKkgNc98giOi7rJnjdqqo4rjQlTd/eEGjImH4RgD1drOlEpH58ojVSy9qeFXJNybgB6ylf7akGvtQrIFaVtPDJefvzx98UER4K5rmBTxnqxGWwUan9zIMF/+G/4eEC4oziVzd1tsMF5rwrLxtnmQzZdbyHnoNVMCct0mXNDr8vSRfCUeOmV35UyYR0vuTWsZ095uSmlThDOaNkuMvwNI8cvxHz+65O474lxrmqsSngS2aHW6SCUJqVeZP/3LnIpoIjNhGJM8PZY3moAE2Ozi0mtaacoJ17ahq44XB005oNF4reIYZZmvroYecRRnviMCWLQ0cbf67hnzTCN485RnueWXgjI5dZR42Cf/YInXjirlhmeLbCukmIMLZiTpFHceyY1YfQNCEXTn6DyITB+GIZKRPxaH4yEFe+CD9M4E5hMjBlrvX8UUk6MrS5Jx+PoZ0Dnlg8PJKGCcVKDqDhpiSyGqVhzyh5cjzfDWl4XinYicilWWR56otMjLKmyL5t7mnHQ6rr+7ObKyeswWHqYNzhuIUWln6i5/7r5O9JlJAJvf1Uom5ZwyFEZcrYKC6rqLIiFz+wjqtenT4e11/KWLh6/A7fVwrg078+h+FIsgulXznK01TE0VzlVUEnd999rYbpYTXw4tTsl4Ba5fkU49ftQXV4YXbJ99QI34nF+cxwmT41Oc5ZLHknhmYOj+q25vRp9vUSLt154VWtYCeitYId1Vk3Xf7UCsC122ff5XKfWqDJlEoaiqzwoX3KOvDkpcOz8BEdrAFtYeLKunliuMclTnDd4bcy77mheG4hrIsJhypajFLRY3PPs5v2jIhFvOcIRwyfE+zTz3dsrqT4uyIQCUkrUkF6UGeAZNhBdE/xjchJdq6j3DF3gM9O8yp+vHOwnogftDZM1yfP9Y1PcFbYZ24u8G3kSsLAeFmrG8O2bw6bUb/hmyiqwoF2tPAcNiy+PQf/pSAd2d65DuheGFNN9IvZ7Je85veceXq7ZqxfS/AO7mJ35L7G2FaY9xcVQlamvlGZY165w4LKUMIzr7FK+886Y52799ky1lu01fAov2cShk0/u3Fx5TSM4eLl4vbu4vLuGpSG2/uXi7vbr/Cp+WNjundW+CnZ36cKyZaeO7boDYsyp3PAJiSCALCzG9SyF2gg0aosKfl8NkiKkrmQFJ2Gmzs0Fl6lWkgv1zPSK1gjeJDO25P/TR5ubs753+frb+fwcHNzd3t/fQ4P9/wv8/d0ffXwcv002r639C/c9t5XOvXcV3XXeB2/v2xNyF3JeNQ+VB+1sF5vQptx520NGU4Mzqpfi9flsftzH3zOut9BKNekLsB/AgAA//+tbOzU" } diff --git a/x-pack/metricbeat/module/oracle/sysmetric/_meta/data.json b/x-pack/metricbeat/module/oracle/sysmetric/_meta/data.json new file mode 100644 index 00000000000..de9dc177a47 --- /dev/null +++ b/x-pack/metricbeat/module/oracle/sysmetric/_meta/data.json @@ -0,0 +1,24 @@ +{ + "@timestamp": "2022-05-27T02:18:55.112Z", + "event": { + "dataset": "oracle.sysmetric", + "module": "oracle", + "duration": 408974115 + }, + "metricset": { + "name": "sysmetric", + "period": 60000 + }, + "oracle": { + "sysmetric": { + "physical_write_total_bytes_per_sec": 15323.3127812031, + "total_table_scans_per_txn": 40, + "physical_read_total_bytes_per_sec": 40680.153307782, + "total_pga_allocated": 2.364416e+08 + } + }, + "service": { + "address": "oracle://localhost:1521/ORCLCDB.localdomain", + "type": "oracle" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/oracle/sysmetric/_meta/docs.asciidoc b/x-pack/metricbeat/module/oracle/sysmetric/_meta/docs.asciidoc new file mode 100644 index 00000000000..ac31c8a2537 --- /dev/null +++ b/x-pack/metricbeat/module/oracle/sysmetric/_meta/docs.asciidoc @@ -0,0 +1,44 @@ +`sysmetric` Metricset includes the system metric values captured for the most current time interval from Oracle System View. + +This metricset dynamically filters metrics based on given input `patterns`. These patterns can be keywords or regular expressions. The `sysmetric` metricset uses the `OR` operator when forming the SQL query internally. Refer https://docs.oracle.com/cd/B12037_01/server.101/b10759/conditions016.htm[here] for more details. `V$SYSMETRIC` table provides different metrics for a short duration (15 seconds, `GROUP_ID` 3) and a long duration (60 seconds, `GROUP_ID` 2). This metricset collects metrics which have `GROUP_ID` 2 i.e. the metrics that are queried for long-duration for DBAs to use the data for historical analysis. + +*Note*: As the value of the metrics queried for long-duration in the `V$SYSMETRIC` table is updated every 60 seconds, it is recommended for the users to set the collection period in the configuration for sysmetric metricset as greater than or equal to 60 seconds so as to avoid duplication of the metrics in the ingested events. + +[float] +=== Required database access + +To ensure that the module has access to the appropriate metrics, the module requires that you configure a user with access to the following tables: + +* V$SYSMETRIC + +[float] +=== Example event + +---- +{ + "@timestamp": "2022-05-27T02:18:55.112Z", + "event": { + "dataset": "oracle.sysmetric", + "module": "oracle", + "duration": 408974115 + }, + "metricset": { + "name": "sysmetric", + "period": 60000 + }, + "oracle": { + "sysmetric": { + "metrics": { + "physical_write_total_bytes_per_sec": 15323.3127812031, + "total_table_scans_per_txn": 40, + "physical_read_total_bytes_per_sec": 40680.153307782, + "total_pga_allocated": 2.364416e+08 + } + } + }, + "service": { + "address": "oracle://localhost:1521/ORCLCDB.localdomain", + "type": "oracle" + } +} +---- diff --git a/x-pack/metricbeat/module/oracle/sysmetric/_meta/fields.yml b/x-pack/metricbeat/module/oracle/sysmetric/_meta/fields.yml new file mode 100644 index 00000000000..4b8d2c4dbd4 --- /dev/null +++ b/x-pack/metricbeat/module/oracle/sysmetric/_meta/fields.yml @@ -0,0 +1,66 @@ +- name: sysmetric + type: group + release: beta + description: > + Sysmetric related metrics. + fields: + - name: session_count + type: long + description: > + Session Count. + - name: average_active_sessions + type: double + description: > + Average Active Sessions. + - name: current_os_load + type: double + description: > + Current OS Load. + - name: physical_reads_per_sec + type: double + description: > + Physical Reads Per Second. + - name: user_transaction_per_sec + type: double + description: > + User Transaction Per Second. + - name: total_table_scans_per_txn + type: double + description: > + Total Table Scans Per Transaction. + - name: physical_writes_per_sec + type: double + description: > + Physical Writes Per Second. + - name: total_index_scans_per_txn + type: double + description: > + Total Index Scans Per Transaction. + - name: host_cpu_utilization_pct + type: double + description: > + Host CPU Utilization (%). + - name: network_traffic_volume_per_sec + type: double + description: > + Network Traffic Volume Per Second. + - name: user_rollbacks_per_sec + type: long + description: > + User Rollbacks Per Second. + - name: cpu_usage_per_sec + type: double + description: > + CPU Usage Per Second. + - name: db_block_changes_per_sec + type: double + description: > + DB Block Changes Per Second. + - name: physical_read_total_bytes_per_sec + type: double + description: > + Physical Read Total Bytes Per Second. + - name: response_time_per_txn + type: double + description: > + Response Time Per Transaction. diff --git a/x-pack/metricbeat/module/oracle/sysmetric/collector.go b/x-pack/metricbeat/module/oracle/sysmetric/collector.go new file mode 100644 index 00000000000..cd1a861ebba --- /dev/null +++ b/x-pack/metricbeat/module/oracle/sysmetric/collector.go @@ -0,0 +1,27 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package sysmetric + +import ( + "context" + "database/sql" +) + +// sysmetricCollectMethod contains the methods needed to collect the necessary information about the performance of the database. +type sysmetricCollectMethod interface { + sysmetricMetric(context.Context) ([]sysmetricMetric, error) +} + +// collectedData contains the necessary system metric information. +type collectedData struct { + sysmetricMetrics []sysmetricMetric +} + +// sysmetricCollector is the implementor of sysmetricCollectMethod. It's implementation are on different Go files +// which refers to the origin of the data for organization purposes. +type sysmetricCollector struct { + db *sql.DB + patterns []interface{} +} diff --git a/x-pack/metricbeat/module/oracle/sysmetric/data.go b/x-pack/metricbeat/module/oracle/sysmetric/data.go new file mode 100644 index 00000000000..d170a475161 --- /dev/null +++ b/x-pack/metricbeat/module/oracle/sysmetric/data.go @@ -0,0 +1,43 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package sysmetric + +import ( + "context" + "fmt" + + "github.com/elastic/beats/v7/metricbeat/mb" +) + +// collect function collects all the system metric information into an instance of collectededData +func (m *MetricSet) collect(ctx context.Context, collector sysmetricCollectMethod) (out *collectedData, err error) { + out = &collectedData{} + if out.sysmetricMetrics, err = collector.sysmetricMetric(ctx); err != nil { + return nil, fmt.Errorf("error getting system metrics %w", err) + } + return out, nil +} + +// collectAndTransform is called by the Fetch method, which is the one +// that "loads" the data into Elasticsearch. +func (m *MetricSet) collectAndTransform(ctx context.Context) ([]mb.Event, error) { + collectedMetricsData, err := m.collect(ctx, m.collector) + if err != nil { + return nil, fmt.Errorf("error collecting data %w", err) + } + return m.transform(collectedMetricsData), nil +} + +// Transform function Transforms the data to create a Kibana/Elasticsearch friendly JSON. +// Data from Oracle is pretty fragmented by design so a lot of data was necessary. +// Data is organized by sysmetric entity that contains metrics details. +func (m *MetricSet) transform(in *collectedData) []mb.Event { + sysMetric := m.addSysmetricData(in.sysmetricMetrics) + events := make([]mb.Event, 0) + for _, v := range sysMetric { + events = append(events, mb.Event{MetricSetFields: v}) + } + return events +} diff --git a/x-pack/metricbeat/module/oracle/sysmetric/data_test.go b/x-pack/metricbeat/module/oracle/sysmetric/data_test.go new file mode 100644 index 00000000000..cefe371ff0e --- /dev/null +++ b/x-pack/metricbeat/module/oracle/sysmetric/data_test.go @@ -0,0 +1,36 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package sysmetric + +import ( + "database/sql" + "testing" +) + +func TestMetricSetTransform(t *testing.T) { + tests := []struct { + in *collectedData + want string + }{ + { + in: &collectedData{ + sysmetricMetrics: []sysmetricMetric{{ + name: sql.NullString{String: "Buffer Cache Hit Ratio", Valid: true}, + value: sql.NullFloat64{Float64: 100, Valid: true}, + }}, + }, + want: `{"buffer_cache_hit_ratio":100}`, + }, + } + for _, tt := range tests { + t.Run("Test func transform()", func(t *testing.T) { + m := &MetricSet{} + got := m.transform(tt.in) + if got[0].MetricSetFields.String() != tt.want { + t.Errorf("MetricSet.transform() = %v, want %v", got[0].MetricSetFields.String(), tt.want) + } + }) + } +} diff --git a/x-pack/metricbeat/module/oracle/sysmetric/metricset.go b/x-pack/metricbeat/module/oracle/sysmetric/metricset.go new file mode 100644 index 00000000000..e5362a3fa4a --- /dev/null +++ b/x-pack/metricbeat/module/oracle/sysmetric/metricset.go @@ -0,0 +1,77 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package sysmetric + +import ( + "context" + "fmt" + + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/oracle" +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("oracle", "sysmetric", New, + mb.WithHostParser(oracle.HostParser)) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + mb.BaseMetricSet + collector sysmetricCollectMethod + connectionDetails oracle.ConnectionDetails +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + config := oracle.ConnectionDetails{} + + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, fmt.Errorf("error parsing config file %w", err) + } + return &MetricSet{ + BaseMetricSet: base, + connectionDetails: config, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) (err error) { + db, err := oracle.NewConnection(m.HostData().URI) + if err != nil { + return fmt.Errorf("error creating connection to Oracle %w", err) + } + defer db.Close() + + m.collector = &sysmetricCollector{db: db, patterns: m.connectionDetails.Patterns} + + events, err := m.collectAndTransform(ctx) + if err != nil { + return err + } + + m.Load(ctx, events, reporter) + + return nil +} + +//Load takes the events and sends them to Elasticsearch. +func (m *MetricSet) Load(ctx context.Context, events []mb.Event, reporter mb.ReporterV2) { + for _, event := range events { + if reported := reporter.Event(event); !reported { + return + } + } +} diff --git a/x-pack/metricbeat/module/oracle/sysmetric/metricset_test.go b/x-pack/metricbeat/module/oracle/sysmetric/metricset_test.go new file mode 100644 index 00000000000..eaac6f9c1aa --- /dev/null +++ b/x-pack/metricbeat/module/oracle/sysmetric/metricset_test.go @@ -0,0 +1,61 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build integration && oracle +// +build integration,oracle + +package sysmetric + +import ( + "testing" + + _ "github.com/godror/godror" + + "github.com/elastic/beats/v7/libbeat/tests/compose" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/oracle" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestData(t *testing.T) { + r := compose.EnsureUp(t, "oracle") + + f := mbtest.NewReportingMetricSetV2WithContext(t, getConfig(r.Host())) + + findKey := func(key string) func(mapstr.M) bool { + return func(in mapstr.M) bool { + _, err := in.GetValue("oracle.sysmetric.metrics." + key) + return err == nil + } + } + + dataFiles := []struct { + keyToFind string + filePath string + }{ + { + keyToFind: "name", + filePath: "./_meta/data.json", + }, + } + + for _, dataFile := range dataFiles { + t.Run(dataFile.filePath, func(t *testing.T) { + if err := mbtest.WriteEventsReporterV2WithContextCond(f, t, dataFile.filePath, findKey(dataFile.keyToFind)); err != nil { + t.Fatal("write", err) + } + }) + } +} + +func getConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": "oracle", + "metricsets": []string{"sysmetric"}, + "hosts": []string{oracle.GetOracleConnectionDetails(host)}, + "patterns": []string{"Session%"}, + "username": "sys", + "password": "Oradoc_db1", + } +} diff --git a/x-pack/metricbeat/module/oracle/sysmetric/sysmetric_metric.go b/x-pack/metricbeat/module/oracle/sysmetric/sysmetric_metric.go new file mode 100644 index 00000000000..bc229d0f277 --- /dev/null +++ b/x-pack/metricbeat/module/oracle/sysmetric/sysmetric_metric.go @@ -0,0 +1,93 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package sysmetric + +import ( + "context" + "database/sql" + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/elastic/beats/v7/x-pack/metricbeat/module/oracle" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +type sysmetricMetric struct { + name sql.NullString + value sql.NullFloat64 +} + +/* + * The following function executes a query that produces the following result + * + * METRIC_NAME VALUE + * PX operations not downgraded Per Sec 0 + * + * Which is parsed into sysmetricMetric instances + */ +func (e *sysmetricCollector) calculateQuery() string { + if len(e.patterns) == 0 { + e.patterns = make([]interface{}, 1) + e.patterns[0] = "%" + } + + // System Metrics Long Duration (group_id = 2): 60 second interval + // Querying for Short Duration (15 seconds interval) will overload the system and may lead to performance issues. + query := "SELECT METRIC_NAME, VALUE FROM V$SYSMETRIC WHERE GROUP_ID = 2 AND METRIC_NAME LIKE :pattern0" + for i := 1; i < len(e.patterns); i++ { + query = query + " OR METRIC_NAME LIKE :pattern" + strconv.Itoa(i) + } + return query +} + +func (e *sysmetricCollector) sysmetricMetric(ctx context.Context) ([]sysmetricMetric, error) { + query := e.calculateQuery() + rows, err := e.db.QueryContext(ctx, query, e.patterns...) + if err != nil { + return nil, fmt.Errorf("error executing query %w", err) + } + + results := make([]sysmetricMetric, 0) + + for rows.Next() { + dest := sysmetricMetric{} + if err = rows.Scan(&dest.name, &dest.value); err != nil { + return nil, err + } + results = append(results, dest) + } + return results, nil +} + +func (m *MetricSet) addSysmetricData(bs []sysmetricMetric) []mapstr.M { + out := make([]mapstr.M, 0) + + ms := mapstr.M{} + + for _, sysmetricMetric := range bs { + metricName := ConvertToSnakeCase(sysmetricMetric.name).String + oracle.SetSqlValue(m.Logger(), ms, metricName, &oracle.Float64Value{NullFloat64: sysmetricMetric.value}) + } + out = append(out, ms) + + return out +} + +// ConvertToSnakeCase function converts a string to snake case to follow +// the Elastic naming conventions in the dynamically mapped fields +func ConvertToSnakeCase(name sql.NullString) sql.NullString { + reg, _ := regexp.Compile("[()/]") // Regex to remove '(', ')' and '/' characters from the string + // Convert to lowercase, replace spaces and hyphens with '_' and replace '%' with 'pct' + str := name.String + str = strings.ToLower(str) + str = strings.ReplaceAll(str, " ", "_") + str = reg.ReplaceAllString(str, "") + str = strings.ReplaceAll(str, "%", "pct") + str = strings.ReplaceAll(str, "-", "_") + name.String = str + return name +} diff --git a/x-pack/metricbeat/module/oracle/sysmetric/sysmetric_metric_test.go b/x-pack/metricbeat/module/oracle/sysmetric/sysmetric_metric_test.go new file mode 100644 index 00000000000..b51874a6bd8 --- /dev/null +++ b/x-pack/metricbeat/module/oracle/sysmetric/sysmetric_metric_test.go @@ -0,0 +1,47 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package sysmetric + +import ( + "testing" +) + +func TestSysmetricCollectorCalculateQuery(t *testing.T) { + type fields struct { + patterns []interface{} + } + strpatterns := []string{"foo%", "%bar", "%foobar%"} + patterns := make([]interface{}, len(strpatterns)) + for i, v := range strpatterns { + patterns[i] = v + } + tests := []struct { + fields fields + want string + }{ + { + // Checks if query is generated properly for given array of patterns. + fields{ + patterns: patterns, + }, + "SELECT METRIC_NAME, VALUE FROM V$SYSMETRIC WHERE GROUP_ID = 2 AND METRIC_NAME LIKE :pattern0 OR METRIC_NAME LIKE :pattern1 OR METRIC_NAME LIKE :pattern2", + }, + { + // Checks if query is generated properly if patterns are not given. + fields{}, + "SELECT METRIC_NAME, VALUE FROM V$SYSMETRIC WHERE GROUP_ID = 2 AND METRIC_NAME LIKE :pattern0", + }, + } + for _, tt := range tests { + t.Run("test func CalculateQuery()", func(t *testing.T) { + e := &sysmetricCollector{ + patterns: tt.fields.patterns, + } + if got := e.calculateQuery(); got != tt.want { + t.Errorf("sysmetricCollector.calculateQuery() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/x-pack/metricbeat/modules.d/oracle.yml.disabled b/x-pack/metricbeat/modules.d/oracle.yml.disabled index e2bfe73a737..d18107df03c 100644 --- a/x-pack/metricbeat/modules.d/oracle.yml.disabled +++ b/x-pack/metricbeat/modules.d/oracle.yml.disabled @@ -1,19 +1,25 @@ # Module: oracle # Docs: https://www.elastic.co/guide/en/beats/metricbeat/main/metricbeat-module-oracle.html -- module: oracle - period: 10m - metricsets: - - tablespace - enabled: true - hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"] -- module: oracle - period: 10s - metricsets: - - performance - enabled: true - hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"] - - # username: "" - # password: "" - +- module: oracle + period: 10m + metricsets: + - tablespace + enabled: true + hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"] +- module: oracle + period: 10s + metricsets: + - performance + enabled: true + hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"] +- module: oracle + period: 60s + metricsets: + - sysmetric + enabled: true + hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"] + # patterns: ["foo%","%bar","%foobar%"] + + # username: "" + # password: "" From 0c60aa89f821b18c0b0d04754fcb02e6179381f2 Mon Sep 17 00:00:00 2001 From: Giuseppe Santoro Date: Wed, 15 Jun 2022 16:53:28 +0100 Subject: [PATCH 15/22] Remote debug metricbeat running on docker/k8s from your laptop (#31748) * extra Dockerfile.debug with instructions on how to use remote debugging to debug metricbeat running on k8s --- metricbeat/Dockerfile.debug | 26 ++++ .../_meta/remote-debugger/README.md | 142 ++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 metricbeat/Dockerfile.debug create mode 100644 metricbeat/module/kubernetes/_meta/remote-debugger/README.md diff --git a/metricbeat/Dockerfile.debug b/metricbeat/Dockerfile.debug new file mode 100644 index 00000000000..b8876e0af79 --- /dev/null +++ b/metricbeat/Dockerfile.debug @@ -0,0 +1,26 @@ +FROM golang:alpine3.15 as builder + +ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin + +ENV CGO_ENABLED=0 + +RUN apk add --no-cache git +RUN go install github.com/go-delve/delve/cmd/dlv@v1.8.3 + +COPY metricbeat /usr/share/metricbeat/metricbeat + +FROM alpine:3.15 + +ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin + +WORKDIR /usr/share/metricbeat + +ENV ELASTICSEARCH_PASSWORD=changeme +ENV ELASTICSEARCH_USERNAME=elastic +ENV ELASTICSEARCH_HOST=elasticsearch + +COPY --from=builder /go/bin/dlv /go/bin/dlv +COPY --from=builder /usr/share/metricbeat/metricbeat /usr/share/metricbeat/metricbeat + +ENTRYPOINT ["dlv", "--headless=true", "--listen=:56268", "--api-version=2", "--log", "exec", "./metricbeat", "--"] +CMD [ "-e" ] diff --git a/metricbeat/module/kubernetes/_meta/remote-debugger/README.md b/metricbeat/module/kubernetes/_meta/remote-debugger/README.md new file mode 100644 index 00000000000..cdebbe225c2 --- /dev/null +++ b/metricbeat/module/kubernetes/_meta/remote-debugger/README.md @@ -0,0 +1,142 @@ +# README + +This readme explain how to remote debug metricbeat running on docker/kubernetes from your laptop with your local IDE. + +## Steps to run on docker: + +0. Move into metricbeat folder from the root folder of this project. Note: For some reason even if we are building in a subfolder, the `substitutePath.to` is still pointing to our root folder. + +```bash +cd metricbeat +``` + +1. cross build image for linux + +```bash +GOOS=linux GOARCH=amd64 go build -gcflags "-N -l" -o metricbeat main.go +``` + +2. buld docker container + +```bash +docker build -t metricbeat-debugger-image -f Dockerfile.debug . +``` + +3. run docker container + +```bash +docker run -p 56268:56268 --network elastic-package-stack_default -v $(pwd)/metric.docker.yml:/usr/share/metricbeat/metricbeat.yml metricbeat-debugger-image -c /usr/share/metricbeat/metricbeat.yml -e +``` + +You can customize the metricbeat configuration by mounting a different file instead of `$(pwd)/metric.docker.yml`. + +4. Attach to the remote debugger via your local IDE. Follow [Attach to remote debugger](./README.md#attach-to-remote-debugger-via-your-local-ide) + + +## Steps to run on kubernetes: + +Steps from 0 to 2 (included) are the same as `Steps to run on docker` + +0. Move into metricbeat folder from the root folder of this project. Note: For some reason even if we are building in a subfolder, the `substitutePath.to` is still pointing to our root folder. + +```bash +cd metricbeat +``` + +1. cross build image for linux + +```bash +GOOS=linux GOARCH=amd64 go build -gcflags "-N -l" -o metricbeat main.go +``` + +2. buld docker container + +```bash +docker build -t metricbeat-debugger-image -f Dockerfile.debug . +``` + +3. load image into Kind in order to run on kubernetes + +```bash +kind load docker-image metricbeat-debugger-image:latest +``` + +4. Edit `deploy/kubernetes/metricbeat-kubernetes.yaml` with these changes + +```yaml +containers: +- name: metricbeat + # image: docker.elastic.co/beats/metricbeat:8.2.0 + image: metricbeat-debugger-image:latest + imagePullPolicy: Never + args: [ + "-c", "/etc/metricbeat.yml", + "-e", + "-system.hostfs=/hostfs", + ] + ports: + - containerPort: 56268 + hostPort: 56268 + protocol: TCP +``` + +Namely you need: +- change the docker image used +- add `imagePullPolicy` to pull the image from inside Kind +- add a `ports` to expose the port in order to remote debug from laptop + +Compared to the docker example, here the metricbeat config is provided in the kubernetes manifest and mounted as a volume. + +5. Apply the changes in kubernetes to run the metricbeat + +```bash +kubectl apply -f metricbeat-kubernetes.yaml +``` + +6. Port forward from k8s to localhost + +```bash +kubectl port-forward -n kube-system 56268:56268 +``` + +where `` is the name of the pod running on k8s + +7. Attach to the remote debugger via your local IDE. Follow [Attach to remote debugger](./README.md#attach-to-remote-debugger-via-your-local-ide) + + +## Attach to remote debugger via your local IDE + +### Visual Studio Code +In order to attach to the remote debugger running on docker container or a k8s pod you need to provide a file `.vscode/launch.json` on your local machine with some configurations. + +You can use the following template, but remember to replace `` with the absolute path of the root folder of your project. + +```json +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Connect to server", + "type": "go", + "request": "attach", + "mode": "remote", + "debugAdapter": "dlv-dap", + "port": 56268, + "host": "127.0.0.1", + "showLog": true, + "trace": "trace", + "cwd": "${workspaceFolder}", + "substitutePath": [ + { + "from": "${workspaceFolder}", + "to": "" + } + ] + } + + ] +} +``` + +## Goland/IntelliJ +More info at [here](https://www.jetbrains.com/help/go/attach-to-running-go-processes-with-debugger.html#attach-to-a-process-on-a-remote-machine) From fd0f27dccf04dc4e985cc3cc2b72565dd41cb3d5 Mon Sep 17 00:00:00 2001 From: Evgeniy Belyi Date: Wed, 15 Jun 2022 19:43:10 +0300 Subject: [PATCH 16/22] Osquerybeat: Implement namespace info as a part of proc osquery_extension (#31918) * Osquerybeat: Implement namespace info as a part of osquery_extension * Refactor to use FS * Disable building on Windows * Code review changes --- .../osquery-extension/pkg/proc/namespace.go | 68 ++++++++++++++++++ .../pkg/proc/namespace_test.go | 71 +++++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 x-pack/osquerybeat/ext/osquery-extension/pkg/proc/namespace.go create mode 100644 x-pack/osquerybeat/ext/osquery-extension/pkg/proc/namespace_test.go diff --git a/x-pack/osquerybeat/ext/osquery-extension/pkg/proc/namespace.go b/x-pack/osquerybeat/ext/osquery-extension/pkg/proc/namespace.go new file mode 100644 index 00000000000..392286e9c24 --- /dev/null +++ b/x-pack/osquerybeat/ext/osquery-extension/pkg/proc/namespace.go @@ -0,0 +1,68 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build !windows +// +build !windows + +package proc + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "syscall" +) + +var ( + ErrInvalidProcNsPidStatContent = errors.New("invalid /proc/ns/pid stat content") + ErrInvalidProcNsPidStatParsedContent = errors.New("invalid /proc/ns/pid stat parsed content") +) + +type NamespaceInfo struct { + Ino uint64 +} + +// ReadNamespace reads process namespace information from /proc//ns/pid. +func ReadNamespace(root string, pid string) (nsInfo NamespaceInfo, err error) { + return ReadNamespaceFS(os.DirFS(root), pid) +} + +func ReadNamespaceFS(fsys fs.FS, pid string) (nsInfo NamespaceInfo, err error) { + // Get the namespace stat + nsStat, err := getNamespaceStat(fsys, pid) + if err != nil { + return + } + + // Set the namespace ino + nsInfo.Ino = nsStat.Ino + + return nsInfo, nil +} + +func getNamespaceStat(fsys fs.FS, pid string) (*syscall.Stat_t, error) { + // Path for the ns pid file + fn := filepath.Join("proc", pid, filepath.Join("ns", "pid")) + + // Calling stat on the ns pid file + stat, err := fs.Stat(fsys, fn) + if err != nil { + return nil, err + } + + // Pull stat data + dataSource := stat.Sys() + if dataSource == nil { + return nil, ErrInvalidProcNsPidStatContent + } + + // Convert pulled stat data into stat structure + dsStat, ok := dataSource.(*syscall.Stat_t) + if !ok { + return nil, ErrInvalidProcNsPidStatParsedContent + } + + return dsStat, nil +} diff --git a/x-pack/osquerybeat/ext/osquery-extension/pkg/proc/namespace_test.go b/x-pack/osquerybeat/ext/osquery-extension/pkg/proc/namespace_test.go new file mode 100644 index 00000000000..8f3c2a78777 --- /dev/null +++ b/x-pack/osquerybeat/ext/osquery-extension/pkg/proc/namespace_test.go @@ -0,0 +1,71 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build !windows +// +build !windows + +package proc + +import ( + "os" + "syscall" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" +) + +var nsIno = uint64(26041992) + +func TestNamespaceFS(t *testing.T) { + pid := "1" + testCases := []struct { + fstest fstest.MapFS + assert func(NamespaceInfo, error) + }{ + { + fstest.MapFS{ + "proc/1/ns/pid": { + Sys: dummyStat(t, nsIno), + }, + }, + func(result NamespaceInfo, err error) { + assert.Nil(t, err) + assert.Equal(t, result.Ino, nsIno) + }}, + { + fstest.MapFS{ + "proc/2/ns/pid": { + Sys: dummyStat(t, nsIno), + }, + }, + func(result NamespaceInfo, err error) { + assert.Error(t, err) + }}, + } + + for _, testCase := range testCases { + result, err := ReadNamespaceFS(testCase.fstest, pid) + testCase.assert(result, err) + } +} + +// Used in order to get a mocked syscall.Stat_t structure with assigned ino +func dummyStat(t *testing.T, ino uint64) *syscall.Stat_t { + name := t.TempDir() + + info, err := os.Stat(name) + if err != nil { + t.Fatal(err) + } + + mockDsStat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + t.Fatal(err) + } + + mockDsStat.Ino = ino + + return mockDsStat +} From 8ec3d99fe7d56011bd49159d2d9f0d9871675c22 Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Wed, 15 Jun 2022 10:27:54 -0700 Subject: [PATCH 17/22] Update shared-monitor-config.asciidoc monitoring.cluster_uuid missing (#31867) (#31934) Co-authored-by: DeDe Morton Co-authored-by: Jennie Soria --- libbeat/docs/monitoring/shared-monitor-config.asciidoc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libbeat/docs/monitoring/shared-monitor-config.asciidoc b/libbeat/docs/monitoring/shared-monitor-config.asciidoc index 609c3c3eb09..e9116008e5f 100644 --- a/libbeat/docs/monitoring/shared-monitor-config.asciidoc +++ b/libbeat/docs/monitoring/shared-monitor-config.asciidoc @@ -32,6 +32,10 @@ The default value is `false`. The {es} instances that you want to ship your {beatname_uc} metrics to. This configuration option contains the following fields: +==== `monitoring.cluster_uuid` + +The `monitoring.cluster_uuid` config identifies the {es} cluster under which the monitoring data will appear in the Stack Monitoring UI. + ===== `api_key` The detail of the API key to be used to send monitoring information to {es}. From c614236230bf342d1a565ac8a12d62b1b229d0e8 Mon Sep 17 00:00:00 2001 From: Craig MacKenzie Date: Wed, 15 Jun 2022 14:05:30 -0400 Subject: [PATCH 18/22] Skip metricbeat/module/mongodb/replstatus.TestFetch (#31943) --- .../module/mongodb/replstatus/replstatus_integration_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go b/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go index 3da78584941..018e81b9fdc 100644 --- a/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go +++ b/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go @@ -40,6 +40,8 @@ import ( ) func TestFetch(t *testing.T) { + t.Skip("Flaky Test: https://github.com/elastic/beats/issues/31768") + service := compose.EnsureUp(t, "mongodb") err := initiateReplicaSet(t, service.Host()) From 1f4b1460a196eac74d1929b13e830b6bd6f101e6 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Thu, 16 Jun 2022 03:08:19 -0400 Subject: [PATCH 19/22] [Automation] Update elastic stack version to 8.4.0-5e6770b1 for testing (#31953) Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index cfcc8908a96..277eab7eea1 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-40cff009-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-5e6770b1-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.4.0-40cff009-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.4.0-5e6770b1-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.4.0-40cff009-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.4.0-5e6770b1-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 125c57b92ce1ca36d954d9f0dbc8b5956a2b53c5 Mon Sep 17 00:00:00 2001 From: Denis Rechkunov Date: Thu, 16 Jun 2022 13:31:44 +0200 Subject: [PATCH 20/22] Add input identification support to the shipper output (#31942) So, we can send the real events to the shipper. --- libbeat/outputs/shipper/api/shipper.pb.go | 288 +++++++++--------- .../outputs/shipper/api/shipper_grpc.pb.go | 24 +- libbeat/outputs/shipper/shipper.go | 78 +++-- libbeat/outputs/shipper/shipper_test.go | 130 +++++++- 4 files changed, 336 insertions(+), 184 deletions(-) diff --git a/libbeat/outputs/shipper/api/shipper.pb.go b/libbeat/outputs/shipper/api/shipper.pb.go index 1e68942a422..8dc82e4f661 100644 --- a/libbeat/outputs/shipper/api/shipper.pb.go +++ b/libbeat/outputs/shipper/api/shipper.pb.go @@ -14,6 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. + package api import ( @@ -89,18 +90,18 @@ type Event struct { // Creation timestamp of the event. Timestamp *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` - // Identifies the input that generated the event. - Input *Input `protobuf:"bytes,2,opt,name=input,proto3" json:"input,omitempty"` + // Source of the generated event. + Source *Source `protobuf:"bytes,2,opt,name=source,proto3" json:"source,omitempty"` // Optional. Producers may specify a unique event ID if the event has a natural unique // identifier to simplify acknowledgement tracking. If no such identifier exists, the queue // offset can be used for acknowledgement tracking. - EventId string `protobuf:"bytes,4,opt,name=event_id,json=eventId,proto3" json:"event_id,omitempty"` + EventId string `protobuf:"bytes,3,opt,name=event_id,json=eventId,proto3" json:"event_id,omitempty"` // Data stream for the event. - DataStream *DataStream `protobuf:"bytes,5,opt,name=data_stream,json=dataStream,proto3" json:"data_stream,omitempty"` + DataStream *DataStream `protobuf:"bytes,4,opt,name=data_stream,json=dataStream,proto3" json:"data_stream,omitempty"` // Metadata JSON object (map[string]google.protobuf.Value) - Metadata *structpb.Struct `protobuf:"bytes,6,opt,name=metadata,proto3" json:"metadata,omitempty"` + Metadata *structpb.Struct `protobuf:"bytes,5,opt,name=metadata,proto3" json:"metadata,omitempty"` // Field JSON object (map[string]google.protobuf.Value) - Fields *structpb.Struct `protobuf:"bytes,7,opt,name=fields,proto3" json:"fields,omitempty"` + Fields *structpb.Struct `protobuf:"bytes,6,opt,name=fields,proto3" json:"fields,omitempty"` } func (x *Event) Reset() { @@ -142,9 +143,9 @@ func (x *Event) GetTimestamp() *timestamppb.Timestamp { return nil } -func (x *Event) GetInput() *Input { +func (x *Event) GetSource() *Source { if x != nil { - return x.Input + return x.Source } return nil } @@ -177,22 +178,21 @@ func (x *Event) GetFields() *structpb.Struct { return nil } -// Input identifiers from the agent policy, see example configuration. -type Input struct { +// Source information required for proper event tracking, processing and routing +type Source struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields // Input ID in the agent policy. - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - // Input name in the agent policy. - Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - // Input type in the agent policy (e.g. logfile, winlog, endpoint, etc.) - Type string `protobuf:"bytes,3,opt,name=type,proto3" json:"type,omitempty"` + InputId string `protobuf:"bytes,1,opt,name=input_id,json=inputId,proto3" json:"input_id,omitempty"` + // Stream ID in the agent policy (Optional, some inputs don't use streams). + // Not to be confused with data streams in Elasticsearch. + StreamId string `protobuf:"bytes,2,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"` } -func (x *Input) Reset() { - *x = Input{} +func (x *Source) Reset() { + *x = Source{} if protoimpl.UnsafeEnabled { mi := &file_shipper_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -200,13 +200,13 @@ func (x *Input) Reset() { } } -func (x *Input) String() string { +func (x *Source) String() string { return protoimpl.X.MessageStringOf(x) } -func (*Input) ProtoMessage() {} +func (*Source) ProtoMessage() {} -func (x *Input) ProtoReflect() protoreflect.Message { +func (x *Source) ProtoReflect() protoreflect.Message { mi := &file_shipper_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -218,43 +218,38 @@ func (x *Input) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use Input.ProtoReflect.Descriptor instead. -func (*Input) Descriptor() ([]byte, []int) { +// Deprecated: Use Source.ProtoReflect.Descriptor instead. +func (*Source) Descriptor() ([]byte, []int) { return file_shipper_proto_rawDescGZIP(), []int{2} } -func (x *Input) GetId() string { - if x != nil { - return x.Id - } - return "" -} - -func (x *Input) GetName() string { +func (x *Source) GetInputId() string { if x != nil { - return x.Name + return x.InputId } return "" } -func (x *Input) GetType() string { +func (x *Source) GetStreamId() string { if x != nil { - return x.Type + return x.StreamId } return "" } +// Elastic data stream // See https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme type DataStream struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Optional. Stable identifier for the data stream, see example configuration. - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - Type string `protobuf:"bytes,2,opt,name=type,proto3" json:"type,omitempty"` - Dataset string `protobuf:"bytes,3,opt,name=dataset,proto3" json:"dataset,omitempty"` - Namespace string `protobuf:"bytes,4,opt,name=namespace,proto3" json:"namespace,omitempty"` + // Generic type describing the data + Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` + // Describes the data ingested and its structure + Dataset string `protobuf:"bytes,2,opt,name=dataset,proto3" json:"dataset,omitempty"` + // User-configurable arbitrary grouping + Namespace string `protobuf:"bytes,3,opt,name=namespace,proto3" json:"namespace,omitempty"` } func (x *DataStream) Reset() { @@ -289,13 +284,6 @@ func (*DataStream) Descriptor() ([]byte, []int) { return file_shipper_proto_rawDescGZIP(), []int{3} } -func (x *DataStream) GetId() string { - if x != nil { - return x.Id - } - return "" -} - func (x *DataStream) GetType() string { if x != nil { return x.Type @@ -323,7 +311,10 @@ type PublishReply struct { unknownFields protoimpl.UnknownFields // Results describing messages successfully published to the queue. - // The output order matches the input order e.g. results[N] maps to events[N]. + // + // The output order matches the input order e.g. `results[N]` maps to `events[N]`, + // however, `results` may contain only the first K results for `events`, when + // not all the events are accepted by the queue, so the list sizes may not match. Results []*EventResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` } @@ -380,7 +371,7 @@ type EventResult struct { EventId string `protobuf:"bytes,3,opt,name=event_id,json=eventId,proto3" json:"event_id,omitempty"` // Optional. List of errors encountered by processors, if any. Processor // errors do not indicate the message failed to send. - ProcessorErrors []*status.Status `protobuf:"bytes,5,rep,name=processor_errors,json=processorErrors,proto3" json:"processor_errors,omitempty"` + ProcessorErrors []*status.Status `protobuf:"bytes,4,rep,name=processor_errors,json=processorErrors,proto3" json:"processor_errors,omitempty"` } func (x *EventResult) Reset() { @@ -448,8 +439,8 @@ type StreamAcksRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Optional. Requests acknowledgements originating only from this input ID. - InputId string `protobuf:"bytes,1,opt,name=input_id,json=inputId,proto3" json:"input_id,omitempty"` + // Optional. Requests acknowledgements originating only from this source. + Source *Source `protobuf:"bytes,1,opt,name=source,proto3" json:"source,omitempty"` // Optional. Requests acknowledgements for events for this data stream only. DataStream *DataStream `protobuf:"bytes,2,opt,name=data_stream,json=dataStream,proto3" json:"data_stream,omitempty"` } @@ -486,11 +477,11 @@ func (*StreamAcksRequest) Descriptor() ([]byte, []int) { return file_shipper_proto_rawDescGZIP(), []int{6} } -func (x *StreamAcksRequest) GetInputId() string { +func (x *StreamAcksRequest) GetSource() *Source { if x != nil { - return x.InputId + return x.Source } - return "" + return nil } func (x *StreamAcksRequest) GetDataStream() *DataStream { @@ -638,96 +629,98 @@ var file_shipper_proto_rawDesc = []byte{ 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x68, 0x69, 0x70, 0x70, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xc0, 0x02, 0x0a, + 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xc3, 0x02, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x12, 0x35, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1f, 0x2e, 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, - 0x73, 0x68, 0x69, 0x70, 0x70, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x70, 0x75, 0x74, - 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x65, 0x76, 0x65, 0x6e, 0x74, - 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x65, 0x76, 0x65, 0x6e, 0x74, - 0x49, 0x64, 0x12, 0x45, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, - 0x63, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x68, 0x69, 0x70, 0x70, 0x65, 0x72, 0x2e, - 0x76, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x0a, 0x64, - 0x61, 0x74, 0x61, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x33, 0x0a, 0x08, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, - 0x72, 0x75, 0x63, 0x74, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x2f, - 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, + 0x12, 0x38, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x20, 0x2e, 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, + 0x2e, 0x73, 0x68, 0x69, 0x70, 0x70, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x52, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x65, 0x76, + 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x65, 0x76, + 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x45, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x65, 0x6c, 0x61, + 0x73, 0x74, 0x69, 0x63, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x68, 0x69, 0x70, 0x70, + 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, + 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x33, 0x0a, 0x08, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x22, - 0x3f, 0x0a, 0x05, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, - 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, - 0x22, 0x68, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x0e, - 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, - 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, - 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x09, - 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0x4f, 0x0a, 0x0c, 0x50, 0x75, - 0x62, 0x6c, 0x69, 0x73, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x3f, 0x0a, 0x07, 0x72, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x65, 0x6c, - 0x61, 0x73, 0x74, 0x69, 0x63, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x68, 0x69, 0x70, - 0x70, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x22, 0xbc, 0x01, 0x0a, 0x0b, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x38, 0x0a, 0x09, 0x74, + 0x2e, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x12, 0x2f, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x17, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, + 0x64, 0x73, 0x22, 0x40, 0x0a, 0x06, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x19, 0x0a, 0x08, + 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x69, 0x6e, 0x70, 0x75, 0x74, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x49, 0x64, 0x22, 0x58, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x53, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, + 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, + 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0x4f, + 0x0a, 0x0c, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x3f, + 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x25, 0x2e, 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, + 0x73, 0x68, 0x69, 0x70, 0x70, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, + 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x22, + 0xbc, 0x01, 0x0a, 0x0b, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, + 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, + 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x19, 0x0a, 0x08, 0x71, 0x75, 0x65, + 0x75, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x71, 0x75, 0x65, + 0x75, 0x65, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, + 0x3d, 0x0a, 0x10, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x6f, 0x72, 0x5f, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0f, 0x70, + 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x6f, 0x72, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x22, 0x94, + 0x01, 0x0a, 0x11, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x41, 0x63, 0x6b, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x38, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x2e, 0x61, + 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x68, 0x69, 0x70, 0x70, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, + 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x45, + 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x2e, 0x61, 0x67, + 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x68, 0x69, 0x70, 0x70, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x44, + 0x61, 0x74, 0x61, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x53, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x22, 0x50, 0x0a, 0x0f, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x41, + 0x63, 0x6b, 0x73, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x3d, 0x0a, 0x04, 0x61, 0x63, 0x6b, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, + 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x68, 0x69, 0x70, 0x70, 0x65, 0x72, 0x2e, 0x76, + 0x31, 0x2e, 0x41, 0x63, 0x6b, 0x6e, 0x6f, 0x77, 0x6c, 0x65, 0x64, 0x67, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x52, 0x04, 0x61, 0x63, 0x6b, 0x73, 0x22, 0xab, 0x01, 0x0a, 0x0f, 0x41, 0x63, 0x6b, 0x6e, + 0x6f, 0x77, 0x6c, 0x65, 0x64, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x19, 0x0a, 0x08, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x71, 0x75, 0x65, 0x75, 0x65, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x3d, 0x0a, 0x10, 0x70, - 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x6f, 0x72, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, - 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x72, - 0x70, 0x63, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0f, 0x70, 0x72, 0x6f, 0x63, 0x65, - 0x73, 0x73, 0x6f, 0x72, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x22, 0x75, 0x0a, 0x11, 0x53, 0x74, - 0x72, 0x65, 0x61, 0x6d, 0x41, 0x63, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x19, 0x0a, 0x08, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x49, 0x64, 0x12, 0x45, 0x0a, 0x0b, 0x64, 0x61, - 0x74, 0x61, 0x5f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x24, 0x2e, 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, - 0x73, 0x68, 0x69, 0x70, 0x70, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, - 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x53, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x22, 0x50, 0x0a, 0x0f, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x41, 0x63, 0x6b, 0x73, 0x52, - 0x65, 0x70, 0x6c, 0x79, 0x12, 0x3d, 0x0a, 0x04, 0x61, 0x63, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x2e, 0x61, 0x67, 0x65, - 0x6e, 0x74, 0x2e, 0x73, 0x68, 0x69, 0x70, 0x70, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x63, - 0x6b, 0x6e, 0x6f, 0x77, 0x6c, 0x65, 0x64, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x04, 0x61, - 0x63, 0x6b, 0x73, 0x22, 0xab, 0x01, 0x0a, 0x0f, 0x41, 0x63, 0x6b, 0x6e, 0x6f, 0x77, 0x6c, 0x65, - 0x64, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x12, 0x19, 0x0a, 0x08, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x71, 0x75, 0x65, 0x75, 0x65, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, - 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, - 0x65, 0x76, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x28, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, - 0x72, 0x32, 0xe1, 0x01, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x65, 0x72, 0x12, 0x61, - 0x0a, 0x0d, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, - 0x28, 0x2e, 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, - 0x73, 0x68, 0x69, 0x70, 0x70, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, - 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x65, 0x6c, 0x61, 0x73, - 0x74, 0x69, 0x63, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x68, 0x69, 0x70, 0x70, 0x65, - 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x52, 0x65, 0x70, 0x6c, - 0x79, 0x12, 0x72, 0x0a, 0x16, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x41, 0x63, 0x6b, 0x6e, 0x6f, - 0x77, 0x6c, 0x65, 0x64, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x2b, 0x2e, 0x65, 0x6c, - 0x61, 0x73, 0x74, 0x69, 0x63, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x68, 0x69, 0x70, - 0x70, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x41, 0x63, 0x6b, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x65, 0x6c, 0x61, 0x73, 0x74, - 0x69, 0x63, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x68, 0x69, 0x70, 0x70, 0x65, 0x72, - 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x41, 0x63, 0x6b, 0x73, 0x52, 0x65, - 0x70, 0x6c, 0x79, 0x30, 0x01, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x2f, 0x65, 0x6c, 0x61, 0x73, - 0x74, 0x69, 0x63, 0x2d, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2d, 0x73, 0x68, 0x69, 0x70, 0x70, 0x65, - 0x72, 0x2f, 0x61, 0x70, 0x69, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x28, 0x09, 0x52, 0x07, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x28, 0x0a, 0x05, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x05, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x32, 0xe1, 0x01, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x64, 0x75, 0x63, + 0x65, 0x72, 0x12, 0x61, 0x0a, 0x0d, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x73, 0x12, 0x28, 0x2e, 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x2e, 0x61, 0x67, + 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x68, 0x69, 0x70, 0x70, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x50, + 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, + 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x68, + 0x69, 0x70, 0x70, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, + 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x72, 0x0a, 0x16, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x41, + 0x63, 0x6b, 0x6e, 0x6f, 0x77, 0x6c, 0x65, 0x64, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, + 0x2b, 0x2e, 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, + 0x73, 0x68, 0x69, 0x70, 0x70, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x41, 0x63, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x65, + 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x68, 0x69, + 0x70, 0x70, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x41, 0x63, + 0x6b, 0x73, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x30, 0x01, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, + 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x2f, + 0x65, 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x2d, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2d, 0x73, 0x68, + 0x69, 0x70, 0x70, 0x65, 0x72, 0x2f, 0x61, 0x70, 0x69, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( @@ -746,7 +739,7 @@ var file_shipper_proto_msgTypes = make([]protoimpl.MessageInfo, 9) var file_shipper_proto_goTypes = []interface{}{ (*PublishRequest)(nil), // 0: elastic.agent.shipper.v1.PublishRequest (*Event)(nil), // 1: elastic.agent.shipper.v1.Event - (*Input)(nil), // 2: elastic.agent.shipper.v1.Input + (*Source)(nil), // 2: elastic.agent.shipper.v1.Source (*DataStream)(nil), // 3: elastic.agent.shipper.v1.DataStream (*PublishReply)(nil), // 4: elastic.agent.shipper.v1.PublishReply (*EventResult)(nil), // 5: elastic.agent.shipper.v1.EventResult @@ -760,26 +753,27 @@ var file_shipper_proto_goTypes = []interface{}{ var file_shipper_proto_depIdxs = []int32{ 1, // 0: elastic.agent.shipper.v1.PublishRequest.events:type_name -> elastic.agent.shipper.v1.Event 9, // 1: elastic.agent.shipper.v1.Event.timestamp:type_name -> google.protobuf.Timestamp - 2, // 2: elastic.agent.shipper.v1.Event.input:type_name -> elastic.agent.shipper.v1.Input + 2, // 2: elastic.agent.shipper.v1.Event.source:type_name -> elastic.agent.shipper.v1.Source 3, // 3: elastic.agent.shipper.v1.Event.data_stream:type_name -> elastic.agent.shipper.v1.DataStream 10, // 4: elastic.agent.shipper.v1.Event.metadata:type_name -> google.protobuf.Struct 10, // 5: elastic.agent.shipper.v1.Event.fields:type_name -> google.protobuf.Struct 5, // 6: elastic.agent.shipper.v1.PublishReply.results:type_name -> elastic.agent.shipper.v1.EventResult 9, // 7: elastic.agent.shipper.v1.EventResult.timestamp:type_name -> google.protobuf.Timestamp 11, // 8: elastic.agent.shipper.v1.EventResult.processor_errors:type_name -> google.rpc.Status - 3, // 9: elastic.agent.shipper.v1.StreamAcksRequest.data_stream:type_name -> elastic.agent.shipper.v1.DataStream - 8, // 10: elastic.agent.shipper.v1.StreamAcksReply.acks:type_name -> elastic.agent.shipper.v1.Acknowledgement - 9, // 11: elastic.agent.shipper.v1.Acknowledgement.timestamp:type_name -> google.protobuf.Timestamp - 11, // 12: elastic.agent.shipper.v1.Acknowledgement.error:type_name -> google.rpc.Status - 0, // 13: elastic.agent.shipper.v1.Producer.PublishEvents:input_type -> elastic.agent.shipper.v1.PublishRequest - 6, // 14: elastic.agent.shipper.v1.Producer.StreamAcknowledgements:input_type -> elastic.agent.shipper.v1.StreamAcksRequest - 4, // 15: elastic.agent.shipper.v1.Producer.PublishEvents:output_type -> elastic.agent.shipper.v1.PublishReply - 7, // 16: elastic.agent.shipper.v1.Producer.StreamAcknowledgements:output_type -> elastic.agent.shipper.v1.StreamAcksReply - 15, // [15:17] is the sub-list for method output_type - 13, // [13:15] is the sub-list for method input_type - 13, // [13:13] is the sub-list for extension type_name - 13, // [13:13] is the sub-list for extension extendee - 0, // [0:13] is the sub-list for field type_name + 2, // 9: elastic.agent.shipper.v1.StreamAcksRequest.source:type_name -> elastic.agent.shipper.v1.Source + 3, // 10: elastic.agent.shipper.v1.StreamAcksRequest.data_stream:type_name -> elastic.agent.shipper.v1.DataStream + 8, // 11: elastic.agent.shipper.v1.StreamAcksReply.acks:type_name -> elastic.agent.shipper.v1.Acknowledgement + 9, // 12: elastic.agent.shipper.v1.Acknowledgement.timestamp:type_name -> google.protobuf.Timestamp + 11, // 13: elastic.agent.shipper.v1.Acknowledgement.error:type_name -> google.rpc.Status + 0, // 14: elastic.agent.shipper.v1.Producer.PublishEvents:input_type -> elastic.agent.shipper.v1.PublishRequest + 6, // 15: elastic.agent.shipper.v1.Producer.StreamAcknowledgements:input_type -> elastic.agent.shipper.v1.StreamAcksRequest + 4, // 16: elastic.agent.shipper.v1.Producer.PublishEvents:output_type -> elastic.agent.shipper.v1.PublishReply + 7, // 17: elastic.agent.shipper.v1.Producer.StreamAcknowledgements:output_type -> elastic.agent.shipper.v1.StreamAcksReply + 16, // [16:18] is the sub-list for method output_type + 14, // [14:16] is the sub-list for method input_type + 14, // [14:14] is the sub-list for extension type_name + 14, // [14:14] is the sub-list for extension extendee + 0, // [0:14] is the sub-list for field type_name } func init() { file_shipper_proto_init() } @@ -813,7 +807,7 @@ func file_shipper_proto_init() { } } file_shipper_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Input); i { + switch v := v.(*Source); i { case 0: return &v.state case 1: diff --git a/libbeat/outputs/shipper/api/shipper_grpc.pb.go b/libbeat/outputs/shipper/api/shipper_grpc.pb.go index d1b55194150..d6919871c02 100644 --- a/libbeat/outputs/shipper/api/shipper_grpc.pb.go +++ b/libbeat/outputs/shipper/api/shipper_grpc.pb.go @@ -34,10 +34,16 @@ const _ = grpc.SupportPackageIsVersion7 // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type ProducerClient interface { - // Publishes an event via the Elastic agent shipper. + // Publishes a list of events via the Elastic agent shipper. + // Blocks until all processing steps complete and data is written to the queue. + // The order of `PublishRequest.events` always matches `PublishReply.results`. // - // Blocks until all processing steps complete and data is written to the queue. Returns a - // RESOURCE_EXHAUSTED gRPC status code if the queue is full. + // Returns the `codes.ResourceExhausted` gRPC status code if the queue is full and none of the events + // can be accepted at the moment. + // + // If the queue could accept some events from the request, this returns a successful response + // containing results for the first K events that were accepted by the queue. + // The client is expected to retry sending the rest of the events in a separate request later. // // Inputs may execute multiple concurrent Produce requests for independent data streams. // The order in which concurrent requests complete is not guaranteed. Use sequential requests to @@ -100,10 +106,16 @@ func (x *producerStreamAcknowledgementsClient) Recv() (*StreamAcksReply, error) // All implementations must embed UnimplementedProducerServer // for forward compatibility type ProducerServer interface { - // Publishes an event via the Elastic agent shipper. + // Publishes a list of events via the Elastic agent shipper. + // Blocks until all processing steps complete and data is written to the queue. + // The order of `PublishRequest.events` always matches `PublishReply.results`. + // + // Returns the `codes.ResourceExhausted` gRPC status code if the queue is full and none of the events + // can be accepted at the moment. // - // Blocks until all processing steps complete and data is written to the queue. Returns a - // RESOURCE_EXHAUSTED gRPC status code if the queue is full. + // If the queue could accept some events from the request, this returns a successful response + // containing results for the first K events that were accepted by the queue. + // The client is expected to retry sending the rest of the events in a separate request later. // // Inputs may execute multiple concurrent Produce requests for independent data streams. // The order in which concurrent requests complete is not guaranteed. Use sequential requests to diff --git a/libbeat/outputs/shipper/shipper.go b/libbeat/outputs/shipper/shipper.go index 1727f54039b..7b27158f6bd 100644 --- a/libbeat/outputs/shipper/shipper.go +++ b/libbeat/outputs/shipper/shipper.go @@ -122,38 +122,14 @@ func (c *shipper) Publish(ctx context.Context, batch publisher.Batch) error { for i, e := range events { - meta, err := convertMapStr(e.Content.Meta) + converted, err := toShipperEvent(e) if err != nil { // conversion errors are not recoverable, so we have to drop the event completely - c.log.Errorf("%d/%d failed to convert event metadata to protobuf, dropped: %w", i+1, len(events), err) + c.log.Errorf("%d/%d: %q, dropped", i+1, len(events), err) continue } - fields, err := convertMapStr(e.Content.Fields) - if err != nil { - c.log.Errorf("%d/%d failed to convert event fields to protobuf, dropped: %w", i+1, len(events), err) - continue - } - - convertedEvents = append(convertedEvents, &sc.Event{ - Timestamp: timestamppb.New(e.Content.Timestamp), - Metadata: meta.GetStructValue(), - Fields: fields.GetStructValue(), - // TODO this contains temporary values, since they are required and not available from the event at the moment - Input: &sc.Input{ - Id: "beats", - Name: "beats", - Type: "beats", - }, - // TODO this contains temporary values, since they are required and not propagated at the moment - DataStream: &sc.DataStream{ - // Id: "none", // not generated at the moment - Type: "shipper.output", - Dataset: "generic", - Namespace: "default", - }, - }) - + convertedEvents = append(convertedEvents, converted) nonDroppedEvents = append(nonDroppedEvents, e) } @@ -171,7 +147,7 @@ func (c *shipper) Publish(ctx context.Context, batch publisher.Batch) error { if status.Code(err) != codes.OK || resp == nil { batch.Cancelled() // does not decrease the TTL st.Cancelled(len(events)) // we cancel the whole batch not just non-dropped events - c.log.Errorf("failed to publish the batch to the shipper, none of the %d events were accepted: %w", len(convertedEvents), err) + c.log.Errorf("failed to publish the batch to the shipper, none of the %d events were accepted: %s", len(convertedEvents), err) return nil } @@ -239,3 +215,49 @@ func convertMapStr(m mapstr.M) (*structpb.Value, error) { return structpb.NewStructValue(s), nil } + +func toShipperEvent(e publisher.Event) (*sc.Event, error) { + meta, err := convertMapStr(e.Content.Meta) + if err != nil { + return nil, fmt.Errorf("failed to convert event metadata to protobuf: %w", err) + } + + fields, err := convertMapStr(e.Content.Fields) + if err != nil { + return nil, fmt.Errorf("failed to convert event fields to protobuf: %w", err) + } + + source := &sc.Source{} + ds := &sc.DataStream{} + + inputIDVal, err := e.Content.Meta.GetValue("input_id") + if err == nil { + source.InputId, _ = inputIDVal.(string) + } + + streamIDVal, err := e.Content.Meta.GetValue("stream_id") + if err == nil { + source.StreamId, _ = streamIDVal.(string) + } + + dsType, err := e.Content.Fields.GetValue("data_stream.type") + if err == nil { + ds.Type, _ = dsType.(string) + } + dsNamespace, err := e.Content.Fields.GetValue("data_stream.namespace") + if err == nil { + ds.Namespace, _ = dsNamespace.(string) + } + dsDataset, err := e.Content.Fields.GetValue("data_stream.dataset") + if err == nil { + ds.Dataset, _ = dsDataset.(string) + } + + return &sc.Event{ + Timestamp: timestamppb.New(e.Content.Timestamp), + Metadata: meta.GetStructValue(), + Fields: fields.GetStructValue(), + Source: source, + DataStream: ds, + }, nil +} diff --git a/libbeat/outputs/shipper/shipper_test.go b/libbeat/outputs/shipper/shipper_test.go index ac6467813d6..ef19bec7c88 100644 --- a/libbeat/outputs/shipper/shipper_test.go +++ b/libbeat/outputs/shipper/shipper_test.go @@ -30,6 +30,7 @@ import ( "google.golang.org/grpc" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/outputs" @@ -40,6 +41,125 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" ) +func TestToShipperEvent(t *testing.T) { + ts := time.Now().Truncate(time.Second) + + cases := []struct { + name string + value publisher.Event + exp *sc.Event + expErr string + }{ + { + name: "successfully converts an event without source and data stream", + value: publisher.Event{ + Content: beat.Event{ + Timestamp: ts, + Meta: mapstr.M{ + "metafield": 42, + }, + Fields: mapstr.M{ + "field": "117", + }, + }, + }, + exp: &sc.Event{ + Timestamp: timestamppb.New(ts), + Source: &sc.Source{}, + DataStream: &sc.DataStream{}, + Metadata: protoStruct(t, map[string]interface{}{ + "metafield": 42, + }), + Fields: protoStruct(t, map[string]interface{}{ + "field": "117", + }), + }, + }, + { + name: "successfully converts an event with source and data stream", + value: publisher.Event{ + Content: beat.Event{ + Timestamp: ts, + Meta: mapstr.M{ + "metafield": 42, + "input_id": "input", + "stream_id": "stream", + }, + Fields: mapstr.M{ + "field": "117", + "data_stream": mapstr.M{ + "type": "ds-type", + "namespace": "ds-namespace", + "dataset": "ds-dataset", + }, + }, + }, + }, + exp: &sc.Event{ + Timestamp: timestamppb.New(ts), + Source: &sc.Source{ + InputId: "input", + StreamId: "stream", + }, + DataStream: &sc.DataStream{ + Type: "ds-type", + Namespace: "ds-namespace", + Dataset: "ds-dataset", + }, + Metadata: protoStruct(t, map[string]interface{}{ + "metafield": 42, + "input_id": "input", + "stream_id": "stream", + }), + Fields: protoStruct(t, map[string]interface{}{ + "field": "117", + "data_stream": map[string]interface{}{ + "type": "ds-type", + "namespace": "ds-namespace", + "dataset": "ds-dataset", + }, + }), + }, + }, + { + name: "returns error if failed to convert metadata", + value: publisher.Event{ + Content: beat.Event{ + Timestamp: ts, + Meta: mapstr.M{ + "metafield": ts, // timestamp is a wrong type + }, + }, + }, + expErr: "failed to convert event metadata", + }, + { + name: "returns error if failed to convert fields", + value: publisher.Event{ + Content: beat.Event{ + Timestamp: ts, + Fields: mapstr.M{ + "field": ts, // timestamp is a wrong type + }, + }, + }, + expErr: "failed to convert event fields", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + converted, err := toShipperEvent(tc.value) + if tc.expErr != "" { + require.Error(t, err) + require.Contains(t, err.Error(), tc.expErr) + require.Nil(t, converted) + return + } + requireEqualProto(t, tc.exp, converted) + }) + } +} + func TestConvertMapStr(t *testing.T) { cases := []struct { name string @@ -54,7 +174,7 @@ func TestConvertMapStr(t *testing.T) { { name: "empty map returns empty struct", value: mapstr.M{}, - exp: protoStruct(t, nil), + exp: protoStructValue(t, nil), }, { name: "returns error when type is not supported", @@ -76,7 +196,7 @@ func TestConvertMapStr(t *testing.T) { }, }, }, - exp: protoStruct(t, map[string]interface{}{ + exp: protoStructValue(t, map[string]interface{}{ "key1": "string", "key2": 42, "key3": 42.2, @@ -275,9 +395,13 @@ func TestPublish(t *testing.T) { }) } -func protoStruct(t *testing.T, values map[string]interface{}) *structpb.Value { +func protoStruct(t *testing.T, values map[string]interface{}) *structpb.Struct { s, err := structpb.NewStruct(values) require.NoError(t, err) + return s +} +func protoStructValue(t *testing.T, values map[string]interface{}) *structpb.Value { + s := protoStruct(t, values) return structpb.NewStructValue(s) } From 330f031b36940beaee882dc1e00b6a6f006f4eb5 Mon Sep 17 00:00:00 2001 From: Lee E Hinman <57081003+leehinman@users.noreply.github.com> Date: Thu, 16 Jun 2022 12:55:34 -0500 Subject: [PATCH 21/22] improve disk queue performance by coalescing writes (#31935) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tested with libbeat/publisher/queue/diskqueue/benchmark_test.go before change go test -bench=. -benchtime 1x -count 10 -timeout 120m -benchmem > baseline.txt after change go test -bench=. -benchtime 1x -count 10 -timeout 120m -benchmem > gather_w.txt benchstat baseline.txt gather_w.txt name old time/op new time/op delta 1M_10-16 38.7s ± 2% 21.8s ± 0% -43.58% (p=0.000 n=10+7) 1M_100-16 38.7s ± 1% 21.8s ± 1% -43.74% (p=0.000 n=10+10) 1M_1k-16 38.6s ± 1% 21.9s ± 1% -43.36% (p=0.000 n=10+10) 1M_10k-16 38.7s ± 2% 21.7s ± 0% -43.78% (p=0.000 n=10+10) name old alloc/op new alloc/op delta 1M_10-16 3.47GB ± 0% 3.43GB ± 0% -1.16% (p=0.000 n=8+10) 1M_100-16 3.47GB ± 0% 3.43GB ± 0% -1.17% (p=0.000 n=10+8) 1M_1k-16 3.47GB ± 0% 3.43GB ± 0% -1.18% (p=0.000 n=10+8) 1M_10k-16 3.47GB ± 0% 3.43GB ± 0% -1.18% (p=0.000 n=10+10) name old allocs/op new allocs/op delta 1M_10-16 43.4M ± 0% 40.6M ± 0% -6.32% (p=0.000 n=8+10) 1M_100-16 43.4M ± 0% 40.6M ± 0% -6.42% (p=0.000 n=10+9) 1M_1k-16 43.4M ± 0% 40.6M ± 0% -6.49% (p=0.000 n=10+10) 1M_10k-16 43.4M ± 0% 40.6M ± 0% -6.49% (p=0.000 n=10+10) Improves speed and lowers allocations --- CHANGELOG.next.asciidoc | 2 +- .../publisher/queue/diskqueue/writer_loop.go | 48 +++++++++++-------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1cb6ccf4ef2..38e968eaad1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -80,7 +80,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] *Affecting all Beats* - Add new config option `timestamp.precision` to configure timestamps. {pull}31682[31682] - +- Improve performance of disk queue by coalescing writes. {pull}31935[31935] *Auditbeat* diff --git a/libbeat/publisher/queue/diskqueue/writer_loop.go b/libbeat/publisher/queue/diskqueue/writer_loop.go index b47f3a2baef..c8290cdfdab 100644 --- a/libbeat/publisher/queue/diskqueue/writer_loop.go +++ b/libbeat/publisher/queue/diskqueue/writer_loop.go @@ -18,6 +18,7 @@ package diskqueue import ( + "bytes" "encoding/binary" "os" "time" @@ -91,9 +92,13 @@ type writerLoop struct { outputFile *os.File currentRetryInterval time.Duration + + // buffer Used to gather write information so there is only one write syscall + buffer *bytes.Buffer } func newWriterLoop(logger *logp.Logger, settings Settings) *writerLoop { + buffer := &bytes.Buffer{} return &writerLoop{ logger: logger, settings: settings, @@ -102,6 +107,7 @@ func newWriterLoop(logger *logp.Logger, settings Settings) *writerLoop { responseChan: make(chan writerLoopResponse), currentRetryInterval: settings.RetryInterval, + buffer: buffer, } } @@ -112,8 +118,8 @@ func (wl *writerLoop) run() { // The request channel is closed, we are done. If there is an active // segment file, finalize its frame count and close it. if wl.outputFile != nil { - writeSegmentHeader(wl.outputFile, wl.currentSegment.frameCount) - wl.outputFile.Sync() + _ = writeSegmentHeader(wl.outputFile, wl.currentSegment.frameCount) + _ = wl.outputFile.Sync() wl.outputFile.Close() wl.outputFile = nil } @@ -156,9 +162,9 @@ outerLoop: if wl.outputFile != nil { // Update the header with the frame count (including the ones we // just wrote), try to sync to disk, then close the file. - writeSegmentHeader(wl.outputFile, + _ = writeSegmentHeader(wl.outputFile, wl.currentSegment.frameCount+curSegmentResponse.framesWritten) - wl.outputFile.Sync() + _ = wl.outputFile.Sync() wl.outputFile.Close() wl.outputFile = nil // We are done with this segment, add the totals to the response and @@ -185,25 +191,25 @@ outerLoop: // to writing this block unless the queue is closed in the meantime. frameSize := uint32(frameRequest.frame.sizeOnDisk()) - // The Write calls below all pass through retryWriter, so they can - // only return an error if the write should be aborted. Thus, all we - // need to do when we see an error is break out of the request loop. - err := binary.Write(retryWriter, binary.LittleEndian, frameSize) - if err != nil { - break - } - _, err = retryWriter.Write(frameRequest.frame.serialized) - if err != nil { - break - } + // The Write calls to wl.buffer are for performance + // reasons, so all the data can be written with one + // Write call to the retryWriter. err is always nil + // for writes to a bytes.Buffer + _ = binary.Write(wl.buffer, binary.LittleEndian, frameSize) + _, _ = wl.buffer.Write(frameRequest.frame.serialized) + // Compute / write the frame's checksum checksum := computeChecksum(frameRequest.frame.serialized) - err = binary.Write(wl.outputFile, binary.LittleEndian, checksum) - if err != nil { - break - } + _ = binary.Write(wl.buffer, binary.LittleEndian, checksum) + // Write the frame footer's (duplicate) length - err = binary.Write(wl.outputFile, binary.LittleEndian, frameSize) + _ = binary.Write(wl.buffer, binary.LittleEndian, frameSize) + + // Write the entire frame to the retryWriter + _, err := retryWriter.Write(wl.buffer.Bytes()) + // Reset the buffer even if there was an error, so + // buffer is setup for writing to next frame + wl.buffer.Reset() if err != nil { break } @@ -228,7 +234,7 @@ outerLoop: } } // Try to sync the written data to disk. - wl.outputFile.Sync() + _ = wl.outputFile.Sync() // If the queue has an ACK listener, notify it the frames were written. if wl.settings.WriteToDiskListener != nil { From bcd4f9f81ab189efc103180736f73e3db8e7acc4 Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Fri, 17 Jun 2022 07:03:58 +0930 Subject: [PATCH 22/22] x-pack/filebeat/module/rabbitmq: add handling of authentication data (#31680) --- CHANGELOG.next.asciidoc | 1 + .../module/rabbitmq/log/ingest/pipeline.yml | 48 +++++++++++++++++++ .../module/rabbitmq/log/test/test.log | 1 + .../rabbitmq/log/test/test.log-expected.json | 45 ++++++++++++++++- 4 files changed, 93 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 38e968eaad1..4962a254681 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -105,6 +105,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Add template helper function for hashing strings. {issue}31613[31613] {pull}31630[31630] - Add extended okta.debug_context.debug_data handling. {pull}31676[31676] - Add `auth.oauth2.google.jwt_json` option to `httpjson` input. {pull}31750[31750] +- Add authentication fields to RabbitMQ module documents. {issue}31159[31159] {pull}31680[31680] *Auditbeat* diff --git a/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml b/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml index e69402c6a95..9a3d30b7a41 100644 --- a/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml @@ -13,6 +13,54 @@ processors: - "%{DATESTAMP:timestamp} \\[%{WORD:log.level}\\] %{ERL_PID:rabbitmq.log.pid} %{GREEDYMULTILINE:message}" ignore_missing: true +- grok: + field: message + patterns: + - "user '%{WORD:user.name}' %{ALLOWED:event.action}" + - "%{DENIED:event.action}: user '%{WORD:user.name}'" + - "%{CLOSING:event.action}%{GREEDYDATA}user: '%{WORD:user.name}'" + pattern_definitions: + ALLOWED: "authenticated and granted access" + DENIED: "access denied" + CLOSING: "closing" + ignore_missing: true + ignore_failure: true +- set: + field: event.action + value: logged-in + if: 'ctx.event?.action == "authenticated and granted access"' +- set: + field: event.action + value: close-connection + if: 'ctx.event?.action == "closing"' +- set: + field: event.outcome + value: success + if: "ctx?.event?.action != null && ['logged-in', 'close-connection'].contains(ctx.event.action)" +- set: + field: event.action + value: logon-failed + if: 'ctx.event?.action == "access denied"' +- set: + field: event.outcome + value: failure + if: 'ctx.event?.action == "logon-failed"' +- append: + field: event.category + value: authentication + if: "ctx?.event?.action != null && ['logged-in', 'logon-failed'].contains(ctx.event.action)" +- append: + field: event.type + value: + - start + - access + if: "ctx?.event?.action != null && ['logged-in', 'logon-failed'].contains(ctx.event.action)" +- append: + field: event.type + value: + - end + - access + if: ctx?.event?.action == 'close-connection' - date: if: "ctx.event.timezone == null" field: timestamp diff --git a/x-pack/filebeat/module/rabbitmq/log/test/test.log b/x-pack/filebeat/module/rabbitmq/log/test/test.log index 91a0d3a63e4..ccfa2277eab 100644 --- a/x-pack/filebeat/module/rabbitmq/log/test/test.log +++ b/x-pack/filebeat/module/rabbitmq/log/test/test.log @@ -76,3 +76,4 @@ 163] 2019-04-12 10:19:14.450 [info] <0.1902.0> Closing connection <0.1345.0> because <<240,159,145,139,240,159,143,190,240,159,144,135,240,159,164,163>> 2019-04-12 10:19:14.451 [info] <0.1345.0> closing AMQP connection <0.1345.0> (127.0.0.1:64875 -> 127.0.0.1:5672, vhost: '/', user: 'guest') +2021-11-22 17:48:20.003 [warning] <0.8084.263> HTTP access denied: user 'guest' - Not monitor user diff --git a/x-pack/filebeat/module/rabbitmq/log/test/test.log-expected.json b/x-pack/filebeat/module/rabbitmq/log/test/test.log-expected.json index 0bdae14b894..d15252dfbbc 100644 --- a/x-pack/filebeat/module/rabbitmq/log/test/test.log-expected.json +++ b/x-pack/filebeat/module/rabbitmq/log/test/test.log-expected.json @@ -310,17 +310,27 @@ }, { "@timestamp": "2019-04-12T10:11:15.101-02:00", + "event.action": "logged-in", + "event.category": [ + "authentication" + ], "event.dataset": "rabbitmq.log", "event.kind": "event", "event.module": "rabbitmq", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "access", + "start" + ], "fileset.name": "log", "input.type": "log", "log.level": "info", "log.offset": 3420, "message": "connection <0.1345.0> (127.0.0.1:64875 -> 127.0.0.1:5672): user 'guest' authenticated and granted access to vhost '/'", "rabbitmq.log.pid": "<0.1345.0>", - "service.type": "rabbitmq" + "service.type": "rabbitmq", + "user.name": "guest" }, { "@timestamp": "2019-04-12T10:19:14.450-02:00", @@ -355,16 +365,47 @@ }, { "@timestamp": "2019-04-12T10:19:14.451-02:00", + "event.action": "close-connection", "event.dataset": "rabbitmq.log", "event.kind": "event", "event.module": "rabbitmq", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "access", + "end" + ], "fileset.name": "log", "input.type": "log", "log.level": "info", "log.offset": 4359, "message": "closing AMQP connection <0.1345.0> (127.0.0.1:64875 -> 127.0.0.1:5672, vhost: '/', user: 'guest')", "rabbitmq.log.pid": "<0.1345.0>", - "service.type": "rabbitmq" + "service.type": "rabbitmq", + "user.name": "guest" + }, + { + "@timestamp": "2021-11-22T17:48:20.003-02:00", + "event.action": "logon-failed", + "event.category": [ + "authentication" + ], + "event.dataset": "rabbitmq.log", + "event.kind": "event", + "event.module": "rabbitmq", + "event.outcome": "failure", + "event.timezone": "-02:00", + "event.type": [ + "access", + "start" + ], + "fileset.name": "log", + "input.type": "log", + "log.level": "warning", + "log.offset": 4499, + "message": "HTTP access denied: user 'guest' - Not monitor user", + "rabbitmq.log.pid": "<0.8084.263>", + "service.type": "rabbitmq", + "user.name": "guest" } ] \ No newline at end of file