diff --git a/docs-requirements.txt b/docs-requirements.txt index 358316fa7e7..cc999f13832 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -5,7 +5,7 @@ sphinx-autodoc-typehints~=1.10.2 # Required by ext packages asgiref~=3.0 ddtrace>=0.34.0 -aiohttp ~= 3.0 +aiohttp~= 3.0 Deprecated>=1.2.6 django>=2.2 PyMySQL~=0.9.3 @@ -18,4 +18,5 @@ pymongo~=3.1 redis>=2.6 sqlalchemy>=1.0 thrift>=0.10.0 -wrapt >=1.0.0,<2.0.0 +wrapt>=1.0.0,<2.0.0 +psutil~=5.7.0 diff --git a/docs/ext/system_metrics/system_metrics.rst b/docs/ext/system_metrics/system_metrics.rst new file mode 100644 index 00000000000..db6f9da70f4 --- /dev/null +++ b/docs/ext/system_metrics/system_metrics.rst @@ -0,0 +1,7 @@ +OpenTelemetry System Metrics Instrumentation +============================================ + +.. automodule:: opentelemetry.ext.system_metrics + :members: + :undoc-members: + :show-inheritance: diff --git a/ext/opentelemetry-ext-system-metrics/CHANGELOG.md b/ext/opentelemetry-ext-system-metrics/CHANGELOG.md new file mode 100644 index 00000000000..2f3c3ab3821 --- /dev/null +++ b/ext/opentelemetry-ext-system-metrics/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog + +## Unreleased + +- Initial release (https://github.com/open-telemetry/opentelemetry-python/pull/652) diff --git a/ext/opentelemetry-ext-system-metrics/LICENSE b/ext/opentelemetry-ext-system-metrics/LICENSE new file mode 100644 index 00000000000..261eeb9e9f8 --- /dev/null +++ b/ext/opentelemetry-ext-system-metrics/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/ext/opentelemetry-ext-system-metrics/MANIFEST.in b/ext/opentelemetry-ext-system-metrics/MANIFEST.in new file mode 100644 index 00000000000..aed3e33273b --- /dev/null +++ b/ext/opentelemetry-ext-system-metrics/MANIFEST.in @@ -0,0 +1,9 @@ +graft src +graft tests +global-exclude *.pyc +global-exclude *.pyo +global-exclude __pycache__/* +include CHANGELOG.md +include MANIFEST.in +include README.rst +include LICENSE diff --git a/ext/opentelemetry-ext-system-metrics/README.rst b/ext/opentelemetry-ext-system-metrics/README.rst new file mode 100644 index 00000000000..796f3f13f26 --- /dev/null +++ b/ext/opentelemetry-ext-system-metrics/README.rst @@ -0,0 +1,24 @@ +OpenTelemetry System Metrics Instrumentation +============================================ + +|pypi| + +.. |pypi| image:: https://badge.fury.io/py/opentelemetry-ext-system-metrics.svg + :target: https://pypi.org/project/opentelemetry-ext-system-metrics/ + +Instrumentation to collect system performance metrics. + + +Installation +------------ + +:: + + pip install opentelemetry-ext-system-metrics + + +References +---------- +* `OpenTelemetry System Metrics Instrumentation `_ +* `OpenTelemetry Project `_ + diff --git a/ext/opentelemetry-ext-system-metrics/setup.cfg b/ext/opentelemetry-ext-system-metrics/setup.cfg new file mode 100644 index 00000000000..25515eaf0d3 --- /dev/null +++ b/ext/opentelemetry-ext-system-metrics/setup.cfg @@ -0,0 +1,51 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +[metadata] +name = opentelemetry-ext-system-metrics +description = OpenTelemetry System Metrics Instrumentation +long_description = file: README.rst +long_description_content_type = text/x-rst +author = OpenTelemetry Authors +author_email = cncf-opentelemetry-contributors@lists.cncf.io +url = https://github.com/open-telemetry/opentelemetry-python/tree/master/ext/opentelemetry-ext-system-metrics +platforms = any +license = Apache-2.0 +classifiers = + Development Status :: 4 - Beta + Intended Audience :: Developers + License :: OSI Approved :: Apache Software License + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + +[options] +python_requires = >=3.4 +package_dir= + =src +packages=find_namespace: +install_requires = + opentelemetry-api == 0.9.dev0 + psutil ~= 5.7.0 + +[options.extras_require] +test = + opentelemetry-test == 0.9.dev0 + +[options.packages.find] +where = src diff --git a/ext/opentelemetry-ext-system-metrics/setup.py b/ext/opentelemetry-ext-system-metrics/setup.py new file mode 100644 index 00000000000..370399094dd --- /dev/null +++ b/ext/opentelemetry-ext-system-metrics/setup.py @@ -0,0 +1,26 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import setuptools + +BASE_DIR = os.path.dirname(__file__) +VERSION_FILENAME = os.path.join( + BASE_DIR, "src", "opentelemetry", "ext", "system_metrics", "version.py" +) +PACKAGE_INFO = {} +with open(VERSION_FILENAME) as f: + exec(f.read(), PACKAGE_INFO) + +setuptools.setup(version=PACKAGE_INFO["__version__"]) diff --git a/ext/opentelemetry-ext-system-metrics/src/opentelemetry/ext/system_metrics/__init__.py b/ext/opentelemetry-ext-system-metrics/src/opentelemetry/ext/system_metrics/__init__.py new file mode 100644 index 00000000000..88f36f4ac48 --- /dev/null +++ b/ext/opentelemetry-ext-system-metrics/src/opentelemetry/ext/system_metrics/__init__.py @@ -0,0 +1,225 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Instrument to report system (CPU, memory, network) and +process (CPU, memory, garbage collection) metrics. By default, the +following metrics are configured: + +"system_memory": ["total", "available", "used", "free"], +"system_cpu": ["user", "system", "idle"], +"network_bytes": ["bytes_recv", "bytes_sent"], +"runtime_memory": ["rss", "vms"], +"runtime_cpu": ["user", "system"], + + +Usage +----- + +.. code:: python + + from opentelemetry.ext.system_metrics import SystemMetrics + from opentelemetry.sdk.metrics.export import ConsoleMetricsExporter + + exporter = ConsoleMetricsExporter() + SystemMetrics(exporter) + + # metrics are collected asynchronously + input("...") + + # to configure custom metrics + configuration = { + "system_memory": ["total", "available", "used", "free", "active", "inactive", "wired"], + "system_cpu": ["user", "nice", "system", "idle"], + "network_bytes": ["bytes_recv", "bytes_sent"], + "runtime_memory": ["rss", "vms"], + "runtime_cpu": ["user", "system"], + } + SystemMetrics(exporter, config=configuration) + +API +--- +""" + +import gc +import os +import typing + +import psutil + +from opentelemetry import metrics +from opentelemetry.sdk.metrics.export import MetricsExporter +from opentelemetry.sdk.metrics.export.controller import PushController + + +class SystemMetrics: + def __init__( + self, + exporter: MetricsExporter, + interval: int = 30, + labels: typing.Optional[typing.Dict[str, str]] = None, + config: typing.Optional[typing.Dict[str, typing.List[str]]] = None, + ): + self._labels = {} if labels is None else labels + self.meter = metrics.get_meter(__name__) + self.controller = PushController( + meter=self.meter, exporter=exporter, interval=interval + ) + if config is None: + self._config = { + "system_memory": ["total", "available", "used", "free"], + "system_cpu": ["user", "system", "idle"], + "network_bytes": ["bytes_recv", "bytes_sent"], + "runtime_memory": ["rss", "vms"], + "runtime_cpu": ["user", "system"], + } + else: + self._config = config + self._proc = psutil.Process(os.getpid()) + self._system_memory_labels = {} + self._system_cpu_labels = {} + self._network_bytes_labels = {} + self._runtime_memory_labels = {} + self._runtime_cpu_labels = {} + self._runtime_gc_labels = {} + # create the label set for each observer once + for key, value in self._labels.items(): + self._system_memory_labels[key] = value + self._system_cpu_labels[key] = value + self._network_bytes_labels[key] = value + self._runtime_memory_labels[key] = value + self._runtime_gc_labels[key] = value + + self.meter.register_observer( + callback=self._get_system_memory, + name="system.mem", + description="System memory", + unit="bytes", + value_type=int, + ) + + self.meter.register_observer( + callback=self._get_system_cpu, + name="system.cpu", + description="System CPU", + unit="seconds", + value_type=float, + ) + + self.meter.register_observer( + callback=self._get_network_bytes, + name="system.net.bytes", + description="System network bytes", + unit="bytes", + value_type=int, + ) + + self.meter.register_observer( + callback=self._get_runtime_memory, + name="runtime.python.mem", + description="Runtime memory", + unit="bytes", + value_type=int, + ) + + self.meter.register_observer( + callback=self._get_runtime_cpu, + name="runtime.python.cpu", + description="Runtime CPU", + unit="seconds", + value_type=float, + ) + + self.meter.register_observer( + callback=self._get_runtime_gc_count, + name="runtime.python.gc.count", + description="Runtime: gc objects", + unit="objects", + value_type=int, + ) + + def _get_system_memory(self, observer: metrics.Observer) -> None: + """Observer callback for memory available + + Args: + observer: the observer to update + """ + system_memory = psutil.virtual_memory() + for metric in self._config["system_memory"]: + self._system_memory_labels["type"] = metric + observer.observe( + getattr(system_memory, metric), self._system_memory_labels + ) + + def _get_system_cpu(self, observer: metrics.Observer) -> None: + """Observer callback for system cpu + + Args: + observer: the observer to update + """ + cpu_times = psutil.cpu_times() + for _type in self._config["system_cpu"]: + self._system_cpu_labels["type"] = _type + observer.observe( + getattr(cpu_times, _type), self._system_cpu_labels + ) + + def _get_network_bytes(self, observer: metrics.Observer) -> None: + """Observer callback for network bytes + + Args: + observer: the observer to update + """ + net_io = psutil.net_io_counters() + for _type in self._config["network_bytes"]: + self._network_bytes_labels["type"] = _type + observer.observe( + getattr(net_io, _type), self._network_bytes_labels + ) + + def _get_runtime_memory(self, observer: metrics.Observer) -> None: + """Observer callback for runtime memory + + Args: + observer: the observer to update + """ + proc_memory = self._proc.memory_info() + for _type in self._config["runtime_memory"]: + self._runtime_memory_labels["type"] = _type + observer.observe( + getattr(proc_memory, _type), self._runtime_memory_labels + ) + + def _get_runtime_cpu(self, observer: metrics.Observer) -> None: + """Observer callback for runtime CPU + + Args: + observer: the observer to update + """ + proc_cpu = self._proc.cpu_times() + for _type in self._config["runtime_cpu"]: + self._runtime_cpu_labels["type"] = _type + observer.observe( + getattr(proc_cpu, _type), self._runtime_cpu_labels + ) + + def _get_runtime_gc_count(self, observer: metrics.Observer) -> None: + """Observer callback for garbage collection + + Args: + observer: the observer to update + """ + gc_count = gc.get_count() + for index, count in enumerate(gc_count): + self._runtime_gc_labels["count"] = str(index) + observer.observe(count, self._runtime_gc_labels) diff --git a/ext/opentelemetry-ext-system-metrics/src/opentelemetry/ext/system_metrics/version.py b/ext/opentelemetry-ext-system-metrics/src/opentelemetry/ext/system_metrics/version.py new file mode 100644 index 00000000000..603bf0b7e5f --- /dev/null +++ b/ext/opentelemetry-ext-system-metrics/src/opentelemetry/ext/system_metrics/version.py @@ -0,0 +1,15 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.9.dev0" diff --git a/ext/opentelemetry-ext-system-metrics/tests/__init__.py b/ext/opentelemetry-ext-system-metrics/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ext/opentelemetry-ext-system-metrics/tests/test_system_metrics.py b/ext/opentelemetry-ext-system-metrics/tests/test_system_metrics.py new file mode 100644 index 00000000000..70ead2c5152 --- /dev/null +++ b/ext/opentelemetry-ext-system-metrics/tests/test_system_metrics.py @@ -0,0 +1,203 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import namedtuple +from unittest import mock + +from opentelemetry import metrics +from opentelemetry.ext.system_metrics import SystemMetrics +from opentelemetry.test.test_base import TestBase + + +class TestSystemMetrics(TestBase): + def setUp(self): + super().setUp() + self.memory_metrics_exporter.clear() + + def test_system_metrics_constructor(self): + # ensure the observers have been registered + meter = metrics.get_meter(__name__) + with mock.patch("opentelemetry.metrics.get_meter") as mock_get_meter: + mock_get_meter.return_value = meter + SystemMetrics(self.memory_metrics_exporter) + self.assertEqual(len(meter.observers), 6) + observer_names = [ + "system.mem", + "system.cpu", + "system.net.bytes", + "runtime.python.mem", + "runtime.python.cpu", + "runtime.python.gc.count", + ] + for observer in meter.observers: + self.assertIn(observer.name, observer_names) + observer_names.remove(observer.name) + + def _assert_metrics(self, observer_name, system_metrics, expected): + system_metrics.controller.tick() + assertions = 0 + for ( + metric + ) in ( + self.memory_metrics_exporter._exported_metrics # pylint: disable=protected-access + ): + if ( + metric.labels in expected + and metric.metric.name == observer_name + ): + self.assertEqual( + metric.aggregator.checkpoint.last, expected[metric.labels], + ) + assertions += 1 + self.assertEqual(len(expected), assertions) + + def _test_metrics(self, observer_name, expected): + meter = self.meter_provider.get_meter(__name__) + + with mock.patch("opentelemetry.metrics.get_meter") as mock_get_meter: + mock_get_meter.return_value = meter + system_metrics = SystemMetrics(self.memory_metrics_exporter) + self._assert_metrics(observer_name, system_metrics, expected) + + @mock.patch("psutil.cpu_times") + def test_system_cpu(self, mock_cpu_times): + CPUTimes = namedtuple("CPUTimes", ["user", "nice", "system", "idle"]) + mock_cpu_times.return_value = CPUTimes( + user=332277.48, nice=0.0, system=309836.43, idle=6724698.94 + ) + + expected = { + (("type", "user"),): 332277.48, + (("type", "system"),): 309836.43, + (("type", "idle"),): 6724698.94, + } + self._test_metrics("system.cpu", expected) + + @mock.patch("psutil.virtual_memory") + def test_system_memory(self, mock_virtual_memory): + VirtualMemory = namedtuple( + "VirtualMemory", + [ + "total", + "available", + "percent", + "used", + "free", + "active", + "inactive", + "wired", + ], + ) + mock_virtual_memory.return_value = VirtualMemory( + total=17179869184, + available=5520928768, + percent=67.9, + used=10263990272, + free=266964992, + active=5282459648, + inactive=5148700672, + wired=4981530624, + ) + + expected = { + (("type", "total"),): 17179869184, + (("type", "used"),): 10263990272, + (("type", "available"),): 5520928768, + (("type", "free"),): 266964992, + } + self._test_metrics("system.mem", expected) + + @mock.patch("psutil.net_io_counters") + def test_network_bytes(self, mock_net_io_counters): + NetworkIO = namedtuple( + "NetworkIO", + ["bytes_sent", "bytes_recv", "packets_recv", "packets_sent"], + ) + mock_net_io_counters.return_value = NetworkIO( + bytes_sent=23920188416, + bytes_recv=46798894080, + packets_sent=53127118, + packets_recv=53205738, + ) + + expected = { + (("type", "bytes_recv"),): 46798894080, + (("type", "bytes_sent"),): 23920188416, + } + self._test_metrics("system.net.bytes", expected) + + def test_runtime_memory(self): + meter = self.meter_provider.get_meter(__name__) + with mock.patch("opentelemetry.metrics.get_meter") as mock_get_meter: + mock_get_meter.return_value = meter + system_metrics = SystemMetrics(self.memory_metrics_exporter) + + with mock.patch.object( + system_metrics._proc, # pylint: disable=protected-access + "memory_info", + ) as mock_runtime_memory: + RuntimeMemory = namedtuple( + "RuntimeMemory", ["rss", "vms", "pfaults", "pageins"], + ) + mock_runtime_memory.return_value = RuntimeMemory( + rss=9777152, vms=4385665024, pfaults=2631, pageins=49 + ) + expected = { + (("type", "rss"),): 9777152, + (("type", "vms"),): 4385665024, + } + self._assert_metrics( + "runtime.python.mem", system_metrics, expected + ) + + def test_runtime_cpu(self): + meter = self.meter_provider.get_meter(__name__) + with mock.patch("opentelemetry.metrics.get_meter") as mock_get_meter: + mock_get_meter.return_value = meter + system_metrics = SystemMetrics(self.memory_metrics_exporter) + + with mock.patch.object( + system_metrics._proc, # pylint: disable=protected-access + "cpu_times", + ) as mock_runtime_cpu_times: + RuntimeCPU = namedtuple( + "RuntimeCPU", ["user", "nice", "system"] + ) + mock_runtime_cpu_times.return_value = RuntimeCPU( + user=100.48, nice=0.0, system=200.43 + ) + + expected = { + (("type", "user"),): 100.48, + (("type", "system"),): 200.43, + } + + self._assert_metrics( + "runtime.python.cpu", system_metrics, expected + ) + + @mock.patch("gc.get_count") + def test_runtime_gc_count(self, mock_gc): + mock_gc.return_value = [ + 100, # gen0 + 50, # gen1 + 10, # gen2 + ] + + expected = { + (("count", "0"),): 100, + (("count", "1"),): 50, + (("count", "2"),): 10, + } + self._test_metrics("runtime.python.gc.count", expected) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/metrics/export/in_memory_metrics_exporter.py b/opentelemetry-sdk/src/opentelemetry/sdk/metrics/export/in_memory_metrics_exporter.py index a5df2bb1d83..58ea6eba2ab 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/metrics/export/in_memory_metrics_exporter.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/metrics/export/in_memory_metrics_exporter.py @@ -41,6 +41,7 @@ def export( ) -> MetricsExportResult: if self._stopped: return MetricsExportResult.FAILURE + with self._lock: self._exported_metrics.extend(metric_records) return MetricsExportResult.SUCCESS diff --git a/tox.ini b/tox.ini index abb1355be96..3cec9fcfbd2 100644 --- a/tox.ini +++ b/tox.ini @@ -114,6 +114,11 @@ envlist = py3{4,5,6,7,8}-test-ext-redis pypy3-test-ext-redis + ; opentelemetry-ext-system-metrics + py3{4,5,6,7,8}-test-ext-system-metrics + ; ext-system-metrics intentionally excluded from pypy3 + ; known limitation: gc.get_count won't work under pypy + ; Coverage is temporarily disabled for pypy3 due to the pytest bug. ; pypy3-coverage @@ -162,6 +167,7 @@ changedir = test-opentracing-shim: ext/opentelemetry-ext-opentracing-shim/tests test-ext-sqlalchemy: ext/opentelemetry-ext-sqlalchemy/tests test-ext-redis: ext/opentelemetry-ext-redis/tests + test-ext-system-metrics: ext/opentelemetry-ext-system-metrics/tests commands_pre = ; Install without -e to test the actual installation @@ -220,6 +226,9 @@ commands_pre = sqlalchemy: pip install {toxinidir}/opentelemetry-auto-instrumentation {toxinidir}/ext/opentelemetry-ext-sqlalchemy + system-metrics: pip install {toxinidir}/opentelemetry-auto-instrumentation + system-metrics: pip install {toxinidir}/ext/opentelemetry-ext-system-metrics[test] + ; In order to get a healthy coverage report, ; we have to install packages in editable mode. coverage: python {toxinidir}/scripts/eachdist.py install --editable @@ -317,6 +326,7 @@ commands_pre = -e {toxinidir}/ext/opentelemetry-ext-pymysql \ -e {toxinidir}/ext/opentelemetry-ext-sqlalchemy \ -e {toxinidir}/ext/opentelemetry-ext-redis \ + -e {toxinidir}/ext/opentelemetry-ext-system-metrics \ -e {toxinidir}/ext/opentelemetry-ext-opencensusexporter docker-compose up -d python check_availability.py