From f34d9dbf6c22805fbf9170480b250395dbdc71e5 Mon Sep 17 00:00:00 2001 From: "allen.k1m" Date: Tue, 12 Sep 2023 23:06:35 +0900 Subject: [PATCH 1/7] add new process metric #1924 --- .../system_metrics/__init__.py | 62 ++++++++++++++++++ .../tests/test_system_metrics.py | 63 ++++++++++++++++++- 2 files changed, 123 insertions(+), 2 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-system-metrics/src/opentelemetry/instrumentation/system_metrics/__init__.py b/instrumentation/opentelemetry-instrumentation-system-metrics/src/opentelemetry/instrumentation/system_metrics/__init__.py index b7bd38907e..42dbdca34e 100644 --- a/instrumentation/opentelemetry-instrumentation-system-metrics/src/opentelemetry/instrumentation/system_metrics/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-system-metrics/src/opentelemetry/instrumentation/system_metrics/__init__.py @@ -105,6 +105,9 @@ "process.runtime.memory": ["rss", "vms"], "process.runtime.cpu.time": ["user", "system"], "process.runtime.gc_count": None, + "process.runtime.thread_count": None, + "process.runtime.cpu.utilization": None, + "process.runtime.context.switches": ["involuntary", "voluntary"], } @@ -150,6 +153,11 @@ def __init__( self._runtime_memory_labels = self._labels.copy() self._runtime_cpu_time_labels = self._labels.copy() self._runtime_gc_count_labels = self._labels.copy() + self._runtime_thread_count_labels = self._labels.copy() + + self._runtime_cpu_utilization_labels = self._labels.copy() + + self._runtime_context_switches_labels = self._labels.copy() def instrumentation_dependencies(self) -> Collection[str]: return _instruments @@ -347,6 +355,29 @@ def _instrument(self, **kwargs): unit="bytes", ) + if "process.runtime.thread_count" in self._config: + self._meter.create_observable_gauge( + name="process.runtime.thread_count", + callbacks=[self._get_runtime_thread_count], + description="Runtime active threads count", + ) + + if "process.runtime.cpu.utilization" in self._config: + self._meter.create_observable_gauge( + name="process.runtime.cpu.utilization", + callbacks=[self._get_runtime_cpu_utilization], + description="Runtime CPU utilization", + unit="1", + ) + + if "process.runtime.context.switches" in self._config: + self._meter.create_observable_counter( + name="process.runtime.context.switches", + callbacks=[self._get_runtime_context_switches], + description="Runtime context switches", + unit="switches", + ) + def _uninstrument(self, **__): pass @@ -646,3 +677,34 @@ def _get_runtime_gc_count( for index, count in enumerate(gc.get_count()): self._runtime_gc_count_labels["count"] = str(index) yield Observation(count, self._runtime_gc_count_labels.copy()) + + def _get_runtime_thread_count( + self, options: CallbackOptions + ) -> Iterable[Observation]: + """Observer callback for runtime active thread count""" + yield Observation( + self._proc.num_threads(), self._runtime_thread_count_labels.copy() + ) + + def _get_runtime_cpu_utilization( + self, options: CallbackOptions + ) -> Iterable[Observation]: + """Observer callback for runtime CPU utilization""" + proc_cpu_percent = self._proc.cpu_percent() + yield Observation( + proc_cpu_percent, + self._runtime_cpu_utilization_labels.copy(), + ) + + def _get_runtime_context_switches( + self, options: CallbackOptions + ) -> Iterable[Observation]: + """Observer callback for runtime context switches""" + ctx_switches = self._proc.num_ctx_switches() + for metric in self._config["process.runtime.context.switches"]: + if hasattr(ctx_switches, metric): + self._runtime_context_switches_labels["type"] = metric + yield Observation( + getattr(ctx_switches, metric), + self._runtime_context_switches_labels.copy(), + ) diff --git a/instrumentation/opentelemetry-instrumentation-system-metrics/tests/test_system_metrics.py b/instrumentation/opentelemetry-instrumentation-system-metrics/tests/test_system_metrics.py index f6dbd6c9a1..91b035e549 100644 --- a/instrumentation/opentelemetry-instrumentation-system-metrics/tests/test_system_metrics.py +++ b/instrumentation/opentelemetry-instrumentation-system-metrics/tests/test_system_metrics.py @@ -96,7 +96,7 @@ def test_system_metrics_instrument(self): for scope_metrics in resource_metrics.scope_metrics: for metric in scope_metrics.metrics: metric_names.append(metric.name) - self.assertEqual(len(metric_names), 18) + self.assertEqual(len(metric_names), 21) observer_names = [ "system.cpu.time", @@ -117,6 +117,9 @@ def test_system_metrics_instrument(self): f"process.runtime.{self.implementation}.memory", f"process.runtime.{self.implementation}.cpu_time", f"process.runtime.{self.implementation}.gc_count", + "process.runtime.thread_count", + "process.runtime.context.switches", + "process.runtime.cpu.utilization", ] for observer in metric_names: @@ -128,6 +131,9 @@ def test_runtime_metrics_instrument(self): "process.runtime.memory": ["rss", "vms"], "process.runtime.cpu.time": ["user", "system"], "process.runtime.gc_count": None, + "process.runtime.thread_count": None, + "process.runtime.cpu.utilization": None, + "process.runtime.context.switches": ["involuntary", "voluntary"], } reader = InMemoryMetricReader() @@ -140,12 +146,15 @@ def test_runtime_metrics_instrument(self): for scope_metrics in resource_metrics.scope_metrics: for metric in scope_metrics.metrics: metric_names.append(metric.name) - self.assertEqual(len(metric_names), 3) + self.assertEqual(len(metric_names), 6) observer_names = [ f"process.runtime.{self.implementation}.memory", f"process.runtime.{self.implementation}.cpu_time", f"process.runtime.{self.implementation}.gc_count", + "process.runtime.thread_count", + "process.runtime.context.switches", + "process.runtime.cpu.utilization", ] for observer in metric_names: @@ -782,3 +791,53 @@ def test_runtime_get_count(self, mock_gc_get_count): self._test_metrics( f"process.runtime.{self.implementation}.gc_count", expected ) + + @mock.patch("psutil.Process.num_ctx_switches") + def test_runtime_context_switches(self, mock_process_num_ctx_switches): + PCtxSwitches = namedtuple("PCtxSwitches", ["voluntary", "involuntary"]) + + mock_process_num_ctx_switches.configure_mock( + **{"return_value": PCtxSwitches(voluntary=1, involuntary=2)} + ) + + expected = [ + _SystemMetricsResult({"type": "voluntary"}, 1), + _SystemMetricsResult({"type": "involuntary"}, 2), + ] + self._test_metrics( + "process.runtime.context.switches", expected + ) + + @mock.patch("psutil.Process.thread_num") + def test_runtime_thread_num(self, mock_process_thread_num): + mock_process_thread_num.configure_mock(**{"return_value": 42}) + + expected = [_SystemMetricsResult({}, 42)] + self._test_metrics( + "process.runtime.thread_num", expected + ) + + @mock.patch("psutil.Process.cpu_percent") + def test_runtime_cpu_percent(self, mock_process_cpu_percent): + mock_process_cpu_percent.configure_mock(**{"return_value": 42}) + + expected = [_SystemMetricsResult({}, 42)] + self._test_metrics( + "process.runtime.cpu.utilization", expected + ) + + @mock.patch("psutil.Process.num_ctx_switches") + def test_runtime_context_switches(self, mock_process_num_ctx_switches): + PCtxSwitches = namedtuple("PCtxSwitches", ["voluntary", "involuntary"]) + + mock_process_num_ctx_switches.configure_mock( + **{"return_value": PCtxSwitches(voluntary=1, involuntary=2)} + ) + + expected = [ + _SystemMetricsResult({"type": "voluntary"}, 1), + _SystemMetricsResult({"type": "involuntary"}, 2), + ] + self._test_metrics( + "process.runtime.context.switches", expected + ) \ No newline at end of file From bc6e4c7646d48300a585a9d978b9c6ca08f00eb9 Mon Sep 17 00:00:00 2001 From: "allen.k1m" Date: Tue, 12 Sep 2023 23:31:32 +0900 Subject: [PATCH 2/7] modify test code --- .../tests/test_system_metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-system-metrics/tests/test_system_metrics.py b/instrumentation/opentelemetry-instrumentation-system-metrics/tests/test_system_metrics.py index 91b035e549..e741f9ed90 100644 --- a/instrumentation/opentelemetry-instrumentation-system-metrics/tests/test_system_metrics.py +++ b/instrumentation/opentelemetry-instrumentation-system-metrics/tests/test_system_metrics.py @@ -808,13 +808,13 @@ def test_runtime_context_switches(self, mock_process_num_ctx_switches): "process.runtime.context.switches", expected ) - @mock.patch("psutil.Process.thread_num") + @mock.patch("psutil.Process.num_threads") def test_runtime_thread_num(self, mock_process_thread_num): mock_process_thread_num.configure_mock(**{"return_value": 42}) expected = [_SystemMetricsResult({}, 42)] self._test_metrics( - "process.runtime.thread_num", expected + "process.runtime.thread_count", expected ) @mock.patch("psutil.Process.cpu_percent") From 8d079956748ce91cb56d147d025a503f9b0265b9 Mon Sep 17 00:00:00 2001 From: "allen.k1m" Date: Wed, 13 Sep 2023 00:10:48 +0900 Subject: [PATCH 3/7] add CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1206844e55..107dc8335d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added +- `opentelemetry-instrumentation-system-metrics` Add support for collecting process metrics + ([#1948](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1948)) + ## Version 1.20.0/0.41b0 (2023-09-01) ### Fixed From 641ca6bb53fcb0fd21daa3b09843a9f64dadb4e2 Mon Sep 17 00:00:00 2001 From: "allen.k1m" Date: Fri, 22 Sep 2023 09:32:03 +0900 Subject: [PATCH 4/7] follow the semantic conventions --- .../system_metrics/__init__.py | 17 ++++--- .../tests/test_system_metrics.py | 49 +++++++------------ 2 files changed, 28 insertions(+), 38 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-system-metrics/src/opentelemetry/instrumentation/system_metrics/__init__.py b/instrumentation/opentelemetry-instrumentation-system-metrics/src/opentelemetry/instrumentation/system_metrics/__init__.py index 42dbdca34e..6ea084d7a2 100644 --- a/instrumentation/opentelemetry-instrumentation-system-metrics/src/opentelemetry/instrumentation/system_metrics/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-system-metrics/src/opentelemetry/instrumentation/system_metrics/__init__.py @@ -36,6 +36,10 @@ "system.thread_count": None "process.runtime.memory": ["rss", "vms"], "process.runtime.cpu.time": ["user", "system"], + "process.runtime.gc_count": None, + "process.runtime.thread_count": None, + "process.runtime.cpu.utilization": None, + "process.runtime.context_switches": ["involuntary", "voluntary"], } Usage @@ -63,6 +67,7 @@ "system.network.io": ["transmit", "receive"], "process.runtime.memory": ["rss", "vms"], "process.runtime.cpu.time": ["user", "system"], + "process.runtime.context_switches": ["involuntary", "voluntary"], } SystemMetricsInstrumentor(config=configuration).instrument() @@ -107,7 +112,7 @@ "process.runtime.gc_count": None, "process.runtime.thread_count": None, "process.runtime.cpu.utilization": None, - "process.runtime.context.switches": ["involuntary", "voluntary"], + "process.runtime.context_switches": ["involuntary", "voluntary"], } @@ -357,22 +362,22 @@ def _instrument(self, **kwargs): if "process.runtime.thread_count" in self._config: self._meter.create_observable_gauge( - name="process.runtime.thread_count", + name=f"process.runtime.{self._python_implementation}.thread_count", callbacks=[self._get_runtime_thread_count], description="Runtime active threads count", ) if "process.runtime.cpu.utilization" in self._config: self._meter.create_observable_gauge( - name="process.runtime.cpu.utilization", + name=f"process.runtime.{self._python_implementation}.cpu.utilization", callbacks=[self._get_runtime_cpu_utilization], description="Runtime CPU utilization", unit="1", ) - if "process.runtime.context.switches" in self._config: + if "process.runtime.context_switches" in self._config: self._meter.create_observable_counter( - name="process.runtime.context.switches", + name=f"process.runtime.{self._python_implementation}.context_switches", callbacks=[self._get_runtime_context_switches], description="Runtime context switches", unit="switches", @@ -701,7 +706,7 @@ def _get_runtime_context_switches( ) -> Iterable[Observation]: """Observer callback for runtime context switches""" ctx_switches = self._proc.num_ctx_switches() - for metric in self._config["process.runtime.context.switches"]: + for metric in self._config["process.runtime.context_switches"]: if hasattr(ctx_switches, metric): self._runtime_context_switches_labels["type"] = metric yield Observation( diff --git a/instrumentation/opentelemetry-instrumentation-system-metrics/tests/test_system_metrics.py b/instrumentation/opentelemetry-instrumentation-system-metrics/tests/test_system_metrics.py index e741f9ed90..e28c437009 100644 --- a/instrumentation/opentelemetry-instrumentation-system-metrics/tests/test_system_metrics.py +++ b/instrumentation/opentelemetry-instrumentation-system-metrics/tests/test_system_metrics.py @@ -18,13 +18,14 @@ from platform import python_implementation from unittest import mock -from opentelemetry.instrumentation.system_metrics import ( - SystemMetricsInstrumentor, -) from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import InMemoryMetricReader from opentelemetry.test.test_base import TestBase +from opentelemetry.instrumentation.system_metrics import ( + SystemMetricsInstrumentor, +) + def _mock_netconnection(): NetConnection = namedtuple( @@ -117,9 +118,9 @@ def test_system_metrics_instrument(self): f"process.runtime.{self.implementation}.memory", f"process.runtime.{self.implementation}.cpu_time", f"process.runtime.{self.implementation}.gc_count", - "process.runtime.thread_count", - "process.runtime.context.switches", - "process.runtime.cpu.utilization", + f"process.runtime.{self.implementation}.thread_count", + f"process.runtime.{self.implementation}.context_switches", + f"process.runtime.{self.implementation}.cpu.utilization", ] for observer in metric_names: @@ -133,7 +134,7 @@ def test_runtime_metrics_instrument(self): "process.runtime.gc_count": None, "process.runtime.thread_count": None, "process.runtime.cpu.utilization": None, - "process.runtime.context.switches": ["involuntary", "voluntary"], + "process.runtime.context_switches": ["involuntary", "voluntary"], } reader = InMemoryMetricReader() @@ -152,9 +153,9 @@ def test_runtime_metrics_instrument(self): f"process.runtime.{self.implementation}.memory", f"process.runtime.{self.implementation}.cpu_time", f"process.runtime.{self.implementation}.gc_count", - "process.runtime.thread_count", - "process.runtime.context.switches", - "process.runtime.cpu.utilization", + f"process.runtime.{self.implementation}.thread_count", + f"process.runtime.{self.implementation}.context_switches", + f"process.runtime.{self.implementation}.cpu.utilization", ] for observer in metric_names: @@ -170,9 +171,9 @@ def _assert_metrics(self, observer_name, reader, expected): for data_point in metric.data.data_points: for expect in expected: if ( - dict(data_point.attributes) - == expect.attributes - and metric.name == observer_name + dict(data_point.attributes) + == expect.attributes + and metric.name == observer_name ): self.assertEqual( data_point.value, @@ -805,7 +806,7 @@ def test_runtime_context_switches(self, mock_process_num_ctx_switches): _SystemMetricsResult({"type": "involuntary"}, 2), ] self._test_metrics( - "process.runtime.context.switches", expected + f"process.runtime.{self.implementation}.context_switches", expected ) @mock.patch("psutil.Process.num_threads") @@ -814,7 +815,7 @@ def test_runtime_thread_num(self, mock_process_thread_num): expected = [_SystemMetricsResult({}, 42)] self._test_metrics( - "process.runtime.thread_count", expected + f"process.runtime.{self.implementation}.thread_count", expected ) @mock.patch("psutil.Process.cpu_percent") @@ -823,21 +824,5 @@ def test_runtime_cpu_percent(self, mock_process_cpu_percent): expected = [_SystemMetricsResult({}, 42)] self._test_metrics( - "process.runtime.cpu.utilization", expected + f"process.runtime.{self.implementation}.cpu.utilization", expected ) - - @mock.patch("psutil.Process.num_ctx_switches") - def test_runtime_context_switches(self, mock_process_num_ctx_switches): - PCtxSwitches = namedtuple("PCtxSwitches", ["voluntary", "involuntary"]) - - mock_process_num_ctx_switches.configure_mock( - **{"return_value": PCtxSwitches(voluntary=1, involuntary=2)} - ) - - expected = [ - _SystemMetricsResult({"type": "voluntary"}, 1), - _SystemMetricsResult({"type": "involuntary"}, 2), - ] - self._test_metrics( - "process.runtime.context.switches", expected - ) \ No newline at end of file From 3de56a59f82d52405febe5263b0b0ae1ff6a5dba Mon Sep 17 00:00:00 2001 From: "allen.k1m" Date: Mon, 25 Sep 2023 20:00:48 +0900 Subject: [PATCH 5/7] add attrs dependency --- .../opentelemetry-instrumentation-remoulade/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/instrumentation/opentelemetry-instrumentation-remoulade/pyproject.toml b/instrumentation/opentelemetry-instrumentation-remoulade/pyproject.toml index 2a3a596700..3311329998 100644 --- a/instrumentation/opentelemetry-instrumentation-remoulade/pyproject.toml +++ b/instrumentation/opentelemetry-instrumentation-remoulade/pyproject.toml @@ -25,6 +25,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] dependencies = [ + "attrs >= 21.2.0", "opentelemetry-api ~= 1.12", "opentelemetry-instrumentation == 0.42b0.dev", "opentelemetry-semantic-conventions == 0.42b0.dev", From 1f6e01b249fc7708adc51995e2e186e82f21be5b Mon Sep 17 00:00:00 2001 From: "allen.k1m" Date: Tue, 26 Sep 2023 17:23:21 +0900 Subject: [PATCH 6/7] exclude from dependencies(https://github.com/wiremind/remoulade/issues/339) --- .../opentelemetry-instrumentation-remoulade/pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/instrumentation/opentelemetry-instrumentation-remoulade/pyproject.toml b/instrumentation/opentelemetry-instrumentation-remoulade/pyproject.toml index 3311329998..2a3a596700 100644 --- a/instrumentation/opentelemetry-instrumentation-remoulade/pyproject.toml +++ b/instrumentation/opentelemetry-instrumentation-remoulade/pyproject.toml @@ -25,7 +25,6 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] dependencies = [ - "attrs >= 21.2.0", "opentelemetry-api ~= 1.12", "opentelemetry-instrumentation == 0.42b0.dev", "opentelemetry-semantic-conventions == 0.42b0.dev", From 8af57ebda41d08937b932506d23653fc201216e9 Mon Sep 17 00:00:00 2001 From: "allen.k1m" Date: Sun, 8 Oct 2023 12:03:08 +0900 Subject: [PATCH 7/7] modify runtime context thread_count metric type --- .../opentelemetry/instrumentation/system_metrics/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-system-metrics/src/opentelemetry/instrumentation/system_metrics/__init__.py b/instrumentation/opentelemetry-instrumentation-system-metrics/src/opentelemetry/instrumentation/system_metrics/__init__.py index 6ea084d7a2..3d6a0c6775 100644 --- a/instrumentation/opentelemetry-instrumentation-system-metrics/src/opentelemetry/instrumentation/system_metrics/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-system-metrics/src/opentelemetry/instrumentation/system_metrics/__init__.py @@ -159,9 +159,7 @@ def __init__( self._runtime_cpu_time_labels = self._labels.copy() self._runtime_gc_count_labels = self._labels.copy() self._runtime_thread_count_labels = self._labels.copy() - self._runtime_cpu_utilization_labels = self._labels.copy() - self._runtime_context_switches_labels = self._labels.copy() def instrumentation_dependencies(self) -> Collection[str]: @@ -361,7 +359,7 @@ def _instrument(self, **kwargs): ) if "process.runtime.thread_count" in self._config: - self._meter.create_observable_gauge( + self._meter.create_observable_up_down_counter( name=f"process.runtime.{self._python_implementation}.thread_count", callbacks=[self._get_runtime_thread_count], description="Runtime active threads count",