From 992ec87a8b9d505048df2cd0ed95384a5beed6ec Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 15 Dec 2022 16:01:52 -0800 Subject: [PATCH 1/9] (WIP) Adds Python.*.Version for some frameworks --- solarwinds_apm/configurator.py | 65 ++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/solarwinds_apm/configurator.py b/solarwinds_apm/configurator.py index 27048ef8..754f6517 100644 --- a/solarwinds_apm/configurator.py +++ b/solarwinds_apm/configurator.py @@ -1,5 +1,6 @@ """Module to initialize OpenTelemetry SDK components and liboboe to work with SolarWinds backend""" +import importlib import logging import os import sys @@ -10,6 +11,10 @@ OTEL_PROPAGATORS, OTEL_TRACES_EXPORTER, ) +from opentelemetry.instrumentation.dependencies import get_dist_dependency_conflicts +from opentelemetry.instrumentation.environment_variables import ( + OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, +) from opentelemetry.instrumentation.propagators import ( set_global_response_propagator, ) @@ -266,6 +271,64 @@ def _initialize_solarwinds_reporter( return NoopReporter(**reporter_kwargs) + def _add_all_instrumented_python_framework_versions( + self, + version_keys, + ) -> dict: + """Updates version_keys with versions of Python frameworks that have been + instrumented with installed (bootstrapped) OTel instrumentation libraries. + Borrowed from opentelemetry-instrumentation sitecustomize. + + Example output: + { + "Python.Requests.Version": "2.28.1", + "Python.Django.Version": "4.1.4", + "Python.Logging.Version": "0.5.1.2", + } + """ + package_to_exclude = os.environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, []) + if isinstance(package_to_exclude, str): + package_to_exclude = package_to_exclude.split(",") + package_to_exclude = [x.strip() for x in package_to_exclude] + + for entry_point in iter_entry_points("opentelemetry_instrumentor"): + if entry_point.name in package_to_exclude: + logger.debug( + "Skipping version lookup for library %s because excluded", entry_point.name + ) + continue + + try: + conflict = get_dist_dependency_conflicts(entry_point.dist) + if conflict: + logger.warning( + "Version lookup for library %s skipped due to conflict: %s", + entry_point.name, + conflict, + ) + continue + except Exception as ex: # pylint: disable=broad-except + logger.warning( + "Version conflict check of %s failed, so skipping: %s", + entry_point.name, + ex, + ) + continue + + instr_key = f"Python.{entry_point.name.capitalize()}.Version" + try: + importlib.import_module(entry_point.name) + version_keys[instr_key] = sys.modules[entry_point.name].__version__ + except (AttributeError, ImportError) as ex: + # could not import package for whatever reason + logger.warning( + "Version lookup of %s failed, so skipping: %s", + entry_point.name, + ex, + ) + + return version_keys + # pylint: disable=too-many-locals def _report_init_event( self, @@ -324,6 +387,8 @@ def _report_init_event( ] = Config.getVersionString() version_keys["APM.Extension.Version"] = Config.getVersionString() + version_keys = self._add_all_instrumented_python_framework_versions(version_keys) + if keys: version_keys.update(keys) From a1fc5ee21db30879170fcf50addecb75eca9dced Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 15 Dec 2022 16:26:06 -0800 Subject: [PATCH 2/9] Add special cases for urllib,sqlite3 --- solarwinds_apm/configurator.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/solarwinds_apm/configurator.py b/solarwinds_apm/configurator.py index 754f6517..f90542bb 100644 --- a/solarwinds_apm/configurator.py +++ b/solarwinds_apm/configurator.py @@ -281,8 +281,11 @@ def _add_all_instrumented_python_framework_versions( Example output: { + "Python.Urllib.Version": "3.9", "Python.Requests.Version": "2.28.1", "Python.Django.Version": "4.1.4", + "Python.Psycopg2.Version": "2.9.5 (dt dec pq3 ext lo64)", + "Python.Sqlite3.Version": "3.34.1", "Python.Logging.Version": "0.5.1.2", } """ @@ -317,8 +320,19 @@ def _add_all_instrumented_python_framework_versions( instr_key = f"Python.{entry_point.name.capitalize()}.Version" try: - importlib.import_module(entry_point.name) - version_keys[instr_key] = sys.modules[entry_point.name].__version__ + # urllib has a rich complex history + if entry_point.name == "urllib": + importlib.import_module(f"{entry_point.name}.request") + else: + importlib.import_module(entry_point.name) + + # some Python frameworks just don't have __version__ + if entry_point.name == "urllib": + version_keys[instr_key] = sys.modules[f"{entry_point.name}.request"].__version__ + elif entry_point.name == "sqlite3": + version_keys[instr_key] = sys.modules[entry_point.name].sqlite_version + else: + version_keys[instr_key] = sys.modules[entry_point.name].__version__ except (AttributeError, ImportError) as ex: # could not import package for whatever reason logger.warning( From a649bd4426f160f9965a3150c8458f695e52c7b6 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Thu, 15 Dec 2022 16:37:04 -0800 Subject: [PATCH 3/9] Formatting --- solarwinds_apm/configurator.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/solarwinds_apm/configurator.py b/solarwinds_apm/configurator.py index f90542bb..552d33ff 100644 --- a/solarwinds_apm/configurator.py +++ b/solarwinds_apm/configurator.py @@ -11,7 +11,9 @@ OTEL_PROPAGATORS, OTEL_TRACES_EXPORTER, ) -from opentelemetry.instrumentation.dependencies import get_dist_dependency_conflicts +from opentelemetry.instrumentation.dependencies import ( + get_dist_dependency_conflicts, +) from opentelemetry.instrumentation.environment_variables import ( OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, ) @@ -278,7 +280,7 @@ def _add_all_instrumented_python_framework_versions( """Updates version_keys with versions of Python frameworks that have been instrumented with installed (bootstrapped) OTel instrumentation libraries. Borrowed from opentelemetry-instrumentation sitecustomize. - + Example output: { "Python.Urllib.Version": "3.9", @@ -289,7 +291,9 @@ def _add_all_instrumented_python_framework_versions( "Python.Logging.Version": "0.5.1.2", } """ - package_to_exclude = os.environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, []) + package_to_exclude = os.environ.get( + OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, [] + ) if isinstance(package_to_exclude, str): package_to_exclude = package_to_exclude.split(",") package_to_exclude = [x.strip() for x in package_to_exclude] @@ -297,7 +301,8 @@ def _add_all_instrumented_python_framework_versions( for entry_point in iter_entry_points("opentelemetry_instrumentor"): if entry_point.name in package_to_exclude: logger.debug( - "Skipping version lookup for library %s because excluded", entry_point.name + "Skipping version lookup for library %s because excluded", + entry_point.name, ) continue @@ -317,22 +322,28 @@ def _add_all_instrumented_python_framework_versions( ex, ) continue - + instr_key = f"Python.{entry_point.name.capitalize()}.Version" try: # urllib has a rich complex history - if entry_point.name == "urllib": + if entry_point.name == "urllib": importlib.import_module(f"{entry_point.name}.request") else: importlib.import_module(entry_point.name) # some Python frameworks just don't have __version__ if entry_point.name == "urllib": - version_keys[instr_key] = sys.modules[f"{entry_point.name}.request"].__version__ + version_keys[instr_key] = sys.modules[ + f"{entry_point.name}.request" + ].__version__ elif entry_point.name == "sqlite3": - version_keys[instr_key] = sys.modules[entry_point.name].sqlite_version + version_keys[instr_key] = sys.modules[ + entry_point.name + ].sqlite_version else: - version_keys[instr_key] = sys.modules[entry_point.name].__version__ + version_keys[instr_key] = sys.modules[ + entry_point.name + ].__version__ except (AttributeError, ImportError) as ex: # could not import package for whatever reason logger.warning( @@ -401,7 +412,9 @@ def _report_init_event( ] = Config.getVersionString() version_keys["APM.Extension.Version"] = Config.getVersionString() - version_keys = self._add_all_instrumented_python_framework_versions(version_keys) + version_keys = self._add_all_instrumented_python_framework_versions( + version_keys + ) if keys: version_keys.update(keys) From a95fd6aa09ef3304d8f1d53b2002e9391d3242d5 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Fri, 23 Dec 2022 15:44:50 -0800 Subject: [PATCH 4/9] Add more cases for Python.*.Version --- solarwinds_apm/configurator.py | 52 ++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/solarwinds_apm/configurator.py b/solarwinds_apm/configurator.py index c2d95f25..312aaa4b 100644 --- a/solarwinds_apm/configurator.py +++ b/solarwinds_apm/configurator.py @@ -26,7 +26,7 @@ from opentelemetry.sdk.resources import SERVICE_NAME, Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor -from pkg_resources import iter_entry_points, load_entry_point +from pkg_resources import iter_entry_points, load_entry_point, get_distribution from solarwinds_apm import apm_logging from solarwinds_apm.apm_config import SolarWindsApmConfig @@ -323,32 +323,60 @@ def _add_all_instrumented_python_framework_versions( ) continue - instr_key = f"Python.{entry_point.name.capitalize()}.Version" + # Set up Instrumented Library Versions KVs with several special cases + entry_point_name = entry_point.name + instr_key = f"Python.{entry_point_name.capitalize()}.Version" try: + # Some OTel instrumentation libraries are named not exactly + # the same as the instrumented libraries! + # https://github.com/open-telemetry/opentelemetry-python-contrib/blob/main/instrumentation/README.md + if entry_point_name == "aiohttp-client": + entry_point_name = "aiohttp" + elif "grpc_" in entry_point_name: + entry_point_name = "grpc" + elif entry_point_name == "system_metrics": + entry_point_name = "psutil" + elif entry_point_name == "tortoiseorm": + entry_point_name = "tortoise" + + # There is no mysql version, but mysql.connector version + if entry_point_name == "mysql": + importlib.import_module(f"{entry_point_name}.connector") # urllib has a rich complex history - if entry_point.name == "urllib": - importlib.import_module(f"{entry_point.name}.request") + elif entry_point_name == "urllib": + importlib.import_module(f"{entry_point_name}.request") else: - importlib.import_module(entry_point.name) + importlib.import_module(entry_point_name) - # some Python frameworks just don't have __version__ - if entry_point.name == "urllib": + # some Python frameworks don't have top-level __version__ + if entry_point_name == "mysql": version_keys[instr_key] = sys.modules[ - f"{entry_point.name}.request" + f"{entry_point_name}.connector" ].__version__ - elif entry_point.name == "sqlite3": + elif entry_point_name == "pyramid": + version_keys[instr_key] = get_distribution(entry_point_name).version + elif entry_point_name == "sqlite3": version_keys[instr_key] = sys.modules[ - entry_point.name + entry_point_name ].sqlite_version + elif entry_point_name == "tornado": + version_keys[instr_key] = sys.modules[ + entry_point_name + ].version + elif entry_point_name == "urllib": + version_keys[instr_key] = sys.modules[ + f"{entry_point_name}.request" + ].__version__ else: version_keys[instr_key] = sys.modules[ - entry_point.name + entry_point_name ].__version__ + except (AttributeError, ImportError) as ex: # could not import package for whatever reason logger.warning( "Version lookup of %s failed, so skipping: %s", - entry_point.name, + entry_point_name, ex, ) From 1e1d3a827c9ad11877efda405591cfe5fb21c42c Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Fri, 23 Dec 2022 15:47:14 -0800 Subject: [PATCH 5/9] Formattin --- solarwinds_apm/configurator.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/solarwinds_apm/configurator.py b/solarwinds_apm/configurator.py index 312aaa4b..4515e4c8 100644 --- a/solarwinds_apm/configurator.py +++ b/solarwinds_apm/configurator.py @@ -26,7 +26,7 @@ from opentelemetry.sdk.resources import SERVICE_NAME, Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor -from pkg_resources import iter_entry_points, load_entry_point, get_distribution +from pkg_resources import get_distribution, iter_entry_points, load_entry_point from solarwinds_apm import apm_logging from solarwinds_apm.apm_config import SolarWindsApmConfig @@ -273,6 +273,7 @@ def _initialize_solarwinds_reporter( return NoopReporter(**reporter_kwargs) + # pylint: disable=too-many-branches def _add_all_instrumented_python_framework_versions( self, version_keys, @@ -354,7 +355,9 @@ def _add_all_instrumented_python_framework_versions( f"{entry_point_name}.connector" ].__version__ elif entry_point_name == "pyramid": - version_keys[instr_key] = get_distribution(entry_point_name).version + version_keys[instr_key] = get_distribution( + entry_point_name + ).version elif entry_point_name == "sqlite3": version_keys[instr_key] = sys.modules[ entry_point_name From 287a8fb3a9834195adffd4010829fd2e13ed1cc9 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Fri, 23 Dec 2022 17:19:07 -0800 Subject: [PATCH 6/9] Add case for elasticsearch --- solarwinds_apm/configurator.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/solarwinds_apm/configurator.py b/solarwinds_apm/configurator.py index 4515e4c8..3a8cc178 100644 --- a/solarwinds_apm/configurator.py +++ b/solarwinds_apm/configurator.py @@ -350,7 +350,13 @@ def _add_all_instrumented_python_framework_versions( importlib.import_module(entry_point_name) # some Python frameworks don't have top-level __version__ - if entry_point_name == "mysql": + # and elasticsearch gives a version as (8, 5, 3) not 8.5.3 + if entry_point_name == "elasticsearch": + version_tuple = sys.modules[entry_point_name].__version__ + version_keys[instr_key] = ".".join( + [str(d) for d in version_tuple] + ) + elif entry_point_name == "mysql": version_keys[instr_key] = sys.modules[ f"{entry_point_name}.connector" ].__version__ From 6a98004d6a791135bf8a97442082933e709e7ef7 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 4 Jan 2023 15:50:53 -0800 Subject: [PATCH 7/9] Init msg KV cap snakecase keys --- solarwinds_apm/configurator.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/solarwinds_apm/configurator.py b/solarwinds_apm/configurator.py index 282b9b55..c3208db1 100644 --- a/solarwinds_apm/configurator.py +++ b/solarwinds_apm/configurator.py @@ -273,7 +273,7 @@ def _initialize_solarwinds_reporter( return NoopReporter(**reporter_kwargs) - # pylint: disable=too-many-branches + # pylint: disable=too-many-branches,too-many-statements def _add_all_instrumented_python_framework_versions( self, version_keys, @@ -326,7 +326,14 @@ def _add_all_instrumented_python_framework_versions( # Set up Instrumented Library Versions KVs with several special cases entry_point_name = entry_point.name - instr_key = f"Python.{entry_point_name.capitalize()}.Version" + instr_key = "" + if "_" in entry_point_name: + instr_key = f"Python.{'_'.join([epn.capitalize() for epn in entry_point_name.split('_')])}.Version" + elif "-" in entry_point_name: + instr_key = f"Python.{'-'.join([epn.capitalize() for epn in entry_point_name.split('-')])}.Version" + else: + instr_key = f"Python.{entry_point_name.capitalize()}.Version" + try: # Some OTel instrumentation libraries are named not exactly # the same as the instrumented libraries! From ef7dfeb323098162531fb23c1737b4191f2334de Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 4 Jan 2023 15:50:58 -0800 Subject: [PATCH 8/9] Revert "Init msg KV cap snakecase keys" This reverts commit 6a98004d6a791135bf8a97442082933e709e7ef7. --- solarwinds_apm/configurator.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/solarwinds_apm/configurator.py b/solarwinds_apm/configurator.py index c3208db1..282b9b55 100644 --- a/solarwinds_apm/configurator.py +++ b/solarwinds_apm/configurator.py @@ -273,7 +273,7 @@ def _initialize_solarwinds_reporter( return NoopReporter(**reporter_kwargs) - # pylint: disable=too-many-branches,too-many-statements + # pylint: disable=too-many-branches def _add_all_instrumented_python_framework_versions( self, version_keys, @@ -326,14 +326,7 @@ def _add_all_instrumented_python_framework_versions( # Set up Instrumented Library Versions KVs with several special cases entry_point_name = entry_point.name - instr_key = "" - if "_" in entry_point_name: - instr_key = f"Python.{'_'.join([epn.capitalize() for epn in entry_point_name.split('_')])}.Version" - elif "-" in entry_point_name: - instr_key = f"Python.{'-'.join([epn.capitalize() for epn in entry_point_name.split('-')])}.Version" - else: - instr_key = f"Python.{entry_point_name.capitalize()}.Version" - + instr_key = f"Python.{entry_point_name.capitalize()}.Version" try: # Some OTel instrumentation libraries are named not exactly # the same as the instrumented libraries! From bffb2c9818fd606a5d1d899f4ecd8d6d49cf0ea4 Mon Sep 17 00:00:00 2001 From: tammy-baylis-swi Date: Wed, 4 Jan 2023 15:58:14 -0800 Subject: [PATCH 9/9] Instrumented library version KVs with untouched components --- solarwinds_apm/configurator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solarwinds_apm/configurator.py b/solarwinds_apm/configurator.py index 282b9b55..20beada3 100644 --- a/solarwinds_apm/configurator.py +++ b/solarwinds_apm/configurator.py @@ -326,7 +326,7 @@ def _add_all_instrumented_python_framework_versions( # Set up Instrumented Library Versions KVs with several special cases entry_point_name = entry_point.name - instr_key = f"Python.{entry_point_name.capitalize()}.Version" + instr_key = f"Python.{entry_point_name}.Version" try: # Some OTel instrumentation libraries are named not exactly # the same as the instrumented libraries!