Merge pull request #422 from solarwinds/NH-88066-upgrade-otel-1_27_0

NH-88066 Upgrade APM Python to Otel 1.27.0/0.48b0

.github/workflows/build_publish_lambda_layer_aarch64.yaml

@@ -45,6 +45,6 @@ jobs:
       artifact-name: solarwinds_apm_lambda_aarch64.zip
       component-version: ${{ needs.build_layer_aarch64.outputs.SW_APM_VERSION }}
       architecture: arm64
-      runtimes: "python3.8 python3.9 python3.10 python3.11"
+      runtimes: "python3.8 python3.9 python3.10 python3.11 python3.12"
       publish-dest: ${{ inputs.publish-dest }}
     secrets: inherit

.github/workflows/build_publish_lambda_layer_x86_64.yaml

@@ -39,6 +39,6 @@ jobs:
       artifact-name: solarwinds_apm_lambda_x86_64.zip
       component-version: ${{ needs.build_layer_x86_64.outputs.SW_APM_VERSION }}
       architecture: x86_64
-      runtimes: "python3.8 python3.9 python3.10 python3.11"
+      runtimes: "python3.8 python3.9 python3.10 python3.11 python3.12"
       publish-dest: ${{ inputs.publish-dest }}
     secrets: inherit

.github/workflows/run_tox_tests.yaml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-minor: ["8", "9", "10", "11"]
+        python-minor: ["8", "9", "10", "11", "12"]
         apm-env: ["nh-staging", "ao-prod"]
     steps:
     - uses: actions/checkout@v4

.github/workflows/verify_install.yaml

@@ -67,6 +67,10 @@ jobs:
           - py3.11-alpine3.17
           - py3.11-debian10
           - py3.11-debian11
+          - py3.12-alpine3.19
+          - py3.12-alpine3.20
+          - py3.12-debian11
+          - py3.12-debian12
         arch:
           - x64
           - arm64
@@ -121,6 +125,14 @@ jobs:
             image: python:3.11-buster
           - hostname: py3.11-debian11
             image: python:3.11-bullseye
+          - hostname: py3.12-alpine3.19
+            image: python:3.12-alpine3.19
+          - hostname: py3.12-alpine3.20
+            image: python:3.12-alpine3.20
+          - hostname: py3.12-debian11
+            image: python:3.12-bullseye
+          - hostname: py3.12-debian12
+            image: python:3.12-bookworm
         # Note: JavaScript Actions (checkout) in Alpine only supported in x64
         exclude:
           - hostname: py3.8-alpine3.12
@@ -145,6 +157,10 @@ jobs:
             arch: arm64
           - hostname: py3.11-alpine3.17
             arch: arm64
+          - hostname: py3.12-alpine3.19
+            arch: arm64
+          - hostname: py3.12-alpine3.20
+            arch: arm64
     container:
       image: "${{ matrix.image }}"
       options: --hostname "${{ matrix.hostname }}"

CONTRIBUTING.md

@@ -44,7 +44,7 @@ docker-compose run aarch64
 
 ### Regression Tests
 
-Automated testing of this repo uses [tox](https://tox.readthedocs.io) and runs in Python 3.8, 3.9, 3.10, and/or 3.11 because these are the versions supported by [OTel Python](https://github.com/open-telemetry/opentelemetry-python/blob/main/tox.ini). Testing can be run inside a build container which provides all dependencies and a compiled C-extension. Here is how to set up then run unit and integration tests locally:
+Automated testing of this repo uses [tox](https://tox.readthedocs.io) and runs in Python 3.8, 3.9, 3.10, 3.11 and/or 3.12 because these are the versions supported by [OTel Python](https://github.com/open-telemetry/opentelemetry-python/blob/main/tox.ini). Testing can be run inside a build container which provides all dependencies and a compiled C-extension. Here is how to set up then run unit and integration tests locally:
 
 1. Create and run a Docker build container as described above.
 2. Inside the build container: `make wrapper`. This downloads the version of a C/C++ dependency defined in `extension/VERSION` from SolarWinds Cloud and builds SWIG bindings.

Makefile

@@ -168,7 +168,7 @@ check-sdist-local:
 # in the corresponding repo of the Docker images: https://github.com/pypa/manylinux#example.
 manylinux-wheels: wrapper
 	@echo -e "Generating python agent package any-linux wheels for 64 bit systems"
-	@set -e; for PYBIN in cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311; do /opt/python/$${PYBIN}/bin/pip -v wheel . -w ./tmp_dist/ --no-deps; done
+	@set -e; for PYBIN in cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312; do /opt/python/$${PYBIN}/bin/pip -v wheel . -w ./tmp_dist/ --no-deps; done
 	@echo -e "Tagging wheels with $(wheel_tag)"
 	@set -e; for whl in ./tmp_dist/*.whl; do auditwheel repair --plat $(wheel_tag) "$$whl" -w ./dist/; done
 	@rm -rf ./tmp_dist
@@ -194,17 +194,17 @@ install-lambda-modules:
 	@echo -e "Install upstream dependencies to include in layer"
 	@/opt/python/cp38-cp38/bin/pip3.8 install -t ${target_dir}/python -r lambda/requirements.txt
 	@echo -e "Install other version-specific .so files for deps"
-	@set -e; for PYBIN in cp39-cp39 cp310-cp310 cp311-cp311; do /opt/python/$${PYBIN}/bin/pip install -t ${target_dir}/$${PYBIN} -r lambda/requirements-so.txt; done
-	@set -e; for PYBIN in cp39-cp39 cp310-cp310 cp311-cp311; do cp ${target_dir}/$${PYBIN}/charset_normalizer/*.so ${target_dir}/python/charset_normalizer/ && cp ${target_dir}/$${PYBIN}/grpc/_cython/*.so ${target_dir}/python/grpc/_cython/ && cp ${target_dir}/$${PYBIN}/wrapt/*.so ${target_dir}/python/wrapt/; done
-	@set -e; for PYBIN in cp39-cp39 cp310-cp310 cp311-cp311; do rm -rf ${target_dir}/$${PYBIN} ; done
+	@set -e; for PYBIN in cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312; do /opt/python/$${PYBIN}/bin/pip install -t ${target_dir}/$${PYBIN} -r lambda/requirements-so.txt; done
+	@set -e; for PYBIN in cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312; do cp ${target_dir}/$${PYBIN}/charset_normalizer/*.so ${target_dir}/python/charset_normalizer/ && cp ${target_dir}/$${PYBIN}/grpc/_cython/*.so ${target_dir}/python/grpc/_cython/ && cp ${target_dir}/$${PYBIN}/wrapt/*.so ${target_dir}/python/wrapt/; done
+	@set -e; for PYBIN in cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312; do rm -rf ${target_dir}/$${PYBIN} ; done
 	@echo -e "Install upstream dependencies without deps to include in layer"
 	@/opt/python/cp38-cp38/bin/pip3.8 install -t ${target_dir}/nodeps -r lambda/requirements-nodeps.txt --no-deps
 	@echo -e "Install solarwinds_apm to be packed up in zip archive to target directory."
 	@/opt/python/cp38-cp38/bin/pip3.8 install . -t ${target_dir}/nodeps --no-deps
 	@echo -e "Removing non-lambda C-extension library files generated by pip install under target directory."
 	@rm -rf ${target_dir}/nodeps/solarwinds_apm/extension/*.so*
 	@echo -e "Building AWS Lambda version of C-extensions for all supported Python versions in target directory."
-	@set -e; for PYBIN in cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311; do /opt/python/$${PYBIN}/bin/python setup.py build_ext -b ${target_dir}/nodeps; done
+	@set -e; for PYBIN in cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312; do /opt/python/$${PYBIN}/bin/python setup.py build_ext -b ${target_dir}/nodeps; done
 	@echo -e "Copying AWS Lambda specific Oboe library liboboe-1.0-lambda-${platform}.so into target directory."
 	@cp solarwinds_apm/extension/liboboe-1.0-lambda-${platform}.so ${target_dir}/nodeps/solarwinds_apm/extension/liboboe.so
 	@echo -e "Moving no-deps dependencies, needed for full opentelemetry/instrumentation path"
@@ -215,10 +215,9 @@ install-lambda-modules:
 	@mkdir ${target_dir}/solarwinds-apm/
 	@cp lambda/solarwinds-apm/wrapper ${target_dir}/solarwinds-apm/wrapper
 	@chmod 755 ${target_dir}/solarwinds-apm/wrapper
-	@echo -e "Removing unnecessary boto, six, setuptools, urllib3 installations"
+	@echo -e "Removing unnecessary boto, six, urllib3 installations"
 	@rm -rf ${target_dir}/python/boto*
 	@rm -rf ${target_dir}/python/six*
-	@rm -rf ${target_dir}/python/setuptools*
 	@rm -rf ${target_dir}/python/urllib3*
 	@find ${target_dir}/python -type d -name '__pycache__' | xargs rm -rf
 

dev-requirements.txt

@@ -1,6 +1,6 @@
-opentelemetry-test-utils==0.46b0
-opentelemetry-instrumentation-flask==0.46b0
-opentelemetry-instrumentation-requests==0.46b0
+opentelemetry-test-utils==0.48b0
+opentelemetry-instrumentation-flask==0.48b0
+opentelemetry-instrumentation-requests==0.48b0
 pytest
 pytest-cov
 pytest-mock

lambda/check_lambda_modules.sh

@@ -57,14 +57,21 @@ expected_otel_files="./python/opentelemetry/exporter/otlp/proto/common/version.p
 ./python/opentelemetry/exporter/otlp/proto/grpc/version.py
 ./python/opentelemetry/exporter/otlp/proto/http/version.py
 ./python/opentelemetry/exporter/otlp/version.py
-./python/opentelemetry/instrumentation/botocore/version.py
-./python/opentelemetry/sdk/version.py"
-found_otel_files=$(find ./python/opentelemetry/exporter ./python/opentelemetry/sdk ./python/opentelemetry/instrumentation/botocore -regextype sed -regex ".*/version.py" | sort -k1)
+./python/opentelemetry/instrumentation/botocore/version.py"
+found_otel_files=$(find ./python/opentelemetry/exporter ./python/opentelemetry/instrumentation/botocore -regextype sed -regex ".*/version.py" | sort -k1)
 if [[ ! "$found_otel_files" =~ $expected_otel_files ]]; then
     echo "FAILED: Missing key opentelemetry dependency version files"
     exit 1
 fi
 
+# An additional check for those packages now following PEP 420
+expected_otel_files_pep420="./python/opentelemetry/sdk/version/__init__.py"
+found_otel_files_pep420=$(find ./python/opentelemetry/sdk -regextype sed -regex ".*/version/__init__.py" | sort -k1)
+if [[ ! "$found_otel_files_pep420" =~ $expected_otel_files_pep420 ]]; then
+    echo "FAILED: Missing key opentelemetry dependency version files (PEP 420)"
+    exit 1
+fi
+
 cd "$pwd"
 echo "Successfully verified lambda files and modules for layer archive"
 exit 0

lambda/requirements-nodeps.txt

@@ -1,30 +1,30 @@
-opentelemetry-instrumentation-aiohttp-client==0.46b0
-opentelemetry-util-http==0.46b0
+opentelemetry-instrumentation-aiohttp-client==0.48b0
+opentelemetry-util-http==0.48b0
 asgiref~=3.8
-opentelemetry-instrumentation-asgi==0.46b0
-opentelemetry-instrumentation-asyncpg==0.46b0
-opentelemetry-instrumentation-boto==0.46b0
-opentelemetry-instrumentation-botocore==0.46b0
-opentelemetry-instrumentation-celery==0.46b0
-opentelemetry-instrumentation-dbapi==0.46b0
-opentelemetry-instrumentation-django==0.46b0
-opentelemetry-instrumentation-elasticsearch==0.46b0
-opentelemetry-instrumentation-fastapi==0.46b0
-opentelemetry-instrumentation-falcon==0.46b0
-opentelemetry-instrumentation-flask==0.46b0
-opentelemetry-instrumentation-grpc==0.46b0
-opentelemetry-instrumentation-jinja2==0.46b0
-opentelemetry-instrumentation-logging==0.46b0
-opentelemetry-instrumentation-mysql==0.46b0
-opentelemetry-instrumentation-psycopg2==0.46b0
-opentelemetry-instrumentation-pymemcache==0.46b0
-opentelemetry-instrumentation-pymongo==0.46b0
-opentelemetry-instrumentation-pymysql==0.46b0
-opentelemetry-instrumentation-pyramid==0.46b0
-opentelemetry-instrumentation-redis==0.46b0
-opentelemetry-instrumentation-requests==0.46b0
-opentelemetry-instrumentation-sqlalchemy==0.46b0
-opentelemetry-instrumentation-sqlite3==0.46b0
-opentelemetry-instrumentation-starlette==0.46b0
-opentelemetry-instrumentation-tornado==0.46b0
-opentelemetry-instrumentation-wsgi==0.46b0
+opentelemetry-instrumentation-asgi==0.48b0
+opentelemetry-instrumentation-asyncpg==0.48b0
+opentelemetry-instrumentation-boto==0.48b0
+opentelemetry-instrumentation-botocore==0.48b0
+opentelemetry-instrumentation-celery==0.48b0
+opentelemetry-instrumentation-dbapi==0.48b0
+opentelemetry-instrumentation-django==0.48b0
+opentelemetry-instrumentation-elasticsearch==0.48b0
+opentelemetry-instrumentation-fastapi==0.48b0
+opentelemetry-instrumentation-falcon==0.48b0
+opentelemetry-instrumentation-flask==0.48b0
+opentelemetry-instrumentation-grpc==0.48b0
+opentelemetry-instrumentation-jinja2==0.48b0
+opentelemetry-instrumentation-logging==0.48b0
+opentelemetry-instrumentation-mysql==0.48b0
+opentelemetry-instrumentation-psycopg2==0.48b0
+opentelemetry-instrumentation-pymemcache==0.48b0
+opentelemetry-instrumentation-pymongo==0.48b0
+opentelemetry-instrumentation-pymysql==0.48b0
+opentelemetry-instrumentation-pyramid==0.48b0
+opentelemetry-instrumentation-redis==0.48b0
+opentelemetry-instrumentation-requests==0.48b0
+opentelemetry-instrumentation-sqlalchemy==0.48b0
+opentelemetry-instrumentation-sqlite3==0.48b0
+opentelemetry-instrumentation-starlette==0.48b0
+opentelemetry-instrumentation-tornado==0.48b0
+opentelemetry-instrumentation-wsgi==0.48b0

lambda/requirements.txt

@@ -1,7 +1,8 @@
-opentelemetry-exporter-otlp==1.25.0
-opentelemetry-exporter-otlp-proto-grpc==1.25.0
-opentelemetry-exporter-otlp-proto-http==1.25.0
-opentelemetry-instrumentation-aws-lambda==0.46b0
+opentelemetry-exporter-otlp==1.27.0
+opentelemetry-exporter-otlp-proto-grpc==1.27.0
+opentelemetry-exporter-otlp-proto-http==1.27.0
+opentelemetry-instrumentation-aws-lambda==0.48b0
 packaging
 jaraco.text
 platformdirs
+setuptools

lambda/tests/requirements.txt

@@ -1,2 +1,2 @@
-opentelemetry-instrumentation-aws-lambda == 0.46b0
-opentelemetry-propagator-aws-xray == 1.0.1
+opentelemetry-instrumentation-aws-lambda == 0.48b0
+opentelemetry-propagator-aws-xray == 1.0.2

lambda/tests/test_instrumentation.py

@@ -13,10 +13,11 @@
 import os
 import subprocess
 import sys
-from importlib import import_module
+from importlib import import_module, reload
 from shutil import which
 from unittest import mock
 
+from opentelemetry import propagate
 from opentelemetry.environment_variables import OTEL_PROPAGATORS
 from opentelemetry.instrumentation.aws_lambda import (
     _HANDLER,
@@ -182,10 +183,12 @@ def test_active_tracing(self):
             {
                 **os.environ,
                 # Using Active tracing
+                OTEL_PROPAGATORS: "xray-lambda",
                 _X_AMZN_TRACE_ID: MOCK_XRAY_TRACE_CONTEXT_SAMPLED,
             },
         )
         test_env_patch.start()
+        reload(propagate)
 
         AwsLambdaInstrumentor().instrument()
 
@@ -235,6 +238,7 @@ def test_parent_context_from_lambda_event(self):
             },
         )
         test_env_patch.start()
+        reload(propagate)
 
         mock_execute_lambda(
             {

pyproject.toml

@@ -34,18 +34,19 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Typing :: Typed",
     "Development Status :: 4 - Beta",
     "Intended Audience :: Developers",
     "Intended Audience :: Information Technology",
 ]
 requires-python = ">=3.8"
 dependencies = [
-    'opentelemetry-api == 1.25.0',
-    'opentelemetry-sdk == 1.25.0',
-    'opentelemetry-exporter-otlp == 1.25.0',
-    'opentelemetry-instrumentation == 0.46b0',
-    'opentelemetry-instrumentation-logging == 0.46b0',
+    'opentelemetry-api == 1.27.0',
+    'opentelemetry-sdk == 1.27.0',
+    'opentelemetry-exporter-otlp == 1.27.0',
+    'opentelemetry-instrumentation == 0.48b0',
+    'opentelemetry-instrumentation-logging == 0.48b0',
 ]
 
 [project.urls]

solarwinds_apm/distro.py

@@ -18,9 +18,6 @@
     OTEL_TRACES_EXPORTER,
 )
 from opentelemetry.instrumentation.distro import BaseDistro
-from opentelemetry.instrumentation.environment_variables import (
-    OTEL_PYTHON_DISABLED_INSTRUMENTATIONS,
-)
 from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
 from opentelemetry.instrumentation.logging.environment_variables import (
     OTEL_PYTHON_LOG_FORMAT,
@@ -142,17 +139,6 @@ def _configure(self, **kwargs):
         # Always opt into new semconv for all instrumentors (if supported)
         environ["OTEL_SEMCONV_STABILITY_OPT_IN"] = self.get_semconv_opt_in()
 
-        # TODO: Bootstrapping and auto-instrumentation ideally
-        # should not load instrumentor nor instrument AWS Lambda if not in lambda
-        if not SolarWindsApmConfig.calculate_is_lambda():
-            # If user has set OTEL_PYTHON_DISABLED_INSTRUMENTATIONS
-            # then they will need to add "aws-lambda" to the list
-            # else instrumentor Version Lookups for attributes may fail
-            environ.setdefault(
-                OTEL_PYTHON_DISABLED_INSTRUMENTATIONS,
-                "aws-lambda",
-            )
-
     def load_instrumentor(self, entry_point: EntryPoint, **kwargs):
         """Takes a collection of instrumentation entry points
         and activates them by instantiating and calling instrument()

solarwinds_apm/version.py

@@ -1 +1 @@
-__version__ = "2.2.1.5"
+__version__ = "2.2.1.6"

tests/docker/install/docker-compose.yml

@@ -229,3 +229,35 @@ services:
     << : [*command-install-test, *workdir, *volumes-codebase]
     environment:
       << : *envvars-install-test
+
+  #--------------------------------------------------------------------
+  # Python 3.12
+  #--------------------------------------------------------------------
+
+  py3.12-install-debian11:
+    hostname: "py3.12-debian11"
+    image: "python:3.12-bullseye"
+    << : [*command-install-test, *workdir, *volumes-codebase]
+    environment:
+      << : *envvars-install-test
+
+  py3.12-install-debian12:
+    hostname: "py3.12-debian12"
+    image: "python:3.12-bookworm"
+    << : [*command-install-test, *workdir, *volumes-codebase]
+    environment:
+      << : *envvars-install-test
+
+  py3.12-install-alpine3.19:
+    hostname: "py3.12-alpine3.19"
+    image: "python:3.12-alpine3.19"
+    << : [*command-install-test, *workdir, *volumes-codebase]
+    environment:
+      << : *envvars-install-test
+
+  py3.12-install-alpine3.20:
+    hostname: "py3.12-alpine3.20"
+    image: "python:3.12-alpine3.20"
+    << : [*command-install-test, *workdir, *volumes-codebase]
+    environment:
+      << : *envvars-install-test

tests/unit/test_distro.py

@@ -13,9 +13,6 @@
     OTEL_PROPAGATORS,
     OTEL_TRACES_EXPORTER
 )
-from opentelemetry.instrumentation.environment_variables import (
-    OTEL_PYTHON_DISABLED_INSTRUMENTATIONS,
-)
 from opentelemetry.sdk.environment_variables import (
     OTEL_EXPORTER_OTLP_LOGS_ENDPOINT,
     OTEL_EXPORTER_OTLP_LOGS_HEADERS,
@@ -300,15 +297,6 @@ def test_configure_env_propagators(self, mocker):
         assert os.environ[OTEL_TRACES_EXPORTER] == "solarwinds_exporter"
         assert os.environ.get("OTEL_SEMCONV_STABILITY_OPT_IN") == "http"
 
-    def test_configure_env_disabled_instrumentations_default(self):
-        distro.SolarWindsDistro()._configure()
-        assert os.environ[OTEL_PYTHON_DISABLED_INSTRUMENTATIONS] == "aws-lambda"
-
-    def test_configure_env_disabled_instrumentations_user_set(self, mocker):
-        mocker.patch.dict(os.environ, {"OTEL_PYTHON_DISABLED_INSTRUMENTATIONS": "foo-bar,only"})
-        distro.SolarWindsDistro()._configure()
-        assert os.environ[OTEL_PYTHON_DISABLED_INSTRUMENTATIONS] == "foo-bar,only"
-
     def test_load_instrumentor_no_commenting(self, mocker):
         mock_instrument = mocker.Mock()
         mock_instrumentor = mocker.Mock()