NH-60984 Add Otel Python SDK-based histogram for response_time

solarwinds_apm/apm_meter_manager.py

@@ -0,0 +1,26 @@
+# © 2023 SolarWinds Worldwide, LLC. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at:http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+
+from opentelemetry import metrics
+
+logger = logging.getLogger(__name__)
+
+
+class SolarWindsMeterManager:
+    """SolarWinds Python OTLP Meter Manager"""
+
+    def __init__(self, **kwargs: int) -> None:
+        # Returns a named `Meter` to handle instrument creation.
+        # A convenience wrapper for MeterProvider.get_meter
+        self.meter = metrics.get_meter("sw.apm.sampling.metrics")
+
+        self.response_time = self.meter.create_histogram(
+            name="trace.service.response_time",
+            description="measures the duration of an inbound HTTP request",
+            unit="ms",
+        )

solarwinds_apm/apm_noop.py

@@ -188,3 +188,15 @@ class Config:
     @staticmethod
     def getVersionString():
         return "No extension loaded."
+
+
+class OtelHistogram:
+    @staticmethod
+    def record(*args, **kwargs):
+        pass
+
+
+class SolarWindsMeterManager:
+    def __init__(self, *args, **kwargs):
+        self.meter = None
+        self.response_time = OtelHistogram()

solarwinds_apm/configurator.py

@@ -45,12 +45,17 @@
     INTL_SWO_SUPPORT_EMAIL,
 )
 from solarwinds_apm.apm_fwkv_manager import SolarWindsFrameworkKvManager
+from solarwinds_apm.apm_meter_manager import SolarWindsMeterManager
 from solarwinds_apm.apm_noop import Reporter
+from solarwinds_apm.apm_noop import SolarWindsMeterManager as NoopMeterManager
 from solarwinds_apm.apm_oboe_codes import OboeReporterCode
 from solarwinds_apm.apm_txname_manager import SolarWindsTxnNameManager
 from solarwinds_apm.inbound_metrics_processor import (
     SolarWindsInboundMetricsSpanProcessor,
 )
+from solarwinds_apm.otlp_metrics_processor import (
+    SolarWindsOTLPMetricsSpanProcessor,
+)
 from solarwinds_apm.response_propagator import (
     SolarWindsTraceResponsePropagator,
 )
@@ -74,9 +79,22 @@ def _configure(self, **kwargs: int) -> None:
         apm_txname_manager = SolarWindsTxnNameManager()
         apm_fwkv_manager = SolarWindsFrameworkKvManager()
         apm_config = SolarWindsApmConfig()
+
+        if not apm_config.get("experimental").get("otel_collector") is True:
+            logger.debug(
+                "Experimental otel_collector flag not configured. Creating meter manager as no-op."
+            )
+            apm_meters = NoopMeterManager()
+        else:
+            apm_meters = SolarWindsMeterManager()
+
         reporter = self._initialize_solarwinds_reporter(apm_config)
         self._configure_otel_components(
-            apm_txname_manager, apm_fwkv_manager, apm_config, reporter
+            apm_txname_manager,
+            apm_fwkv_manager,
+            apm_config,
+            reporter,
+            apm_meters,
         )
         # Report an status event after everything is done.
         self._report_init_event(reporter, apm_config)
@@ -87,6 +105,7 @@ def _configure_otel_components(
         apm_fwkv_manager: SolarWindsFrameworkKvManager,
         apm_config: SolarWindsApmConfig,
         reporter: "Reporter",
+        apm_meters: SolarWindsMeterManager,
     ) -> None:
         """Configure OTel sampler, exporter, propagator, response propagator"""
         self._configure_sampler(apm_config)
@@ -95,6 +114,11 @@ def _configure_otel_components(
                 apm_txname_manager,
                 apm_config,
             )
+            self._configure_otlp_metrics_span_processor(
+                apm_txname_manager,
+                apm_config,
+                apm_meters,
+            )
             self._configure_exporter(
                 reporter,
                 apm_txname_manager,
@@ -154,6 +178,27 @@ def _configure_metrics_span_processor(
             )
         )
 
+    def _configure_otlp_metrics_span_processor(
+        self,
+        apm_txname_manager: SolarWindsTxnNameManager,
+        apm_config: SolarWindsApmConfig,
+        apm_meters: SolarWindsMeterManager,
+    ) -> None:
+        """Configure SolarWindsOTLPMetricsSpanProcessor"""
+        if not apm_config.get("experimental").get("otel_collector") is True:
+            logger.debug(
+                "Experimental otel_collector flag not configured. Not configuring OTLP metrics span processor."
+            )
+            return
+
+        trace.get_tracer_provider().add_span_processor(
+            SolarWindsOTLPMetricsSpanProcessor(
+                apm_txname_manager,
+                apm_config,
+                apm_meters,
+            )
+        )
+
     def _configure_exporter(
         self,
         reporter: "Reporter",
@@ -269,10 +314,19 @@ def _configure_metrics_exporter(
             reader = PeriodicExportingMetricReader(exporter)
             metric_readers.append(reader)
 
-        # Consolidate in #201
-        resource = Resource.create({"service.name": apm_config.service_name})
+        # Use configured Resource attributes then merge with
+        # custom service.name and sw.trace_span_mode
+        resource = trace.get_tracer_provider().get_tracer(__name__).resource
+        sw_resource = Resource.create(
+            {
+                "sw.trace_span_mode": "otel",
+                "service.name": apm_config.service_name,
+            }
+        ).merge(resource)
+
         provider = MeterProvider(
-            resource=resource, metric_readers=metric_readers
+            resource=sw_resource,
+            metric_readers=metric_readers,
         )
         metrics.set_meter_provider(provider)
 

solarwinds_apm/otlp_metrics_processor.py

@@ -0,0 +1,190 @@
+# © 2023 SolarWinds Worldwide, LLC. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at:http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+import random
+from typing import TYPE_CHECKING, Any, Tuple
+
+from opentelemetry.metrics import get_meter_provider
+from opentelemetry.sdk.trace import SpanProcessor
+from opentelemetry.semconv.trace import SpanAttributes
+from opentelemetry.trace import SpanKind, StatusCode
+
+from solarwinds_apm.w3c_transformer import W3CTransformer
+
+if TYPE_CHECKING:
+    from opentelemetry.sdk.trace import ReadableSpan
+
+    from solarwinds_apm.apm_config import SolarWindsApmConfig
+    from solarwinds_apm.apm_meter_manager import SolarWindsMeterManager
+    from solarwinds_apm.apm_txname_manager import SolarWindsTxnNameManager
+
+
+logger = logging.getLogger(__name__)
+
+
+class SolarWindsOTLPMetricsSpanProcessor(SpanProcessor):
+    # TODO Refactor for both inbound and otlp metrics
+    #      https://swicloud.atlassian.net/browse/NH-65061
+    _HTTP_METHOD = SpanAttributes.HTTP_METHOD  # "http.method"
+    _HTTP_ROUTE = SpanAttributes.HTTP_ROUTE  # "http.route"
+    _HTTP_STATUS_CODE = SpanAttributes.HTTP_STATUS_CODE  # "http.status_code"
+    _HTTP_URL = SpanAttributes.HTTP_URL  # "http.url"
+
+    _HTTP_SPAN_STATUS_UNAVAILABLE = 0
+
+    def __init__(
+        self,
+        apm_txname_manager: "SolarWindsTxnNameManager",
+        apm_config: "SolarWindsApmConfig",
+        apm_meters: "SolarWindsMeterManager",
+    ) -> None:
+        self.apm_txname_manager = apm_txname_manager
+        self.service_name = apm_config.service_name
+        self.apm_meters = apm_meters
+
+    # TODO Assumes SolarWindsInboundMetricsSpanProcessor.on_start
+    #      is called to store span ID for any custom txn naming
+    #      https://swicloud.atlassian.net/browse/NH-65061
+
+    def on_end(self, span: "ReadableSpan") -> None:
+        """Calculates and reports OTLP trace metrics"""
+        # Only calculate OTLP metrics for service entry spans
+        parent_span_context = span.parent
+        if (
+            parent_span_context
+            and parent_span_context.is_valid
+            and not parent_span_context.is_remote
+        ):
+            return
+
+        # support ssa and conform to Otel proto common_pb2
+        meter_attrs = {
+            "sw.service_name": self.service_name,
+            "sw.nonce": random.getrandbits(64) >> 1,
+        }
+
+        has_error = self.has_error(span)
+        if has_error:
+            meter_attrs.update({"sw.is_error": "true"})
+        else:
+            meter_attrs.update({"sw.is_error": "false"})
+
+        # trans_name will never be None because always at least span.name
+        trans_name, _ = self.calculate_transaction_names(span)
+
+        is_span_http = self.is_span_http(span)
+        span_time = self.calculate_span_time(
+            span.start_time,
+            span.end_time,
+        )
+
+        if is_span_http:
+            status_code = self.get_http_status_code(span)
+            request_method = span.attributes.get(self._HTTP_METHOD, None)
+            meter_attrs.update(
+                {
+                    self._HTTP_STATUS_CODE: status_code,
+                    self._HTTP_METHOD: request_method,
+                    "sw.transaction": trans_name,
+                }
+            )
+        else:
+            meter_attrs.update({"sw.transaction": trans_name})
+        self.apm_meters.response_time.record(
+            amount=span_time,
+            attributes=meter_attrs,
+        )
+
+        # This does not cache txn_name for span export because
+        # assuming SolarWindsInboundMetricsSpanProcessor does it
+        # for SW-style trace export. This processor is for OTLP-style.
+        # TODO: Cache txn_name for OTLP span export?
+        #       https://swicloud.atlassian.net/browse/NH-65061
+
+        # Force flush metrics after every entry span via flush of all meters
+        # including PeriodicExportingMetricReader
+        logger.debug("Performing MeterProvider force_flush of metrics")
+        get_meter_provider().force_flush()
+
+    # TODO Refactor for both inbound and otlp metrics
+    #      https://swicloud.atlassian.net/browse/NH-65061
+    def is_span_http(self, span: "ReadableSpan") -> bool:
+        """This span from inbound HTTP request if from a SERVER by some http.method"""
+        if span.kind == SpanKind.SERVER and span.attributes.get(
+            self._HTTP_METHOD, None
+        ):
+            return True
+        return False
+
+    # TODO Refactor for both inbound and otlp metrics
+    #      https://swicloud.atlassian.net/browse/NH-65061
+    def has_error(self, span: "ReadableSpan") -> bool:
+        """Calculate if this span instance has_error"""
+        if span.status.status_code == StatusCode.ERROR:
+            return True
+        return False
+
+    # TODO Refactor for both inbound and otlp metrics
+    #      https://swicloud.atlassian.net/browse/NH-65061
+    def get_http_status_code(self, span: "ReadableSpan") -> int:
+        """Calculate HTTP status_code from span or default to UNAVAILABLE"""
+        status_code = span.attributes.get(self._HTTP_STATUS_CODE, None)
+        # Something went wrong in OTel or instrumented service crashed early
+        # if no status_code in attributes of HTTP span
+        if not status_code:
+            # TODO change if refactor
+            status_code = self._HTTP_SPAN_STATUS_UNAVAILABLE
+        return status_code
+
+    # TODO Refactor for both inbound and otlp metrics
+    #      https://swicloud.atlassian.net/browse/NH-65061
+    # Disable pylint for compatibility with Python3.7 else TypeError
+    def calculate_transaction_names(
+        self, span: "ReadableSpan"
+    ) -> Tuple[Any, Any]:  # pylint: disable=deprecated-typing-alias
+        """Get trans_name and url_tran of this span instance."""
+        url_tran = span.attributes.get(self._HTTP_URL, None)
+        http_route = span.attributes.get(self._HTTP_ROUTE, None)
+        trans_name = None
+        custom_trans_name = self.calculate_custom_transaction_name(span)
+
+        if custom_trans_name:
+            trans_name = custom_trans_name
+        elif http_route:
+            trans_name = http_route
+        elif span.name:
+            trans_name = span.name
+        return trans_name, url_tran
+
+    # TODO Refactor for both inbound and otlp metrics
+    #      https://swicloud.atlassian.net/browse/NH-65061
+    def calculate_custom_transaction_name(self, span: "ReadableSpan") -> Any:
+        """Get custom transaction name for trace by trace_id, if any"""
+        trans_name = None
+        trace_span_id = W3CTransformer.trace_and_span_id_from_context(
+            span.context
+        )
+        custom_name = self.apm_txname_manager.get(trace_span_id)
+        if custom_name:
+            trans_name = custom_name
+            # TODO change if refactor
+            # Assume SolarWindsInboundMetricsSpanProcessor is
+            # active and is doing the removal of any
+            # custom txn name from cache if not sampled,
+            # so not doing it here for OTLP
+        return trans_name
+
+    def calculate_span_time(
+        self,
+        start_time: int,
+        end_time: int,
+    ) -> int:
+        """Calculate span time in milliseconds (ms) using start and end time
+        in nanoseconds (ns). OTel span start/end_time are optional."""
+        if not start_time or not end_time:
+            return 0
+        return int((end_time - start_time) // 1e6)

solarwinds_apm/version.py

@@ -5,4 +5,4 @@
 # Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
 
 """Version of SolarWinds Custom-Distro for OpenTelemetry agents"""
-__version__ = "0.18.0.1"
+__version__ = "0.18.0.6"