Merge pull request #439 from solarwinds/NH-88274-sw-export-metrics-en…

…abled

NH-88274 Add support for SW_APM_EXPORT_METRICS_ENABLED

lambda/solarwinds-apm/wrapper

@@ -27,12 +27,23 @@ export LAMBDA_LAYER_PKGS_DIR="/opt/python";
 export PYTHONPATH="$LAMBDA_LAYER_PKGS_DIR:$PYTHONPATH";
 export PYTHONPATH="$LAMBDA_RUNTIME_DIR:$PYTHONPATH";
 
+# Default opt into OTLP metrics export in lambda
+if [ -z "${SW_APM_EXPORT_METRICS_ENABLED}" ]; then
+    export SW_APM_EXPORT_METRICS_ENABLED=true
+fi
+
 # Default OTEL_EXPORTER_OTLP_PROTOCOL for APM Python in lambda
 # for all of logs/trace/metrics is HTTP through otelcollector
 if [ -z "${OTEL_EXPORTER_OTLP_PROTOCOL}" ]; then
     export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
 fi
-# Set default for LOGS specifically because of regular default
+# Set default endpoints for traces, metrics, and logs else would use regular SWO defaults
+if [ -z "${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT}" ]; then
+    export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://0.0.0.0:4318/v1/traces
+fi
+if [ -z "${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT}" ]; then
+    export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://0.0.0.0:4318/v1/metrics
+fi
 if [ -z "${OTEL_EXPORTER_OTLP_LOGS_ENDPOINT}" ]; then
     export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://0.0.0.0:4318/v1/logs
 fi

solarwinds_apm/apm_config.py

@@ -112,6 +112,7 @@ def __init__(
             "transaction_filters": [],
             "transaction_name": None,
             "export_logs_enabled": False,
+            "export_metrics_enabled": False,
         }
         self.is_lambda = self.calculate_is_lambda()
         self.lambda_function_name = os.environ.get("AWS_LAMBDA_FUNCTION_NAME")
@@ -169,6 +170,9 @@ def __init__(
         self.metric_format = self._calculate_metric_format()
         self.certificates = self._calculate_certificates()
         self.__config["export_logs_enabled"] = self._calculate_logs_enabled()
+        self.__config["export_metrics_enabled"] = (
+            self._calculate_metrics_enabled()
+        )
 
         logger.debug("Set ApmConfig as: %s", self)
 
@@ -629,6 +633,21 @@ def _calculate_logs_enabled(self) -> bool:
                 return False
         return self.get("export_logs_enabled")
 
+    def _calculate_metrics_enabled(self) -> bool:
+        """Return if export of metrics telemetry enabled, based on collector.
+        Always False if AO collector, else use current config."""
+        host = self.get("collector")
+        if host:
+            if (
+                INTL_SWO_AO_COLLECTOR in host
+                or INTL_SWO_AO_STG_COLLECTOR in host
+            ):
+                logger.warning(
+                    "AO collector detected. Defaulting to disabled OTLP metrics export."
+                )
+                return False
+        return self.get("export_metrics_enabled")
+
     def mask_service_key(self) -> str:
         """Return masked service key except first 4 and last 4 chars"""
         service_key = self.__config.get("service_key")
@@ -918,7 +937,7 @@ def _set_config_value(self, keys_str: str, val: Any) -> Any:
                 self.__config[key] = val
             elif keys == ["transaction_name"]:
                 self.__config[key] = val
-            elif keys == ["export_logs_enabled"]:
+            elif keys in [["export_logs_enabled"], ["export_metrics_enabled"]]:
                 val = self.convert_to_bool(val)
                 if val not in (True, False):
                     raise ValueError

solarwinds_apm/apm_constants.py

@@ -4,6 +4,9 @@
 #
 # Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
 
+INTL_SWO_DEFAULT_OTLP_COLLECTOR = (
+    "https://otel.collector.na-01.cloud.solarwinds.com:443"
+)
 INTL_SWO_AO_COLLECTOR = "collector.appoptics.com"
 INTL_SWO_AO_STG_COLLECTOR = "collector-stg.appoptics.com"
 INTL_SWO_CURRENT_TRACE_ENTRY_SPAN_ID = "sw-current-trace-entry-span-id"

solarwinds_apm/configurator.py

@@ -313,13 +313,15 @@ def _configure_traces_exporter(
         apm_fwkv_manager: SolarWindsFrameworkKvManager,
         apm_config: SolarWindsApmConfig,
     ) -> None:
-        """Configure SolarWinds OTel span exporters, defaults or environment
-        configured, or none if agent disabled.
+        """Configure traces exporters if agent enabled.
+        Links to global TracerProvider.
 
         Initialization of SolarWinds exporter requires a liboboe reporter
-        Note: if reporter is no-op, the SW exporter will not export spans."""
+        If `reporter` is no-op, the SW exporter will not export spans."""
         if not apm_config.agent_enabled:
-            logger.error("Tracing disabled. Cannot set trace exporter.")
+            logger.error(
+                "APM Python library disabled. Cannot set traces exporters."
+            )
             return
 
         # SolarWindsDistro._configure does setdefault before this is called
@@ -384,8 +386,21 @@ def _configure_metrics_exporter(
         self,
         apm_config: SolarWindsApmConfig,
     ) -> None:
-        """Configure SolarWinds OTel metrics exporters if any configured.
-        Links them to new metric readers and global MeterProvider."""
+        """Configure OTel OTLP metrics exporters if enabled and agent enabled.
+        Settings precedence: OTEL_* > SW_APM_EXPORT_METRICS_ENABLED.
+        Links to new metric readers and global MeterProvider."""
+        if not apm_config.agent_enabled:
+            logger.error(
+                "APM Python library disabled. Cannot set metrics exporters."
+            )
+            return
+
+        if not apm_config.get("export_metrics_enabled"):
+            logger.debug(
+                "APM OTLP metrics export disabled. Skipping init of metrics exporters"
+            )
+            return
+
         # SolarWindsDistro._configure does setdefault before this is called
         environ_exporter = os.environ.get(
             OTEL_METRICS_EXPORTER,
@@ -422,11 +437,19 @@ def _configure_metrics_exporter(
                 "Creating PeriodicExportingMetricReader using %s",
                 exporter_name,
             )
-            # Inf interval to not invoke periodic collection
-            reader = PeriodicExportingMetricReader(
-                exporter,
-                export_interval_millis=math.inf,
-            )
+
+            reader = None
+            if apm_config.is_lambda:
+                # Inf interval to not invoke periodic collection
+                reader = PeriodicExportingMetricReader(
+                    exporter,
+                    export_interval_millis=math.inf,
+                )
+            else:
+                # Use default interval 60s else OTEL_METRIC_EXPORT_INTERVAL
+                reader = PeriodicExportingMetricReader(
+                    exporter,
+                )
             metric_readers.append(reader)
 
         # Use configured Resource attributes then merge with
@@ -450,9 +473,15 @@ def _configure_logs_exporter(
         self,
         apm_config: SolarWindsApmConfig,
     ) -> None:
-        """Configure OTel OTLP logs exporter if enabled.
+        """Configure OTel OTLP logs exporters if enabled and agent enabled.
         Settings precedence: OTEL_* > SW_APM_EXPORT_LOGS_ENABLED.
         Links to new global LoggerProvider."""
+        if not apm_config.agent_enabled:
+            logger.error(
+                "APM Python library disabled. Cannot set logs exporters."
+            )
+            return
+
         otel_ev = os.environ.get(
             _OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED
         )
@@ -469,7 +498,7 @@ def _configure_logs_exporter(
         # then sw_enabled determines logs export setup.
         if not otlp_log_enabled and sw_enabled is False:
             logger.debug(
-                "APM logs exports disabled. Skipping init of logs exporters"
+                "APM OTLP logs export disabled. Skipping init of logs exporters"
             )
             return
 

solarwinds_apm/distro.py

@@ -10,6 +10,7 @@
 import platform
 import sys
 from os import environ
+from typing import Any
 
 from opentelemetry.environment_variables import (
     OTEL_LOGS_EXPORTER,
@@ -27,13 +28,20 @@
     OTEL_EXPORTER_OTLP_LOGS_ENDPOINT,
     OTEL_EXPORTER_OTLP_LOGS_HEADERS,
     OTEL_EXPORTER_OTLP_LOGS_PROTOCOL,
+    OTEL_EXPORTER_OTLP_METRICS_ENDPOINT,
+    OTEL_EXPORTER_OTLP_METRICS_HEADERS,
+    OTEL_EXPORTER_OTLP_METRICS_PROTOCOL,
     OTEL_EXPORTER_OTLP_PROTOCOL,
+    OTEL_EXPORTER_OTLP_TRACES_ENDPOINT,
+    OTEL_EXPORTER_OTLP_TRACES_HEADERS,
+    OTEL_EXPORTER_OTLP_TRACES_PROTOCOL,
 )
 from opentelemetry.sdk.version import __version__ as sdk_version
 from pkg_resources import EntryPoint
 
 from solarwinds_apm.apm_config import SolarWindsApmConfig
 from solarwinds_apm.apm_constants import (
+    INTL_SWO_DEFAULT_OTLP_COLLECTOR,
     INTL_SWO_DEFAULT_OTLP_EXPORTER,
     INTL_SWO_DEFAULT_OTLP_EXPORTER_GRPC,
     INTL_SWO_DEFAULT_PROPAGATORS,
@@ -91,53 +99,120 @@ def _get_token_from_service_key(self):
             return None
         return key_parts[0]
 
-    def _configure(self, **kwargs):
-        """Configure default OTel exporter and propagators"""
-        self._log_runtime()
-
-        # Set defaults for OTLP logs export by HTTP to SWO
-        environ.setdefault(OTEL_EXPORTER_OTLP_LOGS_PROTOCOL, "http/protobuf")
-        environ.setdefault(
-            OTEL_LOGS_EXPORTER, _EXPORTER_BY_OTLP_PROTOCOL["http/protobuf"]
-        )
-        environ.setdefault(
-            OTEL_EXPORTER_OTLP_LOGS_ENDPOINT,
-            "https://otel.collector.na-01.cloud.solarwinds.com:443/v1/logs",
-        )
-        if not SolarWindsApmConfig.calculate_is_lambda():
-            header_token = self._get_token_from_service_key()
-            if not header_token:
-                logger.debug("Setting OTLP logging defaults without SWO token")
+    def _configure_logs_export_env_defaults(
+        self,
+        header_token: str,
+        otlp_protocol: str,
+    ) -> None:
+        """Configure env defaults for OTLP logs signal export by HTTP or gRPC to SWO"""
+        if otlp_protocol in _EXPORTER_BY_OTLP_PROTOCOL:
+            environ.setdefault(OTEL_EXPORTER_OTLP_LOGS_PROTOCOL, otlp_protocol)
             environ.setdefault(
-                OTEL_EXPORTER_OTLP_LOGS_HEADERS,
-                f"authorization=Bearer%20{header_token}",
+                OTEL_LOGS_EXPORTER, _EXPORTER_BY_OTLP_PROTOCOL[otlp_protocol]
             )
+            environ.setdefault(
+                OTEL_EXPORTER_OTLP_LOGS_ENDPOINT,
+                f"{INTL_SWO_DEFAULT_OTLP_COLLECTOR}/v1/logs",
+            )
+            if header_token:
+                environ.setdefault(
+                    OTEL_EXPORTER_OTLP_LOGS_HEADERS,
+                    f"authorization=Bearer%20{header_token}",
+                )
         else:
-            logger.debug("Skipping logs_headers defaults in lambda.")
+            logger.debug(
+                "Tried to setdefault for OTLP logs with invalid protocol. Skipping."
+            )
 
-        otlp_protocol = environ.get(OTEL_EXPORTER_OTLP_PROTOCOL)
+    def _configure_metrics_export_env_defaults(
+        self,
+        header_token: str,
+        otlp_protocol: str,
+    ) -> None:
+        """Configure env defaults for OTLP metrics signal export by HTTP or gRPC to SWO"""
         if otlp_protocol in _EXPORTER_BY_OTLP_PROTOCOL:
-            # If users set OTEL_EXPORTER_OTLP_PROTOCOL
-            # as one of Otel SDK's `http/protobuf` or `grpc`,
-            # then the matching exporters are mapped by default
+            environ.setdefault(
+                OTEL_EXPORTER_OTLP_METRICS_PROTOCOL, otlp_protocol
+            )
             environ.setdefault(
                 OTEL_METRICS_EXPORTER,
                 _EXPORTER_BY_OTLP_PROTOCOL[otlp_protocol],
             )
+            environ.setdefault(
+                OTEL_EXPORTER_OTLP_METRICS_ENDPOINT,
+                f"{INTL_SWO_DEFAULT_OTLP_COLLECTOR}/v1/metrics",
+            )
+            if header_token:
+                environ.setdefault(
+                    OTEL_EXPORTER_OTLP_METRICS_HEADERS,
+                    f"authorization=Bearer%20{header_token}",
+                )
+        else:
+            logger.debug(
+                "Tried to setdefault for OTLP metrics with invalid protocol. Skipping."
+            )
+
+    def _configure_traces_export_env_defaults(
+        self,
+        header_token: str,
+        otlp_protocol: Any = None,
+    ) -> None:
+        """Configure env defaults for OTLP traces signal export by APM protocol
+        to SWO, else follow provided OTLP protocol (HTTP or gRPC)"""
+        if otlp_protocol in _EXPORTER_BY_OTLP_PROTOCOL:
+            environ.setdefault(
+                OTEL_EXPORTER_OTLP_TRACES_PROTOCOL, otlp_protocol
+            )
             environ.setdefault(
                 OTEL_TRACES_EXPORTER, _EXPORTER_BY_OTLP_PROTOCOL[otlp_protocol]
             )
+            environ.setdefault(
+                OTEL_EXPORTER_OTLP_TRACES_ENDPOINT,
+                f"{INTL_SWO_DEFAULT_OTLP_COLLECTOR}/v1/traces",
+            )
+            if header_token:
+                environ.setdefault(
+                    OTEL_EXPORTER_OTLP_TRACES_HEADERS,
+                    f"authorization=Bearer%20{header_token}",
+                )
         else:
-            # Else users need to specify OTEL_METRICS_EXPORTER.
-            # Otherwise, no metrics will generated and no metrics exporter
-            # will be initialized.
+            logger.debug(
+                "Called to setdefault for OTLP traces with empty or invalid protocol. Defaulting to SolarWinds exporter."
+            )
             environ.setdefault(
                 OTEL_TRACES_EXPORTER, INTL_SWO_DEFAULT_TRACES_EXPORTER
             )
 
+    def _configure(self, **kwargs):
+        """Configure default OTel exporters and propagators"""
+        self._log_runtime()
+
+        header_token = None
+        if not SolarWindsApmConfig.calculate_is_lambda():
+            header_token = self._get_token_from_service_key()
+            if not header_token:
+                logger.debug("Setting OTLP export defaults without SWO token")
+        else:
+            logger.debug("Skipping OTLP export headers setdefaults in lambda.")
+
+        # If users set OTEL_EXPORTER_OTLP_PROTOCOL
+        # as one of Otel SDK's `http/protobuf` or `grpc`,
+        # then the matching exporters are mapped
+        otlp_protocol = environ.get(OTEL_EXPORTER_OTLP_PROTOCOL)
+        # For traces, the default is SWO APM - see helper
+        self._configure_traces_export_env_defaults(header_token, otlp_protocol)
+        # For metrics and logs, the default is `http/protobuf`
+        if otlp_protocol not in _EXPORTER_BY_OTLP_PROTOCOL:
+            otlp_protocol = "http/protobuf"
+        self._configure_logs_export_env_defaults(header_token, otlp_protocol)
+        self._configure_metrics_export_env_defaults(
+            header_token, otlp_protocol
+        )
+
         environ.setdefault(
             OTEL_PROPAGATORS, ",".join(INTL_SWO_DEFAULT_PROPAGATORS)
         )
+        # Default for LoggingInstrumentor
         environ.setdefault(
             OTEL_PYTHON_LOG_FORMAT,
             "%(asctime)s %(levelname)s [%(name)s] [%(filename)s:%(lineno)d] [trace_id=%(otelTraceID)s span_id=%(otelSpanID)s trace_flags=%(otelTraceSampled)02d resource.service.name=%(otelServiceName)s] - %(message)s",

solarwinds_apm/version.py

@@ -1 +1 @@
-__version__ = "3.1.0"
+__version__ = "3.2.0.1"