Opt-in config and logic for deferral of instrumentation to only WSGI worker processes

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py

@@ -65,6 +65,7 @@
 DEFAULT_METRIC_EXPORT_INTERVAL = 60000.0
 AWS_LAMBDA_FUNCTION_NAME_CONFIG = "AWS_LAMBDA_FUNCTION_NAME"
 AWS_XRAY_DAEMON_ADDRESS_CONFIG = "AWS_XRAY_DAEMON_ADDRESS"
+OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED_CONFIG = "OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED"
 
 _logger: Logger = getLogger(__name__)
 
@@ -85,6 +86,11 @@ class AwsOpenTelemetryConfigurator(_OTelSDKConfigurator):
     # pylint: disable=no-self-use
     @override
     def _configure(self, **kwargs):
+        if _is_defer_to_workers_enabled() and _is_wsgi_master_process():
+            _logger.info(
+                "Skipping ADOT initialization since deferral to worker is enabled, and this is a master process."
+            )
+            return
         _initialize_components()
 
 
@@ -156,6 +162,27 @@ def _init_tracing(
 # END The OpenTelemetry Authors code
 
 
+def _is_defer_to_workers_enabled():
+    return os.environ.get(OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED_CONFIG, "false").strip().lower() == "true"
+
+
+def _is_wsgi_master_process():
+    # Since the auto-instrumentation loads whenever a process is created and due to known issues with instrumenting
+    # WSGI apps using OTel, we want to skip the instrumentation of master process.
+    # This function is used to identify if the current process is a WSGI server's master process or not.
+    # Typically, a WSGI fork process model server spawns a single master process and multiple worker processes.
+    # When the master process starts, we use an environment variable as a marker. Since child worker processes inherit
+    # the master process environment, checking this marker in worker will tell that master process has been seen.
+    # Note: calling this function more than once in the same master process will return incorrect result.
+    # So use carefully.
+    if os.environ.get("IS_WSGI_MASTER_PROCESS_ALREADY_SEEN", "false").lower() == "true":
+        _logger.info("pid %s identified as a worker process", str(os.getpid()))
+        return False
+    os.environ["IS_WSGI_MASTER_PROCESS_ALREADY_SEEN"] = "true"
+    _logger.info("pid %s identified as a master process", str(os.getpid()))
+    return True
+
+
 def _exclude_urls_for_instrumentations():
     urls_to_exclude_instr = "SamplingTargets,GetSamplingRules"
     requests_excluded_urls = os.environ.pop("OTEL_PYTHON_REQUESTS_EXCLUDED_URLS", "")

aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/test_aws_opentelementry_configurator.py

@@ -16,6 +16,8 @@
     _customize_sampler,
     _customize_span_processors,
     _is_application_signals_enabled,
+    _is_defer_to_workers_enabled,
+    _is_wsgi_master_process,
 )
 from amazon.opentelemetry.distro.aws_opentelemetry_distro import AwsOpenTelemetryDistro
 from amazon.opentelemetry.distro.aws_span_metrics_processor import AwsSpanMetricsProcessor
@@ -305,6 +307,50 @@ def test_application_signals_exporter_provider(self):
         self.assertEqual("127.0.0.1:2000", exporter._udp_exporter._endpoint)
         os.environ.pop("AWS_LAMBDA_FUNCTION_NAME", None)
 
+    def test_is_defer_to_workers_enabled(self):
+        os.environ.setdefault("OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED", "True")
+        self.assertTrue(_is_defer_to_workers_enabled())
+        os.environ.pop("OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED", None)
+
+        os.environ.setdefault("OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED", "False")
+        self.assertFalse(_is_defer_to_workers_enabled())
+        os.environ.pop("OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED", None)
+        self.assertFalse(_is_defer_to_workers_enabled())
+
+    def test_is_wsgi_master_process_first_time(self):
+        self.assertTrue(_is_wsgi_master_process())
+        self.assertEqual(os.environ["IS_WSGI_MASTER_PROCESS_ALREADY_SEEN"], "true")
+        os.environ.pop("IS_WSGI_MASTER_PROCESS_ALREADY_SEEN", None)
+
+    @patch("amazon.opentelemetry.distro.aws_opentelemetry_configurator._initialize_components")
+    def test_initialize_components_skipped_in_master_when_deferred_enabled(self, mock_initialize_components):
+        os.environ.setdefault("OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED", "True")
+        os.environ.pop("IS_WSGI_MASTER_PROCESS_ALREADY_SEEN", None)
+        self.assertTrue(_is_defer_to_workers_enabled())
+        AwsOpenTelemetryConfigurator()._configure()
+        mock_initialize_components.assert_not_called()
+        os.environ.pop("OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED", None)
+        os.environ.pop("IS_WSGI_MASTER_PROCESS_ALREADY_SEEN", None)
+
+    @patch("amazon.opentelemetry.distro.aws_opentelemetry_configurator._initialize_components")
+    def test_initialize_components_called_in_worker_when_deferred_enabled(self, mock_initialize_components):
+        os.environ.setdefault("OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED", "True")
+        os.environ.setdefault("IS_WSGI_MASTER_PROCESS_ALREADY_SEEN", "true")
+        self.assertTrue(_is_defer_to_workers_enabled())
+        self.assertFalse(_is_wsgi_master_process())
+        AwsOpenTelemetryConfigurator()._configure()
+        mock_initialize_components.assert_called_once()
+        os.environ.pop("OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED", None)
+        os.environ.pop("IS_WSGI_MASTER_PROCESS_ALREADY_SEEN", None)
+
+    @patch("amazon.opentelemetry.distro.aws_opentelemetry_configurator._initialize_components")
+    def test_initialize_components_called_when_deferred_disabled(self, mock_initialize_components):
+        os.environ.pop("OTEL_AWS_PYTHON_DEFER_TO_WORKERS_ENABLED", None)
+        self.assertFalse(_is_defer_to_workers_enabled())
+        AwsOpenTelemetryConfigurator()._configure()
+        mock_initialize_components.assert_called_once()
+        os.environ.pop("IS_WSGI_MASTER_PROCESS_ALREADY_SEEN", None)
+
 
 def validate_distro_environ():
     tc: TestCase = TestCase()