feat: add config to influence otel context propagation

- by default, server will not extract trace context from incoming
  Flight RPC headers ==> all spans originating from the server will
  create new trace ID
- enabling the new option, the code will do otelpropagate.extract(headers)
  to obtain the context from it and use it when creating the main
  request handling span ==> all spans originating from the server
  will use trace ID coming over the wire ==> things can be tracked
  e2e

JIRA: CQ-670

gooddata-flight-server/gooddata_flight_server/config/config.py

@@ -38,6 +38,7 @@ class OtelConfig:
     service_name: str
     service_namespace: Optional[str]
     service_instance_id: Optional[str]
+    extract_context_from_headers: bool
 
 
 @dataclass(frozen=True)
@@ -122,6 +123,7 @@ class _Settings:
     OtelServiceName = "otel_service_name"
     OtelServiceNamespace = "otel_service_namespace"
     OtelServiceInstanceId = "otel_service_instance_id"
+    OtelExtractContext = "otel_extract_context"
 
 
 _LOCALHOST = "127.0.0.1"
@@ -408,6 +410,15 @@ def _validate_boolean(val: Any) -> bool:
             "condition": f"{_Settings.OtelServiceInstanceId} must be a non-empty string.",
         },
     ),
+    Validator(
+        _fqsn(_Settings.OtelExtractContext),
+        cast=bool,
+        default=False,
+        condition=_validate_boolean,
+        messages={
+            "condition": f"{_Settings.OtelExtractContext} must be a boolean value.",
+        },
+    ),
 ]
 
 
@@ -491,6 +502,7 @@ def _create_server_config(settings: Dynaconf) -> ServerConfig:
             service_name=server_settings.get(_Settings.OtelServiceName),
             service_namespace=server_settings.get(_Settings.OtelServiceNamespace),
             service_instance_id=server_settings.get(_Settings.OtelServiceInstanceId),
+            extract_context_from_headers=server_settings.get(_Settings.OtelExtractContext),
         ),
     )
 

gooddata-flight-server/gooddata_flight_server/server/flight_rpc/flight_middleware.py

@@ -96,11 +96,17 @@ def call_completed(self, exception: Optional[pyarrow.lib.ArrowException]) -> Non
 class OtelMiddleware(pyarrow.flight.ServerMiddleware):
     MiddlewareName = "otel_middleware"
 
-    def __init__(self, info: pyarrow.flight.CallInfo, headers: dict[str, list[str]]) -> None:
+    def __init__(
+        self, info: pyarrow.flight.CallInfo, headers: dict[str, list[str]], extract_context: bool = False
+    ) -> None:
         super().__init__()
         method_name = info.method.name
 
-        self._otel_ctx = otelpropagate.extract(headers)
+        if extract_context:
+            self._otel_ctx = otelpropagate.extract(headers)
+        else:
+            self._otel_ctx = otelctx.get_current()
+
         self._otel_span = SERVER_TRACER.start_span(
             f"{method_name}",
             kind=SpanKind.SERVER,

gooddata-flight-server/gooddata_flight_server/server/flight_rpc/flight_service.py

@@ -70,10 +70,14 @@ def start_call(
 
 
 class _OtelMiddlewareFactory(pyarrow.flight.ServerMiddlewareFactory):
+    def __init__(self, extract_context: bool) -> None:
+        super().__init__()
+        self._extract_context = extract_context
+
     def start_call(
         self, info: pyarrow.flight.CallInfo, headers: dict[str, list[str]]
     ) -> Optional[pyarrow.flight.ServerMiddleware]:
-        return OtelMiddleware(info, headers)
+        return OtelMiddleware(info, headers, self._extract_context)
 
 
 class FlightRpcService:
@@ -141,7 +145,9 @@ def _initialize_otel_tracing(
         if self._config.otel_config.exporter_type is None:
             return None
 
-        return OtelMiddleware.MiddlewareName, _OtelMiddlewareFactory()
+        return OtelMiddleware.MiddlewareName, _OtelMiddlewareFactory(
+            self._config.otel_config.extract_context_from_headers
+        )
 
     def start(self, ctx: ServerContext) -> None:
         """

gooddata-flight-server/sample-config.toml

@@ -259,6 +259,27 @@ otel_service_namespace = "your-namespace"
 # Default is to use current hostname.
 otel_service_instance_id = "your-service-instance-id"
 
+# optionally specify whether OpenTelemetry integration within the server should look
+# for, extract and use an OpenTelemetry context coming through the Flight RPC
+# request headers.
+#
+# When this option is enabled, the code will use OpenTelemetry's context
+# propagation `extract` method to obtain the context from Flight RPC request headers. This
+# context will then be used when creating OpenTelemetry span representing the Flight
+# RPC call.
+#
+# In simple words:
+#
+# - IF your server is configured to export traces to same backend where your server's clients
+#   also export their traces AND the clients inject the trace context into the Flight RPC headers AND
+#   you turn this option on, THEN you will be able to correlate all calls under one trace ID.
+#
+# - ELSE, you should keep this turned off; each Flight RPC request to your server will create
+#   a new trace ID.
+#
+# Default is false.
+otel_extract_context = false
+
 [flexfun]
 
 # specify one or more modules that contain FlexFun implementations

gooddata-flight-server/tests/config/sample-config.toml

@@ -31,3 +31,4 @@ otel_exporter_type = "otlp-grpc"
 otel_service_name = "your-service-name"
 otel_service_namespace = "your-namespace"
 otel_service_instance_id = "your-service-instance-id"
+otel_extract_context = true

gooddata-flight-server/tests/config/test_config.py

@@ -31,6 +31,7 @@ def test_read_valid():
     assert server_config.otel_config.service_name == "your-service-name"
     assert server_config.otel_config.service_namespace == "your-namespace"
     assert server_config.otel_config.service_instance_id == "your-service-instance-id"
+    assert server_config.otel_config.extract_context_from_headers is True
 
 
 def test_read_empty():
@@ -49,6 +50,7 @@ def test_read_empty():
     assert server_config.otel_config.service_name is None
     assert server_config.otel_config.service_namespace is None
     assert server_config.otel_config.service_instance_id is None
+    assert server_config.otel_config.extract_context_from_headers is False
 
     assert server_config.authentication_method == AuthenticationMethod.NoAuth
     assert server_config.token_header_name is None