NH-11358 Add integration tests

CONTRIBUTING.md

@@ -72,9 +72,13 @@ For more information, run `make` inside the build container.
 
 #### Unit tests
 
-Automated unit testing of this repo uses [tox](https://tox.readthedocs.io) and runs in Python 3.7, 3.8, 3.9, and/or 3.10 because these are the versions supported by [OTel Python](https://github.com/open-telemetry/opentelemetry-python/blob/main/tox.ini). Tests for each Python version can be run against AO prod or NH staging.
+Automated unit testing of this repo uses [tox](https://tox.readthedocs.io) and runs in Python 3.7, 3.8, 3.9, and/or 3.10 because these are the versions supported by [OTel Python](https://github.com/open-telemetry/opentelemetry-python/blob/main/tox.ini). Tests for each Python version can be run against AO prod or NH staging. The unit tests are defined in `tests/unit/`.
 
-The functional tests require a compiled C-extension and should be run inside the build container. Here is how to run tests locally:
+Both unit tests an integration tests are run together. See next section for how to run.
+
+#### Integration tests
+
+The integration tests are defined in `tests/integration/`. They require a compiled C-extension and should be run inside the build container. They are also run under tox as per the unit tests. Here is how to run tests locally:
 
 1. For running tox against both NH staging and AO prod, set two service keys:
   - `SW_APM_SERVICE_KEY_TOX_NH_STAGING`, as `<API_TOKEN>:solarwinds-apm-python-tox-test`
@@ -84,11 +88,8 @@ The functional tests require a compiled C-extension and should be run inside the
 4. To run all tests for a specific version, provide tox options as a string. For example, to run in Python 3.7 against AO prod: `make tox OPTIONS="-e py37-ao-prod"`.
 5. (WARNING: slow!) To run all tests for all supported Python environments: `make tox`
 
-The unit tests are also run on GitHub with the [Run tox tests](https://github.com/appoptics/solarwinds-apm-python/actions/workflows/run_tox_tests.yaml) workflow.
+The unit and integration tests are also run on GitHub with the [Run tox tests](https://github.com/appoptics/solarwinds-apm-python/actions/workflows/run_tox_tests.yaml) workflow.
 
-#### Integration tests
-
-TODO
 
 ### Install tests
 

dev-requirements.txt

@@ -1,5 +1,9 @@
-opentelemetry-test-utils==0.35b0
+opentelemetry-test-utils~=0.35b0
+opentelemetry-instrumentation-flask~=0.35b0
+opentelemetry-instrumentation-requests~=0.35b0
 pytest
 pytest-cov
 pytest-mock
-requests
+requests
+flask
+werkzeug

solarwinds_apm/sampler.py

@@ -278,7 +278,7 @@ def calculate_trace_state(
                 trace_state = trace_state.update(
                     INTL_SWO_TRACESTATE_KEY,
                     W3CTransformer.sw_from_span_and_decision(
-                        parent_span_context.span_id,
+                        parent_span_context.span_id,  # NOTE: This is a placeholder before propagator inject
                         W3CTransformer.trace_flags_from_int(decision["do_sample"])
                     )
                 )

tests/integration/test_base_sw_headers_attrs.py

@@ -0,0 +1,135 @@
+import os
+from pkg_resources import (
+    iter_entry_points,
+    load_entry_point
+)
+import re
+
+import flask
+import requests
+from werkzeug.test import Client
+from werkzeug.wrappers import Response
+
+from opentelemetry import trace as trace_api
+from opentelemetry.instrumentation.flask import FlaskInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
+from opentelemetry.propagate import get_global_textmap
+from opentelemetry.sdk.trace import TracerProvider, export
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
+    InMemorySpanExporter,
+)
+from opentelemetry.test.globals_test import reset_trace_globals
+from opentelemetry.test.test_base import TestBase
+
+from solarwinds_apm.apm_config import SolarWindsApmConfig
+from solarwinds_apm.configurator import SolarWindsConfigurator
+from solarwinds_apm.distro import SolarWindsDistro
+from solarwinds_apm.propagator import SolarWindsPropagator
+from solarwinds_apm.sampler import ParentBasedSwSampler
+
+
+class TestBaseSwHeadersAndAttributes(TestBase):
+    """
+    Base class for testing SolarWinds custom distro header propagation
+    and span attributes calculation from decision and headers.
+    """
+
+    SW_SETTINGS_KEYS = [
+        "BucketCapacity",
+        "BucketRate",
+        "SampleRate",
+        "SampleSource"
+    ]
+
+    @staticmethod
+    def _test_trace():
+        incoming_headers = {}
+        for k, v in flask.request.headers.items():
+            # WSGI capitalizes incoming HTTP headers
+            incoming_headers.update({k.lower(): v.lower()})
+
+        resp = requests.get(f"http://postman-echo.com/headers")
+
+        #  The return type must be a string, dict, tuple, Response instance, or WSGI callable
+        # (not CaseInsensitiveDict)
+        return {
+            "traceparent": resp.request.headers["traceparent"],
+            "tracestate": resp.request.headers["tracestate"],
+            "incoming-headers": incoming_headers,
+        }
+
+    def _setup_endpoints(self):
+        # pylint: disable=no-member
+        self.app.route("/test_trace/")(self._test_trace)
+        # pylint: disable=attribute-defined-outside-init
+        self.client = Client(self.app, Response)
+
+    def setUp(self):
+        """Set up called before each test scenario"""
+        # Based on auto_instrumentation run() and sitecustomize.py
+        # Load OTel env vars entry points
+        argument_otel_environment_variable = {}
+        for entry_point in iter_entry_points(
+            "opentelemetry_environment_variables"
+        ):
+            environment_variable_module = entry_point.load()
+            for attribute in dir(environment_variable_module):
+                if attribute.startswith("OTEL_"):
+                    argument = re.sub(r"OTEL_(PYTHON_)?", "", attribute).lower()
+                    argument_otel_environment_variable[argument] = attribute
+
+        # Set APM service key - not valid, but we mock liboboe anyway
+        os.environ["SW_APM_SERVICE_KEY"] = "foo:bar"
+
+        # Load Distro
+        SolarWindsDistro().configure()
+        assert os.environ["OTEL_PROPAGATORS"] == "tracecontext,baggage,solarwinds_propagator"
+
+        # Load Configurator to Configure SW custom SDK components
+        # except use TestBase InMemorySpanExporter
+        apm_config = SolarWindsApmConfig()
+        configurator = SolarWindsConfigurator()
+        configurator._initialize_solarwinds_reporter(apm_config)
+        configurator._configure_propagator()
+        configurator._configure_response_propagator()
+        # This is done because set_tracer_provider cannot override the
+        # current tracer provider. Has to be done here.
+        reset_trace_globals()
+        sampler = load_entry_point(
+            "solarwinds_apm",
+            "opentelemetry_traces_sampler",
+            configurator._DEFAULT_SW_TRACES_SAMPLER
+        )(apm_config)
+        self.tracer_provider = TracerProvider(sampler=sampler)
+        # Set InMemorySpanExporter for testing
+        # We do NOT use SolarWindsSpanExporter
+        self.memory_exporter = InMemorySpanExporter()
+        span_processor = export.SimpleSpanProcessor(self.memory_exporter)
+        self.tracer_provider.add_span_processor(span_processor)
+        trace_api.set_tracer_provider(self.tracer_provider)
+        self.tracer = self.tracer_provider.get_tracer(__name__)
+
+        # Make sure SW SDK components were set
+        propagators = get_global_textmap()._propagators
+        assert len(propagators) == 3
+        assert isinstance(propagators[2], SolarWindsPropagator)
+        assert isinstance(trace_api.get_tracer_provider().sampler, ParentBasedSwSampler)
+
+        # We need to instrument and create test app for every test
+        self.requests_inst = RequestsInstrumentor()
+        self.flask_inst = FlaskInstrumentor()
+        self.flask_inst.uninstrument()
+        self.flask_inst.instrument(
+            tracer_provider=trace_api.get_tracer_provider()
+        )
+        self.requests_inst.uninstrument()
+        self.requests_inst.instrument(
+            tracer_provider=trace_api.get_tracer_provider()
+        )
+        self.app = flask.Flask(__name__)
+        self._setup_endpoints()
+        self.client = Client(self.app, Response)
+
+    def tearDown(self):
+        """Teardown called after each test scenario"""
+        self.memory_exporter.clear()

tests/integration/test_scenario_1.py

@@ -0,0 +1,131 @@
+import re
+import json
+
+from opentelemetry import trace as trace_api
+from unittest import mock
+
+from .test_base_sw_headers_attrs import TestBaseSwHeadersAndAttributes
+
+class TestScenario1(TestBaseSwHeadersAndAttributes):
+    """
+    Test class for starting a new tracing decision with no input headers.
+    """
+
+    def test_scenario_1_sampled(self):
+        """
+        Scenario #1, sampled:
+        1. Decision to sample is made at root/service entry span (mocked). There is no
+           OTel context extracted from request headers, so this is the root and start
+           of the trace.
+        2. Headers in the original request are not altered by the SW propagator.
+        3. Some traceparent and tracestate are injected into service's outgoing request
+           (done by OTel TraceContextTextMapPropagator).
+        4. Sampling-related attributes are set for the root/service entry span.
+        5. The span_id of the outgoing request span matches the span_id portion in the
+           tracestate header.
+        """
+        # Use in-process test app client and mock to propagate context
+        # and create in-memory trace
+        resp = None
+        # liboboe mocked to guarantee return of "do_sample" (2nd arg)
+        mock_decision = mock.Mock(
+            return_value=(1, 1, 3, 4, 5.0, 6.0, 1, 0, "ok", "ok", 0)
+        )
+        with mock.patch(
+            target="solarwinds_apm.extension.oboe.Context.getDecisions",
+            new=mock_decision,
+        ):
+            # Request to instrumented app, no traceparent/tracestate
+            resp = self.client.get(
+                "/test_trace/",
+                headers={
+                    "some-header": "some-value"
+                }
+            )
+        resp_json = json.loads(resp.data)
+
+        # Verify some-header was not altered by instrumentation
+        try:
+            assert resp_json["incoming-headers"]["some-header"] == "some-value"
+        except KeyError as e:
+            self.fail("KeyError was raised at incoming-headers check: {}".format(e))
+
+        # Verify trace context injected into test app's outgoing postman-echo call
+        # (added to Flask app's response data) includes:
+        #    - traceparent with a trace_id, span_id, and trace_flags for do_sample
+        #    - tracestate with same span_id and trace_flags for do_sample
+        assert "traceparent" in resp_json
+        _TRACEPARENT_HEADER_FORMAT = (
+            "^([0-9a-f]{2})-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$"
+        )
+        _TRACEPARENT_HEADER_FORMAT_RE = re.compile(_TRACEPARENT_HEADER_FORMAT)
+        traceparent_re_result = re.search(
+            _TRACEPARENT_HEADER_FORMAT_RE,
+            resp_json["traceparent"],
+        )
+        new_trace_id = traceparent_re_result.group(2)
+        assert new_trace_id is not None
+        new_span_id = traceparent_re_result.group(3)
+        assert new_span_id is not None
+        new_trace_flags = traceparent_re_result.group(4)
+        assert new_trace_flags == "01"
+
+        assert "tracestate" in resp_json
+        # In this test we know there is only `sw` in tracestate
+        # and its value will be new_span_id and new_trace_flags
+        assert resp_json["tracestate"] == "sw={}-{}".format(new_span_id, new_trace_flags)
+
+        # Verify x-trace response header has same trace_id
+        # though it will have different span ID because of Flask
+        # app's outgoing request
+        assert "x-trace" in resp.headers
+        assert new_trace_id in resp.headers["x-trace"]
+
+        # Verify spans exported: service entry (root) + outgoing request (child with local parent)
+        spans = self.memory_exporter.get_finished_spans()
+        assert len(spans) == 2
+        span_server = spans[1]
+        span_client = spans[0]
+        assert span_server.name == "/test_trace/"
+        assert span_server.kind == trace_api.SpanKind.SERVER
+        assert span_client.name == "HTTP GET"
+        assert span_client.kind == trace_api.SpanKind.CLIENT
+
+        # Check root span tracestate has `sw` key
+        # In this test we know its value will have invalid span_id
+        expected_trace_state = trace_api.TraceState([("sw", "0000000000000000-01")])
+        assert span_server.context.trace_state == expected_trace_state
+
+        # Check root span attributes
+        #   :present:
+        #     service entry internal KVs, which are on all entry spans
+        #   :absent:
+        #     sw.tracestate_parent_id, because cannot be set at root nor without attributes at decision
+        #     SWKeys, because no xtraceoptions in otel context
+        assert all(attr_key in span_server.attributes for attr_key in self.SW_SETTINGS_KEYS)
+        assert span_server.attributes["BucketCapacity"] == "6.0"
+        assert span_server.attributes["BucketRate"] == "5.0"
+        assert span_server.attributes["SampleRate"] == 3
+        assert span_server.attributes["SampleSource"] == 4
+        assert not "sw.tracestate_parent_id" in span_server.attributes
+        assert not "SWKeys" in span_server.attributes
+
+        # Check outgoing request tracestate has `sw` key
+        # In this test we know its value will also have invalid span_id
+        expected_trace_state = trace_api.TraceState([("sw", "0000000000000000-01")])
+        assert span_client.context.trace_state == expected_trace_state
+
+        # Check outgoing request span attributes
+        #   :absent:
+        #     service entry internal KVs, which are only on entry spans
+        #     sw.tracestate_parent_id, because cannot be set without attributes at decision
+        #     SWKeys, because no xtraceoptions in otel context
+        assert not any(attr_key in span_client.attributes for attr_key in self.SW_SETTINGS_KEYS)
+        assert not "sw.tracestate_parent_id" in span_client.attributes
+        assert not "SWKeys" in span_client.attributes
+
+        # Check span_id of the outgoing request span (client span) matches
+        # the span_id portion in the outgoing tracestate header, which
+        # is stored in the test app's response body (new_span_id).
+        # Note: context.span_id needs a 16-byte hex conversion first.
+        assert "{:016x}".format(span_client.context.span_id) == new_span_id