From 7292beefae6ee08c886ca3a8728e8cfe71b43fb8 Mon Sep 17 00:00:00 2001 From: Tammy Baylis <96076570+tammy-baylis-swi@users.noreply.github.com> Date: Fri, 16 Jun 2023 15:40:46 -0700 Subject: [PATCH 1/9] Request Flask attributes passed to Sampler (#1784) * Request Flask attributes passed to Sampler * Update changelog * Lint * Fix botocore test keyerror * Revert "Fix botocore test keyerror" This reverts commit fd03c55a3902b3456afd6a2ecf429afba11b0691. * botocore test does get_queue_url * Revert "botocore test does get_queue_url" This reverts commit 9530cd250dd836b3181a9361decb130e2aae1202. * Update changelog --------- Co-authored-by: Srikanth Chekuri Co-authored-by: Shalev Roda <65566801+shalevr@users.noreply.github.com> --- CHANGELOG.md | 4 ++++ .../instrumentation/flask/__init__.py | 17 ++++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cde00bb9ab..70ba0033de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Make Flask request span attributes available for `start_span`. + ([#1784](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1784)) - Fix falcon instrumentation's usage of Span Status to only set the description if the status code is ERROR. ([#1840](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1840)) - Instrument all httpx versions >= 0.18. ([#1748](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1748)) diff --git a/instrumentation/opentelemetry-instrumentation-flask/src/opentelemetry/instrumentation/flask/__init__.py b/instrumentation/opentelemetry-instrumentation-flask/src/opentelemetry/instrumentation/flask/__init__.py index 73c2f4fe2d..432c6b1fbf 100644 --- a/instrumentation/opentelemetry-instrumentation-flask/src/opentelemetry/instrumentation/flask/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-flask/src/opentelemetry/instrumentation/flask/__init__.py @@ -375,27 +375,26 @@ def _before_request(): flask_request_environ = flask.request.environ span_name = get_default_span_name() + attributes = otel_wsgi.collect_request_attributes( + flask_request_environ + ) + if flask.request.url_rule: + # For 404 that result from no route found, etc, we + # don't have a url_rule. + attributes[SpanAttributes.HTTP_ROUTE] = flask.request.url_rule.rule span, token = _start_internal_or_server_span( tracer=tracer, span_name=span_name, start_time=flask_request_environ.get(_ENVIRON_STARTTIME_KEY), context_carrier=flask_request_environ, context_getter=otel_wsgi.wsgi_getter, + attributes=attributes, ) if request_hook: request_hook(span, flask_request_environ) if span.is_recording(): - attributes = otel_wsgi.collect_request_attributes( - flask_request_environ - ) - if flask.request.url_rule: - # For 404 that result from no route found, etc, we - # don't have a url_rule. - attributes[ - SpanAttributes.HTTP_ROUTE - ] = flask.request.url_rule.rule for key, value in attributes.items(): span.set_attribute(key, value) if span.is_recording() and span.kind == trace.SpanKind.SERVER: From a3a0b2409cc1fe9a03bb73da9c8bbf47b9d3d5ea Mon Sep 17 00:00:00 2001 From: Yashaswi Makula Date: Sat, 17 Jun 2023 04:38:51 +0530 Subject: [PATCH 2/9] Fixed urllib3 instrumentation example in instrumentation documentation (#1793) * corrected instrumentation example in urllib3 * Remove changelog entry --------- Co-authored-by: Shalev Roda <65566801+shalevr@users.noreply.github.com> Co-authored-by: Diego Hurtado --- .../opentelemetry-instrumentation-urllib3/README.rst | 4 ++-- .../src/opentelemetry/instrumentation/urllib3/__init__.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-urllib3/README.rst b/instrumentation/opentelemetry-instrumentation-urllib3/README.rst index 0c53c299a0..4e0089e21e 100644 --- a/instrumentation/opentelemetry-instrumentation-urllib3/README.rst +++ b/instrumentation/opentelemetry-instrumentation-urllib3/README.rst @@ -38,8 +38,8 @@ The hooks can be configured as follows: def response_hook(span, request, response): pass - URLLib3Instrumentor.instrument( - request_hook=request_hook, response_hook=response_hook) + URLLib3Instrumentor().instrument( + request_hook=request_hook, response_hook=response_hook ) Exclude lists diff --git a/instrumentation/opentelemetry-instrumentation-urllib3/src/opentelemetry/instrumentation/urllib3/__init__.py b/instrumentation/opentelemetry-instrumentation-urllib3/src/opentelemetry/instrumentation/urllib3/__init__.py index 809f9b595f..d3016ea5ee 100644 --- a/instrumentation/opentelemetry-instrumentation-urllib3/src/opentelemetry/instrumentation/urllib3/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-urllib3/src/opentelemetry/instrumentation/urllib3/__init__.py @@ -56,8 +56,8 @@ def request_hook(span, request): def response_hook(span, request, response): pass - URLLib3Instrumentor.instrument( - request_hook=request_hook, response_hook=response_hook) + URLLib3Instrumentor().instrument( + request_hook=request_hook, response_hook=response_hook ) Exclude lists From 78040836d2d401c9a0ba5013e73140b768fb58ee Mon Sep 17 00:00:00 2001 From: Iman Shafiei Date: Fri, 16 Jun 2023 16:55:15 -0700 Subject: [PATCH 3/9] Fix Invalid type NoneType for attribute X error | AWS-Lambda instrumentation (#1785) * Add None checking to the aws-lambda logic * Update changelog. * Change .get() check to 'key' in dict check. * Fix consistency issues. * Update changelog. --------- Co-authored-by: Srikanth Chekuri Co-authored-by: Shalev Roda <65566801+shalevr@users.noreply.github.com> --- CHANGELOG.md | 2 + .../instrumentation/aws_lambda/__init__.py | 94 ++++++++++--------- 2 files changed, 53 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70ba0033de..29cecbd899 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fix falcon instrumentation's usage of Span Status to only set the description if the status code is ERROR. ([#1840](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1840)) - Instrument all httpx versions >= 0.18. ([#1748](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1748)) +- Fix `Invalid type NoneType for attribute X (opentelemetry-instrumentation-aws-lambda)` error when some attributes do not exist + ([#1780](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1780)) ## Version 1.18.0/0.39b0 (2023-05-10) diff --git a/instrumentation/opentelemetry-instrumentation-aws-lambda/src/opentelemetry/instrumentation/aws_lambda/__init__.py b/instrumentation/opentelemetry-instrumentation-aws-lambda/src/opentelemetry/instrumentation/aws_lambda/__init__.py index 4404839c66..799becbdcc 100644 --- a/instrumentation/opentelemetry-instrumentation-aws-lambda/src/opentelemetry/instrumentation/aws_lambda/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-aws-lambda/src/opentelemetry/instrumentation/aws_lambda/__init__.py @@ -201,30 +201,35 @@ def _set_api_gateway_v1_proxy_attributes( span.set_attribute( SpanAttributes.HTTP_METHOD, lambda_event.get("httpMethod") ) - span.set_attribute(SpanAttributes.HTTP_ROUTE, lambda_event.get("resource")) if lambda_event.get("headers"): - span.set_attribute( - SpanAttributes.HTTP_USER_AGENT, - lambda_event["headers"].get("User-Agent"), - ) - span.set_attribute( - SpanAttributes.HTTP_SCHEME, - lambda_event["headers"].get("X-Forwarded-Proto"), - ) - span.set_attribute( - SpanAttributes.NET_HOST_NAME, lambda_event["headers"].get("Host") - ) + if "User-Agent" in lambda_event["headers"]: + span.set_attribute( + SpanAttributes.HTTP_USER_AGENT, + lambda_event["headers"]["User-Agent"], + ) + if "X-Forwarded-Proto" in lambda_event["headers"]: + span.set_attribute( + SpanAttributes.HTTP_SCHEME, + lambda_event["headers"]["X-Forwarded-Proto"], + ) + if "Host" in lambda_event["headers"]: + span.set_attribute( + SpanAttributes.NET_HOST_NAME, + lambda_event["headers"]["Host"], + ) + if "resource" in lambda_event: + span.set_attribute(SpanAttributes.HTTP_ROUTE, lambda_event["resource"]) - if lambda_event.get("queryStringParameters"): - span.set_attribute( - SpanAttributes.HTTP_TARGET, - f"{lambda_event.get('resource')}?{urlencode(lambda_event.get('queryStringParameters'))}", - ) - else: - span.set_attribute( - SpanAttributes.HTTP_TARGET, lambda_event.get("resource") - ) + if lambda_event.get("queryStringParameters"): + span.set_attribute( + SpanAttributes.HTTP_TARGET, + f"{lambda_event['resource']}?{urlencode(lambda_event['queryStringParameters'])}", + ) + else: + span.set_attribute( + SpanAttributes.HTTP_TARGET, lambda_event["resource"] + ) return span @@ -237,35 +242,38 @@ def _set_api_gateway_v2_proxy_attributes( More info: https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html """ - span.set_attribute( - SpanAttributes.NET_HOST_NAME, - lambda_event["requestContext"].get("domainName"), - ) - - if lambda_event["requestContext"].get("http"): + if "domainName" in lambda_event["requestContext"]: span.set_attribute( - SpanAttributes.HTTP_METHOD, - lambda_event["requestContext"]["http"].get("method"), - ) - span.set_attribute( - SpanAttributes.HTTP_USER_AGENT, - lambda_event["requestContext"]["http"].get("userAgent"), - ) - span.set_attribute( - SpanAttributes.HTTP_ROUTE, - lambda_event["requestContext"]["http"].get("path"), + SpanAttributes.NET_HOST_NAME, + lambda_event["requestContext"]["domainName"], ) - if lambda_event.get("rawQueryString"): + if lambda_event["requestContext"].get("http"): + if "method" in lambda_event["requestContext"]["http"]: span.set_attribute( - SpanAttributes.HTTP_TARGET, - f"{lambda_event['requestContext']['http'].get('path')}?{lambda_event.get('rawQueryString')}", + SpanAttributes.HTTP_METHOD, + lambda_event["requestContext"]["http"]["method"], ) - else: + if "userAgent" in lambda_event["requestContext"]["http"]: span.set_attribute( - SpanAttributes.HTTP_TARGET, - lambda_event["requestContext"]["http"].get("path"), + SpanAttributes.HTTP_USER_AGENT, + lambda_event["requestContext"]["http"]["userAgent"], + ) + if "path" in lambda_event["requestContext"]["http"]: + span.set_attribute( + SpanAttributes.HTTP_ROUTE, + lambda_event["requestContext"]["http"]["path"], ) + if lambda_event.get("rawQueryString"): + span.set_attribute( + SpanAttributes.HTTP_TARGET, + f"{lambda_event['requestContext']['http']['path']}?{lambda_event['rawQueryString']}", + ) + else: + span.set_attribute( + SpanAttributes.HTTP_TARGET, + lambda_event["requestContext"]["http"]["path"], + ) return span From 1dd17edeeab4cb45477755fb2ef0eef23fbd8289 Mon Sep 17 00:00:00 2001 From: Akochavi <121871419+Akochavi@users.noreply.github.com> Date: Sun, 18 Jun 2023 13:45:00 +0300 Subject: [PATCH 4/9] Add metrics instrumentation celery (#1679) Co-authored-by: Shalev Roda <65566801+shalevr@users.noreply.github.com> --- CHANGELOG.md | 3 + .../instrumentation/celery/__init__.py | 46 ++++++++++- .../tests/test_metrics.py | 76 +++++++++++++++++++ 3 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 instrumentation/opentelemetry-instrumentation-celery/tests/test_metrics.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 29cecbd899..b591e9a143 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased + ### Added - Make Flask request span attributes available for `start_span`. @@ -16,6 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Instrument all httpx versions >= 0.18. ([#1748](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1748)) - Fix `Invalid type NoneType for attribute X (opentelemetry-instrumentation-aws-lambda)` error when some attributes do not exist ([#1780](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1780)) +- Add metric instrumentation for celery + ([#1679](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1679)) ## Version 1.18.0/0.39b0 (2023-05-10) diff --git a/instrumentation/opentelemetry-instrumentation-celery/src/opentelemetry/instrumentation/celery/__init__.py b/instrumentation/opentelemetry-instrumentation-celery/src/opentelemetry/instrumentation/celery/__init__.py index cb265b46f8..a216765fb0 100644 --- a/instrumentation/opentelemetry-instrumentation-celery/src/opentelemetry/instrumentation/celery/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-celery/src/opentelemetry/instrumentation/celery/__init__.py @@ -60,6 +60,7 @@ def add(x, y): """ import logging +from timeit import default_timer from typing import Collection, Iterable from celery import signals # pylint: disable=no-name-in-module @@ -69,6 +70,7 @@ def add(x, y): from opentelemetry.instrumentation.celery.package import _instruments from opentelemetry.instrumentation.celery.version import __version__ from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from opentelemetry.metrics import get_meter from opentelemetry.propagate import extract, inject from opentelemetry.propagators.textmap import Getter from opentelemetry.semconv.trace import SpanAttributes @@ -104,6 +106,11 @@ def keys(self, carrier): class CeleryInstrumentor(BaseInstrumentor): + def __init__(self): + super().__init__() + self.metrics = None + self.task_id_to_start_time = {} + def instrumentation_dependencies(self) -> Collection[str]: return _instruments @@ -113,6 +120,11 @@ def _instrument(self, **kwargs): # pylint: disable=attribute-defined-outside-init self._tracer = trace.get_tracer(__name__, __version__, tracer_provider) + meter_provider = kwargs.get("meter_provider") + meter = get_meter(__name__, __version__, meter_provider) + + self.create_celery_metrics(meter) + signals.task_prerun.connect(self._trace_prerun, weak=False) signals.task_postrun.connect(self._trace_postrun, weak=False) signals.before_task_publish.connect( @@ -139,6 +151,7 @@ def _trace_prerun(self, *args, **kwargs): if task is None or task_id is None: return + self.update_task_duration_time(task_id) request = task.request tracectx = extract(request, getter=celery_getter) or None @@ -153,8 +166,7 @@ def _trace_prerun(self, *args, **kwargs): activation.__enter__() # pylint: disable=E1101 utils.attach_span(task, task_id, (span, activation)) - @staticmethod - def _trace_postrun(*args, **kwargs): + def _trace_postrun(self, *args, **kwargs): task = utils.retrieve_task(kwargs) task_id = utils.retrieve_task_id(kwargs) @@ -178,6 +190,9 @@ def _trace_postrun(*args, **kwargs): activation.__exit__(None, None, None) utils.detach_span(task, task_id) + self.update_task_duration_time(task_id) + labels = {"task": task.name, "worker": task.request.hostname} + self._record_histograms(task_id, labels) def _trace_before_publish(self, *args, **kwargs): task = utils.retrieve_task_from_sender(kwargs) @@ -277,3 +292,30 @@ def _trace_retry(*args, **kwargs): # Use `str(reason)` instead of `reason.message` in case we get # something that isn't an `Exception` span.set_attribute(_TASK_RETRY_REASON_KEY, str(reason)) + + def update_task_duration_time(self, task_id): + cur_time = default_timer() + task_duration_time_until_now = ( + cur_time - self.task_id_to_start_time[task_id] + if task_id in self.task_id_to_start_time + else cur_time + ) + self.task_id_to_start_time[task_id] = task_duration_time_until_now + + def _record_histograms(self, task_id, metric_attributes): + if task_id is None: + return + + self.metrics["flower.task.runtime.seconds"].record( + self.task_id_to_start_time.get(task_id), + attributes=metric_attributes, + ) + + def create_celery_metrics(self, meter) -> None: + self.metrics = { + "flower.task.runtime.seconds": meter.create_histogram( + name="flower.task.runtime.seconds", + unit="seconds", + description="The time it took to run the task.", + ) + } diff --git a/instrumentation/opentelemetry-instrumentation-celery/tests/test_metrics.py b/instrumentation/opentelemetry-instrumentation-celery/tests/test_metrics.py new file mode 100644 index 0000000000..46e39a6046 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-celery/tests/test_metrics.py @@ -0,0 +1,76 @@ +import threading +import time +from timeit import default_timer + +from opentelemetry.instrumentation.celery import CeleryInstrumentor +from opentelemetry.test.test_base import TestBase + +from .celery_test_tasks import app, task_add + + +class TestMetrics(TestBase): + def setUp(self): + super().setUp() + self._worker = app.Worker( + app=app, pool="solo", concurrency=1, hostname="celery@akochavi" + ) + self._thread = threading.Thread(target=self._worker.start) + self._thread.daemon = True + self._thread.start() + + def tearDown(self): + super().tearDown() + self._worker.stop() + self._thread.join() + + def get_metrics(self): + result = task_add.delay(1, 2) + + timeout = time.time() + 60 * 1 # 1 minutes from now + while not result.ready(): + if time.time() > timeout: + break + time.sleep(0.05) + return self.get_sorted_metrics() + + def test_basic_metric(self): + CeleryInstrumentor().instrument() + start_time = default_timer() + task_runtime_estimated = (default_timer() - start_time) * 1000 + + metrics = self.get_metrics() + CeleryInstrumentor().uninstrument() + self.assertEqual(len(metrics), 1) + + task_runtime = metrics[0] + print(task_runtime) + self.assertEqual(task_runtime.name, "flower.task.runtime.seconds") + self.assert_metric_expected( + task_runtime, + [ + self.create_histogram_data_point( + count=1, + sum_data_point=task_runtime_estimated, + max_data_point=task_runtime_estimated, + min_data_point=task_runtime_estimated, + attributes={ + "task": "tests.celery_test_tasks.task_add", + "worker": "celery@akochavi", + }, + ) + ], + est_value_delta=200, + ) + + def test_metric_uninstrument(self): + CeleryInstrumentor().instrument() + metrics = self.get_metrics() + self.assertEqual(len(metrics), 1) + CeleryInstrumentor().uninstrument() + + metrics = self.get_metrics() + self.assertEqual(len(metrics), 1) + + for metric in metrics: + for point in list(metric.data.data_points): + self.assertEqual(point.count, 1) From 5ac58c2ffb4c595c9281f50606306943ce7b5044 Mon Sep 17 00:00:00 2001 From: David Gonoradsky Date: Sun, 18 Jun 2023 16:20:21 +0300 Subject: [PATCH 5/9] Add support for confluent_kafka until 2.1.1 version (#1815) * Add support for confulent_kafka until 2.1.1 version * Include 2.1.1 version * update CHANGELOG.md * run: 'tox -e generate' * resolve comments * update top version to 2.2.0 --------- Co-authored-by: Ran Nozik --- CHANGELOG.md | 2 ++ docs-requirements.txt | 2 +- instrumentation/README.md | 2 +- .../pyproject.toml | 2 +- .../opentelemetry/instrumentation/confluent_kafka/package.py | 2 +- .../src/opentelemetry/instrumentation/bootstrap_gen.py | 2 +- 6 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b591e9a143..0e0229e51b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#1706](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1706)) - `opentelemetry-instrumentation-pymemcache` Update instrumentation to support pymemcache >4 ([#1764](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1764)) +- `opentelemetry-instrumentation-confluent-kafka` Add support for higher versions of confluent_kafka + ([#1815](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1815)) ### Added diff --git a/docs-requirements.txt b/docs-requirements.txt index 6d11d20198..4af8839334 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -26,7 +26,7 @@ boto~=2.0 botocore~=1.0 boto3~=1.0 celery>=4.0 -confluent-kafka>= 1.8.2,< 2.0.0 +confluent-kafka>= 1.8.2,<= 2.2.0 elasticsearch>=2.0,<9.0 flask~=2.0 falcon~=2.0 diff --git a/instrumentation/README.md b/instrumentation/README.md index e69dd6adbd..933e9b763d 100644 --- a/instrumentation/README.md +++ b/instrumentation/README.md @@ -11,7 +11,7 @@ | [opentelemetry-instrumentation-boto3sqs](./opentelemetry-instrumentation-boto3sqs) | boto3 ~= 1.0 | No | [opentelemetry-instrumentation-botocore](./opentelemetry-instrumentation-botocore) | botocore ~= 1.0 | No | [opentelemetry-instrumentation-celery](./opentelemetry-instrumentation-celery) | celery >= 4.0, < 6.0 | No -| [opentelemetry-instrumentation-confluent-kafka](./opentelemetry-instrumentation-confluent-kafka) | confluent-kafka >= 1.8.2, < 2.0.0 | No +| [opentelemetry-instrumentation-confluent-kafka](./opentelemetry-instrumentation-confluent-kafka) | confluent-kafka >= 1.8.2, <= 2.2.0 | No | [opentelemetry-instrumentation-dbapi](./opentelemetry-instrumentation-dbapi) | dbapi | No | [opentelemetry-instrumentation-django](./opentelemetry-instrumentation-django) | django >= 1.10 | Yes | [opentelemetry-instrumentation-elasticsearch](./opentelemetry-instrumentation-elasticsearch) | elasticsearch >= 2.0 | No diff --git a/instrumentation/opentelemetry-instrumentation-confluent-kafka/pyproject.toml b/instrumentation/opentelemetry-instrumentation-confluent-kafka/pyproject.toml index a84f7d959e..2ef1fd6b49 100644 --- a/instrumentation/opentelemetry-instrumentation-confluent-kafka/pyproject.toml +++ b/instrumentation/opentelemetry-instrumentation-confluent-kafka/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ [project.optional-dependencies] instruments = [ - "confluent-kafka >= 1.8.2, < 2.0.0", + "confluent-kafka >= 1.8.2, <= 2.2.0", ] test = [ "opentelemetry-instrumentation-confluent-kafka[instruments]", diff --git a/instrumentation/opentelemetry-instrumentation-confluent-kafka/src/opentelemetry/instrumentation/confluent_kafka/package.py b/instrumentation/opentelemetry-instrumentation-confluent-kafka/src/opentelemetry/instrumentation/confluent_kafka/package.py index eab664d9ee..21d37ebfae 100644 --- a/instrumentation/opentelemetry-instrumentation-confluent-kafka/src/opentelemetry/instrumentation/confluent_kafka/package.py +++ b/instrumentation/opentelemetry-instrumentation-confluent-kafka/src/opentelemetry/instrumentation/confluent_kafka/package.py @@ -13,4 +13,4 @@ # limitations under the License. -_instruments = ("confluent-kafka >= 1.8.2, < 2.0.0",) +_instruments = ("confluent-kafka >= 1.8.2, <= 2.2.0",) diff --git a/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py b/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py index f705324289..1e0b9fd2f5 100644 --- a/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py +++ b/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py @@ -53,7 +53,7 @@ "instrumentation": "opentelemetry-instrumentation-celery==0.40b0.dev", }, "confluent-kafka": { - "library": "confluent-kafka >= 1.8.2, < 2.0.0", + "library": "confluent-kafka >= 1.8.2, <= 2.2.0", "instrumentation": "opentelemetry-instrumentation-confluent-kafka==0.40b0.dev", }, "django": { From 8cc10a0859d0da4773985e1025d261c3e334ea28 Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Sun, 18 Jun 2023 09:55:17 -0400 Subject: [PATCH 6/9] fix redis doc (#1808) doc string rendered at https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation/redis/redis.html refers to `opentelemetry-instrumentation` executable which appears to be a typo Co-authored-by: Shalev Roda <65566801+shalevr@users.noreply.github.com> --- CHANGELOG.md | 1 - .../src/opentelemetry/instrumentation/redis/__init__.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e0229e51b..c096812a60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased - ### Added - Make Flask request span attributes available for `start_span`. diff --git a/instrumentation/opentelemetry-instrumentation-redis/src/opentelemetry/instrumentation/redis/__init__.py b/instrumentation/opentelemetry-instrumentation-redis/src/opentelemetry/instrumentation/redis/__init__.py index 188840c7b8..3c8acdef31 100644 --- a/instrumentation/opentelemetry-instrumentation-redis/src/opentelemetry/instrumentation/redis/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-redis/src/opentelemetry/instrumentation/redis/__init__.py @@ -16,7 +16,7 @@ Instrument `redis`_ to report Redis queries. There are two options for instrumenting code. The first option is to use the -``opentelemetry-instrumentation`` executable which will automatically +``opentelemetry-instrument`` executable which will automatically instrument your Redis client. The second is to programmatically enable instrumentation via the following code: From 60753e2a5528ac34aa415f1daa2fc2db53fc5eb4 Mon Sep 17 00:00:00 2001 From: Iman Shafiei Date: Mon, 19 Jun 2023 13:00:59 -0700 Subject: [PATCH 7/9] Add http.server.response.size metric to ASGI implementation. (#1789) * Add http.server.response.size metric to ASGI implementation. Add new unit tests. * Update changelog. * Fix linting by disabling too-many-nested-blocks * Put new logic in a new method * Refactor the placement of new logic. * Fixed the unit tests in FastAPI and Starlette * Update changelog. * FIx lint errors. * Refactor getting content-length header * Refactor getting content-length header --------- Co-authored-by: Shalev Roda <65566801+shalevr@users.noreply.github.com> Co-authored-by: Diego Hurtado --- CHANGELOG.md | 2 ++ .../instrumentation/asgi/__init__.py | 17 ++++++++++ .../tests/test_asgi_middleware.py | 31 +++++++++++++------ .../tests/test_fastapi_instrumentation.py | 7 ++++- .../tests/test_starlette_instrumentation.py | 4 ++- 5 files changed, 50 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c096812a60..de2ed9475e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#1780](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1780)) - Add metric instrumentation for celery ([#1679](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1679)) +- `opentelemetry-instrumentation-asgi` Add `http.server.response.size` metric + ([#1789](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1789)) ## Version 1.18.0/0.39b0 (2023-05-10) diff --git a/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py b/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py index 010c6accde..1ee25ae7d9 100644 --- a/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py @@ -506,6 +506,11 @@ def __init__( unit="ms", description="measures the duration of the inbound HTTP request", ) + self.server_response_size_histogram = self.meter.create_histogram( + name=MetricInstruments.HTTP_SERVER_RESPONSE_SIZE, + unit="By", + description="measures the size of HTTP response messages (compressed).", + ) self.active_requests_counter = self.meter.create_up_down_counter( name=MetricInstruments.HTTP_SERVER_ACTIVE_REQUESTS, unit="requests", @@ -518,6 +523,7 @@ def __init__( self.server_request_hook = server_request_hook self.client_request_hook = client_request_hook self.client_response_hook = client_response_hook + self.content_length_header = None async def __call__(self, scope, receive, send): """The ASGI application @@ -593,6 +599,10 @@ async def __call__(self, scope, receive, send): self.active_requests_counter.add( -1, active_requests_count_attrs ) + if self.content_length_header: + self.server_response_size_histogram.record( + self.content_length_header, duration_attrs + ) if token: context.detach(token) @@ -660,6 +670,13 @@ async def otel_send(message): setter=asgi_setter, ) + content_length = asgi_getter.get(message, "content-length") + if content_length: + try: + self.content_length_header = int(content_length[0]) + except ValueError: + pass + await send(message) return otel_send diff --git a/instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py b/instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py index f9a5731fd3..0a0c2c301f 100644 --- a/instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py +++ b/instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py @@ -46,10 +46,12 @@ _expected_metric_names = [ "http.server.active_requests", "http.server.duration", + "http.server.response.size", ] _recommended_attrs = { "http.server.active_requests": _active_requests_count_attrs, "http.server.duration": _duration_attrs, + "http.server.response.size": _duration_attrs, } @@ -61,7 +63,10 @@ async def http_app(scope, receive, send): { "type": "http.response.start", "status": 200, - "headers": [[b"Content-Type", b"text/plain"]], + "headers": [ + [b"Content-Type", b"text/plain"], + [b"content-length", b"1024"], + ], } ) await send({"type": "http.response.body", "body": b"*"}) @@ -103,7 +108,10 @@ async def error_asgi(scope, receive, send): { "type": "http.response.start", "status": 200, - "headers": [[b"Content-Type", b"text/plain"]], + "headers": [ + [b"Content-Type", b"text/plain"], + [b"content-length", b"1024"], + ], } ) await send({"type": "http.response.body", "body": b"*"}) @@ -126,7 +134,8 @@ def validate_outputs(self, outputs, error=None, modifiers=None): # Check http response start self.assertEqual(response_start["status"], 200) self.assertEqual( - response_start["headers"], [[b"Content-Type", b"text/plain"]] + response_start["headers"], + [[b"Content-Type", b"text/plain"], [b"content-length", b"1024"]], ) exc_info = self.scope.get("hack_exc_info") @@ -352,6 +361,7 @@ def test_traceresponse_header(self): response_start["headers"], [ [b"Content-Type", b"text/plain"], + [b"content-length", b"1024"], [b"traceresponse", f"{traceresponse}".encode()], [b"access-control-expose-headers", b"traceresponse"], ], @@ -565,6 +575,7 @@ def test_basic_metric_success(self): "http.flavor": "1.0", } metrics_list = self.memory_metrics_reader.get_metrics_data() + # pylint: disable=too-many-nested-blocks for resource_metric in metrics_list.resource_metrics: for scope_metrics in resource_metric.scope_metrics: for metric in scope_metrics.metrics: @@ -575,9 +586,12 @@ def test_basic_metric_success(self): dict(point.attributes), ) self.assertEqual(point.count, 1) - self.assertAlmostEqual( - duration, point.sum, delta=5 - ) + if metric.name == "http.server.duration": + self.assertAlmostEqual( + duration, point.sum, delta=5 + ) + elif metric.name == "http.server.response.size": + self.assertEqual(1024, point.sum) elif isinstance(point, NumberDataPoint): self.assertDictEqual( expected_requests_count_attributes, @@ -602,13 +616,12 @@ async def target_asgi(scope, receive, send): app = otel_asgi.OpenTelemetryMiddleware(target_asgi) self.seed_app(app) self.send_default_request() - metrics_list = self.memory_metrics_reader.get_metrics_data() assertions = 0 for resource_metric in metrics_list.resource_metrics: for scope_metrics in resource_metric.scope_metrics: for metric in scope_metrics.metrics: - if metric.name != "http.server.duration": + if metric.name == "http.server.active_requests": continue for point in metric.data.data_points: if isinstance(point, HistogramDataPoint): @@ -617,7 +630,7 @@ async def target_asgi(scope, receive, send): expected_target, ) assertions += 1 - self.assertEqual(assertions, 1) + self.assertEqual(assertions, 2) def test_no_metric_for_websockets(self): self.scope = { diff --git a/instrumentation/opentelemetry-instrumentation-fastapi/tests/test_fastapi_instrumentation.py b/instrumentation/opentelemetry-instrumentation-fastapi/tests/test_fastapi_instrumentation.py index 9420ba2c0e..7f12d6e3f3 100644 --- a/instrumentation/opentelemetry-instrumentation-fastapi/tests/test_fastapi_instrumentation.py +++ b/instrumentation/opentelemetry-instrumentation-fastapi/tests/test_fastapi_instrumentation.py @@ -44,10 +44,15 @@ _expected_metric_names = [ "http.server.active_requests", "http.server.duration", + "http.server.response.size", ] _recommended_attrs = { "http.server.active_requests": _active_requests_count_attrs, "http.server.duration": {*_duration_attrs, SpanAttributes.HTTP_TARGET}, + "http.server.response.size": { + *_duration_attrs, + SpanAttributes.HTTP_TARGET, + }, } @@ -187,7 +192,7 @@ def test_fastapi_metrics(self): for resource_metric in metrics_list.resource_metrics: self.assertTrue(len(resource_metric.scope_metrics) == 1) for scope_metric in resource_metric.scope_metrics: - self.assertTrue(len(scope_metric.metrics) == 2) + self.assertTrue(len(scope_metric.metrics) == 3) for metric in scope_metric.metrics: self.assertIn(metric.name, _expected_metric_names) data_points = list(metric.data.data_points) diff --git a/instrumentation/opentelemetry-instrumentation-starlette/tests/test_starlette_instrumentation.py b/instrumentation/opentelemetry-instrumentation-starlette/tests/test_starlette_instrumentation.py index 1f4570d293..2a53bdffb7 100644 --- a/instrumentation/opentelemetry-instrumentation-starlette/tests/test_starlette_instrumentation.py +++ b/instrumentation/opentelemetry-instrumentation-starlette/tests/test_starlette_instrumentation.py @@ -49,10 +49,12 @@ _expected_metric_names = [ "http.server.active_requests", "http.server.duration", + "http.server.response.size", ] _recommended_attrs = { "http.server.active_requests": _active_requests_count_attrs, "http.server.duration": _duration_attrs, + "http.server.response.size": _duration_attrs, } @@ -128,7 +130,7 @@ def test_starlette_metrics(self): for resource_metric in metrics_list.resource_metrics: self.assertTrue(len(resource_metric.scope_metrics) == 1) for scope_metric in resource_metric.scope_metrics: - self.assertTrue(len(scope_metric.metrics) == 2) + self.assertTrue(len(scope_metric.metrics) == 3) for metric in scope_metric.metrics: self.assertIn(metric.name, _expected_metric_names) data_points = list(metric.data.data_points) From fe9405730f8d18fe70308db52093b7b5b1e74e0c Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 20 Jun 2023 14:09:53 +0300 Subject: [PATCH 8/9] fix: Update falcon instrumentation to follow semantic conventions (#1824) * fix: Update falcon instrumentation to follow semantic conventions * docs: Update changelog * fix linter errors * Disable falcon.HTTP_200 pylint checck --------- Co-authored-by: Shalev Roda <65566801+shalevr@users.noreply.github.com> Co-authored-by: Srikanth Chekuri --- CHANGELOG.md | 5 +++ .../instrumentation/falcon/__init__.py | 7 +++- .../tests/app.py | 9 +++++ .../tests/test_falcon.py | 33 +++++++++++++++++-- 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de2ed9475e..c83443dc02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Fixed + +- Update falcon instrumentation to follow semantic conventions + ([#1824](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1824)) + ### Added - Make Flask request span attributes available for `start_span`. diff --git a/instrumentation/opentelemetry-instrumentation-falcon/src/opentelemetry/instrumentation/falcon/__init__.py b/instrumentation/opentelemetry-instrumentation-falcon/src/opentelemetry/instrumentation/falcon/__init__.py index 73c005fa17..669f41b0ab 100644 --- a/instrumentation/opentelemetry-instrumentation-falcon/src/opentelemetry/instrumentation/falcon/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-falcon/src/opentelemetry/instrumentation/falcon/__init__.py @@ -428,7 +428,6 @@ def process_resource(self, req, resp, resource, params): resource_name = resource.__class__.__name__ span.set_attribute("falcon.resource", resource_name) - span.update_name(f"{resource_name}.on_{req.method.lower()}") def process_response( self, req, resp, resource, req_succeeded=None @@ -483,6 +482,12 @@ def process_response( response_headers = resp.headers if span.is_recording() and span.kind == trace.SpanKind.SERVER: + # Check if low-cardinality route is available as per semantic-conventions + if req.uri_template: + span.update_name(f"{req.method} {req.uri_template}") + else: + span.update_name(f"{req.method}") + custom_attributes = ( otel_wsgi.collect_custom_response_headers_attributes( response_headers.items() diff --git a/instrumentation/opentelemetry-instrumentation-falcon/tests/app.py b/instrumentation/opentelemetry-instrumentation-falcon/tests/app.py index 3e4c62ec3e..a4d279149d 100644 --- a/instrumentation/opentelemetry-instrumentation-falcon/tests/app.py +++ b/instrumentation/opentelemetry-instrumentation-falcon/tests/app.py @@ -61,6 +61,13 @@ def on_get(self, _, resp): resp.set_header("my-secret-header", "my-secret-value") +class UserResource: + def on_get(self, req, resp, user_id): + # pylint: disable=no-member + resp.status = falcon.HTTP_200 + resp.body = f"Hello user {user_id}" + + def make_app(): _parsed_falcon_version = package_version.parse(falcon.__version__) if _parsed_falcon_version < package_version.parse("3.0.0"): @@ -76,4 +83,6 @@ def make_app(): app.add_route( "/test_custom_response_headers", CustomResponseHeaderResource() ) + app.add_route("/user/{user_id}", UserResource()) + return app diff --git a/instrumentation/opentelemetry-instrumentation-falcon/tests/test_falcon.py b/instrumentation/opentelemetry-instrumentation-falcon/tests/test_falcon.py index cf61a9fd3c..2245dbfd80 100644 --- a/instrumentation/opentelemetry-instrumentation-falcon/tests/test_falcon.py +++ b/instrumentation/opentelemetry-instrumentation-falcon/tests/test_falcon.py @@ -110,7 +110,7 @@ def _test_method(self, method): spans = self.memory_exporter.get_finished_spans() self.assertEqual(len(spans), 1) span = spans[0] - self.assertEqual(span.name, f"HelloWorldResource.on_{method.lower()}") + self.assertEqual(span.name, f"{method} /hello") self.assertEqual(span.status.status_code, StatusCode.UNSET) self.assertEqual( span.status.description, @@ -145,7 +145,7 @@ def test_404(self): spans = self.memory_exporter.get_finished_spans() self.assertEqual(len(spans), 1) span = spans[0] - self.assertEqual(span.name, "GET /does-not-exist") + self.assertEqual(span.name, "GET") self.assertEqual(span.status.status_code, StatusCode.UNSET) self.assertSpanHasAttributes( span, @@ -177,7 +177,7 @@ def test_500(self): spans = self.memory_exporter.get_finished_spans() self.assertEqual(len(spans), 1) span = spans[0] - self.assertEqual(span.name, "ErrorResource.on_get") + self.assertEqual(span.name, "GET /error") self.assertFalse(span.status.is_ok) self.assertEqual(span.status.status_code, StatusCode.ERROR) self.assertEqual( @@ -206,6 +206,33 @@ def test_500(self): span.attributes[SpanAttributes.NET_PEER_IP], "127.0.0.1" ) + def test_url_template(self): + self.client().simulate_get("/user/123") + spans = self.memory_exporter.get_finished_spans() + self.assertEqual(len(spans), 1) + span = spans[0] + self.assertEqual(span.name, "GET /user/{user_id}") + self.assertEqual(span.status.status_code, StatusCode.UNSET) + self.assertEqual( + span.status.description, + None, + ) + self.assertSpanHasAttributes( + span, + { + SpanAttributes.HTTP_METHOD: "GET", + SpanAttributes.HTTP_SERVER_NAME: "falconframework.org", + SpanAttributes.HTTP_SCHEME: "http", + SpanAttributes.NET_HOST_PORT: 80, + SpanAttributes.HTTP_HOST: "falconframework.org", + SpanAttributes.HTTP_TARGET: "/", + SpanAttributes.NET_PEER_PORT: "65133", + SpanAttributes.HTTP_FLAVOR: "1.1", + "falcon.resource": "UserResource", + SpanAttributes.HTTP_STATUS_CODE: 200, + }, + ) + def test_uninstrument(self): self.client().simulate_get(path="/hello") spans = self.memory_exporter.get_finished_spans() From ffc9334dd73d1247f28db92a143ec7cc91624995 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 20 Jun 2023 16:01:36 +0000 Subject: [PATCH 9/9] Bump requests from 2.28.1 to 2.31.0 (#1818) Bumps [requests](https://github.com/psf/requests) from 2.28.1 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.28.1...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Shalev Roda <65566801+shalevr@users.noreply.github.com> --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 8973fb9476..feab6b4b02 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -16,5 +16,5 @@ mypy-protobuf>=1.23 protobuf~=3.13 markupsafe>=2.0.1 codespell==2.1.0 -requests==2.28.1 +requests==2.31.0 ruamel.yaml==0.17.21