From 3fe0931cea53a22f20905cfe266d6f479cb3ba75 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Fri, 4 Nov 2022 17:03:53 -0700 Subject: [PATCH 1/3] Change queue url for botocore>=1.29.0 botocore >=1.29.0 uses sqs.us-east-1.amazonaws.com url instead of queue.amazonaws.com. --- tests/external_botocore/test_botocore_sqs.py | 120 +++++++++---------- tox.ini | 4 +- 2 files changed, 62 insertions(+), 62 deletions(-) diff --git a/tests/external_botocore/test_botocore_sqs.py b/tests/external_botocore/test_botocore_sqs.py index 46482c675..060c1e279 100644 --- a/tests/external_botocore/test_botocore_sqs.py +++ b/tests/external_botocore/test_botocore_sqs.py @@ -14,132 +14,130 @@ import sys import uuid -import pytest import botocore.session import moto +import pytest +from testing_support.fixtures import ( + override_application_settings, + validate_transaction_metrics, +) +from testing_support.validators.validate_span_events import validate_span_events from newrelic.api.background_task import background_task -from testing_support.fixtures import (validate_transaction_metrics, - override_application_settings) -from testing_support.validators.validate_span_events import ( - validate_span_events) +from newrelic.common.package_version_utils import get_package_version -MOTO_VERSION = tuple(int(v) for v in moto.__version__.split('.')[:3]) +MOTO_VERSION = tuple(int(v) for v in moto.__version__.split(".")[:3]) # patch earlier versions of moto to support py37 if sys.version_info >= (3, 7) and MOTO_VERSION <= (1, 3, 1): import re + moto.packages.responses.responses.re._pattern_type = re.Pattern -AWS_ACCESS_KEY_ID = 'AAAAAAAAAAAACCESSKEY' -AWS_SECRET_ACCESS_KEY = 'AAAAAASECRETKEY' -AWS_REGION = 'us-east-1' +url = "sqs.us-east-1.amazonaws.com" +if get_package_version("botocore") < "1.29.0": + url = "queue.amazonaws.com" + +AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY" +AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY" +AWS_REGION = "us-east-1" -TEST_QUEUE = 'python-agent-test-%s' % uuid.uuid4() +TEST_QUEUE = "python-agent-test-%s" % uuid.uuid4() _sqs_scoped_metrics = [ - ('MessageBroker/SQS/Queue/Produce/Named/%s' - % TEST_QUEUE, 2), - ('External/queue.amazonaws.com/botocore/POST', 3), + ("MessageBroker/SQS/Queue/Produce/Named/%s" % TEST_QUEUE, 2), + ("External/%s/botocore/POST" % url, 3), ] _sqs_rollup_metrics = [ - ('MessageBroker/SQS/Queue/Produce/Named/%s' - % TEST_QUEUE, 2), - ('MessageBroker/SQS/Queue/Consume/Named/%s' - % TEST_QUEUE, 1), - ('External/all', 3), - ('External/allOther', 3), - ('External/queue.amazonaws.com/all', 3), - ('External/queue.amazonaws.com/botocore/POST', 3), + ("MessageBroker/SQS/Queue/Produce/Named/%s" % TEST_QUEUE, 2), + ("MessageBroker/SQS/Queue/Consume/Named/%s" % TEST_QUEUE, 1), + ("External/all", 3), + ("External/allOther", 3), + ("External/%s/all" % url, 3), + ("External/%s/botocore/POST" % url, 3), ] _sqs_scoped_metrics_malformed = [ - ('MessageBroker/SQS/Queue/Produce/Named/Unknown', 1), + ("MessageBroker/SQS/Queue/Produce/Named/Unknown", 1), ] _sqs_rollup_metrics_malformed = [ - ('MessageBroker/SQS/Queue/Produce/Named/Unknown', 1), + ("MessageBroker/SQS/Queue/Produce/Named/Unknown", 1), ] -@override_application_settings({'distributed_tracing.enabled': True}) -@validate_span_events(exact_agents={'aws.operation': 'CreateQueue'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'SendMessage'}, count=1) -@validate_span_events( - exact_agents={'aws.operation': 'ReceiveMessage'}, count=1) -@validate_span_events( - exact_agents={'aws.operation': 'SendMessageBatch'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'PurgeQueue'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'DeleteQueue'}, count=1) +@override_application_settings({"distributed_tracing.enabled": True}) +@validate_span_events(exact_agents={"aws.operation": "CreateQueue"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "SendMessage"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "ReceiveMessage"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "SendMessageBatch"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "PurgeQueue"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "DeleteQueue"}, count=1) @validate_transaction_metrics( - 'test_botocore_sqs:test_sqs', - scoped_metrics=_sqs_scoped_metrics, - rollup_metrics=_sqs_rollup_metrics, - background_task=True) + "test_botocore_sqs:test_sqs", + scoped_metrics=_sqs_scoped_metrics, + rollup_metrics=_sqs_rollup_metrics, + background_task=True, +) @background_task() @moto.mock_sqs def test_sqs(): session = botocore.session.get_session() client = session.create_client( - 'sqs', - region_name=AWS_REGION, - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY + "sqs", region_name=AWS_REGION, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY ) # Create queue resp = client.create_queue(QueueName=TEST_QUEUE) - assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # QueueUrl is needed for rest of methods. - QUEUE_URL = resp['QueueUrl'] + QUEUE_URL = resp["QueueUrl"] # Send message - resp = client.send_message(QueueUrl=QUEUE_URL, MessageBody='hello_world') - assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 + resp = client.send_message(QueueUrl=QUEUE_URL, MessageBody="hello_world") + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # Receive message resp = client.receive_message(QueueUrl=QUEUE_URL) - assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # Send message batch messages = [ - {'Id': '1', 'MessageBody': 'message 1'}, - {'Id': '2', 'MessageBody': 'message 2'}, - {'Id': '3', 'MessageBody': 'message 3'}, + {"Id": "1", "MessageBody": "message 1"}, + {"Id": "2", "MessageBody": "message 2"}, + {"Id": "3", "MessageBody": "message 3"}, ] resp = client.send_message_batch(QueueUrl=QUEUE_URL, Entries=messages) - assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # Purge queue resp = client.purge_queue(QueueUrl=QUEUE_URL) - assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # Delete queue resp = client.delete_queue(QueueUrl=QUEUE_URL) - assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 -@override_application_settings({'distributed_tracing.enabled': True}) +@override_application_settings({"distributed_tracing.enabled": True}) @validate_transaction_metrics( - 'test_botocore_sqs:test_sqs_malformed', - scoped_metrics=_sqs_scoped_metrics_malformed, - rollup_metrics=_sqs_rollup_metrics_malformed, - background_task=True) + "test_botocore_sqs:test_sqs_malformed", + scoped_metrics=_sqs_scoped_metrics_malformed, + rollup_metrics=_sqs_rollup_metrics_malformed, + background_task=True, +) @background_task() @moto.mock_sqs def test_sqs_malformed(): session = botocore.session.get_session() client = session.create_client( - 'sqs', - region_name=AWS_REGION, - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY + "sqs", region_name=AWS_REGION, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY ) # Malformed send message, uses arg instead of kwarg with pytest.raises(TypeError): - client.send_message('https://fake-url/', MessageBody='hello_world') + client.send_message("https://fake-url/", MessageBody="hello_world") diff --git a/tox.ini b/tox.ini index e06279107..7834a3998 100644 --- a/tox.ini +++ b/tox.ini @@ -96,7 +96,8 @@ envlist = python-datastore_sqlite-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, memcached-datastore_umemcache-{py27,pypy}, python-external_boto3-{py27,py37,py38,py39,py310,py311}-boto01, - python-external_botocore-{py27,py37,py38,py39,py310,py311}-botocorelatest, + python-external_botocore-{py37,py38,py39,py310,py311}-botocorelatest, + python-external_botocore-{py311}-botocore129, python-external_botocore-py310-botocore0125, python-external_feedparser-py27-feedparser{05,06}, python-external_http-{py27,py37,py38,py39,py310,py311,pypy}, @@ -262,6 +263,7 @@ deps = external_boto3-boto01: moto<2.0 external_boto3-py27: rsa<4.7.1 external_botocore-botocorelatest: botocore + external_botocore-botocore129: botocore<1.29 external_botocore-botocore0125: botocore<1.26 external_botocore-{py37,py38,py39,py310,py311}: moto[awslambda,ec2,iam]<3.0 external_botocore-py27: rsa<4.7.1 From 7ce787931d835d4e0c578e54ed108cbfab51e723 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Mon, 7 Nov 2022 11:14:34 -0800 Subject: [PATCH 2/3] Use tuple version instead of str --- tests/external_botocore/test_botocore_sqs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/external_botocore/test_botocore_sqs.py b/tests/external_botocore/test_botocore_sqs.py index 060c1e279..a009e9f0b 100644 --- a/tests/external_botocore/test_botocore_sqs.py +++ b/tests/external_botocore/test_botocore_sqs.py @@ -36,7 +36,8 @@ moto.packages.responses.responses.re._pattern_type = re.Pattern url = "sqs.us-east-1.amazonaws.com" -if get_package_version("botocore") < "1.29.0": +botocore_version = tuple([int(n) for n in get_package_version("botocore").split(".")]) +if botocore_version < (1, 29, 0): url = "queue.amazonaws.com" AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY" From 29134a8a811fd15b1aaf4663e549818565d7771a Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Mon, 7 Nov 2022 12:01:13 -0800 Subject: [PATCH 3/3] Change botocore129->botocore128 --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 7834a3998..118c22acc 100644 --- a/tox.ini +++ b/tox.ini @@ -97,7 +97,7 @@ envlist = memcached-datastore_umemcache-{py27,pypy}, python-external_boto3-{py27,py37,py38,py39,py310,py311}-boto01, python-external_botocore-{py37,py38,py39,py310,py311}-botocorelatest, - python-external_botocore-{py311}-botocore129, + python-external_botocore-{py311}-botocore128, python-external_botocore-py310-botocore0125, python-external_feedparser-py27-feedparser{05,06}, python-external_http-{py27,py37,py38,py39,py310,py311,pypy}, @@ -263,7 +263,7 @@ deps = external_boto3-boto01: moto<2.0 external_boto3-py27: rsa<4.7.1 external_botocore-botocorelatest: botocore - external_botocore-botocore129: botocore<1.29 + external_botocore-botocore128: botocore<1.29 external_botocore-botocore0125: botocore<1.26 external_botocore-{py37,py38,py39,py310,py311}: moto[awslambda,ec2,iam]<3.0 external_botocore-py27: rsa<4.7.1