diff --git a/.github/actions/update-rpm-config/action.yml b/.github/actions/update-rpm-config/action.yml new file mode 100644 index 000000000..9d19ebba0 --- /dev/null +++ b/.github/actions/update-rpm-config/action.yml @@ -0,0 +1,109 @@ +name: "update-rpm-config" +description: "Set current version of agent in rpm config using API." +inputs: + agent-language: + description: "Language agent to configure (eg. python)" + required: true + default: "python" + target-system: + description: "Target System: prod|staging|all" + required: true + default: "all" + agent-version: + description: "3-4 digit agent version number (eg. 1.2.3) with optional leading v (ignored)" + required: true + dry-run: + description: "Dry Run" + required: true + default: "false" + production-api-key: + description: "API key for New Relic Production" + required: false + staging-api-key: + description: "API key for New Relic Staging" + required: false + +runs: + using: "composite" + steps: + - name: Trim potential leading v from agent version + shell: bash + run: | + AGENT_VERSION=${{ inputs.agent-version }} + echo "AGENT_VERSION=${AGENT_VERSION#"v"}" >> $GITHUB_ENV + + - name: Generate Payload + shell: bash + run: | + echo "PAYLOAD='{ \"system_configuration\": { \"key\": \"${{ inputs.agent-language }}_agent_version\", \"value\": \"${{ env.AGENT_VERSION }}\" } }'" >> $GITHUB_ENV + + - name: Generate Content-Type + shell: bash + run: | + echo "CONTENT_TYPE='Content-Type: application/json'" >> $GITHUB_ENV + + - name: Update Staging system configuration page + shell: bash + if: ${{ inputs.dry-run == 'false' && (inputs.target-system == 'staging' || inputs.target-system == 'all') }} + run: | + curl -X POST 'https://staging-api.newrelic.com/v2/system_configuration.json' \ + -H "X-Api-Key:${{ inputs.staging-api-key }}" -i \ + -H ${{ env.CONTENT_TYPE }} \ + -d ${{ env.PAYLOAD }} + + - name: Update Production system configuration page + shell: bash + if: ${{ inputs.dry-run == 'false' && (inputs.target-system == 'prod' || inputs.target-system == 'all') }} + run: | + curl -X POST 'https://api.newrelic.com/v2/system_configuration.json' \ + -H "X-Api-Key:${{ inputs.production-api-key }}" -i \ + -H ${{ env.CONTENT_TYPE }} \ + -d ${{ env.PAYLOAD }} + + - name: Verify Staging system configuration update + shell: bash + if: ${{ inputs.dry-run == 'false' && (inputs.target-system == 'staging' || inputs.target-system == 'all') }} + run: | + STAGING_VERSION=$(curl -X GET 'https://staging-api.newrelic.com/v2/system_configuration.json' \ + -H "X-Api-Key:${{ inputs.staging-api-key }}" \ + -H "${{ env.CONTENT_TYPE }}" | jq ".system_configurations | from_entries | .${{inputs.agent-language}}_agent_version") + + if [ "${{ env.AGENT_VERSION }}" != "$STAGING_VERSION" ]; then + echo "Staging version mismatch: $STAGING_VERSION" + exit 1 + fi + + - name: Verify Production system configuration update + shell: bash + if: ${{ inputs.dry-run == 'false' && (inputs.target-system == 'prod' || inputs.target-system == 'all') }} + run: | + PROD_VERSION=$(curl -X GET 'https://api.newrelic.com/v2/system_configuration.json' \ + -H "X-Api-Key:${{ inputs.production-api-key }}" \ + -H "${{ env.CONTENT_TYPE }}" | jq ".system_configurations | from_entries | .${{inputs.agent-language}}_agent_version") + + if [ "${{ env.AGENT_VERSION }}" != "$PROD_VERSION" ]; then + echo "Production version mismatch: $PROD_VERSION" + exit 1 + fi + + - name: (dry-run) Update Staging system configuration page + shell: bash + if: ${{ inputs.dry-run != 'false' && (inputs.target-system == 'staging' || inputs.target-system == 'all') }} + run: | + cat << EOF + curl -X POST 'https://staging-api.newrelic.com/v2/system_configuration.json' \ + -H "X-Api-Key:**REDACTED**" -i \ + -H ${{ env.CONTENT_TYPE }} \ + -d ${{ env.PAYLOAD }} + EOF + + - name: (dry-run) Update Production system configuration page + shell: bash + if: ${{ inputs.dry-run != 'false' && (inputs.target-system == 'prod' || inputs.target-system == 'all') }} + run: | + cat << EOF + curl -X POST 'https://api.newrelic.com/v2/system_configuration.json' \ + -H "X-Api-Key:**REDACTED**" -i \ + -H ${{ env.CONTENT_TYPE }} \ + -d ${{ env.PAYLOAD }} + EOF diff --git a/.github/workflows/deploy-python.yml b/.github/workflows/deploy-python.yml index fe16ee485..ca908b825 100644 --- a/.github/workflows/deploy-python.yml +++ b/.github/workflows/deploy-python.yml @@ -80,3 +80,13 @@ jobs: env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + + - name: Update RPM Config + uses: ./.github/actions/update-rpm-config + with: + agent-language: "python" + target-system: "all" + agent-version: "${{ github.ref_name }}" + dry-run: "false" + production-api-key: ${{ secrets.NEW_RELIC_API_KEY_PRODUCTION }}" + staging-api-key: ${{ secrets.NEW_RELIC_API_KEY_STAGING }}" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e3b264a9f..402d0c629 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -62,7 +62,6 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 with: python-version: "3.10" diff --git a/newrelic/common/package_version_utils.py b/newrelic/common/package_version_utils.py index 3152342b4..68320b897 100644 --- a/newrelic/common/package_version_utils.py +++ b/newrelic/common/package_version_utils.py @@ -14,6 +14,44 @@ import sys +try: + from functools import cache as _cache_package_versions +except ImportError: + from functools import wraps + from threading import Lock + + _package_version_cache = {} + _package_version_cache_lock = Lock() + + def _cache_package_versions(wrapped): + """ + Threadsafe implementation of caching for _get_package_version. + + Python 2.7 does not have the @functools.cache decorator, and + must be reimplemented with support for clearing the cache. + """ + + @wraps(wrapped) + def _wrapper(name): + if name in _package_version_cache: + return _package_version_cache[name] + + with _package_version_cache_lock: + if name in _package_version_cache: + return _package_version_cache[name] + + version = _package_version_cache[name] = wrapped(name) + return version + + def cache_clear(): + """Cache clear function to mimic @functools.cache""" + with _package_version_cache_lock: + _package_version_cache.clear() + + _wrapper.cache_clear = cache_clear + return _wrapper + + # Need to account for 4 possible variations of version declaration specified in (rejected) PEP 396 VERSION_ATTRS = ("__version__", "version", "__version_tuple__", "version_tuple") # nosec NULL_VERSIONS = frozenset((None, "", "0", "0.0", "0.0.0", "0.0.0.0", (0,), (0, 0), (0, 0, 0), (0, 0, 0, 0))) # nosec @@ -67,6 +105,7 @@ def int_or_str(value): return version +@_cache_package_versions def _get_package_version(name): module = sys.modules.get(name, None) version = None @@ -75,7 +114,7 @@ def _get_package_version(name): if "importlib" in sys.modules and hasattr(sys.modules["importlib"], "metadata"): try: # In Python3.10+ packages_distribution can be checked for as well - if hasattr(sys.modules["importlib"].metadata, "packages_distributions"): # pylint: disable=E1101 + if hasattr(sys.modules["importlib"].metadata, "packages_distributions"): # pylint: disable=E1101 distributions = sys.modules["importlib"].metadata.packages_distributions() # pylint: disable=E1101 distribution_name = distributions.get(name, name) else: diff --git a/newrelic/hooks/datastore_redis.py b/newrelic/hooks/datastore_redis.py index 6ba192002..bbc517586 100644 --- a/newrelic/hooks/datastore_redis.py +++ b/newrelic/hooks/datastore_redis.py @@ -14,10 +14,11 @@ import re -from newrelic.api.datastore_trace import DatastoreTrace +from newrelic.api.datastore_trace import DatastoreTrace, DatastoreTraceWrapper, wrap_datastore_trace from newrelic.api.time_trace import current_trace from newrelic.api.transaction import current_transaction -from newrelic.common.object_wrapper import function_wrapper, wrap_function_wrapper +from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.common.async_wrapper import coroutine_wrapper, async_generator_wrapper, generator_wrapper _redis_client_sync_methods = { "acl_dryrun", @@ -136,6 +137,7 @@ "client_no_evict", "client_pause", "client_reply", + "client_setinfo", "client_setname", "client_tracking", "client_trackinginfo", @@ -162,7 +164,6 @@ "cluster_reset", "cluster_save_config", "cluster_set_config_epoch", - "client_setinfo", "cluster_setslot", "cluster_slaves", "cluster_slots", @@ -248,7 +249,7 @@ "hmset_dict", "hmset", "hrandfield", - "hscan_inter", + "hscan_iter", "hscan", "hset", "hsetnx", @@ -399,8 +400,8 @@ "syndump", "synupdate", "tagvals", - "tfcall", "tfcall_async", + "tfcall", "tfunction_delete", "tfunction_list", "tfunction_load", @@ -473,6 +474,13 @@ "zunionstore", } +_redis_client_gen_methods = { + "scan_iter", + "hscan_iter", + "sscan_iter", + "zscan_iter", +} + _redis_client_methods = _redis_client_sync_methods.union(_redis_client_async_methods) _redis_multipart_commands = set(["client", "cluster", "command", "config", "debug", "sentinel", "slowlog", "script"]) @@ -498,50 +506,31 @@ def _instance_info(kwargs): def _wrap_Redis_method_wrapper_(module, instance_class_name, operation): - def _nr_wrapper_Redis_method_(wrapped, instance, args, kwargs): - transaction = current_transaction() - - if transaction is None: - return wrapped(*args, **kwargs) - - dt = DatastoreTrace(product="Redis", target=None, operation=operation, source=wrapped) - - transaction._nr_datastore_instance_info = (None, None, None) - - with dt: - result = wrapped(*args, **kwargs) - - host, port_path_or_id, db = transaction._nr_datastore_instance_info - dt.host = host - dt.port_path_or_id = port_path_or_id - dt.database_name = db - - return result - name = "%s.%s" % (instance_class_name, operation) - wrap_function_wrapper(module, name, _nr_wrapper_Redis_method_) + if operation in _redis_client_gen_methods: + async_wrapper = generator_wrapper + else: + async_wrapper = None + wrap_datastore_trace(module, name, product="Redis", target=None, operation=operation, async_wrapper=async_wrapper) -def _wrap_asyncio_Redis_method_wrapper(module, instance_class_name, operation): - @function_wrapper - async def _nr_wrapper_asyncio_Redis_async_method_(wrapped, instance, args, kwargs): - transaction = current_transaction() - if transaction is None: - return await wrapped(*args, **kwargs) - - with DatastoreTrace(product="Redis", target=None, operation=operation): - return await wrapped(*args, **kwargs) +def _wrap_asyncio_Redis_method_wrapper(module, instance_class_name, operation): def _nr_wrapper_asyncio_Redis_method_(wrapped, instance, args, kwargs): from redis.asyncio.client import Pipeline if isinstance(instance, Pipeline): return wrapped(*args, **kwargs) - # Method should be run when awaited, therefore we wrap in an async wrapper. - return _nr_wrapper_asyncio_Redis_async_method_(wrapped)(*args, **kwargs) + # Method should be run when awaited or iterated, therefore we wrap in an async wrapper. + return DatastoreTraceWrapper(wrapped, product="Redis", target=None, operation=operation, async_wrapper=async_wrapper)(*args, **kwargs) name = "%s.%s" % (instance_class_name, operation) + if operation in _redis_client_gen_methods: + async_wrapper = async_generator_wrapper + else: + async_wrapper = coroutine_wrapper + wrap_function_wrapper(module, name, _nr_wrapper_asyncio_Redis_method_) @@ -614,7 +603,15 @@ def _nr_Connection_send_command_wrapper_(wrapped, instance, args, kwargs): except: pass - transaction._nr_datastore_instance_info = (host, port_path_or_id, db) + # Find DatastoreTrace no matter how many other traces are inbetween + trace = current_trace() + while trace is not None and not isinstance(trace, DatastoreTrace): + trace = getattr(trace, "parent", None) + + if trace is not None: + trace.host = host + trace.port_path_or_id = port_path_or_id + trace.database_name = db # Older Redis clients would when sending multi part commands pass # them in as separate arguments to send_command(). Need to therefore @@ -666,7 +663,6 @@ def instrument_asyncio_redis_client(module): if hasattr(class_, operation): _wrap_asyncio_Redis_method_wrapper(module, "Redis", operation) - def instrument_redis_commands_core(module): _instrument_redis_commands_module(module, "CoreCommands") diff --git a/tests/agent_unittests/test_package_version_utils.py b/tests/agent_unittests/test_package_version_utils.py index 30c22cff1..5ed689ea2 100644 --- a/tests/agent_unittests/test_package_version_utils.py +++ b/tests/agent_unittests/test_package_version_utils.py @@ -20,6 +20,7 @@ from newrelic.common.package_version_utils import ( NULL_VERSIONS, VERSION_ATTRS, + _get_package_version, get_package_version, get_package_version_tuple, ) @@ -31,7 +32,7 @@ # such as distribution_packages and removed pkg_resources. IS_PY38_PLUS = sys.version_info[:2] >= (3, 8) -IS_PY310_PLUS = sys.version_info[:2] >= (3,10) +IS_PY310_PLUS = sys.version_info[:2] >= (3, 10) SKIP_IF_NOT_IMPORTLIB_METADATA = pytest.mark.skipif(not IS_PY38_PLUS, reason="importlib.metadata is not supported.") SKIP_IF_IMPORTLIB_METADATA = pytest.mark.skipif( IS_PY38_PLUS, reason="importlib.metadata is preferred over pkg_resources." @@ -46,7 +47,13 @@ def patched_pytest_module(monkeypatch): monkeypatch.delattr(pytest, attr) yield pytest - + + +@pytest.fixture(scope="function", autouse=True) +def cleared_package_version_cache(): + """Ensure cache is empty before every test to exercise code paths.""" + _get_package_version.cache_clear() + # This test only works on Python 3.7 @SKIP_IF_IMPORTLIB_METADATA @@ -123,3 +130,16 @@ def test_mapping_import_to_distribution_packages(): def test_pkg_resources_metadata(): version = get_package_version("pytest") assert version not in NULL_VERSIONS, version + + +def test_version_caching(monkeypatch): + # Add fake module to be deleted later + sys.modules["mymodule"] = sys.modules["pytest"] + setattr(pytest, "__version__", "1.0.0") + version = get_package_version("mymodule") + assert version not in NULL_VERSIONS, version + + # Ensure after deleting that the call to _get_package_version still completes because of caching + del sys.modules["mymodule"] + version = get_package_version("mymodule") + assert version not in NULL_VERSIONS, version diff --git a/tests/datastore_redis/conftest.py b/tests/datastore_redis/conftest.py index 53ff2658d..6747039b4 100644 --- a/tests/datastore_redis/conftest.py +++ b/tests/datastore_redis/conftest.py @@ -15,6 +15,7 @@ import pytest from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixture.event_loop import event_loop as loop # noqa: F401; pylint: disable=W0611 _default_settings = { diff --git a/tests/datastore_redis/test_generators.py b/tests/datastore_redis/test_generators.py new file mode 100644 index 000000000..f747838e1 --- /dev/null +++ b/tests/datastore_redis/test_generators.py @@ -0,0 +1,258 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import redis +from testing_support.db_settings import redis_settings +from testing_support.fixtures import override_application_settings +from testing_support.util import instance_hostname +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.api.datastore_trace import DatastoreTrace +from newrelic.api.time_trace import current_trace +from newrelic.common.package_version_utils import get_package_version_tuple + +DB_SETTINGS = redis_settings()[0] +REDIS_PY_VERSION = get_package_version_tuple("redis") + +# Settings + +_enable_instance_settings = { + "datastore_tracer.instance_reporting.enabled": True, +} +_disable_instance_settings = { + "datastore_tracer.instance_reporting.enabled": False, +} + +# Metrics + +_base_scoped_metrics = ( + ("Datastore/operation/Redis/scan_iter", 1), + ("Datastore/operation/Redis/sscan_iter", 1), + ("Datastore/operation/Redis/zscan_iter", 1), + ("Datastore/operation/Redis/hscan_iter", 1), + ("Datastore/operation/Redis/set", 1), + ("Datastore/operation/Redis/sadd", 1), + ("Datastore/operation/Redis/zadd", 1), + ("Datastore/operation/Redis/hset", 1), +) + +_base_rollup_metrics = ( + ("Datastore/all", 8), + ("Datastore/allOther", 8), + ("Datastore/Redis/all", 8), + ("Datastore/Redis/allOther", 8), + ("Datastore/operation/Redis/scan_iter", 1), + ("Datastore/operation/Redis/sscan_iter", 1), + ("Datastore/operation/Redis/zscan_iter", 1), + ("Datastore/operation/Redis/hscan_iter", 1), + ("Datastore/operation/Redis/set", 1), + ("Datastore/operation/Redis/sadd", 1), + ("Datastore/operation/Redis/zadd", 1), + ("Datastore/operation/Redis/hset", 1), +) + +_disable_rollup_metrics = list(_base_rollup_metrics) +_enable_rollup_metrics = list(_base_rollup_metrics) + +_host = instance_hostname(DB_SETTINGS["host"]) +_port = DB_SETTINGS["port"] + +_instance_metric_name = "Datastore/instance/Redis/%s/%s" % (_host, _port) + +_enable_rollup_metrics.append((_instance_metric_name, 8)) + +_disable_rollup_metrics.append((_instance_metric_name, None)) + +# Operations + + +def exercise_redis(client): + """ + Exercise client generators by iterating on various methods and ensuring they are + non-empty, and that traces are started and stopped with the generator. + """ + + # Set existing values + client.set("scan-key", "value") + client.sadd("sscan-key", "value") + client.zadd("zscan-key", {"value": 1}) + client.hset("hscan-key", "field", "value") + + # Check generators + flag = False + assert not isinstance(current_trace(), DatastoreTrace) # Assert no active DatastoreTrace + for k in client.scan_iter("scan-*"): + assert k == b"scan-key" + assert isinstance(current_trace(), DatastoreTrace) # Assert DatastoreTrace now active + flag = True + assert flag + + flag = False + assert not isinstance(current_trace(), DatastoreTrace) # Assert no active DatastoreTrace + for k in client.sscan_iter("sscan-key"): + assert k == b"value" + assert isinstance(current_trace(), DatastoreTrace) # Assert DatastoreTrace now active + flag = True + assert flag + + flag = False + assert not isinstance(current_trace(), DatastoreTrace) # Assert no active DatastoreTrace + for k, _ in client.zscan_iter("zscan-key"): + assert k == b"value" + assert isinstance(current_trace(), DatastoreTrace) # Assert DatastoreTrace now active + flag = True + assert flag + + flag = False + assert not isinstance(current_trace(), DatastoreTrace) # Assert no active DatastoreTrace + for f, v in client.hscan_iter("hscan-key"): + assert f == b"field" + assert v == b"value" + assert isinstance(current_trace(), DatastoreTrace) # Assert DatastoreTrace now active + flag = True + assert flag + + +async def exercise_redis_async(client): + """ + Exercise client generators by iterating on various methods and ensuring they are + non-empty, and that traces are started and stopped with the generator. + """ + + # Set existing values + await client.set("scan-key", "value") + await client.sadd("sscan-key", "value") + await client.zadd("zscan-key", {"value": 1}) + await client.hset("hscan-key", "field", "value") + + # Check generators + flag = False + assert not isinstance(current_trace(), DatastoreTrace) # Assert no active DatastoreTrace + async for k in client.scan_iter("scan-*"): + assert k == b"scan-key" + assert isinstance(current_trace(), DatastoreTrace) # Assert DatastoreTrace now active + flag = True + assert flag + + flag = False + assert not isinstance(current_trace(), DatastoreTrace) # Assert no active DatastoreTrace + async for k in client.sscan_iter("sscan-key"): + assert k == b"value" + assert isinstance(current_trace(), DatastoreTrace) # Assert DatastoreTrace now active + flag = True + assert flag + + flag = False + assert not isinstance(current_trace(), DatastoreTrace) # Assert no active DatastoreTrace + async for k, _ in client.zscan_iter("zscan-key"): + assert k == b"value" + assert isinstance(current_trace(), DatastoreTrace) # Assert DatastoreTrace now active + flag = True + assert flag + + flag = False + assert not isinstance(current_trace(), DatastoreTrace) # Assert no active DatastoreTrace + async for f, v in client.hscan_iter("hscan-key"): + assert f == b"field" + assert v == b"value" + assert isinstance(current_trace(), DatastoreTrace) # Assert DatastoreTrace now active + flag = True + assert flag + + +# Tests + + +@override_application_settings(_enable_instance_settings) +@validate_transaction_metrics( + "test_generators:test_strict_redis_generator_enable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, +) +@background_task() +def test_strict_redis_generator_enable_instance(): + client = redis.StrictRedis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) + exercise_redis(client) + + +@override_application_settings(_disable_instance_settings) +@validate_transaction_metrics( + "test_generators:test_strict_redis_generator_disable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_disable_rollup_metrics, + background_task=True, +) +@background_task() +def test_strict_redis_generator_disable_instance(): + client = redis.StrictRedis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) + exercise_redis(client) + + +@override_application_settings(_enable_instance_settings) +@validate_transaction_metrics( + "test_generators:test_redis_generator_enable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, +) +@background_task() +def test_redis_generator_enable_instance(): + client = redis.Redis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) + exercise_redis(client) + + +@override_application_settings(_disable_instance_settings) +@validate_transaction_metrics( + "test_generators:test_redis_generator_disable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_disable_rollup_metrics, + background_task=True, +) +@background_task() +def test_redis_generator_disable_instance(): + client = redis.Redis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) + exercise_redis(client) + + +@pytest.mark.skipif(REDIS_PY_VERSION < (4, 2), reason="Redis.asyncio was not added until v4.2") +@override_application_settings(_enable_instance_settings) +@validate_transaction_metrics( + "test_generators:test_redis_async_generator_enable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, +) +@background_task() +def test_redis_async_generator_enable_instance(loop): + client = redis.asyncio.Redis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) + loop.run_until_complete(exercise_redis_async(client)) + + +@pytest.mark.skipif(REDIS_PY_VERSION < (4, 2), reason="Redis.asyncio was not added until v4.2") +@override_application_settings(_disable_instance_settings) +@validate_transaction_metrics( + "test_generators:test_redis_async_generator_disable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_disable_rollup_metrics, + background_task=True, +) +@background_task() +def test_redis_async_generator_disable_instance(loop): + client = redis.asyncio.Redis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) + loop.run_until_complete(exercise_redis_async(client)) diff --git a/tests/datastore_redis/test_uninstrumented_methods.py b/tests/datastore_redis/test_uninstrumented_methods.py index d86f4de95..c0be684b2 100644 --- a/tests/datastore_redis/test_uninstrumented_methods.py +++ b/tests/datastore_redis/test_uninstrumented_methods.py @@ -65,7 +65,6 @@ "get_property", "get_relation", "get_retry", - "hscan_iter", "index_name", "labels", "list_keys", diff --git a/tests/mlmodel_bedrock/_mock_external_bedrock_server.py b/tests/mlmodel_bedrock/_mock_external_bedrock_server.py new file mode 100644 index 000000000..3d200449b --- /dev/null +++ b/tests/mlmodel_bedrock/_mock_external_bedrock_server.py @@ -0,0 +1,2018 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import re + +from testing_support.mock_external_http_server import MockExternalHTTPServer + +# This defines an external server test apps can make requests to instead of +# the real Bedrock backend. This provides 3 features: +# +# 1) This removes dependencies on external websites. +# 2) Provides a better mechanism for making an external call in a test app than +# simple calling another endpoint the test app makes available because this +# server will not be instrumented meaning we don't have to sort through +# transactions to separate the ones created in the test app and the ones +# created by an external call. +# 3) This app runs on a separate thread meaning it won't block the test app. + +RESPONSES = { + "amazon.titan-text-express-v1::Command: Write me a blog about making strong business decisions as a leader.": [ + {}, + { + "inputTextTokenCount": 19, + "results": [ + { + "tokenCount": 128, + "outputText": " Making strong business decisions as a leader requires a combination of strategic thinking, data analysis, and intuition. Here are some tips to help you make informed and effective decisions:\nDefine your goals and vision: Clearly understand your organization's goals and vision, and ensure that all decision-making aligns with these objectives. This will provide a roadmap for your decisions and help you stay focused on the bigger picture.\nGather relevant data and information: Collect and analyze data related to the decision you need to make. Consider multiple sources of information, such as market trends, financial reports, and stakeholder feedback. Use data to inform your decision-making process", + "completionReason": "LENGTH", + } + ], + }, + ], + "anthropic.claude-instant-v1::Human: Write me a blog about making strong business decisions as a leader.": [ + {}, + { + "completion": " Here is a draft blog post on making strong business decisions as a leader:\n\nTitle: 5 Tips for Making Strong Business Decisions as a Leader\n\nBeing a leader means that tough business decisions will inevitably land on your desk. How you handle those decisions can have a huge impact on your company's success. Here are some tips to help you make strong, informed choices that move your business in the right direction.\n\n1. Gather all relevant data. Don't make a call until you've examined the issue from every angle. Seek out useful metrics, get feedback from various stakeholders, and look at historical trends and precedents. The more data you have, the clearer the right path will become. \n\n2. Consider both short and long-term implications. While it's important to address immediate needs, don't lose sight of how a decision may impact the future. Will your choice simply solve today's problem or help build sustainable growth? Carefully weigh short-term gains against potential long-term consequences.\n\n3. Trust your instincts but don't decide alone. Your gut feelings are valuable, but they shouldn't be the sole basis for a leadership decision. Consult with your management team and get differing perspectives. Encourage respectful debate to surface any risks or uncertainties that need discussion. \n\n4. Be willing todelaya decisionif youneed moretime.There'snobenefittomakingarushjudgement beforeallfactorshavebeenweighed.It'sbettertoletyourdecision\"bake\"athirdopinionormoredataratherthanpotentiallyregrettingahastycalllater.\n\n5. Follow through on the outcome. A good decision means little without effective implementation. Clearly communicate the rationale for your choice and gain organizational buy-in. Then follow up to ensure your solution is executed properly and intended goals are achieved. Are any adjustments needed along the way? \n\nLeaders are entrusted to make the calls that steer a business. With care, research and an open yet discerning approach, you can make decisions that propel your company confidently into the future.", + "stop_reason": "stop_sequence", + }, + ], + "ai21.j2-mid-v1::Write me a blog about making strong business decisions as a leader.": [ + {}, + { + "id": 1234, + "prompt": { + "text": "Write me a blog about making strong business decisions as a leader.", + "tokens": [ + { + "generatedToken": { + "token": "\u2581Write", + "logprob": -10.650314331054688, + "raw_logprob": -10.650314331054688, + }, + "topTokens": None, + "textRange": {"start": 0, "end": 5}, + }, + { + "generatedToken": { + "token": "\u2581me", + "logprob": -5.457987308502197, + "raw_logprob": -5.457987308502197, + }, + "topTokens": None, + "textRange": {"start": 5, "end": 8}, + }, + { + "generatedToken": { + "token": "\u2581a\u2581blog", + "logprob": -8.36896800994873, + "raw_logprob": -8.36896800994873, + }, + "topTokens": None, + "textRange": {"start": 8, "end": 15}, + }, + { + "generatedToken": { + "token": "\u2581about\u2581making", + "logprob": -14.223419189453125, + "raw_logprob": -14.223419189453125, + }, + "topTokens": None, + "textRange": {"start": 15, "end": 28}, + }, + { + "generatedToken": { + "token": "\u2581strong", + "logprob": -9.367725372314453, + "raw_logprob": -9.367725372314453, + }, + "topTokens": None, + "textRange": {"start": 28, "end": 35}, + }, + { + "generatedToken": { + "token": "\u2581business\u2581decisions", + "logprob": -7.66295862197876, + "raw_logprob": -7.66295862197876, + }, + "topTokens": None, + "textRange": {"start": 35, "end": 54}, + }, + { + "generatedToken": { + "token": "\u2581as\u2581a\u2581leader", + "logprob": -13.765915870666504, + "raw_logprob": -13.765915870666504, + }, + "topTokens": None, + "textRange": {"start": 54, "end": 66}, + }, + { + "generatedToken": { + "token": ".", + "logprob": -10.953210830688477, + "raw_logprob": -10.953210830688477, + }, + "topTokens": None, + "textRange": {"start": 66, "end": 67}, + }, + ], + }, + "completions": [ + { + "data": { + "text": "\nWhen you are a leader at work, you need to make timely and informed decisions on behalf of your team or company. You have to consider multiple factors and variables, and analyze data in a way to make the best possible choice.\n\nHowever, sometimes things don't turn out the way you intended. Your decision might not work as intended, or act in unforeseen ways. Or, you might find new information or context that causes you to question your decision. That's okay.\n\nIt's important to have courage when you're a leader. This means being willing to think critically, reflect, learn from mistakes, and take action steps moving forward.\n\nThere are three steps that can help you grow as a leader and make better business decisions:\n\nStep 1: Gather information\n\nThe first step to making a good decision is to make sure that you have all of the facts. It's important to know what information you need, and from where to get it.\n\nYou can gather information by doing things like reading reports, talking to stakeholders, and conducting research.\n\nStep 2: Analyze information\n\nOnce you've gathered all of your information, you need to take some time to think about it. You need to analyze the data and identify patterns, trends, and trends that might not be immediately obvious.\n\nThere are a few things you should keep in mind when you're analyzing information:\n\n* Identify the key points: What are the key takeaways from this information? What", + "tokens": [ + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -0.00011955977242905647, + "raw_logprob": -0.00011955977242905647, + }, + "topTokens": None, + "textRange": {"start": 0, "end": 1}, + }, + { + "generatedToken": { + "token": "\u2581When\u2581you\u2581are", + "logprob": -6.066172122955322, + "raw_logprob": -6.066172122955322, + }, + "topTokens": None, + "textRange": {"start": 1, "end": 13}, + }, + { + "generatedToken": { + "token": "\u2581a\u2581leader", + "logprob": -0.8404027223587036, + "raw_logprob": -0.8404027223587036, + }, + "topTokens": None, + "textRange": {"start": 13, "end": 22}, + }, + { + "generatedToken": { + "token": "\u2581at\u2581work", + "logprob": -8.004234313964844, + "raw_logprob": -8.004234313964844, + }, + "topTokens": None, + "textRange": {"start": 22, "end": 30}, + }, + { + "generatedToken": { + "token": ",", + "logprob": -0.07083408534526825, + "raw_logprob": -0.07083408534526825, + }, + "topTokens": None, + "textRange": {"start": 30, "end": 31}, + }, + { + "generatedToken": { + "token": "\u2581you\u2581need\u2581to\u2581make", + "logprob": -2.5708985328674316, + "raw_logprob": -2.5708985328674316, + }, + "topTokens": None, + "textRange": {"start": 31, "end": 48}, + }, + { + "generatedToken": { + "token": "\u2581timely", + "logprob": -9.624330520629883, + "raw_logprob": -9.624330520629883, + }, + "topTokens": None, + "textRange": {"start": 48, "end": 55}, + }, + { + "generatedToken": { + "token": "\u2581and", + "logprob": -1.5508010387420654, + "raw_logprob": -1.5508010387420654, + }, + "topTokens": None, + "textRange": {"start": 55, "end": 59}, + }, + { + "generatedToken": { + "token": "\u2581informed\u2581decisions", + "logprob": -0.5989360809326172, + "raw_logprob": -0.5989360809326172, + }, + "topTokens": None, + "textRange": {"start": 59, "end": 78}, + }, + { + "generatedToken": { + "token": "\u2581on\u2581behalf\u2581of", + "logprob": -5.749756336212158, + "raw_logprob": -5.749756336212158, + }, + "topTokens": None, + "textRange": {"start": 78, "end": 91}, + }, + { + "generatedToken": { + "token": "\u2581your\u2581team", + "logprob": -0.29448866844177246, + "raw_logprob": -0.29448866844177246, + }, + "topTokens": None, + "textRange": {"start": 91, "end": 101}, + }, + { + "generatedToken": { + "token": "\u2581or", + "logprob": -2.9078853130340576, + "raw_logprob": -2.9078853130340576, + }, + "topTokens": None, + "textRange": {"start": 101, "end": 104}, + }, + { + "generatedToken": { + "token": "\u2581company", + "logprob": -0.4439607262611389, + "raw_logprob": -0.4439607262611389, + }, + "topTokens": None, + "textRange": {"start": 104, "end": 112}, + }, + { + "generatedToken": { + "token": ".", + "logprob": -0.004392143338918686, + "raw_logprob": -0.004392143338918686, + }, + "topTokens": None, + "textRange": {"start": 112, "end": 113}, + }, + { + "generatedToken": { + "token": "\u2581You\u2581have", + "logprob": -6.982149600982666, + "raw_logprob": -6.982149600982666, + }, + "topTokens": None, + "textRange": {"start": 113, "end": 122}, + }, + { + "generatedToken": { + "token": "\u2581to\u2581consider", + "logprob": -2.413727283477783, + "raw_logprob": -2.413727283477783, + }, + "topTokens": None, + "textRange": {"start": 122, "end": 134}, + }, + { + "generatedToken": { + "token": "\u2581multiple", + "logprob": -2.61666202545166, + "raw_logprob": -2.61666202545166, + }, + "topTokens": None, + "textRange": {"start": 134, "end": 143}, + }, + { + "generatedToken": { + "token": "\u2581factors", + "logprob": -0.11320021003484726, + "raw_logprob": -0.11320021003484726, + }, + "topTokens": None, + "textRange": {"start": 143, "end": 151}, + }, + { + "generatedToken": { + "token": "\u2581and", + "logprob": -1.4593441486358643, + "raw_logprob": -1.4593441486358643, + }, + "topTokens": None, + "textRange": {"start": 151, "end": 155}, + }, + { + "generatedToken": { + "token": "\u2581variables", + "logprob": -2.3700382709503174, + "raw_logprob": -2.3700382709503174, + }, + "topTokens": None, + "textRange": {"start": 155, "end": 165}, + }, + { + "generatedToken": { + "token": ",", + "logprob": -0.9362450838088989, + "raw_logprob": -0.9362450838088989, + }, + "topTokens": None, + "textRange": {"start": 165, "end": 166}, + }, + { + "generatedToken": { + "token": "\u2581and\u2581analyze", + "logprob": -7.707818031311035, + "raw_logprob": -7.707818031311035, + }, + "topTokens": None, + "textRange": {"start": 166, "end": 178}, + }, + { + "generatedToken": { + "token": "\u2581data\u2581in", + "logprob": -7.114713668823242, + "raw_logprob": -7.114713668823242, + }, + "topTokens": None, + "textRange": {"start": 178, "end": 186}, + }, + { + "generatedToken": { + "token": "\u2581a\u2581way\u2581to\u2581make", + "logprob": -2.1352782249450684, + "raw_logprob": -2.1352782249450684, + }, + "topTokens": None, + "textRange": {"start": 186, "end": 200}, + }, + { + "generatedToken": { + "token": "\u2581the\u2581best\u2581possible", + "logprob": -1.202060341835022, + "raw_logprob": -1.202060341835022, + }, + "topTokens": None, + "textRange": {"start": 200, "end": 218}, + }, + { + "generatedToken": { + "token": "\u2581choice", + "logprob": -0.49673229455947876, + "raw_logprob": -0.49673229455947876, + }, + "topTokens": None, + "textRange": {"start": 218, "end": 225}, + }, + { + "generatedToken": { + "token": ".", + "logprob": -0.08440639078617096, + "raw_logprob": -0.08440639078617096, + }, + "topTokens": None, + "textRange": {"start": 225, "end": 226}, + }, + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -1.4274420738220215, + "raw_logprob": -1.4274420738220215, + }, + "topTokens": None, + "textRange": {"start": 226, "end": 227}, + }, + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -0.002755180699750781, + "raw_logprob": -0.002755180699750781, + }, + "topTokens": None, + "textRange": {"start": 227, "end": 228}, + }, + { + "generatedToken": { + "token": "\u2581However", + "logprob": -2.9974615573883057, + "raw_logprob": -2.9974615573883057, + }, + "topTokens": None, + "textRange": {"start": 228, "end": 235}, + }, + { + "generatedToken": { + "token": ",", + "logprob": -0.0017327546374872327, + "raw_logprob": -0.0017327546374872327, + }, + "topTokens": None, + "textRange": {"start": 235, "end": 236}, + }, + { + "generatedToken": { + "token": "\u2581sometimes", + "logprob": -2.893026113510132, + "raw_logprob": -2.893026113510132, + }, + "topTokens": None, + "textRange": {"start": 236, "end": 246}, + }, + { + "generatedToken": { + "token": "\u2581things", + "logprob": -4.238265037536621, + "raw_logprob": -4.238265037536621, + }, + "topTokens": None, + "textRange": {"start": 246, "end": 253}, + }, + { + "generatedToken": { + "token": "\u2581don't", + "logprob": -2.367069721221924, + "raw_logprob": -2.367069721221924, + }, + "topTokens": None, + "textRange": {"start": 253, "end": 259}, + }, + { + "generatedToken": { + "token": "\u2581turn\u2581out", + "logprob": -1.7048457860946655, + "raw_logprob": -1.7048457860946655, + }, + "topTokens": None, + "textRange": {"start": 259, "end": 268}, + }, + { + "generatedToken": { + "token": "\u2581the\u2581way\u2581you", + "logprob": -2.1934995651245117, + "raw_logprob": -2.1934995651245117, + }, + "topTokens": None, + "textRange": {"start": 268, "end": 280}, + }, + { + "generatedToken": { + "token": "\u2581intended", + "logprob": -3.7538819313049316, + "raw_logprob": -3.7538819313049316, + }, + "topTokens": None, + "textRange": {"start": 280, "end": 289}, + }, + { + "generatedToken": { + "token": ".", + "logprob": -0.41568616032600403, + "raw_logprob": -0.41568616032600403, + }, + "topTokens": None, + "textRange": {"start": 289, "end": 290}, + }, + { + "generatedToken": { + "token": "\u2581Your", + "logprob": -4.143064498901367, + "raw_logprob": -4.143064498901367, + }, + "topTokens": None, + "textRange": {"start": 290, "end": 295}, + }, + { + "generatedToken": { + "token": "\u2581decision", + "logprob": -1.1384129524230957, + "raw_logprob": -1.1384129524230957, + }, + "topTokens": None, + "textRange": {"start": 295, "end": 304}, + }, + { + "generatedToken": { + "token": "\u2581might\u2581not\u2581work", + "logprob": -2.4380242824554443, + "raw_logprob": -2.4380242824554443, + }, + "topTokens": None, + "textRange": {"start": 304, "end": 319}, + }, + { + "generatedToken": { + "token": "\u2581as\u2581intended", + "logprob": -2.9615366458892822, + "raw_logprob": -2.9615366458892822, + }, + "topTokens": None, + "textRange": {"start": 319, "end": 331}, + }, + { + "generatedToken": { + "token": ",", + "logprob": -0.22413745522499084, + "raw_logprob": -0.22413745522499084, + }, + "topTokens": None, + "textRange": {"start": 331, "end": 332}, + }, + { + "generatedToken": { + "token": "\u2581or", + "logprob": -0.4422154128551483, + "raw_logprob": -0.4422154128551483, + }, + "topTokens": None, + "textRange": {"start": 332, "end": 335}, + }, + { + "generatedToken": { + "token": "\u2581act\u2581in", + "logprob": -16.771242141723633, + "raw_logprob": -16.771242141723633, + }, + "topTokens": None, + "textRange": {"start": 335, "end": 342}, + }, + { + "generatedToken": { + "token": "\u2581unforeseen", + "logprob": -2.0343406200408936, + "raw_logprob": -2.0343406200408936, + }, + "topTokens": None, + "textRange": {"start": 342, "end": 353}, + }, + { + "generatedToken": { + "token": "\u2581ways", + "logprob": -0.03732850402593613, + "raw_logprob": -0.03732850402593613, + }, + "topTokens": None, + "textRange": {"start": 353, "end": 358}, + }, + { + "generatedToken": { + "token": ".", + "logprob": -0.07006527483463287, + "raw_logprob": -0.07006527483463287, + }, + "topTokens": None, + "textRange": {"start": 358, "end": 359}, + }, + { + "generatedToken": { + "token": "\u2581Or", + "logprob": -4.574007511138916, + "raw_logprob": -4.574007511138916, + }, + "topTokens": None, + "textRange": {"start": 359, "end": 362}, + }, + { + "generatedToken": { + "token": ",", + "logprob": -0.35941576957702637, + "raw_logprob": -0.35941576957702637, + }, + "topTokens": None, + "textRange": {"start": 362, "end": 363}, + }, + { + "generatedToken": { + "token": "\u2581you\u2581might\u2581find", + "logprob": -3.0860962867736816, + "raw_logprob": -3.0860962867736816, + }, + "topTokens": None, + "textRange": {"start": 363, "end": 378}, + }, + { + "generatedToken": { + "token": "\u2581new\u2581information", + "logprob": -3.0317506790161133, + "raw_logprob": -3.0317506790161133, + }, + "topTokens": None, + "textRange": {"start": 378, "end": 394}, + }, + { + "generatedToken": { + "token": "\u2581or", + "logprob": -3.251086950302124, + "raw_logprob": -3.251086950302124, + }, + "topTokens": None, + "textRange": {"start": 394, "end": 397}, + }, + { + "generatedToken": { + "token": "\u2581context", + "logprob": -4.189438343048096, + "raw_logprob": -4.189438343048096, + }, + "topTokens": None, + "textRange": {"start": 397, "end": 405}, + }, + { + "generatedToken": { + "token": "\u2581that\u2581causes", + "logprob": -4.464134216308594, + "raw_logprob": -4.464134216308594, + }, + "topTokens": None, + "textRange": {"start": 405, "end": 417}, + }, + { + "generatedToken": { + "token": "\u2581you", + "logprob": -0.2493533492088318, + "raw_logprob": -0.2493533492088318, + }, + "topTokens": None, + "textRange": {"start": 417, "end": 421}, + }, + { + "generatedToken": { + "token": "\u2581to\u2581question", + "logprob": -2.251695156097412, + "raw_logprob": -2.251695156097412, + }, + "topTokens": None, + "textRange": {"start": 421, "end": 433}, + }, + { + "generatedToken": { + "token": "\u2581your\u2581decision", + "logprob": -1.989322543144226, + "raw_logprob": -1.989322543144226, + }, + "topTokens": None, + "textRange": {"start": 433, "end": 447}, + }, + { + "generatedToken": { + "token": ".", + "logprob": -0.17142613232135773, + "raw_logprob": -0.17142613232135773, + }, + "topTokens": None, + "textRange": {"start": 447, "end": 448}, + }, + { + "generatedToken": { + "token": "\u2581That's", + "logprob": -5.326101303100586, + "raw_logprob": -5.326101303100586, + }, + "topTokens": None, + "textRange": {"start": 448, "end": 455}, + }, + { + "generatedToken": { + "token": "\u2581okay", + "logprob": -0.7236325740814209, + "raw_logprob": -0.7236325740814209, + }, + "topTokens": None, + "textRange": {"start": 455, "end": 460}, + }, + { + "generatedToken": { + "token": ".", + "logprob": -1.1485638618469238, + "raw_logprob": -1.1485638618469238, + }, + "topTokens": None, + "textRange": {"start": 460, "end": 461}, + }, + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -1.3378857374191284, + "raw_logprob": -1.3378857374191284, + }, + "topTokens": None, + "textRange": {"start": 461, "end": 462}, + }, + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -0.00016985881666187197, + "raw_logprob": -0.00016985881666187197, + }, + "topTokens": None, + "textRange": {"start": 462, "end": 463}, + }, + { + "generatedToken": { + "token": "\u2581It's\u2581important\u2581to", + "logprob": -3.5227854251861572, + "raw_logprob": -3.5227854251861572, + }, + "topTokens": None, + "textRange": {"start": 463, "end": 480}, + }, + { + "generatedToken": { + "token": "\u2581have", + "logprob": -2.9167816638946533, + "raw_logprob": -2.9167816638946533, + }, + "topTokens": None, + "textRange": {"start": 480, "end": 485}, + }, + { + "generatedToken": { + "token": "\u2581courage", + "logprob": -5.581697940826416, + "raw_logprob": -5.581697940826416, + }, + "topTokens": None, + "textRange": {"start": 485, "end": 493}, + }, + { + "generatedToken": { + "token": "\u2581when\u2581you're", + "logprob": -4.5586161613464355, + "raw_logprob": -4.5586161613464355, + }, + "topTokens": None, + "textRange": {"start": 493, "end": 505}, + }, + { + "generatedToken": { + "token": "\u2581a\u2581leader", + "logprob": -0.26272106170654297, + "raw_logprob": -0.26272106170654297, + }, + "topTokens": None, + "textRange": {"start": 505, "end": 514}, + }, + { + "generatedToken": { + "token": ".", + "logprob": -0.3965468406677246, + "raw_logprob": -0.3965468406677246, + }, + "topTokens": None, + "textRange": {"start": 514, "end": 515}, + }, + { + "generatedToken": { + "token": "\u2581This\u2581means", + "logprob": -2.841196298599243, + "raw_logprob": -2.841196298599243, + }, + "topTokens": None, + "textRange": {"start": 515, "end": 526}, + }, + { + "generatedToken": { + "token": "\u2581being", + "logprob": -0.4315812587738037, + "raw_logprob": -0.4315812587738037, + }, + "topTokens": None, + "textRange": {"start": 526, "end": 532}, + }, + { + "generatedToken": { + "token": "\u2581willing\u2581to", + "logprob": -0.03861286863684654, + "raw_logprob": -0.03861286863684654, + }, + "topTokens": None, + "textRange": {"start": 532, "end": 543}, + }, + { + "generatedToken": { + "token": "\u2581think", + "logprob": -7.899557113647461, + "raw_logprob": -7.899557113647461, + }, + "topTokens": None, + "textRange": {"start": 543, "end": 549}, + }, + { + "generatedToken": { + "token": "\u2581critically", + "logprob": -0.6595878601074219, + "raw_logprob": -0.6595878601074219, + }, + "topTokens": None, + "textRange": {"start": 549, "end": 560}, + }, + { + "generatedToken": { + "token": ",", + "logprob": -1.2396876811981201, + "raw_logprob": -1.2396876811981201, + }, + "topTokens": None, + "textRange": {"start": 560, "end": 561}, + }, + { + "generatedToken": { + "token": "\u2581reflect", + "logprob": -6.496954917907715, + "raw_logprob": -6.496954917907715, + }, + "topTokens": None, + "textRange": {"start": 561, "end": 569}, + }, + { + "generatedToken": { + "token": ",", + "logprob": -0.3813382685184479, + "raw_logprob": -0.3813382685184479, + }, + "topTokens": None, + "textRange": {"start": 569, "end": 570}, + }, + { + "generatedToken": { + "token": "\u2581learn\u2581from", + "logprob": -5.863975524902344, + "raw_logprob": -5.863975524902344, + }, + "topTokens": None, + "textRange": {"start": 570, "end": 581}, + }, + { + "generatedToken": { + "token": "\u2581mistakes", + "logprob": -1.1053953170776367, + "raw_logprob": -1.1053953170776367, + }, + "topTokens": None, + "textRange": {"start": 581, "end": 590}, + }, + { + "generatedToken": { + "token": ",", + "logprob": -0.010977472178637981, + "raw_logprob": -0.010977472178637981, + }, + "topTokens": None, + "textRange": {"start": 590, "end": 591}, + }, + { + "generatedToken": { + "token": "\u2581and", + "logprob": -0.5951434373855591, + "raw_logprob": -0.5951434373855591, + }, + "topTokens": None, + "textRange": {"start": 591, "end": 595}, + }, + { + "generatedToken": { + "token": "\u2581take\u2581action", + "logprob": -4.118521690368652, + "raw_logprob": -4.118521690368652, + }, + "topTokens": None, + "textRange": {"start": 595, "end": 607}, + }, + { + "generatedToken": { + "token": "\u2581steps", + "logprob": -8.071130752563477, + "raw_logprob": -8.071130752563477, + }, + "topTokens": None, + "textRange": {"start": 607, "end": 613}, + }, + { + "generatedToken": { + "token": "\u2581moving\u2581forward", + "logprob": -5.662147045135498, + "raw_logprob": -5.662147045135498, + }, + "topTokens": None, + "textRange": {"start": 613, "end": 628}, + }, + { + "generatedToken": { + "token": ".", + "logprob": -0.03737432509660721, + "raw_logprob": -0.03737432509660721, + }, + "topTokens": None, + "textRange": {"start": 628, "end": 629}, + }, + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -1.0259989500045776, + "raw_logprob": -1.0259989500045776, + }, + "topTokens": None, + "textRange": {"start": 629, "end": 630}, + }, + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -0.0006999903125688434, + "raw_logprob": -0.0006999903125688434, + }, + "topTokens": None, + "textRange": {"start": 630, "end": 631}, + }, + { + "generatedToken": { + "token": "\u2581There\u2581are\u2581three", + "logprob": -5.931296348571777, + "raw_logprob": -5.931296348571777, + }, + "topTokens": None, + "textRange": {"start": 631, "end": 646}, + }, + { + "generatedToken": { + "token": "\u2581steps", + "logprob": -2.7536213397979736, + "raw_logprob": -2.7536213397979736, + }, + "topTokens": None, + "textRange": {"start": 646, "end": 652}, + }, + { + "generatedToken": { + "token": "\u2581that\u2581can\u2581help\u2581you", + "logprob": -2.3474459648132324, + "raw_logprob": -2.3474459648132324, + }, + "topTokens": None, + "textRange": {"start": 652, "end": 670}, + }, + { + "generatedToken": { + "token": "\u2581grow", + "logprob": -7.027171611785889, + "raw_logprob": -7.027171611785889, + }, + "topTokens": None, + "textRange": {"start": 670, "end": 675}, + }, + { + "generatedToken": { + "token": "\u2581as\u2581a\u2581leader", + "logprob": -0.40542012453079224, + "raw_logprob": -0.40542012453079224, + }, + "topTokens": None, + "textRange": {"start": 675, "end": 687}, + }, + { + "generatedToken": { + "token": "\u2581and\u2581make", + "logprob": -0.7026352882385254, + "raw_logprob": -0.7026352882385254, + }, + "topTokens": None, + "textRange": {"start": 687, "end": 696}, + }, + { + "generatedToken": { + "token": "\u2581better", + "logprob": -2.1509532928466797, + "raw_logprob": -2.1509532928466797, + }, + "topTokens": None, + "textRange": {"start": 696, "end": 703}, + }, + { + "generatedToken": { + "token": "\u2581business\u2581decisions", + "logprob": -0.24822193384170532, + "raw_logprob": -0.24822193384170532, + }, + "topTokens": None, + "textRange": {"start": 703, "end": 722}, + }, + { + "generatedToken": { + "token": ":", + "logprob": -0.46704334020614624, + "raw_logprob": -0.46704334020614624, + }, + "topTokens": None, + "textRange": {"start": 722, "end": 723}, + }, + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -0.02775048278272152, + "raw_logprob": -0.02775048278272152, + }, + "topTokens": None, + "textRange": {"start": 723, "end": 724}, + }, + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -0.0020361661445349455, + "raw_logprob": -0.0020361661445349455, + }, + "topTokens": None, + "textRange": {"start": 724, "end": 725}, + }, + { + "generatedToken": { + "token": "\u2581Step", + "logprob": -1.442288875579834, + "raw_logprob": -1.442288875579834, + }, + "topTokens": None, + "textRange": {"start": 725, "end": 729}, + }, + { + "generatedToken": { + "token": "\u2581", + "logprob": -0.05165497958660126, + "raw_logprob": -0.05165497958660126, + }, + "topTokens": None, + "textRange": {"start": 729, "end": 730}, + }, + { + "generatedToken": { + "token": "1", + "logprob": -4.792098479811102e-05, + "raw_logprob": -4.792098479811102e-05, + }, + "topTokens": None, + "textRange": {"start": 730, "end": 731}, + }, + { + "generatedToken": { + "token": ":", + "logprob": -0.02608294039964676, + "raw_logprob": -0.02608294039964676, + }, + "topTokens": None, + "textRange": {"start": 731, "end": 732}, + }, + { + "generatedToken": { + "token": "\u2581Gather", + "logprob": -3.0909531116485596, + "raw_logprob": -3.0909531116485596, + }, + "topTokens": None, + "textRange": {"start": 732, "end": 739}, + }, + { + "generatedToken": { + "token": "\u2581information", + "logprob": -0.8507784605026245, + "raw_logprob": -0.8507784605026245, + }, + "topTokens": None, + "textRange": {"start": 739, "end": 751}, + }, + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -0.6048281788825989, + "raw_logprob": -0.6048281788825989, + }, + "topTokens": None, + "textRange": {"start": 751, "end": 752}, + }, + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -0.01351175270974636, + "raw_logprob": -0.01351175270974636, + }, + "topTokens": None, + "textRange": {"start": 752, "end": 753}, + }, + { + "generatedToken": { + "token": "\u2581The\u2581first\u2581step", + "logprob": -2.7363672256469727, + "raw_logprob": -2.7363672256469727, + }, + "topTokens": None, + "textRange": {"start": 753, "end": 767}, + }, + { + "generatedToken": { + "token": "\u2581to", + "logprob": -0.1339748501777649, + "raw_logprob": -0.1339748501777649, + }, + "topTokens": None, + "textRange": {"start": 767, "end": 770}, + }, + { + "generatedToken": { + "token": "\u2581making", + "logprob": -0.3207220137119293, + "raw_logprob": -0.3207220137119293, + }, + "topTokens": None, + "textRange": {"start": 770, "end": 777}, + }, + { + "generatedToken": { + "token": "\u2581a\u2581good", + "logprob": -0.6057114005088806, + "raw_logprob": -0.6057114005088806, + }, + "topTokens": None, + "textRange": {"start": 777, "end": 784}, + }, + { + "generatedToken": { + "token": "\u2581decision", + "logprob": -0.030523210763931274, + "raw_logprob": -0.030523210763931274, + }, + "topTokens": None, + "textRange": {"start": 784, "end": 793}, + }, + { + "generatedToken": { + "token": "\u2581is\u2581to", + "logprob": -3.0425467491149902, + "raw_logprob": -3.0425467491149902, + }, + "topTokens": None, + "textRange": {"start": 793, "end": 799}, + }, + { + "generatedToken": { + "token": "\u2581make\u2581sure\u2581that\u2581you\u2581have", + "logprob": -0.7047816514968872, + "raw_logprob": -0.7047816514968872, + }, + "topTokens": None, + "textRange": {"start": 799, "end": 823}, + }, + { + "generatedToken": { + "token": "\u2581all\u2581of\u2581the", + "logprob": -1.9955559968948364, + "raw_logprob": -1.9955559968948364, + }, + "topTokens": None, + "textRange": {"start": 823, "end": 834}, + }, + { + "generatedToken": { + "token": "\u2581facts", + "logprob": -2.409013271331787, + "raw_logprob": -2.409013271331787, + }, + "topTokens": None, + "textRange": {"start": 834, "end": 840}, + }, + { + "generatedToken": { + "token": ".", + "logprob": -0.2631763517856598, + "raw_logprob": -0.2631763517856598, + }, + "topTokens": None, + "textRange": {"start": 840, "end": 841}, + }, + { + "generatedToken": { + "token": "\u2581It's\u2581important\u2581to", + "logprob": -4.5646491050720215, + "raw_logprob": -4.5646491050720215, + }, + "topTokens": None, + "textRange": {"start": 841, "end": 859}, + }, + { + "generatedToken": { + "token": "\u2581know\u2581what", + "logprob": -6.077958106994629, + "raw_logprob": -6.077958106994629, + }, + "topTokens": None, + "textRange": {"start": 859, "end": 869}, + }, + { + "generatedToken": { + "token": "\u2581information", + "logprob": -2.0120184421539307, + "raw_logprob": -2.0120184421539307, + }, + "topTokens": None, + "textRange": {"start": 869, "end": 881}, + }, + { + "generatedToken": { + "token": "\u2581you\u2581need", + "logprob": -1.7770088911056519, + "raw_logprob": -1.7770088911056519, + }, + "topTokens": None, + "textRange": {"start": 881, "end": 890}, + }, + { + "generatedToken": { + "token": ",", + "logprob": -0.4962013363838196, + "raw_logprob": -0.4962013363838196, + }, + "topTokens": None, + "textRange": {"start": 890, "end": 891}, + }, + { + "generatedToken": { + "token": "\u2581and", + "logprob": -0.8423260450363159, + "raw_logprob": -0.8423260450363159, + }, + "topTokens": None, + "textRange": {"start": 891, "end": 895}, + }, + { + "generatedToken": { + "token": "\u2581from\u2581where", + "logprob": -8.261597633361816, + "raw_logprob": -8.261597633361816, + }, + "topTokens": None, + "textRange": {"start": 895, "end": 906}, + }, + { + "generatedToken": { + "token": "\u2581to\u2581get\u2581it", + "logprob": -0.985969066619873, + "raw_logprob": -0.985969066619873, + }, + "topTokens": None, + "textRange": {"start": 906, "end": 916}, + }, + { + "generatedToken": { + "token": ".", + "logprob": -0.0048598977737128735, + "raw_logprob": -0.0048598977737128735, + }, + "topTokens": None, + "textRange": {"start": 916, "end": 917}, + }, + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -0.5743589401245117, + "raw_logprob": -0.5743589401245117, + }, + "topTokens": None, + "textRange": {"start": 917, "end": 918}, + }, + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -0.000593962671700865, + "raw_logprob": -0.000593962671700865, + }, + "topTokens": None, + "textRange": {"start": 918, "end": 919}, + }, + { + "generatedToken": { + "token": "\u2581You\u2581can", + "logprob": -4.267513275146484, + "raw_logprob": -4.267513275146484, + }, + "topTokens": None, + "textRange": {"start": 919, "end": 926}, + }, + { + "generatedToken": { + "token": "\u2581gather", + "logprob": -0.007923126220703125, + "raw_logprob": -0.007923126220703125, + }, + "topTokens": None, + "textRange": {"start": 926, "end": 933}, + }, + { + "generatedToken": { + "token": "\u2581information", + "logprob": -0.3179577887058258, + "raw_logprob": -0.3179577887058258, + }, + "topTokens": None, + "textRange": {"start": 933, "end": 945}, + }, + { + "generatedToken": { + "token": "\u2581by\u2581doing", + "logprob": -5.132864952087402, + "raw_logprob": -5.132864952087402, + }, + "topTokens": None, + "textRange": {"start": 945, "end": 954}, + }, + { + "generatedToken": { + "token": "\u2581things\u2581like", + "logprob": -2.202630043029785, + "raw_logprob": -2.202630043029785, + }, + "topTokens": None, + "textRange": {"start": 954, "end": 966}, + }, + { + "generatedToken": { + "token": "\u2581reading", + "logprob": -3.232940196990967, + "raw_logprob": -3.232940196990967, + }, + "topTokens": None, + "textRange": {"start": 966, "end": 974}, + }, + { + "generatedToken": { + "token": "\u2581reports", + "logprob": -0.329463928937912, + "raw_logprob": -0.329463928937912, + }, + "topTokens": None, + "textRange": {"start": 974, "end": 982}, + }, + { + "generatedToken": { + "token": ",", + "logprob": -0.002441998338326812, + "raw_logprob": -0.002441998338326812, + }, + "topTokens": None, + "textRange": {"start": 982, "end": 983}, + }, + { + "generatedToken": { + "token": "\u2581talking\u2581to", + "logprob": -0.12298407405614853, + "raw_logprob": -0.12298407405614853, + }, + "topTokens": None, + "textRange": {"start": 983, "end": 994}, + }, + { + "generatedToken": { + "token": "\u2581stakeholders", + "logprob": -2.3864426612854004, + "raw_logprob": -2.3864426612854004, + }, + "topTokens": None, + "textRange": {"start": 994, "end": 1007}, + }, + { + "generatedToken": { + "token": ",", + "logprob": -0.012393603101372719, + "raw_logprob": -0.012393603101372719, + }, + "topTokens": None, + "textRange": {"start": 1007, "end": 1008}, + }, + { + "generatedToken": { + "token": "\u2581and", + "logprob": -0.1544899344444275, + "raw_logprob": -0.1544899344444275, + }, + "topTokens": None, + "textRange": {"start": 1008, "end": 1012}, + }, + { + "generatedToken": { + "token": "\u2581conducting\u2581research", + "logprob": -0.731350839138031, + "raw_logprob": -0.731350839138031, + }, + "topTokens": None, + "textRange": {"start": 1012, "end": 1032}, + }, + { + "generatedToken": { + "token": ".", + "logprob": -0.010276736691594124, + "raw_logprob": -0.010276736691594124, + }, + "topTokens": None, + "textRange": {"start": 1032, "end": 1033}, + }, + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -1.247491478919983, + "raw_logprob": -1.247491478919983, + }, + "topTokens": None, + "textRange": {"start": 1033, "end": 1034}, + }, + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -5.7338023907504976e-05, + "raw_logprob": -5.7338023907504976e-05, + }, + "topTokens": None, + "textRange": {"start": 1034, "end": 1035}, + }, + { + "generatedToken": { + "token": "\u2581Step", + "logprob": -0.43501779437065125, + "raw_logprob": -0.43501779437065125, + }, + "topTokens": None, + "textRange": {"start": 1035, "end": 1039}, + }, + { + "generatedToken": { + "token": "\u2581", + "logprob": -1.1920858014491387e-05, + "raw_logprob": -1.1920858014491387e-05, + }, + "topTokens": None, + "textRange": {"start": 1039, "end": 1040}, + }, + { + "generatedToken": { + "token": "2", + "logprob": -0.00016342257731594145, + "raw_logprob": -0.00016342257731594145, + }, + "topTokens": None, + "textRange": {"start": 1040, "end": 1041}, + }, + { + "generatedToken": { + "token": ":", + "logprob": -0.00010644822759786621, + "raw_logprob": -0.00010644822759786621, + }, + "topTokens": None, + "textRange": {"start": 1041, "end": 1042}, + }, + { + "generatedToken": { + "token": "\u2581Analyze", + "logprob": -0.15760670602321625, + "raw_logprob": -0.15760670602321625, + }, + "topTokens": None, + "textRange": {"start": 1042, "end": 1050}, + }, + { + "generatedToken": { + "token": "\u2581information", + "logprob": -1.612084984779358, + "raw_logprob": -1.612084984779358, + }, + "topTokens": None, + "textRange": {"start": 1050, "end": 1062}, + }, + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -9.583967766957358e-05, + "raw_logprob": -9.583967766957358e-05, + }, + "topTokens": None, + "textRange": {"start": 1062, "end": 1063}, + }, + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -0.00024685196694917977, + "raw_logprob": -0.00024685196694917977, + }, + "topTokens": None, + "textRange": {"start": 1063, "end": 1064}, + }, + { + "generatedToken": { + "token": "\u2581Once\u2581you've", + "logprob": -2.3116512298583984, + "raw_logprob": -2.3116512298583984, + }, + "topTokens": None, + "textRange": {"start": 1064, "end": 1075}, + }, + { + "generatedToken": { + "token": "\u2581gathered", + "logprob": -0.002062814310193062, + "raw_logprob": -0.002062814310193062, + }, + "topTokens": None, + "textRange": {"start": 1075, "end": 1084}, + }, + { + "generatedToken": { + "token": "\u2581all\u2581of\u2581your", + "logprob": -2.685849666595459, + "raw_logprob": -2.685849666595459, + }, + "topTokens": None, + "textRange": {"start": 1084, "end": 1096}, + }, + { + "generatedToken": { + "token": "\u2581information", + "logprob": -0.003219066886231303, + "raw_logprob": -0.003219066886231303, + }, + "topTokens": None, + "textRange": {"start": 1096, "end": 1108}, + }, + { + "generatedToken": { + "token": ",", + "logprob": -3.361645576660521e-05, + "raw_logprob": -3.361645576660521e-05, + }, + "topTokens": None, + "textRange": {"start": 1108, "end": 1109}, + }, + { + "generatedToken": { + "token": "\u2581you\u2581need\u2581to", + "logprob": -1.4020256996154785, + "raw_logprob": -1.4020256996154785, + }, + "topTokens": None, + "textRange": {"start": 1109, "end": 1121}, + }, + { + "generatedToken": { + "token": "\u2581take\u2581some\u2581time\u2581to", + "logprob": -2.1766977310180664, + "raw_logprob": -2.1766977310180664, + }, + "topTokens": None, + "textRange": {"start": 1121, "end": 1139}, + }, + { + "generatedToken": { + "token": "\u2581think\u2581about\u2581it", + "logprob": -0.4216986298561096, + "raw_logprob": -0.4216986298561096, + }, + "topTokens": None, + "textRange": {"start": 1139, "end": 1154}, + }, + { + "generatedToken": { + "token": ".", + "logprob": -0.24139046669006348, + "raw_logprob": -0.24139046669006348, + }, + "topTokens": None, + "textRange": {"start": 1154, "end": 1155}, + }, + { + "generatedToken": { + "token": "\u2581You\u2581need\u2581to", + "logprob": -1.129857063293457, + "raw_logprob": -1.129857063293457, + }, + "topTokens": None, + "textRange": {"start": 1155, "end": 1167}, + }, + { + "generatedToken": { + "token": "\u2581analyze\u2581the", + "logprob": -1.3527189493179321, + "raw_logprob": -1.3527189493179321, + }, + "topTokens": None, + "textRange": {"start": 1167, "end": 1179}, + }, + { + "generatedToken": { + "token": "\u2581data", + "logprob": -1.0173096656799316, + "raw_logprob": -1.0173096656799316, + }, + "topTokens": None, + "textRange": {"start": 1179, "end": 1184}, + }, + { + "generatedToken": { + "token": "\u2581and\u2581identify", + "logprob": -3.182776927947998, + "raw_logprob": -3.182776927947998, + }, + "topTokens": None, + "textRange": {"start": 1184, "end": 1197}, + }, + { + "generatedToken": { + "token": "\u2581patterns", + "logprob": -0.6117339134216309, + "raw_logprob": -0.6117339134216309, + }, + "topTokens": None, + "textRange": {"start": 1197, "end": 1206}, + }, + { + "generatedToken": { + "token": ",", + "logprob": -0.4564504325389862, + "raw_logprob": -0.4564504325389862, + }, + "topTokens": None, + "textRange": {"start": 1206, "end": 1207}, + }, + { + "generatedToken": { + "token": "\u2581trends", + "logprob": -0.0026252351235598326, + "raw_logprob": -0.0026252351235598326, + }, + "topTokens": None, + "textRange": {"start": 1207, "end": 1214}, + }, + { + "generatedToken": { + "token": ",", + "logprob": -2.706014311115723e-05, + "raw_logprob": -2.706014311115723e-05, + }, + "topTokens": None, + "textRange": {"start": 1214, "end": 1215}, + }, + { + "generatedToken": { + "token": "\u2581and", + "logprob": -0.16668428480625153, + "raw_logprob": -0.16668428480625153, + }, + "topTokens": None, + "textRange": {"start": 1215, "end": 1219}, + }, + { + "generatedToken": { + "token": "\u2581trends", + "logprob": -2.091916084289551, + "raw_logprob": -2.091916084289551, + }, + "topTokens": None, + "textRange": {"start": 1219, "end": 1226}, + }, + { + "generatedToken": { + "token": "\u2581that", + "logprob": -2.99127197265625, + "raw_logprob": -2.99127197265625, + }, + "topTokens": None, + "textRange": {"start": 1226, "end": 1231}, + }, + { + "generatedToken": { + "token": "\u2581might\u2581not\u2581be", + "logprob": -2.1681160926818848, + "raw_logprob": -2.1681160926818848, + }, + "topTokens": None, + "textRange": {"start": 1231, "end": 1244}, + }, + { + "generatedToken": { + "token": "\u2581immediately", + "logprob": -0.5720977783203125, + "raw_logprob": -0.5720977783203125, + }, + "topTokens": None, + "textRange": {"start": 1244, "end": 1256}, + }, + { + "generatedToken": { + "token": "\u2581obvious", + "logprob": -0.38135844469070435, + "raw_logprob": -0.38135844469070435, + }, + "topTokens": None, + "textRange": {"start": 1256, "end": 1264}, + }, + { + "generatedToken": { + "token": ".", + "logprob": -0.0025424794293940067, + "raw_logprob": -0.0025424794293940067, + }, + "topTokens": None, + "textRange": {"start": 1264, "end": 1265}, + }, + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -0.005445053335279226, + "raw_logprob": -0.005445053335279226, + }, + "topTokens": None, + "textRange": {"start": 1265, "end": 1266}, + }, + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -1.156323378381785e-05, + "raw_logprob": -1.156323378381785e-05, + }, + "topTokens": None, + "textRange": {"start": 1266, "end": 1267}, + }, + { + "generatedToken": { + "token": "\u2581There\u2581are\u2581a\u2581few", + "logprob": -4.2585649490356445, + "raw_logprob": -4.2585649490356445, + }, + "topTokens": None, + "textRange": {"start": 1267, "end": 1282}, + }, + { + "generatedToken": { + "token": "\u2581things", + "logprob": -2.04957914352417, + "raw_logprob": -2.04957914352417, + }, + "topTokens": None, + "textRange": {"start": 1282, "end": 1289}, + }, + { + "generatedToken": { + "token": "\u2581you\u2581should", + "logprob": -1.8114514350891113, + "raw_logprob": -1.8114514350891113, + }, + "topTokens": None, + "textRange": {"start": 1289, "end": 1300}, + }, + { + "generatedToken": { + "token": "\u2581keep\u2581in\u2581mind", + "logprob": -0.2850663959980011, + "raw_logprob": -0.2850663959980011, + }, + "topTokens": None, + "textRange": {"start": 1300, "end": 1313}, + }, + { + "generatedToken": { + "token": "\u2581when\u2581you're", + "logprob": -0.40983426570892334, + "raw_logprob": -0.40983426570892334, + }, + "topTokens": None, + "textRange": {"start": 1313, "end": 1325}, + }, + { + "generatedToken": { + "token": "\u2581analyzing", + "logprob": -0.049553561955690384, + "raw_logprob": -0.049553561955690384, + }, + "topTokens": None, + "textRange": {"start": 1325, "end": 1335}, + }, + { + "generatedToken": { + "token": "\u2581information", + "logprob": -0.0341101810336113, + "raw_logprob": -0.0341101810336113, + }, + "topTokens": None, + "textRange": {"start": 1335, "end": 1347}, + }, + { + "generatedToken": { + "token": ":", + "logprob": -0.4348779022693634, + "raw_logprob": -0.4348779022693634, + }, + "topTokens": None, + "textRange": {"start": 1347, "end": 1348}, + }, + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -0.006193492095917463, + "raw_logprob": -0.006193492095917463, + }, + "topTokens": None, + "textRange": {"start": 1348, "end": 1349}, + }, + { + "generatedToken": { + "token": "<|newline|>", + "logprob": -0.000358159770257771, + "raw_logprob": -0.000358159770257771, + }, + "topTokens": None, + "textRange": {"start": 1349, "end": 1350}, + }, + { + "generatedToken": { + "token": "\u2581*", + "logprob": -1.0053796768188477, + "raw_logprob": -1.0053796768188477, + }, + "topTokens": None, + "textRange": {"start": 1350, "end": 1351}, + }, + { + "generatedToken": { + "token": "\u2581Identify", + "logprob": -4.100193977355957, + "raw_logprob": -4.100193977355957, + }, + "topTokens": None, + "textRange": {"start": 1351, "end": 1360}, + }, + { + "generatedToken": { + "token": "\u2581the", + "logprob": -1.141700029373169, + "raw_logprob": -1.141700029373169, + }, + "topTokens": None, + "textRange": {"start": 1360, "end": 1364}, + }, + { + "generatedToken": { + "token": "\u2581key\u2581points", + "logprob": -0.9346644282341003, + "raw_logprob": -0.9346644282341003, + }, + "topTokens": None, + "textRange": {"start": 1364, "end": 1375}, + }, + { + "generatedToken": { + "token": ":", + "logprob": -0.29478567838668823, + "raw_logprob": -0.29478567838668823, + }, + "topTokens": None, + "textRange": {"start": 1375, "end": 1376}, + }, + { + "generatedToken": { + "token": "\u2581What\u2581are\u2581the", + "logprob": -0.2456199824810028, + "raw_logprob": -0.2456199824810028, + }, + "topTokens": None, + "textRange": {"start": 1376, "end": 1389}, + }, + { + "generatedToken": { + "token": "\u2581key", + "logprob": -0.8171483278274536, + "raw_logprob": -0.8171483278274536, + }, + "topTokens": None, + "textRange": {"start": 1389, "end": 1393}, + }, + { + "generatedToken": { + "token": "\u2581takeaways", + "logprob": -0.5598645806312561, + "raw_logprob": -0.5598645806312561, + }, + "topTokens": None, + "textRange": {"start": 1393, "end": 1403}, + }, + { + "generatedToken": { + "token": "\u2581from", + "logprob": -1.6096564531326294, + "raw_logprob": -1.6096564531326294, + }, + "topTokens": None, + "textRange": {"start": 1403, "end": 1408}, + }, + { + "generatedToken": { + "token": "\u2581this\u2581information", + "logprob": -1.101968765258789, + "raw_logprob": -1.101968765258789, + }, + "topTokens": None, + "textRange": {"start": 1408, "end": 1425}, + }, + { + "generatedToken": { + "token": "?", + "logprob": -0.0003685271949507296, + "raw_logprob": -0.0003685271949507296, + }, + "topTokens": None, + "textRange": {"start": 1425, "end": 1426}, + }, + { + "generatedToken": { + "token": "\u2581What", + "logprob": -2.42529034614563, + "raw_logprob": -2.42529034614563, + }, + "topTokens": None, + "textRange": {"start": 1426, "end": 1431}, + }, + ], + }, + "finishReason": {"reason": "length", "length": 200}, + } + ], + }, + ], + "cohere.command-text-v14::Write me a blog about making strong business decisions as a leader.": [ + {}, + { + "generations": [ + { + "id": "7449e005-a317-42ab-8e47-6bf0fa119088", + "text": " As a leader, one of the most important things you can do is make strong business decisions. Your choices can make or break your company, so it's essential to take the time to think things through and consider all your options. Here are a few tips for making sound business decisions:\n\n1. Do your research. Before making any decision, it's important to gather as much information as possible. This means talking to your team, looking at data and trends, and considering all of your options. The more information you have, the better equipped you'll be to make a decision.\n\n2. Consider the consequences. Every decision has consequences, so it's important to think about what might happen as a result of your choice. What will the impact be on your team, your company, and your customers? It's also important to think about how your decision might affect your own career and personal life.\n\n3. Seek advice. If you're struggling to make a decision, it", + } + ], + "id": "4e3ebf15-98d2-4aaf-a2da-61d0e262e862", + "prompt": "Write me a blog about making strong business decisions as a leader.", + }, + ], +} + +MODEL_PATH_RE = re.compile(r"/model/([^/]+)/invoke") + + +def simple_get(self): + content_len = int(self.headers.get("content-length")) + content = json.loads(self.rfile.read(content_len).decode("utf-8")) + + model = MODEL_PATH_RE.match(self.path).group(1) + prompt = extract_shortened_prompt(content, model) + if not prompt: + self.send_response(500) + self.end_headers() + self.wfile.write("Could not parse prompt.".encode("utf-8")) + return + + headers, response = ({}, "") + for k, v in RESPONSES.items(): + if prompt.startswith(k): + headers, response = v + break + else: # If no matches found + self.send_response(500) + self.end_headers() + self.wfile.write(("Unknown Prompt:\n%s" % prompt).encode("utf-8")) + return + + # Send response code + self.send_response(200) + + # Send headers + for k, v in headers.items(): + self.send_header(k, v) + self.end_headers() + + # Send response body + self.wfile.write(json.dumps(response).encode("utf-8")) + return + + +def extract_shortened_prompt(content, model): + prompt = content.get("inputText", None) or content.get("prompt", None) + prompt = "::".join((model, prompt)) # Prepend model name to prompt key to keep separate copies + return prompt.lstrip().split("\n")[0] + + +class MockExternalBedrockServer(MockExternalHTTPServer): + # To use this class in a test one needs to start and stop this server + # before and after making requests to the test app that makes the external + # calls. + + def __init__(self, handler=simple_get, port=None, *args, **kwargs): + super(MockExternalBedrockServer, self).__init__(handler=handler, port=port, *args, **kwargs) + + +if __name__ == "__main__": + with MockExternalBedrockServer() as server: + print("MockExternalBedrockServer serving on port %s" % str(server.port)) + while True: + pass # Serve forever diff --git a/tests/mlmodel_bedrock/conftest.py b/tests/mlmodel_bedrock/conftest.py new file mode 100644 index 000000000..0464c2e07 --- /dev/null +++ b/tests/mlmodel_bedrock/conftest.py @@ -0,0 +1,136 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +import pytest +from _mock_external_bedrock_server import ( + MockExternalBedrockServer, + extract_shortened_prompt, +) +from testing_support.fixtures import ( # noqa: F401, pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) + +from newrelic.common.object_wrapper import wrap_function_wrapper + +_default_settings = { + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "ml_insights_event.enabled": True, +} +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (mlmodel_bedrock)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (mlmodel_bedrock)"], +) + +BEDROCK_AUDIT_LOG_FILE = os.path.join(os.path.realpath(os.path.dirname(__file__)), "bedrock_audit.log") +BEDROCK_AUDIT_LOG_CONTENTS = {} + + +@pytest.fixture(autouse=True, scope="session") +def bedrock_server(): + """ + This fixture will either create a mocked backend for testing purposes, or will + set up an audit log file to log responses of the real Bedrock backend to a file. + The behavior can be controlled by setting NEW_RELIC_TESTING_RECORD_BEDROCK_RESPONSES=1 as + an environment variable to run using the real Bedrock backend. (Default: mocking) + """ + import boto3 + + from newrelic.core.config import _environ_as_bool + + if not _environ_as_bool("NEW_RELIC_TESTING_RECORD_BEDROCK_RESPONSES", False): + # Use mocked Bedrock backend and prerecorded responses + with MockExternalBedrockServer() as server: + client = boto3.client( + "bedrock-runtime", + "us-east-1", + endpoint_url="http://localhost:%d" % server.port, + aws_access_key_id="NOT-A-REAL-SECRET", + aws_secret_access_key="NOT-A-REAL-SECRET", + ) + + yield client + else: + # Use real Bedrock backend and record responses + assert ( + os.environ["AWS_ACCESS_KEY_ID"] and os.environ["AWS_SECRET_ACCESS_KEY"] + ), "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are required." + + # Construct real client + client = boto3.client( + "bedrock-runtime", + "us-east-1", + ) + + # Apply function wrappers to record data + wrap_function_wrapper( + "botocore.client", "BaseClient._make_api_call", wrap_botocore_client_BaseClient__make_api_call + ) + yield client # Run tests + + # Write responses to audit log + with open(BEDROCK_AUDIT_LOG_FILE, "w") as audit_log_fp: + json.dump(BEDROCK_AUDIT_LOG_CONTENTS, fp=audit_log_fp, indent=4) + + +# Intercept outgoing requests and log to file for mocking +RECORDED_HEADERS = set(["x-request-id", "contentType"]) + + +def wrap_botocore_client_BaseClient__make_api_call(wrapped, instance, args, kwargs): + from io import BytesIO + + from botocore.response import StreamingBody + + params = bind_make_api_call_params(*args, **kwargs) + if not params: + return wrapped(*args, **kwargs) + + body = json.loads(params["body"]) + model = params["modelId"] + prompt = extract_shortened_prompt(body, model) + + # Send request + result = wrapped(*args, **kwargs) + + # Intercept body data, and replace stream + streamed_body = result["body"].read() + result["body"] = StreamingBody(BytesIO(streamed_body), len(streamed_body)) + + # Clean up data + data = json.loads(streamed_body.decode("utf-8")) + headers = dict(result["ResponseMetadata"].items()) + headers["contentType"] = result["contentType"] + headers = dict( + filter( + lambda k: k[0].lower() in RECORDED_HEADERS or k[0].lower().startswith("x-ratelimit"), + headers.items(), + ) + ) + + # Log response + BEDROCK_AUDIT_LOG_CONTENTS[prompt] = headers, data # Append response data to audit log + return result + + +def bind_make_api_call_params(operation_name, api_params): + return api_params diff --git a/tests/mlmodel_bedrock/test_chat_completion.py b/tests/mlmodel_bedrock/test_chat_completion.py new file mode 100644 index 000000000..ad33001c4 --- /dev/null +++ b/tests/mlmodel_bedrock/test_chat_completion.py @@ -0,0 +1,40 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import pytest + +_test_bedrock_chat_completion_prompt = "Write me a blog about making strong business decisions as a leader." + + +@pytest.mark.parametrize( + "model_id,payload", + [ + ("amazon.titan-text-express-v1", {"inputText": "Command: %s\n\nBlog:"}), + ("anthropic.claude-instant-v1", {"prompt": "Human: %s\n\nAssistant:", "max_tokens_to_sample": 500}), + ("ai21.j2-mid-v1", {"prompt": "%s", "maxTokens": 200}), + ("cohere.command-text-v14", {"prompt": "%s", "max_tokens": 200, "temperature": 0.75}), + ], +) +def test_bedrock_chat_completion(bedrock_server, model_id, payload): + body = json.dumps(payload) % _test_bedrock_chat_completion_prompt + response = bedrock_server.invoke_model( + body=body, + modelId=model_id, + accept="application/json", + contentType="application/json", + ) + response_body = json.loads(response.get("body").read()) + assert response_body diff --git a/tox.ini b/tox.ini index f64eb82b1..8862bcc17 100644 --- a/tox.ini +++ b/tox.ini @@ -140,6 +140,7 @@ envlist = python-framework_starlette-{py37,py38}-starlette{002001}, python-framework_starlette-{py37,py38,py39,py310,py311,pypy38}-starlettelatest, python-framework_strawberry-{py37,py38,py39,py310,py311}-strawberrylatest, + python-mlmodel_bedrock-{py37,py38,py39,py310,py311,pypy38}, python-logger_logging-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, python-logger_loguru-{py37,py38,py39,py310,py311,pypy38}-logurulatest, python-logger_loguru-py39-loguru{06,05}, @@ -343,6 +344,7 @@ deps = framework_tornado: pycurl framework_tornado-tornadolatest: tornado framework_tornado-tornadomaster: https://github.com/tornadoweb/tornado/archive/master.zip + mlmodel_bedrock: boto3 logger_loguru-logurulatest: loguru logger_loguru-loguru06: loguru<0.7 logger_loguru-loguru05: loguru<0.6 @@ -462,6 +464,7 @@ changedir = framework_starlette: tests/framework_starlette framework_strawberry: tests/framework_strawberry framework_tornado: tests/framework_tornado + mlmodel_bedrock: tests/mlmodel_bedrock logger_logging: tests/logger_logging logger_loguru: tests/logger_loguru logger_structlog: tests/logger_structlog