OpenAI ErrorTrace attributes (#941)

* Add openai sync instrumentation.

* Remove commented code.

* Initial openai error commit

* Add example tests and mock error responses

* Changes to attribute collection

* Change error tests to match mock server

* [Mega-Linter] Apply linters fixes

* Trigger tests

* Add dt_enabled decorator to error tests

* Add embedded and async error tests

* [Mega-Linter] Apply linters fixes

* Trigger tests

* Add http.statusCode to span before notice_error call

* Report number of messages in error trace even if 0

* Revert notice_error and add _nr_message attr

* Remove enabled_ml_settings as not needed

* Add stats engine _nr_message test

* [Mega-Linter] Apply linters fixes

* Trigger tests

* Revert black formatting in unicode/byte messages

---------

Co-authored-by: Uma Annamalai <uannamalai@newrelic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Hannah Stepanek <hstepanek@newrelic.com>
Co-authored-by: lrafeei <lrafeei@users.noreply.github.com>
Co-authored-by: hmstepanek <hmstepanek@users.noreply.github.com>

newrelic/api/time_trace.py

@@ -29,7 +29,6 @@
 )
 from newrelic.core.config import is_expected_error, should_ignore_error
 from newrelic.core.trace_cache import trace_cache
-
 from newrelic.packages import six
 
 _logger = logging.getLogger(__name__)
@@ -260,6 +259,11 @@ def _observe_exception(self, exc_info=None, ignore=None, expected=None, status_c
         module, name, fullnames, message_raw = parse_exc_info((exc, value, tb))
         fullname = fullnames[0]
 
+        # In case message is in JSON format for OpenAI models
+        # this will result in a "cleaner" message format
+        if getattr(value, "_nr_message", None):
+            message_raw = value._nr_message
+
         # Check to see if we need to strip the message before recording it.
 
         if settings.strip_exception_messages.enabled and fullname not in settings.strip_exception_messages.allowlist:
@@ -422,23 +426,32 @@ def notice_error(self, error=None, attributes=None, expected=None, ignore=None,
                         input_attributes = {}
                         input_attributes.update(transaction._custom_params)
                         input_attributes.update(attributes)
-                        error_group_name_raw = settings.error_collector.error_group_callback(value, {
-                            "traceback": tb,
-                            "error.class": exc,
-                            "error.message": message_raw,
-                            "error.expected": is_expected,
-                            "custom_params": input_attributes,
-                            "transactionName": getattr(transaction, "name", None),
-                            "response.status": getattr(transaction, "_response_code", None),
-                            "request.method": getattr(transaction, "_request_method", None),
-                            "request.uri": getattr(transaction, "_request_uri", None),
-                        })
+                        error_group_name_raw = settings.error_collector.error_group_callback(
+                            value,
+                            {
+                                "traceback": tb,
+                                "error.class": exc,
+                                "error.message": message_raw,
+                                "error.expected": is_expected,
+                                "custom_params": input_attributes,
+                                "transactionName": getattr(transaction, "name", None),
+                                "response.status": getattr(transaction, "_response_code", None),
+                                "request.method": getattr(transaction, "_request_method", None),
+                                "request.uri": getattr(transaction, "_request_uri", None),
+                            },
+                        )
                         if error_group_name_raw:
                             _, error_group_name = process_user_attribute("error.group.name", error_group_name_raw)
                             if error_group_name is None or not isinstance(error_group_name, six.string_types):
-                                raise ValueError("Invalid attribute value for error.group.name. Expected string, got: %s" % repr(error_group_name_raw))
+                                raise ValueError(
+                                    "Invalid attribute value for error.group.name. Expected string, got: %s"
+                                    % repr(error_group_name_raw)
+                                )
                     except Exception:
-                        _logger.error("Encountered error when calling error group callback:\n%s", "".join(traceback.format_exception(*sys.exc_info())))
+                        _logger.error(
+                            "Encountered error when calling error group callback:\n%s",
+                            "".join(traceback.format_exception(*sys.exc_info())),
+                        )
                         error_group_name = None
 
             transaction._create_error_node(
@@ -595,13 +608,11 @@ def update_async_exclusive_time(self, min_child_start_time, exclusive_duration):
     def process_child(self, node, is_async):
         self.children.append(node)
         if is_async:
-
             # record the lowest start time
             self.min_child_start_time = min(self.min_child_start_time, node.start_time)
 
             # if there are no children running, finalize exclusive time
             if self.child_count == len(self.children):
-
                 exclusive_duration = node.end_time - self.min_child_start_time
 
                 self.update_async_exclusive_time(self.min_child_start_time, exclusive_duration)

newrelic/core/stats_engine.py

@@ -724,6 +724,11 @@ def notice_error(self, error=None, attributes=None, expected=None, ignore=None,
         module, name, fullnames, message_raw = parse_exc_info(error)
         fullname = fullnames[0]
 
+        # In the case case of JSON formatting for OpenAI models
+        # this will result in a "cleaner" message format
+        if getattr(value, "_nr_message", None):
+            message_raw = value._nr_message
+
         # Check to see if we need to strip the message before recording it.
 
         if settings.strip_exception_messages.enabled and fullname not in settings.strip_exception_messages.allowlist:

newrelic/hooks/mlmodel_openai.py

@@ -27,6 +27,28 @@
 OPENAI_VERSION = get_package_version("openai")
 
 
+def openai_error_attributes(exception, request_args):
+    api_key = getattr(openai, "api_key", None)
+    api_key_last_four_digits = f"sk-{api_key[-4:]}" if api_key else ""
+    number_of_messages = len(request_args.get("messages", []))
+
+    error_attributes = {
+        "api_key_last_four_digits": api_key_last_four_digits,
+        "request.model": request_args.get("model") or request_args.get("engine") or "",
+        "request.temperature": request_args.get("temperature", ""),
+        "request.max_tokens": request_args.get("max_tokens", ""),
+        "vendor": "openAI",
+        "ingest_source": "Python",
+        "response.organization": getattr(exception, "organization", ""),
+        "response.number_of_messages": number_of_messages,
+        "http.statusCode": getattr(exception, "http_status", ""),
+        "error.message": getattr(exception, "_message", ""),
+        "error.code": getattr(getattr(exception, "error", ""), "code", ""),
+        "error.param": getattr(exception, "param", ""),
+    }
+    return error_attributes
+
+
 def wrap_embedding_create(wrapped, instance, args, kwargs):
     transaction = current_transaction()
     if not transaction:
@@ -36,7 +58,15 @@ def wrap_embedding_create(wrapped, instance, args, kwargs):
 
     ft_name = callable_name(wrapped)
     with FunctionTrace(ft_name) as ft:
-        response = wrapped(*args, **kwargs)
+        try:
+            response = wrapped(*args, **kwargs)
+        except Exception as exc:
+            error_attributes = openai_error_attributes(exc, kwargs)
+            exc._nr_message = error_attributes.pop("error.message")
+            ft.notice_error(
+                attributes=error_attributes,
+            )
+            raise
 
     if not response:
         return response
@@ -105,7 +135,15 @@ def wrap_chat_completion_create(wrapped, instance, args, kwargs):
 
     ft_name = callable_name(wrapped)
     with FunctionTrace(ft_name) as ft:
-        response = wrapped(*args, **kwargs)
+        try:
+            response = wrapped(*args, **kwargs)
+        except Exception as exc:
+            error_attributes = openai_error_attributes(exc, kwargs)
+            exc._nr_message = error_attributes.pop("error.message")
+            ft.notice_error(
+                attributes=error_attributes,
+            )
+            raise
 
     if not response:
         return response
@@ -250,7 +288,15 @@ async def wrap_embedding_acreate(wrapped, instance, args, kwargs):
 
     ft_name = callable_name(wrapped)
     with FunctionTrace(ft_name) as ft:
-        response = await wrapped(*args, **kwargs)
+        try:
+            response = await wrapped(*args, **kwargs)
+        except Exception as exc:
+            error_attributes = openai_error_attributes(exc, kwargs)
+            exc._nr_message = error_attributes.pop("error.message")
+            ft.notice_error(
+                attributes=error_attributes,
+            )
+            raise
 
     if not response:
         return response
@@ -323,7 +369,15 @@ async def wrap_chat_completion_acreate(wrapped, instance, args, kwargs):
 
     ft_name = callable_name(wrapped)
     with FunctionTrace(ft_name) as ft:
-        response = await wrapped(*args, **kwargs)
+        try:
+            response = await wrapped(*args, **kwargs)
+        except Exception as exc:
+            error_attributes = openai_error_attributes(exc, kwargs)
+            exc._nr_message = error_attributes.pop("error.message")
+            ft.notice_error(
+                attributes=error_attributes,
+            )
+            raise
 
     if not response:
         return response