Added telemetry to ONNX Frontend for reporting unsupported ops

gkrivor · Mar 11, 2024 · 6db94f9 · 6db94f9
1 parent baf1c9d
commit 6db94f9
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 3 deletions.
diff --git a/src/frontends/onnx/frontend/src/core/graph.cpp b/src/frontends/onnx/frontend/src/core/graph.cpp
@@ -21,6 +21,7 @@
 #include "openvino/frontend/onnx/extension/conversion.hpp"
 #include "openvino/frontend/onnx/node_context.hpp"
 #include "openvino/op/util/op_types.hpp"
+#include "openvino/util/common_util.hpp"
 #include "utils/common.hpp"
 
 using namespace ov;
@@ -311,6 +312,34 @@ std::shared_ptr<ov::Model> Graph::create_model() {
 std::shared_ptr<ov::Model> Graph::decode() {
     decode_to_framework_nodes();
     auto function = create_model();
+    if (m_extensions.telemetry) {
+        std::set<std::string> unsupported_operations;
+        std::set<std::string> failures;
+        for (const auto& node : function->get_ordered_ops()) {
+            if (const auto& fw_node = std::dynamic_pointer_cast<frontend::ONNXFrameworkNode>(node)) {
+                const auto& attrs = fw_node->get_attrs();
+                auto node_name = attrs.get_opset_name() + "." + attrs.get_type_name();
+                if (unsupported_operations.count(node_name) > 0) {
+                    continue;
+                }
+                unsupported_operations.insert(node_name);
+            } else if (const auto& fw_node = std::dynamic_pointer_cast<frontend::NotSupportedONNXNode>(node)) {
+                const auto& attrs = fw_node->get_attrs();
+                const auto node_fail = attrs.find(frontend::NotSupportedONNXNode::failed_conversion_key);
+                if (node_fail == attrs.end() || failures.count(node_fail->second) > 0) {
+                    continue;
+                }
+                failures.insert(node_fail->second);
+            }
+            for (const auto& op : unsupported_operations) {
+                m_extensions.telemetry->send_event("error_cause", "onnx_" + op);
+            }
+            for (const auto& str : failures) {
+                m_extensions.telemetry->send_event("error_info",
+                                                   ov::util::filter_lines_by_prefix(str, "[ONNX Frontend] "));
+            }
+        }
+    }
     auto& rt_info = function->get_rt_info();
     rt_info[ONNX_GRAPH_RT_ATTRIBUTE] = shared_from_this();
     return function;

diff --git a/src/frontends/onnx/frontend/src/exceptions.cpp b/src/frontends/onnx/frontend/src/exceptions.cpp
@@ -12,7 +12,7 @@ namespace onnx_error {
 namespace detail {
 std::string get_error_msg_prefix(const ov::frontend::onnx::Node& node) {
     std::stringstream ss;
-    ss << "While validating ONNX node '" << node << "'";
+    ss << "[ONNX Frontend] Node '" << node << "'";
     return ss.str();
 }
 }  // namespace detail

diff --git a/src/frontends/onnx/frontend/src/onnx_framework_node.hpp b/src/frontends/onnx/frontend/src/onnx_framework_node.hpp
@@ -97,11 +97,11 @@ class ONNXSubgraphFrameworkNode : public ONNXFrameworkNode {
 // Be careful with using protobuf references (also ov::frontend::onnx::Node) inside NotSupportedONNXNode
 // which are inserted into ov::Model due to different lifetime and problematic sharing between dynamic libs.
 class NotSupportedONNXNode : public ov::op::util::FrameworkNode {
-    static constexpr const char* failed_conversion_key = "onnx::NotSupportedONNXNode::failed_conversion_key";
-
 public:
     OPENVINO_OP("NotSupportedONNXNode", "util", ov::op::util::FrameworkNode);
 
+    static constexpr const char* failed_conversion_key = "onnx::NotSupportedONNXNode::failed_conversion_key";
+
     NotSupportedONNXNode(const ov::OutputVector& inputs,
                          const size_t output_size,
                          const std::string& domain,