Added openvino infer request API

cmake/developer_package/ncc_naming_style/openvino.style

@@ -3,7 +3,7 @@ CppMethod: '^(operator\W+|[a-z_\d]+|signaling_NaN|quiet_NaN)$'
 # TODO: remove stopwatch|unsupported_op
 ClassName: '^([A-Z][\w]+|b?float16|numeric_limits|ngraph_error|stopwatch|unsupported_op)$'
 # TODO: remove oi_pair
-StructName: '^([A-Z][\w]+|element_type_traits|hash|oi_pair)$'
+StructName: '^([A-Z][\w:]+|element_type_traits|hash|oi_pair)$'
 FunctionName: '^(operator\W+|[a-z_\d]+)$'
 Namespace: '^([a-z\d_]+|InferenceEngine)$'
 NamespaceAlias: '^[a-z\d_]+$'

inference-engine/src/inference_engine/include/openvino/runtime/infer_request.hpp

@@ -0,0 +1,231 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+/**
+ * @brief A header file that provides wrapper classes for infer requests and callbacks.
+ *
+ * @file infer_request.hpp
+ */
+#pragma once
+
+#include <map>
+#include <memory>
+#include <string>
+
+#include "variable_state.hpp"
+
+namespace InferenceEngine {
+class IInferRequestInternal;
+class Blob;
+}  // namespace InferenceEngine
+
+namespace ov {
+namespace runtime {
+/**
+ * @brief This is an interface of asynchronous infer request
+ *
+ * It can throw exceptions safely for the application, where it is properly handled.
+ */
+class INFERENCE_ENGINE_API_CLASS(InferRequest) {
+    InferenceEngine::details::SharedObjectLoader _so;
+    std::shared_ptr<InferenceEngine::IInferRequestInternal> _impl;
+
+    /**
+     * @brief Constructs InferRequest from the initialized std::shared_ptr
+     * @param so Plugin to use. This is required to ensure that InferRequest can work properly even if plugin object is
+     * destroyed.
+     * @param impl Initialized shared pointer
+     */
+    InferRequest(const InferenceEngine::details::SharedObjectLoader& so,
+                 const std::shared_ptr<InferenceEngine::IInferRequestInternal>& impl);
+    friend class ExecutableNetwork;
+
+public:
+    /**
+     * @enum WaitMode
+     * @brief Enumeration to hold wait mode for IInferRequest
+     */
+    enum WaitMode : int64_t {
+        /** Wait until inference result becomes available */
+        RESULT_READY = -1,
+        /** IInferRequest doesn't block or interrupt current thread and immediately returns inference status */
+        STATUS_ONLY = 0,
+    };
+
+    /**
+     * @brief Default constructor
+     */
+    InferRequest() = default;
+
+    /**
+     * @brief Sets input/output data to infer
+     *
+     * @note Memory allocation does not happen
+     * @param name Name of input or output blob.
+     * @param data Reference to input or output blob. The type of a blob must match the network input precision and
+     * size.
+     */
+    void set_blob(const std::string& name, const std::shared_ptr<InferenceEngine::Blob>& data);
+
+    /**
+     * @brief Gets input/output data for inference
+     *
+     * @note Memory allocation does not happen
+     * @param name A name of Blob to get
+     * @return A shared pointer to a Blob with a name @p name. If a blob is not found, an exception is thrown.
+     */
+    std::shared_ptr<InferenceEngine::Blob> get_blob(const std::string& name);
+
+    /**
+     * @brief Sets blob with a pre-process information
+     * @note Returns an error in case if data blob is output
+     * @param name Name of input blob.
+     * @param data A reference to input. The type of Blob must correspond to the network input precision and size.
+     * @param info Preprocess info for blob.
+     */
+    void set_blob(const std::string& name,
+                  const std::shared_ptr<InferenceEngine::Blob>& data,
+                  const InferenceEngine::PreProcessInfo& info);
+
+    /**
+     * @brief Gets pre-process for input data
+     * @param name Name of input blob.
+     * @return pointer to pre-process info of blob with name
+     */
+    const InferenceEngine::PreProcessInfo& get_pre_proccess(const std::string& name) const;
+
+    /**
+     * @brief Infers specified input(s) in synchronous mode
+     *
+     * @note blocks all methods of InferRequest while request is ongoing (running or waiting in queue)
+     *
+     */
+    void infer();
+
+    /**
+     * @brief Cancels inference request
+     */
+    void cancel();
+
+    /**
+     * @brief Queries performance measures per layer to get feedback of what is the most time consuming layer
+     *
+     * @note not all plugins provide meaningful data
+     * @return Map of layer names to profiling information for that layer
+     */
+    std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> get_performance_counts() const;
+
+    /**
+     * @brief Sets input data to infer
+     *
+     * @note Memory allocation doesn't happen
+     * @param inputs A reference to a map of input blobs accessed by input names.
+     *        The type of Blob must correspond to the network input precision and size.
+     */
+    void set_input(const std::map<std::string, std::shared_ptr<InferenceEngine::Blob>>& inputs);
+
+    /**
+     * @brief Sets data that will contain result of the inference
+     *
+     * @note Memory allocation doesn't happen
+     * @param results - a reference to a map of result blobs accessed by output names.
+     *        The type of Blob must correspond to the network output precision and size.
+     */
+    void set_output(const std::map<std::string, std::shared_ptr<InferenceEngine::Blob>>& results);
+
+    /**
+     * @brief Sets new batch size when dynamic batching is enabled in executable network that created this request.
+     *
+     * @param batch new batch size to be used by all the following inference calls for this request.
+     */
+    void set_batch(const int batch);
+
+    /**
+     * @brief Start inference of specified input(s) in asynchronous mode
+     *
+     * @note It returns immediately. Inference starts also immediately.
+     */
+    void start_async();
+
+    /**
+     * @brief Waits for the result to become available. Blocks until specified millis_timeout has elapsed or the result
+     * becomes available, whichever comes first.
+     *
+     *
+     * @param millis_timeout Maximum duration in milliseconds to block for
+     * @note There are special cases when millis_timeout is equal some value of the WaitMode enum:
+     * * STATUS_ONLY - immediately returns inference status (IInferRequest::RequestStatus). It does not block or
+     * interrupt current thread
+     * * RESULT_READY - waits until inference result becomes available
+     * @return A status code of operation
+     */
+    InferenceEngine::StatusCode wait(int64_t millis_timeout = RESULT_READY);
+
+private:
+    void set_complition_callback_impl(std::function<void()>);
+    void set_complition_callback_impl(std::function<void(InferRequest, InferenceEngine::StatusCode)>);
+
+    template <typename T>
+    struct SetCallback {
+        void operator()(std::function<void()> f) {
+            _this.set_complition_callback_impl(std::move(f));
+        }
+        InferRequest& _this;
+    };
+
+public:
+    /**
+     * @brief Sets a callback function that will be called on success or failure of asynchronous request
+     *
+     * @param callback callback object which will be called on when inference finish.
+     */
+    template <typename F>
+    void set_complition_callback(F callback) {
+        SetCallback<F>{*this}(std::move(callback));
+    }
+    /**
+     * @brief Gets state control interface for given infer request.
+     *
+     * State control essential for recurrent networks
+     * @return A vector of Memory State objects
+     */
+    std::vector<VariableState> query_state();
+
+    /**
+     * @brief Checks if current InferRequest object is not initialized
+     * @return true if current InferRequest object is not initialized, false - otherwise
+     */
+    bool operator!() const noexcept;
+
+    /**
+     * @brief Checks if current InferRequest object is initialized
+     * @return true if current InferRequest object is initialized, false - otherwise
+     */
+    explicit operator bool() const noexcept;
+
+    /**
+     * @brief Compares whether this request wraps the same impl underneath
+     * @return true if current InferRequest object doesn't wrap the same impl as the operator's arg
+     */
+    bool operator!=(const InferRequest&) const noexcept;
+
+    /**
+     * @brief Compares whether this request wraps the same impl underneath
+     * @return true if current InferRequest object wraps the same impl as the operator's arg
+     */
+    bool operator==(const InferRequest&) const noexcept;
+};
+
+/**
+ * @private
+ */
+template <>
+struct InferRequest::SetCallback<std::function<void(InferRequest, InferenceEngine::StatusCode)>> {
+    void operator()(std::function<void(InferRequest, InferenceEngine::StatusCode)> f) {
+        _this.set_complition_callback_impl(std::move(f));
+    }
+    InferRequest& _this;
+};
+}  // namespace runtime
+}  // namespace ov

inference-engine/src/inference_engine/include/openvino/runtime/variable_state.hpp

@@ -0,0 +1,79 @@
+// Copyright (C) 2018-2021 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+/**
+ * @brief A header file that provides VariableState
+ *
+ * @file variable_state.hpp
+ */
+
+#pragma once
+
+#include <ie_api.h>
+#include <ie_blob.h>
+
+#include <memory>
+#include <string>
+
+namespace InferenceEngine {
+class IVariableStateInternal;
+class Blob;
+}  // namespace InferenceEngine
+
+namespace ov {
+namespace runtime {
+
+class InferRequest;
+
+/**
+ * @brief VariableState class
+ */
+class INFERENCE_ENGINE_API_CLASS(VariableState) {
+    InferenceEngine::details::SharedObjectLoader _so;
+    std::shared_ptr<InferenceEngine::IVariableStateInternal> _impl;
+
+    /**
+     * @brief Constructs VariableState from the initialized std::shared_ptr
+     * @param impl Initialized shared pointer
+     * @param so Optional: Plugin to use. This is required to ensure that VariableState can work properly even if plugin
+     * object is destroyed.
+     */
+    VariableState(const InferenceEngine::details::SharedObjectLoader& so,
+                  const std::shared_ptr<InferenceEngine::IVariableStateInternal>& impl);
+
+    friend class ov::runtime::InferRequest;
+
+public:
+    /**
+     * @brief Default constructor
+     */
+    VariableState() = default;
+
+    /**
+     * @brief Reset internal variable state for relevant infer request,
+     * to a value specified as default for according ReadValue node
+     */
+    void reset();
+
+    /**
+     * @brief Gets name of current variable state, if length of array is not enough name is truncated by len, null
+     * terminator is inserted as well. As variable state name `variable_id` from according `ReadValue` used.
+     * @return A string representing a state name
+     */
+    std::string get_name() const;
+
+    /**
+     * @brief Returns the value of the variable state.
+     * @return A blob representing a state
+     */
+    std::shared_ptr<const InferenceEngine::Blob> get_state() const;
+
+    /**
+     * @brief Sets the new state for the next inference.
+     * @param state The current state to set
+     */
+    void set_state(const std::shared_ptr<InferenceEngine::Blob>& state);
+};
+}  // namespace runtime
+}  // namespace ov