From 2b63dce5d250181c955714fee531fdcbc0743dc9 Mon Sep 17 00:00:00 2001 From: apankratovantonp Date: Thu, 19 Aug 2021 00:06:23 +0300 Subject: [PATCH] Added openvino infer request API --- .../openvino/runtime/infer_request.hpp | 228 ++++++++++++++++++ .../openvino/runtime/variable_state.hpp | 75 ++++++ .../src/cpp/ie_infer_request.cpp | 139 +++++++++++ .../src/cpp/ie_variable_state.cpp | 42 ++++ .../async_infer_request_test.cpp | 68 ++++++ .../inference_engine/variable_state.cpp | 24 ++ 6 files changed, 576 insertions(+) create mode 100644 inference-engine/src/inference_engine/include/openvino/runtime/infer_request.hpp create mode 100644 inference-engine/src/inference_engine/include/openvino/runtime/variable_state.hpp diff --git a/inference-engine/src/inference_engine/include/openvino/runtime/infer_request.hpp b/inference-engine/src/inference_engine/include/openvino/runtime/infer_request.hpp new file mode 100644 index 00000000000000..23d3da9eaa44df --- /dev/null +++ b/inference-engine/src/inference_engine/include/openvino/runtime/infer_request.hpp @@ -0,0 +1,228 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief A header file that provides wrapper classes for infer requests and callbacks. + * + * @file infer_request.hpp + */ +#pragma once + +#include +#include +#include + +#include "variable_state.hpp" + +namespace InferenceEngine { +class IInferRequestInternal; +class Blob; +} // namespace InferenceEngine + +namespace ov { +namespace runtime { +/** + * @brief This is an interface of asynchronous infer request + * + * It can throw exceptions safely for the application, where it is properly handled. + */ +class INFERENCE_ENGINE_API_CLASS(InferRequest) { + InferenceEngine::details::SharedObjectLoader _so; + std::shared_ptr _impl; + + /** + * @brief Constructs InferRequest from the initialized std::shared_ptr + * @param so Plugin to use. This is required to ensure that InferRequest can work properly even if plugin object is + * destroyed. + * @param impl Initialized shared pointer + */ + InferRequest(const InferenceEngine::details::SharedObjectLoader& so, const std::shared_ptr& impl); + friend class ExecutableNetwork; + +public: + /** + * @enum WaitMode + * @brief Enumeration to hold wait mode for IInferRequest + */ + enum WaitMode : int64_t { + /** Wait until inference result becomes available */ + RESULT_READY = -1, + /** IInferRequest doesn't block or interrupt current thread and immediately returns inference status */ + STATUS_ONLY = 0, + }; + + /** + * @brief Default constructor + */ + InferRequest() = default; + + /** + * @brief Sets input/output data to infer + * + * @note Memory allocation does not happen + * @param name Name of input or output blob. + * @param data Reference to input or output blob. The type of a blob must match the network input precision and + * size. + */ + void set_blob(const std::string& name, const std::shared_ptr& data); + + /** + * @brief Gets input/output data for inference + * + * @note Memory allocation does not happen + * @param name A name of Blob to get + * @return A shared pointer to a Blob with a name @p name. If a blob is not found, an exception is thrown. + */ + std::shared_ptr get_blob(const std::string& name); + + /** + * @brief Sets blob with a pre-process information + * @note Returns an error in case if data blob is output + * @param name Name of input blob. + * @param data A reference to input. The type of Blob must correspond to the network input precision and size. + * @param info Preprocess info for blob. + */ + void set_blob(const std::string& name, const std::shared_ptr& data, const InferenceEngine::PreProcessInfo& info); + + /** + * @brief Gets pre-process for input data + * @param name Name of input blob. + * @return pointer to pre-process info of blob with name + */ + const InferenceEngine::PreProcessInfo& get_pre_proccess(const std::string& name) const; + + /** + * @brief Infers specified input(s) in synchronous mode + * + * @note blocks all methods of InferRequest while request is ongoing (running or waiting in queue) + * + */ + void infer(); + + /** + * @brief Cancels inference request + */ + void cancel(); + + /** + * @brief Queries performance measures per layer to get feedback of what is the most time consuming layer + * + * @note not all plugins provide meaningful data + * @return Map of layer names to profiling information for that layer + */ + std::map get_performance_counts() const; + + /** + * @brief Sets input data to infer + * + * @note Memory allocation doesn't happen + * @param inputs A reference to a map of input blobs accessed by input names. + * The type of Blob must correspond to the network input precision and size. + */ + void set_input(const std::map>& inputs); + + /** + * @brief Sets data that will contain result of the inference + * + * @note Memory allocation doesn't happen + * @param results - a reference to a map of result blobs accessed by output names. + * The type of Blob must correspond to the network output precision and size. + */ + void set_output(const std::map>& results); + + /** + * @brief Sets new batch size when dynamic batching is enabled in executable network that created this request. + * + * @param batch new batch size to be used by all the following inference calls for this request. + */ + void set_batch(const int batch); + + /** + * @brief Start inference of specified input(s) in asynchronous mode + * + * @note It returns immediately. Inference starts also immediately. + */ + void start_async(); + + /** + * @brief Waits for the result to become available. Blocks until specified millis_timeout has elapsed or the result + * becomes available, whichever comes first. + * + * + * @param millis_timeout Maximum duration in milliseconds to block for + * @note There are special cases when millis_timeout is equal some value of the WaitMode enum: + * * STATUS_ONLY - immediately returns inference status (IInferRequest::RequestStatus). It does not block or + * interrupt current thread + * * RESULT_READY - waits until inference result becomes available + * @return A status code of operation + */ + InferenceEngine::StatusCode wait(int64_t millis_timeout = RESULT_READY); + +private: + void set_complition_callback_impl(std::function); + void set_complition_callback_impl(std::function); + + template + struct SetCallback { + void operator()(std::function f) { + _this.set_complition_callback_impl(std::move(f)); + } + InferRequest& _this; + }; + +public: + /** + * @brief Sets a callback function that will be called on success or failure of asynchronous request + * + * @param callback callback object which will be called on when inference finish. + */ + template + void set_complition_callback(F callback) { + SetCallback{*this}(std::move(callback)); + } + /** + * @brief Gets state control interface for given infer request. + * + * State control essential for recurrent networks + * @return A vector of Memory State objects + */ + std::vector query_state(); + + /** + * @brief Checks if current InferRequest object is not initialized + * @return true if current InferRequest object is not initialized, false - otherwise + */ + bool operator!() const noexcept; + + /** + * @brief Checks if current InferRequest object is initialized + * @return true if current InferRequest object is initialized, false - otherwise + */ + explicit operator bool() const noexcept; + + /** + * @brief Compares whether this request wraps the same impl underneath + * @return true if current InferRequest object doesn't wrap the same impl as the operator's arg + */ + bool operator!=(const InferRequest&) const noexcept; + + /** + * @brief Compares whether this request wraps the same impl underneath + * @return true if current InferRequest object wraps the same impl as the operator's arg + */ + bool operator==(const InferRequest&) const noexcept; +}; + +/** + * @private + */ +template <> +struct InferRequest::SetCallback> { + void operator()(std::function f) { + _this.set_complition_callback_impl(std::move(f)); + } + InferRequest& _this; +}; +} // namespace runtime +} // namespace ov diff --git a/inference-engine/src/inference_engine/include/openvino/runtime/variable_state.hpp b/inference-engine/src/inference_engine/include/openvino/runtime/variable_state.hpp new file mode 100644 index 00000000000000..52f0af1e3771bd --- /dev/null +++ b/inference-engine/src/inference_engine/include/openvino/runtime/variable_state.hpp @@ -0,0 +1,75 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief A header file that provides VariableState + * + * @file variable_state.hpp + */ + +#pragma once + +#include +#include +#include +#include + +namespace InferenceEngine { +class IVariableStateInternal; +class Blob; +} // namespace InferenceEngine + +namespace ov { +namespace runtime { + +/** + * @brief VariableState class + */ +class INFERENCE_ENGINE_API_CLASS(VariableState) { + InferenceEngine::details::SharedObjectLoader _so; + std::shared_ptr _impl; + + /** + * @brief Constructs VariableState from the initialized std::shared_ptr + * @param impl Initialized shared pointer + * @param so Optional: Plugin to use. This is required to ensure that VariableState can work properly even if plugin + * object is destroyed. + */ + VariableState(const InferenceEngine::details::SharedObjectLoader& so, const std::shared_ptr& impl); + + friend class InferRequest; + +public: + /** + * @brief Default constructor + */ + VariableState() = default; + + /** + * @brief Reset internal variable state for relevant infer request, + * to a value specified as default for according ReadValue node + */ + void reset(); + + /** + * @brief Gets name of current variable state, if length of array is not enough name is truncated by len, null + * terminator is inserted as well. As variable state name `variable_id` from according `ReadValue` used. + * @return A string representing a state name + */ + std::string get_name() const; + + /** + * @brief Returns the value of the variable state. + * @return A blob representing a state + */ + std::shared_ptr get_state() const; + + /** + * @brief Sets the new state for the next inference. + * @param state The current state to set + */ + void set_state(const std::shared_ptr& state); +}; +} // namespace runtime +} // namespace ov diff --git a/inference-engine/src/inference_engine/src/cpp/ie_infer_request.cpp b/inference-engine/src/inference_engine/src/cpp/ie_infer_request.cpp index 135cf2d3391f6d..0c7408dc0cf95c 100644 --- a/inference-engine/src/inference_engine/src/cpp/ie_infer_request.cpp +++ b/inference-engine/src/inference_engine/src/cpp/ie_infer_request.cpp @@ -12,6 +12,9 @@ #include "cpp_interfaces/interface/ie_iinfer_request_internal.hpp" #include "ie_infer_async_request_base.hpp" #include "ie_remote_context.hpp" +#include "details/ie_so_loader.h" + +#include "openvino/runtime/infer_request.hpp" namespace InferenceEngine { @@ -190,3 +193,139 @@ bool InferRequest::operator==(const InferRequest& r) const noexcept { } } // namespace InferenceEngine + + +namespace ov { +namespace runtime { + +#define OV_INFER_REQ_CALL_STATEMENT(...) \ + if (_impl == nullptr) \ + IE_THROW(NotAllocated) << "Inference Request is not initialized"; \ + try { \ + __VA_ARGS__; \ + } catch (...) { \ + InferenceEngine::details::Rethrow(); \ + } + +InferRequest::InferRequest(const InferenceEngine::details::SharedObjectLoader& so, const InferenceEngine::IInferRequestInternal::Ptr& impl) + : _so{so}, + _impl{impl} { + IE_ASSERT(_impl != nullptr); +} + +void InferRequest::set_blob(const std::string& name, const InferenceEngine::Blob::Ptr& data) { + OV_INFER_REQ_CALL_STATEMENT(_impl->SetBlob(name, data);) +} + +InferenceEngine::Blob::Ptr InferRequest::get_blob(const std::string& name) { + InferenceEngine::Blob::Ptr blobPtr; + OV_INFER_REQ_CALL_STATEMENT(blobPtr = _impl->GetBlob(name);) + std::string error = "Internal error: blob with name `" + name + "` is not allocated!"; + const bool remoteBlobPassed = blobPtr->is(); + if (blobPtr == nullptr) + IE_THROW() << error; + if (!remoteBlobPassed && blobPtr->buffer() == nullptr) + IE_THROW() << error; + return blobPtr; +} + +void InferRequest::set_blob(const std::string& name, const InferenceEngine::Blob::Ptr& data, const InferenceEngine::PreProcessInfo& info) { + OV_INFER_REQ_CALL_STATEMENT(_impl->SetBlob(name, data, info);) +} + +const InferenceEngine::PreProcessInfo& InferRequest::get_pre_proccess(const std::string& name) const { + OV_INFER_REQ_CALL_STATEMENT(return _impl->GetPreProcess(name);) +} + +void InferRequest::infer() { + OV_INFER_REQ_CALL_STATEMENT(_impl->Infer();) +} + +void InferRequest::cancel() { + OV_INFER_REQ_CALL_STATEMENT(_impl->Cancel();) +} + +std::map InferRequest::get_performance_counts() const { + OV_INFER_REQ_CALL_STATEMENT(return _impl->GetPerformanceCounts();) +} + +void InferRequest::set_input(const InferenceEngine::BlobMap& inputs) { + OV_INFER_REQ_CALL_STATEMENT(for (auto&& input : inputs) { _impl->SetBlob(input.first, input.second); }) +} + +void InferRequest::set_output(const InferenceEngine::BlobMap& results) { + OV_INFER_REQ_CALL_STATEMENT(for (auto&& result : results) { _impl->SetBlob(result.first, result.second); }) +} + +void InferRequest::set_batch(const int batch) { + OV_INFER_REQ_CALL_STATEMENT(_impl->SetBatch(batch);) +} + +void InferRequest::start_async() { + OV_INFER_REQ_CALL_STATEMENT(_impl->StartAsync();) +} + +InferenceEngine::StatusCode InferRequest::wait(int64_t millis_timeout) { + OV_INFER_REQ_CALL_STATEMENT(return _impl->Wait(millis_timeout);) +} + +void InferRequest::set_complition_callback_impl(std::function callbackToSet) { + OV_INFER_REQ_CALL_STATEMENT(_impl->SetCallback([callbackToSet](std::exception_ptr) { + callbackToSet(); + });) +} + +void InferRequest::set_complition_callback_impl(std::function callbackToSet) { + OV_INFER_REQ_CALL_STATEMENT( + auto weakThis = + InferRequest{ + _so, + std::shared_ptr{ + _impl.get(), [](InferenceEngine::IInferRequestInternal*) {}}}; + _impl->SetCallback([callbackToSet, weakThis](std::exception_ptr exceptionPtr) { + InferenceEngine::StatusCode statusCode = InferenceEngine::StatusCode::OK; + if (exceptionPtr != nullptr) { + statusCode = [&] { + using namespace InferenceEngine; + try { + std::rethrow_exception(exceptionPtr); + } + CATCH_IE_EXCEPTIONS_RETURN catch (const std::exception&) { + return GENERAL_ERROR; + } + catch (...) { + return UNEXPECTED; + } + }(); + } + callbackToSet(weakThis, statusCode); + });) +} + +std::vector InferRequest::query_state() { + std::vector variable_states; + OV_INFER_REQ_CALL_STATEMENT( + for (auto&& state : _impl->QueryState()) { + variable_states.emplace_back(VariableState{_so, state}); + }) + return variable_states; +} + +bool InferRequest::operator!() const noexcept { + return !_impl; +} + +InferRequest::operator bool() const noexcept { + return (!!_impl); +} + +bool InferRequest::operator!=(const InferRequest& r) const noexcept { + return !(r == *this); +} + +bool InferRequest::operator==(const InferRequest& r) const noexcept { + return r._impl == _impl; +} + +} // namespace runtime +} // namespace ov \ No newline at end of file diff --git a/inference-engine/src/inference_engine/src/cpp/ie_variable_state.cpp b/inference-engine/src/inference_engine/src/cpp/ie_variable_state.cpp index ff8547f13b33da..c17d81372915d5 100644 --- a/inference-engine/src/inference_engine/src/cpp/ie_variable_state.cpp +++ b/inference-engine/src/inference_engine/src/cpp/ie_variable_state.cpp @@ -6,6 +6,9 @@ #include "cpp_interfaces/interface/ie_ivariable_state_internal.hpp" #include "details/ie_so_loader.h" #include "exception2status.hpp" +#include "details/ie_so_loader.h" + +#include "openvino/runtime/variable_state.hpp" #define VARIABLE_CALL_STATEMENT(...) \ if (_impl == nullptr) \ @@ -44,3 +47,42 @@ void VariableState::SetState(Blob::Ptr state) { } } // namespace InferenceEngine + + +namespace ov { +namespace runtime { + +#define OV_VARIABLE_CALL_STATEMENT(...) \ + if (_impl == nullptr) \ + IE_THROW(NotAllocated) << "VariableState was not initialized"; \ + try { \ + __VA_ARGS__; \ + } catch (...) { \ + InferenceEngine::details::Rethrow(); \ + } + +VariableState::VariableState(const InferenceEngine::details::SharedObjectLoader& so, const InferenceEngine::IVariableStateInternal::Ptr& impl) + : _so{so}, + _impl{impl} { + IE_ASSERT(_impl != nullptr); +} + +void VariableState::reset() { + OV_VARIABLE_CALL_STATEMENT(_impl->Reset()); +} + +std::string VariableState::get_name() const { + OV_VARIABLE_CALL_STATEMENT(return _impl->GetName()); +} + +InferenceEngine::Blob::CPtr VariableState::get_state() const { + OV_VARIABLE_CALL_STATEMENT(return _impl->GetState()); +} + +void VariableState::set_state(const InferenceEngine::Blob::Ptr& state) { + OV_VARIABLE_CALL_STATEMENT(_impl->SetState(state)); +} + +} // namespace runtime +} // namespace ov + diff --git a/inference-engine/tests/functional/inference_engine/async_infer_request_test.cpp b/inference-engine/tests/functional/inference_engine/async_infer_request_test.cpp index 6393aa69d16199..51a140be7a4b0f 100644 --- a/inference-engine/tests/functional/inference_engine/async_infer_request_test.cpp +++ b/inference-engine/tests/functional/inference_engine/async_infer_request_test.cpp @@ -5,6 +5,7 @@ #include #include +#include using namespace ::testing; using namespace std; @@ -86,3 +87,70 @@ TEST(InferRequestCPPTests, throwsOnUninitializedQueryState) { InferRequest req; ASSERT_THROW(req.QueryState(), InferenceEngine::NotAllocated); } + + +TEST(InferRequestOVTests, throwsOnUninitializedSetBlob) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.set_blob({}, {}), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedGetBlob) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.get_blob({}), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedSetBlobPreproc) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.set_blob({}, {}, {}), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedGetPreProcess) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.get_pre_proccess({}), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedInfer) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.infer(), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedGetPerformanceCounts) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.get_performance_counts(), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedSetInput) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.set_input({{}}), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedSetOutput) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.set_output({{}}), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedSetBatch) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.set_batch({}), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedStartAsync) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.start_async(), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedWait) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.wait({}), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedSetCompletionCallback) { + ov::runtime::InferRequest req; + std::function f; + ASSERT_THROW(req.set_complition_callback(f), InferenceEngine::NotAllocated); +} + +TEST(InferRequestOVTests, throwsOnUninitializedQueryState) { + ov::runtime::InferRequest req; + ASSERT_THROW(req.query_state(), InferenceEngine::NotAllocated); +} \ No newline at end of file diff --git a/inference-engine/tests/functional/inference_engine/variable_state.cpp b/inference-engine/tests/functional/inference_engine/variable_state.cpp index f17bd599bb6026..0fec5680fa44b1 100644 --- a/inference-engine/tests/functional/inference_engine/variable_state.cpp +++ b/inference-engine/tests/functional/inference_engine/variable_state.cpp @@ -6,6 +6,8 @@ #include +#include + using namespace ::testing; using namespace std; using namespace InferenceEngine; @@ -31,3 +33,25 @@ TEST(VariableStateCPPTests, throwsOnUninitializedSetState) { Blob::Ptr blob; ASSERT_THROW(req.SetState(blob), InferenceEngine::NotAllocated); } + + +TEST(VariableStateOVTests, throwsOnUninitializedReset) { + ov::runtime::VariableState req; + ASSERT_THROW(req.reset(), InferenceEngine::NotAllocated); +} + +TEST(VariableStateOVTests, throwsOnUninitializedGetname) { + ov::runtime::VariableState req; + ASSERT_THROW(req.get_name(), InferenceEngine::NotAllocated); +} + +TEST(VariableStateOVTests, throwsOnUninitializedGetState) { + ov::runtime::VariableState req; + ASSERT_THROW(req.get_state(), InferenceEngine::NotAllocated); +} + +TEST(VariableStateOVTests, throwsOnUninitializedSetState) { + ov::runtime::VariableState req; + Blob::Ptr blob; + ASSERT_THROW(req.set_state(blob), InferenceEngine::NotAllocated); +}