diff --git a/.bazelrc b/.bazelrc index af05ec14cf..af5dab6c70 100644 --- a/.bazelrc +++ b/.bazelrc @@ -106,21 +106,11 @@ try-import %workspace%/.user.bazelrc # Disable native ovms mediapipe support when building in mediapipe repository build --define=MEDIAPIPE_DISABLE=1 -coverage --define=MEDIAPIPE_DISABLE=1 -test --define=MEDIAPIPE_DISABLE=1 build --cxxopt=-DMEDIAPIPE_DISABLE=1 -coverage --cxxopt=-DMEDIAPIPE_DISABLE=1 -test --cxxopt=-DMEDIAPIPE_DISABLE=1 build --define=PYTHON_DISABLE=1 -coverage --define=PYTHON_DISABLE=1 -test --define=PYTHON_DISABLE=1 build --cxxopt=-DPYTHON_DISABLE=1 -coverage --cxxopt=-DPYTHON_DISABLE=1 -test --cxxopt=-DPYTHON_DISABLE=1 build --cxxopt=-DOVMS_DUMP_TO_FILE=0 -coverage --cxxopt=-DOVMS_DUMP_TO_FILE=0 -test --cxxopt=-DOVMS_DUMP_TO_FILE=0 diff --git a/Dockerfile.openvino b/Dockerfile.openvino index ce42f46b31..ab7f905e7c 100644 --- a/Dockerfile.openvino +++ b/Dockerfile.openvino @@ -193,6 +193,7 @@ RUN wget -O video.mp4 "https://www.pexels.com/download/video/3044127/?fps=24.0&h COPY mediapipe /mediapipe/mediapipe/ COPY third_party /mediapipe/third_party/ COPY .bazelrc /mediapipe/ +COPY .user.bazelrc /mediapipe/ COPY .bazelversion /mediapipe/ COPY *.sh /mediapipe/ COPY *.py /mediapipe/ diff --git a/WORKSPACE b/WORKSPACE index ac360ecb0d..9c51f62fd6 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -636,7 +636,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") git_repository( name = "ovms", remote = "https://github.com/openvinotoolkit/model_server", - commit = "ba43198285a1bbad7ec74b672161eb84d27e6adf" # Windows groovy (#2762) + commit = "aa07d47407557781036e2e4f00501bfa5bf3c79b" # Windows groovy (#2762) ) # DEV ovms - adjust local repository path for build @@ -837,15 +837,8 @@ git_repository( patches = ["@ovms//external:mwaitpkg.patch",] ) -load("@rules_foreign_cc//foreign_cc:cmake.bzl", "cmake") -load("@//third_party/model_api:model_api.bzl", "model_api_repository") -model_api_repository(name="_model-api") -new_git_repository( - name = "model_api", - remote = "https:///github.com/openvinotoolkit/model_api/", - build_file = "@_model-api//:BUILD", - commit = "eb9fcfb1e1eebc047ff144707f76203b132e1aa6" # master Jun 24 15:02:17 2024 [cpp] Fix num classes check -) +load("@//third_party/model_api:model_api.bzl", "workspace_model_api") +workspace_model_api() git_repository( name = "nlohmann_json", diff --git a/ci/testOnCommit.groovy b/ci/testOnCommit.groovy index 1915b04bf3..1395a36c43 100644 --- a/ci/testOnCommit.groovy +++ b/ci/testOnCommit.groovy @@ -14,6 +14,8 @@ pipeline { } stage("build image") { steps { + // TODO this should be enabled after Jenkins update + // sh "echo build --remote_cache=${env.OVMS_BAZEL_REMOTE_CACHE_URL} > .user.bazelrc" sh script: "make docker_build OVMS_MEDIA_IMAGE_TAG=${shortCommit}" } } diff --git a/mediapipe/calculators/ovms/modelapiovmsadapter.cc b/mediapipe/calculators/ovms/modelapiovmsadapter.cc index 650548b4b1..e7fa510c5e 100644 --- a/mediapipe/calculators/ovms/modelapiovmsadapter.cc +++ b/mediapipe/calculators/ovms/modelapiovmsadapter.cc @@ -70,31 +70,107 @@ OVMSInferenceAdapter::~OVMSInferenceAdapter() { LOG(INFO) << "OVMSAdapter destr"; } -InferenceOutput OVMSInferenceAdapter::infer(const InferenceInput& input) { +inline std::vector getShapeAcceptableByCAPI(const ov::Shape& shape) { + if (std::any_of(shape.begin(), shape.end(), [](size_t dim) { + return dim > std::numeric_limits::max();})) { + throw std::runtime_error("Cannot use C-API with dimension size greater than int64_t max value"); + } + return std::vector{shape.begin(), shape.end()}; +} + +void OVMSInferenceAdapter::infer(const InferenceInput& input, InferenceOutput& output) { ///////////////////// // PREPARE REQUEST ///////////////////// OVMS_InferenceRequest* request{nullptr}; ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestNew(&request, cserver, servableName.c_str(), servableVersion)); CREATE_GUARD(requestGuard, OVMS_InferenceRequest, request); - - InferenceOutput output; + OVMS_Status* status{nullptr}; + std::vector outputsSet; // PREPARE EACH INPUT // extract single tensor for (const auto& [name, input_tensor] : input) { - const char* realInputName = name.c_str(); - const auto& ovinputShape = input_tensor.get_shape(); - if (std::any_of(ovinputShape.begin(), ovinputShape.end(), [](size_t dim) { - return dim > std::numeric_limits::max();})) { - throw std::runtime_error("Cannot use C-API with dimension size greater than int64_t max value"); + const char* realName = name.c_str(); + const auto& ovShape = input_tensor.get_shape(); + std::vector capiShape = getShapeAcceptableByCAPI(ovShape); + OVMS_DataType inputDataType = OVPrecision2CAPI(input_tensor.get_element_type()); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddInput(request, realName, inputDataType, capiShape.data(), capiShape.size())); + const uint32_t NOT_USED_NUM = 0; + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestInputSetData(request, + realName, + reinterpret_cast(input_tensor.data()), + input_tensor.get_byte_size(), + OVMS_BUFFERTYPE_CPU, + NOT_USED_NUM)); + } + for (const auto& [name, output_tensor] : output) { + outputsSet.emplace_back(name); + const char* realName = name.c_str(); + const auto& ovShape = output_tensor.get_shape(); + std::vector capiShape = getShapeAcceptableByCAPI(ovShape); + OVMS_DataType inputDataType = OVPrecision2CAPI(output_tensor.get_element_type()); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddOutput(request, realName, inputDataType, capiShape.data(), capiShape.size())); + const uint32_t NOT_USED_NUM = 0; + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestOutputSetData(request, + realName, + reinterpret_cast(output_tensor.data()), + output_tensor.get_byte_size(), + OVMS_BUFFERTYPE_CPU, + NOT_USED_NUM)); + + } +#if (OVMS_DUMP_TO_FILE == 1) + dumpOvTensorInput(input,"input"); +#endif + ////////////////// + // INFERENCE + ////////////////// + OVMS_InferenceResponse* response = nullptr; + ASSERT_CAPI_STATUS_NULL(OVMS_Inference(cserver, request, &response)); + CREATE_GUARD(responseGuard, OVMS_InferenceResponse, response); + uint32_t outputCount = 42; + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseOutputCount(response, &outputCount)); + uint32_t parameterCount = 42; + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseParameterCount(response, ¶meterCount)); + const void* voutputData; + size_t bytesize = 42; + OVMS_DataType datatype = (OVMS_DataType)199; + const int64_t* shape{nullptr}; + size_t dimCount = 42; + OVMS_BufferType bufferType = (OVMS_BufferType)199; + uint32_t deviceId = 42; + const char* outputName{nullptr}; + for (size_t i = 0; i < outputCount; ++i) { + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseOutput(response, i, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId)); + if (std::find(outputsSet.begin(), outputsSet.end(), outputName) == outputsSet.end()) { + output.emplace(outputName, std::move(makeOvTensor(datatype, shape, dimCount, voutputData, bytesize))); } - std::vector inputShape{ovinputShape.begin(), ovinputShape.end()}; + } +#if (OVMS_DUMP_TO_FILE == 1) + dumpOvTensorInput(output,"output"); +#endif +} +InferenceOutput OVMSInferenceAdapter::infer(const InferenceInput& input) { + ///////////////////// + // PREPARE REQUEST + ///////////////////// + OVMS_InferenceRequest* request{nullptr}; + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestNew(&request, cserver, servableName.c_str(), servableVersion)); + CREATE_GUARD(requestGuard, OVMS_InferenceRequest, request); + + InferenceOutput output; + OVMS_Status* status{nullptr}; + // PREPARE EACH INPUT + for (const auto& [name, input_tensor] : input) { + const char* realName = name.c_str(); + const auto& ovShape = input_tensor.get_shape(); + std::vector capiShape = getShapeAcceptableByCAPI(ovShape); OVMS_DataType inputDataType = OVPrecision2CAPI(input_tensor.get_element_type()); - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddInput(request, realInputName, inputDataType, inputShape.data(), inputShape.size())); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddInput(request, realName, inputDataType, capiShape.data(), capiShape.size())); const uint32_t NOT_USED_NUM = 0; ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestInputSetData(request, - realInputName, + realName, reinterpret_cast(input_tensor.data()), input_tensor.get_byte_size(), OVMS_BUFFERTYPE_CPU, @@ -135,7 +211,7 @@ InferenceOutput OVMSInferenceAdapter::infer(const InferenceInput& input) { const char* outputName{nullptr}; for (size_t i = 0; i < outputCount; ++i) { ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseOutput(response, i, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId)); - output[outputName] = makeOvTensor(datatype, shape, dimCount, voutputData, bytesize); + output.emplace(outputName, std::move(makeOvTensor(datatype, shape, dimCount, voutputData, bytesize))); } #if (OVMS_DUMP_TO_FILE == 1) dumpOvTensorInput(output,"output"); @@ -169,16 +245,32 @@ void OVMSInferenceAdapter::loadModel(const std::shared_ptr& mod inputMinMax.second.emplace_back(shapeMax[i]); } this->inShapesMinMaxes.insert({tensorName, std::move(inputMinMax)}); + this->inputDatatypes.insert({tensorName, CAPI2OVPrecision(datatype)}); } for (id = 0; id < outputCount; ++id) { ASSERT_CAPI_STATUS_NULL(OVMS_ServableMetadataOutput(servableMetadata, id, &tensorName, &datatype, &dimCount, &shapeMin, &shapeMax)); outputNames.emplace_back(tensorName); + shape_min_max_t outputMinMax; + for (size_t i = 0; i < dimCount; ++i) { + outputMinMax.first.emplace_back(shapeMin[i]); + outputMinMax.second.emplace_back(shapeMax[i]); + } + this->outShapesMinMaxes.insert({tensorName, std::move(outputMinMax)}); + this->outputDatatypes.insert({tensorName, CAPI2OVPrecision(datatype)}); } const ov::AnyMap* servableMetadataRtInfo; ASSERT_CAPI_STATUS_NULL(OVMS_ServableMetadataInfo(servableMetadata, reinterpret_cast(&servableMetadataRtInfo))); this->modelConfig = *servableMetadataRtInfo; } +ov::element::Type_t OVMSInferenceAdapter::getInputDatatype(const std::string& inputName) const { + return inputDatatypes.at(inputName); +} + +ov::element::Type_t OVMSInferenceAdapter::getOutputDatatype(const std::string& outputName) const { + return outputDatatypes.at(outputName); +} + ov::PartialShape OVMSInferenceAdapter::getInputShape(const std::string& inputName) const { auto it = inShapesMinMaxes.find(inputName); if (it == inShapesMinMaxes.end()) { @@ -194,6 +286,21 @@ ov::PartialShape OVMSInferenceAdapter::getInputShape(const std::string& inputNam } return ovShape; } +ov::PartialShape OVMSInferenceAdapter::getOutputShape(const std::string& outputName) const { + auto it = outShapesMinMaxes.find(outputName); + if (it == outShapesMinMaxes.end()) { + LOG(INFO) << "Could not find output:" << outputName; + throw std::runtime_error(std::string("Adapter could not find output:") + outputName); + } + + ov::PartialShape ovShape; + const auto& [minBorder, maxBorder] = it->second; + ovShape.reserve(minBorder.size()); + for (size_t i = 0; i < minBorder.size(); ++i) { + ovShape.emplace_back(ov::Dimension{minBorder[i], maxBorder[i]}); + } + return ovShape; +} std::vector OVMSInferenceAdapter::getInputNames() const { return inputNames; } diff --git a/mediapipe/calculators/ovms/modelapiovmsadapter.hpp b/mediapipe/calculators/ovms/modelapiovmsadapter.hpp index 8453b4be8a..7a315bd950 100644 --- a/mediapipe/calculators/ovms/modelapiovmsadapter.hpp +++ b/mediapipe/calculators/ovms/modelapiovmsadapter.hpp @@ -49,6 +49,9 @@ class OVMSInferenceAdapter : public ::InferenceAdapter { std::vector inputNames; std::vector outputNames; shapes_min_max_t inShapesMinMaxes; + shapes_min_max_t outShapesMinMaxes; + std::unordered_map inputDatatypes; + std::unordered_map outputDatatypes; ov::AnyMap modelConfig; public: @@ -64,6 +67,7 @@ class OVMSInferenceAdapter : public ::InferenceAdapter { } virtual ~OVMSInferenceAdapter(); InferenceOutput infer(const InferenceInput& input) override; + void infer(const InferenceInput& input, InferenceOutput& output) override; void loadModel(const std::shared_ptr& model, ov::Core& core, const std::string& device, const ov::AnyMap& compilationConfig, size_t max_num_requests = 1) override; void inferAsync(const InferenceInput& input, const CallbackData callback_args) override; @@ -73,6 +77,9 @@ class OVMSInferenceAdapter : public ::InferenceAdapter { void awaitAny(); size_t getNumAsyncExecutors() const; ov::PartialShape getInputShape(const std::string& inputName) const override; + ov::PartialShape getOutputShape(const std::string& outputName) const override; + ov::element::Type_t getInputDatatype(const std::string& inputName) const override; + ov::element::Type_t getOutputDatatype(const std::string& outputName) const override; std::vector getInputNames() const override; std::vector getOutputNames() const override; const ov::AnyMap& getModelConfig() const override; diff --git a/third_party/BUILD b/third_party/BUILD index afc5a2afc1..236283d68b 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -158,6 +158,14 @@ cmake( name = "opencv_cmake", # Values to be passed as -Dkey=value on the CMake command line; # here are serving to provide some CMake script configuration options + build_args = [ + "--verbose", + "--", # <- Pass remaining options to the native tool. + # https://github.com/bazelbuild/rules_foreign_cc/issues/329 + # there is no elegant parallel compilation support + "VERBOSE=1", + "-j 32", + ], cache_entries = { "CMAKE_BUILD_TYPE": "Release", # The module list is always sorted alphabetically so that we do not diff --git a/third_party/model_api/model_api.bzl b/third_party/model_api/model_api.bzl index dfebc65452..461120e3d0 100644 --- a/third_party/model_api/model_api.bzl +++ b/third_party/model_api/model_api.bzl @@ -14,6 +14,7 @@ # limitations under the License. # +load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") def _is_windows(ctx): return ctx.os.name.lower().find("windows") != -1 @@ -123,3 +124,13 @@ model_api_repository = repository_rule( implementation = _impl, local=False, ) + +def workspace_model_api(): + model_api_repository(name="_model-api") + new_git_repository( + name = "model_api", + remote = "https:///github.com/openvinotoolkit/model_api/", + build_file = "@_model-api//:BUILD", + commit = "25c88f8fd1ebe08447aca9a959a7a5f37751867e" # master 31th October 2024 Adjust cpp inference adapters for OVMS (#212) + ) +