From bd614c94fd9e9d8820837164b80e598e2f418325 Mon Sep 17 00:00:00 2001 From: sfatima Date: Fri, 28 Feb 2025 16:38:37 +0530 Subject: [PATCH 1/4] Commit Changes --- .../core/providers/openvino/contexts.h | 1 + .../openvino/openvino_provider_factory.cc | 37 ++++++++++++------- .../core/session/provider_bridge_ort.cc | 12 ++++-- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/onnxruntime/core/providers/openvino/contexts.h b/onnxruntime/core/providers/openvino/contexts.h index 216fc5b132696..a1a756a9baef7 100644 --- a/onnxruntime/core/providers/openvino/contexts.h +++ b/onnxruntime/core/providers/openvino/contexts.h @@ -101,6 +101,7 @@ struct ProviderInfo { bool so_context_embed_mode{false}; // ORT session option bool so_share_ep_contexts{false}; // ORT session option fs::path so_context_file_path{}; // ORT session option + const ConfigOptions* config_options{NULL}; }; // Holds context applicable to the entire EP instance. diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index 95e039f8b6d5f..a9b5be2e13deb 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -14,12 +14,14 @@ namespace onnxruntime { namespace openvino_ep { -void ParseConfigOptions(ProviderInfo& pi, const ConfigOptions& config_options) { - pi.so_disable_cpu_ep_fallback = config_options.GetConfigOrDefault(kOrtSessionOptionsDisableCPUEPFallback, "0") == "1"; - pi.so_context_enable = config_options.GetConfigOrDefault(kOrtSessionOptionEpContextEnable, "0") == "1"; - pi.so_context_embed_mode = config_options.GetConfigOrDefault(kOrtSessionOptionEpContextEmbedMode, "0") == "1"; - pi.so_share_ep_contexts = config_options.GetConfigOrDefault(kOrtSessionOptionShareEpContexts, "0") == "1"; - pi.so_context_file_path = config_options.GetConfigOrDefault(kOrtSessionOptionEpContextFilePath, ""); +void ParseConfigOptions(ProviderInfo& pi) { + if(pi.config_options==NULL) + return; + pi.so_disable_cpu_ep_fallback = pi.config_options->GetConfigOrDefault(kOrtSessionOptionsDisableCPUEPFallback, "0") == "1"; + pi.so_context_enable = pi.config_options->GetConfigOrDefault(kOrtSessionOptionEpContextEnable, "0") == "1"; + pi.so_context_embed_mode = pi.config_options->GetConfigOrDefault(kOrtSessionOptionEpContextEmbedMode, "0") == "1"; + pi.so_share_ep_contexts = pi.config_options->GetConfigOrDefault(kOrtSessionOptionShareEpContexts, "0") == "1"; + pi.so_context_file_path = pi.config_options->GetConfigOrDefault(kOrtSessionOptionEpContextFilePath, ""); } void* ParseUint64(const ProviderOptions& provider_options, std::string option_name) { @@ -166,6 +168,7 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { ~OpenVINOProviderFactory() override {} std::unique_ptr CreateProvider() override { + ParseConfigOptions(provider_info_); return std::make_unique(provider_info_, shared_context_); } @@ -184,13 +187,23 @@ struct OpenVINO_Provider : Provider { void* GetInfo() override { return &info_; } std::shared_ptr CreateExecutionProviderFactory(const void* void_params) override { - // Extract the void_params into ProviderOptions and ConfigOptions - using ConfigBuffer = std::pair; - const ConfigBuffer* buffer = reinterpret_cast(void_params); - const auto& provider_options = *buffer->first; - const auto& config_options = buffer->second; + if (void_params == nullptr) { + LOGS_DEFAULT(ERROR) << "[OpenVINO EP] Passed NULL options to CreateExecutionProviderFactory()"; + return nullptr; + } + + std::array pointers_array = *reinterpret_cast*>(void_params); + const ProviderOptions* provider_options_ptr = reinterpret_cast(pointers_array[0]); + const ConfigOptions* config_options = reinterpret_cast(pointers_array[1]); + + if(provider_options_ptr == NULL) { + LOGS_DEFAULT(ERROR) << "[OpenVINO EP] Passed NULL ProviderOptions to CreateExecutionProviderFactory()"; + return nullptr; + } + const ProviderOptions provider_options = *provider_options_ptr; ProviderInfo pi; + pi.config_options = config_options; std::string bool_flag = ""; @@ -326,8 +339,6 @@ struct OpenVINO_Provider : Provider { pi.disable_dynamic_shapes = ParseBooleanOption(provider_options, "disable_dynamic_shapes"); - ParseConfigOptions(pi, config_options); - // Always true for NPU plugin or when passed . if (pi.device_type.find("NPU") != std::string::npos) { pi.disable_dynamic_shapes = true; diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 1d25ceb9af8a3..77c6d4c371f69 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -1998,10 +1998,14 @@ std::shared_ptr QNNProviderFactoryCreator::Create(con std::shared_ptr OpenVINOProviderFactoryCreator::Create( const ProviderOptions* provider_options_map, const SessionOptions* session_options) { // Append session options applicable for EP to EP Provider options. - std::pair config_buffer = {provider_options_map, - session_options->config_options}; - const void* obj = reinterpret_cast(&config_buffer); - return s_library_openvino.Get().CreateExecutionProviderFactory(obj); + const ConfigOptions* config_options = nullptr; + if (session_options !=nullptr) { + config_options = &session_options->config_options; + } + + std::array configs_array = {provider_options_map, config_options}; + const void* arg = reinterpret_cast(&configs_array); + return s_library_openvino.Get().CreateExecutionProviderFactory(arg); } std::shared_ptr DnnlProviderFactoryCreator::Create(const OrtDnnlProviderOptions* dnnl_options) { From e7923285b117849d87feb029afbe9d09d44566a8 Mon Sep 17 00:00:00 2001 From: sfatima Date: Fri, 28 Feb 2025 17:38:04 +0530 Subject: [PATCH 2/4] Changes to make sure to honor SessionOptions API Contract --- .../openvino/openvino_provider_factory.cc | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index a9b5be2e13deb..c4fe16e035241 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -17,11 +17,19 @@ namespace openvino_ep { void ParseConfigOptions(ProviderInfo& pi) { if(pi.config_options==NULL) return; + pi.so_disable_cpu_ep_fallback = pi.config_options->GetConfigOrDefault(kOrtSessionOptionsDisableCPUEPFallback, "0") == "1"; pi.so_context_enable = pi.config_options->GetConfigOrDefault(kOrtSessionOptionEpContextEnable, "0") == "1"; pi.so_context_embed_mode = pi.config_options->GetConfigOrDefault(kOrtSessionOptionEpContextEmbedMode, "0") == "1"; pi.so_share_ep_contexts = pi.config_options->GetConfigOrDefault(kOrtSessionOptionShareEpContexts, "0") == "1"; pi.so_context_file_path = pi.config_options->GetConfigOrDefault(kOrtSessionOptionEpContextFilePath, ""); + + if (pi.so_share_ep_contexts) { + ov::AnyMap map; + map["NPU_COMPILATION_MODE_PARAMS"] = "enable-wd-blockarg-input=true compute-layers-with-higher-precision=Sqrt,Power,ReduceSum"; + pi.load_config["NPU"] = std::move(map); + } + } void* ParseUint64(const ProviderOptions& provider_options, std::string option_name) { @@ -344,13 +352,6 @@ struct OpenVINO_Provider : Provider { pi.disable_dynamic_shapes = true; } - // Append values to config to support weight-as-inputs conversion for shared contexts - if (pi.so_share_ep_contexts) { - ov::AnyMap map; - map["NPU_COMPILATION_MODE_PARAMS"] = "enable-wd-blockarg-input=true compute-layers-with-higher-precision=Sqrt,Power,ReduceSum"; - pi.load_config["NPU"] = std::move(map); - } - return std::make_shared(pi, SharedContext::Get()); } From 7e6ca5ff1c88b31a90ee56bf90e1f093a3b9b2c1 Mon Sep 17 00:00:00 2001 From: sfatima Date: Fri, 28 Feb 2025 16:38:37 +0530 Subject: [PATCH 3/4] Commit Changes Changes to make sure to honor SessionOptions API Contract --- .../core/providers/openvino/contexts.h | 1 + .../openvino/openvino_provider_factory.cc | 52 ++++++++++++------- .../core/session/provider_bridge_ort.cc | 12 +++-- 3 files changed, 41 insertions(+), 24 deletions(-) diff --git a/onnxruntime/core/providers/openvino/contexts.h b/onnxruntime/core/providers/openvino/contexts.h index 216fc5b132696..a1a756a9baef7 100644 --- a/onnxruntime/core/providers/openvino/contexts.h +++ b/onnxruntime/core/providers/openvino/contexts.h @@ -101,6 +101,7 @@ struct ProviderInfo { bool so_context_embed_mode{false}; // ORT session option bool so_share_ep_contexts{false}; // ORT session option fs::path so_context_file_path{}; // ORT session option + const ConfigOptions* config_options{NULL}; }; // Holds context applicable to the entire EP instance. diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index 95e039f8b6d5f..c4fe16e035241 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -14,12 +14,22 @@ namespace onnxruntime { namespace openvino_ep { -void ParseConfigOptions(ProviderInfo& pi, const ConfigOptions& config_options) { - pi.so_disable_cpu_ep_fallback = config_options.GetConfigOrDefault(kOrtSessionOptionsDisableCPUEPFallback, "0") == "1"; - pi.so_context_enable = config_options.GetConfigOrDefault(kOrtSessionOptionEpContextEnable, "0") == "1"; - pi.so_context_embed_mode = config_options.GetConfigOrDefault(kOrtSessionOptionEpContextEmbedMode, "0") == "1"; - pi.so_share_ep_contexts = config_options.GetConfigOrDefault(kOrtSessionOptionShareEpContexts, "0") == "1"; - pi.so_context_file_path = config_options.GetConfigOrDefault(kOrtSessionOptionEpContextFilePath, ""); +void ParseConfigOptions(ProviderInfo& pi) { + if(pi.config_options==NULL) + return; + + pi.so_disable_cpu_ep_fallback = pi.config_options->GetConfigOrDefault(kOrtSessionOptionsDisableCPUEPFallback, "0") == "1"; + pi.so_context_enable = pi.config_options->GetConfigOrDefault(kOrtSessionOptionEpContextEnable, "0") == "1"; + pi.so_context_embed_mode = pi.config_options->GetConfigOrDefault(kOrtSessionOptionEpContextEmbedMode, "0") == "1"; + pi.so_share_ep_contexts = pi.config_options->GetConfigOrDefault(kOrtSessionOptionShareEpContexts, "0") == "1"; + pi.so_context_file_path = pi.config_options->GetConfigOrDefault(kOrtSessionOptionEpContextFilePath, ""); + + if (pi.so_share_ep_contexts) { + ov::AnyMap map; + map["NPU_COMPILATION_MODE_PARAMS"] = "enable-wd-blockarg-input=true compute-layers-with-higher-precision=Sqrt,Power,ReduceSum"; + pi.load_config["NPU"] = std::move(map); + } + } void* ParseUint64(const ProviderOptions& provider_options, std::string option_name) { @@ -166,6 +176,7 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { ~OpenVINOProviderFactory() override {} std::unique_ptr CreateProvider() override { + ParseConfigOptions(provider_info_); return std::make_unique(provider_info_, shared_context_); } @@ -184,13 +195,23 @@ struct OpenVINO_Provider : Provider { void* GetInfo() override { return &info_; } std::shared_ptr CreateExecutionProviderFactory(const void* void_params) override { - // Extract the void_params into ProviderOptions and ConfigOptions - using ConfigBuffer = std::pair; - const ConfigBuffer* buffer = reinterpret_cast(void_params); - const auto& provider_options = *buffer->first; - const auto& config_options = buffer->second; + if (void_params == nullptr) { + LOGS_DEFAULT(ERROR) << "[OpenVINO EP] Passed NULL options to CreateExecutionProviderFactory()"; + return nullptr; + } + + std::array pointers_array = *reinterpret_cast*>(void_params); + const ProviderOptions* provider_options_ptr = reinterpret_cast(pointers_array[0]); + const ConfigOptions* config_options = reinterpret_cast(pointers_array[1]); + + if(provider_options_ptr == NULL) { + LOGS_DEFAULT(ERROR) << "[OpenVINO EP] Passed NULL ProviderOptions to CreateExecutionProviderFactory()"; + return nullptr; + } + const ProviderOptions provider_options = *provider_options_ptr; ProviderInfo pi; + pi.config_options = config_options; std::string bool_flag = ""; @@ -326,20 +347,11 @@ struct OpenVINO_Provider : Provider { pi.disable_dynamic_shapes = ParseBooleanOption(provider_options, "disable_dynamic_shapes"); - ParseConfigOptions(pi, config_options); - // Always true for NPU plugin or when passed . if (pi.device_type.find("NPU") != std::string::npos) { pi.disable_dynamic_shapes = true; } - // Append values to config to support weight-as-inputs conversion for shared contexts - if (pi.so_share_ep_contexts) { - ov::AnyMap map; - map["NPU_COMPILATION_MODE_PARAMS"] = "enable-wd-blockarg-input=true compute-layers-with-higher-precision=Sqrt,Power,ReduceSum"; - pi.load_config["NPU"] = std::move(map); - } - return std::make_shared(pi, SharedContext::Get()); } diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 1d25ceb9af8a3..77c6d4c371f69 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -1998,10 +1998,14 @@ std::shared_ptr QNNProviderFactoryCreator::Create(con std::shared_ptr OpenVINOProviderFactoryCreator::Create( const ProviderOptions* provider_options_map, const SessionOptions* session_options) { // Append session options applicable for EP to EP Provider options. - std::pair config_buffer = {provider_options_map, - session_options->config_options}; - const void* obj = reinterpret_cast(&config_buffer); - return s_library_openvino.Get().CreateExecutionProviderFactory(obj); + const ConfigOptions* config_options = nullptr; + if (session_options !=nullptr) { + config_options = &session_options->config_options; + } + + std::array configs_array = {provider_options_map, config_options}; + const void* arg = reinterpret_cast(&configs_array); + return s_library_openvino.Get().CreateExecutionProviderFactory(arg); } std::shared_ptr DnnlProviderFactoryCreator::Create(const OrtDnnlProviderOptions* dnnl_options) { From 534e66d41a3b49305993abd89362f23dda827a68 Mon Sep 17 00:00:00 2001 From: sfatima Date: Mon, 3 Mar 2025 14:38:46 +0530 Subject: [PATCH 4/4] Commit Lint Errors fix --- .../providers/openvino/openvino_provider_factory.cc | 10 ++-------- onnxruntime/core/session/provider_bridge_ort.cc | 2 +- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index c4fe16e035241..e5a858867e877 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -15,7 +15,7 @@ namespace onnxruntime { namespace openvino_ep { void ParseConfigOptions(ProviderInfo& pi) { - if(pi.config_options==NULL) + if (pi.config_options == nullptr) return; pi.so_disable_cpu_ep_fallback = pi.config_options->GetConfigOrDefault(kOrtSessionOptionsDisableCPUEPFallback, "0") == "1"; @@ -201,15 +201,9 @@ struct OpenVINO_Provider : Provider { } std::array pointers_array = *reinterpret_cast*>(void_params); - const ProviderOptions* provider_options_ptr = reinterpret_cast(pointers_array[0]); + const ProviderOptions provider_options = *reinterpret_cast(pointers_array[0]); const ConfigOptions* config_options = reinterpret_cast(pointers_array[1]); - if(provider_options_ptr == NULL) { - LOGS_DEFAULT(ERROR) << "[OpenVINO EP] Passed NULL ProviderOptions to CreateExecutionProviderFactory()"; - return nullptr; - } - const ProviderOptions provider_options = *provider_options_ptr; - ProviderInfo pi; pi.config_options = config_options; diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 77c6d4c371f69..320f7e1eb0aa1 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -1999,7 +1999,7 @@ std::shared_ptr OpenVINOProviderFactoryCreator::Creat const ProviderOptions* provider_options_map, const SessionOptions* session_options) { // Append session options applicable for EP to EP Provider options. const ConfigOptions* config_options = nullptr; - if (session_options !=nullptr) { + if (session_options != nullptr) { config_options = &session_options->config_options; }