Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[OpenVINO]Session Options Appended After AppendExecutionProvider #23852

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
+35 −24
Open

[OpenVINO]Session Options Appended After AppendExecutionProvider #23852

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
bd614c9
Commit Changes
sfatimar Feb 28, 2025
e792328
Changes to make sure to honor SessionOptions API Contract
sfatimar Feb 28, 2025
7e6ca5f
Commit Changes
sfatimar Feb 28, 2025
534e66d
Commit Lint Errors fix
sfatimar Mar 3, 2025
07720dd
Merge branch 'sahar/session_options' of https://github.com/intel/onnx…
sfatimar Mar 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions onnxruntime/core/providers/openvino/contexts.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ struct ProviderInfo {
bool so_context_embed_mode{false}; // ORT session option
bool so_share_ep_contexts{false}; // ORT session option
fs::path so_context_file_path{}; // ORT session option
const ConfigOptions* config_options{NULL};
};

// Holds context applicable to the entire EP instance.
Expand Down
52 changes: 32 additions & 20 deletions onnxruntime/core/providers/openvino/openvino_provider_factory.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,22 @@

namespace onnxruntime {
namespace openvino_ep {
void ParseConfigOptions(ProviderInfo& pi, const ConfigOptions& config_options) {
pi.so_disable_cpu_ep_fallback = config_options.GetConfigOrDefault(kOrtSessionOptionsDisableCPUEPFallback, "0") == "1";
pi.so_context_enable = config_options.GetConfigOrDefault(kOrtSessionOptionEpContextEnable, "0") == "1";
pi.so_context_embed_mode = config_options.GetConfigOrDefault(kOrtSessionOptionEpContextEmbedMode, "0") == "1";
pi.so_share_ep_contexts = config_options.GetConfigOrDefault(kOrtSessionOptionShareEpContexts, "0") == "1";
pi.so_context_file_path = config_options.GetConfigOrDefault(kOrtSessionOptionEpContextFilePath, "");
void ParseConfigOptions(ProviderInfo& pi) {
if(pi.config_options==NULL)
return;

pi.so_disable_cpu_ep_fallback = pi.config_options->GetConfigOrDefault(kOrtSessionOptionsDisableCPUEPFallback, "0") == "1";
pi.so_context_enable = pi.config_options->GetConfigOrDefault(kOrtSessionOptionEpContextEnable, "0") == "1";
pi.so_context_embed_mode = pi.config_options->GetConfigOrDefault(kOrtSessionOptionEpContextEmbedMode, "0") == "1";
pi.so_share_ep_contexts = pi.config_options->GetConfigOrDefault(kOrtSessionOptionShareEpContexts, "0") == "1";
pi.so_context_file_path = pi.config_options->GetConfigOrDefault(kOrtSessionOptionEpContextFilePath, "");

if (pi.so_share_ep_contexts) {
ov::AnyMap map;
map["NPU_COMPILATION_MODE_PARAMS"] = "enable-wd-blockarg-input=true compute-layers-with-higher-precision=Sqrt,Power,ReduceSum";
pi.load_config["NPU"] = std::move(map);
}

}

void* ParseUint64(const ProviderOptions& provider_options, std::string option_name) {
Expand Down Expand Up @@ -166,6 +176,7 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory {
~OpenVINOProviderFactory() override {}

std::unique_ptr<IExecutionProvider> CreateProvider() override {
ParseConfigOptions(provider_info_);
return std::make_unique<OpenVINOExecutionProvider>(provider_info_, shared_context_);
}

Expand All @@ -184,13 +195,23 @@ struct OpenVINO_Provider : Provider {
void* GetInfo() override { return &info_; }

std::shared_ptr<IExecutionProviderFactory> CreateExecutionProviderFactory(const void* void_params) override {
// Extract the void_params into ProviderOptions and ConfigOptions
using ConfigBuffer = std::pair<const ProviderOptions*, const ConfigOptions&>;
const ConfigBuffer* buffer = reinterpret_cast<const ConfigBuffer*>(void_params);
const auto& provider_options = *buffer->first;
const auto& config_options = buffer->second;
if (void_params == nullptr) {
LOGS_DEFAULT(ERROR) << "[OpenVINO EP] Passed NULL options to CreateExecutionProviderFactory()";
return nullptr;
}

std::array<void*, 2> pointers_array = *reinterpret_cast<const std::array<void*, 2>*>(void_params);
const ProviderOptions* provider_options_ptr = reinterpret_cast<ProviderOptions*>(pointers_array[0]);
const ConfigOptions* config_options = reinterpret_cast<ConfigOptions*>(pointers_array[1]);

if(provider_options_ptr == NULL) {
LOGS_DEFAULT(ERROR) << "[OpenVINO EP] Passed NULL ProviderOptions to CreateExecutionProviderFactory()";
return nullptr;
}
const ProviderOptions provider_options = *provider_options_ptr;

ProviderInfo pi;
pi.config_options = config_options;

std::string bool_flag = "";

Expand Down Expand Up @@ -326,20 +347,11 @@ struct OpenVINO_Provider : Provider {

pi.disable_dynamic_shapes = ParseBooleanOption(provider_options, "disable_dynamic_shapes");

ParseConfigOptions(pi, config_options);

// Always true for NPU plugin or when passed .
if (pi.device_type.find("NPU") != std::string::npos) {
pi.disable_dynamic_shapes = true;
}

// Append values to config to support weight-as-inputs conversion for shared contexts
if (pi.so_share_ep_contexts) {
ov::AnyMap map;
map["NPU_COMPILATION_MODE_PARAMS"] = "enable-wd-blockarg-input=true compute-layers-with-higher-precision=Sqrt,Power,ReduceSum";
pi.load_config["NPU"] = std::move(map);
}

return std::make_shared<OpenVINOProviderFactory>(pi, SharedContext::Get());
}

Expand Down
12 changes: 8 additions & 4 deletions onnxruntime/core/session/provider_bridge_ort.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1998,10 +1998,14 @@ std::shared_ptr<IExecutionProviderFactory> QNNProviderFactoryCreator::Create(con
std::shared_ptr<IExecutionProviderFactory> OpenVINOProviderFactoryCreator::Create(
const ProviderOptions* provider_options_map, const SessionOptions* session_options) {
// Append session options applicable for EP to EP Provider options.
std::pair<const ProviderOptions*, const ConfigOptions&> config_buffer = {provider_options_map,
session_options->config_options};
const void* obj = reinterpret_cast<const void*>(&config_buffer);
return s_library_openvino.Get().CreateExecutionProviderFactory(obj);
const ConfigOptions* config_options = nullptr;
if (session_options !=nullptr) {
config_options = &session_options->config_options;
}

std::array<const void*, 2> configs_array = {provider_options_map, config_options};
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how about declare a struct type?

struct OpenVINOProviderFactoryCreateParams {
  const ProviderOptions* provider_options_map;
  const ConfigOptions* config_options;
};

This would be much easier to read

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Earlier we had a similar structure but looking at this code, https://github.com/microsoft/onnxruntime/blob/99c51a326e0ff54a56e7b194204d459932084408/onnxruntime/core/session/provider_bridge_ort.cc#L1992C2-L1992C3
I desired to have consistency.

const void* arg = reinterpret_cast<const void*>(&configs_array);
return s_library_openvino.Get().CreateExecutionProviderFactory(arg);
}

std::shared_ptr<IExecutionProviderFactory> DnnlProviderFactoryCreator::Create(const OrtDnnlProviderOptions* dnnl_options) {
Expand Down
Loading