Skip to content

Commit

Permalink
Mark mandatory conformance tests for HW plugins (openvinotoolkit#27197)
Browse files Browse the repository at this point in the history
### Details:
- *Mark OVHoldersTestOnImportedNetwork and OVClassQueryModelTest
mandatory for HW plugin and optional for meta-plugin*
 - *Remove the meta-plugin skipping in test cases*

### Tickets:
 - *CVS-130212*

---------

Signed-off-by: Chen, Peter <peter.chen@intel.com>
Co-authored-by: panhaiqi <haiqi.pan@intel.com>
  • Loading branch information
peterchen-intel and panhaiqi authored Oct 26, 2024
1 parent 0c07136 commit f3e3f75
Show file tree
Hide file tree
Showing 8 changed files with 35 additions and 39 deletions.
2 changes: 1 addition & 1 deletion src/plugins/hetero/tests/functional/skip_tests_config.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,6 @@ std::vector<std::string> disabledTestPatterns() {
R"(.*smoke_(Multi|Auto|Hetero)_BehaviorTests.*OVPropertiesTests.*SetCorrectProperties.*)",
R"(.*smoke_(Multi|Auto|Hetero)_BehaviorTests.*OVPropertiesTests.*canSetPropertyAndCheckGetProperty.*)",
R"(.*OVInferRequestCheckTensorPrecision.*get(Input|Output|Inputs|Outputs)From.*FunctionWith(Single|Several).*type=(u4|u1|i4|boolean).*)",
};
R"(.*OVGetMetricPropsTest.*OVGetMetricPropsTest.*GetMetricAndPrintNoThrow_AVAILABLE_DEVICES.*)"};
return retVector;
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
#include "behavior/ov_plugin/query_model.hpp"
#include "openvino/runtime/core.hpp"
#include "ov_api_conformance_helpers.hpp"
#include "behavior/ov_plugin/properties_tests.hpp"
#include "base/ov_behavior_test_utils.hpp"

using namespace ov::test::behavior;
using namespace ov::test::conformance;
Expand All @@ -15,17 +17,17 @@ namespace {
// OV Class Common tests with <pluginName, deviceName params>
//

INSTANTIATE_TEST_SUITE_P(ov_plugin_mandatory,
OVClassModelTestP,
::testing::Values(ov::test::utils::target_device));
INSTANTIATE_TEST_SUITE_P(ov_plugin_mandatory, OVClassModelTestP, ::testing::Values(ov::test::utils::target_device));

INSTANTIATE_TEST_SUITE_P(ov_plugin,
OVClassModelOptionalTestP,
::testing::Values(ov::test::utils::target_device));

// OV Class Query network

INSTANTIATE_TEST_SUITE_P(ov_plugin_mandatory,
static std::string (*NULL_getTestCaseName)(const testing::TestParamInfo<std::string>& info) = NULL;
INSTANTIATE_TEST_SUITE_P(ov_plugin,
OVClassQueryModelTest,
::testing::Values(ov::test::utils::target_device));
::testing::Values(ov::test::utils::target_device),
MARK_MANDATORY_API_FOR_HW_DEVICE_WITHOUT_PARAM());
} // namespace
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

#include "behavior/ov_plugin/life_time.hpp"
#include "ov_api_conformance_helpers.hpp"
#include "base/ov_behavior_test_utils.hpp"

using namespace ov::test::behavior;
using namespace ov::test::conformance;
Expand All @@ -14,7 +15,8 @@ INSTANTIATE_TEST_SUITE_P(ov_plugin_mandatory, OVHoldersTest,
::testing::Values(ov::test::utils::target_device),
OVHoldersTest::getTestCaseName);

INSTANTIATE_TEST_SUITE_P(ov_plugin_mandatory, OVHoldersTestOnImportedNetwork,
::testing::Values(ov::test::utils::target_device),
OVHoldersTestOnImportedNetwork::getTestCaseName);
INSTANTIATE_TEST_SUITE_P(ov_plugin,
OVHoldersTestOnImportedNetwork,
::testing::Values(ov::test::utils::target_device),
MARK_MANDATORY_API_FOR_HW_DEVICE_WITH_PARAM(OVHoldersTestOnImportedNetwork::getTestCaseName));
} // namespace
Original file line number Diff line number Diff line change
Expand Up @@ -105,17 +105,19 @@ INSTANTIATE_TEST_SUITE_P(ov_plugin, OVCheckMetricsPropsTests_ModelDependceProps,
// OV Class GetMetric
//

INSTANTIATE_TEST_SUITE_P(
ov_plugin_mandatory, OVGetMetricPropsTest,
::testing::Values(ov::test::utils::target_device));
INSTANTIATE_TEST_SUITE_P(ov_plugin,
OVGetMetricPropsTest,
::testing::Values(ov::test::utils::target_device),
MARK_MANDATORY_API_FOR_HW_DEVICE_WITHOUT_PARAM());

INSTANTIATE_TEST_SUITE_P(
ov_plugin, OVGetMetricPropsOptionalTest,
::testing::Values(ov::test::utils::target_device));

INSTANTIATE_TEST_SUITE_P(
ov_plugin_mandatory, OVGetAvailableDevicesPropsTest,
::testing::Values(ov::test::utils::target_device));
INSTANTIATE_TEST_SUITE_P(ov_plugin,
OVGetAvailableDevicesPropsTest,
::testing::Values(ov::test::utils::target_device),
MARK_MANDATORY_API_FOR_HW_DEVICE_WITHOUT_PARAM());

//
// OV Class GetConfig
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,23 @@
#include "common_test_utils/subgraph_builders/concat_with_params.hpp"
#include "common_test_utils/subgraph_builders/split_concat.hpp"

#define MARK_MANDATORY_PROPERTY_FOR_HW_DEVICE(GET_TEST_NAME) \
[](const testing::TestParamInfo<PropertiesParams>& info) { \
std::string name = GET_TEST_NAME(info); \
#define MARK_MANDATORY_PROPERTY_FOR_HW_DEVICE(GET_TEST_NAME) \
[](const testing::TestParamInfo<PropertiesParams>& info) { \
std::string name = GET_TEST_NAME(info); \
return (sw_plugin_in_target_device(ov::test::utils::target_device) ? "" : "mandatory_") + name; \
}

#define MARK_MANDATORY_API_FOR_HW_DEVICE_WITH_PARAM(GET_TEST_NAME) \
[](const testing::TestParamInfo<std::string>& info) { \
std::string name = GET_TEST_NAME(info); \
return (sw_plugin_in_target_device(ov::test::utils::target_device) ? "" : "mandatory_") + name; \
}

#define MARK_MANDATORY_API_FOR_HW_DEVICE_WITHOUT_PARAM() \
[](const testing::TestParamInfo<std::string>& info) { \
return sw_plugin_in_target_device(ov::test::utils::target_device) ? "" : "mandatory_"; \
}

namespace ov {
namespace test {
namespace behavior {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,6 @@ TEST_P(OVClassQueryModelTest, QueryModelWithMatMul) {
}

TEST_P(OVClassQueryModelTest, QueryModelHETEROWithDeviceIDNoThrow) {
if (sw_plugin_in_target_device(target_device)) {
return;
}
ov::Core ie = ov::test::utils::create_core();

auto deviceIDs = ie.get_property(target_device, ov::available_devices);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -148,9 +148,6 @@ void OVHoldersTestOnImportedNetwork::TearDown() {
}

TEST_P(OVHoldersTestOnImportedNetwork, LoadedTensor) {
if (sw_plugin_in_target_device(target_device)) {
return;
}
ov::Core core = ov::test::utils::create_core();
std::stringstream stream;
{
Expand All @@ -163,9 +160,6 @@ TEST_P(OVHoldersTestOnImportedNetwork, LoadedTensor) {
}

TEST_P(OVHoldersTestOnImportedNetwork, CreateRequestWithCoreRemoved) {
if (sw_plugin_in_target_device(target_device)) {
return;
}
ov::Core core = ov::test::utils::create_core();
std::stringstream stream;
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -588,9 +588,6 @@ TEST_P(OVCheckMetricsPropsTests_ModelDependceProps, ChangeCorrectDevicePropertie
TEST_P(OVClassSetDefaultDeviceIDPropTest, SetDefaultDeviceIDNoThrow) {
ov::Core ie = ov::test::utils::create_core();
// sw plugins are not requested to support `ov::available_devices` and ` ov::device::id` property
if (sw_plugin_in_target_device(target_device)) {
return;
}
auto deviceIDs = ie.get_property(target_device, ov::available_devices);
if (std::find(deviceIDs.begin(), deviceIDs.end(), deviceID) == deviceIDs.end()) {
GTEST_FAIL();
Expand All @@ -613,9 +610,6 @@ TEST_P(OVSpecificDeviceSetConfigTest, GetConfigSpecificDeviceNoThrow) {
deviceID = target_device.substr(pos + 1, target_device.size());
}
// sw plugins are not requested to support `ov::available_devices`, `ov::device::id` and `ov::num_streams` property
if (sw_plugin_in_target_device(target_device)) {
return;
}
auto deviceIDs = ie.get_property(clear_target_device, ov::available_devices);
if (std::find(deviceIDs.begin(), deviceIDs.end(), deviceID) == deviceIDs.end()) {
GTEST_FAIL() << "No DeviceID" << std::endl;
Expand Down Expand Up @@ -672,9 +666,6 @@ TEST_P(OVSpecificDeviceGetConfigTest, GetConfigSpecificDeviceNoThrow) {
}

TEST_P(OVGetAvailableDevicesPropsTest, GetAvailableDevicesNoThrow) {
if (sw_plugin_in_target_device(target_device)) {
return;
}
ov::Core ie = ov::test::utils::create_core();
std::vector<std::string> devices;

Expand Down Expand Up @@ -727,9 +718,6 @@ TEST_P(OVClassSetDevicePriorityConfigPropsTest, SetConfigAndCheckGetConfigNoThro
}

TEST_P(OVGetMetricPropsTest, GetMetricAndPrintNoThrow_AVAILABLE_DEVICES) {
if (sw_plugin_in_target_device(target_device)) {
return;
}
ov::Core ie = ov::test::utils::create_core();
std::vector<std::string> device_ids;

Expand Down

0 comments on commit f3e3f75

Please sign in to comment.