Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Clean meta plugin tests from CPU/GPU plugin #24477

Merged
Merged
Show file tree
Hide file tree
Changes from 21 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
cef4d27
[CPU Test] clean BATCH plugin test in cpu plugin test
zhaixuejun1993 May 13, 2024
4924831
[CPU Test] clean HETERO plugin test in cpu plugin test
zhaixuejun1993 May 13, 2024
8bbdbc0
Fix CI error
zhaixuejun1993 May 13, 2024
38a5e90
[CPU Test] clean BATCH plugin test in GPU plugin test
zhaixuejun1993 May 13, 2024
18f70f0
clean test skip list
zhaixuejun1993 May 13, 2024
9f61039
[CPU Test] clean HETERO plugin test in GPU plugin test
zhaixuejun1993 May 13, 2024
a7c0119
Fix CI format issue
zhaixuejun1993 May 14, 2024
36dc56d
Fix CI format issue
zhaixuejun1993 May 14, 2024
d36978d
Fix CI error
zhaixuejun1993 May 14, 2024
b78f4f3
Fix CI format issue
zhaixuejun1993 May 14, 2024
93dc950
Fix CI error
zhaixuejun1993 May 16, 2024
0de0458
Merge branch 'master' into xuejun/clean_mata_tests
zhaixuejun1993 May 17, 2024
3beb4ff
Merge branch 'master' into xuejun/clean_mata_tests
zhaixuejun1993 May 19, 2024
39d3924
Fix error
zhaixuejun1993 May 20, 2024
630f1dd
Fix CI format issue
zhaixuejun1993 May 20, 2024
e47fd13
Merge branch 'master' into xuejun/clean_mata_tests
peterchen-intel May 21, 2024
5479047
Merge branch 'master' into xuejun/clean_mata_tests
zhaixuejun1993 May 28, 2024
5c0ccae
Merge branch 'master' into xuejun/clean_mata_tests
zhaixuejun1993 May 28, 2024
5f863b0
Hetero -> ov::test::utils::DEVICE_HETERO
zhaixuejun1993 Jun 25, 2024
fd846b2
fix review comments
zhaixuejun1993 Jun 25, 2024
19e4574
Fix format issues
zhaixuejun1993 Jun 26, 2024
8cff5e1
Merge branch 'master' into xuejun/clean_mata_tests
zhaixuejun1993 Jul 10, 2024
438be5e
Merge branch 'master' into xuejun/clean_mata_tests
peterchen-intel Jul 29, 2024
fd551c7
Merge branch 'master' into xuejun/clean_mata_tests
peterchen-intel Jul 31, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,29 @@

#include "behavior/compiled_model/compiled_model_base.hpp"

using namespace ov::test::behavior;
namespace {
auto autoBatchConfigs = []() {
return std::vector<ov::AnyMap>{
// explicit batch size 4 to avoid fallback to no auto-batching
{{ov::device::priorities.name(), std::string(ov::test::utils::DEVICE_TEMPLATE) + "(4)"},
// no timeout to avoid increasing the test time
{ov::auto_batch_timeout.name(), "0"}}};
};
namespace ov {
namespace test {
namespace behavior {
auto autoBatchConfigs =
std::vector<ov::AnyMap>{// explicit batch size 4 to avoid fallback to no auto-batching
{{ov::device::priorities.name(), std::string(ov::test::utils::DEVICE_TEMPLATE) + "(4)"},
// no timeout to avoid increasing the test time
{ov::auto_batch_timeout.name(), "0"}}};

INSTANTIATE_TEST_SUITE_P(smoke_AutoBatch_BehaviorTests, OVCompiledModelBaseTest,
::testing::Combine(
::testing::Values(ov::test::utils::DEVICE_BATCH),
::testing::ValuesIn(autoBatchConfigs())),
INSTANTIATE_TEST_SUITE_P(smoke_AutoBatch_BehaviorTests,
OVCompiledModelBaseTest,
::testing::Combine(::testing::Values(ov::test::utils::DEVICE_BATCH),
::testing::ValuesIn(autoBatchConfigs)),
OVCompiledModelBaseTest::getTestCaseName);

} // namespace
std::vector<ov::element::Type> convert_types = {ov::element::f16, ov::element::i64};

INSTANTIATE_TEST_SUITE_P(smoke_AutoBatchBehaviorTests,
CompiledModelSetType,
::testing::Combine(::testing::ValuesIn(convert_types),
::testing::Values(ov::test::utils::DEVICE_BATCH),
::testing::ValuesIn(autoBatchConfigs)),
CompiledModelSetType::getTestCaseName);
} // namespace behavior
} // namespace test
} // namespace ov
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
// Copyright (C) 2018-2024 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//

#include "behavior/compiled_model/properties.hpp"

using namespace ov::test::behavior;
#include "behavior/ov_plugin/properties_tests.hpp"

namespace {
namespace ov {
namespace test {
namespace behavior {

const std::vector<ov::AnyMap> auto_batch_inproperties = {
{ov::num_streams(-100)},
Expand All @@ -33,4 +34,28 @@ INSTANTIATE_TEST_SUITE_P(smoke_AutoBatch_BehaviorTests,
::testing::ValuesIn(auto_batch_properties)),
OVClassCompiledModelPropertiesTests::getTestCaseName);

} // namespace
INSTANTIATE_TEST_SUITE_P(nightly_OVClassCompiledModelGetPropertyTest,
OVClassCompiledModelGetPropertyTest,
::testing::Values("BATCH:GPU"));

INSTANTIATE_TEST_SUITE_P(nightly_OVClassCompiledModelGetIncorrectPropertyTest,
OVClassCompiledModelGetIncorrectPropertyTest,
::testing::Values("BATCH:GPU"));

const std::vector<ov::AnyMap> batchCorrectConfigs = {{}};

INSTANTIATE_TEST_SUITE_P(nightly_Auto_Batch_OVClassCompileModelWithCorrectPropertiesAutoBatchingTest,
OVClassCompileModelWithCorrectPropertiesTest,
::testing::Combine(::testing::Values("BATCH:GPU"), ::testing::ValuesIn(batchCorrectConfigs)));

const std::vector<std::tuple<std::string, std::pair<ov::AnyMap, std::string>>> GetMetricTest_ExecutionDevice_GPU = {
{"BATCH:GPU", std::make_pair(ov::AnyMap{}, "GPU.0")}};

INSTANTIATE_TEST_SUITE_P(nightly_OVClassCompiledModelGetPropertyTest,
OVClassCompiledModelGetPropertyTest_EXEC_DEVICES,
::testing::ValuesIn(GetMetricTest_ExecutionDevice_GPU));

INSTANTIATE_TEST_SUITE_P(nightly_HeteroAutoBatchOVGetMetricPropsTest, OVGetMetricPropsTest, ::testing::Values("BATCH"));
} // namespace behavior
} // namespace test
} // namespace ov
2 changes: 1 addition & 1 deletion src/plugins/hetero/tests/functional/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ ov_add_test_target(
$<TARGET_PROPERTY:openvino_hetero_plugin,SOURCE_DIR>/src
ADD_CLANG_FORMAT
LABELS
OV UNIT HETERO
OV HETERO
)

target_compile_definitions(${TARGET_NAME} PRIVATE CI_BUILD_NUMBER=\"mock_version\")
Expand Down
27 changes: 14 additions & 13 deletions src/plugins/hetero/tests/functional/compile_model_tests.cpp
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Copyright (C) 2018-2024 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "common_test_utils/test_constants.hpp"
#include "hetero_tests.hpp"
#include "openvino/runtime/exec_model_info.hpp"
#include "openvino/runtime/internal_properties.hpp"
Expand All @@ -23,45 +24,45 @@ TEST_F(HeteroTests, get_available_devices) {

TEST_F(HeteroTests, compile_with_registered_devices) {
// Change device priority
core.set_property("HETERO", ov::device::priorities("MOCK0,MOCK1"));
core.set_property(ov::test::utils::DEVICE_HETERO, ov::device::priorities("MOCK0,MOCK1"));
auto model = create_model_with_reshape();
EXPECT_NO_THROW(core.compile_model(model, "HETERO"));
EXPECT_NO_THROW(core.compile_model(model, ov::test::utils::DEVICE_HETERO));
}

TEST_F(HeteroTests, compile_with_unregistered_devices_throw) {
// Change device priority
core.set_property("HETERO", ov::device::priorities("MOCK2,MOCK3"));
core.set_property(ov::test::utils::DEVICE_HETERO, ov::device::priorities("MOCK2,MOCK3"));
auto model = create_model_with_reshape();
EXPECT_THROW(core.compile_model(model, "HETERO"), ov::Exception);
EXPECT_THROW(core.compile_model(model, ov::test::utils::DEVICE_HETERO), ov::Exception);
}

TEST_F(HeteroTests, compile_without_device_priorities_throw) {
// Change device priority
core.set_property("HETERO", ov::device::priorities(""));
core.set_property(ov::test::utils::DEVICE_HETERO, ov::device::priorities(""));
auto model = create_model_with_reshape();
EXPECT_THROW(core.compile_model(model, "HETERO"), ov::Exception);
EXPECT_THROW(core.compile_model(model, ov::test::utils::DEVICE_HETERO), ov::Exception);
}

TEST_F(HeteroTests, compile_dynamic_model_fail) {
// Change device priority
core.set_property("HETERO", ov::device::priorities("MOCK0,MOCK1"));
core.set_property(ov::test::utils::DEVICE_HETERO, ov::device::priorities("MOCK0,MOCK1"));
auto model = create_model_with_subtract_reshape(true);
EXPECT_THROW(core.compile_model(model, "HETERO"), ov::Exception);
EXPECT_THROW(core.compile_model(model, ov::test::utils::DEVICE_HETERO), ov::Exception);
}

TEST_F(HeteroTests, compile_model_shapeof) {
// Change device priority
core.set_property("HETERO", ov::device::priorities("MOCK0,MOCK1"));
core.set_property(ov::test::utils::DEVICE_HETERO, ov::device::priorities("MOCK0,MOCK1"));
auto model = create_model_with_subtract_shapeof_reshape();
EXPECT_NO_THROW(core.compile_model(model, "HETERO"));
EXPECT_NO_THROW(core.compile_model(model, ov::test::utils::DEVICE_HETERO));
}

TEST_F(HeteroTests, compile_with_device_properties) {
ov::AnyMap config = {ov::device::priorities("MOCK0,MOCK1"),
ov::device::properties("MOCK0", ov::num_streams(4), ov::enable_profiling(false)),
ov::device::properties("MOCK1", ov::num_streams(6), ov::enable_profiling(true))};
auto model = create_model_with_subtract_reshape();
auto compiled_model = core.compile_model(model, "HETERO", config);
auto compiled_model = core.compile_model(model, ov::test::utils::DEVICE_HETERO, config);
EXPECT_THROW(compiled_model.get_property(ov::num_streams), ov::Exception);
EXPECT_THROW(compiled_model.get_property(ov::enable_profiling), ov::Exception);
auto device_properties = compiled_model.get_property(ov::device::properties.name()).as<ov::AnyMap>();
Expand All @@ -85,7 +86,7 @@ TEST_F(HeteroTests, compile_with_device_properties_no_exclusive) {
ov::device::properties("MOCK0", ov::num_streams(4)),
ov::device::properties("MOCK1", ov::num_streams(6))};
auto model = create_model_with_subtract_reshape();
auto compiled_model = core.compile_model(model, "HETERO", config);
auto compiled_model = core.compile_model(model, ov::test::utils::DEVICE_HETERO, config);
EXPECT_THROW(compiled_model.get_property(ov::num_streams), ov::Exception);
auto device_properties = compiled_model.get_property(ov::device::properties.name()).as<ov::AnyMap>();
ASSERT_TRUE(device_properties.count("MOCK0.0"));
Expand All @@ -105,7 +106,7 @@ TEST_F(HeteroTests, get_runtime_model) {
for (auto& op : model->get_ordered_ops()) {
original_names.insert(op->get_friendly_name());
}
auto compiled_model = core.compile_model(model, "HETERO", config);
auto compiled_model = core.compile_model(model, ov::test::utils::DEVICE_HETERO, config);
auto runtime_model = compiled_model.get_runtime_model();
for (auto& op : runtime_model->get_ordered_ops()) {
auto& info = op->get_rt_info();
Expand Down
18 changes: 13 additions & 5 deletions src/plugins/hetero/tests/functional/import_model_tests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,24 @@
// SPDX-License-Identifier: Apache-2.0
//

#include "common_test_utils/test_constants.hpp"
#include "hetero_tests.hpp"

using namespace ov::hetero::tests;
namespace ov {
namespace hetero {
namespace tests {

// IR frontend is needed for import
#ifdef IR_FRONTEND_ENABLED
TEST_F(HeteroTests, import_single_plugins) {
std::stringstream model_stream;
auto model = create_model_with_reshape();
{
auto compiled_model = core.compile_model(model, "HETERO", ov::device::priorities("MOCK0"));
auto compiled_model =
core.compile_model(model, ov::test::utils::DEVICE_HETERO, ov::device::priorities("MOCK0"));
compiled_model.export_model(model_stream);
}
auto compiled_model = core.import_model(model_stream, "HETERO", {});
auto compiled_model = core.import_model(model_stream, ov::test::utils::DEVICE_HETERO, {});
EXPECT_EQ(1, compiled_model.inputs().size());
EXPECT_EQ(1, compiled_model.outputs().size());
auto infer_request = compiled_model.create_infer_request();
Expand All @@ -32,10 +36,11 @@ TEST_F(HeteroTests, import_several_plugins) {
std::stringstream model_stream;
auto model = create_model_with_subtract();
{
auto compiled_model = core.compile_model(model, "HETERO", ov::device::priorities("MOCK0,MOCK1"));
auto compiled_model =
core.compile_model(model, ov::test::utils::DEVICE_HETERO, ov::device::priorities("MOCK0,MOCK1"));
compiled_model.export_model(model_stream);
}
auto compiled_model = core.import_model(model_stream, "HETERO", {});
auto compiled_model = core.import_model(model_stream, ov::test::utils::DEVICE_HETERO, {});
EXPECT_EQ(1, compiled_model.inputs().size());
EXPECT_EQ(1, compiled_model.outputs().size());
auto infer_request = compiled_model.create_infer_request();
Expand All @@ -49,3 +54,6 @@ TEST_F(HeteroTests, import_several_plugins) {
EXPECT_EQ(memcmp(input_tensor.data(), output_tensor.data(), input_tensor.get_byte_size()), 0);
}
#endif
} // namespace tests
} // namespace hetero
} // namespace ov
35 changes: 22 additions & 13 deletions src/plugins/hetero/tests/functional/properties_tests.cpp
Original file line number Diff line number Diff line change
@@ -1,19 +1,22 @@
// Copyright (C) 2018-2024 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//
#include "common_test_utils/test_constants.hpp"
#include "hetero_tests.hpp"
#include "openvino/runtime/internal_properties.hpp"
#include "properties.hpp"

using namespace ov::hetero::tests;
namespace ov {
namespace hetero {
namespace tests {

TEST_F(HeteroTests, get_property_supported_properties) {
const std::vector<ov::PropertyName> supported_properties = {ov::supported_properties,
ov::device::full_name,
ov::device::capabilities,
ov::device::priorities,
ov::hint::model_distribution_policy};
auto actual_supported_properties = core.get_property("HETERO", ov::supported_properties);
auto actual_supported_properties = core.get_property(ov::test::utils::DEVICE_HETERO, ov::supported_properties);
EXPECT_EQ(supported_properties.size(), actual_supported_properties.size());
for (auto& supported_property : supported_properties) {
ASSERT_TRUE(std::find(actual_supported_properties.begin(),
Expand All @@ -24,7 +27,8 @@ TEST_F(HeteroTests, get_property_supported_properties) {

TEST_F(HeteroTests, get_property_internal_supported_properties) {
const std::vector<ov::PropertyName> supported_properties = {ov::internal::caching_properties};
auto actual_supported_properties = core.get_property("HETERO", ov::internal::supported_properties);
auto actual_supported_properties =
core.get_property(ov::test::utils::DEVICE_HETERO, ov::internal::supported_properties);
EXPECT_EQ(supported_properties.size(), actual_supported_properties.size());
for (auto& supported_property : supported_properties) {
ASSERT_TRUE(std::find(actual_supported_properties.begin(),
Expand All @@ -34,28 +38,33 @@ TEST_F(HeteroTests, get_property_internal_supported_properties) {
}

TEST_F(HeteroTests, get_property_ro_properties) {
EXPECT_EQ("HETERO", core.get_property("HETERO", ov::device::full_name));
EXPECT_EQ(ov::test::utils::DEVICE_HETERO, core.get_property(ov::test::utils::DEVICE_HETERO, ov::device::full_name));
EXPECT_EQ(std::vector<std::string>{ov::device::capability::EXPORT_IMPORT},
core.get_property("HETERO", ov::device::capabilities));
core.get_property(ov::test::utils::DEVICE_HETERO, ov::device::capabilities));
}

TEST_F(HeteroTests, set_property_device_priorities) {
EXPECT_EQ("", core.get_property("HETERO", ov::device::priorities));
core.set_property("HETERO", ov::device::priorities("MOCK0,MOCK1"));
EXPECT_EQ("MOCK0,MOCK1", core.get_property("HETERO", ov::device::priorities));
EXPECT_EQ("", core.get_property(ov::test::utils::DEVICE_HETERO, ov::device::priorities));
core.set_property(ov::test::utils::DEVICE_HETERO, ov::device::priorities("MOCK0,MOCK1"));
EXPECT_EQ("MOCK0,MOCK1", core.get_property(ov::test::utils::DEVICE_HETERO, ov::device::priorities));
}

TEST_F(HeteroTests, set_property_ModelDistributionPolicy) {
std::set<ov::hint::ModelDistributionPolicy> value = {};
std::set<ov::hint::ModelDistributionPolicy> model_policy = {ov::hint::ModelDistributionPolicy::PIPELINE_PARALLEL};

ASSERT_NO_THROW(core.set_property("HETERO", ov::hint::model_distribution_policy(model_policy)));
ASSERT_NO_THROW(value = core.get_property("HETERO", ov::hint::model_distribution_policy));
ASSERT_NO_THROW(
core.set_property(ov::test::utils::DEVICE_HETERO, ov::hint::model_distribution_policy(model_policy)));
ASSERT_NO_THROW(value = core.get_property(ov::test::utils::DEVICE_HETERO, ov::hint::model_distribution_policy));
ASSERT_EQ(model_policy, value);

model_policy = {};

ASSERT_NO_THROW(core.set_property("HETERO", ov::hint::model_distribution_policy(model_policy)));
ASSERT_NO_THROW(value = core.get_property("HETERO", ov::hint::model_distribution_policy));
ASSERT_NO_THROW(
core.set_property(ov::test::utils::DEVICE_HETERO, ov::hint::model_distribution_policy(model_policy)));
ASSERT_NO_THROW(value = core.get_property(ov::test::utils::DEVICE_HETERO, ov::hint::model_distribution_policy));
ASSERT_EQ(model_policy, value);
}
}
} // namespace tests
} // namespace hetero
} // namespace ov
Loading
Loading