From f8591ffee50c7a2e25a606152e543e4df1ebf06a Mon Sep 17 00:00:00 2001 From: Ashwin Bhat Date: Thu, 10 Nov 2022 18:08:45 -0800 Subject: [PATCH 01/14] Upgrade to Catch v2.13.10 Update to Catch v2.13.10 to use BENCHMARK tests. --- source/MaterialXTest/CMakeLists.txt | 3 ++ source/MaterialXTest/External/Catch/catch.hpp | 28 +++++++++++-------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/source/MaterialXTest/CMakeLists.txt b/source/MaterialXTest/CMakeLists.txt index e66db4bd2e..4351e91e6e 100644 --- a/source/MaterialXTest/CMakeLists.txt +++ b/source/MaterialXTest/CMakeLists.txt @@ -121,6 +121,9 @@ set_target_properties( VERSION "${MATERIALX_LIBRARY_VERSION}" SOVERSION "${MATERIALX_MAJOR_VERSION}") +# enable Catch Benchmarking +target_compile_definitions(MaterialXTest PRIVATE -DCATCH_CONFIG_ENABLE_BENCHMARKING) + target_link_libraries( MaterialXTest ${CMAKE_DL_LIBS}) diff --git a/source/MaterialXTest/External/Catch/catch.hpp b/source/MaterialXTest/External/Catch/catch.hpp index d2a12427b2..9b309bddc6 100644 --- a/source/MaterialXTest/External/Catch/catch.hpp +++ b/source/MaterialXTest/External/Catch/catch.hpp @@ -1,6 +1,6 @@ /* - * Catch v2.13.9 - * Generated: 2022-04-12 22:37:23.260201 + * Catch v2.13.10 + * Generated: 2022-10-16 11:01:23.452308 * ---------------------------------------------------------- * This file has been merged from multiple headers. Please don't edit it directly * Copyright (c) 2022 Two Blue Cubes Ltd. All rights reserved. @@ -15,7 +15,7 @@ #define CATCH_VERSION_MAJOR 2 #define CATCH_VERSION_MINOR 13 -#define CATCH_VERSION_PATCH 9 +#define CATCH_VERSION_PATCH 10 #ifdef __clang__ # pragma clang system_header @@ -7395,8 +7395,6 @@ namespace Catch { template struct ObjectStorage { - using TStorage = typename std::aligned_storage::value>::type; - ObjectStorage() : data() {} ObjectStorage(const ObjectStorage& other) @@ -7439,7 +7437,7 @@ namespace Catch { return *static_cast(static_cast(&data)); } - TStorage data; + struct { alignas(T) unsigned char data[sizeof(T)]; } data; }; } @@ -7949,7 +7947,7 @@ namespace Catch { #if defined(__i386__) || defined(__x86_64__) #define CATCH_TRAP() __asm__("int $3\n" : : ) /* NOLINT */ #elif defined(__aarch64__) - #define CATCH_TRAP() __asm__(".inst 0xd4200000") + #define CATCH_TRAP() __asm__(".inst 0xd43e0000") #endif #elif defined(CATCH_PLATFORM_IPHONE) @@ -13558,7 +13556,7 @@ namespace Catch { // Handle list request if( Option listed = list( m_config ) ) - return static_cast( *listed ); + return (std::min) (MaxExitCode, static_cast(*listed)); TestGroup tests { m_config }; auto const totals = tests.execute(); @@ -15391,7 +15389,7 @@ namespace Catch { } Version const& libraryVersion() { - static Version version( 2, 13, 9, "", 0 ); + static Version version( 2, 13, 10, "", 0 ); return version; } @@ -17526,12 +17524,20 @@ namespace Catch { #ifndef __OBJC__ +#ifndef CATCH_INTERNAL_CDECL +#ifdef _MSC_VER +#define CATCH_INTERNAL_CDECL __cdecl +#else +#define CATCH_INTERNAL_CDECL +#endif +#endif + #if defined(CATCH_CONFIG_WCHAR) && defined(CATCH_PLATFORM_WINDOWS) && defined(_UNICODE) && !defined(DO_NOT_USE_WMAIN) // Standard C/C++ Win32 Unicode wmain entry point -extern "C" int wmain (int argc, wchar_t * argv[], wchar_t * []) { +extern "C" int CATCH_INTERNAL_CDECL wmain (int argc, wchar_t * argv[], wchar_t * []) { #else // Standard C/C++ main entry point -int main (int argc, char * argv[]) { +int CATCH_INTERNAL_CDECL main (int argc, char * argv[]) { #endif return Catch::Session().run( argc, argv ); From 3ba7f910ea7247c655ee5bd504844b0c57a8b456 Mon Sep 17 00:00:00 2001 From: Ashwin Bhat Date: Thu, 12 Oct 2023 15:40:14 -0700 Subject: [PATCH 02/14] New performance measurement test Add a new "GenShader: ShaderGen Performance" test to measure time take to load, validate and generate shaders --- .../MaterialXGenGlsl/GenGlsl.cpp | 9 +++ .../MaterialXGenShader/GenShaderUtil.cpp | 79 +++++++++++++++++++ .../MaterialXGenShader/GenShaderUtil.h | 3 + 3 files changed, 91 insertions(+) diff --git a/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp b/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp index 339d3853cb..305859512c 100644 --- a/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp +++ b/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp @@ -119,6 +119,15 @@ TEST_CASE("GenShader: Bind Light Shaders", "[genglsl]") REQUIRE_NOTHROW(mx::HwShaderGenerator::bindLightShader(*spotLightShader, 66, context)); } +TEST_CASE("GenShader: ShaderGen Performance", "[genglsl]") +{ + mx::GenContext context(mx::GlslShaderGenerator::create()); + BENCHMARK("Load documents, validate and generate shader") + { + return GenShaderUtil::shaderGenPerformanceTest(context); + }; +} + enum class GlslType { Glsl400, diff --git a/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp b/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp index 261aacdefe..2ca8afa86c 100644 --- a/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp +++ b/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp @@ -314,6 +314,85 @@ void testUniqueNames(mx::GenContext& context, const std::string& stage) REQUIRE(sgNode1->getOutput()->getVariable() == "unique_names_out"); } +// Test ShaderGen performance +void shaderGenPerformanceTest(mx::GenContext& context) +{ + mx::DocumentPtr nodeLibrary = mx::createDocument(); + mx::FilePath currentPath = mx::FilePath::getCurrentPath(); + const mx::FileSearchPath libSearchPath(currentPath); + + // Load the standard libraries. + loadLibraries({ "libraries" }, libSearchPath, nodeLibrary); + context.registerSourceCodeSearchPath(libSearchPath); + + // Enable Color Management + mx::ColorManagementSystemPtr colorManagementSystem = + mx::DefaultColorManagementSystem::create(context.getShaderGenerator().getTarget()); + + REQUIRE(colorManagementSystem); + if (colorManagementSystem) + { + context.getShaderGenerator().setColorManagementSystem(colorManagementSystem); + colorManagementSystem->loadLibrary(nodeLibrary); + } + + // Enable Unit System + mx::UnitSystemPtr unitSystem = mx::UnitSystem::create(context.getShaderGenerator().getTarget()); + REQUIRE(unitSystem); + if (unitSystem) + { + context.getShaderGenerator().setUnitSystem(unitSystem); + unitSystem->loadLibrary(nodeLibrary); + // Setup Unit converters + unitSystem->setUnitConverterRegistry(mx::UnitConverterRegistry::create()); + mx::UnitTypeDefPtr distanceTypeDef = nodeLibrary->getUnitTypeDef("distance"); + unitSystem->getUnitConverterRegistry()->addUnitConverter(distanceTypeDef, mx::LinearUnitConverter::create(distanceTypeDef)); + mx::UnitTypeDefPtr angleTypeDef = nodeLibrary->getUnitTypeDef("angle"); + unitSystem->getUnitConverterRegistry()->addUnitConverter(angleTypeDef, mx::LinearUnitConverter::create(angleTypeDef)); + context.getOptions().targetDistanceUnit = "meter"; + } + + // Read mtlx documents + mx::FilePathVec testRootPaths; + testRootPaths.push_back("resources/Materials/Examples/StandardSurface"); + + std::vector loadedDocuments; + mx::StringVec documentsPaths; + mx::StringVec errorLog; + + for (const auto& testRoot : testRootPaths) + { + mx::loadDocuments(testRoot, libSearchPath, {}, {}, loadedDocuments, documentsPaths, + nullptr, &errorLog); + } + + REQUIRE(loadedDocuments.size() > 0); + REQUIRE(loadedDocuments.size() == documentsPaths.size()); + + // Shuffle the order of documents and perform document library import validatation and shadergen + std::random_device random_dev; + std::mt19937 generator(random_dev()); + std::shuffle(loadedDocuments.begin(), loadedDocuments.end(), generator); + for (const auto& doc : loadedDocuments) + { + doc->importLibrary(nodeLibrary); + std::vector elements = mx::findRenderableElements(doc); + + REQUIRE(elements.size() > 0); + + std::string message; + bool docValid = doc->validate(&message); + + REQUIRE(docValid == true); + + mx::StringVec sourceCode; + mx::ShaderPtr shader = nullptr; + shader = context.getShaderGenerator().generate(elements[0]->getName(), elements[0], context); + + REQUIRE(shader != nullptr); + REQUIRE(shader->getSourceCode(mx::Stage::PIXEL).length() > 0); + } +} void ShaderGeneratorTester::checkImplementationUsage(const mx::StringSet& usedImpls, const mx::GenContext& context, diff --git a/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.h b/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.h index ea34170d71..9c471466aa 100644 --- a/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.h +++ b/source/MaterialXTest/MaterialXGenShader/GenShaderUtil.h @@ -52,6 +52,9 @@ void checkImplementations(mx::GenContext& context, // Utility test to check unique name generation on a shader generator void testUniqueNames(mx::GenContext& context, const std::string& stage); +// Utility to perfrom simple performance test to load, validate and generate shaders +void shaderGenPerformanceTest(mx::GenContext& context); + // // Render validation options. Reflects the _options.mtlx // file in the test suite area. From 0089b607b10eda3288293eac240520a3683edac8 Mon Sep 17 00:00:00 2001 From: Jonathan Stone Date: Fri, 13 Oct 2023 13:52:00 -0700 Subject: [PATCH 03/14] Minor formatting fix --- source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp b/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp index 305859512c..927d23bd1c 100644 --- a/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp +++ b/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp @@ -124,7 +124,7 @@ TEST_CASE("GenShader: ShaderGen Performance", "[genglsl]") mx::GenContext context(mx::GlslShaderGenerator::create()); BENCHMARK("Load documents, validate and generate shader") { - return GenShaderUtil::shaderGenPerformanceTest(context); + return GenShaderUtil::shaderGenPerformanceTest(context); }; } From 83535fd026a436aa59694a76db3bb63661baf2ff Mon Sep 17 00:00:00 2001 From: Ashwin Bhat Date: Mon, 16 Oct 2023 13:18:42 -0700 Subject: [PATCH 04/14] Add toggle for Performance Test Add the Perfromance test only if MATERIALX_DYNAMIC_ANALYSIS is enabled. Note that test implmentation is always compiled to catch any code/api changes to keep it updated, it is disabled from the default run. Example output: 100% tests passed, 0 tests failed out of 72 Total Test time (real) = 121.03 sec The following tests did not run: 51 - MaterialXTest_PerformanceTest_ShaderGen_and_Validation (Disabled) --- source/MaterialXTest/CMakeLists.txt | 4 ++++ source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/source/MaterialXTest/CMakeLists.txt b/source/MaterialXTest/CMakeLists.txt index 4351e91e6e..2646e5fb14 100644 --- a/source/MaterialXTest/CMakeLists.txt +++ b/source/MaterialXTest/CMakeLists.txt @@ -16,6 +16,10 @@ function(add_tests _sources) add_test(NAME "MaterialXTest_${test_safe_name}" COMMAND MaterialXTest ${test_name} WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) + # Disable Performance test + if(${test_name} MATCHES "^PerformanceTest" AND NOT MATERIALX_DYNAMIC_ANALYSIS) + set_tests_properties(MaterialXTest_${test_safe_name} PROPERTIES DISABLED TRUE) + endif() if(MATERIALX_BUILD_OIIO AND MSVC) # Add path to OIIO library so it can be found for the test. # On windows we have to escape the semicolons, otherwise only diff --git a/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp b/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp index 927d23bd1c..7488e6c20f 100644 --- a/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp +++ b/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp @@ -119,7 +119,7 @@ TEST_CASE("GenShader: Bind Light Shaders", "[genglsl]") REQUIRE_NOTHROW(mx::HwShaderGenerator::bindLightShader(*spotLightShader, 66, context)); } -TEST_CASE("GenShader: ShaderGen Performance", "[genglsl]") +TEST_CASE("PerformanceTest: ShaderGen and Validation", "[performancetest]") { mx::GenContext context(mx::GlslShaderGenerator::create()); BENCHMARK("Load documents, validate and generate shader") From 312f3a31a38ef80e52bab24a97225c22ff3bba3a Mon Sep 17 00:00:00 2001 From: Ashwin Bhat Date: Mon, 16 Oct 2023 22:47:53 -0700 Subject: [PATCH 05/14] Enable benchmarks conditionally Enable benchmarks only when MATERIALX_DYNAMIC_ANALYSIS is defined. MATERIALX_DYNAMIC_ANALYSIS = OFF: 51: Test command: C:\source\MaterialX-adsk-fork-ext\build\bin\Release\MaterialXTest.exe "PerformanceTest: ShaderGen and Validation" 51: Working Directory: C:/source/MaterialX-adsk-fork-ext/build/bin 51: Test timeout computed to be: 10000000 51: Filters: PerformanceTest: ShaderGen and Validation 51: 0.000 s: PerformanceTest: ShaderGen and Validation 51: =============================================================================== 51: All tests passed (1 assertion in 1 test case) With MATERIALX_DYNAMIC_ANALYSIS = ON: test 51 Start 51: MaterialXTest_PerformanceTest_ShaderGen_and_Validation 51: Test command: C:\source\MaterialX-adsk-fork-ext\build\bin\Release\MaterialXTest.exe "PerformanceTest: ShaderGen and Validation" 51: Working Directory: C:/source/MaterialX-adsk-fork-ext/build/bin 51: Test timeout computed to be: 10000000 51: Filters: PerformanceTest: ShaderGen and Validation 51: 51: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 51: MaterialXTest.exe is a Catch v2.13.10 host application. 51: Run with -? for options 51: 51: ------------------------------------------------------------------------------- 51: PerformanceTest: ShaderGen and Validation 51: ------------------------------------------------------------------------------- 51: C:\source\MaterialX-adsk-fork-ext\source\MaterialXTest\MaterialXGenGlsl\GenGlsl.cpp(123) 51: ............................................................................... 51: 51: benchmark name samples iterations estimated 51: mean low mean high mean 51: std dev low std dev high std dev 51: ------------------------------------------------------------------------------- 51: Load documents, validate and 51: generate shader 100 1 2.34307 m 51: 1.38572 s 1.38087 s 1.39103 s 51: 25.8656 ms 23.34 ms 28.815 ms 51: 51: 51: 142.748 s: PerformanceTest: ShaderGen and Validation 51: =============================================================================== 51: All tests passed (8888 assertions in 1 test case) 51: 51/73 Test #51: MaterialXTest_PerformanceTest_ShaderGen_and_Validation ....... Passed 142.94 sec --- CMakeLists.txt | 5 +++++ source/MaterialXTest/CMakeLists.txt | 4 ---- source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp | 5 +++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index de6aa7442f..4de78fdda2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -186,6 +186,11 @@ if (MATERIALX_BUILD_GEN_MDL) add_definitions(-DMATERIALX_INSTALL_MDL_MODULE_PATH=\"${MATERIALX_INSTALL_MDL_MODULE_PATH}\") endif() +# Enable additional benchmark tests +if (MATERIALX_DYNAMIC_ANALYSIS) + add_definitions(-DMATERIALX_DYNAMIC_ANALYSIS) +endif() + # Adjust the default installation path if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set(CMAKE_INSTALL_PREFIX "${CMAKE_BINARY_DIR}/installed" CACHE PATH "Default install path" FORCE) diff --git a/source/MaterialXTest/CMakeLists.txt b/source/MaterialXTest/CMakeLists.txt index 2646e5fb14..4351e91e6e 100644 --- a/source/MaterialXTest/CMakeLists.txt +++ b/source/MaterialXTest/CMakeLists.txt @@ -16,10 +16,6 @@ function(add_tests _sources) add_test(NAME "MaterialXTest_${test_safe_name}" COMMAND MaterialXTest ${test_name} WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) - # Disable Performance test - if(${test_name} MATCHES "^PerformanceTest" AND NOT MATERIALX_DYNAMIC_ANALYSIS) - set_tests_properties(MaterialXTest_${test_safe_name} PROPERTIES DISABLED TRUE) - endif() if(MATERIALX_BUILD_OIIO AND MSVC) # Add path to OIIO library so it can be found for the test. # On windows we have to escape the semicolons, otherwise only diff --git a/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp b/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp index 7488e6c20f..96abecfa86 100644 --- a/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp +++ b/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp @@ -119,13 +119,18 @@ TEST_CASE("GenShader: Bind Light Shaders", "[genglsl]") REQUIRE_NOTHROW(mx::HwShaderGenerator::bindLightShader(*spotLightShader, 66, context)); } + TEST_CASE("PerformanceTest: ShaderGen and Validation", "[performancetest]") { +#ifdef MATERIALX_DYNAMIC_ANALYSIS mx::GenContext context(mx::GlslShaderGenerator::create()); BENCHMARK("Load documents, validate and generate shader") { return GenShaderUtil::shaderGenPerformanceTest(context); }; +#else + SUCCEED("PerformanceTest - skipped"); +#endif // MATERIALX_DYNAMIC_ANALYSIS } enum class GlslType From 96a57b43ff19e1ff0c5c653803de9f968461e80f Mon Sep 17 00:00:00 2001 From: Ashwin Bhat Date: Tue, 17 Oct 2023 21:28:47 -0700 Subject: [PATCH 06/14] Disable PerformanceTest: ShaderGen and Validation for Analysis builds --- source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp b/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp index 96abecfa86..0663ad0ae2 100644 --- a/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp +++ b/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp @@ -123,13 +123,13 @@ TEST_CASE("GenShader: Bind Light Shaders", "[genglsl]") TEST_CASE("PerformanceTest: ShaderGen and Validation", "[performancetest]") { #ifdef MATERIALX_DYNAMIC_ANALYSIS + SUCCEED("PerformanceTest - skipped"); +#else mx::GenContext context(mx::GlslShaderGenerator::create()); BENCHMARK("Load documents, validate and generate shader") { return GenShaderUtil::shaderGenPerformanceTest(context); }; -#else - SUCCEED("PerformanceTest - skipped"); #endif // MATERIALX_DYNAMIC_ANALYSIS } From 2b4a2dcb9e5d03e7de0d5e96f1b3d8affe27f707 Mon Sep 17 00:00:00 2001 From: Jonathan Stone Date: Wed, 18 Oct 2023 08:50:19 -0700 Subject: [PATCH 07/14] Minor refactor --- CMakeLists.txt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4de78fdda2..833d5f0ab5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -177,6 +177,9 @@ endif() if(MATERIALX_TEST_RENDER) add_definitions(-DMATERIALX_TEST_RENDER) endif() +if (MATERIALX_DYNAMIC_ANALYSIS) + add_definitions(-DMATERIALX_DYNAMIC_ANALYSIS) +endif() if (MATERIALX_BUILD_GEN_MDL) add_definitions(-DMATERIALX_MDLC_EXECUTABLE=\"${MATERIALX_MDLC_EXECUTABLE}\") @@ -186,11 +189,6 @@ if (MATERIALX_BUILD_GEN_MDL) add_definitions(-DMATERIALX_INSTALL_MDL_MODULE_PATH=\"${MATERIALX_INSTALL_MDL_MODULE_PATH}\") endif() -# Enable additional benchmark tests -if (MATERIALX_DYNAMIC_ANALYSIS) - add_definitions(-DMATERIALX_DYNAMIC_ANALYSIS) -endif() - # Adjust the default installation path if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set(CMAKE_INSTALL_PREFIX "${CMAKE_BINARY_DIR}/installed" CACHE PATH "Default install path" FORCE) From 327cbcda16b8d0cb9c77669804cf32e6935ba501 Mon Sep 17 00:00:00 2001 From: Jonathan Stone Date: Wed, 18 Oct 2023 09:39:56 -0700 Subject: [PATCH 08/14] Update letter case --- source/MaterialXTest/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/MaterialXTest/CMakeLists.txt b/source/MaterialXTest/CMakeLists.txt index 4351e91e6e..1ca7241fc0 100644 --- a/source/MaterialXTest/CMakeLists.txt +++ b/source/MaterialXTest/CMakeLists.txt @@ -121,7 +121,7 @@ set_target_properties( VERSION "${MATERIALX_LIBRARY_VERSION}" SOVERSION "${MATERIALX_MAJOR_VERSION}") -# enable Catch Benchmarking +# Enable Catch benchmarking target_compile_definitions(MaterialXTest PRIVATE -DCATCH_CONFIG_ENABLE_BENCHMARKING) target_link_libraries( From 78171c47a7b5b59528e94bbaf5749691d881c43a Mon Sep 17 00:00:00 2001 From: Jonathan Stone Date: Wed, 18 Oct 2023 10:30:19 -0700 Subject: [PATCH 09/14] Minor refactor --- source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp b/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp index 0663ad0ae2..b8b8f0190a 100644 --- a/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp +++ b/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp @@ -119,18 +119,17 @@ TEST_CASE("GenShader: Bind Light Shaders", "[genglsl]") REQUIRE_NOTHROW(mx::HwShaderGenerator::bindLightShader(*spotLightShader, 66, context)); } - -TEST_CASE("PerformanceTest: ShaderGen and Validation", "[performancetest]") +TEST_CASE("GenShader: Performance Test, "[genglsl]") { #ifdef MATERIALX_DYNAMIC_ANALYSIS - SUCCEED("PerformanceTest - skipped"); + SUCCEED("Skipping performance test in dynamic analysis build"); #else mx::GenContext context(mx::GlslShaderGenerator::create()); BENCHMARK("Load documents, validate and generate shader") { return GenShaderUtil::shaderGenPerformanceTest(context); }; -#endif // MATERIALX_DYNAMIC_ANALYSIS +#endif } enum class GlslType From 26c5dfe9dc5a4b0b044fcea71b1fb6105d73aa20 Mon Sep 17 00:00:00 2001 From: Jonathan Stone Date: Wed, 18 Oct 2023 11:06:31 -0700 Subject: [PATCH 10/14] Fix typo --- source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp b/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp index b8b8f0190a..ac5699e072 100644 --- a/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp +++ b/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp @@ -119,7 +119,7 @@ TEST_CASE("GenShader: Bind Light Shaders", "[genglsl]") REQUIRE_NOTHROW(mx::HwShaderGenerator::bindLightShader(*spotLightShader, 66, context)); } -TEST_CASE("GenShader: Performance Test, "[genglsl]") +TEST_CASE("GenShader: Performance Test", "[genglsl]") { #ifdef MATERIALX_DYNAMIC_ANALYSIS SUCCEED("Skipping performance test in dynamic analysis build"); From d801b8a1af8344d6f01145e4f686319fbc6188bf Mon Sep 17 00:00:00 2001 From: Ashwin Bhat Date: Wed, 18 Oct 2023 22:00:59 -0700 Subject: [PATCH 11/14] Introduce MaterialX Benchmarks Adds a new benchmarking macro MATERIALX_BUILD_BENCHMARK_TESTS to enable GenShader Performance Tests using Catch2 Benchmarking system. --- CMakeLists.txt | 5 +++++ source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 833d5f0ab5..5fe109fdbc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,6 +45,7 @@ option(MATERIALX_BUILD_GEN_MSL "Build the MSL shader generator back-end." ON) option(MATERIALX_BUILD_RENDER "Build the MaterialX Render modules." ON) option(MATERIALX_BUILD_OIIO "Build OpenImageIO support for MaterialXRender." OFF) option(MATERIALX_BUILD_TESTS "Build unit tests." ON) +option(MATERIALX_BUILD_BENCHMARK_TESTS "Build benchmark tests." OFF) option(MATERIALX_BUILD_SHARED_LIBS "Build MaterialX libraries as shared rather than static." OFF) option(MATERIALX_PYTHON_LTO "Enable link-time optimizations for MaterialX Python." ON) @@ -131,6 +132,7 @@ mark_as_advanced(MATERIALX_BUILD_GEN_MSL) mark_as_advanced(MATERIALX_BUILD_RENDER) mark_as_advanced(MATERIALX_BUILD_OIIO) mark_as_advanced(MATERIALX_BUILD_TESTS) +mark_as_advanced(MATERIALX_BUILD_BENCHMARK_TESTS) mark_as_advanced(MATERIALX_BUILD_SHARED_LIBS) mark_as_advanced(MATERIALX_NAMESPACE_SUFFIX) mark_as_advanced(MATERIALX_LIBNAME_SUFFIX) @@ -180,6 +182,9 @@ endif() if (MATERIALX_DYNAMIC_ANALYSIS) add_definitions(-DMATERIALX_DYNAMIC_ANALYSIS) endif() +if (MATERIALX_BUILD_BENCHMARK_TESTS) + add_definitions(-DMATERIALX_BUILD_BENCHMARK_TESTS) +endif() if (MATERIALX_BUILD_GEN_MDL) add_definitions(-DMATERIALX_MDLC_EXECUTABLE=\"${MATERIALX_MDLC_EXECUTABLE}\") diff --git a/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp b/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp index ac5699e072..0e1fd00c6a 100644 --- a/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp +++ b/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp @@ -121,14 +121,14 @@ TEST_CASE("GenShader: Bind Light Shaders", "[genglsl]") TEST_CASE("GenShader: Performance Test", "[genglsl]") { -#ifdef MATERIALX_DYNAMIC_ANALYSIS - SUCCEED("Skipping performance test in dynamic analysis build"); -#else +#ifdef MATERIALX_BUILD_BENCHMARK_TESTS mx::GenContext context(mx::GlslShaderGenerator::create()); BENCHMARK("Load documents, validate and generate shader") { return GenShaderUtil::shaderGenPerformanceTest(context); }; +#else + SUCCEED("Performance tests are disabled"); #endif } From ea91fc15536d3e9a28f60644f0a81966156e05ca Mon Sep 17 00:00:00 2001 From: Ashwin Bhat Date: Fri, 20 Oct 2023 09:37:45 -0700 Subject: [PATCH 12/14] Minor configuration updates. --- CMakeLists.txt | 3 --- source/MaterialXTest/CMakeLists.txt | 4 +++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5fe109fdbc..0d6a64d6b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -179,9 +179,6 @@ endif() if(MATERIALX_TEST_RENDER) add_definitions(-DMATERIALX_TEST_RENDER) endif() -if (MATERIALX_DYNAMIC_ANALYSIS) - add_definitions(-DMATERIALX_DYNAMIC_ANALYSIS) -endif() if (MATERIALX_BUILD_BENCHMARK_TESTS) add_definitions(-DMATERIALX_BUILD_BENCHMARK_TESTS) endif() diff --git a/source/MaterialXTest/CMakeLists.txt b/source/MaterialXTest/CMakeLists.txt index 1ca7241fc0..9cbf2b8ed3 100644 --- a/source/MaterialXTest/CMakeLists.txt +++ b/source/MaterialXTest/CMakeLists.txt @@ -122,7 +122,9 @@ set_target_properties( SOVERSION "${MATERIALX_MAJOR_VERSION}") # Enable Catch benchmarking -target_compile_definitions(MaterialXTest PRIVATE -DCATCH_CONFIG_ENABLE_BENCHMARKING) +if(MATERIALX_BUILD_BENCHMARK_TESTS) + target_compile_definitions(MaterialXTest PRIVATE -DCATCH_CONFIG_ENABLE_BENCHMARKING) +endif() target_link_libraries( MaterialXTest From a8bb5a03f072a62b53e495984e87170af2034616 Mon Sep 17 00:00:00 2001 From: Jonathan Stone Date: Tue, 24 Oct 2023 09:27:15 -0700 Subject: [PATCH 13/14] Remove extra comment --- source/MaterialXTest/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/source/MaterialXTest/CMakeLists.txt b/source/MaterialXTest/CMakeLists.txt index 9cbf2b8ed3..862a176b50 100644 --- a/source/MaterialXTest/CMakeLists.txt +++ b/source/MaterialXTest/CMakeLists.txt @@ -121,7 +121,6 @@ set_target_properties( VERSION "${MATERIALX_LIBRARY_VERSION}" SOVERSION "${MATERIALX_MAJOR_VERSION}") -# Enable Catch benchmarking if(MATERIALX_BUILD_BENCHMARK_TESTS) target_compile_definitions(MaterialXTest PRIVATE -DCATCH_CONFIG_ENABLE_BENCHMARKING) endif() From e12a3709491a48e02d0bccfac89eafff2952bf5e Mon Sep 17 00:00:00 2001 From: Jonathan Stone Date: Tue, 24 Oct 2023 14:02:57 -0700 Subject: [PATCH 14/14] Slightly simplify preprocessor wrapper --- source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp b/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp index 0e1fd00c6a..2cc4d7ccf4 100644 --- a/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp +++ b/source/MaterialXTest/MaterialXGenGlsl/GenGlsl.cpp @@ -119,18 +119,16 @@ TEST_CASE("GenShader: Bind Light Shaders", "[genglsl]") REQUIRE_NOTHROW(mx::HwShaderGenerator::bindLightShader(*spotLightShader, 66, context)); } +#ifdef MATERIALX_BUILD_BENCHMARK_TESTS TEST_CASE("GenShader: Performance Test", "[genglsl]") { -#ifdef MATERIALX_BUILD_BENCHMARK_TESTS mx::GenContext context(mx::GlslShaderGenerator::create()); BENCHMARK("Load documents, validate and generate shader") { return GenShaderUtil::shaderGenPerformanceTest(context); }; -#else - SUCCEED("Performance tests are disabled"); -#endif } +#endif enum class GlslType {