Use rapids-cmake testing to run tests in parallel (#3697)

By switching to rapids-cmake testing infrastructure we can both run all the tests in parallel from the build environment and the test/install env. Parallel testing performance. - serial baseline: 820sec - serial + improved memory allocation: 798sec - 1 gpu j8: 220sec Authors: - Robert Maynard (https://github.com/robertmaynard) - Matt Joux (https://github.com/MatthiasKohl) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Brad Rees (https://github.com/BradReesWork) - Ray Douglass (https://github.com/raydouglass) URL: #3697
rapidsai · Jul 25, 2023 · e68d1dd · e68d1dd
1 parent 4535d38
commit e68d1dd
Show file tree

Hide file tree

Showing 6 changed files with 118 additions and 67 deletions.
diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh
@@ -41,19 +41,17 @@ EXITCODE=0
 trap "EXITCODE=1" ERR
 set +e
 
+export GTEST_OUTPUT=xml:${RAPIDS_TESTS_DIR}/
+
 # Run libcugraph gtests from libcugraph-tests package
 rapids-logger "Run gtests"
-for gt in "$CONDA_PREFIX"/bin/gtests/libcugraph/* ; do
-    test_name=$(basename ${gt})
-    echo "Running gtest $test_name"
-    ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR}
-done
-
-for ct in "$CONDA_PREFIX"/bin/gtests/libcugraph_c/CAPI_*_TEST ; do
-    test_name=$(basename ${ct})
-    echo "Running C API test $test_name"
-    ${ct}
-done
+cd "$CONDA_PREFIX"/bin/gtests/libcugraph/
+ctest -j10 --output-on-failure
+
+if [ -d "$CONDA_PREFIX"/bin/gtests/libcugraph_c/ ]; then
+  cd "$CONDA_PREFIX"/bin/gtests/libcugraph_c/
+  ctest -j10 --output-on-failure
+fi
 
 rapids-logger "Test script exiting with value: $EXITCODE"
 exit ${EXITCODE}
diff --git a/conda/recipes/libcugraph/install_libcugraph_tests.sh b/conda/recipes/libcugraph/install_libcugraph_tests.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
-# Copyright (c) 2022, NVIDIA CORPORATION.
+# Copyright (c) 2022-2023, NVIDIA CORPORATION.
 
 cmake --install cpp/build --component testing
+cmake --install cpp/build --component testing_c
+cmake --install cpp/build --component testing_mg
 cmake --install cpp/libcugraph_etl/build --component testing
diff --git a/cpp/src/c_api/array.cpp b/cpp/src/c_api/array.cpp
@@ -356,6 +356,7 @@ extern "C" cugraph_error_code_t cugraph_type_erased_device_array_view_copy_to_ho
                       reinterpret_cast<byte_t const*>(internal_pointer->data_),
                       internal_pointer->num_bytes(),
                       p_handle->handle_->get_stream());
+    p_handle->handle_->sync_stream();
 
     return CUGRAPH_SUCCESS;
   } catch (std::exception const& ex) {

diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt
@@ -14,6 +14,10 @@
 # limitations under the License.
 #
 #=============================================================================
+enable_testing()
+
+include(rapids-test)
+rapids_test_init()
 
 ###################################################################################################
 # - common test utils -----------------------------------------------------------------------------
@@ -28,7 +32,7 @@ add_library(cugraphtestutil STATIC
             centrality/betweenness_centrality_validate.cu
             community/egonet_validate.cu
             cores/k_core_validate.cu
-	    structure/induced_subgraph_validate.cu
+            structure/induced_subgraph_validate.cu
             sampling/random_walks_check_sg.cu
             ../../thirdparty/mmio/mmio.c)
 
@@ -59,7 +63,22 @@ target_link_libraries(cugraphtestutil
 # - compiler function -----------------------------------------------------------------------------
 
 function(ConfigureTest CMAKE_TEST_NAME)
-    add_executable(${CMAKE_TEST_NAME} ${ARGN})
+    set(options)
+    set(one_value GPUS PERCENT)
+    set(multi_value)
+    cmake_parse_arguments(_CUGRAPH_TEST "${options}" "${one_value}" "${multi_value}" ${ARGN})
+    if(NOT DEFINED _CUGRAPH_TEST_GPUS AND NOT DEFINED _CUGRAPH_TEST_PERCENT)
+        set(_CUGRAPH_TEST_GPUS 1)
+        set(_CUGRAPH_TEST_PERCENT 25)
+    endif()
+    if(NOT DEFINED _CUGRAPH_TEST_GPUS)
+        set(_CUGRAPH_TEST_GPUS 1)
+    endif()
+    if(NOT DEFINED _CUGRAPH_TEST_PERCENT)
+        set(_CUGRAPH_TEST_PERCENT 100)
+    endif()
+
+    add_executable(${CMAKE_TEST_NAME} ${_CUGRAPH_TEST_UNPARSED_ARGUMENTS})
 
     target_link_libraries(${CMAKE_TEST_NAME}
         PRIVATE
@@ -69,18 +88,18 @@ function(ConfigureTest CMAKE_TEST_NAME)
             GTest::gtest_main
             NCCL::NCCL
     )
-
-    add_test(NAME ${CMAKE_TEST_NAME} COMMAND ${CMAKE_TEST_NAME})
-
     set_target_properties(
         ${CMAKE_TEST_NAME}
             PROPERTIES INSTALL_RPATH "\$ORIGIN/../../../lib")
 
-    install(
-        TARGETS ${CMAKE_TEST_NAME}
-        COMPONENT testing
-        DESTINATION bin/gtests/libcugraph
-        EXCLUDE_FROM_ALL)
+    rapids_test_add(
+        NAME ${CMAKE_TEST_NAME}
+        COMMAND ${CMAKE_TEST_NAME}
+        GPUS ${_CUGRAPH_TEST_GPUS}
+        PERCENT ${_CUGRAPH_TEST_PERCENT}
+        INSTALL_COMPONENT_SET testing
+    )
+    set_tests_properties(${CMAKE_TEST_NAME} PROPERTIES LABELS "CUGRAPH")
 endfunction()
 
 function(ConfigureTestMG CMAKE_TEST_NAME)
@@ -96,29 +115,44 @@ function(ConfigureTestMG CMAKE_TEST_NAME)
         NCCL::NCCL
         MPI::MPI_CXX
     )
+    set_target_properties(
+        ${CMAKE_TEST_NAME}
+            PROPERTIES INSTALL_RPATH "\$ORIGIN/../../../lib")
 
-    add_test(NAME ${CMAKE_TEST_NAME}
-             COMMAND ${MPIEXEC_EXECUTABLE}
+    rapids_test_add(
+        NAME ${CMAKE_TEST_NAME}
+        COMMAND ${MPIEXEC_EXECUTABLE}
              "--noprefix"
              ${MPIEXEC_NUMPROC_FLAG}
              ${GPU_COUNT}
              ${MPIEXEC_PREFLAGS}
              ${CMAKE_TEST_NAME}
-             ${MPIEXEC_POSTFLAGS})
-
-    set_target_properties(
-        ${CMAKE_TEST_NAME}
-            PROPERTIES INSTALL_RPATH "\$ORIGIN/../../../lib")
+             ${MPIEXEC_POSTFLAGS}
+        GPUS ${GPU_COUNT}
+        PERCENT 100
+        INSTALL_COMPONENT_SET testing_mg
+    )
+    set_tests_properties(${CMAKE_TEST_NAME} PROPERTIES LABELS "CUGRAPH_MG")
 
-    install(
-        TARGETS ${CMAKE_TEST_NAME}
-        COMPONENT testing
-        DESTINATION bin/gtests/libcugraph_mg
-        EXCLUDE_FROM_ALL)
 endfunction()
 
 function(ConfigureCTest CMAKE_TEST_NAME)
-    add_executable(${CMAKE_TEST_NAME} ${ARGN})
+    set(options)
+    set(one_value GPUS PERCENT)
+    set(multi_value)
+    cmake_parse_arguments(_CUGRAPH_TEST "${options}" "${one_value}" "${multi_value}" ${ARGN})
+    if(NOT DEFINED _CUGRAPH_TEST_GPUS AND NOT DEFINED _CUGRAPH_TEST_PERCENT)
+        set(_CUGRAPH_TEST_GPUS 1)
+        set(_CUGRAPH_TEST_PERCENT 25)
+    endif()
+    if(NOT DEFINED _CUGRAPH_TEST_GPUS)
+        set(_CUGRAPH_TEST_GPUS 1)
+    endif()
+    if(NOT DEFINED _CUGRAPH_TEST_PERCENT)
+        set(_CUGRAPH_TEST_PERCENT 100)
+    endif()
+
+    add_executable(${CMAKE_TEST_NAME} ${_CUGRAPH_TEST_UNPARSED_ARGUMENTS})
 
     target_link_libraries(${CMAKE_TEST_NAME}
         PRIVATE
@@ -127,18 +161,19 @@ function(ConfigureCTest CMAKE_TEST_NAME)
             GTest::gtest
             GTest::gtest_main
     )
-
-    add_test(NAME ${CMAKE_TEST_NAME} COMMAND ${CMAKE_TEST_NAME})
-
     set_target_properties(
         ${CMAKE_TEST_NAME}
             PROPERTIES INSTALL_RPATH "\$ORIGIN/../../../lib")
 
-    install(
-        TARGETS ${CMAKE_TEST_NAME}
-        COMPONENT testing
-        DESTINATION bin/gtests/libcugraph_c
-        EXCLUDE_FROM_ALL)
+    rapids_test_add(
+        NAME ${CMAKE_TEST_NAME}
+        COMMAND ${CMAKE_TEST_NAME}
+        GPUS ${_CUGRAPH_TEST_GPUS}
+        PERCENT ${_CUGRAPH_TEST_PERCENT}
+        INSTALL_COMPONENT_SET testing_c
+    )
+    set_tests_properties(${CMAKE_TEST_NAME} PROPERTIES LABELS "CUGRAPH_C")
+
 endfunction()
 
 function(ConfigureCTestMG CMAKE_TEST_NAME)
@@ -154,25 +189,26 @@ function(ConfigureCTestMG CMAKE_TEST_NAME)
             NCCL::NCCL
             MPI::MPI_CXX
     )
+    set_target_properties(
+        ${CMAKE_TEST_NAME}
+            PROPERTIES INSTALL_RPATH "\$ORIGIN/../../../lib")
 
-    add_test(NAME ${CMAKE_TEST_NAME}
-             COMMAND ${MPIEXEC_EXECUTABLE}
+    rapids_test_add(
+        NAME ${CMAKE_TEST_NAME}
+        COMMAND ${MPIEXEC_EXECUTABLE}
              "--noprefix"
              ${MPIEXEC_NUMPROC_FLAG}
              ${GPU_COUNT}
              ${MPIEXEC_PREFLAGS}
              ${CMAKE_TEST_NAME}
-             ${MPIEXEC_POSTFLAGS})
+             ${MPIEXEC_POSTFLAGS}
+        GPUS ${GPU_COUNT}
+        PERCENT 100
+        INSTALL_COMPONENT_SET testing_mg
+    )
+    set_tests_properties(${CMAKE_TEST_NAME} PROPERTIES LABELS "CUGRAPH_C_MG")
 
-    set_target_properties(
-        ${CMAKE_TEST_NAME}
-            PROPERTIES INSTALL_RPATH "\$ORIGIN/../../../lib")
 
-    install(
-        TARGETS ${CMAKE_TEST_NAME}
-        COMPONENT testing
-        DESTINATION bin/gtests/libcugraph_c
-        EXCLUDE_FROM_ALL)
 endfunction()
 
 if(NOT USE_CUGRAPH_OPS)
@@ -230,7 +266,7 @@ ConfigureTest(BALANCED_TEST community/balanced_edge_test.cpp)
 
 ###################################################################################################
 # - EGO tests -------------------------------------------------------------------------------------
-ConfigureTest(EGO_TEST community/egonet_test.cpp)
+ConfigureTest(EGO_TEST community/egonet_test.cpp GPUS 1 PERCENT 75)
 
 ###################################################################################################
 # - FORCE ATLAS 2  tests --------------------------------------------------------------------------
@@ -258,7 +294,8 @@ ConfigureTest(GENERATE_RMAT_TEST generators/generate_rmat_test.cpp)
 
 ###################################################################################################
 # - Bipartite R-mat graph generation tests --------------------------------------------------------
-ConfigureTest(GENERATE_BIPARTITE_RMAT_TEST generators/generate_bipartite_rmat_test.cpp)
+ConfigureTest(GENERATE_BIPARTITE_RMAT_TEST generators/generate_bipartite_rmat_test.cpp
+    GPUS 1 PERCENT 100)
 
 ###################################################################################################
 # - Graph mask tests ------------------------------------------------------------------------------
@@ -360,10 +397,7 @@ target_link_libraries(RENUMBER_SAMPLED_EDGELIST_TEST PRIVATE cuco::cuco)
 
 ###################################################################################################
 # - Renumber tests --------------------------------------------------------------------------------
-set(RENUMBERING_TEST_SRCS
-    "${CMAKE_CURRENT_SOURCE_DIR}/structure/renumbering_test.cpp")
-
-ConfigureTest(RENUMBERING_TEST "${RENUMBERING_TEST_SRCS}")
+ConfigureTest(RENUMBERING_TEST structure/renumbering_test.cpp)
 
 ###################################################################################################
 # - Core Number tests -----------------------------------------------------------------------------
@@ -381,6 +415,11 @@ ConfigureTest(TRIANGLE_COUNT_TEST community/triangle_count_test.cpp)
 # - K-hop Neighbors tests -------------------------------------------------------------------------
 ConfigureTest(K_HOP_NBRS_TEST traversal/k_hop_nbrs_test.cpp)
 
+
+###################################################################################################
+# - install tests ---------------------------------------------------------------------------------
+rapids_test_install_relocatable(INSTALL_COMPONENT_SET testing DESTINATION bin/gtests/libcugraph)
+
 ###################################################################################################
 # - MG tests --------------------------------------------------------------------------------------
 
@@ -626,6 +665,8 @@ if(BUILD_CUGRAPH_MG_TESTS)
     ConfigureCTestMG(MG_CAPI_INDUCED_SUBGRAPH_TEST c_api/mg_induced_subgraph_test.c)
     ConfigureCTestMG(MG_CAPI_EGONET_TEST c_api/mg_egonet_test.c)
     ConfigureCTestMG(MG_CAPI_TWO_HOP_NEIGHBORS_TEST c_api/mg_two_hop_neighbors_test.c)
+
+    rapids_test_install_relocatable(INSTALL_COMPONENT_SET testing_mg DESTINATION bin/gtests/libcugraph_mg)
 endif()
 
 ###################################################################################################
@@ -684,8 +725,4 @@ ConfigureCTest(CAPI_INDUCED_SUBGRAPH_TEST c_api/induced_subgraph_test.c)
 ConfigureCTest(CAPI_EGONET_TEST c_api/egonet_test.c)
 ConfigureCTest(CAPI_TWO_HOP_NEIGHBORS_TEST c_api/two_hop_neighbors_test.c)
 
-###################################################################################################
-### enable testing ################################################################################
-###################################################################################################
-
-enable_testing()
+rapids_test_install_relocatable(INSTALL_COMPONENT_SET testing_c DESTINATION bin/gtests/libcugraph_c)
diff --git a/cpp/tests/utilities/base_fixture.hpp b/cpp/tests/utilities/base_fixture.hpp
@@ -69,7 +69,14 @@ inline auto make_managed() { return std::make_shared<rmm::mr::managed_memory_res
 
 inline auto make_pool()
 {
-  return rmm::mr::make_owning_wrapper<rmm::mr::pool_memory_resource>(make_cuda());
+  // Reduce the default pool allocation to 1/6th of the GPU memory so that we can
+  // run more than 2 tests in parallel at the same time. Changes to this value could
+  // effect the maximum amount of parallel tests, and therefore `tests/CMakeLists.txt`
+  // `_CUGRAPH_TEST_PERCENT` default value will need to be audited.
+  auto const [free, total] = rmm::detail::available_device_memory();
+  auto const min_alloc =
+    rmm::detail::align_down(std::min(free, total / 6), rmm::detail::CUDA_ALLOCATION_ALIGNMENT);
+  return rmm::mr::make_owning_wrapper<rmm::mr::pool_memory_resource>(make_cuda(), min_alloc);
 }
 
 inline auto make_binning()

diff --git a/dependencies.yaml b/dependencies.yaml
@@ -35,6 +35,7 @@ files:
     output: none
     includes:
       - cudatoolkit
+      - test_cpp
   test_notebooks:
     output: none
     includes:
@@ -223,7 +224,7 @@ dependencies:
     common:
       - output_types: [conda, pyproject]
         packages:
-          - cmake>=3.26.4
+          - &cmake_ver cmake>=3.26.4
           - ninja
   cpp_build:
     common:
@@ -406,6 +407,11 @@ dependencies:
           - sphinx-markdown-tables
           - sphinx-copybutton
           - pylibcugraphops==23.8.*
+  test_cpp:
+    common:
+      - output_types: conda
+        packages:
+          - *cmake_ver
   test_notebook:
     common:
       - output_types: [conda, requirements]