Merge pull request #262 from lanl/JoshuaSBrown/fix-code-cov-test

Fix code coverage test
parthenon-hpc-lab · Aug 17, 2020 · 37ee9d4 · 37ee9d4
2 parents 5979a09 + 7398a76
commit 37ee9d4
Show file tree

Hide file tree

Showing 18 changed files with 180 additions and 112 deletions.
diff --git a/.gitlab-ci-ias.yml b/.gitlab-ci-ias.yml
@@ -1,6 +1,6 @@
 variables:
   OMPI_VER: '3.1.6'
-  CMAKE_VER: '3.11.4'
+  CMAKE_VER: '3.12.4'
   HDF5_VER: '1.10.6'
   UCX_VER: '1.8.1'
 

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -5,7 +5,7 @@ before_script:
     - export OMP_PROC_BIND=close
     - export OMP_PLACES=cores
     - export OMP_NUM_THREADS=1
-    - export CMAKE_VERSION=3.10.3
+    - export CMAKE_VERSION=3.12.4
     - export CTEST_OUTPUT_ON_FAILURE=1
     - export J=$(( $(nproc --all) / 4  + 1 )) && echo Using ${J} cores during build
     - wget -qO- http://www.cmake.org/files/v${CMAKE_VERSION:0:4}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz | tar -xz

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -11,7 +11,7 @@
 # the public, perform publicly and display publicly, and to permit others to do so.
 #=========================================================================================
 
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.12)
 project(parthenon VERSION 0.1.0 LANGUAGES C CXX)
 
 include(CTest)
@@ -67,6 +67,7 @@ endif()
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
+include(cmake/CodeCov.cmake)
 if(CODE_COVERAGE)
   if( NOT COVERAGE_NAME)
     SET(COVERAGE_NAME "coverage_reports")
@@ -255,7 +256,7 @@ endif()
 add_subdirectory(src)
 add_subdirectory(example)
 
-include(cmake/CodeCov.cmake)
+create_pathenon_coverage_targets()
 include(cmake/CheckCopyright.cmake)
 
 # Currently Ctest/Cmake doesn't ensure that tests are not stale

diff --git a/cmake/CodeCov.cmake b/cmake/CodeCov.cmake
@@ -11,79 +11,130 @@
 # the public, perform publicly and display publicly, and to permit others to do so.
 #=========================================================================================
 
-if(CODE_COVERAGE)
+# Function will add the coverage label to all tests provided
+# The function can be called by invoking
+#
+# list(APPEND all_tests test1 test2)
+#
+# add_coverage_label("${all_tests}")
+#
+# You also have the option of excluding tests with specific labels if desired
+# by passing a second argument
+#
+# list(APPEND all_tests test1 test2)
+# list(APPEND exclude_tests_with_these_labels "performance;CGS")
+#
+# add_coverage_label("${all_tests}" "${exclude_tests_with_these_labels}")
+# 
+# This will only add the coverage label to tests that do not contain the 
+# performance and CGS labels
+function(add_coverage_label tests )
+  if( CODE_COVERAGE )
+    foreach( CHECK_TEST ${tests})
+      set(exclude FALSE)
+      get_test_property(${CHECK_TEST} LABELS TEST_LABELS)
+      foreach( exclude_if_contains_this_label ${ARGN})
+        if( ${exclude_if_contains_this_label} IN_LIST TEST_LABELS)
+          set(exclude TRUE) 
+          continue()
+        endif()
+      endforeach()
+      if(${exclude})
+        continue()
+      endif()
+      set_property(TEST "${CHECK_TEST}" APPEND PROPERTY LABELS "coverage")
+    endforeach()
+  endif()
+endfunction()
+
+# This function creates the code coverage targets for parthenon:
+#
+# coverage - this target will build the coverage reports by running all tests 
+#            with the coverage label.
+#
+# make coverage
+#
+# coverage-upload - this target will upload the reports to code cov for remote viewing
+#
+# make coverage-upload
+#
+# The 'create_parthenon_coverage_targets' should only be called after the parthenon 
+# library has been defined. 
+function(create_pathenon_coverage_targets)
+  if(CODE_COVERAGE)
 
-  find_program( PATH_GCOV gcov )
-  find_program( PATH_LCOV lcov )
+    find_program( PATH_GCOV gcov )
+    find_program( PATH_LCOV lcov )
 
-  if(NOT PATH_GCOV)
-      message(FATAL_ERROR "Unable to build with code coverage gcov was not found.")
-  endif() 
+    if(NOT PATH_GCOV)
+        message(FATAL_ERROR "Unable to build with code coverage gcov was not found.")
+    endif() 
 
-  if(NOT PATH_LCOV)
-      message(FATAL_ERROR "Unable to build with code coverage lcov was not found.")
-  endif() 
+    if(NOT PATH_LCOV)
+        message(FATAL_ERROR "Unable to build with code coverage lcov was not found.")
+    endif() 
 
-  if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
-      message(WARNING "Using code coverage with an optimized build is discouraged, as it may lead to misleading results.")
-  endif() 
+    if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+        message(WARNING "Using code coverage with an optimized build is discouraged, as it may lead to misleading results.")
+    endif() 
 
-  message(STATUS "Coverage reports will be placed in ${COVERAGE_PATH}/${COVERAGE_NAME}")
-
-  get_target_property(PARTHENON_SOURCES parthenon SOURCES)
-  get_target_property(UNIT_TEST_SOURCES unit_tests SOURCES)
+    message(STATUS "Coverage reports will be placed in ${COVERAGE_PATH}/${COVERAGE_NAME}")
+   
+    get_target_property(PARTHENON_SOURCES parthenon SOURCES)
+    get_target_property(UNIT_TEST_SOURCES unit_tests SOURCES)
 
-  add_custom_target(coverage)
-  add_custom_command(TARGET coverage
+    add_custom_target(coverage)
+    add_custom_command(TARGET coverage
 
-    COMMAND echo "====================== Code Coverage ======================"
-    COMMAND mkdir -p ${COVERAGE_PATH}/${COVERAGE_NAME}
-    COMMAND ${PATH_LCOV} --version
-    # Clean
-    COMMAND ${PATH_LCOV} --gcov-tool ${PATH_GCOV} --directory ${PROJECT_BINARY_DIR} -b ${PROJECT_SOURCE_DIR} --zerocounters
-    # Base report
-    COMMAND ctest -L coverage --verbose
-    COMMAND ${PATH_LCOV} --gcov-tool ${PATH_GCOV} -c -i -d ${PROJECT_BINARY_DIR} -b ${PROJECT_SOURCE_DIR} -o ${COVERAGE_PATH}/${COVERAGE_NAME}/report.base.old
-    # Remove Kokkos and tst info from code coverage
-    COMMAND ${PATH_LCOV} --remove ${COVERAGE_PATH}/${COVERAGE_NAME}/report.base.old "*/Kokkos/*" "*/tst/*" -o ${COVERAGE_PATH}/${COVERAGE_NAME}/report.base
+      COMMAND echo "====================== Code Coverage ======================"
+      COMMAND mkdir -p ${COVERAGE_PATH}/${COVERAGE_NAME}
+      COMMAND ${PATH_LCOV} --version
+      # Clean
+      COMMAND ${PATH_LCOV} --gcov-tool ${PATH_GCOV} --directory ${PROJECT_BINARY_DIR} -b ${PROJECT_SOURCE_DIR} --zerocounters
+      # Base report
+      COMMAND ctest -L coverage --verbose
+      COMMAND ${PATH_LCOV} --gcov-tool ${PATH_GCOV} -c -i -d ${PROJECT_BINARY_DIR} -b ${PROJECT_SOURCE_DIR} -o ${COVERAGE_PATH}/${COVERAGE_NAME}/report.base.old
+      # Remove Kokkos and tst info from code coverage
+      COMMAND ${PATH_LCOV} --remove ${COVERAGE_PATH}/${COVERAGE_NAME}/report.base.old "*/Kokkos/*" "*/tst/*" -o ${COVERAGE_PATH}/${COVERAGE_NAME}/report.base
 
-    # Capture information from test runs
-    COMMAND ${PATH_LCOV} --gcov-tool ${PATH_GCOV} --directory ${PROJECT_BINARY_DIR} -b ${PROJECT_SOURCE_DIR} --capture --output-file ${COVERAGE_PATH}/${COVERAGE_NAME}/report.test.old
-    # Remove Kokkos and tst info from code coverage
-    COMMAND ${PATH_LCOV} --remove ${COVERAGE_PATH}/${COVERAGE_NAME}/report.test.old "*/Kokkos/*" "*/tst/*" -o ${COVERAGE_PATH}/${COVERAGE_NAME}/report.test
-
-    # Combining base line counters with counters from running tests
-    COMMAND ${PATH_LCOV} --gcov-tool ${PATH_GCOV} -a ${COVERAGE_PATH}/${COVERAGE_NAME}/report.base -a ${COVERAGE_PATH}/${COVERAGE_NAME}/report.test --output-file ${COVERAGE_PATH}/${COVERAGE_NAME}/report.all
-    # Remove unneeded reports
-    COMMAND rm ${COVERAGE_PATH}/${COVERAGE_NAME}/report.test.old
-    COMMAND rm ${COVERAGE_PATH}/${COVERAGE_NAME}/report.base.old
-    COMMAND rm ${COVERAGE_PATH}/${COVERAGE_NAME}/report.base
-    COMMAND rm ${COVERAGE_PATH}/${COVERAGE_NAME}/report.test
-    WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
-    )
+      # Capture information from test runs
+      COMMAND ${PATH_LCOV} --gcov-tool ${PATH_GCOV} --directory ${PROJECT_BINARY_DIR} -b ${PROJECT_SOURCE_DIR} --capture --output-file ${COVERAGE_PATH}/${COVERAGE_NAME}/report.test.old
+      # Remove Kokkos and tst info from code coverage
+      COMMAND ${PATH_LCOV} --remove ${COVERAGE_PATH}/${COVERAGE_NAME}/report.test.old "*/Kokkos/*" "*/tst/*" -o ${COVERAGE_PATH}/${COVERAGE_NAME}/report.test
+      
+      # Combining base line counters with counters from running tests
+      COMMAND ${PATH_LCOV} --gcov-tool ${PATH_GCOV} -a ${COVERAGE_PATH}/${COVERAGE_NAME}/report.base -a ${COVERAGE_PATH}/${COVERAGE_NAME}/report.test --output-file ${COVERAGE_PATH}/${COVERAGE_NAME}/report.all
+      # Remove unneeded reports
+      COMMAND rm ${COVERAGE_PATH}/${COVERAGE_NAME}/report.test.old
+      COMMAND rm ${COVERAGE_PATH}/${COVERAGE_NAME}/report.base.old
+      COMMAND rm ${COVERAGE_PATH}/${COVERAGE_NAME}/report.base
+      COMMAND rm ${COVERAGE_PATH}/${COVERAGE_NAME}/report.test
+      WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+      )
 
-  set(UPLOAD_COMMAND "bash <\(curl -s https://codecov.io/bash\) \|\| echo \"code coverage failed to upload\"")
-  add_custom_target(coverage-upload) 
-  add_custom_command(TARGET coverage-upload
-    COMMAND echo "================ Uploading Code Coverage =================="
-    # Upload coverage report
-    COMMAND ${PROJECT_SOURCE_DIR}/scripts/combine_coverage.sh ${PATH_LCOV} ${PATH_GCOV} ${COVERAGE_PATH}
-    COMMAND curl -s https://codecov.io/bash > ${COVERAGE_PATH}/CombinedCoverage/script.coverage
-    COMMAND cat ${COVERAGE_PATH}/CombinedCoverage/script.coverage
-    COMMAND cd ${COVERAGE_PATH}/CombinedCoverage && bash ${COVERAGE_PATH}/CombinedCoverage/script.coverage -p ${PROJECT_BINARY_DIR} -s ${COVERAGE_PATH}/CombinedCoverage
-    WORKING_DIRECTORY ${COVERAGE_PATH}
-    )
+    set(UPLOAD_COMMAND "bash <\(curl -s https://codecov.io/bash\) \|\| echo \"code coverage failed to upload\"")
+    add_custom_target(coverage-upload) 
+    add_custom_command(TARGET coverage-upload
+      COMMAND echo "================ Uploading Code Coverage =================="
+      # Upload coverage report
+      COMMAND ${PROJECT_SOURCE_DIR}/scripts/combine_coverage.sh ${PATH_LCOV} ${PATH_GCOV} ${COVERAGE_PATH}
+      COMMAND curl -s https://codecov.io/bash > ${COVERAGE_PATH}/CombinedCoverage/script.coverage
+      COMMAND cat ${COVERAGE_PATH}/CombinedCoverage/script.coverage
+      COMMAND cd ${COVERAGE_PATH}/CombinedCoverage && bash ${COVERAGE_PATH}/CombinedCoverage/script.coverage -p ${PROJECT_BINARY_DIR} -s ${COVERAGE_PATH}/CombinedCoverage
+      WORKING_DIRECTORY ${COVERAGE_PATH}
+      )
 
-  if(ENABLE_UNIT_TESTS)
-    add_dependencies(coverage unit_tests)
-  endif()
+    if(ENABLE_UNIT_TESTS)
+      add_dependencies(coverage unit_tests)
+    endif()
 
-else()
-  add_custom_target(coverage)
-  add_custom_command(TARGET coverage
-    COMMAND echo "====================== Code Coverage ======================"
-    COMMENT "Code coverage has not been enabled"
-    )
-endif()
+  else()
+    add_custom_target(coverage)
+    add_custom_command(TARGET coverage
+      COMMAND echo "====================== Code Coverage ======================"
+      COMMENT "Code coverage has not been enabled"
+      )
+  endif()
+endfunction()
 
 
diff --git a/cmake/Lint.cmake b/cmake/Lint.cmake
@@ -25,12 +25,20 @@ function(lint_file SOURCE_DIR INPUT OUTPUT)
         set(MKDIR_COMMAND COMMAND ${CMAKE_COMMAND} -E make_directory ${OUTPUT_DIR})
     endif()
 
+    if( EXISTS ${INPUT} )
+      set(FILE_TO_LINT ${INPUT} )
+    elseif( EXISTS ${SOURCE_DIR}/${INPUT})
+      set(FILE_TO_LINT ${SOURCE_DIR}/${INPUT})
+    else()
+      message(WARNING "Cannot lint file ${INPUT} does not appear to exist.")
+    endif()
+
     add_custom_command(
         OUTPUT ${OUTPUT}
         COMMAND
             ${PROJECT_SOURCE_DIR}/tst/style/cpplint.py
                 --counting=detailed
-                --quiet ${SOURCE_DIR}/${INPUT}
+                --quiet ${FILE_TO_LINT}
         ${MKDIR_COMMAND}
         COMMAND ${CMAKE_COMMAND} -E touch ${OUTPUT}
         DEPENDS ${INPUT}

diff --git a/cmake/TestSetup.cmake b/cmake/TestSetup.cmake
@@ -42,14 +42,16 @@ endfunction()
 # test output will be sent to /tst/regression/outputs/dir_cov
 # test property labels: regression, mpi-no; coverage
 function(setup_test_coverage dir arg)
-  separate_arguments(arg) 
-  add_test( NAME regression_coverage_test:${dir} COMMAND python "${CMAKE_CURRENT_SOURCE_DIR}/run_test.py" 
-    ${arg} 
-    --coverage
-    --test_dir "${CMAKE_CURRENT_SOURCE_DIR}/test_suites/${dir}"
-    --output_dir "${PROJECT_BINARY_DIR}/tst/regression/outputs/${dir}_cov")
-  set_tests_properties(regression_coverage_test:${dir} PROPERTIES LABELS "regression;coverage;mpi-no" )
-  record_driver("${arg}")
+  if( CODE_COVERAGE )
+    separate_arguments(arg) 
+    add_test( NAME regression_coverage_test:${dir} COMMAND python "${CMAKE_CURRENT_SOURCE_DIR}/run_test.py" 
+      ${arg} 
+      --coverage
+      --test_dir "${CMAKE_CURRENT_SOURCE_DIR}/test_suites/${dir}"
+      --output_dir "${PROJECT_BINARY_DIR}/tst/regression/outputs/${dir}_cov")
+    set_tests_properties(regression_coverage_test:${dir} PROPERTIES LABELS "regression;coverage;mpi-no" )
+    record_driver("${arg}")
+  endif()
 endfunction()
 
 # Adds test that will run in parallel with mpi
@@ -75,19 +77,21 @@ endfunction()
 # test output will be sent to /tst/regression/outputs/dir_mpi_cov
 # test property labels: regression, mpi-yes, coverage
 function(setup_test_mpi_coverage nproc dir arg)
-  if( MPI_FOUND )
-    separate_arguments(arg) 
-    add_test( NAME regression_mpi_coverage_test:${dir} COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/run_test.py
-      --coverage
-      --mpirun ${MPIEXEC_EXECUTABLE} 
-      --mpirun_opts=${MPIEXEC_NUMPROC_FLAG} --mpirun_opts=${nproc}
-      --mpirun_opts=${MPIEXEC_PREFLAGS} ${arg}
-      --test_dir ${CMAKE_CURRENT_SOURCE_DIR}/test_suites/${dir}
-      --output_dir "${PROJECT_BINARY_DIR}/tst/regression/outputs/${dir}_mpi_cov"
-      )
-    set_tests_properties(regression_mpi_coverage_test:${dir} PROPERTIES LABELS "regression;coverage;mpi-yes" RUN_SERIAL ON )
-  else()
-    message(STATUS "MPI not found, not building coverage regression tests with mpi")
+  if( CODE_COVERAGE )
+    if( MPI_FOUND )
+      separate_arguments(arg) 
+      add_test( NAME regression_mpi_coverage_test:${dir} COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/run_test.py
+        --coverage
+        --mpirun ${MPIEXEC_EXECUTABLE} 
+        --mpirun_opts=${MPIEXEC_NUMPROC_FLAG} --mpirun_opts=${nproc}
+        --mpirun_opts=${MPIEXEC_PREFLAGS} ${arg}
+        --test_dir ${CMAKE_CURRENT_SOURCE_DIR}/test_suites/${dir}
+        --output_dir "${PROJECT_BINARY_DIR}/tst/regression/outputs/${dir}_mpi_cov"
+        )
+      set_tests_properties(regression_mpi_coverage_test:${dir} PROPERTIES LABELS "regression;coverage;mpi-yes" RUN_SERIAL ON )
+    else()
+      message(STATUS "MPI not found, not building coverage regression tests with mpi")
+    endif()
   endif()
 endfunction()
 

diff --git a/tst/unit/CMakeLists.txt b/tst/unit/CMakeLists.txt
@@ -17,7 +17,6 @@
 ##========================================================================================
 
 list(APPEND unit_tests_SOURCES
-
     test_taskid.cpp
     test_unit_face_variables.cpp
     test_unit_params.cpp
@@ -29,15 +28,21 @@ list(APPEND unit_tests_SOURCES
     test_container_iterator.cpp
     test_required_desired.cpp
     test_error_checking.cpp
-
 )
 
-add_executable(unit_tests ${unit_tests_SOURCES})
+add_executable(unit_tests "${unit_tests_SOURCES}")
+
 target_link_libraries(unit_tests PRIVATE parthenon catch2_define Kokkos::kokkos)
+
 if (TEST_ERROR_CHECKING)
    message(WARNING "\tTesting error checking. This test will FAIL.")
    target_compile_definitions(unit_tests PRIVATE PARTHENON_TEST_ERROR_CHECKING)
 endif()
+
 lint_target(unit_tests)
 
 ParseAndAddCatchTests(unit_tests)
+
+get_property(ALL_TESTS DIRECTORY . PROPERTY TESTS)
+
+add_coverage_label("${ALL_TESTS}" "performance")
diff --git a/tst/unit/kokkos_abstraction.cpp b/tst/unit/kokkos_abstraction.cpp
@@ -204,7 +204,7 @@ bool test_wrapper_4d(T loop_pattern, DevExecSpace exec_space) {
   return all_same;
 }
 
-TEST_CASE("par_for loops", "[wrapper][coverage]") {
+TEST_CASE("par_for loops", "[wrapper]") {
   auto default_exec_space = DevExecSpace();
 
   SECTION("1D loops") {
@@ -401,7 +401,7 @@ bool test_wrapper_nested_4d(OuterLoopPattern outer_loop_pattern,
   return max_rel_err < rel_tol;
 }
 
-TEST_CASE("nested par_for loops", "[wrapper][coverage]") {
+TEST_CASE("nested par_for loops", "[wrapper]") {
   auto default_exec_space = DevExecSpace();
 
   SECTION("3D nested loops") {

diff --git a/tst/unit/test_container_iterator.cpp b/tst/unit/test_container_iterator.cpp
@@ -63,8 +63,7 @@ bool intervals_intersect(const std::pair<int, int> &i1, const std::pair<int, int
   return false;
 }
 
-TEST_CASE("Can pull variables from containers based on Metadata",
-          "[ContainerIterator][coverage]") {
+TEST_CASE("Can pull variables from containers based on Metadata", "[ContainerIterator]") {
   GIVEN("A Container with a set of variables initialized to zero") {
     Container<Real> rc;
     Metadata m_in({Metadata::Independent, Metadata::FillGhost});

diff --git a/tst/unit/test_error_checking.cpp b/tst/unit/test_error_checking.cpp
@@ -21,7 +21,7 @@
 
 #include "utils/error_checking.hpp"
 
-TEST_CASE("Parthenon Error Checking", "[ErrorChecking][Kokkos][coverage]") {
+TEST_CASE("Parthenon Error Checking", "[ErrorChecking][Kokkos]") {
   SECTION("PARTHENON_REQUIRE passes if condition true") {
     PARTHENON_REQUIRE(true, "This shouldn't fail");
   }