Merge pull request #401 from bartlettroscoe/154-improve-cdash-github-…

…interaction

Improve GitHub Actions and CDash integration (#154)

* Links are provided from GitHub Actions Jobs to results on CDash (through CDash URL implemented manually in
   tribits_ctest_drdiver())

* Links on CDash builds are provided back to the GitHub PRs (through setting CTEST_CHANGE_ID=<pr-id>)

* The builds on CDash are prefixed by the PR ID with pr_<pr-id>_ so CDash queries can be easily constructed that
   show all of these.

* The Git repo SHA1 is shown on CDash (which shows merge commits for PRs but shows commits on 'master' for
   'Continuous' and 'Nightly' builds).

.github/workflows/tribits_testing.yml

@@ -83,3 +83,9 @@ jobs:
             --python-ver ${{ matrix.config.python }} \
             --cxx-compiler-and-ver ${{ matrix.config.cxx }} \
             --fortran-compiler-and-ver ${{ matrix.config.fc }}
+      - name: URL to results on CDash
+        run: |
+          cd ..
+          echo "Result on CDash are posted at:"
+          echo
+          cat tribits-build/BUILD/CDashBuildUrl.txt

cmake/ctest/github_actions/ctest_github_actions_serial_debug.cmake

@@ -3,7 +3,8 @@ include("${CMAKE_CURRENT_LIST_DIR}/../TribitsProjCTestDriver.cmake")
 set(CTEST_BUILD_NAME $ENV{CTEST_BUILD_NAME})
 
 set(CTEST_TEST_TIMEOUT 60)
-set(CTEST_DO_UPDATES OFF)
+set(CTEST_DO_UPDATES TRUE)
+set(CTEST_UPDATE_VERSION_ONLY TRUE)
 set_default_and_from_env(CTEST_BUILD_FLAGS "-j2")
 set_default_and_from_env(CTEST_PARALLEL_LEVEL "2")
 

cmake/ctest/github_actions/run_github_actions_ctest_driver.sh

@@ -173,6 +173,7 @@ if [[ "${GITHUB_EVENT_PATH}" != "" ]] ; then
   echo "pull_number = '${pull_number}'"
   if [[ "${pull_number}" != "null" ]] ; then
     export CTEST_BUILD_NAME="pr-${pull_number}_${CTEST_BUILD_NAME}${CTEST_BUILD_NAME_SUFFIX}"
+    export CTEST_CHANGE_ID=${pull_number}
   fi
 fi
 echo "CTEST_BUILD_NAME = '${CTEST_BUILD_NAME}'"
@@ -206,7 +207,6 @@ export TriBITS_TRACK
 echo "CTEST_TEST_TYPE = '${CTEST_TEST_TYPE}'"
 echo "TriBITS_TRACK = '${TriBITS_TRACK}'"
 
-
 #
 # C) Run the local configure, build, test and submit using exported vars above
 #

test/ctest_driver/CMakeLists.txt

@@ -6,7 +6,7 @@ tribits_add_advanced_test( CTestDriverUnitTests
       -D${PROJECT_NAME}_TRIBITS_DIR=${${PROJECT_NAME}_TRIBITS_DIR}
       -P "${CMAKE_CURRENT_SOURCE_DIR}/CTestDriverUnitTests.cmake"
     PASS_REGULAR_EXPRESSION_ALL
-      "Final UnitTests Result: num_run = 2"
+      "Final UnitTests Result: num_run = 6"
       "Final UnitTests Result: PASSED"
   )
 

test/ctest_driver/CTestDriverUnitTests.cmake

@@ -47,28 +47,93 @@ set( CMAKE_MODULE_PATH
   )
 
 include(GlobalSet)
-include(TribitsReadTagFile)
 include(UnitTestHelpers)
 
+include(TribitsReadTagFile)
+include(TribitsGetCDashUrlFromTagFile)
+
 
-function(unittest_read_ctest_tag_file)
+function(unittest_tribits_read_ctest_tag_file)
 
   message("\n***")
   message("*** Testing tribits_read_ctest_tag_file()")
   message("***\n")
 
   set(TAG_FILE_IN "${CMAKE_CURRENT_LIST_DIR}/data/dummy_build_dir/Testing/TAG")
 
-  tribits_read_ctest_tag_file(${TAG_FILE_IN} BUILD_START_TIME_OUT CDASH_TRACK_OUT)
+  tribits_read_ctest_tag_file(${TAG_FILE_IN} buildStartTime cdashGroup cdashModel)
 
-  unittest_compare_const(BUILD_START_TIME_OUT
+  unittest_compare_const(buildStartTime
     "20101015-1112")
+  unittest_compare_const(cdashGroup
+    "My CDash Group")  # NOTE: Spaces are important to test here!
+  unittest_compare_const(cdashModel
+    "The Model")  # NOTE: Spaces are important to test here!
+
+endfunction()
+
+
+function(unittest_tribits_get_cdash_index_php_from_drop_site_and_location)
+
+  message("\n***")
+  message("*** Testing tribits_get_cdash_index_php_from_drop_site_and_location()")
+  message("***\n")
 
-  unittest_compare_const(CDASH_TRACK_OUT
-    "My CDash Track")  # NOTE: Spaces are important to test here!
+  tribits_get_cdash_index_php_from_drop_site_and_location(
+    CTEST_DROP_SITE "some.site.com"
+    CTEST_DROP_LOCATION "/cdash/submit.php?project=SomeProject"
+    INDEX_PHP_URL_OUT indexPhpUrl
+    )
+
+  unittest_compare_const(indexPhpUrl "some.site.com/cdash/index.php")
 
 endfunction()
 
+
+function(unittest_tribits_get_cdash_build_url_from_parts)
+
+  message("\n***")
+  message("*** Testing tribits_get_cdash_build_url_from_parts()")
+  message("***\n")
+
+  tribits_get_cdash_build_url_from_parts(
+    INDEX_PHP_URL "mycdash/index.php"
+    PROJECT_NAME "my project"
+    SITE_NAME "my site"
+    BUILD_NAME "my buildname g++-2.5"
+    BUILD_STAMP "20210729-0024-My Group"
+    CDASH_BUILD_URL_OUT cdashBuildUrlOut
+    )
+
+  unittest_compare_const(cdashBuildUrlOut
+     "mycdash/index.php?project=my%20project&filtercount=3&showfilters=1&filtercombine=and&field1=site&compare1=61&value1=my%20site&field2=buildname&compare2=61&value2=my%20buildname%20g%2B%2B-2.5&field3=buildstamp&compare3=61&value3=20210729-0024-My%20Group") 
+
+endfunction()
+
+
+function(unittest_tribits_get_cdash_build_url_from_tag_file)
+
+  message("\n***")
+  message("*** Testing tribits_get_cdash_build_url_from_tag_file()")
+  message("***\n")
+
+  set(TAG_FILE "${CMAKE_CURRENT_LIST_DIR}/data/dummy_build_dir/Testing/TAG")
+
+  tribits_get_cdash_build_url_from_tag_file(
+    INDEX_PHP_URL "mycdash/index.php"
+    PROJECT_NAME "my project"
+    SITE_NAME "my site"
+    BUILD_NAME "my buildname g++-2.5"
+    TAG_FILE "${TAG_FILE}"
+    CDASH_BUILD_URL_OUT cdashBuildUrl
+    )
+
+  unittest_compare_const(cdashBuildUrl
+     "mycdash/index.php?project=my%20project&filtercount=3&showfilters=1&filtercombine=and&field1=site&compare1=61&value1=my%20site&field2=buildname&compare2=61&value2=my%20buildname%20g%2B%2B-2.5&field3=buildstamp&compare3=61&value3=20101015-1112-My%20CDash%20Group") 
+
+endfunction()
+
+
 #
 # Execute the unit tests
 #
@@ -79,11 +144,14 @@ global_set(UNITTEST_OVERALL_NUMPASSED 0)
 global_set(UNITTEST_OVERALL_NUMRUN 0)
 
 # Run the unit test functions
-unittest_read_ctest_tag_file()
+unittest_tribits_read_ctest_tag_file()
+unittest_tribits_get_cdash_index_php_from_drop_site_and_location()
+unittest_tribits_get_cdash_build_url_from_parts()
+unittest_tribits_get_cdash_build_url_from_tag_file()
 
 message("\n***")
 message("*** Determine final result of all unit tests")
 message("***\n")
 
 # Pass in the number of expected tests that must pass!
-unittest_final_result(2)
+unittest_final_result(6)

test/ctest_driver/TribitsExampleMetaProject/CMakeLists.txt

@@ -264,7 +264,7 @@ tribits_add_advanced_test( CTestDriver_TribitsExMetaProj_clone_default_branch_re
     ALWAYS_FAIL_ON_NONZERO_RETURN
 
   TEST_11
-    MESSAGE "Run ctest driver with an on branch 'for-testing'"
+    MESSAGE "Run ctest driver on branch 'for-testing'"
     CMND env
     ARGS
       CTEST_DASHBOARD_ROOT=PWD
@@ -315,7 +315,76 @@ tribits_add_advanced_test( CTestDriver_TribitsExMetaProj_clone_default_branch_re
   # * Makes sure some git commit SHA1s are shown in 'Old/New revision of
   #   repository' printouts from ctest_update().
   #
-  # * ???
+  # NOTE: The reason that so many tests are done in a single ctest test is the
+  # cost of the initial configure which checks the compilers which is very
+  # expensive.  The other runs use
+  # CTEST_START_WITH_EMPTY_BINARY_DIRECTORY=FALSE and CTEST_WIPE_CACHE=FALSE
+  # to avoid this cost.
+
+
+tribits_add_advanced_test( CTestDriver_TribitsExMetaProj_clone_default_branch_remote_CTEST_UPDATE_VERSION_ONLY
+  OVERALL_WORKING_DIRECTORY TEST_NAME
+  OVERALL_NUM_MPI_PROCS 1
+  EXCLUDE_IF_NOT_TRUE ${PROJECT_NAME}_ENABLE_Fortran
+
+  TEST_0
+    MESSAGE "Run ctest driver with intial clone only of the repos on the default branch 'master' and remote 'origin'"
+    CMND env
+    ARGS
+      CTEST_DASHBOARD_ROOT=PWD
+      ${COMMON_ENV_ARGS}
+      TribitsExMetaProj_ENABLE_SECONDARY_TESTED_CODE=TRUE
+      CTEST_DO_CONFIGURE=OFF
+      CTEST_DO_BUILD=OFF
+      CTEST_DO_TEST=OFF
+      CTEST_BUILD_NAME=CTestDriver_TribitsExMetaProj_clone_default_branch_remote_CTEST_UPDATE_VERSION_ONLY
+      ${CTEST_S_SCRIPT_ARGS}
+    PASS_REGULAR_EXPRESSION_ALL
+      "CTEST_NOTES_FILES=''"
+      "First perform the initial checkout: .*/git. clone -o origin .*github.com.tribits/TribitsExampleMetaProject.git"
+      "Perform checkout in directory: .*/TriBITS_CTestDriver_TribitsExMetaProj_clone_default_branch_remote_CTEST_UPDATE_VERSION_ONLY"
+      "Calling ctest_update[(][)] to update base source repo '.*/TriBITS_CTestDriver_TribitsExMetaProj_clone_default_branch_remote_CTEST_UPDATE_VERSION_ONLY/TribitsExampleMetaProject"
+      "Old revision of repository is: [a-z0-9]+"
+      "New revision of repository is: [a-z0-9]+"
+      "ctest_update[(][.][.][.][)] returned '0'"
+      "cmake -P tribits_ctest_update_commands[.]cmake"
+      "Git Update PASSED!"
+      "TribitsExampleProject: Doing initial GIT clone/checkout from URL '.*github[.]com.tribits/TribitsExampleProject.git' to dir 'TribitsExampleProject' [.][.][.]"
+      "Cloning into 'TribitsExampleProject'[.][.][.]"
+      "TribitsExampleProjectAddons: Doing initial GIT clone/checkout from URL '.*github[.]com.tribits/TribitsExampleProjectAddons.git' to dir 'TribitsExampleProjectAddons' [.][.][.]"
+      "Cloning into 'TribitsExampleProjectAddons'[.][.][.]"
+      "CTEST_NOTES_FILES_WO_CACHE='.*/TriBITS_CTestDriver_TribitsExMetaProj_clone_default_branch_remote_CTEST_UPDATE_VERSION_ONLY/BUILD/UpdateCommandsOutput.txt'"
+      "TRIBITS_CTEST_DRIVER: OVERALL: ALL PASSED"
+    ALWAYS_FAIL_ON_NONZERO_RETURN
+
+  TEST_1
+    MESSAGE "Run ctest driver again but this time with CTEST_UPDATE_VERSION_ONLY=ON"
+    CMND env
+    ARGS
+      CTEST_DASHBOARD_ROOT=PWD
+      ${COMMON_ENV_ARGS}
+      CTEST_START_WITH_EMPTY_BINARY_DIRECTORY=FALSE
+      CTEST_WIPE_CACHE=FALSE
+      TribitsExMetaProj_ENABLE_SECONDARY_TESTED_CODE=TRUE
+      CTEST_UPDATE_VERSION_ONLY=ON
+      CTEST_BUILD_NAME=CTestDriver_TribitsExMetaProj_clone_default_branch_remote_CTEST_UPDATE_VERSION_ONLY
+      ${CTEST_S_SCRIPT_ARGS}
+    PASS_REGULAR_EXPRESSION_ALL
+      "-- CTEST_NOTES_FILES=''"
+      "-- ENV_CTEST_UPDATE_VERSION_ONLY='ON'"
+      "-- CTEST_UPDATE_VERSION_ONLY='ON'"
+      "Calling ctest_update[(][)] to update base source repo '.*/TriBITS_CTestDriver_TribitsExMetaProj_clone_default_branch_remote_CTEST_UPDATE_VERSION_ONLY/TribitsExampleMetaProject"
+      "ctest_update[(][.][.][.][)] returned '0'"
+      "Configure PASSED!"
+      "Build PASSED!"
+      "100% tests passed, 0 tests failed out of 11"
+    ALWAYS_FAIL_ON_NONZERO_RETURN
+
+  )
+  # The above test checks:
+  #
+  # * Tests the option CTEST_UPDATE_VERSION_ONLY=ON that requires all of the
+  #   repos to be cloned and already be in the correct git state.
   #
   # NOTE: The reason that so many tests are done in a single ctest test is the
   # cost of the initial configure which checks the compilers which is very
@@ -453,7 +522,7 @@ tribits_add_advanced_test( CTestDriver_TribitsExMetaProj_clone_custom_branch_rem
       TribitsExMetaProj_ENABLE_SECONDARY_TESTED_CODE=TRUE
       TribitsExMetaProj_GIT_REPOSITORY_REMOTE=github
       TribitsExMetaProj_BRANCH=for-testing
-      CTEST_BUILD_NAME=CTestDriver_TribitsExMetaProj_clone_default_branch_remote
+      CTEST_BUILD_NAME=CTestDriver_TribitsExMetaProj_clone_custom_branch_remote
       ${CTEST_S_SCRIPT_ARGS}
     PASS_REGULAR_EXPRESSION_ALL
       "-- ENV_TribitsExMetaProj_BRANCH='for-testing'"

test/ctest_driver/TribitsExampleProject/CMakeLists.txt

@@ -1207,6 +1207,60 @@ function(generate_aao_tests)
     # CONFIGURE_OPTIONS list is checked to make sure that it does not change
     # without being noticed.  Follow-on tests will not check all of that.
 
+  set(cdash_url_expected_regex
+    "https://cdash[.]site[.]com/cdash/index[.]php[?]project=CustomTribitsExProj&filtercount=3&showfilters=1&filtercombine=and&field1=site&compare1=61&value1=My%20Site&field2=buildname&compare2=61&value2=TriBITS_CTestDriver_AAOP_CDASH_URL&field3=buildstamp&compare3=61&value3=[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]-Custom%20CDash%20Group" )
+  # NOTE: Above, we have to repeat [0-9] 8 times and 4 times for the regex for
+  # the number of digits in the buildstarttime.  CMake regex does not support
+  # \d{8}-\d{4} :-(
+
+  tribits_add_advanced_test( CTestDriver_AAO${AAO_POSTFIX}_CDASH_URL
+    OVERALL_WORKING_DIRECTORY TEST_NAME
+    OVERALL_NUM_MPI_PROCS 1
+    EXCLUDE_IF_NOT_TRUE ${PROJECT_NAME}_ENABLE_Fortran
+    TEST_0
+      MESSAGE "Symlink TribitsExampleProject so no need to clone (which would not work)."
+      CMND ln
+      ARGS -s ${TribitsExProj_DIR} .
+    TEST_1
+      MESSAGE "Run ctest driver testing everyting but no push."
+      CMND env
+      ARGS
+        CTEST_DASHBOARD_ROOT=PWD
+        ${AAO_COMMON_ENV_ARGS}
+        CTEST_PARALLEL_LEVEL=3
+        CTEST_DO_SUBMIT=OFF  # Never submit this
+        CTEST_SITE="My Site"
+        CTEST_DROP_SITE="cdash.site.com"
+        CTEST_PROJECT_NAME="CustomTribitsExProj"
+        CTEST_DROP_LOCATION="/cdash/submit.php?project=CustomTribitsExProj"
+        TribitsExProj_PACKAGES=SimpleCxx
+        CTEST_TEST_TYPE=Continuous
+        TribitsExProj_TRACK="Custom CDash Group" # Does not need to exist!
+        CTEST_BUILD_NAME=${PACKAGE_NAME}_CTestDriver_AAO${AAO_POSTFIX}_CDASH_URL
+        ${CTEST_S_SCRIPT_ARGS}
+      PASS_REGULAR_EXPRESSION_ALL
+        "NONE does not exist, skipping extra repositories"
+        "Final set of enabled packages:  SimpleCxx 1"
+        "Final set of enabled SE packages:  SimpleCxx 1"
+        "Results will be submitted on CDash at:"
+        "File '' does NOT exist so all tests passed"
+        "See results submitted on CDash at:"
+        "${cdash_url_expected_regex}"
+        "TRIBITS_CTEST_DRIVER: OVERALL: ALL PASSED"
+      ALWAYS_FAIL_ON_NONZERO_RETURN
+    TEST_2
+      MESSAGE "Check that CDashBuildUrl.txt exists and lists the right URL."
+      CMND cat ARGS BUILD/CDashBuildUrl.txt
+      PASS_REGULAR_EXPRESSION_ALL
+        "${cdash_url_expected_regex}"
+    )
+    # NOTE: The above test is never actually submitted to CDash, even when
+    # ${PACKAGE_NAME}_CTEST_DRIVER_SUBMIT_TO is set.  This is so we can
+    # exactly control the CDash location vars and do a good test of the
+    # construction of the CDash URL.  Note that we don't know the actual
+    # buildstamp because the buildstarttime changes everytime the test runs.
+    # We only know it is [8 digits]-[4 digits].
+
 
   tribits_add_advanced_test( CTestDriver_AAO${AAO_POSTFIX}_ST_SKIP_CTEST_ADD_TEST
     OVERALL_WORKING_DIRECTORY TEST_NAME

test/ctest_driver/data/dummy_build_dir/Testing/TAG

@@ -1,2 +1,3 @@
 20101015-1112
-My CDash Track
+My CDash Group
+The Model