Merge Pull Request #7269 from e10harvey/Trilinos/atdv-271-atdm-ats1

Automatically Merged using Trilinos Pull Request AutoTester
PR Title: ATDM/ats1: Add initial ats1 env (ATDV-271)
PR Author: e10harvey

cmake/ctest/drivers/atdm/TrilinosCTestDriverCore.atdm.cmake

@@ -34,7 +34,7 @@ MACRO(TRILINOS_SYSTEM_SPECIFIC_CTEST_DRIVER)
   # Always assume the PWD is the root directory
   SET(CTEST_DASHBOARD_ROOT PWD)
 
-  # Must set the Jenkins JOB_NAME which will be the CDash build name 
+  # Must set the Jenkins JOB_NAME which will be the CDash build name
   SET(CTEST_BUILD_NAME "$ENV{JOB_NAME}")
 
   # Add this script and the shiller env script to the notes
@@ -44,9 +44,9 @@ MACRO(TRILINOS_SYSTEM_SPECIFIC_CTEST_DRIVER)
     "${THIS_FILE_LIST_DIR}/$ENV{ATDM_CONFIG_SYSTEM_NAME}/drivers/$ENV{JOB_NAME}.sh"
     "${CMAKE_CURRENT_LIST_FILE}"
     )
-  
+
   SET(CTEST_PARALLEL_LEVEL "$ENV{ATDM_CONFIG_CTEST_PARALLEL_LEVEL}")
-  
+
   IF ($ENV{ATDM_CONFIG_USE_NINJA})
     SET(CTEST_CMAKE_GENERATOR Ninja)
     IF ("$ENV{ATDM_CONFIG_BUILD_COUNT}" GREATER "0")
@@ -62,7 +62,7 @@ MACRO(TRILINOS_SYSTEM_SPECIFIC_CTEST_DRIVER)
   ATDM_SET_CACHE(CTEST_BUILD_FLAGS "${CTEST_BUILD_FLAGS}" CACHE STRING)
   # NOTE: Above, we need to set this as a cache var because this var is also
   # set as a cache var in ATDMDevEnvSettings.cmake that gets included below.
-  
+
   SET(EXTRA_CONFIGURE_OPTIONS)
 
   # See if to enable all of the packages
@@ -139,7 +139,7 @@ MACRO(TRILINOS_SYSTEM_SPECIFIC_CTEST_DRIVER)
   # basis.
   SET_DEFAULT_AND_FROM_ENV(CTEST_TEST_TYPE Nightly)
   SET_DEFAULT(Trilinos_TRACK Specialized)
-  
+
   IF (CTEST_TEST_TYPE STREQUAL "Experimental")
     # For "Experimental" builds, set the CDash site name to the real hostname.
     # This is done so that using queryTests.php will not pick up tests from

cmake/ctest/drivers/atdm/ats1/drivers/Trilinos-atdm-ats1-hsw_intel-18.0.5_mpich-7.7.6_openmp_static_dbg.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+if [ "${Trilinos_TRACK}" == "" ] ; then
+  export Trilinos_TRACK=Experimental
+fi
+export SBATCH_TIME_LIMIT_MINUTES=1440 # 24 hour time limit
+$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ats1/local-driver.sh

cmake/ctest/drivers/atdm/ats1/drivers/Trilinos-atdm-ats1-hsw_intel-18.0.5_mpich-7.7.6_openmp_static_opt.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+if [ "${Trilinos_TRACK}" == "" ] ; then
+  export Trilinos_TRACK=Experimental
+fi
+export SBATCH_TIME_LIMIT_MINUTES=1440 # 24 hour time limit
+$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ats1/local-driver.sh

cmake/ctest/drivers/atdm/ats1/drivers/Trilinos-atdm-ats1-hsw_intel-19.0.4_mpich-7.7.6_openmp_static_dbg.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+if [ "${Trilinos_TRACK}" == "" ] ; then
+  export Trilinos_TRACK=Experimental
+fi
+export SBATCH_TIME_LIMIT_MINUTES=1440 # 24 hour time limit
+$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ats1/local-driver.sh

cmake/ctest/drivers/atdm/ats1/drivers/Trilinos-atdm-ats1-hsw_intel-19.0.4_mpich-7.7.6_openmp_static_opt.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+if [ "${Trilinos_TRACK}" == "" ] ; then
+  export Trilinos_TRACK=Experimental
+fi
+export SBATCH_TIME_LIMIT_MINUTES=1440 # 24 hour time limit
+$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ats1/local-driver.sh

cmake/ctest/drivers/atdm/ats1/drivers/Trilinos-atdm-ats1-knl_intel-18.0.5_mpich-7.7.6_openmp_static_dbg.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+if [ "${Trilinos_TRACK}" == "" ] ; then
+  export Trilinos_TRACK=Experimental
+fi
+export SBATCH_TIME_LIMIT_MINUTES=900 # 15 hour time limit
+$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ats1/local-driver.sh

cmake/ctest/drivers/atdm/ats1/drivers/Trilinos-atdm-ats1-knl_intel-18.0.5_mpich-7.7.6_openmp_static_opt.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+if [ "${Trilinos_TRACK}" == "" ] ; then
+  export Trilinos_TRACK=Experimental
+fi
+export SBATCH_TIME_LIMIT_MINUTES=900 # 15 hour time limit
+$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ats1/local-driver.sh

cmake/ctest/drivers/atdm/ats1/drivers/Trilinos-atdm-ats1-knl_intel-19.0.4_mpich-7.7.6_openmp_static_dbg.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+if [ "${Trilinos_TRACK}" == "" ] ; then
+  export Trilinos_TRACK=Experimental
+fi
+export SBATCH_TIME_LIMIT_MINUTES=900 # 15 hour time limit
+$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ats1/local-driver.sh

cmake/ctest/drivers/atdm/ats1/drivers/Trilinos-atdm-ats1-knl_intel-19.0.4_mpich-7.7.6_openmp_static_opt.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+if [ "${Trilinos_TRACK}" == "" ] ; then
+  export Trilinos_TRACK=Experimental
+fi
+export SBATCH_TIME_LIMIT_MINUTES=900 # 15 hour time limit
+$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ats1/local-driver.sh

cmake/ctest/drivers/atdm/ats1/local-driver-on-allocation.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+export CTEST_SITE=${ATDM_CONFIG_CDASH_HOSTNAME}
+srun -N1  $WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ctest-s-driver.sh

cmake/ctest/drivers/atdm/ats1/local-driver.sh

@@ -0,0 +1,22 @@
+#!/bin/bash -l
+
+if [ "${SBATCH_TIME_LIMIT_MINUTES}" == "" ] ; then
+  SBATCH_TIME_LIMIT_MINUTES=180  # Default limit is 3 hours
+fi
+
+if [ "${Trilinos_CTEST_DO_ALL_AT_ONCE}" == "" ] ; then
+  export Trilinos_CTEST_DO_ALL_AT_ONCE=TRUE
+fi
+
+# Load environment on the login node
+source $WORKSPACE/Trilinos/cmake/std/atdm/load-env.sh $JOB_NAME
+
+set -x
+
+#export CTEST_SITE=${ATDM_CONFIG_CDASH_HOSTNAME}
+
+# Run configure, build, and test on the compute node
+atdm_run_script_on_compute_node \
+  $WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ats1/local-driver-on-allocation.sh \
+  $PWD/ctest-s-driver.out \
+  ${SBATCH_TIME_LIMIT_MINUTES}

cmake/std/atdm/ATDMDevEnvSettings.cmake

@@ -50,6 +50,7 @@ ENDIF()
 ATDM_SET_ATDM_VAR_FROM_ENV_AND_DEFAULT(PARALLEL_COMPILE_JOBS_LIMIT "")
 ATDM_SET_ATDM_VAR_FROM_ENV_AND_DEFAULT(PARALLEL_LINK_JOBS_LIMIT "")
 
+ATDM_SET_ATDM_VAR_FROM_ENV_AND_DEFAULT(Trilinos_LINK_SEARCH_START_STATIC OFF)
 ATDM_SET_ATDM_VAR_FROM_ENV_AND_DEFAULT(USE_OPENMP OFF)
 ATDM_SET_ATDM_VAR_FROM_ENV_AND_DEFAULT(USE_PTHREADS OFF)
 ATDM_SET_ATDM_VAR_FROM_ENV_AND_DEFAULT(USE_CUDA OFF)
@@ -278,6 +279,7 @@ ATDM_SET_CACHE(Trilinos_ENABLE_EXPLICIT_INSTANTIATION ON CACHE BOOL)
 ATDM_SET_CACHE(Trilinos_ENABLE_INSTALL_CMAKE_CONFIG_FILES ON CACHE BOOL)
 ATDM_SET_CACHE(Trilinos_ENABLE_DEVELOPMENT_MODE OFF CACHE BOOL)
 ATDM_SET_CACHE(Trilinos_ASSERT_MISSING_PACKAGES ON CACHE BOOL)
+ATDM_SET_CACHE(Trilinos_LINK_SEARCH_START_STATIC "${ATDM_Trilinos_LINK_SEARCH_START_STATIC}" CACHE BOOL)
 ATDM_SET_CACHE(Trilinos_ENABLE_OpenMP "${ATDM_USE_OPENMP}" CACHE BOOL)
 ATDM_SET_CACHE(Trilinos_ENABLE_COMPLEX "${ATDM_COMPLEX}" CACHE BOOL)
 ATDM_SET_CACHE(Kokkos_ENABLE_OPENMP "${ATDM_USE_OPENMP}" CACHE BOOL)

cmake/std/atdm/CMakeLists.txt

@@ -2,4 +2,5 @@ TRIBITS_PACKAGE(TrilinosATDMConfigTests)
 TRIBITS_ADD_TEST_DIRECTORIES(test)
 TRIBITS_ADD_TEST_DIRECTORIES(cee-rhel6)
 TRIBITS_ADD_TEST_DIRECTORIES(van1-tx2)
+TRIBITS_ADD_TEST_DIRECTORIES(ats1)
 TRIBITS_PACKAGE_POSTPROCESS()

cmake/std/atdm/README.md

@@ -138,7 +138,7 @@ functionality could be supported in the future if needed.
 **`<compiler>`:** The following **lower case** `<build-name>` keywords specify
 the `<COMPILER>` variable include:
 
-* `default`: Auto-select the default compiler and version for the given system 
+* `default`: Auto-select the default compiler and version for the given system
 * `gnu`: Use the GCC compilers (`<COMPILER>=GNU`)
   - `gnu-4.9.3`: Use GNU GCC 4.9.3 compilers
   - `gnu-7.2.0`: Use GNU GCC 7.2.0 compilers
@@ -241,7 +241,7 @@ checking etc.):
 * `debug` or `dbg`: (`<BUILD_TYPE>=DEBUG`, DEFAULT)
   * Set `CMAKE_BULD_TYPE=DEBUG` (i.e. `-O0 -g` compiler options)
   * Turn **ON** runtime debug checking
-  * NOTE: This build supports running in a debugger. 
+  * NOTE: This build supports running in a debugger.
 
 <a name="pt"/>
 
@@ -328,7 +328,7 @@ which after installation can then be sourced by clients using:
 
 ```
 $ source <install-prefix>/load_matching_env.sh
-``` 
+```
 
 Sourcing this file loads the compilers, MPI, and TPLs and sets up the various
 `ATDM_CONG_` environment variables described above.  It also sets the
@@ -609,6 +609,7 @@ example, skip the configure, skip the build, skip running tests, etc.
 * <a href="#waterman">waterman</a>
 * <a href="#ats-2">ATS-2</a>
 * <a href="#astra-vanguard-arm-system">ASTRA (Vanguard ARM System)</a>
+* <a href="#ats-1">ATS-1</a>
 
 
 ### ride/white
@@ -754,6 +755,8 @@ $ salloc -N1 --time=0:20:00 --account=<YOUR_WCID> \
 
 ### mutrino
 
+The default environment on mutrino is now ats1. Please see 
+<a href="#ats-1">ATS-1</a>.
 Once logged on to 'mutrino', one can directly configure and build on the login
 node (being careful not to overload the node) using the `mutrino` env.  But to
 run the tests, one must run on the compute nodes using the `salloc` command.
@@ -764,7 +767,7 @@ For example, to configure, build and run the tests for say `MueLu` on
 ```
 $ cd <some_build_dir>/
 
-$ source $TRILINOS_DIR/cmake/std/atdm/load-env.sh intel-opt-openmp
+$ source $TRILINOS_DIR/cmake/std/atdm/load-env.sh mutrino-intel-opt-openmp
 
 $ cmake \
   -DTrilinos_CONFIGURE_OPTIONS_FILE:STRING=cmake/std/atdm/ATDMDevEnv.cmake \
@@ -1064,7 +1067,7 @@ $ cmake -GNinja \
 # Get a node allocation and log on to launch node
 $ bsub -J <job-name> -W 6:00 -Is bash
 
-# Build on the allocated compute node 
+# Build on the allocated compute node
 $ lrun -n 1 make NP=32
 
 # Run the test suite from the launch node
@@ -1177,7 +1180,7 @@ $ cd <some_build_dir>/
 # List available environments
 $ source $TRILINOS_DIR/cmake/std/atdm/load-env.sh help
 
-# Load hsw env and configure on the login node
+# Load arm env and configure on the login node
 $ source $TRILINOS_DIR/cmake/std/atdm/load-env.sh arm-20.0
 $ cmake -G Ninja \
   -DTrilinos_CONFIGURE_OPTIONS_FILE:STRING=cmake/std/atdm/ATDMDevEnv.cmake \
@@ -1244,6 +1247,65 @@ fail.)
   shell directly on a compute node using `srun ... bash' an` then run `mpirun`
   from there.
 
+### ATS-1
+
+Once logged onto a supported ATS-1 system (called system 'ats1')
+system like 'mutrino', one can build and configure on a login node.
+
+To configure, build and run the tests for the default `intel-19` build for
+`Kokkos` (after cloning Trilinos on the 'develop' branch), run the following
+from a login node on 'mutrino':
+
+```bash
+$ cd <some_build_dir>/
+
+# List available environments
+$ source $TRILINOS_DIR/cmake/std/atdm/load-env.sh ats1-help
+
+# Load HSW env and configure on the login node
+$ source $TRILINOS_DIR/cmake/std/atdm/load-env.sh intel-19
+$ cmake -G Ninja \
+  -DTrilinos_CONFIGURE_OPTIONS_FILE:STRING=cmake/std/atdm/ATDMDevEnv.cmake \
+  -DTrilinos_ENABLE_TESTS=ON \
+  -DTrilinos_ENABLE_Tpetra=ON \
+  $TRILINOS_DIR
+
+$ make NP=8  # This is a shared node!
+
+# Get a node allocation and run ctest
+$ salloc -N 1 --time=2:00:00  -p short,batch --account=<YOUR_WCID> ctest -j1
+```
+
+One can also run tests in the background on a compute node using sbatch:
+```
+echo '#!/bin/bash' > do-test.sh
+echo 'ctest -j1' >> do-test.sh
+$ sbatch --output=do-test.out -N1 --time=2:00:00 -J ats1-do-test --account=<YOUR_WCID> do-test.sh
+```
+
+To drop to a compute node for an interactive session:
+```
+$ salloc -N1 --account=<YOUR_WCID>
+```
+
+To use the `ctest-s-local-test-driver.sh` script, one must set one's WCID
+account using:
+
+```
+$ export ATDM_CONFIG_WCID_ACCOUNT=<YOUR_WCID>
+```
+
+If `ATDM_CONFIG_WCID_ACCOUNT` is not set, then a default account will be used.
+(But if the user is not approved for that account, then the allocation will
+fail.)
+
+**NOTES:**
+- **IMPORTANT** Do not use anything but `ctest -j1` for running tests.
+- One cannot run more than one instance of srun at a time on a given compute node.
+- To get information on <YOUR_WCID> used above, there is a WC tool tab on
+  computing.sandia.gov.
+- CTest runs everything using the `srun` command and this must be run from
+  inside a `salloc` or `sbatch` allocation.
 
 ## Building and installing Trilinos for ATDM Applications
 

cmake/std/atdm/ats1/CMakeLists.txt

@@ -0,0 +1,11 @@
+TRIBITS_ADD_ADVANCED_TEST( ats1_custom_builds_unit_tests
+  OVERALL_WORKING_DIRECTORY TEST_NAME
+  OVERALL_NUM_MPI_PROCS 1
+  TEST_0 CMND env
+    ARGS SHUNIT_COLOR=none ${CMAKE_CURRENT_SOURCE_DIR}/custom_builds_unit_tests.sh
+    PASS_REGULAR_EXPRESSION_ALL
+      "ERROR: A supported compiler was not selected for 'ats1' env - anything-intel"
+      "Ran 1 test"
+      "[^]OK[$]"
+    ALWAYS_FAIL_ON_NONZERO_RETURN
+  )

cmake/std/atdm/ats1/all_supported_builds.sh

@@ -0,0 +1,13 @@
+
+export ATDM_CONFIG_CTEST_S_BUILD_NAME_PREFIX=Trilinos-atdm-
+
+export ATDM_CONFIG_ALL_SUPPORTED_BUILDS=(
+  ats1-hsw_intel-19.0.4_mpich-7.7.6_openmp_static_opt
+  ats1-hsw_intel-19.0.4_mpich-7.7.6_openmp_static_dbg
+  ats1-knl_intel-19.0.4_mpich-7.7.6_openmp_static_opt
+  ats1-knl_intel-19.0.4_mpich-7.7.6_openmp_static_dbg
+  ats1-hsw_intel-18.0.5_mpich-7.7.6_openmp_static_opt
+  ats1-hsw_intel-18.0.5_mpich-7.7.6_openmp_static_dbg
+  ats1-knl_intel-18.0.5_mpich-7.7.6_openmp_static_opt
+  ats1-knl_intel-18.0.5_mpich-7.7.6_openmp_static_dbg
+  )

cmake/std/atdm/ats1/custom_builds.sh

@@ -0,0 +1,41 @@
+#
+# Custom builds for 'ats1' env
+#
+# NOTE: This file gets sourced in atdm/utils/set_build_options.sh before the
+# default grep logic is applied.
+#
+
+# Custom compiler selection logic
+
+if atdm_match_any_buildname_keyword \
+     intel-19.0.4-mpich-7.7.6 \
+     intel-19.0.4_mpich-7.7.6 \
+     intel-19.0.4 \
+     intel-19 \
+  ; then
+  export ATDM_CONFIG_COMPILER=INTEL-19.0.4_MPICH-7.7.6
+# Note: matching "intel" below is dangerous. The user could easily make a typo
+# and wind up with the intel-18 env. Leaving "intel" as a match below due to
+# customer request.
+elif atdm_match_any_buildname_keyword \
+       intel-18.0.5-mpich-7.7.6 \
+       intel-18.0.5_mpich-7.7.6 \
+       intel-18.0.5 \
+       intel-18 \
+       default \
+       intel \
+  ; then
+  export ATDM_CONFIG_COMPILER=INTEL-18.0.5_MPICH-7.7.6
+else
+  echo
+  echo "***"
+  echo "*** ERROR: A supported compiler was not selected for 'ats1' env - $ATDM_CONFIG_BUILD_NAME"
+  echo "***"
+  echo "*** Supported compilers include:"
+  echo "***"
+  echo "****  intel-19.0.4-mpich-7.7.6                  (intel-19)"
+  echo "****  intel-18.0.5-mpich-7.7.6                  (intel-18, intel, default)"
+  echo "***"
+  return
+
+fi