ATDM/cts1empire: Create cts1empire and set as default env (ATDV-367)

cmake/ctest/drivers/atdm/cts1empire/drivers/Trilinos-atdm-cts1empire-intel-18.0.2_openmpi-4.0.1_openmp_static_dbg.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+#export SLURM_CTEST_TIMEOUT=1:00:00
+if [ "${Trilinos_TRACK}" == "" ] ; then
+  export Trilinos_TRACK=Specialized
+fi
+$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/cts1/local-driver.sh

cmake/ctest/drivers/atdm/cts1empire/drivers/Trilinos-atdm-cts1empire-intel-18.0.2_openmpi-4.0.1_openmp_static_opt.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+#export SLURM_CTEST_TIMEOUT=1:00:00
+if [ "${Trilinos_TRACK}" == "" ] ; then
+  export Trilinos_TRACK=Specialized
+fi
+$WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/cts1/local-driver.sh

cmake/ctest/drivers/atdm/cts1empire/local-driver.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -l
+
+if [ "${SLURM_CTEST_TIMEOUT}" == "" ] ; then
+  SLURM_CTEST_TIMEOUT=1:20:00
+  # This is just running tests, not the entire build!
+fi
+
+if [ "${Trilinos_CTEST_DO_ALL_AT_ONCE}" == "" ] ; then
+  export Trilinos_CTEST_DO_ALL_AT_ONCE=TRUE
+fi
+
+set -x
+
+source $WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ctest-s-driver-config-build.sh
+
+set -x
+
+atdm_run_script_on_compute_node \
+  $WORKSPACE/Trilinos/cmake/ctest/drivers/atdm/ctest-s-driver-test.sh \
+  $PWD/ctest-s-driver-test.out \
+  ${SLURM_CTEST_TIMEOUT}

cmake/std/atdm/cts1empire/all_supported_builds.sh

@@ -0,0 +1,7 @@
+
+export ATDM_CONFIG_CTEST_S_BUILD_NAME_PREFIX=Trilinos-atdm-
+
+export ATDM_CONFIG_ALL_SUPPORTED_BUILDS=(
+  cts1empire-intel-18.0.2_openmpi-4.0.1_openmp_static_dbg
+  cts1empire-intel-18.0.2_openmpi-4.0.1_openmp_static_opt
+  )

cmake/std/atdm/cts1empire/custom_builds.sh

@@ -0,0 +1,28 @@
+#
+# Custom builds for 'cts1empire' env
+#
+# NOTE: This file gets sourced in atdm/utils/set_build_options.sh before the
+# default grep logic is applied.
+#
+
+# Custom compiler selection logic
+
+if [[ $ATDM_CONFIG_BUILD_NAME == *"intel-18.0.2-openmpi-4.0.1"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"intel-18.0.2_openmpi-4.0.1"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"intel-18.0.2"* ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"intel" ]] \
+  || [[ $ATDM_CONFIG_BUILD_NAME == *"default" ]] \
+  ; then
+  export ATDM_CONFIG_COMPILER=INTEL-18.0.2_OPENMPI-4.0.1
+else
+  echo
+  echo "***"
+  echo "*** ERROR: A supported compiler was not selected for 'cts1empire' env"
+  echo "***"
+  echo "*** Supported compilers include:"
+  echo "***"
+  echo "****  intel-18.0.2_openmpi-4.0.1   (default)"
+  echo "***"
+  return
+
+fi

cmake/std/atdm/cts1empire/environment.sh

@@ -0,0 +1,31 @@
+################################################################################
+#
+# Set up env on cts1 for ATMD builds of Trilinos
+#
+# This source script gets the settings from the ATDM_CONFIG_BUILD_NAME var.
+#
+################################################################################
+
+if [ "$ATDM_CONFIG_KOKKOS_ARCH" == "DEFAULT" ] ; then
+  export ATDM_CONFIG_KOKKOS_ARCH=BDW
+fi
+
+if [ "$ATDM_CONFIG_KOKKOS_ARCH" != "BDW" ] ; then
+  echo "***"
+  echo "*** ERROR: KOKKOS_ARCH=$ATDM_CONFIG_KOKKOS_ARCH is not a valid option on this system."
+  echo "*** '$ATDM_CONFIG_KOKKOS_ARCH' appears in $ATDM_CONFIG_BUILD_NAME which then sets the KOKKOS_ARCH"
+  echo "*** on cts1 'BDW' is the only valid KOKKOS_ARCH. If no KOKKOS_ARCH is specified then"
+  echo "*** 'BDW' will be used by default"
+  echo "***"
+  return
+fi
+
+export ATDM_CONFIG_SPARC_TPL_BASE=/projects/sparc/tpls/cts1-bdw
+
+export ATDM_CONFIG_SBATCH_DEFAULT_TIMEOUT=4:00:00
+
+export ATDM_CONFIG_SLURM_DEFAULT_ACCOUNT=fy150090
+
+source $ATDM_SCRIPT_DIR/cts1empire/environment_new.sh
+
+export ATDM_CONFIG_COMPLETED_ENV_SETUP=TRUE

cmake/std/atdm/cts1empire/environment_new.sh

@@ -0,0 +1,98 @@
+################################################################################
+#
+# Set up cts1 env for EMPIRE ATDM builds of Trilinos
+#
+# This source script gets the settings from the ATDM_CONFIG_BUILD_NAME var.
+#
+################################################################################
+
+echo "Using $ATDM_CONFIG_SYSTEM_NAME toss3 compiler stack $ATDM_CONFIG_COMPILER to build $ATDM_CONFIG_BUILD_TYPE code with Kokkos node type $ATDM_CONFIG_NODE_TYPE"
+
+export ATDM_CONFIG_ENABLE_SPARC_SETTINGS=ON
+export ATDM_CONFIG_USE_NINJA=ON
+export ATDM_CONFIG_BUILD_COUNT=8
+# export ATDM_CONFIG_CMAKE_JOB_POOL_LINK=2
+# NOTE: Above, currently setting CMAKE_JOB_POOL_LINK results in a build
+# failures with Ninja.  See https://gitlab.kitware.com/snl/project-1/issues/60
+
+# We do this twice since sems modules are wacked and we get errors to the screen on a purge
+# The second purge will catch any real errors with purging ...
+module purge &> /dev/null
+module purge
+. /projects/sems/modulefiles/utils/sems-modules-init.sh
+module load sems-env
+module load sems-ninja_fortran/1.8.2
+module load sems-git/2.10.1
+
+if [ "$ATDM_CONFIG_NODE_TYPE" == "OPENMP" ] ; then
+  export ATDM_CONFIG_CTEST_PARALLEL_LEVEL=8
+  export OMP_NUM_THREADS=2
+else
+  export ATDM_CONFIG_CTEST_PARALLEL_LEVEL=16
+fi
+
+sparc_tpl_base=${ATDM_CONFIG_SPARC_TPL_BASE}
+module load cmake/3.12.2
+
+if [ "$ATDM_CONFIG_COMPILER" == "INTEL-18.0.2_OPENMPI-4.0.1" ]; then
+    module load intel/18.0.2.199
+    module load mkl/18.0.5.274 # Needed to address defect in MKL (#5316, #3992, #3914)
+    module load openmpi-intel/4.0
+
+    sparc_tpl_ext=cts1-bdw_intel-19.0.5
+    sparc_tpl_mpi_ext=cts1-bdw_intel-19.0.5_openmpi-4.0.1
+
+    export PATH=/usr/tce/packages/gcc/gcc-6.1.0/bin:${PATH}
+    export LD_LIBRARY_PATH=/usr/tce/packages/gcc/gcc-6.1.0/lib64:${LD_LIBRARY_PATH}
+else
+    echo
+    echo "***"
+    echo "*** ERROR: COMPILER=$ATDM_CONFIG_COMPILER is not supported on this system!"
+    echo "***"
+    return
+fi
+
+export OMPI_CXX=`which icpc`
+export OMPI_CC=`which icc`
+export OMPI_FC=`which ifort`
+export ATDM_CONFIG_LAPACK_LIBS="-mkl"
+export ATDM_CONFIG_BLAS_LIBS="-mkl"
+
+export BOOST_ROOT=${sparc_tpl_base}/boost-1.72.0/00000000/${sparc_tpl_ext}
+export HDF5_ROOT=${sparc_tpl_base}/hdf5-1.10.5/00000000/${sparc_tpl_mpi_ext}
+export CGNS_ROOT=${sparc_tpl_base}/cgns-c09a5cd/d313cc2f822078e47c7dbdee074ecb0431e573eb/${sparc_tpl_mpi_ext}
+export PNETCDF_ROOT=${sparc_tpl_base}/pnetcdf-1.12.1/6144dc67b2041e4093063a04e89fc1e33398bd09/${sparc_tpl_mpi_ext}
+export NETCDF_ROOT=${sparc_tpl_base}/netcdf-4.7.0/24baa07a3fa1ff9dbc8e70dc591ebbdec56783b2/${sparc_tpl_mpi_ext}
+export PARMETIS_ROOT=${sparc_tpl_base}/parmetis-4.0.3/00000000/${sparc_tpl_mpi_ext}
+export METIS_ROOT=${sparc_tpl_base}/parmetis-4.0.3/00000000/${sparc_tpl_mpi_ext}
+export LIBHIO_ROOT=${sparc_tpl_base}/libhio-1.4.1.2/00000000/${sparc_tpl_mpi_ext}
+export EUCLID_ROOT=${sparc_tpl_base}/euclid-20.23/8b68b12f72b59648c9a0a962a6d55ea978199860/${sparc_tpl_mpi_ext}
+export SGM_ROOT=${sparc_tpl_base}/sgm-20.23/00000000/${sparc_tpl_mpi_ext}
+export SUPERLUDIST_ROOT=${sparc_tpl_base}/superlu_dist-5.4.0/a3121eaff44f7bf7d44e625c3b3d2a9911e58876/${sparc_tpl_mpi_ext}
+
+export ATDM_CONFIG_USE_HWLOC=OFF
+export ATDM_CONFIG_HDF5_LIBS="-L${HDF5_ROOT}/lib;${HDF5_ROOT}/lib/libhdf5_hl.a;${HDF5_ROOT}/lib/libhdf5.a;-lz;-ldl"
+export ATDM_CONFIG_NETCDF_LIBS="-L${NETCDF_ROOT}/lib;-L${PNETCDF_ROOT}/lib;${NETCDF_ROOT}/lib/libnetcdf.a;${PNETCDF_ROOT}/lib/libpnetcdf.a;${ATDM_CONFIG_HDF5_LIBS};-lcurl"
+export ATDM_CONFIG_SUPERLUDIST_INCLUDE_DIRS=${SUPERLUDIST_ROOT}/include
+export ATDM_CONFIG_SUPERLUDIST_LIBS=${SUPERLUDIST_ROOT}/lib64/libsuperlu_dist.a
+export ATDM_CONFIG_BINUTILS_LIBS="/usr/lib64/libbfd.so;/usr/lib64/libiberty.a"
+
+# Set MPI wrappers
+export MPICC=`which mpicc`
+export MPICXX=`which mpicxx`
+export MPIF90=`which mpif90`
+
+export ATDM_CONFIG_MPI_EXEC=srun
+export ATDM_CONFIG_MPI_PRE_FLAGS="--mpi=pmi2;--ntasks-per-node;36"
+export ATDM_CONFIG_MPI_EXEC_NUMPROCS_FLAG=--ntasks
+
+# Set the default compilers
+export CC=mpicc
+export CXX=mpicxx
+export FC=mpif77
+export F90=mpif90
+
+# Define function atdm_run_script_on_compute_node
+source $ATDM_SCRIPT_DIR/common/define_run_on_slurm_compute_node_func.sh
+
+export ATDM_CONFIG_COMPLETED_ENV_SETUP=TRUE

cmake/std/atdm/cts1empire/tweaks/INTEL_DEBUG_OPENMP_BDW.cmake

@@ -0,0 +1,2 @@
+# Disable test that is failing or timing out in this build (see #2751)
+ATDM_SET_ENABLE(PanzerAdaptersSTK_MixedPoissonExample-ConvTest-Hex-Order-3_DISABLE ON)

cmake/std/atdm/cts1empire/tweaks/INTEL_RELEASE_OPENMP_BDW.cmake

@@ -0,0 +1,2 @@
+# Disable test that is failing or timing out in this build (see #2751)
+ATDM_SET_ENABLE(PanzerAdaptersSTK_MixedPoissonExample-ConvTest-Hex-Order-3_DISABLE ON)

cmake/std/atdm/utils/get_known_system_info.sh

@@ -32,15 +32,18 @@ source ${ATDM_CONFIG_SCRIPT_DIR}/utils/get_system_info_utils.sh
 
 realHostname=`hostname`
 if [[ "${ATDM_CONFIG_GET_KNOW_SYSTEM_INFO_REAL_HOSTNAME_OVERRIDE_FOR_UNIT_TESTING}" ]] ; then
-  echo
-  echo "***"
-  echo "*** WARNING: realHostname=$realHostname overriden to value of"
-  echo "*** ATDM_CONFIG_GET_KNOW_SYSTEM_INFO_REAL_HOSTNAME_OVERRIDE_FOR_UNIT_TESTING='${ATDM_CONFIG_GET_KNOW_SYSTEM_INFO_REAL_HOSTNAME_OVERRIDE_FOR_UNIT_TESTING}'"
-  echo "*** in <trilinos-dir>/cmake/std/atdm/utils/get_known_system_info.sh."
-  echo "*** This variable should only be set for unit testing purposes!"
-  echo "***"
-  echo
+  if [[ -z $ATDM_CONFIG_DISABLE_WARNINGS ]]; then
+    echo
+    echo "***"
+    echo "*** WARNING: realHostname=$realHostname overriden to value of"
+    echo "*** ATDM_CONFIG_GET_KNOW_SYSTEM_INFO_REAL_HOSTNAME_OVERRIDE_FOR_UNIT_TESTING='${ATDM_CONFIG_GET_KNOW_SYSTEM_INFO_REAL_HOSTNAME_OVERRIDE_FOR_UNIT_TESTING}'"
+    echo "*** in <trilinos-dir>/cmake/std/atdm/utils/get_known_system_info.sh."
+    echo "*** This variable should only be set for unit testing purposes!"
+    echo "***"
+    echo
+  fi
   realHostname=${ATDM_CONFIG_GET_KNOW_SYSTEM_INFO_REAL_HOSTNAME_OVERRIDE_FOR_UNIT_TESTING}
+  hostNameOverride=true
 fi
 #echo "Hostname = '$realHostname'"
 
@@ -65,6 +68,7 @@ ATDM_KNOWN_SYSTEM_NAMES_LIST=(
   waterman
   ats2
   van1-tx2
+  cts1empire
   cts1
   tlcc2
   sems-rhel7
@@ -150,6 +154,10 @@ fi
 
 # CTS1 systems
 if [[ $SNLSYSTEM == "cts1" ]] ; then
+  # Make cts1empire the default environment
+  systemNameTypeMatchedList+=(cts1empire)
+  systemNameTypeMatchedListHostNames[cts1empire]=$SNLCLUSTER
+  # Add cts1 to the list for the D.1 branch, below
   systemNameTypeMatchedList+=(cts1)
   systemNameTypeMatchedListHostNames[cts1]=$SNLCLUSTER
 fi
@@ -171,7 +179,8 @@ elif [[ "${SNLSYSTEM}" == "astra" || \
         "${SNLSYSTEM}" == "vortex" ]] ; then
   echo "Don't call get-platform on 'astra' systems" > /dev/null
   # Above logic avoids an 'ERROR: Unrecognized cluster <name>' on these systems
-elif [[ -f /projects/sems/modulefiles/utils/get-platform ]] ; then
+elif [[ -f /projects/sems/modulefiles/utils/get-platform &&
+        -z $hostNameOverride ]] ; then
   ATDM_SYSTEM_NAME=`source /projects/sems/modulefiles/utils/get-platform`
   if [[ $ATDM_SYSTEM_NAME == "rhel6-x86_64" ]] ; then
     systemNameTypeMatchedList+=(sems-rhel6)

cmake/std/atdm/utils/util_unit_tests.sh

@@ -1,13 +1,73 @@
 #!/bin/bash
 
 CURRENT_SCRIPTS_DIR=`echo $BASH_SOURCE | sed "s/\(.*\)\/.*\.sh/\1/g"`
-#ATDM_CONFIG_SCRIPT_DIR=".."
-ATDM_CONFIG_SCRIPT_DIR=`readlink -f ${CURRENT_SCRIPTS_DIR}/..`
-#ATDM_UTIL_SCRIPT="atdm_config_helper_funcs.sh"
-ATDM_UTIL_SCRIPT=`readlink -f ${CURRENT_SCRIPTS_DIR}/atdm_config_helper_funcs.sh`
 
-source ${ATDM_UTIL_SCRIPT}
+if [[ "$(uname)" == "Darwin" ]]; then
+  ATDM_CONFIG_SCRIPT_DIR=".."
+  ATDM_UTIL_SCRIPT_ATDM_CONFIG_HELPER_FUNCS="atdm_config_helper_funcs.sh"
+  ATDM_UTIL_SCRIPT_GET_KNOWN_SYSTEM_INFO="get_known_system_info.sh"
+  SHUNIT2_DIR="${ATDM_CONFIG_SCRIPT_DIR}/../../../commonTools/test/shunit2"
+else
+  ATDM_CONFIG_SCRIPT_DIR=`readlink -f ${CURRENT_SCRIPTS_DIR}/..`
+  ATDM_UTIL_SCRIPT_ATDM_CONFIG_HELPER_FUNCS=`readlink -f ${CURRENT_SCRIPTS_DIR}/atdm_config_helper_funcs.sh`
+  ATDM_UTIL_SCRIPT_GET_KNOWN_SYSTEM_INFO=`readlink -f ${CURRENT_SCRIPTS_DIR}/get_known_system_info.sh`
+  SHUNIT2_DIR=`readlink -f ${ATDM_CONFIG_SCRIPT_DIR}/../../../commonTools/test/shunit2`
+fi
 
+#
+# Test atdm get_known_system_info script
+#
+test_atdm_get_known_system_info() {
+   # TODO: D.2 needs to be tested by setting these values:
+  SNLSYSTEM=
+  SEMS_PLATFORM=
+  ATDM_SYSTEM_NAME=
+  SNLCLUSTER=
+
+  # Get the good ATDM_KNOWN_SYSTEM_NAMES_LIST
+  ATDM_CONFIG_BUILD_NAME=unit_test
+  ATDM_CONFIG_DISABLE_WARNINGS=ON
+  source ${ATDM_UTIL_SCRIPT_GET_KNOWN_SYSTEM_INFO}
+
+  # Check that all known system names pass
+  for ATDM_CONFIG_BUILD_NAME in ${ATDM_KNOWN_SYSTEM_NAMES_LIST[@]}; do
+    ATDM_CONFIG_GET_KNOW_SYSTEM_INFO_REAL_HOSTNAME_OVERRIDE_FOR_UNIT_TESTING=$ATDM_CONFIG_BUILD_NAME
+    RET=$(source ${ATDM_UTIL_SCRIPT_GET_KNOWN_SYSTEM_INFO})
+    assertEquals "Hostname '$ATDM_CONFIG_BUILD_NAME' matches known ATDM host '$ATDM_CONFIG_BUILD_NAME' and system '$ATDM_CONFIG_BUILD_NAME'" "$RET"
+  done
+
+  # Set the bad ATDM_KNOWN_SYSTEM_NAMES
+  ATDM_KNOWN_SYSTEM_NAMES_LIST=(
+    dne-name1
+    dne-name2
+    dne-name3
+
+  )
+
+  # Check a few bad systems names for failure
+  for ATDM_CONFIG_BUILD_NAME in ${ATDM_KNOWN_SYSTEM_NAMES_LIST[@]}; do
+    ATDM_CONFIG_GET_KNOW_SYSTEM_INFO_REAL_HOSTNAME_OVERRIDE_FOR_UNIT_TESTING=$ATDM_CONFIG_BUILD_NAME
+    source ${ATDM_UTIL_SCRIPT_GET_KNOWN_SYSTEM_INFO}
+    assertEquals "" "$ATDM_SYSTEM_NAME"
+    assertEquals "$ATDM_CONFIG_BUILD_NAME" "$realHostname"
+  done
+
+  # Ensure that cts1empire is the default on cts1
+  ATDM_CONFIG_BUILD_NAME=default
+  ATDM_CONFIG_GET_KNOW_SYSTEM_INFO_REAL_HOSTNAME_OVERRIDE_FOR_UNIT_TESTING=$ATDM_CONFIG_BUILD_NAME
+  SNLSYSTEM=cts1
+  RET=$(source ${ATDM_UTIL_SCRIPT_GET_KNOWN_SYSTEM_INFO})
+  assertEquals "Hostname '$ATDM_CONFIG_BUILD_NAME' matches known ATDM host '$ATDM_CONFIG_BUILD_NAME' and system 'cts1empire'" "$RET"
+
+  # Ensure that cts1 is still selected when it's in the build name
+  ATDM_CONFIG_BUILD_NAME=cts1-default
+  ATDM_CONFIG_GET_KNOW_SYSTEM_INFO_REAL_HOSTNAME_OVERRIDE_FOR_UNIT_TESTING=$ATDM_CONFIG_BUILD_NAME
+  SNLSYSTEM=cts1
+  RET=$(source ${ATDM_UTIL_SCRIPT_GET_KNOWN_SYSTEM_INFO})
+  assertEquals "Hostname '$ATDM_CONFIG_BUILD_NAME' matches known ATDM host '$ATDM_CONFIG_BUILD_NAME' and system 'cts1'" "$RET"
+}
+
+source ${ATDM_UTIL_SCRIPT_ATDM_CONFIG_HELPER_FUNCS}
 #
 # Test atdm utility functions
 #
@@ -59,18 +119,11 @@ test_atdm_remove_substrings_from_env_var() {
 
   for DELIM in ";" "." "/" " "; do
     RET=$(atdm_remove_substrings_from_env_var ENV_VAR "$DELIM" "$STRINGS")
-    assertEquals "$RET" "ERROR: atdm_remove_substrings_from_env_var: \"$DELIM\" is an invalid delimiter."
+    assertEquals "ERROR: atdm_remove_substrings_from_env_var: \"$DELIM\" is an invalid delimiter." "$RET"
   done
 }
 
-testAll() {
-  test_atdm_remove_substrings_from_env_var
-}
-
-
 #
 # Run the unit tests
 #
-SHUNIT2_DIR=`readlink -f ${ATDM_CONFIG_SCRIPT_DIR}/../../../commonTools/test/shunit2`
-#SHUNIT2_DIR="${ATDM_CONFIG_SCRIPT_DIR}/../../../commonTools/test/shunit2"
 . ${SHUNIT2_DIR}/shunit2