Merge pull request #3610 from vicentebolea/enable-olcf-build

ci,ascent: enable ascent builds

.gitlab/gitlab-ci-ascent.yml

@@ -0,0 +1,130 @@
+# Ad-hoc build that runs in the ECP Hardware, concretely in OLCF Ascent.
+.setup_env_ecpci: &setup_env_ecpci |
+  module purge
+  module load ${JOB_MODULES}
+  module list
+  export PATH="/gpfs/wolf/csc303/scratch/vbolea/ci/utils:$PATH"
+
+.ascent-common:
+  except:
+    - schedules
+  tags:
+    - batch
+  interruptible: true
+  variables:
+    CCACHE_BASEDIR:       "/gpfs/wolf/"
+    CCACHE_DIR:           "/gpfs/wolf/csc303/scratch/vbolea/ci/ccache"
+    # -isystem= is not affected by CCACHE_BASEDIR, thus we must ignore it
+    CCACHE_IGNOREOPTIONS: "-isystem=*"
+    CCACHE_NOHASHDIR:     "true"
+
+    CUDAHOSTCXX:          "g++"
+    CUSTOM_CI_BUILDS_DIR: "/gpfs/wolf/csc303/scratch/vbolea/ci/adios2"
+    GITLAB_SITE:          "OLCF Ascent"
+    SCHEDULER_PARAMETERS: -P CSC303 -W 1:00 -nnodes 1 -alloc_flags gpudefault
+  before_script:
+    - *setup_env_ecpci
+    - ccache -z
+  script:
+    - bash scripts/ci/gitlab-ci/run.sh update
+    - bash scripts/ci/gitlab-ci/run.sh configure
+    - jsrun -n1 -a1 -g1 -c40 -bpacked:40 bash scripts/ci/gitlab-ci/run.sh build
+    - jsrun -n1 -a1 -g1 -c2 bash scripts/ci/gitlab-ci/run.sh test
+  after_script:
+    - *setup_env_ecpci
+    - bash scripts/ci/gitlab-ci/run.sh submit
+    - ccache -s
+
+ascent-cuda:
+  variables:
+    # Order matters
+    JOB_MODULES: >-
+      DefApps
+      zstd
+      cuda/11.4.2
+      git
+      gcc/10.2.0
+      ninja
+      spectrum-mpi
+      lsf-tools
+      libffi
+      hdf5
+      cmake
+  extends:
+    - .ascent-common
+
+ascent-kokkos-cuda:
+  variables:
+    # Order matters
+    JOB_MODULES: >-
+      DefApps
+      zstd
+      cuda/11.4.2
+      git
+      gcc/10.2.0
+      ninja
+      spectrum-mpi
+      lsf-tools
+      libffi
+      hdf5
+      cmake
+    KOKKOS_OPTS: >-
+      -DKokkos_ARCH_POWER9=ON
+      -DKokkos_ARCH_VOLTA70=ON
+      -DKokkos_ENABLE_CUDA=ON
+      -DKokkos_ENABLE_CUDA_LAMBDA=ON
+      -DCMAKE_INSTALL_PREFIX:PATH=$CI_BUILDS_DIR/kokkos_install
+
+    Kokkos_DIR: $CI_BUILDS_DIR/kokkos_install
+  before_script:
+    - *setup_env_ecpci
+    - ccache -z
+    - .gitlab/kokkos.sh "$CI_BUILDS_DIR" "3.7.01" $KOKKOS_OPTS
+  extends:
+    - .ascent-common
+
+ascent-nvhpc:
+  variables:
+    # Order matters
+    JOB_MODULES: >-
+      DefApps
+      zstd
+      nvhpc
+      git
+      spectrum-mpi
+      lsf-tools
+      libffi
+      hdf5
+      cmake
+  extends:
+    - .ascent-common
+
+ascent-xl:
+  variables:
+    # Order matters
+    JOB_MODULES: >-
+      DefApps
+      zstd
+      cuda/11.4.2
+      git
+      xl
+      ninja
+      spectrum-mpi
+      lsf-tools
+      libffi
+      hdf5
+      cmake
+  extends:
+    - .ascent-common
+
+sync-github-prs:
+  tags:
+    - nobatch
+  only:
+    - schedules
+  variables:
+    GIT_STRATEGY: none
+    CUSTOM_CI_BUILDS_DIR: "/gpfs/wolf/csc303/scratch/vbolea/ci/adios2"
+  script:
+    - export PATH="/gpfs/wolf/csc303/scratch/vbolea/ci/utils:$PATH"
+    - SpackCIBridge.py ornladios/ADIOS2 git@code.ornl.gov:ecpcitest/adios2.git https://code.ornl.gov/ ecpcitest/adios2 --prereq-check=format --prereq-check=git_checks

.gitlab/kokkos.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -x
+
+WORKDIR="$1"
+VERSION="$2"
+
+shift 2
+
+if [ ! -d "$WORKDIR" ] || [ -z "$VERSION" ]
+then
+  echo "[E] missing args: Invoke as .gitlab/ci/config/kokkos.sh <WORKDIR> <VERSION> [extra_args]"
+  exit 1
+fi
+
+# Build and install Kokkos
+curl -L "https://github.com/kokkos/kokkos/archive/refs/tags/$VERSION.tar.gz" \
+  | tar -C "$WORKDIR" -xzf -
+
+cmake -S "$WORKDIR/kokkos-$VERSION" -B "$WORKDIR/kokkos_build" \
+  "-DBUILD_SHARED_LIBS=ON" \
+  "-DCMAKE_BUILD_TYPE:STRING=release" \
+  "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache" \
+  "-DCMAKE_CXX_STANDARD:STRING=17" \
+  "-DCMAKE_CXX_EXTENSIONS:BOOL=OFF" \
+  "-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON" \
+  "-DCMAKE_CXX_COMPILER:STRING=$WORKDIR/kokkos-$VERSION/bin/nvcc_wrapper" \
+  $*
+
+cmake --build "$WORKDIR/kokkos_build"
+cmake --install "$WORKDIR/kokkos_build"

.shellcheck_exclude_paths

@@ -8,7 +8,6 @@ scripts/ci/gh-actions/get-changed-files.sh
 scripts/ci/gh-actions/linux-setup.sh
 scripts/ci/gh-actions/macos-setup.sh
 scripts/ci/gh-actions/run.sh
-scripts/ci/gitlab-ci/run.sh
 scripts/ci/images-v2/build-base.sh
 scripts/ci/images-v2/build-clang-base.sh
 scripts/ci/images-v2/build-clang.sh

CTestCustom.cmake.in

@@ -22,6 +22,8 @@ list(APPEND CTEST_CUSTOM_WARNING_EXCEPTION
   "warning: template parameter ... is not used in declaring the parameter types of function template"
   "warning: command-line option '.*' is valid for Fortran but not for C"
   "Warning #20208-D: '.*' is treated as '.*' in device code"
+  "Warning: '.*' is treated as '.*' in device code"
+  ".*was specified as both a system and non-system include directory.*"
 )
 list(APPEND CTEST_CUSTOM_COVERAGE_EXCLUDE
   ".*/thirdparty/.*"

scripts/ci/cmake-v2/ci-ascent-cuda.cmake

@@ -0,0 +1,27 @@
+# Client maintainer: vicente.bolea@kitware.com
+
+set(dashboard_cache "
+ADIOS2_USE_BZip2:BOOL=OFF
+ADIOS2_USE_CUDA:BOOL=ON
+ADIOS2_USE_DataMan:BOOL=ON
+ADIOS2_USE_Fortran:BOOL=ON
+ADIOS2_USE_MPI:BOOL=OFF
+ADIOS2_USE_PNG:BOOL=OFF
+ADIOS2_USE_Python:BOOL=OFF
+ADIOS2_USE_SST:BOOL=ON
+
+CMAKE_C_COMPILER_LAUNCHER=ccache
+CMAKE_CXX_COMPILER_LAUNCHER=ccache
+CMAKE_CUDA_COMPILER_LAUNCHER=ccache
+CMAKE_DISABLE_FIND_PACKAGE_BISON=ON
+CMAKE_DISABLE_FIND_PACKAGE_FLEX=ON
+CMAKE_Fortran_FLAGS:STRING=-Wall
+")
+
+set(CTEST_TEST_ARGS
+  PARALLEL_LEVEL 8
+  EXCLUDE ".*/BPWRCUDA.ADIOS2BPCUDAWrong/.*BP4.Serial|.*/BPWRCUDA.ADIOS2BPCUDAMemSel/.*BP4.Serial"
+  )
+set(CTEST_CMAKE_GENERATOR "Ninja")
+list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
+include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)

scripts/ci/cmake-v2/ci-ascent-kokkos-cuda.cmake

@@ -0,0 +1,32 @@
+# Client maintainer: vicente.bolea@kitware.com
+
+set(kokkos_install_path $ENV{Kokkos_DIR})
+
+set(ENV{CC}  gcc)
+set(ENV{CXX} g++)
+set(ENV{FC}  gfortran)
+
+set(dashboard_cache "
+ADIOS2_USE_BZip2:BOOL=OFF
+ADIOS2_USE_DataMan:BOOL=ON
+ADIOS2_USE_Fortran:BOOL=OFF
+ADIOS2_USE_MPI:BOOL=OFF
+ADIOS2_USE_PNG:BOOL=OFF
+ADIOS2_USE_Python:BOOL=OFF
+ADIOS2_USE_SST:BOOL=ON
+ADIOS2_USE_Kokkos=ON
+
+CMAKE_C_COMPILER_LAUNCHER=ccache
+CMAKE_CXX_COMPILER_LAUNCHER=ccache
+CMAKE_CUDA_COMPILER_LAUNCHER=ccache
+CMAKE_DISABLE_FIND_PACKAGE_BISON=ON
+CMAKE_DISABLE_FIND_PACKAGE_FLEX=ON
+")
+
+set(CTEST_TEST_ARGS
+  PARALLEL_LEVEL 8
+  EXCLUDE ".*/BPWRCUDA.ADIOS2BPCUDAWrong/.*BP4.Serial|.*/BPWRCUDA.ADIOS2BPCUDAMemSel/.*BP4.Serial|Engine.Staging.TestThreads.*"
+  )
+set(CTEST_CMAKE_GENERATOR "Ninja")
+list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
+include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)

scripts/ci/cmake-v2/ci-ascent-nvhpc.cmake

@@ -0,0 +1,30 @@
+# Client maintainer: vicente.bolea@kitware.com
+
+set(ENV{CC}  nvc)
+set(ENV{CXX} nvc++)
+set(ENV{FC}  nvfortran)
+
+set(dashboard_cache "
+ADIOS2_USE_BZip2:BOOL=OFF
+ADIOS2_USE_CUDA:BOOL=ON
+ADIOS2_USE_DataMan:BOOL=ON
+ADIOS2_USE_Fortran:BOOL=ON
+ADIOS2_USE_MPI:BOOL=OFF
+ADIOS2_USE_PNG:BOOL=OFF
+ADIOS2_USE_Python:BOOL=OFF
+ADIOS2_USE_SST:BOOL=ON
+
+CMAKE_C_COMPILER_LAUNCHER=ccache
+CMAKE_CXX_COMPILER_LAUNCHER=ccache
+CMAKE_DISABLE_FIND_PACKAGE_BISON=ON
+CMAKE_DISABLE_FIND_PACKAGE_FLEX=ON
+CMAKE_NINJA_FORCE_RESPONSE_FILE=OFF
+")
+
+set(CTEST_TEST_ARGS
+  PARALLEL_LEVEL 8
+  EXCLUDE ".*/BPWRCUDA.ADIOS2BPCUDAWrong/.*BP4.Serial|.*/BPWRCUDA.ADIOS2BPCUDAMemSel/.*BP4.Serial|Install.*Fortran"
+  )
+set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
+list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
+include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)

scripts/ci/cmake/ci-ascent-xl.cmake → scripts/ci/cmake-v2/ci-ascent-xl.cmake

@@ -1,36 +1,30 @@
-# Client maintainer: chuck.atkins@kitware.com
-
-find_package(EnvModules REQUIRED)
-
-env_module(purge)
-env_module(load git)
-env_module(load xl)
-env_module(load hdf5)
-env_module(load libfabric)
-env_module(load python/3.7.0)
-env_module(load zfp)
-env_module(load zeromq)
+# Client maintainer: vicente.bolea@kitware.com
 
 set(ENV{CC}  xlc)
 set(ENV{CXX} xlc++)
 set(ENV{FC}  xlf)
 
 set(dashboard_cache "
 ADIOS2_USE_BZip2:BOOL=OFF
-ADIOS2_USE_Blosc:BOOL=OFF
 ADIOS2_USE_DataMan:BOOL=ON
-ADIOS2_USE_Fortran:BOOL=ON
-ADIOS2_USE_HDF5:BOOL=ON
+ADIOS2_USE_Fortran:BOOL=OFF
 ADIOS2_USE_MPI:BOOL=OFF
+ADIOS2_USE_PNG:BOOL=OFF
 ADIOS2_USE_Python:BOOL=OFF
 ADIOS2_USE_SST:BOOL=ON
-ADIOS2_USE_SZ:BOOL=OFF
-ADIOS2_USE_ZeroMQ:STRING=ON
+ADIOS2_USE_ZeroMQ:STRING=OFF
 ADIOS2_USE_ZFP:BOOL=OFF
+ADIOS2_USE_SZ:BOOL=OFF
+ADIOS2_USE_Blosc:BOOL=OFF
+
+CMAKE_C_COMPILER_LAUNCHER=ccache
+CMAKE_CXX_COMPILER_LAUNCHER=ccache
+CMAKE_DISABLE_FIND_PACKAGE_BISON=ON
+CMAKE_DISABLE_FIND_PACKAGE_FLEX=ON
 ")
 
 set(NCPUS 4)
 set(CTEST_TEST_ARGS PARALLEL_LEVEL 8)
-set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
+set(CTEST_CMAKE_GENERATOR "Ninja")
 list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
 include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)