Skip to content

Commit

Permalink
Merge pull request #3610 from vicentebolea/enable-olcf-build
Browse files Browse the repository at this point in the history
ci,ascent: enable ascent builds
  • Loading branch information
vicentebolea authored May 12, 2023
2 parents 765a02c + cf7a879 commit c4af0cf
Show file tree
Hide file tree
Showing 15 changed files with 326 additions and 143 deletions.
130 changes: 130 additions & 0 deletions .gitlab/gitlab-ci-ascent.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
# Ad-hoc build that runs in the ECP Hardware, concretely in OLCF Ascent.
.setup_env_ecpci: &setup_env_ecpci |
module purge
module load ${JOB_MODULES}
module list
export PATH="/gpfs/wolf/csc303/scratch/vbolea/ci/utils:$PATH"

.ascent-common:
except:
- schedules
tags:
- batch
interruptible: true
variables:
CCACHE_BASEDIR: "/gpfs/wolf/"
CCACHE_DIR: "/gpfs/wolf/csc303/scratch/vbolea/ci/ccache"
# -isystem= is not affected by CCACHE_BASEDIR, thus we must ignore it
CCACHE_IGNOREOPTIONS: "-isystem=*"
CCACHE_NOHASHDIR: "true"

CUDAHOSTCXX: "g++"
CUSTOM_CI_BUILDS_DIR: "/gpfs/wolf/csc303/scratch/vbolea/ci/adios2"
GITLAB_SITE: "OLCF Ascent"
SCHEDULER_PARAMETERS: -P CSC303 -W 1:00 -nnodes 1 -alloc_flags gpudefault
before_script:
- *setup_env_ecpci
- ccache -z
script:
- bash scripts/ci/gitlab-ci/run.sh update
- bash scripts/ci/gitlab-ci/run.sh configure
- jsrun -n1 -a1 -g1 -c40 -bpacked:40 bash scripts/ci/gitlab-ci/run.sh build
- jsrun -n1 -a1 -g1 -c2 bash scripts/ci/gitlab-ci/run.sh test
after_script:
- *setup_env_ecpci
- bash scripts/ci/gitlab-ci/run.sh submit
- ccache -s

ascent-cuda:
variables:
# Order matters
JOB_MODULES: >-
DefApps
zstd
cuda/11.4.2
git
gcc/10.2.0
ninja
spectrum-mpi
lsf-tools
libffi
hdf5
cmake
extends:
- .ascent-common

ascent-kokkos-cuda:
variables:
# Order matters
JOB_MODULES: >-
DefApps
zstd
cuda/11.4.2
git
gcc/10.2.0
ninja
spectrum-mpi
lsf-tools
libffi
hdf5
cmake
KOKKOS_OPTS: >-
-DKokkos_ARCH_POWER9=ON
-DKokkos_ARCH_VOLTA70=ON
-DKokkos_ENABLE_CUDA=ON
-DKokkos_ENABLE_CUDA_LAMBDA=ON
-DCMAKE_INSTALL_PREFIX:PATH=$CI_BUILDS_DIR/kokkos_install
Kokkos_DIR: $CI_BUILDS_DIR/kokkos_install
before_script:
- *setup_env_ecpci
- ccache -z
- .gitlab/kokkos.sh "$CI_BUILDS_DIR" "3.7.01" $KOKKOS_OPTS
extends:
- .ascent-common

ascent-nvhpc:
variables:
# Order matters
JOB_MODULES: >-
DefApps
zstd
nvhpc
git
spectrum-mpi
lsf-tools
libffi
hdf5
cmake
extends:
- .ascent-common

ascent-xl:
variables:
# Order matters
JOB_MODULES: >-
DefApps
zstd
cuda/11.4.2
git
xl
ninja
spectrum-mpi
lsf-tools
libffi
hdf5
cmake
extends:
- .ascent-common

sync-github-prs:
tags:
- nobatch
only:
- schedules
variables:
GIT_STRATEGY: none
CUSTOM_CI_BUILDS_DIR: "/gpfs/wolf/csc303/scratch/vbolea/ci/adios2"
script:
- export PATH="/gpfs/wolf/csc303/scratch/vbolea/ci/utils:$PATH"
- SpackCIBridge.py ornladios/ADIOS2 git@code.ornl.gov:ecpcitest/adios2.git https://code.ornl.gov/ ecpcitest/adios2 --prereq-check=format --prereq-check=git_checks
6 changes: 0 additions & 6 deletions .gitlab/gitlab-ci-gitlabdotcom.yml

This file was deleted.

28 changes: 0 additions & 28 deletions .gitlab/gitlab-ci-olcf.yml

This file was deleted.

30 changes: 30 additions & 0 deletions .gitlab/kokkos.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/usr/bin/env bash
set -x

WORKDIR="$1"
VERSION="$2"

shift 2

if [ ! -d "$WORKDIR" ] || [ -z "$VERSION" ]
then
echo "[E] missing args: Invoke as .gitlab/ci/config/kokkos.sh <WORKDIR> <VERSION> [extra_args]"
exit 1
fi

# Build and install Kokkos
curl -L "https://github.com/kokkos/kokkos/archive/refs/tags/$VERSION.tar.gz" \
| tar -C "$WORKDIR" -xzf -

cmake -S "$WORKDIR/kokkos-$VERSION" -B "$WORKDIR/kokkos_build" \
"-DBUILD_SHARED_LIBS=ON" \
"-DCMAKE_BUILD_TYPE:STRING=release" \
"-DCMAKE_CXX_COMPILER_LAUNCHER=ccache" \
"-DCMAKE_CXX_STANDARD:STRING=17" \
"-DCMAKE_CXX_EXTENSIONS:BOOL=OFF" \
"-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON" \
"-DCMAKE_CXX_COMPILER:STRING=$WORKDIR/kokkos-$VERSION/bin/nvcc_wrapper" \
$*

cmake --build "$WORKDIR/kokkos_build"
cmake --install "$WORKDIR/kokkos_build"
1 change: 0 additions & 1 deletion .shellcheck_exclude_paths
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ scripts/ci/gh-actions/get-changed-files.sh
scripts/ci/gh-actions/linux-setup.sh
scripts/ci/gh-actions/macos-setup.sh
scripts/ci/gh-actions/run.sh
scripts/ci/gitlab-ci/run.sh
scripts/ci/images-v2/build-base.sh
scripts/ci/images-v2/build-clang-base.sh
scripts/ci/images-v2/build-clang.sh
Expand Down
2 changes: 2 additions & 0 deletions CTestCustom.cmake.in
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ list(APPEND CTEST_CUSTOM_WARNING_EXCEPTION
"warning: template parameter ... is not used in declaring the parameter types of function template"
"warning: command-line option '.*' is valid for Fortran but not for C"
"Warning #20208-D: '.*' is treated as '.*' in device code"
"Warning: '.*' is treated as '.*' in device code"
".*was specified as both a system and non-system include directory.*"
)
list(APPEND CTEST_CUSTOM_COVERAGE_EXCLUDE
".*/thirdparty/.*"
Expand Down
27 changes: 27 additions & 0 deletions scripts/ci/cmake-v2/ci-ascent-cuda.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Client maintainer: vicente.bolea@kitware.com

set(dashboard_cache "
ADIOS2_USE_BZip2:BOOL=OFF
ADIOS2_USE_CUDA:BOOL=ON
ADIOS2_USE_DataMan:BOOL=ON
ADIOS2_USE_Fortran:BOOL=ON
ADIOS2_USE_MPI:BOOL=OFF
ADIOS2_USE_PNG:BOOL=OFF
ADIOS2_USE_Python:BOOL=OFF
ADIOS2_USE_SST:BOOL=ON
CMAKE_C_COMPILER_LAUNCHER=ccache
CMAKE_CXX_COMPILER_LAUNCHER=ccache
CMAKE_CUDA_COMPILER_LAUNCHER=ccache
CMAKE_DISABLE_FIND_PACKAGE_BISON=ON
CMAKE_DISABLE_FIND_PACKAGE_FLEX=ON
CMAKE_Fortran_FLAGS:STRING=-Wall
")

set(CTEST_TEST_ARGS
PARALLEL_LEVEL 8
EXCLUDE ".*/BPWRCUDA.ADIOS2BPCUDAWrong/.*BP4.Serial|.*/BPWRCUDA.ADIOS2BPCUDAMemSel/.*BP4.Serial"
)
set(CTEST_CMAKE_GENERATOR "Ninja")
list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)
32 changes: 32 additions & 0 deletions scripts/ci/cmake-v2/ci-ascent-kokkos-cuda.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# Client maintainer: vicente.bolea@kitware.com

set(kokkos_install_path $ENV{Kokkos_DIR})

set(ENV{CC} gcc)
set(ENV{CXX} g++)
set(ENV{FC} gfortran)

set(dashboard_cache "
ADIOS2_USE_BZip2:BOOL=OFF
ADIOS2_USE_DataMan:BOOL=ON
ADIOS2_USE_Fortran:BOOL=OFF
ADIOS2_USE_MPI:BOOL=OFF
ADIOS2_USE_PNG:BOOL=OFF
ADIOS2_USE_Python:BOOL=OFF
ADIOS2_USE_SST:BOOL=ON
ADIOS2_USE_Kokkos=ON
CMAKE_C_COMPILER_LAUNCHER=ccache
CMAKE_CXX_COMPILER_LAUNCHER=ccache
CMAKE_CUDA_COMPILER_LAUNCHER=ccache
CMAKE_DISABLE_FIND_PACKAGE_BISON=ON
CMAKE_DISABLE_FIND_PACKAGE_FLEX=ON
")

set(CTEST_TEST_ARGS
PARALLEL_LEVEL 8
EXCLUDE ".*/BPWRCUDA.ADIOS2BPCUDAWrong/.*BP4.Serial|.*/BPWRCUDA.ADIOS2BPCUDAMemSel/.*BP4.Serial|Engine.Staging.TestThreads.*"
)
set(CTEST_CMAKE_GENERATOR "Ninja")
list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)
30 changes: 30 additions & 0 deletions scripts/ci/cmake-v2/ci-ascent-nvhpc.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# Client maintainer: vicente.bolea@kitware.com

set(ENV{CC} nvc)
set(ENV{CXX} nvc++)
set(ENV{FC} nvfortran)

set(dashboard_cache "
ADIOS2_USE_BZip2:BOOL=OFF
ADIOS2_USE_CUDA:BOOL=ON
ADIOS2_USE_DataMan:BOOL=ON
ADIOS2_USE_Fortran:BOOL=ON
ADIOS2_USE_MPI:BOOL=OFF
ADIOS2_USE_PNG:BOOL=OFF
ADIOS2_USE_Python:BOOL=OFF
ADIOS2_USE_SST:BOOL=ON
CMAKE_C_COMPILER_LAUNCHER=ccache
CMAKE_CXX_COMPILER_LAUNCHER=ccache
CMAKE_DISABLE_FIND_PACKAGE_BISON=ON
CMAKE_DISABLE_FIND_PACKAGE_FLEX=ON
CMAKE_NINJA_FORCE_RESPONSE_FILE=OFF
")

set(CTEST_TEST_ARGS
PARALLEL_LEVEL 8
EXCLUDE ".*/BPWRCUDA.ADIOS2BPCUDAWrong/.*BP4.Serial|.*/BPWRCUDA.ADIOS2BPCUDAMemSel/.*BP4.Serial|Install.*Fortran"
)
set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)
Original file line number Diff line number Diff line change
@@ -1,36 +1,30 @@
# Client maintainer: chuck.atkins@kitware.com

find_package(EnvModules REQUIRED)

env_module(purge)
env_module(load git)
env_module(load xl)
env_module(load hdf5)
env_module(load libfabric)
env_module(load python/3.7.0)
env_module(load zfp)
env_module(load zeromq)
# Client maintainer: vicente.bolea@kitware.com

set(ENV{CC} xlc)
set(ENV{CXX} xlc++)
set(ENV{FC} xlf)

set(dashboard_cache "
ADIOS2_USE_BZip2:BOOL=OFF
ADIOS2_USE_Blosc:BOOL=OFF
ADIOS2_USE_DataMan:BOOL=ON
ADIOS2_USE_Fortran:BOOL=ON
ADIOS2_USE_HDF5:BOOL=ON
ADIOS2_USE_Fortran:BOOL=OFF
ADIOS2_USE_MPI:BOOL=OFF
ADIOS2_USE_PNG:BOOL=OFF
ADIOS2_USE_Python:BOOL=OFF
ADIOS2_USE_SST:BOOL=ON
ADIOS2_USE_SZ:BOOL=OFF
ADIOS2_USE_ZeroMQ:STRING=ON
ADIOS2_USE_ZeroMQ:STRING=OFF
ADIOS2_USE_ZFP:BOOL=OFF
ADIOS2_USE_SZ:BOOL=OFF
ADIOS2_USE_Blosc:BOOL=OFF
CMAKE_C_COMPILER_LAUNCHER=ccache
CMAKE_CXX_COMPILER_LAUNCHER=ccache
CMAKE_DISABLE_FIND_PACKAGE_BISON=ON
CMAKE_DISABLE_FIND_PACKAGE_FLEX=ON
")

set(NCPUS 4)
set(CTEST_TEST_ARGS PARALLEL_LEVEL 8)
set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
set(CTEST_CMAKE_GENERATOR "Ninja")
list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)
38 changes: 0 additions & 38 deletions scripts/ci/cmake/ci-ascent-gcc.cmake

This file was deleted.

Loading

0 comments on commit c4af0cf

Please sign in to comment.