From d765ae3c88315b932d9836424c13a84714c396b5 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Fri, 23 Aug 2024 11:07:38 +0200 Subject: [PATCH 01/71] Env script for Stages/2024 --- env/jsc.2024_Intel.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 env/jsc.2024_Intel.sh diff --git a/env/jsc.2024_Intel.sh b/env/jsc.2024_Intel.sh new file mode 100644 index 0000000..9c5eb1d --- /dev/null +++ b/env/jsc.2024_Intel.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +# Load modules +module --force purge +module use $OTHERSTAGES +module load Stages/2024 +module load Intel/2023.2.1 +module load ParaStationMPI/5.9.2-1 +# +module load Hypre/2.29.0 +module load Silo/4.11.1 +module load Tcl/8.6.13 +# +module load ecCodes/2.31.0 +module load HDF5/1.14.2 +module load netCDF/4.9.2 +module load netCDF-Fortran/4.6.1 +module load PnetCDF/1.12.3 +module load cURL/8.0.1 +module load Szip/.2.1.1 +module load Python/3.11.3 +module load NCO/5.1.8 +module load CMake/3.26.3 +module load git/2.41.0-nodocs + +module li + +# Set default compilers +export CC=mpicc +export FC=mpif90 +export CXX=mpicxx +export MPI_HOME=$EBROOTPSMPI From a0f7242a1957796513af4397254763828a2c4504 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Fri, 23 Aug 2024 11:49:05 +0200 Subject: [PATCH 02/71] TSMP2 CI build test --- .github/workflows/CI.yml | 100 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 .github/workflows/CI.yml diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000..4546d0b --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,100 @@ +name: eCLM CI Test + +# Controls when the action will run. +on: [push, pull_request] + +jobs: + eclm_build_job: + name: ${{ matrix.config.name }} + runs-on: ubuntu-22.04 + + strategy: + matrix: + config: + - { + name: "eCLM-ParFlow", + } + + env: + CC: mpicc + FC: mpifort + BUILD_DIR: bld + INSTALL_DIR: install + OASIS_SRC: oasis3-mct + OASIS_TAG: tsmp-patches-v0.1 + eCLM_SRC: eCLM + PARFLOW_SRC: parflow + PARFLOW_TAG: v3.12.0 + CMAKE_BUILD_PARALLEL_LEVEL: 4 + + steps: + - uses: actions/checkout@v4 + - uses: awalsh128/cache-apt-pkgs-action@latest + with: + packages: libxml2-utils pylint wget cmake netcdf-bin libnetcdf-dev libnetcdff-dev libpnetcdf-dev + version: 1.0 + execute_install_scripts: true + + - name: Download MPI Fortran compiler + run: sudo apt-get install gfortran openmpi-bin libopenmpi-dev + + - name: Cache OASIS3-MCT ${{ env.OASIS_TAG }} + uses: actions/cache@v4 + id: cache-oasis + env: + cache-name: cache-oasis + with: + path: "~/$OASIS_SRC" + key: cache-${{ matrix.config.name }} + + - if: steps.cache-oasis.outputs.cache-hit != 'true' + name: Download OASIS3-MCT ${{ env.OASIS_TAG }} + run: | + git clone -b $OASIS_TAG https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/oasis3-mct.git + + - name: Cache eCLM + uses: actions/cache@v4 + id: cache-eclm + env: + cache-name: cache-eclm + with: + path: "~/$eCLM_SRC" + key: cache-${{ matrix.config.name }} + + - if: steps.cache-eclm.outputs.cache-hit != 'true' + name: Download eCLM + run: | + git clone https://github.com/HPSCTerrSys/eCLM.git + + - name: Cache ParFlow ${{ env.PARFLOW_TAG }} + uses: actions/cache@v4 + id: cache-parflow + env: + cache-name: cache-parflow + with: + path: "~/$PARFLOW_SRC" + key: cache-${{ matrix.config.name }} + + - if: steps.cache-parflow.outputs.cache-hit != 'true' + name: Download ParFlow ${{ env.PARFLOW_TAG }} + run: | + git clone -b $PARFLOW_TAG https://github.com/parflow/parflow.git + + - name: Configure TSMP2 + run: | + cmake -S . -B $BUILD_DIR + -DCMAKE_BUILD_TYPE="RELEASE" + -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR + -DCMAKE_C_COMPILER=$CC + -DCMAKE_Fortran_COMPILER=$FC + -DOASIS_SRC=${OASIS_SRC} + -DeCLM_SRC=${eCLM_SRC} + -DPARFLOW_SRC=${PARFLOW_SRC} + + - name: Build TSMP2 + run: | + cmake --build $BUILD_DIR + + - name: Install TSMP2 + run: | + cmake --install $BUILD_DIR From 1ef6b2b265d6084907adba9d08775d1382b54d8e Mon Sep 17 00:00:00 2001 From: kvrigor Date: Fri, 23 Aug 2024 16:59:55 +0200 Subject: [PATCH 03/71] MPI Fortran fix for Ubuntu --- cmake/BuildOASIS3MCT.cmake | 2 +- cmake/FindMPIFortran.cmake | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cmake/BuildOASIS3MCT.cmake b/cmake/BuildOASIS3MCT.cmake index 878d35c..a4618d6 100644 --- a/cmake/BuildOASIS3MCT.cmake +++ b/cmake/BuildOASIS3MCT.cmake @@ -22,7 +22,7 @@ file(APPEND ${OASIS_MAKE_INC} "BUILD_DIR = ${OASIS_BLD_DIR}\n") file(APPEND ${OASIS_MAKE_INC} "ARCHDIR = ${OASIS_INSTALL_PREFIX}\n") file(APPEND ${OASIS_MAKE_INC} "NETCDF_INCLUDE = ${NetCDF_ROOT}/include\n") file(APPEND ${OASIS_MAKE_INC} "NETCDF_LIBRARY = ${NetCDF_LIBRARIES}\n") -file(APPEND ${OASIS_MAKE_INC} "MPI_INCLUDE = ${MPI_Fortran_INCLUDE_DIRS}\n") +file(APPEND ${OASIS_MAKE_INC} "MPI_INCLUDE = ${MPI_Fortran_MODULE_DIR}\n") file(APPEND ${OASIS_MAKE_INC} "MAKE = make\n") file(APPEND ${OASIS_MAKE_INC} "F90 = ${CMAKE_Fortran_COMPILER}\n") file(APPEND ${OASIS_MAKE_INC} "F = $(F90)\n") diff --git a/cmake/FindMPIFortran.cmake b/cmake/FindMPIFortran.cmake index a0f13dc..ffcf473 100644 --- a/cmake/FindMPIFortran.cmake +++ b/cmake/FindMPIFortran.cmake @@ -10,18 +10,20 @@ find_package(MPI REQUIRED) find_package(PkgConfig QUIET) include(FindPackageHandleStandardArgs) -if (NOT MPI_Fortran_INCLUDE_DIRS) - pkg_check_modules(MPICH_Fortran REQUIRED mpich) - if (MPICH_Fortran_FOUND) - pkg_get_variable(MPI_Fortran_INCLUDE_DIRS mpich includedir) - pkg_get_variable(MPI_Fortran_LIB_DIR mpich libdir) - endif() +if (MPI_Fortran_MODULE_DIR) + message(STATUS "MPI_Fortran_MODULE_DIR=${MPI_Fortran_MODULE_DIR}") +endif() + +if (MPI_mpi_LIBRARY) + message(STATUS "MPI_mpi_LIBRARY=${MPI_mpi_LIBRARY}") + get_filename_component(MPI_Fortran_LIB_DIR ${MPI_mpi_LIBRARY} DIRECTORY) + message(STATUS "MPI_Fortran_LIB_DIR=${MPI_Fortran_LIB_DIR}") endif() find_package_handle_standard_args(MPIFortran REQUIRED_VARS MPI_Fortran_FOUND MPI_Fortran_COMPILER - MPI_Fortran_INCLUDE_DIRS + MPI_Fortran_MODULE_DIR MPI_Fortran_LIB_DIR MPIEXEC_EXECUTABLE MPIEXEC_NUMPROC_FLAG From 62827a535377a8ed785346c171609f660bea24a9 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Fri, 23 Aug 2024 17:06:41 +0200 Subject: [PATCH 04/71] Added Hypre installation in CI --- .github/workflows/CI.yml | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 4546d0b..1c41fd1 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -1,4 +1,4 @@ -name: eCLM CI Test +name: TSMP2 Build # Controls when the action will run. on: [push, pull_request] @@ -25,7 +25,9 @@ jobs: eCLM_SRC: eCLM PARFLOW_SRC: parflow PARFLOW_TAG: v3.12.0 + HYPRE_TAG: 2.26.0 CMAKE_BUILD_PARALLEL_LEVEL: 4 + CMAKE_PREFIX_PATH: $HOME/.local steps: - uses: actions/checkout@v4 @@ -45,12 +47,14 @@ jobs: cache-name: cache-oasis with: path: "~/$OASIS_SRC" - key: cache-${{ matrix.config.name }} + key: cache-${{ matrix.config.name }}-${{ env.OASIS_TAG }} - if: steps.cache-oasis.outputs.cache-hit != 'true' name: Download OASIS3-MCT ${{ env.OASIS_TAG }} run: | git clone -b $OASIS_TAG https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/oasis3-mct.git + cd $OASIS_SRC + echo "OASIS3-MCT version: $(git describe --tags --always)" - name: Cache eCLM uses: actions/cache@v4 @@ -65,6 +69,8 @@ jobs: name: Download eCLM run: | git clone https://github.com/HPSCTerrSys/eCLM.git + cd eCLM + echo "eCLM version: $(git describe --tags --always)" - name: Cache ParFlow ${{ env.PARFLOW_TAG }} uses: actions/cache@v4 @@ -73,12 +79,32 @@ jobs: cache-name: cache-parflow with: path: "~/$PARFLOW_SRC" - key: cache-${{ matrix.config.name }} + key: cache-${{ matrix.config.name }}-${{ env.PARFLOW_TAG }} - if: steps.cache-parflow.outputs.cache-hit != 'true' name: Download ParFlow ${{ env.PARFLOW_TAG }} run: | git clone -b $PARFLOW_TAG https://github.com/parflow/parflow.git + cd parflow + echo "ParFlow version: $(git describe --tags --always)" + + - name: Cache Hypre ${{ env.HYPRE_TAG }} + uses: actions/cache@v4 + id: cache-hypre + env: + cache-name: cache-hypre + with: + path: "~/$CMAKE_PREFIX_PATH" + key: cache-${{ matrix.config.name }}-${{ env.HYPRE_TAG }} + + - if: steps.cache-hypre.outputs.cache-hit != 'true' + name: Install Hypre ${{ env.HYPRE_TAG }} + run: | + wget https://github.com/hypre-space/hypre/archive/v${HYPRE_TAG}.tar.gz + tar xf v${HYPRE_TAG}.tar.gz + cd hypre-${HYPRE_TAG}/src + ./configure --prefix=$CMAKE_PREFIX_PATH + make -j4 install - name: Configure TSMP2 run: | From e4935d5996328d17dcafd9ffb20d227c6c64e6ba Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 9 Oct 2024 12:59:09 +0200 Subject: [PATCH 05/71] Switched to IntelMPI; Loaded 2024 env by default --- build_tsmp2.sh | 2 +- env/jsc.2024_Intel.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build_tsmp2.sh b/build_tsmp2.sh index 7ee6eed..20cf2f6 100755 --- a/build_tsmp2.sh +++ b/build_tsmp2.sh @@ -205,7 +205,7 @@ build_log="$(dirname ${cmake_build_dir})/${model_id}_$(date +%Y-%m-%d_%H-%M).log ## source environment message "source environment" if [ -z "${tsmp2_env}" ]; then - tsmp2_env="${cmake_tsmp2_dir}/env/jsc.2023_Intel.sh" + tsmp2_env="${cmake_tsmp2_dir}/env/jsc.2024_Intel.sh" else tsmp2_env="${tsmp2_env}" fi # tsmp2_env diff --git a/env/jsc.2024_Intel.sh b/env/jsc.2024_Intel.sh index 9c5eb1d..70e6cf7 100644 --- a/env/jsc.2024_Intel.sh +++ b/env/jsc.2024_Intel.sh @@ -5,7 +5,7 @@ module --force purge module use $OTHERSTAGES module load Stages/2024 module load Intel/2023.2.1 -module load ParaStationMPI/5.9.2-1 +module load IntelMPI/2021.10.0 # module load Hypre/2.29.0 module load Silo/4.11.1 From 94d04747845498b7d4e7be0e0ad46d9c83945f68 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 9 Oct 2024 13:55:33 +0200 Subject: [PATCH 06/71] MPI Fortran Finder fixes for ParaStationMPI --- cmake/FindMPIFortran.cmake | 15 +++++++++------ env/jsc.2024_Intel.sh | 4 +++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/cmake/FindMPIFortran.cmake b/cmake/FindMPIFortran.cmake index ffcf473..689c02d 100644 --- a/cmake/FindMPIFortran.cmake +++ b/cmake/FindMPIFortran.cmake @@ -10,14 +10,17 @@ find_package(MPI REQUIRED) find_package(PkgConfig QUIET) include(FindPackageHandleStandardArgs) -if (MPI_Fortran_MODULE_DIR) - message(STATUS "MPI_Fortran_MODULE_DIR=${MPI_Fortran_MODULE_DIR}") +if (NOT MPI_Fortran_INCLUDE_DIRS) + # TODO: MPICH is specific to ParaStationMPI; another approach is necessary for IntelMPI + pkg_check_modules(MPICH_Fortran REQUIRED mpich) + if (MPICH_Fortran_FOUND) + pkg_get_variable(MPI_Fortran_INCLUDE_DIRS mpich includedir) + pkg_get_variable(MPI_Fortran_LIB_DIR mpich libdir) + endif() endif() -if (MPI_mpi_LIBRARY) - message(STATUS "MPI_mpi_LIBRARY=${MPI_mpi_LIBRARY}") - get_filename_component(MPI_Fortran_LIB_DIR ${MPI_mpi_LIBRARY} DIRECTORY) - message(STATUS "MPI_Fortran_LIB_DIR=${MPI_Fortran_LIB_DIR}") +if (NOT MPI_Fortran_MODULE_DIR) + find_path(MPI_Fortran_MODULE_DIR NAMES "mpi.mod" "mpi_f08.mod" HINTS MPI_Fortran_INCLUDE_DIRS) endif() find_package_handle_standard_args(MPIFortran diff --git a/env/jsc.2024_Intel.sh b/env/jsc.2024_Intel.sh index 70e6cf7..a310489 100644 --- a/env/jsc.2024_Intel.sh +++ b/env/jsc.2024_Intel.sh @@ -5,7 +5,9 @@ module --force purge module use $OTHERSTAGES module load Stages/2024 module load Intel/2023.2.1 -module load IntelMPI/2021.10.0 +module load ParaStationMPI/5.9.2-1 +#module load IntelMPI/2021.10.0 + # module load Hypre/2.29.0 module load Silo/4.11.1 From e2936d9bb9041eaffb3c5764cd4c44ad13f2ef20 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 9 Oct 2024 14:17:16 +0200 Subject: [PATCH 07/71] Only source env file when using JSC supercomputers --- build_tsmp2.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/build_tsmp2.sh b/build_tsmp2.sh index 20cf2f6..b949764 100755 --- a/build_tsmp2.sh +++ b/build_tsmp2.sh @@ -186,7 +186,7 @@ else fi # set INSTALL and BUILD DIR (neccesary for building) -if [ -z "${SYSTEMNAME}" ]; then SYSTEMNAME="UNKN"; fi +if [ -z "${SYSTEMNAME}" ]; then SYSTEMNAME=$(hostname); fi if [ -z "${build_dir}" ]; then cmake_build_dir="${cmake_tsmp2_dir}/bld/${SYSTEMNAME^^}_${model_id}" @@ -203,13 +203,15 @@ fi # install_dir build_log="$(dirname ${cmake_build_dir})/${model_id}_$(date +%Y-%m-%d_%H-%M).log" ## source environment -message "source environment" -if [ -z "${tsmp2_env}" ]; then +if [[ $SYSTEMNAME = "jurecadc" || $SYSTEMNAME = "juwels" || $SYSTEMNAME = "jusuf" ]]; then + message "source environment" + if [ -z "${tsmp2_env}" ]; then tsmp2_env="${cmake_tsmp2_dir}/env/jsc.2024_Intel.sh" -else - tsmp2_env="${tsmp2_env}" -fi # tsmp2_env -source $tsmp2_env + else + tsmp2_env="${tsmp2_env}" + fi # tsmp2_env + source $tsmp2_env +fi ## CMAKE config # rm -rf ${cmake_build_dir} From e2b2e8a66a7baf5d7126dc6e90664fe57ce8661b Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 9 Oct 2024 15:16:18 +0200 Subject: [PATCH 08/71] Fixed FindMPIFortran for Ubuntu --- build_tsmp2.sh | 2 +- cmake/BuildOASIS3MCT.cmake | 2 +- cmake/FindMPIFortran.cmake | 32 ++++++++++++++++++++++---------- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/build_tsmp2.sh b/build_tsmp2.sh index b949764..7b0aae1 100755 --- a/build_tsmp2.sh +++ b/build_tsmp2.sh @@ -186,7 +186,7 @@ else fi # set INSTALL and BUILD DIR (neccesary for building) -if [ -z "${SYSTEMNAME}" ]; then SYSTEMNAME=$(hostname); fi +if [ -z "${SYSTEMNAME}" ]; then export SYSTEMNAME=$(hostname); fi if [ -z "${build_dir}" ]; then cmake_build_dir="${cmake_tsmp2_dir}/bld/${SYSTEMNAME^^}_${model_id}" diff --git a/cmake/BuildOASIS3MCT.cmake b/cmake/BuildOASIS3MCT.cmake index 20df3c7..d3cc924 100644 --- a/cmake/BuildOASIS3MCT.cmake +++ b/cmake/BuildOASIS3MCT.cmake @@ -22,7 +22,7 @@ file(APPEND ${OASIS_MAKE_INC} "BUILD_DIR = ${OASIS_BLD_DIR}\n") file(APPEND ${OASIS_MAKE_INC} "ARCHDIR = ${OASIS_INSTALL_PREFIX}\n") file(APPEND ${OASIS_MAKE_INC} "NETCDF_INCLUDE = ${NetCDF_ROOT}/include\n") file(APPEND ${OASIS_MAKE_INC} "NETCDF_LIBRARY = ${NetCDF_LIBRARIES}\n") -file(APPEND ${OASIS_MAKE_INC} "MPI_INCLUDE = ${MPI_Fortran_MODULE_DIR}\n") +file(APPEND ${OASIS_MAKE_INC} "MPI_INCLUDE = ${MPI_Fortran_INCLUDE_DIRS}\n") file(APPEND ${OASIS_MAKE_INC} "MAKE = make\n") file(APPEND ${OASIS_MAKE_INC} "F90 = ${CMAKE_Fortran_COMPILER}\n") file(APPEND ${OASIS_MAKE_INC} "F = $(F90)\n") diff --git a/cmake/FindMPIFortran.cmake b/cmake/FindMPIFortran.cmake index 689c02d..72bf359 100644 --- a/cmake/FindMPIFortran.cmake +++ b/cmake/FindMPIFortran.cmake @@ -3,31 +3,43 @@ # FindMPI.cmake is supposed to automatically set MPI_Fortran_INCLUDE_DIRS, but for some reason # it doesn't despite knowing where the MPI compilers are. This module serves as a workaround # for this issue. +# +# Updates +# ------- +# 2024-10-09: Built-in FindMPI still unreliable for various MPI Fortran implementations. +# +# [1]: https://gitlab.kitware.com/cmake/cmake/-/blob/master/Modules/FindMPI.cmake # -# [1]: https://gitlab.kitware.com/cmake/cmake/blob/v3.21.1/Modules/FindMPI.cmake) find_package(MPI REQUIRED) find_package(PkgConfig QUIET) include(FindPackageHandleStandardArgs) -if (NOT MPI_Fortran_INCLUDE_DIRS) - # TODO: MPICH is specific to ParaStationMPI; another approach is necessary for IntelMPI - pkg_check_modules(MPICH_Fortran REQUIRED mpich) +if (NOT MPI_Fortran_INCLUDE_DIRS OR NOT MPI_Fortran_LIB_DIR) + # TODO: Preferred MPI implementation should be introspected + # from host system environment. For now we only consider + # JSC (ParaStationMPI) and Ubuntu machines (OpenMPI). + + # MPICH + pkg_check_modules(MPICH_Fortran mpich) if (MPICH_Fortran_FOUND) pkg_get_variable(MPI_Fortran_INCLUDE_DIRS mpich includedir) pkg_get_variable(MPI_Fortran_LIB_DIR mpich libdir) + else() + # OpenMPI + pkg_check_modules(OpenMPI_Fortran REQUIRED mpi) + if (OpenMPI_Fortran_FOUND) + pkg_get_variable(MPI_Fortran_INCLUDE_DIRS mpi includedir) + pkg_get_variable(MPI_Fortran_LIB_DIR mpi libdir) + endif() endif() endif() -if (NOT MPI_Fortran_MODULE_DIR) - find_path(MPI_Fortran_MODULE_DIR NAMES "mpi.mod" "mpi_f08.mod" HINTS MPI_Fortran_INCLUDE_DIRS) -endif() - find_package_handle_standard_args(MPIFortran REQUIRED_VARS MPI_Fortran_FOUND MPI_Fortran_COMPILER - MPI_Fortran_MODULE_DIR + MPI_Fortran_INCLUDE_DIRS MPI_Fortran_LIB_DIR MPIEXEC_EXECUTABLE MPIEXEC_NUMPROC_FLAG - VERSION_VAR MPI_Fortran_VERSION) \ No newline at end of file + VERSION_VAR MPI_Fortran_VERSION) From cf063bf96dc7d37a21a90f7b97b0ebd0ba9508bd Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 9 Oct 2024 15:28:01 +0200 Subject: [PATCH 09/71] Added -DeCLM=ON and -DParFlow=ON --- .github/workflows/CI.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 1c41fd1..88f50c6 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -114,7 +114,9 @@ jobs: -DCMAKE_C_COMPILER=$CC -DCMAKE_Fortran_COMPILER=$FC -DOASIS_SRC=${OASIS_SRC} + -DeCLM="ON" -DeCLM_SRC=${eCLM_SRC} + -DParFlow="ON" -DPARFLOW_SRC=${PARFLOW_SRC} - name: Build TSMP2 From 79659237b866266ed5a25e0b4cd09c0e2a646269 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 9 Oct 2024 15:41:57 +0200 Subject: [PATCH 10/71] CI.yml: Set SYSTEMNAME and eCLM_TAG --- .github/workflows/CI.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 88f50c6..c291f9c 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -23,11 +23,13 @@ jobs: OASIS_SRC: oasis3-mct OASIS_TAG: tsmp-patches-v0.1 eCLM_SRC: eCLM + eCLM_TAG: beta-0.1 PARFLOW_SRC: parflow PARFLOW_TAG: v3.12.0 HYPRE_TAG: 2.26.0 CMAKE_BUILD_PARALLEL_LEVEL: 4 CMAKE_PREFIX_PATH: $HOME/.local + SYSTEMNAME: ${{ runner.os }} steps: - uses: actions/checkout@v4 @@ -56,7 +58,7 @@ jobs: cd $OASIS_SRC echo "OASIS3-MCT version: $(git describe --tags --always)" - - name: Cache eCLM + - name: Cache eCLM ${{ env.eCLM_TAG }} uses: actions/cache@v4 id: cache-eclm env: @@ -66,9 +68,9 @@ jobs: key: cache-${{ matrix.config.name }} - if: steps.cache-eclm.outputs.cache-hit != 'true' - name: Download eCLM + name: Download eCLM ${{ env.eCLM_TAG }} run: | - git clone https://github.com/HPSCTerrSys/eCLM.git + git clone -b $eCLM_TAG https://github.com/HPSCTerrSys/eCLM.git cd eCLM echo "eCLM version: $(git describe --tags --always)" From 5980881d8a3f8e8802dffc4cc5400b79b7706ed7 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 9 Oct 2024 15:48:13 +0200 Subject: [PATCH 11/71] CI.yml: Used matrix.os instead of runner.os --- .github/workflows/CI.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index c291f9c..f1fa8c3 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -6,10 +6,10 @@ on: [push, pull_request] jobs: eclm_build_job: name: ${{ matrix.config.name }} - runs-on: ubuntu-22.04 - + runs-on: ${{ matrix.os }} strategy: matrix: + os: [ ubuntu-22.04 ] config: - { name: "eCLM-ParFlow", @@ -29,7 +29,7 @@ jobs: HYPRE_TAG: 2.26.0 CMAKE_BUILD_PARALLEL_LEVEL: 4 CMAKE_PREFIX_PATH: $HOME/.local - SYSTEMNAME: ${{ runner.os }} + SYSTEMNAME: ${{ matrix.os }} steps: - uses: actions/checkout@v4 From 1b7a5585ed565419e4fb270bdc6164b0af1d85a5 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 9 Oct 2024 15:56:14 +0200 Subject: [PATCH 12/71] CI.yml: YAML trick to wrap multiline command into a single line --- .github/workflows/CI.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f1fa8c3..8ac5cea 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -109,12 +109,10 @@ jobs: make -j4 install - name: Configure TSMP2 - run: | + run: >- cmake -S . -B $BUILD_DIR -DCMAKE_BUILD_TYPE="RELEASE" -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR - -DCMAKE_C_COMPILER=$CC - -DCMAKE_Fortran_COMPILER=$FC -DOASIS_SRC=${OASIS_SRC} -DeCLM="ON" -DeCLM_SRC=${eCLM_SRC} From 7efb5157c5d38379c3698c357d37bcb9b929a972 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 9 Oct 2024 16:04:28 +0200 Subject: [PATCH 13/71] CI.yml: YAML is sensitive to whitespace... --- .github/workflows/CI.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 8ac5cea..6495ef6 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -110,19 +110,19 @@ jobs: - name: Configure TSMP2 run: >- - cmake -S . -B $BUILD_DIR - -DCMAKE_BUILD_TYPE="RELEASE" - -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR - -DOASIS_SRC=${OASIS_SRC} - -DeCLM="ON" - -DeCLM_SRC=${eCLM_SRC} - -DParFlow="ON" - -DPARFLOW_SRC=${PARFLOW_SRC} + cmake -S . -B ${BUILD_DIR} + -DCMAKE_BUILD_TYPE="RELEASE" + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} + -DOASIS_SRC=${OASIS_SRC} + -DeCLM="ON" + -DeCLM_SRC=${eCLM_SRC} + -DParFlow="ON" + -DPARFLOW_SRC=${PARFLOW_SRC} - name: Build TSMP2 run: | - cmake --build $BUILD_DIR + cmake --build ${BUILD_DIR} - name: Install TSMP2 run: | - cmake --install $BUILD_DIR + cmake --install ${BUILD_DIR} From de3577c43292bf5234ea2fbf44089333c8d962ca Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Fri, 18 Oct 2024 11:18:33 +0200 Subject: [PATCH 14/71] include `CMAKE_CXX_COMPILER_ID` in `BuildPDAF.cmake` In `Stages/2024`, the `CMAKE_CXX_COMPILER_ID` is `IntelLLVM` instead of `Intel`. Without the fix, this leads to ``` CMake Error at cmake/BuildPDAF.cmake:116 (message): Unsupported CMAKE_CXX_COMPILER_ID: IntelLLVM Call Stack (most recent call first): CMakeLists.txt:110 (include) ``` --- cmake/BuildPDAF.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/BuildPDAF.cmake b/cmake/BuildPDAF.cmake index 4e785ae..0fdce11 100644 --- a/cmake/BuildPDAF.cmake +++ b/cmake/BuildPDAF.cmake @@ -73,7 +73,8 @@ list(JOIN PDAF_LINK_LIBS " " PDAF_LINK_LIBS) # Set PDAF_OPTIM for Makefile header # ---------------------------------- -if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") +if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel" + or CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") # using Intel Compiler if (CMAKE_BUILD_TYPE STREQUAL "RELEASE") From 029a8d211e374e02ee4d196741e5e95c899ffb43 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Tue, 29 Oct 2024 12:54:41 +0100 Subject: [PATCH 15/71] Fixed hardcoded compiler-specific OpenMP flag --- cmake/BuildCOSMO.cmake | 3 ++- cmake/BuildOASIS3MCT.cmake | 11 +++++++++-- cmake/BuildParFlow.cmake | 8 ++++---- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/cmake/BuildCOSMO.cmake b/cmake/BuildCOSMO.cmake index fc923df..4bac495 100644 --- a/cmake/BuildCOSMO.cmake +++ b/cmake/BuildCOSMO.cmake @@ -1,5 +1,6 @@ find_package(ecCodes REQUIRED) find_package(NetCDF REQUIRED) +find_package(OpenMP REQUIRED) set(COSMO_BLD_DIR ${CMAKE_BINARY_DIR}/COSMO5_1/bld) set(COSMO_Fopts ${COSMO_BLD_DIR}/Fopts) @@ -15,7 +16,7 @@ file(APPEND ${COSMO_Fopts} "COMFLG4 = $(COMFLG1)\n") file(APPEND ${COSMO_Fopts} "COMFLG = $(COMFLG1)\n") file(APPEND ${COSMO_Fopts} "COMFLG5 = $(COMFLG1)\n") file(APPEND ${COSMO_Fopts} "LDPAR = ${CMAKE_Fortran_COMPILER}\n") -file(APPEND ${COSMO_Fopts} "LDFLG = -qopenmp\n") +file(APPEND ${COSMO_Fopts} "LDFLG = ${OpenMP_Fortran_FLAGS}\n") file(APPEND ${COSMO_Fopts} "PROGRAM = lmparbin\n") file(APPEND ${COSMO_Fopts} "LIBPATH = ${ecCodes_LIBRARIES} ${NetCDF_LIBRARIES} ${OASIS_LIBRARIES}\n") diff --git a/cmake/BuildOASIS3MCT.cmake b/cmake/BuildOASIS3MCT.cmake index d3cc924..9c1ca8f 100644 --- a/cmake/BuildOASIS3MCT.cmake +++ b/cmake/BuildOASIS3MCT.cmake @@ -1,4 +1,5 @@ find_package(NetCDF REQUIRED) +find_package(OpenMP REQUIRED) if (CMAKE_BUILD_TYPE STREQUAL "DEBUG") set(OPTIM "-g") @@ -39,8 +40,14 @@ if(${PDAF}) else() file(APPEND ${OASIS_MAKE_INC} "CPPDEF = -Duse_netCDF -Duse_comm_$(CHAN) -D__VERBOSE -DTREAT_OVERLAY\n") endif() -file(APPEND ${OASIS_MAKE_INC} "FCBASEFLAGS = ${OPTIM} -xCORE-AVX2 -I. -assume byterecl -mt_mpi -qopenmp\n") -file(APPEND ${OASIS_MAKE_INC} "CCBASEFLAGS = ${OPTIM} -qopenmp\n") +if(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") + file(APPEND ${OASIS_MAKE_INC} "FCBASEFLAGS = ${OPTIM} -I. -ffree-line-length-none -fallow-argument-mismatch ${OpenMP_Fortran_FLAGS}\n") +elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "Intel" OR COMPILER STREQUAL "IntelLLVM") + file(APPEND ${OASIS_MAKE_INC} "FCBASEFLAGS = ${OPTIM} -I. -xCORE-AVX2 -assume byterecl -mt_mpi ${OpenMP_Fortran_FLAGS}\n") +else() + message(FATAL_ERROR "Fortran compiler '${CMAKE_Fortran_COMPILER_ID}' is not supported.") +endif() +file(APPEND ${OASIS_MAKE_INC} "CCBASEFLAGS = ${OPTIM} ${OpenMP_Fortran_FLAGS}\n") file(APPEND ${OASIS_MAKE_INC} "MCT_DEBUGFLAG = ${MCT_DEBUGFLAG}\n") file(APPEND ${OASIS_MAKE_INC} "FLIBS = ${NetCDF_LIBRARIES}\n") file(APPEND ${OASIS_MAKE_INC} "INC_DIRS = -I$(ARCHDIR)/include -I$(NETCDF_INCLUDE)\n") diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake index 834b58a..76f7fdb 100644 --- a/cmake/BuildParFlow.cmake +++ b/cmake/BuildParFlow.cmake @@ -1,9 +1,9 @@ -# TODO: Properly implement these flags! -set(PF_CFLAGS "-qopenmp -Wall -Werror") -set(PF_LDFLAGS "-lcudart -lcusparse -lcurand") - find_package(NetCDF REQUIRED) find_package(Hypre REQUIRED) +find_package(OpenMP REQUIRED) + +set(PF_CFLAGS "${OpenMP_Fortran_FLAGS} -Wall -Werror") +set(PF_LDFLAGS "-lcudart -lcusparse -lcurand") if(DEFINED eCLM_SRC) list(APPEND PF_CLM_FLAGS -DPARFLOW_AMPS_LAYER=oas3 From 1d14c8ff8c6816eaaec6981d640bc2cdc667abd7 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Tue, 29 Oct 2024 13:06:58 +0100 Subject: [PATCH 16/71] BuildParFlow: Disabled CUDA linker flags when on GCC-OpenMPI system --- cmake/BuildOASIS3MCT.cmake | 2 +- cmake/BuildParFlow.cmake | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/cmake/BuildOASIS3MCT.cmake b/cmake/BuildOASIS3MCT.cmake index 9c1ca8f..a2f36d6 100644 --- a/cmake/BuildOASIS3MCT.cmake +++ b/cmake/BuildOASIS3MCT.cmake @@ -42,7 +42,7 @@ else() endif() if(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") file(APPEND ${OASIS_MAKE_INC} "FCBASEFLAGS = ${OPTIM} -I. -ffree-line-length-none -fallow-argument-mismatch ${OpenMP_Fortran_FLAGS}\n") -elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "Intel" OR COMPILER STREQUAL "IntelLLVM") +elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "Intel" OR CMAKE_Fortran_COMPILER_ID STREQUAL "IntelLLVM") file(APPEND ${OASIS_MAKE_INC} "FCBASEFLAGS = ${OPTIM} -I. -xCORE-AVX2 -assume byterecl -mt_mpi ${OpenMP_Fortran_FLAGS}\n") else() message(FATAL_ERROR "Fortran compiler '${CMAKE_Fortran_COMPILER_ID}' is not supported.") diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake index 76f7fdb..474e1bb 100644 --- a/cmake/BuildParFlow.cmake +++ b/cmake/BuildParFlow.cmake @@ -2,9 +2,18 @@ find_package(NetCDF REQUIRED) find_package(Hypre REQUIRED) find_package(OpenMP REQUIRED) -set(PF_CFLAGS "${OpenMP_Fortran_FLAGS} -Wall -Werror") -set(PF_LDFLAGS "-lcudart -lcusparse -lcurand") +if(CMAKE_C_COMPILER_ID STREQUAL "GNU") + # Flags were based from https://github.com/parflow/parflow/blob/77316043227b95215744e58fe9005d35145432ab/.github/workflows/linux.yml#L305 + set(PF_CFLAGS "${OpenMP_Fortran_FLAGS} -Wall -Werror -Wno-unused-result -Wno-unused-function") +elseif(CMAKE_C_COMPILER_ID STREQUAL "Intel" OR CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") + set(PF_CFLAGS "${OpenMP_Fortran_FLAGS} -Wall -Werror") + set(PF_LDFLAGS "-lcudart -lcusparse -lcurand") #TODO: These linker flags are specific to JSC system! +else() + message(FATAL_ERROR "C compiler '${CMAKE_C_COMPILER_ID}' is not supported.") +endif() + +set(PF_CFLAGS "${OpenMP_Fortran_FLAGS} ${OpenMP_Fortran_FLAGS}") if(DEFINED eCLM_SRC) list(APPEND PF_CLM_FLAGS -DPARFLOW_AMPS_LAYER=oas3 -DOAS3_ROOT=${OASIS_ROOT} From 6ac4c48f308a474a2cf4d564433958bd5390fc5c Mon Sep 17 00:00:00 2001 From: kvrigor Date: Tue, 29 Oct 2024 16:27:34 +0100 Subject: [PATCH 17/71] Various fixes for building ParFlow in Ubuntu NOTE: Not yet fully fixed! --- cmake/BuildParFlow.cmake | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake index 474e1bb..c929c1a 100644 --- a/cmake/BuildParFlow.cmake +++ b/cmake/BuildParFlow.cmake @@ -1,19 +1,26 @@ find_package(NetCDF REQUIRED) find_package(Hypre REQUIRED) find_package(OpenMP REQUIRED) +find_package(TCL REQUIRED) if(CMAKE_C_COMPILER_ID STREQUAL "GNU") # Flags were based from https://github.com/parflow/parflow/blob/77316043227b95215744e58fe9005d35145432ab/.github/workflows/linux.yml#L305 set(PF_CFLAGS "${OpenMP_Fortran_FLAGS} -Wall -Werror -Wno-unused-result -Wno-unused-function") + set(PF_FFLAGS "-ffree-line-length-none -ffixed-line-length-none -ffree-form") + list(APPEND JSC_FLAGS -DPARFLOW_ENABLE_SLURM=OFF) elseif(CMAKE_C_COMPILER_ID STREQUAL "Intel" OR CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") set(PF_CFLAGS "${OpenMP_Fortran_FLAGS} -Wall -Werror") - set(PF_LDFLAGS "-lcudart -lcusparse -lcurand") #TODO: These linker flags are specific to JSC system! + set(PF_FFLAGS "-free") + + #TODO: These flags are specific to JSC system. These flags must be machine-agnostic! + list(APPEND JSC_FLAGS -DPARFLOW_ENABLE_SLURM=ON) + list(APPEND JSC_FLAGS -DMPIEXEC_EXECUTABLE=${MPIEXEC_EXECUTABLE}) + list(APPEND JSC_FLAGS -DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG}) + list(APPEND JSC_FLAGS -DCMAKE_EXE_LINKER_FLAGS="-lcudart -lcusparse -lcurand") else() message(FATAL_ERROR "C compiler '${CMAKE_C_COMPILER_ID}' is not supported.") endif() - -set(PF_CFLAGS "${OpenMP_Fortran_FLAGS} ${OpenMP_Fortran_FLAGS}") if(DEFINED eCLM_SRC) list(APPEND PF_CLM_FLAGS -DPARFLOW_AMPS_LAYER=oas3 -DOAS3_ROOT=${OASIS_ROOT} @@ -37,16 +44,15 @@ ExternalProject_Add(ParFlow -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} - ${PF_CLM_FLAGS} -DNETCDF_DIR=${NetCDF_ROOT} -DPARFLOW_AMPS_SEQUENTIAL_IO=on -DHYPRE_ROOT=${HYPRE_ROOT} -DPARFLOW_ENABLE_TIMING=TRUE - -DPARFLOW_ENABLE_SLURM=TRUE - -DMPIEXEC_EXECUTABLE=${MPIEXEC_EXECUTABLE} - -DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG} -DCMAKE_C_FLAGS=${PF_CFLAGS} - -DCMAKE_EXE_LINKER_FLAGS=${PF_LDFLAGS} + -DCMAKE_Fortran_FLAGS=${PF_FFLAGS} + -DTCL_TCLSH=${TCL_TCLSH} + ${PF_CLM_FLAGS} + ${JSC_FLAGS} DEPENDS ${MODEL_DEPENDENCIES} ) From 3640f41fe9539af13f3e9da5a3accc80aa236d71 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Tue, 29 Oct 2024 16:28:23 +0100 Subject: [PATCH 18/71] CI.yml: Update cache location --- .github/workflows/CI.yml | 52 +++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 6495ef6..381a4bf 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -20,83 +20,85 @@ jobs: FC: mpifort BUILD_DIR: bld INSTALL_DIR: install - OASIS_SRC: oasis3-mct + MODEL_DIR: component-models OASIS_TAG: tsmp-patches-v0.1 - eCLM_SRC: eCLM eCLM_TAG: beta-0.1 - PARFLOW_SRC: parflow - PARFLOW_TAG: v3.12.0 + PARFLOW_TAG: v3.13.0 HYPRE_TAG: 2.26.0 CMAKE_BUILD_PARALLEL_LEVEL: 4 - CMAKE_PREFIX_PATH: $HOME/.local + CMAKE_PREFIX_PATH: dependencies SYSTEMNAME: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: awalsh128/cache-apt-pkgs-action@latest with: - packages: libxml2-utils pylint wget cmake netcdf-bin libnetcdf-dev libnetcdff-dev libpnetcdf-dev + packages: libxml2-utils pylint wget cmake netcdf-bin libnetcdf-mpi-dev libnetcdff-dev libpnetcdf-dev tcl-dev tk-dev version: 1.0 execute_install_scripts: true - name: Download MPI Fortran compiler run: sudo apt-get install gfortran openmpi-bin libopenmpi-dev - - name: Cache OASIS3-MCT ${{ env.OASIS_TAG }} + - name: Create directories for component model and dependencies + run: | + mkdir -p ${CMAKE_PREFIX_PATH} ${MODEL_DIR} + + - name: Restore cached OASIS3-MCT ${{ env.OASIS_TAG }} uses: actions/cache@v4 id: cache-oasis env: cache-name: cache-oasis with: - path: "~/$OASIS_SRC" + path: ${{ env.MODEL_DIR }}/oasis3-mct key: cache-${{ matrix.config.name }}-${{ env.OASIS_TAG }} - if: steps.cache-oasis.outputs.cache-hit != 'true' name: Download OASIS3-MCT ${{ env.OASIS_TAG }} + working-directory: ${{ env.MODEL_DIR }} run: | git clone -b $OASIS_TAG https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/oasis3-mct.git - cd $OASIS_SRC - echo "OASIS3-MCT version: $(git describe --tags --always)" + echo "OASIS3-MCT version: $(cd oasis3-mct && git describe --tags --always)" - - name: Cache eCLM ${{ env.eCLM_TAG }} + - name: Restore cached eCLM ${{ env.eCLM_TAG }} uses: actions/cache@v4 id: cache-eclm env: cache-name: cache-eclm with: - path: "~/$eCLM_SRC" + path: ${{ env.MODEL_DIR }}/eCLM key: cache-${{ matrix.config.name }} - if: steps.cache-eclm.outputs.cache-hit != 'true' name: Download eCLM ${{ env.eCLM_TAG }} + working-directory: ${{ env.MODEL_DIR }} run: | git clone -b $eCLM_TAG https://github.com/HPSCTerrSys/eCLM.git - cd eCLM - echo "eCLM version: $(git describe --tags --always)" + echo "eCLM version: $(cd eCLM && git describe --tags --always)" - - name: Cache ParFlow ${{ env.PARFLOW_TAG }} + - name: Restore cached ParFlow ${{ env.PARFLOW_TAG }} uses: actions/cache@v4 id: cache-parflow env: cache-name: cache-parflow with: - path: "~/$PARFLOW_SRC" + path: ${{ env.MODEL_DIR }}/parflow key: cache-${{ matrix.config.name }}-${{ env.PARFLOW_TAG }} - if: steps.cache-parflow.outputs.cache-hit != 'true' name: Download ParFlow ${{ env.PARFLOW_TAG }} + working-directory: ${{ env.MODEL_DIR }} run: | git clone -b $PARFLOW_TAG https://github.com/parflow/parflow.git - cd parflow - echo "ParFlow version: $(git describe --tags --always)" + echo "ParFlow version: $(cd parflow && git describe --tags --always)" - - name: Cache Hypre ${{ env.HYPRE_TAG }} + - name: Restore cached Hypre ${{ env.HYPRE_TAG }} uses: actions/cache@v4 id: cache-hypre env: cache-name: cache-hypre with: - path: "~/$CMAKE_PREFIX_PATH" + path: ${{ env.CMAKE_PREFIX_PATH }} key: cache-${{ matrix.config.name }}-${{ env.HYPRE_TAG }} - if: steps.cache-hypre.outputs.cache-hit != 'true' @@ -105,19 +107,19 @@ jobs: wget https://github.com/hypre-space/hypre/archive/v${HYPRE_TAG}.tar.gz tar xf v${HYPRE_TAG}.tar.gz cd hypre-${HYPRE_TAG}/src - ./configure --prefix=$CMAKE_PREFIX_PATH - make -j4 install + ./configure --prefix=${CMAKE_PREFIX_PATH} + make -j${CMAKE_BUILD_PARALLEL_LEVEL} install - name: Configure TSMP2 run: >- cmake -S . -B ${BUILD_DIR} -DCMAKE_BUILD_TYPE="RELEASE" -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} - -DOASIS_SRC=${OASIS_SRC} + -DOASIS_SRC=${MODEL_DIR}/oasis3-mct -DeCLM="ON" - -DeCLM_SRC=${eCLM_SRC} + -DeCLM_SRC=${MODEL_DIR}/eCLM -DParFlow="ON" - -DPARFLOW_SRC=${PARFLOW_SRC} + -DPARFLOW_SRC=${MODEL_DIR}/parflow - name: Build TSMP2 run: | From baf1c315f08d69c8ee8a549f8c8959eaf0a319df Mon Sep 17 00:00:00 2001 From: kvrigor Date: Tue, 29 Oct 2024 16:29:11 +0100 Subject: [PATCH 19/71] README: Set default stage to 2024; added tag when cloning eCLM --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 09fb098..9b5c57e 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ cd $TSMP2_DIR 2. Load the environment variables required for the build. ```bash -source env/jsc.2023_Intel.sh +source env/jsc.2024_Intel.sh ``` 3. Specify build and install directories. @@ -81,7 +81,7 @@ mkdir -p ${BUILD_DIR} ${INSTALL_DIR} ## NOTE: Download only the component models that you need! ## # eCLM -git clone https://github.com/HPSCTerrSys/eCLM.git models/eCLM +git clone -b beta-0.1 https://github.com/HPSCTerrSys/eCLM.git models/eCLM eCLM_SRC=`realpath models/eCLM` # ICON From 2ae0b48742cd8d7cee66d2220ceca4487dcd634c Mon Sep 17 00:00:00 2001 From: kvrigor Date: Tue, 29 Oct 2024 16:40:41 +0100 Subject: [PATCH 20/71] Set absolute path to Hypre install dir --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 381a4bf..d01b38c 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -107,7 +107,7 @@ jobs: wget https://github.com/hypre-space/hypre/archive/v${HYPRE_TAG}.tar.gz tar xf v${HYPRE_TAG}.tar.gz cd hypre-${HYPRE_TAG}/src - ./configure --prefix=${CMAKE_PREFIX_PATH} + ./configure --prefix=$(realpath $CMAKE_PREFIX_PATH) make -j${CMAKE_BUILD_PARALLEL_LEVEL} install - name: Configure TSMP2 From e4bf44751bb944ae268498d87b8580a82cb9a530 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 30 Oct 2024 07:54:27 +0100 Subject: [PATCH 21/71] Check model and dependency directories --- .github/workflows/CI.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d01b38c..f8983d7 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -59,6 +59,7 @@ jobs: run: | git clone -b $OASIS_TAG https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/oasis3-mct.git echo "OASIS3-MCT version: $(cd oasis3-mct && git describe --tags --always)" + realpath oasis3-mct - name: Restore cached eCLM ${{ env.eCLM_TAG }} uses: actions/cache@v4 @@ -75,6 +76,7 @@ jobs: run: | git clone -b $eCLM_TAG https://github.com/HPSCTerrSys/eCLM.git echo "eCLM version: $(cd eCLM && git describe --tags --always)" + realpath eCLM - name: Restore cached ParFlow ${{ env.PARFLOW_TAG }} uses: actions/cache@v4 @@ -91,6 +93,7 @@ jobs: run: | git clone -b $PARFLOW_TAG https://github.com/parflow/parflow.git echo "ParFlow version: $(cd parflow && git describe --tags --always)" + realpath parflow - name: Restore cached Hypre ${{ env.HYPRE_TAG }} uses: actions/cache@v4 @@ -110,6 +113,11 @@ jobs: ./configure --prefix=$(realpath $CMAKE_PREFIX_PATH) make -j${CMAKE_BUILD_PARALLEL_LEVEL} install + - name: Check for component model and dependency directories + run: | + tree ${CMAKE_PREFIX_PATH} + tree ${MODEL_DIR} + - name: Configure TSMP2 run: >- cmake -S . -B ${BUILD_DIR} From f9dc2e2e355e552713d0dea7146c4b3128a2e217 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 30 Oct 2024 10:55:53 +0100 Subject: [PATCH 22/71] FindNetCDF.cmake and CI.yml fixes FindNetCDF: - Split NetCDF_ROOT into NetCDF_F90_ROOT and NetCDF_C_ROOT. C and Fortran NetCDF root directories can differ; moreover there could be multiple NetCDF C implementations installed on a system, e.g. serial NetCDF and parallel NetCDF. - In particular, ParFlow requires parallel NetCDF header (`netcdf_par.h`) CI: - Forced cache component model repos - Specified full path to dependencies and component models --- .github/workflows/CI.yml | 84 ++++++++++++++++++++++++++------------ cmake/BuildPDAFMODEL.cmake | 2 +- cmake/BuildParFlow.cmake | 2 +- cmake/FindNetCDF.cmake | 26 ++++++++++-- 4 files changed, 82 insertions(+), 32 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f8983d7..ff2ea1c 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -20,13 +20,13 @@ jobs: FC: mpifort BUILD_DIR: bld INSTALL_DIR: install - MODEL_DIR: component-models + MODEL_DIR: ${{ github.workspace }}/component-models OASIS_TAG: tsmp-patches-v0.1 eCLM_TAG: beta-0.1 PARFLOW_TAG: v3.13.0 HYPRE_TAG: 2.26.0 CMAKE_BUILD_PARALLEL_LEVEL: 4 - CMAKE_PREFIX_PATH: dependencies + CMAKE_PREFIX_PATH: ${{ github.workspace }}/dependencies SYSTEMNAME: ${{ matrix.os }} steps: @@ -44,16 +44,17 @@ jobs: run: | mkdir -p ${CMAKE_PREFIX_PATH} ${MODEL_DIR} + # + # OASIS3-MCT + # - name: Restore cached OASIS3-MCT ${{ env.OASIS_TAG }} - uses: actions/cache@v4 - id: cache-oasis - env: - cache-name: cache-oasis + uses: actions/cache/restore@v4 + id: cache-oasis-restore with: path: ${{ env.MODEL_DIR }}/oasis3-mct key: cache-${{ matrix.config.name }}-${{ env.OASIS_TAG }} - - if: steps.cache-oasis.outputs.cache-hit != 'true' + - if: steps.cache-oasis-restore.outputs.cache-hit != 'true' name: Download OASIS3-MCT ${{ env.OASIS_TAG }} working-directory: ${{ env.MODEL_DIR }} run: | @@ -61,16 +62,24 @@ jobs: echo "OASIS3-MCT version: $(cd oasis3-mct && git describe --tags --always)" realpath oasis3-mct + - if: steps.cache-oasis-restore.outputs.cache-hit != 'true' + name: Cache OASIS3-MCT ${{ env.OASIS_TAG }} + uses: actions/cache/save@v4 + with: + path: ${{ env.MODEL_DIR }}/oasis3-mct + key: cache-${{ matrix.config.name }}-${{ env.OASIS_TAG }} + + # + # eCLM + # - name: Restore cached eCLM ${{ env.eCLM_TAG }} - uses: actions/cache@v4 - id: cache-eclm - env: - cache-name: cache-eclm + uses: actions/cache/restore@v4 + id: cache-eclm-restore with: path: ${{ env.MODEL_DIR }}/eCLM - key: cache-${{ matrix.config.name }} + key: cache-${{ matrix.config.name }}-${{ env.eCLM_TAG }} - - if: steps.cache-eclm.outputs.cache-hit != 'true' + - if: steps.cache-eclm-restore.outputs.cache-hit != 'true' name: Download eCLM ${{ env.eCLM_TAG }} working-directory: ${{ env.MODEL_DIR }} run: | @@ -78,16 +87,24 @@ jobs: echo "eCLM version: $(cd eCLM && git describe --tags --always)" realpath eCLM + - if: steps.cache-eclm-restore.outputs.cache-hit != 'true' + name: Cache eCLM ${{ env.eCLM_TAG }} + uses: actions/cache/save@v4 + with: + path: ${{ env.MODEL_DIR }}/eCLM + key: cache-${{ matrix.config.name }}-${{ env.eCLM_TAG }} + + # + # ParFlow + # - name: Restore cached ParFlow ${{ env.PARFLOW_TAG }} - uses: actions/cache@v4 - id: cache-parflow - env: - cache-name: cache-parflow + uses: actions/cache/restore@v4 + id: cache-parflow-restore with: path: ${{ env.MODEL_DIR }}/parflow key: cache-${{ matrix.config.name }}-${{ env.PARFLOW_TAG }} - - if: steps.cache-parflow.outputs.cache-hit != 'true' + - if: steps.cache-parflow-restore.outputs.cache-hit != 'true' name: Download ParFlow ${{ env.PARFLOW_TAG }} working-directory: ${{ env.MODEL_DIR }} run: | @@ -95,16 +112,24 @@ jobs: echo "ParFlow version: $(cd parflow && git describe --tags --always)" realpath parflow + - if: steps.cache-parflow-restore.outputs.cache-hit != 'true' + name: Cache ParFlow ${{ env.PARFLOW_TAG }} + uses: actions/cache/save@v4 + with: + path: ${{ env.MODEL_DIR }}/parflow + key: cache-${{ matrix.config.name }}-${{ env.PARFLOW_TAG }} + + # + # Hypre + # - name: Restore cached Hypre ${{ env.HYPRE_TAG }} - uses: actions/cache@v4 - id: cache-hypre - env: - cache-name: cache-hypre + uses: actions/cache/restore@v4 + id: cache-hypre-restore with: path: ${{ env.CMAKE_PREFIX_PATH }} key: cache-${{ matrix.config.name }}-${{ env.HYPRE_TAG }} - - if: steps.cache-hypre.outputs.cache-hit != 'true' + - if: steps.cache-hypre-restore.outputs.cache-hit != 'true' name: Install Hypre ${{ env.HYPRE_TAG }} run: | wget https://github.com/hypre-space/hypre/archive/v${HYPRE_TAG}.tar.gz @@ -113,11 +138,16 @@ jobs: ./configure --prefix=$(realpath $CMAKE_PREFIX_PATH) make -j${CMAKE_BUILD_PARALLEL_LEVEL} install - - name: Check for component model and dependency directories - run: | - tree ${CMAKE_PREFIX_PATH} - tree ${MODEL_DIR} + - if: steps.cache-hypre-restore.outputs.cache-hit != 'true' + name: Cache Hypre ${{ env.HYPRE_TAG }} + uses: actions/cache/save@v4 + with: + path: ${{ env.CMAKE_PREFIX_PATH }} + key: cache-${{ matrix.config.name }}-${{ env.HYPRE_TAG }} + # + # TSMP2 + # - name: Configure TSMP2 run: >- cmake -S . -B ${BUILD_DIR} diff --git a/cmake/BuildPDAFMODEL.cmake b/cmake/BuildPDAFMODEL.cmake index f730933..81fe91b 100644 --- a/cmake/BuildPDAFMODEL.cmake +++ b/cmake/BuildPDAFMODEL.cmake @@ -17,7 +17,7 @@ set(TSMPPDAFLIBDIR "${CMAKE_INSTALL_PREFIX}/lib") # Include directories # ------------------- # General include dirs -list(APPEND PDAF_INCLUDES "-I${NetCDF_ROOT}/include") +list(APPEND PDAF_INCLUDES "-I${NetCDF_F90_ROOT}/include") # DA include dirs list(APPEND PDAF_INCLUDES "-I${PDAF_SRC}/interface/model") diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake index c929c1a..9f5a1cb 100644 --- a/cmake/BuildParFlow.cmake +++ b/cmake/BuildParFlow.cmake @@ -44,7 +44,7 @@ ExternalProject_Add(ParFlow -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} - -DNETCDF_DIR=${NetCDF_ROOT} + -DNETCDF_DIR=${NetCDF_C_ROOT} -DPARFLOW_AMPS_SEQUENTIAL_IO=on -DHYPRE_ROOT=${HYPRE_ROOT} -DPARFLOW_ENABLE_TIMING=TRUE diff --git a/cmake/FindNetCDF.cmake b/cmake/FindNetCDF.cmake index 80b8ac4..db04742 100644 --- a/cmake/FindNetCDF.cmake +++ b/cmake/FindNetCDF.cmake @@ -3,20 +3,40 @@ include(FindPackageHandleStandardArgs) find_package(NetCDF_C QUIET NAMES netCDF) if(NetCDF_C_FOUND) + # Set default NetCDF C paths to the first found NetCDF C library. set(NetCDF_C_LIB_DIR "${netCDF_LIB_DIR}") + set(NetCDF_C_ROOT "${netCDF_INSTALL_PREFIX}") + + # However, if the default NetCDF C library doesn't have parallel support, try searching for another one. + if(NOT ${netCDF_HAS_PARALLEL}) + # Option 1 (Ubuntu-specific): `sudo apt-get install libnetcdf-mpi-dev` provides NetCDF with parallel support. + pkg_check_modules(NetCDF_C_MPI QUIET netcdf-mpi) + if (NetCDF_C_MPI_FOUND) + pkg_get_variable(NetCDF_C_ROOT netcdf-mpi prefix) + set(NetCDF_C_LIB_DIR "${NetCDF_C_ROOT}/lib") + set(netCDF_HAS_PARALLEL "ON") + endif() + endif() endif() pkg_check_modules(NetCDF_F90 REQUIRED netcdf-fortran) if (NetCDF_F90_FOUND) - pkg_get_variable(NetCDF_F90_LIB_DIR netcdf-fortran libdir) pkg_get_variable(NetCDF_F90_ROOT netcdf-fortran prefix) + pkg_get_variable(NetCDF_F90_LIB_DIR netcdf-fortran libdir) + pkg_get_variable(NetCDF_F90_INCLUDEDIR netcdf-fortran fmoddir) endif() find_package_handle_standard_args(NetCDF - REQUIRED_VARS NetCDF_C_LIB_DIR NetCDF_F90_LIB_DIR + REQUIRED_VARS NetCDF_C_ROOT NetCDF_F90_ROOT NetCDF_C_LIB_DIR NetCDF_F90_LIB_DIR NetCDF_F90_INCLUDEDIR VERSION_VAR NetCDF_F90_VERSION) if(NetCDF_FOUND) set(NetCDF_LIBRARIES "-L${NetCDF_C_LIB_DIR} -L${NetCDF_F90_LIB_DIR} -lnetcdff -lnetcdf" CACHE STRING "NetCDF linker options") - set(NetCDF_ROOT ${NetCDF_F90_ROOT} CACHE PATH "Full path to the root directory containing NetCDF include files and libraries.") + set(NetCDF_F90_ROOT ${NetCDF_F90_ROOT} CACHE PATH "Path to NetCDF-Fortran directory which contains its include header files and libraries.") + set(NetCDF_C_ROOT ${NetCDF_C_ROOT} CACHE PATH "Path to NetCDF-C directory which contains its include header files and libraries.") + if (netCDF_HAS_PARALLEL) + message(STATUS "TSMP2 found NetCDF C built with parallel I/O support.") + else() + message(WARNING "TSMP2 is using NetCDF C without parallel I/O support.") + endif() endif() \ No newline at end of file From 816c0e4230bd8b098c5e8ab628c4a0cfcb2ff5a9 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 30 Oct 2024 12:58:55 +0100 Subject: [PATCH 23/71] More NetCDF fixes BuildParFlow.cmake - NETCDF_INCLUDE_DIR and NETCDF_LIBRARY aren't properly set even NETCDF_DIR is supplied. So we set them explicitly. - Reordered options for readability BuildOASIS3MCT.cmake - Forgot to change NetCDF_ROOT to NetCDF_F90_ROOT --- .github/workflows/CI.yml | 2 ++ cmake/BuildOASIS3MCT.cmake | 2 +- cmake/BuildParFlow.cmake | 54 ++++++++++++++++++++++---------------- 3 files changed, 34 insertions(+), 24 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ff2ea1c..dcdce0a 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -17,7 +17,9 @@ jobs: env: CC: mpicc + CXX: mpicxx FC: mpifort + F77: mpif77 BUILD_DIR: bld INSTALL_DIR: install MODEL_DIR: ${{ github.workspace }}/component-models diff --git a/cmake/BuildOASIS3MCT.cmake b/cmake/BuildOASIS3MCT.cmake index a2f36d6..c66840f 100644 --- a/cmake/BuildOASIS3MCT.cmake +++ b/cmake/BuildOASIS3MCT.cmake @@ -21,7 +21,7 @@ file(WRITE ${OASIS_MAKE_INC} "CHAN = MPI1\n") file(APPEND ${OASIS_MAKE_INC} "COUPLE = ${OASIS_SRC}\n") file(APPEND ${OASIS_MAKE_INC} "BUILD_DIR = ${OASIS_BLD_DIR}\n") file(APPEND ${OASIS_MAKE_INC} "ARCHDIR = ${OASIS_INSTALL_PREFIX}\n") -file(APPEND ${OASIS_MAKE_INC} "NETCDF_INCLUDE = ${NetCDF_ROOT}/include\n") +file(APPEND ${OASIS_MAKE_INC} "NETCDF_INCLUDE = ${NetCDF_F90_ROOT}/include\n") file(APPEND ${OASIS_MAKE_INC} "NETCDF_LIBRARY = ${NetCDF_LIBRARIES}\n") file(APPEND ${OASIS_MAKE_INC} "MPI_INCLUDE = ${MPI_Fortran_INCLUDE_DIRS}\n") file(APPEND ${OASIS_MAKE_INC} "MAKE = make\n") diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake index 9f5a1cb..fb4565f 100644 --- a/cmake/BuildParFlow.cmake +++ b/cmake/BuildParFlow.cmake @@ -3,24 +3,9 @@ find_package(Hypre REQUIRED) find_package(OpenMP REQUIRED) find_package(TCL REQUIRED) -if(CMAKE_C_COMPILER_ID STREQUAL "GNU") - # Flags were based from https://github.com/parflow/parflow/blob/77316043227b95215744e58fe9005d35145432ab/.github/workflows/linux.yml#L305 - set(PF_CFLAGS "${OpenMP_Fortran_FLAGS} -Wall -Werror -Wno-unused-result -Wno-unused-function") - set(PF_FFLAGS "-ffree-line-length-none -ffixed-line-length-none -ffree-form") - list(APPEND JSC_FLAGS -DPARFLOW_ENABLE_SLURM=OFF) -elseif(CMAKE_C_COMPILER_ID STREQUAL "Intel" OR CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") - set(PF_CFLAGS "${OpenMP_Fortran_FLAGS} -Wall -Werror") - set(PF_FFLAGS "-free") - - #TODO: These flags are specific to JSC system. These flags must be machine-agnostic! - list(APPEND JSC_FLAGS -DPARFLOW_ENABLE_SLURM=ON) - list(APPEND JSC_FLAGS -DMPIEXEC_EXECUTABLE=${MPIEXEC_EXECUTABLE}) - list(APPEND JSC_FLAGS -DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG}) - list(APPEND JSC_FLAGS -DCMAKE_EXE_LINKER_FLAGS="-lcudart -lcusparse -lcurand") -else() - message(FATAL_ERROR "C compiler '${CMAKE_C_COMPILER_ID}' is not supported.") -endif() - +# +# Choose which CLM to enable +# if(DEFINED eCLM_SRC) list(APPEND PF_CLM_FLAGS -DPARFLOW_AMPS_LAYER=oas3 -DOAS3_ROOT=${OASIS_ROOT} @@ -35,22 +20,45 @@ else() -DPARFLOW_HAVE_CLM=ON) endif() +# # TODO: Add compile switches for ParFlow GPU +# +if(CMAKE_C_COMPILER_ID STREQUAL "GNU") + # Flags were based from https://github.com/parflow/parflow/blob/77316043227b95215744e58fe9005d35145432ab/.github/workflows/linux.yml#L305 + set(PF_CFLAGS "${OpenMP_Fortran_FLAGS} -Wall -Werror -Wno-unused-result -Wno-unused-function") + #set(PF_FFLAGS "-ffree-line-length-none -ffixed-line-length-none -ffree-form") +elseif(CMAKE_C_COMPILER_ID STREQUAL "Intel" OR CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") + set(PF_CFLAGS "${OpenMP_Fortran_FLAGS} -Wall -Werror") + #set(PF_FFLAGS "-free") + + #TODO: These flags are specific to JSC system. This should be set in an env or build script! + list(APPEND JSC_FLAGS -DPARFLOW_ENABLE_SLURM=ON) + list(APPEND JSC_FLAGS -DCMAKE_EXE_LINKER_FLAGS="-lcudart -lcusparse -lcurand") +else() + message(FATAL_ERROR "C compiler '${CMAKE_C_COMPILER_ID}' is not supported.") +endif() + +# +# Pass options to ParFlow CMake +# ExternalProject_Add(ParFlow PREFIX ParFlow SOURCE_DIR ${PARFLOW_SRC} CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_C_FLAGS=${PF_CFLAGS} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} + -DCMAKE_Fortran_FLAGS=${PF_FFLAGS} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} - -DNETCDF_DIR=${NetCDF_C_ROOT} - -DPARFLOW_AMPS_SEQUENTIAL_IO=on + -DTCL_TCLSH=${TCL_TCLSH} -DHYPRE_ROOT=${HYPRE_ROOT} + -DNETCDF_INCLUDE_DIR=${NetCDF_C_ROOT}/include + -DNETCDF_LIBRARY=${NetCDF_C_ROOT}/lib + -DPARFLOW_AMPS_SEQUENTIAL_IO=on -DPARFLOW_ENABLE_TIMING=TRUE - -DCMAKE_C_FLAGS=${PF_CFLAGS} - -DCMAKE_Fortran_FLAGS=${PF_FFLAGS} - -DTCL_TCLSH=${TCL_TCLSH} + -DMPIEXEC_EXECUTABLE=${MPIEXEC_EXECUTABLE} + -DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG} ${PF_CLM_FLAGS} ${JSC_FLAGS} DEPENDS ${MODEL_DEPENDENCIES} From 134c7899e5f14323e59c91bd05fb60149f57412d Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 30 Oct 2024 13:08:25 +0100 Subject: [PATCH 24/71] Reenabled free/fixed flags and disabled free-form flag - `-ffree-line-length-none -ffixed-line-length-none` required for F90 source files - `-ffree-form` must be disabled to correctly compile F77 (`.F`) source files --- cmake/BuildParFlow.cmake | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake index fb4565f..5ffda52 100644 --- a/cmake/BuildParFlow.cmake +++ b/cmake/BuildParFlow.cmake @@ -26,11 +26,9 @@ endif() if(CMAKE_C_COMPILER_ID STREQUAL "GNU") # Flags were based from https://github.com/parflow/parflow/blob/77316043227b95215744e58fe9005d35145432ab/.github/workflows/linux.yml#L305 set(PF_CFLAGS "${OpenMP_Fortran_FLAGS} -Wall -Werror -Wno-unused-result -Wno-unused-function") - #set(PF_FFLAGS "-ffree-line-length-none -ffixed-line-length-none -ffree-form") + set(PF_FFLAGS "-ffree-line-length-none -ffixed-line-length-none") elseif(CMAKE_C_COMPILER_ID STREQUAL "Intel" OR CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") set(PF_CFLAGS "${OpenMP_Fortran_FLAGS} -Wall -Werror") - #set(PF_FFLAGS "-free") - #TODO: These flags are specific to JSC system. This should be set in an env or build script! list(APPEND JSC_FLAGS -DPARFLOW_ENABLE_SLURM=ON) list(APPEND JSC_FLAGS -DCMAKE_EXE_LINKER_FLAGS="-lcudart -lcusparse -lcurand") From 03c8c8b6333e90ce607d5f087e71a38e814d176d Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 30 Oct 2024 13:20:52 +0100 Subject: [PATCH 25/71] Disabled parallel build Parallel build jumbles the build log. --- .github/workflows/CI.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index dcdce0a..76003b5 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -27,7 +27,6 @@ jobs: eCLM_TAG: beta-0.1 PARFLOW_TAG: v3.13.0 HYPRE_TAG: 2.26.0 - CMAKE_BUILD_PARALLEL_LEVEL: 4 CMAKE_PREFIX_PATH: ${{ github.workspace }}/dependencies SYSTEMNAME: ${{ matrix.os }} @@ -138,7 +137,7 @@ jobs: tar xf v${HYPRE_TAG}.tar.gz cd hypre-${HYPRE_TAG}/src ./configure --prefix=$(realpath $CMAKE_PREFIX_PATH) - make -j${CMAKE_BUILD_PARALLEL_LEVEL} install + make -j4 install - if: steps.cache-hypre-restore.outputs.cache-hit != 'true' name: Cache Hypre ${{ env.HYPRE_TAG }} From a983f30f0377dfbe5a9f61e75b4e2d27fcf97689 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 30 Oct 2024 13:23:35 +0100 Subject: [PATCH 26/71] BuildParFlow: NETCDF_LIBRARY actually requires a library file, not a directory --- cmake/BuildParFlow.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake index 5ffda52..c397cb1 100644 --- a/cmake/BuildParFlow.cmake +++ b/cmake/BuildParFlow.cmake @@ -52,7 +52,7 @@ ExternalProject_Add(ParFlow -DTCL_TCLSH=${TCL_TCLSH} -DHYPRE_ROOT=${HYPRE_ROOT} -DNETCDF_INCLUDE_DIR=${NetCDF_C_ROOT}/include - -DNETCDF_LIBRARY=${NetCDF_C_ROOT}/lib + -DNETCDF_LIBRARY=${NetCDF_C_LIB_DIR}/libnetcdf.so # TODO: Fix this hardcoded library file! -DPARFLOW_AMPS_SEQUENTIAL_IO=on -DPARFLOW_ENABLE_TIMING=TRUE -DMPIEXEC_EXECUTABLE=${MPIEXEC_EXECUTABLE} From 5dbfa7fe9da9cfc36ea831aafcdea6c712a4e142 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 30 Oct 2024 13:51:35 +0100 Subject: [PATCH 27/71] AMPS_CPU_TICKS_PER_SEC fix in kvrigor/parflow --- .github/workflows/CI.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 76003b5..7e1e0eb 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -25,7 +25,7 @@ jobs: MODEL_DIR: ${{ github.workspace }}/component-models OASIS_TAG: tsmp-patches-v0.1 eCLM_TAG: beta-0.1 - PARFLOW_TAG: v3.13.0 + PARFLOW_TAG: oasis3-ubuntu22.04 #v3.13.0 HYPRE_TAG: 2.26.0 CMAKE_PREFIX_PATH: ${{ github.workspace }}/dependencies SYSTEMNAME: ${{ matrix.os }} @@ -61,7 +61,6 @@ jobs: run: | git clone -b $OASIS_TAG https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/oasis3-mct.git echo "OASIS3-MCT version: $(cd oasis3-mct && git describe --tags --always)" - realpath oasis3-mct - if: steps.cache-oasis-restore.outputs.cache-hit != 'true' name: Cache OASIS3-MCT ${{ env.OASIS_TAG }} @@ -86,7 +85,6 @@ jobs: run: | git clone -b $eCLM_TAG https://github.com/HPSCTerrSys/eCLM.git echo "eCLM version: $(cd eCLM && git describe --tags --always)" - realpath eCLM - if: steps.cache-eclm-restore.outputs.cache-hit != 'true' name: Cache eCLM ${{ env.eCLM_TAG }} @@ -105,13 +103,13 @@ jobs: path: ${{ env.MODEL_DIR }}/parflow key: cache-${{ matrix.config.name }}-${{ env.PARFLOW_TAG }} + # TODO: Must use upstream repo https://github.com/parflow/parflow.git - if: steps.cache-parflow-restore.outputs.cache-hit != 'true' name: Download ParFlow ${{ env.PARFLOW_TAG }} working-directory: ${{ env.MODEL_DIR }} run: | - git clone -b $PARFLOW_TAG https://github.com/parflow/parflow.git + git clone -b $PARFLOW_TAG https://github.com/kvrigor/parflow.git echo "ParFlow version: $(cd parflow && git describe --tags --always)" - realpath parflow - if: steps.cache-parflow-restore.outputs.cache-hit != 'true' name: Cache ParFlow ${{ env.PARFLOW_TAG }} From 4389d6c277458c02a8105bb15fa60b95670c1b3a Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 30 Oct 2024 14:12:34 +0100 Subject: [PATCH 28/71] Reinstall all Ubuntu apt packages --- .github/workflows/CI.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 7e1e0eb..542cc1e 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -32,14 +32,11 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: awalsh128/cache-apt-pkgs-action@latest - with: - packages: libxml2-utils pylint wget cmake netcdf-bin libnetcdf-mpi-dev libnetcdff-dev libpnetcdf-dev tcl-dev tk-dev - version: 1.0 - execute_install_scripts: true - - name: Download MPI Fortran compiler - run: sudo apt-get install gfortran openmpi-bin libopenmpi-dev + - name: Install TSMP2 dependencies on Ubuntu + run: | + sudo apt-get update + sudo apt-get install libxml2-utils wget gfortran cmake openmpi-bin libopenmpi-dev libnetcdf-mpi-dev libnetcdff-dev libpnetcdf-dev tcl-dev tk-dev - name: Create directories for component model and dependencies run: | From dce8160d1f7d951bcea9003f97ed698c04d20cc2 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 30 Oct 2024 14:42:11 +0100 Subject: [PATCH 29/71] NetCDF fixes on ParFlow FindNetCDF.cmake - `NETCDF_C_ROOT` still pointing to serial version of NetCDF. Changed the logical structure such that `NETCDF_C_ROOT` will always point to the parallel-aware NetCDF if it is found. BuildParFlow.cmake - Removed `find_package(TCL)`. This is already done inside ParFlow. --- cmake/BuildParFlow.cmake | 2 -- cmake/FindNetCDF.cmake | 15 +++++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake index c397cb1..0c8fff3 100644 --- a/cmake/BuildParFlow.cmake +++ b/cmake/BuildParFlow.cmake @@ -1,7 +1,6 @@ find_package(NetCDF REQUIRED) find_package(Hypre REQUIRED) find_package(OpenMP REQUIRED) -find_package(TCL REQUIRED) # # Choose which CLM to enable @@ -49,7 +48,6 @@ ExternalProject_Add(ParFlow -DCMAKE_Fortran_FLAGS=${PF_FFLAGS} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} - -DTCL_TCLSH=${TCL_TCLSH} -DHYPRE_ROOT=${HYPRE_ROOT} -DNETCDF_INCLUDE_DIR=${NetCDF_C_ROOT}/include -DNETCDF_LIBRARY=${NetCDF_C_LIB_DIR}/libnetcdf.so # TODO: Fix this hardcoded library file! diff --git a/cmake/FindNetCDF.cmake b/cmake/FindNetCDF.cmake index db04742..c3d18b5 100644 --- a/cmake/FindNetCDF.cmake +++ b/cmake/FindNetCDF.cmake @@ -3,18 +3,21 @@ include(FindPackageHandleStandardArgs) find_package(NetCDF_C QUIET NAMES netCDF) if(NetCDF_C_FOUND) - # Set default NetCDF C paths to the first found NetCDF C library. - set(NetCDF_C_LIB_DIR "${netCDF_LIB_DIR}") - set(NetCDF_C_ROOT "${netCDF_INSTALL_PREFIX}") - - # However, if the default NetCDF C library doesn't have parallel support, try searching for another one. - if(NOT ${netCDF_HAS_PARALLEL}) + if (netCDF_HAS_PARALLEL) + set(NetCDF_C_ROOT "${netCDF_INSTALL_PREFIX}") + set(NetCDF_C_LIB_DIR "${netCDF_LIB_DIR}") + else() + # If the default NetCDF C library doesn't have parallel support, try searching for another one. # Option 1 (Ubuntu-specific): `sudo apt-get install libnetcdf-mpi-dev` provides NetCDF with parallel support. pkg_check_modules(NetCDF_C_MPI QUIET netcdf-mpi) if (NetCDF_C_MPI_FOUND) pkg_get_variable(NetCDF_C_ROOT netcdf-mpi prefix) set(NetCDF_C_LIB_DIR "${NetCDF_C_ROOT}/lib") set(netCDF_HAS_PARALLEL "ON") + else() + # Set default NetCDF C paths to the first found NetCDF C library. + set(NetCDF_C_ROOT "${netCDF_INSTALL_PREFIX}") + set(NetCDF_C_LIB_DIR "${netCDF_LIB_DIR}") endif() endif() endif() From 90e015dd40ffc6b0160b1b03403ee78db996d0fd Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 30 Oct 2024 14:42:51 +0100 Subject: [PATCH 30/71] Re-enabled Ubuntu apt caching --- .github/workflows/CI.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 542cc1e..c55fb50 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -34,9 +34,16 @@ jobs: - uses: actions/checkout@v4 - name: Install TSMP2 dependencies on Ubuntu + uses: awalsh128/cache-apt-pkgs-action@latest + with: + packages: libxml2-utils pylint wget cmake gfortran openmpi-bin libopenmpi-dev libnetcdf-mpi-dev libnetcdff-dev libpnetcdf-dev tcl-dev tk-dev + version: 1.0 + execute_install_scripts: true + + - name: Set default MPI compilers on Ubuntu run: | sudo apt-get update - sudo apt-get install libxml2-utils wget gfortran cmake openmpi-bin libopenmpi-dev libnetcdf-mpi-dev libnetcdff-dev libpnetcdf-dev tcl-dev tk-dev + sudo apt-get install --reinstall openmpi-bin - name: Create directories for component model and dependencies run: | From 5c47ed31f1fe79f09d3b36a3e1fa9f83ed9c3fdf Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 30 Oct 2024 14:52:04 +0100 Subject: [PATCH 31/71] FindNetCDF.cmake: Inject logging statements --- cmake/FindNetCDF.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake/FindNetCDF.cmake b/cmake/FindNetCDF.cmake index c3d18b5..fb52940 100644 --- a/cmake/FindNetCDF.cmake +++ b/cmake/FindNetCDF.cmake @@ -14,6 +14,7 @@ if(NetCDF_C_FOUND) pkg_get_variable(NetCDF_C_ROOT netcdf-mpi prefix) set(NetCDF_C_LIB_DIR "${NetCDF_C_ROOT}/lib") set(netCDF_HAS_PARALLEL "ON") + message(STATUS "FindNETCDF1: NetCDF_C_ROOT=${NetCDF_C_ROOT}") else() # Set default NetCDF C paths to the first found NetCDF C library. set(NetCDF_C_ROOT "${netCDF_INSTALL_PREFIX}") @@ -42,4 +43,5 @@ if(NetCDF_FOUND) else() message(WARNING "TSMP2 is using NetCDF C without parallel I/O support.") endif() + message(STATUS "FindNETCDF2: NetCDF_C_ROOT=${NetCDF_C_ROOT}") endif() \ No newline at end of file From 64801d3e1c80539061c164423d3ef2df7a8293f2 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 30 Oct 2024 14:57:27 +0100 Subject: [PATCH 32/71] Reinstalled MPI fortran compilers --- .github/workflows/CI.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index c55fb50..5293e58 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -36,14 +36,12 @@ jobs: - name: Install TSMP2 dependencies on Ubuntu uses: awalsh128/cache-apt-pkgs-action@latest with: - packages: libxml2-utils pylint wget cmake gfortran openmpi-bin libopenmpi-dev libnetcdf-mpi-dev libnetcdff-dev libpnetcdf-dev tcl-dev tk-dev + packages: libxml2-utils pylint wget cmake libnetcdf-mpi-dev libnetcdff-dev libpnetcdf-dev tcl-dev tk-dev version: 1.0 execute_install_scripts: true - - name: Set default MPI compilers on Ubuntu - run: | - sudo apt-get update - sudo apt-get install --reinstall openmpi-bin + - name: Download MPI Fortran compiler + run: sudo apt-get install gfortran openmpi-bin libopenmpi-dev - name: Create directories for component model and dependencies run: | From 387ebfe4dd97d437f29afd353e22a06c2ff65194 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 30 Oct 2024 15:04:08 +0100 Subject: [PATCH 33/71] Manually verify netcdf-mpi prefix path --- .github/workflows/CI.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5293e58..39e41da 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -1,7 +1,11 @@ name: TSMP2 Build # Controls when the action will run. -on: [push, pull_request] +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] jobs: eclm_build_job: @@ -47,6 +51,11 @@ jobs: run: | mkdir -p ${CMAKE_PREFIX_PATH} ${MODEL_DIR} + - name: Introspect netcdf-mpi + run: | + NETCDF_PREFIX=$(pkg-config --variable=prefix netcdf-mpi) + ls ${NETCDF_PREFIX}/include + # # OASIS3-MCT # From e3241326a63641e57a87a23bdc9d955283a79283 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 30 Oct 2024 15:16:20 +0100 Subject: [PATCH 34/71] Add more NetCDF debugging logs --- .github/workflows/CI.yml | 6 ++++-- cmake/FindNetCDF.cmake | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 39e41da..5659166 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -53,8 +53,10 @@ jobs: - name: Introspect netcdf-mpi run: | - NETCDF_PREFIX=$(pkg-config --variable=prefix netcdf-mpi) - ls ${NETCDF_PREFIX}/include + NETCDF_P_PREFIX=$(pkg-config --variable=prefix netcdf-mpi) + echo "NETCDF_P_PREFIX=${NETCDF_PREFIX}" && ls ${NETCDF_P_PREFIX}/include + NETCDF_PREFIX=$(pkg-config --variable=prefix netcdf) + echo "NETCDF_PREFIX=${NETCDF_PREFIX}" && ls ${NETCDF_PREFIX}/include # # OASIS3-MCT diff --git a/cmake/FindNetCDF.cmake b/cmake/FindNetCDF.cmake index fb52940..ec10228 100644 --- a/cmake/FindNetCDF.cmake +++ b/cmake/FindNetCDF.cmake @@ -6,15 +6,17 @@ if(NetCDF_C_FOUND) if (netCDF_HAS_PARALLEL) set(NetCDF_C_ROOT "${netCDF_INSTALL_PREFIX}") set(NetCDF_C_LIB_DIR "${netCDF_LIB_DIR}") + message(STATUS "FindNETCDF_C: NetCDF_C_ROOT=${NetCDF_C_ROOT}") + message(STATUS "FindNETCDF_C: netCDF_LIB_DIR=${netCDF_LIB_DIR}") else() # If the default NetCDF C library doesn't have parallel support, try searching for another one. # Option 1 (Ubuntu-specific): `sudo apt-get install libnetcdf-mpi-dev` provides NetCDF with parallel support. - pkg_check_modules(NetCDF_C_MPI QUIET netcdf-mpi) + pkg_check_modules(NetCDF_C_MPI netcdf-mpi) if (NetCDF_C_MPI_FOUND) pkg_get_variable(NetCDF_C_ROOT netcdf-mpi prefix) set(NetCDF_C_LIB_DIR "${NetCDF_C_ROOT}/lib") set(netCDF_HAS_PARALLEL "ON") - message(STATUS "FindNETCDF1: NetCDF_C_ROOT=${NetCDF_C_ROOT}") + message(STATUS "FindNETCDF_MPI: NetCDF_C_ROOT=${NetCDF_C_ROOT}") else() # Set default NetCDF C paths to the first found NetCDF C library. set(NetCDF_C_ROOT "${netCDF_INSTALL_PREFIX}") From 41e1508577fc0f1ab35e82bbbeda86cf9d40beac Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 30 Oct 2024 15:21:13 +0100 Subject: [PATCH 35/71] netCDF_HAS_PARALLEL seems to be incorrectly parsed --- .github/workflows/CI.yml | 2 +- cmake/FindNetCDF.cmake | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5659166..c3fcbef 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -54,7 +54,7 @@ jobs: - name: Introspect netcdf-mpi run: | NETCDF_P_PREFIX=$(pkg-config --variable=prefix netcdf-mpi) - echo "NETCDF_P_PREFIX=${NETCDF_PREFIX}" && ls ${NETCDF_P_PREFIX}/include + echo "NETCDF_P_PREFIX=${NETCDF_P_PREFIX}" && ls ${NETCDF_P_PREFIX}/include NETCDF_PREFIX=$(pkg-config --variable=prefix netcdf) echo "NETCDF_PREFIX=${NETCDF_PREFIX}" && ls ${NETCDF_PREFIX}/include diff --git a/cmake/FindNetCDF.cmake b/cmake/FindNetCDF.cmake index ec10228..66758ab 100644 --- a/cmake/FindNetCDF.cmake +++ b/cmake/FindNetCDF.cmake @@ -3,7 +3,7 @@ include(FindPackageHandleStandardArgs) find_package(NetCDF_C QUIET NAMES netCDF) if(NetCDF_C_FOUND) - if (netCDF_HAS_PARALLEL) + if (${netCDF_HAS_PARALLEL}) set(NetCDF_C_ROOT "${netCDF_INSTALL_PREFIX}") set(NetCDF_C_LIB_DIR "${netCDF_LIB_DIR}") message(STATUS "FindNETCDF_C: NetCDF_C_ROOT=${NetCDF_C_ROOT}") @@ -40,7 +40,7 @@ if(NetCDF_FOUND) set(NetCDF_LIBRARIES "-L${NetCDF_C_LIB_DIR} -L${NetCDF_F90_LIB_DIR} -lnetcdff -lnetcdf" CACHE STRING "NetCDF linker options") set(NetCDF_F90_ROOT ${NetCDF_F90_ROOT} CACHE PATH "Path to NetCDF-Fortran directory which contains its include header files and libraries.") set(NetCDF_C_ROOT ${NetCDF_C_ROOT} CACHE PATH "Path to NetCDF-C directory which contains its include header files and libraries.") - if (netCDF_HAS_PARALLEL) + if (${netCDF_HAS_PARALLEL}) message(STATUS "TSMP2 found NetCDF C built with parallel I/O support.") else() message(WARNING "TSMP2 is using NetCDF C without parallel I/O support.") From 80680d134451a33af73ee9f6a37c97fefc170480 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 30 Oct 2024 15:28:31 +0100 Subject: [PATCH 36/71] Check value of netCDF_HAS_PARALLEL --- cmake/FindNetCDF.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/FindNetCDF.cmake b/cmake/FindNetCDF.cmake index 66758ab..ac2b0a2 100644 --- a/cmake/FindNetCDF.cmake +++ b/cmake/FindNetCDF.cmake @@ -3,6 +3,7 @@ include(FindPackageHandleStandardArgs) find_package(NetCDF_C QUIET NAMES netCDF) if(NetCDF_C_FOUND) + message(STATUS "netCDF_HAS_PARALLEL=${netCDF_HAS_PARALLEL}") if (${netCDF_HAS_PARALLEL}) set(NetCDF_C_ROOT "${netCDF_INSTALL_PREFIX}") set(NetCDF_C_LIB_DIR "${netCDF_LIB_DIR}") From 7233837a010d6cc7d6d80bccd3d1ab6cd0197290 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 30 Oct 2024 15:41:14 +0100 Subject: [PATCH 37/71] Be more verbose how NetCDF is being searched --- cmake/FindNetCDF.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/FindNetCDF.cmake b/cmake/FindNetCDF.cmake index ac2b0a2..88aa775 100644 --- a/cmake/FindNetCDF.cmake +++ b/cmake/FindNetCDF.cmake @@ -1,7 +1,7 @@ find_package(PkgConfig QUIET) include(FindPackageHandleStandardArgs) -find_package(NetCDF_C QUIET NAMES netCDF) +find_package(NetCDF_C NAMES netCDF) if(NetCDF_C_FOUND) message(STATUS "netCDF_HAS_PARALLEL=${netCDF_HAS_PARALLEL}") if (${netCDF_HAS_PARALLEL}) From 06d8e689b0a63995820cc4afa5f4bc662a5edf97 Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Wed, 30 Oct 2024 15:52:30 +0100 Subject: [PATCH 38/71] BuildPDAF: fix syntax error `OR` --- cmake/BuildPDAF.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/BuildPDAF.cmake b/cmake/BuildPDAF.cmake index 0fdce11..7a1f5bf 100644 --- a/cmake/BuildPDAF.cmake +++ b/cmake/BuildPDAF.cmake @@ -74,7 +74,7 @@ list(JOIN PDAF_LINK_LIBS " " PDAF_LINK_LIBS) # Set PDAF_OPTIM for Makefile header # ---------------------------------- if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel" - or CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") + OR CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") # using Intel Compiler if (CMAKE_BUILD_TYPE STREQUAL "RELEASE") From 1f225b68b77c7b58bfed8e96eebf53518a2d908d Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Thu, 31 Oct 2024 09:43:47 +0100 Subject: [PATCH 39/71] BuildPDAF: yet another `CMAKE_CXX_COMPILER_ID`-related fix --- cmake/BuildPDAF.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/BuildPDAF.cmake b/cmake/BuildPDAF.cmake index 3cce4e2..34c3434 100644 --- a/cmake/BuildPDAF.cmake +++ b/cmake/BuildPDAF.cmake @@ -120,7 +120,8 @@ list(JOIN PDAF_OPTIM " " PDAF_OPTIM) # Set PDAF_DOUBLEPRECISION for Makefile header # -------------------------------------------- -if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") +if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel" + OR CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") list(APPEND PDAF_DOUBLEPRECISION "-r8") From 21be0b3bc90470bc67572f1ab04f4fc75c8cb942 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Thu, 31 Oct 2024 12:45:48 +0100 Subject: [PATCH 40/71] Open SSH port to debug CI runner See https://github.com/lhotari/action-upterm --- .github/workflows/CI.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index c3fcbef..a0b5d16 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -171,6 +171,13 @@ jobs: -DParFlow="ON" -DPARFLOW_SRC=${MODEL_DIR}/parflow + - name: Set debugging session + uses: lhotari/action-upterm@v1 + with: + limit-access-to-actor: true + limit-access-to-users: kvrigor + wait-timeout-minutes: 5 + - name: Build TSMP2 run: | cmake --build ${BUILD_DIR} From 0f65fbba8eeb29b12c87f0192beeeb37541fe381 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Thu, 31 Oct 2024 18:26:59 +0100 Subject: [PATCH 41/71] Appended path to parallel-aware NetCDF to CMAKE_PREFIX_PATH; increased cmake logging verbosity --- .github/workflows/CI.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a0b5d16..2cf1f0e 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -51,12 +51,10 @@ jobs: run: | mkdir -p ${CMAKE_PREFIX_PATH} ${MODEL_DIR} - - name: Introspect netcdf-mpi + # This workaround helps TSMP2 to locate parallel-aware NetCDF more easily + - name: Add parallel-aware NetCDF C to CMAKE_PREFIX_PATH run: | - NETCDF_P_PREFIX=$(pkg-config --variable=prefix netcdf-mpi) - echo "NETCDF_P_PREFIX=${NETCDF_P_PREFIX}" && ls ${NETCDF_P_PREFIX}/include - NETCDF_PREFIX=$(pkg-config --variable=prefix netcdf) - echo "NETCDF_PREFIX=${NETCDF_PREFIX}" && ls ${NETCDF_PREFIX}/include + echo "CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:$(pkg-config --variable=prefix netcdf-mpi)" >> $GITHUB_ENV # # OASIS3-MCT @@ -162,7 +160,7 @@ jobs: # - name: Configure TSMP2 run: >- - cmake -S . -B ${BUILD_DIR} + cmake -S . -B ${BUILD_DIR} --log-level=DEBUG -DCMAKE_BUILD_TYPE="RELEASE" -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -DOASIS_SRC=${MODEL_DIR}/oasis3-mct From 7baec0457c5da59c4fd5e0eca063304226c800b3 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Thu, 31 Oct 2024 18:30:42 +0100 Subject: [PATCH 42/71] Yet another attempt at taming the ill-behaved FindNetCDF Note: Not yet fully working! --- cmake/FindNetCDF.cmake | 97 ++++++++++++++++++++++++++++++++---------- 1 file changed, 75 insertions(+), 22 deletions(-) diff --git a/cmake/FindNetCDF.cmake b/cmake/FindNetCDF.cmake index 88aa775..96c92aa 100644 --- a/cmake/FindNetCDF.cmake +++ b/cmake/FindNetCDF.cmake @@ -1,31 +1,81 @@ find_package(PkgConfig QUIET) include(FindPackageHandleStandardArgs) -find_package(NetCDF_C NAMES netCDF) -if(NetCDF_C_FOUND) - message(STATUS "netCDF_HAS_PARALLEL=${netCDF_HAS_PARALLEL}") - if (${netCDF_HAS_PARALLEL}) - set(NetCDF_C_ROOT "${netCDF_INSTALL_PREFIX}") - set(NetCDF_C_LIB_DIR "${netCDF_LIB_DIR}") - message(STATUS "FindNETCDF_C: NetCDF_C_ROOT=${NetCDF_C_ROOT}") - message(STATUS "FindNETCDF_C: netCDF_LIB_DIR=${netCDF_LIB_DIR}") - else() - # If the default NetCDF C library doesn't have parallel support, try searching for another one. - # Option 1 (Ubuntu-specific): `sudo apt-get install libnetcdf-mpi-dev` provides NetCDF with parallel support. - pkg_check_modules(NetCDF_C_MPI netcdf-mpi) - if (NetCDF_C_MPI_FOUND) - pkg_get_variable(NetCDF_C_ROOT netcdf-mpi prefix) - set(NetCDF_C_LIB_DIR "${NetCDF_C_ROOT}/lib") - set(netCDF_HAS_PARALLEL "ON") - message(STATUS "FindNETCDF_MPI: NetCDF_C_ROOT=${NetCDF_C_ROOT}") +# ************************ +# 1. Find NetCDF C library +# ************************ + +# Attempt 1: Prioritize finding a parallel-aware NetCDF. +find_path(NetCDF_C_INCLUDEDIR NAMES netcdf_par.h PATH_SUFFIXES include) + +if(NetCDF_C_INCLUDEDIR) + # Attempt 1 succeeded! + get_filename_component(NetCDF_C_ROOT ${NetCDF_C_INCLUDEDIR} DIRECTORY) + set(NetCDF_C_LIB_DIR "${NetCDF_C_ROOT}/lib") + set(NetCDF_HAS_PARALLEL TRUE) + + # Run `cmake ... --log-level=DEBUG` to see these debugging information. + message(DEBUG "FindNETCDF_1: NetCDF_C_ROOT=${NetCDF_C_ROOT}") + message(DEBUG "FindNETCDF_1: NetCDF_C_LIB_DIR=${NetCDF_C_LIB_DIR}") + message(DEBUG "FindNETCDF_1: NetCDF_C_INCLUDEDIR=${NetCDF_C_INCLUDEDIR}") +else() + # Attempt 2: Use find_package() to find NetCDF. The complicated logical + # structure below reflects the voodoo behavior of + # find_package() + buggy NetCDF CMake config file + platform-specific filesystem layout + find_package(NetCDF_C NAMES netCDF) + if(NetCDF_C_FOUND) + set(NetCDF_C_ROOT ${netCDF_INSTALL_PREFIX}) + set(NetCDF_C_LIB_DIR ${netCDF_LIB_DIR}) + message(DEBUG "FindNETCDF_2: NetCDF_C_ROOT=${NetCDF_C_ROOT}") + message(DEBUG "FindNETCDF_2: NetCDF_C_LIB_DIR=${NetCDF_C_LIB_DIR}") + + # Don't trust what netCDF_HAS_PARALLEL reports. The existence of netcdf_par.h header + # is the surest way to know that netCDF was built with parallel support. + find_path(NetCDF_C_INCLUDEDIR NAMES netcdf_par.h PATH_SUFFIXES ${NetCDF_C_ROOT}/include) + + if(NetCDF_C_INCLUDEDIR) + # Result 2.1: find_package() gets lucky in finding parallel-aware NetCDF + set(NetCDF_HAS_PARALLEL TRUE) + message(DEBUG "FindNETCDF_21: NetCDF_C_INCLUDEDIR=${NetCDF_C_INCLUDEDIR}") else() - # Set default NetCDF C paths to the first found NetCDF C library. - set(NetCDF_C_ROOT "${netCDF_INSTALL_PREFIX}") - set(NetCDF_C_LIB_DIR "${netCDF_LIB_DIR}") + # Result 2.2: find_package() only finds serial-aware NetCDF + set(NetCDF_HAS_PARALLEL FALSE) + set(NetCDF_C_INCLUDEDIR ${NetCDF_C_ROOT}/include) + message(DEBUG "FindNETCDF_22: NetCDF_C_INCLUDEDIR=${NetCDF_C_INCLUDEDIR}") + + # TODO: It's possible to keep searching for a parallel-aware NetCDF, but + # I'm not yet aware of any method that isn't too platform-specific. endif() + + # --------------------------------------------------------------------------- + # 31-Oct-2024 kvrigor: The snippet below works for Ubuntu. The package + # `libnetcdf-mpi-dev` is a parallel-aware NetCDF library, but it takes + # some trickery to extract the correct prefix path, which doesn't really + # work 100% of the time (e.g. see [1]). Baking such fragile system-specific + # manouevre here doesn't seem to be worth it; anyway I'm leaving this piece + # of code here in case somebody would like to investigate this further. + # + # [1]: https://github.com/Unidata/netcdf-c/issues/2069 + # --------------------------------------------------------------------------- + # + # pkg_check_modules(NetCDF_C_MPI netcdf-mpi) + # if (NetCDF_C_MPI_FOUND) + # pkg_get_variable(NetCDF_C_ROOT netcdf-mpi prefix) + # set(NetCDF_C_LIB_DIR "${NetCDF_C_ROOT}/lib") + # set(NetCDF_C_INCLUDEDIR "${NetCDF_C_ROOT}/lib") + # set(NetCDF_HAS_PARALLEL "ON") + # message(DEBUG "FindNETCDF_MPI: NetCDF_C_ROOT=${NetCDF_C_ROOT}") + # message(DEBUG "FindNETCDF_MPI: NetCDF_C_LIB_DIR=${NetCDF_C_LIB_DIR}") + # message(DEBUG "FindNETCDF_MPI: NetCDF_C_INCLUDEDIR=${NetCDF_C_ROOT}") + # endif() + # + # -------------------------------------------------------------------- endif() endif() +# ************************ +# 2. Find NetCDF Fortran +# ************************ pkg_check_modules(NetCDF_F90 REQUIRED netcdf-fortran) if (NetCDF_F90_FOUND) pkg_get_variable(NetCDF_F90_ROOT netcdf-fortran prefix) @@ -33,6 +83,10 @@ if (NetCDF_F90_FOUND) pkg_get_variable(NetCDF_F90_INCLUDEDIR netcdf-fortran fmoddir) endif() + +# ************************************* +# 3. Set paths to NetCDF C and Fortran +# ************************************* find_package_handle_standard_args(NetCDF REQUIRED_VARS NetCDF_C_ROOT NetCDF_F90_ROOT NetCDF_C_LIB_DIR NetCDF_F90_LIB_DIR NetCDF_F90_INCLUDEDIR VERSION_VAR NetCDF_F90_VERSION) @@ -41,10 +95,9 @@ if(NetCDF_FOUND) set(NetCDF_LIBRARIES "-L${NetCDF_C_LIB_DIR} -L${NetCDF_F90_LIB_DIR} -lnetcdff -lnetcdf" CACHE STRING "NetCDF linker options") set(NetCDF_F90_ROOT ${NetCDF_F90_ROOT} CACHE PATH "Path to NetCDF-Fortran directory which contains its include header files and libraries.") set(NetCDF_C_ROOT ${NetCDF_C_ROOT} CACHE PATH "Path to NetCDF-C directory which contains its include header files and libraries.") - if (${netCDF_HAS_PARALLEL}) + if (${NetCDF_HAS_PARALLEL}) message(STATUS "TSMP2 found NetCDF C built with parallel I/O support.") else() message(WARNING "TSMP2 is using NetCDF C without parallel I/O support.") endif() - message(STATUS "FindNETCDF2: NetCDF_C_ROOT=${NetCDF_C_ROOT}") endif() \ No newline at end of file From 0d95e23ac2feec8771c8bd8cae50bc6ca09e974c Mon Sep 17 00:00:00 2001 From: kvrigor Date: Thu, 31 Oct 2024 18:46:50 +0100 Subject: [PATCH 43/71] Append to CMAKE_PREFIX_PATH only after Hypre installation --- .github/workflows/CI.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 2cf1f0e..4f3cf0e 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -51,11 +51,6 @@ jobs: run: | mkdir -p ${CMAKE_PREFIX_PATH} ${MODEL_DIR} - # This workaround helps TSMP2 to locate parallel-aware NetCDF more easily - - name: Add parallel-aware NetCDF C to CMAKE_PREFIX_PATH - run: | - echo "CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:$(pkg-config --variable=prefix netcdf-mpi)" >> $GITHUB_ENV - # # OASIS3-MCT # @@ -155,6 +150,12 @@ jobs: path: ${{ env.CMAKE_PREFIX_PATH }} key: cache-${{ matrix.config.name }}-${{ env.HYPRE_TAG }} + # This workaround helps TSMP2 to locate parallel-aware NetCDF more easily + - name: Add parallel-aware NetCDF C to CMAKE_PREFIX_PATH + run: | + echo "CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:$(pkg-config --variable=prefix netcdf-mpi)" >> $GITHUB_ENV + echo "CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}" + # # TSMP2 # From 62a99681dd65013643e37ef6fc5ec4b4e0361174 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Fri, 8 Nov 2024 15:31:05 +0100 Subject: [PATCH 44/71] CI: Installed NetCDF C and Fortran The libnetcdf* situation in Ubuntu is hopeless --- .github/workflows/CI.yml | 83 +++++++++++++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 14 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 4f3cf0e..ce7d791 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -31,6 +31,8 @@ jobs: eCLM_TAG: beta-0.1 PARFLOW_TAG: oasis3-ubuntu22.04 #v3.13.0 HYPRE_TAG: 2.26.0 + VER_NETCDF_C: 4.9.2 + VER_NETCDF_F90: 4.6.1 CMAKE_PREFIX_PATH: ${{ github.workspace }}/dependencies SYSTEMNAME: ${{ matrix.os }} @@ -40,17 +42,73 @@ jobs: - name: Install TSMP2 dependencies on Ubuntu uses: awalsh128/cache-apt-pkgs-action@latest with: - packages: libxml2-utils pylint wget cmake libnetcdf-mpi-dev libnetcdff-dev libpnetcdf-dev tcl-dev tk-dev + packages: libxml++2.6-dev pylint wget cmake libpnetcdf-dev tcl-dev tk-dev version: 1.0 execute_install_scripts: true - - name: Download MPI Fortran compiler - run: sudo apt-get install gfortran openmpi-bin libopenmpi-dev + - name: Download HDF5 headers and MPI Fortran compiler + run: sudo apt-get install gfortran openmpi-bin libopenmpi-dev libhdf5-openmpi-dev libhdf5-openmpi-103 hdf5-helpers - name: Create directories for component model and dependencies run: | mkdir -p ${CMAKE_PREFIX_PATH} ${MODEL_DIR} + # + # NetCDF C + # + - name: Restore cached NetCDF C ${{ env.VER_NETCDF_C }} + uses: actions/cache/restore@v4 + id: cache-netcdf-restore + with: + path: ${{ env.CMAKE_PREFIX_PATH }} + key: cache-${{ matrix.config.name }}-${{ env.VER_NETCDF_C }} + + - if: steps.cache-netcdf-restore.outputs.cache-hit != 'true' + name: Install NetCDF C ${{ env.VER_NETCDF_C }} + run: | + wget https://github.com/Unidata/netcdf-c/archive/v${VER_NETCDF_C}.tar.gz + tar xf v${VER_NETCDF_C}.tar.gz + cd netcdf-c-${VER_NETCDF_C} + export CPPFLAGS=-I/usr/include/hdf5/openmpi + export LDFLAGS=-L/usr/lib/x86_64-linux-gnu/hdf5/openmpi + ./configure --prefix=$(realpath $CMAKE_PREFIX_PATH) + make -j4 install + + - if: steps.cache-netcdf-restore.outputs.cache-hit != 'true' + name: Cache NetCDF ${{ env.VER_NETCDF_C }} + uses: actions/cache/save@v4 + with: + path: ${{ env.CMAKE_PREFIX_PATH }} + key: cache-${{ matrix.config.name }}-${{ env.VER_NETCDF_C }} + + # + # NetCDF Fortran + # + - name: Restore cached NetCDF Fortran ${{ env.VER_NETCDF_F90 }} + uses: actions/cache/restore@v4 + id: cache-netcdff90-restore + with: + path: ${{ env.CMAKE_PREFIX_PATH }} + key: cache-${{ matrix.config.name }}-${{ env.VER_NETCDF_F90 }} + + - if: steps.cache-netcdff90-restore.outputs.cache-hit != 'true' + name: Install NetCDF Fortran ${{ env.VER_NETCDF_F90 }} + run: | + wget https://github.com/Unidata/netcdf-fortran/archive/v${VER_NETCDF_F90}.tar.gz + tar xf v${VER_NETCDF_F90}.tar.gz + cd netcdf-fortran-${VER_NETCDF_F90} + export CPPFLAGS=-I${CMAKE_PREFIX_PATH} + export LDFLAGS=-L${CMAKE_PREFIX_PATH} + ./configure --prefix=$(realpath $CMAKE_PREFIX_PATH) + make -j4 install + + - if: steps.cache-netcdff90-restore.outputs.cache-hit != 'true' + name: Cache NetCDF Fortran ${{ env.VER_NETCDF_F90 }} + uses: actions/cache/save@v4 + with: + path: ${{ env.CMAKE_PREFIX_PATH }} + key: cache-${{ matrix.config.name }}-${{ env.VER_NETCDF_F90 }} + # # OASIS3-MCT # @@ -150,11 +208,8 @@ jobs: path: ${{ env.CMAKE_PREFIX_PATH }} key: cache-${{ matrix.config.name }}-${{ env.HYPRE_TAG }} - # This workaround helps TSMP2 to locate parallel-aware NetCDF more easily - - name: Add parallel-aware NetCDF C to CMAKE_PREFIX_PATH - run: | - echo "CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:$(pkg-config --variable=prefix netcdf-mpi)" >> $GITHUB_ENV - echo "CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}" + - name: Check CMAKE_PREFIX_PATH + run: tree -FUCh --du --filelimit 20 -L 2 ${CMAKE_PREFIX_PATH} # # TSMP2 @@ -170,12 +225,12 @@ jobs: -DParFlow="ON" -DPARFLOW_SRC=${MODEL_DIR}/parflow - - name: Set debugging session - uses: lhotari/action-upterm@v1 - with: - limit-access-to-actor: true - limit-access-to-users: kvrigor - wait-timeout-minutes: 5 + # - name: Set debugging session + # uses: lhotari/action-upterm@v1 + # with: + # limit-access-to-actor: true + # limit-access-to-users: kvrigor + # wait-timeout-minutes: 5 - name: Build TSMP2 run: | From a39f8893d737e58b582bdb3e7a72887c36be4bfe Mon Sep 17 00:00:00 2001 From: kvrigor Date: Fri, 8 Nov 2024 15:37:24 +0100 Subject: [PATCH 45/71] CI: Changed version naming convention; fixed NetCDF F90 configure error --- .github/workflows/CI.yml | 64 ++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ce7d791..4293681 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -27,10 +27,10 @@ jobs: BUILD_DIR: bld INSTALL_DIR: install MODEL_DIR: ${{ github.workspace }}/component-models - OASIS_TAG: tsmp-patches-v0.1 - eCLM_TAG: beta-0.1 - PARFLOW_TAG: oasis3-ubuntu22.04 #v3.13.0 - HYPRE_TAG: 2.26.0 + VER_OASIS: tsmp-patches-v0.1 + VER_eCLM: beta-0.1 + VER_PARFLOW: oasis3-ubuntu22.04 #v3.13.0 + VER_HYPRE: 2.26.0 VER_NETCDF_C: 4.9.2 VER_NETCDF_F90: 4.6.1 CMAKE_PREFIX_PATH: ${{ github.workspace }}/dependencies @@ -97,8 +97,8 @@ jobs: wget https://github.com/Unidata/netcdf-fortran/archive/v${VER_NETCDF_F90}.tar.gz tar xf v${VER_NETCDF_F90}.tar.gz cd netcdf-fortran-${VER_NETCDF_F90} - export CPPFLAGS=-I${CMAKE_PREFIX_PATH} - export LDFLAGS=-L${CMAKE_PREFIX_PATH} + export CPPFLAGS=-I${CMAKE_PREFIX_PATH}/include + export LDFLAGS=-L${CMAKE_PREFIX_PATH}/lib ./configure --prefix=$(realpath $CMAKE_PREFIX_PATH) make -j4 install @@ -112,101 +112,101 @@ jobs: # # OASIS3-MCT # - - name: Restore cached OASIS3-MCT ${{ env.OASIS_TAG }} + - name: Restore cached OASIS3-MCT ${{ env.VER_OASIS }} uses: actions/cache/restore@v4 id: cache-oasis-restore with: path: ${{ env.MODEL_DIR }}/oasis3-mct - key: cache-${{ matrix.config.name }}-${{ env.OASIS_TAG }} + key: cache-${{ matrix.config.name }}-${{ env.VER_OASIS }} - if: steps.cache-oasis-restore.outputs.cache-hit != 'true' - name: Download OASIS3-MCT ${{ env.OASIS_TAG }} + name: Download OASIS3-MCT ${{ env.VER_OASIS }} working-directory: ${{ env.MODEL_DIR }} run: | - git clone -b $OASIS_TAG https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/oasis3-mct.git + git clone -b $VER_OASIS https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/oasis3-mct.git echo "OASIS3-MCT version: $(cd oasis3-mct && git describe --tags --always)" - if: steps.cache-oasis-restore.outputs.cache-hit != 'true' - name: Cache OASIS3-MCT ${{ env.OASIS_TAG }} + name: Cache OASIS3-MCT ${{ env.VER_OASIS }} uses: actions/cache/save@v4 with: path: ${{ env.MODEL_DIR }}/oasis3-mct - key: cache-${{ matrix.config.name }}-${{ env.OASIS_TAG }} + key: cache-${{ matrix.config.name }}-${{ env.VER_OASIS }} # # eCLM # - - name: Restore cached eCLM ${{ env.eCLM_TAG }} + - name: Restore cached eCLM ${{ env.VER_eCLM }} uses: actions/cache/restore@v4 id: cache-eclm-restore with: path: ${{ env.MODEL_DIR }}/eCLM - key: cache-${{ matrix.config.name }}-${{ env.eCLM_TAG }} + key: cache-${{ matrix.config.name }}-${{ env.VER_eCLM }} - if: steps.cache-eclm-restore.outputs.cache-hit != 'true' - name: Download eCLM ${{ env.eCLM_TAG }} + name: Download eCLM ${{ env.VER_eCLM }} working-directory: ${{ env.MODEL_DIR }} run: | - git clone -b $eCLM_TAG https://github.com/HPSCTerrSys/eCLM.git + git clone -b $VER_eCLM https://github.com/HPSCTerrSys/eCLM.git echo "eCLM version: $(cd eCLM && git describe --tags --always)" - if: steps.cache-eclm-restore.outputs.cache-hit != 'true' - name: Cache eCLM ${{ env.eCLM_TAG }} + name: Cache eCLM ${{ env.VER_eCLM }} uses: actions/cache/save@v4 with: path: ${{ env.MODEL_DIR }}/eCLM - key: cache-${{ matrix.config.name }}-${{ env.eCLM_TAG }} + key: cache-${{ matrix.config.name }}-${{ env.VER_eCLM }} # # ParFlow # - - name: Restore cached ParFlow ${{ env.PARFLOW_TAG }} + - name: Restore cached ParFlow ${{ env.VER_PARFLOW }} uses: actions/cache/restore@v4 id: cache-parflow-restore with: path: ${{ env.MODEL_DIR }}/parflow - key: cache-${{ matrix.config.name }}-${{ env.PARFLOW_TAG }} + key: cache-${{ matrix.config.name }}-${{ env.VER_PARFLOW }} # TODO: Must use upstream repo https://github.com/parflow/parflow.git - if: steps.cache-parflow-restore.outputs.cache-hit != 'true' - name: Download ParFlow ${{ env.PARFLOW_TAG }} + name: Download ParFlow ${{ env.VER_PARFLOW }} working-directory: ${{ env.MODEL_DIR }} run: | - git clone -b $PARFLOW_TAG https://github.com/kvrigor/parflow.git + git clone -b $VER_PARFLOW https://github.com/kvrigor/parflow.git echo "ParFlow version: $(cd parflow && git describe --tags --always)" - if: steps.cache-parflow-restore.outputs.cache-hit != 'true' - name: Cache ParFlow ${{ env.PARFLOW_TAG }} + name: Cache ParFlow ${{ env.VER_PARFLOW }} uses: actions/cache/save@v4 with: path: ${{ env.MODEL_DIR }}/parflow - key: cache-${{ matrix.config.name }}-${{ env.PARFLOW_TAG }} + key: cache-${{ matrix.config.name }}-${{ env.VER_PARFLOW }} # # Hypre # - - name: Restore cached Hypre ${{ env.HYPRE_TAG }} + - name: Restore cached Hypre ${{ env.VER_HYPRE }} uses: actions/cache/restore@v4 id: cache-hypre-restore with: path: ${{ env.CMAKE_PREFIX_PATH }} - key: cache-${{ matrix.config.name }}-${{ env.HYPRE_TAG }} + key: cache-${{ matrix.config.name }}-${{ env.VER_HYPRE }} - if: steps.cache-hypre-restore.outputs.cache-hit != 'true' - name: Install Hypre ${{ env.HYPRE_TAG }} + name: Install Hypre ${{ env.VER_HYPRE }} run: | - wget https://github.com/hypre-space/hypre/archive/v${HYPRE_TAG}.tar.gz - tar xf v${HYPRE_TAG}.tar.gz - cd hypre-${HYPRE_TAG}/src + wget https://github.com/hypre-space/hypre/archive/v${VER_HYPRE}.tar.gz + tar xf v${VER_HYPRE}.tar.gz + cd hypre-${VER_HYPRE}/src ./configure --prefix=$(realpath $CMAKE_PREFIX_PATH) make -j4 install - if: steps.cache-hypre-restore.outputs.cache-hit != 'true' - name: Cache Hypre ${{ env.HYPRE_TAG }} + name: Cache Hypre ${{ env.VER_HYPRE }} uses: actions/cache/save@v4 with: path: ${{ env.CMAKE_PREFIX_PATH }} - key: cache-${{ matrix.config.name }}-${{ env.HYPRE_TAG }} + key: cache-${{ matrix.config.name }}-${{ env.VER_HYPRE }} - name: Check CMAKE_PREFIX_PATH run: tree -FUCh --du --filelimit 20 -L 2 ${CMAKE_PREFIX_PATH} From 10eed1b40608598069063d683d42511ab2e73cb9 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Fri, 8 Nov 2024 16:08:50 +0100 Subject: [PATCH 46/71] BuildParFlow: Reverted NETCDF_DIR setting --- cmake/BuildParFlow.cmake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake index 0c8fff3..450c65a 100644 --- a/cmake/BuildParFlow.cmake +++ b/cmake/BuildParFlow.cmake @@ -49,8 +49,7 @@ ExternalProject_Add(ParFlow -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} -DHYPRE_ROOT=${HYPRE_ROOT} - -DNETCDF_INCLUDE_DIR=${NetCDF_C_ROOT}/include - -DNETCDF_LIBRARY=${NetCDF_C_LIB_DIR}/libnetcdf.so # TODO: Fix this hardcoded library file! + -DNETCDF_DIR=${NetCDF_C_ROOT} -DPARFLOW_AMPS_SEQUENTIAL_IO=on -DPARFLOW_ENABLE_TIMING=TRUE -DMPIEXEC_EXECUTABLE=${MPIEXEC_EXECUTABLE} From f08cd23325c9744a637bae5299d8ca5ca740fade Mon Sep 17 00:00:00 2001 From: kvrigor Date: Fri, 8 Nov 2024 16:10:02 +0100 Subject: [PATCH 47/71] CI: Removed debugging statements --- .github/workflows/CI.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 4293681..0c2c0ca 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -29,7 +29,7 @@ jobs: MODEL_DIR: ${{ github.workspace }}/component-models VER_OASIS: tsmp-patches-v0.1 VER_eCLM: beta-0.1 - VER_PARFLOW: oasis3-ubuntu22.04 #v3.13.0 + VER_PARFLOW: oasis3-ubuntu22.04 VER_HYPRE: 2.26.0 VER_NETCDF_C: 4.9.2 VER_NETCDF_F90: 4.6.1 @@ -208,9 +208,6 @@ jobs: path: ${{ env.CMAKE_PREFIX_PATH }} key: cache-${{ matrix.config.name }}-${{ env.VER_HYPRE }} - - name: Check CMAKE_PREFIX_PATH - run: tree -FUCh --du --filelimit 20 -L 2 ${CMAKE_PREFIX_PATH} - # # TSMP2 # @@ -225,13 +222,6 @@ jobs: -DParFlow="ON" -DPARFLOW_SRC=${MODEL_DIR}/parflow - # - name: Set debugging session - # uses: lhotari/action-upterm@v1 - # with: - # limit-access-to-actor: true - # limit-access-to-users: kvrigor - # wait-timeout-minutes: 5 - - name: Build TSMP2 run: | cmake --build ${BUILD_DIR} From 463f61c5d5c344fc4f6705694ea2d63cc9749ca5 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Fri, 8 Nov 2024 17:58:52 +0100 Subject: [PATCH 48/71] CI: Removed --log-level=DEBUG --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 0c2c0ca..be4e8f4 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -213,7 +213,7 @@ jobs: # - name: Configure TSMP2 run: >- - cmake -S . -B ${BUILD_DIR} --log-level=DEBUG + cmake -S . -B ${BUILD_DIR} -DCMAKE_BUILD_TYPE="RELEASE" -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -DOASIS_SRC=${MODEL_DIR}/oasis3-mct From 4667d42f19b383e4482eef1d91589ca92ec19f3f Mon Sep 17 00:00:00 2001 From: kvrigor Date: Mon, 11 Nov 2024 07:51:28 +0100 Subject: [PATCH 49/71] BuildParFlow: Fixed mis-parsed linker flags --- cmake/BuildParFlow.cmake | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake index 450c65a..3878e1a 100644 --- a/cmake/BuildParFlow.cmake +++ b/cmake/BuildParFlow.cmake @@ -30,7 +30,8 @@ elseif(CMAKE_C_COMPILER_ID STREQUAL "Intel" OR CMAKE_C_COMPILER_ID STREQUAL "Int set(PF_CFLAGS "${OpenMP_Fortran_FLAGS} -Wall -Werror") #TODO: These flags are specific to JSC system. This should be set in an env or build script! list(APPEND JSC_FLAGS -DPARFLOW_ENABLE_SLURM=ON) - list(APPEND JSC_FLAGS -DCMAKE_EXE_LINKER_FLAGS="-lcudart -lcusparse -lcurand") + set(PF_LDFLAGS "-lcudart -lcusparse -lcurand") + list(APPEND JSC_FLAGS -DCMAKE_EXE_LINKER_FLAGS=${PF_LDFLAGS}) else() message(FATAL_ERROR "C compiler '${CMAKE_C_COMPILER_ID}' is not supported.") endif() @@ -60,4 +61,4 @@ ExternalProject_Add(ParFlow ) get_model_version(${PARFLOW_SRC} PARFLOW_VERSION) -list(APPEND TSMP2_MODEL_VERSIONS "ParFlow: ${PARFLOW_VERSION}") \ No newline at end of file +list(APPEND TSMP2_MODEL_VERSIONS "ParFlow: ${PARFLOW_VERSION}") From c0388204e6b96d83303e06bf02bbb597d38cce12 Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Wed, 13 Nov 2024 17:25:31 +0100 Subject: [PATCH 50/71] BuildPDAF: Separate Fortran/C compiler options - `PDAF_FOPT` - `PDAF_COPT` --- cmake/BuildPDAF.cmake | 80 +++++++++++++++++++++++++++------- cmake/BuildPDAFFRAMEWORK.cmake | 4 +- cmake/BuildPDAFMODEL.cmake | 4 +- 3 files changed, 70 insertions(+), 18 deletions(-) diff --git a/cmake/BuildPDAF.cmake b/cmake/BuildPDAF.cmake index 0720a40..a947b6d 100644 --- a/cmake/BuildPDAF.cmake +++ b/cmake/BuildPDAF.cmake @@ -71,51 +71,97 @@ list(APPEND PDAF_LINK_LIBS "${NetCDF_F90_STATIC_LDFLAGS}") # "-lnetcdf", "-lnetc # Join list list(JOIN PDAF_LINK_LIBS " " PDAF_LINK_LIBS) -# Set PDAF_OPTIM for Makefile header +# Set PDAF_FOPT for Makefile header # ---------------------------------- if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") # using Intel Compiler if (CMAKE_BUILD_TYPE STREQUAL "RELEASE") # Release optimization flags - list(APPEND PDAF_OPTIM "-O2") + list(APPEND PDAF_FOPT "-O2") elseif (CMAKE_BUILD_TYPE STREQUAL "DEBUG") # Debug optimization flags - list(APPEND PDAF_OPTIM "-O0") - list(APPEND PDAF_OPTIM "-g") - list(APPEND PDAF_OPTIM "-traceback") + list(APPEND PDAF_FOPT "-O0") + list(APPEND PDAF_FOPT "-g") + list(APPEND PDAF_FOPT "-traceback") else() message(FATAL_ERROR "Unsupported CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") endif() - list(APPEND PDAF_OPTIM "-xHost") + list(APPEND PDAF_FOPT "-xHost") elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") # using GCC (experimental) if (CMAKE_BUILD_TYPE STREQUAL "RELEASE") # Release optimization flags - list(APPEND PDAF_OPTIM "-O2") + list(APPEND PDAF_FOPT "-O2") elseif (CMAKE_BUILD_TYPE STREQUAL "DEBUG") # Debug optimization flags - list(APPEND PDAF_OPTIM "-O0") - list(APPEND PDAF_OPTIM "-g") - list(APPEND PDAF_OPTIM "-fbacktrace") + list(APPEND PDAF_FOPT "-O0") + list(APPEND PDAF_FOPT "-g") + list(APPEND PDAF_FOPT "-fbacktrace") else() message(FATAL_ERROR "Unsupported CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") endif() - list(APPEND PDAF_OPTIM "-falign-commons") - list(APPEND PDAF_OPTIM "-fno-automatic") - list(APPEND PDAF_OPTIM "-finit-local-zero") - list(APPEND PDAF_OPTIM "-mcmodel=large") + list(APPEND PDAF_FOPT "-falign-commons") + list(APPEND PDAF_FOPT "-fno-automatic") + list(APPEND PDAF_FOPT "-finit-local-zero") + list(APPEND PDAF_FOPT "-mcmodel=large") else() message(FATAL_ERROR "Unsupported CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}") endif() # Join list -list(JOIN PDAF_OPTIM " " PDAF_OPTIM) +list(JOIN PDAF_FOPT " " PDAF_FOPT) + +# Set PDAF_COPT for Makefile header +# ---------------------------------- +if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + + # using Intel Compiler + if (CMAKE_BUILD_TYPE STREQUAL "RELEASE") + # Release optimization flags + list(APPEND PDAF_COPT "-O2") + elseif (CMAKE_BUILD_TYPE STREQUAL "DEBUG") + # Debug optimization flags + list(APPEND PDAF_COPT "-O0") + list(APPEND PDAF_COPT "-g") + list(APPEND PDAF_COPT "-traceback") + else() + message(FATAL_ERROR "Unsupported CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") + endif() + + list(APPEND PDAF_COPT "-xHost") + +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + + # using GCC (experimental) + if (CMAKE_BUILD_TYPE STREQUAL "RELEASE") + # Release optimization flags + list(APPEND PDAF_COPT "-O2") + elseif (CMAKE_BUILD_TYPE STREQUAL "DEBUG") + # Debug optimization flags + list(APPEND PDAF_COPT "-O0") + list(APPEND PDAF_COPT "-g") + list(APPEND PDAF_COPT "-fbacktrace") + else() + message(FATAL_ERROR "Unsupported CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") + endif() + + list(APPEND PDAF_COPT "-falign-commons") + list(APPEND PDAF_COPT "-fno-automatic") + list(APPEND PDAF_COPT "-finit-local-zero") + list(APPEND PDAF_COPT "-mcmodel=large") + +else() + message(FATAL_ERROR "Unsupported CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}") +endif() + +# Join list +list(JOIN PDAF_COPT " " PDAF_COPT) # Set PDAF_DOUBLEPRECISION for Makefile header # -------------------------------------------- @@ -158,7 +204,9 @@ list(JOIN PDAF_CPP_DEFS " " PDAF_CPP_DEFS) list(APPEND PDAF_ENV_VARS PDAF_ARCH=${PDAF_ARCH}) list(APPEND PDAF_ENV_VARS PDAF_DIR=${PDAF_DIR}) list(APPEND PDAF_ENV_VARS TSMPPDAFLINK_LIBS=${PDAF_LINK_LIBS}) -list(APPEND PDAF_ENV_VARS TSMPPDAFOPTIM=${PDAF_OPTIM}) +list(APPEND PDAF_ENV_VARS TSMPPDAFOPTIM=${PDAF_FOPT}) #backward compatibility +list(APPEND PDAF_ENV_VARS TSMPPDAFFOPT=${PDAF_FOPT}) +list(APPEND PDAF_ENV_VARS TSMPPDAFCOPT=${PDAF_COPT}) list(APPEND PDAF_ENV_VARS TSMPPDAFDOUBLEPRECISION=${PDAF_DOUBLEPRECISION}) list(APPEND PDAF_ENV_VARS TSMPPDAFMPI_INC=${PDAF_MPI_INC}) list(APPEND PDAF_ENV_VARS TSMPPDAFCPP_DEFS=${PDAF_CPP_DEFS}) diff --git a/cmake/BuildPDAFFRAMEWORK.cmake b/cmake/BuildPDAFFRAMEWORK.cmake index 283d6b0..8b530b4 100644 --- a/cmake/BuildPDAFFRAMEWORK.cmake +++ b/cmake/BuildPDAFFRAMEWORK.cmake @@ -13,7 +13,9 @@ set(TSMPPDAFBINDIR "${CMAKE_INSTALL_PREFIX}/bin") list(APPEND PDAFFRAMEWORK_ENV_VARS PDAF_ARCH=${PDAF_ARCH}) list(APPEND PDAFFRAMEWORK_ENV_VARS PDAF_DIR=${PDAF_DIR}) list(APPEND PDAFFRAMEWORK_ENV_VARS TSMPPDAFLINK_LIBS=${PDAF_LINK_LIBS}) -list(APPEND PDAFFRAMEWORK_ENV_VARS TSMPPDAFOPTIM=${PDAF_OPTIM}) +list(APPEND PDAFFRAMEWORK_ENV_VARS TSMPPDAFOPTIM=${PDAF_FOPT}) #backward compatibility +list(APPEND PDAFFRAMEWORK_ENV_VARS TSMPPDAFFOPT=${PDAF_FOPT}) +list(APPEND PDAFFRAMEWORK_ENV_VARS TSMPPDAFCOPT=${PDAF_COPT}) list(APPEND PDAFFRAMEWORK_ENV_VARS TSMPPDAFDOUBLEPRECISION=${PDAF_DOUBLEPRECISION}) list(APPEND PDAFFRAMEWORK_ENV_VARS TSMPPDAFMPI_INC=${PDAF_MPI_INC}) list(APPEND PDAFFRAMEWORK_ENV_VARS TSMPPDAFCPP_DEFS=${PDAF_CPP_DEFS}) diff --git a/cmake/BuildPDAFMODEL.cmake b/cmake/BuildPDAFMODEL.cmake index a23f387..0018b18 100644 --- a/cmake/BuildPDAFMODEL.cmake +++ b/cmake/BuildPDAFMODEL.cmake @@ -158,7 +158,9 @@ list(JOIN PDAF_DEFS " " PDAF_DEFS) list(APPEND PDAFMODEL_ENV_VARS PDAF_ARCH=${PDAF_ARCH}) list(APPEND PDAFMODEL_ENV_VARS PDAF_DIR=${PDAF_DIR}) list(APPEND PDAFMODEL_ENV_VARS TSMPPDAFLINK_LIBS=${PDAF_LINK_LIBS}) -list(APPEND PDAFMODEL_ENV_VARS TSMPPDAFOPTIM=${PDAF_OPTIM}) +list(APPEND PDAFMODEL_ENV_VARS TSMPPDAFOPTIM=${PDAF_FOPT}) #backward compatibility +list(APPEND PDAFMODEL_ENV_VARS TSMPPDAFFOPT=${PDAF_FOPT}) +list(APPEND PDAFMODEL_ENV_VARS TSMPPDAFCOPT=${PDAF_FCOPT}) list(APPEND PDAFMODEL_ENV_VARS TSMPPDAFDOUBLEPRECISION=${PDAF_DOUBLEPRECISION}) list(APPEND PDAFMODEL_ENV_VARS TSMPPDAFMPI_INC=${PDAF_MPI_INC}) list(APPEND PDAFMODEL_ENV_VARS TSMPPDAFCPP_DEFS=${PDAF_CPP_DEFS}) From b140a083eaac46fc68e2720b2522220e81fea342 Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Wed, 13 Nov 2024 17:41:46 +0100 Subject: [PATCH 51/71] BuildPDAF: Add Fortran debug compiler options like for eCLM --- cmake/BuildPDAF.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake/BuildPDAF.cmake b/cmake/BuildPDAF.cmake index 97d2d0a..ed837f5 100644 --- a/cmake/BuildPDAF.cmake +++ b/cmake/BuildPDAF.cmake @@ -85,6 +85,8 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel" list(APPEND PDAF_FOPT "-O0") list(APPEND PDAF_FOPT "-g") list(APPEND PDAF_FOPT "-traceback") + list(APPEND PDAF_FOPT "-fpe0") # compare eCLM debug flags + list(APPEND PDAF_FOPT "-check all") # compare eCLM debug flags else() message(FATAL_ERROR "Unsupported CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") endif() From f1336d8760e720306ee0b327a46489e110feaf3f Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Wed, 13 Nov 2024 17:47:08 +0100 Subject: [PATCH 52/71] eCLM: Update to stages-2024 branch --- models/eCLM | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/eCLM b/models/eCLM index 4582f14..922109d 160000 --- a/models/eCLM +++ b/models/eCLM @@ -1 +1 @@ -Subproject commit 4582f148490edf621cbc1a6f11692017bc344469 +Subproject commit 922109d62f8bc4e490c92b3b2d5323211d5d7eb9 From 86d1ae31261cf867599314cf35571d86acc150ac Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Wed, 13 Nov 2024 17:47:19 +0100 Subject: [PATCH 53/71] pdaf: update to stages 2024 branch --- models/pdaf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/pdaf b/models/pdaf index be688eb..5471753 160000 --- a/models/pdaf +++ b/models/pdaf @@ -1 +1 @@ -Subproject commit be688ebb6813a17a6f07bc8a60610d2fa0360733 +Subproject commit 5471753077ffe28ec5963c3a4703c53e075e9611 From f6422f16d62de420cbf241ccac42a3d03944b5a3 Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Thu, 14 Nov 2024 07:25:36 +0100 Subject: [PATCH 54/71] pdaf: fix for adding IntelLLVM for Stages/2024 --- cmake/BuildPDAF.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/BuildPDAF.cmake b/cmake/BuildPDAF.cmake index ed837f5..509d444 100644 --- a/cmake/BuildPDAF.cmake +++ b/cmake/BuildPDAF.cmake @@ -122,7 +122,8 @@ list(JOIN PDAF_FOPT " " PDAF_FOPT) # Set PDAF_COPT for Makefile header # ---------------------------------- -if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") +if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel" + OR CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") # using Intel Compiler if (CMAKE_BUILD_TYPE STREQUAL "RELEASE") From fdacc35de1907e5bec1a57b41801d168a42b0ccd Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Thu, 14 Nov 2024 08:57:12 +0100 Subject: [PATCH 55/71] update eCLM to head of `master` --- models/eCLM | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/eCLM b/models/eCLM index 922109d..c9f3348 160000 --- a/models/eCLM +++ b/models/eCLM @@ -1 +1 @@ -Subproject commit 922109d62f8bc4e490c92b3b2d5323211d5d7eb9 +Subproject commit c9f334838b4584e8543dc3e6f7e50661433b2055 From c97a9a4a745ac589b8ce62ad67b96000455a8173 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Thu, 14 Nov 2024 11:58:47 +0100 Subject: [PATCH 56/71] CI: Changed ParFlow source to upstream ParFlow PR#563 has been merged: https://github.com/parflow/parflow/pull/563 --- .github/workflows/CI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index be4e8f4..a870261 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -29,7 +29,7 @@ jobs: MODEL_DIR: ${{ github.workspace }}/component-models VER_OASIS: tsmp-patches-v0.1 VER_eCLM: beta-0.1 - VER_PARFLOW: oasis3-ubuntu22.04 + VER_PARFLOW: 499ea4465e91aab96261679f3c26b3d73b159892 # points to commit "Upgrade OASIS3-MCT CI to Ubuntu22.04" VER_HYPRE: 2.26.0 VER_NETCDF_C: 4.9.2 VER_NETCDF_F90: 4.6.1 @@ -172,7 +172,7 @@ jobs: name: Download ParFlow ${{ env.VER_PARFLOW }} working-directory: ${{ env.MODEL_DIR }} run: | - git clone -b $VER_PARFLOW https://github.com/kvrigor/parflow.git + git clone -b $VER_PARFLOW https://github.com/parflow/parflow.git echo "ParFlow version: $(cd parflow && git describe --tags --always)" - if: steps.cache-parflow-restore.outputs.cache-hit != 'true' From 6536361c0f637dbcb5c39a704cf0ec28eaea6dad Mon Sep 17 00:00:00 2001 From: kvrigor Date: Thu, 14 Nov 2024 12:07:50 +0100 Subject: [PATCH 57/71] CI: Clone from ParFlow master and then checkout specific commit --- .github/workflows/CI.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a870261..d54d176 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -172,7 +172,8 @@ jobs: name: Download ParFlow ${{ env.VER_PARFLOW }} working-directory: ${{ env.MODEL_DIR }} run: | - git clone -b $VER_PARFLOW https://github.com/parflow/parflow.git + git clone https://github.com/parflow/parflow.git + git checkout $VER_PARFLOW echo "ParFlow version: $(cd parflow && git describe --tags --always)" - if: steps.cache-parflow-restore.outputs.cache-hit != 'true' From ae99465dc47aa66dbdac3268217404ab138e7ff3 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Thu, 14 Nov 2024 14:34:26 +0100 Subject: [PATCH 58/71] CI: cd parflow then git checkout --- .github/workflows/CI.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d54d176..c4e321c 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -173,8 +173,9 @@ jobs: working-directory: ${{ env.MODEL_DIR }} run: | git clone https://github.com/parflow/parflow.git + cd parflow git checkout $VER_PARFLOW - echo "ParFlow version: $(cd parflow && git describe --tags --always)" + echo "ParFlow version: $(git describe --tags --always)" - if: steps.cache-parflow-restore.outputs.cache-hit != 'true' name: Cache ParFlow ${{ env.VER_PARFLOW }} From 4dfdfd4bfcc0b897ed4841f8f73dcf1fd73d16ca Mon Sep 17 00:00:00 2001 From: kvrigor Date: Thu, 14 Nov 2024 18:41:50 +0100 Subject: [PATCH 59/71] BuildParFlow: Fixed misparsed LDFLAGS and ignore unused variable warnings --- cmake/BuildParFlow.cmake | 10 +++++++--- models/parflow | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake index 3878e1a..17be4e2 100644 --- a/cmake/BuildParFlow.cmake +++ b/cmake/BuildParFlow.cmake @@ -26,12 +26,14 @@ if(CMAKE_C_COMPILER_ID STREQUAL "GNU") # Flags were based from https://github.com/parflow/parflow/blob/77316043227b95215744e58fe9005d35145432ab/.github/workflows/linux.yml#L305 set(PF_CFLAGS "${OpenMP_Fortran_FLAGS} -Wall -Werror -Wno-unused-result -Wno-unused-function") set(PF_FFLAGS "-ffree-line-length-none -ffixed-line-length-none") + #TODO: These flags are specific to JSC system. This should be set in an env or build script! + set(PF_LDFLAGS "") + set(ENABLE_SLURM "OFF") elseif(CMAKE_C_COMPILER_ID STREQUAL "Intel" OR CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") - set(PF_CFLAGS "${OpenMP_Fortran_FLAGS} -Wall -Werror") + set(PF_CFLAGS "${OpenMP_Fortran_FLAGS} -Wall -Werror -Wno-unused-function -Wno-unused-variable") #TODO: These flags are specific to JSC system. This should be set in an env or build script! - list(APPEND JSC_FLAGS -DPARFLOW_ENABLE_SLURM=ON) set(PF_LDFLAGS "-lcudart -lcusparse -lcurand") - list(APPEND JSC_FLAGS -DCMAKE_EXE_LINKER_FLAGS=${PF_LDFLAGS}) + set(ENABLE_SLURM "ON") else() message(FATAL_ERROR "C compiler '${CMAKE_C_COMPILER_ID}' is not supported.") endif() @@ -55,6 +57,8 @@ ExternalProject_Add(ParFlow -DPARFLOW_ENABLE_TIMING=TRUE -DMPIEXEC_EXECUTABLE=${MPIEXEC_EXECUTABLE} -DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG} + -DPARFLOW_ENABLE_SLURM=${ENABLE_SLURM} + -DCMAKE_EXE_LINKER_FLAGS=${PF_LDFLAGS} ${PF_CLM_FLAGS} ${JSC_FLAGS} DEPENDS ${MODEL_DEPENDENCIES} diff --git a/models/parflow b/models/parflow index 6ea2098..499ea44 160000 --- a/models/parflow +++ b/models/parflow @@ -1 +1 @@ -Subproject commit 6ea209875f6478450ec0dbbff313ca7a97291b92 +Subproject commit 499ea4465e91aab96261679f3c26b3d73b159892 From a40d52e7b4500c0d5cc2a0a064d90b4bd5811a79 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Fri, 15 Nov 2024 18:32:22 +0100 Subject: [PATCH 60/71] Align y/n input to the same line as question --- build_tsmp2.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build_tsmp2.sh b/build_tsmp2.sh index 2a0d499..8bc419c 100755 --- a/build_tsmp2.sh +++ b/build_tsmp2.sh @@ -72,8 +72,7 @@ if [ -n "${comp_namey}" ] && [ -z "${comp_srcname}" ];then submodule_name=$(echo "models/"${sub_srcname}) fi if [ "$( ls -A ${cmake_tsmp2_dir}/${submodule_name} | wc -l)" -ne 0 ];then - echo "submodule ${submodule_name} aleady exist. Do you want overwrite it? (y/n)" - read yn + read -p "submodule ${submodule_name} aleady exists. Do you want overwrite it? (y/n) " yn if [ "${yn,}" = "y" ];then message "Overwrite submodule ${submodule_name}" git submodule update --init --force -- ${submodule_name} From 3285e163bc66ffce10e58017cb952a1f900e291d Mon Sep 17 00:00:00 2001 From: Johannes Keller Date: Tue, 19 Nov 2024 13:18:43 +0100 Subject: [PATCH 61/71] PDAF commit to head of `tsmp-pdaf-patched` --- models/pdaf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/pdaf b/models/pdaf index 5471753..b975d61 160000 --- a/models/pdaf +++ b/models/pdaf @@ -1 +1 @@ -Subproject commit 5471753077ffe28ec5963c3a4703c53e075e9611 +Subproject commit b975d61e11af05a9c60af028b1d8e5bc9acfda44 From da38a7c3f51cd4537835820000c3a354541bc2e6 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Thu, 21 Nov 2024 11:18:46 +0100 Subject: [PATCH 62/71] CI: Update apt before downloading packages --- .github/workflows/CI.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index c4e321c..8c4ea6f 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -47,7 +47,9 @@ jobs: execute_install_scripts: true - name: Download HDF5 headers and MPI Fortran compiler - run: sudo apt-get install gfortran openmpi-bin libopenmpi-dev libhdf5-openmpi-dev libhdf5-openmpi-103 hdf5-helpers + run: | + sudo apt-get update + sudo apt-get install gfortran openmpi-bin libopenmpi-dev libhdf5-openmpi-dev libhdf5-openmpi-103 hdf5-helpers - name: Create directories for component model and dependencies run: | From d9f33dd66031eedba9dadd4ce088c1e264ae5033 Mon Sep 17 00:00:00 2001 From: Paul Rigor Date: Tue, 26 Nov 2024 13:42:53 +0100 Subject: [PATCH 63/71] CI: Point to ParFlow commit with LLVM fixes See https://github.com/parflow/parflow/pull/567 --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 8c4ea6f..41b9cd6 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -29,7 +29,7 @@ jobs: MODEL_DIR: ${{ github.workspace }}/component-models VER_OASIS: tsmp-patches-v0.1 VER_eCLM: beta-0.1 - VER_PARFLOW: 499ea4465e91aab96261679f3c26b3d73b159892 # points to commit "Upgrade OASIS3-MCT CI to Ubuntu22.04" + VER_PARFLOW: ed3974987902ef24c1f7519eb0ba5f8f66380c8c # points to commit "Supported Ubuntu 24.04 and added LLVM to CI" VER_HYPRE: 2.26.0 VER_NETCDF_C: 4.9.2 VER_NETCDF_F90: 4.6.1 From 4052bf77b9bfe14da7f9ae6dfc02efe66f4c0e6d Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 27 Nov 2024 14:00:21 +0100 Subject: [PATCH 64/71] Point to ParFlow commit with LLVM fix (ed39749) --- models/parflow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/parflow b/models/parflow index 499ea44..ed39749 160000 --- a/models/parflow +++ b/models/parflow @@ -1 +1 @@ -Subproject commit 499ea4465e91aab96261679f3c26b3d73b159892 +Subproject commit ed3974987902ef24c1f7519eb0ba5f8f66380c8c From a799767ed4f8882e67dc79cffd8d72bae40ea2ea Mon Sep 17 00:00:00 2001 From: kvrigor Date: Wed, 27 Nov 2024 14:01:50 +0100 Subject: [PATCH 65/71] Use Intel-built Hypre --- env/jsc.2024_Intel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env/jsc.2024_Intel.sh b/env/jsc.2024_Intel.sh index a310489..492449f 100644 --- a/env/jsc.2024_Intel.sh +++ b/env/jsc.2024_Intel.sh @@ -9,7 +9,7 @@ module load ParaStationMPI/5.9.2-1 #module load IntelMPI/2021.10.0 # -module load Hypre/2.29.0 +module load Hypre/2.31.0-cpu module load Silo/4.11.1 module load Tcl/8.6.13 # From 157ebdcdd04d289dd822e114908f4ecf22f9f364 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Thu, 28 Nov 2024 15:24:58 +0100 Subject: [PATCH 66/71] Add EOL to FindNetCDF.cmake Cheatsheet: ```sh cat -e cmake/FindNetCDF.cmake sed -i -e '$a\' cmake/FindNetCDF.cmake ``` Sources: https://www.shellhacks.com/find-out-text-file-line-endings-lf-or-clrf/ https://unix.stackexchange.com/a/31955 --- cmake/FindNetCDF.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/FindNetCDF.cmake b/cmake/FindNetCDF.cmake index 96c92aa..9257d54 100644 --- a/cmake/FindNetCDF.cmake +++ b/cmake/FindNetCDF.cmake @@ -100,4 +100,4 @@ if(NetCDF_FOUND) else() message(WARNING "TSMP2 is using NetCDF C without parallel I/O support.") endif() -endif() \ No newline at end of file +endif() From 2f285d6a41e1db4e05dd1f8905dd180230b7adfb Mon Sep 17 00:00:00 2001 From: Johannes Keller <16795031+jjokella@users.noreply.github.com> Date: Mon, 2 Dec 2024 14:29:10 +0100 Subject: [PATCH 67/71] Stages/2024: PDAF-related updates and fixes (#43) Updates and bugfixes in the component models - ParFlow: Update to version `v3.13.0` and add PDAF-fixes producing new tag `v3.13.0-pdaf` (https://github.com/HPSCTerrSys/parflow/releases/tag/v3.13.0-pdaf) - CLM3.5: Bugfix and new tag `tsmp-patches-v0.3` (https://github.com/HPSCTerrSys/CLM3.5/releases/tag/tsmp-patches-v0.3) - PDAF: Update with general developments. --- .gitmodules | 2 +- README.md | 2 +- models/CLM3.5 | 2 +- models/parflow_pdaf | 2 +- models/pdaf | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 4d8c3de..182b1ce 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,7 +9,7 @@ [submodule "models/parflow_pdaf"] path = models/parflow_pdaf url = https://github.com/HPSCTerrSys/parflow - branch = tsmp-pdaf + branch = tsmp-pdaf-patched [submodule "models/pdaf"] path = models/pdaf url = https://github.com/HPSCTerrSys/pdaf.git diff --git a/README.md b/README.md index a0c02ea..e966e26 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ git clone -b v3.12.0 https://github.com/parflow/parflow.git models/parflow PARFLOW_SRC=`realpath models/parflow` # ParFlow (PDAF-patched) -git clone -b v3.12.0-tsmp https://github.com/HPSCTerrSys/parflow models/parflow_pdaf +git clone -b v3.13.0-pdaf https://github.com/HPSCTerrSys/parflow models/parflow_pdaf PARFLOW_SRC=`realpath models/parflow_pdaf` # CLM3.5 diff --git a/models/CLM3.5 b/models/CLM3.5 index 6301079..15dba17 160000 --- a/models/CLM3.5 +++ b/models/CLM3.5 @@ -1 +1 @@ -Subproject commit 6301079b68e08297a5eb7814bc90ad4f39e770c6 +Subproject commit 15dba177f27142c781fbd845770598c8db9c8be7 diff --git a/models/parflow_pdaf b/models/parflow_pdaf index d67aa13..24fd719 160000 --- a/models/parflow_pdaf +++ b/models/parflow_pdaf @@ -1 +1 @@ -Subproject commit d67aa13c831087d6f74ea07a1c5b6504d59b3d82 +Subproject commit 24fd719aa5c20356b44ef902db8a6d0b7b945847 diff --git a/models/pdaf b/models/pdaf index b975d61..555d5b4 160000 --- a/models/pdaf +++ b/models/pdaf @@ -1 +1 @@ -Subproject commit b975d61e11af05a9c60af028b1d8e5bc9acfda44 +Subproject commit 555d5b4ed561e30043b30bcf7fe9817b40da601f From df0c9388136ae64059d8c6fb5c31c65a52199020 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Tue, 3 Dec 2024 16:56:58 +0100 Subject: [PATCH 68/71] Only source env file if on JSC or --tsmp2_env is supplied --- build_tsmp2.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/build_tsmp2.sh b/build_tsmp2.sh index feee791..fcae238 100755 --- a/build_tsmp2.sh +++ b/build_tsmp2.sh @@ -209,14 +209,14 @@ fi # Makefile verbosity build_log="$(dirname ${cmake_build_dir})/${model_id}_$(date +%Y-%m-%d_%H-%M).log" -## source environment -if [[ $SYSTEMNAME = "jurecadc" || $SYSTEMNAME = "juwels" || $SYSTEMNAME = "jusuf" ]]; then - message "source environment" - if [ -z "${tsmp2_env}" ]; then +## source environment if on JSC or env file is provided +if [[ -z "${tsmp2_env}" && ($SYSTEMNAME = "jurecadc" || $SYSTEMNAME = "juwels" || $SYSTEMNAME = "jusuf") ]]; then tsmp2_env="${cmake_tsmp2_dir}/env/jsc.2024_Intel.sh" - else - tsmp2_env="${tsmp2_env}" - fi # tsmp2_env +else + tsmp2_env="${tsmp2_env}" +fi # tsmp2_env +if [ ! -z "{tsmp2_env}" ]; then + message "source environment" source $tsmp2_env fi From c82093a4477289be8b88d4e965676cd87c1922b8 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Tue, 3 Dec 2024 17:00:23 +0100 Subject: [PATCH 69/71] Fixed typo: {tsmp2_env} -> ${tsmp2_env} --- build_tsmp2.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_tsmp2.sh b/build_tsmp2.sh index fcae238..86c6bf9 100755 --- a/build_tsmp2.sh +++ b/build_tsmp2.sh @@ -215,7 +215,8 @@ if [[ -z "${tsmp2_env}" && ($SYSTEMNAME = "jurecadc" || $SYSTEMNAME = "juwels" | else tsmp2_env="${tsmp2_env}" fi # tsmp2_env -if [ ! -z "{tsmp2_env}" ]; then +if [ ! -z "${tsmp2_env}" ]; then + echo "tsmp2_env=${tsmp2_env}" message "source environment" source $tsmp2_env fi From 0925143bf021ce6d58c4cc0329336c46a1b1d922 Mon Sep 17 00:00:00 2001 From: kvrigor Date: Tue, 3 Dec 2024 17:10:04 +0100 Subject: [PATCH 70/71] Removed debugging statement --- build_tsmp2.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/build_tsmp2.sh b/build_tsmp2.sh index 86c6bf9..f906078 100755 --- a/build_tsmp2.sh +++ b/build_tsmp2.sh @@ -216,7 +216,6 @@ else tsmp2_env="${tsmp2_env}" fi # tsmp2_env if [ ! -z "${tsmp2_env}" ]; then - echo "tsmp2_env=${tsmp2_env}" message "source environment" source $tsmp2_env fi From 1a7d5ed7b7a1466a6bf608b699f6b26aad85725a Mon Sep 17 00:00:00 2001 From: Paul Rigor Date: Wed, 4 Dec 2024 17:39:01 +0100 Subject: [PATCH 71/71] README: Stages/2024 requires cloning ParFlow master instead of v3.12. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e966e26..a0a886e 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ git clone https://icg4geo.icg.kfa-juelich.de/spoll/icon2.6.4_oascoup.git models/ ICON_SRC=`realpath models/icon` # ParFlow -git clone -b v3.12.0 https://github.com/parflow/parflow.git models/parflow +git clone https://github.com/parflow/parflow.git models/parflow PARFLOW_SRC=`realpath models/parflow` # ParFlow (PDAF-patched)