diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000..41b9cd6 --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,235 @@ +name: TSMP2 Build + +# Controls when the action will run. +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + eclm_build_job: + name: ${{ matrix.config.name }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-22.04 ] + config: + - { + name: "eCLM-ParFlow", + } + + env: + CC: mpicc + CXX: mpicxx + FC: mpifort + F77: mpif77 + BUILD_DIR: bld + INSTALL_DIR: install + MODEL_DIR: ${{ github.workspace }}/component-models + VER_OASIS: tsmp-patches-v0.1 + VER_eCLM: beta-0.1 + VER_PARFLOW: ed3974987902ef24c1f7519eb0ba5f8f66380c8c # points to commit "Supported Ubuntu 24.04 and added LLVM to CI" + VER_HYPRE: 2.26.0 + VER_NETCDF_C: 4.9.2 + VER_NETCDF_F90: 4.6.1 + CMAKE_PREFIX_PATH: ${{ github.workspace }}/dependencies + SYSTEMNAME: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + + - name: Install TSMP2 dependencies on Ubuntu + uses: awalsh128/cache-apt-pkgs-action@latest + with: + packages: libxml++2.6-dev pylint wget cmake libpnetcdf-dev tcl-dev tk-dev + version: 1.0 + execute_install_scripts: true + + - name: Download HDF5 headers and MPI Fortran compiler + run: | + sudo apt-get update + sudo apt-get install gfortran openmpi-bin libopenmpi-dev libhdf5-openmpi-dev libhdf5-openmpi-103 hdf5-helpers + + - name: Create directories for component model and dependencies + run: | + mkdir -p ${CMAKE_PREFIX_PATH} ${MODEL_DIR} + + # + # NetCDF C + # + - name: Restore cached NetCDF C ${{ env.VER_NETCDF_C }} + uses: actions/cache/restore@v4 + id: cache-netcdf-restore + with: + path: ${{ env.CMAKE_PREFIX_PATH }} + key: cache-${{ matrix.config.name }}-${{ env.VER_NETCDF_C }} + + - if: steps.cache-netcdf-restore.outputs.cache-hit != 'true' + name: Install NetCDF C ${{ env.VER_NETCDF_C }} + run: | + wget https://github.com/Unidata/netcdf-c/archive/v${VER_NETCDF_C}.tar.gz + tar xf v${VER_NETCDF_C}.tar.gz + cd netcdf-c-${VER_NETCDF_C} + export CPPFLAGS=-I/usr/include/hdf5/openmpi + export LDFLAGS=-L/usr/lib/x86_64-linux-gnu/hdf5/openmpi + ./configure --prefix=$(realpath $CMAKE_PREFIX_PATH) + make -j4 install + + - if: steps.cache-netcdf-restore.outputs.cache-hit != 'true' + name: Cache NetCDF ${{ env.VER_NETCDF_C }} + uses: actions/cache/save@v4 + with: + path: ${{ env.CMAKE_PREFIX_PATH }} + key: cache-${{ matrix.config.name }}-${{ env.VER_NETCDF_C }} + + # + # NetCDF Fortran + # + - name: Restore cached NetCDF Fortran ${{ env.VER_NETCDF_F90 }} + uses: actions/cache/restore@v4 + id: cache-netcdff90-restore + with: + path: ${{ env.CMAKE_PREFIX_PATH }} + key: cache-${{ matrix.config.name }}-${{ env.VER_NETCDF_F90 }} + + - if: steps.cache-netcdff90-restore.outputs.cache-hit != 'true' + name: Install NetCDF Fortran ${{ env.VER_NETCDF_F90 }} + run: | + wget https://github.com/Unidata/netcdf-fortran/archive/v${VER_NETCDF_F90}.tar.gz + tar xf v${VER_NETCDF_F90}.tar.gz + cd netcdf-fortran-${VER_NETCDF_F90} + export CPPFLAGS=-I${CMAKE_PREFIX_PATH}/include + export LDFLAGS=-L${CMAKE_PREFIX_PATH}/lib + ./configure --prefix=$(realpath $CMAKE_PREFIX_PATH) + make -j4 install + + - if: steps.cache-netcdff90-restore.outputs.cache-hit != 'true' + name: Cache NetCDF Fortran ${{ env.VER_NETCDF_F90 }} + uses: actions/cache/save@v4 + with: + path: ${{ env.CMAKE_PREFIX_PATH }} + key: cache-${{ matrix.config.name }}-${{ env.VER_NETCDF_F90 }} + + # + # OASIS3-MCT + # + - name: Restore cached OASIS3-MCT ${{ env.VER_OASIS }} + uses: actions/cache/restore@v4 + id: cache-oasis-restore + with: + path: ${{ env.MODEL_DIR }}/oasis3-mct + key: cache-${{ matrix.config.name }}-${{ env.VER_OASIS }} + + - if: steps.cache-oasis-restore.outputs.cache-hit != 'true' + name: Download OASIS3-MCT ${{ env.VER_OASIS }} + working-directory: ${{ env.MODEL_DIR }} + run: | + git clone -b $VER_OASIS https://icg4geo.icg.kfa-juelich.de/ExternalReposPublic/oasis3-mct.git + echo "OASIS3-MCT version: $(cd oasis3-mct && git describe --tags --always)" + + - if: steps.cache-oasis-restore.outputs.cache-hit != 'true' + name: Cache OASIS3-MCT ${{ env.VER_OASIS }} + uses: actions/cache/save@v4 + with: + path: ${{ env.MODEL_DIR }}/oasis3-mct + key: cache-${{ matrix.config.name }}-${{ env.VER_OASIS }} + + # + # eCLM + # + - name: Restore cached eCLM ${{ env.VER_eCLM }} + uses: actions/cache/restore@v4 + id: cache-eclm-restore + with: + path: ${{ env.MODEL_DIR }}/eCLM + key: cache-${{ matrix.config.name }}-${{ env.VER_eCLM }} + + - if: steps.cache-eclm-restore.outputs.cache-hit != 'true' + name: Download eCLM ${{ env.VER_eCLM }} + working-directory: ${{ env.MODEL_DIR }} + run: | + git clone -b $VER_eCLM https://github.com/HPSCTerrSys/eCLM.git + echo "eCLM version: $(cd eCLM && git describe --tags --always)" + + - if: steps.cache-eclm-restore.outputs.cache-hit != 'true' + name: Cache eCLM ${{ env.VER_eCLM }} + uses: actions/cache/save@v4 + with: + path: ${{ env.MODEL_DIR }}/eCLM + key: cache-${{ matrix.config.name }}-${{ env.VER_eCLM }} + + # + # ParFlow + # + - name: Restore cached ParFlow ${{ env.VER_PARFLOW }} + uses: actions/cache/restore@v4 + id: cache-parflow-restore + with: + path: ${{ env.MODEL_DIR }}/parflow + key: cache-${{ matrix.config.name }}-${{ env.VER_PARFLOW }} + + # TODO: Must use upstream repo https://github.com/parflow/parflow.git + - if: steps.cache-parflow-restore.outputs.cache-hit != 'true' + name: Download ParFlow ${{ env.VER_PARFLOW }} + working-directory: ${{ env.MODEL_DIR }} + run: | + git clone https://github.com/parflow/parflow.git + cd parflow + git checkout $VER_PARFLOW + echo "ParFlow version: $(git describe --tags --always)" + + - if: steps.cache-parflow-restore.outputs.cache-hit != 'true' + name: Cache ParFlow ${{ env.VER_PARFLOW }} + uses: actions/cache/save@v4 + with: + path: ${{ env.MODEL_DIR }}/parflow + key: cache-${{ matrix.config.name }}-${{ env.VER_PARFLOW }} + + # + # Hypre + # + - name: Restore cached Hypre ${{ env.VER_HYPRE }} + uses: actions/cache/restore@v4 + id: cache-hypre-restore + with: + path: ${{ env.CMAKE_PREFIX_PATH }} + key: cache-${{ matrix.config.name }}-${{ env.VER_HYPRE }} + + - if: steps.cache-hypre-restore.outputs.cache-hit != 'true' + name: Install Hypre ${{ env.VER_HYPRE }} + run: | + wget https://github.com/hypre-space/hypre/archive/v${VER_HYPRE}.tar.gz + tar xf v${VER_HYPRE}.tar.gz + cd hypre-${VER_HYPRE}/src + ./configure --prefix=$(realpath $CMAKE_PREFIX_PATH) + make -j4 install + + - if: steps.cache-hypre-restore.outputs.cache-hit != 'true' + name: Cache Hypre ${{ env.VER_HYPRE }} + uses: actions/cache/save@v4 + with: + path: ${{ env.CMAKE_PREFIX_PATH }} + key: cache-${{ matrix.config.name }}-${{ env.VER_HYPRE }} + + # + # TSMP2 + # + - name: Configure TSMP2 + run: >- + cmake -S . -B ${BUILD_DIR} + -DCMAKE_BUILD_TYPE="RELEASE" + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} + -DOASIS_SRC=${MODEL_DIR}/oasis3-mct + -DeCLM="ON" + -DeCLM_SRC=${MODEL_DIR}/eCLM + -DParFlow="ON" + -DPARFLOW_SRC=${MODEL_DIR}/parflow + + - name: Build TSMP2 + run: | + cmake --build ${BUILD_DIR} + + - name: Install TSMP2 + run: | + cmake --install ${BUILD_DIR} diff --git a/.gitmodules b/.gitmodules index 4d8c3de..182b1ce 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,7 +9,7 @@ [submodule "models/parflow_pdaf"] path = models/parflow_pdaf url = https://github.com/HPSCTerrSys/parflow - branch = tsmp-pdaf + branch = tsmp-pdaf-patched [submodule "models/pdaf"] path = models/pdaf url = https://github.com/HPSCTerrSys/pdaf.git diff --git a/README.md b/README.md index 0caaa86..a0a886e 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ cd $TSMP2_DIR 2. Load the environment variables required for the build. ```bash -source env/jsc.2023_Intel.sh +source env/jsc.2024_Intel.sh ``` 3. Specify build and install directories. @@ -82,7 +82,7 @@ mkdir -p ${BUILD_DIR} ${INSTALL_DIR} ## NOTE: Download only the component models that you need! ## # eCLM -git clone https://github.com/HPSCTerrSys/eCLM.git models/eCLM +git clone -b beta-0.1 https://github.com/HPSCTerrSys/eCLM.git models/eCLM eCLM_SRC=`realpath models/eCLM` # ICON @@ -90,11 +90,11 @@ git clone https://icg4geo.icg.kfa-juelich.de/spoll/icon2.6.4_oascoup.git models/ ICON_SRC=`realpath models/icon` # ParFlow -git clone -b v3.12.0 https://github.com/parflow/parflow.git models/parflow +git clone https://github.com/parflow/parflow.git models/parflow PARFLOW_SRC=`realpath models/parflow` # ParFlow (PDAF-patched) -git clone -b v3.12.0-tsmp https://github.com/HPSCTerrSys/parflow models/parflow_pdaf +git clone -b v3.13.0-pdaf https://github.com/HPSCTerrSys/parflow models/parflow_pdaf PARFLOW_SRC=`realpath models/parflow_pdaf` # CLM3.5 diff --git a/build_tsmp2.sh b/build_tsmp2.sh index 7d16939..80ea280 100755 --- a/build_tsmp2.sh +++ b/build_tsmp2.sh @@ -72,8 +72,7 @@ if [ -n "${comp_namey}" ] && [ -z "${comp_srcname}" ];then submodule_name=$(echo "models/"${sub_srcname}) fi if [ "$( ls -A ${cmake_tsmp2_dir}/${submodule_name} | wc -l)" -ne 0 ];then - echo "submodule ${submodule_name} aleady exist. Do you want overwrite it? (y/n)" - read yn + read -p "submodule ${submodule_name} aleady exists. Do you want overwrite it? (y/n) " yn if [ "${yn,}" = "y" ];then message "Overwrite submodule ${submodule_name}" git submodule update --init --force -- ${submodule_name} @@ -188,7 +187,7 @@ else fi # set INSTALL and BUILD DIR (neccesary for building) -if [ -z "${SYSTEMNAME}" ]; then SYSTEMNAME="UNKN"; fi +if [ -z "${SYSTEMNAME}" ]; then export SYSTEMNAME=$(hostname); fi if [ -z "${build_dir}" ]; then cmake_build_dir="${cmake_tsmp2_dir}/bld/${SYSTEMNAME^^}_${model_id}" @@ -210,14 +209,16 @@ fi # Makefile verbosity build_log="$(dirname ${cmake_build_dir})/${model_id}_$(date +%Y-%m-%d_%H-%M).log" -## source environment -message "source environment" -if [ -z "${tsmp2_env}" ]; then - tsmp2_env="${cmake_tsmp2_dir}/env/jsc.2023_Intel.sh" +## source environment if on JSC or env file is provided +if [[ -z "${tsmp2_env}" && ($SYSTEMNAME = "jurecadc" || $SYSTEMNAME = "juwels" || $SYSTEMNAME = "jusuf") ]]; then + tsmp2_env="${cmake_tsmp2_dir}/env/jsc.2024_Intel.sh" else tsmp2_env="$(realpath ${tsmp2_env})" fi # tsmp2_env -source $tsmp2_env +if [ ! -z "${tsmp2_env}" ]; then + message "source environment" + source $tsmp2_env +fi ## CMAKE config # rm -rf ${cmake_build_dir} diff --git a/cmake/BuildCOSMO.cmake b/cmake/BuildCOSMO.cmake index fc923df..4bac495 100644 --- a/cmake/BuildCOSMO.cmake +++ b/cmake/BuildCOSMO.cmake @@ -1,5 +1,6 @@ find_package(ecCodes REQUIRED) find_package(NetCDF REQUIRED) +find_package(OpenMP REQUIRED) set(COSMO_BLD_DIR ${CMAKE_BINARY_DIR}/COSMO5_1/bld) set(COSMO_Fopts ${COSMO_BLD_DIR}/Fopts) @@ -15,7 +16,7 @@ file(APPEND ${COSMO_Fopts} "COMFLG4 = $(COMFLG1)\n") file(APPEND ${COSMO_Fopts} "COMFLG = $(COMFLG1)\n") file(APPEND ${COSMO_Fopts} "COMFLG5 = $(COMFLG1)\n") file(APPEND ${COSMO_Fopts} "LDPAR = ${CMAKE_Fortran_COMPILER}\n") -file(APPEND ${COSMO_Fopts} "LDFLG = -qopenmp\n") +file(APPEND ${COSMO_Fopts} "LDFLG = ${OpenMP_Fortran_FLAGS}\n") file(APPEND ${COSMO_Fopts} "PROGRAM = lmparbin\n") file(APPEND ${COSMO_Fopts} "LIBPATH = ${ecCodes_LIBRARIES} ${NetCDF_LIBRARIES} ${OASIS_LIBRARIES}\n") diff --git a/cmake/BuildOASIS3MCT.cmake b/cmake/BuildOASIS3MCT.cmake index d3cc924..c66840f 100644 --- a/cmake/BuildOASIS3MCT.cmake +++ b/cmake/BuildOASIS3MCT.cmake @@ -1,4 +1,5 @@ find_package(NetCDF REQUIRED) +find_package(OpenMP REQUIRED) if (CMAKE_BUILD_TYPE STREQUAL "DEBUG") set(OPTIM "-g") @@ -20,7 +21,7 @@ file(WRITE ${OASIS_MAKE_INC} "CHAN = MPI1\n") file(APPEND ${OASIS_MAKE_INC} "COUPLE = ${OASIS_SRC}\n") file(APPEND ${OASIS_MAKE_INC} "BUILD_DIR = ${OASIS_BLD_DIR}\n") file(APPEND ${OASIS_MAKE_INC} "ARCHDIR = ${OASIS_INSTALL_PREFIX}\n") -file(APPEND ${OASIS_MAKE_INC} "NETCDF_INCLUDE = ${NetCDF_ROOT}/include\n") +file(APPEND ${OASIS_MAKE_INC} "NETCDF_INCLUDE = ${NetCDF_F90_ROOT}/include\n") file(APPEND ${OASIS_MAKE_INC} "NETCDF_LIBRARY = ${NetCDF_LIBRARIES}\n") file(APPEND ${OASIS_MAKE_INC} "MPI_INCLUDE = ${MPI_Fortran_INCLUDE_DIRS}\n") file(APPEND ${OASIS_MAKE_INC} "MAKE = make\n") @@ -39,8 +40,14 @@ if(${PDAF}) else() file(APPEND ${OASIS_MAKE_INC} "CPPDEF = -Duse_netCDF -Duse_comm_$(CHAN) -D__VERBOSE -DTREAT_OVERLAY\n") endif() -file(APPEND ${OASIS_MAKE_INC} "FCBASEFLAGS = ${OPTIM} -xCORE-AVX2 -I. -assume byterecl -mt_mpi -qopenmp\n") -file(APPEND ${OASIS_MAKE_INC} "CCBASEFLAGS = ${OPTIM} -qopenmp\n") +if(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") + file(APPEND ${OASIS_MAKE_INC} "FCBASEFLAGS = ${OPTIM} -I. -ffree-line-length-none -fallow-argument-mismatch ${OpenMP_Fortran_FLAGS}\n") +elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "Intel" OR CMAKE_Fortran_COMPILER_ID STREQUAL "IntelLLVM") + file(APPEND ${OASIS_MAKE_INC} "FCBASEFLAGS = ${OPTIM} -I. -xCORE-AVX2 -assume byterecl -mt_mpi ${OpenMP_Fortran_FLAGS}\n") +else() + message(FATAL_ERROR "Fortran compiler '${CMAKE_Fortran_COMPILER_ID}' is not supported.") +endif() +file(APPEND ${OASIS_MAKE_INC} "CCBASEFLAGS = ${OPTIM} ${OpenMP_Fortran_FLAGS}\n") file(APPEND ${OASIS_MAKE_INC} "MCT_DEBUGFLAG = ${MCT_DEBUGFLAG}\n") file(APPEND ${OASIS_MAKE_INC} "FLIBS = ${NetCDF_LIBRARIES}\n") file(APPEND ${OASIS_MAKE_INC} "INC_DIRS = -I$(ARCHDIR)/include -I$(NETCDF_INCLUDE)\n") diff --git a/cmake/BuildPDAF.cmake b/cmake/BuildPDAF.cmake index 0e1a7da..dea776a 100644 --- a/cmake/BuildPDAF.cmake +++ b/cmake/BuildPDAF.cmake @@ -86,7 +86,8 @@ list(JOIN PDAF_LINK_LIBS " " PDAF_LINK_LIBS) # Set PDAF_FOPT for Makefile header # ---------------------------------- -if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") +if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel" + OR CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") # using Intel Compiler if (CMAKE_BUILD_TYPE STREQUAL "RELEASE") @@ -97,6 +98,8 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") list(APPEND PDAF_FOPT "-O0") list(APPEND PDAF_FOPT "-g") list(APPEND PDAF_FOPT "-traceback") + list(APPEND PDAF_FOPT "-fpe0") # compare eCLM debug flags + list(APPEND PDAF_FOPT "-check all") # compare eCLM debug flags else() message(FATAL_ERROR "Unsupported CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") endif() @@ -132,7 +135,8 @@ list(JOIN PDAF_FOPT " " PDAF_FOPT) # Set PDAF_COPT for Makefile header # ---------------------------------- -if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") +if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel" + OR CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") # using Intel Compiler if (CMAKE_BUILD_TYPE STREQUAL "RELEASE") @@ -178,7 +182,8 @@ list(JOIN PDAF_COPT " " PDAF_COPT) # Set PDAF_DOUBLEPRECISION for Makefile header # -------------------------------------------- -if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") +if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel" + OR CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") list(APPEND PDAF_DOUBLEPRECISION "-r8") diff --git a/cmake/BuildPDAFMODEL.cmake b/cmake/BuildPDAFMODEL.cmake index 0018b18..54e9f0d 100644 --- a/cmake/BuildPDAFMODEL.cmake +++ b/cmake/BuildPDAFMODEL.cmake @@ -17,7 +17,7 @@ set(TSMPPDAFLIBDIR "${CMAKE_INSTALL_PREFIX}/lib") # Include directories # ------------------- # General include dirs -list(APPEND PDAF_INCLUDES "-I${NetCDF_ROOT}/include") +list(APPEND PDAF_INCLUDES "-I${NetCDF_F90_ROOT}/include") # DA include dirs list(APPEND PDAF_INCLUDES "-I${PDAF_SRC}/interface/model") diff --git a/cmake/BuildParFlow.cmake b/cmake/BuildParFlow.cmake index 834b58a..17be4e2 100644 --- a/cmake/BuildParFlow.cmake +++ b/cmake/BuildParFlow.cmake @@ -1,10 +1,10 @@ -# TODO: Properly implement these flags! -set(PF_CFLAGS "-qopenmp -Wall -Werror") -set(PF_LDFLAGS "-lcudart -lcusparse -lcurand") - find_package(NetCDF REQUIRED) find_package(Hypre REQUIRED) +find_package(OpenMP REQUIRED) +# +# Choose which CLM to enable +# if(DEFINED eCLM_SRC) list(APPEND PF_CLM_FLAGS -DPARFLOW_AMPS_LAYER=oas3 -DOAS3_ROOT=${OASIS_ROOT} @@ -19,27 +19,50 @@ else() -DPARFLOW_HAVE_CLM=ON) endif() +# # TODO: Add compile switches for ParFlow GPU +# +if(CMAKE_C_COMPILER_ID STREQUAL "GNU") + # Flags were based from https://github.com/parflow/parflow/blob/77316043227b95215744e58fe9005d35145432ab/.github/workflows/linux.yml#L305 + set(PF_CFLAGS "${OpenMP_Fortran_FLAGS} -Wall -Werror -Wno-unused-result -Wno-unused-function") + set(PF_FFLAGS "-ffree-line-length-none -ffixed-line-length-none") + #TODO: These flags are specific to JSC system. This should be set in an env or build script! + set(PF_LDFLAGS "") + set(ENABLE_SLURM "OFF") +elseif(CMAKE_C_COMPILER_ID STREQUAL "Intel" OR CMAKE_C_COMPILER_ID STREQUAL "IntelLLVM") + set(PF_CFLAGS "${OpenMP_Fortran_FLAGS} -Wall -Werror -Wno-unused-function -Wno-unused-variable") + #TODO: These flags are specific to JSC system. This should be set in an env or build script! + set(PF_LDFLAGS "-lcudart -lcusparse -lcurand") + set(ENABLE_SLURM "ON") +else() + message(FATAL_ERROR "C compiler '${CMAKE_C_COMPILER_ID}' is not supported.") +endif() + +# +# Pass options to ParFlow CMake +# ExternalProject_Add(ParFlow PREFIX ParFlow SOURCE_DIR ${PARFLOW_SRC} CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_C_FLAGS=${PF_CFLAGS} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} + -DCMAKE_Fortran_FLAGS=${PF_FFLAGS} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} - ${PF_CLM_FLAGS} - -DNETCDF_DIR=${NetCDF_ROOT} - -DPARFLOW_AMPS_SEQUENTIAL_IO=on -DHYPRE_ROOT=${HYPRE_ROOT} + -DNETCDF_DIR=${NetCDF_C_ROOT} + -DPARFLOW_AMPS_SEQUENTIAL_IO=on -DPARFLOW_ENABLE_TIMING=TRUE - -DPARFLOW_ENABLE_SLURM=TRUE -DMPIEXEC_EXECUTABLE=${MPIEXEC_EXECUTABLE} -DMPIEXEC_NUMPROC_FLAG=${MPIEXEC_NUMPROC_FLAG} - -DCMAKE_C_FLAGS=${PF_CFLAGS} + -DPARFLOW_ENABLE_SLURM=${ENABLE_SLURM} -DCMAKE_EXE_LINKER_FLAGS=${PF_LDFLAGS} + ${PF_CLM_FLAGS} + ${JSC_FLAGS} DEPENDS ${MODEL_DEPENDENCIES} ) get_model_version(${PARFLOW_SRC} PARFLOW_VERSION) -list(APPEND TSMP2_MODEL_VERSIONS "ParFlow: ${PARFLOW_VERSION}") \ No newline at end of file +list(APPEND TSMP2_MODEL_VERSIONS "ParFlow: ${PARFLOW_VERSION}") diff --git a/cmake/FindMPIFortran.cmake b/cmake/FindMPIFortran.cmake index a0f13dc..72bf359 100644 --- a/cmake/FindMPIFortran.cmake +++ b/cmake/FindMPIFortran.cmake @@ -3,18 +3,35 @@ # FindMPI.cmake is supposed to automatically set MPI_Fortran_INCLUDE_DIRS, but for some reason # it doesn't despite knowing where the MPI compilers are. This module serves as a workaround # for this issue. +# +# Updates +# ------- +# 2024-10-09: Built-in FindMPI still unreliable for various MPI Fortran implementations. +# +# [1]: https://gitlab.kitware.com/cmake/cmake/-/blob/master/Modules/FindMPI.cmake # -# [1]: https://gitlab.kitware.com/cmake/cmake/blob/v3.21.1/Modules/FindMPI.cmake) find_package(MPI REQUIRED) find_package(PkgConfig QUIET) include(FindPackageHandleStandardArgs) -if (NOT MPI_Fortran_INCLUDE_DIRS) - pkg_check_modules(MPICH_Fortran REQUIRED mpich) +if (NOT MPI_Fortran_INCLUDE_DIRS OR NOT MPI_Fortran_LIB_DIR) + # TODO: Preferred MPI implementation should be introspected + # from host system environment. For now we only consider + # JSC (ParaStationMPI) and Ubuntu machines (OpenMPI). + + # MPICH + pkg_check_modules(MPICH_Fortran mpich) if (MPICH_Fortran_FOUND) pkg_get_variable(MPI_Fortran_INCLUDE_DIRS mpich includedir) pkg_get_variable(MPI_Fortran_LIB_DIR mpich libdir) + else() + # OpenMPI + pkg_check_modules(OpenMPI_Fortran REQUIRED mpi) + if (OpenMPI_Fortran_FOUND) + pkg_get_variable(MPI_Fortran_INCLUDE_DIRS mpi includedir) + pkg_get_variable(MPI_Fortran_LIB_DIR mpi libdir) + endif() endif() endif() @@ -25,4 +42,4 @@ find_package_handle_standard_args(MPIFortran MPI_Fortran_LIB_DIR MPIEXEC_EXECUTABLE MPIEXEC_NUMPROC_FLAG - VERSION_VAR MPI_Fortran_VERSION) \ No newline at end of file + VERSION_VAR MPI_Fortran_VERSION) diff --git a/cmake/FindNetCDF.cmake b/cmake/FindNetCDF.cmake index 80b8ac4..9257d54 100644 --- a/cmake/FindNetCDF.cmake +++ b/cmake/FindNetCDF.cmake @@ -1,22 +1,103 @@ find_package(PkgConfig QUIET) include(FindPackageHandleStandardArgs) -find_package(NetCDF_C QUIET NAMES netCDF) -if(NetCDF_C_FOUND) - set(NetCDF_C_LIB_DIR "${netCDF_LIB_DIR}") +# ************************ +# 1. Find NetCDF C library +# ************************ + +# Attempt 1: Prioritize finding a parallel-aware NetCDF. +find_path(NetCDF_C_INCLUDEDIR NAMES netcdf_par.h PATH_SUFFIXES include) + +if(NetCDF_C_INCLUDEDIR) + # Attempt 1 succeeded! + get_filename_component(NetCDF_C_ROOT ${NetCDF_C_INCLUDEDIR} DIRECTORY) + set(NetCDF_C_LIB_DIR "${NetCDF_C_ROOT}/lib") + set(NetCDF_HAS_PARALLEL TRUE) + + # Run `cmake ... --log-level=DEBUG` to see these debugging information. + message(DEBUG "FindNETCDF_1: NetCDF_C_ROOT=${NetCDF_C_ROOT}") + message(DEBUG "FindNETCDF_1: NetCDF_C_LIB_DIR=${NetCDF_C_LIB_DIR}") + message(DEBUG "FindNETCDF_1: NetCDF_C_INCLUDEDIR=${NetCDF_C_INCLUDEDIR}") +else() + # Attempt 2: Use find_package() to find NetCDF. The complicated logical + # structure below reflects the voodoo behavior of + # find_package() + buggy NetCDF CMake config file + platform-specific filesystem layout + find_package(NetCDF_C NAMES netCDF) + if(NetCDF_C_FOUND) + set(NetCDF_C_ROOT ${netCDF_INSTALL_PREFIX}) + set(NetCDF_C_LIB_DIR ${netCDF_LIB_DIR}) + message(DEBUG "FindNETCDF_2: NetCDF_C_ROOT=${NetCDF_C_ROOT}") + message(DEBUG "FindNETCDF_2: NetCDF_C_LIB_DIR=${NetCDF_C_LIB_DIR}") + + # Don't trust what netCDF_HAS_PARALLEL reports. The existence of netcdf_par.h header + # is the surest way to know that netCDF was built with parallel support. + find_path(NetCDF_C_INCLUDEDIR NAMES netcdf_par.h PATH_SUFFIXES ${NetCDF_C_ROOT}/include) + + if(NetCDF_C_INCLUDEDIR) + # Result 2.1: find_package() gets lucky in finding parallel-aware NetCDF + set(NetCDF_HAS_PARALLEL TRUE) + message(DEBUG "FindNETCDF_21: NetCDF_C_INCLUDEDIR=${NetCDF_C_INCLUDEDIR}") + else() + # Result 2.2: find_package() only finds serial-aware NetCDF + set(NetCDF_HAS_PARALLEL FALSE) + set(NetCDF_C_INCLUDEDIR ${NetCDF_C_ROOT}/include) + message(DEBUG "FindNETCDF_22: NetCDF_C_INCLUDEDIR=${NetCDF_C_INCLUDEDIR}") + + # TODO: It's possible to keep searching for a parallel-aware NetCDF, but + # I'm not yet aware of any method that isn't too platform-specific. + endif() + + # --------------------------------------------------------------------------- + # 31-Oct-2024 kvrigor: The snippet below works for Ubuntu. The package + # `libnetcdf-mpi-dev` is a parallel-aware NetCDF library, but it takes + # some trickery to extract the correct prefix path, which doesn't really + # work 100% of the time (e.g. see [1]). Baking such fragile system-specific + # manouevre here doesn't seem to be worth it; anyway I'm leaving this piece + # of code here in case somebody would like to investigate this further. + # + # [1]: https://github.com/Unidata/netcdf-c/issues/2069 + # --------------------------------------------------------------------------- + # + # pkg_check_modules(NetCDF_C_MPI netcdf-mpi) + # if (NetCDF_C_MPI_FOUND) + # pkg_get_variable(NetCDF_C_ROOT netcdf-mpi prefix) + # set(NetCDF_C_LIB_DIR "${NetCDF_C_ROOT}/lib") + # set(NetCDF_C_INCLUDEDIR "${NetCDF_C_ROOT}/lib") + # set(NetCDF_HAS_PARALLEL "ON") + # message(DEBUG "FindNETCDF_MPI: NetCDF_C_ROOT=${NetCDF_C_ROOT}") + # message(DEBUG "FindNETCDF_MPI: NetCDF_C_LIB_DIR=${NetCDF_C_LIB_DIR}") + # message(DEBUG "FindNETCDF_MPI: NetCDF_C_INCLUDEDIR=${NetCDF_C_ROOT}") + # endif() + # + # -------------------------------------------------------------------- + endif() endif() +# ************************ +# 2. Find NetCDF Fortran +# ************************ pkg_check_modules(NetCDF_F90 REQUIRED netcdf-fortran) if (NetCDF_F90_FOUND) - pkg_get_variable(NetCDF_F90_LIB_DIR netcdf-fortran libdir) pkg_get_variable(NetCDF_F90_ROOT netcdf-fortran prefix) + pkg_get_variable(NetCDF_F90_LIB_DIR netcdf-fortran libdir) + pkg_get_variable(NetCDF_F90_INCLUDEDIR netcdf-fortran fmoddir) endif() + +# ************************************* +# 3. Set paths to NetCDF C and Fortran +# ************************************* find_package_handle_standard_args(NetCDF - REQUIRED_VARS NetCDF_C_LIB_DIR NetCDF_F90_LIB_DIR + REQUIRED_VARS NetCDF_C_ROOT NetCDF_F90_ROOT NetCDF_C_LIB_DIR NetCDF_F90_LIB_DIR NetCDF_F90_INCLUDEDIR VERSION_VAR NetCDF_F90_VERSION) if(NetCDF_FOUND) set(NetCDF_LIBRARIES "-L${NetCDF_C_LIB_DIR} -L${NetCDF_F90_LIB_DIR} -lnetcdff -lnetcdf" CACHE STRING "NetCDF linker options") - set(NetCDF_ROOT ${NetCDF_F90_ROOT} CACHE PATH "Full path to the root directory containing NetCDF include files and libraries.") -endif() \ No newline at end of file + set(NetCDF_F90_ROOT ${NetCDF_F90_ROOT} CACHE PATH "Path to NetCDF-Fortran directory which contains its include header files and libraries.") + set(NetCDF_C_ROOT ${NetCDF_C_ROOT} CACHE PATH "Path to NetCDF-C directory which contains its include header files and libraries.") + if (${NetCDF_HAS_PARALLEL}) + message(STATUS "TSMP2 found NetCDF C built with parallel I/O support.") + else() + message(WARNING "TSMP2 is using NetCDF C without parallel I/O support.") + endif() +endif() diff --git a/env/jsc.2024_Intel.sh b/env/jsc.2024_Intel.sh new file mode 100644 index 0000000..492449f --- /dev/null +++ b/env/jsc.2024_Intel.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Load modules +module --force purge +module use $OTHERSTAGES +module load Stages/2024 +module load Intel/2023.2.1 +module load ParaStationMPI/5.9.2-1 +#module load IntelMPI/2021.10.0 + +# +module load Hypre/2.31.0-cpu +module load Silo/4.11.1 +module load Tcl/8.6.13 +# +module load ecCodes/2.31.0 +module load HDF5/1.14.2 +module load netCDF/4.9.2 +module load netCDF-Fortran/4.6.1 +module load PnetCDF/1.12.3 +module load cURL/8.0.1 +module load Szip/.2.1.1 +module load Python/3.11.3 +module load NCO/5.1.8 +module load CMake/3.26.3 +module load git/2.41.0-nodocs + +module li + +# Set default compilers +export CC=mpicc +export FC=mpif90 +export CXX=mpicxx +export MPI_HOME=$EBROOTPSMPI diff --git a/models/CLM3.5 b/models/CLM3.5 index 6301079..15dba17 160000 --- a/models/CLM3.5 +++ b/models/CLM3.5 @@ -1 +1 @@ -Subproject commit 6301079b68e08297a5eb7814bc90ad4f39e770c6 +Subproject commit 15dba177f27142c781fbd845770598c8db9c8be7 diff --git a/models/eCLM b/models/eCLM index 4582f14..c9f3348 160000 --- a/models/eCLM +++ b/models/eCLM @@ -1 +1 @@ -Subproject commit 4582f148490edf621cbc1a6f11692017bc344469 +Subproject commit c9f334838b4584e8543dc3e6f7e50661433b2055 diff --git a/models/parflow b/models/parflow index 6ea2098..ed39749 160000 --- a/models/parflow +++ b/models/parflow @@ -1 +1 @@ -Subproject commit 6ea209875f6478450ec0dbbff313ca7a97291b92 +Subproject commit ed3974987902ef24c1f7519eb0ba5f8f66380c8c diff --git a/models/parflow_pdaf b/models/parflow_pdaf index d67aa13..24fd719 160000 --- a/models/parflow_pdaf +++ b/models/parflow_pdaf @@ -1 +1 @@ -Subproject commit d67aa13c831087d6f74ea07a1c5b6504d59b3d82 +Subproject commit 24fd719aa5c20356b44ef902db8a6d0b7b945847 diff --git a/models/pdaf b/models/pdaf index a2b6a47..555d5b4 160000 --- a/models/pdaf +++ b/models/pdaf @@ -1 +1 @@ -Subproject commit a2b6a4718f5eac9f04960ddf36a89d4420b18aa3 +Subproject commit 555d5b4ed561e30043b30bcf7fe9817b40da601f