Merge branch 'to4.3' into ruisdael

merging for gfortran 10 compatibility

CHANGELOG.md

@@ -7,7 +7,8 @@ Version 4.3 - 2020
 This version introduces a library interface to DALES, defined in `daleslib.f90.` With this interface,
 DALES can be used with the [OMUSE](https://omuse.readthedocs.io/en/latest/) framework, which provides
 a Python interface to DALES. Another new feature is the support of iterative Poisson solvers, see
-the [Wiki](https://github.com/dalesteam/dales/wiki/Iterative-Poisson-solver)
+the [Wiki](https://github.com/dalesteam/dales/wiki/Iterative-Poisson-solver) and also an
+alternative FFT-based solver using the FFTW library.
 
 
 ### Improvements

CMakeLists.txt

@@ -28,9 +28,10 @@ elseif("$ENV{SYST}" STREQUAL "HYDRA")
   set(CMAKE_Fortran_FLAGS_DEBUG "-traceback -fpe1 -O0 -g -check all" CACHE STRING "")
 elseif("$ENV{SYST}" STREQUAL "FEDORA")
   set(CMAKE_Fortran_COMPILER "mpif90")
-  set(CMAKE_Fortran_FLAGS "-finit-real=nan -fdefault-real-8 -ffree-line-length-none -I /usr/lib64/gfortran/modules/mpich/" CACHE STRING "")
+  set(CMAKE_Fortran_FLAGS "-finit-real=nan -fdefault-real-8 -ffree-line-length-none -I /usr/lib64/gfortran/modules/mpich/ -std=legacy" CACHE STRING "")
   set (CMAKE_Fortran_FLAGS_RELEASE "-funroll-all-loops -fno-f2c -O3" CACHE STRING "")
   set (CMAKE_Fortran_FLAGS_DEBUG   "-fbounds-check -fbacktrace -fno-f2c -O0 -g -ffpe-trap=invalid,zero,overflow" CACHE STRING "")
+  set (CMAKE_Fortran_FLAGS_DEV     "-fbounds-check -fbacktrace -fno-f2c -march=native -O3 -g -ffpe-trap=invalid,zero,overflow" CACHE STRING "")
 elseif("$ENV{SYST}" STREQUAL "ECMWF")
   set(CMAKE_Fortran_COMPILER "ftn")
   set(CMAKE_Fortran_FLAGS "-s real64" CACHE STRING "")
@@ -48,7 +49,7 @@ elseif("$ENV{SYST}" STREQUAL "ECMWF-gnu")
   set(CMAKE_Fortran_FLAGS_DEBUG   "-fbounds-check -fbacktrace -fno-f2c -O0 -g -ffpe-trap=invalid,zero,overflow" CACHE STRING "")
 elseif("$ENV{SYST}" STREQUAL "gnu-fast")
   set(CMAKE_Fortran_COMPILER "mpif90")
-  set(CMAKE_Fortran_FLAGS "-finit-real=nan -W -Wall -fdefault-real-8 -ffree-line-length-none" CACHE STRING "")
+  set(CMAKE_Fortran_FLAGS "-finit-real=nan -W -Wall -fdefault-real-8 -ffree-line-length-none -std=legacy" CACHE STRING "")
   set (CMAKE_Fortran_FLAGS_RELEASE "-funroll-all-loops -fno-f2c -Ofast -march=native -g -fbacktrace" CACHE STRING "")
   set (CMAKE_Fortran_FLAGS_DEBUG   "-fbounds-check -fbacktrace -fno-f2c -O0 -g -ffpe-trap=invalid,zero,overflow" CACHE STRING "")
 elseif("$ENV{SYST}" STREQUAL "lisa-intel")
@@ -58,11 +59,16 @@ elseif("$ENV{SYST}" STREQUAL "lisa-intel")
    set(CMAKE_Fortran_FLAGS_DEBUG "-traceback -fpe1 -O0 -g -check all" CACHE STRING "")
 else()
   set(CMAKE_Fortran_COMPILER "mpif90")
-  set(CMAKE_Fortran_FLAGS "-finit-real=nan -fdefault-real-8 -ffree-line-length-none " CACHE STRING "")
+  set(CMAKE_Fortran_FLAGS "-finit-real=nan -fdefault-real-8 -ffree-line-length-none -std=legacy" CACHE STRING "")
   set (CMAKE_Fortran_FLAGS_RELEASE "-funroll-all-loops -fno-f2c -O3" CACHE STRING "")
   set (CMAKE_Fortran_FLAGS_DEBUG   "-fbounds-check -fbacktrace -fno-f2c -O0 -g -ffpe-trap=invalid,zero,overflow" CACHE STRING "")
 endif()
 
+# On compiler options
+# June 2020: added -std=legacy to gfortran for compatibility with gfortran 10, which introduced stricter type checking
+# the old FFT code from Netlib/FFTPACK does not compile with gfortran 10 without this flag.
+
+
 ## Project parameters
 PROJECT(DALES Fortran)
 cmake_minimum_required(VERSION 2.6)
@@ -122,7 +128,8 @@ FIND_LIBRARY(NETCDF_FORTRAN_LIB netcdff
   DOC "NetCDF Fortran library"
 )
 
-FIND_LIBRARY(HYPRE_LIB libHYPRE.a
+# libHYPRE.a for static library
+FIND_LIBRARY(HYPRE_LIB HYPRE
   PATHS
   $ENV{HYPRE_LIB}
   ${ADDMODULEPATH}/lib
@@ -134,12 +141,34 @@ FIND_LIBRARY(HYPRE_LIB libHYPRE.a
   DOC "Iterative solver library"
 )
 
+FIND_PATH(FFTW_INCLUDE_DIR fftw3.f03
+  PATHS
+  $ENV{FFTW_INCLUDE_DIR}
+  /usr/include
+  DOC "FFTW include files (fftw3.f03)"
+)
+
+#libfftw3.a for static library
+FIND_LIBRARY(FFTW_LIB fftw3
+  PATHS
+  $ENV{FFTW_LIB}
+  /usr/lib
+  /usr/lib64
+  DOC "FFTW static library"
+)
+
 if(NETCDF_INCLUDE_DIR)
   include_directories(${NETCDF_INCLUDE_DIR})
 else(NETCDF_INCLUDE_DIR)
   MESSAGE(STATUS "WARNING: No NETCDF bindings are found.")
 endif(NETCDF_INCLUDE_DIR)
 
+if(FFTW_INCLUDE_DIR)
+  include_directories(${FFTW_INCLUDE_DIR})
+else(FFTW_INCLUDE_DIR)
+  MESSAGE(STATUS "WARNING: No FFTW3 bindings are found (fftw3.f03).")
+endif(FFTW_INCLUDE_DIR)
+
 if(NETCDF_C_LIB)
   set(NETCDF_LIBS ${NETCDF_C_LIB})
 else(NETCDF_C_LIB)
@@ -155,6 +184,9 @@ endif(NETCDF_FORTRAN_LIB)
 ### Iterative solver (HYPRE)
 OPTION(USE_HYPRE "Also build iterative solver (optional, needs HYPRE)" OFF)
 
+### FFTW based poisson solver (FFTW)
+OPTION(USE_FFTW "Also build FFTW based poisson solver (optional, needs FFTW3)" OFF)
+
 ### Documentation
 INCLUDE(FindDoxygen)
 if(DOXYGEN)

cases/benchmark/ruisdael/namoptions-1728.001

@@ -11,7 +11,7 @@ peclet = 0.1
 irandom = 1
 randthl = 0.01
 randqt = 2.5e-5
-nsv = 3
+nsv = 2
 ! nprocx = 3
 ! nprocy = 8
 /

cases/benchmark/ruisdael/namoptions-192.001

@@ -11,7 +11,7 @@ peclet = 0.1
 irandom = 1
 randthl = 0.01
 randqt = 2.5e-5
-nsv = 3
+nsv = 2
 ! nprocx = 3
 ! nprocy = 8
 /

cases/benchmark/ruisdael/namoptions-96.001

@@ -11,7 +11,7 @@ peclet = 0.1
 irandom = 1
 randthl = 0.01
 randqt = 2.5e-5
-nsv = 3
+nsv = 2
 ! nprocx = 3
 ! nprocy = 8
 /

src/CMakeLists.txt

@@ -5,6 +5,8 @@ set(target_lib dales)
 # This will also add some unwated files, which we will remove again below
 FILE(GLOB sourcefiles "*.f90")
 
+list(REMOVE_ITEM sourcefiles ${CMAKE_CURRENT_SOURCE_DIR}/test_transposes.f90)
+
 # Use git-version.cmake to create modversion.f90, containing a version string from git, e.g. "4.2-34-g62b85a-dirty"
 add_custom_target(tag_git_version ALL
   COMMAND ${CMAKE_COMMAND} -D TARGET_DIR=${CMAKE_CURRENT_BINARY_DIR} -P ${CMAKE_CURRENT_SOURCE_DIR}/git-version.cmake
@@ -15,31 +17,30 @@ add_custom_target(tag_git_version ALL
 if(USE_HYPRE)
   # Build the iterative solver
   list(REMOVE_ITEM sourcefiles ${CMAKE_CURRENT_SOURCE_DIR}/modnohypre.f90)
-
-  # Stand-alone DALES program
-  add_executable(${target_name} ${sourcefiles} ${CMAKE_CURRENT_BINARY_DIR}/modversion.f90) 
-  target_link_libraries(${target_name} ${NETCDF_LIBS} ${HYPRE_LIB})
-
+  list(APPEND OPTIONAL_LIBS ${HYPRE_LIB})
 else(USE_HYPRE)
   # Do not build the iterative solver, but use (empty) fallback functions
   list(REMOVE_ITEM sourcefiles ${CMAKE_CURRENT_SOURCE_DIR}/modhypre.f90)
+endif(USE_HYPRE)
 
-  # Stand-alone DALES program
-  add_executable(${target_name} ${sourcefiles} ${CMAKE_CURRENT_BINARY_DIR}/modversion.f90) 
-  target_link_libraries(${target_name} ${NETCDF_LIBS})
+if(USE_FFTW)
+  # Build the iterative solver
+  list(REMOVE_ITEM sourcefiles ${CMAKE_CURRENT_SOURCE_DIR}/modnofftw.f90)
+  list(APPEND OPTIONAL_LIBS ${FFTW_LIB})
+else(USE_FFTW)
+  # Do not build the iterative solver, but use (empty) fallback functions
+  list(REMOVE_ITEM sourcefiles ${CMAKE_CURRENT_SOURCE_DIR}/modfftw.f90)
+endif(USE_FFTW)
 
-endif(USE_HYPRE)
+# Stand-alone DALES program
+add_executable(${target_name} ${sourcefiles} ${CMAKE_CURRENT_BINARY_DIR}/modversion.f90)
+target_link_libraries(${target_name} ${NETCDF_LIBS} ${OPTIONAL_LIBS})
 
 # DALES library, e.g. for use with OMUSE
 add_library(${target_lib} ${sourcefiles} ${CMAKE_CURRENT_BINARY_DIR}/modversion.f90)
 
-
 # set separate module directories for the program and library targets, to avoid problems during parallel builds (make -j 8)
 set_target_properties(${target_name} PROPERTIES Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/program_modules")
 set_target_properties(${target_lib} PROPERTIES Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/library_modules")
 
-target_link_libraries(${target_name} ${NETCDF_LIBS})
-
 install(TARGETS dales4 dales DESTINATION ${CMAKE_BINARY_DIR})
-
-

src/modAGScross.f90

@@ -51,6 +51,7 @@ module modAGScross
 contains
 !> Initializing AGScross. Read out the namelist, initializing the variables
   subroutine initAGScross
+    use mpi
     use modmpi,   only :myid,my_real,mpierr,comm3d,mpi_logical,cmyid
     use modglobal,only :imax,jmax,ifnamopt,fname_options,dtmax, dtav_glob,ladaptive,dt_lim,cexpnr,tres,btime,checknamelisterror
     use modstat_nc,only : open_nc, define_nc,ncinfo,writestat_dims_nc,nctiminfo

src/modbudget.f90

@@ -75,6 +75,7 @@ module modbudget
 contains
 !> Initialization routine, reads namelists and inits variables
   subroutine initbudget
+    use mpi
     use modmpi,    only : myid,mpierr, comm3d,my_real, mpi_logical
     use modglobal, only : dtmax,k1,ifnamopt,fname_options, ifoutput,cexpnr,dtav_glob,timeav_glob,&
     ladaptive,dt_lim,btime,tres,lwarmstart,checknamelisterror

src/modbulkmicrostat.f90

@@ -83,6 +83,7 @@ module modbulkmicrostat
 contains
 !> Initialization routine, reads namelists and inits variables
 subroutine initbulkmicrostat
+    use mpi
     use modmpi,    only  : myid, mpi_logical, my_real, comm3d, mpierr
     use modglobal, only  : ifnamopt, fname_options, cexpnr, ifoutput, &
          dtav_glob, timeav_glob, ladaptive, k1, dtmax,btime,tres,lwarmstart,checknamelisterror

src/modcanopy.f90

@@ -55,6 +55,7 @@ module modcanopy
 contains
 !-----------------------------------------------------------------------------------------
   SUBROUTINE initcanopy
+    use mpi
     use modmpi,    only : myid, mpi_logical, mpi_integer, my_real, comm3d, mpierr
     use modglobal, only : kmax, ifnamopt, fname_options, ifinput, cexpnr, zh, dzh, dzf, checknamelisterror
 

src/modcape.f90

@@ -47,6 +47,7 @@ module modcape
 
 !> Initializing cape crossections. Read out the namelist, initializing the variables
   subroutine initcape
+    use mpi
     use modmpi,   only :myid,my_real,mpierr,comm3d,mpi_logical,cmyid
     use modglobal,only :imax,jmax,ifnamopt,fname_options,dtmax,dtav_glob,ladaptive,dt_lim,cexpnr,tres,btime,checknamelisterror
     use modstat_nc,only : lnetcdf,open_nc, define_nc, redefine_nc,ncinfo,nctiminfo,writestat_dims_nc

src/modchecksim.f90

@@ -45,6 +45,7 @@ module modchecksim
 !> Initializing Checksim. Read out the namelist, initializing the variables
   subroutine initchecksim
     use modglobal, only : ifnamopt, fname_options,dtmax,ladaptive,btime,tres,checknamelisterror
+    use mpi
     use modmpi,    only : myid,my_real,comm3d,mpierr
     implicit none
     integer :: ierr
@@ -97,6 +98,7 @@ end subroutine checksim
   subroutine calccourant
     use modglobal, only : i1,j1,kmax,dx,dy,dzh
     use modfields, only : u0,v0,w0
+    use mpi
     use modmpi,    only : myid,comm3d,mpierr,mpi_max,my_real
     implicit none
 
@@ -136,6 +138,7 @@ subroutine calcpeclet
 
     use modglobal, only : i1,j1,k1,kmax,dx,dy,dzh
     use modsubgrid,only : ekm
+    use mpi
     use modmpi,    only : myid,comm3d,mpierr,mpi_max,my_real
     implicit none
 
@@ -164,6 +167,7 @@ subroutine chkdiv
 
     use modglobal, only : i1,j1,kmax,dx,dy,dzf,dt_reason
     use modfields, only : u0,v0,w0,rhobf,rhobh
+    use mpi
     use modmpi,    only : myid,comm3d,mpi_sum,mpi_max,my_real,mpierr
     implicit none
 

src/modchem.f90

@@ -239,6 +239,7 @@ module modchem
 !-----------------------------------------------------------------------------------------
 SUBROUTINE initchem
   use modglobal,   only : i1,j1,nsv, ifnamopt, fname_options, ifoutput, cexpnr,timeav_glob,btime,tres,lwarmstart,checknamelisterror
+  use mpi
   use modmpi,      only : myid, mpi_logical, mpi_integer, my_real, comm3d, mpierr
   use modsurfdata, only : lCHon
   implicit none
@@ -1136,6 +1137,7 @@ SUBROUTINE twostep2(y)
 !c
 use modglobal, only : ih,i1,jh,j1,k1,kmax,rtimee,rdt,timee,timeav_glob,ifoutput,cexpnr,dz,ijtot
 use modfields, only : qt0
+use mpi
 use modmpi, only: comm3d, mpierr,mpi_max,mpi_min,mpi_sum,my_real,myid,nprocs
 use modtimestat, only: we, zi, ziold, calcblheight
 
@@ -1786,6 +1788,7 @@ subroutine ratech
   use modglobal, only : i1,j1,kmax,pi,xtime,timee,rtimee,xday,xlat,xlon, &
                         zf,dzf,ijtot,ifoutput,cexpnr
   use modfields, only : qt0, ql0 ,rhof
+  use mpi
   use modmpi,    only : myid, comm3d, mpierr, mpi_max, my_real, mpi_sum
   use modsurfdata,only: taufield, lrsAgs
   use modraddata,only: iradiation, irad_par

src/modcloudfield.f90

@@ -42,7 +42,7 @@ module modcloudfield
 contains
 !> Initializing Cloudfield. Read out the namelist, initializing the variables
   subroutine initcloudfield
-
+    use mpi
     use modmpi,   only :myid,my_real,mpierr,comm3d,mpi_logical
     use modglobal,only :ifnamopt,fname_options,dtmax,dtav_glob,btime,ladaptive,tres,checknamelisterror
     implicit none

src/modcrosssection.f90

@@ -67,6 +67,7 @@ module modcrosssection
 contains
 !> Initializing Crosssection. Read out the namelist, initializing the variables
   subroutine initcrosssection
+    use mpi
     use modmpi,   only :myid,my_real,mpierr,comm3d,mpi_logical,mpi_integer,cmyid,myidx,myidy
     use modglobal,only :imax,jmax,ifnamopt,fname_options,dtmax,dtav_glob,ladaptive,j1,kmax,i1,dt_lim,cexpnr,tres,btime,checknamelisterror
     use modstat_nc,only : lnetcdf,open_nc, define_nc,ncinfo,nctiminfo,writestat_dims_nc