Merge remote-tracking branch 'frans/ruisdael' into ruisdael

CMakeLists.txt

@@ -60,7 +60,7 @@ elseif("$ENV{SYST}" STREQUAL "lisa-intel")
 elseif("$ENV{SYST}" STREQUAL "NO_OVERRIDES")
 else()
   set(CMAKE_Fortran_COMPILER "mpif90")
-  set(CMAKE_Fortran_FLAGS "-fdefault-real-8 -fdefault-double-8 -march=native -malign-double -ffree-line-length-none -std=legacy" CACHE STRING "")
+  set(CMAKE_Fortran_FLAGS "-fdefault-real-8 -fdefault-double-8 -march=native -ffree-line-length-none -std=legacy" CACHE STRING "")
   set (CMAKE_Fortran_FLAGS_RELEASE "-funroll-all-loops -fno-f2c -O3" CACHE STRING "")
   set (CMAKE_Fortran_FLAGS_DEBUG   "-finit-real=nan -fbounds-check -fbacktrace -fno-f2c -O0 -g -ffpe-trap=invalid,zero,overflow" CACHE STRING "")
 endif()

src/modboundary.f90

@@ -147,7 +147,7 @@ end subroutine cyclicm
 !! to infinity at the bottom of the sponge layer.
 !! \endlatexonly
  subroutine grwdamp
-  use modglobal, only : i1,j1,kmax,cu,cv,lcoriol,igrw_damp,geodamptime,nsv,rdt,unudge,dzf
+  use modglobal, only : i1,j1,kmax,cu,cv,lcoriol,igrw_damp,geodamptime,nsv,rdt,unudge,dzf,lopenbc
   use modfields, only : up,vp,wp,thlp,qtp,u0,v0,w0,thl0,qt0,sv0,ug,vg &
                         ,thl0av,qt0av,sv0av,u0av,v0av
   implicit none
@@ -196,12 +196,13 @@ subroutine grwdamp
   ! Additional to gravity wave damping, set qt, thl and sv0(:) equal to slabaverage
   ! at level kmax.
   ! Originally done in subroutine tqaver, now using averages from modthermodynamics
-
-  thl0(2:i1,2:j1,kmax) = thl0av(kmax)
-  qt0 (2:i1,2:j1,kmax) = qt0av(kmax)
-  do n=1,nsv
-    sv0(2:i1,2:j1,kmax,n) = sv0av(kmax,n)
-  end do
+  if(.not.lopenbc) then
+    thl0(2:i1,2:j1,kmax) = thl0av(kmax)
+    qt0 (2:i1,2:j1,kmax) = qt0av(kmax)
+    do n=1,nsv
+      sv0(2:i1,2:j1,kmax,n) = sv0av(kmax,n)
+    end do
+  endif
 
   return
   end subroutine grwdamp

src/modchecksim.f90

@@ -31,15 +31,15 @@ module modchecksim
 
   implicit none
   private
-  public initchecksim,checksim
+  public initchecksim,checksim,chkdiv
 
   real    :: tcheck = 0.
   integer(kind=longint) :: tnext = 3600.,itcheck
   real    :: dtmn =0.,ndt =0.
 
   ! explanations for dt_limit, determined in tstep_update()
   character (len=15) :: dt_reasons(0:5) = [character(len=15):: "initial step", "timee", "dt_lim" , "idtmax", "velocity", "diffusion"]
-  
+
   save
 contains
 !> Initializing Checksim. Read out the namelist, initializing the variables
@@ -204,9 +204,8 @@ subroutine chkdiv
       write(6 ,'(A,2ES11.2,A,A)')'divmax, divtot = ', divmax, divtot,  '       dt limited by ', dt_reasons(dt_reason)
    end if
 
-   return    
-    
+   return
+
   end subroutine chkdiv
 
 end module modchecksim
-

src/modglobal.f90

@@ -230,9 +230,6 @@ module modglobal
       real :: dxint=-1.,dyint=-1.,dzint=-1.,tauh=60.,taum=0.,tau=60.,lambda,lambdas=-1.,lambdas_x=-1.,lambdas_y=-1.,lambdas_z=-1.
       integer :: nmodes=100,ntboundary=1,pbc = 3,iturb=0
       real,dimension(:),allocatable :: tboundary
-      type(solver_type) :: initsolver
-      integer :: solver_id_init = -1, precond_id_init = -1, maxiter_init = -1, n_pre_init = -1, n_post_init = -1, maxiter_precond_init = 1
-      real :: tolerance_init = -1
 
       ! modphsgrd.f90
 

src/modopenboundary.f90

@@ -430,24 +430,18 @@ subroutine openboundary_divcorr()
     use mpi
     use modmpi, only : myid,comm3d,mpierr,MY_REAL
     use modglobal, only : imax,jmax,kmax,dzf,dy,dx,xsize,ysize,zh,k1,lwarmstart, &
-      i1,i2,j1,j2,initsolver,solver_type,k1,solver_id,maxiter,tolerance,precond_id, &
-      n_pre,n_post,solver_id_init,maxiter_init,tolerance_init,precond_id_init,maxiter_precond_init, &
-      n_pre_init,n_post_init,ih,jh,dzh
+      i1,i2,j1,j2,k1,dzh
     use modfields, only : u0,um,v0,vm,w0,wm,rhobf,rhobh
-    use modhypre, only : inithypre_grid, inithypre_solver, solve_hypre,set_zero_guess,exithypre_solver
-    !use modchecksim, only : chkdiv
     implicit none
-    real :: sumdiv,divold,divnew,div,divpart
-    integer :: i,j,k,it,iter
-    real,parameter :: maxdiv = 1e-10
-    real,dimension(i2,j2,k1) :: pcorr
-    logical :: converged
-    real :: divmax,divmaxl,divtot,divtotl
+    real :: sumdiv,div,divpart,divnew,divold
+    integer :: i,j,k,it,icalc
+    ! Create 1/int(rho)
+    allocate(rhointi(k1))
+    rhointi = 1./(rhobf*dzf)
     ! Divergence correction
-    if(myid==0) print *, "Start divergence correction"
+    if(myid==0) print *, "Start divergence correction boundaries"
     do it = 1,ntboundary
-      iter = 0
-      do while(.True.)
+      do icalc=1,2
         ! Calculate divergence
         div = 0.
         if(lboundary(1)) then
@@ -486,17 +480,14 @@ subroutine openboundary_divcorr()
           end do
         endif
         call MPI_ALLREDUCE(div,sumdiv,1,MY_REAL,MPI_SUM,comm3d,mpierr)
-        if(iter==0) then
-          divold = sumdiv
-        else
-          divnew = sumdiv
-        endif
-        if(abs(sumdiv)<maxdiv .or. iter>=20) then
-          if(myid==0) print *, 't,input,corrected,niter',tboundary(it),divold,divnew,iter
-          exit
-        endif
-        iter = iter+1
-        ! Start correction, spread divergence over lateral boundaries
+        if(icalc==1) then
+           divold=sumdiv
+         else
+           divnew=sumdiv
+           if(myid==0) print *, 't,input,corrected;',tboundary(it),divold,divnew
+           exit
+         endif
+        ! Apply correction, spread divergence over lateral boundaries
         if(lboundary(1)) then
           do k = 1,kmax
             divpart = sumdiv*ysize*dzf(k)/(2*xsize*zh(k1)+2*ysize*zh(k1))
@@ -523,7 +514,7 @@ subroutine openboundary_divcorr()
         endif
       end do
     end do
-    if(myid==0) print *, "Finished divergence correction"
+    if(myid==0) print *, "Finished divergence correction boundaries"
     ! Copy data to boundary information
     if(.not.lwarmstart) then
       if(lboundary(1).and..not.lperiodic(1)) then
@@ -567,78 +558,6 @@ subroutine openboundary_divcorr()
         end do
       endif
     endif
-    ! Create 1/int(rho)
-    allocate(rhointi(k1))
-    rhointi = 1./(rhobf*dzf)
-    if(linithetero) then
-      if(myid==0) print *, "Start divergence correction initial field"
-      if(solver_id_init == -1) solver_id_init = solver_id
-      if(maxiter_init == -1) maxiter_init = maxiter
-      if(tolerance_init == -1) tolerance_init = tolerance
-      if(precond_id_init == -1) precond_id_init = precond_id
-      if(n_pre_init == -1) n_pre_init = n_pre
-      if(n_post_init == -1) n_post_init = n_post
-      if(solver_id == 100) then
-         call inithypre_grid ! if FFTW is used during run, need to init hypre grid here(?)
-      end if
-      call inithypre_solver(initsolver,solver_id_init,maxiter_init,tolerance_init,precond_id_init,n_pre_init,n_post_init,maxiter_precond_init)
-      iter = 0
-      do while(.True.)
-        divmax = 0.
-        divtot = 0.
-        divmaxl= 0.
-        divtotl= 0.
-        do k=1,kmax
-          do j=2,j1
-            do i=2,i1
-              pcorr(i,j,k)  =  rhobf(k)*(( um(i+1,j,k)-um(i,j,k) ) / dx &
-                                        +( vm(i,j+1,k)-vm(i,j,k) ) / dy ) &
-                                        +( wm(i,j,k+1)*rhobh(k+1)-wm(i,j,k)*rhobh(k)) / dzf(k)
-              divmaxl = max(divmaxl,abs(pcorr(i,j,k)))
-              divtotl = divtotl + pcorr(i,j,k)*dx*dy*dzf(k)
-            end do
-          end do
-        end do
-        call MPI_ALLREDUCE(divtotl, divtot, 1,    MY_REAL, &
-                              MPI_SUM, comm3d,mpierr)
-        call MPI_ALLREDUCE(divmaxl, divmax, 1,    MY_REAL, &
-                              MPI_MAX, comm3d,mpierr)
-        if(iter==0) divold = divmax
-        if(divmax<1e-5.or. iter>=5) then
-          if(myid==0) print *, "Finished divergence correction initial field, old/new maximum divergence and iter",divold,divmax,iter
-          exit
-        endif
-        call set_zero_guess()
-        call solve_hypre(initsolver,pcorr,converged)
-        call set_zero_guess()
-        call openboundary_excjs(pcorr   , 2,i1,2,j1,1,kmax,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4))
-        if(lboundary(1).and. .not. lperiodic(1)) pcorr(1,:,:) = pcorr(2,:,:)
-        if(lboundary(2).and. .not. lperiodic(2)) pcorr(i2,:,:) = pcorr(i1,:,:)
-        if(lboundary(3).and. .not. lperiodic(3)) pcorr(:,1,:) = pcorr(:,2,:)
-        if(lboundary(4).and. .not. lperiodic(4)) pcorr(:,j2,:) = pcorr(:,j1,:)
-        do k=1,kmax
-          do j=2,j1
-            do i=2,i1
-              um(i,j,k) = um(i,j,k)-(pcorr(i,j,k)-pcorr(i-1,j,k))/dx
-              vm(i,j,k) = vm(i,j,k)-(pcorr(i,j,k)-pcorr(i,j-1,k))/dy
-            end do
-          end do
-        end do
-        do k=2,kmax
-          do j=2,j1
-            do i=2,i1
-              wm(i,j,k) = wm(i,j,k)-(pcorr(i,j,k)-pcorr(i,j,k-1))/dzh(k)
-            end do
-          end do
-        end do
-        call openboundary_excjs(um   , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4))
-        call openboundary_excjs(vm   , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4))
-        call openboundary_excjs(wm   , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4))
-        iter = iter+1
-      end do
-      u0 = um; v0 = vm; w0 = wm
-      call exithypre_solver(initsolver)
-    endif
   end subroutine openboundary_divcorr
 
   subroutine openboundary_ghost
@@ -1436,4 +1355,5 @@ subroutine handle_err(errcode)
   stop 2
 
   end subroutine handle_err
+
 end module modopenboundary

src/modstartup.f90

@@ -61,16 +61,15 @@ subroutine startup(path)
                                   ibas_prf,lambda_crit,iadv_mom,iadv_tke,iadv_thl,iadv_qt,iadv_sv,courant,peclet,ladaptive,author,lnoclouds,lrigidlid,unudge, &
                                   solver_id, maxiter, maxiter_precond, tolerance, n_pre, n_post, precond_id, checknamelisterror, &
                                   lopenbc,linithetero,lperiodic,dxint,dyint,dzint,taum,tauh,pbc,lsynturb,nmodes,tau,lambda,lambdas,lambdas_x,lambdas_y,lambdas_z,iturb, &
-                                  solver_id_init, maxiter_init, tolerance_init, n_pre_init, n_post_init, precond_id_init, maxiter_precond_init, &
-                                  hypre_logging, lconstexner
+                                  hypre_logging,rdt,rk3step,i1,j1,k1,ih,jh,lboundary,lconstexner
     use modforces,         only : lforce_user
     use modsurfdata,       only : z0,ustin,wtsurf,wqsurf,wsvsurf,ps,thls,isurf
     use modsurface,        only : initsurface
     use moddatetime,       only : initdatetime
     use modemission,       only : initemission
     use modlsm,            only : initlsm, kmax_soil
-    use modfields,         only : initfields
-    use modpois,           only : initpois
+    use modfields,         only : initfields,um,vm,wm,u0,v0,w0,up,vp,wp
+    use modpois,           only : initpois,poisson
     use modradiation,      only : initradiation
     use modraddata,        only : irad,iradiation,&
                                   rad_ls,rad_longw,rad_shortw,rad_smoke,useMcICA,&
@@ -86,12 +85,14 @@ subroutine startup(path)
     use modmpi,            only : initmpi,commwrld,my_real,myid,nprocx,nprocy,mpierr,periods
     use modchem,           only : initchem
     use modversion,        only : git_version
-    use modopenboundary,   only : initopenboundary,openboundary_divcorr
+    use modopenboundary,   only : initopenboundary,openboundary_divcorr,openboundary_excjs
+    use modchecksim,       only : chkdiv
 
     implicit none
     integer :: ierr
     logical,dimension(2) :: lper = .false.
     character(256), optional, intent(in) :: path
+    real rk3coef
 
     !declare namelists
     namelist/RUN/ &
@@ -115,8 +116,7 @@ subroutine startup(path)
     namelist/SOLVER/ &
         solver_id, maxiter, tolerance, n_pre, n_post, precond_id, maxiter_precond, hypre_logging
     namelist/OPENBC/ &
-        lopenbc,linithetero,lper,dxint,dyint,dzint,taum,tauh,pbc,lsynturb,iturb,tau,lambda,nmodes,lambdas,lambdas_x,lambdas_y,lambdas_z, &
-        solver_id_init,maxiter_init,tolerance_init,precond_id_init,n_pre_init,n_post_init,maxiter_precond_init
+        lopenbc,linithetero,lper,dxint,dyint,dzint,taum,tauh,pbc,lsynturb,iturb,tau,lambda,nmodes,lambdas,lambdas_x,lambdas_y,lambdas_z
 
 
     ! get myid
@@ -303,13 +303,6 @@ subroutine startup(path)
     call MPI_BCAST(lambdas_x,1,MY_REAL   ,0,commwrld,mpierr)
     call MPI_BCAST(lambdas_y,1,MY_REAL   ,0,commwrld,mpierr)
     call MPI_BCAST(lambdas_z,1,MY_REAL   ,0,commwrld,mpierr)
-    call MPI_BCAST(solver_id_init,1,MPI_INTEGER,0,commwrld,mpierr)
-    call MPI_BCAST(maxiter_init,1,MPI_INTEGER,0,commwrld,mpierr)
-    call MPI_BCAST(n_pre_init,1,MPI_INTEGER,0,commwrld,mpierr)
-    call MPI_BCAST(n_post_init,1,MPI_INTEGER,0,commwrld,mpierr)
-    call MPI_BCAST(tolerance_init,1,MY_REAL,0,commwrld,mpierr)
-    call MPI_BCAST(precond_id_init,1,MPI_INTEGER,0,commwrld,mpierr)
-    call MPI_BCAST(maxiter_precond_init,1,MPI_INTEGER,0,commwrld,mpierr)
 
     call testwctime
     ! Allocate and initialize core modules
@@ -335,7 +328,28 @@ subroutine startup(path)
     call readinitfiles ! moved to obtain the correct btime for the timedependent forcings in case of a warmstart
     call inittimedep !depends on modglobal,modfields, modmpi, modsurf, modradiation
     call initpois ! hypre solver needs grid and baseprofiles
-    if(lopenbc) call openboundary_divcorr
+    if(lopenbc) then  ! Correct boundaries and initial field for divergence
+      call chkdiv
+      call openboundary_divcorr ! Remove divergence from large scale input
+      ! Use poisson solver to get rid of divergence in initial field, needs to
+      ! be here to avoid cross dependencies between modopenbondaries and modpois
+      if(myid==0) print *, 'Start divergence correction initial field'
+      call chkdiv
+      up = 0.; vp = 0.; wp = 0. ! Set tendencies to zero
+      call poisson
+      rk3coef = rdt / (4. - dble(rk3step))
+      um = um + rk3coef * up
+      vm = vm + rk3coef * vp
+      wm = wm + rk3coef * wp
+      call openboundary_excjs(um   , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4))
+      call openboundary_excjs(vm   , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4))
+      call openboundary_excjs(wm   , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4))
+      u0 = um; v0 = vm; w0 = wm
+      call chkdiv
+      ! Reset tendencies
+      up = 0.; vp = 0.; wp = 0.
+      if(myid==0) print *, 'Finished divergence correction initial field'
+    endif
 
     call checkinitvalues