From 031b97b17eb16425e6cf45129907dd03e3a65c2f Mon Sep 17 00:00:00 2001 From: Frans Liqui Lung Date: Thu, 15 Sep 2022 10:29:21 +0000 Subject: [PATCH 1/4] fixed timing issue in modsynturb --- src/modsynturb.f90 | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/modsynturb.f90 b/src/modsynturb.f90 index 144ce3f0..743b30f1 100644 --- a/src/modsynturb.f90 +++ b/src/modsynturb.f90 @@ -323,7 +323,7 @@ subroutine synturb_all() end subroutine synturb_all subroutine sepsim() - use modglobal, only : rtimee,rdt,lmoist + use modglobal, only : rtimee,lmoist use modmpi, only : myid implicit none integer :: ib @@ -335,7 +335,7 @@ subroutine sepsim() itimestep = itimestep+1 end do if(abs(tturb(itimestep)-real(rtimee))>0.01 .and. real(rtimee)>5.) then - print *, 'mistake in time', itimestep,tturb(itimestep),real(rtimee),rdt + print *, 'mistake in time', itimestep,tturb(itimestep),real(rtimee) stop endif boundary(ib)%uturb = uturbin(:,:,itimestep) @@ -358,7 +358,7 @@ subroutine sepsim() end subroutine sepsim subroutine calc_eigdec(ib) - use modglobal, only : rtimee,rdt,tboundary,ntboundary + use modglobal, only : rtimee,tboundary,ntboundary implicit none integer, intent(in) :: ib real*8,dimension(3,3) :: r,eigvec @@ -368,15 +368,15 @@ subroutine calc_eigdec(ib) ! Interpolate covariance to current time itm = 1 if(ntboundary>1) then - do while(rtimee-rdt>tboundary(itm)) + do while(rtimee>tboundary(itm)) itm = itm+1 end do - if (rtimee-rdt>tboundary(1)) then + if (rtimee>tboundary(1)) then itm = itm-1 end if itp = itm+1 - fm = (tboundary(itp)-rtimee+rdt)/(tboundary(itp)-tboundary(itm)) - fp = (rtimee-rdt-tboundary(itm))/(tboundary(itp)-tboundary(itm)) + fm = (tboundary(itp)-rtimee)/(tboundary(itp)-tboundary(itm)) + fp = (rtimee-tboundary(itm))/(tboundary(itp)-tboundary(itm)) else itm = 1; itp = 1; fp = 1.; fm = 1. endif @@ -402,7 +402,7 @@ subroutine calc_eigdec(ib) end subroutine calc_eigdec subroutine calc_pert(ib,x,y,z,nx,ny,nz,uturb,iuturb) - use modglobal, only : rtimee, rdt,dxint,dyint + use modglobal, only : rtimee,dxint,dyint use modmpi, only : myidx,myidy implicit none real, dimension(:), intent(in) :: x,y,z @@ -414,7 +414,7 @@ subroutine calc_pert(ib,x,y,z,nx,ny,nz,uturb,iuturb) real, dimension(3) :: xx,ci real, dimension(3,3) :: eigvec real :: t,utemp,vtemp,wtemp - t = real(rtimee-rdt) + t = rtimee select case(ib) case(1,2) pi1 => j; pi2 => k; pi1patch => jpatch; pi2patch => kpatch @@ -453,7 +453,7 @@ subroutine calc_pert(ib,x,y,z,nx,ny,nz,uturb,iuturb) end subroutine calc_pert subroutine calc_pert2(ib,x,y,z,nx,ny,nz,uturb,iuturb,thlturb,qtturb) - use modglobal, only : rtimee, rdt,tboundary,ntboundary + use modglobal, only : rtimee,tboundary,ntboundary use modmpi, only : myidx,myidy implicit none real, dimension(:), intent(in) :: x,y,z @@ -466,7 +466,7 @@ subroutine calc_pert2(ib,x,y,z,nx,ny,nz,uturb,iuturb,thlturb,qtturb) real, dimension(3,3) :: eigvec real :: wthl,wqt,w2,thl2,qt2,utemp,vtemp,wtemp,wturbf,thltemp,qttemp real :: t,fp,fm,rho - t = real(rtimee-rdt) + t = rtimee itm = 1 ! Interpolate covariance to current time if(ntboundary>1) then From 09498f6fc97b870da885605509f4d78746051bec Mon Sep 17 00:00:00 2001 From: Frans Liqui Lung Date: Sun, 2 Oct 2022 20:26:12 +0000 Subject: [PATCH 2/4] removed print from modsynturb and removed hard set top layer of temperature in modboundary --- src/modboundary.f90 | 15 ++++++++------- src/modsynturb.f90 | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/modboundary.f90 b/src/modboundary.f90 index effcbeba..b511db7a 100644 --- a/src/modboundary.f90 +++ b/src/modboundary.f90 @@ -147,7 +147,7 @@ end subroutine cyclicm !! to infinity at the bottom of the sponge layer. !! \endlatexonly subroutine grwdamp - use modglobal, only : i1,j1,kmax,cu,cv,lcoriol,igrw_damp,geodamptime,nsv,rdt,unudge,dzf + use modglobal, only : i1,j1,kmax,cu,cv,lcoriol,igrw_damp,geodamptime,nsv,rdt,unudge,dzf,lopenbc use modfields, only : up,vp,wp,thlp,qtp,u0,v0,w0,thl0,qt0,sv0,ug,vg & ,thl0av,qt0av,sv0av,u0av,v0av implicit none @@ -196,12 +196,13 @@ subroutine grwdamp ! Additional to gravity wave damping, set qt, thl and sv0(:) equal to slabaverage ! at level kmax. ! Originally done in subroutine tqaver, now using averages from modthermodynamics - - thl0(2:i1,2:j1,kmax) = thl0av(kmax) - qt0 (2:i1,2:j1,kmax) = qt0av(kmax) - do n=1,nsv - sv0(2:i1,2:j1,kmax,n) = sv0av(kmax,n) - end do + if(.not.lopenbc) then + thl0(2:i1,2:j1,kmax) = thl0av(kmax) + qt0 (2:i1,2:j1,kmax) = qt0av(kmax) + do n=1,nsv + sv0(2:i1,2:j1,kmax,n) = sv0av(kmax,n) + end do + endif return end subroutine grwdamp diff --git a/src/modsynturb.f90 b/src/modsynturb.f90 index 743b30f1..98f2830f 100644 --- a/src/modsynturb.f90 +++ b/src/modsynturb.f90 @@ -392,7 +392,7 @@ subroutine calc_eigdec(ib) r(2,1) = r(1,2); r(3,1) = r(1,3); r(3,2) = r(2,3) call DSYEVJ3(r,eigvec,eigval) do ii = 1,3 - if(eigval(ii)<-1.e-8) print *,"warning negative eigenvalue ",eigval(ii), " value set to 1e-8" + !if(eigval(ii)<-1.e-8) print *,"warning negative eigenvalue ",eigval(ii), " value set to 1e-8" eigval(ii) = max(eigval(ii),1.e-8) end do boundary(ib)%eigvec(i,j,:,:) = real(eigvec) From 41edb3088c4b63ab51465874dd4e2ada411861ff Mon Sep 17 00:00:00 2001 From: Frans Liqui Lung Date: Tue, 13 Dec 2022 11:36:33 +1100 Subject: [PATCH 3/4] adjusted initial convergence correction to be compatible with cosine fourier solver, implemented in modstartup to avoid cross dependency conflict between modpois and modopenboundaries --- CMakeLists.txt | 2 +- src/modchecksim.f90 | 9 +- src/modopenboundary.f90 | 195 +++++++++++++++++++++------------------- src/modstartup.f90 | 33 +++++-- 4 files changed, 137 insertions(+), 102 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index be2cfbe7..da3a24fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,7 +60,7 @@ elseif("$ENV{SYST}" STREQUAL "lisa-intel") elseif("$ENV{SYST}" STREQUAL "NO_OVERRIDES") else() set(CMAKE_Fortran_COMPILER "mpif90") - set(CMAKE_Fortran_FLAGS "-fdefault-real-8 -fdefault-double-8 -march=native -malign-double -ffree-line-length-none -std=legacy" CACHE STRING "") + set(CMAKE_Fortran_FLAGS "-fdefault-real-8 -fdefault-double-8 -march=native -ffree-line-length-none -std=legacy" CACHE STRING "") set (CMAKE_Fortran_FLAGS_RELEASE "-funroll-all-loops -fno-f2c -O3" CACHE STRING "") set (CMAKE_Fortran_FLAGS_DEBUG "-finit-real=nan -fbounds-check -fbacktrace -fno-f2c -O0 -g -ffpe-trap=invalid,zero,overflow" CACHE STRING "") endif() diff --git a/src/modchecksim.f90 b/src/modchecksim.f90 index 4b2bfea8..411e7264 100644 --- a/src/modchecksim.f90 +++ b/src/modchecksim.f90 @@ -31,7 +31,7 @@ module modchecksim implicit none private - public initchecksim,checksim + public initchecksim,checksim,chkdiv real :: tcheck = 0. integer(kind=longint) :: tnext = 3600.,itcheck @@ -39,7 +39,7 @@ module modchecksim ! explanations for dt_limit, determined in tstep_update() character (len=15) :: dt_reasons(0:5) = [character(len=15):: "initial step", "timee", "dt_lim" , "idtmax", "velocity", "diffusion"] - + save contains !> Initializing Checksim. Read out the namelist, initializing the variables @@ -204,9 +204,8 @@ subroutine chkdiv write(6 ,'(A,2ES11.2,A,A)')'divmax, divtot = ', divmax, divtot, ' dt limited by ', dt_reasons(dt_reason) end if - return - + return + end subroutine chkdiv end module modchecksim - diff --git a/src/modopenboundary.f90 b/src/modopenboundary.f90 index bd2ceb4f..359f7c6d 100644 --- a/src/modopenboundary.f90 +++ b/src/modopenboundary.f90 @@ -428,6 +428,8 @@ end subroutine openboundary_readboundary subroutine openboundary_divcorr() ! Correct for any integrated divergence present in the boundary input use mpi + !use modfftw, only: fftwf, fftwb + !use modpois, only: solmpj use modmpi, only : myid,comm3d,mpierr,MY_REAL use modglobal, only : imax,jmax,kmax,dzf,dy,dx,xsize,ysize,zh,k1,lwarmstart, & i1,i2,j1,j2,initsolver,solver_type,k1,solver_id,maxiter,tolerance,precond_id, & @@ -435,19 +437,21 @@ subroutine openboundary_divcorr() n_pre_init,n_post_init,ih,jh,dzh use modfields, only : u0,um,v0,vm,w0,wm,rhobf,rhobh use modhypre, only : inithypre_grid, inithypre_solver, solve_hypre,set_zero_guess,exithypre_solver - !use modchecksim, only : chkdiv implicit none - real :: sumdiv,divold,divnew,div,divpart - integer :: i,j,k,it,iter + real :: sumdiv,div,divpart + integer :: i,j,k,it,iter,icalc real,parameter :: maxdiv = 1e-10 real,dimension(i2,j2,k1) :: pcorr logical :: converged - real :: divmax,divmaxl,divtot,divtotl + real :: divmax,divmaxl,divtot,divtotl,divnew_max,divnew_sum,divold_max,divold_sum,divnew,divold + !real, pointer :: Fp(:,:,:) + ! Create 1/int(rho) + allocate(rhointi(k1)) + rhointi = 1./(rhobf*dzf) ! Divergence correction - if(myid==0) print *, "Start divergence correction" + if(myid==0) print *, "Start divergence correction boundaries" do it = 1,ntboundary - iter = 0 - do while(.True.) + do icalc=1,2 ! Calculate divergence div = 0. if(lboundary(1)) then @@ -486,17 +490,14 @@ subroutine openboundary_divcorr() end do endif call MPI_ALLREDUCE(div,sumdiv,1,MY_REAL,MPI_SUM,comm3d,mpierr) - if(iter==0) then - divold = sumdiv - else - divnew = sumdiv - endif - if(abs(sumdiv)=20) then - if(myid==0) print *, 't,input,corrected,niter',tboundary(it),divold,divnew,iter - exit - endif - iter = iter+1 - ! Start correction, spread divergence over lateral boundaries + if(icalc==1) then + divold=sumdiv + else + divnew=sumdiv + if(myid==0) print *, 't,input,corrected;',tboundary(it),divold,divnew + exit + endif + ! Apply correction, spread divergence over lateral boundaries if(lboundary(1)) then do k = 1,kmax divpart = sumdiv*ysize*dzf(k)/(2*xsize*zh(k1)+2*ysize*zh(k1)) @@ -523,7 +524,7 @@ subroutine openboundary_divcorr() endif end do end do - if(myid==0) print *, "Finished divergence correction" + if(myid==0) print *, "Finished divergence correction boundaries" ! Copy data to boundary information if(.not.lwarmstart) then if(lboundary(1).and..not.lperiodic(1)) then @@ -567,78 +568,89 @@ subroutine openboundary_divcorr() end do endif endif - ! Create 1/int(rho) - allocate(rhointi(k1)) - rhointi = 1./(rhobf*dzf) - if(linithetero) then - if(myid==0) print *, "Start divergence correction initial field" - if(solver_id_init == -1) solver_id_init = solver_id - if(maxiter_init == -1) maxiter_init = maxiter - if(tolerance_init == -1) tolerance_init = tolerance - if(precond_id_init == -1) precond_id_init = precond_id - if(n_pre_init == -1) n_pre_init = n_pre - if(n_post_init == -1) n_post_init = n_post - if(solver_id == 100) then - call inithypre_grid ! if FFTW is used during run, need to init hypre grid here(?) - end if - call inithypre_solver(initsolver,solver_id_init,maxiter_init,tolerance_init,precond_id_init,n_pre_init,n_post_init,maxiter_precond_init) - iter = 0 - do while(.True.) - divmax = 0. - divtot = 0. - divmaxl= 0. - divtotl= 0. - do k=1,kmax - do j=2,j1 - do i=2,i1 - pcorr(i,j,k) = rhobf(k)*(( um(i+1,j,k)-um(i,j,k) ) / dx & - +( vm(i,j+1,k)-vm(i,j,k) ) / dy ) & - +( wm(i,j,k+1)*rhobh(k+1)-wm(i,j,k)*rhobh(k)) / dzf(k) - divmaxl = max(divmaxl,abs(pcorr(i,j,k))) - divtotl = divtotl + pcorr(i,j,k)*dx*dy*dzf(k) - end do - end do - end do - call MPI_ALLREDUCE(divtotl, divtot, 1, MY_REAL, & - MPI_SUM, comm3d,mpierr) - call MPI_ALLREDUCE(divmaxl, divmax, 1, MY_REAL, & - MPI_MAX, comm3d,mpierr) - if(iter==0) divold = divmax - if(divmax<1e-5.or. iter>=5) then - if(myid==0) print *, "Finished divergence correction initial field, old/new maximum divergence and iter",divold,divmax,iter - exit - endif - call set_zero_guess() - call solve_hypre(initsolver,pcorr,converged) - call set_zero_guess() - call openboundary_excjs(pcorr , 2,i1,2,j1,1,kmax,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) - if(lboundary(1).and. .not. lperiodic(1)) pcorr(1,:,:) = pcorr(2,:,:) - if(lboundary(2).and. .not. lperiodic(2)) pcorr(i2,:,:) = pcorr(i1,:,:) - if(lboundary(3).and. .not. lperiodic(3)) pcorr(:,1,:) = pcorr(:,2,:) - if(lboundary(4).and. .not. lperiodic(4)) pcorr(:,j2,:) = pcorr(:,j1,:) - do k=1,kmax - do j=2,j1 - do i=2,i1 - um(i,j,k) = um(i,j,k)-(pcorr(i,j,k)-pcorr(i-1,j,k))/dx - vm(i,j,k) = vm(i,j,k)-(pcorr(i,j,k)-pcorr(i,j-1,k))/dy - end do - end do - end do - do k=2,kmax - do j=2,j1 - do i=2,i1 - wm(i,j,k) = wm(i,j,k)-(pcorr(i,j,k)-pcorr(i,j,k-1))/dzh(k) - end do - end do - end do - call openboundary_excjs(um , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) - call openboundary_excjs(vm , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) - call openboundary_excjs(wm , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) - iter = iter+1 - end do - u0 = um; v0 = vm; w0 = wm - call exithypre_solver(initsolver) - endif + ! if(linithetero) then + ! if(myid==0) print *, "Start divergence correction initial field" + ! if(solver_id_init == -1) solver_id_init = solver_id + ! if(maxiter_init == -1) maxiter_init = maxiter + ! if(tolerance_init == -1) tolerance_init = tolerance + ! if(precond_id_init == -1) precond_id_init = precond_id + ! if(n_pre_init == -1) n_pre_init = n_pre + ! if(n_post_init == -1) n_post_init = n_post + ! if(solver_id /= 100) then ! Use iterative solver (HYPRE) + ! call inithypre_solver(initsolver,solver_id_init,maxiter_init,tolerance_init,precond_id_init,n_pre_init,n_post_init,maxiter_precond_init) + ! else + ! call inithypre_grid ! if FFTW is used during run, need to init hypre grid here(?) + ! call inithypre_solver(initsolver,solver_id_init,maxiter_init,tolerance_init,precond_id_init,n_pre_init,n_post_init,maxiter_precond_init) + ! end if + ! do icalc=1,2 + ! do k=1,kmax + ! do j=2,j1 + ! do i=2,i1 + ! pcorr(i,j,k) = rhobf(k)*(( um(i+1,j,k)-um(i,j,k) ) / dx & + ! +( vm(i,j+1,k)-vm(i,j,k) ) / dy ) & + ! +( wm(i,j,k+1)*rhobh(k+1)-wm(i,j,k)*rhobh(k)) / dzf(k) + ! divmaxl = max(divmaxl,abs(pcorr(i,j,k))) + ! divtotl = divtotl + pcorr(i,j,k)*dx*dy*dzf(k) + ! end do + ! end do + ! end do + ! call MPI_ALLREDUCE(divtotl, divtot, 1, MY_REAL, & + ! MPI_SUM, comm3d,mpierr) + ! call MPI_ALLREDUCE(divmaxl, divmax, 1, MY_REAL, & + ! MPI_MAX, comm3d,mpierr) + ! if(icalc==1) then + ! divold_max = divmax + ! divold_sum = divtot + ! else + ! divnew_max = divmax + ! divnew_sum = divtot + ! if(myid==0) then + ! print *, "input max,input sum,corrected max, corrected sum;",divold_max,divold_sum,divnew_max,divnew_sum + ! print *, "Finished divergence correction initial field" + ! endif + ! exit + ! endif + ! ! Calculate correction using pressure solver + ! if(solver_id==100) then ! Use fourrier solver + ! call set_zero_guess() + ! call solve_hypre(initsolver,pcorr,converged) + ! ! call fftwf(pcorr, Fp) + ! ! call solmpj ! Cross dependency conflict + ! ! call fftwb(pcorr, Fp) + ! else ! Use iterative solver (HYPRE) + ! call set_zero_guess() + ! call solve_hypre(initsolver,pcorr,converged) + ! endif + ! call openboundary_excjs(pcorr , 2,i1,2,j1,1,kmax,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + ! if(lboundary(1).and. .not. lperiodic(1)) pcorr(1,:,:) = pcorr(2,:,:) + ! if(lboundary(2).and. .not. lperiodic(2)) pcorr(i2,:,:) = pcorr(i1,:,:) + ! if(lboundary(3).and. .not. lperiodic(3)) pcorr(:,1,:) = pcorr(:,2,:) + ! if(lboundary(4).and. .not. lperiodic(4)) pcorr(:,j2,:) = pcorr(:,j1,:) + ! ! Apply correction + ! do k=1,kmax + ! do j=2,j1 + ! do i=2,i1 + ! um(i,j,k) = um(i,j,k)-(pcorr(i,j,k)-pcorr(i-1,j,k))/dx + ! vm(i,j,k) = vm(i,j,k)-(pcorr(i,j,k)-pcorr(i,j-1,k))/dy + ! end do + ! end do + ! end do + ! do k=2,kmax + ! do j=2,j1 + ! do i=2,i1 + ! wm(i,j,k) = wm(i,j,k)-(pcorr(i,j,k)-pcorr(i,j,k-1))/dzh(k) + ! end do + ! end do + ! end do + ! call openboundary_excjs(um , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + ! call openboundary_excjs(vm , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + ! call openboundary_excjs(wm , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + ! u0 = um; v0 = vm; w0 = wm + ! enddo + ! !if(solver_id/=100) call exithypre_solver(initsolver) + ! if(solver_id/=100) call exithypre_solver(initsolver) + ! !nullify(Fp) + ! endif end subroutine openboundary_divcorr subroutine openboundary_ghost @@ -1436,4 +1448,5 @@ subroutine handle_err(errcode) stop 2 end subroutine handle_err + end module modopenboundary diff --git a/src/modstartup.f90 b/src/modstartup.f90 index 5995160e..bf647cd8 100644 --- a/src/modstartup.f90 +++ b/src/modstartup.f90 @@ -62,15 +62,15 @@ subroutine startup(path) solver_id, maxiter, maxiter_precond, tolerance, n_pre, n_post, precond_id, checknamelisterror, & lopenbc,linithetero,lperiodic,dxint,dyint,dzint,taum,tauh,pbc,lsynturb,nmodes,tau,lambda,lambdas,lambdas_x,lambdas_y,lambdas_z,iturb, & solver_id_init, maxiter_init, tolerance_init, n_pre_init, n_post_init, precond_id_init, maxiter_precond_init, & - hypre_logging + hypre_logging,rdt,rk3step,i1,j1,k1,ih,jh,lboundary use modforces, only : lforce_user use modsurfdata, only : z0,ustin,wtsurf,wqsurf,wsvsurf,ps,thls,isurf use modsurface, only : initsurface use moddatetime, only : initdatetime use modemission, only : initemission use modlsm, only : initlsm, kmax_soil - use modfields, only : initfields - use modpois, only : initpois + use modfields, only : initfields,um,vm,wm,u0,v0,w0,up,vp,wp + use modpois, only : initpois,poisson use modradiation, only : initradiation use modraddata, only : irad,iradiation,& rad_ls,rad_longw,rad_shortw,rad_smoke,useMcICA,& @@ -86,12 +86,14 @@ subroutine startup(path) use modmpi, only : initmpi,commwrld,my_real,myid,nprocx,nprocy,mpierr,periods use modchem, only : initchem use modversion, only : git_version - use modopenboundary, only : initopenboundary,openboundary_divcorr + use modopenboundary, only : initopenboundary,openboundary_divcorr,openboundary_excjs + use modchecksim, only : chkdiv implicit none integer :: ierr logical,dimension(2) :: lper = .false. character(256), optional, intent(in) :: path + real rk3coef !declare namelists namelist/RUN/ & @@ -333,7 +335,28 @@ subroutine startup(path) call readinitfiles ! moved to obtain the correct btime for the timedependent forcings in case of a warmstart call inittimedep !depends on modglobal,modfields, modmpi, modsurf, modradiation call initpois ! hypre solver needs grid and baseprofiles - if(lopenbc) call openboundary_divcorr + if(lopenbc) then ! Correct boundaries and initial field for divergence + call chkdiv + call openboundary_divcorr ! Remove divergence from large scale input + ! Use poisson solver to get rid of divergence in initial field, needs to + ! be here to avoid cross dependencies between modopenbondaries and modpois + if(myid==0) print *, 'Start divergence correction initial field' + call chkdiv + up = 0.; vp = 0.; wp = 0. ! Set tendencies to zero + call poisson + rk3coef = rdt / (4. - dble(rk3step)) + um = um + rk3coef * up + vm = vm + rk3coef * vp + wm = wm + rk3coef * wp + call openboundary_excjs(um , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + call openboundary_excjs(vm , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + call openboundary_excjs(wm , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) + u0 = um; v0 = vm; w0 = wm + call chkdiv + ! Reset tendencies + up = 0.; vp = 0.; wp = 0. + if(myid==0) print *, 'Finished divergence correction initial field' + endif call checkinitvalues From cc84edf89eac4499295ffab8adb3fb0fceec93aa Mon Sep 17 00:00:00 2001 From: Frans Liqui Lung Date: Tue, 13 Dec 2022 11:44:49 +1100 Subject: [PATCH 4/4] removed old code to allow for different initial pressure solver --- src/modglobal.f90 | 3 -- src/modopenboundary.f90 | 99 ++--------------------------------------- src/modstartup.f90 | 11 +---- 3 files changed, 4 insertions(+), 109 deletions(-) diff --git a/src/modglobal.f90 b/src/modglobal.f90 index f93ac86d..ad55a154 100644 --- a/src/modglobal.f90 +++ b/src/modglobal.f90 @@ -229,9 +229,6 @@ module modglobal real :: dxint=-1.,dyint=-1.,dzint=-1.,tauh=60.,taum=0.,tau=60.,lambda,lambdas=-1.,lambdas_x=-1.,lambdas_y=-1.,lambdas_z=-1. integer :: nmodes=100,ntboundary=1,pbc = 3,iturb=0 real,dimension(:),allocatable :: tboundary - type(solver_type) :: initsolver - integer :: solver_id_init = -1, precond_id_init = -1, maxiter_init = -1, n_pre_init = -1, n_post_init = -1, maxiter_precond_init = 1 - real :: tolerance_init = -1 ! modphsgrd.f90 diff --git a/src/modopenboundary.f90 b/src/modopenboundary.f90 index 359f7c6d..1d50f29f 100644 --- a/src/modopenboundary.f90 +++ b/src/modopenboundary.f90 @@ -428,23 +428,13 @@ end subroutine openboundary_readboundary subroutine openboundary_divcorr() ! Correct for any integrated divergence present in the boundary input use mpi - !use modfftw, only: fftwf, fftwb - !use modpois, only: solmpj use modmpi, only : myid,comm3d,mpierr,MY_REAL use modglobal, only : imax,jmax,kmax,dzf,dy,dx,xsize,ysize,zh,k1,lwarmstart, & - i1,i2,j1,j2,initsolver,solver_type,k1,solver_id,maxiter,tolerance,precond_id, & - n_pre,n_post,solver_id_init,maxiter_init,tolerance_init,precond_id_init,maxiter_precond_init, & - n_pre_init,n_post_init,ih,jh,dzh + i1,i2,j1,j2,k1,dzh use modfields, only : u0,um,v0,vm,w0,wm,rhobf,rhobh - use modhypre, only : inithypre_grid, inithypre_solver, solve_hypre,set_zero_guess,exithypre_solver implicit none - real :: sumdiv,div,divpart - integer :: i,j,k,it,iter,icalc - real,parameter :: maxdiv = 1e-10 - real,dimension(i2,j2,k1) :: pcorr - logical :: converged - real :: divmax,divmaxl,divtot,divtotl,divnew_max,divnew_sum,divold_max,divold_sum,divnew,divold - !real, pointer :: Fp(:,:,:) + real :: sumdiv,div,divpart,divnew,divold + integer :: i,j,k,it,icalc ! Create 1/int(rho) allocate(rhointi(k1)) rhointi = 1./(rhobf*dzf) @@ -568,89 +558,6 @@ subroutine openboundary_divcorr() end do endif endif - ! if(linithetero) then - ! if(myid==0) print *, "Start divergence correction initial field" - ! if(solver_id_init == -1) solver_id_init = solver_id - ! if(maxiter_init == -1) maxiter_init = maxiter - ! if(tolerance_init == -1) tolerance_init = tolerance - ! if(precond_id_init == -1) precond_id_init = precond_id - ! if(n_pre_init == -1) n_pre_init = n_pre - ! if(n_post_init == -1) n_post_init = n_post - ! if(solver_id /= 100) then ! Use iterative solver (HYPRE) - ! call inithypre_solver(initsolver,solver_id_init,maxiter_init,tolerance_init,precond_id_init,n_pre_init,n_post_init,maxiter_precond_init) - ! else - ! call inithypre_grid ! if FFTW is used during run, need to init hypre grid here(?) - ! call inithypre_solver(initsolver,solver_id_init,maxiter_init,tolerance_init,precond_id_init,n_pre_init,n_post_init,maxiter_precond_init) - ! end if - ! do icalc=1,2 - ! do k=1,kmax - ! do j=2,j1 - ! do i=2,i1 - ! pcorr(i,j,k) = rhobf(k)*(( um(i+1,j,k)-um(i,j,k) ) / dx & - ! +( vm(i,j+1,k)-vm(i,j,k) ) / dy ) & - ! +( wm(i,j,k+1)*rhobh(k+1)-wm(i,j,k)*rhobh(k)) / dzf(k) - ! divmaxl = max(divmaxl,abs(pcorr(i,j,k))) - ! divtotl = divtotl + pcorr(i,j,k)*dx*dy*dzf(k) - ! end do - ! end do - ! end do - ! call MPI_ALLREDUCE(divtotl, divtot, 1, MY_REAL, & - ! MPI_SUM, comm3d,mpierr) - ! call MPI_ALLREDUCE(divmaxl, divmax, 1, MY_REAL, & - ! MPI_MAX, comm3d,mpierr) - ! if(icalc==1) then - ! divold_max = divmax - ! divold_sum = divtot - ! else - ! divnew_max = divmax - ! divnew_sum = divtot - ! if(myid==0) then - ! print *, "input max,input sum,corrected max, corrected sum;",divold_max,divold_sum,divnew_max,divnew_sum - ! print *, "Finished divergence correction initial field" - ! endif - ! exit - ! endif - ! ! Calculate correction using pressure solver - ! if(solver_id==100) then ! Use fourrier solver - ! call set_zero_guess() - ! call solve_hypre(initsolver,pcorr,converged) - ! ! call fftwf(pcorr, Fp) - ! ! call solmpj ! Cross dependency conflict - ! ! call fftwb(pcorr, Fp) - ! else ! Use iterative solver (HYPRE) - ! call set_zero_guess() - ! call solve_hypre(initsolver,pcorr,converged) - ! endif - ! call openboundary_excjs(pcorr , 2,i1,2,j1,1,kmax,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) - ! if(lboundary(1).and. .not. lperiodic(1)) pcorr(1,:,:) = pcorr(2,:,:) - ! if(lboundary(2).and. .not. lperiodic(2)) pcorr(i2,:,:) = pcorr(i1,:,:) - ! if(lboundary(3).and. .not. lperiodic(3)) pcorr(:,1,:) = pcorr(:,2,:) - ! if(lboundary(4).and. .not. lperiodic(4)) pcorr(:,j2,:) = pcorr(:,j1,:) - ! ! Apply correction - ! do k=1,kmax - ! do j=2,j1 - ! do i=2,i1 - ! um(i,j,k) = um(i,j,k)-(pcorr(i,j,k)-pcorr(i-1,j,k))/dx - ! vm(i,j,k) = vm(i,j,k)-(pcorr(i,j,k)-pcorr(i,j-1,k))/dy - ! end do - ! end do - ! end do - ! do k=2,kmax - ! do j=2,j1 - ! do i=2,i1 - ! wm(i,j,k) = wm(i,j,k)-(pcorr(i,j,k)-pcorr(i,j,k-1))/dzh(k) - ! end do - ! end do - ! end do - ! call openboundary_excjs(um , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) - ! call openboundary_excjs(vm , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) - ! call openboundary_excjs(wm , 2,i1,2,j1,1,k1,ih,jh,.not.lboundary(1:4).or.lperiodic(1:4)) - ! u0 = um; v0 = vm; w0 = wm - ! enddo - ! !if(solver_id/=100) call exithypre_solver(initsolver) - ! if(solver_id/=100) call exithypre_solver(initsolver) - ! !nullify(Fp) - ! endif end subroutine openboundary_divcorr subroutine openboundary_ghost diff --git a/src/modstartup.f90 b/src/modstartup.f90 index bf647cd8..34f68bf7 100644 --- a/src/modstartup.f90 +++ b/src/modstartup.f90 @@ -61,7 +61,6 @@ subroutine startup(path) ibas_prf,lambda_crit,iadv_mom,iadv_tke,iadv_thl,iadv_qt,iadv_sv,courant,peclet,ladaptive,author,lnoclouds,lrigidlid,unudge, & solver_id, maxiter, maxiter_precond, tolerance, n_pre, n_post, precond_id, checknamelisterror, & lopenbc,linithetero,lperiodic,dxint,dyint,dzint,taum,tauh,pbc,lsynturb,nmodes,tau,lambda,lambdas,lambdas_x,lambdas_y,lambdas_z,iturb, & - solver_id_init, maxiter_init, tolerance_init, n_pre_init, n_post_init, precond_id_init, maxiter_precond_init, & hypre_logging,rdt,rk3step,i1,j1,k1,ih,jh,lboundary use modforces, only : lforce_user use modsurfdata, only : z0,ustin,wtsurf,wqsurf,wsvsurf,ps,thls,isurf @@ -115,8 +114,7 @@ subroutine startup(path) namelist/SOLVER/ & solver_id, maxiter, tolerance, n_pre, n_post, precond_id, maxiter_precond, hypre_logging namelist/OPENBC/ & - lopenbc,linithetero,lper,dxint,dyint,dzint,taum,tauh,pbc,lsynturb,iturb,tau,lambda,nmodes,lambdas,lambdas_x,lambdas_y,lambdas_z, & - solver_id_init,maxiter_init,tolerance_init,precond_id_init,n_pre_init,n_post_init,maxiter_precond_init + lopenbc,linithetero,lper,dxint,dyint,dzint,taum,tauh,pbc,lsynturb,iturb,tau,lambda,nmodes,lambdas,lambdas_x,lambdas_y,lambdas_z ! get myid @@ -303,13 +301,6 @@ subroutine startup(path) call MPI_BCAST(lambdas_x,1,MY_REAL ,0,commwrld,mpierr) call MPI_BCAST(lambdas_y,1,MY_REAL ,0,commwrld,mpierr) call MPI_BCAST(lambdas_z,1,MY_REAL ,0,commwrld,mpierr) - call MPI_BCAST(solver_id_init,1,MPI_INTEGER,0,commwrld,mpierr) - call MPI_BCAST(maxiter_init,1,MPI_INTEGER,0,commwrld,mpierr) - call MPI_BCAST(n_pre_init,1,MPI_INTEGER,0,commwrld,mpierr) - call MPI_BCAST(n_post_init,1,MPI_INTEGER,0,commwrld,mpierr) - call MPI_BCAST(tolerance_init,1,MY_REAL,0,commwrld,mpierr) - call MPI_BCAST(precond_id_init,1,MPI_INTEGER,0,commwrld,mpierr) - call MPI_BCAST(maxiter_precond_init,1,MPI_INTEGER,0,commwrld,mpierr) call testwctime ! Allocate and initialize core modules