Open boundary conditions and FFTW

FFTW can be used also with open lateral boundary conditions.
Pressure solver boundary condition is Neumann on the open boundaries.
The following modifications are made for the direction(s) with open boundaries:
* use Discrete Cosine Transform instead of FFT.
  - FFTW_REDFT_10 forward
  - FFTW_REDFT_01 back
* normalization constant is different, a factor 2 per direction
* the eigenvalues xrt, yrt are different

By Pedro Simões Costa and Fredrik Jansson

src/modfftw.f90

@@ -65,7 +65,7 @@ module modfftw
   subroutine fftwinit(p, Fp, d, xyrt, ps,pe,qs,qe)
 
     use modmpi, only   : nprocx, nprocy
-    use modglobal, only: itot, jtot, imax, jmax, i1, j1, ih, jh, kmax
+    use modglobal, only: itot, jtot, imax, jmax, i1, j1, ih, jh, kmax, lperiodic
     use, intrinsic  :: iso_c_binding
     implicit none
 
@@ -146,7 +146,11 @@ subroutine fftwinit(p, Fp, d, xyrt, ps,pe,qs,qe)
     ! Prepare 1d FFT transforms
     ! TODO: in plan_many, skip part where k > kmax
     embed(1) = itot
-    kinds(1) = FFTW_R2HC
+    if (lperiodic(1)) then
+       kinds(1) = FFTW_R2HC
+    else
+       kinds(1) = FFTW_REDFT10 ! DCT for Neumann BC for pressure
+    end if
     planx = fftw_plan_many_r2r( &
       1, &           ! rank
       embed, &       ! n (size)  [array]
@@ -164,7 +168,11 @@ subroutine fftwinit(p, Fp, d, xyrt, ps,pe,qs,qe)
     )
 
     embed(1) = itot
-    kinds(1) = FFTW_HC2R
+    if (lperiodic(1)) then
+       kinds(1) = FFTW_HC2R
+    else
+       kinds(1) = FFTW_REDFT01 ! Inverse DCT
+    end if
     planxi = fftw_plan_many_r2r( &
       1, &           ! rank
       embed, &       ! n (size)  [array]
@@ -182,7 +190,11 @@ subroutine fftwinit(p, Fp, d, xyrt, ps,pe,qs,qe)
     )
 
     embed(1) = jtot
-    kinds(1) = FFTW_R2HC
+    if (lperiodic(3)) then
+       kinds(1) = FFTW_R2HC
+    else
+       kinds(1) = FFTW_REDFT10
+    end if
     plany = fftw_plan_many_r2r( &
       1, &           ! rank
       embed, &       ! n (size)  [array]
@@ -200,7 +212,11 @@ subroutine fftwinit(p, Fp, d, xyrt, ps,pe,qs,qe)
     )
 
     embed(1) = jtot
-    kinds(1) = FFTW_HC2R
+    if (lperiodic(3)) then
+       kinds(1) = FFTW_HC2R
+    else
+       kinds(1) = FFTW_REDFT01 ! Inverse DCT
+    end if
     planyi = fftw_plan_many_r2r( &
       1, &           ! rank
       embed, &       ! n (size)  [array]
@@ -246,6 +262,9 @@ subroutine fftwinit(p, Fp, d, xyrt, ps,pe,qs,qe)
 
     kinds(1) = FFTW_R2HC
     kinds(2) = FFTW_R2HC
+    if (.not. lperiodic(1)) kinds(1) = FFTW_REDFT10
+    if (.not. lperiodic(3)) kinds(2) = FFTW_REDFT10
+
     planxy = fftw_plan_guru_r2r(&
       2, &             ! rank
       dimij, &         ! dims
@@ -259,6 +278,8 @@ subroutine fftwinit(p, Fp, d, xyrt, ps,pe,qs,qe)
 
     kinds(1) = FFTW_HC2R
     kinds(2) = FFTW_HC2R
+    if (.not. lperiodic(1)) kinds(1) = FFTW_REDFT01
+    if (.not. lperiodic(3)) kinds(2) = FFTW_REDFT01
     planxyi = fftw_plan_guru_r2r(&
       2, &             ! rank
       dimij, &         ! dims
@@ -644,20 +665,21 @@ subroutine fftwf(p, Fp)
       stop 'Illegal method in fftwsolver.'
     endif
 
-    Fp(:,:,:) = Fp(:,:,:) / sqrt(ijtot)
+    !Fp(:,:,:) = Fp(:,:,:) / sqrt(ijtot)  ! moving normalization to fftwb
   end subroutine
 
   subroutine fftwb(p, Fp)
 
-    use modglobal, only : ijtot, i1, j1, ih, jh, kmax
+    use modglobal, only : ijtot, i1, j1, ih, jh, kmax, lperiodic
     use modmpi, only    : myidx,myidy,nprocx
 
     implicit none
 
     real, pointer :: p(:,:,:)
     real, pointer :: Fp(:,:,:)
+    real :: norm
 
-    Fp(:,:,:) = Fp(:,:,:) / sqrt(ijtot)
+    !Fp(:,:,:) = Fp(:,:,:) / sqrt(ijtot)
 
     if (method == 1) then
       call transpose_a3inv(p201, Fp)
@@ -673,12 +695,17 @@ subroutine fftwb(p, Fp)
     else
       stop 'Illegal method in fftwsolver.'
     endif
+
+    norm = 1.0/ijtot
+    if (.not. lperiodic(1)) norm = norm / 2 ! different normalization for the DCT
+    if (.not. lperiodic(3)) norm = norm / 2
+    p(:,:,:) = p(:,:,:) * norm ! do all normalization at once
   end subroutine
 
 
 ! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
   subroutine fftwinit_factors(xyrt)
-    use modglobal, only : i1,j1,kmax,imax,jmax,itot,jtot,dxi,dyi,pi,ih,jh
+    use modglobal, only : i1,j1,kmax,imax,jmax,itot,jtot,dxi,dyi,pi,ih,jh,lperiodic
     use modmpi, only    : myidx, myidy
 
     implicit none
@@ -707,27 +734,19 @@ subroutine fftwinit_factors(xyrt)
 
   ! I --> direction
     fac = 1./(2.*itot)
-    do i=2,(itot/2)
+    if (.not. lperiodic(1)) fac = 1./(4.*itot)
+    do i=2,itot
       xrt(i)=-4.*dxi*dxi*(sin(float(2*(i-1))*pi*fac))**2
-      xrt(itot - i + 2) = xrt(i)
     end do
     xrt(1) = 0.
-    if (mod(itot,2) == 0) then
-      ! Nyquist frequency
-      xrt(1 + itot/2) = -4.*dxi*dxi
-    endif
 
   ! J --> direction
     fac = 1./(2.*jtot)
-    do j=2,(jtot/2)
+    if (.not. lperiodic(3)) fac = 1./(4.*jtot)
+    do j=2,jtot
       yrt(j)=-4.*dyi*dyi*(sin(float(2*(j-1))*pi*fac))**2
-      yrt(jtot - j + 2) = yrt(j)
     end do
     yrt(1) = 0.
-    if (mod(jtot,2) == 0) then
-      ! Nyquist frequency
-      yrt(1 + jtot/2) = -4.*dyi*dyi
-    endif
 
   ! Combine I and J directions
   ! Note that:

src/modglobal.f90

@@ -225,7 +225,7 @@ module modglobal
       end type
       type(boundary_type), dimension(5) :: boundary
       logical, dimension(5) :: lboundary = .false.
-      logical, dimension(5) :: lperiodic = .false.
+      logical, dimension(5) :: lperiodic =  (/.true., .true., .true., .true., .false./)
       real :: dxint=-1.,dyint=-1.,dzint=-1.,tauh=60.,taum=0.,tau=60.,lambda,lambdas=-1.,lambdas_x=-1.,lambdas_y=-1.,lambdas_z=-1.
       integer :: nmodes=100,ntboundary=1,pbc = 3,iturb=0
       real,dimension(:),allocatable :: tboundary

src/modopenboundary.f90

@@ -57,7 +57,7 @@ subroutine initopenboundary
 
     if(.not.lopenbc) return
     ! Check for conflicting options
-    if(solver_id == 0 .or. solver_id == 100) stop 'Openboundaries only possible with HYPRE pressure solver, change solver_id'
+    if(solver_id == 0) stop 'Openboundaries only possible with HYPRE or FFTW pressure solver, change solver_id'
     if(iadv_mom /=2) stop 'Only second order advection scheme supported with openboundaries, change iadv_mom to 2'
     if(iadv_thl /=2) stop 'Only second order advection scheme supported with openboundaries, change iadv_thl to 2'
     if(iadv_qt  /=2) stop 'Only second order advection scheme supported with openboundaries, change iadv_qt to 2'
@@ -434,7 +434,7 @@ subroutine openboundary_divcorr()
       n_pre,n_post,solver_id_init,maxiter_init,tolerance_init,precond_id_init,maxiter_precond_init, &
       n_pre_init,n_post_init,ih,jh,dzh
     use modfields, only : u0,um,v0,vm,w0,wm,rhobf,rhobh
-    use modhypre, only : inithypre_solver, solve_hypre,set_zero_guess,exithypre_solver
+    use modhypre, only : inithypre_grid, inithypre_solver, solve_hypre,set_zero_guess,exithypre_solver
     !use modchecksim, only : chkdiv
     implicit none
     real :: sumdiv,divold,divnew,div,divpart
@@ -578,6 +578,9 @@ subroutine openboundary_divcorr()
       if(precond_id_init == -1) precond_id_init = precond_id
       if(n_pre_init == -1) n_pre_init = n_pre
       if(n_post_init == -1) n_post_init = n_post
+      if(solver_id == 100) then
+         call inithypre_grid ! if FFTW is used during run, need to init hypre grid here(?)
+      end if
       call inithypre_solver(initsolver,solver_id_init,maxiter_init,tolerance_init,precond_id_init,n_pre_init,n_post_init,maxiter_precond_init)
       iter = 0
       do while(.True.)