From 6901981a0b29f3ed7bb7cfc45d7aac0e6afb9174 Mon Sep 17 00:00:00 2001 From: Jun Wang <37633869+junwang-noaa@users.noreply.github.com> Date: Wed, 2 Feb 2022 10:45:22 -0500 Subject: [PATCH] Add 2d decomposition on the write grid comp (#470) * update write grid comp to have 2D decomposition on output grid. * combine PR#468: Bug fix in FV3GFS_io.F90 for allocation of temp2d Co-authored-by: Ted Mansell --- io/FV3GFS_io.F90 | 2 +- io/module_wrt_grid_comp.F90 | 26 +++++++++++++++++++------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/io/FV3GFS_io.F90 b/io/FV3GFS_io.F90 index b45190f3d..b38084585 100644 --- a/io/FV3GFS_io.F90 +++ b/io/FV3GFS_io.F90 @@ -196,7 +196,7 @@ subroutine FV3GFS_GFS_checksum (Model, GFS_Data, Atm_block) nsfcprop2d = nsfcprop2d + 16 endif - allocate (temp2d(isc:iec,jsc:jec,nsfcprop2d+Model%ntot3d+Model%nctp)) + allocate (temp2d(isc:iec,jsc:jec,nsfcprop2d+Model%ntot2d+Model%nctp)) allocate (temp3d(isc:iec,jsc:jec,1:lev,14+Model%ntot3d+2*ntr)) allocate (temp3dlevsp1(isc:iec,jsc:jec,1:lev+1,3)) diff --git a/io/module_wrt_grid_comp.F90 b/io/module_wrt_grid_comp.F90 index 97c9c8885..507e35a4d 100644 --- a/io/module_wrt_grid_comp.F90 +++ b/io/module_wrt_grid_comp.F90 @@ -61,6 +61,7 @@ module module_wrt_grid_comp integer,save :: lead_write_task !<-- Rank of the first write task in the write group integer,save :: last_write_task !<-- Rank of the last write task in the write group integer,save :: ntasks !<-- # of write tasks in the current group + integer,save :: itasks, jtasks !<-- # of write tasks in i/j direction in the current group integer,save :: mytile !<-- the tile number in write task integer,save :: wrt_mpi_comm !<-- the mpi communicator in the write comp @@ -294,11 +295,19 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) print *,'output_grid=',trim(output_grid) end if + call ESMF_ConfigGetAttribute(config=CF, value=itasks,default=1,label ='itasks:',rc=rc) + jtasks = ntasks + if(itasks > 0 ) jtasks = ntasks/itasks + if( itasks*jtasks /= ntasks ) then + itasks = 1 + jtasks = ntasks + endif + if(trim(output_grid) == 'gaussian_grid' .or. trim(output_grid) == 'global_latlon') then call ESMF_ConfigGetAttribute(config=CF, value=imo, label ='imo:',rc=rc) call ESMF_ConfigGetAttribute(config=CF, value=jmo, label ='jmo:',rc=rc) if (lprnt) then - print *,'imo=',imo,'jmo=',jmo + print *,'imo=',imo,'jmo=',jmo,'itasks=',itasks,'jtasks=',jtasks end if else if(trim(output_grid) == 'regional_latlon') then call ESMF_ConfigGetAttribute(config=CF, value=lon1, label ='lon1:',rc=rc) @@ -422,7 +431,7 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) else if ( trim(output_grid) == 'gaussian_grid') then wrtgrid = ESMF_GridCreate1PeriDim(minIndex=(/1,1/), & - maxIndex=(/imo,jmo/), regDecomp=(/1,ntasks/), & + maxIndex=(/imo,jmo/), regDecomp=(/itasks,jtasks/), & indexflag=ESMF_INDEX_GLOBAL, & name='wrt_grid',rc=rc) ! indexflag=ESMF_INDEX_GLOBAL, coordSys=ESMF_COORDSYS_SPH_DEG @@ -462,8 +471,9 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) latPtr(i,j) = lat(j) enddo enddo -! print *,'aft wrtgrd, Gaussian, dimi,i=',lbound(lonPtr,1),ubound(lonPtr,1), & -! ' j=',lbound(lonPtr,2),ubound(lonPtr,2),'imo=',imo,'jmo=',jmo + if(lprnt) print *,'aft wrtgrd, Gaussian, dimi,i=',lbound(lonPtr,1),ubound(lonPtr,1), & + lbound(lonPtr,2),ubound(lonPtr,2),'j(i)=',lbound(latPtr,1),ubound(latPtr,1),& + ' j(j)=',lbound(latPtr,2),ubound(latPtr,2),'imo=',imo,'jmo=',jmo ! if(wrt_int_state%mype==0) print *,'aft wrtgrd, lon=',lonPtr(1:5,1), & ! 'lat=',latPtr(1,1:5),'imo,jmo=',imo,jmo ! lonPtr(lbound(lonPtr,1),ubound(lonPtr,2)),'lat=',latPtr(lbound(lonPtr,1),lbound(lonPtr,2)), & @@ -479,7 +489,9 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) call mpi_allgather(wrt_int_state%lat_end, 1,MPI_INTEGER, & wrt_int_state%lat_end_wrtgrp, 1, MPI_INTEGER, wrt_mpi_comm, rc) if( lprnt ) print *,'aft wrtgrd, Gaussian, dimj_start=',wrt_int_state%lat_start_wrtgrp, & - 'dimj_end=',wrt_int_state%lat_end_wrtgrp, 'wrt_group=',n_group + 'dimj_end=',wrt_int_state%lat_end_wrtgrp, 'wrt_group=',n_group, & + 'lon_start,end=',wrt_int_state%lon_start,wrt_int_state%lon_end, & + 'lat_start,end=',wrt_int_state%lat_start, wrt_int_state%lat_end allocate( wrt_int_state%latPtr(wrt_int_state%lon_start:wrt_int_state%lon_end, & wrt_int_state%lat_start:wrt_int_state%lat_end)) allocate( wrt_int_state%lonPtr(wrt_int_state%lon_start:wrt_int_state%lon_end, & @@ -497,7 +509,7 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) deallocate(slat) else if ( trim(output_grid) == 'global_latlon') then wrtgrid = ESMF_GridCreate1PeriDim(minIndex=(/1,1/), & - maxIndex=(/imo,jmo/), regDecomp=(/1,ntasks/), & + maxIndex=(/imo,jmo/), regDecomp=(/itasks,jtasks/), & indexflag=ESMF_INDEX_GLOBAL, name='wrt_grid',rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -581,7 +593,7 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) trim(output_grid) == 'lambert_conformal' ) then wrtgrid = ESMF_GridCreate1PeriDim(minIndex=(/1,1/), & - maxIndex=(/imo,jmo/), regDecomp=(/1,ntasks/), & + maxIndex=(/imo,jmo/), regDecomp=(/itasks,jtasks/), & indexflag=ESMF_INDEX_GLOBAL, & name='wrt_grid',rc=rc)