diff --git a/atmos_model.F90 b/atmos_model.F90 index 6725b1809..2fa6788cd 100644 --- a/atmos_model.F90 +++ b/atmos_model.F90 @@ -2923,19 +2923,19 @@ subroutine setup_exportdata(rc) call block_data_copy(datar82d, GFS_data(nb)%coupling%v10mi_cpl, Atm_block, nb, rc=localrc) ! Instantaneous Zonal compt of momentum flux (N/m**2) case ('inst_zonal_moment_flx') - call block_data_copy(datar82d, GFS_data(nb)%coupling%dusfci_cpl, Atm_block, nb, rc=localrc) + call block_data_copy(datar82d, GFS_data(nb)%coupling%dusfci_cpl, Atm_block, nb, -one, spval, rc=localrc) ! Instantaneous Merid compt of momentum flux (N/m**2) case ('inst_merid_moment_flx') - call block_data_copy(datar82d, GFS_data(nb)%coupling%dvsfci_cpl, Atm_block, nb, rc=localrc) + call block_data_copy(datar82d, GFS_data(nb)%coupling%dvsfci_cpl, Atm_block, nb, -one, spval, rc=localrc) ! Instantaneous Sensible heat flux (W/m**2) case ('inst_sensi_heat_flx') - call block_data_copy(datar82d, GFS_data(nb)%coupling%dtsfci_cpl, Atm_block, nb, rc=localrc) + call block_data_copy(datar82d, GFS_data(nb)%coupling%dtsfci_cpl, Atm_block, nb, -one, spval, rc=localrc) ! Instantaneous Latent heat flux (W/m**2) case ('inst_laten_heat_flx') - call block_data_copy(datar82d, GFS_data(nb)%coupling%dqsfci_cpl, Atm_block, nb, rc=localrc) + call block_data_copy(datar82d, GFS_data(nb)%coupling%dqsfci_cpl, Atm_block, nb, -one, spval, rc=localrc) ! Instantaneous Evap flux (kg/m**2/s) case ('inst_evap_rate') - call block_data_copy(datar82d, GFS_data(nb)%coupling%dqsfci_cpl, Atm_block, nb, revap, spval, rc=localrc) + call block_data_copy(datar82d, GFS_data(nb)%coupling%dqsfci_cpl, Atm_block, nb, -revap, spval, rc=localrc) ! Instantaneous Downward long wave radiation flux (W/m**2) case ('inst_down_lw_flx') call block_data_copy(datar82d, GFS_data(nb)%coupling%dlwsfci_cpl, Atm_block, nb, rc=localrc) @@ -2993,19 +2993,19 @@ subroutine setup_exportdata(rc) !--- Mean quantities ! MEAN Zonal compt of momentum flux (N/m**2) case ('mean_zonal_moment_flx_atm') - call block_data_copy(datar82d, GFS_data(nb)%coupling%dusfc_cpl, Atm_block, nb, rtime, spval, rc=localrc) + call block_data_copy(datar82d, GFS_data(nb)%coupling%dusfc_cpl, Atm_block, nb, -rtime, spval, rc=localrc) ! MEAN Merid compt of momentum flux (N/m**2) case ('mean_merid_moment_flx_atm') - call block_data_copy(datar82d, GFS_data(nb)%coupling%dvsfc_cpl, Atm_block, nb, rtime, spval, rc=localrc) + call block_data_copy(datar82d, GFS_data(nb)%coupling%dvsfc_cpl, Atm_block, nb, -rtime, spval, rc=localrc) ! MEAN Sensible heat flux (W/m**2) case ('mean_sensi_heat_flx') - call block_data_copy(datar82d, GFS_data(nb)%coupling%dtsfc_cpl, Atm_block, nb, rtime, spval, rc=localrc) + call block_data_copy(datar82d, GFS_data(nb)%coupling%dtsfc_cpl, Atm_block, nb, -rtime, spval, rc=localrc) ! MEAN Latent heat flux (W/m**2) case ('mean_laten_heat_flx') - call block_data_copy(datar82d, GFS_data(nb)%coupling%dqsfc_cpl, Atm_block, nb, rtime, spval, rc=localrc) + call block_data_copy(datar82d, GFS_data(nb)%coupling%dqsfc_cpl, Atm_block, nb, -rtime, spval, rc=localrc) ! MEAN Evap rate (kg/m**2/s) case ('mean_evap_rate') - call block_data_copy(datar82d, GFS_data(nb)%coupling%dqsfc_cpl, Atm_block, nb, rtime*revap, rc=localrc) + call block_data_copy(datar82d, GFS_data(nb)%coupling%dqsfc_cpl, Atm_block, nb, -rtime*revap, spval, rc=localrc) ! MEAN Downward LW heat flux (W/m**2) case ('mean_down_lw_flx') call block_data_copy(datar82d, GFS_data(nb)%coupling%dlwsfc_cpl, Atm_block, nb, rtime, spval, rc=localrc) diff --git a/ccpp/physics b/ccpp/physics index 61b419f13..be2e7bbea 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 61b419f132ea24cbe191e34f1bbfcedf2c66e0dd +Subproject commit be2e7bbea91d8e7bf999957f9d48981147120562 diff --git a/fv3_cap.F90 b/fv3_cap.F90 index 50ad49104..1bca9b004 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -209,7 +209,7 @@ subroutine InitializeAdvertise(gcomp, rc) integer :: wrttasks_per_group_from_parent, wrtLocalPet, num_threads character(len=64) :: rh_filename logical :: use_saved_routehandles, rh_file_exist - logical :: fieldbundle_is_restart = .false. + logical :: fieldbundle_uses_redist = .false. integer :: sloc type(ESMF_StaggerLoc) :: staggerloc @@ -698,11 +698,12 @@ subroutine InitializeAdvertise(gcomp, rc) if(mype == 0) print *,'af get wrtfb=',"output_"//trim(fcstItemNameList(j)),' rc=',rc if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - fieldbundle_is_restart = .false. + fieldbundle_uses_redist = .false. + ! if (fcstItemNameList(j)(1:8) == "restart_" .or. fcstItemNameList(j)(1:18) == "cubed_sphere_grid_") then if (fcstItemNameList(j)(1:8) == "restart_") then ! restart output forecast bundles, no need to set regridmethod ! Redist will be used instead of Regrid - fieldbundle_is_restart = .true. + fieldbundle_uses_redist = .true. else ! history output forecast bundles ! determine regridmethod @@ -739,7 +740,7 @@ subroutine InitializeAdvertise(gcomp, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return else ! this is a Store() for the first wrtComp -> must do the Store() - if (fieldbundle_is_restart) then + if (fieldbundle_uses_redist) then call ESMF_TraceRegionEnter("ESMF_FieldBundleRedistStore()", rc=rc) call ESMF_FieldBundleRedistStore(fcstFB(j), wrtFB(j,1), & routehandle=routehandle(j,1), & @@ -1097,6 +1098,7 @@ subroutine ModelAdvance_phase2(gcomp, rc) character(240) :: msgString type(ESMF_Clock) :: clock, clock_out + integer :: fieldCount !----------------------------------------------------------------------------- @@ -1147,12 +1149,17 @@ subroutine ModelAdvance_phase2(gcomp, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return endif - ! execute the routehandle from fcstFB -> wrtFB (either Regrid() or Redist()) - call ESMF_FieldBundleSMM(fcstFB(j), wrtFB(j,n_group), & - routehandle=routehandle(j, n_group), & - termorderflag=(/ESMF_TERMORDER_SRCSEQ/), rc=rc) + ! execute the routehandle from fcstFB -> wrtFB (either Regrid() or Redist()), only if there are fields in the bundle + call ESMF_FieldBundleGet(fcstFB(j), fieldCount=fieldCount, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if (fieldCount > 0) then + call ESMF_FieldBundleSMM(fcstFB(j), wrtFB(j,n_group), & + routehandle=routehandle(j, n_group), & + termorderflag=(/ESMF_TERMORDER_SRCSEQ/), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + end if + enddo call ESMF_VMEpochExit(rc=rc) diff --git a/io/module_write_netcdf.F90 b/io/module_write_netcdf.F90 index 380ea5975..4b0506549 100644 --- a/io/module_write_netcdf.F90 +++ b/io/module_write_netcdf.F90 @@ -11,7 +11,7 @@ module module_write_netcdf use netcdf use module_fv3_io_def,only : ideflate, nbits, & ichunk2d,jchunk2d,ichunk3d,jchunk3d,kchunk3d, & - output_grid,dx,dy,lon1,lat1,lon2,lat2, & + dx,dy,lon1,lat1,lon2,lat2, & time_unlimited use mpi @@ -95,6 +95,7 @@ subroutine write_netcdf(wrtfb, filename, & integer, dimension(:), allocatable :: deToTileMap, localDeToDeMap logical :: do_io integer :: par_access + character(len=ESMF_MAXSTR) :: output_grid_name ! is_cubed_sphere = .false. tileCount = 0 @@ -106,13 +107,15 @@ subroutine write_netcdf(wrtfb, filename, & do_io = par .or. (mype==0) call ESMF_FieldBundleGet(wrtfb, fieldCount=fieldCount, rc=rc); ESMF_ERR_RETURN(rc) + call ESMF_AttributeGet(wrtfb, convention="NetCDF", purpose="FV3", & + name='grid', value=output_grid_name, rc=rc); ESMF_ERR_RETURN(rc) allocate(compress_err(fieldCount)); compress_err=-999. allocate(fldlev(fieldCount)) ; fldlev = 0 allocate(fcstField(fieldCount)) allocate(varids(fieldCount)) - call ESMF_FieldBundleGet(wrtfb, fieldList=fcstField, grid=wrtGrid, & + call ESMF_FieldBundleGet(wrtfb, fieldList=fcstField, grid=wrtgrid, & ! itemorderflag=ESMF_ITEMORDER_ADDORDER, & rc=rc); ESMF_ERR_RETURN(rc) @@ -162,6 +165,10 @@ subroutine write_netcdf(wrtfb, filename, & start_i = 1 start_j = 1 end if + if (is_cubed_sphere) then + start_i = mod(start_i, im) + start_j = mod(start_j, jm) + end if end if if (fieldDimCount > gridDimCount) then @@ -240,21 +247,18 @@ subroutine write_netcdf(wrtfb, filename, & ncerr = nf90_put_att(ncid, timeiso_varid, "_Encoding", "UTF-8"); NC_ERR_STOP(ncerr) ! coordinate variable attributes based on output_grid type - if (trim(output_grid(grid_id)) == 'gaussian_grid' .or. & - trim(output_grid(grid_id)) == 'global_latlon' .or. & - trim(output_grid(grid_id)) == 'regional_latlon' .or. & - trim(output_grid(grid_id)) == 'regional_latlon_moving') then + if (trim(output_grid_name) == 'gaussian' .or. & + trim(output_grid_name) == 'latlon') then ncerr = nf90_put_att(ncid, im_varid, "long_name", "T-cell longitude"); NC_ERR_STOP(ncerr) ncerr = nf90_put_att(ncid, im_varid, "units", "degrees_E"); NC_ERR_STOP(ncerr) ncerr = nf90_put_att(ncid, jm_varid, "long_name", "T-cell latiitude"); NC_ERR_STOP(ncerr) ncerr = nf90_put_att(ncid, jm_varid, "units", "degrees_N"); NC_ERR_STOP(ncerr) - else if (trim(output_grid(grid_id)) == 'rotated_latlon' .or. & - trim(output_grid(grid_id)) == 'rotated_latlon_moving') then + else if (trim(output_grid_name) == 'rotated_latlon') then ncerr = nf90_put_att(ncid, im_varid, "long_name", "rotated T-cell longiitude"); NC_ERR_STOP(ncerr) ncerr = nf90_put_att(ncid, im_varid, "units", "degrees"); NC_ERR_STOP(ncerr) ncerr = nf90_put_att(ncid, jm_varid, "long_name", "rotated T-cell latiitude"); NC_ERR_STOP(ncerr) ncerr = nf90_put_att(ncid, jm_varid, "units", "degrees"); NC_ERR_STOP(ncerr) - else if (trim(output_grid(grid_id)) == 'lambert_conformal') then + else if (trim(output_grid_name) == 'lambert_conformal') then ncerr = nf90_put_att(ncid, im_varid, "long_name", "x-coordinate of projection"); NC_ERR_STOP(ncerr) ncerr = nf90_put_att(ncid, im_varid, "units", "meters"); NC_ERR_STOP(ncerr) ncerr = nf90_put_att(ncid, jm_varid, "long_name", "y-coordinate of projection"); NC_ERR_STOP(ncerr) @@ -466,10 +470,10 @@ subroutine write_netcdf(wrtfb, filename, & ! write lon (lon_varid) if (par) then - call ESMF_GridGetCoord(wrtGrid, coordDim=1, farrayPtr=array_r8, rc=rc); ESMF_ERR_RETURN(rc) + call ESMF_GridGetCoord(wrtgrid, coordDim=1, farrayPtr=array_r8, rc=rc); ESMF_ERR_RETURN(rc) ncerr = nf90_put_var(ncid, lon_varid, values=array_r8, start=start_idx); NC_ERR_STOP(ncerr) else - call ESMF_GridGetCoord(wrtGrid, coordDim=1, array=array, rc=rc); ESMF_ERR_RETURN(rc) + call ESMF_GridGetCoord(wrtgrid, coordDim=1, array=array, rc=rc); ESMF_ERR_RETURN(rc) if (is_cubed_sphere) then do t=1,tileCount call ESMF_ArrayGather(array, array_r8_cube(:,:,t), rootPet=0, tile=t, rc=rc); ESMF_ERR_RETURN(rc) @@ -491,39 +495,35 @@ subroutine write_netcdf(wrtfb, filename, & ! write grid_xt (im_varid) if (do_io) then allocate (x(im)) - if (trim(output_grid(grid_id)) == 'gaussian_grid' .or. & - trim(output_grid(grid_id)) == 'global_latlon' .or. & - trim(output_grid(grid_id)) == 'regional_latlon' .or. & - trim(output_grid(grid_id)) == 'regional_latlon_moving') then + if (trim(output_grid_name) == 'gaussian' .or. trim(output_grid_name) == 'latlon') then ncerr = nf90_put_var(ncid, im_varid, values=array_r8(:,jstart), start=[istart], count=[iend-istart+1]); NC_ERR_STOP(ncerr) - else if (trim(output_grid(grid_id)) == 'rotated_latlon' .or. & - trim(output_grid(grid_id)) == 'rotated_latlon_moving') then + else if (trim(output_grid_name) == 'rotated_latlon') then do i=1,im x(i) = lon1(grid_id) + (lon2(grid_id)-lon1(grid_id))/(im-1) * (i-1) end do ncerr = nf90_put_var(ncid, im_varid, values=x); NC_ERR_STOP(ncerr) - else if (trim(output_grid(grid_id)) == 'lambert_conformal') then + else if (trim(output_grid_name) == 'lambert_conformal') then do i=1,im x(i) = dx(grid_id) * (i-1) end do ncerr = nf90_put_var(ncid, im_varid, values=x); NC_ERR_STOP(ncerr) - else if (trim(output_grid(grid_id)) == 'cubed_sphere_grid') then + else if (trim(output_grid_name) == 'cubed_sphere') then do i=1,im x(i) = i end do ncerr = nf90_put_var(ncid, im_varid, values=x); NC_ERR_STOP(ncerr) else - if (mype==0) write(0,*)'unknown output_grid ', trim(output_grid(grid_id)) + if (mype==0) write(0,*)'unknown output_grid ', trim(output_grid_name) call ESMF_Finalize(endflag=ESMF_END_ABORT) end if end if ! write lat (lat_varid) if (par) then - call ESMF_GridGetCoord(wrtGrid, coordDim=2, farrayPtr=array_r8, rc=rc); ESMF_ERR_RETURN(rc) + call ESMF_GridGetCoord(wrtgrid, coordDim=2, farrayPtr=array_r8, rc=rc); ESMF_ERR_RETURN(rc) ncerr = nf90_put_var(ncid, lat_varid, values=array_r8, start=start_idx); NC_ERR_STOP(ncerr) else - call ESMF_GridGetCoord(wrtGrid, coordDim=2, array=array, rc=rc); ESMF_ERR_RETURN(rc) + call ESMF_GridGetCoord(wrtgrid, coordDim=2, array=array, rc=rc); ESMF_ERR_RETURN(rc) if (is_cubed_sphere) then do t=1,tileCount call ESMF_ArrayGather(array, array_r8_cube(:,:,t), rootPet=0, tile=t, rc=rc); ESMF_ERR_RETURN(rc) @@ -542,29 +542,25 @@ subroutine write_netcdf(wrtfb, filename, & ! write grid_yt (jm_varid) if (do_io) then allocate (y(jm)) - if (trim(output_grid(grid_id)) == 'gaussian_grid' .or. & - trim(output_grid(grid_id)) == 'global_latlon' .or. & - trim(output_grid(grid_id)) == 'regional_latlon' .or. & - trim(output_grid(grid_id)) == 'regional_latlon_moving') then + if (trim(output_grid_name) == 'gaussian' .or. trim(output_grid_name) == 'latlon') then ncerr = nf90_put_var(ncid, jm_varid, values=array_r8(istart,:), start=[jstart], count=[jend-jstart+1]); NC_ERR_STOP(ncerr) - else if (trim(output_grid(grid_id)) == 'rotated_latlon' .or. & - trim(output_grid(grid_id)) == 'rotated_latlon_moving') then + else if (trim(output_grid_name) == 'rotated_latlon') then do j=1,jm y(j) = lat1(grid_id) + (lat2(grid_id)-lat1(grid_id))/(jm-1) * (j-1) end do ncerr = nf90_put_var(ncid, jm_varid, values=y); NC_ERR_STOP(ncerr) - else if (trim(output_grid(grid_id)) == 'lambert_conformal') then + else if (trim(output_grid_name) == 'lambert_conformal') then do j=1,jm y(j) = dy(grid_id) * (j-1) end do ncerr = nf90_put_var(ncid, jm_varid, values=y); NC_ERR_STOP(ncerr) - else if (trim(output_grid(grid_id)) == 'cubed_sphere_grid') then + else if (trim(output_grid_name) == 'cubed_sphere') then do j=1,jm y(j) = j end do ncerr = nf90_put_var(ncid, jm_varid, values=y); NC_ERR_STOP(ncerr) else - if (mype==0) write(0,*)'unknown output_grid ', trim(output_grid(grid_id)) + if (mype==0) write(0,*)'unknown output_grid ', trim(output_grid_name) call ESMF_Finalize(endflag=ESMF_END_ABORT) end if end if diff --git a/io/module_wrt_grid_comp.F90 b/io/module_wrt_grid_comp.F90 index 3cd17002f..97dcf2d1b 100644 --- a/io/module_wrt_grid_comp.F90 +++ b/io/module_wrt_grid_comp.F90 @@ -163,6 +163,8 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, type(ESMF_DELayout) :: delayout type(ESMF_Grid) :: fcstGrid type(ESMF_Grid), allocatable :: wrtGrid(:) + type(ESMF_Grid) :: wrtGrid_cubed_sphere + logical :: create_wrtGrid_cubed_sphere = .true. type(ESMF_Grid) :: actualWrtGrid type(ESMF_Array) :: array type(ESMF_Field) :: field_work, field @@ -208,6 +210,9 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, type(ESMF_DistGrid) :: acceptorDG, newAcceptorDG integer :: grid_id + + logical :: history_file_on_native_grid + character(len=esmf_maxstr) :: output_grid_name ! !----------------------------------------------------------------------- !*********************************************************************** @@ -480,10 +485,35 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return endif + call ESMF_ConfigGetAttribute(config=CF, value=history_file_on_native_grid, default=.false., & + label='history_file_on_native_grid:', rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + +#if 1 + if (n == 1 .and. top_parent_is_global .and. history_file_on_native_grid) then + do tl=1,6 + decomptile(1,tl) = 1 + decomptile(2,tl) = jidx + decompflagPTile(:,tl) = (/ESMF_DECOMP_SYMMEDGEMAX,ESMF_DECOMP_SYMMEDGEMAX/) + enddo + call ESMF_AttributeGet(imp_state_write, convention="NetCDF", purpose="FV3", & + name="gridfile", value=gridfile, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + wrtGrid_cubed_sphere = ESMF_GridCreateMosaic(filename="INPUT/"//trim(gridfile), & + regDecompPTile=decomptile,tileFilePath="INPUT/", & + decompflagPTile=decompflagPTile, & + staggerlocList=(/ESMF_STAGGERLOC_CENTER, ESMF_STAGGERLOC_CORNER/), & + name='wrt_grid', rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + create_wrtGrid_cubed_sphere = .false. + endif +#endif + if ( trim(output_grid(n)) == 'cubed_sphere_grid' ) then !*** Create cubed sphere grid from file - if (top_parent_is_global .and. n==1) then - gridfile = 'grid_spec.nc' ! global top-level parent + if (top_parent_is_global .and. n == 1) then do tl=1,6 decomptile(1,tl) = 1 decomptile(2,tl) = jidx @@ -493,7 +523,6 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, name="gridfile", value=gridfile, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_LogWrite("wrtComp: gridfile:"//trim(gridfile),ESMF_LOGMSG_INFO,rc=rc) wrtGrid(n) = ESMF_GridCreateMosaic(filename="INPUT/"//trim(gridfile), & regDecompPTile=decomptile,tileFilePath="INPUT/", & decompflagPTile=decompflagPTile, & @@ -528,8 +557,6 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - if (lprnt) print *,'in nested/regional cubed_sphere grid, regDecomp=',regDecomp,' PetMap=',petMap(1),petMap(ntasks), & - 'gridfile=',trim(gridfile) deallocate(petMap) endif else ! non 'cubed_sphere_grid' @@ -869,29 +896,51 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, call ESMF_StateAdd(imp_state_write, (/mirrorFB/), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return -! copy the fcstFB Attributes to the 'mirror_' FieldBundle + ! copy the fcstFB Attributes to the 'mirror_' FieldBundle call ESMF_AttributeCopy(fcstFB, mirrorFB, attcopy=ESMF_ATTCOPY_REFERENCE, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return endif ! deal with all of the Fields inside this fcstFB - call ESMF_FieldBundleGet(fcstFB, fieldCount=fieldCount, grid=fcstGrid, rc=rc) + call ESMF_FieldBundleGet(fcstFB, fieldCount=fieldCount, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return if (fieldCount > 0) then + call ESMF_FieldBundleGet(fcstFB, grid=fcstGrid, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + allocate(fcstField(fieldCount)) call ESMF_FieldBundleGet(fcstFB, fieldList=fcstField, & itemorderflag=ESMF_ITEMORDER_ADDORDER, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - actualWrtGrid = wrtGrid(grid_id) + if (fcstItemNameList(i)(1:18) == 'cubed_sphere_grid_') then + + if (create_wrtGrid_cubed_sphere) then + ! create a grid from fcstGrid on forecast grid comp, by rebalancing distgrid to the local PETs + ! access the acceptor DistGrid + call ESMF_GridGet(fcstGrid, distgrid=acceptorDG, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + ! rebalance the acceptor DistGrid across the local PETs + newAcceptorDG = ESMF_DistGridCreate(acceptorDG, balanceflag=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + wrtGrid_cubed_sphere = ESMF_GridCreate(fcstGrid, newAcceptorDG, copyAttributes=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + create_wrtGrid_cubed_sphere = .false. + end if + + actualWrtGrid = wrtGrid_cubed_sphere + call ESMF_AttributeSet(fieldbundle, convention="NetCDF", purpose="FV3-nooutput", name="output_grid", value="cubed_sphere_grid", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + else if (fcstItemNameList(i)(1:8) == 'restart_') then + ! If this is a 'restart' bundle the actual grid that the output field ('field_work' below) is created on + ! must be the same grid as forecast grid, not the output grid for this grid_id (wrtGrid(grid_id)). + ! For 'cubed_sphere_grid' these are the same, but for all other output grids (like Lambert) they are not. - ! If this is a 'restart' bundle the actual grid that the output field ('field_work' below) is created on - ! must be the same grid as forecast grid, not the output grid for this grid_id (wrtGrid(grid_id)). - ! For 'cubed_sphere_grid' these are the same, but for all other output grids (like Lambert) they are not. - if (fcstItemNameList(i)(1:8) == 'restart_') then ! create a grid from fcstGrid on forecast grid comp, by rebalancing distgrid to the local PETs ! access the acceptor DistGrid call ESMF_GridGet(fcstGrid, distgrid=acceptorDG, rc=rc) @@ -901,7 +950,11 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return actualWrtGrid = ESMF_GridCreate(fcstGrid, newAcceptorDG, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - end if ! end of setting actualWrtGrid for restart bundle + else + actualWrtGrid = wrtGrid(grid_id) + call ESMF_AttributeSet(fieldbundle, convention="NetCDF", purpose="FV3-nooutput", name="output_grid", value=output_grid(grid_id), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + end if do j=1, fieldCount @@ -925,7 +978,7 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, ! 'gridToFieldMap=',gridToFieldMap,'ungriddedLBound=',ungriddedLBound, & ! 'ungriddedUBound=',ungriddedUBound,'rc=',rc -! create the output field on output grid + ! create the output field on output grid field_work = ESMF_FieldCreate(actualWrtGrid, typekind, name=fieldName, & ! use actualWrtGrid instead of wrtGrid(grid_id) staggerloc=staggerloc, & gridToFieldMap=gridToFieldMap, & @@ -936,7 +989,7 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, call ESMF_AttributeCopy(fcstField(j), field_work, attcopy=ESMF_ATTCOPY_REFERENCE, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return -! get output file name + ! get output file name call ESMF_AttributeGet(fcstField(j), convention="NetCDF", purpose="FV3", & name="output_file", value=outfile_name, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -947,13 +1000,13 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, endif call ESMF_LogWrite("af fcstfield, get output_file",ESMF_LOGMSG_INFO,rc=RC) -! if (lprnt) print *,' i=',i,' j=',j,' outfilename=',trim(outfilename(j,i)) + ! if (lprnt) print *,' i=',i,' j=',j,' outfilename=',trim(outfilename(j,i)) -! add the output field to the 'output_' FieldBundle + ! add the output field to the 'output_' FieldBundle call ESMF_FieldBundleAdd(fieldbundle, (/field_work/), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return -! deal with grids for which 'is_moving' is .true. + ! deal with grids for which 'is_moving' is .true. if (is_moving(grid_id)) then ! create an empty field that will serve as acceptor for GridTransfer of fcstGrid field_work = ESMF_FieldEmptyCreate(name=fieldName, rc=rc) @@ -981,11 +1034,10 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, endif -! local garbage collection + ! local garbage collection deallocate(gridToFieldMap, ungriddedLBound, ungriddedUBound) enddo -! - ! call ESMF_AttributeCopy(fcstGrid, wrtGrid(grid_id), & + call ESMF_AttributeCopy(fcstGrid, actualWrtGrid , & attcopy=ESMF_ATTCOPY_REFERENCE, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -1000,15 +1052,13 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, return endif -!end FBCount - enddo -! -!loop over all items in the imp_state_write and count output FieldBundles + enddo !FBCount + + !loop over all items in the imp_state_write and count output FieldBundles call get_outfile(FBCount, outfilename, FBlist_outfilename, noutfile) wrt_int_state%FBCount = noutfile -! -!create output field bundles + !create output field bundles allocate(wrt_int_state%wrtFB(wrt_int_state%FBCount)) ! if (lprnt) write(0,*)'wrt_initialize_p1: allocated ',wrt_int_state%FBCount, ' wrt_int_state%wrtFB' @@ -1016,7 +1066,7 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, wrt_int_state%wrtFB(i) = ESMF_FieldBundleCreate(name=trim(FBlist_outfilename(i)), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - ! if (lprnt) write(0,*)'wrt_initialize_p1: created wrtFB ',i, ' with name ', trim(wrt_int_state%wrtFB_names(i)) + ! if (lprnt) write(0,*)'wrt_initialize_p1: created wrtFB ',i, ' with name ', trim(FBlist_outfilename(i)) ! if (lprnt) write(0,*)'wrt_initialize_p1: loop over ', FBCount, ' forecast bundles' do n=1, FBCount @@ -1029,9 +1079,9 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, ! if (lprnt) write(0,*)'wrt_initialize_p1: is ', trim(fcstItemNameList(n)), ' == ', trim(FBlist_outfilename(i)) if (trim_regridmethod_suffix(fcstItemNameList(n)) == trim_regridmethod_suffix(FBlist_outfilename(i))) then -! -! copy the fcstfield bundle Attributes to the output field bundle - ! if (lprnt) write(0,*)'wrt_initialize_p1: copy atts/fields from ', "output_"//trim(fcstItemNameList(n)), ' to ', trim(wrt_int_state%wrtFB_names(i)) + + ! copy the fcstfield bundle Attributes to the output field bundle + ! if (lprnt) write(0,*)'wrt_initialize_p1: copy atts/fields from ', "output_"//trim(fcstItemNameList(n)), ' to ', trim(FBlist_outfilename(i)) call ESMF_AttributeCopy(fcstFB, wrt_int_state%wrtFB(i), & attcopy=ESMF_ATTCOPY_REFERENCE, rc=rc) @@ -1059,10 +1109,6 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return -! if (lprnt) print *,'in wrt,add field,i=',i,'n=',n,' j=',j, & -! 'fieldname=',trim(fieldnamelist(j)), ' outfile_name=',trim(outfile_name), & -! ' field bundle name, FBlist_outfilename(i)=',trim(FBlist_outfilename(i)) - if( trim(outfile_name) == trim(FBlist_outfilename(i))) then call ESMF_FieldBundleAdd(wrt_int_state%wrtFB(i), (/fcstField(j)/), rc=rc) @@ -1074,21 +1120,26 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, endif ! index(trim(fcstItemNameList(n)),trim(FBlist_outfilename(i))) - enddo ! end FBCount + enddo ! FBCount + + ! add output grid related attributes, only for history files(bundles), skip restart + if (FBlist_outfilename(i)(1:8) /= 'restart_') then -! add output grid related attributes + call ESMF_AttributeGet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3-nooutput", & + name="output_grid", value=output_grid_name, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return call ESMF_AttributeAdd(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & attrList=(/"source","grid "/), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & name="source", value="FV3GFS", rc=rc) - if (trim(output_grid(grid_id)) == 'cubed_sphere_grid') then + if (trim(output_grid_name) == 'cubed_sphere_grid') then call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & name="grid", value="cubed_sphere", rc=rc) - else if (trim(output_grid(grid_id)) == 'gaussian_grid') then + else if (trim(output_grid_name) == 'gaussian_grid') then call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & name="grid", value="gaussian", rc=rc) @@ -1099,9 +1150,9 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & name="jm", value=jmo(grid_id), rc=rc) - else if (trim(output_grid(grid_id)) == 'regional_latlon' & - .or. trim(output_grid(grid_id)) == 'regional_latlon_moving' & - .or. trim(output_grid(grid_id)) == 'global_latlon') then + else if (trim(output_grid_name) == 'regional_latlon' & + .or. trim(output_grid_name) == 'regional_latlon_moving' & + .or. trim(output_grid_name) == 'global_latlon') then ! for 'regional_latlon_moving' lon1/2 and lat1/2 will be overwritten in run phase call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & @@ -1112,7 +1163,7 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, name="dlon", value=dlon(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & name="dlat", value=dlat(grid_id), rc=rc) - if (trim(output_grid(grid_id)) /= 'regional_latlon_moving') then + if (trim(output_grid_name) /= 'regional_latlon_moving') then call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & name="lon1", value=lon1(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & @@ -1122,8 +1173,8 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & name="lat2", value=lat2(grid_id), rc=rc) endif - else if (trim(output_grid(grid_id)) == 'rotated_latlon' & - .or. trim(output_grid(grid_id)) == 'rotated_latlon_moving') then + else if (trim(output_grid_name) == 'rotated_latlon' & + .or. trim(output_grid_name) == 'rotated_latlon_moving') then call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & name="grid", value="rotated_latlon", rc=rc) @@ -1145,7 +1196,7 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, name="dlon", value=dlon(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & name="dlat", value=dlat(grid_id), rc=rc) - if (trim(output_grid(grid_id)) /= 'rotated_latlon_moving') then + if (trim(output_grid_name) /= 'rotated_latlon_moving') then call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & name="lon1", value=lon1(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & @@ -1155,7 +1206,7 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & name="lat2", value=lat2(grid_id), rc=rc) endif - else if (trim(output_grid(grid_id)) == 'lambert_conformal') then + else if (trim(output_grid_name) == 'lambert_conformal') then call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & name="grid", value="lambert_conformal", rc=rc) @@ -1192,6 +1243,7 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, name="dy", value=dy(grid_id), rc=rc) end if + end if enddo ! end wrt_int_state%FBCount ! @@ -1235,14 +1287,19 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, endif enddo - do n = 1, ngrids -! add the transfer attributes from importState to grid + ! add the transfer attributes from importState to grid call ESMF_AttributeAdd(wrtGrid(n), convention="NetCDF", purpose="FV3", & attrList=attNameList(1:j-1), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + ! add the transfer attributes from importState to special cubed_sphere grid + if (n == 1 .and. top_parent_is_global .and. history_file_on_native_grid) then + call ESMF_AttributeAdd(wrtGrid_cubed_sphere, convention="NetCDF", purpose="FV3", & + attrList=attNameList(1:j-1), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + endif + ! loop over the added attributes, access the value (only scalar allowed), ! and set them on the grid do i=1, j-1 @@ -1266,9 +1323,14 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, endif call ESMF_AttributeSet(wrtGrid(n), convention="NetCDF", purpose="FV3", & name=trim(attNameList(i)), value=valueS, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if (n == 1 .and. top_parent_is_global .and. history_file_on_native_grid) then + call ESMF_AttributeSet(wrtGrid_cubed_sphere, convention="NetCDF", purpose="FV3", & + name=trim(attNameList(i)), value=valueS, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + endif + else if (typekindList(i) == ESMF_TYPEKIND_I4) then call ESMF_AttributeGet(imp_state_write, & convention="NetCDF", purpose="FV3", & @@ -1804,6 +1866,7 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) fieldbundle=mirror_bundle, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + ! if (fcstItemNameList(i)(1:8) == "restart_" .or. fcstItemNameList(i)(1:18) == 'cubed_sphere_grid_') then if (fcstItemNameList(i)(1:8) == "restart_") then ! restart output forecast bundles, use Redist instead of Regrid @@ -1979,6 +2042,11 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) !recover fields from cartesian vector and sfc pressure call recover_fields(file_bundle,rc) + ! FIXME rrfs_smoke_conus13km_fast_phy32_qr crashes with teh following error in recover_fields + ! 20230720 121647.816 ERROR PET147 ESMF_Grid.F90:20442 ESMF_GridGetCoord2DR8 Arguments are incompatible - - farrayPtr typekind does not match Grid typekind + ! 20230720 121647.816 ERROR PET147 module_wrt_grid_comp.F90:2450 Arguments are incompatible - Passing error in return code + + ! if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return enddo ! !----------------------------------------------------------------------- @@ -2004,6 +2072,7 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__,file=__FILE__)) return if (wrtFBName(1:8) == 'restart_') cycle + if (wrtFBName(1:18) == 'cubed_sphere_grid_') cycle call mask_fields(wrt_int_state%wrtFB(nbdl),rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -2426,10 +2495,14 @@ subroutine recover_fields(file_bundle,rc) real(ESMF_KIND_R8) :: coslon, sinlon, sinlat ! ! get filed count - call ESMF_FieldBundleGet(file_bundle, fieldCount=fieldCount, & - grid=fieldGrid, rc=rc) + call ESMF_FieldBundleGet(file_bundle, fieldCount=fieldCount, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return -! + + if (fieldCount == 0) return + + call ESMF_FieldBundleGet(file_bundle, grid=fieldGrid, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + call ESMF_LogWrite("call recover field on wrt comp",ESMF_LOGMSG_INFO,rc=RC) call ESMF_GridGet(fieldgrid, dimCount=gridDimCount, rc=rc) @@ -3290,7 +3363,7 @@ subroutine ioCompRun(comp, importState, exportState, clock, rc) trim(tileFileName), ESMF_LOGMSG_INFO, rc=rc) if (status == ESMF_FILESTATUS_OLD) then - ! This writes the vectical coordinates and the time dimension into the + ! This writes the vertical coordinates and the time dimension into the ! file. Doing this before the large data sets are written, assuming that ! the first time coming into ioCompRun() with this tileFileName, only ! the grid info is written. Second time in, with ESMF_FILESTATUS_OLD, @@ -3314,7 +3387,7 @@ subroutine ioCompRun(comp, importState, exportState, clock, rc) ncerr = nf90_open(tileFileName, NF90_WRITE, ncid=ncid) if (ESMF_LogFoundNetCDFError(ncerr, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return - ! loop over all the fields in the bundle and handle their vectical dims + ! loop over all the fields in the bundle and handle their vertical dims thereAreVerticals = .false. do i=1, fieldCount @@ -3447,7 +3520,6 @@ subroutine ioCompRun(comp, importState, exportState, clock, rc) attName = attNameList(i) call ESMF_AttributeGet(grid, convention="NetCDF", purpose="FV3", & name=trim(attNameList(i)), typekind=typekind, rc=rc) -! print *,'in esmf call, att name=',trim(attNameList(i)) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -3455,7 +3527,7 @@ subroutine ioCompRun(comp, importState, exportState, clock, rc) call ESMF_AttributeGet(grid, & convention="NetCDF", purpose="FV3", & name=trim(attNameList(i)), value=valueS, rc=rc) -! print *,'in esmf call, att string value=',trim(valueS) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ncerr = nf90_put_att(ncid, varid, & @@ -3468,7 +3540,7 @@ subroutine ioCompRun(comp, importState, exportState, clock, rc) call ESMF_AttributeGet(grid, & convention="NetCDF", purpose="FV3", & name=trim(attNameList(i)), value=valueI4, rc=rc) -! print *,'in esmf call, att I4 value=',valueR8 + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ncerr = nf90_put_att(ncid, varid, & trim(attName(6:len(attName))), values=valueI4) @@ -3479,7 +3551,6 @@ subroutine ioCompRun(comp, importState, exportState, clock, rc) call ESMF_AttributeGet(grid, & convention="NetCDF", purpose="FV3", & name=trim(attNameList(i)), value=valueR4, rc=rc) -! print *,'in esmf call, att r4 value=',valueR8 if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -3492,7 +3563,6 @@ subroutine ioCompRun(comp, importState, exportState, clock, rc) call ESMF_AttributeGet(grid, & convention="NetCDF", purpose="FV3", & name=trim(attNameList(i)), value=valueR8, rc=rc) -! print *,'in esmf call, att r8 value=',valueR8 if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ncerr = nf90_put_att(ncid, varid, & diff --git a/io/post_fv3.F90 b/io/post_fv3.F90 index 7cc6ab45e..696a6b026 100644 --- a/io/post_fv3.F90 +++ b/io/post_fv3.F90 @@ -187,7 +187,7 @@ subroutine post_run_fv3(wrt_int_state,grid_id,mype,mpicomp,lead_write, & call set_outflds(kth,th,kpv,pv) if(allocated(datapd))deallocate(datapd) allocate(datapd(ite-its+1,jte-jts+1,nrecout+100)) -!$omp parallel do default(none),private(i,j,k),shared(nrecout,jend,jsta,im,datapd,ista,iend) +!$omp parallel do default(none),private(i,j,k),shared(nrecout,jend,jsta,datapd,ista,iend) do k=1,nrecout+100 do j=1,jend+1-jsta do i=1,iend+1-ista @@ -258,6 +258,7 @@ subroutine post_getattr_fv3(wrt_int_state,grid_id) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__,file=__FILE__)) return if (wrtFBName(1:8) == 'restart_') cycle + if (wrtFBName(1:18) == 'cubed_sphere_grid_') cycle ! set grid spec: ! if(mype==0) print*,'in post_getattr_lam,output_grid=',trim(output_grid(grid_id)),'nfb=',nfb @@ -494,6 +495,8 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) ! Apr 2022 W. Meng Unify set_postvars_gfs and ! set_postvars_regional to set_postvars_fv3 ! Apr 2023 W. Meng Sync RRFS and GFS changes from off-line post +! Jun 2023 W. Meng Remove duplicate initialization; +! relocate computation of aerosol fields ! !----------------------------------------------------------------------- !*** set up post fields from nmint_state @@ -542,12 +545,13 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) sfcvxi, t10m, t10avg, psfcavg, akhsavg, akmsavg, & albedo, tg, prate_max, pwat, snow_acm, snow_bkt, & acgraup, graup_bucket, acfrain, frzrn_bucket, & - ltg1_max, ltg2_max, ltg3_max, aodtot, ebb, hwp, & + ltg1_max, ltg2_max, ltg3_max, ebb, hwp, & aod550,du_aod550,ss_aod550,su_aod550,oc_aod550, & bc_aod550,maod, & dustpm10, dustcb, bccb, occb, sulfcb, sscb, & dustallcb, ssallcb, dustpm, sspm, pp25cb, pp10cb, & - no3cb, nh4cb, dusmass, ducmass, dusmass25,ducmass25 + no3cb, nh4cb, dusmass, ducmass, dusmass25,ducmass25, & + snownc, graupelnc, qrmax use soil, only: sldpth, sh2o, smc, stc, sllevel use masks, only: lmv, lmh, htm, vtm, gdlat, gdlon, dx, dy, hbm2, sm, sice use ctlblk_mod, only: im, jm, lm, lp1, jsta, jend, jsta_2l, jend_2u, jsta_m,jend_m, & @@ -603,9 +607,8 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) real,dimension(:), allocatable :: slat,qstl real,external::FPVSNEW real,dimension(:,:),allocatable :: dummy, p2d, t2d, q2d, qs2d, & - cw2d, cfr2d, accswe_ice, accswe_land, & - snacc_land, snacc_ice - real,dimension(:,:,:),allocatable :: extsmoke, extdust + cw2d, cfr2d, snacc_land, snacc_ice + real,dimension(:,:,:),allocatable :: ext550 character(len=80) :: fieldname, wrtFBName, flatlon, & VarName type(ESMF_Grid) :: wrtGrid @@ -670,7 +673,7 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) bk5(i) = wrt_int_state%bk(i) enddo -!$omp parallel do default(none) private(i,j) shared(jsta,jend,im,f,gdlat,ista,iend) +!$omp parallel do default(none) private(i,j) shared(jsta,jend,f,gdlat,ista,iend) do j=jsta,jend do i=ista,iend f(I,J) = 1.454441e-4*sin(gdlat(i,j)*dtr) ! 2*omeg*sin(phi) @@ -679,18 +682,6 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) ! pt = ak5(1) -! GFS does not have surface specific humidity -! inst sensible heat flux -! inst latent heat flux -!$omp parallel do default(none),private(i,j),shared(jsta,jend,im,spval,qs,twbs,qwbs,ths,ista,iend) - do j=jsta,jend - do i=ista,iend - qs(i,j) = SPVAL - twbs(i,j) = SPVAL - qwbs(i,j) = SPVAL - enddo - enddo - ! GFS set up DT to compute accumulated fields, set it to one dtq2 = wrt_int_state%dtp nphs = 2. @@ -698,57 +689,26 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) !Allocate for regional models only if(modelname=='FV3R') then - allocate(extsmoke(ista:iend,jsta:jend,lm)) - allocate(extdust(ista:iend,jsta:jend,lm)) - allocate(accswe_ice(ista:iend,jsta:jend)) - allocate(accswe_land(ista:iend,jsta:jend)) + allocate(ext550(ista:iend,jsta:jend,lm)) allocate(snacc_ice(ista:iend,jsta:jend)) allocate(snacc_land(ista:iend,jsta:jend)) - endif -! -! GFS does not have convective cloud efficiency -! similated precip -! 10 m theta -! 10 m humidity -! snow free albedo -!$omp parallel do default(none), private(i,j), shared(jsta,jend,im,spval,ista,iend), & -!$omp& shared(cldefi,lspa,th10,q10,albase) - do j=jsta,jend - do i=ista,iend - cldefi(i,j) = SPVAL - lspa(i,j) = SPVAL - th10(i,j) = SPVAL - q10(i,j) = SPVAL - albase(i,j) = SPVAL - enddo - enddo + do j=jsta,jend + do i=ista,iend + snacc_ice(i,j)=spval + snacc_land(i,j)=spval + end do + end do -! GFS does not have convective precip -!$omp parallel do default(none) private(i,j) shared(jsta,jend,im,cprate,ista,iend) - do j=jsta,jend - do i=ista,iend - cprate(i,j) = 0. - enddo - enddo + do l=1,lm + do j=jsta,jend + do i=ista,iend + ext550(i,j,l)=spval + end do + end do + end do + endif -! GFS probably does not use zenith angle, czen, czmean -! inst surface outgoing longwave, radot -! inst cloud fraction for high, middle, and low cloud, -! cfrach -! inst ground heat flux, grnflx -!$omp parallel do default(none) private(i,j) shared(jsta,jend,im,spval,ista,iend), & -!$omp& shared(czen,czmean,radot,cfrach,cfracl,cfracm,grnflx) - do j=jsta,jend - do i=ista,iend - czen(i,j) = SPVAL - czmean(i,j) = SPVAL - cfrach(i,j) = SPVAL - cfracl(i,j) = SPVAL - cfracm(i,j) = SPVAL - grnflx(i,j) = SPVAL - enddo - enddo ! ! GFS doesn not yet output soil layer thickness, assign SLDPTH to be the same as nam sldpth(1) = 0.10 @@ -756,27 +716,13 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) sldpth(3) = 0.6 sldpth(4) = 1.0 -! GFS does not output time averaged convective and strat cloud fraction, set acfrcv to spval, n -! cfrcv to 1 -! time averaged cloud fraction, set acfrst to spval, ncfrst to 1 -! UNDERGROUND RUNOFF, bgroff -! inst incoming sfc longwave -! inst incoming sfc shortwave, rswin -! inst incoming clear sky sfc shortwave, rswinc -! inst outgoing sfc shortwave, rswout -! snow phase change heat flux, snopcx -! GFS does not use total momentum flux,sfcuvx -!$omp parallel do default(none),private(i,j),shared(jsta,jend,im,spval,ista,iend), & -!$omp& shared(acfrcv,ncfrcv,acfrst,ncfrst,bgroff,rswin,rswinc,rswout,snopcx,sfcuvx,& -!$omp& ltg1_max,ltg2_max,ltg3_max) +! set ncfrcv to 1, ncfrst to 1 +!$omp parallel do default(none),private(i,j),shared(jsta,jend,spval,ista,iend), & +!$omp& shared(ncfrcv,ncfrst) do j=jsta,jend do i=ista,iend - acfrcv(i,j) = spval ncfrcv(i,j) = 1.0 - acfrst(i,j) = spval ncfrst(i,j) = 1.0 - bgroff(i,j) = spval - rswinc(i,j) = spval enddo enddo @@ -787,77 +733,11 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) ! GFS surface flux has been averaged, set ASRFC to 1 asrfc = 1.0 -! GFS does not have temperature tendency due to long wave radiation -! temperature tendency due to short wave radiation -! temperature tendency due to latent heating from convection -! temperature tendency due to latent heating from grid scale - do l=1,lm -!$omp parallel do default(none),private(i,j),shared(jsta_2l,jend_2u,im,spval,l,ista_2l,iend_2u), & -!$omp& shared(rlwtt,rswtt,tcucn,tcucns,train) - do j=jsta_2l,jend_2u - do i=ista_2l,iend_2u - rlwtt(i,j,l) = spval - rswtt(i,j,l) = spval - tcucn(i,j,l) = spval - tcucns(i,j,l) = spval - train(i,j,l) = spval - enddo - enddo - enddo - ! set avrain to 1 avrain = 1.0 avcnvc = 1.0 theat = 6.0 ! just in case GFS decides to output T tendency -! GFS does not have temperature tendency due to latent heating from grid scale - train = spval - -! GFS does not have soil moisture availability, smstav -! accumulated surface evaporatio, sfcevp -! averaged accumulated snow, acsnow -! snow melt,acsnom -! humidity at roughness length, qz0 -! u at roughness length, uz0 -! v at roughness length, vz0 -! shelter rh max, maxrhshltr -! shelter rh min, minrhshltr -!$omp parallel do default(none),private(i,j),shared(jsta_2l,jend_2u,im,spval,ista_2l,iend_2u), & -!$omp& shared(sfcevp,acsnom,qz0,uz0,vz0,maxrhshltr,minrhshltr) - do j=jsta_2l,jend_2u - do i=ista_2l,iend_2u - sfcevp(i,j) = spval - acsnom(i,j) = spval - qz0(i,j) = spval - uz0(i,j) = spval - vz0(i,j) = spval - enddo - enddo - -! GFS does not have mixing length,el_pbl -! exchange coefficient, exch_h - do l=1,lm -!$omp parallel do default(none),private(i,j),shared(jsta_2l,jend_2u,im,l,spval,el_pbl,exch_h,ista_2l,iend_2u) - do j=jsta_2l,jend_2u - do i=ista_2l,iend_2u - el_pbl(i,j,l) = spval - exch_h(i,j,l) = spval - enddo - enddo - enddo - -! GFS does not have deep convective cloud top and bottom fields -!$omp parallel do default(none),private(i,j),shared(jsta_2l,jend_2u,im,spval,ista_2l,iend_2u), & -!$omp& shared(htopd,hbotd,htops,hbots,cuppt) - do j=jsta_2l,jend_2u - do i=ista_2l,iend_2u - htopd(i,j) = SPVAL - hbotd(i,j) = SPVAL - htops(i,j) = SPVAL - hbots(i,j) = SPVAL - cuppt(i,j) = SPVAL - enddo - enddo ! ! get inital date sdat(1) = wrt_int_state%idate(2) !month @@ -896,7 +776,7 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__,file=__FILE__)) return if (wrtFBName(1:8) == 'restart_') cycle - + if (wrtFBName(1:18) == 'cubed_sphere_grid_') cycle call ESMF_AttributeGet(wrt_int_state%wrtFB(ibdl), convention="NetCDF", purpose="FV3", & name="grid_id", value=bundle_grid_id, rc=rc) @@ -987,6 +867,7 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) line=__LINE__, file=__FILE__)) return ! bail out if (wrtFBName(1:8) == 'restart_') cycle + if (wrtFBName(1:18) == 'cubed_sphere_grid_') cycle call ESMF_AttributeGet(wrt_int_state%wrtFB(ibdl), convention="NetCDF", purpose="FV3", & name="grid_id", value=bundle_grid_id, rc=rc) @@ -1114,17 +995,6 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) enddo endif - ! total aod - if(trim(fieldname)=='aodtot') then - !$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,aodtot,arrayr42d,fillValue,spval) - do j=jsta,jend - do i=ista, iend - aodtot(i,j)=arrayr42d(i,j) - if(abs(arrayr42d(i,j)-fillValue) < small) aodtot(i,j)=spval - enddo - enddo - endif - ! biomass burning emissions if(trim(fieldname)=='ebb_smoke_hr') then !$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,ebb,arrayr42d,fillValue,spval) @@ -1367,6 +1237,28 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) enddo endif + !time step snow (in m) + if(trim(fieldname)=='snow') then + !$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,spval,snownc,arrayr42d,sm,fillValue) + do j=jsta,jend + do i=ista, iend + snownc(i,j) = arrayr42d(i,j) + if (abs(arrayr42d(i,j)-fillValue) < small) snownc(i,j) = spval + enddo + enddo + endif + + !time step graupel (in m) + if(trim(fieldname)=='graupel') then + !$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,spval,graupelnc,arrayr42d,sm,fillValue) + do j=jsta,jend + do i=ista, iend + graupelnc(i,j) = arrayr42d(i,j) + if (abs(arrayr42d(i,j)-fillValue) < small) graupelnc(i,j) = spval + enddo + enddo + endif + ! max hourly surface precipitation rate if(trim(fieldname)=='pratemax') then !$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,spval,prate_max,arrayr42d,sm,fillValue) @@ -1736,6 +1628,69 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) enddo endif + if(nsoil==9) then + ! liquid volumetric soil mpisture in fraction + if(trim(fieldname)=='soill5') then + !$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,spval,sh2o,arrayr42d,sm,fillValue) + do j=jsta,jend + do i=ista, iend + sh2o(i,j,5) = arrayr42d(i,j) + if( abs(arrayr42d(i,j)-fillValue) < small) sh2o(i,j,5) = spval + if (sm(i,j) /= 0.0) sh2o(i,j,5) = spval + enddo + enddo + endif + + ! liquid volumetric soil mpisture in fraction + if(trim(fieldname)=='soill6') then + !$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,spval,sh2o,arrayr42d,sm,fillValue) + do j=jsta,jend + do i=ista, iend + sh2o(i,j,6) = arrayr42d(i,j) + if( abs(arrayr42d(i,j)-fillValue) < small) sh2o(i,j,6) = spval + if (sm(i,j) /= 0.0) sh2o(i,j,6) = spval + enddo + enddo + endif + + ! liquid volumetric soil mpisture in fraction + if(trim(fieldname)=='soill7') then + !$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,spval,sh2o,arrayr42d,sm,fillValue) + do j=jsta,jend + do i=ista, iend + sh2o(i,j,7) = arrayr42d(i,j) + if( abs(arrayr42d(i,j)-fillValue) < small) sh2o(i,j,7) = spval + if (sm(i,j) /= 0.0) sh2o(i,j,7) = spval + enddo + enddo + endif + + ! liquid volumetric soil mpisture in fraction + if(trim(fieldname)=='soill8') then + !$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,spval,sh2o,arrayr42d,sm,fillValue) + do j=jsta,jend + do i=ista, iend + sh2o(i,j,8) = arrayr42d(i,j) + if( abs(arrayr42d(i,j)-fillValue) < small) sh2o(i,j,8) = spval + if (sm(i,j) /= 0.0) sh2o(i,j,8) = spval + enddo + enddo + endif + + ! liquid volumetric soil mpisture in fraction + if(trim(fieldname)=='soill9') then + !$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,spval,sh2o,arrayr42d,sm,fillValue) + do j=jsta,jend + do i=ista, iend + sh2o(i,j,9) = arrayr42d(i,j) + if( abs(arrayr42d(i,j)-fillValue) < small) sh2o(i,j,9) = spval + if (sm(i,j) /= 0.0) sh2o(i,j,9) = spval + enddo + enddo + endif + + endif !nsoil + ! volumetric soil moisture if(trim(fieldname)=='soilw1') then !$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,spval,smc,arrayr42d,sm,fillValue) @@ -2350,25 +2305,6 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) endif if(modelname=='FV3R')then - !acsnow - if(trim(fieldname)=='accswe_land') then - !$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,accswe_land,arrayr42d,fillvalue,spval) - do j=jsta,jend - do i=ista, iend - accswe_land(i,j) = arrayr42d(i,j) - if(abs(arrayr42d(i,j)-fillvalue) null() logical :: top_parent_is_global + logical :: history_file_on_native_grid integer :: num_restart_interval, restart_starttime real,dimension(:),allocatable :: restart_interval @@ -985,12 +985,10 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) ! Create FieldBundle for Fields that need to be regridded bilinear if( quilting ) then - allocate(fieldbundle(ngrids)) - nbdlphys = 2 - allocate(fieldbundlephys(nbdlphys,ngrids)) + call ESMF_ConfigGetAttribute(config=CF, value=history_file_on_native_grid, default=.false., label='history_file_on_native_grid:', rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - allocate(fieldbundle_dyn_restart(ngrids,3)) ! fv_core.res fv_srf_wnd.res fv_tracer.res - allocate(fieldbundle_phy_restart(ngrids,2)) ! phy_data sfc_data + nbdlphys = 2 do n=1,ngrids bundle_grid='' @@ -1005,69 +1003,41 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) name_FB = trim(filename_base(i)) // trim(bundle_grid) ! - if( i==1 ) then -! for dyn + if (i == 1) then ! for dyn name_FB1 = trim(name_FB)//'_bilinear' - fieldbundle(n) = ESMF_FieldBundleCreate(name=trim(name_FB1),rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_AttributeAdd(fieldbundle(n), convention="NetCDF", purpose="FV3", & - attrList=(/"grid_id"/), rc=rc) + call create_bundle_and_add_it_to_state(trim(name_FB1), tempState, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_AttributeSet(fieldbundle(n), convention="NetCDF", purpose="FV3", & - name="grid_id", value=n, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_AttributeAdd(fieldbundle(n), convention="NetCDF", purpose="FV3-nooutput", & - attrList=(/"frestart"/), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_AttributeSet(fieldbundle(n), convention="NetCDF", purpose="FV3-nooutput", & - name="frestart", valueList=frestart, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_StateAdd(tempState, (/fieldbundle(n)/), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if (n == 1 .AND. top_parent_is_global .AND. history_file_on_native_grid) then + call create_bundle_and_add_it_to_state('cubed_sphere_grid_'//trim(name_FB1), tempState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + end if call ESMF_GridCompInitialize(fcstGridComp(n), importState=tempState,& - exportState=exportState, phase=1, userrc=urc, rc=rc) + exportState=exportState, phase=1, userrc=urc, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return - else if( i==2 ) then -! for phys + else if (i == 2) then ! for phys + do j=1, nbdlphys - if( j==1 ) then + if (j == 1) then name_FB1 = trim(name_FB)//'_nearest_stod' else name_FB1 = trim(name_FB)//'_bilinear' endif - fieldbundlephys(j,n) = ESMF_FieldBundleCreate(name=trim(name_FB1),rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_AttributeAdd(fieldbundlephys(j,n), convention="NetCDF", purpose="FV3", & - attrList=(/"grid_id"/), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_AttributeSet(fieldbundlephys(j,n), convention="NetCDF", purpose="FV3", & - name="grid_id", value=n, rc=rc) + call create_bundle_and_add_it_to_state(trim(name_FB1), tempState, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_AttributeAdd(fieldbundlephys(j,n), convention="NetCDF", purpose="FV3-nooutput", & - attrList=(/"frestart"/), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_AttributeSet(fieldbundlephys(j,n), convention="NetCDF", purpose="FV3-nooutput", & - name="frestart", valueList=frestart, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if (n == 1 .AND. top_parent_is_global .AND. history_file_on_native_grid) then + call create_bundle_and_add_it_to_state('cubed_sphere_grid_'//trim(name_FB1), tempState, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + endif - call ESMF_StateAdd(tempState, (/fieldbundlephys(j,n)/), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return enddo call ESMF_GridCompInitialize(fcstGridComp(n), importState=tempState,& - exportState=exportState, phase=2, userrc=urc, rc=rc) + exportState=exportState, phase=2, userrc=urc, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return @@ -1106,26 +1076,7 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) name_FB = trim(name_FB)//nest_suffix endif - fieldbundle_dyn_restart(n,i) = ESMF_FieldBundleCreate(name=trim(name_FB),rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_AttributeAdd(fieldbundle_dyn_restart(n,i), convention="NetCDF", purpose="FV3", & - attrList=(/"grid_id"/), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_AttributeSet(fieldbundle_dyn_restart(n,i), convention="NetCDF", purpose="FV3", & - name="grid_id", value=n, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_AttributeAdd(fieldbundle_dyn_restart(n,i), convention="NetCDF", purpose="FV3-nooutput", & - attrList=(/"frestart"/), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_AttributeSet(fieldbundle_dyn_restart(n,i), convention="NetCDF", purpose="FV3-nooutput", & - name="frestart", valueList=frestart, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_StateAdd(tempState, (/fieldbundle_dyn_restart(n,i)/), rc=rc) + call create_bundle_and_add_it_to_state(trim(name_FB), tempState, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return call ESMF_GridCompInitialize(fcstGridComp(n), importState=tempState, & @@ -1157,26 +1108,7 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) name_FB = trim(name_FB)//nest_suffix endif - fieldbundle_phy_restart(n,i) = ESMF_FieldBundleCreate(name=trim(name_FB),rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_AttributeAdd(fieldbundle_phy_restart(n,i), convention="NetCDF", purpose="FV3", & - attrList=(/"grid_id"/), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_AttributeSet(fieldbundle_phy_restart(n,i), convention="NetCDF", purpose="FV3", & - name="grid_id", value=n, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_AttributeAdd(fieldbundle_phy_restart(n,i), convention="NetCDF", purpose="FV3-nooutput", & - attrList=(/"frestart"/), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_AttributeSet(fieldbundle_phy_restart(n,i), convention="NetCDF", purpose="FV3-nooutput", & - name="frestart", valueList=frestart, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_StateAdd(tempState, (/fieldbundle_phy_restart(n,i)/), rc=rc) + call create_bundle_and_add_it_to_state(trim(name_FB), tempState, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return call ESMF_GridCompInitialize(fcstGridComp(n), importState=tempState, & @@ -1192,11 +1124,7 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) enddo ! ngrids - ! total number of field bundles created is ngrids * (1(atm) + 2(phy) + 3(dyn_rest) +2(phy_rest) - if (mype == 0) write(*,*)'fcst_initialize: total number of field bundles: ', ngrids*(1+2+0+2) - -!end qulting - endif + endif ! quilting call get_atmos_model_ungridded_dim(nlev=numLevels, & nsoillev=numSoilLayers, & @@ -1206,6 +1134,36 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) ! !----------------------------------------------------------------------- ! + contains + + subroutine create_bundle_and_add_it_to_state(name_fb, state, rc) + + character(len=*), intent(in) :: name_fb + type(ESMF_State), intent(inout) :: state + integer, intent(out) :: rc + + type(ESMF_FieldBundle) :: fieldbundle + + fieldbundle = ESMF_FieldBundleCreate(name=trim(name_fb), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_AttributeAdd(fieldbundle, convention="NetCDF", purpose="FV3", attrList=(/"grid_id"/), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_AttributeSet(fieldbundle, convention="NetCDF", purpose="FV3", name="grid_id", value=n, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_AttributeAdd(fieldbundle, convention="NetCDF", purpose="FV3-nooutput", attrList=(/"frestart"/), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_AttributeSet(fieldbundle, convention="NetCDF", purpose="FV3-nooutput", name="frestart", valueList=frestart, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_StateAdd(state, (/fieldbundle/), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + end subroutine create_bundle_and_add_it_to_state + end subroutine fcst_initialize ! !----------------------------------------------------------------------- diff --git a/upp b/upp index dccb32176..baa77515f 160000 --- a/upp +++ b/upp @@ -1 +1 @@ -Subproject commit dccb32176930676ef2a258eb65571ab4e3f7e7a4 +Subproject commit baa77515f182eb6d6f2f8b57bd1e0ffb742761e8