From 9929dcd96c000586a9b1856e4fce5523380b943b Mon Sep 17 00:00:00 2001 From: Dusan Jovic <48258889+DusanJovic-NOAA@users.noreply.github.com> Date: Tue, 15 Feb 2022 11:22:57 -0500 Subject: [PATCH] Multiple output grids (#480) Update fv3 cap and write grid component to enable outputting multiple domains. This is done be creating an array of fcstGrids, and array of rout handles where each element of these arrays correspond to one atm domain. In the write grid component updates were made to allow grid spec parameters for each output grid to be specified separately. Co-authored-by: Gerhard Theurich --- atmos_cubed_sphere | 2 +- atmos_model.F90 | 17 +- fv3_cap.F90 | 58 +- io/FV3GFS_io.F90 | 9 +- io/inline_post.F90 | 30 +- io/module_fv3_io_def.F90 | 26 +- io/module_write_netcdf.F90 | 99 ++-- io/module_wrt_grid_comp.F90 | 1116 ++++++++++++++++------------------- io/post_regional.F90 | 87 +-- module_fcst_grid_comp.F90 | 522 ++++++++++------ 10 files changed, 1033 insertions(+), 933 deletions(-) diff --git a/atmos_cubed_sphere b/atmos_cubed_sphere index 5193c6b60..348cee70d 160000 --- a/atmos_cubed_sphere +++ b/atmos_cubed_sphere @@ -1 +1 @@ -Subproject commit 5193c6b60c02c744b1ffe27078eccbeed2a22ad8 +Subproject commit 348cee70d9c0252e4191833b3aa3a87b313efa22 diff --git a/atmos_model.F90 b/atmos_model.F90 index 0e398154f..16e595dd6 100644 --- a/atmos_model.F90 +++ b/atmos_model.F90 @@ -74,7 +74,7 @@ module atmos_model_mod use atmosphere_mod, only: atmosphere_scalar_field_halo use atmosphere_mod, only: atmosphere_get_bottom_layer use atmosphere_mod, only: set_atmosphere_pelist -use atmosphere_mod, only: Atm, mygrid +use atmosphere_mod, only: Atm, mygrid, get_nth_domain_info use block_control_mod, only: block_control_type, define_blocks_packed use DYCORE_typedefs, only: DYCORE_data_type, DYCORE_diag_type @@ -113,6 +113,7 @@ module atmos_model_mod public atmos_model_exchange_phase_1, atmos_model_exchange_phase_2 public atmos_model_restart public get_atmos_model_ungridded_dim +public atmos_model_get_nth_domain_info public addLsmask2grid public setup_exportdata !----------------------------------------------------------------------- @@ -125,6 +126,8 @@ module atmos_model_mod integer :: layout(2) ! computer task laytout logical :: regional ! true if domain is regional logical :: nested ! true if there is a nest + integer :: ngrids ! + integer :: mygrid ! integer :: mlon, mlat integer :: iau_offset ! iau running window length logical :: pe ! current pe. @@ -526,7 +529,7 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step) call atmosphere_resolution (nlon, nlat, global=.false.) call atmosphere_resolution (mlon, mlat, global=.true.) call alloc_atmos_data_type (nlon, nlat, Atmos) - call atmosphere_domain (Atmos%domain, Atmos%layout, Atmos%regional, Atmos%nested, Atmos%pelist) + call atmosphere_domain (Atmos%domain, Atmos%layout, Atmos%regional, Atmos%nested, Atmos%ngrids, Atmos%mygrid, Atmos%pelist) call atmosphere_diag_axes (Atmos%axes) call atmosphere_etalvls (Atmos%ak, Atmos%bk, flip=flip_vc) call atmosphere_grid_bdry (Atmos%lon_bnd, Atmos%lat_bnd, global=.false.) @@ -2480,7 +2483,6 @@ subroutine assign_importdata(jdat, rc) rc=0 ! - if (mpp_pe() == mpp_root_pe()) print *,'end of assign_importdata' end subroutine assign_importdata ! @@ -2879,5 +2881,14 @@ subroutine addLsmask2grid(fcstGrid, rc) end subroutine addLsmask2grid !------------------------------------------------------------------------------ + subroutine atmos_model_get_nth_domain_info(n, layout, nx, ny, pelist) + integer, intent(in) :: n + integer, intent(out) :: layout(2) + integer, intent(out) :: nx, ny + integer, pointer, intent(out) :: pelist(:) + + call get_nth_domain_info(n, layout, nx, ny, pelist) + + end subroutine atmos_model_get_nth_domain_info end module atmos_model_mod diff --git a/fv3_cap.F90 b/fv3_cap.F90 index 157106797..87dbe0e69 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -37,12 +37,11 @@ module fv3gfs_cap_mod num_files, filename_base, & wrttasks_per_group, n_group, & lead_wrttask, last_wrttask, & - output_grid, output_file, & nsout_io, iau_offset, lflname_fulltime ! use module_fcst_grid_comp, only: fcstSS => SetServices, & fcstGrid, numLevels, numSoilLayers, & - numTracers + numTracers, mygrid, grid_number_on_all_pets use module_wrt_grid_comp, only: wrtSS => SetServices ! @@ -187,7 +186,6 @@ subroutine InitializeAdvertise(gcomp, rc) integer :: i, j, k, urc, ist integer :: noutput_fh, nfh, nfh2 integer :: petcount - integer :: num_output_file integer :: nfhmax_hf real :: nfhmax real :: output_startfh, outputfh, outputfh2(2) @@ -218,7 +216,7 @@ subroutine InitializeAdvertise(gcomp, rc) call NUOPC_ModelGet(gcomp, driverClock=clock, importState=importState, exportState=exportState, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_AttributeGet(gcomp, name="ProfileMemory", value=value, defaultValue="true", & + call ESMF_AttributeGet(gcomp, name="ProfileMemory", value=value, defaultValue="false", & convention="NUOPC", purpose="Instance", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return profile_memory = (trim(value)/="false") @@ -296,33 +294,6 @@ subroutine InitializeAdvertise(gcomp, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return enddo - allocate(output_file(num_files)) - num_output_file = ESMF_ConfigGetLen(config=CF, label ='output_file:',rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - if (num_files == num_output_file) then - call ESMF_ConfigGetAttribute(CF,valueList=output_file,label='output_file:', & - count=num_files, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - do i = 1, num_files - if(output_file(i) /= "netcdf" .and. output_file(i) /= "netcdf_parallel") then - write(0,*)"fv3_cap.F90: only netcdf and netcdf_parallel are allowed for multiple values of output_file" - call ESMF_Finalize(endflag=ESMF_END_ABORT) - endif - enddo - else if ( num_output_file == 1) then - call ESMF_ConfigGetAttribute(CF,valuelist=output_file,label='output_file:', count=1, rc=rc) - output_file(1:num_files) = output_file(1) - else - output_file(1:num_files) = 'netcdf' - endif - if(mype == 0) then - print *,'af nems config,num_files=',num_files - do i=1,num_files - print *,'num_file=',i,'filename_base= ',trim(filename_base(i)),& - ' output_file= ',trim(output_file(i)) - enddo - endif -! ! variables for output call ESMF_ConfigGetAttribute(config=CF, value=nfhout, label ='nfhout:', default=-1,rc=rc) call ESMF_ConfigGetAttribute(config=CF, value=nfhmax_hf,label ='nfhmax_hf:',default=-1,rc=rc) @@ -428,7 +399,9 @@ subroutine InitializeAdvertise(gcomp, rc) ! pull out the item names and item types from fcstState call ESMF_StateGet(fcstState, itemNameList=fcstItemNameList, & - itemTypeList=fcstItemTypeList, rc=rc) + itemTypeList=fcstItemTypeList, & + !itemorderflag=ESMF_ITEMORDER_ADDORDER, & + rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! loop over all items in the fcstState and collect all FieldBundles @@ -556,8 +529,8 @@ subroutine InitializeAdvertise(gcomp, rc) if (i==1) then ! this is a Store() for the first wrtComp -> must do the Store() - call ESMF_FieldBundleRegridStore(fcstFB(j), wrtFB(j,i), & - regridMethod=regridmethod, routehandle=routehandle(j,i), & + call ESMF_FieldBundleRegridStore(fcstFB(j), wrtFB(j,1), & + regridMethod=regridmethod, routehandle=routehandle(j,1), & unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, & srcTermProcessing=isrcTermProcessing, rc=rc) @@ -584,10 +557,10 @@ subroutine InitializeAdvertise(gcomp, rc) endif write(msgString,"(A,I2.2,',',I2.2,A)") "... returned from wrtFB(",j,i, ") FieldBundleRegridStore()." call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) - enddo + enddo ! j=1, FBcount ! end write_groups - enddo + enddo ! i=1, write_groups if(mype==0) print *,'in fv3cap init, time wrtcrt/regrdst',MPI_Wtime()-timerhs deallocate(petList) deallocate(originPetList) @@ -793,7 +766,7 @@ subroutine InitializeRealize(gcomp, rc) if (isPetLocal) then ! -- realize connected fields in exportState - call realizeConnectedCplFields(exportState, fcstGrid, & + call realizeConnectedCplFields(exportState, fcstGrid(mygrid), & numLevels, numSoilLayers, numTracers, & exportFieldsInfo, 'FV3 Export', exportFields, 0.0_ESMF_KIND_R8, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -803,7 +776,7 @@ subroutine InitializeRealize(gcomp, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! -- realize connected fields in importState - call realizeConnectedCplFields(importState, fcstGrid, & + call realizeConnectedCplFields(importState, fcstGrid(mygrid), & numLevels, numSoilLayers, numTracers, & importFieldsInfo, 'FV3 Import', importFields, 9.99e20_ESMF_KIND_R8, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -896,7 +869,7 @@ subroutine ModelAdvance_phase2(gcomp, rc) type(ESMF_Time) :: startTime type(ESMF_TimeInterval) :: time_elapsed - integer :: na, i, urc + integer :: na, j, urc integer :: nfseconds logical :: fcstpe character(len=*),parameter :: subname='(fv3_cap:ModelAdvance_phase2)' @@ -944,10 +917,10 @@ subroutine ModelAdvance_phase2(gcomp, rc) call ESMF_VMEpochEnter(epoch=ESMF_VMEpoch_Buffer, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - do i=1, FBCount + do j=1, FBCount - call ESMF_FieldBundleRegrid(fcstFB(i), wrtFB(i,n_group), & - routehandle=routehandle(i, n_group), & + call ESMF_FieldBundleRegrid(fcstFB(j), wrtFB(j,n_group), & + routehandle=routehandle(j, n_group), & termorderflag=(/ESMF_TERMORDER_SRCSEQ/), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! @@ -956,7 +929,6 @@ subroutine ModelAdvance_phase2(gcomp, rc) call ESMF_VMEpochExit(rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - ! if(mype==0 .or. mype==lead_wrttask(1)) print *,'on wrt bf wrt run, na=',na call ESMF_LogWrite('Model Advance: before wrtcomp run ', ESMF_LOGMSG_INFO, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return diff --git a/io/FV3GFS_io.F90 b/io/FV3GFS_io.F90 index b38084585..041a2d46b 100644 --- a/io/FV3GFS_io.F90 +++ b/io/FV3GFS_io.F90 @@ -3081,7 +3081,7 @@ end subroutine store_data3D ! #ifdef use_WRTCOMP - subroutine fv_phys_bundle_setup(Diag, axes, phys_bundle, fcst_grid, quilting, nbdlphys) + subroutine fv_phys_bundle_setup(Diag, axes, phys_bundle, fcst_grid, quilting, nbdlphys, rc) ! !------------------------------------------------------------- !*** set esmf bundle for phys output fields @@ -3098,9 +3098,11 @@ subroutine fv_phys_bundle_setup(Diag, axes, phys_bundle, fcst_grid, quilting, nb type(ESMF_Grid),intent(inout) :: fcst_grid logical,intent(in) :: quilting integer, intent(in) :: nbdlphys + integer,intent(out) :: rc + ! !*** local variables - integer i, j, k, n, rc, idx, ibdl, nbdl + integer i, j, k, n, idx, ibdl, nbdl integer id, axis_length, direction, edges, axis_typ integer num_attributes, num_field_dyn integer currdate(6) @@ -3227,6 +3229,7 @@ subroutine fv_phys_bundle_setup(Diag, axes, phys_bundle, fcst_grid, quilting, nb name="vertical_dim_labels", valueList=axis_name_vert, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return endif + deallocate(axis_name_vert) endif !*** add attributes @@ -3341,6 +3344,8 @@ subroutine fv_phys_bundle_setup(Diag, axes, phys_bundle, fcst_grid, quilting, nb endif enddo + deallocate(axis_name) + deallocate(all_axes) end subroutine fv_phys_bundle_setup ! diff --git a/io/inline_post.F90 b/io/inline_post.F90 index b51e2e7ac..2e123346e 100644 --- a/io/inline_post.F90 +++ b/io/inline_post.F90 @@ -16,7 +16,7 @@ module inline_post contains - subroutine inline_post_run(wrt_int_state,mypei,mpicomp,lead_write, & + subroutine inline_post_run(wrt_int_state,grid_id,mypei,mpicomp,lead_write, & mynfhr,mynfmin,mynfsec) ! ! revision history: @@ -30,6 +30,7 @@ subroutine inline_post_run(wrt_int_state,mypei,mpicomp,lead_write, & !----------------------------------------------------------------------- ! type(wrt_internal_state),intent(in) :: wrt_int_state + integer,intent(in) :: grid_id integer,intent(in) :: mypei integer,intent(in) :: mpicomp integer,intent(in) :: lead_write @@ -37,14 +38,14 @@ subroutine inline_post_run(wrt_int_state,mypei,mpicomp,lead_write, & integer,intent(in) :: mynfmin integer,intent(in) :: mynfsec ! - if(mypei == 0) print *,'inline_post_run, output_grid=',trim(output_grid) - if(trim(output_grid) == 'gaussian_grid' & - .or. trim(output_grid) == 'global_latlon') then + if(mypei == 0) print *,'inline_post_run, output_grid=',trim(output_grid(grid_id)) + if(trim(output_grid(grid_id)) == 'gaussian_grid' & + .or. trim(output_grid(grid_id)) == 'global_latlon') then call post_run_gfs(wrt_int_state, mypei, mpicomp, lead_write, & mynfhr, mynfmin,mynfsec) - else if( trim(output_grid) == 'regional_latlon' & - .or. trim(output_grid) == 'rotated_latlon' & - .or. trim(output_grid) == 'lambert_conformal') then + else if( trim(output_grid(grid_id)) == 'regional_latlon' & + .or. trim(output_grid(grid_id)) == 'rotated_latlon' & + .or. trim(output_grid(grid_id)) == 'lambert_conformal') then if(mypei == 0) print *,'inline_post_run, call post_run_regional' call post_run_regional(wrt_int_state, mypei, mpicomp, lead_write, & mynfhr, mynfmin,mynfsec) @@ -55,21 +56,22 @@ end subroutine inline_post_run ! !----------------------------------------------------------------------- ! - subroutine inline_post_getattr(wrt_int_state) + subroutine inline_post_getattr(wrt_int_state,grid_id) ! use esmf ! implicit none ! type(wrt_internal_state),intent(inout) :: wrt_int_state + integer, intent(in) :: grid_id ! - if(trim(output_grid) == 'gaussian_grid' & - .or. trim(output_grid) == 'global_latlon') then + if(trim(output_grid(grid_id)) == 'gaussian_grid' & + .or. trim(output_grid(grid_id)) == 'global_latlon') then call post_getattr_gfs(wrt_int_state) - else if( trim(output_grid) == 'regional_latlon' & - .or. trim(output_grid) == 'rotated_latlon' & - .or. trim(output_grid) == 'lambert_conformal') then - call post_getattr_regional(wrt_int_state) + else if( trim(output_grid(grid_id)) == 'regional_latlon' & + .or. trim(output_grid(grid_id)) == 'rotated_latlon' & + .or. trim(output_grid(grid_id)) == 'lambert_conformal') then + call post_getattr_regional(wrt_int_state,grid_id) endif ! end subroutine inline_post_getattr diff --git a/io/module_fv3_io_def.F90 b/io/module_fv3_io_def.F90 index 65d2b926b..dda5310ad 100644 --- a/io/module_fv3_io_def.F90 +++ b/io/module_fv3_io_def.F90 @@ -1,4 +1,4 @@ - module module_fv3_io_def +module module_fv3_io_def ! !*** fv3 io related configration variables ! @@ -9,24 +9,26 @@ module module_fv3_io_def ! use esmf, only : esmf_maxstr implicit none -! + integer :: num_pes_fcst integer :: wrttasks_per_group, write_groups integer :: n_group integer :: num_files - character(len=esmf_maxstr) :: app_domain - character(len=esmf_maxstr) :: output_grid - integer :: imo,jmo - integer :: ichunk2d,jchunk2d,ichunk3d,jchunk3d,kchunk3d integer :: nbdlphys - integer :: nsout_io, iau_offset, ideflate, nbits + integer :: nsout_io, iau_offset logical :: lflname_fulltime - real :: cen_lon, cen_lat, lon1, lat1, lon2, lat2, dlon, dlat - real :: stdlat1, stdlat2, dx, dy + character(len=esmf_maxstr),dimension(:),allocatable :: filename_base character(len=esmf_maxstr),dimension(:),allocatable :: output_file -! + integer,dimension(:),allocatable :: lead_wrttask, last_wrttask -! - end module module_fv3_io_def + character(len=esmf_maxstr),dimension(:),allocatable :: output_grid + integer,dimension(:),allocatable :: imo,jmo + real,dimension(:),allocatable :: cen_lon, cen_lat + real,dimension(:),allocatable :: lon1, lat1, lon2, lat2, dlon, dlat + real,dimension(:),allocatable :: stdlat1, stdlat2, dx, dy + integer,dimension(:),allocatable :: ideflate, nbits + integer,dimension(:),allocatable :: ichunk2d, jchunk2d, ichunk3d, jchunk3d, kchunk3d + +end module module_fv3_io_def diff --git a/io/module_write_netcdf.F90 b/io/module_write_netcdf.F90 index f7f518f3a..1445d5e04 100644 --- a/io/module_write_netcdf.F90 +++ b/io/module_write_netcdf.F90 @@ -10,6 +10,7 @@ module module_write_netcdf use esmf use netcdf use module_fv3_io_def,only : ideflate, nbits, & + ichunk2d,jchunk2d,ichunk3d,jchunk3d,kchunk3d, & output_grid,dx,dy,lon1,lat1,lon2,lat2 use mpi @@ -29,14 +30,14 @@ module module_write_netcdf !---------------------------------------------------------------------------------------- subroutine write_netcdf(wrtfb, filename, & use_parallel_netcdf, mpi_comm, mype, & - ichunk2d, jchunk2d, ichunk3d, jchunk3d, kchunk3d, rc) + grid_id, rc) ! type(ESMF_FieldBundle), intent(in) :: wrtfb character(*), intent(in) :: filename logical, intent(in) :: use_parallel_netcdf integer, intent(in) :: mpi_comm integer, intent(in) :: mype - integer, intent(in) :: ichunk2d,jchunk2d,ichunk3d,jchunk3d,kchunk3d + integer, intent(in) :: grid_id integer, optional,intent(out) :: rc ! !** local vars @@ -71,7 +72,7 @@ subroutine write_netcdf(wrtfb, filename, & character(len=ESMF_MAXSTR) :: attName, fldName integer :: varival - real(4) :: varr4val, scale_fact, offset, dataMin, dataMax + real(4) :: varr4val, dataMin, dataMax real(4), allocatable, dimension(:) :: compress_err real(8) :: varr8val character(len=ESMF_MAXSTR) :: varcval @@ -232,19 +233,19 @@ subroutine write_netcdf(wrtfb, filename, & end if ! coordinate variable attributes based on output_grid type - if (trim(output_grid) == 'gaussian_grid' .or. & - trim(output_grid) == 'global_latlon' .or. & - trim(output_grid) == 'regional_latlon') then + if (trim(output_grid(grid_id)) == 'gaussian_grid' .or. & + trim(output_grid(grid_id)) == 'global_latlon' .or. & + trim(output_grid(grid_id)) == 'regional_latlon') then ncerr = nf90_put_att(ncid, im_varid, "long_name", "T-cell longitude"); NC_ERR_STOP(ncerr) ncerr = nf90_put_att(ncid, im_varid, "units", "degrees_E"); NC_ERR_STOP(ncerr) ncerr = nf90_put_att(ncid, jm_varid, "long_name", "T-cell latiitude"); NC_ERR_STOP(ncerr) ncerr = nf90_put_att(ncid, jm_varid, "units", "degrees_N"); NC_ERR_STOP(ncerr) - else if (trim(output_grid) == 'rotated_latlon') then + else if (trim(output_grid(grid_id)) == 'rotated_latlon') then ncerr = nf90_put_att(ncid, im_varid, "long_name", "rotated T-cell longiitude"); NC_ERR_STOP(ncerr) ncerr = nf90_put_att(ncid, im_varid, "units", "degrees"); NC_ERR_STOP(ncerr) ncerr = nf90_put_att(ncid, jm_varid, "long_name", "rotated T-cell latiitude"); NC_ERR_STOP(ncerr) ncerr = nf90_put_att(ncid, jm_varid, "units", "degrees"); NC_ERR_STOP(ncerr) - else if (trim(output_grid) == 'lambert_conformal') then + else if (trim(output_grid(grid_id)) == 'lambert_conformal') then ncerr = nf90_put_att(ncid, im_varid, "long_name", "x-coordinate of projection"); NC_ERR_STOP(ncerr) ncerr = nf90_put_att(ncid, im_varid, "units", "meters"); NC_ERR_STOP(ncerr) ncerr = nf90_put_att(ncid, jm_varid, "long_name", "y-coordinate of projection"); NC_ERR_STOP(ncerr) @@ -303,24 +304,24 @@ subroutine write_netcdf(wrtfb, filename, & ! define variables if (rank == 2) then if (typekind == ESMF_TYPEKIND_R4) then - if (ideflate > 0) then - if (ichunk2d < 0 .or. jchunk2d < 0) then + if (ideflate(grid_id) > 0) then + if (ichunk2d(grid_id) < 0 .or. jchunk2d(grid_id) < 0) then ! let netcdf lib choose chunksize ! shuffle filter on for 2d fields (lossless compression) ncerr = nf90_def_var(ncid, trim(fldName), NF90_FLOAT, & dimids_2d, varids(i), & - shuffle=.true.,deflate_level=ideflate); NC_ERR_STOP(ncerr) + shuffle=.true.,deflate_level=ideflate(grid_id)); NC_ERR_STOP(ncerr) else if (is_cubed_sphere) then ncerr = nf90_def_var(ncid, trim(fldName), NF90_FLOAT, & dimids_2d, varids(i), & - shuffle=.true.,deflate_level=ideflate,& - chunksizes=[ichunk2d,jchunk2d,tileCount,1]); NC_ERR_STOP(ncerr) + shuffle=.true.,deflate_level=ideflate(grid_id),& + chunksizes=[ichunk2d(grid_id),jchunk2d(grid_id),tileCount,1]); NC_ERR_STOP(ncerr) else ncerr = nf90_def_var(ncid, trim(fldName), NF90_FLOAT, & dimids_2d, varids(i), & - shuffle=.true.,deflate_level=ideflate,& - chunksizes=[ichunk2d,jchunk2d, 1]); NC_ERR_STOP(ncerr) + shuffle=.true.,deflate_level=ideflate(grid_id),& + chunksizes=[ichunk2d(grid_id),jchunk2d(grid_id), 1]); NC_ERR_STOP(ncerr) end if end if ! compression filters require collective access. @@ -338,29 +339,29 @@ subroutine write_netcdf(wrtfb, filename, & end if else if (rank == 3) then if (typekind == ESMF_TYPEKIND_R4) then - if (ideflate > 0) then + if (ideflate(grid_id) > 0) then ! shuffle filter off for 3d fields using lossy compression - if (nbits > 0) then + if (nbits(grid_id) > 0) then shuffle=.false. else shuffle=.true. end if - if (ichunk3d < 0 .or. jchunk3d < 0 .or. kchunk3d < 0) then + if (ichunk3d(grid_id) < 0 .or. jchunk3d(grid_id) < 0 .or. kchunk3d(grid_id) < 0) then ! let netcdf lib choose chunksize ncerr = nf90_def_var(ncid, trim(fldName), NF90_FLOAT, & dimids_3d, varids(i), & - shuffle=shuffle,deflate_level=ideflate); NC_ERR_STOP(ncerr) + shuffle=shuffle,deflate_level=ideflate(grid_id)); NC_ERR_STOP(ncerr) else if (is_cubed_sphere) then ncerr = nf90_def_var(ncid, trim(fldName), NF90_FLOAT, & dimids_3d, varids(i), & - shuffle=shuffle,deflate_level=ideflate,& - chunksizes=[ichunk3d,jchunk3d,kchunk3d,tileCount,1]); NC_ERR_STOP(ncerr) + shuffle=shuffle,deflate_level=ideflate(grid_id),& + chunksizes=[ichunk3d(grid_id),jchunk3d(grid_id),kchunk3d(grid_id),tileCount,1]); NC_ERR_STOP(ncerr) else ncerr = nf90_def_var(ncid, trim(fldName), NF90_FLOAT, & dimids_3d, varids(i), & - shuffle=shuffle,deflate_level=ideflate,& - chunksizes=[ichunk3d,jchunk3d,kchunk3d, 1]); NC_ERR_STOP(ncerr) + shuffle=shuffle,deflate_level=ideflate(grid_id),& + chunksizes=[ichunk3d(grid_id),jchunk3d(grid_id),kchunk3d(grid_id), 1]); NC_ERR_STOP(ncerr) end if end if ! compression filters require collective access. @@ -480,27 +481,27 @@ subroutine write_netcdf(wrtfb, filename, & ! write grid_xt (im_varid) if (do_io) then allocate (x(im)) - if (trim(output_grid) == 'gaussian_grid' .or. & - trim(output_grid) == 'global_latlon' .or. & - trim(output_grid) == 'regional_latlon') then + if (trim(output_grid(grid_id)) == 'gaussian_grid' .or. & + trim(output_grid(grid_id)) == 'global_latlon' .or. & + trim(output_grid(grid_id)) == 'regional_latlon') then ncerr = nf90_put_var(ncid, im_varid, values=array_r8(:,jstart), start=[istart], count=[iend-istart+1]); NC_ERR_STOP(ncerr) - else if (trim(output_grid) == 'rotated_latlon') then + else if (trim(output_grid(grid_id)) == 'rotated_latlon') then do i=1,im - x(i) = lon1 + (lon2-lon1)/(im-1) * (i-1) + x(i) = lon1(grid_id) + (lon2(grid_id)-lon1(grid_id))/(im-1) * (i-1) end do ncerr = nf90_put_var(ncid, im_varid, values=x); NC_ERR_STOP(ncerr) - else if (trim(output_grid) == 'lambert_conformal') then + else if (trim(output_grid(grid_id)) == 'lambert_conformal') then do i=1,im - x(i) = dx * (i-1) + x(i) = dx(grid_id) * (i-1) end do ncerr = nf90_put_var(ncid, im_varid, values=x); NC_ERR_STOP(ncerr) - else if (trim(output_grid) == 'cubed_sphere_grid') then + else if (trim(output_grid(grid_id)) == 'cubed_sphere_grid') then do i=1,im x(i) = i end do ncerr = nf90_put_var(ncid, im_varid, values=x); NC_ERR_STOP(ncerr) else - write(0,*)'unknown output_grid ', trim(output_grid) + write(0,*)'unknown output_grid ', trim(output_grid(grid_id)) call ESMF_Finalize(endflag=ESMF_END_ABORT) end if end if @@ -529,27 +530,27 @@ subroutine write_netcdf(wrtfb, filename, & ! write grid_yt (jm_varid) if (do_io) then allocate (y(jm)) - if (trim(output_grid) == 'gaussian_grid' .or. & - trim(output_grid) == 'global_latlon' .or. & - trim(output_grid) == 'regional_latlon') then + if (trim(output_grid(grid_id)) == 'gaussian_grid' .or. & + trim(output_grid(grid_id)) == 'global_latlon' .or. & + trim(output_grid(grid_id)) == 'regional_latlon') then ncerr = nf90_put_var(ncid, jm_varid, values=array_r8(istart,:), start=[jstart], count=[jend-jstart+1]); NC_ERR_STOP(ncerr) - else if (trim(output_grid) == 'rotated_latlon') then + else if (trim(output_grid(grid_id)) == 'rotated_latlon') then do j=1,jm - y(j) = lat1 + (lat2-lat1)/(jm-1) * (j-1) + y(j) = lat1(grid_id) + (lat2(grid_id)-lat1(grid_id))/(jm-1) * (j-1) end do ncerr = nf90_put_var(ncid, jm_varid, values=y); NC_ERR_STOP(ncerr) - else if (trim(output_grid) == 'lambert_conformal') then + else if (trim(output_grid(grid_id)) == 'lambert_conformal') then do j=1,jm - y(j) = dy * (j-1) + y(j) = dy(grid_id) * (j-1) end do ncerr = nf90_put_var(ncid, jm_varid, values=y); NC_ERR_STOP(ncerr) - else if (trim(output_grid) == 'cubed_sphere_grid') then + else if (trim(output_grid(grid_id)) == 'cubed_sphere_grid') then do j=1,jm y(j) = j end do ncerr = nf90_put_var(ncid, jm_varid, values=y); NC_ERR_STOP(ncerr) else - write(0,*)'unknown output_grid ', trim(output_grid) + write(0,*)'unknown output_grid ', trim(output_grid(grid_id)) call ESMF_Finalize(endflag=ESMF_END_ABORT) end if end if @@ -631,12 +632,12 @@ subroutine write_netcdf(wrtfb, filename, & if (typekind == ESMF_TYPEKIND_R4) then if (par) then call ESMF_FieldGet(fcstField(i), localDe=0, farrayPtr=array_r4_3d, rc=rc); ESMF_ERR_RETURN(rc) - if (ideflate > 0 .and. nbits > 0) then + if (ideflate(grid_id) > 0 .and. nbits(grid_id) > 0) then dataMax = maxval(array_r4_3d) dataMin = minval(array_r4_3d) call mpi_allreduce(mpi_in_place,dataMax,1,mpi_real4,mpi_max,mpi_comm,ierr) call mpi_allreduce(mpi_in_place,dataMin,1,mpi_real4,mpi_min,mpi_comm,ierr) - call quantize_array(array_r4_3d, dataMin, dataMax, nbits, compress_err(i)) + call quantize_array(array_r4_3d, dataMin, dataMax, nbits(grid_id), compress_err(i)) call mpi_allreduce(mpi_in_place,compress_err(i),1,mpi_real4,mpi_max,mpi_comm,ierr) end if ncerr = nf90_put_var(ncid, varids(i), values=array_r4_3d, start=start_idx); NC_ERR_STOP(ncerr) @@ -647,16 +648,16 @@ subroutine write_netcdf(wrtfb, filename, & call ESMF_ArrayGather(array, array_r4_3d_cube(:,:,:,t), rootPet=0, tile=t, rc=rc); ESMF_ERR_RETURN(rc) end do if (mype==0) then - if (ideflate > 0 .and. nbits > 0) then - call quantize_array(array_r4_3d_cube, minval(array_r4_3d_cube), maxval(array_r4_3d_cube), nbits, compress_err(i)) + if (ideflate(grid_id) > 0 .and. nbits(grid_id) > 0) then + call quantize_array(array_r4_3d_cube, minval(array_r4_3d_cube), maxval(array_r4_3d_cube), nbits(grid_id), compress_err(i)) end if ncerr = nf90_put_var(ncid, varids(i), values=array_r4_3d_cube, start=start_idx); NC_ERR_STOP(ncerr) end if else call ESMF_FieldGather(fcstField(i), array_r4_3d, rootPet=0, rc=rc); ESMF_ERR_RETURN(rc) if (mype==0) then - if (ideflate > 0 .and. nbits > 0) then - call quantize_array(array_r4_3d, minval(array_r4_3d), maxval(array_r4_3d), nbits, compress_err(i)) + if (ideflate(grid_id) > 0 .and. nbits(grid_id) > 0) then + call quantize_array(array_r4_3d, minval(array_r4_3d), maxval(array_r4_3d), nbits(grid_id), compress_err(i)) end if ncerr = nf90_put_var(ncid, varids(i), values=array_r4_3d, start=start_idx); NC_ERR_STOP(ncerr) end if @@ -693,12 +694,12 @@ subroutine write_netcdf(wrtfb, filename, & end do ! end fieldCount - if (ideflate > 0 .and. nbits > 0 .and. do_io) then + if (ideflate(grid_id) > 0 .and. nbits(grid_id) > 0 .and. do_io) then ncerr = nf90_redef(ncid=ncid); NC_ERR_STOP(ncerr) do i=1, fieldCount if (compress_err(i) > 0) then ncerr = nf90_put_att(ncid, varids(i), 'max_abs_compression_error', compress_err(i)); NC_ERR_STOP(ncerr) - ncerr = nf90_put_att(ncid, varids(i), 'nbits', nbits); NC_ERR_STOP(ncerr) + ncerr = nf90_put_att(ncid, varids(i), 'nbits', nbits(grid_id)); NC_ERR_STOP(ncerr) end if end do ncerr = nf90_enddef(ncid=ncid); NC_ERR_STOP(ncerr) diff --git a/io/module_wrt_grid_comp.F90 b/io/module_wrt_grid_comp.F90 index 5acc70931..0bbf2a221 100644 --- a/io/module_wrt_grid_comp.F90 +++ b/io/module_wrt_grid_comp.F90 @@ -31,7 +31,7 @@ module module_wrt_grid_comp use write_internal_state use module_fv3_io_def, only : num_pes_fcst, & - n_group, num_files, app_domain, & + n_group, num_files, & filename_base, output_grid, output_file, & imo,jmo,ichunk2d,jchunk2d, & ichunk3d,jchunk3d,kchunk3d,nbits, & @@ -54,32 +54,22 @@ module module_wrt_grid_comp ! !----------------------------------------------------------------------- ! - real, parameter :: rdgas=287.04, grav=9.80 - real, parameter :: stndrd_atmos_ps = 101325. - real, parameter :: stndrd_atmos_lapse = 0.0065 ! integer,save :: lead_write_task !<-- Rank of the first write task in the write group integer,save :: last_write_task !<-- Rank of the last write task in the write group integer,save :: ntasks !<-- # of write tasks in the current group integer,save :: itasks, jtasks !<-- # of write tasks in i/j direction in the current group - integer,save :: mytile !<-- the tile number in write task integer,save :: wrt_mpi_comm !<-- the mpi communicator in the write comp integer,save :: idate(7) logical,save :: write_nsflip - logical,save :: first_init=.false. - logical,save :: first_run=.false. - logical,save :: first_getlatlon=.true. - logical,save :: first_getmaskwrt=.true. !<-- for mask the output grid of the write comp logical,save :: change_wrtidate=.false. ! !----------------------------------------------------------------------- ! - type(wrt_internal_state),pointer :: wrt_int_state ! The internal state pointer. type(ESMF_FieldBundle) :: gridFB integer :: FBcount character(len=esmf_maxstr),allocatable :: fcstItemNameList(:) - real(ESMF_KIND_R4), dimension(:,:), allocatable :: maskwrt ! !----------------------------------------------------------------------- REAL(KIND=8) :: btim,btim0 @@ -113,15 +103,15 @@ subroutine SetServices(wrt_comp, rc) call ESMF_GridCompSetEntryPoint(wrt_comp, ESMF_METHOD_INITIALIZE, & userRoutine=wrt_initialize, rc=rc) - if(rc/=0) write(*,*)'Error: write grid comp, initial' -! + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + call ESMF_GridCompSetEntryPoint(wrt_comp, ESMF_METHOD_RUN, & userRoutine=wrt_run, rc=rc) - if(rc/=0) write(*,*)'Error: write grid comp, run' -! + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + call ESMF_GridCompSetEntryPoint(wrt_comp, ESMF_METHOD_FINALIZE, & userRoutine=wrt_finalize, rc=rc) - if(rc/=0) write(*,*)'Error: write grid comp, run' + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return end subroutine SetServices ! @@ -146,21 +136,20 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) type(write_wrap) :: WRAP type(wrt_internal_state),pointer :: wrt_int_state - integer :: ISTAT, tl, i, j, n, k + integer :: tl, i, j, n, k integer,dimension(2,6) :: decomptile integer,dimension(2) :: regDecomp !define delayout for the nest grid integer :: fieldCount integer :: vm_mpi_comm - character(40) :: fieldName, axesname,longname - type(ESMF_Config) :: cf + character(40) :: fieldName + type(ESMF_Config) :: cf, cf_output_grid type(ESMF_DELayout) :: delayout - type(ESMF_Grid) :: wrtGrid, fcstGrid + type(ESMF_Grid) :: fcstGrid + type(ESMF_Grid), allocatable :: wrtGrid(:) type(ESMF_Array) :: array - type(ESMF_FieldBundle) :: fieldbdl_work type(ESMF_Field) :: field_work, field type(ESMF_Decomp_Flag) :: decompflagPTile(2,6) - character(len=80) :: attrValueSList(2) type(ESMF_StateItem_Flag), allocatable :: fcstItemTypeList(:) type(ESMF_FieldBundle) :: fcstFB, fieldbundle type(ESMF_Field), allocatable :: fcstField(:) @@ -179,7 +168,7 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) real(ESMF_KIND_R4) :: valueR4 real(ESMF_KIND_R8) :: valueR8 - integer :: attCount, axeslen, jidx, idx, noutfile + integer :: attCount, jidx, idx, noutfile character(19) :: newdate character(128) :: FBlist_outfilename(100), outfile_name character(128),dimension(:,:), allocatable :: outfilename @@ -189,19 +178,17 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) real(ESMF_KIND_R8) :: rot_lon, rot_lat real(ESMF_KIND_R8) :: geo_lon, geo_lat real(ESMF_KIND_R8) :: lon1_r8, lat1_r8 - real(ESMF_KIND_R8) :: x1, y1, x, y, delat + real(ESMF_KIND_R8) :: x1, y1, x, y, delat, delon type(ESMF_TimeInterval) :: IAU_offsetTI - type(ESMF_DataCopy_Flag) :: copyflag=ESMF_DATACOPY_REFERENCE -! real(8),parameter :: PI=3.14159265358979d0 + character(256) :: cf_open, cf_close character(256) :: gridfile integer :: num_output_file - ! - logical,save :: first=.true. logical :: lprnt -!test - real(ESMF_KIND_R8),dimension(:,:), pointer :: glatPtr, glonPtr + + integer :: ngrids, grid_id + logical :: top_parent_is_global ! !----------------------------------------------------------------------- !*********************************************************************** @@ -243,6 +230,7 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) ! lead_write_task,'last_write_task=',last_write_task, & ! 'mype=',wrt_int_state%mype,'jidx=',jidx,' comm=',wrt_mpi_comm ! + !----------------------------------------------------------------------- !*** get configuration variables !----------------------------------------------------------------------- @@ -265,34 +253,108 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, file=__FILE__)) return + if( wrt_int_state%write_dopost ) then +#ifdef NO_INLINE_POST + rc = ESMF_RC_NOT_IMPL + print *,'inline post not available on this machine' + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return +#endif + call esmf_configgetattribute(cf,wrt_int_state%post_nlunit,default=777,label='nlunit:',rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return + call ESMF_ConfigGetAttribute(config=CF,value=wrt_int_state%post_namelist,default='itag', & + label ='post_namelist:',rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, file=__FILE__)) return + endif + + allocate(output_file(num_files)) + num_output_file = ESMF_ConfigGetLen(config=CF, label ='output_file:',rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if (num_files == num_output_file) then + call ESMF_ConfigGetAttribute(CF,valueList=output_file,label='output_file:', & + count=num_files, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + do i = 1, num_files + if(output_file(i) /= "netcdf" .and. output_file(i) /= "netcdf_parallel") then + write(0,*)"Only netcdf and netcdf_parallel are allowed for multiple values of output_file" + call ESMF_Finalize(endflag=ESMF_END_ABORT) + endif + enddo + else if ( num_output_file == 1) then + call ESMF_ConfigGetAttribute(CF,valuelist=output_file,label='output_file:', count=1, rc=rc) + output_file(1:num_files) = output_file(1) + else + output_file(1:num_files) = 'netcdf' + endif + if(lprnt) then + print *,'num_files=',num_files + do i=1,num_files + print *,'num_file=',i,'filename_base= ',trim(filename_base(i)),' output_file= ',trim(output_file(i)) + enddo + endif + + call ESMF_AttributeGet(imp_state_write, convention="NetCDF", purpose="FV3", & + name="ngrids", value=ngrids, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - ! chunksizes for netcdf_parallel - call ESMF_ConfigGetAttribute(config=CF,value=ichunk2d,default=0,label ='ichunk2d:',rc=rc) - call ESMF_ConfigGetAttribute(config=CF,value=jchunk2d,default=0,label ='jchunk2d:',rc=rc) - call ESMF_ConfigGetAttribute(config=CF,value=ichunk3d,default=0,label ='ichunk3d:',rc=rc) - call ESMF_ConfigGetAttribute(config=CF,value=jchunk3d,default=0,label ='jchunk3d:',rc=rc) - call ESMF_ConfigGetAttribute(config=CF,value=kchunk3d,default=0,label ='kchunk3d:',rc=rc) - - ! zlib compression flag - call ESMF_ConfigGetAttribute(config=CF,value=ideflate,default=0,label ='ideflate:',rc=rc) - if (ideflate < 0) ideflate=0 - - call ESMF_ConfigGetAttribute(config=CF,value=nbits,default=0,label ='nbits:',rc=rc) - if (lprnt) then - print *,'ideflate=',ideflate,' nbits=',nbits - end if - ! nbits quantization level for lossy compression (must be between 1 and 31) - ! 1 is most compression, 31 is least. If outside this range, set to zero - ! which means use lossless compression. - if (nbits < 1 .or. nbits > 31) nbits=0 ! lossless compression (no quantization) -! variables for I/O options - call ESMF_ConfigGetAttribute(config=CF,value=app_domain, default="global", & - label ='app_domain:',rc=rc) + call ESMF_AttributeGet(imp_state_write, convention="NetCDF", purpose="FV3", & + name="top_parent_is_global", value=top_parent_is_global, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_ConfigGetAttribute(config=CF, value=output_grid, label ='output_grid:',rc=rc) + allocate(wrtGrid(ngrids)) + + allocate(output_grid(ngrids)) + + allocate(imo(ngrids)) + allocate(jmo(ngrids)) + + allocate(cen_lon(ngrids)) + allocate(cen_lat(ngrids)) + allocate(lon1(ngrids)) + allocate(lat1(ngrids)) + allocate(lon2(ngrids)) + allocate(lat2(ngrids)) + allocate(dlon(ngrids)) + allocate(dlat(ngrids)) + + allocate(stdlat1(ngrids)) + allocate(stdlat2(ngrids)) + allocate(dx(ngrids)) + allocate(dy(ngrids)) + + allocate(ichunk2d(ngrids)) + allocate(jchunk2d(ngrids)) + allocate(ichunk3d(ngrids)) + allocate(jchunk3d(ngrids)) + allocate(kchunk3d(ngrids)) + allocate(ideflate(ngrids)) + allocate(nbits(ngrids)) + + do n=1, ngrids + + if (n == 1) then + ! for top level domain look directly in cf + cf_output_grid = cf + else + ! for nest domains, look under specific section + write(cf_open,'("")') n + write(cf_close,'("")') n + cf_output_grid = ESMF_ConfigCreate(cf, openLabel=trim(cf_open), closeLabel=trim(cf_close), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + end if + + if (allocated(wrt_int_state%lat_start_wrtgrp)) deallocate (wrt_int_state%lat_start_wrtgrp) + if (allocated(wrt_int_state%lat_end_wrtgrp )) deallocate (wrt_int_state%lat_end_wrtgrp ) + if (allocated(wrt_int_state%lon_start_wrtgrp)) deallocate (wrt_int_state%lon_start_wrtgrp) + if (allocated(wrt_int_state%lon_end_wrtgrp )) deallocate (wrt_int_state%lon_end_wrtgrp ) + if (allocated(wrt_int_state%latPtr) ) deallocate (wrt_int_state%latPtr) + if (allocated(wrt_int_state%lonPtr) ) deallocate (wrt_int_state%lonPtr) + + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=output_grid(n), label ='output_grid:',rc=rc) if (lprnt) then - print *,'output_grid=',trim(output_grid) + print *,'grid_id= ', n, ' output_grid= ', trim(output_grid(n)) end if call ESMF_ConfigGetAttribute(config=CF, value=itasks,default=1,label ='itasks:',rc=rc) @@ -303,88 +365,94 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) jtasks = ntasks endif - if(trim(output_grid) == 'gaussian_grid' .or. trim(output_grid) == 'global_latlon') then - call ESMF_ConfigGetAttribute(config=CF, value=imo, label ='imo:',rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=jmo, label ='jmo:',rc=rc) + if(trim(output_grid(n)) == 'gaussian_grid' .or. trim(output_grid(n)) == 'global_latlon') then + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=imo(n), label ='imo:',rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=jmo(n), label ='jmo:',rc=rc) if (lprnt) then - print *,'imo=',imo,'jmo=',jmo,'itasks=',itasks,'jtasks=',jtasks + print *,'imo=',imo(n),'jmo=',jmo(n) end if - else if(trim(output_grid) == 'regional_latlon') then - call ESMF_ConfigGetAttribute(config=CF, value=lon1, label ='lon1:',rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=lat1, label ='lat1:',rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=lon2, label ='lon2:',rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=lat2, label ='lat2:',rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=dlon, label ='dlon:',rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=dlat, label ='dlat:',rc=rc) - imo = (lon2-lon1)/dlon + 1 - jmo = (lat2-lat1)/dlat + 1 + else if(trim(output_grid(n)) == 'regional_latlon') then + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=lon1(n), label ='lon1:',rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=lat1(n), label ='lat1:',rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=lon2(n), label ='lon2:',rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=lat2(n), label ='lat2:',rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=dlon(n), label ='dlon:',rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=dlat(n), label ='dlat:',rc=rc) + imo(n) = (lon2(n)-lon1(n))/dlon(n) + 1 + jmo(n) = (lat2(n)-lat1(n))/dlat(n) + 1 if (lprnt) then - print *,'lon1=',lon1,' lat1=',lat1 - print *,'lon2=',lon2,' lat2=',lat2 - print *,'dlon=',dlon,' dlat=',dlat - print *,'imo =',imo, ' jmo=',jmo + print *,'lon1=',lon1(n),' lat1=',lat1(n) + print *,'lon2=',lon2(n),' lat2=',lat2(n) + print *,'dlon=',dlon(n),' dlat=',dlat(n) + print *,'imo =',imo(n), ' jmo =',jmo(n) end if - else if (trim(output_grid) == 'rotated_latlon') then - call ESMF_ConfigGetAttribute(config=CF, value=cen_lon, label ='cen_lon:',rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=cen_lat, label ='cen_lat:',rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=lon1, label ='lon1:', rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=lat1, label ='lat1:', rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=lon2, label ='lon2:', rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=lat2, label ='lat2:', rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=dlon, label ='dlon:', rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=dlat, label ='dlat:', rc=rc) - imo = (lon2-lon1)/dlon + 1 - jmo = (lat2-lat1)/dlat + 1 + else if (trim(output_grid(n)) == 'rotated_latlon') then + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=cen_lon(n), label ='cen_lon:',rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=cen_lat(n), label ='cen_lat:',rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=lon1(n), label ='lon1:', rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=lat1(n), label ='lat1:', rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=lon2(n), label ='lon2:', rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=lat2(n), label ='lat2:', rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=dlon(n), label ='dlon:', rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=dlat(n), label ='dlat:', rc=rc) + imo(n) = (lon2(n)-lon1(n))/dlon(n) + 1 + jmo(n) = (lat2(n)-lat1(n))/dlat(n) + 1 if (lprnt) then - print *,'lon1=',lon1,' lat1=',lat1 - print *,'lon2=',lon2,' lat2=',lat2 - print *,'dlon=',dlon,' dlat=',dlat - print *,'imo =',imo, ' jmo=',jmo + print *,'lon1=',lon1(n),' lat1=',lat1(n) + print *,'lon2=',lon2(n),' lat2=',lat2(n) + print *,'dlon=',dlon(n),' dlat=',dlat(n) + print *,'imo =',imo(n), ' jmo =',jmo(n) end if - else if (trim(output_grid) == 'lambert_conformal') then - call ESMF_ConfigGetAttribute(config=CF, value=cen_lon, label ='cen_lon:',rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=cen_lat, label ='cen_lat:',rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=stdlat1, label ='stdlat1:',rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=stdlat2, label ='stdlat2:',rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=imo, label ='nx:', rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=jmo, label ='ny:', rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=lon1, label ='lon1:', rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=lat1, label ='lat1:', rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=dx, label ='dx:', rc=rc) - call ESMF_ConfigGetAttribute(config=CF, value=dy, label ='dy:', rc=rc) + else if (trim(output_grid(n)) == 'lambert_conformal') then + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=cen_lon(n), label ='cen_lon:',rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=cen_lat(n), label ='cen_lat:',rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=stdlat1(n), label ='stdlat1:',rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=stdlat2(n), label ='stdlat2:',rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=imo(n), label ='nx:', rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=jmo(n), label ='ny:', rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=lon1(n), label ='lon1:', rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=lat1(n), label ='lat1:', rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=dx(n), label ='dx:', rc=rc) + call ESMF_ConfigGetAttribute(config=cf_output_grid, value=dy(n), label ='dy:', rc=rc) if (lprnt) then - print *,'cen_lon=',cen_lon,' cen_lat=',cen_lat - print *,'stdlat1=',stdlat1,' stdlat2=',stdlat2 - print *,'lon1=',lon1,' lat1=',lat1 - print *,'nx=',imo, ' ny=',jmo - print *,'dx=',dx,' dy=',dy + print *,'cen_lon=',cen_lon(n),' cen_lat=',cen_lat(n) + print *,'stdlat1=',stdlat1(n),' stdlat2=',stdlat2(n) + print *,'lon1=',lon1(n),' lat1=',lat1(n) + print *,'nx=',imo(n), ' ny=',jmo(n) + print *,'dx=',dx(n),' dy=',dy(n) endif endif ! output_grid - if( wrt_int_state%write_dopost ) then -#ifdef NO_INLINE_POST - rc = ESMF_RC_NOT_IMPL - print *,'inline post not available on this machine' - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, file=__FILE__)) return -#endif - call esmf_configgetattribute(cf,wrt_int_state%post_nlunit,default=777,label='nlunit:',rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, file=__FILE__)) return - call ESMF_ConfigGetAttribute(config=CF,value=wrt_int_state%post_namelist,default='itag', & - label ='post_namelist:',rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, file=__FILE__)) return + + ! chunksizes for netcdf_parallel + call ESMF_ConfigGetAttribute(config=CF,value=ichunk2d(n),default=0,label ='ichunk2d:',rc=rc) + call ESMF_ConfigGetAttribute(config=CF,value=jchunk2d(n),default=0,label ='jchunk2d:',rc=rc) + call ESMF_ConfigGetAttribute(config=CF,value=ichunk3d(n),default=0,label ='ichunk3d:',rc=rc) + call ESMF_ConfigGetAttribute(config=CF,value=jchunk3d(n),default=0,label ='jchunk3d:',rc=rc) + call ESMF_ConfigGetAttribute(config=CF,value=kchunk3d(n),default=0,label ='kchunk3d:',rc=rc) + + ! zlib compression flag + call ESMF_ConfigGetAttribute(config=CF,value=ideflate(n),default=0,label ='ideflate:',rc=rc) + if (ideflate(n) < 0) ideflate(n)=0 + + call ESMF_ConfigGetAttribute(config=CF,value=nbits(n),default=0,label ='nbits:',rc=rc) + if (lprnt) then + print *,'ideflate=',ideflate(n),' nbits=',nbits(n) + end if + ! nbits quantization level for lossy compression (must be between 1 and 31) + ! 1 is most compression, 31 is least. If outside this range, set to zero + ! which means use lossless compression. + if (nbits(n) < 1 .or. nbits(n) > 31) nbits(n)=0 ! lossless compression (no quantization) + + if (cf_output_grid /= cf) then + ! destroy the temporary config object created for nest domains + call ESMF_ConfigDestroy(config=cf_output_grid, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return endif -! -!----------------------------------------------------------------------- -!*** Create the cubed sphere grid with field on PETs -!*** first try: Create cubed sphere grid from file -!----------------------------------------------------------------------- -! - if ( trim(output_grid) == 'cubed_sphere_grid' ) then - mytile = mod(wrt_int_state%mype,ntasks)+1 - if ( trim(app_domain) == 'global' ) then + if ( trim(output_grid(n)) == 'cubed_sphere_grid' ) then + !*** Create cubed sphere grid from file + if (top_parent_is_global .and. n==1) then + gridfile = 'grid_spec.nc' ! global top-level parent do tl=1,6 decomptile(1,tl) = 1 decomptile(2,tl) = jidx @@ -394,19 +462,23 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) name="gridfile", value=gridfile, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - CALL ESMF_LogWrite("wrtComp: gridfile:"//trim(gridfile),ESMF_LOGMSG_INFO,rc=rc) - wrtgrid = ESMF_GridCreateMosaic(filename="INPUT/"//trim(gridfile), & + call ESMF_LogWrite("wrtComp: gridfile:"//trim(gridfile),ESMF_LOGMSG_INFO,rc=rc) + wrtGrid(n) = ESMF_GridCreateMosaic(filename="INPUT/"//trim(gridfile), & regDecompPTile=decomptile,tileFilePath="INPUT/", & decompflagPTile=decompflagPTile, & staggerlocList=(/ESMF_STAGGERLOC_CENTER, ESMF_STAGGERLOC_CORNER/), & name='wrt_grid', rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return else - if(trim(app_domain) == 'nested') then - gridfile='grid.nest02.tile7.nc' - else if(trim(app_domain) == 'regional') then - gridfile='grid.tile7.halo0.nc' - endif + if (top_parent_is_global) then + write(gridfile,'(A,I2.2,A,I1,A)') 'grid.nest', n, '.tile', n+5, '.nc' + else + if (n == 1) then + gridfile='grid.tile7.halo0.nc' ! regional top-level parent + else + write(gridfile,'(A,I2.2,A,I1,A)') 'grid.nest', n, '.tile', n, '.nc' + endif + end if regDecomp(1) = 1 regDecomp(2) = ntasks allocate(petMap(ntasks)) @@ -417,57 +489,57 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! create the nest Grid by reading it from file but use DELayout - wrtGrid = ESMF_GridCreate(filename="INPUT/"//trim(gridfile), & + call ESMF_LogWrite("wrtComp: gridfile:"//trim(gridfile),ESMF_LOGMSG_INFO,rc=rc) + wrtGrid(n) = ESMF_GridCreate(filename="INPUT/"//trim(gridfile), & fileformat=ESMF_FILEFORMAT_GRIDSPEC, regDecomp=regDecomp, & decompflag=(/ESMF_DECOMP_SYMMEDGEMAX,ESMF_DECOMP_SYMMEDGEMAX/), & delayout=delayout, isSphere=.false., indexflag=ESMF_INDEX_DELOCAL, & rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - print *,'in nested/regional cubed_sphere grid, regDecomp=',regDecomp,' PetMap=',petMap(1),petMap(ntasks), & + if (lprnt) print *,'in nested/regional cubed_sphere grid, regDecomp=',regDecomp,' PetMap=',petMap(1),petMap(ntasks), & 'gridfile=',trim(gridfile) deallocate(petMap) endif - else if ( trim(output_grid) == 'gaussian_grid') then + else if ( trim(output_grid(n)) == 'gaussian_grid') then - wrtgrid = ESMF_GridCreate1PeriDim(minIndex=(/1,1/), & - maxIndex=(/imo,jmo/), regDecomp=(/itasks,jtasks/), & - indexflag=ESMF_INDEX_GLOBAL, & - name='wrt_grid',rc=rc) -! indexflag=ESMF_INDEX_GLOBAL, coordSys=ESMF_COORDSYS_SPH_DEG + wrtGrid(n) = ESMF_GridCreate1PeriDim(minIndex=(/1,1/), & + maxIndex=(/imo(n),jmo(n)/), regDecomp=(/itasks,jtasks/), & + indexflag=ESMF_INDEX_GLOBAL, & + name='wrt_grid',rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_GridAddCoord(wrtgrid, staggerLoc=ESMF_STAGGERLOC_CENTER, rc=rc) + call ESMF_GridAddCoord(wrtGrid(n), staggerLoc=ESMF_STAGGERLOC_CENTER, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_GridGetCoord(wrtgrid, coordDim=1, farrayPtr=lonPtr, rc=rc) + call ESMF_GridGetCoord(wrtGrid(n), coordDim=1, farrayPtr=lonPtr, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_GridGetCoord(wrtgrid, coordDim=2, farrayPtr=latPtr, rc=rc) + call ESMF_GridGetCoord(wrtGrid(n), coordDim=2, farrayPtr=latPtr, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! - allocate(slat(jmo), lat(jmo), lon(imo)) - call splat(4, jmo, slat) + allocate(slat(jmo(n)), lat(jmo(n)), lon(imo(n))) + call splat(4, jmo(n), slat) if(write_nsflip) then - do j=1,jmo + do j=1,jmo(n) lat(j) = asin(slat(j)) * radi enddo else - do j=1,jmo - lat(jmo-j+1) = asin(slat(j)) * radi + do j=1,jmo(n) + lat(jmo(n)-j+1) = asin(slat(j)) * radi enddo endif wrt_int_state%latstart = lat(1) - wrt_int_state%latlast = lat(jmo) - do j=1,imo - lon(j) = 360.d0/real(imo,8) *real(j-1,8) + wrt_int_state%latlast = lat(jmo(n)) + do j=1,imo(n) + lon(j) = 360.d0/real(imo(n),8) *real(j-1,8) enddo wrt_int_state%lonstart = lon(1) - wrt_int_state%lonlast = lon(imo) + wrt_int_state%lonlast = lon(imo(n)) do j=lbound(latPtr,2),ubound(latPtr,2) do i=lbound(lonPtr,1),ubound(lonPtr,1) - lonPtr(i,j) = 360.d0/real(imo,8) * real(i-1,8) + lonPtr(i,j) = 360.d0/real(imo(n),8) * real(i-1,8) latPtr(i,j) = lat(j) enddo enddo @@ -508,60 +580,63 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) wrt_int_state%lonPtr(i,j) = lonPtr(i,j) enddo enddo - wrt_int_state%im = imo - wrt_int_state%jm = jmo + wrt_int_state%im = imo(n) + wrt_int_state%jm = jmo(n) wrt_int_state%post_maptype = 4 - deallocate(slat) - else if ( trim(output_grid) == 'global_latlon') then - wrtgrid = ESMF_GridCreate1PeriDim(minIndex=(/1,1/), & - maxIndex=(/imo,jmo/), regDecomp=(/itasks,jtasks/), & - indexflag=ESMF_INDEX_GLOBAL, name='wrt_grid',rc=rc) + deallocate(slat, lat, lon) + + else if ( trim(output_grid(n)) == 'global_latlon') then + wrtGrid(n) = ESMF_GridCreate1PeriDim(minIndex=(/1,1/), & + maxIndex=(/imo(n),jmo(n)/), regDecomp=(/itasks,jtasks/), & + indexflag=ESMF_INDEX_GLOBAL, & + name='wrt_grid',rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_GridAddCoord(wrtgrid, staggerLoc=ESMF_STAGGERLOC_CENTER, rc=rc) + call ESMF_GridAddCoord(wrtGrid(n), staggerLoc=ESMF_STAGGERLOC_CENTER, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_GridGetCoord(wrtgrid, coordDim=1, farrayPtr=lonPtr, rc=rc) + call ESMF_GridGetCoord(wrtGrid(n), coordDim=1, farrayPtr=lonPtr, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_GridGetCoord(wrtgrid, coordDim=2, farrayPtr=latPtr, rc=rc) + call ESMF_GridGetCoord(wrtGrid(n), coordDim=2, farrayPtr=latPtr, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! - allocate(lat(jmo), lon(imo)) - if (mod(jmo,2) == 0) then + allocate(lat(jmo(n)), lon(imo(n))) + if (mod(jmo(n),2) == 0) then ! if jmo even, lats do not include poles and equator - delat = 180.d0/real(jmo,8) + delat = 180.d0/real(jmo(n),8) if(write_nsflip) then - do j=1,jmo + do j=1,jmo(n) lat(j) = 90.d0 - 0.5*delat - real(j-1,8)*delat enddo else - do j=1,jmo + do j=1,jmo(n) lat(j) = -90.d0 + 0.5*delat + real(j-1,8)*delat enddo endif else ! if jmo odd, lats include poles and equator - delat = 180.d0/real(jmo-1,8) + delat = 180.d0/real(jmo(n)-1,8) if(write_nsflip) then - do j=1,jmo + do j=1,jmo(n) lat(j) = 90.d0 - real(j-1,8)*delat enddo else - do j=1,jmo + do j=1,jmo(n) lat(j) = -90.d0 + real(j-1,8)*delat enddo endif endif wrt_int_state%latstart = lat(1) - wrt_int_state%latlast = lat(jmo) - do i=1,imo - lon(i) = 360.d0/real(imo,8) *real(i-1,8) + wrt_int_state%latlast = lat(jmo(n)) + delon = 360.d0/real(imo(n),8) + do i=1,imo(n) + lon(i) = real(i-1,8)*delon enddo wrt_int_state%lonstart = lon(1) - wrt_int_state%lonlast = lon(imo) + wrt_int_state%lonlast = lon(imo(n)) do j=lbound(latPtr,2),ubound(latPtr,2) do i=lbound(lonPtr,1),ubound(lonPtr,1) lonPtr(i,j) = lon(i) @@ -572,6 +647,12 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) wrt_int_state%lat_end = ubound(latPtr,2) wrt_int_state%lon_start = lbound(lonPtr,1) wrt_int_state%lon_end = ubound(lonPtr,1) + lon1(n) = wrt_int_state%lonstart + lon2(n) = wrt_int_state%lonlast + lat1(n) = wrt_int_state%latstart + lat2(n) = wrt_int_state%latlast + dlon(n) = delon + dlat(n) = delat allocate( wrt_int_state%lat_start_wrtgrp(wrt_int_state%petcount)) allocate( wrt_int_state%lat_end_wrtgrp (wrt_int_state%petcount)) allocate( wrt_int_state%lon_start_wrtgrp(wrt_int_state%petcount)) @@ -596,62 +677,64 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) wrt_int_state%lonPtr(i,j) = lonPtr(i,j) enddo enddo - wrt_int_state%im = imo - wrt_int_state%jm = jmo + wrt_int_state%im = imo(n) + wrt_int_state%jm = jmo(n) wrt_int_state%post_maptype = 0 - else if ( trim(output_grid) == 'regional_latlon' .or. & - trim(output_grid) == 'rotated_latlon' .or. & - trim(output_grid) == 'lambert_conformal' ) then + deallocate(lat, lon) - wrtgrid = ESMF_GridCreate1PeriDim(minIndex=(/1,1/), & - maxIndex=(/imo,jmo/), regDecomp=(/itasks,jtasks/), & - indexflag=ESMF_INDEX_GLOBAL, & - name='wrt_grid',rc=rc) + else if ( trim(output_grid(n)) == 'regional_latlon' .or. & + trim(output_grid(n)) == 'rotated_latlon' .or. & + trim(output_grid(n)) == 'lambert_conformal' ) then + + wrtGrid(n) = ESMF_GridCreateNoPeriDim(minIndex=(/1,1/), & + maxIndex=(/imo(n),jmo(n)/), regDecomp=(/itasks,jtasks/), & + indexflag=ESMF_INDEX_GLOBAL, & + name='wrt_grid',rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_GridAddCoord(wrtgrid, staggerLoc=ESMF_STAGGERLOC_CENTER, rc=rc) + call ESMF_GridAddCoord(wrtGrid(n), staggerLoc=ESMF_STAGGERLOC_CENTER, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_GridGetCoord(wrtgrid, coordDim=1, farrayPtr=lonPtr, rc=rc) + call ESMF_GridGetCoord(wrtGrid(n), coordDim=1, farrayPtr=lonPtr, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_GridGetCoord(wrtgrid, coordDim=2, farrayPtr=latPtr, rc=rc) + call ESMF_GridGetCoord(wrtGrid(n), coordDim=2, farrayPtr=latPtr, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - wrt_int_state%im = imo - wrt_int_state%jm = jmo - if ( trim(output_grid) == 'regional_latlon' ) then + wrt_int_state%im = imo(n) + wrt_int_state%jm = jmo(n) + if ( trim(output_grid(n)) == 'regional_latlon' ) then do j=lbound(lonPtr,2),ubound(lonPtr,2) do i=lbound(lonPtr,1),ubound(lonPtr,1) - lonPtr(i,j) = lon1 + (lon2-lon1)/(imo-1) * (i-1) - latPtr(i,j) = lat1 + (lat2-lat1)/(jmo-1) * (j-1) + lonPtr(i,j) = lon1(n) + (lon2(n)-lon1(n))/(imo(n)-1) * (i-1) + latPtr(i,j) = lat1(n) + (lat2(n)-lat1(n))/(jmo(n)-1) * (j-1) enddo enddo wrt_int_state%post_maptype = 0 - else if ( trim(output_grid) == 'rotated_latlon' ) then + else if ( trim(output_grid(n)) == 'rotated_latlon' ) then do j=lbound(lonPtr,2),ubound(lonPtr,2) do i=lbound(lonPtr,1),ubound(lonPtr,1) - rot_lon = lon1 + (lon2-lon1)/(imo-1) * (i-1) - rot_lat = lat1 + (lat2-lat1)/(jmo-1) * (j-1) - call rtll(rot_lon, rot_lat, geo_lon, geo_lat, dble(cen_lon), dble(cen_lat)) + rot_lon = lon1(n) + (lon2(n)-lon1(n))/(imo(n)-1) * (i-1) + rot_lat = lat1(n) + (lat2(n)-lat1(n))/(jmo(n)-1) * (j-1) + call rtll(rot_lon, rot_lat, geo_lon, geo_lat, dble(cen_lon(n)), dble(cen_lat(n))) if (geo_lon < 0.0) geo_lon = geo_lon + 360.0 lonPtr(i,j) = geo_lon latPtr(i,j) = geo_lat enddo enddo wrt_int_state%post_maptype = 207 - else if ( trim(output_grid) == 'lambert_conformal' ) then - lon1_r8 = dble(lon1) - lat1_r8 = dble(lat1) - call lambert(dble(stdlat1),dble(stdlat2),dble(cen_lat),dble(cen_lon), & + else if ( trim(output_grid(n)) == 'lambert_conformal' ) then + lon1_r8 = dble(lon1(n)) + lat1_r8 = dble(lat1(n)) + call lambert(dble(stdlat1(n)),dble(stdlat2(n)),dble(cen_lat(n)),dble(cen_lon(n)), & lon1_r8,lat1_r8,x1,y1, 1) do j=lbound(lonPtr,2),ubound(lonPtr,2) do i=lbound(lonPtr,1),ubound(lonPtr,1) - x = x1 + dx * (i-1) - y = y1 + dy * (j-1) - call lambert(dble(stdlat1),dble(stdlat2),dble(cen_lat),dble(cen_lon), & + x = x1 + dx(n) * (i-1) + y = y1 + dy(n) * (j-1) + call lambert(dble(stdlat1(n)),dble(stdlat2(n)),dble(cen_lat(n)),dble(cen_lon(n)), & geo_lon,geo_lat,x,y,-1) if (geo_lon <0.0) geo_lon = geo_lon + 360.0 lonPtr(i,j) = geo_lon @@ -690,11 +773,13 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) else - write(0,*)"wrt_initialize: Unknown output_grid ", trim(output_grid) - call ESMF_LogWrite("wrt_initialize: Unknown output_grid "//trim(output_grid),ESMF_LOGMSG_ERROR,rc=RC) + write(0,*)"wrt_initialize: Unknown output_grid ", trim(output_grid(n)) + call ESMF_LogWrite("wrt_initialize: Unknown output_grid "//trim(output_grid(n)),ESMF_LOGMSG_ERROR,rc=RC) call ESMF_Finalize(endflag=ESMF_END_ABORT) endif + + end do ! n = 1, ngrids ! !----------------------------------------------------------------------- !*** get write grid component initial time from clock @@ -718,17 +803,9 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) m=idate(5),s=idate(6),rc=rc) wrt_int_state%idate = idate change_wrtidate = .true. - if (lprnt) print *,'in wrt initial, with iau, io_baseline time=',idate,'rc=',rc + if (lprnt) print *,'in wrt initial, with iau, io_baseline time=',idate,'rc=',rc endif ! -! Create field bundle -!------------------------------------------------------------------- -! -!--- check grid dim count first - call ESMF_GridGet(wrtgrid, dimCount=gridDimCount, rc=rc) - - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return -! !--- Look at the incoming FieldBundles in the imp_state_write, and mirror them ! call ESMF_StateGet(imp_state_write, itemCount=FBCount, rc=rc) @@ -748,12 +825,14 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) outfilename = '' call ESMF_StateGet(imp_state_write, itemNameList=fcstItemNameList, & - itemTypeList=fcstItemTypeList, rc=rc) + itemTypeList=fcstItemTypeList, & + !itemorderflag=ESMF_ITEMORDER_ADDORDER, & + rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return !loop over all items in the imp_state_write and collect all FieldBundles - do i=1, FBcount + do i=1, FBCount if (fcstItemTypeList(i) == ESMF_STATEITEM_FIELDBUNDLE) then @@ -762,6 +841,15 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + call ESMF_AttributeGet(fcstFB, convention="NetCDF", purpose="FV3", & + name="grid_id", value=grid_id, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + +!--- check grid dim count first + call ESMF_GridGet(wrtGrid(grid_id), dimCount=gridDimCount, rc=rc) + + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + ! create a mirror FieldBundle and add it to importState fieldbundle = ESMF_FieldBundleCreate(name="mirror_"//trim(fcstItemNameList(i)), rc=rc) @@ -804,7 +892,7 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) call ESMF_FieldGet(fcstField(j), gridToFieldMap=gridToFieldMap, & ungriddedLBound=ungriddedLBound, ungriddedUBound=ungriddedUBound, & rc=rc) - CALL ESMF_LogWrite("after field create on wrt comp",ESMF_LOGMSG_INFO,rc=RC) + call ESMF_LogWrite("after field create on wrt comp",ESMF_LOGMSG_INFO,rc=RC) ! if (lprnt) print *,'in wrt,fcstfld,fieldname=', & ! trim(fieldname),'fieldDimCount=',fieldDimCount,'gridDimCount=',gridDimCount, & @@ -815,12 +903,12 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) ! create the mirror field - CALL ESMF_LogWrite("call field create on wrt comp",ESMF_LOGMSG_INFO,rc=RC) - field_work = ESMF_FieldCreate(wrtGrid, typekind, name=fieldName, & + call ESMF_LogWrite("call field create on wrt comp",ESMF_LOGMSG_INFO,rc=RC) + field_work = ESMF_FieldCreate(wrtGrid(grid_id), typekind, name=fieldName, & gridToFieldMap=gridToFieldMap, & ungriddedLBound=ungriddedLBound, & ungriddedUBound=ungriddedUBound, rc=rc) - CALL ESMF_LogWrite("aft call field create on wrt comp",ESMF_LOGMSG_INFO,rc=RC) + call ESMF_LogWrite("aft call field create on wrt comp",ESMF_LOGMSG_INFO,rc=RC) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -835,11 +923,11 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) name="output_file", value=outfile_name, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - CALL ESMF_LogWrite("bf fcstfield, get output_file "//trim(outfile_name)//" "//trim(fieldName),ESMF_LOGMSG_INFO,rc=RC) + call ESMF_LogWrite("bf fcstfield, get output_file "//trim(outfile_name)//" "//trim(fieldName),ESMF_LOGMSG_INFO,rc=RC) if (trim(outfile_name) /= '') then outfilename(j,i) = trim(outfile_name) endif - CALL ESMF_LogWrite("af fcstfield, get output_file",ESMF_LOGMSG_INFO,rc=RC) + call ESMF_LogWrite("af fcstfield, get output_file",ESMF_LOGMSG_INFO,rc=RC) ! if (lprnt) print *,' i=',i,' j=',j,' outfilename=',trim(outfilename(j,i)) @@ -852,7 +940,7 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) deallocate(gridToFieldMap, ungriddedLBound, ungriddedUBound) enddo ! - call ESMF_AttributeCopy(fcstGrid, wrtGrid, & + call ESMF_AttributeCopy(fcstGrid, wrtGrid(grid_id), & attcopy=ESMF_ATTCOPY_REFERENCE, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -877,7 +965,7 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) ! !create output field bundles - allocate(wrt_int_state%wrtFB(num_files)) + allocate(wrt_int_state%wrtFB(wrt_int_state%FBcount)) do i=1, wrt_int_state%FBcount wrt_int_state%wrtFB_names(i) = trim(FBlist_outfilename(i)) @@ -890,13 +978,17 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) call ESMF_StateGet(imp_state_write, itemName="mirror_"//trim(fcstItemNameList(n)), & fieldbundle=fcstFB, rc=rc) - if( index(trim(fcstItemNameList(n)),trim(FBlist_outfilename(i))) > 0 ) then + if( index(trim(fcstItemNameList(n)),trim(FBlist_outfilename(i))) == 1 ) then ! ! copy the mirror fcstfield bundle Attributes to the output field bundle call ESMF_AttributeCopy(fcstFB, wrt_int_state%wrtFB(i), & attcopy=ESMF_ATTCOPY_REFERENCE, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_AttributeGet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & + name="grid_id", value=grid_id, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! call ESMF_FieldBundleGet(fcstFB, fieldCount=fieldCount, rc=rc) @@ -937,57 +1029,43 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & name="source", value="FV3GFS", rc=rc) - if (trim(output_grid) == 'cubed_sphere_grid') then + if (trim(output_grid(grid_id)) == 'cubed_sphere_grid') then call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & name="grid", value="cubed_sphere", rc=rc) - else if (trim(output_grid) == 'gaussian_grid') then + else if (trim(output_grid(grid_id)) == 'gaussian_grid') then call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & name="grid", value="gaussian", rc=rc) call ESMF_AttributeAdd(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & attrList=(/"im","jm"/), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="im", value=imo, rc=rc) + name="im", value=imo(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="jm", value=jmo, rc=rc) + name="jm", value=jmo(grid_id), rc=rc) - else if (trim(output_grid) == 'global_latlon') then - - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="grid", value="latlon", rc=rc) - call ESMF_AttributeAdd(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - attrList=(/"lonstart","latstart","lonlast ","latlast "/), rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lonstart", value=wrt_int_state%lonstart, rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="latstart", value=wrt_int_state%latstart, rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lonlast", value=wrt_int_state%lonlast, rc=rc) - call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="latlast", value=wrt_int_state%latlast, rc=rc) - - else if (trim(output_grid) == 'regional_latlon') then + else if (trim(output_grid(grid_id)) == 'regional_latlon' & + .or. trim(output_grid(grid_id)) == 'global_latlon') then call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & name="grid", value="latlon", rc=rc) call ESMF_AttributeAdd(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & attrList=(/"lon1","lat1","lon2","lat2","dlon","dlat"/), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lon1", value=lon1, rc=rc) + name="lon1", value=lon1(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lat1", value=lat1, rc=rc) + name="lat1", value=lat1(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lon2", value=lon2, rc=rc) + name="lon2", value=lon2(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lat2", value=lat2, rc=rc) + name="lat2", value=lat2(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="dlon", value=dlon, rc=rc) + name="dlon", value=dlon(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="dlat", value=dlat, rc=rc) + name="dlat", value=dlat(grid_id), rc=rc) - else if (trim(output_grid) == 'rotated_latlon') then + else if (trim(output_grid(grid_id)) == 'rotated_latlon') then call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & name="grid", value="rotated_latlon", rc=rc) @@ -1001,23 +1079,23 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) "dlon ",& "dlat "/), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="cen_lon", value=cen_lon, rc=rc) + name="cen_lon", value=cen_lon(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="cen_lat", value=cen_lat, rc=rc) + name="cen_lat", value=cen_lat(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lon1", value=lon1, rc=rc) + name="lon1", value=lon1(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lat1", value=lat1, rc=rc) + name="lat1", value=lat1(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lon2", value=lon2, rc=rc) + name="lon2", value=lon2(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lat2", value=lat2, rc=rc) + name="lat2", value=lat2(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="dlon", value=dlon, rc=rc) + name="dlon", value=dlon(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="dlat", value=dlat, rc=rc) + name="dlat", value=dlat(grid_id), rc=rc) - else if (trim(output_grid) == 'lambert_conformal') then + else if (trim(output_grid(grid_id)) == 'lambert_conformal') then call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & name="grid", value="lambert_conformal", rc=rc) @@ -1033,25 +1111,25 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) "dx ",& "dy "/), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="cen_lon", value=cen_lon, rc=rc) + name="cen_lon", value=cen_lon(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="cen_lat", value=cen_lat, rc=rc) + name="cen_lat", value=cen_lat(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="stdlat1", value=stdlat1, rc=rc) + name="stdlat1", value=stdlat1(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="stdlat2", value=stdlat2, rc=rc) + name="stdlat2", value=stdlat2(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="nx", value=imo, rc=rc) + name="nx", value=imo(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="ny", value=jmo, rc=rc) + name="ny", value=jmo(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lat1", value=lat1, rc=rc) + name="lat1", value=lat1(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="lon1", value=lon1, rc=rc) + name="lon1", value=lon1(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="dx", value=dx, rc=rc) + name="dx", value=dx(grid_id), rc=rc) call ESMF_AttributeSet(wrt_int_state%wrtFB(i), convention="NetCDF", purpose="FV3", & - name="dy", value=dy, rc=rc) + name="dy", value=dy(grid_id), rc=rc) end if @@ -1098,8 +1176,10 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) endif enddo + + do n = 1, ngrids ! add the transfer attributes from importState to grid - call ESMF_AttributeAdd(wrtgrid, convention="NetCDF", purpose="FV3", & + call ESMF_AttributeAdd(wrtGrid(n), convention="NetCDF", purpose="FV3", & attrList=attNameList(1:j-1), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -1125,7 +1205,7 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) if(lprnt) print *,'in write grid comp, new time:unit=',trim(valueS) endif endif - call ESMF_AttributeSet(wrtgrid, convention="NetCDF", purpose="FV3", & + call ESMF_AttributeSet(wrtGrid(n), convention="NetCDF", purpose="FV3", & name=trim(attNameList(i)), value=valueS, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -1137,7 +1217,7 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_AttributeSet(wrtgrid, convention="NetCDF", purpose="FV3", & + call ESMF_AttributeSet(wrtGrid(n), convention="NetCDF", purpose="FV3", & name=trim(attNameList(i)), value=valueI4, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -1149,7 +1229,7 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_AttributeSet(wrtgrid, convention="NetCDF", purpose="FV3", & + call ESMF_AttributeSet(wrtGrid(n), convention="NetCDF", purpose="FV3", & name=trim(attNameList(i)), value=valueR4, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -1161,7 +1241,7 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_AttributeSet(wrtgrid, convention="NetCDF", purpose="FV3", & + call ESMF_AttributeSet(wrtGrid(n), convention="NetCDF", purpose="FV3", & name=trim(attNameList(i)), value=valueR8, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -1170,17 +1250,16 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) ! Add special attribute that holds names of "time" related attributes ! for faster access during Run(). - call ESMF_AttributeAdd(wrtgrid, convention="NetCDF", purpose="FV3", & + call ESMF_AttributeAdd(wrtGrid(n), convention="NetCDF", purpose="FV3", & attrList=(/"TimeAttributes"/), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_AttributeSet(wrtgrid, convention="NetCDF", purpose="FV3", & + call ESMF_AttributeSet(wrtGrid(n), convention="NetCDF", purpose="FV3", & name="TimeAttributes", valueList=attNameList2(1:k-1), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - deallocate(attNameList, attNameList2, typekindList) ! !*** create temporary field bundle for axes information @@ -1190,20 +1269,14 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_AttributeGet(wrtGrid, convention="NetCDF", purpose="FV3", & - name="ESMF:gridded_dim_labels", valueList=attrValueSList, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_GridGetCoord(wrtGrid, coordDim=1, & + call ESMF_GridGetCoord(wrtGrid(n), coordDim=1, & staggerloc=ESMF_STAGGERLOC_CENTER, array=array, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return -! write(0,*) 'create gridFB,fieldname=',trim(attrValueSList(1)),trim(attrValueSList(2)), & -! 'lon value=',array(1:5) - - field = ESMF_FieldCreate(wrtGrid, array, name=trim(attrValueSList(1)), rc=rc) + field = ESMF_FieldCreate(wrtGrid(n), array, name="grid_xt", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -1245,15 +1318,12 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! ! get 2nd dimension - call ESMF_GridGetCoord(wrtGrid, coordDim=2, & + call ESMF_GridGetCoord(wrtGrid(n), coordDim=2, & staggerloc=ESMF_STAGGERLOC_CENTER, array=array, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return -! write(0,*) 'create gridFB,fieldname=',trim(attrValueSList(1)),trim(attrValueSList(2)), & -! 'lat value=',array(1:5,1),array(1,1:5) - - field = ESMF_FieldCreate(wrtGrid, array, name=trim(attrValueSList(2)), rc=rc) + field = ESMF_FieldCreate(wrtGrid(n), array, name="grid_yt", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return !add attribute info @@ -1292,6 +1362,10 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) call ESMF_FieldBundleAdd(gridFB, (/field/), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + end do ! n=1, ngrids + + deallocate(attNameList, attNameList2, typekindList) ! !----------------------------------------------------------------------- !*** SET THE FIRST HISTORY FILE'S TIME INDEX. @@ -1306,17 +1380,9 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) call ESMF_LogWrite("before initialize for POST", ESMF_LOGMSG_INFO, rc=rc) if (lprnt) print *,'in wrt grid comp, dopost=',wrt_int_state%write_dopost if( wrt_int_state%write_dopost ) then - call inline_post_getattr(wrt_int_state) + call inline_post_getattr(wrt_int_state,1) endif ! -!----------------------------------------------------------------------- -! - IF(RC /= ESMF_SUCCESS) THEN - WRITE(0,*)"FAIL: Write_Initialize." -! ELSE -! WRITE(0,*)"PASS: Write_Initialize." - ENDIF -! ! write_init_tim = MPI_Wtime() - btim0 ! !----------------------------------------------------------------------- @@ -1344,9 +1410,7 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) TYPE(ESMF_VM) :: VM type(ESMF_FieldBundle) :: file_bundle type(ESMF_Time) :: currtime - type(ESMF_TypeKind_Flag) :: datatype - type(ESMF_Field) :: field_work - type(ESMF_Grid) :: fbgrid, wrtgrid + type(ESMF_Grid) :: fbgrid, wrtGrid type(ESMF_State),save :: stateGridFB type(optimizeT), save :: optimize(4) type(ESMF_GridComp), save, allocatable :: compsGridFB(:) @@ -1354,34 +1418,27 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) type(write_wrap) :: wrap type(wrt_internal_state),pointer :: wrt_int_state ! - integer :: i,j,n,mype,nolog + integer :: i,j,n,mype,nolog, grid_id ! integer :: nf_hours,nf_seconds, nf_minutes, & nseconds,nseconds_num,nseconds_den ! - integer :: id - integer :: nbdl, idx, date(6), ndig + integer :: nbdl, date(6), ndig, nnnn integer :: step=1 ! logical :: opened logical :: lmask_fields - logical,save :: first=.true. - logical,save :: file_first=.true. ! - character(esmf_maxstr) :: filename,compname,bundle_name + character(esmf_maxstr) :: filename,compname character(40) :: cfhour, cform real(ESMF_KIND_R8) :: time ! - real(kind=8) :: wait_time, MPI_Wtime - real(kind=8) :: times,times2,etim - character(10) :: timeb - real(kind=8) :: tbeg,tend + real(kind=8) :: MPI_Wtime + real(kind=8) :: tbeg real(kind=8) :: wbeg,wend - real(kind=ESMF_KIND_R8), dimension(:,:,:), pointer :: datar8 - real(kind=ESMF_KIND_R8), dimension(:,:), pointer :: datar82d -! - logical lprnt + logical :: use_parallel_netcdf + logical :: lprnt ! !----------------------------------------------------------------------- !*********************************************************************** @@ -1391,16 +1448,11 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) rc = esmf_success ! !----------------------------------------------------------------------- -!*** get the current write grid comp name, id, and internal state +!*** get the current write grid comp name, and internal state ! call ESMF_GridCompGet(wrt_comp, name=compname, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return -! print *,'in wrt run. compname=',trim(compname),' rc=',rc - -! instance id from name - read(compname(10:11),"(I2)") id - ! Provide log message indicating which wrtComp is active call ESMF_LogWrite("Write component activated: "//trim(compname), & ESMF_LOGMSG_INFO, rc=rc) @@ -1456,7 +1508,7 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) nf_seconds = nf_hours*3600+nf_minuteS*60+nseconds+real(nseconds_num)/real(nseconds_den) wrt_int_state%nfhour = nf_seconds/3600. nf_hours = int(nf_seconds/3600.) - if(mype == lead_write_task) print *,'in write grid comp, nf_hours=',nf_hours + if(lprnt) print *,'in write grid comp, nf_hours=',nf_hours ! if iau_offset > nf_hours, don't write out anything if (nf_hours < 0) return @@ -1480,25 +1532,20 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) ! ' nf_seconds=',nf_seconds,wrt_int_state%nfhour ! access the time Attribute which is updated by the driver each time - call ESMF_LogWrite("before Write component get time", ESMF_LOGMSG_INFO, rc=rc) call ESMF_AttributeGet(imp_state_write, convention="NetCDF", purpose="FV3", & name="time", value=time, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_LogWrite("before Write component af get time", ESMF_LOGMSG_INFO, rc=rc) ! !----------------------------------------------------------------------- !*** loop on the files that need to write out !----------------------------------------------------------------------- do i=1, FBCount - call ESMF_LogWrite("before Write component get mirror file bundle", ESMF_LOGMSG_INFO, rc=rc) call ESMF_StateGet(imp_state_write, itemName="mirror_"//trim(fcstItemNameList(i)), & fieldbundle=file_bundle, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_LogWrite("before Write component af get mirror file bundle", ESMF_LOGMSG_INFO, rc=rc) + !recover fields from cartesian vector and sfc pressure call recover_fields(file_bundle,rc) enddo @@ -1510,26 +1557,25 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) if( wrt_int_state%write_dopost ) then ! wbeg = MPI_Wtime() - if (trim(output_grid) == 'regional_latlon' .or. & - trim(output_grid) == 'rotated_latlon' .or. & - trim(output_grid) == 'lambert_conformal') then + if (trim(output_grid(1)) == 'regional_latlon' .or. & + trim(output_grid(1)) == 'rotated_latlon' .or. & + trim(output_grid(1)) == 'lambert_conformal') then !mask fields according to sfc pressure do nbdl=1, wrt_int_state%FBCount - call ESMF_LogWrite("before mask_fields for wrt field bundle", ESMF_LOGMSG_INFO, rc=rc) call mask_fields(wrt_int_state%wrtFB(nbdl),rc) - call ESMF_LogWrite("after mask_fields for wrt field bundle", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return enddo lmask_fields = .true. endif - call inline_post_run(wrt_int_state, mype, wrt_mpi_comm, lead_write_task, & + call inline_post_run(wrt_int_state, 1, mype, wrt_mpi_comm, lead_write_task, & nf_hours, nf_minutes,nseconds) wend = MPI_Wtime() if (lprnt) then write(*,'(A,F10.5,A,I4.2,A,I2.2)')' actual inline post Time is ',wend-wbeg & ,' at Fcst ',nf_hours,':',nf_minutes - endif + endif endif ! @@ -1541,52 +1587,60 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) file_loop_all: do nbdl=1, wrt_int_state%FBCount ! + ! get grid_id + call ESMF_AttributeGet(wrt_int_state%wrtFB(nbdl), convention="NetCDF", purpose="FV3", & + name="grid_id", value=grid_id, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if(step == 1) then file_bundle = wrt_int_state%wrtFB(nbdl) endif + ! FIXME map nbdl to [1:num_files], only used for output_file + nnnn = mod(nbdl-1, num_files) + 1 + ! set default chunksizes for netcdf output ! (use MPI decomposition size). ! if chunksize parameter set to negative value, ! netcdf library default is used. - if (output_file(nbdl)(1:6) == 'netcdf') then - if (ichunk2d == 0) then + if (output_file(nnnn)(1:6) == 'netcdf') then + if (ichunk2d(grid_id) == 0) then if( wrt_int_state%mype == 0 ) & - ichunk2d = wrt_int_state%lon_end-wrt_int_state%lon_start+1 - call mpi_bcast(ichunk2d,1,mpi_integer,0,wrt_mpi_comm,rc) + ichunk2d(grid_id) = wrt_int_state%lon_end-wrt_int_state%lon_start+1 + call mpi_bcast(ichunk2d(grid_id),1,mpi_integer,0,wrt_mpi_comm,rc) endif - if (jchunk2d == 0) then + if (jchunk2d(grid_id) == 0) then if( wrt_int_state%mype == 0 ) & - jchunk2d = wrt_int_state%lat_end-wrt_int_state%lat_start+1 - call mpi_bcast(jchunk2d,1,mpi_integer,0,wrt_mpi_comm,rc) + jchunk2d(grid_id) = wrt_int_state%lat_end-wrt_int_state%lat_start+1 + call mpi_bcast(jchunk2d(grid_id),1,mpi_integer,0,wrt_mpi_comm,rc) endif - if (ichunk3d == 0) then + if (ichunk3d(grid_id) == 0) then if( wrt_int_state%mype == 0 ) & - ichunk3d = wrt_int_state%lon_end-wrt_int_state%lon_start+1 - call mpi_bcast(ichunk3d,1,mpi_integer,0,wrt_mpi_comm,rc) + ichunk3d(grid_id) = wrt_int_state%lon_end-wrt_int_state%lon_start+1 + call mpi_bcast(ichunk3d(grid_id),1,mpi_integer,0,wrt_mpi_comm,rc) endif - if (jchunk3d == 0) then + if (jchunk3d(grid_id) == 0) then if( wrt_int_state%mype == 0 ) & - jchunk3d = wrt_int_state%lat_end-wrt_int_state%lat_start+1 - call mpi_bcast(jchunk3d,1,mpi_integer,0,wrt_mpi_comm,rc) + jchunk3d(grid_id) = wrt_int_state%lat_end-wrt_int_state%lat_start+1 + call mpi_bcast(jchunk3d(grid_id),1,mpi_integer,0,wrt_mpi_comm,rc) endif - if (kchunk3d == 0 .and. nbdl == 1) then + if (kchunk3d(grid_id) == 0 .and. nbdl == 1) then if( wrt_int_state%mype == 0 ) then - call ESMF_FieldBundleGet(wrt_int_state%wrtFB(nbdl), grid=wrtgrid) - call ESMF_AttributeGet(wrtgrid, convention="NetCDF", purpose="FV3", & + call ESMF_FieldBundleGet(wrt_int_state%wrtFB(nbdl), grid=wrtGrid) + call ESMF_AttributeGet(wrtGrid, convention="NetCDF", purpose="FV3", & attnestflag=ESMF_ATTNEST_OFF, name='pfull', & - itemCount=kchunk3d, rc=rc) + itemCount=kchunk3d(grid_id), rc=rc) endif - call mpi_bcast(kchunk3d,1,mpi_integer,0,wrt_mpi_comm,rc) + call mpi_bcast(kchunk3d(grid_id),1,mpi_integer,0,wrt_mpi_comm,rc) endif if (wrt_int_state%mype == 0) then - print *,'ichunk2d,jchunk2d',ichunk2d,jchunk2d - print *,'ichunk3d,jchunk3d,kchunk3d',ichunk3d,jchunk3d,kchunk3d + print *,'ichunk2d,jchunk2d',ichunk2d(grid_id),jchunk2d(grid_id) + print *,'ichunk3d,jchunk3d,kchunk3d',ichunk3d(grid_id),jchunk3d(grid_id),kchunk3d(grid_id) endif endif filename = trim(wrt_int_state%wrtFB_names(nbdl))//'f'//trim(cfhour)//'.nc' -! if(mype == lead_write_task) print *,'in wrt run,filename=',trim(filename) + if(mype == lead_write_task) print *,'in wrt run,filename= ',nbdl,trim(filename) ! ! set the time Attribute on the grid to carry it into the lower levels @@ -1605,189 +1659,86 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - if (trim(output_grid) == 'cubed_sphere_grid') then + if (trim(output_file(nnnn)) == 'netcdf') then + use_parallel_netcdf = .false. + else if (trim(output_file(nnnn)) == 'netcdf_parallel') then + use_parallel_netcdf = .true. + else + call ESMF_LogWrite("wrt_run: Unknown output_file",ESMF_LOGMSG_ERROR,rc=RC) + call ESMF_Finalize(endflag=ESMF_END_ABORT) + endif + + if (trim(output_grid(grid_id)) == 'cubed_sphere_grid') then wbeg = MPI_Wtime() - if (trim(output_file(nbdl)) == 'netcdf_parallel') then + if (trim(output_file(nnnn)) == 'netcdf_parallel') then call write_netcdf(wrt_int_state%wrtFB(nbdl),trim(filename), & .true., wrt_mpi_comm,wrt_int_state%mype, & - ichunk2d,jchunk2d,ichunk3d,jchunk3d,kchunk3d,rc) + grid_id,rc) else - call ESMFproto_FieldBundleWrite(gridFB, filename=trim(filename), & - convention="NetCDF", purpose="FV3", & - status=ESMF_FILESTATUS_REPLACE, & - state=stateGridFB, comps=compsGridFB,rc=rc) + call ESMFproto_FieldBundleWrite(gridFB, filename=trim(filename), & + convention="NetCDF", purpose="FV3", & + status=ESMF_FILESTATUS_REPLACE, & + state=stateGridFB, comps=compsGridFB,rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMFproto_FieldBundleWrite(wrt_int_state%wrtFB(nbdl), & - filename=trim(filename), convention="NetCDF", & - purpose="FV3", status=ESMF_FILESTATUS_OLD, & - timeslice=step, state=optimize(nbdl)%state, & - comps=optimize(nbdl)%comps, rc=rc) + call ESMFproto_FieldBundleWrite(wrt_int_state%wrtFB(nbdl), & + filename=trim(filename), convention="NetCDF", & + purpose="FV3", status=ESMF_FILESTATUS_OLD, & + timeslice=step, state=optimize(nbdl)%state, & + comps=optimize(nbdl)%comps, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return end if wend = MPI_Wtime() if (lprnt) then - write(*,'(A,F10.5,A,I4.2,A,I2.2)')' actual netcdf Write Time is ',wend-wbeg & + write(*,'(A15,A,F10.5,A,I4.2,A,I2.2,1X,A)')trim(output_file(nnnn)),' Write Time is ',wend-wbeg & ,' at Fcst ',NF_HOURS,':',NF_MINUTES endif - else if (trim(output_grid) == 'gaussian_grid') then - - if (trim(output_file(nbdl)) == 'netcdf') then - - wbeg = MPI_Wtime() - call write_netcdf(wrt_int_state%wrtFB(nbdl),trim(filename), & - .false., wrt_mpi_comm,wrt_int_state%mype, & - ichunk2d,jchunk2d,ichunk3d,jchunk3d,kchunk3d,rc) - wend = MPI_Wtime() - if (lprnt) then - write(*,'(A,F10.5,A,I4.2,A,I2.2)')' netcdf Write Time is ',wend-wbeg & - ,' at Fcst ',NF_HOURS,':',NF_MINUTES - endif - - else if (trim(output_file(nbdl)) == 'netcdf_parallel') then - -#ifdef NO_PARALLEL_NETCDF - rc = ESMF_RC_NOT_IMPL - print *,'netcdf_parallel not available on this machine' - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, file=__FILE__)) return -#endif - wbeg = MPI_Wtime() - call write_netcdf(wrt_int_state%wrtFB(nbdl),trim(filename), & - .true., wrt_mpi_comm,wrt_int_state%mype, & - ichunk2d,jchunk2d,ichunk3d,jchunk3d,kchunk3d,rc) - wend = MPI_Wtime() - if (lprnt) then - write(*,'(A,F10.5,A,I4.2,A,I2.2)')' parallel netcdf Write Time is ',wend-wbeg & - ,' at Fcst ',NF_HOURS,':',NF_MINUTES - endif - - else if (trim(output_file(nbdl)) == 'netcdf_esmf') then - - wbeg = MPI_Wtime() - call ESMFproto_FieldBundleWrite(gridFB, filename=trim(filename), & - convention="NetCDF", purpose="FV3", & - status=ESMF_FILESTATUS_REPLACE, state=stateGridFB, comps=compsGridFB,rc=rc) - - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMFproto_FieldBundleWrite(wrt_int_state%wrtFB(nbdl), & - filename=trim(filename), convention="NetCDF", & - purpose="FV3", status=ESMF_FILESTATUS_OLD, & - timeslice=step, state=optimize(nbdl)%state, & - comps=optimize(nbdl)%comps, rc=rc) - - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - wend = MPI_Wtime() - if (lprnt) then - write(*,'(A,F10.5,A,I4.2,A,I2.2)')' netcdf_esmf Write Time is ',wend-wbeg & - ,' at Fcst ',NF_HOURS,':',NF_MINUTES - endif - endif - - else if (trim(output_grid) == 'global_latlon') then - - if (trim(output_file(nbdl)) == 'netcdf') then - - wbeg = MPI_Wtime() - call write_netcdf(wrt_int_state%wrtFB(nbdl),trim(filename), & - .false., wrt_mpi_comm,wrt_int_state%mype, & - ichunk2d,jchunk2d,ichunk3d,jchunk3d,kchunk3d,rc) - wend = MPI_Wtime() - if (lprnt) then - write(*,'(A,F10.5,A,I4.2,A,I2.2)')' netcdf Write Time is ',wend-wbeg & - ,' at Fcst ',NF_HOURS,':',NF_MINUTES - endif - - else if (trim(output_file(nbdl)) == 'netcdf_parallel') then - -#ifdef NO_PARALLEL_NETCDF - rc = ESMF_RC_NOT_IMPL - print *,'netcdf_parallel not available on this machine' - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, file=__FILE__)) return -#endif - wbeg = MPI_Wtime() - call write_netcdf(wrt_int_state%wrtFB(nbdl),trim(filename), & - .true., wrt_mpi_comm,wrt_int_state%mype, & - ichunk2d,jchunk2d,ichunk3d,jchunk3d,kchunk3d,rc) - wend = MPI_Wtime() - if (lprnt) then - write(*,'(A,F10.5,A,I4.2,A,I2.2)')' parallel netcdf Write Time is ',wend-wbeg & - ,' at Fcst ',NF_HOURS,':',NF_MINUTES - endif - - else ! unknown output_file - - call ESMF_LogWrite("wrt_run: Unknown output_file",ESMF_LOGMSG_ERROR,rc=RC) - call ESMF_Finalize(endflag=ESMF_END_ABORT) + else if (trim(output_grid(grid_id)) == 'gaussian_grid' .or. & + trim(output_grid(grid_id)) == 'global_latlon') then + wbeg = MPI_Wtime() + call write_netcdf(wrt_int_state%wrtFB(nbdl),trim(filename), & + use_parallel_netcdf, wrt_mpi_comm,wrt_int_state%mype, & + grid_id,rc) + wend = MPI_Wtime() + if (lprnt) then + write(*,'(A15,A,F10.5,A,I4.2,A,I2.2,1X,A)')trim(output_file(nnnn)),' Write Time is ',wend-wbeg & + ,' at Fcst ',NF_HOURS,':',NF_MINUTES endif - else if (trim(output_grid) == 'regional_latlon' .or. & - trim(output_grid) == 'rotated_latlon' .or. & - trim(output_grid) == 'lambert_conformal') then + else if (trim(output_grid(grid_id)) == 'regional_latlon' .or. & + trim(output_grid(grid_id)) == 'rotated_latlon' .or. & + trim(output_grid(grid_id)) == 'lambert_conformal') then !mask fields according to sfc pressure - !if (mype == lead_write_task) print *,'before mask_fields' if( .not. lmask_fields ) then wbeg = MPI_Wtime() - call ESMF_LogWrite("before mask_fields for wrt field bundle", ESMF_LOGMSG_INFO, rc=rc) - !call mask_fields(wrt_int_state%wrtFB(nbdl),rc) call mask_fields(file_bundle,rc) - !if (mype == lead_write_task) print *,'after mask_fields' - call ESMF_LogWrite("after mask_fields for wrt field bundle", ESMF_LOGMSG_INFO, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return wend = MPI_Wtime() - if (mype == lead_write_task) then + if (lprnt) then write(*,'(A,F10.5,A,I4.2,A,I2.2)')' mask_fields time is ',wend-wbeg endif endif - if (trim(output_file(nbdl)) == 'netcdf' .and. nbits==0) then - - wbeg = MPI_Wtime() - call write_netcdf(wrt_int_state%wrtFB(nbdl),trim(filename), & - .false., wrt_mpi_comm,wrt_int_state%mype, & - ichunk2d,jchunk2d,ichunk3d,jchunk3d,kchunk3d,rc) - wend = MPI_Wtime() - if (mype == lead_write_task) then - write(*,'(A,F10.5,A,I4.2,A,I2.2)')' netcdf Write Time is ',wend-wbeg & - ,' at Fcst ',NF_HOURS,':',NF_MINUTES - endif - - else if (trim(output_file(nbdl)) == 'netcdf_parallel' .and. nbits==0) then - -#ifdef NO_PARALLEL_NETCDF - rc = ESMF_RC_NOT_IMPL - print *,'netcdf_parallel not available on this machine' - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, file=__FILE__)) return -#endif - wbeg = MPI_Wtime() - call write_netcdf(wrt_int_state%wrtFB(nbdl),trim(filename), & - .true., wrt_mpi_comm,wrt_int_state%mype, & - ichunk2d,jchunk2d,ichunk3d,jchunk3d,kchunk3d,rc) - wend = MPI_Wtime() - if (lprnt) then - write(*,'(A,F10.5,A,I4.2,A,I2.2)')' parallel netcdf Write Time is ',wend-wbeg & - ,' at Fcst ',NF_HOURS,':',NF_MINUTES - endif - else ! unknown output_file - - if( nbits /= 0) then - call ESMF_LogWrite("wrt_run: lossy compression is not supported for regional grids",ESMF_LOGMSG_ERROR,rc=RC) - call ESMF_Finalize(endflag=ESMF_END_ABORT) - else - call ESMF_LogWrite("wrt_run: Unknown output_file",ESMF_LOGMSG_ERROR,rc=RC) - call ESMF_Finalize(endflag=ESMF_END_ABORT) - endif + if (nbits(grid_id) /= 0) then + call ESMF_LogWrite("wrt_run: lossy compression is not supported for regional grids",ESMF_LOGMSG_ERROR,rc=RC) + call ESMF_Finalize(endflag=ESMF_END_ABORT) + end if + wbeg = MPI_Wtime() + call write_netcdf(wrt_int_state%wrtFB(nbdl),trim(filename), & + use_parallel_netcdf, wrt_mpi_comm,wrt_int_state%mype, & + grid_id,rc) + wend = MPI_Wtime() + if (lprnt) then + write(*,'(A15,A,F10.5,A,I4.2,A,I2.2,1X,A)')trim(output_file(nnnn)),' Write Time is ',wend-wbeg & + ,' at Fcst ',NF_HOURS,':',NF_MINUTES endif else ! unknown output_grid @@ -1804,7 +1755,7 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) ! !** write out log file ! - if(mype == lead_write_task) then + if (mype == lead_write_task) then do n=701,900 inquire(n,opened=OPENED) if(.not.opened)then @@ -1823,6 +1774,7 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) !----------------------------------------------------------------------- ! call ESMF_VMBarrier(VM, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! write_run_tim = MPI_Wtime() - tbeg ! @@ -1830,12 +1782,6 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) WRITE(*,'(A,F10.5,A,I4.2,A,I2.2)')' total Write Time is ',write_run_tim & ,' at Fcst ',NF_HOURS,':',NF_MINUTES ENDIF -! - IF(RC /= ESMF_SUCCESS) THEN - WRITE(0,*)"FAIL: WRITE_RUN" -! ELSE -! WRITE(0,*)"PASS: WRITE_RUN" - ENDIF ! !----------------------------------------------------------------------- ! @@ -1878,21 +1824,14 @@ subroutine wrt_finalize(wrt_comp, imp_state_write, exp_state_write, clock, rc) !----------------------------------------------------------------------- ! call ESMF_GridCompGetInternalState(wrt_comp, wrap, rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + deallocate(wrap%write_int_state,stat=stat) -! if (ESMF_LogFoundDeallocError(statusToCheck=stat, & msg="Deallocation of internal state memory failed.", & line=__LINE__, file=__FILE__)) return ! !----------------------------------------------------------------------- -! - IF(RC /= ESMF_SUCCESS)THEN - WRITE(0,*)'FAIL: Write_Finalize.' -! ELSE -! WRITE(0,*)'PASS: Write_Finalize.' - ENDIF -! -!----------------------------------------------------------------------- ! end subroutine wrt_finalize ! @@ -1903,8 +1842,12 @@ subroutine recover_fields(file_bundle,rc) type(ESMF_FieldBundle), intent(in) :: file_bundle integer, intent(out), optional :: rc ! + real, parameter :: rdgas = 287.04, grav = 9.80 + real, parameter :: stndrd_atmos_ps = 101325. + real, parameter :: stndrd_atmos_lapse = 0.0065 + integer i,j,k,ifld,fieldCount,nstt,nend,fieldDimCount,gridDimCount - integer istart,iend,jstart,jend,kstart,kend,km + integer istart,iend,jstart,jend,kstart,kend logical uPresent, vPresent type(ESMF_Grid) fieldGrid type(ESMF_Field) ufield, vfield @@ -1918,68 +1861,63 @@ subroutine recover_fields(file_bundle,rc) real(ESMF_KIND_R4), dimension(:,:,:), pointer :: uwind3dr4,vwind3dr4 real(ESMF_KIND_R4), dimension(:,:,:), pointer :: cart3dPtr2dr4 real(ESMF_KIND_R4), dimension(:,:,:,:), pointer :: cart3dPtr3dr4 - real(ESMF_KIND_R8), dimension(:,:,:,:), pointer :: cart3dPtr3dr8 - save lonloc, latloc real(ESMF_KIND_R8) :: coslon, sinlon, sinlat ! ! get filed count call ESMF_FieldBundleGet(file_bundle, fieldCount=fieldCount, & grid=fieldGrid, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! - CALL ESMF_LogWrite("call recover field on wrt comp",ESMF_LOGMSG_INFO,rc=RC) + call ESMF_LogWrite("call recover field on wrt comp",ESMF_LOGMSG_INFO,rc=RC) call ESMF_GridGet(fieldgrid, dimCount=gridDimCount, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - if( first_getlatlon ) then - CALL ESMF_LogWrite("call recover field get coord 1",ESMF_LOGMSG_INFO,rc=RC) + call ESMF_LogWrite("call recover field get coord 1",ESMF_LOGMSG_INFO,rc=RC) - call ESMF_GridGetCoord(fieldgrid, coordDim=1, farrayPtr=lon, rc=rc) + call ESMF_GridGetCoord(fieldgrid, coordDim=1, farrayPtr=lon, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - allocate(lonloc(lbound(lon,1):ubound(lon,1),lbound(lon,2):ubound(lon,2))) - istart = lbound(lon,1) - iend = ubound(lon,1) - jstart = lbound(lon,2) - jend = ubound(lon,2) + allocate(lonloc(lbound(lon,1):ubound(lon,1),lbound(lon,2):ubound(lon,2))) + istart = lbound(lon,1) + iend = ubound(lon,1) + jstart = lbound(lon,2) + jend = ubound(lon,2) !$omp parallel do default(none) shared(lon,lonloc,jstart,jend,istart,iend) & !$omp private(i,j) - do j=jstart,jend - do i=istart,iend - lonloc(i,j) = lon(i,j) * pi/180. - enddo - enddo + do j=jstart,jend + do i=istart,iend + lonloc(i,j) = lon(i,j) * pi/180. + enddo + enddo - CALL ESMF_LogWrite("call recover field get coord 2",ESMF_LOGMSG_INFO,rc=RC) + call ESMF_LogWrite("call recover field get coord 2",ESMF_LOGMSG_INFO,rc=RC) - call ESMF_GridGetCoord(fieldgrid, coordDim=2, farrayPtr=lat, rc=rc) + call ESMF_GridGetCoord(fieldgrid, coordDim=2, farrayPtr=lat, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - allocate(latloc(lbound(lat,1):ubound(lat,1),lbound(lat,2):ubound(lat,2))) - istart = lbound(lat,1) - iend = ubound(lat,1) - jstart = lbound(lat,2) - jend = ubound(lat,2) + allocate(latloc(lbound(lat,1):ubound(lat,1),lbound(lat,2):ubound(lat,2))) + istart = lbound(lat,1) + iend = ubound(lat,1) + jstart = lbound(lat,2) + jend = ubound(lat,2) !$omp parallel do default(none) shared(lat,latloc,jstart,jend,istart,iend) & !$omp private(i,j) - do j=jstart,jend - do i=istart,iend - latloc(i,j) = lat(i,j) * pi/180.d0 - enddo - enddo - first_getlatlon = .false. - endif + do j=jstart,jend + do i=istart,iend + latloc(i,j) = lat(i,j) * pi/180.d0 + enddo + enddo ! allocate(fcstField(fieldCount)) - CALL ESMF_LogWrite("call recover field get fcstField",ESMF_LOGMSG_INFO,rc=RC) + call ESMF_LogWrite("call recover field get fcstField",ESMF_LOGMSG_INFO,rc=RC) call ESMF_FieldBundleGet(file_bundle, fieldList=fcstField, itemorderflag=ESMF_ITEMORDER_ADDORDER, rc=rc) ! do ifld=1,fieldCount - CALL ESMF_LogWrite("call recover field get fieldname, type dimcount",ESMF_LOGMSG_INFO,rc=RC) + call ESMF_LogWrite("call recover field get fieldname, type dimcount",ESMF_LOGMSG_INFO,rc=RC) call ESMF_FieldGet(fcstField(ifld),name=fieldName,typekind=typekind,dimCount=fieldDimCount, rc=rc) ! convert back wind @@ -1995,7 +1933,7 @@ subroutine recover_fields(file_bundle,rc) endif ! print *,'in get 3D vector wind, uwindname=',trim(uwindname),' v=', trim(vwindname),' fieldname=',trim(fieldname) ! get u , v wind - CALL ESMF_LogWrite("call recover field get u, v field",ESMF_LOGMSG_INFO,rc=RC) + call ESMF_LogWrite("call recover field get u, v field",ESMF_LOGMSG_INFO,rc=RC) call ESMF_FieldBundleGet(file_bundle,trim(uwindname),field=ufield,isPresent=uPresent,rc=rc) call ESMF_FieldBundleGet(file_bundle,trim(vwindname),field=vfield,isPresent=vPresent,rc=rc) if(.not. uPresent .or. .not.vPresent) then @@ -2007,7 +1945,7 @@ subroutine recover_fields(file_bundle,rc) ! get field data if ( typekind == ESMF_TYPEKIND_R4 ) then if( fieldDimCount > gridDimCount+1 ) then - CALL ESMF_LogWrite("call recover field get 3d card wind farray",ESMF_LOGMSG_INFO,rc=RC) + call ESMF_LogWrite("call recover field get 3d card wind farray",ESMF_LOGMSG_INFO,rc=RC) call ESMF_FieldGet(fcstField(ifld), localDe=0, farrayPtr=cart3dPtr3dr4, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return if( ubound(cart3dPtr3dr4,1)-lbound(cart3dPtr3dr4,1)+1/=3) then @@ -2044,11 +1982,11 @@ subroutine recover_fields(file_bundle,rc) enddo else call ESMF_FieldGet(fcstField(ifld), localDe=0, farrayPtr=cart3dPtr2dr4, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return if( ubound(cart3dPtr2dr4,1)-lbound(cart3dPtr2dr4,1)+1 /= 3) then rc=991 - print *,'ERROR, 2D the vector dimension /= 3, rc=',rc - exit + write(0,*) 'ERROR, 2D the vector dimension /= 3, rc=',rc + call ESMF_Finalize(endflag=ESMF_END_ABORT) endif istart = lbound(cart3dPtr2dr4,2) iend = ubound(cart3dPtr2dr4,2) @@ -2105,8 +2043,8 @@ subroutine mask_fields(file_bundle,rc) type(ESMF_FieldBundle), intent(in) :: file_bundle integer, intent(out), optional :: rc ! - integer i,j,k,ifld,fieldCount,nstt,nend,fieldDimCount,gridDimCount - integer istart,iend,jstart,jend,kstart,kend,km + integer i,j,k,ifld,fieldCount,fieldDimCount,gridDimCount + integer istart,iend,jstart,jend,kstart,kend type(ESMF_Grid) fieldGrid type(ESMF_TypeKind_Flag) typekind type(ESMF_TypeKind_Flag) attTypeKind @@ -2123,8 +2061,6 @@ subroutine mask_fields(file_bundle,rc) real(ESMF_KIND_R8) :: missing_value_r8=9.99e20 character(len=ESMF_MAXSTR) :: msg - save maskwrt - call ESMF_LogWrite("call mask field on wrt comp",ESMF_LOGMSG_INFO,rc=RC) ! get fieldCount @@ -2142,8 +2078,6 @@ subroutine mask_fields(file_bundle,rc) call ESMF_FieldBundleGet(file_bundle, fieldList=fcstField, itemorderflag=ESMF_ITEMORDER_ADDORDER, rc=rc) ! generate the maskwrt according to surface pressure - if( first_getmaskwrt ) then - do ifld=1,fieldCount !call ESMF_LogWrite("call mask field get fieldname, type dimcount",ESMF_LOGMSG_INFO,rc=RC) call ESMF_FieldGet(fcstField(ifld),name=fieldName,typekind=typekind,dimCount=fieldDimCount, rc=rc) @@ -2180,9 +2114,6 @@ subroutine mask_fields(file_bundle,rc) exit endif enddo - first_getmaskwrt = .false. - - endif !first_getmaskwrt ! loop to mask all fields according to maskwrt do ifld=1,fieldCount @@ -2202,8 +2133,8 @@ subroutine mask_fields(file_bundle,rc) line=__LINE__, file=__FILE__)) return ! bail out if( ubound(vect4dPtr3dr4,1)-lbound(vect4dPtr3dr4,1)+1/=3 ) then rc=991 - print *,'ERROR, 3D the vector dimension /= 3, rc=',rc - exit + write(0,*) 'ERROR, 3D the vector dimension /= 3, rc=',rc + call ESMF_Finalize(endflag=ESMF_END_ABORT) endif ! Get the _FillValue from the field attribute if exists call ESMF_AttributeGet(fcstField(ifld), convention="NetCDF", purpose="FV3", & @@ -2245,8 +2176,8 @@ subroutine mask_fields(file_bundle,rc) line=__LINE__, file=__FILE__)) return ! bail out if( ubound(vect3dPtr2dr4,1)-lbound(vect3dPtr2dr4,1)+1 /= 3 ) then rc=991 - print *,'ERROR, 2D the vector dimension /= 3, rc=',rc - exit + write(0,*) 'ERROR, 2D the vector dimension /= 3, rc=',rc + call ESMF_Finalize(endflag=ESMF_END_ABORT) endif ! Get the _FillValue from the field attribute if exists call ESMF_AttributeGet(fcstField(ifld), convention="NetCDF", purpose="FV3", & @@ -2356,6 +2287,7 @@ subroutine mask_fields(file_bundle,rc) endif enddo ! + deallocate(maskwrt) deallocate(fcstField) rc = 0 @@ -3384,12 +3316,12 @@ subroutine splat4(idrt,jmax,aslat) 121.737742088d0, 124.879308913d0, 128.020877005d0, 131.162446275d0, & 134.304016638d0, 137.445588020d0, 140.587160352d0, 143.728733573d0, & 146.870307625d0, 150.011882457d0, 153.153458019d0, 156.295034268d0 / - real(8) :: dlt,d1=1.d0 - integer :: jhe,jho,j0=0 + real(8) :: dlt + integer :: jhe,jho ! real(8),parameter :: PI=3.14159265358979d0,C=(1.d0-(2.d0/PI)**2)*0.25d0 real(8),parameter :: C=(1.d0-(2.d0/PI)**2)*0.25d0 real(8) r - integer jh,js,n,j + integer jh,n,j ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ! GAUSSIAN LATITUDES IF(IDRT.EQ.4) THEN @@ -3494,12 +3426,12 @@ subroutine splat8(idrt,jmax,aslat) 121.737742088d0, 124.879308913d0, 128.020877005d0, 131.162446275d0, & 134.304016638d0, 137.445588020d0, 140.587160352d0, 143.728733573d0, & 146.870307625d0, 150.011882457d0, 153.153458019d0, 156.295034268d0 / - real(8) :: dlt,d1=1.d0 - integer(4) :: jhe,jho,j0=0 + real(8) :: dlt + integer(4) :: jhe,jho ! real(8),parameter :: PI=3.14159265358979d0,C=(1.d0-(2.d0/PI)**2)*0.25d0 real(8),parameter :: C=(1.d0-(2.d0/PI)**2)*0.25d0 real(8) r - integer jh,js,n,j + integer jh,n,j ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ! GAUSSIAN LATITUDES IF(IDRT.EQ.4) THEN @@ -3649,7 +3581,7 @@ subroutine lambert(stlat1,stlat2,c_lat,c_lon,glon,glat,x,y,inv) ! inv == 1 (glon,glat) ---> (x,y) lat/lon to grid ! inv == -1 (x,y) ---> (glon,glat) grid to lat/lon - real(ESMF_KIND_R8) :: en,f,rho,rho0, dlon, theta, xp, yp + real(ESMF_KIND_R8) :: en,f,rho,rho0, dlon, theta IF (stlat1 == stlat2) THEN en=sin(stlat1*dtor) @@ -3691,7 +3623,7 @@ subroutine get_outfile(nfl, filename, outfile_name,noutfile) character(*), intent(inout) :: outfile_name(:) integer, intent(inout) :: noutfile - integer :: i,j,n,idx + integer :: i,j,n logical :: found ! noutfile = 0 diff --git a/io/post_regional.F90 b/io/post_regional.F90 index 44ea99b2e..a42e10475 100644 --- a/io/post_regional.F90 +++ b/io/post_regional.F90 @@ -220,7 +220,7 @@ end subroutine post_run_regional ! !----------------------------------------------------------------------- ! - subroutine post_getattr_regional(wrt_int_state) + subroutine post_getattr_regional(wrt_int_state,grid_id) ! use esmf use ctlblk_mod, only: im, jm, mpi_comm_comp,gdsdegr,spval @@ -236,6 +236,7 @@ subroutine post_getattr_regional(wrt_int_state) implicit none ! type(wrt_internal_state),intent(inout) :: wrt_int_state + integer, intent(in) :: grid_id ! ! local variable integer i,j,k,n,kz, attcount, nfb @@ -254,92 +255,92 @@ subroutine post_getattr_regional(wrt_int_state) fldbundle = wrt_int_state%wrtFB(nfb) ! set grid spec: -! if(mype==0) print*,'in post_getattr_lam,output_grid=',trim(output_grid),'nfb=',nfb +! if(mype==0) print*,'in post_getattr_lam,output_grid=',trim(output_grid(grid_id)),'nfb=',nfb ! if(mype==0) print*,'in post_getattr_lam, lon1=',lon1,lon2,lat1,lat2,dlon,dlat gdsdegr = 1000000. - if(trim(output_grid) == 'regional_latlon') then + if(trim(output_grid(grid_id)) == 'regional_latlon') then MAPTYPE=0 gridtype='A' - if( lon1<0 ) then - lonstart = nint((lon1+360.)*gdsdegr) + if( lon1(grid_id)<0 ) then + lonstart = nint((lon1(grid_id)+360.)*gdsdegr) else - lonstart = nint(lon1*gdsdegr) + lonstart = nint(lon1(grid_id)*gdsdegr) endif - if( lon2<0 ) then - lonlast = nint((lon2+360.)*gdsdegr) + if( lon2(grid_id)<0 ) then + lonlast = nint((lon2(grid_id)+360.)*gdsdegr) else - lonlast = nint(lon2*gdsdegr) + lonlast = nint(lon2(grid_id)*gdsdegr) endif - latstart = nint(lat1*gdsdegr) - latlast = nint(lat2*gdsdegr) + latstart = nint(lat1(grid_id)*gdsdegr) + latlast = nint(lat2(grid_id)*gdsdegr) - dxval = dlon*gdsdegr - dyval = dlat*gdsdegr + dxval = dlon(grid_id)*gdsdegr + dyval = dlat(grid_id)*gdsdegr ! if(mype==0) print*,'lonstart,latstart,dyval,dxval', & ! lonstart,lonlast,latstart,latlast,dyval,dxval - else if(trim(output_grid) == 'lambert_conformal') then + else if(trim(output_grid(grid_id)) == 'lambert_conformal') then MAPTYPE=1 GRIDTYPE='A' - if( cen_lon<0 ) then - cenlon = nint((cen_lon+360.)*gdsdegr) + if( cen_lon(grid_id)<0 ) then + cenlon = nint((cen_lon(grid_id)+360.)*gdsdegr) else - cenlon = nint(cen_lon*gdsdegr) + cenlon = nint(cen_lon(grid_id)*gdsdegr) endif - cenlat = cen_lat*gdsdegr - if( lon1<0 ) then - lonstart = nint((lon1+360.)*gdsdegr) + cenlat = cen_lat(grid_id)*gdsdegr + if( lon1(grid_id)<0 ) then + lonstart = nint((lon1(grid_id)+360.)*gdsdegr) else - lonstart = nint(lon1*gdsdegr) + lonstart = nint(lon1(grid_id)*gdsdegr) endif - latstart = nint(lat1*gdsdegr) + latstart = nint(lat1(grid_id)*gdsdegr) - truelat1 = nint(stdlat1*gdsdegr) - truelat2 = nint(stdlat2*gdsdegr) + truelat1 = nint(stdlat1(grid_id)*gdsdegr) + truelat2 = nint(stdlat2(grid_id)*gdsdegr) - if(dxin null() + logical :: top_parent_is_global integer :: num_restart_interval, restart_starttime real,dimension(:),allocatable :: restart_interval + + integer :: urc + type(ESMF_State) :: tempState + type(ESMF_Info) :: info + type(time_type) :: Time_init, Time, Time_step, Time_end, & Time_restart, Time_step_restart type(time_type) :: iautime @@ -441,10 +620,8 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) call mpp_close(unit, MPP_DELETE) endif ! -! !----------------------------------------------------------------------- -!*** create grid for output fields -!*** first try: Create cubed sphere grid from file +!*** create grid for output fields, using FV3 parameters !----------------------------------------------------------------------- ! call mpp_error(NOTE, 'before create fcst grid') @@ -455,157 +632,102 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) call read_data("INPUT/grid_spec.nc", "atm_mosaic_file", gridfile) endif - if (mpp_pe() == mpp_root_pe()) & - write(*, *) 'create fcst grid: mype,regional,nested=',mype,Atmos%regional,Atmos%nested - - ! regional-only without nests - if( Atmos%regional .and. .not. Atmos%nested ) then - - call atmosphere_control_data (isc, iec, jsc, jec, nlev) - - domain = Atmos%domain - fcstNpes = Atmos%layout(1)*Atmos%layout(2) - allocate(isl(fcstNpes), iel(fcstNpes), jsl(fcstNpes), jel(fcstNpes)) - allocate(deBlockList(2,2,fcstNpes)) - call mpp_get_compute_domains(domain,xbegin=isl,xend=iel,ybegin=jsl,yend=jel) - do n=1,fcstNpes - deBlockList(:,1,n) = (/ isl(n),iel(n) /) - deBlockList(:,2,n) = (/ jsl(n),jel(n) /) - end do - delayout = ESMF_DELayoutCreate(petMap=(/(i,i=0,fcstNpes-1)/), rc=rc); ESMF_ERR_ABORT(rc) - distgrid = ESMF_DistGridCreate(minIndex=(/1,1/), & - maxIndex=(/Atmos%mlon,Atmos%mlat/), & - delayout=delayout, & - deBlockList=deBlockList, rc=rc); ESMF_ERR_ABORT(rc) - - fcstGrid = ESMF_GridCreateNoPeriDim(regDecomp=(/Atmos%layout(1),Atmos%layout(2)/), & - minIndex=(/1,1/), & - maxIndex=(/Atmos%mlon,Atmos%mlat/), & - gridAlign=(/-1,-1/), & - decompflag=(/ESMF_DECOMP_SYMMEDGEMAX,ESMF_DECOMP_SYMMEDGEMAX/), & - name="fcst_grid", & - indexflag=ESMF_INDEX_DELOCAL, & - rc=rc); ESMF_ERR_ABORT(rc) - - ! add and define "center" coordinate values - call ESMF_GridAddCoord(fcstGrid, staggerLoc=ESMF_STAGGERLOC_CENTER, rc=rc); ESMF_ERR_ABORT(rc) - call ESMF_GridGetCoord(fcstGrid, coordDim=1, staggerLoc=ESMF_STAGGERLOC_CENTER, & - farrayPtr=glonPtr, rc=rc); ESMF_ERR_ABORT(rc) - call ESMF_GridGetCoord(fcstGrid, coordDim=2, staggerLoc=ESMF_STAGGERLOC_CENTER, & - farrayPtr=glatPtr, rc=rc); ESMF_ERR_ABORT(rc) + ngrids = Atmos%ngrids + mygrid = Atmos%mygrid + allocate(grid_number_on_all_pets(fcst_ntasks)) + call mpi_allgather(mygrid, 1, MPI_INTEGER, & + grid_number_on_all_pets, 1, MPI_INTEGER, & + fcst_mpi_comm, rc) + + allocate (fcstGrid(ngrids),fcstGridComp(ngrids)) + do n=1,ngrids + + pelist => null() + call atmos_model_get_nth_domain_info(n, layout, nx, ny, pelist) + call ESMF_VMBroadcast(vm, bcstData=layout, count=2, rootPet=pelist(1), rc=rc); ESMF_ERR_ABORT(rc) + + if (n==1) then + ! on grid==1 (top level parent) determine if the domain is global or regional + top_parent_is_global = .true. + if(mygrid==1) then + if (Atmos%regional) top_parent_is_global = .false. + endif + call mpi_bcast(top_parent_is_global, 1, MPI_LOGICAL, 0, fcst_mpi_comm, rc) + endif - do j = jsc, jec - do i = isc, iec - glonPtr(i-isc+1,j-jsc+1) = Atmos%lon(i-isc+1,j-jsc+1) * dtor - glatPtr(i-isc+1,j-jsc+1) = Atmos%lat(i-isc+1,j-jsc+1) * dtor - enddo - enddo + if (n==1 .and. top_parent_is_global) then - ! add and define "corner" coordinate values - call ESMF_GridAddCoord(fcstGrid, staggerLoc=ESMF_STAGGERLOC_CORNER, & - rc=rc); ESMF_ERR_ABORT(rc) - call ESMF_GridGetCoord(fcstGrid, coordDim=1, staggerLoc=ESMF_STAGGERLOC_CORNER, & - totalLBound=tlb, totalUBound=tub, & - farrayPtr=glonPtr, rc=rc); ESMF_ERR_ABORT(rc) - glonPtr(tlb(1):tub(1),tlb(2):tub(2)) = & - Atmos%lon_bnd(tlb(1):tub(1),tlb(2):tub(2)) * dtor - call ESMF_GridGetCoord(fcstGrid, coordDim=2, staggerLoc=ESMF_STAGGERLOC_CORNER, & - totalLBound=tlb, totalUBound=tub, & - farrayPtr=glatPtr, rc=rc); ESMF_ERR_ABORT(rc) - glatPtr(tlb(1):tub(1),tlb(2):tub(2)) = & - Atmos%lat_bnd(tlb(1):tub(1),tlb(2):tub(2)) * dtor - - call mpp_error(NOTE, 'after create fcst grid for regional-only') - - else ! not regional only - - if (.not. Atmos%regional .and. .not. Atmos%nested ) then !! global only - - do tl=1,6 - decomptile(1,tl) = Atmos%layout(1) - decomptile(2,tl) = Atmos%layout(2) - decompflagPTile(:,tl) = (/ESMF_DECOMP_SYMMEDGEMAX,ESMF_DECOMP_SYMMEDGEMAX/) - enddo - fcstGrid = ESMF_GridCreateMosaic(filename="INPUT/"//trim(gridfile), & - regDecompPTile=decomptile,tileFilePath="INPUT/", & - decompflagPTile=decompflagPTile, & - staggerlocList=(/ESMF_STAGGERLOC_CENTER, ESMF_STAGGERLOC_CORNER/), & - name='fcst_grid', rc=rc) + fcstGridComp(n) = ESMF_GridCompCreate(name="global", petList=pelist, rc=rc); ESMF_ERR_ABORT(rc) + + call ESMF_InfoGetFromHost(fcstGridComp(n), info=info, rc=rc); ESMF_ERR_ABORT(rc) + call ESMF_InfoSet(info, key="layout", values=layout, rc=rc); ESMF_ERR_ABORT(rc) + call ESMF_InfoSet(info, key="tilesize", value=Atmos%mlon, rc=rc); ESMF_ERR_ABORT(rc) + + call ESMF_GridCompSetServices(fcstGridComp(n), SetServicesNest, userrc=urc, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return - call mpp_error(NOTE, 'after create fcst grid for global-only with INPUT/'//trim(gridfile)) + else - else !! global-nesting or regional-nesting + allocate(petListNest(layout(1)*layout(2))) + k=pelist(1) + do j=1,layout(2) + do i=1,layout(1) + petListNest(k-pelist(1)+1) = k + k = k + 1 + end do + end do - if (mype == 0) TileLayout = Atmos%layout - call ESMF_VMBroadcast(vm, bcstData=TileLayout, count=2, & - rootPet=0, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + fcstGridComp(n) = ESMF_GridCompCreate(name="nest", petList=petListNest, rc=rc); ESMF_ERR_ABORT(rc) + + call ESMF_InfoGetFromHost(fcstGridComp(n), info=info, rc=rc); ESMF_ERR_ABORT(rc) + call ESMF_InfoSet(info, key="layout", values=layout, rc=rc); ESMF_ERR_ABORT(rc) + call ESMF_InfoSet(info, key="nx", value=nx, rc=rc); ESMF_ERR_ABORT(rc) + call ESMF_InfoSet(info, key="ny", value=ny, rc=rc); ESMF_ERR_ABORT(rc) - if (mype == 0) npes(1) = mpp_npes() - call ESMF_VMBroadcast(vm, bcstData=npes, count=1, & - rootPet=0, rc=rc) + call ESMF_GridCompSetServices(fcstGridComp(n), SetServicesNest, userrc=urc, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return - if ( npes(1) == TileLayout(1) * TileLayout(2) * 6 ) then - ! global-nesting - nestRootPet = npes(1) - gridfile="grid.nest02.tile7.nc" - else if ( npes(1) == TileLayout(1) * TileLayout(2) ) then - ! regional-nesting - nestRootPet = npes(1) - gridfile="grid.nest02.tile2.nc" - else - call mpp_error(FATAL, 'Inconsistent nestRootPet and Atmos%layout') - endif - if (mype == nestRootPet) then - if (nestRootPet /= Atmos%pelist(1)) then - write(0,*)'error in fcst_initialize: nestRootPet /= Atmos%pelist(1)' - write(0,*)'error in fcst_initialize: nestRootPet = ',nestRootPet - write(0,*)'error in fcst_initialize: Atmos%pelist(1) = ',Atmos%pelist(1) - ESMF_ERR_ABORT(100) - endif - endif + deallocate(petListNest) - ! nest rootPet shares peList with others - if (mype == nestRootPet) peListSize(1) = size(Atmos%pelist) - call ESMF_VMBroadcast(vm, bcstData=peListSize, count=1, rootPet=nestRootPet, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + end if - ! nest rootPet shares layout with others - if (mype == nestRootPet) regDecomp = Atmos%layout - call ESMF_VMBroadcast(vm, bcstData=regDecomp, count=2, rootPet=nestRootPet, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if (ESMF_GridCompIsPetLocal(fcstGridComp(n), rc=rc)) then + call ESMF_GridCompGet(fcstGridComp(n), grid=fcstGrid(n), rc=rc); ESMF_ERR_ABORT(rc) - ! prepare petMap variable - allocate(petMap(peListSize(1))) - if (mype == nestRootPet) petMap = Atmos%pelist - ! do the actual broadcast of the petMap - call ESMF_VMBroadcast(vm, bcstData=petMap, count=peListSize(1), rootPet=nestRootPet, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + call ESMF_GridAddCoord(fcstGrid(n), staggerLoc=ESMF_STAGGERLOC_CENTER, rc=rc); ESMF_ERR_ABORT(rc) + call ESMF_GridAddCoord(fcstGrid(n), staggerLoc=ESMF_STAGGERLOC_CORNER, rc=rc); ESMF_ERR_ABORT(rc) - ! create the DELayout that maps DEs to the PETs in the petMap - delayout = ESMF_DELayoutCreate(petMap=petMap, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + ! define "center" coordinate values + call ESMF_GridGetCoord(fcstGrid(n), coordDim=1, staggerLoc=ESMF_STAGGERLOC_CENTER, & + totalLBound=tlb, totalUBound=tub, & + farrayPtr=glonPtr, rc=rc); ESMF_ERR_ABORT(rc) + glonPtr(tlb(1):tub(1),tlb(2):tub(2)) = Atmos%lon(tlb(1):tub(1),tlb(2):tub(2)) - ! create the nest Grid by reading it from file but use DELayout - fcstGrid = ESMF_GridCreate(filename="INPUT/"//trim(gridfile), & - fileformat=ESMF_FILEFORMAT_GRIDSPEC, regDecomp=regDecomp, & - decompflag=(/ESMF_DECOMP_SYMMEDGEMAX,ESMF_DECOMP_SYMMEDGEMAX/), & - delayout=delayout, isSphere=.false., indexflag=ESMF_INDEX_DELOCAL, & - rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + call ESMF_GridGetCoord(fcstGrid(n), coordDim=2, staggerLoc=ESMF_STAGGERLOC_CENTER, & + totalLBound=tlb, totalUBound=tub, & + farrayPtr=glatPtr, rc=rc); ESMF_ERR_ABORT(rc) + glatPtr(tlb(1):tub(1),tlb(2):tub(2)) = Atmos%lat(tlb(1):tub(1),tlb(2):tub(2)) - call mpp_error(NOTE, 'after create fcst grid with INPUT/'//trim(gridfile)) + ! define "corner" coordinate values + call ESMF_GridGetCoord(fcstGrid(n), coordDim=1, staggerLoc=ESMF_STAGGERLOC_CORNER, & + totalLBound=tlb, totalUBound=tub, & + farrayPtr=glonPtr, rc=rc); ESMF_ERR_ABORT(rc) + glonPtr(tlb(1):tub(1),tlb(2):tub(2)) = Atmos%lon_bnd(tlb(1):tub(1),tlb(2):tub(2)) - endif + call ESMF_GridGetCoord(fcstGrid(n), coordDim=2, staggerLoc=ESMF_STAGGERLOC_CORNER, & + totalLBound=tlb, totalUBound=tub, & + farrayPtr=glatPtr, rc=rc); ESMF_ERR_ABORT(rc) + glatPtr(tlb(1):tub(1),tlb(2):tub(2)) = Atmos%lat_bnd(tlb(1):tub(1),tlb(2):tub(2)) + end if ! IsPetLocal - endif + end do ! !! FIXME if ( .not. Atmos%nested ) then !! global only - call addLsmask2grid(fcstGrid, rc=rc) + call addLsmask2grid(fcstGrid(mygrid), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! print *,'call addLsmask2grid after fcstGrid, rc=',rc endif @@ -619,7 +741,7 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) ! ! Write grid to netcdf file if( cplprint_flag ) then - call wrt_fcst_grid(fcstGrid, "diagnostic_FV3_fcstGrid.nc", & + call wrt_fcst_grid(fcstGrid(mygrid), "diagnostic_FV3_fcstGrid.nc", & regridArea=.TRUE., rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return endif @@ -633,15 +755,39 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) name="gridfile", value=trim(gridfile), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return +! Add total number of domains(grids) Attribute to the exportState + call ESMF_AttributeAdd(exportState, convention="NetCDF", purpose="FV3", & + attrList=(/"ngrids"/), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_AttributeSet(exportState, convention="NetCDF", purpose="FV3", & + name="ngrids", value=ngrids, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + +! Add top_parent_is_global Attribute to the exportState + call ESMF_AttributeAdd(exportState, convention="NetCDF", purpose="FV3", & + attrList=(/"top_parent_is_global"/), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_AttributeSet(exportState, convention="NetCDF", purpose="FV3", & + name="top_parent_is_global", value=top_parent_is_global, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + ! Add dimension Attributes to Grid - call ESMF_AttributeAdd(fcstGrid, convention="NetCDF", purpose="FV3", & + do n=1,ngrids + if (ESMF_GridCompIsPetLocal(fcstGridComp(n), rc=rc)) then + + call ESMF_AttributeAdd(fcstGrid(n), convention="NetCDF", purpose="FV3", & attrList=(/"ESMF:gridded_dim_labels"/), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_AttributeSet(fcstGrid, convention="NetCDF", purpose="FV3", & + call ESMF_AttributeSet(fcstGrid(n), convention="NetCDF", purpose="FV3", & name="ESMF:gridded_dim_labels", valueList=(/"grid_xt", "grid_yt"/), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return -! + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + endif + end do + ! Add time Attribute to the exportState call ESMF_AttributeAdd(exportState, convention="NetCDF", purpose="FV3", & attrList=(/ "time ", & @@ -690,49 +836,73 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) ! Create FieldBundle for Fields that need to be regridded bilinear if( quilting ) then + allocate(fieldbundle(ngrids)) + nbdlphys = 2 + allocate(fieldbundlephys(nbdlphys,ngrids)) + + do n=1,ngrids + bundle_grid='' + if (ngrids > 1 .and. n >= 2) then + write(bundle_grid,'(A5,I2.2,A1)') '.nest', n, '.' + endif + do i=1,num_files ! - name_FB = filename_base(i) + tempState = ESMF_StateCreate(rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + name_FB = trim(filename_base(i)) // trim(bundle_grid) ! if( i==1 ) then ! for dyn name_FB1 = trim(name_FB)//'_bilinear' - fieldbundle = ESMF_FieldBundleCreate(name=trim(name_FB1),rc=rc) + fieldbundle(n) = ESMF_FieldBundleCreate(name=trim(name_FB1),rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_AttributeAdd(fieldbundle(n), convention="NetCDF", purpose="FV3", & + attrList=(/"grid_id"/), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - if (mype == 0) write(*,*)'af create fcst fieldbundle, name=',trim(name_FB),'rc=',rc - call fv_dyn_bundle_setup(Atmos%axes, & - fieldbundle, fcstGrid, quilting, rc=rc) + call ESMF_AttributeSet(fieldbundle(n), convention="NetCDF", purpose="FV3", & + name="grid_id", value=n, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - ! Add the field to the importState so parent can connect to it - call ESMF_StateAdd(exportState, (/fieldbundle/), rc=rc) + call ESMF_StateAdd(tempState, (/fieldbundle(n)/), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + call ESMF_GridCompInitialize(fcstGridComp(n), importState=tempState,& + exportState=exportState, phase=1, userrc=urc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return + else if( i==2 ) then ! for phys - nbdlphys = 2 - allocate(fieldbundlephys(nbdlphys)) do j=1, nbdlphys if( j==1 ) then name_FB1 = trim(name_FB)//'_nearest_stod' else name_FB1 = trim(name_FB)//'_bilinear' endif - fieldbundlephys(j) = ESMF_FieldBundleCreate(name=trim(name_FB1),rc=rc) + fieldbundlephys(j,n) = ESMF_FieldBundleCreate(name=trim(name_FB1),rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - if (mype == 0) write(*,*)'af create fcst fieldbundle, name=',trim(name_FB1),'rc=',rc - enddo -! - call fv_phys_bundle_setup(Atmos%diag, Atmos%axes, & - fieldbundlephys, fcstGrid, quilting, nbdlphys) -! - ! Add the field to the importState so parent can connect to it - do j=1,nbdlphys - call ESMF_StateAdd(exportState, (/fieldbundlephys(j)/), rc=rc) + + call ESMF_AttributeAdd(fieldbundlephys(j,n), convention="NetCDF", purpose="FV3", & + attrList=(/"grid_id"/), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_AttributeSet(fieldbundlephys(j,n), convention="NetCDF", purpose="FV3", & + name="grid_id", value=n, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_StateAdd(tempState, (/fieldbundlephys(j,n)/), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return enddo + call ESMF_GridCompInitialize(fcstGridComp(n), importState=tempState,& + exportState=exportState, phase=2, userrc=urc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return + else write(0,*)' unknown name_FB ', trim(name_FB) @@ -740,7 +910,11 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) endif ! - enddo + call ESMF_StateDestroy(tempState, noGarbage=.true., rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + enddo ! num_files + enddo ! ngrids !end qulting endif