Merge pull request #12 from climbfuji/update_dtc_develop_from_ncar_ma…

…ster_20200606

Update dtc develop from ncar master 2020/06/06

driver/fvGFS/atmosphere.F90

@@ -270,30 +270,13 @@ module atmosphere_mod
 !! and diagnostics.  
  subroutine atmosphere_init (Time_init, Time, Time_step, Grid_box, area)
 #ifdef CCPP
-#ifdef STATIC
-! For static builds, the ccpp_physics_{init,run,finalize} calls
-! are not pointing to code in the CCPP framework, but to auto-generated
-! ccpp_suite_cap and ccpp_group_*_cap modules behind a ccpp_static_api
-   use ccpp_api,          only: ccpp_init
    use ccpp_static_api,   only: ccpp_physics_init
-#else
-   use iso_c_binding,     only: c_loc
-   use ccpp_api,          only: ccpp_init,           &
-                                ccpp_physics_init,   &
-                                ccpp_field_add,      &
-                                ccpp_error
-#endif
    use CCPP_data,         only: ccpp_suite,          &
                                 cdata => cdata_tile, &
                                 CCPP_interstitial
 #ifdef OPENMP
    use omp_lib
 #endif
-#ifndef STATIC
-! Begin include auto-generated list of modules for ccpp
-#include "ccpp_modules_fast_physics.inc"
-! End include auto-generated list of modules for ccpp
-#endif
 #endif
    type (time_type),    intent(in)    :: Time_init, Time, Time_step
    type(grid_box_type), intent(inout) :: Grid_box
@@ -443,15 +426,8 @@ subroutine atmosphere_init (Time_init, Time, Time_step, Grid_box, area)
 #ifdef CCPP
    ! Do CCPP fast physics initialization before call to adiabatic_init (since this calls fv_dynamics)
 
-   ! Initialize the cdata structure
-   call ccpp_init(trim(ccpp_suite), cdata, ierr)
-   if (ierr/=0) then
-      cdata%errmsg = ' atmosphere_dynamics: error in ccpp_init: ' // trim(cdata%errmsg)
-      call mpp_error (FATAL, cdata%errmsg)
-   end if
-
-   ! For fast physics running over the entire domain, block and thread
-   ! number are not used; set to safe values
+   ! For fast physics running over the entire domain, block
+   ! and thread number are not used; set to safe values
    cdata%blk_no = 1
    cdata%thrd_no = 1
 
@@ -468,6 +444,7 @@ subroutine atmosphere_init (Time_init, Time, Time_step, Grid_box, area)
                                  Atm(mytile)%npz, Atm(mytile)%ng,                                              &
                                  dt_atmos, p_split, Atm(mytile)%flagstruct%k_split,                            &
                                  zvir, Atm(mytile)%flagstruct%p_ref, Atm(mytile)%ak, Atm(mytile)%bk,           &
+                                 liq_wat>0, ice_wat>0, rainwat>0, snowwat>0, graupel>0,                        &
                                  cld_amt>0, kappa, Atm(mytile)%flagstruct%hydrostatic,                         &
                                  Atm(mytile)%flagstruct%do_sat_adj,                                            &
                                  Atm(mytile)%delp, Atm(mytile)%delz, Atm(mytile)%gridstruct%area_64,           &
@@ -487,18 +464,9 @@ subroutine atmosphere_init (Time_init, Time, Time_step, Grid_box, area)
 #endif
                                  mpirank=mpp_pe(), mpiroot=mpp_root_pe())
 
-#ifndef STATIC
-! Populate cdata structure with fields required to run fast physics (auto-generated).
-#include "ccpp_fields_fast_physics.inc"
-#endif
-
    if (Atm(mytile)%flagstruct%do_sat_adj) then
       ! Initialize fast physics
-#ifdef STATIC
       call ccpp_physics_init(cdata, suite_name=trim(ccpp_suite), group_name="fast_physics", ierr=ierr)
-#else
-      call ccpp_physics_init(cdata, group_name="fast_physics", ierr=ierr)
-#endif
       if (ierr/=0) then
          cdata%errmsg = ' atmosphere_dynamics: error in ccpp_physics_init for group fast_physics: ' // trim(cdata%errmsg)
          call mpp_error (FATAL, cdata%errmsg)
@@ -715,15 +683,8 @@ end subroutine atmosphere_dynamics
 !! FV3 dynamical core responsible for writing out a restart and final diagnostic state.
  subroutine atmosphere_end (Time, Grid_box, restart_endfcst)
 #ifdef CCPP
-#ifdef STATIC
-! For static builds, the ccpp_physics_{init,run,finalize} calls
-! are not pointing to code in the CCPP framework, but to auto-generated
-! ccpp_suite_cap and ccpp_group_*_cap modules behind a ccpp_static_api
    use ccpp_static_api,   only: ccpp_physics_finalize
    use CCPP_data,         only: ccpp_suite
-#else
-   use ccpp_api,          only: ccpp_physics_finalize
-#endif
    use CCPP_data,         only: cdata => cdata_tile
 #endif
    type (time_type),      intent(in)    :: Time
@@ -735,11 +696,7 @@ subroutine atmosphere_end (Time, Grid_box, restart_endfcst)
 
    if (Atm(mytile)%flagstruct%do_sat_adj) then
       ! Finalize fast physics
-#ifdef STATIC
       call ccpp_physics_finalize(cdata, suite_name=trim(ccpp_suite), group_name="fast_physics", ierr=ierr)
-#else
-      call ccpp_physics_finalize(cdata, group_name="fast_physics", ierr=ierr)
-#endif
       if (ierr/=0) then
          cdata%errmsg = ' atmosphere_dynamics: error in ccpp_physics_finalize for group fast_physics: ' // trim(cdata%errmsg)
          call mpp_error (FATAL, cdata%errmsg)
@@ -1269,7 +1226,6 @@ subroutine atmosphere_get_bottom_layer (Atm_block, DYCORE_Data)
    rrg  = rdgas / grav
 
    if (first_time) then
-     if (mpp_pe() == mpp_root_pe()) print *, 'calculating slp kr value'
      ! determine 0.8 sigma reference level
      sigtop = Atm(mytile)%ak(1)/pstd_mks+Atm(mytile)%bk(1)
      do k = 1, npz

makefile

@@ -23,7 +23,7 @@ endif
 
 LIBRARY  = libfv3core.a
 
-FFLAGS   += -I$(FMS_DIR) -I../gfsphysics -I../ipd -I../io -I../namphysics
+FFLAGS   += -I$(FMS_DIR) -I../gfsphysics -I../ipd -I../io
 
 SRCS_f   =
 

model/fv_arrays.F90

@@ -974,6 +974,10 @@ module fv_arrays_mod
                              !< Useful for perturbing initial conditions. -1 by default;
                              !< disabled if 0 or negative.
 
+   logical :: butterfly_effect = .false.   !< Flip the least-significant-bit of the lowest level temperature
+                                           !< at the center of the domain (the center of tile 1), if set to .true.
+                                           !< The default value is .false.
+
    integer :: a2b_ord = 4   !< Order of interpolation used by the pressure gradient force
                             !< to interpolate cell-centered (A-grid) values to the grid corners. 
                             !< The default value is 4 (recommended), which uses fourth-order 

model/fv_control.F90

@@ -316,6 +316,7 @@ module fv_control_mod
    logical , pointer :: make_hybrid_z  
    logical , pointer :: nudge_qv
    real,     pointer :: add_noise
+   logical , pointer :: butterfly_effect
 
    integer , pointer :: a2b_ord 
    integer , pointer :: c2l_ord 
@@ -669,7 +670,7 @@ subroutine run_setup(Atm, dt_atmos, grids_on_this_pe, p_split)
                          pnats, dnats, a2b_ord, remap_t, p_ref, d2_bg_k1, d2_bg_k2,  &
                          c2l_ord, dx_const, dy_const, umax, deglat,      &
                          deglon_start, deglon_stop, deglat_start, deglat_stop, &
-                         phys_hydrostatic, use_hydro_pressure, make_hybrid_z, old_divg_damp, add_noise, &
+                         phys_hydrostatic, use_hydro_pressure, make_hybrid_z, old_divg_damp, add_noise, butterfly_effect, &
                          nested, twowaynest, parent_grid_num, parent_tile, nudge_qv, &
                          refinement, nestbctype, nestupdate, nsponge, s_weight, &
                          ioffset, joffset, check_negative, nudge_ic, halo_update_type, gfs_phil, agrid_vel_rst,     &
@@ -1328,6 +1329,7 @@ subroutine setup_pointers(Atm)
      make_hybrid_z                 => Atm%flagstruct%make_hybrid_z
      nudge_qv                      => Atm%flagstruct%nudge_qv
      add_noise                     => Atm%flagstruct%add_noise
+     butterfly_effect              => Atm%flagstruct%butterfly_effect
      a2b_ord                       => Atm%flagstruct%a2b_ord
      c2l_ord                       => Atm%flagstruct%c2l_ord
      ndims                         => Atm%flagstruct%ndims

model/fv_mapz.F90

@@ -95,17 +95,10 @@ module fv_mapz_mod
 #ifndef CCPP
   use fv_cmp_mod,        only: qs_init, fv_sat_adj
 #else
-#ifdef STATIC
-! For static builds, the ccpp_physics_{init,run,finalize} calls
-! are not pointing to code in the CCPP framework, but to auto-generated
-! ccpp_suite_cap and ccpp_group_*_cap modules behind a ccpp_static_api
-  use ccpp_api,          only: ccpp_initialized
   use ccpp_static_api,   only: ccpp_physics_run
   use CCPP_data,         only: ccpp_suite
-#else
-  use ccpp_api,          only: ccpp_initialized, ccpp_physics_run
-#endif
-  use CCPP_data,         only: cdata => cdata_tile, CCPP_interstitial
+  use CCPP_data,         only: cdata => cdata_tile
+  use CCPP_data,         only: CCPP_interstitial
 #endif
 #ifdef MULTI_GASES
   use multi_gases_mod,  only:  virq, virqd, vicpqd, vicvqd, num_gas
@@ -259,7 +252,7 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk,   &
        endif
 
 !$OMP parallel do default(none) shared(is,ie,js,je,km,pe,ptop,kord_tm,hydrostatic, &
-!$OMP                                  pt,pk,rg,peln,q,nwat,liq_wat,rainwat,ice_wat,snowwat,    &
+!$OMP                                  pt,pk,rg,peln,q,nwat,liq_wat,rainwat,ice_wat,snowwat, &
 !$OMP                                  graupel,q_con,sphum,cappa,r_vir,rcp,k1k,delp, &
 !$OMP                                  delz,akap,pkz,te,u,v,ps, gridstruct, last_step, &
 !$OMP                                  ak,bk,nq,isd,ied,jsd,jed,kord_tr,fill, adiabatic, &
@@ -627,9 +620,7 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk,   &
 !$OMP                               ng,gridstruct,E_Flux,pdt,dtmp,reproduce_sum,q,             &
 !$OMP                               mdt,cld_amt,cappa,dtdt,out_dt,rrg,akap,do_sat_adj,         &
 !$OMP                               kord_tm,cdata,CCPP_interstitial)                           &
-#ifdef STATIC
 !$OMP                        shared(ccpp_suite)                                                &
-#endif
 #ifdef MULTI_GASES
 !$OMP                        shared(num_gas)                                                   &
 #endif
@@ -643,9 +634,7 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk,   &
 !$OMP                               ng,gridstruct,E_Flux,pdt,dtmp,reproduce_sum,q,             &
 !$OMP                               mdt,cld_amt,cappa,dtdt,out_dt,rrg,akap,do_sat_adj,         &
 !$OMP                               fast_mp_consv,kord_tm,cdata, CCPP_interstitial)            &
-#ifdef STATIC
 !$OMP                        shared(ccpp_suite)                                                &
-#endif
 #ifdef MULTI_GASES
 !$OMP                        shared(num_gas)                                                   &
 #endif
@@ -808,15 +797,11 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk,   &
   if ( do_sat_adj ) then
                                            call timing_on('sat_adj2')
 #ifdef CCPP
-    if (ccpp_initialized(cdata)) then
-#ifdef STATIC
+    if (cdata%initialized()) then
       call ccpp_physics_run(cdata, suite_name=trim(ccpp_suite), group_name='fast_physics', ierr=ierr)
-#else
-      call ccpp_physics_run(cdata, group_name='fast_physics', ierr=ierr)
-#endif
       if (ierr/=0) call mpp_error(FATAL, "Call to ccpp_physics_run for group 'fast_physics' failed")
     else
-      call mpp_error (FATAL, 'Lagrangian_to_Eulerian: can not call CCPP fast physics because cdata not initialized')
+      call mpp_error (FATAL, 'Lagrangian_to_Eulerian: can not call CCPP fast physics because CCPP not initialized')
     endif
 #else
 !$OMP do

tools/fv_restart.F90

@@ -222,6 +222,8 @@ subroutine fv_restart(fv_domain, Atm, dt_atmos, seconds, days, cold_start, grid_
     integer :: npts
     real    :: sumpertn
 
+    integer :: i_butterfly, j_butterfly
+
     rgrav = 1. / grav
 
     if(.not.module_is_initialized) call mpp_error(FATAL, 'You must call fv_restart_init.')
@@ -627,6 +629,25 @@ subroutine fv_restart(fv_domain, Atm, dt_atmos, seconds, days, cold_start, grid_
         call mpp_error(NOTE, errstring)
      endif
 
+     if (Atm(n)%flagstruct%butterfly_effect) then
+        if (n==1 .and. Atm(n)%tile == 1) then
+           i_butterfly = Atm(n)%npx / 2
+           j_butterfly = Atm(n)%npy / 2
+           if (isc <= i_butterfly .and. i_butterfly <= iec) then
+           if (jsc <= j_butterfly .and. j_butterfly <= jec) then
+
+              write(*,'(A, I0, A, I0)') "Adding butterfly effect at (i,j) ", i_butterfly, ", ", j_butterfly
+              write(*,'(A, E24.17)') "pt (before) :", Atm(n)%pt(i_butterfly,j_butterfly,Atm(n)%npz)
+
+              Atm(n)%pt(i_butterfly,j_butterfly,Atm(n)%npz) = nearest(Atm(n)%pt(i_butterfly,j_butterfly,Atm(n)%npz), -1.0)
+
+              write(*,'(A, E24.17)') "pt (after)  :", Atm(n)%pt(i_butterfly,j_butterfly,Atm(n)%npz)
+
+           endif
+           endif
+        endif
+     endif
+
       if (Atm(n)%grid_number > 1) then
          write(gn,'(A2, I1)') " g", Atm(n)%grid_number
       else