CAPS physics modifications and update to trunk

CMakeLists.txt

@@ -152,6 +152,9 @@ elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "Intel")
     string(REPLACE "-xCORE-AVX2" "-xCORE-AVX-I"
            CMAKE_Fortran_FLAGS_LOPT
            "${CMAKE_Fortran_FLAGS_LOPT}")
+    string(REPLACE "-axSSE4.2,AVX,CORE-AVX2" "-axSSE4.2,AVX,CORE-AVX-I"
+           CMAKE_Fortran_FLAGS_LOPT
+           "${CMAKE_Fortran_FLAGS_LOPT}")
     SET_SOURCE_FILES_PROPERTIES(./physics/radiation_aerosols.f
                                 PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS_LOPT} -r8 -ftz")
     # Force consistent results of math calculations for MG microphysics;
@@ -175,6 +178,39 @@ elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "Intel")
       SET_SOURCE_FILES_PROPERTIES(./physics/gfdl_fv_sat_adj.F90
                                   PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS_OPT32BIT}")
     endif (DYN32)
+
+    # For CCPP acceptance: selective reduction of optimization flags, hopefully
+    # to be removed once established that this is not a reasonable approach.
+    if (TRANSITION)
+      # Replace "CORE-AVX2" with "CORE-AVX-I"
+      set(CMAKE_Fortran_FLAGS_LOPT ${CMAKE_Fortran_FLAGS})
+      string(REPLACE "-xCORE-AVX2" "-xCORE-AVX-I"
+             CMAKE_Fortran_FLAGS_LOPT
+             "${CMAKE_Fortran_FLAGS_LOPT}")
+      string(REPLACE "-axSSE4.2,AVX,CORE-AVX2" "-axSSE4.2,AVX,CORE-AVX-I"
+             CMAKE_Fortran_FLAGS_LOPT
+             "${CMAKE_Fortran_FLAGS_LOPT}")
+      SET_SOURCE_FILES_PROPERTIES(./physics/sflx.f
+                                  PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS_LOPT}")
+      # Replace "-no-prec-div -no-prec-sqrt" with "-prec-div -prec-sqrt",
+      # replace "CORE-AVX2" with "CORE-AVX-I"
+      set(CMAKE_Fortran_FLAGS_LOPT2 ${CMAKE_Fortran_FLAGS})
+      string(REPLACE "-no-prec-div" "-prec-div"
+             CMAKE_Fortran_FLAGS_LOPT2
+             "${CMAKE_Fortran_FLAGS_LOPT2}")
+      string(REPLACE "-no-prec-sqrt" "-prec-sqrt"
+             CMAKE_Fortran_FLAGS_LOPT2
+             "${CMAKE_Fortran_FLAGS_LOPT2}")
+      string(REPLACE "-xCORE-AVX2" "-xCORE-AVX-I"
+             CMAKE_Fortran_FLAGS_LOPT2
+             "${CMAKE_Fortran_FLAGS_LOPT2}")
+      string(REPLACE "-axSSE4.2,AVX,CORE-AVX2" "-axSSE4.2,AVX,CORE-AVX-I"
+             CMAKE_Fortran_FLAGS_LOPT2
+             "${CMAKE_Fortran_FLAGS_LOPT2}")
+      SET_SOURCE_FILES_PROPERTIES(./physics/satmedmfvdif.F
+                                  PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS_LOPT2}")
+    endif (TRANSITION)
+
   else (PROJECT STREQUAL "CCPP-FV3")
     SET_SOURCE_FILES_PROPERTIES(./physics/module_bfmicrophysics.f ./physics/rascnvv2.f ./physics/sflx.f ./physics/sfc_diff.f ./physics/sfc_diag.f PROPERTIES COMPILE_FLAGS -r8)
     SET_SOURCE_FILES_PROPERTIES(./physics/module_nst_model.f90 ./physics/calpreciptype.f90 PROPERTIES COMPILE_FLAGS "-r8 -free")
@@ -263,19 +299,22 @@ set_target_properties(ccppphys PROPERTIES VERSION ${PROJECT_VERSION}
                                      COMPILE_FLAGS "${CMAKE_Fortran_FLAGS}"
                                      LINK_FLAGS "${CMAKE_Fortran_FLAGS}")
 
-# DH* hack for PGI compiler: rename objects in scheme cap object files for ISO_C compliancy
+# DH* Hack for PGI compiler: rename objects in scheme cap object files for ISO_C compliancy,
+# this is only needed for dynamics builds - static build generates plain Fortran code.
 if (${CMAKE_Fortran_COMPILER_ID} STREQUAL "PGI")
-set(CAPOBJS)
-foreach(cap ${CAPS})
-    string(REPLACE "_cap.F90" "_cap.F90.o" capobj "./${CMAKE_FILES_DIRECTORY}/ccppphys.dir/${cap}")
-    list(APPEND CAPOBJS ${capobj})
-endforeach(cap)
-
-add_custom_command(TARGET ccppphys
-                   PRE_LINK
-                   COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/pgifix.py --cmake ${CAPOBJS}
-                   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-                   COMMENT "Running pgifix_wrapper.py over all scheme caps")
+  if (NOT STATIC)
+  set(CAPOBJS)
+  foreach(cap ${CAPS})
+      string(REPLACE "_cap.F90" "_cap.F90.o" capobj "./${CMAKE_FILES_DIRECTORY}/ccppphys.dir/${cap}")
+      list(APPEND CAPOBJS ${capobj})
+  endforeach(cap)
+
+  add_custom_command(TARGET ccppphys
+                     PRE_LINK
+                     COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/pgifix.py --cmake ${CAPOBJS}
+                     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+                     COMMENT "Running pgifix_wrapper.py over all scheme caps")
+  endif (NOT STATIC)
 endif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "PGI")
 # *DH end hack for PGI compiler
 

physics/GFS_DCNV_generic.F90

@@ -169,7 +169,13 @@ subroutine GFS_DCNV_generic_post_run (im, levs, lssav, ldiag3d, lgocart, ras, cs
       real(kind=kind_phys), dimension(:,:), intent(inout) :: upd_mf, dwn_mf, det_mf
       ! dqdti, cnvqci, upd_mfi, dwn_mfi, det_mfi only allocated if ldiag3d == .true. or lgocart == .true.
       real(kind=kind_phys), dimension(:,:), intent(inout) :: dqdti, cnvqci, upd_mfi, dwn_mfi, det_mfi
-      real(kind=kind_phys), dimension(im,levs), intent(inout) :: cnvw, cnvc, cnvw_phy_f3d, cnvc_phy_f3d
+      real(kind=kind_phys), dimension(im,levs), intent(inout) :: cnvw, cnvc
+      ! DH* The following arrays may not be allocated, depending on certain flags and microphysics schemes.
+      ! Since Intel 15 crashes when passing unallocated arrays to arrays defined with explicit shape,
+      ! use assumed-shape arrays. Note that Intel 18 and GNU 6.2.0-8.1.0 tolerate explicit-shape arrays
+      ! as long as these do not get used when not allocated (it is still invalid Fortran code, though).
+      real(kind=kind_phys), dimension(:,:), intent(inout) :: cnvw_phy_f3d, cnvc_phy_f3d
+      ! *DH
 
       character(len=*), intent(out) :: errmsg
       integer, intent(out) :: errflg

physics/GFS_MP_generic.F90

@@ -104,6 +104,7 @@ end subroutine GFS_MP_generic_post_init
 !! | ntrac            | number_of_tracers                                                       | number of tracers                                                       | count       |    0 | integer    |           | in     | F        |
 !! | imp_physics      | flag_for_microphysics_scheme                                            | choice of microphysics scheme                                           | flag        |    0 | integer    |           | in     | F        |
 !! | imp_physics_gfdl | flag_for_gfdl_microphysics_scheme                                       | choice of GFDL microphysics scheme                                      | flag        |    0 | integer    |           | in     | F        |
+!! | imp_physics_thompson | flag_for_thompson_microphysics_scheme                               | choice of Thompson microphysics scheme                                  | flag        |    0 | integer    |           | in     | F        |
 !! | cal_pre          | flag_for_precipitation_type_algorithm                                   | flag controls precip type algorithm                                     | flag        |    0 | logical    |           | in     | F        |
 !! | lssav            | flag_diagnostics                                                        | logical flag for storing diagnostics                                    | flag        |    0 | logical    |           | in     | F        |
 !! | ldiag3d          | flag_diagnostics_3D                                                     | flag for 3d diagnostic fields                                           | flag        |    0 | logical    |           | in     | F        |
@@ -158,23 +159,31 @@ end subroutine GFS_MP_generic_post_init
 !! | dtdtc            | tendency_of_air_temperature_due_to_radiative_heating_assuming_clear_sky | clear sky radiative (shortwave + longwave) heating rate at current time | K s-1       |    2 | real       | kind_phys | in     | F        |
 !! | drain_cpl        | tendency_of_lwe_thickness_of_precipitation_amount_for_coupling          | change in rain_cpl (coupling_type)                                      | m           |    1 | real       | kind_phys | inout  | F        |
 !! | dsnow_cpl        | tendency_of_lwe_thickness_of_snow_amount_for_coupling                   | change in show_cpl (coupling_type)                                      | m           |    1 | real       | kind_phys | inout  | F        |
+!! | lsm              | flag_for_land_surface_scheme                                            | flag for land surface model                                             | flag        |    0 | integer    |           | in     | F        |
+!! | lsm_ruc          | flag_for_ruc_land_surface_scheme                                        | flag for RUC land surface model                                         | flag        |    0 | integer    |           | in     | F        |
+!! | raincprv         | lwe_thickness_of_explicit_rainfall_amount_from_previous_timestep        | explicit rainfall from previous timestep                                | m           |    1 | real       | kind_phys | inout  | F        |
+!! | rainncprv        | lwe_thickness_of_convective_precipitation_amount_from_previous_timestep | convective_precipitation_amount from previous timestep                  | m           |    1 | real       | kind_phys | inout  | F        |
+!! | iceprv           | lwe_thickness_of_ice_amount_from_previous_timestep                      | ice amount from previous timestep                                       | m           |    1 | real       | kind_phys | inout  | F        |
+!! | snowprv          | lwe_thickness_of_snow_amount_from_previous_timestep                     | snow amount from previous timestep                                      | m           |    1 | real       | kind_phys | inout  | F        |
+!! | graupelprv       | lwe_thickness_of_graupel_amount_from_previous_timestep                  | graupel amount from previous timestep                                   | m           |    1 | real       | kind_phys | inout  | F        |
 !! | errmsg           | ccpp_error_message                                                      | error message for error handling in CCPP                                | none        |    0 | character  | len=*     | out    | F        |
 !! | errflg           | ccpp_error_flag                                                         | error flag for error handling in CCPP                                   | flag        |    0 | integer    |           | out    | F        |
 !!
 !> \section gfs_mp_gen GFS MP Generic Post General Algorithm
 !! @{
       subroutine GFS_MP_generic_post_run(im, ix, levs, kdt, nrcm, ncld, nncl, ntcw, ntrac, imp_physics, imp_physics_gfdl, &
-        cal_pre, lssav, ldiag3d, cplflx, cplchm, con_g, dtf, frain, rainc, rain1, rann, xlat, xlon, gt0, gq0,             &
-        prsl, prsi, phii, tsfc, ice, snow, graupel, save_t, save_qv, ice0, snow0, graupel0, del,                          &
+        imp_physics_thompson, cal_pre, lssav, ldiag3d, cplflx, cplchm, con_g, dtf, frain, rainc, rain1, rann, xlat, xlon, &
+        gt0, gq0, prsl, prsi, phii, tsfc, ice, snow, graupel, save_t, save_qv, ice0, snow0, graupel0, del,                &
         rain, domr_diag, domzr_diag, domip_diag, doms_diag, tprcp, srflag, totprcp, totice, totsnw,                       &
         totgrp, totprcpb, toticeb, totsnwb, totgrpb, dt3dt, dq3dt, rain_cpl, rainc_cpl, snow_cpl, pwat,                   &
-        do_sppt, dtdtr, dtdtc, drain_cpl, dsnow_cpl, errmsg, errflg)
+        do_sppt, dtdtr, dtdtc, drain_cpl, dsnow_cpl, lsm, lsm_ruc, raincprv, rainncprv, iceprv, snowprv, graupelprv,      &
+        errmsg, errflg)
 !
-      use machine,               only: kind_phys
+      use machine, only: kind_phys
 
       implicit none
 
-      integer, intent(in) :: im, ix, levs, kdt, nrcm, ncld, nncl, ntcw, ntrac, imp_physics, imp_physics_gfdl
+      integer, intent(in) :: im, ix, levs, kdt, nrcm, ncld, nncl, ntcw, ntrac, imp_physics, imp_physics_gfdl, imp_physics_thompson
       logical, intent(in) :: cal_pre, lssav, ldiag3d, cplflx, cplchm
 
       real(kind=kind_phys),                           intent(in)    :: dtf, frain, con_g
@@ -197,6 +206,14 @@ subroutine GFS_MP_generic_post_run(im, ix, levs, kdt, nrcm, ncld, nncl, ntcw, nt
       real(kind=kind_phys), dimension(im),      intent(inout) :: drain_cpl
       real(kind=kind_phys), dimension(im),      intent(inout) :: dsnow_cpl
 
+      ! Rainfall variables previous time step (update for RUC LSM)
+      integer, intent(in) :: lsm, lsm_ruc
+      real(kind=kind_phys), dimension(im),      intent(inout) :: raincprv
+      real(kind=kind_phys), dimension(im),      intent(inout) :: rainncprv
+      real(kind=kind_phys), dimension(im),      intent(inout) :: iceprv
+      real(kind=kind_phys), dimension(im),      intent(inout) :: snowprv
+      real(kind=kind_phys), dimension(im),      intent(inout) :: graupelprv
+
       ! CCPP error handling
       character(len=*), intent(out) :: errmsg
       integer, intent(out) :: errflg
@@ -214,7 +231,7 @@ subroutine GFS_MP_generic_post_run(im, ix, levs, kdt, nrcm, ncld, nncl, ntcw, nt
       onebg = 1.0d0/con_g
 
       do i = 1, im
-          rain(i) = rainc(i) + frain * rain1(i)
+          rain(i) = rainc(i) + frain * rain1(i) ! time-step convective plus explicit
       enddo
 
 !> - If requested (e.g. Zhao-Carr MP scheme), call calpreciptype() to calculate dominant 
@@ -226,12 +243,29 @@ subroutine GFS_MP_generic_post_run(im, ix, levs, kdt, nrcm, ncld, nncl, ntcw, nt
       ! physics timestep, while Diag%{rain,rainc} and all totprecip etc
       ! are on the dynamics timestep. Totally confusing and wrong. *DH
       if (imp_physics == imp_physics_gfdl) then
+        tprcp   = max(0., rain)               ! clu: rain -> tprcp
         !graupel = frain*graupel0
         !ice     = frain*ice0
         !snow    = frain*snow0
         graupel = graupel0
         ice     = ice0
         snow    = snow0
+      ! Do it right from the beginning for Thompson
+      else if (imp_physics == imp_physics_thompson) then
+        tprcp   = max (0.,rainc + frain * rain1) ! time-step convective and explicit precip
+        graupel = frain*graupel0              ! time-step graupel
+        ice     = frain*ice0                  ! time-step ice
+        snow    = frain*snow0                 ! time-step snow
+      end if
+
+      if (lsm==lsm_ruc) then
+        if (imp_physics == imp_physics_gfdl .or. imp_physics == imp_physics_thompson) then
+            raincprv(:)   = rainc(:)
+            rainncprv(:)  = frain * rain1(:)
+            iceprv(:)     = ice(:)
+            snowprv(:)    = snow(:)
+            graupelprv(:) = graupel(:)
+        end if
       end if
 
       if (cal_pre) then       ! hchuang: add dominant precipitation type algorithm
@@ -252,7 +286,7 @@ subroutine GFS_MP_generic_post_run(im, ix, levs, kdt, nrcm, ncld, nncl, ntcw, nt
 !       end do
 !       HCHUANG: use new precipitation type to decide snow flag for LSM snow accumulation
 
-        if (imp_physics /= imp_physics_gfdl) then
+        if (imp_physics /= imp_physics_gfdl .and. imp_physics /= imp_physics_thompson) then
           do i=1,im
             tprcp(i)  = max(0.0, rain(i) )
             if(doms(i) > 0.0 .or. domip(i) > 0.0) then
@@ -309,15 +343,15 @@ subroutine GFS_MP_generic_post_run(im, ix, levs, kdt, nrcm, ncld, nncl, ntcw, nt
         enddo
       enddo
 
-!> - For GFDL cloud MP scheme, determine convective snow by surface temperature;
+!> - For GFDL and Thompson MP scheme, determine convective snow by surface temperature;
 !! and determine explicit rain/snow by snow/ice/graupel coming out directly from MP
 !! and convective rainfall from the cumulus scheme if the surface temperature is below
 !! \f$0^oC\f$.
-      if (imp_physics == imp_physics_gfdl) then
+      if (imp_physics == imp_physics_gfdl .or. imp_physics == imp_physics_thompson) then
 ! determine convective rain/snow by surface temperature
 ! determine large-scale rain/snow by rain/snow coming out directly from MP
         do i = 1, im
-          tprcp(i)  = max(0.0, rain(i) )! clu: rain -> tprcp
+          !tprcp(i)  = max(0.0, rain(i) )! clu: rain -> tprcp ! DH now lines 245-250
           srflag(i) = 0.                     ! clu: default srflag as 'rain' (i.e. 0)
           if (tsfc(i) >= 273.15) then
             crain = rainc(i)