Merge NCAR:dtc/develop into develop 2020/04/14 (#98)

CMakeLists.txt

@@ -113,6 +113,7 @@ add_library(
 
 if(CCPP)
 target_include_directories(fv3cap PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/ccpp/driver/mod)
+add_dependencies(fv3cap ccppdriver ccppphys)
 endif()
 target_include_directories(fv3cap PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/stochastic_physics)
 target_compile_definitions(fv3cap PRIVATE -DESMF_VERSION_MAJOR=${ESMF_VERSION_MAJOR})

ccpp/CMakeLists.txt

@@ -1,21 +1,15 @@
-cmake_minimum_required(VERSION 2.8.11)
+cmake_minimum_required(VERSION 3.0)
 
 # Use rpaths on MacOSX
 set(CMAKE_MACOSX_RPATH 1)
-
-if(POLICY CMP0048)
-    cmake_policy(SET CMP0048 NEW)
-endif(POLICY CMP0048)
-
 if(POLICY CMP0042)
     cmake_policy(SET CMP0042 NEW)
 endif(POLICY CMP0042)
 
-PROJECT(CCPP-FV3)
+project(CCPP-FV3
+        LANGUAGES C CXX Fortran)
 set(PROJECT "CCPP-FV3")
 
-ENABLE_LANGUAGE(Fortran)
-
 include(CMakeForceCompiler)
 
 #------------------------------------------------------------------------------

ccpp/build_ccpp.sh

@@ -211,21 +211,21 @@ make ${CCPP_MAKE_FLAGS} install
 
 # Generate ESMF makefile fragment
 
-# Explicitly append libxml2, with or without path
-CCPP_XML2_LIB="${LIBXML2_LIB_DIR:+-L${LIBXML2_LIB_DIR} }-lxml2"
 set -u
 if ( echo "${MAKE_OPT}" | grep STATIC=Y ) ; then
   # Set linker flags for static build
-  CCPP_LINK_OBJS="-L${PATH_CCPP_LIB} -lccpp -lccppphys ${CCPP_XML2_LIB}"
+  CCPP_LINK_OBJS="-L${PATH_CCPP_LIB} -lccpp -lccppphys"
 else
+  # Explicitly append libxml2, with or without path
+  CCPP_XML2_LIB="${LIBXML2_LIB_DIR:+-L${LIBXML2_LIB_DIR} }-lxml2"
   # Set link objects
   if ( echo "$MACHINE_ID" | grep gaea ) ; then
-    CCPP_LINK_OBJS="-dynamic -L${PATH_CCPP_LIB} -lccpp ${CCPP_XML2_LIB} ${CRAY_PMI_POST_LINK_OPTS} -lpmi"
+    CCPP_LINK_OBJS="-dynamic -L${PATH_CCPP_LIB} -lccpp -lccppphys ${CCPP_XML2_LIB} ${CRAY_PMI_POST_LINK_OPTS} -lpmi"
   else
-    CCPP_LINK_OBJS="-L${PATH_CCPP_LIB} -lccpp ${CCPP_XML2_LIB}"
+    CCPP_LINK_OBJS="-L${PATH_CCPP_LIB} -lccpp -lccppphys ${CCPP_XML2_LIB}"
   fi
 fi
-echo "ESMF_DEP_INCPATH=${PATH_CCPP_INC}" > ${CCPP_MK}
+echo "ESMF_DEP_INCPATH=${PATH_CCPP_INC} ${PATH_CCPP_BUILD}/physics" > ${CCPP_MK}
 echo "ESMF_DEP_LINK_OBJS=${CCPP_LINK_OBJS}" >> ${CCPP_MK}
 
 if [ $clean_after = YES ]; then

ccpp/config/ccpp_prebuild_config.py

@@ -9,15 +9,21 @@
 
 HOST_MODEL_IDENTIFIER = "FV3"
 
-# Add all files with metadata tables on the host model side,
-# relative to basedir = top-level directory of host model
+# Add all files with metadata tables on the host model side and in CCPP,
+# relative to basedir = top-level directory of host model. This includes
+# kind and type definitions used in CCPP physics.
 VARIABLE_DEFINITION_FILES = [
     'FV3/ccpp/physics/physics/machine.F',
     'FV3/ccpp/physics/physics/radsw_param.f',
     'FV3/ccpp/physics/physics/radlw_param.f',
     'FV3/gfsphysics/CCPP_layer/CCPP_typedefs.F90',
     'FV3/gfsphysics/GFS_layer/GFS_typedefs.F90',
     'FV3/gfsphysics/CCPP_layer/CCPP_data.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rrtmgp/mo_gas_optics_rrtmgp.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rrtmgp/mo_gas_concentrations.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rte/mo_optical_props.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/extensions/cloud_optics/mo_cloud_optics.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rte/mo_source_functions.F90',
     ]
 
 TYPEDEFS_NEW_METADATA = {
@@ -56,6 +62,27 @@
         'GFS_stateout_type'     : 'GFS_Data(cdata%blk_no)%Stateout',
         'GFS_typedefs' : '',
         },
+    'mo_gas_concentrations' : {
+        'ty_gas_concs' : '',
+        'mo_gas_concentrations' : '',
+        },
+    'mo_gas_optics_rrtmgp' : {
+        'ty_gas_optics_rrtmgp' : '',
+        'mo_gas_optics_rrtmgp' : '',
+        },
+    'mo_optical_props' : {
+        'ty_optical_props_1scl' : '',
+        'ty_optical_props_2str' : '',
+        'mo_optical_props' : '',
+        },
+    'mo_cloud_optics' : {
+        'ty_cloud_optics' : '',
+        'mo_cloud_optics' : '',
+        },
+    'mo_source_functions' : {
+        'ty_source_func_lw' : '',
+        'mo_source_functions' : '',
+        },
     }
 
 # Add all physics scheme dependencies relative to basedir - note that the CCPP
@@ -144,6 +171,33 @@
     'FV3/ccpp/physics/physics/namelist_soilveg_ruc.F90',
     'FV3/ccpp/physics/physics/set_soilveg_ruc.F90',
     'FV3/ccpp/physics/physics/module_soil_pre.F90',
+    # RRTMGP
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rrtmgp/mo_gas_concentrations.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rrtmgp/mo_gas_optics.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rrtmgp/mo_gas_optics_rrtmgp.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rrtmgp/mo_rrtmgp_constants.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rrtmgp/mo_rrtmgp_util_reorder.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rrtmgp/mo_rrtmgp_util_string.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rrtmgp/kernels/mo_gas_optics_kernels.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rrtmgp/kernels/mo_rrtmgp_util_reorder_kernels.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rte/mo_fluxes.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rte/mo_rte_util_array.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rte/mo_optical_props.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rte/mo_rte_kind.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rte/mo_rte_lw.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rte/mo_rte_sw.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rte/mo_source_functions.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rte/kernels/mo_fluxes_broadband_kernels.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rte/kernels/mo_optical_props_kernels.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/rte/kernels/mo_rte_solver_kernels.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/extensions/mo_compute_bc.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/extensions/mo_fluxes_byband.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/extensions/mo_fluxes_byband_kernels.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/extensions/mo_fluxes_bygpoint.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/extensions/mo_heating_rates.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/extensions/mo_rrtmgp_clr_all_sky.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/extensions/cloud_optics/mo_cloud_optics.F90',
+    'FV3/ccpp/physics/physics/rte-rrtmgp/extensions/cloud_optics/mo_cloud_sampling.F90',
     # derived data type definitions
     'FV3/gfsphysics/GFS_layer/GFS_typedefs.F90',
     'FV3/gfsphysics/CCPP_layer/CCPP_typedefs.F90',
@@ -241,12 +295,37 @@
     'FV3/ccpp/physics/physics/mp_fer_hires.F90'                  : [ 'slow_physics' ],
     # for testing the <init> and <finalize> sections
     'FV3/ccpp/physics/physics/GFS_suite_init_finalize_test.F90'  : [ 'slow_physics' ],
+    # RRTMGP
+    'FV3/ccpp/physics/physics/rrtmg_lw_cloud_optics.F90'         : [ 'slow_physics' ],
+    'FV3/ccpp/physics/physics/rrtmg_sw_cloud_optics.F90'         : [ 'slow_physics' ],
+    'FV3/ccpp/physics/physics/rrtmgp_aux.F90'                    : [ 'slow_physics' ],
+    'FV3/ccpp/physics/physics/rrtmgp_lw_gas_optics.F90'          : [ 'slow_physics' ],
+    'FV3/ccpp/physics/physics/rrtmgp_lw_cloud_optics.F90'        : [ 'slow_physics' ],
+    'FV3/ccpp/physics/physics/rrtmgp_sw_gas_optics.F90'          : [ 'slow_physics' ],
+    'FV3/ccpp/physics/physics/rrtmgp_sw_cloud_optics.F90'        : [ 'slow_physics' ],
+    'FV3/ccpp/physics/physics/rrtmgp_sw_aerosol_optics.F90'      : [ 'slow_physics' ],
+    'FV3/ccpp/physics/physics/rrtmgp_lw_rte.F90'                 : [ 'slow_physics' ],
+    'FV3/ccpp/physics/physics/rrtmgp_lw_cloud_sampling.F90'      : [ 'slow_physics' ],
+    'FV3/ccpp/physics/physics/rrtmgp_sw_rte.F90'                 : [ 'slow_physics' ],
+    'FV3/ccpp/physics/physics/rrtmgp_sw_cloud_sampling.F90'      : [ 'slow_physics' ],
+    'FV3/ccpp/physics/physics/rrtmgp_lw_aerosol_optics.F90'      : [ 'slow_physics' ],
+    'FV3/ccpp/physics/physics/GFS_rrtmgp_setup.F90'              : [ 'slow_physics' ],
+    'FV3/ccpp/physics/physics/GFS_rrtmgp_pre.F90'                : [ 'slow_physics' ],
+    'FV3/ccpp/physics/physics/rrtmgp_lw_pre.F90'                 : [ 'slow_physics' ],
+    'FV3/ccpp/physics/physics/GFS_rrtmgp_sw_pre.F90'             : [ 'slow_physics' ],
+    'FV3/ccpp/physics/physics/GFS_rrtmgp_lw_post.F90'            : [ 'slow_physics' ],
+    'FV3/ccpp/physics/physics/GFS_rrtmgp_sw_post.F90'            : [ 'slow_physics' ],
     }
 
 # Default build dir, relative to current working directory,
 # if not specified as command-line argument
 DEFAULT_BUILD_DIR = 'FV3'
 
+# Auto-generated makefile/cmakefile snippets that contain all type definitions
+TYPEDEFS_MAKEFILE   = '{build_dir}/ccpp/physics/CCPP_TYPEDEFS.mk'
+TYPEDEFS_CMAKEFILE  = '{build_dir}/ccpp/physics/CCPP_TYPEDEFS.cmake'
+TYPEDEFS_SOURCEFILE = '{build_dir}/ccpp/physics/CCPP_TYPEDEFS.sh'
+
 # Auto-generated makefile/cmakefile snippets that contain all schemes
 SCHEMES_MAKEFILE   = '{build_dir}/ccpp/physics/CCPP_SCHEMES.mk'
 SCHEMES_CMAKEFILE  = '{build_dir}/ccpp/physics/CCPP_SCHEMES.cmake'
@@ -276,9 +355,40 @@
 # if no entry is made here. Possible values are: 'all', 'none',
 # or a list of standard_names: [ 'var1', 'var3' ].
 OPTIONAL_ARGUMENTS = {
+    'rrtmgp_sw_rte' : {
+         'rrtmgp_sw_rte_run' : [
+             'components_of_surface_downward_shortwave_fluxes',
+             'sw_fluxes_sfc',
+             'sw_fluxes_toa',
+             ],
+         },
+    'GFS_rrtmgp_sw_post' : {
+         'GFS_rrtmgp_sw_post_run' : [
+             'components_of_surface_downward_shortwave_fluxes',
+             'sw_fluxes_sfc',
+             'sw_fluxes_toa',
+             ],
+         },
+    'rrtmgp_lw_rte' : {
+         'rrtmgp_lw_rte_run' : [
+             'lw_fluxes_sfc',
+             'lw_fluxes_toa',
+             ],
+        },
+    'GFS_rrtmgp_lw_post' : {
+         'GFS_rrtmgp_lw_post_run' : [
+             'lw_fluxes_sfc',
+             'lw_fluxes_toa',
+             ],
+         },
+    'GFS_rrtmgp_post' : {
+         'GFS_rrtmgp_post_run' : [
+             'components_of_surface_downward_shortwave_fluxes',
+             ],
+         },
     'rrtmg_sw' : {
         'rrtmg_sw_run' : [
-            'tendency_of_air_temperature_due_to_shortwave_heating_assuming_clear_sky_on_radiation_time_step',
+            'tendency_of_air_temperature_due_to_shortwave_heating_assuming_clear_sky_on_radiation_time_step_and_radiation_levels',
             'components_of_surface_downward_shortwave_fluxes',
             'cloud_liquid_water_path',
             'mean_effective_radius_for_liquid_cloud',
@@ -292,7 +402,7 @@
         },
     'rrtmg_lw' : {
         'rrtmg_lw_run' : [
-            'tendency_of_air_temperature_due_to_longwave_heating_assuming_clear_sky_on_radiation_time_step',
+            'tendency_of_air_temperature_due_to_longwave_heating_assuming_clear_sky_on_radiation_time_step_and_radiation_levels',
             'cloud_liquid_water_path',
             'mean_effective_radius_for_liquid_cloud',
             'cloud_ice_water_path',

ccpp/suites/suite_FV3_GFS_2017_RRTMGP.xml

@@ -0,0 +1,95 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<suite name="FV3_GFS_2017_RRTMGP" lib="ccppphys" ver="3">
+  <!-- <init></init> -->
+  <group name="time_vary">
+    <subcycle loop="1">
+      <scheme>GFS_time_vary_pre</scheme>
+      <scheme>GFS_rrtmgp_setup</scheme>
+      <scheme>GFS_rad_time_vary</scheme>
+      <scheme>GFS_phys_time_vary</scheme>
+    </subcycle>
+  </group>
+  <group name="radiation">
+    <subcycle loop="1">
+      <scheme>GFS_suite_interstitial_rad_reset</scheme>
+      <scheme>GFS_rrtmgp_pre</scheme>
+      <scheme>GFS_rrtmgp_sw_pre</scheme>
+      <scheme>rrtmgp_sw_gas_optics</scheme>
+      <scheme>rrtmgp_sw_aerosol_optics</scheme>
+      <scheme>rrtmgp_sw_cloud_optics</scheme>
+      <scheme>rrtmgp_sw_cloud_sampling</scheme>
+      <scheme>rrtmgp_sw_rte</scheme>
+      <scheme>GFS_rrtmgp_sw_post</scheme>
+      <scheme>rrtmgp_lw_pre</scheme>
+      <scheme>rrtmgp_lw_gas_optics</scheme>
+      <scheme>rrtmgp_lw_aerosol_optics</scheme>
+      <scheme>rrtmgp_lw_cloud_optics</scheme>
+      <scheme>rrtmgp_lw_cloud_sampling</scheme>
+      <scheme>rrtmgp_lw_rte</scheme>
+      <scheme>GFS_rrtmgp_lw_post</scheme>
+    </subcycle>
+  </group>
+  <group name="physics">
+    <subcycle loop="1">
+      <scheme>GFS_suite_interstitial_phys_reset</scheme>
+      <scheme>GFS_suite_stateout_reset</scheme>
+      <scheme>get_prs_fv3</scheme>
+      <scheme>GFS_suite_interstitial_1</scheme>
+      <scheme>GFS_surface_generic_pre</scheme>
+      <scheme>GFS_surface_composites_pre</scheme>
+      <scheme>dcyc2t3</scheme>
+      <scheme>GFS_surface_composites_inter</scheme>
+      <scheme>GFS_suite_interstitial_2</scheme>
+    </subcycle>
+    <!-- Surface iteration loop -->
+    <subcycle loop="2">
+      <scheme>sfc_diff</scheme>
+      <scheme>GFS_surface_loop_control_part1</scheme>
+      <scheme>sfc_nst_pre</scheme>
+      <scheme>sfc_nst</scheme>
+      <scheme>sfc_nst_post</scheme>
+      <scheme>lsm_noah</scheme>
+      <scheme>sfc_sice</scheme>
+      <scheme>GFS_surface_loop_control_part2</scheme>
+    </subcycle>
+    <!-- End of surface iteration loop -->
+    <subcycle loop="1">
+      <scheme>GFS_surface_composites_post</scheme>
+      <scheme>sfc_diag</scheme>
+      <scheme>sfc_diag_post</scheme>
+      <scheme>GFS_surface_generic_post</scheme>
+      <scheme>GFS_PBL_generic_pre</scheme>
+      <scheme>hedmf</scheme>
+      <scheme>GFS_PBL_generic_post</scheme>
+      <scheme>GFS_GWD_generic_pre</scheme>
+      <scheme>cires_ugwp</scheme>
+      <scheme>cires_ugwp_post</scheme>
+      <scheme>GFS_GWD_generic_post</scheme>
+      <scheme>rayleigh_damp</scheme>
+      <scheme>GFS_suite_stateout_update</scheme>
+      <scheme>ozphys</scheme>
+      <scheme>GFS_DCNV_generic_pre</scheme>
+      <scheme>get_phi_fv3</scheme>
+      <scheme>GFS_suite_interstitial_3</scheme>
+      <scheme>samfdeepcnv</scheme>
+      <scheme>GFS_DCNV_generic_post</scheme>
+      <scheme>GFS_SCNV_generic_pre</scheme>
+      <scheme>samfshalcnv</scheme>
+      <scheme>GFS_SCNV_generic_post</scheme>
+      <scheme>GFS_suite_interstitial_4</scheme>
+      <scheme>cnvc90</scheme>
+      <scheme>GFS_MP_generic_pre</scheme>
+      <scheme>zhaocarr_gscond</scheme>
+      <scheme>zhaocarr_precpd</scheme>
+      <scheme>GFS_MP_generic_post</scheme>
+      <scheme>maximum_hourly_diagnostics</scheme>
+    </subcycle>
+  </group>
+  <group name="stochastics">
+    <subcycle loop="1">
+      <scheme>GFS_stochastics</scheme>
+    </subcycle>
+  </group>
+  <!-- <finalize></finalize> -->
+</suite>

gfsphysics/CMakeLists.txt

@@ -18,26 +18,20 @@ set(CCPP_SOURCES
     physics/mersenne_twister.f
     physics/namelist_soilveg.f
     physics/physparam.f
-    physics/radlw_param.f
-    physics/radsw_param.f
     physics/set_soilveg.f
 
     physics/noahmp_tables.f90
 
-    physics/machine.F
-
     physics/GFDL_parse_tracers.F90
     physics/physcons.F90
 
-    CCPP_layer/CCPP_typedefs.F90
     CCPP_layer/CCPP_data.F90
     ${CMAKE_BINARY_DIR}/FV3/ccpp/physics/ccpp_static_api.F90
 
     GFS_layer/GFS_abstraction_layer.F90
     GFS_layer/GFS_diagnostics.F90
     GFS_layer/GFS_driver.F90
     GFS_layer/GFS_restart.F90
-    GFS_layer/GFS_typedefs.F90
 )
 
 set(IPD_SOURCES

gfsphysics/GFS_layer/GFS_physics_driver.F90

@@ -4949,6 +4949,7 @@ subroutine GFS_physics_driver                                 &
 !       write(1000+me,*)' maxwatncb=',maxval(Stateout%gq0(1:im,k,ntlnc)),' k=',k,' kdt',kdt
 !       enddo
 
+!## CCPP ##* m_micro.F90/m_micro_run
         call m_micro_driver (im, ix, levs, Model%flipv, dtp,  Statein%prsl,      &
                              Statein%prsi, Statein%phil, Statein%phii,           &
                              Statein%vvl, clw(1,1,2), QLCN, clw(1,1,1), QICN,    &
@@ -4973,6 +4974,7 @@ subroutine GFS_physics_driver                                 &
 !                            ipr, kdt, Grid%xlat, Grid%xlon)
                              Model%mg_alf, Model%mg_qcmin, Model%pdfflag,        &
                              ipr, kdt, Grid%xlat, Grid%xlon, rhc)
+!*## CCPP ##
 !     do k=1,levs
 !     write(1000+me,*)' maxwatnca=',maxval(Stateout%gq0(1:im,k,ntlnc)),' k=',k,' kdt=',kdt
 !     enddo