Merge remote-tracking branch 'ESCOMP/cam_development' into history-bu…

…gfixes

.gitmodules

@@ -36,7 +36,7 @@
 [submodule "atmos_phys"]
 	path = src/atmos_phys
 	url = https://github.com/ESCOMP/atmospheric_physics
-        fxtag = atmos_phys0_05_001
+        fxtag = atmos_phys0_06_000
         fxrequired = AlwaysRequired
 	fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics
 
@@ -144,21 +144,21 @@ fxDONOTUSEurl = https://github.com/ESCOMP/mizuRoute
 [submodule "ccs_config"]
 path = ccs_config
 url = https://github.com/ESMCI/ccs_config_cesm.git
-fxtag = ccs_config_cesm1.0.7
+fxtag = ccs_config_cesm1.0.8
 fxrequired = ToplevelRequired
 fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git
 
 [submodule "cime"]
 path = cime
 url = https://github.com/ESMCI/cime
-fxtag = cime6.1.29
+fxtag = cime6.1.41
 fxrequired = ToplevelRequired
 fxDONOTUSEurl = https://github.com/ESMCI/cime
 
 [submodule "cmeps"]
 path = components/cmeps
 url = https://github.com/ESCOMP/CMEPS.git
-fxtag = cmeps1.0.16
+fxtag = cmeps1.0.22
 fxrequired = ToplevelRequired
 fxDONOTUSEurl = https://github.com/ESCOMP/CMEPS.git
 

bld/configure

@@ -2335,6 +2335,7 @@ sub write_filepath
     # be overridden by modules from directories that occur earlier
     # in the list of filepaths.
     print $fh "$camsrcdir/src/physics/cam\n";
+    print $fh "$camsrcdir/src/atmos_phys/to_be_ccppized\n";
 
     #Add the CCPP'ized subdirectories
     print $fh "$camsrcdir/src/atmos_phys/schemes/tropopause_find\n";

cime_config/config_pes.xml

@@ -164,14 +164,14 @@
 	  <ntasks_cpl>-4</ntasks_cpl>
 	</ntasks>
 	<nthrds>
-	  <nthrds_atm>2</nthrds_atm>
-	  <nthrds_lnd>2</nthrds_lnd>
-	  <nthrds_rof>2</nthrds_rof>
-	  <nthrds_ice>2</nthrds_ice>
-	  <nthrds_ocn>2</nthrds_ocn>
-	  <nthrds_glc>2</nthrds_glc>
-	  <nthrds_wav>2</nthrds_wav>
-	  <nthrds_cpl>2</nthrds_cpl>
+	  <nthrds_atm>1</nthrds_atm>
+	  <nthrds_lnd>1</nthrds_lnd>
+	  <nthrds_rof>1</nthrds_rof>
+	  <nthrds_ice>1</nthrds_ice>
+	  <nthrds_ocn>1</nthrds_ocn>
+	  <nthrds_glc>1</nthrds_glc>
+	  <nthrds_wav>1</nthrds_wav>
+	  <nthrds_cpl>1</nthrds_cpl>
 	</nthrds>
 	<rootpe>
 	  <rootpe_atm>0</rootpe_atm>

cime_config/testdefs/testlist_cam.xml

@@ -1489,7 +1489,7 @@
 <!--                    (unsupported)                         -->
 <!--              @@@@@@@@@@@@@@@@@@@@@@@@@@@                 -->
 
-  <test compset="F2000dev" grid="ne30pg3_ne30pg3_mg17" name="ERS_Ln9_G4-a100-openacc" testmods="cam/outfrq9s_mg3_default">
+  <test compset="F2000dev" grid="ne30pg3_ne30pg3_mg17" name="ERS_Ln9" testmods="cam/outfrq9s_gpu_default">
     <machines>
       <machine name="derecho" compiler="nvhpc" category="derecho_gpu"/>
       <machine name="derecho" compiler="nvhpc" category="aux_cam"/>
@@ -1498,7 +1498,7 @@
       <option name="wallclock">00:30:00</option>
     </options>
   </test>
-  <test compset="F2000dev" grid="ne30pg3_ne30pg3_mg17" name="ERS_Ln9_G4-a100-openacc" testmods="cam/outfrq9s_mg3_pcols760">
+  <test compset="F2000dev" grid="ne30pg3_ne30pg3_mg17" name="ERS_Ln9" testmods="cam/outfrq9s_gpu_pcols760">
     <machines>
       <machine name="derecho" compiler="nvhpc" category="derecho_gpu"/>
       <machine name="derecho" compiler="nvhpc" category="prealpha"/>

cime_config/testdefs/testmods_dirs/cam/outfrq9s_gpu_default/shell_commands

@@ -0,0 +1,12 @@
+./xmlchange NTASKS=128
+./xmlchange NTHRDS=1
+./xmlchange ROOTPE='0'
+./xmlchange ROF_NCPL=`./xmlquery --value ATM_NCPL`
+./xmlchange GLC_NCPL=`./xmlquery --value ATM_NCPL`
+./xmlchange CAM_CONFIG_OPTS=' -microphys mg3 -rad rrtmg' --append
+./xmlchange TIMER_DETAIL='6'
+./xmlchange TIMER_LEVEL='999'
+./xmlchange GPU_TYPE=a100
+./xmlchange OPENACC_GPU_OFFLOAD=TRUE
+./xmlchange OVERSUBSCRIBE_GPU=TRUE
+./xmlchange NGPUS_PER_NODE=4

cime_config/testdefs/testmods_dirs/cam/outfrq9s_gpu_default/user_nl_cam

@@ -0,0 +1,4 @@
+mfilt=1,1,1,1,1,1
+ndens=1,1,1,1,1,1
+nhtfrq=9,9,9,9,9,9
+inithist='ENDOFRUN'

cime_config/testdefs/testmods_dirs/cam/outfrq9s_gpu_default/user_nl_clm

@@ -0,0 +1,26 @@
+!----------------------------------------------------------------------------------
+! Users should add all user specific namelist changes below in the form of 
+! namelist_var = new_namelist_value 
+!
+! Include namelist variables for drv_flds_in ONLY if -megan and/or -drydep options
+! are set in the CLM_NAMELIST_OPTS env variable.
+!
+! EXCEPTIONS: 
+! Set use_cndv           by the compset you use and the CLM_BLDNML_OPTS -dynamic_vegetation setting
+! Set use_vichydro       by the compset you use and the CLM_BLDNML_OPTS -vichydro           setting
+! Set use_cn             by the compset you use and CLM_BLDNML_OPTS -bgc  setting
+! Set use_crop           by the compset you use and CLM_BLDNML_OPTS -crop setting
+! Set spinup_state       by the CLM_BLDNML_OPTS -bgc_spinup      setting
+! Set irrigate           by the CLM_BLDNML_OPTS -irrig           setting
+! Set dtime              with L_NCPL                             option
+! Set fatmlndfrc         with LND_DOMAIN_PATH/LND_DOMAIN_FILE    options
+! Set finidat            with RUN_REFCASE/RUN_REFDATE/RUN_REFTOD options for hybrid or branch cases
+!                        (includes $inst_string for multi-ensemble cases)
+! Set glc_grid           with CISM_GRID                          option
+! Set glc_smb            with GLC_SMB                            option
+! Set maxpatch_glcmec    with GLC_NEC                            option
+! Set glc_do_dynglacier  with GLC_TWO_WAY_COUPLING               env variable
+!----------------------------------------------------------------------------------
+hist_nhtfrq = 9
+hist_mfilt  = 1
+hist_ndens  = 1

cime_config/testdefs/testmods_dirs/cam/outfrq9s_gpu_pcols760/shell_commands

@@ -0,0 +1,12 @@
+./xmlchange NTASKS=64
+./xmlchange NTHRDS=1
+./xmlchange ROOTPE='0'
+./xmlchange ROF_NCPL=`./xmlquery --value ATM_NCPL`
+./xmlchange GLC_NCPL=`./xmlquery --value ATM_NCPL`
+./xmlchange CAM_CONFIG_OPTS=' -microphys mg3 -rad rrtmg -pcols 760 ' --append
+./xmlchange TIMER_DETAIL='6'
+./xmlchange TIMER_LEVEL='999'
+./xmlchange GPU_TYPE=a100
+./xmlchange OPENACC_GPU_OFFLOAD=TRUE
+./xmlchange OVERSUBSCRIBE_GPU=TRUE
+./xmlchange NGPUS_PER_NODE=4

cime_config/testdefs/testmods_dirs/cam/outfrq9s_gpu_pcols760/user_nl_cam

@@ -0,0 +1,4 @@
+mfilt=1,1,1,1,1,1
+ndens=1,1,1,1,1,1
+nhtfrq=9,9,9,9,9,9
+inithist='ENDOFRUN'

cime_config/testdefs/testmods_dirs/cam/outfrq9s_gpu_pcols760/user_nl_clm

@@ -0,0 +1,26 @@
+!----------------------------------------------------------------------------------
+! Users should add all user specific namelist changes below in the form of 
+! namelist_var = new_namelist_value 
+!
+! Include namelist variables for drv_flds_in ONLY if -megan and/or -drydep options
+! are set in the CLM_NAMELIST_OPTS env variable.
+!
+! EXCEPTIONS: 
+! Set use_cndv           by the compset you use and the CLM_BLDNML_OPTS -dynamic_vegetation setting
+! Set use_vichydro       by the compset you use and the CLM_BLDNML_OPTS -vichydro           setting
+! Set use_cn             by the compset you use and CLM_BLDNML_OPTS -bgc  setting
+! Set use_crop           by the compset you use and CLM_BLDNML_OPTS -crop setting
+! Set spinup_state       by the CLM_BLDNML_OPTS -bgc_spinup      setting
+! Set irrigate           by the CLM_BLDNML_OPTS -irrig           setting
+! Set dtime              with L_NCPL                             option
+! Set fatmlndfrc         with LND_DOMAIN_PATH/LND_DOMAIN_FILE    options
+! Set finidat            with RUN_REFCASE/RUN_REFDATE/RUN_REFTOD options for hybrid or branch cases
+!                        (includes $inst_string for multi-ensemble cases)
+! Set glc_grid           with CISM_GRID                          option
+! Set glc_smb            with GLC_SMB                            option
+! Set maxpatch_glcmec    with GLC_NEC                            option
+! Set glc_do_dynglacier  with GLC_TWO_WAY_COUPLING               env variable
+!----------------------------------------------------------------------------------
+hist_nhtfrq = 9
+hist_mfilt  = 1
+hist_ndens  = 1