feature/stack

Merge branch 'develop' into feature/stack Fixes ufs-community#248
GeorgeGayno-NOAA · Mar 2, 2021 · 6da547e · 6da547e
2 parents dfc85d7 + 5bb62fb
commit 6da547e
Show file tree

Hide file tree

Showing 40 changed files with 1,294 additions and 824 deletions.
diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in
@@ -402,13 +402,13 @@ EXTRACT_ALL            = NO
 # be included in the documentation.
 # The default value is: NO.
 
-EXTRACT_PRIVATE        = NO
+EXTRACT_PRIVATE        = YES
 
 # If the EXTRACT_PACKAGE tag is set to YES all members with package or internal
 # scope will be included in the documentation.
 # The default value is: NO.
 
-EXTRACT_PACKAGE        = NO
+EXTRACT_PACKAGE        = YES
 
 # If the EXTRACT_STATIC tag is set to YES all static members of a file will be
 # included in the documentation.

diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -54,6 +54,8 @@
     'sphinxcontrib.bibtex'
 ]
 
+bibtex_bibfiles = ['references.bib']
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 

diff --git a/reg_tests/grid_gen/driver.cray.sh b/reg_tests/grid_gen/driver.cray.sh
@@ -66,10 +66,17 @@ bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.uniform -W 0:15
 bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J gfdl.regional -W 0:10 -M 2400 \
         -w 'ended(c96.uniform)' -extsched 'CRAYLINUX[]' "export NODES=1; $PWD/gfdl.regional.sh"
 
+#-----------------------------------------------------------------------------
+# ESG regional grid
+#-----------------------------------------------------------------------------
+
+bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J esg.regional -W 0:10 -M 2400 \
+        -w 'ended(gfdl.regional)' -extsched 'CRAYLINUX[]' "export NODES=1; $PWD/esg.regional.sh"
+
 #-----------------------------------------------------------------------------
 # Create summary log.
 #-----------------------------------------------------------------------------
 
-bsub -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J summary -R "rusage[mem=100]" -W 0:01 -w 'ended(gfdl.regional)' "grep -a '<<<' $LOG_FILE >> $SUM_FILE"
+bsub -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J summary -R "rusage[mem=100]" -W 0:01 -w 'ended(esg.regional)' "grep -a '<<<' $LOG_FILE >> $SUM_FILE"
 
 exit
diff --git a/reg_tests/grid_gen/driver.dell.sh b/reg_tests/grid_gen/driver.dell.sh
@@ -63,9 +63,16 @@ bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J c96.uniform -W 0:15
 bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J gfdl.regional -W 0:10 -x -n 24 -w 'ended(c96.uniform)' \
         -R "span[ptile=24]" -R "affinity[core(1):distribute=balance]" "$PWD/gfdl.regional.sh"
 
+#-----------------------------------------------------------------------------
+# ESG regional grid
+#-----------------------------------------------------------------------------
+
+bsub -e $LOG_FILE -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J esg.regional -W 0:10 -x -n 24 -w 'ended(gfdl.regional)' \
+        -R "span[ptile=24]" -R "affinity[core(1):distribute=balance]" "$PWD/esg.regional.sh"
+
 #-----------------------------------------------------------------------------
 # Create summary log.
 #-----------------------------------------------------------------------------
 
 bsub -o $LOG_FILE -q $QUEUE -P $PROJECT_CODE -J summary -R "affinity[core(1)]" -R "rusage[mem=100]" -W 0:01 \
-     -w 'ended(gfdl.regional)' "grep -a '<<<' $LOG_FILE >> $SUM_FILE"
+     -w 'ended(esg.regional)' "grep -a '<<<' $LOG_FILE >> $SUM_FILE"
diff --git a/reg_tests/grid_gen/driver.hera.sh b/reg_tests/grid_gen/driver.hera.sh
@@ -68,12 +68,19 @@ TEST1=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:15:00 -A $PROJECT_
 TEST2=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J gfdl.regional \
       -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST1 ./gfdl.regional.sh)
 
+#-----------------------------------------------------------------------------
+# esg regional grid
+#-----------------------------------------------------------------------------
+
+TEST3=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J esg.regional \
+      -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST2 ./esg.regional.sh)
+
 #-----------------------------------------------------------------------------
 # Create summary log.
 #-----------------------------------------------------------------------------
 
 sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J grid_summary -o $LOG_FILE -e $LOG_FILE \
-       --open-mode=append -q $QUEUE -d afterok:$TEST2 << EOF
+       --open-mode=append -q $QUEUE -d afterok:$TEST3 << EOF
 #!/bin/bash
 grep -a '<<<' $LOG_FILE  > $SUM_FILE
 EOF
diff --git a/reg_tests/grid_gen/driver.jet.sh b/reg_tests/grid_gen/driver.jet.sh
@@ -66,12 +66,19 @@ TEST1=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:15:00 -A $PROJECT_
 TEST2=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J gfdl.regional \
       --partition=xjet -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST1 ./gfdl.regional.sh)
 
+#-----------------------------------------------------------------------------
+# ESG regional grid
+#-----------------------------------------------------------------------------
+
+TEST3=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J esg.regional \
+      --partition=xjet -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST2 ./esg.regional.sh)
+
 #-----------------------------------------------------------------------------
 # Create summary log.
 #-----------------------------------------------------------------------------
 
 sbatch --partition=xjet --nodes=1  -t 0:01:00 -A $PROJECT_CODE -J grid_summary -o $LOG_FILE -e $LOG_FILE \
-       --open-mode=append -q $QUEUE -d afterok:$TEST2 << EOF
+       --open-mode=append -q $QUEUE -d afterok:$TEST3 << EOF
 #!/bin/bash
 grep -a '<<<' $LOG_FILE  > $SUM_FILE
 EOF
diff --git a/reg_tests/grid_gen/driver.orion.sh b/reg_tests/grid_gen/driver.orion.sh
@@ -56,18 +56,25 @@ TEST1=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:15:00 -A $PROJECT_
       -o $LOG_FILE -e $LOG_FILE ./c96.uniform.sh)
 
 #-----------------------------------------------------------------------------
-# C96 regional grid
+# GFDL regional grid
 #-----------------------------------------------------------------------------
 
 TEST2=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J gfdl.regional \
       --open-mode=append -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST1 ./gfdl.regional.sh)
 
+#-----------------------------------------------------------------------------
+# ESG regional grid
+#-----------------------------------------------------------------------------
+
+TEST3=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J esg.regional \
+      --open-mode=append -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST2 ./esg.regional.sh)
+
 #-----------------------------------------------------------------------------
 # Create summary log.
 #-----------------------------------------------------------------------------
 
 sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J grid_summary -o $LOG_FILE -e $LOG_FILE \
-       --open-mode=append -q $QUEUE -d afterok:$TEST2 << EOF
+       --open-mode=append -q $QUEUE -d afterok:$TEST3 << EOF
 #!/bin/bash
 grep -a '<<<' $LOG_FILE  > $SUM_FILE
 EOF
diff --git a/reg_tests/grid_gen/esg.regional.sh b/reg_tests/grid_gen/esg.regional.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+
+#-----------------------------------------------------------------------
+# Create a regional esg grid.  Compare output to a set
+# of baseline files using the 'nccmp' utility.  This script is
+# run by the machine specific driver script.
+#-----------------------------------------------------------------------
+
+set -x
+
+export TEMP_DIR=${WORK_DIR}/esg.regional.work
+export out_dir=${WORK_DIR}/esg.regional
+
+export gtype=regional_esg
+export target_lon=-97.5      # Center longitude of the highest resolution tile
+export target_lat=35.5       # Center latitude of the highest resolution tile
+export idim=1301             # Dimension of grid in 'i' direction
+export jdim=600              # Dimension of grid in 'j' direction
+export delx=0.0145           # Grid spacing in degrees in 'i' direction
+export dely=0.0145           # Grid spacing in degrees in 'j' direction
+export halo=3
+
+NCCMP=${NCCMP:-$(which nccmp)}
+
+#-----------------------------------------------------------------------
+# Start script.
+#-----------------------------------------------------------------------
+
+echo "Starting at: " `date`
+
+$home_dir/ush/fv3gfs_driver_grid.sh
+
+iret=$?
+if [ $iret -ne 0 ]; then
+  set +x
+  echo "<<< ESG REGIONAL TEST FAILED. <<<"
+  exit $iret
+fi
+
+echo "Ending at: " `date`
+
+#-----------------------------------------------------------------------------
+# Compare output to baseline set of data.
+#-----------------------------------------------------------------------------
+
+cd $out_dir/C3113
+
+test_failed=0
+for files in *tile*.nc ./fix_sfc/*tile*.nc
+do
+  if [ -f $files ]; then
+    echo CHECK $files
+    $NCCMP -dmfqS $files $HOMEreg/esg.regional/$files
+    iret=$?
+    if [ $iret -ne 0 ]; then
+      test_failed=1
+    fi
+  fi
+done
+
+set +x
+if [ $test_failed -ne 0 ]; then
+  echo "<<< ESG REGIONAL TEST FAILED. >>>"
+else
+  echo "<<< ESG REGIONAL TEST PASSED. >>>"
+fi
+
+exit 0
diff --git a/sorc/chgres_cube.fd/CMakeLists.txt b/sorc/chgres_cube.fd/CMakeLists.txt
@@ -9,12 +9,12 @@ set(fortran_src
     grib2_util.F90
     input_data.F90
     model_grid.F90
-    program_setup.f90
-    search_util.f90
+    program_setup.F90
+    search_util.F90
     static_data.F90
     surface.F90
     thompson_mp_climo_data.F90
-    utils.f90
+    utils.F90
     write_data.F90)
 
 if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel)$")

diff --git a/sorc/chgres_cube.fd/atmosphere.F90 b/sorc/chgres_cube.fd/atmosphere.F90
@@ -1,7 +1,8 @@
 !> @file
 !! @brief Process atmospheric fields
-!!
-!! Process atmospheric fields:  Horizontally interpolate input
+!! @author George Gayno NCEP/EMC
+
+!> Process atmospheric fields:  Horizontally interpolate input
 !! fields to the target grid.  Adjust surface pressure according to
 !! terrain difference between input and target grids.  Vertically
 !! interpolate to target grid vertical levels.  Processing based on
@@ -14,7 +15,7 @@
 !! 'west' edge of the grid box.  Otherwise, fields are at the center
 !! of the grid box.
 !!
-!! @author gayno NCEP/EMC
+!! @author George Gayno NCEP/EMC
  module atmosphere
 
  use esmf

diff --git a/sorc/chgres_cube.fd/grib2_util.F90 b/sorc/chgres_cube.fd/grib2_util.F90
@@ -1,6 +1,9 @@
 !> @file
 !! @brief Utilities for use when reading grib2 data.
-!!
+!! @author George Gayno NCEP/EMC
+
+!> Utilities for use when reading grib2 data.
+!! @author George Gayno NCEP/EMC
 module grib2_util
 
 use esmf
@@ -85,30 +88,4 @@ subroutine convert_omega(omega,p,t,q,clb,cub)
 
 end subroutine convert_omega
 
-!> Convert string from lower to uppercase.
-!! @author Clive Page
-!!
-!! Adapted from http://www.star.le.ac.uk/~cgp/fortran.html (25 May 2012)
-!!
-!! @param[in] strIn   string to convert
-!! @return strOut string in uppercase
-function to_upper(strIn) result(strOut)
-
-     implicit none
-
-     character(len=*), intent(in) :: strIn
-     character(len=len(strIn)) :: strOut
-     integer :: i,j
-
-     do i = 1, len(strIn)
-          j = iachar(strIn(i:i))
-          if (j>= iachar("a") .and. j<=iachar("z") ) then
-               strOut(i:i) = achar(iachar(strIn(i:i))-32)
-          else
-               strOut(i:i) = strIn(i:i)
-          end if
-     end do
-
-end function to_upper
-
  end module grib2_util
diff --git a/sorc/chgres_cube.fd/input_data.F90 b/sorc/chgres_cube.fd/input_data.F90
@@ -1,15 +1,16 @@
 !> @file
 !! @brief Read input data
 !! @author George Gayno NCEP/EMC
-!!
-!! Read atmospheric, surface and nst data on the input grid.
+
+!> Read atmospheric, surface and nst data on the input grid.
 !! Supported formats include fv3 tiled 'restart' files, fv3 tiled 
 !! 'history' files, fv3 gaussian history files, spectral gfs
 !! gaussian nemsio files, and spectral gfs sigio/sfcio files.
 !!
 !! Public variables are defined below: "input" indicates field
 !! associated with the input grid.
 !!
+!! @author George Gayno NCEP/EMC
  module input_data
 
  use esmf
@@ -4579,7 +4580,6 @@ end subroutine read_input_sfc_netcdf_file
  subroutine read_input_sfc_grib2_file(localpet)
 
    use wgrib2api
-   use grib2_util, only    : to_upper
    use program_setup, only : vgtyp_from_climo, sotyp_from_climo
    use model_grid, only    : input_grid_type
    use search_util
@@ -4592,9 +4592,9 @@ subroutine read_input_sfc_grib2_file(localpet)
    character(len=250)                    :: the_file
    character(len=250)                    :: geo_file
    character(len=20)                     :: vname, vname_file,slev
-
-   character(len=50)                      :: method
-
+   character(len=50)                     :: method
+   character(len=20)                     :: to_upper
+ 
    integer                               :: rc, varnum, iret, i, j,k
    integer                               :: ncid2d, varid, varsize
    integer, parameter                    :: icet_default = 265.0

diff --git a/sorc/chgres_cube.fd/model_grid.F90 b/sorc/chgres_cube.fd/model_grid.F90
@@ -1,7 +1,8 @@
 !> @file
 !! @brief Specify input and target model grids.
-!!
-!! Specify input and target model grids via ESMF grid objects.
+!! @author George Gayno NCEP/EMC
+
+!> Specify input and target model grids via ESMF grid objects.
 !!
 !! @author George Gayno NCEP/EMC
  module model_grid

diff --git a/sorc/chgres_cube.fd/program_setup.f90 → sorc/chgres_cube.fd/program_setup.F90 b/sorc/chgres_cube.fd/program_setup.f90 → sorc/chgres_cube.fd/program_setup.F90
@@ -1,7 +1,9 @@
 !> @file
-!! @brief Set up program execution
-!!
-!! Set up program execution
+!! @brief Set up program execution.
+!! @author George Gayno NCEP/EMC
+
+!> This module contains code to read the setup namelist file, handle
+!! the varmap file for GRIB2 data, and calculate the soil parameters.
 !!
 !! @author George Gayno NCEP/EMC
  module program_setup
@@ -144,7 +146,8 @@ module program_setup
 
 !> Reads program configuration namelist.
 !!
-!! @param filename the name of the configuration file (defaults to ./fort.41).
+!! @param filename the name of the configuration file (defaults to
+!! ./fort.41).
 !! @author George Gayno NCEP/EMC
  subroutine read_setup_namelist(filename)
  implicit none
@@ -443,6 +446,27 @@ end subroutine get_var_cond
 !> Driver routine to compute soil parameters for each
 !! soil type. Works for Zobler and STATSGO soil categories.
 !!
+!! The calculations are those used in the Noah Land Surface Model. For
+!! more information see <a
+!! href="https://doi.org/10.1029/2002JD003296">Implementation of Noah
+!! land surface model advances in the National Centers for
+!! Environmental Prediction operational mesoscale Eta model</a>.
+!!
+!! For more details about the soil parameters/properties see <a
+!! href="https://journals.ametsoc.org/view/journals/mwre/129/4/1520-0493_2001_129_0569_caalsh_2.0.co_2.xml">Coupling
+!! an Advanced Land Surface–Hydrology Model with the Penn State–NCAR
+!! MM5 Modeling System. Part I: Model Implementation and
+!! Sensitivity</a>.
+!!
+!! The original source for soil properties is here:
+!!
+!! Cosby, B. J., G. M. Hornberger, R. B. Clapp, and T. R. Ginn, 1984:
+!! <a
+!! href="https://agupubs.onlinelibrary.wiley.com/doi/10.1029/WR020i006p00682">A
+!! statistical exploration of the relationships of soil moisture
+!! characteristics to the physical properties of soils</a>. Water
+!! Resour. Res.,20, 682–690.
+!!
 !! @param [in] localpet  ESMF local persistent execution thread
 !! @author George Gayno NCEP/EMC
  subroutine calc_soil_params_driver(localpet)
@@ -588,8 +612,8 @@ subroutine calc_soil_params_driver(localpet)
 
  end subroutine calc_soil_params_driver
 
-!> Compute soil parameters.  Will be used to rescale soil moisture
-!! differences in soil type between the input grid and target
+!> Compute soil parameters. These will be used to rescale soil
+!! moisture differences in soil type between the input grid and target
 !! model grid.
 !!
 !! @param [in] num_soil_cats  number of soil type categories