Merge pull request #4 from NCAR/dtc/develop

update
climbfuji · Jul 14, 2020 · 334fa69 · 334fa69
2 parents 644b2f2 + c22f9db
commit 334fa69
Show file tree

Hide file tree

Showing 85 changed files with 98,778 additions and 3,266 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -101,89 +101,92 @@ add_library(
     FMS/amip_interp/amip_interp.F90
     FMS/astronomy/astronomy.F90
     FMS/axis_utils/axis_utils.F90
+    FMS/axis_utils/axis_utils2.F90
     FMS/block_control/block_control.F90
     FMS/column_diagnostics/column_diagnostics.F90
     FMS/constants/constants.F90
     FMS/coupler/atmos_ocean_fluxes.F90
     FMS/coupler/coupler_types.F90
     FMS/coupler/ensemble_manager.F90
     FMS/data_override/data_override.F90
+    FMS/diag_integral/diag_integral.F90
     FMS/diag_manager/diag_axis.F90
     FMS/diag_manager/diag_data.F90
     FMS/diag_manager/diag_grid.F90
     FMS/diag_manager/diag_manager.F90
-    FMS/diag_manager/diag_manifest.F90
     FMS/diag_manager/diag_output.F90
     FMS/diag_manager/diag_table.F90
     FMS/diag_manager/diag_util.F90
     FMS/drifters/cloud_interpolator.F90
+    FMS/drifters/drifters.F90
     FMS/drifters/drifters_comm.F90
     FMS/drifters/drifters_core.F90
-    FMS/drifters/drifters.F90
     FMS/drifters/drifters_input.F90
     FMS/drifters/drifters_io.F90
     FMS/drifters/quicksort.F90
     FMS/exchange/stock_constants.F90
     FMS/exchange/xgrid.F90
-    FMS/fft/fft99.F90
-    FMS/fft/fft.F90
     FMS/field_manager/field_manager.F90
     FMS/field_manager/fm_util.F90
     FMS/fms/fms.F90
     FMS/fms/fms_io.F90
+    FMS/fms2_io/blackboxio.F90
+    FMS/fms2_io/fms2_io.F90
+    FMS/fms2_io/fms_io_utils.F90
+    FMS/fms2_io/fms_netcdf_domain_io.F90
+    FMS/fms2_io/fms_netcdf_unstructured_domain_io.F90
+    FMS/fms2_io/netcdf_io.F90
+    FMS/horiz_interp/horiz_interp.F90
     FMS/horiz_interp/horiz_interp_bicubic.F90
     FMS/horiz_interp/horiz_interp_bilinear.F90
     FMS/horiz_interp/horiz_interp_conserve.F90
-    FMS/horiz_interp/horiz_interp.F90
     FMS/horiz_interp/horiz_interp_spherical.F90
     FMS/horiz_interp/horiz_interp_type.F90
     FMS/interpolator/interpolator.F90
     FMS/memutils/memutils.F90
+    FMS/monin_obukhov/monin_obukhov.F90
+    FMS/monin_obukhov/monin_obukhov_kernel.F90
     FMS/mosaic/gradient.F90
     FMS/mosaic/grid.F90
     FMS/mosaic/mosaic.F90
+    FMS/mosaic/mosaic2.F90
+    FMS/mpp/mpp.F90
     FMS/mpp/mpp_data.F90
     FMS/mpp/mpp_domains.F90
     FMS/mpp/mpp_efp.F90
-    FMS/mpp/mpp.F90
     FMS/mpp/mpp_io.F90
     FMS/mpp/mpp_memutils.F90
     FMS/mpp/mpp_parameter.F90
     FMS/mpp/mpp_pset.F90
     FMS/mpp/mpp_utilities.F90
-    FMS/oda_tools/oda_core_ecda.F90
-    FMS/oda_tools/oda_core.F90
-    FMS/oda_tools/oda_types.F90
-    FMS/oda_tools/write_ocean_data.F90
-    FMS/oda_tools/xbt_drop_rate_adjust.f90
     FMS/platform/platform.F90
     FMS/random_numbers/MersenneTwister.F90
     FMS/random_numbers/random_numbers.F90
     FMS/sat_vapor_pres/sat_vapor_pres.F90
     FMS/sat_vapor_pres/sat_vapor_pres_k.F90
-    FMS/station_data/station_data.F90
-    FMS/time_interp/time_interp_external.F90
     FMS/time_interp/time_interp.F90
+    FMS/time_interp/time_interp_external.F90
+    FMS/time_interp/time_interp_external2.F90
     FMS/time_manager/get_cal_time.F90
     FMS/time_manager/time_manager.F90
     FMS/topography/gaussian_topog.F90
     FMS/topography/topography.F90
     FMS/tracer_manager/tracer_manager.F90
     FMS/tridiagonal/tridiagonal.F90
-    FMS/memutils/memuse.c
+
     FMS/mosaic/create_xgrid.c
     FMS/mosaic/gradient_c2l.c
     FMS/mosaic/interp.c
     FMS/mosaic/mosaic_util.c
     FMS/mosaic/read_mosaic.c
-    FMS/affinity/affinity.c
+    FMS/mpp/mpp_memuse.c
     FMS/mpp/nsclock.c
     FMS/mpp/threadloc.c
 )
 # stupid cmake can not figure out dependency of fft.F90 on fft99.F90 because 'use fft99_mod' is inside ifdefs
 set_property(SOURCE FMS/fft/fft.F90 APPEND_STRING PROPERTY COMPILE_FLAGS "-DSGICRAY=0 -DNAGFFT=0")
 
-target_include_directories(fms PUBLIC ${PROJECT_SOURCE_DIR}/FMS/include FMS/include FMS/fms FMS/mpp/include)
+target_include_directories(fms PUBLIC ${PROJECT_SOURCE_DIR}/FMS/include FMS/include FMS/fms FMS/fms2_io/include FMS/mpp/include)
 target_include_directories(fms PRIVATE ${NETCDF_INC_DIR})
 
 set_target_properties(fms PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/FMS)

diff --git a/FMS b/FMS
diff --git a/FV3 b/FV3
diff --git a/build.sh b/build.sh
@@ -3,8 +3,7 @@ set -eu
 
 MYDIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P)
 
-export COMPILER=${COMPILER:?"Please set COMPILER environment variable [gnu|intel]"}
-export CMAKE_Platform=linux.${COMPILER}
+export CMAKE_Platform=${CMAKE_Platform:?"Please set the CMAKE_Platform environment variable, e.g. [macosx.gnu|linux.gnu|linux.intel|hera.intel|...]"}
 export CMAKE_C_COMPILER=${CMAKE_C_COMPILER:-mpicc}
 export CMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER:-mpicxx}
 export CMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER:-mpif90}

diff --git a/cmake/GNU.cmake b/cmake/GNU.cmake
@@ -61,3 +61,17 @@ if(INLINE_POST)
 else()
     message("INLINE_POST is disabled")
 endif()
+
+if(QUAD_PRECISION)
+    message("QUAD_PRECISION is ENABLED")
+    add_definitions(-DENABLE_QUAD_PRECISION)
+else()
+    message("QUAD_PRECISION is disabled")
+endif()
+
+if(MULTI_GASES)
+    message("MULTI_GASES is ENABLED")
+    add_definitions(-DMULTI_GASES)
+else()
+    message("MULTI_GASES is disabled")
+endif()
diff --git a/cmake/Intel.cmake b/cmake/Intel.cmake
@@ -50,6 +50,14 @@ if(OPENMP)
     add_definitions(-DOPENMP)
 endif()
 
+if(QUAD_PRECISION)
+    add_definitions(-DENABLE_QUAD_PRECISION)
+endif()
+
+if(MULTI_GASES)
+    add_definitions(-DMULTI_GASES)
+endif()
+
 
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__IFC -sox -fp-model source")
 
@@ -90,3 +98,13 @@ if(INLINE_POST)
 else()
     message("INLINE_POST is disabled")
 endif()
+if(QUAD_PRECISION)
+    message("QUAD_PRECISION is ENABLED")
+else()
+    message("QUAD_PRECISION is disabled")
+endif()
+if(MULTI_GASES)
+    message("MULTI_GASES is ENABLED")
+else()
+    message("MULTI_GASES is disabled")
+endif()
diff --git a/cmake/PGI.cmake b/cmake/PGI.cmake
@@ -60,3 +60,18 @@ if(INLINE_POST)
 else()
     message("INLINE_POST is disabled")
 endif()
+
+if(QUAD_PRECISION)
+    message("QUAD_PRECISION is ENABLED")
+    add_definitions(-DENABLE_QUAD_PRECISION)
+else()
+    message("QUAD_PRECISION is disabled")
+endif()
+
+if(MULTI_GASES)
+    message("MULTI_GASES is ENABLED")
+    add_definitions(-DMULTI_GASES)
+else()
+    message("MULTI_GASES is disabled")
+endif()
+
diff --git a/cmake/configure_cheyenne.gnu.cmake b/cmake/configure_cheyenne.gnu.cmake
@@ -16,6 +16,8 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" OFF)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
+option(MULTI_GASES  "Enable MULTI_GASES" OFF)
 
 option(INLINE_POST "Enable inline post" OFF)
 

diff --git a/cmake/configure_cheyenne.intel-impi.cmake b/cmake/configure_cheyenne.intel-impi.cmake
@@ -16,6 +16,8 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" ON)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
+option(MULTI_GASES  "Enable MULTI_GASES" OFF)
 
 option(INLINE_POST "Enable inline post" OFF)
 

diff --git a/cmake/configure_cheyenne.intel.cmake b/cmake/configure_cheyenne.intel.cmake
@@ -16,6 +16,8 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" ON)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
+option(MULTI_GASES  "Enable MULTI_GASES" OFF)
 
 option(INLINE_POST "Enable inline post" OFF)
 

diff --git a/cmake/configure_gaea.cmake b/cmake/configure_gaea.cmake
@@ -16,6 +16,8 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" ON)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
+option(MULTI_GASES  "Enable MULTI_GASES" OFF)
 
 option(INLINE_POST "Enable inline post" OFF)
 

diff --git a/cmake/configure_hera.gnu.cmake b/cmake/configure_hera.gnu.cmake
@@ -16,6 +16,8 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" OFF)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
+option(MULTI_GASES  "Enable MULTI_GASES" OFF)
 
 option(INLINE_POST "Enable inline post" OFF)
 

diff --git a/cmake/configure_hera.intel.cmake b/cmake/configure_hera.intel.cmake
@@ -16,6 +16,8 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" ON)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
+option(MULTI_GASES  "Enable MULTI_GASES" OFF)
 
 option(INLINE_POST "Enable inline post" ON)
 

diff --git a/cmake/configure_jet.intel.cmake b/cmake/configure_jet.intel.cmake
@@ -16,14 +16,18 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" OFF)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
+option(MULTI_GASES  "Enable MULTI_GASES" OFF)
 
 option(INLINE_POST "Enable inline post" OFF)
 
 include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake )
 
-message("AVX2 is   ENABLED on Jet (multi-tagret executable)")
+message("AVX2 is   ENABLED on Jet (multi-target executable)")
 string (REPLACE "-xHOST" "-axSSE4.2,AVX,CORE-AVX2" CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}")
 string (REPLACE "-xHOST" "-axSSE4.2,AVX,CORE-AVX2" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
+# For CCPP
+option(SIMDMULTIARCH "Enable multi-target SIMD instruction sets" ON)
 
 string(REPLACE "-i_dynamic" "-shared-intel"
        CMAKE_SHARED_LIBRARY_LINK_Fortran_FLAGS

diff --git a/cmake/configure_linux.gnu.cmake b/cmake/configure_linux.gnu.cmake
@@ -16,6 +16,8 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" OFF)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
+option(MULTI_GASES  "Enable MULTI_GASES" OFF)
 
 option(INLINE_POST "Enable inline post" OFF)
 

diff --git a/cmake/configure_linux.intel.cmake b/cmake/configure_linux.intel.cmake
@@ -16,6 +16,8 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" OFF)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
+option(MULTI_GASES  "Enable MULTI_GASES" OFF)
 
 option(INLINE_POST "Enable inline post" OFF)
 

diff --git a/cmake/configure_macosx.gnu.cmake b/cmake/configure_macosx.gnu.cmake
@@ -21,6 +21,8 @@ else()
 option(OPENMP  "Enable OpenMP threading" ON)
 endif()
 option(AVX2    "Enable AVX2 instruction set" OFF)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
+option(MULTI_GASES  "Enable MULTI_GASES" OFF)
 
 option(INLINE_POST "Enable inline post" OFF)
 

diff --git a/cmake/configure_orion.intel.cmake b/cmake/configure_orion.intel.cmake
@@ -16,6 +16,8 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" ON)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
+option(MULTI_GASES  "Enable MULTI_GASES" OFF)
 
 option(INLINE_POST "Enable inline post" ON)
 

diff --git a/cmake/configure_stampede.intel.cmake b/cmake/configure_stampede.intel.cmake
@@ -16,6 +16,8 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" ON)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
+option(MULTI_GASES  "Enable MULTI_GASES" OFF)
 
 option(INLINE_POST "Enable inline post" OFF)
 

diff --git a/cmake/configure_wcoss_cray.cmake b/cmake/configure_wcoss_cray.cmake
@@ -16,6 +16,8 @@ option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" ON)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
+option(MULTI_GASES  "Enable MULTI_GASES" OFF)
 
 option(INLINE_POST "Enable inline post" ON)
 

diff --git a/cmake/configure_wcoss_dell_p3.cmake b/cmake/configure_wcoss_dell_p3.cmake
@@ -15,7 +15,9 @@ option(REPRO   "Enable REPRO mode" OFF)
 option(VERBOSE "Enable VERBOSE mode" OFF)
 option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
-option(AVX2    "Enable AVX2 instruction set" OFF)
+option(AVX2    "Enable AVX2 instruction set" ON)
+option(QUAD_PRECISION  "Enable QUAD_PRECISION (for certain grid metric terms in dycore)" ON)
+option(MULTI_GASES  "Enable MULTI_GASES" OFF)
 
 option(INLINE_POST "Enable inline post" ON)
 

diff --git a/conf/configure.fv3.cheyenne.gnu b/conf/configure.fv3.cheyenne.gnu
@@ -56,6 +56,8 @@ AVX2 = Y
 HYDRO = N
 CCPP = N
 SION = N
+QUAD_PRECISION = Y
+MULTI_GASES = N
 
 include       $(ESMFMKFILE)
 ESMF_INC    = $(ESMF_F90COMPILEPATHS)
@@ -118,6 +120,10 @@ ifeq ($(MULTI_GASES),Y)
 CPPDEFS += -DMULTI_GASES
 endif
 
+ifeq ($(QUAD_PRECISION),Y)
+CPPDEFS += -DENABLE_QUAD_PRECISION
+endif
+
 FFLAGS_OPT = -O2 -fno-range-check
 FFLAGS_REPRO = -O2 -g -fbacktrace -fno-range-check
 FFLAGS_DEBUG = -g -O0 -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -ffpe-trap=invalid,zero,overflow -fbounds-check -fbacktrace -fno-range-check -Wall

diff --git a/conf/configure.fv3.cheyenne.intel b/conf/configure.fv3.cheyenne.intel
@@ -56,6 +56,8 @@ AVX2 = Y
 HYDRO = N
 CCPP = N
 SION = N
+QUAD_PRECISION = Y
+MULTI_GASES = N
 
 include       $(ESMFMKFILE)
 ESMF_INC    = $(ESMF_F90COMPILEPATHS)
@@ -127,6 +129,10 @@ ifeq ($(MULTI_GASES),Y)
 CPPDEFS += -DMULTI_GASES
 endif
 
+ifeq ($(QUAD_PRECISION),Y)
+CPPDEFS += -DENABLE_QUAD_PRECISION
+endif
+
 FFLAGS_OPT = -O2 -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3
 FFLAGS_REPRO = -O2 -debug minimal -fp-model consistent -qoverride-limits -g -traceback
 FFLAGS_DEBUG = -g -O0 -check all -check noarg_temp_created -check nopointer -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -traceback -ftrapuv

diff --git a/conf/configure.fv3.cheyenne.intel-impi b/conf/configure.fv3.cheyenne.intel-impi
@@ -52,6 +52,8 @@ AVX2 = Y
 HYDRO = N
 CCPP = N
 SION = N
+QUAD_PRECISION = Y
+MULTI_GASES = N
 
 include       $(ESMFMKFILE)
 ESMF_INC    = $(ESMF_F90COMPILEPATHS)
@@ -123,6 +125,10 @@ ifeq ($(MULTI_GASES),Y)
 CPPDEFS += -DMULTI_GASES
 endif
 
+ifeq ($(QUAD_PRECISION),Y)
+CPPDEFS += -DENABLE_QUAD_PRECISION
+endif
+
 FFLAGS_OPT = -O2 -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3
 FFLAGS_REPRO = -O2 -debug minimal -fp-model consistent -qoverride-limits -g -traceback
 FFLAGS_DEBUG = -g -O0 -check all -check noarg_temp_created -check nopointer -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -traceback -ftrapuv