Merge May 16 ufs-community develop to gsl/develop

.gitmodules

@@ -1,7 +1,7 @@
 [submodule "FV3"]
   path = FV3
-  url = https://github.com/NOAA-GSL/fv3atm
-  branch = gsl/develop
+  url = ssh://git@github.com/SamuelTrahanNOAA/fv3atm
+  branch = gsl/merge-community-to-develop
 [submodule "WW3"]
   path = WW3
   url = https://github.com/NOAA-EMC/WW3
@@ -38,3 +38,7 @@
   path = GOCART
   url = https://github.com/GEOS-ESM/GOCART
   branch = develop
+[submodule "AQM"]
+  path = AQM
+  url = https://github.com/NOAA-EMC/AQM
+  branch = develop

CDEPS-interface/CMakeLists.txt

@@ -1,6 +1,9 @@
 # CDEPS compiler flags
 if(CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
   set(CMAKE_Fortran_FLAGS "-g -fbacktrace -ffree-line-length-none")
+  if(${CMAKE_Fortran_COMPILER_VERSION} VERSION_GREATER_EQUAL 10)
+    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch -fallow-invalid-boz")
+  endif()
   set(CMAKE_Fortran_FLAGS_RELEASE "-O2")
   set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -fcheck=bounds -ffpe-trap=invalid,zero,overflow,underflow" )
 elseif(CMAKE_Fortran_COMPILER_ID MATCHES "Intel")
@@ -69,7 +72,7 @@ add_dependencies(docn share streams dshr)
 set_target_properties(docn PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
 target_include_directories(docn PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>)
 target_link_libraries(docn PRIVATE dshr share streams)
-target_link_libraries(docn PRIVATE esmf)
+target_link_libraries(docn PRIVATE esmf PIO::PIO_Fortran)
 
 # CDEPS/drof
 add_library(drof OBJECT ${cdeps_drof_files})

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.19)
 
 project(ufs
         VERSION 1.0
@@ -16,14 +16,15 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/CMakeModules/Modules)
 ###############################################################################
 
 # Valid applications and choices
-list(APPEND VALID_APPS ATM ATMAERO ATMW S2S S2SW HAFS HAFSW HAFS-ALL NG-GODAS)
+list(APPEND VALID_APPS ATM ATMAERO ATMAQ ATMW S2S S2SA S2SW S2SWA HAFS HAFSW HAFS-ALL NG-GODAS)
 set(APP NONE CACHE BOOL "Application Name")
 if(NOT (APP IN_LIST VALID_APPS))
   message(FATAL_ERROR "${APP} is not a valid application.\nValid Applications are: ${VALID_APPS}")
 endif()
 
 set(FMS             OFF CACHE BOOL "Enable FMS")
 set(FV3             OFF CACHE BOOL "Enable FV3")
+set(AQM             OFF CACHE BOOL "Enable AQM")
 set(UFS_GOCART      OFF CACHE BOOL "Enable GOCART")
 set(MOM6            OFF CACHE BOOL "Enable MOM6")
 set(HYCOM           OFF CACHE BOOL "Enable HYCOM")
@@ -40,6 +41,7 @@ include(cmake/configure_apps.cmake)
 message("")
 message("FMS .............. ${FMS}")
 message("FV3 .............. ${FV3}")
+message("AQM .............. ${AQM}")
 message("GOCART ........... ${UFS_GOCART}")
 message("MOM6 ............. ${MOM6}")
 message("HYCOM ............ ${HYCOM}")
@@ -60,9 +62,9 @@ set(DEBUG           OFF CACHE BOOL "Enable DEBUG mode")
 set(DEBUG_LINKMPI   ON  CACHE BOOL "Enable linkmpi option when DEBUG mode is on")
 set(INLINE_POST     OFF CACHE BOOL "Enable inline post")
 set(MULTI_GASES     OFF CACHE BOOL "Enable MULTI_GASES")
+set(MOVING_NEST     OFF CACHE BOOL "Enable moving nest code")
 set(OPENMP          ON  CACHE BOOL "Enable OpenMP threading")
 set(PARALLEL_NETCDF OFF CACHE BOOL "Enable parallel NetCDF")
-set(REPRO           OFF CACHE BOOL "Enable REPRO mode")
 set(JEDI_DRIVER     OFF CACHE BOOL "Enable JEDI as top level driver")
 
 set(CMAKE_Platform $ENV{CMAKE_Platform})
@@ -84,9 +86,9 @@ message("DEBUG ............ ${DEBUG}")
 message("DEBUG_LINKMPI .... ${DEBUG_LINKMPI}")
 message("INLINE_POST ...... ${INLINE_POST}")
 message("MULTI_GASES ...... ${MULTI_GASES}")
+message("MOVING_NEST ...... ${MOVING_NEST}")
 message("OPENMP ........... ${OPENMP}")
 message("PARALLEL_NETCDF .. ${PARALLEL_NETCDF}")
-message("REPRO ............ ${REPRO}")
 message("JEDI_DRIVER ...... ${JEDI_DRIVER}")
 message("")
 
@@ -144,6 +146,13 @@ if(FV3)
   add_subdirectory(FV3)
 endif()
 
+###############################################################################
+### AQM
+###############################################################################
+if(AQM)
+  add_subdirectory(AQM)
+endif()
+
 ###############################################################################
 ### GOCART
 ###############################################################################
@@ -155,27 +164,10 @@ endif()
 ### Wave components [WW3]
 ###############################################################################
 if(WW3)
-  if(CMAKE_Platform)
-    if(${CMAKE_Platform} MATCHES "linux*" OR ${CMAKE_Platform} MATCHES "macosx*")
-      if (CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
-        set(WW3_COMP "gnu")
-      elseif (CMAKE_Fortran_COMPILER_ID MATCHES "Intel")
-        set(WW3_COMP "intel")
-      endif()
-    else()
-      set(WW3_COMP ${CMAKE_Platform})
-    endif()
-  endif()
-  message("Build WW3:")
-  message("   run: ${CMAKE_BUILD_TOOL} WW3_PARCOMPN=4 WW3_COMP=${WW3_COMP} ww3_nemslibonly")
-  message("   in:  ${CMAKE_CURRENT_SOURCE_DIR}/WW3/model/esmf")
-  message("")
-
-  add_custom_target(ww3_nems
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/WW3/model/esmf
-    COMMAND ${CMAKE_BUILD_TOOL} WW3_PARCOMPN=4 WW3_COMP=${WW3_COMP} ww3_nemslibonly > ${CMAKE_CURRENT_BINARY_DIR}/ww3_make.log 2>&1)
-
-  set(WW3_LIBS ${CMAKE_CURRENT_SOURCE_DIR}/WW3/model/obj/libww3_multi_esmf.a)
+  # Set options for WW3 build
+  set(SWITCH "multi_esmf" CACHE STRING "Default to switch_multi_esmf")
+  set(MULTI_ESMF ON CACHE BOOL "Enable ESMF for WW3")
+  add_subdirectory(WW3)
 endif()
 
 ###############################################################################
@@ -220,17 +212,21 @@ if(JEDI_DRIVER)
   list(APPEND _ufs_defs_private JEDI_DRIVER=ON)
 endif()
 
+if(AQM)
+  add_dependencies(ufs aqm)
+  list(APPEND _ufs_defs_private FRONT_AQM=aqm)
+  list(APPEND _ufs_libs_public aqm)
+endif()
+
 if(UFS_GOCART)
   add_dependencies(ufs UFS_Aerosols)
   list(APPEND _ufs_defs_private FRONT_GOCART=Aerosol_Cap)
   list(APPEND _ufs_libs_public UFS_Aerosols)
 endif()
 
 if(WW3)
-  add_dependencies(ufs ww3_nems)
   list(APPEND _ufs_defs_private FRONT_WW3=WMESMFMD)
-  target_include_directories(ufs PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/WW3/model/mod)
-  list(APPEND _ufs_libs_public ${WW3_LIBS})
+  target_link_libraries(ufs PUBLIC WW3::WW3)
 endif()
 
 list(APPEND _ufs_libs_public esmf)

README.md

@@ -2,7 +2,7 @@
 
 # ufs-weather-model
 
-This is the UFS weather model source code.
+This is the UFS weather model source code
 
 # Where to find information
 

cmake/GNU.cmake

@@ -12,24 +12,7 @@ if(DEBUG)
     add_definitions(-DDEBUG)
     set(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -O0 -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -ffpe-trap=invalid,zero,overflow -fbounds-check")
     set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0")
-elseif(REPRO)
-    add_definitions(-DREPRO)
-    if (APPLE)
-        set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O0")
-        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O0")
-    else()
-        set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2")
-        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2")
-    endif()
 else()
-    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2")
+    set(CMAKE_Fortran_FLAGS_RELEASE "-O2")
+    set(CMAKE_C_FLAGS_RELEASE "-O2")
 endif()
-
-# For ccpp
-set(CMAKE_C_FLAGS_RELEASE       "-O2 -fPIC" CACHE STRING "" FORCE)
-set(CMAKE_CXX_FLAGS_RELEASE     "-O2 -fPIC" CACHE STRING "" FORCE)
-set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE)
-set(CMAKE_C_FLAGS_BITFORBIT       "-O2 -fPIC" CACHE STRING "" FORCE)
-set(CMAKE_CXX_FLAGS_BITFORBIT     "-O2 -fPIC" CACHE STRING "" FORCE)
-set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE)

cmake/Intel.cmake

@@ -16,38 +16,23 @@ if(DEBUG)
       endif()
     endif()
     set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -ftrapuv")
-elseif(REPRO)
-    add_definitions(-DREPRO)
-    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2 -debug minimal -fp-model consistent -qoverride-limits")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -debug minimal")
 else()
-    if(32BIT)
-      set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2 -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3")
-    else()
-      set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2 -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3 -no-prec-div -no-prec-sqrt")
-    endif()
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -debug minimal")
+    set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -debug minimal -qoverride-limits")
+    set(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -fp-model consistent")
+    set(CMAKE_C_FLAGS_RELEASE       "-O2 -debug minimal")
     set(FAST "-fast-transcendentals")
     if(AVX2)
-        set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -march=core-avx2")
-        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=core-avx2")
+        set(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -march=core-avx2")
+        set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -march=core-avx2")
     elseif(SIMDMULTIARCH)
-        set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -axSSE4.2,CORE-AVX2")
-        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -axSSE4.2,CORE-AVX2")
-     elseif(AVX)
-        set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -march=core-avx-i")
-        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=core-avx-i")
+        set(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -axSSE4.2,CORE-AVX2")
+        set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -axSSE4.2,CORE-AVX2")
+    elseif(AVX)
+        set(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -march=core-avx-i")
+        set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -march=core-avx-i")
     endif()
 endif()
 
-# For ccpp
-set(CMAKE_C_FLAGS_RELEASE       "-O2 -fPIC" CACHE STRING "" FORCE)
-set(CMAKE_CXX_FLAGS_RELEASE     "-O2 -fPIC" CACHE STRING "" FORCE)
-set(CMAKE_Fortran_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE)
-set(CMAKE_C_FLAGS_BITFORBIT       "-O2 -fPIC" CACHE STRING "" FORCE)
-set(CMAKE_CXX_FLAGS_BITFORBIT     "-O2 -fPIC" CACHE STRING "" FORCE)
-set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE)
-
 if(APPLE)
   # The linker on macOS does not include `common symbols` by default
   # Passing the -c flag includes them and fixes an error with undefined symbols

cmake/configure_apps.cmake

@@ -13,13 +13,16 @@
 ###############################################################################
 ### Configure Application Components
 ###############################################################################
-if(APP MATCHES "^(ATM|ATMW)$")
+if(APP MATCHES "^(ATM|ATMW|ATMAQ)$")
   set(FMS        ON  CACHE BOOL "Enable FMS"                 FORCE)
   set(FV3        ON  CACHE BOOL "Enable FV3"                 FORCE)
   set(STOCH_PHYS ON  CACHE BOOL "Enable Stochastic Physics"  FORCE)
   if(APP MATCHES "ATMW")
     set(WW3      ON  CACHE BOOL "Enable WAVEWATCH III"       FORCE)
     message("Configuring UFS app in Atmosphere with Waves mode")
+  elseif(APP MATCHES "ATMAQ")
+    set(AQM      ON  CACHE BOOL "Enable AQM"                 FORCE)
+    message("Configuring UFS app in Atmosphere with Air Quality mode")
   else()
     message("Configuring UFS app in Atmosphere Only mode")
   endif()
@@ -35,19 +38,27 @@ if(APP MATCHES "^(NG-GODAS)$")
   message("Configuring UFS app in (CDEPS) Data Atmosphere mode")
 endif()
 
-if(APP MATCHES "^(S2S|S2SW)$")
+if(APP MATCHES "^(S2S|S2SA|S2SW|S2SWA)$")
+  set(APP_MSG "Configuring UFS app in S2S")
   set(CMEPS      ON  CACHE BOOL "Enable CMEPS"               FORCE)
   set(FMS        ON  CACHE BOOL "Enable FMS"                 FORCE)
   set(FV3        ON  CACHE BOOL "Enable FV3"                 FORCE)
   set(MOM6       ON  CACHE BOOL "Enable MOM6"                FORCE)
   set(CICE6      ON  CACHE BOOL "Enable CICE6"               FORCE)
   set(STOCH_PHYS ON  CACHE BOOL "Enable Stochastic Physics"  FORCE)
-  if(APP MATCHES "S2SW")
+  if(APP MATCHES "^S2SW")
     set(WW3      ON  CACHE BOOL "Enable WAVEWATCH III"       FORCE)
-    message("Configuring UFS app in S2S with Waves mode")
-  else()
-    message("Configuring UFS app in S2S mode")
+    string(CONCAT APP_MSG ${APP_MSG} " with Waves")
+  endif()
+  if(APP MATCHES "A$")
+    set(UFS_GOCART ON  CACHE BOOL "Enable GOCART"            FORCE)
+    if(WW3)
+      string(CONCAT APP_MSG ${APP_MSG} " and Aerosols")
+    else()
+      string(CONCAT APP_MSG ${APP_MSG} " with Aerosols")
+    endif()
   endif()
+  message("${APP_MSG} mode")
 endif()
 
 if(APP MATCHES "^(HAFS|HAFSW|HAFS-ALL)$")

doc/UsersGuide/source/CodeOverview.rst

@@ -40,13 +40,13 @@ The ufs-weather-model repository supports the short- and medium-range UFS applic
     - https://github.com/NOAA-EMC/CICE
   * - NOAA/NCEP WAVEWATCH III Model (WW3)
     - https://github.com/NOAA-EMC/WW3
+  * - The Goddard Chemistry Aerosol Radiation and Transport (GOCART)
+    - https://github.com/GEOS-ESM/GOCART 
   * - NUOPC Community Mediator for Earth Prediction Systems (CMEPS)
     - https://github.com/NOAA-EMC/CMEPS
   * - Community Data Models for Earth Prediction Systems (CDEPS)
     - https://github.com/NOAA-EMC/CDEPS
 
-.. TODO:: GOCART information in progress
-
 In the table, the left column contains a description of each repository, and the right column shows the component repositories which are pointing to (or will point to) the authoritative repositories. The ufs-weather-model currently uses git submodule to manage the sub-components.
 
 The umbrella repository for the UFS Weather Model is named ufs-weather-model.  Under this repository reside a number of submodules that are nested in specific directories under the parent repository’s working directory.  When the ufs-weather-model repository is cloned, the *.gitmodules* file creates the following directories:
@@ -68,6 +68,7 @@ The umbrella repository for the UFS Weather Model is named ufs-weather-model.  U
    ├── CICE-interface
    │    └── CICE                               https://github.com/NOAA-EMC/CICE
    ├── WW3                                     https://github.com/NOAA-EMC/WW3
+   ├── GOCART                                  https://github.com/GEOS-ESM/GOCART
    ├── CMEPS-interface
    │    └── CMEPS                              https://github.com/NOAA-EMC/CMEPS
    ├── CDEPS-interface
@@ -118,6 +119,8 @@ When the ufs-weather-model is cloned, the basic directory structure will be simi
    ├── WW3
    │    └── model                                --------- WW3 model
    │        └── esmf                             --------- NUOPC WW3 cap
+   ├── GOCART
+   │    └── ESMF                                 --------- GOCART model
    ├── CDEPS-interface
    │     └── CDEPS
    │         ├── datm                            --------- CDEPS DATM
@@ -126,9 +129,7 @@ When the ufs-weather-model is cloned, the basic directory structure will be simi
    ├── tests                -------- regression test infrastructure
    │   └── parm
    │   └── tests
-   │   └── fv3_conf
-
-.. TODO:: GOCART information in progress
+   │   └── fv3_conf   
 
 The physics subdirectory in the *gfsphysics* directory  is not used or supported
 as part of this release (all physics is available through the :term:`CCPP` using