Skip to content

Commit

Permalink
Merge NCAR:dtc/develop into develop 2020/04/14 (#102)
Browse files Browse the repository at this point in the history
* add s4 rules

* update submodule pointers

* update the branch in gitmodules to point to NEMS::feature/s4

* Update .gitmodules and submodule pointer for ccpp-physics for code review and testing

* Merge NCAR:dtc/develop into ufs-community:develop

- for macOS with LLVM clang, turn off OpenMP (from UFS public release)
- switch to Intel 18 on wcoss_cray
- new regression tests with RRTMGP using current control as basis

Co-authored-by: Rahul Mahajan <rmahajan@s4-submit.ssec.wisc.edu>
Co-authored-by: Rahul Mahajan <aerorahul@users.noreply.github.com>
  • Loading branch information
3 people committed Apr 17, 2020
1 parent 7c3cdca commit 3b16cad
Show file tree
Hide file tree
Showing 43 changed files with 259,439 additions and 99,779 deletions.
6 changes: 4 additions & 2 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
cmake_minimum_required(VERSION 2.8.12)
cmake_minimum_required(VERSION 3.0)

foreach(env_var IN ITEMS
CMAKE_C_COMPILER CMAKE_CXX_COMPILER CMAKE_Fortran_COMPILER
Expand All @@ -13,7 +13,9 @@ set(CMAKE_CXX_COMPILER $ENV{CMAKE_CXX_COMPILER})
set(CMAKE_Fortran_COMPILER $ENV{CMAKE_Fortran_COMPILER})
set(CMAKE_Platform $ENV{CMAKE_Platform})

project(NEMSfv3gfs C CXX Fortran)
project(ufs-weather-model
VERSION 1.0
LANGUAGES C CXX Fortran)

list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake)
find_package(MPI REQUIRED)
Expand Down
5 changes: 5 additions & 0 deletions cmake/configure_macosx.gnu.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,12 @@ option(DEBUG "Enable DEBUG mode" OFF)
option(REPRO "Enable REPRO mode" OFF)
option(VERBOSE "Enable VERBOSE mode" OFF)
option(32BIT "Enable 32BIT (single precision arithmetic in dycore)" OFF)
# OpenMP broken for clang compiler
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang*")
option(OPENMP "Enable OpenMP threading" OFF)
else()
option(OPENMP "Enable OpenMP threading" ON)
endif()
option(AVX2 "Enable AVX2 instruction set" OFF)

option(INLINE_POST "Enable inline post" OFF)
Expand Down
4 changes: 2 additions & 2 deletions compsets/hera.input
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ platform hera.intel {
# INPUTS is the input directory, which should contain fix and parm
# files, plus any restarts or other inputs.

BASELINE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200323/INTEL"
BASELINE_TEMPLATE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200323/INTEL"
BASELINE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200413/INTEL"
BASELINE_TEMPLATE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200413/INTEL"
INPUTS="@[BASELINE]"

default_resources={
Expand Down
16 changes: 8 additions & 8 deletions compsets/wcoss.input
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ platform wcoss.phase2 {
LONG_TEST_QUEUE='&LONGQ;'
BUILD_QUEUE='&BUILDQ;'
MACHINE_ID='wcoss.phase2'
BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200323"
BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200323"
BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200413"
BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200413"

default_resources={
TASKS=156
Expand Down Expand Up @@ -126,8 +126,8 @@ platform wcoss.phase1 {
C768_THRD=4

MACHINE_ID='wcoss.phase1'
BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200323"
BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200323"
BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200413"
BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200413"

# WCOSS Phase 2 has been slow of late.
DEFAULT_TEST_WALLTIME=2700
Expand Down Expand Up @@ -213,8 +213,8 @@ platform wcoss_dell_p3 {
LONG_TEST_QUEUE='&LONGQ;'
BUILD_QUEUE='&BUILDQ;'

BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200323/"
BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200323/"
BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200413/"
BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200413/"

default_resources={
TASKS=156
Expand Down Expand Up @@ -314,8 +314,8 @@ platform wcoss.cray {
BUILD_WALLTIME="3600"
DEFAULT_TEST_WALLTIME=1800

BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200323"
BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200323"
BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200413"
BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200413"

execution_time_modules=[[[
module load alps
Expand Down
193 changes: 193 additions & 0 deletions conf/configure.fv3.s4.intel
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
## NEMS configuration file
##
## Platform: SSEC Wisconsin S4
## Compiler: Intel with IntelMPI

SHELL=/bin/sh

################################################################################
## Include the common configuration parts

ifdef InNemsMakefile
include $(TOP)/conf/configure.nems.NUOPC
endif

############
# commands #
############
FC = mpiifort
CC = mpiicc
CXX = mpiicpc
LD = mpiifort -mkl=sequential

#########
# flags #
#########
# default is 64-bit OpenMP non-hydrostatic build using AVX2
DEBUG =
REPRO =
VERBOSE =
OPENMP = Y
AVX2 = N
HYDRO = N
CCPP = N
STATIC = N
INLINE_POST=N

include $(ESMFMKFILE)
ESMF_INC = $(ESMF_F90COMPILEPATHS)

NEMSIOINC = -I$(NEMSIO_INC)
NCEPLIBS = $(NEMSIO_LIB) $(BACIO_LIB4) $(SP_LIBd) $(W3EMC_LIBd) $(W3NCO_LIBd)

##############################################
# Need to use at least GNU Make version 3.81 #
##############################################
need := 3.81
ok := $(filter $(need),$(firstword $(sort $(MAKE_VERSION) $(need))))
ifneq ($(need),$(ok))
$(error Need at least make version $(need). Load module gmake/3.81)
endif

NETCDF_ROOT = $(NETCDF_DIR)
NETCDF = $(NETCDF_DIR)
INCLUDE = -I$(NETCDF_ROOT)/include
NETCDF_INC = -I$(NETCDF_ROOT)/include
ifneq ($(findstring netcdf/4,$(LOADEDMODULES)),)
NETCDF_LIB += -L$(NETCDF)/lib -lnetcdff -lnetcdf
else
NETCDF_LIB = -L$(NETCDF)/lib -lnetcdff -lnetcdf
endif

FPPFLAGS := -fpp -Wp,-w $(INCLUDE)
CFLAGS := $(INCLUDE)

FFLAGS := $(INCLUDE) -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -nowarn -sox -align array64byte

CPPDEFS += -Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO -DUSE_GFSL63 -DGFS_PHYS -Duse_WRTCOMP
CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML

ifeq ($(INLINE_POST),Y)
CPPDEFS +=
NCEPLIBS = $(POST_LIB) $(NEMSIO_LIB) $(G2_LIB4) $(G2TMPL_LIB) $(BACIO_LIB4) $(SP_LIBd) $(W3EMC_LIBd) $(W3NCO_LIBd) $(CRTM_LIB) $(PNG_LIB) $(JASPER_LIB) $(Z_LIB)
else
CPPDEFS += -DNO_INLINE_POST
endif

ifeq ($(HYDRO),Y)
CPPDEFS +=
else
CPPDEFS += -DMOIST_CAPPA -DUSE_COND
endif

ifeq ($(NAM_phys),Y)
CPPDEFS += -DNAM_phys
endif

ifeq ($(32BIT),Y)
CPPDEFS += -DOVERLOAD_R4 -DOVERLOAD_R8
FFLAGS += -i4 -real-size 32
else
ifeq ($(REPRO),Y)
FFLAGS += -i4 -real-size 64
else
FFLAGS += -i4 -real-size 64 -no-prec-div -no-prec-sqrt
endif
endif

ifeq ($(REPRO),Y)
FFLAGS += -qno-opt-dynamic-align
CFLAGS += -qno-opt-dynamic-align
else
ifeq ($(AVX2),Y)
FFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align
CFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align
else
FFLAGS += -xHOST -qno-opt-dynamic-align
CFLAGS += -xHOST -qno-opt-dynamic-align
endif
endif

ifeq ($(MULTI_GASES),Y)
CPPDEFS += -DMULTI_GASES
endif

FFLAGS_OPT = -O2 -debug minimal -fp-model strict -qoverride-limits -qopt-prefetch=3 -g -traceback
FFLAGS_REPRO = -O2 -debug minimal -fp-model strict -qoverride-limits -g -traceback
FFLAGS_DEBUG = -g -O0 -check -check noarg_temp_created -check nopointer -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -traceback -ftrapuv

TRANSCENDENTALS := -fast-transcendentals
FFLAGS_OPENMP = -qopenmp
FFLAGS_VERBOSE = -v -V -what

CFLAGS += -D__IFC -sox -fp-model strict

CFLAGS_OPT = -O2 -debug minimal
CFLAGS_REPRO = -O2 -debug minimal
CFLAGS_OPENMP = -qopenmp
CFLAGS_DEBUG = -O0 -g -ftrapuv -traceback

# Optional Testing compile flags. Mutually exclusive from DEBUG, REPRO, and OPT
# *_TEST will match the production if no new option(s) is(are) to be tested.
FFLAGS_TEST = -O2 -debug minimal -fp-model strict -qoverride-limits
CFLAGS_TEST = -O2

LDFLAGS :=
LDFLAGS_OPENMP := -qopenmp
LDFLAGS_VERBOSE := -Wl,-V,--verbose,-cref,-M

# start with blank LIBS
LIBS :=

ifeq ($(REPRO),Y)
CPPDEFS += -DREPRO
CFLAGS += $(CFLAGS_REPRO)
FFLAGS += $(FFLAGS_REPRO)
FAST :=
else ifeq ($(DEBUG),Y)
CPPDEFS += -DDEBUG
CFLAGS += $(CFLAGS_DEBUG)
FFLAGS += $(FFLAGS_DEBUG)
FAST :=
else ifeq ($(TEST),Y)
CFLAGS += $(CFLAGS_TEST)
FFLAGS += $(FFLAGS_TEST)
FAST :=
else
CFLAGS += $(CFLAGS_OPT)
FFLAGS += $(FFLAGS_OPT)
FAST := $(TRANSCENDENTALS)
endif

ifeq ($(OPENMP),Y)
CPPDEFS += -DOPENMP
CFLAGS += $(CFLAGS_OPENMP)
FFLAGS += $(FFLAGS_OPENMP)
LDFLAGS += $(LDFLAGS_OPENMP)
endif

ifeq ($(VERBOSE),Y)
CFLAGS += $(CFLAGS_VERBOSE)
FFLAGS += $(FFLAGS_VERBOSE)
LDFLAGS += $(LDFLAGS_VERBOSE)
endif

ifeq ($(CCPP),Y)
CPPDEFS += -DCCPP
CFLAGS += -I$(PATH_CCPP)/include
FFLAGS += -I$(PATH_CCPP)/include
ifeq ($(STATIC),Y)
CPPDEFS += -DSTATIC
LDFLAGS += -L$(PATH_CCPP)/lib -lccppphys -lccpp $(NCEPLIBS) -lxml2
else
LDFLAGS += -L$(PATH_CCPP)/lib -lccpp
endif
endif

LDFLAGS += $(LIBS)

ifdef InNemsMakefile
FFLAGS += $(ESMF_INC)
CPPFLAGS += -traditional
EXTLIBS = $(NCEPLIBS) $(ESMF_LIB) $(LDFLAGS) $(NETCDF_LIB)
endif
Loading

0 comments on commit 3b16cad

Please sign in to comment.