Changes to move SRW app towards NCO complaince (#337)

* Build to exec directory instead of bin. * Rename src to sorc * Add separate building of rrfs components. * Temporarily change bin directory from "exec" to "bin" * Remove --rrfs option from test/build, print packages to build. * Update hash of regional_workflow. * Minor changes. * Set default list for direct cmake build path. * Separte clean,build and install steps. * Bug fix remove vs clean * Remove custom install step cause it is same as all target anyway. * Update for EMC_EXEC_DIR removal from UPP. * Use modulefiles of component instead of top-level modulefile. * Make modulefile name search flexible. * Remove GSI dependency of rrfs_utl assuming gsi is already built. * Fallback to SRW modules when we cant find modules of sub-component.
ufs-community · Aug 30, 2022 · 032afc2 · 032afc2
1 parent 09028a4
commit 032afc2
Show file tree

Hide file tree

Showing 6 changed files with 315 additions and 130 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -45,15 +45,14 @@ FUNCTION(is_enabled feature ret_val)
   ENDIF(${feature})
 ENDFUNCTION()
 
-# User options
-option(ENABLE_RRFS "Build RRFS components" OFF)
-
 # Turn on testing.
 enable_testing()
 include(CTest)
 
-# Build in this subdirectory.
-#add_subdirectory(src)
+# Default bin directory
+if(NOT DEFINED CMAKE_INSTALL_BINDIR)
+    set(CMAKE_INSTALL_BINDIR bin)
+endif()
 
 #####
 # Configure and print the ufs-srweather-app.settings file.
@@ -122,7 +121,7 @@ MESSAGE(${UFS-SRWEATHER-APP_SETTINGS})
 # Install ufs_srweather_app.settings file into same location
 # as the app.
 INSTALL(FILES "${CMAKE_BINARY_DIR}/ufs_srweather_app.settings"
-  DESTINATION bin)
+  DESTINATION ${CMAKE_INSTALL_BINDIR})
 
 #####
 # Create 'ufs_srweather_app_meta.h' include file.
@@ -133,4 +132,6 @@ configure_file(
 
 FILE(COPY "${CMAKE_CURRENT_BINARY_DIR}/ufs_srweather_app_meta.h" DESTINATION include)
 
-add_subdirectory(src)
+add_custom_target(build DEPENDS sorc/pbuild)
+
+add_subdirectory(sorc)
diff --git a/Externals.cfg b/Externals.cfg
@@ -3,7 +3,7 @@ protocol = git
 repo_url = https://github.com/ufs-community/regional_workflow
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = 3a2cb82
+hash = b17ac12
 local_path = regional_workflow
 required = True
 
@@ -13,7 +13,7 @@ repo_url = https://github.com/ufs-community/UFS_UTILS
 # Specify either a branch name or a hash but not both.
 #branch = develop
 hash = c5cfcf4
-local_path = src/UFS_UTILS
+local_path = sorc/UFS_UTILS
 required = True
 
 [ufs-weather-model]
@@ -22,7 +22,7 @@ repo_url = https://github.com/ufs-community/ufs-weather-model
 # Specify either a branch name or a hash but not both.
 #branch = develop
 hash = 65cf401
-local_path = src/ufs-weather-model
+local_path = sorc/ufs-weather-model
 required = True
 
 [UPP]
@@ -31,7 +31,7 @@ repo_url = https://github.com/NOAA-EMC/UPP
 # Specify either a branch name or a hash but not both.
 #branch = develop
 hash = 8767929
-local_path = src/UPP
+local_path = sorc/UPP
 required = True
 
 [GSI]
@@ -40,7 +40,7 @@ repo_url = https://github.com/NOAA-EMC/GSI.git
 # Specify either a branch name or a hash but not both.
 #branch = develop
 hash = ad84f17
-local_path = src/gsi
+local_path = sorc/gsi
 externals = None
 required = True
 
@@ -50,7 +50,7 @@ repo_url = https://github.com/NOAA-GSL/rrfs_utl.git
 # Specify either a branch name or a hash but not both.
 #branch = develop
 hash = 8d21f25
-local_path = src/rrfs_utl
+local_path = sorc/rrfs_utl
 required = True
 
 [externals_description]

diff --git a/devbuild.sh b/devbuild.sh
@@ -3,7 +3,7 @@
 # usage instructions
 usage () {
 cat << EOF_USAGE
-Usage: $0 --platform=PLATFORM [OPTIONS]...
+Usage: $0 --platform=PLATFORM [OPTIONS] ... [TARGETS]
 
 OPTIONS
   -h, --help
@@ -19,8 +19,6 @@ OPTIONS
       (e.g. ATM | ATMW | S2S | S2SW)
   --ccpp="CCPP_SUITE1,CCPP_SUITE2..."
       CCPP suites (CCPP_SUITES) to include in build; delimited with ','
-  --rrfs
-      build rrfs system components: GSI, rrfs_utl
   --enable-options="OPTION1,OPTION2,..."
       enable ufs-weather-model options; delimited with ','
       (e.g. 32BIT | INLINE_POST | UFS_GOCART | MOM6 | CICE6 | WW3 | CMEPS)
@@ -29,20 +27,33 @@ OPTIONS
       (e.g. 32BIT | INLINE_POST | UFS_GOCART | MOM6 | CICE6 | WW3 | CMEPS)
   --continue
       continue with existing build
-  --clean
+  --remove
       removes existing build; overrides --continue
+  --clean
+      does a "make clean"
+  --build
+      does a "make" (build only)
   --build-dir=BUILD_DIR
       build directory
   --install-dir=INSTALL_DIR
       installation prefix
+  --bin-dir=BIN_DIR
+      installation binary directory name ("bin" or "exec")
   --build-type=BUILD_TYPE
       build type; defaults to RELEASE
       (e.g. DEBUG | RELEASE | RELWITHDEBINFO)
   --build-jobs=BUILD_JOBS
       number of build jobs; defaults to 4
+  --use-sub-modules
+      Use sub-component modules instead of top-level level SRW modules
   -v, --verbose
       build with verbose output
 
+TARGETS
+   default = builds the default list of apps (also not passing any target does the same)
+   all = builds all apps
+   Or any combinations of (ufs, ufs_utils, upp, gsi, rrfs_utils)
+
 NOTE: This script is for internal developer use only;
 See User's Guide for detailed build instructions
 
@@ -57,17 +68,23 @@ Settings:
   SRW_DIR=${SRW_DIR}
   BUILD_DIR=${BUILD_DIR}
   INSTALL_DIR=${INSTALL_DIR}
+  BIN_DIR=${BIN_DIR}
   PLATFORM=${PLATFORM}
   COMPILER=${COMPILER}
   APP=${APPLICATION}
   CCPP=${CCPP_SUITES}
   ENABLE_OPTIONS=${ENABLE_OPTIONS}
   DISABLE_OPTIONS=${DISABLE_OPTIONS}
-  CLEAN=${CLEAN}
+  REMOVE=${REMOVE}
   CONTINUE=${CONTINUE}
   BUILD_TYPE=${BUILD_TYPE}
   BUILD_JOBS=${BUILD_JOBS}
   VERBOSE=${VERBOSE}
+  BUILD_UFS=${BUILD_UFS}
+  BUILD_UFS_UTILS=${BUILD_UFS_UTILS}
+  BUILD_UPP=${BUILD_UPP}
+  BUILD_GSI=${BUILD_GSI}
+  BUILD_RRFS_UTILS=${BUILD_RRFS_UTILS}
 
 EOF_SETTINGS
 }
@@ -85,18 +102,31 @@ SRW_DIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P)
 MACHINE_SETUP=${SRW_DIR}/src/UFS_UTILS/sorc/machine-setup.sh
 BUILD_DIR="${SRW_DIR}/build"
 INSTALL_DIR=${SRW_DIR}
+BIN_DIR="bin" #change this to "exec" for NCO mode later
 COMPILER=""
 APPLICATION=""
 CCPP_SUITES=""
-RRFS="off"
 ENABLE_OPTIONS=""
 DISABLE_OPTIONS=""
 BUILD_TYPE="RELEASE"
 BUILD_JOBS=4
-CLEAN=false
+REMOVE=false
 CONTINUE=false
 VERBOSE=false
 
+# Turn off all apps to build and choose default later
+DEFAULT_BUILD=true 
+BUILD_UFS="off"
+BUILD_UFS_UTILS="off"
+BUILD_UPP="off"
+BUILD_GSI="off"
+BUILD_RRFS_UTILS="off"
+
+# Make options
+CLEAN=false
+BUILD=false
+USE_SUB_MODULES=false #change default to true later
+
 # process required arguments
 if [[ ("$1" == "--help") || ("$1" == "-h") ]]; then
   usage
@@ -115,32 +145,53 @@ while :; do
     --app|--app=|-a|-a=) usage_error "$1 requires argument." ;;
     --ccpp=?*) CCPP_SUITES=${1#*=} ;;
     --ccpp|--ccpp=) usage_error "$1 requires argument." ;;
-    --rrfs) RRFS="on" ;;
-    --rrfs=*) usage_error "$1 argument ignored." ;;
     --enable-options=?*) ENABLE_OPTIONS=${1#*=} ;;
     --enable-options|--enable-options=) usage_error "$1 requires argument." ;;
     --disable-options=?*) DISABLE_OPTIONS=${1#*=} ;;
     --disable-options|--disable-options=) usage_error "$1 requires argument." ;;
-    --clean) CLEAN=true ;;
-    --clean=?*|--clean=) usage_error "$1 argument ignored." ;;
+    --remove) REMOVE=true ;;
+    --remove=?*|--remove=) usage_error "$1 argument ignored." ;;
     --continue) CONTINUE=true ;;
     --continue=?*|--continue=) usage_error "$1 argument ignored." ;;
+    --clean) CLEAN=true ;;
+    --build) BUILD=true ;;
     --build-dir=?*) BUILD_DIR=${1#*=} ;;
     --build-dir|--build-dir=) usage_error "$1 requires argument." ;;
     --install-dir=?*) INSTALL_DIR=${1#*=} ;;
     --install-dir|--install-dir=) usage_error "$1 requires argument." ;;
+    --bin-dir=?*) BIN_DIR=${1#*=} ;;
+    --bin-dir|--bin-dir=) usage_error "$1 requires argument." ;;
     --build-type=?*) BUILD_TYPE=${1#*=} ;;
     --build-type|--build-type=) usage_error "$1 requires argument." ;;
     --build-jobs=?*) BUILD_JOBS=$((${1#*=})) ;;
     --build-jobs|--build-jobs=) usage_error "$1 requires argument." ;;
     --verbose|-v) VERBOSE=true ;;
     --verbose=?*|--verbose=) usage_error "$1 argument ignored." ;;
+    --use-sub-modules) USE_SUB_MODULES=true ;;
+    # targets
+    default) ;;
+    all) DEFAULT_BUILD=false; BUILD_UFS="on";
+         BUILD_UFS_UTILS="on"; BUILD_UPP="on";
+         BUILD_GSI="on"; BUILD_RRFS_UTILS="on";;
+    ufs) DEFAULT_BUILD=false; BUILD_UFS="on" ;;
+    ufs_utils) DEFAULT_BUILD=false; BUILD_UFS_UTILS="on" ;;
+    upp) DEFAULT_BUILD=false; BUILD_UPP="on" ;;
+    gsi) DEFAULT_BUILD=false; BUILD_GSI="on" ;;
+    rrfs_utils) DEFAULT_BUILD=false; BUILD_RRFS_UTILS="on" ;;
+    # unknown
     -?*|?*) usage_error "Unknown option $1" ;;
     *) break
   esac
   shift
 done
 
+# choose default apps to build
+if [ "${DEFAULT_BUILD}" = true ]; then
+  BUILD_UFS="on"
+  BUILD_UFS_UTILS="on"
+  BUILD_UPP="on"
+fi
+
 # Ensure uppercase / lowercase ============================================
 APPLICATION="${APPLICATION^^}"
 PLATFORM="${PLATFORM,,}"
@@ -197,7 +248,7 @@ fi
 printf "MODULE_FILE=${MODULE_FILE}\n" >&2
 
 # if build directory already exists then exit
-if [ "${CLEAN}" = true ]; then
+if [ "${REMOVE}" = true ]; then
   printf "Remove build directory\n"
   printf "  BUILD_DIR=${BUILD_DIR}\n\n"
   rm -rf ${BUILD_DIR}
@@ -231,17 +282,22 @@ else
 fi
 
 # cmake settings
-CMAKE_SETTINGS="-DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}"
-CMAKE_SETTINGS="${CMAKE_SETTINGS} -DCMAKE_BUILD_TYPE=${BUILD_TYPE}"
+CMAKE_SETTINGS="\
+ -DCMAKE_BUILD_TYPE=${BUILD_TYPE}\
+ -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}\
+ -DCMAKE_INSTALL_BINDIR=${BIN_DIR}\
+ -DBUILD_UFS=${BUILD_UFS}\
+ -DBUILD_UFS_UTILS=${BUILD_UFS_UTILS}\
+ -DBUILD_UPP=${BUILD_UPP}\
+ -DBUILD_GSI=${BUILD_GSI}\
+ -DBUILD_RRFS_UTILS=${BUILD_RRFS_UTILS}"
+
 if [ ! -z "${APPLICATION}" ]; then
   CMAKE_SETTINGS="${CMAKE_SETTINGS} -DAPP=${APPLICATION}"
 fi
 if [ ! -z "${CCPP_SUITES}" ]; then
   CMAKE_SETTINGS="${CMAKE_SETTINGS} -DCCPP_SUITES=${CCPP_SUITES}"
 fi
-if [ ! -z "${RRFS}" ]; then
-  CMAKE_SETTINGS="${CMAKE_SETTINGS} -DENABLE_RRFS=${RRFS}"
-fi
 if [ ! -z "${ENABLE_OPTIONS}" ]; then
   CMAKE_SETTINGS="${CMAKE_SETTINGS} -DENABLE_OPTIONS=${ENABLE_OPTIONS}"
 fi
@@ -260,14 +316,90 @@ source ${SRW_DIR}/etc/lmod-setup.sh $MACHINE
 
 # source the module file for this platform/compiler combination, then build the code
 printf "... Load MODULE_FILE and create BUILD directory ...\n"
-module use ${SRW_DIR}/modulefiles
-module load ${MODULE_FILE}
+
+if [ $USE_SUB_MODULES = true ]; then
+    #helper to try and load module
+    function load_module() {
+
+        set +e
+        #try most specialized modulefile first
+        MODF="$1${PLATFORM}.${COMPILER}"
+        if [ $BUILD_TYPE != "RELEASE" ]; then
+            MODF="${MODF}.debug"
+        else
+            MODF="${MODF}.release"
+        fi
+        module is-avail ${MODF}
+        if [ $? -eq 0 ]; then
+            module load ${MODF}
+            return
+        fi
+        # without build type
+        MODF="$1${PLATFORM}.${COMPILER}"
+        module is-avail ${MODF}
+        if [ $? -eq 0 ]; then
+            module load ${MODF}
+            return
+        fi
+        # without compiler
+        MODF="$1${PLATFORM}"
+        module is-avail ${MODF}
+        if [ $? -eq 0 ]; then
+            module load ${MODF}
+            return
+        fi
+        set -e
+
+        # else fallback on app level modulefile
+        printf "... Fall back to app level modulefile ...\n"
+        module use ${SRW_DIR}/modulefiles
+        module load ${MODULE_FILE}
+    }
+    if [ $BUILD_UFS = "on" ]; then
+        printf "... Loading UFS modules ...\n"
+        module use ${SRW_DIR}/sorc/ufs-weather-model/modulefiles
+        load_module "ufs_"
+    fi
+    if [ $BUILD_UFS_UTILS = "on" ]; then
+        printf "... Loading UFS_UTILS modules ...\n"
+        module use ${SRW_DIR}/sorc/UFS_UTILS/modulefiles
+        load_module "build."
+    fi
+    if [ $BUILD_UPP = "on" ]; then
+        printf "... Loading UPP modules ...\n"
+        module use ${SRW_DIR}/sorc/UPP/modulefiles
+        load_module ""
+    fi
+    if [ $BUILD_GSI = "on" ]; then
+        printf "... Loading GSI modules ...\n"
+        module use ${SRW_DIR}/sorc/gsi/modulefiles
+        load_module "gsi_"
+    fi
+    if [ $BUILD_RRFS_UTILS = "on" ]; then
+        printf "... Loading RRFS_UTILS modules ...\n"
+        load_module ""
+    fi
+else
+    module use ${SRW_DIR}/modulefiles
+    module load ${MODULE_FILE}
+fi
 module list
+
 mkdir -p ${BUILD_DIR}
 cd ${BUILD_DIR}
+
 printf "... Generate CMAKE configuration ...\n"
 cmake ${SRW_DIR} ${CMAKE_SETTINGS} 2>&1 | tee log.cmake
-printf "... Compile executables ...\n"
-make ${MAKE_SETTINGS} 2>&1 | tee log.make
+
+if [ "${CLEAN}" = true ]; then
+    printf "... Clean executables ...\n"
+    make ${MAKE_SETTINGS} clean 2>&1 | tee log.make
+elif [ "${BUILD}" = true ]; then
+    printf "... Compile executables ...\n"
+    make ${MAKE_SETTINGS} build 2>&1 | tee log.make
+else
+    printf "... Compile and install executables ...\n"
+    make ${MAKE_SETTINGS} install 2>&1 | tee log.make
+fi
 
 exit 0