CI JJOB Tests using CMake

CMakeLists.txt

@@ -0,0 +1,27 @@
+# ------------------------------------------------------------------------- #
+# Global Workflow
+# ------------------------------------------------------------------------- #
+
+# Check for minimum cmake requirement
+cmake_minimum_required( VERSION 3.20 FATAL_ERROR )
+
+project(global_workflow VERSION 1.0.0)
+
+include(GNUInstallDirs)
+enable_testing()
+
+# Build type.
+if(NOT CMAKE_BUILD_TYPE MATCHES "^(Debug|Release|RelWithDebInfo|MinSizeRel)$")
+  message(STATUS "Setting build type to 'Release' as none was specified.")
+  set(CMAKE_BUILD_TYPE
+      "Release"
+      CACHE STRING "Choose the type of build." FORCE)
+  set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release"
+                                               "MinSizeRel" "RelWithDebInfo")
+endif()
+
+# Build global-workflow source codes
+# add_subdirectory(sorc)
+
+# Setup tests
+add_subdirectory(ctests)

ci/platforms/config.hera

@@ -2,6 +2,8 @@
 
 export GFS_CI_ROOT=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT
 export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR
+
+export STAGED_FUNCTIONAL_TEST_DIR=${GFS_CI_ROOT}/STAGED_FUNCTIONAL_TEST_DIR
 export HPC_ACCOUNT=nems
 export max_concurrent_cases=5
 export max_concurrent_pr=4

ci/platforms/config.orion

@@ -3,6 +3,7 @@
 export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/ORION
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
 export HPC_ACCOUNT=nems
+export STAGED_FUNCTIONAL_TEST_DIR=/work/noaa/stmp/GFS_CI_ROOT/ORION/STAGED_FUNCTIONAL_TEST_DIR
 export max_concurrent_cases=5
 export max_concurrent_pr=4
 

ctests/CMakeLists.txt

@@ -0,0 +1,89 @@
+if (NOT DEFINED HOMEgfs)
+  set(HOMEgfs ${PROJECT_SOURCE_DIR})
+endif()
+
+if (DEFINED ENV{RUNTESTS} AND NOT DEFINED RUNTESTS)
+  set(RUNTESTS $ENV{RUNTESTS} CACHE FILEPATH "Path to global-workflow RUNTESTS directory")
+else()
+  set(RUNTESTS ${CMAKE_CURRENT_BINARY_DIR}/RUNTESTS)
+endif()
+
+if (DEFINED ENV{HPC_ACCOUNT} AND NOT DEFINED HPC_ACCOUNT)
+  set(HPC_ACCOUNT $ENV{HPC_ACCOUNT} CACHE STRING "Local location of glowbal-workflow initialconditions")
+endif()
+
+if (NOT DEFINED HPC_ACCOUNT)
+  message(FATAL_ERROR "HPC_ACCOUNT must be set")
+  return()
+endif()
+
+if (DEFINED ENV{ISCDIR_ROOT} AND NOT DEFINED ICSDIR_ROOT)
+  set(ICSDIR_ROOT $ENV{HPC_ACCOUNT} CACHE STRING "Local path to global-workflow initialconditions")
+endif()
+if(NOT DEFINED ICSDIR_ROOT)
+  message(FATAL_ERROR "ICSDIR_ROOT must be set")
+  return()
+endif()
+
+if (DEFINED ENV{STAGED_FUNCTIONAL_TEST_DIR} AND NOT DEFINED STAGED_FUNCTIONAL_TEST_DIR)
+	set(STAGED_FUNCTIONAL_TEST_DIR $ENV{STAGED_FUNCTIONAL_TEST_DIR} CACHE STRING "Local path to global-workflow staged data for functional test")
+endif()
+if(NOT DEFINED STAGED_FUNCTIONAL_TEST_DIR)
+  message(FATAL_ERROR "STATED_FUNCTIONAL_TEST_DIR must be set")
+  return()
+endif()
+
+message(STATUS "MACHINE_ID: '${MACHINE_ID}'")
+message(STATUS "gw: global-workflow baselines will be used from: '${HOMEgfs}'")
+message(STATUS "gw: global-workflow tests will be run at: '${RUNTESTS}'")
+message(STATUS "gw: global-workflow tests will use the allocation: '${HPC_ACCOUNT}'")
+message(STATUS "gw: global-workflow tests will use ICSDIR_ROOT: '${ICSDIR_ROOT}'")
+
+# Prepare test scripts
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/setup.sh.in
+               ${CMAKE_CURRENT_BINARY_DIR}/scripts/setup.sh @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/stage.sh.in
+               ${CMAKE_CURRENT_BINARY_DIR}/scripts/stage.sh @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/execute.sh.in
+               ${CMAKE_CURRENT_BINARY_DIR}/scripts/execute.sh @ONLY)
+
+function(AddJJOBTest)
+
+  set(prefix     ARG)
+  set(novals     NOTRAPFPE NOVALGRIND)
+  set(singlevals CASE JOB TEST_DATE)
+  set(multivals  TEST_DEPENDS)
+
+  cmake_parse_arguments(${prefix}
+                       "${novals}" "${singlevals}" "${multivals}"
+                        ${ARGN})
+
+  set(TEST_NAME ${ARG_CASE}_${ARG_JOB})
+  set(CASE_PATH ${HOMEgfs}/ci/cases/pr)
+  set(CASE_YAML ${CASE_PATH}/${ARG_CASE}.yaml)
+
+  add_test(NAME test_${TEST_NAME}_setup
+    COMMAND ./setup.sh ${TEST_NAME} ${CASE_YAML} ${ARG_TEST_DATE}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
+  set_tests_properties(test_${TEST_NAME}_setup PROPERTIES LABELS "${ARG_CASE};${ARG_JOB}")
+
+  add_test(NAME test_${TEST_NAME}_stage
+      COMMAND ./stage.sh ${TEST_NAME} ${ARG_TEST_DATE}
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
+  set_tests_properties(test_${TEST_NAME}_stage PROPERTIES DEPENDS test_${TEST_NAME}_setup LABELS "${ARG_CASE};${ARG_JOB}")
+
+  add_test(NAME test_${TEST_NAME}_execute
+      COMMAND ./execute.sh ${TEST_NAME} ${ARG_JOB} ${ARG_TEST_DATE}
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
+  set_tests_properties(test_${TEST_NAME}_execute PROPERTIES DEPENDS test_${TEST_NAME}_stage LABELS "${ARG_CASE};${ARG_JOB}")
+
+  add_test(NAME test_${TEST_NAME}_validate
+      COMMAND bash -c "${GW_BASELINE_DIR}/ci/ctests/scripts/validate.sh ${TEST_NAME}")
+  set_tests_properties(test_${TEST_NAME}_validate PROPERTIES DEPENDS test_${TEST_NAME}_execute LABELS "${ARG_CASE};${ARG_JOB}")
+endfunction()
+
+AddJJOBTest(
+  CASE "C48_ATM"
+  JOB  "gfs_fcst_seg0"
+  TEST_DATE "2021032312"
+)

ctests/cases/C48_ATM_gfs_fcst_seg0.yaml

@@ -0,0 +1,17 @@
+input_files:
+    mkdir:
+        - "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}//gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input"
+    copy:
+        - ["{{ 'STAGED_FUNCTIONAL_TEST' | getenv }}/{{ 'TEST_NAME' | getenv }}/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_ctrl.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}//gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_ctrl.nc"]
+        - ["{{ 'STAGED_FUNCTIONAL_TEST' | getenv }}/{{ 'TEST_NAME' | getenv }}/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile1.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}//gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile1.nc"]
+        - ["{{ 'STAGED_FUNCTIONAL_TEST' | getenv }}/{{ 'TEST_NAME' | getenv }}/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile2.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}//gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile2.nc"]
+        - ["{{ 'STAGED_FUNCTIONAL_TEST' | getenv }}/{{ 'TEST_NAME' | getenv }}/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile3.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile3.nc"]
+        - ["{{ 'STAGED_FUNCTIONAL_TEST' | getenv }}/{{ 'TEST_NAME' | getenv }}/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile4.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}//gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile4.nc"]
+        - ["{{ 'STAGED_FUNCTIONAL_TEST' | getenv }}/{{ 'TEST_NAME' | getenv }}/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile5.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}///gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile5.nc"]
+        - ["{{ 'STAGED_FUNCTIONAL_TEST' | getenv }}/{{ 'TEST_NAME' | getenv }}/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile6.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile6.nc"]
+        - ["{{ 'STAGED_FUNCTIONAL_TEST' | getenv }}/{{ 'TEST_NAME' | getenv }}/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile1.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile1.nc"]
+        - ["{{ 'STAGED_FUNCTIONAL_TEST' | getenv }}/{{ 'TEST_NAME' | getenv }}/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile2.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile2.nc"]
+        - ["{{ 'STAGED_FUNCTIONAL_TEST' | getenv }}/{{ 'TEST_NAME' | getenv }}/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile3.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile3.nc"]
+        - ["{{ 'STAGED_FUNCTIONAL_TEST' | getenv }}/{{ 'TEST_NAME' | getenv }}/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile4.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile4.nc"]
+        - ["{{ 'STAGED_FUNCTIONAL_TEST' | getenv }}/{{ 'TEST_NAME' | getenv }}/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile5.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile5.nc"]
+        - ["{{ 'STAGED_FUNCTIONAL_TEST' | getenv }}/{{ 'TEST_NAME' | getenv }}/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile6.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile6.nc"]

ctests/scripts/execute.sh.in

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+set -xe
+
+TEST_NAME=${1:?"Name of the test is required"}
+JOB=${2:?"Job name is required"}
+# TODO - adding idate by hand for now, need to get this from the test somehow 
+idate=$3
+
+#TODO - add rocotoboot_dryrun to repo some how
+rocotoboot_dryrun=/work2/noaa/global/mterry/rocoto_dryrun/bin/rocotoboot
+CASEDIR="@CMAKE_CURRENT_BINARY_DIR@/RUNTESTS/EXPDIR/${TEST_NAME}"
+cd "${CASEDIR}"
+rm -f ./*.db
+rm -f ./jobcard                                                                             
+
+job_card=$(yes | "${rocotoboot_dryrun}" -d "${TEST_NAME}.db" -w "${TEST_NAME}.xml" -v 10 -c "${idate}00" -t "${JOB}" 2> jobcard) || true
+cat jobcard | sed '/^{{\|^}}/d' | sed '1d'  > "${TEST_NAME}.sub" || true
+
+#TODO - Generalize for batch system (hard coded to slurm)
+
+output=$(sbatch "${TEST_NAME}.sub")
+job_id=$(echo "${output}" | awk '{print $4}')
+echo "Job ${job_id} submitted for test ${TEST_NAME} with job name ${JOB}"
+stdout_file=$(scontrol show job "${job_id}" | grep StdOut | awk -F= '{print $2}') || true
+
+# Check the status of the Slurm job
+check_stdout_file() {
+    last_line=$(tail -n 1 "${stdout_file}")
+    if [[ "${last_line}" == "End"* && "${last_line}" == *"error code 0"* ]]; then
+        echo "Job ${job_id} completed successfully with END and error code 0."
+        exit 0
+    else
+        echo "Job ${job_id} did not complete successfully. Last line: ${last_line}"
+        exit 1
+    fi
+}
+
+# Check the status of the Slurm job
+timeout=0
+TIMEOUT=60
+while true; do
+    job_status=$(squeue --job "${job_id}" --noheader --format "%T")
+    if [[ "${job_status}" == "COMPLETED" ]]; then
+        echo "Job ${job_id} completed successfully."
+        check_stdout_file
+    elif [[ "${job_status}" == "FAILED" || "${job_status}" == "CANCELLED" || "${job_status}" == "TIMEOUT" ]]; then
+        echo "Job ${job_id} failed with status: ${job_status}."
+        exit 1
+    elif [[ -z "${job_status}" ]]; then
+        echo "Job ${job_id} is no longer in the queue. Checking stdout file."
+        check_stdout_file
+    else
+        echo "Job ${job_id} is still running with status: ${job_status}."
+        sleep 60  # Check every 60 seconds
+        timeout=$((timeout+1))
+        if [[ "${timeout}" -gt "${TIMEOUT}" ]]; then
+            echo "Job ${job_id} has been running for more than ${TIMEOUT} minutes. Exiting."
+            exit 1
+        fi
+    fi
+done

ctests/scripts/setup.sh.in

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+set -ux
+
+TEST_NAME=${1:?"Name of the test is required"}
+YAML_FILE=${2:?"Name of the CI yaml file for the test"}
+TEST_DATE=${3:?"idate of the test is required"}
+
+# CMake to fill these variables
+HOMEgfs="@PROJECT_SOURCE_DIR@"
+RUNTESTS="@RUNTESTS@"
+ICSDIR_ROOT="@ICSDIR_ROOT@"
+HPC_ACCOUNT="@HPC_ACCOUNT@"
+
+set +x
+source "${HOMEgfs}/workflow/gw_setup.sh"
+set -x
+
+pslot="${TEST_NAME}" \
+RUNTESTS="${RUNTESTS}" \
+ICSDIR_ROOT="${ICSDIR_ROOT}" \
+HPC_ACCOUNT="${HPC_ACCOUNT}" \
+"${HOMEgfs}/workflow/create_experiment.py" --yaml "${YAML_FILE}" --overwrite
+rc=$?
+if [[ "${rc}" -ne 0 ]]; then
+    set +x
+    echo "Failed to create test experiment for '${TEST_NAME}' with yaml file '${YAML_FILE}'"
+    set -x
+    exit "${rc}"
+fi
+
+exit 0

ctests/scripts/stage.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+import os, sys
+import shutil
+_here = os.path.dirname(__file__)
+_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../../..'))
+sys.path.insert(0, _top)
+
+from argparse import ArgumentParser
+from pathlib import Path
+from wxflow import parse_j2yaml,  FileHandler, AttrDict, Logger
+
+logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=False)
+
+def parse_args():
+    """
+    Parse command line arguments.
+
+    Returns
+    -------
+    argparse.Namespace
+        The parsed command line arguments.
+    """
+    description = """Arguments for creating and updating error log files
+    """
+    parser = ArgumentParser(description=description)
+
+    parser.add_argument('-y', '--yaml', help='full path to yaml file describing the job test configuration', type=Path, required=True)
+    return parser.parse_args()
+
+if __name__ == '__main__':
+
+    # Parse command line arguments
+    args = parse_args()
+    data = {'TEST_DATE': os.environ.get('TEST_DATE')}
+    case_cfg = parse_j2yaml(path=args.yaml, data=data)
+    FileHandler(case_cfg.input_files).sync()

ctests/scripts/stage.sh.in

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -ux
+
+TEST_NAME=${1:?"Name of the test is required"}
+TEST_DATE=${2:?"idate of the test is required"}
+
+# CMake to fill these variables
+STAGED_FUNCTIONAL_TEST_DIR="@STAGED_FUNCTIONAL_TEST_DIR@"
+RUNTESTS="@RUNTESTS@"
+HOMEgfs="@PROJECT_SOURCE_DIR@"
+
+# Load the runtime environment for this script (needs wxflow and its dependencies)
+set +x
+source "${HOMEgfs}/workflow/gw_setup.sh"
+rc=$?
+[[ "${rc}" -ne 0 ]] && exit "${status}"
+set -x
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/sorc/wxflow/src"
+export PYTHONPATH
+
+INPUTS_YAML="${HOMEgfs}/ctests/cases/${TEST_NAME}.yaml"
+
+"${HOMEgfs}/ctests/scripts/stage.py" --yaml "${INPUTS_YAML}"
+rc=$?
+if [[ "${rc}" -ne 0 ]]; then
+    set +x
+    echo "Failed to stage inputs for '${TEST_NAME}' with '${INPUTS_YAML}'"
+    set -x
+    exit "${rc}"
+fi
+
+exit 0

ctests/scripts/validate.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+echo "This is a stub for the validate script."

modulefiles/module_gwsetup.orion.lua

@@ -10,11 +10,13 @@ prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-st
 
 local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0"
 local python_ver=os.getenv("python_ver") or "3.11.6"
+local cmake_ver=os.getenv("cmake_ver") or "3.23.1"
 
 load(pathJoin("stack-intel", stack_intel_ver))
 load(pathJoin("python", python_ver))
 load("py-jinja2")
 load("py-pyyaml")
 load("py-numpy")
+load(pathJoin("cmake", cmake_ver))
 
 whatis("Description: GFS run setup environment")