CI JJOB Tests using CMake

ci/ctests/CMakeLists.txt

@@ -0,0 +1,111 @@
+cmake_minimum_required(VERSION 3.10)
+
+project(FunctionalTests)
+enable_testing()
+set(CMAKE_VERBOSE_MAKEFILE ON)
+
+function(set_from_env_or_default VAR_NAME ENV_VAR DEFAULT_VALUE)
+  if (DEFINED ENV{${ENV_VAR}} AND NOT DEFINED ${VAR_NAME})
+    set(${VAR_NAME} $ENV{${ENV_VAR}} CACHE STRING "Set from environment variable ${ENV_VAR}")
+  elseif(NOT DEFINED ${VAR_NAME})
+    set(${VAR_NAME} ${DEFAULT_VALUE} CACHE STRING "Default value for ${VAR_NAME}")
+  endif()
+endfunction()
+
+if (DEFINED ENV{GW_BASELINE_DIR} AND NOT DEFINED GW_BASELINE_DIR)
+  set(GW_BASELINE_DIR $ENV{GW_BASELINE_DIR} CACHE FILEPATH "Path to global-workflow baselines")
+else()
+  get_filename_component(GW_BASELINE_DIR ${CMAKE_SOURCE_DIR}/../.. ABSOLUTE)
+endif()
+
+# If GW_BASELINE_DIR is not defined, tests will not be run, return
+if(NOT DEFINED GW_BASELINE_DIR)
+  message(WARNING "gw: GW_BASELINE_DIR is not defined. Tests will not be run.")
+  return()
+endif()
+
+set_from_env_or_default(RUNTESTS RUNTESTS "${CMAKE_BINARY_DIR}/RUNTESTS")
+
+set_from_env_or_default(MACHINE_ID MACHINE_ID "NOT_DEFINED")
+if(${MACHINE_ID} STREQUAL "NOT_DEFINED")
+  execute_process(
+    COMMAND bash -c "source ${GW_BASELINE_DIR}/ush/detect_machine.sh && echo \$MACHINE_ID"
+    OUTPUT_VARIABLE MACHINE_ID
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+endif()
+
+set_from_env_or_default(HPC_ACCOUNT HPC_ACCOUNT "NOT_DEFINED")
+if(${HPC_ACCOUNT} STREQUAL "NOT_DEFINED")
+  execute_process(
+    COMMAND bash -c "source ${GW_BASELINE_DIR}/ci/platforms/config.${MACHINE_ID} && echo \$HPC_ACCOUNT"
+    OUTPUT_VARIABLE HPC_ACCOUNT
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+endif()
+
+set_from_env_or_default(ICSDIR_ROOT ICSDIR_ROOT "NOT_DEFINED")
+if(${ICSDIR_ROOT} STREQUAL "NOT_DEFINED")
+  execute_process(
+    COMMAND bash -c "source ${GW_BASELINE_DIR}/ci/platforms/config.${MACHINE_ID} && echo \$ICSDIR_ROOT"
+    OUTPUT_VARIABLE ICSDIR_ROOT
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+  set(ICSDIR_ROOT ${ICSDIR_ROOT} CACHE FILEPATH "Path to ICS directory")
+endif()
+
+message(STATUS "MACHINE_ID: '${MACHINE_ID}'")
+message(STATUS "gw: global-workflow baselines will be used from: '${GW_BASELINE_DIR}'")
+message(STATUS "gw: global-workflow tests will be run at: '${RUNTESTS}'")
+message(STATUS "gw: global-workflow tests will use the allocation: '${HPC_ACCOUNT}'")
+message(STATUS "gw: global-workflow tests will use ICSDIR_ROOT: '${ICSDIR_ROOT}'")
+
+# Prepare test scripts
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/setup.sh.in
+               ${CMAKE_CURRENT_BINARY_DIR}/scripts/setup.sh @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/stage.sh.in
+               ${CMAKE_CURRENT_BINARY_DIR}/scripts/stage.sh @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/execute.sh.in
+               ${CMAKE_CURRENT_BINARY_DIR}/scripts/execute.sh @ONLY)
+
+function(AddFunctionalTest)
+
+  set(prefix     ARG)
+  set(novals     NOTRAPFPE NOVALGRIND)
+  set(singlevals CASE JOB)
+  set(multivals  TEST_DEPENDS)
+
+  cmake_parse_arguments(${prefix}
+                       "${novals}" "${singlevals}" "${multivals}"
+                        ${ARGN})
+
+  set(TEST_NAME ${ARG_CASE}_${ARG_JOB})
+  set(CASE_PATH ${GW_BASELINE_DIR}/ci/cases/pr)
+  set(CASE_YAML ${CASE_PATH}/${ARG_CASE}.yaml)
+
+  add_test(NAME test_${TEST_NAME}_setup
+    COMMAND ./setup.sh ${TEST_NAME} ${CASE_YAML}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
+  set_tests_properties(test_${TEST_NAME}_setup PROPERTIES LABELS "${ARG_CASE};${ARG_JOB}")
+
+  add_test(NAME test_${TEST_NAME}_stage
+      COMMAND ./stage.sh ${TEST_NAME}
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
+  set_tests_properties(test_${TEST_NAME}_stage PROPERTIES DEPENDS test_${TEST_NAME}_setup LABELS "${ARG_CASE};${ARG_JOB}")
+
+  add_test(NAME test_${TEST_NAME}_execute
+      #TODO Need to get the idate for rocotoboot get the right cycle for the test (hard coded for now)
+      #TODO Need to add a loop to check for when job has been completed
+      COMMAND ./execute.sh ${TEST_NAME} ${ARG_JOB} 2021032312
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
+  set_tests_properties(test_${TEST_NAME}_execute PROPERTIES DEPENDS test_${TEST_NAME}_stage LABELS "${ARG_CASE};${ARG_JOB}")
+
+  add_test(NAME test_${TEST_NAME}_validate
+      COMMAND bash -c "${GW_BASELINE_DIR}/ci/ctests/scripts/validate.sh ${TEST_NAME}")
+  set_tests_properties(test_${TEST_NAME}_validate PROPERTIES DEPENDS test_${TEST_NAME}_execute LABELS "${ARG_CASE};${ARG_JOB}")
+endfunction()
+
+AddFunctionalTest(
+  CASE "C48_ATM"
+  JOB  "gfs_fcst_seg0"
+)

ci/ctests/cases/C48_ATM_gfs_fcst_seg0.yaml

@@ -0,0 +1,17 @@
+input_files:
+    mkdir:
+        - "{{RUNTESTS}}/COMROOT/{{ TEST_NAME }}//gfs.{{PDY}}/{{HH}}/model/atmos/input"
+    copy:
+        - ["{{ STAGED_FUNCTIONAL_TEST_DIR }}/{{ TEST_NAME }}/gfs.{{ PDY }}/{{ HH }}/model/atmos/input/gfs_ctrl.nc", "{{RUNTESTS}}/COMROOT/{{ TEST_NAME }}//gfs.{{PDY}}/{{HH}}/model/atmos/input/gfs_ctrl.nc"]
+        - ["{{ STAGED_FUNCTIONAL_TEST_DIR }}/{{ TEST_NAME }}/gfs.{{ PDY }}/{{ HH }}/model/atmos/input/gfs_data.tile1.nc", "{{RUNTESTS}}/COMROOT/{{ TEST_NAME }}//gfs.{{PDY}}/{{HH}}/model/atmos/input/gfs_data.tile1.nc"]
+        - ["{{ STAGED_FUNCTIONAL_TEST_DIR }}/{{ TEST_NAME }}/gfs.{{ PDY }}/{{ HH }}/model/atmos/input/gfs_data.tile2.nc", "{{RUNTESTS}}/COMROOT/{{ TEST_NAME }}//gfs.{{PDY}}/{{HH}}/model/atmos/input/gfs_data.tile2.nc"]
+        - ["{{ STAGED_FUNCTIONAL_TEST_DIR }}/{{ TEST_NAME }}/gfs.{{ PDY }}/{{ HH }}/model/atmos/input/gfs_data.tile3.nc", "{{RUNTESTS}}/COMROOT/{{ TEST_NAME }}/gfs.{{PDY}}/{{HH}}/model/atmos/input/gfs_data.tile3.nc"]
+        - ["{{ STAGED_FUNCTIONAL_TEST_DIR }}/{{ TEST_NAME }}/gfs.{{ PDY }}/{{ HH }}/model/atmos/input/gfs_data.tile4.nc", "{{RUNTESTS}}/COMROOT/{{ TEST_NAME }}//gfs.{{PDY}}/{{HH}}/model/atmos/input/gfs_data.tile4.nc"]
+        - ["{{ STAGED_FUNCTIONAL_TEST_DIR }}/{{ TEST_NAME }}/gfs.{{ PDY }}/{{ HH }}/model/atmos/input/gfs_data.tile5.nc", "{{RUNTESTS}}/COMROOT/{{ TEST_NAME }}///gfs.{{PDY}}/{{HH}}/model/atmos/input/gfs_data.tile5.nc"]
+        - ["{{ STAGED_FUNCTIONAL_TEST_DIR }}/{{ TEST_NAME }}/gfs.{{ PDY }}/{{ HH }}/model/atmos/input/gfs_data.tile6.nc", "{{RUNTESTS}}/COMROOT/{{ TEST_NAME }}/gfs.{{PDY}}/{{HH}}/model/atmos/input/gfs_data.tile6.nc"]
+        - ["{{ STAGED_FUNCTIONAL_TEST_DIR }}/{{ TEST_NAME }}/gfs.{{ PDY }}/{{ HH }}/model/atmos/input/sfc_data.tile1.nc", "{{RUNTESTS}}/COMROOT/{{ TEST_NAME }}/gfs.{{PDY}}/{{HH}}/model/atmos/input/sfc_data.tile1.nc"]
+        - ["{{ STAGED_FUNCTIONAL_TEST_DIR }}/{{ TEST_NAME }}/gfs.{{ PDY }}/{{ HH }}/model/atmos/input/sfc_data.tile2.nc", "{{RUNTESTS}}/COMROOT/{{ TEST_NAME }}/gfs.{{PDY}}/{{HH}}/model/atmos/input/sfc_data.tile2.nc"]
+        - ["{{ STAGED_FUNCTIONAL_TEST_DIR }}/{{ TEST_NAME }}/gfs.{{ PDY }}/{{ HH }}/model/atmos/input/sfc_data.tile3.nc", "{{RUNTESTS}}/COMROOT/{{ TEST_NAME }}/gfs.{{PDY}}/{{HH}}/model/atmos/input/sfc_data.tile3.nc"]
+        - ["{{ STAGED_FUNCTIONAL_TEST_DIR }}/{{ TEST_NAME }}/gfs.{{ PDY }}/{{ HH }}/model/atmos/input/sfc_data.tile4.nc", "{{RUNTESTS}}/COMROOT/{{ TEST_NAME }}/gfs.{{PDY}}/{{HH}}/model/atmos/input/sfc_data.tile4.nc"]
+        - ["{{ STAGED_FUNCTIONAL_TEST_DIR }}/{{ TEST_NAME }}/gfs.{{ PDY }}/{{ HH }}/model/atmos/input/sfc_data.tile5.nc", "{{RUNTESTS}}/COMROOT/{{ TEST_NAME }}/gfs.{{PDY}}/{{HH}}/model/atmos/input/sfc_data.tile5.nc"]
+        - ["{{ STAGED_FUNCTIONAL_TEST_DIR }}/{{ TEST_NAME }}/gfs.{{ PDY }}/{{ HH }}/model/atmos/input/sfc_data.tile6.nc", "{{RUNTESTS}}/COMROOT/{{ TEST_NAME }}/gfs.{{PDY}}/{{HH}}/model/atmos/input/sfc_data.tile6.nc"]

ci/ctests/scripts/execute.sh.in

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+set -xe
+
+TEST_NAME=${1:?"Name of the test is required"}
+JOB=${2:?"Job name is required"}
+# TODO - adding idate by hand for now, need to get this from the test somehow 
+idate=$3
+
+#TODO - add rocotoboot_dryrun to repo some how
+rocotoboot_dryrun=/work2/noaa/global/mterry/rocoto_dryrun/bin/rocotoboot
+CASEDIR="@CMAKE_CURRENT_BINARY_DIR@/RUNTESTS/EXPDIR/${TEST_NAME}"
+cd "${CASEDIR}"
+rm -f *.db
+rm -f jobcard                                                                             
+
+job_card=$(yes | ${rocotoboot_dryrun} -d ${TEST_NAME}.db -w ${TEST_NAME}.xml -v 10 -c ${idate}00 -t ${JOB} 2> jobcard)
+cat jobcard | sed '/^{{\|^}}/d' | sed '1d'  > "${TEST_NAME}.sub"
+
+#TODO - Generalize for batch system (hard coded to slurm)
+
+output=$(sbatch "${TEST_NAME}.sub")
+job_id=$(echo $output | awk '{print $4}')
+echo "Job ${job_id} submitted for test ${TEST_NAME} with job name ${JOB}"
+stdout_file=$(scontrol show job $job_id | grep StdOut | awk -F= '{print $2}')
+
+# Check the status of the Slurm job
+check_stdout_file() {
+    last_line=$(tail -n 1 "$stdout_file")
+    if [[ "$last_line" == "End"* && "$last_line" == *"error code 0"* ]]; then
+        echo "Job ${job_id} completed successfully with END and error code 0."
+        exit 0
+    else
+        echo "Job ${job_id} did not complete successfully. Last line: $last_line"
+        exit -1
+    fi
+}
+
+# Check the status of the Slurm job
+timeout=0
+TIMEOUT=60
+while true; do
+    job_status=$(squeue --job $job_id --noheader --format "%T")
+    if [[ "$job_status" == "COMPLETED" ]]; then
+        echo "Job ${job_id} completed successfully."
+        check_stdout_file
+    elif [[ "$job_status" == "FAILED" || "$job_status" == "CANCELLED" || "$job_status" == "TIMEOUT" ]]; then
+        echo "Job ${job_id} failed with status: $job_status."
+        exit -1
+    elif [[ -z "$job_status" ]]; then
+        echo "Job ${job_id} is no longer in the queue. Checking stdout file."
+        check_stdout_file
+    else
+        echo "Job ${job_id} is still running with status: $job_status."
+        sleep 60  # Check every 60 seconds
+        timeout=$((timeout+1))
+        if [[ $timeout -gt $TIMEOUT ]]; then
+            echo "Job ${job_id} has been running for more than $TIMEOUT minutes. Exiting."
+            exit -1
+        fi
+    fi
+done

ci/ctests/scripts/setup.sh.in

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -ux
+
+TEST_NAME=${1:?"Name of the test is required"}
+YAML_FILE=${2:?"Name of the CI yaml file for the test"}
+
+# CMake to fill these variables
+HOMEgfs="@GW_BASELINE_DIR@"
+RUNTESTS="@RUNTESTS@"
+ICSDIR_ROOT="@ICSDIR_ROOT@"
+HPC_ACCOUNT="@HPC_ACCOUNT@"
+
+set +x
+source "${HOMEgfs}/workflow/gw_setup.sh"
+set -x
+
+pslot="${TEST_NAME}" \
+RUNTESTS="${RUNTESTS}" \
+ICSDIR_ROOT="${ICSDIR_ROOT}" \
+HPC_ACCOUNT="${HPC_ACCOUNT}" \
+"${HOMEgfs}/workflow/create_experiment.py" --yaml "${YAML_FILE}" --overwrite
+rc=$?
+if [[ "${rc}" -ne 0 ]]; then
+    set +x
+    echo "Failed to create test experiment for '${TEST_NAME}' with yaml file '${YAML_FILE}'"
+    set -x
+    exit "${rc}"
+fi
+
+exit 0

ci/ctests/scripts/stage.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+
+import os, sys
+import shutil
+_here = os.path.dirname(__file__)
+_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../../..'))
+sys.path.insert(0, _top)
+
+from argparse import ArgumentParser
+from pathlib import Path
+from wxflow import Configuration, AttrDict, parse_j2yaml, Logger, logit, which, CommandNotFoundError, ProcessError, FileHandler
+from workflow.hosts import Host
+
+logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=False)
+
+def parse_args():
+    """
+    Parse command line arguments.
+
+    Returns
+    -------
+    argparse.Namespace
+        The parsed command line arguments.
+    """
+    description = """Arguments for creating and updating error log files
+    """
+    parser = ArgumentParser(description=description)
+
+    parser.add_argument('--build_dir', help='CMake build directory', required=False, type=Path, default=None)
+    parser.add_argument('-y', '--yaml', help='full path to yaml file describing the job test configuration', type=Path, required=True)
+    return parser.parse_args()
+
+if __name__ == '__main__':
+
+    # Parse command line arguments
+    args = parse_args()
+    data = AttrDict(HOMEgfs=_top)
+    data.update(os.environ)
+
+    # Initialize host and platform configuration for getting PATH to staged data per machine
+    host = Host()
+    cfg = Configuration(f'{data.HOMEgfs}/ci/platforms')
+    platform_config = cfg.parse_config(f'config.{host.machine.lower()}')
+    data.update(platform_config)
+
+    #pr_case_cfg = parse_j2yaml(path=pr_case_yaml_path, data=data)
+    #data["PDY"]=str(pr_case_cfg.arguments.idate)[0:8]
+    #data["HH"]=str(pr_case_cfg.arguments.idate)[8:10]
+
+    # TODO get idate in lue of above
+    idate = "2021032312"
+
+    data["PDY"]=str(idate)[0:8]
+    data["HH"]=str(idate)[8:10]
+    data["RUNTESTS"]=Path.joinpath(args.build_dir,"RUNTESTS")
+    data["TEST_NAME"]=args.yaml.stem
+    case_cfg = parse_j2yaml(path=args.yaml, data=data)
+    FileHandler(case_cfg.input_files).sync()

ci/ctests/scripts/stage.sh.in

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -ux
+
+TEST_NAME=${1:?"Name of the test is required"}
+
+# CMake to fill these variables
+HOMEgfs="@GW_BASELINE_DIR@"
+
+# Load the runtime environment for this script (needs wxflow and its dependencies)
+set +x
+source "${HOMEgfs}/workflow/gw_setup.sh"
+rc=$?
+[[ "${rc}" -ne 0 ]] && exit "${status}"
+set -x
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/sorc/wxflow/src"
+export PYTHONPATH
+
+INPUTS_YAML="${HOMEgfs}/ci/ctests/cases/${TEST_NAME}.yaml"
+
+"${HOMEgfs}/ci/ctests/scripts/stage.py" --build_dir "@CMAKE_CURRENT_BINARY_DIR@" --yaml "${INPUTS_YAML}" 
+rc=$?
+if [[ "${rc}" -ne 0 ]]; then
+    set +x
+    echo "Failed to stage inputs for '${TEST_NAME}' with '${INPUTS_YAML}'"
+    set -x
+    exit "${rc}"
+fi
+
+exit 0

ci/ctests/scripts/validate.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+echo "This is a stub for the validate script."

ci/platforms/config.hera

@@ -2,6 +2,8 @@
 
 export GFS_CI_ROOT=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT
 export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR
+
+export STAGED_FUNCTIONAL_TEST_DIR=${GFS_CI_ROOT}/STAGED_FUNCTIONAL_TEST_DIR
 export HPC_ACCOUNT=nems
 export max_concurrent_cases=5
 export max_concurrent_pr=4

ci/platforms/config.orion

@@ -3,6 +3,7 @@
 export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/ORION
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
 export HPC_ACCOUNT=nems
+export STAGED_FUNCTIONAL_TEST_DIR=/work/noaa/stmp/GFS_CI_ROOT/ORION/STAGED_FUNCTIONAL_TEST_DIR
 export max_concurrent_cases=5
 export max_concurrent_pr=4