[develop] Move all unittest tests to a common area.

.cicd/Jenkinsfile

@@ -122,6 +122,13 @@ pipeline {
                         }
                     }
 
+                    // Run the unittest functional tests that require an HPC platform
+                    stage('Functional Tests') {
+                        steps {
+                            echo "Running unittest on retrieve_data.py"
+                            sh 'bash --login "${WORKSPACE}/.cicd/scripts/srw_unittest.sh"'
+                        }
+                    }
                     // Run the unified build script; if successful create a tarball of the build and upload to S3
                     stage('Build') {
                         steps {

.cicd/scripts/srw_unittest.sh

@@ -0,0 +1,32 @@
+#!/user/bin/env bash
+#
+# A test script for running the SRW application unittest tests that
+# should be tested on-prem.
+#
+set -e -u -x
+
+script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+
+# Get repository root from Jenkins WORKSPACE variable if set, otherwise,
+# set relative to script directory.
+declare workspace
+if [[ -n "${WORKSPACE}" ]]; then
+  workspace="${WORKSPACE}"
+else
+  workspace="$(cd -- "${script_dir}/../.." && pwd)"
+fi
+
+# Only run this on machines with hpss access
+hpss_machines=( jet hera )
+if [ $hpss_machines =~ ${SRW_PLATFORM} ] ; then
+
+  module load hpss
+  module use ${workspace}/modulefiles
+  module load wflow_${SRW_PLATFORM}
+
+  conda activate regional_workflow
+
+  export PYTHONPATH=${workspace}/ush
+  python -m unittest $workspace/tests/test_python/test_retrieve_data.py
+
+fi

.github/workflows/python_func_tests.yaml

@@ -30,11 +30,10 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install python3 python3-pip netcdf-bin
-          sudo pip3 install pyyaml jinja2 f90nml
-          sudo pip3 install numpy matplotlib basemap
+          sudo pip3 install pyyaml
 
       # Run python functional tests
       - name: Run python functional tests
         run: |
-          cd ush
-          python3 -m unittest -b test_retrieve_data.FunctionalTesting
+          export PYTHONPATH=${PWD}/ush
+          python3 -m unittest -b tests/test_python/test_retrieve_data.py

.github/workflows/python_linter.yaml

@@ -0,0 +1,37 @@
+name: Python linting
+on:
+  push:
+    branches:
+      - develop
+      - 'release/*'
+  pull_request:
+    branches:
+      - develop
+      - 'release/*'
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+jobs:
+
+  python_linter:
+    name: Python unittests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      # Install dependencies
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install python3 python3-pip netcdf-bin
+          sudo pip3 install pylint==2.6
+
+      # Run python unittests
+      - name: Lint the test directory
+        run: |
+          export PYTHONPATH=${PWD}/ush
+          pylint --ignore-imports=yes tests/test_python/

.github/workflows/python_unittests.yaml

@@ -10,6 +10,9 @@ on:
       - 'release/*'
   workflow_dispatch:
 
+env:
+  UNIT_TEST: True
+
 defaults:
   run:
     shell: bash
@@ -35,10 +38,7 @@ jobs:
       - name: Run python unittests
         run: |
           ./manage_externals/checkout_externals ufs-weather-model
-          cd ush
-          python3 -m unittest -b python_utils/test_python_utils.py
           # exclude test_retrieve_data that is tested in functional test
-          files=$(find . -maxdepth 1 -name '*.py' -a ! -name 'test_retrieve_data.py' -exec basename {} \;)
-          files=$(echo $(echo "${files[@]}"))
-          python3 -m unittest -b ${files[@]}
+          export PYTHONPATH=${PWD}/ush
+          python3 -m unittest -b tests/test_python/*.py
 

tests/test_python/test_calculate_cost.py

@@ -0,0 +1,17 @@
+""" Tests for calculate_cost.py"""
+
+#pylint: disable=invalid-name
+import os
+import unittest
+
+from calculate_cost import calculate_cost
+
+class Testing(unittest.TestCase):
+    """ Define the tests"""
+    def test_calculate_cost(self):
+        """ Test that the function returns the expected value for a
+        given config file."""
+        test_dir = os.path.dirname(os.path.abspath(__file__))
+        USHdir = os.path.join(test_dir, "..", "..", "ush")
+        params = calculate_cost(os.path.join(USHdir, 'config.community.yaml'))
+        self.assertCountEqual(params, [180, 28689, 180, 28689])

tests/test_python/test_check_ruc_lsm.py

@@ -0,0 +1,27 @@
+""" Test for the define_ruc_lsm script. """
+
+#pylint: disable=invalid-name
+
+import os
+import unittest
+
+from python_utils import set_env_var
+
+from check_ruc_lsm import check_ruc_lsm
+
+class Testing(unittest.TestCase):
+    """ Define the tests"""
+
+    def test_check_ruc_lsm(self):
+        """" Read in a CCPP suite definition file and check that it is
+        using RUC LSM as part of the suite. """
+        test_dir = os.path.dirname(os.path.abspath(__file__))
+        USHdir = os.path.join(test_dir, "..", "..", "ush")
+        self.assertTrue(
+            check_ruc_lsm(
+                ccpp_phys_suite_fp=f"{USHdir}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml"
+            )
+        )
+
+    def setUp(self):
+        set_env_var("DEBUG", True)

tests/test_python/test_create_diag_table_file.py

@@ -0,0 +1,31 @@
+""" Tests for create_diag_table_file.py """
+
+#pylint: disable=invalid-name
+import os
+import unittest
+
+from python_utils import set_env_var
+
+from create_diag_table_file import create_diag_table_file
+
+class Testing(unittest.TestCase):
+    """ Define the tests """
+    def test_create_diag_table_file(self):
+        """ Test that when called with user config settings, the
+        function returns True """
+        path = os.path.join(os.getenv("USHdir"), "test_data")
+        self.assertTrue(create_diag_table_file(run_dir=path))
+
+    def setUp(self):
+        test_dir = os.path.dirname(os.path.abspath(__file__))
+        USHdir = os.path.join(test_dir, "..", "..", "ush")
+        PARMdir = os.path.join(USHdir, "..", "parm")
+        diag_table_fn = "diag_table"
+        diag_table_tmpl_fp = os.path.join(PARMdir, f"{diag_table_fn}.FV3_GFS_v15p2")
+        set_env_var("DEBUG", True)
+        set_env_var("VERBOSE", True)
+        set_env_var("USHdir", USHdir)
+        set_env_var("DIAG_TABLE_FN", diag_table_fn)
+        set_env_var("DIAG_TABLE_TMPL_FP", diag_table_tmpl_fp)
+        set_env_var("CRES", "C48")
+        set_env_var("CDATE", "2021010106")

tests/test_python/test_create_model_configure_file.py

@@ -0,0 +1,63 @@
+""" Tests for create_model_configure_file.py"""
+
+#pylint: disable=invalid-name
+from datetime import datetime
+import os
+import unittest
+
+from python_utils import set_env_var
+
+from create_model_configure_file import create_model_configure_file
+
+
+class Testing(unittest.TestCase):
+    """ Define tests"""
+
+    def test_create_model_configure_file(self):
+        """ Test that the function returns True when configured with
+        valid input data. """
+        path = os.path.join(os.getenv("USHdir"), "test_data")
+        self.assertTrue(
+            create_model_configure_file(
+                run_dir=path,
+                cdate=datetime(2021, 1, 1),
+                fcst_len_hrs=72,
+                fhrot=0,
+                sub_hourly_post=True,
+                dt_subhourly_post_mnts=4,
+                dt_atmos=1,
+            )
+        )
+
+    def setUp(self):
+        test_dir = os.path.dirname(os.path.abspath(__file__))
+        USHdir = os.path.join(test_dir, "..", "..", "ush")
+        PARMdir = os.path.join(USHdir, "..", "parm")
+        MODEL_CONFIG_FN = "model_configure"
+        MODEL_CONFIG_TMPL_FP = os.path.join(PARMdir, MODEL_CONFIG_FN)
+
+        set_env_var("DEBUG", True)
+        set_env_var("VERBOSE", True)
+        set_env_var("QUILTING", True)
+        set_env_var("WRITE_DOPOST", True)
+        set_env_var("USHdir", USHdir)
+        set_env_var("MODEL_CONFIG_FN", MODEL_CONFIG_FN)
+        set_env_var("MODEL_CONFIG_TMPL_FP", MODEL_CONFIG_TMPL_FP)
+        set_env_var("FCST_LEN_HRS", 72)
+        set_env_var("FHROT", 0)
+        set_env_var("DT_ATMOS", 1)
+        set_env_var("RESTART_INTERVAL", 4)
+
+        set_env_var("WRTCMP_write_groups", 1)
+        set_env_var("WRTCMP_write_tasks_per_group", 2)
+        set_env_var("WRTCMP_output_grid", "lambert_conformal")
+        set_env_var("WRTCMP_cen_lon", -97.5)
+        set_env_var("WRTCMP_cen_lat", 35.0)
+        set_env_var("WRTCMP_stdlat1", 35.0)
+        set_env_var("WRTCMP_stdlat2", 35.0)
+        set_env_var("WRTCMP_nx", 199)
+        set_env_var("WRTCMP_ny", 111)
+        set_env_var("WRTCMP_lon_lwr_left", -121.23349066)
+        set_env_var("WRTCMP_lat_lwr_left", 23.41731593)
+        set_env_var("WRTCMP_dx", 3000.0)
+        set_env_var("WRTCMP_dy", 3000.0)

tests/test_python/test_generate_FV3LAM_wflow.py

@@ -0,0 +1,79 @@
+""" Defines an integration test for generate_FV3LAM_wflow script in the
+ush directory """
+
+#pylint: disable=invalid-name, no-self-use
+import os
+import sys
+import unittest
+from multiprocessing import Process
+
+from python_utils import (
+    load_config_file,
+    update_dict,
+    cp_vrfy,
+    run_command,
+    define_macos_utilities,
+    cfg_to_yaml_str,
+    set_env_var,
+    get_env_var,
+)
+
+from generate_FV3LAM_wflow import generate_FV3LAM_wflow
+
+class Testing(unittest.TestCase):
+    """ Class to run the tests. """
+    def test_generate_FV3LAM_wflow(self):
+
+        """ Test that a community and nco sample config can successfully
+        lead to the creation of an experiment directory. No jobs are
+        submitted. """
+
+        # run workflows in separate process to avoid conflict between community and nco settings
+        def run_workflow(USHdir, logfile):
+            p = Process(target=generate_FV3LAM_wflow, args=(USHdir, logfile))
+            p.start()
+            p.join()
+            exit_code = p.exitcode
+            if exit_code != 0:
+                sys.exit(exit_code)
+
+        test_dir = os.path.dirname(os.path.abspath(__file__))
+        USHdir = os.path.join(test_dir, "..", "..", "ush")
+        logfile = "log.generate_FV3LAM_wflow"
+        sed = get_env_var("SED")
+
+        # community test case
+        cp_vrfy(f"{USHdir}/config.community.yaml", f"{USHdir}/config.yaml")
+        run_command(
+            f"""{sed} -i 's/MACHINE: hera/MACHINE: linux/g' {USHdir}/config.yaml"""
+        )
+        run_workflow(USHdir, logfile)
+
+        # nco test case
+        nco_test_config = load_config_file(f"{USHdir}/config.nco.yaml")
+        # Since we don't have a pre-gen grid dir on a generic linux
+        # platform, turn the make_* tasks on for this test.
+        cfg_updates = {
+            "user": {
+                "MACHINE": "linux",
+            },
+            "rocoto": {
+                "tasks": {
+                    "taskgroups": \
+                        """'{{ ["parm/wflow/prep.yaml",
+                                "parm/wflow/coldstart.yaml",
+                                "parm/wflow/post.yaml"]|include }}'"""
+                },
+            },
+        }
+        update_dict(cfg_updates, nco_test_config)
+
+        with open(f"{USHdir}/config.yaml", "w") as cfg_file:
+            cfg_file.write(cfg_to_yaml_str(nco_test_config))
+
+        run_workflow(USHdir, logfile)
+
+    def setUp(self):
+        define_macos_utilities()
+        set_env_var("DEBUG", False)
+        set_env_var("VERBOSE", False)

tests/test_python/test_get_crontab_contents.py

@@ -0,0 +1,18 @@
+""" Tests for get_crontab_contents.py"""
+
+import unittest
+
+from python_utils import define_macos_utilities, set_env_var
+from get_crontab_contents import get_crontab_contents
+
+class Testing(unittest.TestCase):
+    """ Define the tests"""
+    def test_get_crontab_contents(self):
+        """ Call the function and make sure it doesn't fail. """
+        crontab_cmd, _ = get_crontab_contents(called_from_cron=True)
+        self.assertEqual(crontab_cmd, "crontab")
+
+    def setUp(self):
+        define_macos_utilities()
+        set_env_var("DEBUG", False)
+        set_env_var("MACHINE", "HERA")