Update develop-ref after #860 (#863)

ci/actions/run_tests/entrypoint.sh

@@ -48,10 +48,10 @@ fi
 
 # install Pillow library needed for diff testing
 # this will be replaced with better image diffing package used by METplotpy
-pip_command="pip3 install Pillow"
+pip_command="pip3 install Pillow; yum -y install poppler-utils; pip3 install pdf2image"
 
 # build command to run
-command="./ci/jobs/run_use_cases_docker.py ${CATEGORIES} ${SUBSETLIST}"
+command="./ci/jobs/run_use_cases.py ${CATEGORIES} ${SUBSETLIST}"
 
 # add input volumes to run command
 # keep track of --volumes-from arguments to docker run command

ci/jobs/create_output_data_volumes.sh

@@ -1,5 +1,9 @@
 #! /bin/bash
 
+# Run by GitHub Actions (in .github/workflows/main.yml) to create
+# Docker data volumes from output data to create a "truth"
+# data set to use in difference tests.
+
 if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then
   echo This is a pull request, so skip this setp
   exit 0

ci/jobs/diff_output.py

@@ -1,5 +1,8 @@
 #! /usr/bin/env python3
 
+# Run by GitHub Actions (in ci/jobs/run_use_cases.py) to
+# trigger difference tests.
+
 import sys
 import os
 

ci/jobs/docker_setup.sh

@@ -1,5 +1,13 @@
 #! /bin/bash
 
+# Run by GitHub Actions (in .github/workflows/main.yml) to build
+# METplus Docker image and put it up to DockerHub so it can be
+# used by the use case tests.
+# If GitHub Actions run is triggered by a fork that does not have
+# permissions to push Docker images to DockerHub, the script is
+# is also called (in ci/actions/run_tests/entrypoint.sh) to
+# build the Docker image to use for each use case test group
+
 branch_name=`${GITHUB_WORKSPACE}/ci/jobs/print_branch_name.py`
 if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then
   branch_name=${branch_name}-pull_request

ci/jobs/docker_update_data_volumes.py

@@ -1,5 +1,10 @@
 #! /usr/bin/env python3
 
+# Run by GitHub Actions (in .github/workflows/main.yml) check DTCenter web
+# server for any input data tarfiles that have been updated and need to be
+# regenerated as Docker data volumes to be used in use case tests.
+# Push new/updated data volumes up to DockerHub
+
 import sys
 import os
 import shlex
@@ -65,6 +70,9 @@ def main():
         print("Could not get current branch. Exiting.")
         sys.exit(1)
 
+    if branch_name.endswith('-ref'):
+        branch_name = branch_name[0:-4]
+
     if not os.environ.get('GITHUB_WORKSPACE'):
         print("GITHUB_WORKSPACE not set. Exiting.")
         sys.exit(1)

ci/jobs/docker_utils.py

@@ -1,6 +1,11 @@
 import os
 import re
 
+# Utilities used by various CI jobs. Functionality includes:
+#  - Check if Docker data volumes need to be updated.
+#  - Get appropriate branch name to use to obtain/create Docker
+#    images. This is needed for pull request runs.
+
 # repository used for storing input data for development branches
 DOCKERHUB_DATA_REPO = 'dtcenter/metplus-data-dev'
 

ci/jobs/get_artifact_name.sh

@@ -1,5 +1,9 @@
 #! /bin/bash
 
+# Run by GitHub Actions (in .github/workflows/main.yml and
+# ci/actions/run_tests/entrypoint.sh) to get properly
+# formatted artifact name for use case output
+
 artifact_name=$1
 # strip of :NEW if found at end of name
 if [ ${artifact_name: -4} == ":NEW" ]; then

ci/jobs/get_data_volumes.py

@@ -1,5 +1,10 @@
 #! /usr/bin/env python3
 
+# Run by GitHub Actions (in ci/actions/run_tests/entrypoint.sh)
+# to obtain Docker data volumes for input and output data, create
+# an alias name for the volumes, and generate --volumes-from arguments
+# that are added to the Docker run command to make data available
+
 import sys
 import os
 import subprocess

ci/jobs/get_use_case_commands.py

@@ -1,5 +1,9 @@
 #! /usr/bin/env python3
 
+# Script to obtain commands needed to run use case groups including
+# scripts or pip commands to obtain external Python dependencies
+# Run by GitHub Actions (in ci/jobs/run_use_cases.py) to run use case tests
+
 import sys
 import os
 
@@ -19,6 +23,7 @@ def handle_requirements(requirements, work_dir):
         script_path = os.path.join(work_dir,
                                    'ci',
                                    'jobs',
+                                   'python_requirements',
                                     f'get_{requirement.lower()}.sh')
         print(f"Looking for script: {script_path}")
         if os.path.exists(script_path):

ci/jobs/print_branch_name.py

@@ -1,5 +1,9 @@
 #! /usr/bin/env python3
 
+# Script to easily get branch name from docker_utils function
+# Run by GitHub Actions (in ci/actions/run_tests/entrypoint.sh,
+# ci/jobs/create_output_data_volumes.sh, and ci/jobs/docker_setup.sh)
+
 from docker_utils import get_branch_name
 
 print(get_branch_name())

ci/jobs/print_python_version.py

@@ -1,5 +1,8 @@
 #! /usr/bin/env python3
 
+# Script to easily obtain minimum python version requirement
+# Used in GitHub Actions (in ci/jobs/python_requirements/get_miniconda.sh)
+
 import sys
 import os
 

ci/jobs/get_metcalcpy.sh → ci/jobs/python_requirements/get_metcalcpy.sh

@@ -5,7 +5,7 @@ pip3 install scipy
 pip3 install pingouin
 
 basedir=$(dirname "$0")
-work_dir=$basedir/../..
+work_dir=$basedir/../../..
 
 # run manage externals to obtain METcalcpy
 ${work_dir}/manage_externals/checkout_externals -e ${work_dir}/ci/parm/Externals_metcalcpy.cfg

ci/jobs/get_metplotpy.sh → ci/jobs/python_requirements/get_metplotpy.sh

@@ -5,7 +5,7 @@ pip3 install scipy
 pip3 install cmocean
 
 basedir=$(dirname "$0")
-work_dir=$basedir/../..
+work_dir=$basedir/../../..
 
 # run manage externals to obtain METcalcpy
 ${work_dir}/manage_externals/checkout_externals -e ${work_dir}/ci/parm/Externals_metplotpy.cfg

ci/jobs/get_miniconda.sh → ci/jobs/python_requirements/get_miniconda.sh

@@ -2,7 +2,7 @@
 
 script_dir=$(dirname "$0")
 
-python_version=`${script_dir}/print_python_version.py`
+python_version=`${script_dir}/../print_python_version.py`
 
 # these are used to obtain version of MiniConda3
 # the version determines the default version of Python

ci/jobs/get_pygrib.sh → ci/jobs/python_requirements/get_pygrib.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 
 #shell script to install pygrib with dependencies
-# todd arbetter: arbetter@ucar.edu
 
 yum -y install eccodes-devel
 pip3 install numpy