Feature 839 conda envs in automation (#967)

.github/actions/run_tests/Dockerfile.gempak

@@ -0,0 +1,11 @@
+ARG METPLUS_ENV_TAG=metplus_base
+ARG METPLUS_IMG_TAG=develop
+
+FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env
+
+ARG METPLUS_IMG_TAG=develop
+FROM dtcenter/metplus-dev:${METPLUS_IMG_TAG}
+
+COPY --from=env /usr/lib/jvm/jre /usr/lib/jvm/jre/
+COPY --from=env /usr/share/javazi-1.8/tzdb.dat /usr/share/javazi-1.8/
+COPY --from=env /data/input/GempakToCF.jar /data/input/GempakToCF.jar

.github/actions/run_tests/Dockerfile.run

@@ -0,0 +1,11 @@
+ARG METPLUS_ENV_TAG=metplus_base
+ARG METPLUS_IMG_TAG=develop
+
+FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env
+
+ARG METPLUS_IMG_TAG=develop
+FROM dtcenter/metplus-dev:${METPLUS_IMG_TAG}
+
+COPY --from=env /usr/local/envs /usr/local/envs/
+
+COPY --from=env /usr/local/bin/conda /usr/local/bin/conda

ci/actions/run_tests/action.yml → .github/actions/run_tests/action.yml

@@ -6,14 +6,9 @@ inputs:
   categories:
     description: 'Use case category or categories to run (separate by comma)'
     required: true
-  run_diff:
-    description: 'Obtain truth data and run diffing logic if true'
-    required: false
-    default: false
 
 runs:
   using: "docker"
   image: "Dockerfile"
   args:
     - ${{ inputs.categories }}
-    - ${{ inputs.run_diff }}

.github/actions/run_tests/entrypoint.sh

@@ -0,0 +1,92 @@
+#! /bin/bash
+
+# The repo source code is cloned to $RUNNER_WORKSPACE/$REPO_NAME
+# Setup the workspace path to that for easier access later
+REPO_NAME=$(basename $RUNNER_WORKSPACE)
+WS_PATH=$RUNNER_WORKSPACE/$REPO_NAME
+
+# set CI jobs directory variable to easily move it
+CI_JOBS_DIR=.github/jobs
+
+source ${GITHUB_WORKSPACE}/${CI_JOBS_DIR}/bash_functions.sh
+
+# get branch name for push or pull request events
+# add -pull_request if pull request event to keep separated
+branch_name=`${GITHUB_WORKSPACE}/${CI_JOBS_DIR}/print_branch_name.py`
+if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then
+  branch_name=${branch_name}-pull_request
+fi
+
+# try to pull image from DockerHub
+DOCKERHUBTAG=dtcenter/metplus-dev:${branch_name}
+time_command docker pull $DOCKERHUBTAG
+
+# if unsuccessful (i.e. pull request from a fork)
+# then build image locally
+docker inspect --type=image $DOCKERHUBTAG > /dev/null
+if [ $? != 0 ]; then
+   # if docker pull fails, build locally
+   echo docker pull failed. Building Docker image locally...
+   ${GITHUB_WORKSPACE}/${CI_JOBS_DIR}/docker_setup.sh
+fi
+
+#
+# running unit tests (pytests)
+#
+if [ "$INPUT_CATEGORIES" == "pytests" ]; then
+  export METPLUS_ENV_TAG="pytest"
+  export METPLUS_IMG_TAG=${branch_name}
+  echo METPLUS_ENV_TAG=${METPLUS_ENV_TAG}
+  echo METPLUS_IMG_TAG=${METPLUS_IMG_TAG}
+
+  export RUN_TAG=metplus-run-env
+
+  # use BuildKit to build image
+  export DOCKER_BUILDKIT=1
+
+  start_seconds=$SECONDS
+
+  # build an image with the pytest conda env and the METplus branch image
+  # Note: adding --build-arg <arg-name> without any value tells docker to
+  #  use value from local environment (export METPLUS_IMG_TAG)
+  time_command docker build -t $RUN_TAG \
+	 --build-arg METPLUS_IMG_TAG \
+	 --build-arg METPLUS_ENV_TAG \
+	 -f .github/actions/run_tests/Dockerfile.run \
+	 .
+
+  echo Running Pytests
+  command="export METPLUS_PYTEST_HOST=docker; cd internal_tests/pytests; /usr/local/envs/pytest/bin/pytest -vv --cov=../../metplus"
+  time_command docker run -v $WS_PATH:$GITHUB_WORKSPACE --workdir $GITHUB_WORKSPACE $RUN_TAG bash -c "$command"
+  exit $?
+fi
+
+#
+# running use case tests
+#
+
+# split apart use case category and subset list from input
+CATEGORIES=`echo $INPUT_CATEGORIES | awk -F: '{print $1}'`
+SUBSETLIST=`echo $INPUT_CATEGORIES | awk -F: '{print $2}'`
+
+# run all cases if no subset list specified
+if [ -z "${SUBSETLIST}" ]; then
+    SUBSETLIST="all"
+fi
+
+# get METviewer if used in any use cases
+all_requirements=`./${CI_JOBS_DIR}/get_requirements.py ${CATEGORIES} ${SUBSETLIST}`
+echo All requirements: $all_requirements
+NETWORK_ARG=""
+if [[ "$all_requirements" =~ .*"metviewer".* ]]; then
+  echo "Setting up METviewer"
+  ${GITHUB_WORKSPACE}/${CI_JOBS_DIR}/get_metviewer.sh
+  NETWORK_ARG=--network="container:mysql_mv"
+fi
+
+# export network arg so it can be read by setup_and_run_use_cases.py
+export NETWORK_ARG
+
+# call script to loop over use case groups to
+# get data volumes, set up run image, and run use cases
+./${CI_JOBS_DIR}/setup_and_run_use_cases.py ${CATEGORIES} ${SUBSETLIST}

.github/jobs/bash_functions.sh

@@ -0,0 +1,16 @@
+#! /bin/bash
+
+# utility function to run command get log the time it took to run
+function time_command {
+  local start_seconds=$SECONDS
+  echo "RUNNING: $*"
+  "$@"
+  local error=$?
+
+  local duration=$(( SECONDS - start_seconds ))
+  echo "TIMING: Command took `printf '%02d' $(($duration / 60))`:`printf '%02d' $(($duration % 60))` (MM:SS): '$*'"
+  if [ ${error} -ne 0 ]; then
+    echo "ERROR: '$*' exited with status = ${error}"
+  fi
+  return $error
+}

.github/jobs/build_documentation.sh

@@ -22,6 +22,9 @@ if [ -s $warning_file ]; then
   cp -r ${DOCS_DIR}/_build/warnings.log artifact/doc_warnings.log
   cp artifact/doc_warnings.log artifact/documentation
   echo ERROR: Warnings/Errors found in documentation
+  echo Summary:
+  grep WARNING ${DOCS_DIR}/_build/warnings.log
+  grep ERROR ${DOCS_DIR}/_build/warnings.log
   echo Review this log file or download documentation_warnings.log artifact
   exit 1
 fi

.github/jobs/copy_error_logs.py

@@ -0,0 +1,47 @@
+#! /usr/bin/env python3
+
+################################################################################
+# Used in GitHub Actions (in .github/workflows/testing.yml) to copy logs for
+# use cases that reported errors to another directory
+
+import os
+import sys
+import shutil
+
+def main(output_data_dir, error_logs_dir):
+    """! Copy log output to error log directory if any use case failed """
+    for use_case_dir in os.listdir(output_data_dir):
+        log_dir = os.path.join(output_data_dir,
+                               use_case_dir,
+                               'logs')
+        if not os.path.isdir(log_dir):
+            continue
+
+        # check if there are errors in the metplus.log file and
+        # only copy directory if there are any errors
+        metplus_log = os.path.join(log_dir, 'metplus.log')
+        found_errors = False
+        with open(metplus_log, 'r') as file_handle:
+            if 'ERROR:' in file_handle.read():
+                found_errors = True
+
+        if not found_errors:
+            continue
+
+        output_dir = os.path.join(error_logs_dir,
+                                  use_case_dir)
+        log_files = os.listdir(log_dir)
+        for log_file in log_files:
+            log_path = os.path.join(log_dir, log_file)
+            output_path = os.path.join(output_dir, log_file)
+            print(f"Copying {log_path} to {output_path}")
+            # create output directory if it doesn't exist
+            output_dir = os.path.dirname(output_path)
+            if not os.path.exists(output_dir):
+                os.makedirs(output_dir)
+            shutil.copyfile(log_path, output_path)
+
+if __name__ == '__main__':
+    output_data_dir = sys.argv[1]
+    error_logs_dir = sys.argv[2]
+    main(output_data_dir, error_logs_dir)

ci/jobs/create_output_data_volumes.sh → .github/jobs/create_output_data_volumes.sh

@@ -9,7 +9,7 @@ if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then
   exit 0
 fi
 
-branch_name=`${GITHUB_WORKSPACE}/ci/jobs/print_branch_name.py`
+branch_name=`${GITHUB_WORKSPACE}/.github/jobs/print_branch_name.py`
 
 if [ "${branch_name: -4}" != "-ref" ]; then
   echo Branch ${branch_name} is not a reference branch, so skip this step

ci/jobs/docker_setup.sh → .github/jobs/docker_setup.sh

@@ -5,71 +5,51 @@
 # used by the use case tests.
 # If GitHub Actions run is triggered by a fork that does not have
 # permissions to push Docker images to DockerHub, the script is
-# is also called (in ci/actions/run_tests/entrypoint.sh) to
+# is also called (in .github/actions/run_tests/entrypoint.sh) to
 # build the Docker image to use for each use case test group
 
-branch_name=`${GITHUB_WORKSPACE}/ci/jobs/print_branch_name.py`
+source ${GITHUB_WORKSPACE}/.github/jobs/bash_functions.sh
+
+branch_name=`${GITHUB_WORKSPACE}/.github/jobs/print_branch_name.py`
 if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then
   branch_name=${branch_name}-pull_request
 fi
 
-#DOCKERHUB_TAG=dtcenter/metplus-dev:${DOCKER_IMAGE}
 DOCKERHUB_TAG=dtcenter/metplus-dev:${branch_name}
 
 echo Get Docker image: ${DOCKERHUB_TAG}
-echo 'doing docker build'
-# Note: adding --build-arg <arg-name> without any value tells docker to
-#  use value from local environment (export DO_GIT_CLONE)
 
-echo Timing docker pull...
+# can't use time_command function for this command because it contains redirection
 start_seconds=$SECONDS
 
 # pipe result to true because it will fail if image has not yet been built
 docker pull ${DOCKERHUB_TAG} &> /dev/null || true
 
 duration=$(( SECONDS - start_seconds ))
-echo TIMING docker_setup
-echo "Docker pull took $(($duration / 60)) minutes and $(($duration % 60)) seconds."
-
-echo Timing docker build with --cache-from...
-start_seconds=$SECONDS
+echo "TIMING: docker pull ${DOCKERHUB_TAG} took `printf '%02d' $(($duration / 60))`:`printf '%02d' $(($duration % 60))` (MM:SS)"
 
 # set DOCKERFILE_PATH that is used by docker hook script get_met_version
 export DOCKERFILE_PATH=${GITHUB_WORKSPACE}/ci/docker/Dockerfile
 
 MET_TAG=`${GITHUB_WORKSPACE}/ci/docker/hooks/get_met_version`
-echo Running docker build with MET_TAG=$MET_TAG
 
-docker build --pull --cache-from ${DOCKERHUB_TAG} \
+echo Setting DOCKER_BUILDKIT=1
+export DOCKER_BUILDKIT=1
+
+time_command docker build --pull --cache-from ${DOCKERHUB_TAG} \
 -t ${DOCKERHUB_TAG} \
---build-arg OBTAIN_SOURCE_CODE='copy' \
+--build-arg OBTAIN_SOURCE_CODE=copy \
 --build-arg MET_TAG=$MET_TAG \
 -f ${DOCKERFILE_PATH} ${GITHUB_WORKSPACE}
 
-duration=$(( SECONDS - start_seconds ))
-echo TIMING docker_setup
-echo "Docker build took $(($duration / 60)) minutes and $(($duration % 60)) seconds."
-echo
-
 # skip docker push if credentials are not set
 if [ -z ${DOCKER_USERNAME+x} ] || [ -z ${DOCKER_PASSWORD+x} ]; then
     echo "DockerHub credentials not set. Skipping docker push"
     exit 0
 fi
 
-echo Timing docker push...
-start_seconds=$SECONDS
-
 echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
-docker push ${DOCKERHUB_TAG}
+time_command docker push ${DOCKERHUB_TAG}
 
-duration=$(( SECONDS - start_seconds ))
-echo TIMING docker_setup
-echo "Docker push took $(($duration / 60)) minutes and $(($duration % 60)) seconds."
-echo
-
-echo DOCKER IMAGES after DOCKER_SETUP
+echo Running docker images
 docker images
-echo
-
-echo 'done'

ci/jobs/get_artifact_name.sh → .github/jobs/get_artifact_name.sh

@@ -1,7 +1,7 @@
 #! /bin/bash
 
 # Run by GitHub Actions (in .github/workflows/testing.yml and
-# ci/actions/run_tests/entrypoint.sh) to get properly
+# .github/actions/run_tests/entrypoint.sh) to get properly
 # formatted artifact name for use case output
 
 artifact_name=$1

ci/jobs/get_data_volumes.py → .github/jobs/get_data_volumes.py

@@ -1,6 +1,6 @@
 #! /usr/bin/env python3
 
-# Run by GitHub Actions (in ci/actions/run_tests/entrypoint.sh)
+# Run by GitHub Actions (in .github/actions/run_tests/entrypoint.sh)
 # to obtain Docker data volumes for input and output data, create
 # an alias name for the volumes, and generate --volumes-from arguments
 # that are added to the Docker run command to make data available
@@ -13,23 +13,25 @@
 from docker_utils import docker_get_volumes_last_updated, get_branch_name
 from docker_utils import get_data_repo, DOCKERHUB_METPLUS_DATA_DEV
 
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__),
+def main(args):
+    # get METplus version
+    version_file = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                 os.pardir,
-                                                os.pardir,)))
-
-from metplus import __version__
-
-# METPLUS_VERSION should be set to develop or a release version, i.e. X.Y
-# if version is set to X.Y without -betaZ or -dev, use that version
-# otherwise use develop
-if len(__version__.split('-')) == 1:
-    # only get first 2 numbers from version, i.e. X.Y.Z will use X.Y
-    METPLUS_VERSION = '.'.join(__version__.split('.')[:2])
-
-else:
-    METPLUS_VERSION = 'develop'
+                                                os.pardir,
+                                                'metplus',
+                                                'VERSION'))
+    with open(version_file, 'r') as file_handle:
+        version = file_handle.read().strip()
+
+    # version should be set to develop or a release version, i.e. X.Y
+    # if version is set to X.Y without -betaZ or -dev, use that version
+    # otherwise use develop
+    if len(version.split('-')) == 1:
+        # only get first 2 numbers from version, i.e. X.Y.Z will use X.Y
+        metplus_version = '.'.join(version.split('.')[:2])
+    else:
+        metplus_version = 'develop'
 
-def main(args):
     volume_list = []
 
     # get the name of the current branch
@@ -44,7 +46,7 @@ def main(args):
 
     # if running development version, use metplus-data-dev
     # if released version, i.e. X.Y.Z, use metplus-data
-    data_repo = get_data_repo(METPLUS_VERSION)
+    data_repo = get_data_repo(metplus_version)
 
     if branch_name.startswith('main_v'):
         branch_name = branch_name[5:]
@@ -60,7 +62,7 @@ def main(args):
 
         # if getting all input data, set volume name to METplus version
         if model_app_name == 'all_metplus_data':
-            volume_name = METPLUS_VERSION
+            volume_name = metplus_version
 
         # requested data volume is output data
         # should match output-{pr_dest_branch}-use_cases_{dataset_id}
@@ -82,11 +84,11 @@ def main(args):
 
         # if using development version and branch data volume is available
         # use it, otherwise use develop version of data volume
-        elif (METPLUS_VERSION == 'develop' and
+        elif (metplus_version == 'develop' and
               f'{branch_name}-{model_app_name}' in available_volumes):
                 volume_name = f'{branch_name}-{model_app_name}'
         else:
-            volume_name = f'{METPLUS_VERSION}-{model_app_name}'
+            volume_name = f'{metplus_version}-{model_app_name}'
 
         cmd = f'docker pull {repo_to_use}:{volume_name}'
         ret = subprocess.run(shlex.split(cmd), stdout=subprocess.DEVNULL)
@@ -110,6 +112,7 @@ def main(args):
 if __name__ == "__main__":
     # split up command line args that have commas before passing into main
     args = []
+
     for arg in sys.argv[1:]:
         args.extend(arg.split(','))
     out = main(args)