Feature 1546 CI testing (#2029)

Co-authored-by: John Halley Gotway <johnhg@ucar.edu>

.github/jobs/Dockerfile.truth

@@ -0,0 +1,17 @@
+FROM centos:7
+MAINTAINER George McCabe <mccabe@ucar.edu>
+
+ENV OUTPUT_DIR /data/output
+RUN mkdir -p ${OUTPUT_DIR}
+
+ARG TRUTH_DIR
+
+COPY ${TRUTH_DIR} ${OUTPUT_DIR}/
+
+ARG TRUTH_DIR
+
+# Define the volume mount point
+VOLUME ${OUTPUT_DIR}/${TRUTH_DIR}
+
+USER root
+CMD ["true"]

.github/jobs/bash_functions.sh

@@ -0,0 +1,27 @@
+#! /bin/bash
+
+# utility function to run command get log the time it took to run
+# if CMD_LOGFILE is set, send output to that file and unset var
+function time_command {
+  local start_seconds=$SECONDS
+  echo "RUNNING: $*"
+
+  local error
+  # pipe output to log file if set
+  if [ "x$CMD_LOGFILE" == "x" ]; then
+      "$@"
+      error=$?
+  else
+      echo "Logging to ${CMD_LOGFILE}"
+      "$@" &>> $CMD_LOGFILE
+      error=$?
+      unset CMD_LOGFILE
+  fi
+
+  local duration=$(( SECONDS - start_seconds ))
+  echo "TIMING: Command took `printf '%02d' $(($duration / 60))`:`printf '%02d' $(($duration % 60))` (MM:SS): '$*'"
+  if [ ${error} -ne 0 ]; then
+    echo "ERROR: '$*' exited with status = ${error}"
+  fi
+  return $error
+}

.github/jobs/build_docker_image.sh

@@ -0,0 +1,14 @@
+#! /bin/bash
+
+source ${GITHUB_WORKSPACE}/.github/jobs/bash_functions.sh
+
+DOCKERHUB_TAG=${DOCKERHUB_REPO}:${SOURCE_BRANCH}
+
+DOCKERFILE_PATH=${GITHUB_WORKSPACE}/scripts/docker/Dockerfile.copy
+
+CMD_LOGFILE=${GITHUB_WORKSPACE}/docker_build.log
+
+time_command docker build -t ${DOCKERHUB_TAG} \
+    --build-arg SOURCE_BRANCH \
+    --build-arg MET_BASE_IMAGE \
+    -f $DOCKERFILE_PATH ${GITHUB_WORKSPACE}

.github/jobs/copy_diff_files.py

@@ -0,0 +1,80 @@
+#! /usr/bin/env python3
+
+import os
+import shutil
+
+OUTPUT_DIR = os.environ['MET_TEST_OUTPUT']
+TRUTH_DIR = os.environ['MET_TEST_TRUTH']
+DIFF_DIR = os.environ['MET_TEST_DIFF']
+
+LOG_DIR = '/met/logs'
+
+def get_files_with_diffs(log_file):
+    files_to_copy = set()
+
+    with open(log_file, 'r') as file_handle:
+        file_content = file_handle.read()
+
+    missing_section, *test_sections = file_content.split(
+      '\n# # # # # # # # # # # # # # # # # # # # # # # # # # # # # #\n'
+    )
+
+    # parse list of missing files
+    if 'ERROR:' in missing_section:
+        for missing_group in missing_section.split('ERROR:')[1:]:
+            dir_str, *rel_paths = missing_group.splitlines()
+            dir_str = dir_str.split()[1]
+            if OUTPUT_DIR in dir_str:
+                top_dir = dir_str.replace(OUTPUT_DIR, TRUTH_DIR)
+            elif TRUTH_DIR in dir_str:
+                top_dir = dir_str.replace(TRUTH_DIR, OUTPUT_DIR)
+            else:
+                print("ERROR: SOMETHING WENT WRONG PARSING COMP_DIR OUTPUT")
+                continue
+            for rel_path in rel_paths:
+                files_to_copy.add(os.path.join(top_dir, rel_path.strip()))
+
+    # parse file paths out of sections that have errors
+    error_sections = [item for item in test_sections if 'ERROR:' in item]
+    for error_section in error_sections:
+        for line in error_section.splitlines():
+            for item in line.split():
+                if OUTPUT_DIR in item or TRUTH_DIR in item:
+                    files_to_copy.add(item)
+
+    return files_to_copy
+
+def copy_files_to_diff_dir(files_to_copy):
+
+    print(f"Found {len(files_to_copy)} diff files")
+
+    # add extension for output/truth and copy files to diff directory
+    for filename in files_to_copy:
+        output_path, extension = os.path.splitext(filename)
+        if OUTPUT_DIR in output_path:
+            output_path = f'{output_path}_OUTPUT{extension}'
+            output_path = output_path.replace(OUTPUT_DIR, DIFF_DIR)
+        elif TRUTH_DIR in output_path:
+            output_path = f'{output_path}_TRUTH{extension}'
+            output_path = output_path.replace(TRUTH_DIR, DIFF_DIR)
+        else:
+            continue
+
+        # change bad char - this can be removed once test output is changed
+        output_path = output_path.replace(':', '_')
+
+        print(f"Copy {filename} to {output_path}")
+        output_dir = os.path.dirname(output_path)
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+        shutil.copyfile(filename, output_path)
+
+def main():
+    log_file = os.path.join(LOG_DIR, 'comp_dir.log')
+    print(f"Parsing {log_file}")
+    all_files_to_copy = get_files_with_diffs(log_file)
+
+    copy_files_to_diff_dir(all_files_to_copy)
+
+if __name__ == "__main__":
+    main()

.github/jobs/create_docker_truth.sh

@@ -0,0 +1,18 @@
+#! /bin/bash
+
+source ${GITHUB_WORKSPACE}/.github/jobs/bash_functions.sh
+
+image_name=dtcenter/met-data-output:${TRUTH_DATA_VERSION}
+
+time_command docker build -t ${image_name} \
+       --build-arg TRUTH_DIR=met_test_truth \
+       -f ${GITHUB_WORKSPACE}/.github/jobs/Dockerfile.truth \
+       ${RUNNER_WORKSPACE}
+if [ $? != 0 ]; then
+  echo "ERROR: Docker build failed"
+  exit 1
+fi
+
+echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
+
+time_command docker push ${image_name}

.github/jobs/get_branch_name.sh

@@ -0,0 +1,15 @@
+#! /bin/bash
+
+# If pull request, use GitHub head ref and add -PR to end
+# Otherwise use GitHub ref
+
+if [ "${GITHUB_EVENT_NAME}" == "pull_request" ] ; then
+  branch_name=${GITHUB_HEAD_REF}-PR
+else
+  branch_name=${GITHUB_REF}
+fi
+
+branch_name=${branch_name#"refs/heads/"}
+
+echo ::set-output name=branch_name::$branch_name
+echo branch_name: $branch_name

.github/jobs/get_test_input_data.sh

@@ -0,0 +1,7 @@
+#! /bin/bash
+
+source ${GITHUB_WORKSPACE}/.github/jobs/bash_functions.sh
+
+DATA_VERSION=$1
+
+time_command docker create --name met_input dtcenter/met-data-dev:${DATA_VERSION}

.github/jobs/get_test_truth_data.sh

@@ -0,0 +1,11 @@
+#! /bin/bash
+
+source ${GITHUB_WORKSPACE}/.github/jobs/bash_functions.sh
+
+DATA_VERSION=$1
+
+time_command docker create --name met_truth dtcenter/met-data-output:${DATA_VERSION}
+if [ $? != 0 ]; then
+  echo "Image tag ${DATA_VERSION} does not exist. Using develop..."
+  time_command docker create --name met_truth dtcenter/met-data-output:develop
+fi

.github/jobs/pull_docker_image.sh

@@ -0,0 +1,7 @@
+#! /bin/bash
+
+source ${GITHUB_WORKSPACE}/.github/jobs/bash_functions.sh
+
+DOCKERHUB_TAG=$1
+
+time_command docker pull ${DOCKERHUB_TAG}

.github/jobs/push_docker_image.sh

@@ -0,0 +1,15 @@
+#! /bin/bash
+
+source ${GITHUB_WORKSPACE}/.github/jobs/bash_functions.sh
+
+DOCKERHUB_TAG=${DOCKERHUB_REPO}:${SOURCE_BRANCH}
+
+# skip docker push if credentials are not set
+if [ -z ${DOCKER_USERNAME+x} ] || [ -z ${DOCKER_PASSWORD+x} ]; then
+    echo "DockerHub credentials not set. Skipping docker push"
+    exit 0
+fi
+
+echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
+
+time_command docker push ${DOCKERHUB_TAG}

.github/jobs/run_diff_docker.sh

@@ -0,0 +1,48 @@
+#! /bin/bash
+
+source ${GITHUB_WORKSPACE}/.github/jobs/bash_functions.sh
+
+DOCKERHUB_TAG=${DOCKERHUB_REPO}:${SOURCE_BRANCH}
+
+# Get truth output data
+${GITHUB_WORKSPACE}/.github/jobs/get_test_truth_data.sh ${TRUTH_DATA_VERSION}
+
+# Set up directories to mount
+LOCAL_OUTPUT_DIR=${RUNNER_WORKSPACE}/output
+DOCKER_OUTPUT_DIR=/data/output/met_test_output
+
+LOCAL_DIFF_DIR=${RUNNER_WORKSPACE}/diff
+DOCKER_DIFF_DIR=/data/output/met_test_diff
+
+LOCAL_LOG_DIR=${RUNNER_WORKSPACE}/logs
+DOCKER_LOG_DIR=/met/logs
+
+# Create local directories to store output
+mkdir -p ${LOCAL_LOG_DIR}
+mkdir -p ${LOCAL_DIFF_DIR}
+
+# mount output and log dirs, mount GitHub files into MET_REPO_DIR
+mount_args="-v ${LOCAL_OUTPUT_DIR}:${DOCKER_OUTPUT_DIR} -v ${LOCAL_DIFF_DIR}:${DOCKER_DIFF_DIR} -v ${LOCAL_LOG_DIR}:${DOCKER_LOG_DIR}"
+
+# Set up data volumes
+volumes_from="--volumes-from met_truth"
+
+# run unit test script inside Docker, mount MET output and truth data
+# set MET_REPO_DIR env var in Docker to mounted directory
+cmd="\${MET_REPO_DIR}/.github/jobs/run_diff_tests.sh"
+time_command docker run ${volumes_from} ${mount_args} ${DOCKERHUB_TAG} bash -c \"${cmd}\"
+if [ $? != 0 ]; then
+  exit 1
+fi
+
+if [ "$(ls -A ${LOCAL_DIFF_DIR})" ]; then
+  echo "ERROR: Differences exist in the output"
+
+  # only exit non-zero (job fails) if not updating truth data
+  # this makes difference output available when updating truth data
+  # so it is easier to see what changed with the update
+  if [ "${RUN_UPDATE_TRUTH}" != "true" ]; then
+    exit 1
+  fi
+
+fi

.github/jobs/run_diff_tests.sh

@@ -0,0 +1,33 @@
+#! /bin/bash
+
+source ${MET_REPO_DIR}/.github/jobs/bash_functions.sh
+
+###
+# Set environment variables needed to run unit tests
+###
+
+source ${MET_REPO_DIR}/.github/jobs/test_env_vars.sh
+
+###
+# Run comparison of MET unit test output
+###
+
+echo "Running comparison on test output"
+CMD_LOGFILE=/met/logs/comp_dir.log
+time_command ${MET_TEST_BASE}/bin/comp_dir.sh ${MET_TEST_TRUTH} ${MET_TEST_OUTPUT}
+if [ $? != 0 ]; then
+    echo "ERROR: Test output comparison failed"
+    cat /met/logs/comp_dir.log
+    exit 1
+fi
+
+echo "Running copy_diff_files.py"
+CMD_LOGFILE=/met/logs/copy_diff_files.log
+time_command ${MET_REPO_DIR}/.github/jobs/copy_diff_files.py
+if [ $? != 0 ]; then
+    echo "ERROR: Copy diff files script failed"
+    cat /met/logs/copy_diff_files.log
+    exit 1
+fi
+
+echo "Success"

.github/jobs/run_unit_docker.sh

@@ -0,0 +1,37 @@
+#! /bin/bash
+
+source ${GITHUB_WORKSPACE}/.github/jobs/bash_functions.sh
+
+DOCKERHUB_TAG=${DOCKERHUB_REPO}:${SOURCE_BRANCH}
+
+# Pull MET Image from DockerHub
+${GITHUB_WORKSPACE}/.github/jobs/pull_docker_image.sh ${DOCKERHUB_TAG}
+
+# Get test input data if needed
+volumes_from=""
+if [ "${INPUT_DATA_VERSION}" != "none" ]; then
+  ${GITHUB_WORKSPACE}/.github/jobs/get_test_input_data.sh ${INPUT_DATA_VERSION}
+  volumes_from=${volumes_from}"--volumes-from met_input"
+fi
+
+# Set up directories to mount
+LOCAL_OUTPUT_DIR=${RUNNER_WORKSPACE}/output
+DOCKER_OUTPUT_DIR=/data/output/met_test_output
+
+LOCAL_LOG_DIR=${RUNNER_WORKSPACE}/logs
+DOCKER_LOG_DIR=/met/logs
+
+# Create local directories to store output
+mkdir -p ${LOCAL_LOG_DIR}
+mkdir -p ${LOCAL_OUTPUT_DIR}
+
+mount_args="-v ${LOCAL_OUTPUT_DIR}:${DOCKER_OUTPUT_DIR} -v ${LOCAL_LOG_DIR}:${DOCKER_LOG_DIR}"
+
+export TESTS_TO_RUN=$TESTS
+
+# run unit test script inside Docker, mount MET input and truth data
+cmd="\${MET_REPO_DIR}/.github/jobs/run_unit_tests.sh"
+time_command docker run -e TESTS_TO_RUN ${volumes_from} ${mount_args} ${DOCKERHUB_TAG} bash -c \"${cmd}\"
+if [ $? != 0 ]; then
+  exit 1
+fi