diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1a374d892..000000000 --- a/.travis.yml +++ /dev/null @@ -1,105 +0,0 @@ -dist: focal -os: linux - -git: - depth: 150 - -services: - - docker - -env: - global: - - PYTHONIOENCODING=UTF8 - - DOCKER_REGISTRY="quay.io" - - DOCKER_REPO_PROD="quay.io/broadinstitute/viral-pipelines" - - DOCKER_REPO_DEV="quay.io/broadinstitute/viral-pipelines-dev" - - MODULE_VERSIONS="requirements-modules.txt" - -stages: - - validate - - build - -jobs: - fast_finish: true - include: - - language: python - stage: validate - python: "3.8" - env: - - TRAVIS_JOB=validate_wdl_miniwdl - before_install: - - sudo apt-get update - - sudo apt-get -y install shellcheck - install: - - pip3 -q install miniwdl - script: - - set -e - - travis/check-wdl-runtimes.sh - - miniwdl check pipes/WDL/workflows/*.wdl - - - language: java - jdk: openjdk11 - stage: validate - env: - - TRAVIS_JOB=validate_wdl_womtool - install: - - travis/install-wdl.sh - script: - - set -e - - travis/check-wdl-runtimes.sh - - travis/validate-wdl-womtool.sh - - - language: python - stage: validate - env: TRAVIS_JOB=test_docs - install: travis/install-pip-docs.sh - script: travis/build-docs.sh - ## note: this build_docs job does not actually produce the output on readthedocs - ## readthedocs does its own build trigger. this travis job exists simply to alert us - ## of build failures of the docs because otherwise we would never know. - - - language: java - jdk: openjdk11 - stage: build - env: - - TRAVIS_JOB=test_cromwell - install: - - travis/install-wdl.sh - script: - - set -e - - travis/relative-wdl-paths.sh - - travis/tests-cromwell.sh - - - language: python - stage: build - python: "3.8" - env: - - TRAVIS_JOB=test_miniwdl - before_install: - - sudo apt-get update - - sudo apt-get -y install jq - install: - - pip3 -q install miniwdl - script: - - travis/tests-miniwdl.sh - - - language: java - jdk: openjdk11 - stage: build - if: branch = dnanexus - env: - - TRAVIS_JOB=deploy_dnanexus - # DX_API_TOKEN (for DNAnexus builds) -- token #3, expires 12/31/2024 - - secure: "zAAxwXWwSDcwnLYHawc0+gbshV3UURAZWSyk8g99kXgFCRcgytacvUpRxd44mC+rlgL2RtS8+RPT11hO2zJ+4y0J0FBaC8d9R7aTzEAGKLHk11cfhMg63QMVBt/8Gz9HToeE0BJ8/JbNZr+IZ01dD8GNDRA4FcFj+8maTpp4nrQ5/leDdmUwzkV1XfMiE+0/wR3KttKvqzHy6+6gRTAKbTeSxPICvWpHKlIYkIxFcEoH+3SNxgwWU9jZqG29MRt3/ikzZMBW/T2O9g+tY/5vKRqxm9aqWChFgjrpWRZDwvZqvwdxizokoVfReLZES3Ls//YSKzfFJG2GZ6a2vhAbfjcUkbRnyPOcZPwK+sE91Pq7FgFwL4N8BV63j6GQ5VrHZxe0b6spw8gvlkEjjhBgN1178lcxnp75QANNJ8AprZBqi3MzTQOTlgRZRAsDWTLgpXODrcDib76pg3af+JK5PQb0ncRVps9Z09u2EJ8fk9MGCDB6iw7XFedPPCxlx6MyIp2ywF4v9mkQXUBafYo67KRyPaojUUDGFkJsylYxzGHvvp0PrU93cp8zvDg4NqUkrbgEdGVvPF+qmXxT9MpfLc/yZ4vfhtKfZuQUGmNw+5WfTa3RYbJDEowN6noWCdXBZ67SrusKrok3TxIJV25vxXrFVhD/u/3ermysu7xA9ss=" - - DX_PROJECT=project-F8PQ6380xf5bK0Qk0YPjB17P - before_install: - - sudo apt-get -y install python2-minimal - install: - - travis/install-wdl.sh - script: - - set -e - - travis/relative-wdl-paths.sh - - travis/build-dx.sh - - travis/tests-dx.sh - - diff --git a/travis/build-docs.sh b/travis/build-docs.sh deleted file mode 100755 index a4b6379cb..000000000 --- a/travis/build-docs.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -e - -pushd docs -make html && echo "Docs built successfully!" || echo "Docs did NOT build successfully." -popd diff --git a/travis/build-dx.sh b/travis/build-dx.sh deleted file mode 100755 index cd7ea823d..000000000 --- a/travis/build-dx.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash -set -e -o pipefail - -if [ -z "$DX_API_TOKEN" ]; then - echo "ERROR: DX_API_TOKEN is not set, this is needed to build dxWDL workflows." - exit 1 -fi - -# obtain version tag -VERSION=`travis/list-docker-tags.sh | tail -1 | sed 's/:/\//'` - -# log in to DNAnexus -source dx-toolkit/environment -dx login --token "$DX_API_TOKEN" --noprojects -dx select $DX_PROJECT - -# compile with dxWDL -COMPILE_SUCCESS="dxWDL-compile_all-success.txt" -touch $COMPILE_SUCCESS -for workflow in pipes/WDL/workflows/*.wdl; do - if [ -n "$(grep DX_SKIP_WORKFLOW $workflow)" ]; then - echo "Skipping $workflow due to the presence of the DX_SKIP_WORKFLOW tag" - else - workflow_name=`basename $workflow .wdl` - echo "Building $workflow to DNAnexus: /build/$VERSION/$workflow_name" - - defaults_json="pipes/dnax/dx-defaults-$workflow_name.json" - if [ -f "$defaults_json" ]; then - CMD_DEFAULTS="-defaults $defaults_json" - else - CMD_DEFAULTS="" - fi - - extras_json="pipes/dnax/dx-extras.json" - CMD_DEFAULTS+=" -extras $extras_json" - - dx_id=$(java -jar dxWDL.jar compile \ - $workflow $CMD_DEFAULTS -f -verbose \ - -leaveWorkflowsOpen \ - -imports pipes/WDL/tasks/ \ - -project $DX_PROJECT \ - -destination /build/$VERSION/$workflow_name) - if [ $? -eq 0 ]; then - echo "Succeeded: $workflow_name = $dx_id" - else - echo "Failed to build: $workflow_name" - exit $? - fi - echo -e "$workflow_name\t$dx_id" >> $COMPILE_SUCCESS - fi -done - -# build consolidate_run_tarballs (native DNAnexus applet) applet -pushd pipes/dnax/dx-launcher -cp consolidate_run_tarballs.yml consolidate_run_tarballs_dxapp.yml -consolidate_tarballs_dx_id=$(./dx-yml-build consolidate_run_tarballs_dxapp.yml -a --destination /build/$VERSION/ | jq -r ".id") -popd -echo -e "consolidate_run_tarballs\t$dx_id" >> $COMPILE_SUCCESS - -# Special case: build demux launchers (native DNAnexus applets), embedding the -# demux workflow ID as a default input -demux_workflows_to_build="demux_plus demux_only" -for wf_name in $(echo "${demux_workflows_to_build}"); do - echo "Building applet ${wf_name}..." - demux_workflow_id=$(grep "^${wf_name}\s" $COMPILE_SUCCESS | cut -f 2) - pushd pipes/dnax/dx-launcher - sed "s/DEFAULT_DEMUX_WORKFLOW_ID/$demux_workflow_id/" demux_launcher.yml \ - | sed "s/DEFAULT_DEMUX_WORKFLOW_NAME/${wf_name}_launcher/" \ - | sed "s/DEFAULT_CONSOLIDATE_RUN_TARBALLS_APPLET_ID/$consolidate_tarballs_dx_id/" > "${wf_name}_dxapp.yml" - dx_id=$(./dx-yml-build ${wf_name}_dxapp.yml -a --destination /build/$VERSION/ | jq -r ".id") - popd - echo -e "${wf_name}_launcher\t$dx_id" >> $COMPILE_SUCCESS -done - -# the presence of this file in the project denotes successful build -dx upload --brief --no-progress --destination /build/$VERSION/ $COMPILE_SUCCESS diff --git a/travis/check-wdl-runtimes.sh b/travis/check-wdl-runtimes.sh deleted file mode 100755 index 047e65631..000000000 --- a/travis/check-wdl-runtimes.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# requires $MODULE_VERSIONS to be set to point to a text file with equal-sign-separated values -# export MODULE_VERSIONS="./requirements-modules.txt" && ./github_actions_ci/check-wdl-runtimes.sh - -echo "Checking wdl container versions against ${MODULE_VERSIONS}" - -# this is the newer script that simply validates existing version strings -should_error=false -for task_file in $(ls -1 pipes/WDL/tasks/*.wdl); do - echo "Checking ${task_file}" - while IFS='=' read module version; do - OLD_TAG=$module - NEW_TAG="$module:$version" - - offending_lines="$(grep -nE "^[^#]*$OLD_TAG" "${task_file}" | grep -v $NEW_TAG)" - - # if the expected tag is not seen, let us know the file and exit - if [ $? -eq 0 ]; then - offending_lines="$(echo "${offending_lines}" | sed 's/^/ /g')" - echo " \"$NEW_TAG\" needed in \"${task_file}\":" - echo "$offending_lines" - should_error=true #exit 1 eventually, but only after printing all of the problems - fi - offending_lines="" - - done < "${MODULE_VERSIONS}" -done -if [ "$should_error" = true ]; then - exit 1 -fi \ No newline at end of file diff --git a/travis/dockstoreyml.sh b/travis/dockstoreyml.sh deleted file mode 100755 index a1abb2944..000000000 --- a/travis/dockstoreyml.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -set -e -o pipefail - -echo "version: 1.2" -echo "workflows:" -for WDL in $*; do - echo " - name: $(basename $WDL .wdl)" - echo " subclass: WDL" - echo " primaryDescriptorPath: $WDL" - echo " testParameterFiles:" - echo " - empty.json" -done diff --git a/travis/install-pip-docs.sh b/travis/install-pip-docs.sh deleted file mode 100755 index dccee7330..000000000 --- a/travis/install-pip-docs.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -echo "pip installing test-related packages (Sphinx, etc.)" -pip install --quiet -r docs/requirements.txt diff --git a/travis/install-wdl.sh b/travis/install-wdl.sh deleted file mode 100755 index 8fd820752..000000000 --- a/travis/install-wdl.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -e -o pipefail - -fetch_jar_from_github () { - _github_org=$1 - _repo_name=$2 - _tool_name=$3 - _jar_version=$4 - _jar_fname="$_tool_name-$_jar_version.jar" - echo "Fetching $_jar_fname" - wget --quiet https://github.com/$_github_org/$_repo_name/releases/download/$_jar_version/$_jar_fname - ln -s $_jar_fname $_tool_name.jar -} - -fetch_jar_from_github broadinstitute cromwell womtool 61 -fetch_jar_from_github broadinstitute cromwell cromwell 61 -fetch_jar_from_github dnanexus dxWDL dxWDL v1.50 - -TGZ=dx-toolkit-v0.298.1-ubuntu-20.04-amd64.tar.gz -echo "Fetching $TGZ" -wget --quiet https://dnanexus-sdk.s3.amazonaws.com/$TGZ -tar -xzpf $TGZ \ No newline at end of file diff --git a/travis/list-docker-tags.sh b/travis/list-docker-tags.sh deleted file mode 100755 index 2bb3fcf75..000000000 --- a/travis/list-docker-tags.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -# -# This script lists all docker tags that should apply to this particular -# deploy, including short tags (like repo:latest) and long tags (like -# repo:full-version-number). Tags are emitted one per line. -# -# Important note: the last tag is considered the definitive, uniquely -# identifiable tag that should be used for all build steps that depend on -# the docker image (e.g. check-wdl-runtimes.sh). All others are aliases. - -set -e -o pipefail - -if [ -z "$DOCKER_REPO_PROD" -o -z "$DOCKER_REPO_DEV" -o -z "$TRAVIS_BRANCH" ]; then - echo "This script requires the following environment variables set: DOCKER_REPO_PROD, DOCKER_REPO_DEV, and TRAVIS_BRANCH." - exit 1 -fi - -# figure out appropriate tag names -if [ -n "$TRAVIS_TAG" ]; then - # this is an official tagged release - DOCKER_REPO=$DOCKER_REPO_PROD - DOCKER_SHORT_TAG="latest" - DOCKER_LONG_TAG="$(echo $TRAVIS_TAG | sed 's/^v//')" - PUSH_TO_MIRROR=true -elif [ -n "$TRAVIS_PULL_REQUEST_BRANCH" ]; then - # this is a PR build (TRAVIS_BRANCH=target of PR, TRAVIS_PULL_REQUEST_BRANCH=source of PR) - DOCKER_REPO=$DOCKER_REPO_DEV - BRANCH_NAME="$TRAVIS_PULL_REQUEST_BRANCH" - DOCKER_SHORT_TAG="$BRANCH_NAME-pull_request" - DOCKER_LONG_TAG="$(git describe --tags --always | sed s/^v//)-$(echo $DOCKER_SHORT_TAG)" - #DOCKER_LONG_TAG="$(git describe --tags --always | perl -lape 's/^v?(\S+)-(\d+)-g(\S+)/$1-beta$2-g$3/')-$(echo $BRANCH_NAME | sed 's/-/_/g')" -elif [[ "$TRAVIS_BRANCH" == "master" ]]; then - # this is a master branch commit (e.g. merged pull request) - DOCKER_REPO=$DOCKER_REPO_PROD - DOCKER_SHORT_TAG="latest" - DOCKER_LONG_TAG="$(git describe --tags --always | perl -lape 's/^v?(\S+)-(\d+)-g\S+/$1-rc$2/')" - PUSH_TO_MIRROR=true -else - # this is an normal non-master branch commit - DOCKER_REPO=$DOCKER_REPO_DEV - BRANCH_NAME="$TRAVIS_BRANCH" - DOCKER_SHORT_TAG="$BRANCH_NAME" - DOCKER_LONG_TAG="$(git describe --tags --always | sed s/^v//)-$(echo $DOCKER_SHORT_TAG)" - #DOCKER_LONG_TAG="$(git describe --tags --always | perl -lape 's/^v?(\S+)-(\d+)-g(\S+)/$1-dev$2-g$3/')-$(echo $DOCKER_SHORT_TAG)" -fi - -if [ -n "$DOCKER_REPO_PROD_MIRROR" -a -n "$PUSH_TO_MIRROR" ]; then - # if this should be sent to prod, also echo the mirror registry tags - echo $DOCKER_REPO_PROD_MIRROR:$DOCKER_SHORT_TAG - echo $DOCKER_REPO_PROD_MIRROR:$DOCKER_LONG_TAG -fi - -echo $DOCKER_REPO:$DOCKER_SHORT_TAG -echo $DOCKER_REPO:$DOCKER_LONG_TAG diff --git a/travis/relative-wdl-paths.sh b/travis/relative-wdl-paths.sh deleted file mode 100755 index bbb63439f..000000000 --- a/travis/relative-wdl-paths.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -e -o pipefail - -echo Replacing relative paths to same directory -sed -i -- 's|import \"../tasks/|import \"|g' pipes/WDL/workflows/*.wdl diff --git a/travis/tests-cromwell.sh b/travis/tests-cromwell.sh deleted file mode 100755 index d9daf5fac..000000000 --- a/travis/tests-cromwell.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -set -e # intentionally allow for pipe failures below - -mkdir -p workflows -cp *.jar pipes/WDL/workflows/*.wdl pipes/WDL/tasks/*.wdl workflows -cp -r test workflows/ -cd workflows - -for workflow in ../pipes/WDL/workflows/*.wdl; do - workflow_name=$(basename $workflow .wdl) - input_json="test/input/WDL/cromwell-local/test_inputs-$workflow_name-local.json" - if [ -f $input_json ]; then - date - echo "Executing $workflow_name using Cromwell on local instance" - # the "cat" is to allow a pipe failure (otherwise it halts because of set -e) - java -Dconfig.file=../pipes/cromwell/cromwell.local-travis.conf \ - -jar cromwell.jar run \ - $workflow_name.wdl \ - -i $input_json | tee cromwell.out - if [ ${PIPESTATUS[0]} -gt 0 ]; then - echo "error running $workflow_name" - error_logs=$(grep stderr cromwell.out | perl -lape 's/.*\s(\S+)$/$1/g') - for log in $error_logs; do - echo "contents of stderr ($log):" - cat `dirname $log`/stderr | sed "s/^/[STDERR] /" - echo "contents of stdout ($log):" - cat `dirname $log`/stdout | sed "s/^/[STDOUT] /" - done - sync; sleep 20; exit 1 - fi - fi -done - -cd - -date -echo "note: there is no testing of output correctness yet..." diff --git a/travis/tests-dx.sh b/travis/tests-dx.sh deleted file mode 100755 index 7bee86348..000000000 --- a/travis/tests-dx.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/bash -set -e -o pipefail - -# obtain version tag -VERSION=`travis/list-docker-tags.sh | tail -1 | sed 's/:/\//'` - -# log in to DNAnexus -source dx-toolkit/environment -if [ -n "$DX_API_TOKEN" ]; then - dx login --token "$DX_API_TOKEN" --noprojects - dx select $DX_PROJECT -fi - -COMPILE_SUCCESS="dxWDL-compile_all-success.txt" -if [ ! -f $COMPILE_SUCCESS ]; then - dx download --no-progress /build/$VERSION/$COMPILE_SUCCESS -fi - -function dx_run_timeout_args { - # - # Construct command-line arguments for 'dx run' command - # to set a timeout on the applets it runs - # - - local dx_workflow_id="$1" - local dx_extra_applet_id="$2" - - local dx_workflow_applet_ids=$(dx describe $dx_workflow_id | grep applet- | awk '{print $2;}') - local dx_applet_ids="$dx_workflow_applet_ids $dx_extra_applet_id" - local comma="" - local timeout_args="{\"timeoutPolicyByExecutable\":{" - for dx_applet_id in $dx_applet_ids - do - timeout_args="${timeout_args}${comma}\"$dx_applet_id\":{\"*\":{\"hours\":3}}" - comma="," - done - timeout_args="$timeout_args}}" - echo $timeout_args -} - -TEST_LAUNCH_ALL="dxWDL-execute_all-launched.txt" -touch $TEST_LAUNCH_ALL -for workflow in pipes/WDL/workflows/*.wdl; do - echo "testing $workflow..." - if [ -n "$(grep DX_SKIP_WORKFLOW $workflow)" ]; then - echo "Skipping $workflow due to the presence of the DX_SKIP_WORKFLOW tag" - else - workflow_name=`basename $workflow .wdl` - input_json="test/input/WDL/test_inputs-$workflow_name-dnanexus.dx.json" - if [ -f $input_json ]; then - # launch simple test cases on DNAnexus CI project - dx_workflow_id=$(grep -w "^$workflow_name" $COMPILE_SUCCESS | cut -f 2) - timeout_args=$(dx_run_timeout_args $dx_workflow_id) - echo "running test $workflow_name - $dx_workflow_id -y --brief -f $input_json --extra-args $timeout_args" - dx_job_id=$(dx run \ - $dx_workflow_id -y --brief \ - -f $input_json \ - --name "$VERSION $workflow_name" \ - --destination /tests/$VERSION/$workflow_name \ - #--extra-args $timeout_args \ - ) - if [ $? -eq 0 ]; then - echo "Launched $workflow_name as $dx_job_id" - else - echo "Failed to build: $workflow_name" - fi - echo "Launched $workflow_name as $dx_job_id" - echo -e "$workflow_name\t$dx_workflow_id\t$dx_job_id" >> $TEST_LAUNCH_ALL - fi - fi -done - -# only run demux_plus if this is on master or tagged branch -if [ "$TRAVIS_BRANCH" = "master" -o -n "$TRAVIS_TAG" ]; then - demux_name="demux_plus" -else - # otherwise just run the (faster) demux_only - demux_name="demux_only" -fi - -# Special case: run test for the demux_(plus|only)_launcher native applet (which invokes -# the demux_(plus|only) WDL workflow) -demux_launcher_id=$(grep "^${demux_name}_launcher\s" $COMPILE_SUCCESS | cut -f 2) - -demux_workflow_id=$(grep "^${demux_name}\s" $COMPILE_SUCCESS | cut -f 2) - -timeout_args=$(dx_run_timeout_args $demux_workflow_id $demux_launcher_id) -dx_job_id=$(dx run "${demux_launcher_id}" \ - -y --brief \ - -i upload_sentinel_record=record-Bv8qkgQ0jy198GK0QVz2PV8Y \ - -i demux_workflow_id=${demux_workflow_id} \ - --name "$VERSION ${demux_name}_launcher" \ - -i folder=/tests/$VERSION/${demux_name}_launcher \ - --extra-args $timeout_args \ - ) -echo "Launched ${demux_name}_launcher as $dx_job_id" -echo -e "${demux_name}_launcher\t$demux_launcher_id\t$dx_job_id" >> $TEST_LAUNCH_ALL - -# the presence of this file in the project denotes all tests launched -dx upload --brief --no-progress --destination /build/$VERSION/ $TEST_LAUNCH_ALL - -# -# Cleanup folders w/ files that are 30 days old (@yihchii) -# -THIRTY_DAYS_FILES="30d_old_files.json" -dx find data --class file --created-before=-30d --path=/tests/ --json > $THIRTY_DAYS_FILES -[ "$(cat $THIRTY_DAYS_FILES)" != "[]" ] && dx rm -r -f $(jq -r '(.[].describe.folder + "/")' $THIRTY_DAYS_FILES | sort -u) -rm -rf $THIRTY_DAYS_FILES - -# Cleanup empty folders -# - Loop over all versioned sub-folders and clean them up if they are empty -ALL_VERSION_FOLDERS="all_version_folders.txt" -CURRENT_FOLDER="/tests/$VERSION/" -dx ls --folders --full $(dirname $CURRENT_FOLDER) > $ALL_VERSION_FOLDERS -for folder in $(cat $ALL_VERSION_FOLDERS); do - content=$(dx find data --path $folder --brief) - [ -z "$content" ] && dx rm -rf $folder -done -rm -rf $ALL_VERSION_FOLDERS \ No newline at end of file diff --git a/travis/tests-miniwdl.sh b/travis/tests-miniwdl.sh deleted file mode 100755 index 818024c98..000000000 --- a/travis/tests-miniwdl.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -set -ex -o pipefail - -starting_dir="$(pwd)" -test_dir="miniwdl_testing" - -function cleanup(){ - echo "Cleaning up from miniwdl run; exit code: $?" - cd "$starting_dir" - if [ -d "$test_dir" ]; then - rm -r "$test_dir" - fi -} -trap cleanup EXIT SIGINT SIGQUIT SIGTERM - -mkdir $test_dir -cp -r test $test_dir -cd $test_dir - -docker --version - -# make sure our system has everything it needs to perform "miniwdl run" (e.g. docker swarm works) -miniwdl run_self_test - -for workflow in ../pipes/WDL/workflows/*.wdl; do - workflow_name=$(basename $workflow .wdl) - input_json="test/input/WDL/miniwdl-local/test_inputs-$workflow_name-local.json" - expected_output_json="test/input/WDL/miniwdl-local/test_outputs-$workflow_name-local.json" - if [ -f $input_json ]; then - date - echo "Executing $workflow_name using miniWDL on local instance" - # the following invocation with -d $workflow_name/. tells miniwdl not to create - # a timestamped subdirectory within $workflow_name/ - time miniwdl run -i $input_json -d $workflow_name/. --error-json --verbose $workflow - # the existence of $workflow_name/outputs.json is a guarantee of successful execution - if [ -f $workflow_name/outputs.json ]; then - echo "$workflow_name SUCCESS -- outputs:" - cat $workflow_name/outputs.json - if [ -f $expected_output_json ]; then - echo "$workflow_name -- validating outputs" - touch expected actual - # here we create a key=value text table for the output in "actual" for only the - # subset of keys that are specified in $expected_output_json, ignoring any - # of the actual output values that aren't specified there. - for k in `cat $expected_output_json | jq -r 'keys[]'`; do - echo -n "$k=" >> expected - echo -n "$k=" >> actual - cat $expected_output_json | jq -r '.["'$k'"]' >> expected - cat $workflow_name/outputs.json | jq -r '.["'$k'"]' >> actual - done - # diff returns non-zero (and triggers failure by set -e) if these are non-equal - # and prints all differences between expected and actual for this workflow at the same time - diff expected actual - fi - else - echo "$workflow_name FAILED" - exit 1 - fi - fi -done - -cd "$starting_dir" -date diff --git a/travis/validate-wdl-womtool.sh b/travis/validate-wdl-womtool.sh deleted file mode 100755 index f5433b75d..000000000 --- a/travis/validate-wdl-womtool.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -e -o pipefail - -for workflow in pipes/WDL/workflows/*.wdl; do - echo -n "validating $workflow ... " - if $(hash -r womtool &> /dev/null); then - womtool validate $workflow - else - java -jar womtool.jar validate $workflow - fi -done diff --git a/travis/version-wdl-runtimes.sh b/travis/version-wdl-runtimes.sh deleted file mode 100755 index 9b7c25656..000000000 --- a/travis/version-wdl-runtimes.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# use sed to replace version strings of docker images based on versions defined in txt file - -# requires $MODULE_VERSIONS to be set to point to a text file with equal-sign-separated values -# export MODULE_VERSIONS="./requirements-modules.txt" && ./travis/check-wdl-runtimes.sh - -while IFS='=' read module version; do - OLD_TAG=$module - NEW_TAG="$module:$version" - echo Replacing $OLD_TAG with $NEW_TAG in all task WDL files - sed -i '' "s|$OLD_TAG[^\"\']*|$NEW_TAG|g" pipes/WDL/tasks/*.wdl -done < $MODULE_VERSIONS \ No newline at end of file