From cc2173b80606ffbb7bf017fcc57c0efc081bb8e7 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Mon, 4 Dec 2023 22:19:39 -0500 Subject: [PATCH 01/13] Is this faster? --- .../lots_of_inputs/lots_of_inputs.wdl | 7 +++++++ src/ci/bin/testCentaurHoricromtalPapiV2beta.sh | 12 +++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/centaur/src/main/resources/standardTestCases/lots_of_inputs/lots_of_inputs.wdl b/centaur/src/main/resources/standardTestCases/lots_of_inputs/lots_of_inputs.wdl index 73a483cb7f9..4c6e5b06ab5 100644 --- a/centaur/src/main/resources/standardTestCases/lots_of_inputs/lots_of_inputs.wdl +++ b/centaur/src/main/resources/standardTestCases/lots_of_inputs/lots_of_inputs.wdl @@ -7,8 +7,14 @@ task do_nothing { output { String o = read_string(stdout()) } + + # We are localizing 10,000 tiny files, so the default disk size of 10 GB SSD [0] is NOT + # going to cut it, having a measly IO per second rating of just 10 x 30 = 300 IOPS [1] + # [0] https://cromwell.readthedocs.io/en/stable/RuntimeAttributes/ + # [1] https://cloud.google.com/compute/docs/disks/performance#zonal runtime { docker: "python:latest" + disks: "local-disk 1000 SSD" } } @@ -28,6 +34,7 @@ task make_array { } runtime { docker: "python:latest" + disks: "local-disk 1000 SSD" } } diff --git a/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh b/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh index 36b215a2f52..ef01da73f3b 100755 --- a/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh +++ b/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh @@ -20,11 +20,13 @@ cromwell::build::build_cromwell_docker cromwell::build::run_centaur \ -p 500 \ - -e localdockertest \ - -e relative_output_paths \ - -e relative_output_paths_colliding \ - -e standard_output_paths_colliding_prevented \ - -e papi_v2alpha1_gcsa \ + -i lots_of_inputs \ +# -e localdockertest \ +# -e relative_output_paths \ +# -e relative_output_paths_colliding \ +# -e standard_output_paths_colliding_prevented \ +# -e papi_v2alpha1_gcsa \ + cromwell::build::generate_code_coverage From 56fc2b55d5d2a9627a710aee344d854188c41b7b Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Mon, 4 Dec 2023 23:28:24 -0500 Subject: [PATCH 02/13] Revert --- src/ci/bin/testCentaurHoricromtalPapiV2beta.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh b/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh index ef01da73f3b..84dd89ed3e6 100755 --- a/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh +++ b/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh @@ -20,12 +20,11 @@ cromwell::build::build_cromwell_docker cromwell::build::run_centaur \ -p 500 \ - -i lots_of_inputs \ -# -e localdockertest \ -# -e relative_output_paths \ -# -e relative_output_paths_colliding \ -# -e standard_output_paths_colliding_prevented \ -# -e papi_v2alpha1_gcsa \ + -e localdockertest \ + -e relative_output_paths \ + -e relative_output_paths_colliding \ + -e standard_output_paths_colliding_prevented \ + -e papi_v2alpha1_gcsa \ cromwell::build::generate_code_coverage From c650c0a596f40de7fd6bf3efb78c96958ec6574d Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Mon, 4 Dec 2023 23:28:55 -0500 Subject: [PATCH 03/13] Oh come on --- src/ci/bin/testCentaurHoricromtalPapiV2beta.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh b/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh index 84dd89ed3e6..36b215a2f52 100755 --- a/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh +++ b/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh @@ -26,7 +26,6 @@ cromwell::build::run_centaur \ -e standard_output_paths_colliding_prevented \ -e papi_v2alpha1_gcsa \ - cromwell::build::generate_code_coverage cromwell::build::print_workflow_statistics From 0131686b34c8d90b75e13aed97df7de982ad551b Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Tue, 5 Dec 2023 16:03:30 -0500 Subject: [PATCH 04/13] Exclude not-that-many inputs --- src/ci/bin/testCentaurHoricromtalPapiV2beta.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh b/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh index 36b215a2f52..07158ecc490 100755 --- a/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh +++ b/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh @@ -25,6 +25,8 @@ cromwell::build::run_centaur \ -e relative_output_paths_colliding \ -e standard_output_paths_colliding_prevented \ -e papi_v2alpha1_gcsa \ + -e lots_of_inputs \ + # `lots_of_inputs` is redundant to, and a less rigorous version of, `lots_of_inputs_papiv2` cromwell::build::generate_code_coverage From 9b75117a46961ca0175b0b208667c7c1493acbcf Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Tue, 5 Dec 2023 17:36:09 -0500 Subject: [PATCH 05/13] Clean up old PAPI configs --- .../testCentaurEngineUpgradePapiV2alpha1.sh | 24 ------ ...aurHoricromtalEngineUpgradePapiV2alpha1.sh | 26 ------ .../bin/testCentaurHoricromtalPapiV2alpha1.sh | 29 ------- ...taurPapiUpgradeNewWorkflowsPapiV2alpha1.sh | 4 - .../bin/testCentaurPapiUpgradePapiV2alpha1.sh | 3 - src/ci/bin/testCentaurPapiV2alpha1.sh | 27 ------- src/ci/bin/testDockerScripts.sh | 27 ------- src/ci/bin/testMetadataComparisonPython.sh | 34 -------- .../papi_v1_v2alpha1_provider_config.inc.conf | 20 ----- .../resources/papi_v2alpha1_application.conf | 81 ------------------- ...v2alpha1_conformance_expected_failures.txt | 29 ------- ...papi_v2alpha1_horicromtal_application.conf | 9 --- ...i_v2alpha1_v2beta_upgrade_application.conf | 32 -------- 13 files changed, 345 deletions(-) delete mode 100755 src/ci/bin/testCentaurEngineUpgradePapiV2alpha1.sh delete mode 100755 src/ci/bin/testCentaurHoricromtalEngineUpgradePapiV2alpha1.sh delete mode 100755 src/ci/bin/testCentaurHoricromtalPapiV2alpha1.sh delete mode 100755 src/ci/bin/testCentaurPapiUpgradeNewWorkflowsPapiV2alpha1.sh delete mode 100755 src/ci/bin/testCentaurPapiUpgradePapiV2alpha1.sh delete mode 100755 src/ci/bin/testCentaurPapiV2alpha1.sh delete mode 100755 src/ci/bin/testDockerScripts.sh delete mode 100755 src/ci/bin/testMetadataComparisonPython.sh delete mode 100644 src/ci/resources/papi_v1_v2alpha1_provider_config.inc.conf delete mode 100644 src/ci/resources/papi_v2alpha1_application.conf delete mode 100644 src/ci/resources/papi_v2alpha1_conformance_expected_failures.txt delete mode 100644 src/ci/resources/papi_v2alpha1_horicromtal_application.conf delete mode 100644 src/ci/resources/papi_v2alpha1_v2beta_upgrade_application.conf diff --git a/src/ci/bin/testCentaurEngineUpgradePapiV2alpha1.sh b/src/ci/bin/testCentaurEngineUpgradePapiV2alpha1.sh deleted file mode 100755 index 7a471ce9e34..00000000000 --- a/src/ci/bin/testCentaurEngineUpgradePapiV2alpha1.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -o nounset -o pipefail -export CROMWELL_BUILD_REQUIRES_SECURE=true -export CROMWELL_BUILD_REQUIRES_PRIOR_VERSION=true -# import in shellcheck / CI / IntelliJ compatible ways -# shellcheck source=/dev/null -source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh -# shellcheck source=/dev/null -source "${BASH_SOURCE%/*}/test_papi.inc.sh" || source test_papi.inc.sh - -cromwell::build::setup_common_environment - -cromwell::build::setup_centaur_environment - -cromwell::build::papi::setup_papi_centaur_environment - -cromwell::build::assemble_jars - -cromwell::build::run_centaur \ - -s "centaur.EngineUpgradeTestCaseSpec" \ - -e localdockertest \ - -cromwell::build::generate_code_coverage diff --git a/src/ci/bin/testCentaurHoricromtalEngineUpgradePapiV2alpha1.sh b/src/ci/bin/testCentaurHoricromtalEngineUpgradePapiV2alpha1.sh deleted file mode 100755 index f4050548f38..00000000000 --- a/src/ci/bin/testCentaurHoricromtalEngineUpgradePapiV2alpha1.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -o nounset -o pipefail -export CROMWELL_BUILD_REQUIRES_SECURE=true -export CROMWELL_BUILD_REQUIRES_PRIOR_VERSION=true -# import in shellcheck / CI / IntelliJ compatible ways -# shellcheck source=/dev/null -source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh -# shellcheck source=/dev/null -source "${BASH_SOURCE%/*}/test_papi.inc.sh" || source test_papi.inc.sh - -cromwell::build::setup_common_environment - -cromwell::build::setup_centaur_environment - -cromwell::build::papi::setup_papi_centaur_environment - -cromwell::build::assemble_jars - -cromwell::build::build_cromwell_docker - -cromwell::build::run_centaur \ - -s "centaur.EngineUpgradeTestCaseSpec" \ - -e localdockertest - -cromwell::build::generate_code_coverage diff --git a/src/ci/bin/testCentaurHoricromtalPapiV2alpha1.sh b/src/ci/bin/testCentaurHoricromtalPapiV2alpha1.sh deleted file mode 100755 index 3e8453c9d83..00000000000 --- a/src/ci/bin/testCentaurHoricromtalPapiV2alpha1.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -o nounset -o pipefail -export CROMWELL_BUILD_REQUIRES_SECURE=true -# import in shellcheck / CI / IntelliJ compatible ways -# shellcheck source=/dev/null -source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh -# shellcheck source=/dev/null -source "${BASH_SOURCE%/*}/test_papi.inc.sh" || source test_papi.inc.sh - -cromwell::build::setup_common_environment - -cromwell::build::setup_centaur_environment - -cromwell::build::papi::setup_papi_centaur_environment - -cromwell::build::assemble_jars - -cromwell::build::build_cromwell_docker - -cromwell::build::run_centaur \ - -p 100 \ - -e localdockertest \ - -e relative_output_paths \ - -e relative_output_paths_colliding \ - -e standard_output_paths_colliding_prevented \ - -e papi_v2beta_gcsa \ - -cromwell::build::generate_code_coverage diff --git a/src/ci/bin/testCentaurPapiUpgradeNewWorkflowsPapiV2alpha1.sh b/src/ci/bin/testCentaurPapiUpgradeNewWorkflowsPapiV2alpha1.sh deleted file mode 100755 index 5b5ffe0ddc3..00000000000 --- a/src/ci/bin/testCentaurPapiUpgradeNewWorkflowsPapiV2alpha1.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -echo "$(tput setab 1)$(tput blink)BT-81: This sub-build was doing no useful work; it built Cromwell and launched Centaur but never found tests to run. However, tests for functionality implied by this sub-build's name may be desirable, see BT-81 for further information.$(tput sgr 0)" -exit 0 diff --git a/src/ci/bin/testCentaurPapiUpgradePapiV2alpha1.sh b/src/ci/bin/testCentaurPapiUpgradePapiV2alpha1.sh deleted file mode 100755 index 8c46a8223e7..00000000000 --- a/src/ci/bin/testCentaurPapiUpgradePapiV2alpha1.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -echo "$(tput setab 1)$(tput blink)BT-81: This sub-build was doing no useful work; it built Cromwell and launched Centaur but never found tests to run. However, tests for functionality implied by this sub-build's name may be desirable, see BT-81 for further information.$(tput sgr 0)"exit 0 diff --git a/src/ci/bin/testCentaurPapiV2alpha1.sh b/src/ci/bin/testCentaurPapiV2alpha1.sh deleted file mode 100755 index 706d13c0acb..00000000000 --- a/src/ci/bin/testCentaurPapiV2alpha1.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -o nounset -o pipefail -export CROMWELL_BUILD_REQUIRES_SECURE=true -# import in shellcheck / CI / IntelliJ compatible ways -# shellcheck source=/dev/null -source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh -# shellcheck source=/dev/null -source "${BASH_SOURCE%/*}/test_papi.inc.sh" || source test_papi.inc.sh - -cromwell::build::setup_common_environment - -cromwell::build::setup_centaur_environment - -cromwell::build::papi::setup_papi_centaur_environment - -cromwell::build::assemble_jars - -cromwell::build::run_centaur \ - -p 100 \ - -e localdockertest \ - -e relative_output_paths \ - -e relative_output_paths_colliding \ - -e standard_output_paths_colliding_prevented \ - -e papi_v2beta_gcsa \ - -cromwell::build::generate_code_coverage diff --git a/src/ci/bin/testDockerScripts.sh b/src/ci/bin/testDockerScripts.sh deleted file mode 100755 index e9f670bba54..00000000000 --- a/src/ci/bin/testDockerScripts.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -o nounset -o pipefail -# import in shellcheck / CI / IntelliJ compatible ways -# shellcheck source=/dev/null -source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh - -cromwell::build::setup_common_environment - -# When running locally, an `ADD .` in the Dockerfile might pull in files the local developer does not expect. -# Until there is further review of what goes into the docker image (ex: rendered vault secrets!) do not push it yet. -docker_tag="broadinstitute/cromwell-docker-develop:test-only-do-not-push" - -docker build -t "${docker_tag}" scripts/docker-develop - -echo "What tests would you like, my dear McMuffins?" - -echo "1. Testing for install of sbt" -docker run --rm "${docker_tag}" which sbt - -echo "2. Testing sbt assembly" -docker run \ - --rm \ - --volume "${CROMWELL_BUILD_ROOT_DIRECTORY}:${CROMWELL_BUILD_ROOT_DIRECTORY}" \ - --workdir "${CROMWELL_BUILD_ROOT_DIRECTORY}" \ - "${docker_tag}" \ - sbt -Dsbt.supershell=false --warn assembly diff --git a/src/ci/bin/testMetadataComparisonPython.sh b/src/ci/bin/testMetadataComparisonPython.sh deleted file mode 100755 index f12b9ee5fe7..00000000000 --- a/src/ci/bin/testMetadataComparisonPython.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -o nounset -o pipefail - -export CROMWELL_BUILD_REQUIRES_SECURE=true - -# import in shellcheck / CI / IntelliJ compatible ways -# shellcheck source=/dev/null -source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh - -cromwell::build::setup_common_environment - -# Our Python scripts can only be run by Python version >= 3.6, because we use string interpolation which was introduced -# in Python 3.6. But Travis environment is Ubuntu 16.04 Xenial LTS, which officially supports only Python <= 3.5. -# Thus we have to run Python tests in Docker container. -GOOGLE_CENTAUR_SERVICE_ACCOUNT_JSON="cromwell-centaur-service-account.json" -export GOOGLE_APPLICATION_CREDENTIALS="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/${GOOGLE_CENTAUR_SERVICE_ACCOUNT_JSON}" -export DIGESTER_TEST_GCS=true - -docker run -it --rm \ - -e GOOGLE_APPLICATION_CREDENTIALS \ - -e DIGESTER_TEST_GCS \ - -v "${CROMWELL_BUILD_RESOURCES_DIRECTORY}:${CROMWELL_BUILD_RESOURCES_DIRECTORY}" \ - -v "${CROMWELL_BUILD_ROOT_DIRECTORY}/scripts/metadata_comparison:/metadata_comparison" \ - python:3 /bin/bash -c " - -pip3 install --upgrade requests -pip3 install --upgrade google-api-python-client -pip3 install --upgrade google-cloud -pip3 install --upgrade google-cloud-storage -pip3 install --upgrade gitpython -pip3 install --upgrade python-dateutil -python3 -m unittest discover -v /metadata_comparison -" diff --git a/src/ci/resources/papi_v1_v2alpha1_provider_config.inc.conf b/src/ci/resources/papi_v1_v2alpha1_provider_config.inc.conf deleted file mode 100644 index 4b786c6ec2c..00000000000 --- a/src/ci/resources/papi_v1_v2alpha1_provider_config.inc.conf +++ /dev/null @@ -1,20 +0,0 @@ -project = "broad-dsde-cromwell-dev" -root = "gs://cloud-cromwell-dev-self-cleaning/cromwell_execution/ci" -maximum-polling-interval = 600 -concurrent-job-limit = 1000 - -genomics { - auth = "service_account" - endpoint-url = "https://genomics.googleapis.com/" -} - -filesystems { - gcs { - auth = "service_account" - project = "broad-dsde-cromwell-dev" - } -} - -slow-job-warning-time: 20 minutes - -checkpointing-interval: "10 seconds" diff --git a/src/ci/resources/papi_v2alpha1_application.conf b/src/ci/resources/papi_v2alpha1_application.conf deleted file mode 100644 index cc12d36b27d..00000000000 --- a/src/ci/resources/papi_v2alpha1_application.conf +++ /dev/null @@ -1,81 +0,0 @@ -include "papi_v2_shared_application.inc.conf" - -backend { - providers { - Papi { - actor-factory = "cromwell.backend.google.pipelines.v2alpha1.PipelinesApiLifecycleActorFactory" - config { - include "papi_v1_v2alpha1_provider_config.inc.conf" - } - } - Papiv2 { - actor-factory = "cromwell.backend.google.pipelines.v2alpha1.PipelinesApiLifecycleActorFactory" - config { - include "papi_v1_v2alpha1_provider_config.inc.conf" - } - } - Papiv2USADockerhub { - actor-factory = "cromwell.backend.google.pipelines.v2alpha1.PipelinesApiLifecycleActorFactory" - config { - include "papi_v1_v2alpha1_provider_config.inc.conf" - } - } - Papiv2NoDockerHubConfig { - actor-factory = "cromwell.backend.google.pipelines.v2alpha1.PipelinesApiLifecycleActorFactory" - config { - include "papi_v1_v2alpha1_provider_config.inc.conf" - } - } - Papiv2RequesterPays { - actor-factory = "cromwell.backend.google.pipelines.v2alpha1.PipelinesApiLifecycleActorFactory" - config { - include "papi_v1_v2alpha1_provider_config.inc.conf" - - # Need this again (even though it's in the "papiv2_shared" file because the provider_config include overrides it. - filesystems.gcs.auth = "requester_pays_service_account" - } - } - Papi-Caching-No-Copy { - actor-factory = "cromwell.backend.google.pipelines.v2alpha1.PipelinesApiLifecycleActorFactory" - config { - include "papi_v1_v2alpha1_provider_config.inc.conf" - } - } - Papiv2-Virtual-Private-Cloud-Labels { - actor-factory = "cromwell.backend.google.pipelines.v2alpha1.PipelinesApiLifecycleActorFactory" - config { - include "papi_v1_v2alpha1_provider_config.inc.conf" - } - } - Papiv2-Virtual-Private-Cloud-Literals { - actor-factory = "cromwell.backend.google.pipelines.v2alpha1.PipelinesApiLifecycleActorFactory" - config { - include "papi_v1_v2alpha1_provider_config.inc.conf" - } - } - papi-v2-gcsa { - actor-factory = "cromwell.backend.google.pipelines.v2alpha1.PipelinesApiLifecycleActorFactory" - config { - include "papi_v1_v2alpha1_provider_config.inc.conf" - } - } - papi-v2-usa { - actor-factory = "cromwell.backend.google.pipelines.v2alpha1.PipelinesApiLifecycleActorFactory" - config { - include "papi_v1_v2alpha1_provider_config.inc.conf" - } - } - Papiv2ParallelCompositeUploads { - actor-factory = "cromwell.backend.google.pipelines.v2alpha1.PipelinesApiLifecycleActorFactory" - config { - include "papi_v1_v2alpha1_provider_config.inc.conf" - } - } - Papiv2-Reference-Disk-Localization { - actor-factory = "cromwell.backend.google.pipelines.v2beta.PipelinesApiLifecycleActorFactory" - config { - include "papi_v2beta_provider_config.inc.conf" - } - } - } -} diff --git a/src/ci/resources/papi_v2alpha1_conformance_expected_failures.txt b/src/ci/resources/papi_v2alpha1_conformance_expected_failures.txt deleted file mode 100644 index 4caa25c75a6..00000000000 --- a/src/ci/resources/papi_v2alpha1_conformance_expected_failures.txt +++ /dev/null @@ -1,29 +0,0 @@ -86 -87 -101 -102 -104 -107 -136 -137 -140 -142 -152 -153 -154 -156 -158 -159 -160 -161 -163 -164 -165 -166 -170 -171 -173 -189 -190 -191 -196 diff --git a/src/ci/resources/papi_v2alpha1_horicromtal_application.conf b/src/ci/resources/papi_v2alpha1_horicromtal_application.conf deleted file mode 100644 index dacfbd63877..00000000000 --- a/src/ci/resources/papi_v2alpha1_horicromtal_application.conf +++ /dev/null @@ -1,9 +0,0 @@ -include "papi_v2alpha1_application.conf" - -system.max-workflow-launch-count=1 -system.new-workflow-poll-rate=10 -system.max-concurrent-workflows=30 -system.cromwell_id_random_suffix=false - -# Turn off token logging to reduce log volume -system.hog-safety.token-log-interval-seconds = 0 diff --git a/src/ci/resources/papi_v2alpha1_v2beta_upgrade_application.conf b/src/ci/resources/papi_v2alpha1_v2beta_upgrade_application.conf deleted file mode 100644 index 6059508a592..00000000000 --- a/src/ci/resources/papi_v2alpha1_v2beta_upgrade_application.conf +++ /dev/null @@ -1,32 +0,0 @@ -include required(classpath("application.conf")) -include "build_application.inc.conf" -include "papi_application.inc.conf" - -# We now have two types of PAPI backend to monitor: -services.HealthMonitor.config.check-papi-backends: [ Papiv2alpha1, Papiv2beta ] - -backend { - default = "Papiv2alpha1" - enabled = ["Papiv2alpha1", "Papiv2beta"] - providers { - # Pre-existing PAPI v2alpha1 backend - Papiv2alpha1 { - actor-factory = "cromwell.backend.google.pipelines.v2alpha1.PipelinesApiLifecycleActorFactory" - config { - include "papi_v1_v2alpha1_provider_config.inc.conf" - include "dockerhub_provider_config_v2.inc.conf" - genomics.compute-service-account = "centaur@broad-dsde-cromwell-dev.iam.gserviceaccount.com" - name-for-call-caching-purposes = "Papi" - } - } - Papiv2beta { - actor-factory = "cromwell.backend.google.pipelines.v2beta.PipelinesApiLifecycleActorFactory" - config { - include "papi_v2beta_provider_config.inc.conf" - include "dockerhub_provider_config_v2.inc.conf" - genomics.compute-service-account = "centaur@broad-dsde-cromwell-dev.iam.gserviceaccount.com" - name-for-call-caching-purposes = "Papi" - } - } - } -} From a249b005b76528e355fbada6835a7e26b92c3233 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Tue, 5 Dec 2023 18:16:20 -0500 Subject: [PATCH 06/13] Fix --- src/ci/bin/testCentaurPapiV2beta.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ci/bin/testCentaurPapiV2beta.sh b/src/ci/bin/testCentaurPapiV2beta.sh index 43d5a7d62aa..c349c7d0537 100755 --- a/src/ci/bin/testCentaurPapiV2beta.sh +++ b/src/ci/bin/testCentaurPapiV2beta.sh @@ -23,6 +23,8 @@ cromwell::build::run_centaur \ -e relative_output_paths_colliding \ -e standard_output_paths_colliding_prevented \ -e papi_v2alpha1_gcsa \ + -e lots_of_inputs \ + # `lots_of_inputs` is redundant to, and a less rigorous version of, `lots_of_inputs_papiv2` cromwell::build::generate_code_coverage From cb45489e48a6d6fbdc30533aa4642dbf5f2972d0 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Tue, 5 Dec 2023 20:00:26 -0500 Subject: [PATCH 07/13] Undo changes that didn't help --- .../standardTestCases/lots_of_inputs/lots_of_inputs.wdl | 7 ------- 1 file changed, 7 deletions(-) diff --git a/centaur/src/main/resources/standardTestCases/lots_of_inputs/lots_of_inputs.wdl b/centaur/src/main/resources/standardTestCases/lots_of_inputs/lots_of_inputs.wdl index 4c6e5b06ab5..73a483cb7f9 100644 --- a/centaur/src/main/resources/standardTestCases/lots_of_inputs/lots_of_inputs.wdl +++ b/centaur/src/main/resources/standardTestCases/lots_of_inputs/lots_of_inputs.wdl @@ -7,14 +7,8 @@ task do_nothing { output { String o = read_string(stdout()) } - - # We are localizing 10,000 tiny files, so the default disk size of 10 GB SSD [0] is NOT - # going to cut it, having a measly IO per second rating of just 10 x 30 = 300 IOPS [1] - # [0] https://cromwell.readthedocs.io/en/stable/RuntimeAttributes/ - # [1] https://cloud.google.com/compute/docs/disks/performance#zonal runtime { docker: "python:latest" - disks: "local-disk 1000 SSD" } } @@ -34,7 +28,6 @@ task make_array { } runtime { docker: "python:latest" - disks: "local-disk 1000 SSD" } } From d3583bb9bbbc7db6e6dc84f8dc4a2a410c7b75f1 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Wed, 6 Dec 2023 10:05:34 -0500 Subject: [PATCH 08/13] Cleanup --- src/ci/bin/test.inc.sh | 287 +---------------------------------------- 1 file changed, 4 insertions(+), 283 deletions(-) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 6f76ee8c05c..4833212d31b 100755 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -45,50 +45,13 @@ cromwell::private::check_debug() { fi } -cromwell::private::set_variable_if_only_some_files_changed() { - files_changed_regex=${1} - variable_to_set=${2} - - if [[ "${TRAVIS_EVENT_TYPE:-unset}" != "pull_request" ]]; then - export "${variable_to_set}=false" - else - # Large changesets seem to trigger the situation described in [1] where a `git diff` pipelined to `grep` can cause - # `grep` to exit 0 on the first match while `git diff` is still writing to the pipe. When this happens `git diff` - # is killed with a SIGPIPE and exits with code 141. With `set -o pipefail` this causes the entire pipeline to exit - # with code 141, which sets `$variable_to_set` to `true` when it probably should have been set to `false`. - # - # Instead of composing these commands into a pipeline write to a temporary file. - # - # [1] https://stackoverflow.com/a/19120674 - # [2] https://gist.github.com/mohanpedala/1e2ff5661761d3abd0385e8223e16425#set--o-pipefail - - files_changed_temporary_file=$(mktemp) - - git diff --name-only "origin/${TRAVIS_BRANCH}" > "${files_changed_temporary_file}" 2>&1 && \ - grep -E -q --invert-match "${files_changed_regex}" "${files_changed_temporary_file}" - RESULT=$? - - if [[ $RESULT -eq 0 ]]; then - export "${variable_to_set}=false" - else - export "${variable_to_set}=true" - fi - - rm "${files_changed_temporary_file}" - fi -} - # Exports environment variables used for scripts. cromwell::private::create_build_variables() { - CROMWELL_BUILD_PROVIDER_TRAVIS="travis" CROMWELL_BUILD_PROVIDER_JENKINS="jenkins" - CROMWELL_BUILD_PROVIDER_CIRCLE="circle" CROMWELL_BUILD_PROVIDER_GITHUB="github" CROMWELL_BUILD_PROVIDER_UNKNOWN="unknown" - if [[ "${TRAVIS-false}" == "true" ]]; then - CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_TRAVIS}" - elif [[ "${JENKINS-false}" == "true" ]]; then + if [[ "${JENKINS-false}" == "true" ]]; then CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_JENKINS}" elif [[ "${CIRCLECI-false}" == "true" ]]; then CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_CIRCLE}" @@ -157,76 +120,14 @@ cromwell::private::create_build_variables() { CROMWELL_BUILD_GIT_HASH_SUFFIX="gUNKNOWN" fi - # Value of the `TRAVIS_BRANCH` variable depends on type of Travis build: if it is pull request build, the value - # will be the name of the branch targeted by the pull request, and for push builds it will be the name of the - # branch. So, in case of push builds `git diff` will always return empty result. This is why we only use this short - # circuiting logic for pull request builds - # PR #6790 disabled the conditional that skips tests for documentation-only PRs, because # those PRs (and only those PRs) were uniformly failing tests with a nondescript error. # https://broadinstitute.slack.com/archives/GHYJZ2ZE0/p1656625952888149?thread_ts=1656620572.975059&cid=GHYJZ2ZE0 case "${CROMWELL_BUILD_PROVIDER}" in - "${CROMWELL_BUILD_PROVIDER_TRAVIS}") - CROMWELL_BUILD_IS_CI=true - CROMWELL_BUILD_IS_SECURE="${TRAVIS_SECURE_ENV_VARS}" - - CROMWELL_BUILD_TYPE="${BUILD_TYPE}" - CROMWELL_BUILD_BRANCH="${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}}" - CROMWELL_BUILD_EVENT="${TRAVIS_EVENT_TYPE}" - CROMWELL_BUILD_TAG="${TRAVIS_TAG}" - CROMWELL_BUILD_NUMBER="${TRAVIS_JOB_NUMBER}" - CROMWELL_BUILD_URL="https://travis-ci.com/${TRAVIS_REPO_SLUG}/jobs/${TRAVIS_JOB_ID}" - CROMWELL_BUILD_GIT_USER_EMAIL="travis@travis-ci.com" - CROMWELL_BUILD_GIT_USER_NAME="Travis CI" - CROMWELL_BUILD_HEARTBEAT_PATTERN="…" - CROMWELL_BUILD_GENERATE_COVERAGE=true - - local travis_commit_message - local travis_force_tests - local travis_minimal_tests - if [[ -n "${TRAVIS_COMMIT_RANGE:+set}" ]]; then - # The commit message to analyze should be the last one in the commit range. - # This works for both pull_request and push builds, unlike using 'git log HEAD' which - # gives a merge commit message on pull requests. - travis_commit_message="$(git log --reverse "${TRAVIS_COMMIT_RANGE}" | tail -n1 2>/dev/null || true)" - fi - - if [[ -z "${travis_commit_message:-}" ]]; then - travis_commit_message="$(git log --format=%B --max-count=1 HEAD 2>/dev/null || true)" - fi - - if [[ "${travis_commit_message}" == *"[force ci]"* ]]; then - travis_force_tests=true - travis_minimal_tests=false - elif [[ "${travis_commit_message}" == *"[minimal ci]"* ]]; then - travis_force_tests=false - travis_minimal_tests=true - else - travis_force_tests=false - travis_minimal_tests=false - fi - - echo "Building for commit message='${travis_commit_message}' with force=${travis_force_tests} and minimal=${travis_minimal_tests}" - - # For solely documentation updates run only checkPublish. Otherwise always run sbt, even for 'push'. - # This allows quick sanity checks before starting PRs *and* publishing after merges into develop. - if [[ "${travis_force_tests}" == "true" ]]; then - CROMWELL_BUILD_RUN_TESTS=true - elif [[ "${travis_minimal_tests}" == "true" ]] && \ - [[ "${TRAVIS_EVENT_TYPE}" != "push" ]]; then - CROMWELL_BUILD_RUN_TESTS=false - elif [[ "${TRAVIS_EVENT_TYPE}" == "push" ]] && \ - [[ "${BUILD_TYPE}" != "sbt" ]]; then - CROMWELL_BUILD_RUN_TESTS=false - else - CROMWELL_BUILD_RUN_TESTS=true - fi - ;; "${CROMWELL_BUILD_PROVIDER_JENKINS}") # External variables must be passed through in the ENVIRONMENT of src/ci/docker-compose/docker-compose.yml CROMWELL_BUILD_IS_CI=true - CROMWELL_BUILD_IS_SECURE=true CROMWELL_BUILD_TYPE="${JENKINS_BUILD_TYPE}" CROMWELL_BUILD_BRANCH="${GIT_BRANCH#origin/}" CROMWELL_BUILD_EVENT="" @@ -239,73 +140,8 @@ cromwell::private::create_build_variables() { CROMWELL_BUILD_GENERATE_COVERAGE=false CROMWELL_BUILD_RUN_TESTS=true ;; - "${CROMWELL_BUILD_PROVIDER_CIRCLE}") - CROMWELL_BUILD_IS_CI=true - CROMWELL_BUILD_TYPE="${BUILD_TYPE}" - CROMWELL_BUILD_NUMBER="${CIRCLE_BUILD_NUM}" - CROMWELL_BUILD_URL="${CIRCLE_BUILD_URL}" - CROMWELL_BUILD_GIT_USER_EMAIL="builds@circleci.com" - CROMWELL_BUILD_GIT_USER_NAME="CircleCI" - CROMWELL_BUILD_HEARTBEAT_PATTERN="…" - CROMWELL_BUILD_GENERATE_COVERAGE=true - CROMWELL_BUILD_BRANCH="${CIRCLE_BRANCH:-${CIRCLE_TAG}}" - CROMWELL_BUILD_TAG="${CIRCLE_TAG:-}" - - local circle_github_repository - circle_github_repository="${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}" - - if [[ "${circle_github_repository}" == "broadinstitute/cromwell" ]]; then - CROMWELL_BUILD_IS_SECURE=true - else - CROMWELL_BUILD_IS_SECURE=false - fi - - CROMWELL_BUILD_EVENT="pull_request" - - local circle_commit_message - local circle_force_tests - local circle_minimal_tests - if [[ -n "${CIRCLE_COMMIT_RANGE:+set}" ]]; then - # The commit message to analyze should be the last one in the commit range. - # This works for both pull_request and push builds, unlike using 'git log HEAD' which - # gives a merge commit message on pull requests. - circle_commit_message="$(git log --reverse "${CIRCLE_COMMIT_RANGE}" | tail -n1 2>/dev/null || true)" - fi - - if [[ -z "${circle_commit_message:-}" ]]; then - circle_commit_message="$(git log --format=%B --max-count=1 HEAD 2>/dev/null || true)" - fi - - if [[ "${circle_commit_message}" == *"[force ci]"* ]]; then - circle_force_tests=true - circle_minimal_tests=false - elif [[ "${circle_commit_message}" == *"[minimal ci]"* ]]; then - circle_force_tests=false - circle_minimal_tests=true - else - circle_force_tests=false - circle_minimal_tests=false - fi - - echo "Building for commit message='${circle_commit_message}' with force=${circle_force_tests} and minimal=${circle_minimal_tests}" - - # For solely documentation updates run only checkPublish. Otherwise always run sbt, even for 'push'. - # This allows quick sanity checks before starting PRs *and* publishing after merges into develop. - if [[ "${circle_force_tests}" == "true" ]]; then - CROMWELL_BUILD_RUN_TESTS=true - elif [[ "${circle_minimal_tests}" == "true" ]] && \ - [[ "${CROMWELL_BUILD_EVENT}" != "push" ]]; then - CROMWELL_BUILD_RUN_TESTS=false - elif [[ "${CROMWELL_BUILD_EVENT}" == "push" ]] && \ - [[ "${BUILD_TYPE}" != "sbt" ]]; then - CROMWELL_BUILD_RUN_TESTS=false - else - CROMWELL_BUILD_RUN_TESTS=true - fi - ;; "${CROMWELL_BUILD_PROVIDER_GITHUB}") CROMWELL_BUILD_IS_CI=true - CROMWELL_BUILD_IS_SECURE=true CROMWELL_BUILD_TYPE="${BUILD_TYPE}" CROMWELL_BUILD_BRANCH="${GITHUB_REF_NAME}" CROMWELL_BUILD_EVENT="${GITHUB_EVENT_NAME}" @@ -320,7 +156,6 @@ cromwell::private::create_build_variables() { ;; *) CROMWELL_BUILD_IS_CI=false - CROMWELL_BUILD_IS_SECURE=true CROMWELL_BUILD_TYPE="unknown" CROMWELL_BUILD_BRANCH="unknown" CROMWELL_BUILD_EVENT="unknown" @@ -371,21 +206,9 @@ cromwell::private::create_build_variables() { CROMWELL_BUILD_SBT_EXCLUDE="${BUILD_SBT_EXCLUDE:-}" case "${CROMWELL_BUILD_TYPE}" in - centaurPapiUpgradePapiV2alpha1*) - CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/papi_v2alpha1_v2beta_upgrade_application.conf" - ;; - centaurPapiUpgradeNewWorkflowsPapiV2alpha1*) - CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/papi_v2alpha1_v2beta_upgrade_application.conf" - ;; - centaurHoricromtalPapiV2alpha1*) - CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/papi_v2alpha1_horicromtal_application.conf" - ;; centaurHoricromtalPapiV2beta*) CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/papi_v2beta_horicromtal_application.conf" ;; - centaurHoricromtalEngineUpgrade*) - CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/papi_v2alpha1_horicromtal_application.conf" - ;; centaurBlob*) CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/centaur_blob_test.conf" ;; @@ -405,7 +228,6 @@ cromwell::private::create_build_variables() { CROMWELL_BUILD_DOCKER_TAG="${CROMWELL_BUILD_DOCKER_TAG:0:128}" CROMWELL_BUILD_DOCKER_TAG="${CROMWELL_BUILD_DOCKER_TAG//[^a-zA-Z0-9.-]/_}" - CROMWELL_BUILD_REQUIRES_SECURE="${CROMWELL_BUILD_REQUIRES_SECURE-false}" CROMWELL_BUILD_REQUIRES_PRIOR_VERSION="${CROMWELL_BUILD_REQUIRES_PRIOR_VERSION-false}" local hours_to_minutes @@ -432,7 +254,6 @@ cromwell::private::create_build_variables() { export CROMWELL_BUILD_HOME_DIRECTORY export CROMWELL_BUILD_IS_CI export CROMWELL_BUILD_IS_HOTFIX - export CROMWELL_BUILD_IS_SECURE export CROMWELL_BUILD_IS_VIRTUAL_ENV export CROMWELL_BUILD_LOG_DIRECTORY export CROMWELL_BUILD_NUMBER @@ -441,9 +262,7 @@ cromwell::private::create_build_variables() { export CROMWELL_BUILD_OS_LINUX export CROMWELL_BUILD_PRIOR_VERSION_NUMBER export CROMWELL_BUILD_PROVIDER - export CROMWELL_BUILD_PROVIDER_CIRCLE export CROMWELL_BUILD_PROVIDER_JENKINS - export CROMWELL_BUILD_PROVIDER_TRAVIS export CROMWELL_BUILD_PROVIDER_UNKNOWN export CROMWELL_BUILD_REQUIRES_PRIOR_VERSION export CROMWELL_BUILD_REQUIRES_SECURE @@ -470,8 +289,6 @@ cromwell::private::create_build_variables() { cromwell::private::echo_build_variables() { echo "CROMWELL_BUILD_IS_CI='${CROMWELL_BUILD_IS_CI}'" - echo "CROMWELL_BUILD_IS_SECURE='${CROMWELL_BUILD_IS_SECURE}'" - echo "CROMWELL_BUILD_REQUIRES_SECURE='${CROMWELL_BUILD_REQUIRES_SECURE}'" echo "CROMWELL_BUILD_TYPE='${CROMWELL_BUILD_TYPE}'" echo "CROMWELL_BUILD_BRANCH='${CROMWELL_BUILD_BRANCH}'" echo "CROMWELL_BUILD_IS_HOTFIX='${CROMWELL_BUILD_IS_HOTFIX}'" @@ -492,8 +309,6 @@ cromwell::private::create_database_variables() { CROMWELL_BUILD_DATABASE_SCHEMA="cromwell_test" case "${CROMWELL_BUILD_PROVIDER}" in - "${CROMWELL_BUILD_PROVIDER_TRAVIS}"|\ - "${CROMWELL_BUILD_PROVIDER_CIRCLE}"|\ "${CROMWELL_BUILD_PROVIDER_GITHUB}") CROMWELL_BUILD_MARIADB_HOSTNAME="localhost" CROMWELL_BUILD_MARIADB_PORT="23306" @@ -654,8 +469,6 @@ cromwell::private::create_centaur_variables() { # Pick **one** of the databases to run Centaur against case "${CROMWELL_BUILD_PROVIDER}" in - "${CROMWELL_BUILD_PROVIDER_TRAVIS}"|\ - "${CROMWELL_BUILD_PROVIDER_CIRCLE}"|\ "${CROMWELL_BUILD_PROVIDER_GITHUB}") if [[ -n "${CROMWELL_BUILD_MYSQL_DOCKER_TAG:+set}" ]]; then @@ -706,19 +519,9 @@ cromwell::private::create_centaur_variables() { ;; esac - # When upgrading to the MariaDB driver, start with MySQL then switch to MariaDB. - if [[ "${CROMWELL_BUILD_PROVIDER}" == "${CROMWELL_BUILD_PROVIDER_TRAVIS}" ]] && \ - [[ -n "${CROMWELL_BUILD_MARIADB_DOCKER_TAG:+set}" ]]; then - - CROMWELL_BUILD_CENTAUR_PRIOR_SLICK_PROFILE="slick.jdbc.MySQLProfile$" - CROMWELL_BUILD_CENTAUR_PRIOR_JDBC_DRIVER="com.mysql.cj.jdbc.Driver" - CROMWELL_BUILD_CENTAUR_PRIOR_JDBC_URL="jdbc:mysql://${CROMWELL_BUILD_MARIADB_HOSTNAME}:${CROMWELL_BUILD_MARIADB_PORT}/${CROMWELL_BUILD_DATABASE_SCHEMA}?allowPublicKeyRetrieval=true&useSSL=false&rewriteBatchedStatements=true&serverTimezone=UTC&useInformationSchema=true" - else - - CROMWELL_BUILD_CENTAUR_PRIOR_SLICK_PROFILE="${CROMWELL_BUILD_CENTAUR_PRIOR_SLICK_PROFILE-${CROMWELL_BUILD_CENTAUR_SLICK_PROFILE}}" - CROMWELL_BUILD_CENTAUR_PRIOR_JDBC_DRIVER="${CROMWELL_BUILD_CENTAUR_PRIOR_JDBC_DRIVER-${CROMWELL_BUILD_CENTAUR_JDBC_DRIVER}}" - CROMWELL_BUILD_CENTAUR_PRIOR_JDBC_URL="${CROMWELL_BUILD_CENTAUR_PRIOR_JDBC_URL-${CROMWELL_BUILD_CENTAUR_JDBC_URL}}" - fi + CROMWELL_BUILD_CENTAUR_PRIOR_SLICK_PROFILE="${CROMWELL_BUILD_CENTAUR_PRIOR_SLICK_PROFILE-${CROMWELL_BUILD_CENTAUR_SLICK_PROFILE}}" + CROMWELL_BUILD_CENTAUR_PRIOR_JDBC_DRIVER="${CROMWELL_BUILD_CENTAUR_PRIOR_JDBC_DRIVER-${CROMWELL_BUILD_CENTAUR_JDBC_DRIVER}}" + CROMWELL_BUILD_CENTAUR_PRIOR_JDBC_URL="${CROMWELL_BUILD_CENTAUR_PRIOR_JDBC_URL-${CROMWELL_BUILD_CENTAUR_JDBC_URL}}" CROMWELL_BUILD_CENTAUR_256_BITS_KEY="$(dd bs=1 count=32 if=/dev/urandom 2> /dev/null | base64 | tr -d '\n')" @@ -743,38 +546,12 @@ cromwell::private::create_centaur_variables() { export CROMWELL_BUILD_DOCKER_TAG } -cromwell::private::verify_secure_build() { - case "${CROMWELL_BUILD_PROVIDER}" in - "${CROMWELL_BUILD_PROVIDER_TRAVIS}") - if [[ "${CROMWELL_BUILD_IS_SECURE}" != "true" ]] && \ - [[ "${CROMWELL_BUILD_REQUIRES_SECURE}" == "true" ]]; then - echo "********************************************************" - echo "********************************************************" - echo "** **" - echo "** WARNING: Encrypted keys are unavailable. Exiting. **" - echo "** **" - echo "********************************************************" - echo "********************************************************" - exit 0 - fi - ;; - *) - ;; - esac -} - cromwell::private::exec_test_script() { local upper_build_type upper_build_type="$(tr '[:lower:]' '[:upper:]' <<< "${CROMWELL_BUILD_TYPE:0:1}")${CROMWELL_BUILD_TYPE:1}" exec "${CROMWELL_BUILD_SCRIPTS_DIRECTORY}/test${upper_build_type}.sh" } -cromwell::private::stop_travis_defaults() { - # https://stackoverflow.com/questions/27382295/how-to-stop-services-on-travis-ci-running-by-default#answer-27410479 - sudo /etc/init.d/mysql stop || true - sudo /etc/init.d/postgresql stop || true -} - cromwell::private::delete_boto_config() { # https://github.com/travis-ci/travis-ci/issues/7940#issuecomment-310759657 sudo rm -f /etc/boto.cfg @@ -788,30 +565,6 @@ cromwell::private::delete_sbt_boot() { rm -rf ~/.sbt/boot/ } -cromwell::private::install_adoptopenjdk() { - # https://adoptopenjdk.net/installation.html#linux-pkg-deb - sudo apt-get install -y wget apt-transport-https gnupg - wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | - sudo apt-key add - - echo "deb https://adoptopenjdk.jfrog.io/adoptopenjdk/deb $( - grep UBUNTU_CODENAME /etc/os-release | cut -d = -f 2 - ) main" | - sudo tee /etc/apt/sources.list.d/adoptopenjdk.list - sudo apt-get update - sudo apt-get install -y adoptopenjdk-11-hotspot - sudo update-java-alternatives --set adoptopenjdk-11-hotspot-amd64 -} - -cromwell::private::install_sbt_launcher() { - # Install sbt launcher - # Non-deb package installation instructions adapted from - # - https://github.com/sbt/sbt/releases/tag/v1.4.9 - # - https://github.com/broadinstitute/scala-baseimage/pull/4/files - curl --location --fail --silent --show-error "https://github.com/sbt/sbt/releases/download/v1.5.5/sbt-1.5.5.tgz" | - sudo tar zxf - -C /usr/share - sudo update-alternatives --install /usr/bin/sbt sbt /usr/share/sbt/bin/sbt 1 -} - cromwell::private::install_docker_compose() { # Install or upgrade docker-compose so that we get the correct exit codes # https://docs.docker.com/compose/release-notes/#1230 @@ -824,12 +577,6 @@ cromwell::private::install_docker_compose() { sudo chmod +x /usr/local/bin/docker-compose } -cromwell::private::setup_pyenv_python_latest() { - # Make `python` whatever the most recent version of python installed - # Fixes cases where someone has set pyenv to override `python` to use an older `python2` instead of `python3` - pyenv global "$(pyenv versions --bare --skip-aliases | sort -t '.' -k1,1n -k2,2n -k3,3n | tail -n 1)" -} - cromwell::private::pip_install() { local pip_package pip_package="${1:?pip_install called without a package}"; shift @@ -1269,7 +1016,6 @@ cromwell::build::setup_common_environment() { cromwell::private::check_debug cromwell::private::create_build_variables cromwell::private::echo_build_variables - cromwell::private::verify_secure_build cromwell::private::make_build_directories cromwell::private::install_git_secrets cromwell::private::install_minnie_kenny @@ -1277,31 +1023,6 @@ cromwell::build::setup_common_environment() { cromwell::private::create_database_variables case "${CROMWELL_BUILD_PROVIDER}" in - "${CROMWELL_BUILD_PROVIDER_TRAVIS}") - cromwell::private::stop_travis_defaults - # Try to login to vault, and if successful then use vault creds to login to docker. - # For those committers with vault access this avoids pull rate limits reported in BT-143. - cromwell::private::install_vault - cromwell::private::login_vault - cromwell::private::login_docker - cromwell::private::install_adoptopenjdk - cromwell::private::install_sbt_launcher - cromwell::private::install_docker_compose - cromwell::private::delete_boto_config - cromwell::private::delete_sbt_boot - cromwell::private::upgrade_pip - cromwell::private::start_docker_databases - ;; - "${CROMWELL_BUILD_PROVIDER_CIRCLE}") - # Try to login to vault, and if successful then use vault creds to login to docker. - # For those committers with vault access this avoids pull rate limits reported in BT-143. - cromwell::private::install_vault - cromwell::private::login_vault - cromwell::private::login_docker - cromwell::private::install_adoptopenjdk - cromwell::private::setup_pyenv_python_latest - cromwell::private::start_docker_databases - ;; "${CROMWELL_BUILD_PROVIDER_GITHUB}") # Try to login to vault, and if successful then use vault creds to login to docker. # For those committers with vault access this avoids pull rate limits reported in BT-143. From 071c32c70c553ee09f0c2258fa0b6f4d804ef8d4 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Wed, 6 Dec 2023 16:24:40 -0500 Subject: [PATCH 09/13] Oops --- src/ci/bin/testCentaurPapiV2beta.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/bin/testCentaurPapiV2beta.sh b/src/ci/bin/testCentaurPapiV2beta.sh index 3f92bf0982e..108c6cca388 100755 --- a/src/ci/bin/testCentaurPapiV2beta.sh +++ b/src/ci/bin/testCentaurPapiV2beta.sh @@ -23,8 +23,8 @@ cromwell::build::run_centaur \ -e relative_output_paths_colliding \ -e standard_output_paths_colliding_prevented \ -e papi_v2alpha1_gcsa \ - -e lots_of_inputs \ -e restart \ + -e lots_of_inputs \ # `lots_of_inputs` is redundant to, and a less rigorous version of, `lots_of_inputs_papiv2` cromwell::build::generate_code_coverage From 0e841ba70f752e4d82d09b7f8fee122a00f4a780 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Wed, 6 Dec 2023 17:39:11 -0500 Subject: [PATCH 10/13] Clean up `test.inc.sh` --- src/ci/bin/test.inc.sh | 96 ++---------------------------------------- 1 file changed, 4 insertions(+), 92 deletions(-) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index 4833212d31b..71577ce1e4c 100755 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -47,15 +47,12 @@ cromwell::private::check_debug() { # Exports environment variables used for scripts. cromwell::private::create_build_variables() { - CROMWELL_BUILD_PROVIDER_JENKINS="jenkins" CROMWELL_BUILD_PROVIDER_GITHUB="github" CROMWELL_BUILD_PROVIDER_UNKNOWN="unknown" - if [[ "${JENKINS-false}" == "true" ]]; then - CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_JENKINS}" - elif [[ "${CIRCLECI-false}" == "true" ]]; then - CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_CIRCLE}" - elif [[ "${GITHUB_ACTIONS-false}" == "true" ]]; then + # CI providers set an env like `GITHUB_ACTIONS` that indicates to jobs where they're running + # https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables + if [[ "${GITHUB_ACTIONS-false}" == "true" ]]; then CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_GITHUB}" else CROMWELL_BUILD_PROVIDER="${CROMWELL_BUILD_PROVIDER_UNKNOWN}" @@ -125,21 +122,6 @@ cromwell::private::create_build_variables() { # https://broadinstitute.slack.com/archives/GHYJZ2ZE0/p1656625952888149?thread_ts=1656620572.975059&cid=GHYJZ2ZE0 case "${CROMWELL_BUILD_PROVIDER}" in - "${CROMWELL_BUILD_PROVIDER_JENKINS}") - # External variables must be passed through in the ENVIRONMENT of src/ci/docker-compose/docker-compose.yml - CROMWELL_BUILD_IS_CI=true - CROMWELL_BUILD_TYPE="${JENKINS_BUILD_TYPE}" - CROMWELL_BUILD_BRANCH="${GIT_BRANCH#origin/}" - CROMWELL_BUILD_EVENT="" - CROMWELL_BUILD_TAG="" - CROMWELL_BUILD_NUMBER="${BUILD_NUMBER}" - CROMWELL_BUILD_URL="${BUILD_URL}" - CROMWELL_BUILD_GIT_USER_EMAIL="jenkins@jenkins.io" - CROMWELL_BUILD_GIT_USER_NAME="Jenkins CI" - CROMWELL_BUILD_HEARTBEAT_PATTERN="…\n" - CROMWELL_BUILD_GENERATE_COVERAGE=false - CROMWELL_BUILD_RUN_TESTS=true - ;; "${CROMWELL_BUILD_PROVIDER_GITHUB}") CROMWELL_BUILD_IS_CI=true CROMWELL_BUILD_TYPE="${BUILD_TYPE}" @@ -262,7 +244,6 @@ cromwell::private::create_build_variables() { export CROMWELL_BUILD_OS_LINUX export CROMWELL_BUILD_PRIOR_VERSION_NUMBER export CROMWELL_BUILD_PROVIDER - export CROMWELL_BUILD_PROVIDER_JENKINS export CROMWELL_BUILD_PROVIDER_UNKNOWN export CROMWELL_BUILD_REQUIRES_PRIOR_VERSION export CROMWELL_BUILD_REQUIRES_SECURE @@ -329,29 +310,6 @@ cromwell::private::create_database_variables() { CROMWELL_BUILD_POSTGRESQL_LATEST_PORT="15432" CROMWELL_BUILD_POSTGRESQL_LATEST_TAG="${BUILD_POSTGRESQL_LATEST-}" ;; - "${CROMWELL_BUILD_PROVIDER_JENKINS}") - # NOTE: Jenkins uses src/ci/docker-compose/docker-compose.yml. - # We don't define a docker tag because the docker-compose has already spun up the database containers by the - # time this script is run. Other variables here must match the database service names and settings the yaml. - CROMWELL_BUILD_MARIADB_HOSTNAME="mariadb-db" - CROMWELL_BUILD_MARIADB_PORT="3306" - CROMWELL_BUILD_MARIADB_DOCKER_TAG="" - CROMWELL_BUILD_MARIADB_LATEST_HOSTNAME="mariadb-db-latest" - CROMWELL_BUILD_MARIADB_LATEST_PORT="3306" - CROMWELL_BUILD_MARIADB_LATEST_TAG="" - CROMWELL_BUILD_MYSQL_HOSTNAME="mysql-db" - CROMWELL_BUILD_MYSQL_PORT="3306" - CROMWELL_BUILD_MYSQL_DOCKER_TAG="" - CROMWELL_BUILD_MYSQL_LATEST_HOSTNAME="mysql-db-latest" - CROMWELL_BUILD_MYSQL_LATEST_PORT="3306" - CROMWELL_BUILD_MYSQL_LATEST_TAG="" - CROMWELL_BUILD_POSTGRESQL_HOSTNAME="postgresql-db" - CROMWELL_BUILD_POSTGRESQL_PORT="5432" - CROMWELL_BUILD_POSTGRESQL_DOCKER_TAG="" - CROMWELL_BUILD_POSTGRESQL_LATEST_HOSTNAME="postgresql-db-latest" - CROMWELL_BUILD_POSTGRESQL_LATEST_PORT="3306" - CROMWELL_BUILD_POSTGRESQL_LATEST_TAG="" - ;; *) if [[ -z "${CROMWELL_BUILD_DOCKER_LOCALHOST-}" ]]; then CROMWELL_BUILD_DOCKER_LOCALHOST="localhost" @@ -494,12 +452,6 @@ cromwell::private::create_centaur_variables() { CROMWELL_BUILD_CENTAUR_TEST_ADDITIONAL_PARAMETERS= ;; - "${CROMWELL_BUILD_PROVIDER_JENKINS}") - CROMWELL_BUILD_CENTAUR_SLICK_PROFILE="slick.jdbc.MySQLProfile$" - CROMWELL_BUILD_CENTAUR_JDBC_DRIVER="com.mysql.cj.jdbc.Driver" - CROMWELL_BUILD_CENTAUR_JDBC_URL="${mysql_jdbc_url}" - CROMWELL_BUILD_CENTAUR_TEST_ADDITIONAL_PARAMETERS="${CENTAUR_TEST_ADDITIONAL_PARAMETERS-}" - ;; *) CROMWELL_BUILD_CENTAUR_SLICK_PROFILE="${CROMWELL_BUILD_CENTAUR_SLICK_PROFILE-slick.jdbc.MySQLProfile\$}" CROMWELL_BUILD_CENTAUR_JDBC_DRIVER="${CROMWELL_BUILD_CENTAUR_JDBC_DRIVER-com.mysql.cj.jdbc.Driver}" @@ -813,27 +765,7 @@ cromwell::private::render_secure_resources() { fi } -cromwell::private::copy_all_resources() { - # Only copy the CI resources. Secure resources are not rendered. - sbt -Dsbt.supershell=false --warn copyCiResources -} - -cromwell::private::setup_secure_resources() { - case "${CROMWELL_BUILD_PROVIDER}" in - "${CROMWELL_BUILD_PROVIDER_JENKINS}") - # Jenkins secret resources should have already been rendered outside the CI's docker-compose container. - cromwell::private::copy_all_resources - ;; - *) - cromwell::private::render_secure_resources - ;; - esac -} - cromwell::private::make_build_directories() { - if [[ "${CROMWELL_BUILD_PROVIDER}" == "${CROMWELL_BUILD_PROVIDER_JENKINS}" ]]; then - sudo chmod -R a+w . - fi mkdir -p "${CROMWELL_BUILD_LOG_DIRECTORY}" mkdir -p "${CROMWELL_BUILD_RESOURCES_DIRECTORY}" } @@ -1038,12 +970,11 @@ cromwell::build::setup_common_environment() { cromwell::private::upgrade_pip cromwell::private::start_docker_databases ;; - "${CROMWELL_BUILD_PROVIDER_JENKINS}"|\ *) ;; esac - cromwell::private::setup_secure_resources + cromwell::private::render_secure_resources cromwell::private::start_build_heartbeat } @@ -1225,25 +1156,6 @@ cromwell::build::print_workflow_statistics() { LIMIT 20;" } -cromwell::build::exec_retry_function() { - local retried_function - local retry_count - local attempt - local exit_status - - retried_function="${1:?exec_retry_function called without a function}"; shift - retry_count="${1:-3}"; shift || true - sleep_seconds="${1:-15}"; shift || true - - # https://unix.stackexchange.com/a/82610 - # https://stackoverflow.com/a/17336953 - for attempt in $(seq 0 "${retry_count}"); do - [[ ${attempt} -gt 0 ]] && sleep "${sleep_seconds}" - ${retried_function} && exit_status=0 && break || exit_status=$? - done - return ${exit_status} -} - cromwell::build::exec_silent_function() { local silent_function silent_function="${1:?exec_silent_function called without a function}"; shift From a5414767dece8fc33c83f65b8c4148f245f00a0f Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Wed, 6 Dec 2023 17:56:47 -0500 Subject: [PATCH 11/13] Fix --- src/ci/bin/test_gcpbatch.inc.sh | 4 ---- src/ci/bin/test_papi.inc.sh | 4 ---- src/ci/bin/test_unit.inc.sh | 12 ------------ 3 files changed, 20 deletions(-) diff --git a/src/ci/bin/test_gcpbatch.inc.sh b/src/ci/bin/test_gcpbatch.inc.sh index 1570bdb5f76..97dce8b878d 100644 --- a/src/ci/bin/test_gcpbatch.inc.sh +++ b/src/ci/bin/test_gcpbatch.inc.sh @@ -29,10 +29,6 @@ cromwell::private::batch::setup_batch_gcloud() { export CROMWELL_BUILD_BATCH_GCR_IMAGES export CROMWELL_BUILD_BATCH_PROJECT_ID - if [[ "${CROMWELL_BUILD_PROVIDER}" == "${CROMWELL_BUILD_PROVIDER_TRAVIS}" ]]; then - cromwell::private::batch::install_gcloud - fi - # All `gcloud` commands should use this configuration directory. # https://stackoverflow.com/questions/34883810/how-to-authenticate-google-apis-with-different-service-account-credentials # https://github.com/googleapis/google-auth-library-java/issues/58 diff --git a/src/ci/bin/test_papi.inc.sh b/src/ci/bin/test_papi.inc.sh index 98d77820435..2fd5c8352f6 100644 --- a/src/ci/bin/test_papi.inc.sh +++ b/src/ci/bin/test_papi.inc.sh @@ -29,10 +29,6 @@ cromwell::private::papi::setup_papi_gcloud() { export CROMWELL_BUILD_PAPI_GCR_IMAGES export CROMWELL_BUILD_PAPI_PROJECT_ID - if [[ "${CROMWELL_BUILD_PROVIDER}" == "${CROMWELL_BUILD_PROVIDER_TRAVIS}" ]]; then - cromwell::private::papi::install_gcloud - fi - # All `gcloud` commands should use this configuration directory. # https://stackoverflow.com/questions/34883810/how-to-authenticate-google-apis-with-different-service-account-credentials # https://github.com/googleapis/google-auth-library-java/issues/58 diff --git a/src/ci/bin/test_unit.inc.sh b/src/ci/bin/test_unit.inc.sh index edef0fda323..5ff2b69de3e 100644 --- a/src/ci/bin/test_unit.inc.sh +++ b/src/ci/bin/test_unit.inc.sh @@ -18,12 +18,6 @@ source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh cromwell::build::unit::setup_scale_factor() { case "${CROMWELL_BUILD_PROVIDER}" in - "${CROMWELL_BUILD_PROVIDER_TRAVIS}") - CROMWELL_BUILD_UNIT_SPAN_SCALE_FACTOR=2 - ;; - "${CROMWELL_BUILD_PROVIDER_JENKINS}") - CROMWELL_BUILD_UNIT_SPAN_SCALE_FACTOR=10 - ;; *) CROMWELL_BUILD_UNIT_SPAN_SCALE_FACTOR=1 ;; @@ -33,12 +27,6 @@ cromwell::build::unit::setup_scale_factor() { cromwell::build::unit::setup_exclude_tags() { case "${CROMWELL_BUILD_PROVIDER}" in - "${CROMWELL_BUILD_PROVIDER_TRAVIS}") - CROMWELL_BUILD_UNIT_EXCLUDE_TAGS="AwsTest,CromwellIntegrationTest,DbmsTest,GcsIntegrationTest" - ;; - "${CROMWELL_BUILD_PROVIDER_JENKINS}") - CROMWELL_BUILD_UNIT_EXCLUDE_TAGS="AwsTest,CromwellIntegrationTest,DockerTest,DbmsTest,GcsIntegrationTest" - ;; *) # Use the full list of excludes listed in Testing.scala CROMWELL_BUILD_UNIT_EXCLUDE_TAGS="" From 811ac03012c39152c52e4655158f9ab21cb095f3 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Wed, 6 Dec 2023 18:06:50 -0500 Subject: [PATCH 12/13] Fix --- src/ci/bin/test_gcpbatch.inc.sh | 4 ---- src/ci/bin/test_papi.inc.sh | 4 ---- 2 files changed, 8 deletions(-) diff --git a/src/ci/bin/test_gcpbatch.inc.sh b/src/ci/bin/test_gcpbatch.inc.sh index 97dce8b878d..641cd86d612 100644 --- a/src/ci/bin/test_gcpbatch.inc.sh +++ b/src/ci/bin/test_gcpbatch.inc.sh @@ -102,9 +102,5 @@ cromwell::private::batch::setup_batch_service_account() { cromwell::build::batch::setup_batch_centaur_environment() { cromwell::private::batch::setup_batch_gcloud - if [[ "${CROMWELL_BUILD_PROVIDER}" != "${CROMWELL_BUILD_PROVIDER_JENKINS}" ]] - then - cromwell::private::batch::setup_batch_gcr - fi cromwell::private::batch::setup_batch_service_account } diff --git a/src/ci/bin/test_papi.inc.sh b/src/ci/bin/test_papi.inc.sh index 2fd5c8352f6..c17536693d7 100644 --- a/src/ci/bin/test_papi.inc.sh +++ b/src/ci/bin/test_papi.inc.sh @@ -111,10 +111,6 @@ cromwell::private::papi::setup_papi_endpoint_url() { cromwell::build::papi::setup_papi_centaur_environment() { cromwell::private::papi::setup_papi_gcloud - if [[ "${CROMWELL_BUILD_PROVIDER}" != "${CROMWELL_BUILD_PROVIDER_JENKINS}" ]] - then - cromwell::private::papi::setup_papi_gcr - fi cromwell::private::papi::setup_papi_service_account cromwell::private::papi::setup_papi_endpoint_url } From 51a5fa680a7cb55b7c5e9b27e42300398c2443e0 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Thu, 7 Dec 2023 16:44:55 -0500 Subject: [PATCH 13/13] That was the wrong branch of that conditional... --- src/ci/bin/test_papi.inc.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/bin/test_papi.inc.sh b/src/ci/bin/test_papi.inc.sh index c17536693d7..0b4ed151664 100644 --- a/src/ci/bin/test_papi.inc.sh +++ b/src/ci/bin/test_papi.inc.sh @@ -111,6 +111,7 @@ cromwell::private::papi::setup_papi_endpoint_url() { cromwell::build::papi::setup_papi_centaur_environment() { cromwell::private::papi::setup_papi_gcloud + cromwell::private::papi::setup_papi_gcr cromwell::private::papi::setup_papi_service_account cromwell::private::papi::setup_papi_endpoint_url }