From a26ca25d240bdd5dfb96e1a350fb1bae78e9d2f9 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Mon, 4 Dec 2023 11:27:35 -0500 Subject: [PATCH 1/5] Standalone test --- core/src/main/resources/reference.conf | 4 +++- src/ci/bin/testCentaurHoricromtalPapiV2beta.sh | 2 +- src/ci/resources/papi_v2beta_horicromtal_application.conf | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 6ec05cf6025..391234eb74f 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -118,7 +118,9 @@ system { max-concurrent-workflows = 5000 # Cromwell will launch up to N submitted workflows at a time, regardless of how many open workflow slots exist - max-workflow-launch-count = 50 + # Deviating from 1 is not recommended for multiS-runner setups due to possible deadlocks. [BW-962] + # This has been the Terra configuration for + max-workflow-launch-count = 1 # Workflows will be grouped by the value of the specified field in their workflow options. # diff --git a/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh b/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh index 9dba2f6f8ce..3b4dc0057b5 100755 --- a/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh +++ b/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh @@ -19,7 +19,7 @@ cromwell::build::assemble_jars cromwell::build::build_cromwell_docker cromwell::build::run_centaur \ - -p 100 \ + -p 500 \ -e localdockertest \ -e relative_output_paths \ -e relative_output_paths_colliding \ diff --git a/src/ci/resources/papi_v2beta_horicromtal_application.conf b/src/ci/resources/papi_v2beta_horicromtal_application.conf index c6bc9176c16..f0f2c45f381 100644 --- a/src/ci/resources/papi_v2beta_horicromtal_application.conf +++ b/src/ci/resources/papi_v2beta_horicromtal_application.conf @@ -1,8 +1,8 @@ include "papi_v2beta_application.conf" -system.max-workflow-launch-count=1 -system.new-workflow-poll-rate=10 -system.max-concurrent-workflows=30 +system.max-workflow-launch-count=10 +system.new-workflow-poll-rate=1 +system.max-concurrent-workflows=500 system.cromwell_id_random_suffix=false # Turn off token logging to reduce log volume From 213ee021cf5b2cb80bb8a42083056810c4ed5d20 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Mon, 4 Dec 2023 13:42:07 -0500 Subject: [PATCH 2/5] Improved statistics --- core/src/main/resources/reference.conf | 1 - src/ci/bin/test.inc.sh | 36 +++++++++++++++++++ .../bin/testCentaurHoricromtalPapiV2beta.sh | 2 ++ src/ci/bin/testCentaurPapiV2beta.sh | 2 ++ 4 files changed, 40 insertions(+), 1 deletion(-) diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 391234eb74f..77ae784dd27 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -119,7 +119,6 @@ system { # Cromwell will launch up to N submitted workflows at a time, regardless of how many open workflow slots exist # Deviating from 1 is not recommended for multiS-runner setups due to possible deadlocks. [BW-962] - # This has been the Terra configuration for max-workflow-launch-count = 1 # Workflows will be grouped by the value of the specified field in their workflow options. diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index eef3a9fab6b..e27679d16e5 100755 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -1607,6 +1607,42 @@ cromwell::build::generate_code_coverage() { fi } +cromwell::build::print_workflow_statistics() { + echo "Total workflows" + mysql --host=127.0.0.1 --user=cromwell --password=test cromwell_test -e \ + "SELECT COUNT(*) as total_workflows_run FROM WORKFLOW_METADATA_SUMMARY_ENTRY;" + + echo "Late starters" + mysql --host=127.0.0.1 --user=cromwell --password=test cromwell_test -e \ + "SELECT WORKFlOW_NAME as name, + TIMESTAMPDIFF(MINUTE, START_TIMESTAMP, END_TIMESTAMP) as runtime_minutes, + START_TIMESTAMP as start, + END_TIMESTAMP as end + FROM WORKFLOW_METADATA_SUMMARY_ENTRY + ORDER BY START_TIMESTAMP DESC + LIMIT 20;" + + echo "Late finishers" + mysql --host=127.0.0.1 --user=cromwell --password=test cromwell_test -e \ + "SELECT WORKFlOW_NAME as name, + TIMESTAMPDIFF(MINUTE, START_TIMESTAMP, END_TIMESTAMP) as runtime_minutes, + START_TIMESTAMP as start, + END_TIMESTAMP as end + FROM WORKFLOW_METADATA_SUMMARY_ENTRY + ORDER BY END_TIMESTAMP DESC + LIMIT 20;" + + echo "Long duration" + mysql --host=127.0.0.1 --user=cromwell --password=test cromwell_test -e \ + "SELECT WORKFlOW_NAME as name, + TIMESTAMPDIFF(MINUTE, START_TIMESTAMP, END_TIMESTAMP) as runtime_minutes, + START_TIMESTAMP as start, + END_TIMESTAMP as end + FROM WORKFLOW_METADATA_SUMMARY_ENTRY + ORDER BY runtime_minutes DESC + LIMIT 20;" +} + cromwell::build::exec_retry_function() { local retried_function local retry_count diff --git a/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh b/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh index 3b4dc0057b5..36b215a2f52 100755 --- a/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh +++ b/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh @@ -27,3 +27,5 @@ cromwell::build::run_centaur \ -e papi_v2alpha1_gcsa \ cromwell::build::generate_code_coverage + +cromwell::build::print_workflow_statistics diff --git a/src/ci/bin/testCentaurPapiV2beta.sh b/src/ci/bin/testCentaurPapiV2beta.sh index b3a1ef2a61e..43d5a7d62aa 100755 --- a/src/ci/bin/testCentaurPapiV2beta.sh +++ b/src/ci/bin/testCentaurPapiV2beta.sh @@ -25,3 +25,5 @@ cromwell::build::run_centaur \ -e papi_v2alpha1_gcsa \ cromwell::build::generate_code_coverage + +cromwell::build::print_workflow_statistics From 055936223a8fee921f39e7f372f6c8683366ad34 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Mon, 4 Dec 2023 13:51:44 -0500 Subject: [PATCH 3/5] Fix typo --- core/src/main/resources/reference.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 77ae784dd27..3fa11b458ff 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -118,7 +118,7 @@ system { max-concurrent-workflows = 5000 # Cromwell will launch up to N submitted workflows at a time, regardless of how many open workflow slots exist - # Deviating from 1 is not recommended for multiS-runner setups due to possible deadlocks. [BW-962] + # Deviating from 1 is not recommended for multi-runner setups due to possible deadlocks. [BW-962] max-workflow-launch-count = 1 # Workflows will be grouped by the value of the specified field in their workflow options. From 1e01f0ee449c7560337b96e3b8efd9839bbc6025 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Mon, 4 Dec 2023 15:16:36 -0500 Subject: [PATCH 4/5] Emphasis --- src/ci/bin/test.inc.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index e27679d16e5..b6b26678a00 100755 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -1616,7 +1616,7 @@ cromwell::build::print_workflow_statistics() { mysql --host=127.0.0.1 --user=cromwell --password=test cromwell_test -e \ "SELECT WORKFlOW_NAME as name, TIMESTAMPDIFF(MINUTE, START_TIMESTAMP, END_TIMESTAMP) as runtime_minutes, - START_TIMESTAMP as start, + START_TIMESTAMP as START, END_TIMESTAMP as end FROM WORKFLOW_METADATA_SUMMARY_ENTRY ORDER BY START_TIMESTAMP DESC @@ -1627,7 +1627,7 @@ cromwell::build::print_workflow_statistics() { "SELECT WORKFlOW_NAME as name, TIMESTAMPDIFF(MINUTE, START_TIMESTAMP, END_TIMESTAMP) as runtime_minutes, START_TIMESTAMP as start, - END_TIMESTAMP as end + END_TIMESTAMP as END FROM WORKFLOW_METADATA_SUMMARY_ENTRY ORDER BY END_TIMESTAMP DESC LIMIT 20;" @@ -1635,11 +1635,11 @@ cromwell::build::print_workflow_statistics() { echo "Long duration" mysql --host=127.0.0.1 --user=cromwell --password=test cromwell_test -e \ "SELECT WORKFlOW_NAME as name, - TIMESTAMPDIFF(MINUTE, START_TIMESTAMP, END_TIMESTAMP) as runtime_minutes, + TIMESTAMPDIFF(MINUTE, START_TIMESTAMP, END_TIMESTAMP) as RUNTIME_MINUTES, START_TIMESTAMP as start, END_TIMESTAMP as end FROM WORKFLOW_METADATA_SUMMARY_ENTRY - ORDER BY runtime_minutes DESC + ORDER BY RUNTIME_MINUTES DESC LIMIT 20;" } From 5836edfc73e400f9feba2cca9b0c7349fdfa46cb Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Mon, 4 Dec 2023 19:38:36 -0500 Subject: [PATCH 5/5] Exclude subworkflows --- src/ci/bin/test.inc.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh index b6b26678a00..7da8290b9a1 100755 --- a/src/ci/bin/test.inc.sh +++ b/src/ci/bin/test.inc.sh @@ -1619,6 +1619,7 @@ cromwell::build::print_workflow_statistics() { START_TIMESTAMP as START, END_TIMESTAMP as end FROM WORKFLOW_METADATA_SUMMARY_ENTRY + WHERE PARENT_WORKFLOW_EXECUTION_UUID IS NULL # exclude subworkflows ORDER BY START_TIMESTAMP DESC LIMIT 20;" @@ -1629,6 +1630,7 @@ cromwell::build::print_workflow_statistics() { START_TIMESTAMP as start, END_TIMESTAMP as END FROM WORKFLOW_METADATA_SUMMARY_ENTRY + WHERE PARENT_WORKFLOW_EXECUTION_UUID IS NULL ORDER BY END_TIMESTAMP DESC LIMIT 20;" @@ -1639,6 +1641,7 @@ cromwell::build::print_workflow_statistics() { START_TIMESTAMP as start, END_TIMESTAMP as end FROM WORKFLOW_METADATA_SUMMARY_ENTRY + WHERE PARENT_WORKFLOW_EXECUTION_UUID IS NULL ORDER BY RUNTIME_MINUTES DESC LIMIT 20;" }