Skip to content

Commit

Permalink
Merge pull request #22 from henriqueribeiro/cromwell-83
Browse files Browse the repository at this point in the history
Cromwell 83
  • Loading branch information
henriqueribeiro authored Aug 16, 2022
2 parents 3262085 + d40342a commit 1d78426
Show file tree
Hide file tree
Showing 571 changed files with 4,970 additions and 9,864 deletions.
4 changes: 0 additions & 4 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -88,10 +88,6 @@ workflows:
name: testCentaurAws
build-type: "centaurAws"
build-mysql: "5.7"
- test:
name: testCentaurBcs
build-type: "centaurBcs"
build-mysql: "5.7"
- test:
name: testCentaurDummy
build-type: "centaurDummy"
Expand Down
1 change: 1 addition & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@
*.MD text
*.java text
*.html text
docs/api/RESTAPI.md linguist-generated=true
24 changes: 5 additions & 19 deletions .github/issue_template.md
Original file line number Diff line number Diff line change
@@ -1,27 +1,13 @@
###
### IMPORTANT: Please file new issues over in our Jira issue tracker!
###
### https://broadworkbench.atlassian.net/projects/BA/issues
###
### You may need to create an account before you can view/create issues.
###

<!--
Hi! Thanks for taking the time to report feedback.
Before posting an issue over in Jira tracker, please check whether your question is already answered in our:
forum https://gatkforums.broadinstitute.org/wdl/categories/ask-the-wdl-team
documentation http://cromwell.readthedocs.io/en/develop/
Other forums:
FireCloud https://gatkforums.broadinstitute.org/firecloud/categories/ask-the-firecloud-team
WDL https://gatkforums.broadinstitute.org/wdl/categories/ask-the-wdl-team
CWL https://www.biostars.org/
Please check whether your question is already answered in our:
Documentation http://cromwell.readthedocs.io/en/develop/
Bioinformatics Stack Exchange https://bioinformatics.stackexchange.com/search?q=cromwell
Slack https://join.slack.com/t/cromwellhq/shared_invite/zt-dxmmrtye-JHxwKE53rfKE_ZWdOHIB4g
-->

<!-- Are you seeing something that looks like a bug? Then great! You're almost in the right place. -->

<!-- You'll want to go to https://broadworkbench.atlassian.net/projects/BA/issues and then tell us: -->
<!-- Are you seeing something that looks like a bug? Please attach as much information as possible. -->

<!-- Which backend are you running? -->

Expand Down
71 changes: 71 additions & 0 deletions .github/workflows/chart_update_on_merge.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
name: chart-update-on-merge

on:
pull_request:
types:
- closed

jobs:
chart-update:
name: Cromwhelm Chart Auto Updater
if: github.event.pull_request.merged == true
runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners
steps:
- name: Clone Cromwell
uses: actions/checkout@v2
with:
repository: broadinstitute/cromwell
token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # Has to be set at checkout AND later when pushing to work
path: cromwell
- uses: olafurpg/setup-scala@v10
with:
java-version: adopt@1.11
- name: Clone Cromwhelm
uses: actions/checkout@v2
with:
repository: broadinstitute/cromwhelm
token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # Has to be set at checkout AND later when pushing to work
path: cromwhelm
- name: Find Cromwell short SHA
run: |
set -e
cd cromwell
echo "CROMWELL_SHORT_SHA=`git rev-parse --short $GITHUB_SHA`" >> $GITHUB_ENV
- name: Find Cromwell release number
run: |
set -e
previous_version=$(curl -X GET https://api.github.com/repos/broadinstitute/cromwell/releases/latest | jq .tag_name | xargs)
if ! [[ "${previous_version}" =~ ^[0-9][0-9]+$ ]]; then
exit 1
fi
echo "CROMWELL_NUMBER=$((previous_version + 1))" >> $GITHUB_ENV
- name: Save complete image ID
run: |
echo "CROMWELL_SNAP_VERSION=`echo "$CROMWELL_NUMBER-$CROMWELL_SHORT_SHA-SNAP"`" >> $GITHUB_ENV
# `DSDEJENKINS_PASSWORD` auto syncs from vault with https://github.com/broadinstitute/terraform-ap-deployments/pull/614
- name: Login to Docker Hub
uses: docker/login-action@v1
with:
username: dsdejenkins
password: ${{ secrets.DSDEJENKINS_PASSWORD }}
- name: Build Cromwell Docker
run: |
set -e
cd cromwell
sbt server/docker
docker push broadinstitute/cromwell:$CROMWELL_SNAP_VERSION
- name: Edit & push chart
env:
BROADBOT_GITHUB_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
run: |
set -e
cd cromwhelm
git checkout main
ls -la
sed -i "s/appVersion.*/appVersion: \"$CROMWELL_SNAP_VERSION\"/" cromwell-helm/Chart.yaml
sed -i "s/image: broadinstitute\/cromwell.*/image: broadinstitute\/cromwell:$CROMWELL_SNAP_VERSION/" cromwell-helm/templates/cromwell.yaml
git diff
git config --global user.name "broadbot"
git config --global user.email "broadbot@broadinstitute.org"
git commit -am "Auto update to Cromwell $CROMWELL_SNAP_VERSION"
git push https://broadbot:$BROADBOT_GITHUB_TOKEN@github.com/broadinstitute/cromwhelm.git main
73 changes: 73 additions & 0 deletions .github/workflows/make_publish_prs.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
name: Make publish PRs

on:
workflow_dispatch:
inputs:
old_cromwell_version:
description: 'What version of Cromwell to update from'
required: true
new_cromwell_version:
description: 'What version of Cromwell to update to'
required: true
jira_ticket:
description: 'The JIRA ticket to link the update PR to'
required: true

jobs:
make-firecloud-develop-pr:
name: Create firecloud-develop PR
runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners
steps:
- name: Clone firecloud-develop
uses: actions/checkout@v2
with:
repository: broadinstitute/firecloud-develop
token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # Has to be set at checkout AND later when pushing to work
path: firecloud-develop
- name: Update & push to firecloud-develop
id: update-and-push
env:
BROADBOT_GITHUB_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
run: |
cd firecloud-develop
JIRA_TICKET=${{ github.event.inputs.jira_ticket }}
OLD_CROMWELL_V=${{ github.event.inputs.old_cromwell_version }}
NEW_CROMWELL_V=${{ github.event.inputs.new_cromwell_version }}
NEW_BRANCH_NAME="${JIRA_TICKET}_cromwell_${NEW_CROMWELL_V}"
TEMP=$(mktemp)
git checkout -b "${NEW_BRANCH_NAME}"
SED_STRING="s/${OLD_CROMWELL_V}/${NEW_CROMWELL_V}/g"
FILES=( "base-configs/cromwell/cromwell.conf.ctmpl" "run-context/fiab/scripts/FiaB_images.env" )
for i in "${FILES[@]}"
do
echo "Updating ${i}"
sed "${SED_STRING}" ${i} > ${TEMP}
mv ${TEMP} ${i}
git add ${i}
done
echo "Pushing to firecloud-develop branch ${NEW_BRANCH_NAME}"
git config --global user.name "broadbot"
git config --global user.email "broadbot@broadinstitute.org"
git commit -m "Updating Cromwell version to ${NEW_CROMWELL_V}"
git push https://broadbot:$BROADBOT_GITHUB_TOKEN@github.com/broadinstitute/firecloud-develop.git ${NEW_BRANCH_NAME}
echo ::set-output name=NEW_BRANCH_NAME::${NEW_BRANCH_NAME}
- name: Create firecloud-develop PR
uses: actions/github-script@v6
with:
github-token: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
script: |
const result = await github.rest.pulls.create({
title: '${{ github.event.inputs.jira_ticket }}: Update Cromwell version to ${{ github.event.inputs.new_cromwell_version }}',
owner: 'broadinstitute',
repo: 'firecloud-develop',
head: '${{ steps.update-and-push.outputs.NEW_BRANCH_NAME }}',
base: 'dev',
body: [
'This PR is auto-generated by',
'[Cromwell actions/make_publish_prs](https://github.com/broadinstitute/cromwell/blob/develop/.github/workflows/make_publish_prs.yml), using',
'[github actions/github-script](https://github.com/actions/github-script).',
'',
'It updates cromwell from version ${{ github.event.inputs.old_cromwell_version }} to ${{ github.event.inputs.new_cromwell_version }}.'
].join('\n')
});
2 changes: 0 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,6 @@ scripts/docker-compose-mysql/compose/mysql/data
# Vault rendered resources
artifactory_credentials.properties
aws_credentials
bcs_application.conf
bcs_login.inc.sh
centaur_secure.inc.conf
cromwell-centaur-requester-pays-service-account.json
cromwell-centaur-service-account.json
Expand Down
2 changes: 1 addition & 1 deletion .scala-steward.conf
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
#
# Default: @asap
#
pullRequests.frequency = "0 0 1 1,4,7,10 ?" # Run at 00:00 on the 1st day of Jan,Apr,Jul,Oct (whatever day that is)
pullRequests.frequency = "0 0 1 3,6,9,12 ?" # Run at 00:00 on the 1st day of Mar,Jun,Sep,Dec (whatever day of the week that is)

# Only these dependencies which match the given patterns are updated.
#
Expand Down
12 changes: 0 additions & 12 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,6 @@ env:
- >-
BUILD_TYPE=centaurAws
BUILD_MYSQL=5.7
- >-
BUILD_TYPE=centaurBcs
BUILD_MYSQL=5.7
- >-
BUILD_TYPE=centaurDummy
BUILD_MYSQL=5.7
Expand Down Expand Up @@ -75,15 +72,6 @@ env:
BUILD_MYSQL=5.7
- >-
BUILD_TYPE=checkPublish
- >-
BUILD_TYPE=conformanceLocal
BUILD_MYSQL=5.7
- >-
BUILD_TYPE=conformancePapiV2beta
BUILD_MYSQL=5.7
- >-
BUILD_TYPE=conformanceTesk
BUILD_MYSQL=5.7
- >-
BUILD_TYPE=horicromtalDeadlock
- >-
Expand Down
83 changes: 83 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,88 @@
# Cromwell Change Log

## 82 Release Notes

* Restored missing example configuration file
* Upgraded to latest version of the Google Cloud Storage NIO library (0.124.8)
* Cromwell will now finitely retry the following Google Cloud Storage I/O error.
* Response code `400` bad request, message `User project specified in the request is invalid`
* The default retry count is `5` and may be customized with `system.io.number-of-attempts`.

## 81 Release Notes

### Workflow labels in TES tasks

Beginning in Cromwell 81 we will populate the `tags` field of tasks created by the TES backend
with the labels applied to the workflow at creation time. No guarantee is made about labels
added while the workflow is running.

### Alibaba BCS backend and OSS filesystem removed

The BCS backend and OSS filesystem (both of which support Alibaba Cloud) have been removed.

## 80 Release Notes

### Direct WES support in Cromwell

Cromwell 80 no longer supports the wes2cromwell project within the Cromwell repository.

In the previous release, 3 Wes2Cromwell endpoints in the Cromwell project were implemented and documented in the Swagger API. Three new endpoints,
located within the wes2cromwell project, will also be moved, implemented, and documented within Cromwell. As a result of this, we can safely remove
and deprecate the wes2cromwell project from the repo.

Previous endpoints:

| HTTP verb | Endpoint path | Description |
| --------- | ------------- |---------------|
| GET | /api/ga4gh/wes/v1/service-info | Server info |
| POST | /api/ga4gh/wes/v1/runs/{run_id}/cancel | Abort workflow |
| GET | /api/ga4gh/wes/v1/runs/{run_id}/status | Workflow status |

Newly implemented endpoints:

| HTTP verb | Endpoint path | Description |
| --------- | ------------- |-----------------|
| GET | /api/ga4gh/wes/v1/runs | List workflows |
| POST | /api/ga4gh/wes/v1/runs | Submit workflow |
| GET | /api/ga4gh/wes/v1/runs/{run_id} | Workflow details |

## 79 Release Notes

### Last release with CWL support

Cromwell 79 is the last release with CWL. Support will be removed in Cromwell 80 and above.

CWL will be re-introduced at a later date in the [Terra platform](https://terra.bio/), using a solution other than Cromwell. See the blog post ["Terra’s roadmap to supporting more workflow languages"](https://terra.bio/terras-roadmap-to-supporting-more-workflow-languages/) for details.

| Product | Language | Support |
|-------------------------------------------|----------|-----------------------|
| Cromwell standalone | WDL | :white_check_mark: |
| Cromwell standalone | CWL | :x: |
| [Terra SaaS platform](https://terra.bio/) | WDL | :white_check_mark: |
| [Terra SaaS platform](https://terra.bio/) | CWL | Future support planned |

### Last release with Alibaba Cloud

The BCS backend and OSS filesystem (both of which support Alibaba Cloud) will be removed in version 80.

### WES endpoints preview

As a means to stay on top of endpoints within our repo, 3 new Workflow Execution Service (WES) endpoints are now documented in the Cromwell Swagger (others to follow as part of later work):

| HTTP verb | Endpoint path | Description |
| --------- | ------------- |---------------|
| GET | /api/ga4gh/wes/v1/service-info | Server info |
| POST | /api/ga4gh/wes/v1/runs/{run_id}/cancel | Abort workflow |
| GET | /api/ga4gh/wes/v1/runs/{run_id}/status | Workflow status |

### Scala 2.13

Cromwell is now built with Scala version 2.13. This change should not be noticeable to users but may be of interest to developers of Cromwell backend implementations.

### Bug Fixes

* Fixed a call caching bug in which an invalid cache entry could cause a valid cache entry to be ignored.

## 75 Release Notes

### New `AwaitingCloudQuota` backend status
Expand Down
10 changes: 9 additions & 1 deletion CromIAM/src/main/scala/cromiam/cromwell/CromwellClient.scala
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,8 @@ class CromwellClient(scheme: String, interface: String, port: Int, log: LoggingA

def forwardToCromwell(httpRequest: HttpRequest): FailureResponseOrT[HttpResponse] = {
val future = {
val headers = httpRequest.headers.filterNot(_.name == TimeoutAccessHeader)
// See CromwellClient's companion object for info on these header modifications
val headers = httpRequest.headers.filterNot(header => header.name == TimeoutAccessHeader || header.name == HostHeader)
val cromwellRequest = httpRequest
.copy(uri = httpRequest.uri.withAuthority(interface, port).withScheme(scheme))
.withHeaders(headers)
Expand Down Expand Up @@ -112,6 +113,13 @@ object CromwellClient {
// See: https://github.com/akka/akka-http/issues/64
val TimeoutAccessHeader = "Timeout-Access"

// Header that all clients will add before they send a request.
// If we don't strip this header out and let Akka replace it automatically before forwarding
// requests to Cromwell, any host-based routing in front of Cromwell will fail because the
// header will still contain CromIAM's host, not Cromwell's.
// See: https://broadworkbench.atlassian.net/browse/DDO-2190
val HostHeader = "Host"

final case class CromwellConnectionFailure(f: Throwable) extends Exception(s"Unable to connect to Cromwell (${f.getMessage})", f)

implicit class EnhancedWorkflowLabels(val wl: WorkflowLabels) extends AnyVal {
Expand Down
2 changes: 1 addition & 1 deletion CromIAM/src/main/scala/cromiam/server/CromIamServer.scala
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ object CromIamServer extends HttpApp with CromIamApiService with SwaggerService

override val routes: Route = allRoutes ~ swaggerUiResourceRoute

override val statusService: StatusService = new StatusService(() => Map(Cromwell -> cromwellClient.subsystemStatus, Sam -> samClient.subsystemStatus))
override val statusService: StatusService = new StatusService(() => Map(Cromwell -> cromwellClient.subsystemStatus(), Sam -> samClient.subsystemStatus()))

// Override default shutdownsignal which was just "hit return/enter"
override def waitForShutdownSignal(actorSystem: ActorSystem)(implicit executionContext: ExecutionContext): Future[Done] = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import common.validation.Validation._
import cromiam.server.config.CromIamServerConfig._
import net.ceedubs.ficus.Ficus._

import scala.collection.JavaConverters._
import scala.jdk.CollectionConverters._
import scala.util.{Failure, Success, Try}

final case class CromIamServerConfig(cromIamConfig: CromIamConfig,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ trait StatusCheckedSubsystem {
* error messages, otherwise OK = false and include the response body
*/
def subsystemStatus()(implicit ec: ExecutionContext): Future[SubsystemStatus] = {
sttp.get(statusUri).send map { x =>
sttp.get(statusUri).send() map { x =>
x.body match {
case Right(_) => SubsystemStatus(true, None)
case Left(errors) => SubsystemStatus(false, Option(List(errors)))
Expand Down
3 changes: 2 additions & 1 deletion CromIAM/src/main/scala/cromiam/webservice/QuerySupport.scala
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package cromiam.webservice

import akka.event.LoggingAdapter
import akka.http.scaladsl.model.Uri.Query
import akka.http.scaladsl.model._
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.server._
Expand Down Expand Up @@ -86,7 +87,7 @@ trait QuerySupport extends RequestSupport {
// DO NOT REMOVE THE NEXT LINE WITHOUT READING THE SCALADOC ON ensureNoLabelOrs
ensureNoLabelOrs(user, labelOrs)

val newQueryBuilder = query.newBuilder
val newQueryBuilder = Query.newBuilder
newQueryBuilder ++= query

val collectionLabels = userCollectionLabels(user, collections)
Expand Down
Loading

0 comments on commit 1d78426

Please sign in to comment.