Skip to content

Commit

Permalink
Merge branch 'WiddlerQueryUpdates'
Browse files Browse the repository at this point in the history
  • Loading branch information
Amr Abouelleil committed Aug 17, 2018
2 parents dff51d7 + 32ec39a commit 77a52d4
Show file tree
Hide file tree
Showing 5 changed files with 147 additions and 67 deletions.
1 change: 1 addition & 0 deletions resources/hello_remote.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
{
"hello.infile": "",
"hello.onprem_download_path": "/cil/shed/sandboxes/amr/",
"hello.fofn": "gs://broad-cil-devel-bucket/input_data/hello_remote.fofn",
"hello.name": "amr",
"hello.sleep": 1
Expand Down
20 changes: 10 additions & 10 deletions resources/hello_world.wdl
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ workflow hello {
File fofn
String name
Int sleep
Array[File] file_array
# Array[File] file_array
File ? optional_file

String onprem_download_path = "/cil/shed/resources/jenkins_tests/"
Expand All @@ -58,14 +58,14 @@ workflow hello {
}
}

call print_contents {
input: input_file = fofn, optional_file = optional_file
}

scatter(f in file_array) {
call print_contents as pc2 {
input: input_file = f
}
}
# call print_contents {
# input: input_file = fofn, optional_file = optional_file
# }
#
# scatter(f in file_array) {
# call print_contents as pc2 {
# input: input_file = f
# }
# }
}
51 changes: 0 additions & 51 deletions tests/test_cromwell.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,27 +30,6 @@ def test_start_workflow(self):
self.assertEqual(len(wfid), 36)
self.cromwell.stop_workflow(wfid)

def test_query_status(self):
wf = self._initiate_workflow()
wfid = wf['id']
result = self.cromwell.query_status(wfid)
self.assertTrue('id' in result and 'status' in result)
self.cromwell.stop_workflow(wfid)

def test_query_metadata(self):
wf = self._initiate_workflow()
wfid = wf['id']
result = self.cromwell.query_metadata(wfid)
self.assertTrue('id' in result and 'submission' in result)
self.cromwell.stop_workflow(wfid)

def test_query_logs(self):
wf = self._initiate_workflow()
wfid = wf['id']
result = self.cromwell.query_logs(wfid)
self.assertTrue('id' in result)
self.cromwell.stop_workflow(wfid)

def test_build_long_url(self):
wf = self._initiate_workflow()
wfid = wf['id']
Expand All @@ -65,42 +44,13 @@ def test_build_long_url(self):
self.assertEquals(r.status_code, 200)
self.cromwell.stop_workflow(wfid)

def test_query(self):
wf = self._initiate_workflow()
wfid = wf['id']
url_dict = {
'name': 'gatk',
'id': [wfid],
'start': datetime.datetime.now() - datetime.timedelta(days=1),
'end': datetime.datetime.now()
}
result = self.cromwell.query(url_dict)
self.assertTrue(isinstance(result['results'], list), True)
self.cromwell.stop_workflow(wfid)

def test_label_workflow(self):
wf = self._initiate_workflow()
wfid = wf['id']
r = self.cromwell.label_workflow(wfid, self.labels)
self.assertEquals(r.status_code, 200)
self.cromwell.stop_workflow(wfid)

def test_query_labels(self):
wf = self._initiate_workflow()
wfid = wf['id']
labels = {'username': 'amr', 'foo': 'bar'}
self.cromwell.label_workflow(wfid, self.labels)
# This sleep is needed to make sure the label workflow completes before we query for it.
time.sleep(5)
r = self.cromwell.query_labels(labels)
# Here, the most recent workflow that matches the query will be the last item so we can use that to check
# this assertion.
self.assertTrue(wfid in r['results'][-1]['id'])
self.cromwell.stop_workflow(wfid)

def test_query_backend(self):
self.assertTrue('defaultBackend' in self.cromwell.query_backend())

def test_explain(self):
wf = self._initiate_workflow()
wfid = wf['id']
Expand All @@ -116,7 +66,6 @@ def test_stop_workflow(self):
print(result)
self.cromwell.stop_workflow(wfid)


@classmethod
def tearDownClass(self):
print("Done!")
130 changes: 130 additions & 0 deletions tests/test_queries.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
__author__ = 'Amr Abouelleil'
import unittest
import os
import time
from src.Cromwell import Cromwell
import src.config as c
import datetime
import requests


class QueryUnitTests(unittest.TestCase):
@classmethod
def setUpClass(self):
resources = c.resource_dir
self.cromwell = Cromwell(host='btl-cromwell')
self.json = os.path.join(resources, 'hw.json')
self.wdl = os.path.join(resources, 'hw.wdl')
self.labels = {'username': 'amr', 'foo': 'bar'}

def _initiate_workflow(self):
wf = self.cromwell.jstart_workflow(self.wdl, self.json)
time.sleep(5)
return wf

def test_query_status(self):
wf = self._initiate_workflow()
wfid = wf['id']
result = self.cromwell.query_status(wfid)
self.assertTrue('id' in result and 'status' in result)
self.cromwell.stop_workflow(wfid)

def test_query_metadata(self):
wf = self._initiate_workflow()
wfid = wf['id']
result = self.cromwell.query_metadata(wfid)
self.assertTrue('id' in result and 'submission' in result)
self.cromwell.stop_workflow(wfid)

def test_query_logs(self):
wf = self._initiate_workflow()
wfid = wf['id']
result = self.cromwell.query_logs(wfid)
self.assertTrue('id' in result)
self.cromwell.stop_workflow(wfid)

def test_build_long_url(self):
wf = self._initiate_workflow()
wfid = wf['id']
url_dict = {
'name': 'test_build_long_url',
'id': wfid,
'start': datetime.datetime.now() - datetime.timedelta(days=1),
'end': datetime.datetime.now()
}
query_url = self.cromwell.build_query_url('http://btl-cromwell:9000/api/workflows/v1/query?', url_dict)
r = requests.get(query_url)
self.assertEquals(r.status_code, 200)
self.cromwell.stop_workflow(wfid)

def test_query(self):
wf = self._initiate_workflow()
wfid = wf['id']
url_dict = {
'name': 'gatk',
'id': [wfid],
'start': datetime.datetime.now() - datetime.timedelta(days=1),
'end': datetime.datetime.now()
}
result = self.cromwell.query(url_dict)
self.assertTrue(isinstance(result['results'], list), True)
self.cromwell.stop_workflow(wfid)

def test_label_workflow(self):
wf = self._initiate_workflow()
wfid = wf['id']
r = self.cromwell.label_workflow(wfid, self.labels)
self.assertEquals(r.status_code, 200)
self.cromwell.stop_workflow(wfid)

def test_query_labels(self):
wf = self._initiate_workflow()
wfid = wf['id']
labels = {'username': 'amr', 'foo': 'bar'}
self.cromwell.label_workflow(wfid, self.labels)
# This sleep is needed to make sure the label workflow completes before we query for it.
time.sleep(5)
r = self.cromwell.query_labels(labels)
# Here, the most recent workflow that matches the query will be the last item so we can use that to check
# this assertion.
self.assertTrue(wfid in r['results'][-1]['id'])
self.cromwell.stop_workflow(wfid)

def test_query_filter_by_statuses(self):
from argparse import Namespace
from widdler import call_list
wf = self._initiate_workflow()
wfid = wf['id']
result = call_list(Namespace(server="btl-cromwell", all=False, no_notify=True, verbose=True, interval=None,
username="*", days=1, filter=['Succeeded', 'Failed']))
statuses = set(d['status'] for d in result)
self.assertEqual(len(statuses), 2)
self.assertIn('Succeeded', statuses)
self.assertIn('Failed', statuses)
self.cromwell.stop_workflow(wfid)

def test_query_filter_by_name(self):
from argparse import Namespace
from widdler import call_list
user_result = call_list(Namespace(server="btl-cromwell", all=False, no_notify=True, verbose=True, interval=None,
username="amr", days=1, filter=None))
user_wfids = set(d['id'] for d in user_result)
all_result = call_list(Namespace(server="btl-cromwell", all=False, no_notify=True, verbose=True, interval=None,
username="*", days=1, filter=None))
all_wfids = set(d['id'] for d in all_result)
self.assertGreater(len(all_wfids), len(user_wfids))

def test_query_filter_by_days(self):
from argparse import Namespace
from widdler import call_list
result = call_list(Namespace(server="btl-cromwell", all=False, no_notify=True, verbose=True, interval=None,
username="*", days=1, filter=None))
all_dates = set(d['start'].split('T')[0] for d in result)
self.assertEqual(len(all_dates), 1)

def test_query_backend(self):
self.assertTrue('defaultBackend' in self.cromwell.query_backend())

@classmethod
def tearDownClass(self):
print("Done!")
12 changes: 6 additions & 6 deletions widdler.py
Original file line number Diff line number Diff line change
Expand Up @@ -308,12 +308,14 @@ def my_safe_repr(object, context, maxlevels, level):
q = m.get_user_workflows(raw=True, start_time=start_date_str)
try:
result = q["results"]
if args.filter:
result = [res for res in result if res['status'] in args.filter]
result = map(lambda j: process_job(j), result)
printer = pprint.PrettyPrinter()
printer.format = my_safe_repr
printer.pprint(result)
args.monitor = True
return None
return result
except KeyError as e:
logger.critical('KeyError: Unable to find key {}'.format(e))

Expand Down Expand Up @@ -372,7 +374,7 @@ def call_log(args):
command = command + key + ":\n\n"
command = command + command_log + "\n\n"

print(command) #print to stdout
print(command) # print to stdout
return None


Expand Down Expand Up @@ -491,7 +493,7 @@ def call_upload(args):
query.add_argument('-s', '--status', action='store_true', default=False, help='Print status for workflow to stdout')
query.add_argument('-m', '--metadata', action='store_true', default=False, help='Print metadata for workflow to stdout')
query.add_argument('-l', '--logs', action='store_true', default=False, help='Print logs for workflow to stdout')
query.add_argument('-u', '--username', action='store', default=getpass.getuser(), help='Owner of workflows to monitor.')
query.add_argument('-u', '--username', action='store', default=getpass.getuser(), help='Owner of workflows to query.')
query.add_argument('-L', '--label', action='append', help='Query status of all workflows with specific label(s).')
query.add_argument('-d', '--days', action='store', default=7, help='Last n days to query.')
query.add_argument('-S', '--server', action='store', required=True, type=str, choices=c.servers,
Expand Down Expand Up @@ -583,10 +585,8 @@ def call_upload(args):
upload.set_defaults(func=call_upload)


args = parser.parse_args()


def main():
args = parser.parse_args()
# Get user's username so we can tag workflows and logs for them.
user = getpass.getuser()
try:
Expand Down

0 comments on commit 77a52d4

Please sign in to comment.