From 77ca0a355219eb4bc23d99e943fd61181e170e2e Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 4 Oct 2016 18:45:57 +0200 Subject: [PATCH 001/588] Ignore hidden project and system folders --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a9fecf7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.idea +**/.DS_Store \ No newline at end of file From aa0cfb8040d50586eb555033782a9e572a74812f Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 7 Oct 2016 15:33:24 +0200 Subject: [PATCH 002/588] Added initial version --- workflow_tester.py | 354 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 354 insertions(+) create mode 100755 workflow_tester.py diff --git a/workflow_tester.py b/workflow_tester.py new file mode 100755 index 0000000..bc19bee --- /dev/null +++ b/workflow_tester.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python + +import os +import json +import uuid +import yaml +import logging +import optparse +from bioblend.galaxy.objects import GalaxyInstance +from bioblend.galaxy.workflows import WorkflowClient +from bioblend.galaxy.histories import HistoryClient + +# Galaxy ENV variable names +ENV_KEY_GALAXY_URL = "BIOBLEND_GALAXY_URL" +ENV_KEY_GALAXY_API_KEY = "BIOBLEND_GALAXY_API_KEY" + +# Default settings +DEFAULT_HISTORY_NAME_PREFIX = "_TestHistory" +DEFAULT_WORKFLOW_NAME_PREFIX = "_WorkflowTest" +DEFAULT_OUTPUT_FOLDER = "results" +DEFAULT_CONFIG_FILENAME = "workflows.yml" +DEFAULT_WORKFLOW_CONFIG = { + "file": "workflow.ga", + "inputs": { + "Input Dataset": {"name": "Input Dataset", "file": ["input"]} + }, + "outputs": { + "output1": {"file": "expected_output", "comparator": "filecmp.cmp", "name": "output1"}, + "output2": {"file": "expected_output", "comparator": "filecmp.cmp", "name": "output2"} + } +} + +# configure logger +logger = logging.getLogger("WorkflowTest") +logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s') +logger.setLevel(logging.DEBUG) + + +class WorkflowTestSuite(): + def __init__(self, galaxy_url=None, galaxy_api_key=None): + self._workflows = {} + self._workflows_tests = [] + self._galaxy_instance = None + self._galaxy_workflows = None + # + if galaxy_url: + self._galaxy_url = galaxy_url + elif os.environ.has_key(ENV_KEY_GALAXY_URL): + self._galaxy_url = os.environ[ENV_KEY_GALAXY_URL] + else: + raise ValueError("GALAXY URL not defined!!!") + # + if galaxy_api_key: + self._galaxy_api_key = galaxy_api_key + elif os.environ.has_key(ENV_KEY_GALAXY_API_KEY): + self._galaxy_api_key = os.environ[ENV_KEY_GALAXY_API_KEY] + else: + raise ValueError("GALAXY API KEY not defined!!!") + + # initialize the galaxy instance + self._galaxy_instance = GalaxyInstance(self._galaxy_url, self._galaxy_api_key) + self._galaxy_workflows = WorkflowClient(self._galaxy_instance.gi) + + @property + def galaxy_url(self): + return self._galaxy_url + + @property + def galaxy_api_key(self): + return self._galaxy_api_key + + @property + def galaxy_instance(self): + return self._galaxy_instance + + @property + def workflows(self): + return self.galaxy_instance.workflows.list() + + def run_tests(self, workflow_tests_config): + results = [] + for test_config in workflow_tests_config["workflows"].values(): + workflow = self.create_test_runner(test_config) + test_case = workflow.run_test(test_config["inputs"], test_config["outputs"], + workflow_tests_config["output_folder"]) + results.append(test_case) + return results + + def create_test_runner(self, workflow_test_config): + workflow = self._load_work_flow(workflow_test_config["file"]) + runner = WorkflowTestRunner(self.galaxy_instance, workflow, workflow_test_config) + self._workflows_tests.append(runner) + return runner + + def cleanup(self): + logger.debug("Cleaning save histories ...") + hslist = self.galaxy_instance.histories.list() + for history in [h for h in hslist if DEFAULT_HISTORY_NAME_PREFIX in h.name]: + self.galaxy_instance.histories.delete(history.id) + + logger.debug("Cleaning workflow library ...") + wflist = self.galaxy_instance.workflows.list() + workflows = [w for w in wflist if DEFAULT_WORKFLOW_NAME_PREFIX in w.name] + for wf in workflows: + self._unload_workflow(wf.id) + + def _load_work_flow(self, workflow_filename, workflow_name=None): + with open(workflow_filename) as f: + wf_json = json.load(f) + if not self._workflows.has_key(wf_json["name"]): + wf_name = wf_json["name"] + wf_json["name"] = workflow_name if workflow_name else "_".join([DEFAULT_WORKFLOW_NAME_PREFIX, wf_name]) + wf_info = self._galaxy_workflows.import_workflow_json(wf_json) + workflow = self.galaxy_instance.workflows.get(wf_info["id"]) + self._workflows[wf_name] = workflow + else: + workflow = self._workflows[wf_json["name"]] + return workflow + + def _unload_workflow(self, workflow_id): + self._galaxy_workflows.delete_workflow(workflow_id) + + +class WorkflowTestRunner(): + def __init__(self, galaxy_instance, galaxy_workflow, workflow_test_config): + self._galaxy_instance = galaxy_instance + self._galaxy_workflow = galaxy_workflow + self._workflow_test_config = workflow_test_config + self._galaxy_history_client = HistoryClient(galaxy_instance.gi) + self._test_cases = {} + + def check_required_tools(self): + logger.debug("Checking required tools ...") + available_tools = self._galaxy_instance.tools.list() + missing_tools = [] + for order, step in self._galaxy_workflow.steps.items(): + if step.tool_id and len( + filter(lambda t: t.id == step.tool_id and t.version == step.tool_version, available_tools)) == 0: + missing_tools.append((step.tool_id, step.tool_version)) + logger.debug("Missing tools: {0}".format("None" + if len(missing_tools) == 0 + else ", ".join(["{0} (version {1})" + .format(x[0], x[1]) for x in missing_tools]))) + logger.debug("Checking required tools: DONE") + return missing_tools + + def run_test(self, input_map=None, expected_output_map=None, output_folder=DEFAULT_OUTPUT_FOLDER): + + # check input_map + if not input_map: + if self._workflow_test_config.has_key("inputs"): + input_map = self._workflow_test_config["inputs"] + else: + raise ValueError("No input configured !!!") + # check expected_output_map + if not expected_output_map: + if self._workflow_test_config.has_key("outputs"): + expected_output_map = self._workflow_test_config["outputs"] + else: + raise ValueError("No output configured !!!") + + # uuid of the current test + test_uuid = DEFAULT_HISTORY_NAME_PREFIX + str(uuid.uuid1()) + + # check tools + missing_tools = self.check_required_tools() + if len(missing_tools) == 0: + + # create a new history for the current test + history_info = self._galaxy_history_client.create_history(test_uuid) + history = self._galaxy_instance.histories.get(history_info["id"]) + logger.info("Create a history '%s' (id: %r)", history.name, history.id) + + # upload input data to the current history + # and generate the datamap INPUT --> DATASET + datamap = {} + for label, config in input_map.items(): + datamap[label] = [] + for filename in config["file"]: + datamap[label].append(history.upload_dataset(filename)) + + # run the workflow + logger.info("Workflow '%s' (id: %s) running ...", self._galaxy_workflow.name, self._galaxy_workflow.id) + outputs, output_history = self._galaxy_workflow.run(datamap, history, wait=True, polling_interval=0.5) + logger.info("Workflow '%s' (id: %s) executed", self._galaxy_workflow.name, self._galaxy_workflow.id) + + # test output ... + test_case = WorkflowTestCase(self._galaxy_workflow, test_uuid, input_map, + outputs, expected_output_map, output_history, missing_tools, output_folder) + else: + # test output ... + test_case = WorkflowTestCase(self._galaxy_workflow, test_uuid, input_map, [], + expected_output_map, None, missing_tools, output_folder) + self._test_cases[test_uuid] = test_case + test_case.check_outputs() + return test_case + + def cleanup(self): + for test_id, test_case in self._test_cases.items(): + test_case.cleanup() + + +class WorkflowTestCase(): + def __init__(self, workflow, test_id, input_map, outputs, expected_output_map, output_history, + missing_tools, output_folder=DEFAULT_OUTPUT_FOLDER): + self.workflow = workflow + self.test_id = test_id + self.inputs = input_map + self.outputs = outputs + self.output_history = output_history + self.expected_output_map = expected_output_map + self.output_folder = output_folder + self.missing_tools = missing_tools + self.output_file_map = {} + self.results = None + + if not os.path.isdir(output_folder): + os.makedirs(output_folder) + + def __str__(self): + return "Test {0}: workflow {1}, intputs=[{2}], outputs=[{3}]" \ + .format(self.test_id, self.workflow.name, + ",".join([i for i in self.inputs]), + ", ".join(["{0}: {1}".format(x[0], "OK" if x[1] else "ERROR") + for x in self.results.items()])) + + def __repr__(self): + return self.__str__() + + def check_output(self, output, force=False): + logger.debug("Checking OUTPUT '%s' ...", output.name) + if not self.results or not self.results.has_key(output.name) or force: + output_filename = os.path.join(self.output_folder, "output_" + str(self.outputs.index(output))) + with open(output_filename, "w") as out_file: + output.download(out_file) + self.output_file_map[output.name] = {"dataset": output, "filename": output_filename} + logger.debug("Downloaded output {0}: dataset_id '{1}', filename '{2}'".format(output.name, output.id, + output_filename)) + config = self.expected_output_map[output.name] + comparator = _load_comparator(config["comparator"]) + result = comparator(config["file"], output_filename) + logger.debug("Output '{0}' {1} the expected: dataset '{2}', actual-output '{3}', expected-output '{4}'" + .format(output.name, "is equal to" if result else "differs from", + output.id, output_filename, config["file"])) + self.results[output.name] = result + logger.debug("Checking OUTPUT '%s': DONE", output.name) + return self.results[output.name] + + def check_outputs(self, force=False): + logger.info("Checking test output: ...") + if not self.results or force: + self.results = {} + for output in self.outputs: + self.results[output.name] = self.check_output(output) + logger.info("Checking test output: DONE") + return self.results + + def clean_up(self): + self.output_history + + +def load_configuration(filename=DEFAULT_CONFIG_FILENAME): + config = {} + if os.path.exists(filename): + with open(filename, "r") as config_file: + workflows_conf = yaml.load(config_file) + config["galaxy_url"] = workflows_conf["galaxy_url"] + config["galaxy_api_key"] = workflows_conf["galaxy_api_key"] + config["workflows"] = {} + for workflow in workflows_conf.get("workflows").items(): + w = DEFAULT_WORKFLOW_CONFIG.copy() + w["name"] = workflow[0] + w.update(workflow[1]) + # parse inputs + w["inputs"] = _parse_yaml_list(w["inputs"]) + # parse outputs + w["outputs"] = _parse_yaml_list(w["outputs"]) + # add the workflow + config["workflows"][w["name"]] = w + else: + config["workflows"] = {"unknown": DEFAULT_WORKFLOW_CONFIG.copy()} + return config + + +def load_workflow_test_configuration(workflow_test_name, filename=DEFAULT_CONFIG_FILENAME): + config = load_configuration(filename) + if config["workflows"].has_key(workflow_test_name): + return config["workflows"][workflow_test_name] + else: + raise KeyError("WorkflowTest with name '%s' not found" % workflow_test_name) + + +def _parse_yaml_list(ylist): + objs = {} + if isinstance(ylist, list): + for obj in ylist: + obj_data = obj.items() + obj_name = obj_data[0][0] + obj_file = obj_data[0][1] + objs[obj_name] = { + "name": obj_name, + "file": obj_file + } + elif isinstance(ylist, dict): + for obj_name, obj_data in ylist.items(): + obj_data["name"] = obj_name + objs[obj_name] = obj_data + return objs + + +def _load_comparator(fully_qualified_comparator_function): + components = fully_qualified_comparator_function.split('.') + mod = __import__(components[0]) + for comp in components[1:]: + mod = getattr(mod, comp) + return mod + + +def _parse_cli_options(): + parser = optparse.OptionParser() + parser.add_option('--server', help='Galaxy server URL') + parser.add_option('--api-key', help='Galaxy server API KEY') + parser.add_option('-o', '--output', help='absolute path of the folder to download workflow outputs') + parser.add_option('-f', '--file', default=DEFAULT_CONFIG_FILENAME, help='YAML configuration file of workflow tests') + (options, args) = parser.parse_args() + return (options, args) + + +def main(clean_up=True): + options, args = _parse_cli_options() + config = load_configuration(options.file) + + config["galaxy_url"] = options.server \ + if options.server \ + else config["galaxy_url"] if config.has_key("galaxy_url") else None + + config["galaxy_api_key"] = options.api_key \ + if options.api_key \ + else config["galaxy_api_key"] if config.has_key("galaxy_api_key") else None + + config["output_folder"] = options.output \ + if options.output \ + else config["output_folder"] if config.has_key("output_folder") else DEFAULT_OUTPUT_FOLDER + + test_suite = WorkflowTestSuite(config["galaxy_url"], config["galaxy_api_key"]) + test_suite.run_tests(config) + if clean_up: + test_suite.cleanup() + + return (config, test_suite) + + +if __name__ == '__main__': + main() From 39cb90aa2c43a9e6d931ef9ce2740ae4ba984f7a Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 11 Oct 2016 15:25:27 +0200 Subject: [PATCH 003/588] Fixed imports --- workflow_tester.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index bc19bee..01eacfc 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -1,14 +1,18 @@ #!/usr/bin/env python -import os -import json -import uuid -import yaml -import logging -import optparse -from bioblend.galaxy.objects import GalaxyInstance -from bioblend.galaxy.workflows import WorkflowClient -from bioblend.galaxy.histories import HistoryClient +import os as _os + +import logging as _logging +import unittest as _unittest +import optparse as _optparse + +from json import load as _json_load +from yaml import load as _yaml_load +from uuid import uuid1 as _uuid1 + +from bioblend.galaxy.objects import GalaxyInstance as _GalaxyInstance +from bioblend.galaxy.workflows import WorkflowClient as _WorkflowClient +from bioblend.galaxy.histories import HistoryClient as _HistoryClient # Galaxy ENV variable names ENV_KEY_GALAXY_URL = "BIOBLEND_GALAXY_URL" From 85ec88af04d56bfc7dc409eadd7db4668c2cdeae Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 11 Oct 2016 15:25:43 +0200 Subject: [PATCH 004/588] Changed default prefixes --- workflow_tester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 01eacfc..ea219c1 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -19,8 +19,8 @@ ENV_KEY_GALAXY_API_KEY = "BIOBLEND_GALAXY_API_KEY" # Default settings -DEFAULT_HISTORY_NAME_PREFIX = "_TestHistory" -DEFAULT_WORKFLOW_NAME_PREFIX = "_WorkflowTest" +DEFAULT_HISTORY_NAME_PREFIX = "_WorkflowTestHistory_" +DEFAULT_WORKFLOW_NAME_PREFIX = "_WorkflowTest_" DEFAULT_OUTPUT_FOLDER = "results" DEFAULT_CONFIG_FILENAME = "workflows.yml" DEFAULT_WORKFLOW_CONFIG = { From 6c7151dd7394069179561c7d9d69da27308e7937 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 11 Oct 2016 15:26:23 +0200 Subject: [PATCH 005/588] Updated base configuration of the module logger --- workflow_tester.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index ea219c1..4d7c44d 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -35,9 +35,8 @@ } # configure logger -logger = logging.getLogger("WorkflowTest") -logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s') -logger.setLevel(logging.DEBUG) +_logger = _logging.getLogger("WorkflowTest") +_logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s') class WorkflowTestSuite(): From 91588c4f0e38f6b5859215e5ee8cbfbe2a5ce12c Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 11 Oct 2016 15:28:35 +0200 Subject: [PATCH 006/588] Added new CLI options to enable/disable {logger, debug_mode, assertions, cleanup} --- workflow_tester.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/workflow_tester.py b/workflow_tester.py index 4d7c44d..bb72608 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -320,9 +320,13 @@ def _load_comparator(fully_qualified_comparator_function): def _parse_cli_options(): - parser = optparse.OptionParser() + parser = _optparse.OptionParser() parser.add_option('--server', help='Galaxy server URL') parser.add_option('--api-key', help='Galaxy server API KEY') + parser.add_option('--enable-logger', help='Enable log messages', action='store_true') + parser.add_option('--debug', help='Enable debug mode', action='store_true') + parser.add_option('--disable-cleanup', help='Disable cleanup', action='store_false') + parser.add_option('--disable-assertions', help='Disable assertions', action='store_false') parser.add_option('-o', '--output', help='absolute path of the folder to download workflow outputs') parser.add_option('-f', '--file', default=DEFAULT_CONFIG_FILENAME, help='YAML configuration file of workflow tests') (options, args) = parser.parse_args() @@ -345,6 +349,24 @@ def main(clean_up=True): if options.output \ else config["output_folder"] if config.has_key("output_folder") else DEFAULT_OUTPUT_FOLDER + config["enable_logger"] = options.enable_logger \ + if options.enable_logger \ + else config["enable_logger"] if config.has_key("enable_logger") else False + + config["disable_cleanup"] = options.disable_cleanup + config["disable_assertions"] = options.disable_assertions + + for test_config in config["workflows"].values(): + test_config["disable_cleanup"] = config["disable_cleanup"] + test_config["disable_assertions"] = config["disable_assertions"] + + # enable the logger with the proper detail level + if config["enable_logger"]: + config["logger_level"] = _logging.DEBUG if options.debug else _logging.INFO + _logger.setLevel(config["logger_level"]) + + # log the current configuration + _logger.debug("Configuration: %r", config) test_suite = WorkflowTestSuite(config["galaxy_url"], config["galaxy_api_key"]) test_suite.run_tests(config) if clean_up: From 74660d56578ee9c5802d9345515df2bccb45b970 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 11 Oct 2016 15:32:17 +0200 Subject: [PATCH 007/588] Added support for running workflow tests with the Python unittest framework --- workflow_tester.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index bb72608..12e1ace 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -89,6 +89,17 @@ def run_tests(self, workflow_tests_config): results.append(test_case) return results + def run_test_suite(self, workflow_tests_config): + suite = _unittest.TestSuite() + for test_config in workflow_tests_config["workflows"].values(): + workflow = self.create_test_runner(test_config) + suite.addTest(workflow) + _RUNNER = _unittest.TextTestRunner(verbosity=2) + _RUNNER.run((suite)) + # cleanup + if not workflow_tests_config["disable_cleanup"]: + self.cleanup() + def create_test_runner(self, workflow_test_config): workflow = self._load_work_flow(workflow_test_config["file"]) runner = WorkflowTestRunner(self.galaxy_instance, workflow, workflow_test_config) @@ -124,8 +135,9 @@ def _unload_workflow(self, workflow_id): self._galaxy_workflows.delete_workflow(workflow_id) -class WorkflowTestRunner(): +class WorkflowTestRunner(_unittest.TestCase): def __init__(self, galaxy_instance, galaxy_workflow, workflow_test_config): + self._galaxy_instance = galaxy_instance self._galaxy_workflow = galaxy_workflow self._workflow_test_config = workflow_test_config @@ -367,12 +379,12 @@ def main(clean_up=True): # log the current configuration _logger.debug("Configuration: %r", config) + + # create and run the configured test suite test_suite = WorkflowTestSuite(config["galaxy_url"], config["galaxy_api_key"]) - test_suite.run_tests(config) - if clean_up: - test_suite.cleanup() + test_suite.run_test_suite(config) - return (config, test_suite) + return (test_suite, config) if __name__ == '__main__': From c77609ac59330ecc87a6f592d034151e84130cd6 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 11 Oct 2016 15:35:07 +0200 Subject: [PATCH 008/588] Refactored WorkflowTestResult class to only host test results and workflow info --- workflow_tester.py | 48 +++++++++++++++------------------------------- 1 file changed, 15 insertions(+), 33 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 12e1ace..2d5ff2f 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -215,22 +215,23 @@ def cleanup(self): test_case.cleanup() -class WorkflowTestCase(): - def __init__(self, workflow, test_id, input_map, outputs, expected_output_map, output_history, - missing_tools, output_folder=DEFAULT_OUTPUT_FOLDER): - self.workflow = workflow +class WorkflowTestResult(): + def __init__(self, test_id, workflow, input_map, outputs, output_history, expected_output_map, + missing_tools, results, output_file_map, output_folder=DEFAULT_OUTPUT_FOLDER): self.test_id = test_id + self.workflow = workflow self.inputs = input_map self.outputs = outputs self.output_history = output_history self.expected_output_map = expected_output_map self.output_folder = output_folder self.missing_tools = missing_tools - self.output_file_map = {} - self.results = None + self.output_file_map = output_file_map + self.results = results - if not os.path.isdir(output_folder): - os.makedirs(output_folder) + self.failed_outputs = {out[0]: out[1] + for out in self.results.items() + if not out[1]} def __str__(self): return "Test {0}: workflow {1}, intputs=[{2}], outputs=[{3}]" \ @@ -242,37 +243,18 @@ def __str__(self): def __repr__(self): return self.__str__() + def failed(self): + return len(self.failed_outputs) > 0 + + def passed(self): + return not self.failed() + def check_output(self, output, force=False): - logger.debug("Checking OUTPUT '%s' ...", output.name) - if not self.results or not self.results.has_key(output.name) or force: - output_filename = os.path.join(self.output_folder, "output_" + str(self.outputs.index(output))) - with open(output_filename, "w") as out_file: - output.download(out_file) - self.output_file_map[output.name] = {"dataset": output, "filename": output_filename} - logger.debug("Downloaded output {0}: dataset_id '{1}', filename '{2}'".format(output.name, output.id, - output_filename)) - config = self.expected_output_map[output.name] - comparator = _load_comparator(config["comparator"]) - result = comparator(config["file"], output_filename) - logger.debug("Output '{0}' {1} the expected: dataset '{2}', actual-output '{3}', expected-output '{4}'" - .format(output.name, "is equal to" if result else "differs from", - output.id, output_filename, config["file"])) - self.results[output.name] = result - logger.debug("Checking OUTPUT '%s': DONE", output.name) return self.results[output.name] def check_outputs(self, force=False): - logger.info("Checking test output: ...") - if not self.results or force: - self.results = {} - for output in self.outputs: - self.results[output.name] = self.check_output(output) - logger.info("Checking test output: DONE") return self.results - def clean_up(self): - self.output_history - def load_configuration(filename=DEFAULT_CONFIG_FILENAME): config = {} From 7e8384b1fdb5fee8ab6282ce9c9a766fcf143765 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 11 Oct 2016 15:37:32 +0200 Subject: [PATCH 009/588] Updated references to imported modules --- workflow_tester.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 2d5ff2f..3295e60 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -44,25 +44,25 @@ def __init__(self, galaxy_url=None, galaxy_api_key=None): self._workflows = {} self._workflows_tests = [] self._galaxy_instance = None - self._galaxy_workflows = None + self._galaxy_workflow_client = None # if galaxy_url: self._galaxy_url = galaxy_url - elif os.environ.has_key(ENV_KEY_GALAXY_URL): - self._galaxy_url = os.environ[ENV_KEY_GALAXY_URL] + elif _os.environ.has_key(ENV_KEY_GALAXY_URL): + self._galaxy_url = _os.environ[ENV_KEY_GALAXY_URL] else: raise ValueError("GALAXY URL not defined!!!") # if galaxy_api_key: self._galaxy_api_key = galaxy_api_key - elif os.environ.has_key(ENV_KEY_GALAXY_API_KEY): - self._galaxy_api_key = os.environ[ENV_KEY_GALAXY_API_KEY] + elif _os.environ.has_key(ENV_KEY_GALAXY_API_KEY): + self._galaxy_api_key = _os.environ[ENV_KEY_GALAXY_API_KEY] else: raise ValueError("GALAXY API KEY not defined!!!") # initialize the galaxy instance - self._galaxy_instance = GalaxyInstance(self._galaxy_url, self._galaxy_api_key) - self._galaxy_workflows = WorkflowClient(self._galaxy_instance.gi) + self._galaxy_instance = _GalaxyInstance(self._galaxy_url, self._galaxy_api_key) + self._galaxy_workflow_client = _WorkflowClient(self._galaxy_instance.gi) @property def galaxy_url(self): @@ -107,12 +107,12 @@ def create_test_runner(self, workflow_test_config): return runner def cleanup(self): - logger.debug("Cleaning save histories ...") + _logger.debug("Cleaning save histories ...") hslist = self.galaxy_instance.histories.list() for history in [h for h in hslist if DEFAULT_HISTORY_NAME_PREFIX in h.name]: self.galaxy_instance.histories.delete(history.id) - logger.debug("Cleaning workflow library ...") + _logger.debug("Cleaning workflow library ...") wflist = self.galaxy_instance.workflows.list() workflows = [w for w in wflist if DEFAULT_WORKFLOW_NAME_PREFIX in w.name] for wf in workflows: @@ -120,7 +120,7 @@ def cleanup(self): def _load_work_flow(self, workflow_filename, workflow_name=None): with open(workflow_filename) as f: - wf_json = json.load(f) + wf_json = _json_load(f) if not self._workflows.has_key(wf_json["name"]): wf_name = wf_json["name"] wf_json["name"] = workflow_name if workflow_name else "_".join([DEFAULT_WORKFLOW_NAME_PREFIX, wf_name]) From ee17ccb1a44e79d13acf6f554eedf4cb4e5ef1c1 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 11 Oct 2016 15:40:37 +0200 Subject: [PATCH 010/588] Renamed method for finding missing tools --- workflow_tester.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 3295e60..a5749b7 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -144,19 +144,19 @@ def __init__(self, galaxy_instance, galaxy_workflow, workflow_test_config): self._galaxy_history_client = HistoryClient(galaxy_instance.gi) self._test_cases = {} - def check_required_tools(self): - logger.debug("Checking required tools ...") + def find_missing_tools(self): + _logger.debug("Checking required tools ...") available_tools = self._galaxy_instance.tools.list() missing_tools = [] for order, step in self._galaxy_workflow.steps.items(): if step.tool_id and len( filter(lambda t: t.id == step.tool_id and t.version == step.tool_version, available_tools)) == 0: missing_tools.append((step.tool_id, step.tool_version)) - logger.debug("Missing tools: {0}".format("None" - if len(missing_tools) == 0 - else ", ".join(["{0} (version {1})" - .format(x[0], x[1]) for x in missing_tools]))) - logger.debug("Checking required tools: DONE") + _logger.debug("Missing tools: {0}".format("None" + if len(missing_tools) == 0 + else ", ".join(["{0} (version {1})" + .format(x[0], x[1]) for x in missing_tools]))) + _logger.debug("Checking required tools: DONE") return missing_tools def run_test(self, input_map=None, expected_output_map=None, output_folder=DEFAULT_OUTPUT_FOLDER): From 0cabc0a6a8cd24e235f878eb288bc92be5087e13 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 11 Oct 2016 15:46:59 +0200 Subject: [PATCH 011/588] Fixed generation of the test ID --- workflow_tester.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/workflow_tester.py b/workflow_tester.py index a5749b7..20b2449 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -143,6 +143,10 @@ def __init__(self, galaxy_instance, galaxy_workflow, workflow_test_config): self._workflow_test_config = workflow_test_config self._galaxy_history_client = HistoryClient(galaxy_instance.gi) self._test_cases = {} + def _get_test_uuid(self, update=False): + if not self._test_uuid or update: + self._test_uuid = str(_uuid1()) + return self._test_uuid def find_missing_tools(self): _logger.debug("Checking required tools ...") @@ -159,7 +163,11 @@ def find_missing_tools(self): _logger.debug("Checking required tools: DONE") return missing_tools - def run_test(self, input_map=None, expected_output_map=None, output_folder=DEFAULT_OUTPUT_FOLDER): + def run_test(self, base_path=None, input_map=None, expected_output_map=None, + output_folder=DEFAULT_OUTPUT_FOLDER, assertions=None, cleanup=None): + + # set basepath + base_path = self._base_path if not base_path else base_path # check input_map if not input_map: From 2ce303be5847a68a68732e90103feddd740d61ff Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 11 Oct 2016 15:50:03 +0200 Subject: [PATCH 012/588] Added support for disable-{cleanup,assertions} options --- workflow_tester.py | 99 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 82 insertions(+), 17 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 20b2449..70bfd4d 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -139,10 +139,17 @@ class WorkflowTestRunner(_unittest.TestCase): def __init__(self, galaxy_instance, galaxy_workflow, workflow_test_config): self._galaxy_instance = galaxy_instance - self._galaxy_workflow = galaxy_workflow self._workflow_test_config = workflow_test_config - self._galaxy_history_client = HistoryClient(galaxy_instance.gi) + self._galaxy_workflow = galaxy_workflow + self._galaxy_history_client = _HistoryClient(galaxy_instance.gi) + self._disable_cleanup = workflow_test_config.get("disable_cleanup", False) + self._disable_assertions = workflow_test_config.get("disable_assertions", False) + self._base_path = workflow_test_config.get("base_path", "") self._test_cases = {} + self._test_uuid = None + + setattr(self, "test_" + workflow_test_config["name"], self.run_test) + super(WorkflowTestRunner, self).__init__("test_" + workflow_test_config["name"]) def _get_test_uuid(self, update=False): if not self._test_uuid or update: self._test_uuid = str(_uuid1()) @@ -182,17 +189,24 @@ def run_test(self, base_path=None, input_map=None, expected_output_map=None, else: raise ValueError("No output configured !!!") + # update config options + disable_cleanup = self._disable_cleanup if not cleanup else not cleanup + disable_assertions = self._disable_assertions if not assertions else not assertions + # uuid of the current test - test_uuid = DEFAULT_HISTORY_NAME_PREFIX + str(uuid.uuid1()) + test_uuid = self._get_test_uuid(True) + + # store the current message + error_msg = None # check tools - missing_tools = self.check_required_tools() + missing_tools = self.find_missing_tools() if len(missing_tools) == 0: # create a new history for the current test - history_info = self._galaxy_history_client.create_history(test_uuid) + history_info = self._galaxy_history_client.create_history(DEFAULT_HISTORY_NAME_PREFIX + test_uuid) history = self._galaxy_instance.histories.get(history_info["id"]) - logger.info("Create a history '%s' (id: %r)", history.name, history.id) + _logger.info("Create a history '%s' (id: %r)", history.name, history.id) # upload input data to the current history # and generate the datamap INPUT --> DATASET @@ -203,20 +217,71 @@ def run_test(self, base_path=None, input_map=None, expected_output_map=None, datamap[label].append(history.upload_dataset(filename)) # run the workflow - logger.info("Workflow '%s' (id: %s) running ...", self._galaxy_workflow.name, self._galaxy_workflow.id) + _logger.info("Workflow '%s' (id: %s) running ...", self._galaxy_workflow.name, self._galaxy_workflow.id) outputs, output_history = self._galaxy_workflow.run(datamap, history, wait=True, polling_interval=0.5) - logger.info("Workflow '%s' (id: %s) executed", self._galaxy_workflow.name, self._galaxy_workflow.id) + _logger.info("Workflow '%s' (id: %s) executed", self._galaxy_workflow.name, self._galaxy_workflow.id) + + # check outputs + results, output_file_map = self._check_outputs(base_path, outputs, expected_output_map, output_folder) + + # instantiate the result object + test_result = WorkflowTestResult(test_uuid, self._galaxy_workflow, input_map, outputs, output_history, + expected_output_map, missing_tools, results, output_file_map, + output_folder) + if test_result.failed(): + error_msg = "Some outputs differ from the expected ones: {0}".format( + ", ".join(test_result.failed_outputs)) - # test output ... - test_case = WorkflowTestCase(self._galaxy_workflow, test_uuid, input_map, - outputs, expected_output_map, output_history, missing_tools, output_folder) else: - # test output ... - test_case = WorkflowTestCase(self._galaxy_workflow, test_uuid, input_map, [], - expected_output_map, None, missing_tools, output_folder) - self._test_cases[test_uuid] = test_case - test_case.check_outputs() - return test_case + # instantiate the result object + test_result = WorkflowTestResult(test_uuid, self._galaxy_workflow, input_map, [], None, + expected_output_map, missing_tools, [], {}, output_folder) + error_msg = "Some workflow tools are not available in Galaxy: {0}".format(", ".join(missing_tools)) + + # store + self._test_cases[test_uuid] = test_result + + # cleanup + if not disable_cleanup: + self.cleanup() + + # raise error message + if error_msg: + _logger.error(error_msg) + if not disable_assertions: + raise AssertionError(error_msg) + + return test_result + + def _check_outputs(self, base_path, outputs, expected_output_map, output_folder): + results = {} + output_file_map = {} + + if not _os.path.isdir(output_folder): + _os.makedirs(output_folder) + + _logger.info("Checking test output: ...") + for output in outputs: + _logger.debug("Checking OUTPUT '%s' ...", output.name) + output_filename = _os.path.join(output_folder, "output_" + str(outputs.index(output))) + with open(output_filename, "w") as out_file: + output.download(out_file) + output_file_map[output.name] = {"dataset": output, "filename": output_filename} + _logger.debug( + "Downloaded output {0}: dataset_id '{1}', filename '{2}'".format(output.name, output.id, + output_filename)) + config = expected_output_map[output.name] + comparator = _load_comparator(config["comparator"]) + expected_output_filename = _os.path.join(base_path, config["file"]) + result = comparator(output_filename, expected_output_filename) + _logger.debug( + "Output '{0}' {1} the expected: dataset '{2}', actual-output '{3}', expected-output '{4}'" + .format(output.name, "is equal to" if result else "differs from", + output.id, output_filename, expected_output_filename)) + results[output.name] = result + _logger.debug("Checking OUTPUT '%s': DONE", output.name) + _logger.info("Checking test output: DONE") + return (results, output_file_map) def cleanup(self): for test_id, test_case in self._test_cases.items(): From 8664a0fc98a0036366b801ef3c02bc86fde5fda8 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 11 Oct 2016 16:02:19 +0200 Subject: [PATCH 013/588] Updated workflow configuration: paths of workflows, inputs and expected_outputs are now relative to the path of the workflow test configuration file (i.e., workflows.yml) --- workflow_tester.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 70bfd4d..59b7d33 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -101,7 +101,10 @@ def run_test_suite(self, workflow_tests_config): self.cleanup() def create_test_runner(self, workflow_test_config): - workflow = self._load_work_flow(workflow_test_config["file"]) + workflow_filename = workflow_test_config["file"] \ + if not workflow_test_config.has_key("base_path") \ + else _os.path.join(workflow_test_config["base_path"], workflow_test_config["file"]) + workflow = self._load_work_flow(workflow_filename, workflow_test_config["name"]) runner = WorkflowTestRunner(self.galaxy_instance, workflow, workflow_test_config) self._workflows_tests.append(runner) return runner @@ -214,7 +217,7 @@ def run_test(self, base_path=None, input_map=None, expected_output_map=None, for label, config in input_map.items(): datamap[label] = [] for filename in config["file"]: - datamap[label].append(history.upload_dataset(filename)) + datamap[label].append(history.upload_dataset(_os.path.join(base_path, filename))) # run the workflow _logger.info("Workflow '%s' (id: %s) running ...", self._galaxy_workflow.name, self._galaxy_workflow.id) @@ -331,7 +334,8 @@ def check_outputs(self, force=False): def load_configuration(filename=DEFAULT_CONFIG_FILENAME): config = {} - if os.path.exists(filename): + if _os.path.exists(filename): + base_path = _os.path.dirname(_os.path.abspath(filename)) with open(filename, "r") as config_file: workflows_conf = yaml.load(config_file) config["galaxy_url"] = workflows_conf["galaxy_url"] @@ -345,6 +349,8 @@ def load_configuration(filename=DEFAULT_CONFIG_FILENAME): w["inputs"] = _parse_yaml_list(w["inputs"]) # parse outputs w["outputs"] = _parse_yaml_list(w["outputs"]) + # add base path + w["base_path"] = base_path # add the workflow config["workflows"][w["name"]] = w else: From 2a7bcff30c7ba0b299a281c50c6a0aac581e48e3 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 11 Oct 2016 16:03:36 +0200 Subject: [PATCH 014/588] Updated textual representation of the WorkflowTestRunner class --- workflow_tester.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/workflow_tester.py b/workflow_tester.py index 59b7d33..a61f9f5 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -153,6 +153,17 @@ def __init__(self, galaxy_instance, galaxy_workflow, workflow_test_config): setattr(self, "test_" + workflow_test_config["name"], self.run_test) super(WorkflowTestRunner, self).__init__("test_" + workflow_test_config["name"]) + + def __str__(self): + return "Workflow Test '{0}': testId={1}, workflow='{2}', input=[{3}], output=[{4}]" \ + .format(self._workflow_test_config["name"], + self._get_test_uuid(), + self._workflow_test_config["name"], + ",".join(self._workflow_test_config[ + "inputs"]), + ",".join(self._workflow_test_config[ + "outputs"])) + def _get_test_uuid(self, update=False): if not self._test_uuid or update: self._test_uuid = str(_uuid1()) From 97fce486c4b63e6c27d15cccfa235d8d5955e4fd Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 11 Oct 2016 16:04:57 +0200 Subject: [PATCH 015/588] Minor changes --- workflow_tester.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index a61f9f5..74874bd 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -126,8 +126,8 @@ def _load_work_flow(self, workflow_filename, workflow_name=None): wf_json = _json_load(f) if not self._workflows.has_key(wf_json["name"]): wf_name = wf_json["name"] - wf_json["name"] = workflow_name if workflow_name else "_".join([DEFAULT_WORKFLOW_NAME_PREFIX, wf_name]) - wf_info = self._galaxy_workflows.import_workflow_json(wf_json) + wf_json["name"] = DEFAULT_WORKFLOW_NAME_PREFIX + (workflow_name if workflow_name else wf_name) + wf_info = self._galaxy_workflow_client.import_workflow_json(wf_json) workflow = self.galaxy_instance.workflows.get(wf_info["id"]) self._workflows[wf_name] = workflow else: @@ -135,7 +135,7 @@ def _load_work_flow(self, workflow_filename, workflow_name=None): return workflow def _unload_workflow(self, workflow_id): - self._galaxy_workflows.delete_workflow(workflow_id) + self._galaxy_workflow_client.delete_workflow(workflow_id) class WorkflowTestRunner(_unittest.TestCase): @@ -169,6 +169,14 @@ def _get_test_uuid(self, update=False): self._test_uuid = str(_uuid1()) return self._test_uuid + @property + def worflow_test_name(self): + return self._workflow_test_config["name"] + + @property + def workflow_name(self): + return self._galaxy_workflow.name + def find_missing_tools(self): _logger.debug("Checking required tools ...") available_tools = self._galaxy_instance.tools.list() @@ -298,8 +306,9 @@ def _check_outputs(self, base_path, outputs, expected_output_map, output_folder) return (results, output_file_map) def cleanup(self): - for test_id, test_case in self._test_cases.items(): - test_case.cleanup() + for test_uuid, test_result in self._test_cases.items(): + if test_result.output_history: + self._galaxy_instance.histories.delete(test_result.output_history.id) class WorkflowTestResult(): @@ -348,9 +357,10 @@ def load_configuration(filename=DEFAULT_CONFIG_FILENAME): if _os.path.exists(filename): base_path = _os.path.dirname(_os.path.abspath(filename)) with open(filename, "r") as config_file: - workflows_conf = yaml.load(config_file) + workflows_conf = _yaml_load(config_file) config["galaxy_url"] = workflows_conf["galaxy_url"] config["galaxy_api_key"] = workflows_conf["galaxy_api_key"] + config["enable_logger"] = workflows_conf["enable_logger"] config["workflows"] = {} for workflow in workflows_conf.get("workflows").items(): w = DEFAULT_WORKFLOW_CONFIG.copy() @@ -366,6 +376,7 @@ def load_configuration(filename=DEFAULT_CONFIG_FILENAME): config["workflows"][w["name"]] = w else: config["workflows"] = {"unknown": DEFAULT_WORKFLOW_CONFIG.copy()} + config["output_folder"] = DEFAULT_OUTPUT_FOLDER return config @@ -417,9 +428,9 @@ def _parse_cli_options(): return (options, args) -def main(clean_up=True): +def run_tests(config=None): options, args = _parse_cli_options() - config = load_configuration(options.file) + config = load_configuration(options.file) if not config else config config["galaxy_url"] = options.server \ if options.server \ @@ -460,4 +471,4 @@ def main(clean_up=True): if __name__ == '__main__': - main() + run_tests() From f77e261dd5682ef492a27cdeb0e28c3636fabbc3 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 11 Oct 2016 16:06:15 +0200 Subject: [PATCH 016/588] Added requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..50406b1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +bioblend>=0.8.0 \ No newline at end of file From 6f93b31ed04d6963b0f3edf8db3a8ccd84326060 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 11 Oct 2016 23:20:01 +0200 Subject: [PATCH 017/588] Eliminated the use of 'has_key' --- workflow_tester.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 74874bd..59868b4 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -34,7 +34,7 @@ } } -# configure logger +# configure module logger _logger = _logging.getLogger("WorkflowTest") _logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s') @@ -48,14 +48,14 @@ def __init__(self, galaxy_url=None, galaxy_api_key=None): # if galaxy_url: self._galaxy_url = galaxy_url - elif _os.environ.has_key(ENV_KEY_GALAXY_URL): + elif ENV_KEY_GALAXY_URL in _os.environ: self._galaxy_url = _os.environ[ENV_KEY_GALAXY_URL] else: raise ValueError("GALAXY URL not defined!!!") # if galaxy_api_key: self._galaxy_api_key = galaxy_api_key - elif _os.environ.has_key(ENV_KEY_GALAXY_API_KEY): + elif ENV_KEY_GALAXY_API_KEY in _os.environ: self._galaxy_api_key = _os.environ[ENV_KEY_GALAXY_API_KEY] else: raise ValueError("GALAXY API KEY not defined!!!") @@ -102,7 +102,7 @@ def run_test_suite(self, workflow_tests_config): def create_test_runner(self, workflow_test_config): workflow_filename = workflow_test_config["file"] \ - if not workflow_test_config.has_key("base_path") \ + if not "base_path" in workflow_test_config \ else _os.path.join(workflow_test_config["base_path"], workflow_test_config["file"]) workflow = self._load_work_flow(workflow_filename, workflow_test_config["name"]) runner = WorkflowTestRunner(self.galaxy_instance, workflow, workflow_test_config) @@ -114,7 +114,6 @@ def cleanup(self): hslist = self.galaxy_instance.histories.list() for history in [h for h in hslist if DEFAULT_HISTORY_NAME_PREFIX in h.name]: self.galaxy_instance.histories.delete(history.id) - _logger.debug("Cleaning workflow library ...") wflist = self.galaxy_instance.workflows.list() workflows = [w for w in wflist if DEFAULT_WORKFLOW_NAME_PREFIX in w.name] @@ -124,7 +123,7 @@ def cleanup(self): def _load_work_flow(self, workflow_filename, workflow_name=None): with open(workflow_filename) as f: wf_json = _json_load(f) - if not self._workflows.has_key(wf_json["name"]): + if not wf_json["name"] in self._workflows: wf_name = wf_json["name"] wf_json["name"] = DEFAULT_WORKFLOW_NAME_PREFIX + (workflow_name if workflow_name else wf_name) wf_info = self._galaxy_workflow_client.import_workflow_json(wf_json) @@ -200,13 +199,13 @@ def run_test(self, base_path=None, input_map=None, expected_output_map=None, # check input_map if not input_map: - if self._workflow_test_config.has_key("inputs"): + if "inputs" in self._workflow_test_config: input_map = self._workflow_test_config["inputs"] else: raise ValueError("No input configured !!!") # check expected_output_map if not expected_output_map: - if self._workflow_test_config.has_key("outputs"): + if "outputs" in self._workflow_test_config: expected_output_map = self._workflow_test_config["outputs"] else: raise ValueError("No output configured !!!") @@ -382,7 +381,7 @@ def load_configuration(filename=DEFAULT_CONFIG_FILENAME): def load_workflow_test_configuration(workflow_test_name, filename=DEFAULT_CONFIG_FILENAME): config = load_configuration(filename) - if config["workflows"].has_key(workflow_test_name): + if workflow_test_name in config["workflows"]: return config["workflows"][workflow_test_name] else: raise KeyError("WorkflowTest with name '%s' not found" % workflow_test_name) @@ -434,19 +433,19 @@ def run_tests(config=None): config["galaxy_url"] = options.server \ if options.server \ - else config["galaxy_url"] if config.has_key("galaxy_url") else None + else config["galaxy_url"] if "galaxy_url" in config else None config["galaxy_api_key"] = options.api_key \ if options.api_key \ - else config["galaxy_api_key"] if config.has_key("galaxy_api_key") else None + else config["galaxy_api_key"] if "galaxy_api_key" in config else None config["output_folder"] = options.output \ if options.output \ - else config["output_folder"] if config.has_key("output_folder") else DEFAULT_OUTPUT_FOLDER + else config["output_folder"] if "output_folder" in config else DEFAULT_OUTPUT_FOLDER config["enable_logger"] = options.enable_logger \ if options.enable_logger \ - else config["enable_logger"] if config.has_key("enable_logger") else False + else config["enable_logger"] if "enable_logger" in config else False config["disable_cleanup"] = options.disable_cleanup config["disable_assertions"] = options.disable_assertions From 6a853973c582526f9b7fc012fe0c3818bd19a286 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 12 Oct 2016 16:14:11 +0200 Subject: [PATCH 018/588] Added new WorkflowTestConfiguration responsible for loading the configuration of workflow tests --- workflow_tester.py | 102 +++++++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 50 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 59868b4..b73ecec 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -18,27 +18,63 @@ ENV_KEY_GALAXY_URL = "BIOBLEND_GALAXY_URL" ENV_KEY_GALAXY_API_KEY = "BIOBLEND_GALAXY_API_KEY" -# Default settings -DEFAULT_HISTORY_NAME_PREFIX = "_WorkflowTestHistory_" -DEFAULT_WORKFLOW_NAME_PREFIX = "_WorkflowTest_" -DEFAULT_OUTPUT_FOLDER = "results" -DEFAULT_CONFIG_FILENAME = "workflows.yml" -DEFAULT_WORKFLOW_CONFIG = { - "file": "workflow.ga", - "inputs": { - "Input Dataset": {"name": "Input Dataset", "file": ["input"]} - }, - "outputs": { - "output1": {"file": "expected_output", "comparator": "filecmp.cmp", "name": "output1"}, - "output2": {"file": "expected_output", "comparator": "filecmp.cmp", "name": "output2"} - } -} - # configure module logger _logger = _logging.getLogger("WorkflowTest") _logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s') +class WorkflowTestConfiguration: + # Default settings + DEFAULT_HISTORY_NAME_PREFIX = "_WorkflowTestHistory_" + DEFAULT_WORKFLOW_NAME_PREFIX = "_WorkflowTest_" + DEFAULT_OUTPUT_FOLDER = "results" + DEFAULT_CONFIG_FILENAME = "workflows.yml" + DEFAULT_WORKFLOW_CONFIG = { + "file": "workflow.ga", + "inputs": { + "Input Dataset": {"name": "Input Dataset", "file": ["input"]} + }, + "outputs": { + "output1": {"file": "expected_output", "comparator": "filecmp.cmp", "name": "output1"}, + "output2": {"file": "expected_output", "comparator": "filecmp.cmp", "name": "output2"} + } + } + + @staticmethod + def load(filename=DEFAULT_CONFIG_FILENAME, workflow_test_name=None): + config = {} + if _os.path.exists(filename): + base_path = _os.path.dirname(_os.path.abspath(filename)) + with open(filename, "r") as config_file: + workflows_conf = _yaml_load(config_file) + config["galaxy_url"] = workflows_conf["galaxy_url"] + config["galaxy_api_key"] = workflows_conf["galaxy_api_key"] + config["enable_logger"] = workflows_conf["enable_logger"] + config["workflows"] = {} + for workflow in workflows_conf.get("workflows").items(): + w = WorkflowTestConfiguration.DEFAULT_WORKFLOW_CONFIG.copy() + w["name"] = workflow[0] + w.update(workflow[1]) + # parse inputs + w["inputs"] = _parse_yaml_list(w["inputs"]) + # parse outputs + w["outputs"] = _parse_yaml_list(w["outputs"]) + # add base path + w["base_path"] = base_path + # add the workflow + config["workflows"][w["name"]] = w + # returns the current workflow test config + # if its name matches the 'workflow_test_name' param + if workflow_test_name and w["name"] == workflow_test_name: + return w + # raise an exception if the workflow test we are searching for + # cannot be found within the configuration file. + if workflow_test_name: + raise KeyError("WorkflowTest with name '%s' not found" % workflow_test_name) + else: + config["workflows"] = {"unknown": WorkflowTestConfiguration.DEFAULT_WORKFLOW_CONFIG.copy()} + config["output_folder"] = WorkflowTestConfiguration.DEFAULT_OUTPUT_FOLDER + return config class WorkflowTestSuite(): def __init__(self, galaxy_url=None, galaxy_api_key=None): self._workflows = {} @@ -351,40 +387,6 @@ def check_outputs(self, force=False): return self.results -def load_configuration(filename=DEFAULT_CONFIG_FILENAME): - config = {} - if _os.path.exists(filename): - base_path = _os.path.dirname(_os.path.abspath(filename)) - with open(filename, "r") as config_file: - workflows_conf = _yaml_load(config_file) - config["galaxy_url"] = workflows_conf["galaxy_url"] - config["galaxy_api_key"] = workflows_conf["galaxy_api_key"] - config["enable_logger"] = workflows_conf["enable_logger"] - config["workflows"] = {} - for workflow in workflows_conf.get("workflows").items(): - w = DEFAULT_WORKFLOW_CONFIG.copy() - w["name"] = workflow[0] - w.update(workflow[1]) - # parse inputs - w["inputs"] = _parse_yaml_list(w["inputs"]) - # parse outputs - w["outputs"] = _parse_yaml_list(w["outputs"]) - # add base path - w["base_path"] = base_path - # add the workflow - config["workflows"][w["name"]] = w - else: - config["workflows"] = {"unknown": DEFAULT_WORKFLOW_CONFIG.copy()} - config["output_folder"] = DEFAULT_OUTPUT_FOLDER - return config - - -def load_workflow_test_configuration(workflow_test_name, filename=DEFAULT_CONFIG_FILENAME): - config = load_configuration(filename) - if workflow_test_name in config["workflows"]: - return config["workflows"][workflow_test_name] - else: - raise KeyError("WorkflowTest with name '%s' not found" % workflow_test_name) def _parse_yaml_list(ylist): From 661c65985dabd47bf62f72b3af12c63969c29165 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 12 Oct 2016 16:18:30 +0200 Subject: [PATCH 019/588] Added new class WorkflowTestLoader responsible for loading/unloading workflow to/from Galaxy --- workflow_tester.py | 84 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 69 insertions(+), 15 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index b73ecec..2492ce0 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -75,6 +75,62 @@ def load(filename=DEFAULT_CONFIG_FILENAME, workflow_test_name=None): config["workflows"] = {"unknown": WorkflowTestConfiguration.DEFAULT_WORKFLOW_CONFIG.copy()} config["output_folder"] = WorkflowTestConfiguration.DEFAULT_OUTPUT_FOLDER return config + + +class WorkflowLoader: + _instance = None + + @staticmethod + def get_instance(): + if not WorkflowLoader._instance: + WorkflowLoader._instance = WorkflowLoader() + return WorkflowLoader._instance + + def __init__(self, galaxy_instance=None): + self._galaxy_instance = galaxy_instance + self._galaxy_workflow_client = None + self._workflows = {} + # if galaxy_instance exists, complete initialization + if galaxy_instance: + self.initialize() + + def initialize(self, galaxy_url=None, galaxy_api_key=None): + if not self._galaxy_instance: + # initialize the galaxy instance + self._galaxy_instance = _get_galaxy_instance(galaxy_url, galaxy_api_key) + # initialize the workflow client + self._galaxy_workflow_client = _WorkflowClient(self._galaxy_instance.gi) + + def load_workflow(self, workflow_test_config, workflow_name=None): + workflow_filename = workflow_test_config["file"] \ + if not "base_path" in workflow_test_config \ + else _os.path.join(workflow_test_config["base_path"], workflow_test_config["file"]) + return self.load_workflow_by_filename(workflow_filename, workflow_name) + + def load_workflow_by_filename(self, workflow_filename, workflow_name=None): + with open(workflow_filename) as f: + wf_json = _json_load(f) + # TODO: register workflow by ID (equal to UUID?) + if not wf_json["name"] in self._workflows: + wf_name = wf_json["name"] + wf_json["name"] = WorkflowTestConfiguration.DEFAULT_WORKFLOW_NAME_PREFIX \ + + (workflow_name if workflow_name else wf_name) + wf_info = self._galaxy_workflow_client.import_workflow_json(wf_json) + workflow = self._galaxy_instance.workflows.get(wf_info["id"]) + self._workflows[wf_name] = workflow + else: + workflow = self._workflows[wf_json["name"]] + return workflow + + def unload_workflow(self, workflow_id): + self._galaxy_workflow_client.delete_workflow(workflow_id) + # TODO: remove workflow from the list + + def unload_workflows(self): + for wf_name, wf in self._workflows.items(): + self.unload_workflow(wf[id]) + + class WorkflowTestSuite(): def __init__(self, galaxy_url=None, galaxy_api_key=None): self._workflows = {} @@ -107,6 +163,8 @@ def galaxy_url(self): @property def galaxy_api_key(self): return self._galaxy_api_key + # initialize the workflow loader + self._workflow_loader = WorkflowLoader(self._galaxy_instance) @property def galaxy_instance(self): @@ -114,6 +172,9 @@ def galaxy_instance(self): @property def workflows(self): + def workflow_loader(self): + return self._workflow_loader + return self.galaxy_instance.workflows.list() def run_tests(self, workflow_tests_config): @@ -156,27 +217,13 @@ def cleanup(self): for wf in workflows: self._unload_workflow(wf.id) - def _load_work_flow(self, workflow_filename, workflow_name=None): - with open(workflow_filename) as f: - wf_json = _json_load(f) - if not wf_json["name"] in self._workflows: - wf_name = wf_json["name"] - wf_json["name"] = DEFAULT_WORKFLOW_NAME_PREFIX + (workflow_name if workflow_name else wf_name) - wf_info = self._galaxy_workflow_client.import_workflow_json(wf_json) - workflow = self.galaxy_instance.workflows.get(wf_info["id"]) - self._workflows[wf_name] = workflow - else: - workflow = self._workflows[wf_json["name"]] - return workflow - - def _unload_workflow(self, workflow_id): - self._galaxy_workflow_client.delete_workflow(workflow_id) class WorkflowTestRunner(_unittest.TestCase): def __init__(self, galaxy_instance, galaxy_workflow, workflow_test_config): self._galaxy_instance = galaxy_instance + self._workflow_loader = workflow_loader self._workflow_test_config = workflow_test_config self._galaxy_workflow = galaxy_workflow self._galaxy_history_client = _HistoryClient(galaxy_instance.gi) @@ -226,6 +273,10 @@ def find_missing_tools(self): .format(x[0], x[1]) for x in missing_tools]))) _logger.debug("Checking required tools: DONE") return missing_tools + def get_galaxy_workflow(self): + if not self._galaxy_workflow: + self._galaxy_workflow = self._workflow_loader.load_workflow(self._workflow_test_config) + return self._galaxy_workflow def run_test(self, base_path=None, input_map=None, expected_output_map=None, output_folder=DEFAULT_OUTPUT_FOLDER, assertions=None, cleanup=None): @@ -344,6 +395,9 @@ def cleanup(self): for test_uuid, test_result in self._test_cases.items(): if test_result.output_history: self._galaxy_instance.histories.delete(test_result.output_history.id) + if self._galaxy_workflow: + self._workflow_loader.unload_workflow(self._galaxy_workflow.id) + self._galaxy_workflow = None class WorkflowTestResult(): From 225a1cae705b0f1dbb53fd403aacc0c5830a98cc Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 12 Oct 2016 16:22:11 +0200 Subject: [PATCH 020/588] Wrapped GalaxyInstance configuration within a utility function --- workflow_tester.py | 39 +++++++++++++++------------------------ 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 2492ce0..dad42b8 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -137,32 +137,9 @@ def __init__(self, galaxy_url=None, galaxy_api_key=None): self._workflows_tests = [] self._galaxy_instance = None self._galaxy_workflow_client = None - # - if galaxy_url: - self._galaxy_url = galaxy_url - elif ENV_KEY_GALAXY_URL in _os.environ: - self._galaxy_url = _os.environ[ENV_KEY_GALAXY_URL] - else: - raise ValueError("GALAXY URL not defined!!!") - # - if galaxy_api_key: - self._galaxy_api_key = galaxy_api_key - elif ENV_KEY_GALAXY_API_KEY in _os.environ: - self._galaxy_api_key = _os.environ[ENV_KEY_GALAXY_API_KEY] - else: - raise ValueError("GALAXY API KEY not defined!!!") # initialize the galaxy instance - self._galaxy_instance = _GalaxyInstance(self._galaxy_url, self._galaxy_api_key) - self._galaxy_workflow_client = _WorkflowClient(self._galaxy_instance.gi) - - @property - def galaxy_url(self): - return self._galaxy_url - - @property - def galaxy_api_key(self): - return self._galaxy_api_key + self._galaxy_instance = _get_galaxy_instance(galaxy_url, galaxy_api_key) # initialize the workflow loader self._workflow_loader = WorkflowLoader(self._galaxy_instance) @@ -441,6 +418,20 @@ def check_outputs(self, force=False): return self.results +def _get_galaxy_instance(galaxy_url=None, galaxy_api_key=None): + if not galaxy_url: + if ENV_KEY_GALAXY_URL in _os.environ: + galaxy_url = _os.environ[ENV_KEY_GALAXY_URL] + else: + raise ValueError("GALAXY URL not defined!!!") + # set the galaxy api key + if not galaxy_api_key: + if ENV_KEY_GALAXY_API_KEY in _os.environ: + galaxy_api_key = _os.environ[ENV_KEY_GALAXY_API_KEY] + else: + raise ValueError("GALAXY API KEY not defined!!!") + # initialize the galaxy instance + return _GalaxyInstance(galaxy_url, galaxy_api_key) def _parse_yaml_list(ylist): From 4070ae5de698ead8608105a67e2f4851840b1f21 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 12 Oct 2016 16:28:24 +0200 Subject: [PATCH 021/588] Added support for storing test results within the WorkflowTestSuite object --- workflow_tester.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index dad42b8..7c5bc66 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -134,7 +134,8 @@ def unload_workflows(self): class WorkflowTestSuite(): def __init__(self, galaxy_url=None, galaxy_api_key=None): self._workflows = {} - self._workflows_tests = [] + self._workflow_runners = [] + self._workflow_test_results = [] self._galaxy_instance = None self._galaxy_workflow_client = None @@ -154,6 +155,12 @@ def workflow_loader(self): return self.galaxy_instance.workflows.list() + def get_workflow_test_results(self, workflow_id=None): + return list([w for w in self._workflow_test_results if w.id == workflow_id] if workflow_id + else self._workflow_test_results) + + def _add_test_result(self, test_result): + self._workflow_test_results.append(test_result) def run_tests(self, workflow_tests_config): results = [] for test_config in workflow_tests_config["workflows"].values(): @@ -197,12 +204,11 @@ def cleanup(self): class WorkflowTestRunner(_unittest.TestCase): - def __init__(self, galaxy_instance, galaxy_workflow, workflow_test_config): - + def __init__(self, galaxy_instance, workflow_loader, workflow_test_config, test_suite=None): self._galaxy_instance = galaxy_instance self._workflow_loader = workflow_loader self._workflow_test_config = workflow_test_config - self._galaxy_workflow = galaxy_workflow + self._test_suite = test_suite self._galaxy_history_client = _HistoryClient(galaxy_instance.gi) self._disable_cleanup = workflow_test_config.get("disable_cleanup", False) self._disable_assertions = workflow_test_config.get("disable_assertions", False) @@ -323,8 +329,10 @@ def run_test(self, base_path=None, input_map=None, expected_output_map=None, expected_output_map, missing_tools, [], {}, output_folder) error_msg = "Some workflow tools are not available in Galaxy: {0}".format(", ".join(missing_tools)) - # store + # store result self._test_cases[test_uuid] = test_result + if self._test_suite: + self._test_suite._add_test_result(test_result) # cleanup if not disable_cleanup: From 6456fb8433d0b510d9f08bf5214979ade25e54bf Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 12 Oct 2016 16:30:31 +0200 Subject: [PATCH 022/588] Updated 'run_tests' function to support new configuration parameters (i.e., debug, cleanup, assertions) --- workflow_tester.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 7c5bc66..9904403 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -482,9 +482,9 @@ def _parse_cli_options(): return (options, args) -def run_tests(config=None): +def run_tests(config=None, debug=None, cleanup=None, assertions=None): options, args = _parse_cli_options() - config = load_configuration(options.file) if not config else config + config = WorkflowTestConfiguration.load(options.file) if not config else config config["galaxy_url"] = options.server \ if options.server \ @@ -496,14 +496,12 @@ def run_tests(config=None): config["output_folder"] = options.output \ if options.output \ - else config["output_folder"] if "output_folder" in config else DEFAULT_OUTPUT_FOLDER + else config["output_folder"] if "output_folder" in config else WorkflowTestConfiguration.DEFAULT_OUTPUT_FOLDER - config["enable_logger"] = options.enable_logger \ - if options.enable_logger \ - else config["enable_logger"] if "enable_logger" in config else False - - config["disable_cleanup"] = options.disable_cleanup - config["disable_assertions"] = options.disable_assertions + config["enable_logger"] = True if options.enable_logger else config.get("enable_logger", False) + config["debug"] = options.debug if not debug else debug + config["disable_cleanup"] = options.disable_cleanup if not cleanup else cleanup + config["disable_assertions"] = options.disable_assertions if not assertions else assertions for test_config in config["workflows"].values(): test_config["disable_cleanup"] = config["disable_cleanup"] @@ -511,11 +509,11 @@ def run_tests(config=None): # enable the logger with the proper detail level if config["enable_logger"]: - config["logger_level"] = _logging.DEBUG if options.debug else _logging.INFO + config["logger_level"] = _logging.DEBUG if debug or options.debug else _logging.INFO _logger.setLevel(config["logger_level"]) # log the current configuration - _logger.debug("Configuration: %r", config) + _logger.info("Configuration: %r", config) # create and run the configured test suite test_suite = WorkflowTestSuite(config["galaxy_url"], config["galaxy_api_key"]) From cc784d168f53cacb9c84499e318cc855e5089623 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 12 Oct 2016 16:33:10 +0200 Subject: [PATCH 023/588] Added factory method to instantiate WorkflowTestRunner objects not associated with any suite --- workflow_tester.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/workflow_tester.py b/workflow_tester.py index 9904403..68ee90d 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -219,6 +219,13 @@ def __init__(self, galaxy_instance, workflow_loader, workflow_test_config, test_ setattr(self, "test_" + workflow_test_config["name"], self.run_test) super(WorkflowTestRunner, self).__init__("test_" + workflow_test_config["name"]) + @staticmethod + def new_instance(workflow_test_config, galaxy_url=None, galaxy_api_key=None): + # initialize the galaxy instance + galaxy_instance = _get_galaxy_instance(galaxy_url, galaxy_api_key) + workflow_loader = WorkflowLoader(galaxy_instance) + # return the runner instance + return WorkflowTestRunner(galaxy_instance, workflow_loader, workflow_test_config) def __str__(self): return "Workflow Test '{0}': testId={1}, workflow='{2}', input=[{3}], output=[{4}]" \ .format(self._workflow_test_config["name"], From 6e8347785154fe7b786fe4fb8b6c2801450bdd1f Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 12 Oct 2016 16:36:44 +0200 Subject: [PATCH 024/588] Minor code refactoring --- workflow_tester.py | 114 +++++++++++++++++++++++---------------------- 1 file changed, 58 insertions(+), 56 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 68ee90d..81110db 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -149,10 +149,10 @@ def galaxy_instance(self): return self._galaxy_instance @property - def workflows(self): def workflow_loader(self): return self._workflow_loader + def get_workflows(self): return self.galaxy_instance.workflows.list() def get_workflow_test_results(self, workflow_id=None): @@ -161,46 +161,42 @@ def get_workflow_test_results(self, workflow_id=None): def _add_test_result(self, test_result): self._workflow_test_results.append(test_result) + + def _create_test_runner(self, workflow_test_config): + runner = WorkflowTestRunner(self.galaxy_instance, self.workflow_loader, workflow_test_config, self) + self._workflow_runners.append(runner) + return runner + def run_tests(self, workflow_tests_config): results = [] for test_config in workflow_tests_config["workflows"].values(): - workflow = self.create_test_runner(test_config) - test_case = workflow.run_test(test_config["inputs"], test_config["outputs"], - workflow_tests_config["output_folder"]) - results.append(test_case) + runner = self._create_test_runner(test_config) + result = runner.run_test(test_config["inputs"], test_config["outputs"], + workflow_tests_config["output_folder"]) + results.append(result) return results def run_test_suite(self, workflow_tests_config): suite = _unittest.TestSuite() for test_config in workflow_tests_config["workflows"].values(): - workflow = self.create_test_runner(test_config) - suite.addTest(workflow) + runner = self._create_test_runner(test_config) + suite.addTest(runner) _RUNNER = _unittest.TextTestRunner(verbosity=2) _RUNNER.run((suite)) # cleanup if not workflow_tests_config["disable_cleanup"]: self.cleanup() - def create_test_runner(self, workflow_test_config): - workflow_filename = workflow_test_config["file"] \ - if not "base_path" in workflow_test_config \ - else _os.path.join(workflow_test_config["base_path"], workflow_test_config["file"]) - workflow = self._load_work_flow(workflow_filename, workflow_test_config["name"]) - runner = WorkflowTestRunner(self.galaxy_instance, workflow, workflow_test_config) - self._workflows_tests.append(runner) - return runner - def cleanup(self): _logger.debug("Cleaning save histories ...") hslist = self.galaxy_instance.histories.list() - for history in [h for h in hslist if DEFAULT_HISTORY_NAME_PREFIX in h.name]: + for history in [h for h in hslist if WorkflowTestConfiguration.DEFAULT_HISTORY_NAME_PREFIX in h.name]: self.galaxy_instance.histories.delete(history.id) _logger.debug("Cleaning workflow library ...") wflist = self.galaxy_instance.workflows.list() - workflows = [w for w in wflist if DEFAULT_WORKFLOW_NAME_PREFIX in w.name] + workflows = [w for w in wflist if WorkflowTestConfiguration.DEFAULT_WORKFLOW_NAME_PREFIX in w.name] for wf in workflows: - self._unload_workflow(wf.id) - + self._workflow_loader.unload_workflow(wf.id) class WorkflowTestRunner(_unittest.TestCase): @@ -215,6 +211,7 @@ def __init__(self, galaxy_instance, workflow_loader, workflow_test_config, test_ self._base_path = workflow_test_config.get("base_path", "") self._test_cases = {} self._test_uuid = None + self._galaxy_workflow = None setattr(self, "test_" + workflow_test_config["name"], self.run_test) super(WorkflowTestRunner, self).__init__("test_" + workflow_test_config["name"]) @@ -226,6 +223,11 @@ def new_instance(workflow_test_config, galaxy_url=None, galaxy_api_key=None): workflow_loader = WorkflowLoader(galaxy_instance) # return the runner instance return WorkflowTestRunner(galaxy_instance, workflow_loader, workflow_test_config) + + @property + def worflow_test_name(self): + return self._workflow_test_config["name"] + def __str__(self): return "Workflow Test '{0}': testId={1}, workflow='{2}', input=[{3}], output=[{4}]" \ .format(self._workflow_test_config["name"], @@ -241,39 +243,20 @@ def _get_test_uuid(self, update=False): self._test_uuid = str(_uuid1()) return self._test_uuid - @property - def worflow_test_name(self): - return self._workflow_test_config["name"] - - @property - def workflow_name(self): - return self._galaxy_workflow.name - - def find_missing_tools(self): - _logger.debug("Checking required tools ...") - available_tools = self._galaxy_instance.tools.list() - missing_tools = [] - for order, step in self._galaxy_workflow.steps.items(): - if step.tool_id and len( - filter(lambda t: t.id == step.tool_id and t.version == step.tool_version, available_tools)) == 0: - missing_tools.append((step.tool_id, step.tool_version)) - _logger.debug("Missing tools: {0}".format("None" - if len(missing_tools) == 0 - else ", ".join(["{0} (version {1})" - .format(x[0], x[1]) for x in missing_tools]))) - _logger.debug("Checking required tools: DONE") - return missing_tools def get_galaxy_workflow(self): if not self._galaxy_workflow: self._galaxy_workflow = self._workflow_loader.load_workflow(self._workflow_test_config) return self._galaxy_workflow def run_test(self, base_path=None, input_map=None, expected_output_map=None, - output_folder=DEFAULT_OUTPUT_FOLDER, assertions=None, cleanup=None): + output_folder=WorkflowTestConfiguration.DEFAULT_OUTPUT_FOLDER, assertions=None, cleanup=None): # set basepath base_path = self._base_path if not base_path else base_path + # load workflow + workflow = self.get_galaxy_workflow() + # check input_map if not input_map: if "inputs" in self._workflow_test_config: @@ -302,7 +285,8 @@ def run_test(self, base_path=None, input_map=None, expected_output_map=None, if len(missing_tools) == 0: # create a new history for the current test - history_info = self._galaxy_history_client.create_history(DEFAULT_HISTORY_NAME_PREFIX + test_uuid) + history_info = self._galaxy_history_client.create_history( + WorkflowTestConfiguration.DEFAULT_HISTORY_NAME_PREFIX + test_uuid) history = self._galaxy_instance.histories.get(history_info["id"]) _logger.info("Create a history '%s' (id: %r)", history.name, history.id) @@ -315,25 +299,25 @@ def run_test(self, base_path=None, input_map=None, expected_output_map=None, datamap[label].append(history.upload_dataset(_os.path.join(base_path, filename))) # run the workflow - _logger.info("Workflow '%s' (id: %s) running ...", self._galaxy_workflow.name, self._galaxy_workflow.id) - outputs, output_history = self._galaxy_workflow.run(datamap, history, wait=True, polling_interval=0.5) - _logger.info("Workflow '%s' (id: %s) executed", self._galaxy_workflow.name, self._galaxy_workflow.id) + _logger.info("Workflow '%s' (id: %s) running ...", workflow.name, workflow.id) + outputs, output_history = workflow.run(datamap, history, wait=True, polling_interval=0.5) + _logger.info("Workflow '%s' (id: %s) executed", workflow.name, workflow.id) # check outputs results, output_file_map = self._check_outputs(base_path, outputs, expected_output_map, output_folder) # instantiate the result object - test_result = WorkflowTestResult(test_uuid, self._galaxy_workflow, input_map, outputs, output_history, - expected_output_map, missing_tools, results, output_file_map, - output_folder) + test_result = _WorkflowTestResult(test_uuid, workflow, input_map, outputs, output_history, + expected_output_map, missing_tools, results, output_file_map, + output_folder) if test_result.failed(): - error_msg = "Some outputs differ from the expected ones: {0}".format( + error_msg = "The following outputs differ from the expected ones: {0}".format( ", ".join(test_result.failed_outputs)) else: # instantiate the result object - test_result = WorkflowTestResult(test_uuid, self._galaxy_workflow, input_map, [], None, - expected_output_map, missing_tools, [], {}, output_folder) + test_result = _WorkflowTestResult(test_uuid, workflow, input_map, [], None, + expected_output_map, missing_tools, [], {}, output_folder) error_msg = "Some workflow tools are not available in Galaxy: {0}".format(", ".join(missing_tools)) # store result @@ -353,6 +337,22 @@ def run_test(self, base_path=None, input_map=None, expected_output_map=None, return test_result + def find_missing_tools(self, workflow=None): + _logger.debug("Checking required tools ...") + workflow = self.get_galaxy_workflow() if not workflow else workflow + available_tools = self._galaxy_instance.tools.list() + missing_tools = [] + for order, step in workflow.steps.items(): + if step.tool_id and len( + filter(lambda t: t.id == step.tool_id and t.version == step.tool_version, available_tools)) == 0: + missing_tools.append((step.tool_id, step.tool_version)) + _logger.debug("Missing tools: {0}".format("None" + if len(missing_tools) == 0 + else ", ".join(["{0} (version {1})" + .format(x[0], x[1]) for x in missing_tools]))) + _logger.debug("Checking required tools: DONE") + return missing_tools + def _check_outputs(self, base_path, outputs, expected_output_map, output_folder): results = {} output_file_map = {} @@ -392,9 +392,10 @@ def cleanup(self): self._galaxy_workflow = None -class WorkflowTestResult(): +class _WorkflowTestResult(): def __init__(self, test_id, workflow, input_map, outputs, output_history, expected_output_map, - missing_tools, results, output_file_map, output_folder=DEFAULT_OUTPUT_FOLDER): + missing_tools, results, output_file_map, + output_folder=WorkflowTestConfiguration.DEFAULT_OUTPUT_FOLDER): self.test_id = test_id self.workflow = workflow self.inputs = input_map @@ -484,7 +485,8 @@ def _parse_cli_options(): parser.add_option('--disable-cleanup', help='Disable cleanup', action='store_false') parser.add_option('--disable-assertions', help='Disable assertions', action='store_false') parser.add_option('-o', '--output', help='absolute path of the folder to download workflow outputs') - parser.add_option('-f', '--file', default=DEFAULT_CONFIG_FILENAME, help='YAML configuration file of workflow tests') + parser.add_option('-f', '--file', default=WorkflowTestConfiguration.DEFAULT_CONFIG_FILENAME, + help='YAML configuration file of workflow tests') (options, args) = parser.parse_args() return (options, args) From 5e6858cd3349a020a194edcc7efed62c15d4b4a3 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 12 Oct 2016 16:50:55 +0200 Subject: [PATCH 025/588] Updated 'check_output' method to support the output label as parameter --- workflow_tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow_tester.py b/workflow_tester.py index 81110db..8819d6d 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -428,7 +428,7 @@ def passed(self): return not self.failed() def check_output(self, output, force=False): - return self.results[output.name] + return self.results[output if isinstance(output, str) else output.name] def check_outputs(self, force=False): return self.results From 22d34491747c0afa65a8c09f0f014b8f94e88622 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 13 Oct 2016 09:52:49 +0200 Subject: [PATCH 026/588] Updated imports --- workflow_tester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 8819d6d..87c5167 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -6,9 +6,9 @@ import unittest as _unittest import optparse as _optparse -from json import load as _json_load -from yaml import load as _yaml_load from uuid import uuid1 as _uuid1 +from yaml import load as _yaml_load, dump as _yaml_dump +from json import load as _json_load, dump as _json_dump from bioblend.galaxy.objects import GalaxyInstance as _GalaxyInstance from bioblend.galaxy.workflows import WorkflowClient as _WorkflowClient From 1c7ecf2f2fa286a07b9e2c0f54f99f2c976dfd90 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 13 Oct 2016 09:54:44 +0200 Subject: [PATCH 027/588] Added new 'WorkflowTestConfiguration' class to programmatically handle the configuration of the workflow test. --- workflow_tester.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/workflow_tester.py b/workflow_tester.py index 87c5167..0d13d61 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -40,6 +40,93 @@ class WorkflowTestConfiguration: } } + def __init__(self, base_path=".", filename="workflow.ga", name=None, inputs={}, expected_outputs={}, + cleanup=True, assertions=True): + # init properties + self._base_path = None + self._filename = None + self._inputs = {} + self._expected_outputs = {} + + # set parameters + self.name = name + self.set_base_path(base_path) + self.set_filename(filename) + self.set_inputs(inputs) + self.set_expected_outputs(expected_outputs) + self.disable_cleanup = not cleanup + self.disable_assertions = not assertions + + def __str__(self): + return "WorkflowTestConfig: name={0}, file={1}, inputs=[{2}], expected_outputs=[{3}]".format( + self.name, self.filename, ",".join(self.inputs.keys()), ",".join(self.expected_outputs.keys())) + + def __repr__(self): + return self.__str__() + + @property + def base_path(self): + return self._base_path + + def set_base_path(self, base_path): + self._base_path = base_path + + @property + def filename(self): + return self._filename + + def set_filename(self, filename): + self._filename = filename + + @property + def inputs(self): + return self._inputs + + def set_inputs(self, inputs): + print inputs + for name, config in inputs.items(): + self.add_input(name, config["file"]) + + def add_input(self, name, file): + if not name: + raise ValueError("Input name not defined") + self._inputs[name] = {"name": name, "file": file if isinstance(file, list) else [file]} + + def remove_input(self, name): + if name in self._inputs: + del self._inputs[name] + + def get_input(self, name): + return self._inputs.get(name, None) + + @property + def expected_outputs(self): + return self._expected_outputs + + def set_expected_outputs(self, expected_outputs): + for name, config in expected_outputs.items(): + self.add_expected_output(name, config["file"], config["comparator"]) + + def add_expected_output(self, name, filename, comparator="filecmp.cmp"): + if not name: + raise ValueError("Input name not defined") + self._expected_outputs[name] = {"name": name, "file": filename, "comparator": comparator} + + def remove_expected_output(self, name): + if name in self._expected_outputs: + del self._expected_outputs[name] + + def get_expected_output(self, name): + return self._expected_outputs.get(name, None) + + def to_json(self): + return dict({ + "name": self.name, + "file": self.filename, + "inputs": self.inputs, + "outputs": self.expected_outputs + }) + @staticmethod def load(filename=DEFAULT_CONFIG_FILENAME, workflow_test_name=None): config = {} From 638d06653275747c3b56b6bfbac15ffaeaf81720 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 13 Oct 2016 09:57:18 +0200 Subject: [PATCH 028/588] Updated suite configuration to use WorkflowTestConfiguration objects --- workflow_tester.py | 61 +++++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 0d13d61..236b580 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -138,21 +138,14 @@ def load(filename=DEFAULT_CONFIG_FILENAME, workflow_test_name=None): config["galaxy_api_key"] = workflows_conf["galaxy_api_key"] config["enable_logger"] = workflows_conf["enable_logger"] config["workflows"] = {} - for workflow in workflows_conf.get("workflows").items(): - w = WorkflowTestConfiguration.DEFAULT_WORKFLOW_CONFIG.copy() - w["name"] = workflow[0] - w.update(workflow[1]) - # parse inputs - w["inputs"] = _parse_yaml_list(w["inputs"]) - # parse outputs - w["outputs"] = _parse_yaml_list(w["outputs"]) - # add base path - w["base_path"] = base_path + for wf_name, wf_config in workflows_conf.get("workflows").items(): # add the workflow - config["workflows"][w["name"]] = w + w = WorkflowTestConfiguration(base_path=base_path, filename=wf_config["file"], name=wf_name, + inputs=wf_config["inputs"], expected_outputs=wf_config["outputs"]) + config["workflows"][wf_name] = w # returns the current workflow test config # if its name matches the 'workflow_test_name' param - if workflow_test_name and w["name"] == workflow_test_name: + if workflow_test_name and wf_name == workflow_test_name: return w # raise an exception if the workflow test we are searching for # cannot be found within the configuration file. @@ -189,9 +182,9 @@ def initialize(self, galaxy_url=None, galaxy_api_key=None): self._galaxy_workflow_client = _WorkflowClient(self._galaxy_instance.gi) def load_workflow(self, workflow_test_config, workflow_name=None): - workflow_filename = workflow_test_config["file"] \ - if not "base_path" in workflow_test_config \ - else _os.path.join(workflow_test_config["base_path"], workflow_test_config["file"]) + workflow_filename = workflow_test_config.filename \ + if not workflow_test_config.base_path \ + else _os.path.join(workflow_test_config.base_path, workflow_test_config.filename) return self.load_workflow_by_filename(workflow_filename, workflow_name) def load_workflow_by_filename(self, workflow_filename, workflow_name=None): @@ -293,15 +286,15 @@ def __init__(self, galaxy_instance, workflow_loader, workflow_test_config, test_ self._workflow_test_config = workflow_test_config self._test_suite = test_suite self._galaxy_history_client = _HistoryClient(galaxy_instance.gi) - self._disable_cleanup = workflow_test_config.get("disable_cleanup", False) - self._disable_assertions = workflow_test_config.get("disable_assertions", False) - self._base_path = workflow_test_config.get("base_path", "") + self._disable_cleanup = workflow_test_config.disable_cleanup + self._disable_assertions = workflow_test_config.disable_assertions + self._base_path = workflow_test_config.base_path self._test_cases = {} self._test_uuid = None self._galaxy_workflow = None - setattr(self, "test_" + workflow_test_config["name"], self.run_test) - super(WorkflowTestRunner, self).__init__("test_" + workflow_test_config["name"]) + setattr(self, "test_" + workflow_test_config.name, self.run_test) + super(WorkflowTestRunner, self).__init__("test_" + workflow_test_config.name) @staticmethod def new_instance(workflow_test_config, galaxy_url=None, galaxy_api_key=None): @@ -311,19 +304,21 @@ def new_instance(workflow_test_config, galaxy_url=None, galaxy_api_key=None): # return the runner instance return WorkflowTestRunner(galaxy_instance, workflow_loader, workflow_test_config) + @property + def workflow_test_config(self): + return self._workflow_test_config + @property def worflow_test_name(self): - return self._workflow_test_config["name"] + return self._workflow_test_config.name def __str__(self): return "Workflow Test '{0}': testId={1}, workflow='{2}', input=[{3}], output=[{4}]" \ - .format(self._workflow_test_config["name"], + .format(self._workflow_test_config.name, self._get_test_uuid(), - self._workflow_test_config["name"], - ",".join(self._workflow_test_config[ - "inputs"]), - ",".join(self._workflow_test_config[ - "outputs"])) + self._workflow_test_config.name, + ",".join(self._workflow_test_config.inputs), + ",".join(self._workflow_test_config.expected_outputs)) def _get_test_uuid(self, update=False): if not self._test_uuid or update: @@ -346,14 +341,14 @@ def run_test(self, base_path=None, input_map=None, expected_output_map=None, # check input_map if not input_map: - if "inputs" in self._workflow_test_config: - input_map = self._workflow_test_config["inputs"] + if len(self._workflow_test_config.inputs) > 0: + input_map = self._workflow_test_config.inputs else: raise ValueError("No input configured !!!") # check expected_output_map if not expected_output_map: - if "outputs" in self._workflow_test_config: - expected_output_map = self._workflow_test_config["outputs"] + if len(self._workflow_test_config.expected_outputs) > 0: + expected_output_map = self._workflow_test_config.expected_outputs else: raise ValueError("No output configured !!!") @@ -600,8 +595,8 @@ def run_tests(config=None, debug=None, cleanup=None, assertions=None): config["disable_assertions"] = options.disable_assertions if not assertions else assertions for test_config in config["workflows"].values(): - test_config["disable_cleanup"] = config["disable_cleanup"] - test_config["disable_assertions"] = config["disable_assertions"] + test_config.disable_cleanup = config["disable_cleanup"] + test_config.disable_assertions = config["disable_assertions"] # enable the logger with the proper detail level if config["enable_logger"]: From 9106364eafac313d050f5139c17bcc129a929922 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 13 Oct 2016 09:57:47 +0200 Subject: [PATCH 029/588] Minor code refactoring --- workflow_tester.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 236b580..f4ecf25 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -475,15 +475,15 @@ def cleanup(self): class _WorkflowTestResult(): - def __init__(self, test_id, workflow, input_map, outputs, output_history, expected_output_map, + def __init__(self, test_id, workflow, inputs, outputs, output_history, expected_outputs, missing_tools, results, output_file_map, output_folder=WorkflowTestConfiguration.DEFAULT_OUTPUT_FOLDER): self.test_id = test_id self.workflow = workflow - self.inputs = input_map + self.inputs = inputs self.outputs = outputs self.output_history = output_history - self.expected_output_map = expected_output_map + self.expected_outputs = expected_outputs self.output_folder = output_folder self.missing_tools = missing_tools self.output_file_map = output_file_map From de03ca9db48821c42ebff91f54a05e92c864a1ad Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 13 Oct 2016 09:58:50 +0200 Subject: [PATCH 030/588] Added new method to save a configuration of workflow tests --- workflow_tester.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/workflow_tester.py b/workflow_tester.py index f4ecf25..fa117a8 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -156,6 +156,18 @@ def load(filename=DEFAULT_CONFIG_FILENAME, workflow_test_name=None): config["output_folder"] = WorkflowTestConfiguration.DEFAULT_OUTPUT_FOLDER return config + @staticmethod + def dump(filename, worflow_test_list): + workflows = {} + config = {"workflows": workflows} + worflow_test_list = worflow_test_list.values() if isinstance(worflow_test_list, dict) else worflow_test_list + print worflow_test_list + for worlflow in worflow_test_list: + workflows[worlflow.name] = worlflow.to_json() + with open(filename, "w") as f: + _yaml_dump(config, f) + return config + class WorkflowLoader: _instance = None From 365994d729b6e3be582c2105713dc81b1506b03c Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 13 Oct 2016 18:06:37 +0200 Subject: [PATCH 031/588] Initial documentation --- workflow_tester.py | 494 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 472 insertions(+), 22 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index fa117a8..d1561fb 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -24,6 +24,9 @@ class WorkflowTestConfiguration: + """ + Utility class for programmatically handle a workflow test configuration. + """ # Default settings DEFAULT_HISTORY_NAME_PREFIX = "_WorkflowTestHistory_" DEFAULT_WORKFLOW_NAME_PREFIX = "_WorkflowTest_" @@ -42,6 +45,53 @@ class WorkflowTestConfiguration: def __init__(self, base_path=".", filename="workflow.ga", name=None, inputs={}, expected_outputs={}, cleanup=True, assertions=True): + """ + Create a new class instance and initialize its initial properties. + + :type base_path: str + :param base_path: base path for workflow and datasets files; the current path is assumed as default + + :type filename: str + :param filename: the path (relative to the basepath) of the file containing the workflow definition + + :type name: str + :param name: the name of the workflow test + + :type inputs: dict + :param inputs: a map : (e.g., {"input_name" : {"file": ...}} + + :type expected_outputs: dict + :param expected_outputs: maps actual to expected outputs. + Each output requires a dict containing the path of the expected output filename + and the fully qualified name of a function which will be used to compare the expected + to the actual output. Such a function takes ``actual_output_filename`` and ``expected_output_filename`` + as parameters and returns ``True`` if the comparison succeeds, ``False``otherwise. + + Example of expected_outputs: + + :Example: + + {'output1': {'comparator': 'filecmp.cmp', + 'file': 'change_case_1/expected_output_1', + 'name': 'output1'}} + + Comparator function signature: + + :Example: + + def compare_outputs(actual_output_filename, expected_output_filename): + .... + return True | False + + :type cleanup: bool + :param cleanup: ``True`` (default) to perform a cleanup (Galaxy workflow, history, datasets) + after the workflow test execution; ``False`` otherwise. + + :type assertions: bool + :param assertions: ``True`` (default) to disable assertions during the workflow test execution; + ``False`` otherwise. + """ + # init properties self._base_path = None self._filename = None @@ -69,6 +119,11 @@ def base_path(self): return self._base_path def set_base_path(self, base_path): + """ Set the base path to ``base_path``. + + :type name: str + :param base_path: base path for workflow and datasets files; the current path is assumed as default + """ self._base_path = base_path @property @@ -76,6 +131,11 @@ def filename(self): return self._filename def set_filename(self, filename): + """ Set the filename of the workflow definition. + + :type filename: str + :param filename: the path (relative to the basepath) of the file containing the workflow definition + """ self._filename = filename @property @@ -83,20 +143,50 @@ def inputs(self): return self._inputs def set_inputs(self, inputs): - print inputs + """ + Add a set of inputs. + + :param inputs: dict + :return: a map : (e.g., {"input_name" : {"file": ...}} + """ for name, config in inputs.items(): self.add_input(name, config["file"]) def add_input(self, name, file): + """ + Add a new input. + + :type name: str + :param name: the Galaxy label of the input + + :type file: str + :param file: the path (relative to the basepath) of the file containing the input dataset + """ if not name: raise ValueError("Input name not defined") self._inputs[name] = {"name": name, "file": file if isinstance(file, list) else [file]} def remove_input(self, name): + """ + Remove an input. + + :type name: str + :param name: the Galaxy label of the input + + """ if name in self._inputs: del self._inputs[name] def get_input(self, name): + """ + Return the input configuration. + + :type name: str + :param name: the Galaxy label of the input + + :rtype: dict + :return: input configuration as dict (e.g., {'name': 'Input Dataset', 'file': "input.txt"}) + """ return self._inputs.get(name, None) @property @@ -104,22 +194,76 @@ def expected_outputs(self): return self._expected_outputs def set_expected_outputs(self, expected_outputs): + """ + Add a set of expected outputs which are intended to map actual to expected outputs. + + :type expected_outputs: dict + :param expected_outputs: map actual to expected outputs. + Each output requires a dict containing the path of the expected output filename + and the fully qualified name of a function which will be used to compare the expected + to the actual output. Such a function takes ``actual_output_filename`` and ``expected_output_filename`` + as parameters and returns ``True`` if the comparison succeeds, ``False``otherwise. + + .. example: {'output1': {'comparator': 'filecmp.cmp', + 'file': 'change_case_1/expected_output_1', + 'name': 'output1'}} + """ for name, config in expected_outputs.items(): self.add_expected_output(name, config["file"], config["comparator"]) def add_expected_output(self, name, filename, comparator="filecmp.cmp"): + """ + Add a new expected output to the workflow test configuration. + + :type name: str + :param name: the Galaxy name of the output which the expected output has to be mapped. + + :type filename: str + :param filename: the path (relative to the basepath) of the file containing the expected_output dataset + + :type comparator: str + :param comparator: a fully qualified name of a `comparator`function + + :Example: + + def compare_outputs(actual_output_filename, expected_output_filename): + .... + return True | False + + """ if not name: raise ValueError("Input name not defined") self._expected_outputs[name] = {"name": name, "file": filename, "comparator": comparator} def remove_expected_output(self, name): + """ + Remove an expected output from the workflow test configuration. + + :type name: str + :param name: the Galaxy name of the output which the expected output has to be mapped + """ if name in self._expected_outputs: del self._expected_outputs[name] def get_expected_output(self, name): + """ + Return the configuration of an expected output. + + :type name: str + :param name: the Galaxy name of the output which the expected output has to be mapped. + + :rtype: dict + :return + """ return self._expected_outputs.get(name, None) def to_json(self): + """ + Return a dict representation of the current class instance. + + :rtype: dict + :return: + """ return dict({ "name": self.name, "file": self.filename, @@ -129,6 +273,19 @@ def to_json(self): @staticmethod def load(filename=DEFAULT_CONFIG_FILENAME, workflow_test_name=None): + """ + Load the configuration of a workflow test suite or a single workflow test from a YAML file. + + :type filename: str + :param filename: the path of the file containing the suite definition + + :type workflow_test_name: str + :param workflow_test_name: the name of a workflow test name + + :rtype: dict + :return: + """ + config = {} if _os.path.exists(filename): base_path = _os.path.dirname(_os.path.abspath(filename)) @@ -158,6 +315,17 @@ def load(filename=DEFAULT_CONFIG_FILENAME, workflow_test_name=None): @staticmethod def dump(filename, worflow_test_list): + """ + Write the configuration of a workflow test suite to a YAML file. + + :type filename: str + :param filename: the absolute path of the YAML file + + :type worflow_test_list: dict + :param worflow_test_list: a dictionary which maps a workflow test name + to the corresponding configuration (:class:`WorkflowTestConfiguration`) + """ + workflows = {} config = {"workflows": workflows} worflow_test_list = worflow_test_list.values() if isinstance(worflow_test_list, dict) else worflow_test_list @@ -170,15 +338,34 @@ def dump(filename, worflow_test_list): class WorkflowLoader: + """ + Utility class responsible for loading/unloading workflows to a Galaxy server. + """ + _instance = None @staticmethod def get_instance(): + """ + Return the singleton instance of this class. + + :rtype: :class:`WorkflowLoader` + :return: a workflow loader instance + """ if not WorkflowLoader._instance: WorkflowLoader._instance = WorkflowLoader() return WorkflowLoader._instance def __init__(self, galaxy_instance=None): + """ + Create a new instance of this class + It requires a ``galaxy_instance`` (:class:`bioblend.GalaxyInstance`) which can be provided + as a constructor parameter (if it has already been instantiated) or configured (and instantiated) + by means of the method ``initialize``. + + :type galaxy_instance: :class:`bioblend.GalaxyInstance` + :param galaxy_instance: a galaxy instance object + """ self._galaxy_instance = galaxy_instance self._galaxy_workflow_client = None self._workflows = {} @@ -187,6 +374,15 @@ def __init__(self, galaxy_instance=None): self.initialize() def initialize(self, galaxy_url=None, galaxy_api_key=None): + """ + Initialize the required ``galaxy_instance``. + + :type galaxy_url: str + :param galaxy_url: the URL of the Galaxy server + + :type galaxy_api_key: str + :param galaxy_api_key: a registered Galaxy API KEY + """ if not self._galaxy_instance: # initialize the galaxy instance self._galaxy_instance = _get_galaxy_instance(galaxy_url, galaxy_api_key) @@ -194,12 +390,31 @@ def initialize(self, galaxy_url=None, galaxy_api_key=None): self._galaxy_workflow_client = _WorkflowClient(self._galaxy_instance.gi) def load_workflow(self, workflow_test_config, workflow_name=None): + """ + Load a workflow defined in a :class:`WorkflowTestConfig` instance to the configured Galaxy server. + + :type workflow_test_config: :class:`WorkflowTestConfig` + :param workflow_test_config: the configuration of the workflow test + + :type workflow_name: str + :param workflow_name: an optional name which overrides the workflow name + """ workflow_filename = workflow_test_config.filename \ if not workflow_test_config.base_path \ else _os.path.join(workflow_test_config.base_path, workflow_test_config.filename) return self.load_workflow_by_filename(workflow_filename, workflow_name) def load_workflow_by_filename(self, workflow_filename, workflow_name=None): + """ + Load a workflow defined within the `workflow_filename` file to the configured Galaxy server. + + :type workflow_filename: str + :param workflow_filename: the path of workflow definition file + + :type workflow_name: str + :param workflow_name: an optional name which overrides the workflow name + :return: + """ with open(workflow_filename) as f: wf_json = _json_load(f) # TODO: register workflow by ID (equal to UUID?) @@ -215,6 +430,12 @@ def load_workflow_by_filename(self, workflow_filename, workflow_name=None): return workflow def unload_workflow(self, workflow_id): + """ + Unload a workflow from the configured Galaxy server. + + :type workflow_id: str + :param workflow_id: the ID of the workflow to unload. + """ self._galaxy_workflow_client.delete_workflow(workflow_id) # TODO: remove workflow from the list @@ -223,14 +444,26 @@ def unload_workflows(self): self.unload_workflow(wf[id]) -class WorkflowTestSuite(): - def __init__(self, galaxy_url=None, galaxy_api_key=None): +class WorkflowTestSuite: + """ + Define a test suite. + """ + + def __init__(self, galaxy_url, galaxy_api_key): + """ + Create an instance of :class:`WorkflowTestSuite`. + + :type galaxy_url: str + :param galaxy_url: the URL of a Galaxy server (e.g., http://192.168.64.6:30700) + + :type galaxy_api_key: str + :param galaxy_api_key: the API KEY registered in the Galaxy server. + """ self._workflows = {} self._workflow_runners = [] self._workflow_test_results = [] self._galaxy_instance = None self._galaxy_workflow_client = None - # initialize the galaxy instance self._galaxy_instance = _get_galaxy_instance(galaxy_url, galaxy_api_key) # initialize the workflow loader @@ -244,31 +477,81 @@ def galaxy_instance(self): def workflow_loader(self): return self._workflow_loader - def get_workflows(self): - return self.galaxy_instance.workflows.list() + # def get_workflows(self): + # """ + # + # :rtype: list + # :return: list of :class:`bioblend:Workflow + # """ + # return self.galaxy_instance.workflows.list() def get_workflow_test_results(self, workflow_id=None): + """ + Return the list of :class:`WorkflowTestResult` instances resulting by the executed workflow tests. + Such a list can be filtered by workflow, specified as `workflow_id`. + + :type workflow_id: str + :param workflow_id: the optional ID of a workflow + + :rtype: list + :return: a list of :class:`WorkflowTestResult` + """ return list([w for w in self._workflow_test_results if w.id == workflow_id] if workflow_id else self._workflow_test_results) def _add_test_result(self, test_result): + """ + Private method to publish a test result. + + :type test_result: :class:'WorkflowTestResult' + :param test_result: an instance of :class:'WorkflowTestResult' + """ self._workflow_test_results.append(test_result) def _create_test_runner(self, workflow_test_config): + """ + Private method which creates a test runner associated to this suite. + + :type workflow_test_config: :class:'WorkflowTestConfig' + :param workflow_test_config: + + :rtype: :class:'WorkflowTestRunner' + :return: the created :class:'WorkflowTestResult' instance + """ runner = WorkflowTestRunner(self.galaxy_instance, self.workflow_loader, workflow_test_config, self) self._workflow_runners.append(runner) return runner def run_tests(self, workflow_tests_config): + """ + Execute tests associated to this suite and return the corresponding results. + + :type workflow_tests_config: dict + :param workflow_tests_config: a suite configuration as produced + by the `WorkflowTestConfiguration.load(...)` method + + :rtype: list + :return: the list of :class:'WorkflowTestResult' instances + """ results = [] for test_config in workflow_tests_config["workflows"].values(): runner = self._create_test_runner(test_config) result = runner.run_test(test_config["inputs"], test_config["outputs"], workflow_tests_config["output_folder"]) results.append(result) + # cleanup + if not workflow_tests_config["disable_cleanup"]: + self.cleanup() return results def run_test_suite(self, workflow_tests_config): + """ + Execute tests associated to this suite using the unittest framework. + + :type workflow_tests_config: dict + :param workflow_tests_config: a suite configuration as produced + by the `WorkflowTestConfiguration.load(...)` method + """ suite = _unittest.TestSuite() for test_config in workflow_tests_config["workflows"].values(): runner = self._create_test_runner(test_config) @@ -280,6 +563,9 @@ def run_test_suite(self, workflow_tests_config): self.cleanup() def cleanup(self): + """ + Perform a cleanup unloading workflows and deleting temporary histories. + """ _logger.debug("Cleaning save histories ...") hslist = self.galaxy_instance.histories.list() for history in [h for h in hslist if WorkflowTestConfiguration.DEFAULT_HISTORY_NAME_PREFIX in h.name]: @@ -292,6 +578,10 @@ def cleanup(self): class WorkflowTestRunner(_unittest.TestCase): + """ + Class responsible for launching tests. + """ + def __init__(self, galaxy_instance, workflow_loader, workflow_test_config, test_suite=None): self._galaxy_instance = galaxy_instance self._workflow_loader = workflow_loader @@ -310,6 +600,21 @@ def __init__(self, galaxy_instance, workflow_loader, workflow_test_config, test_ @staticmethod def new_instance(workflow_test_config, galaxy_url=None, galaxy_api_key=None): + """ + Factory method to create and intialize an instance of :class:`WorkflowTestRunner`. + + :type workflow_test_config: :class:`WorkflowTestConfiguration` + :param workflow_test_config: the configuration of a workflow test + + :type galaxy_url: str + :param galaxy_url: the URL of the Galaxy server + + :type galaxy_api_key: str + :param galaxy_api_key: a registered Galaxy API KEY + + :rtype: :class:`WorkflowTestRunner` + :return: + """ # initialize the galaxy instance galaxy_instance = _get_galaxy_instance(galaxy_url, galaxy_api_key) workflow_loader = WorkflowLoader(galaxy_instance) @@ -333,18 +638,78 @@ def __str__(self): ",".join(self._workflow_test_config.expected_outputs)) def _get_test_uuid(self, update=False): + """ + Get the current UUID or generate a new one. + + :type update: bool + :param update: ``True`` to force the generation of a new UUID + + :rtype: str + :return: a generated UUID + """ if not self._test_uuid or update: self._test_uuid = str(_uuid1()) return self._test_uuid def get_galaxy_workflow(self): + """ + Return the bioblend workflow instance associated to this runner. + + :rtype: :class:`bioblend.galaxy.objects.wrappers.Workflow` + :return: bioblend workflow instance + """ if not self._galaxy_workflow: self._galaxy_workflow = self._workflow_loader.load_workflow(self._workflow_test_config) return self._galaxy_workflow - def run_test(self, base_path=None, input_map=None, expected_output_map=None, + def run_test(self, base_path=None, inputs=None, expected_outputs=None, output_folder=WorkflowTestConfiguration.DEFAULT_OUTPUT_FOLDER, assertions=None, cleanup=None): + """ + Run the test with the given inputs and expected_outputs. + + :type base_path: str + :param base_path: base path for workflow and datasets files; the current path is assumed as default + + :type inputs: dict + :param inputs: a map : (e.g., {"input_name" : {"file": ...}} + + :type expected_outputs: dict + :param expected_outputs: maps actual to expected outputs. + Each output requires a dict containing the path of the expected output filename + and the fully qualified name of a function which will be used to compare the expected + to the actual output. Such a function takes ``actual_output_filename`` and ``expected_output_filename`` + as parameters and returns ``True`` if the comparison succeeds, ``False``otherwise. + Example of expected_outputs: + + :Example: + + {'output1': {'comparator': 'filecmp.cmp', + 'file': 'change_case_1/expected_output_1', + 'name': 'output1'}} + + Comparator function signature: + + :Example: + + def compare_outputs(actual_output_filename, expected_output_filename): + .... + return True | False + + :type output_folder: str + :param output_folder: the path of folder to temporary store intermediate results + + :type cleanup: bool + :param cleanup: ``True`` (default) to perform a cleanup (Galaxy workflow, history, datasets) + after the workflow test execution; ``False`` otherwise. + + :type assertions: bool + :param assertions: ``True`` (default) to disable assertions during the workflow test execution; + ``False`` otherwise. + + :rtype: :class:``WorkflowTestResult`` + :return: workflow test result + """ # set basepath base_path = self._base_path if not base_path else base_path @@ -352,15 +717,15 @@ def run_test(self, base_path=None, input_map=None, expected_output_map=None, workflow = self.get_galaxy_workflow() # check input_map - if not input_map: + if not inputs: if len(self._workflow_test_config.inputs) > 0: - input_map = self._workflow_test_config.inputs + inputs = self._workflow_test_config.inputs else: raise ValueError("No input configured !!!") # check expected_output_map - if not expected_output_map: + if not expected_outputs: if len(self._workflow_test_config.expected_outputs) > 0: - expected_output_map = self._workflow_test_config.expected_outputs + expected_outputs = self._workflow_test_config.expected_outputs else: raise ValueError("No output configured !!!") @@ -387,7 +752,7 @@ def run_test(self, base_path=None, input_map=None, expected_output_map=None, # upload input data to the current history # and generate the datamap INPUT --> DATASET datamap = {} - for label, config in input_map.items(): + for label, config in inputs.items(): datamap[label] = [] for filename in config["file"]: datamap[label].append(history.upload_dataset(_os.path.join(base_path, filename))) @@ -398,11 +763,11 @@ def run_test(self, base_path=None, input_map=None, expected_output_map=None, _logger.info("Workflow '%s' (id: %s) executed", workflow.name, workflow.id) # check outputs - results, output_file_map = self._check_outputs(base_path, outputs, expected_output_map, output_folder) + results, output_file_map = self._check_outputs(base_path, outputs, expected_outputs, output_folder) # instantiate the result object - test_result = _WorkflowTestResult(test_uuid, workflow, input_map, outputs, output_history, - expected_output_map, missing_tools, results, output_file_map, + test_result = _WorkflowTestResult(test_uuid, workflow, inputs, outputs, output_history, + expected_outputs, missing_tools, results, output_file_map, output_folder) if test_result.failed(): error_msg = "The following outputs differ from the expected ones: {0}".format( @@ -410,8 +775,8 @@ def run_test(self, base_path=None, input_map=None, expected_output_map=None, else: # instantiate the result object - test_result = _WorkflowTestResult(test_uuid, workflow, input_map, [], None, - expected_output_map, missing_tools, [], {}, output_folder) + test_result = _WorkflowTestResult(test_uuid, workflow, inputs, [], None, + expected_outputs, missing_tools, [], {}, output_folder) error_msg = "Some workflow tools are not available in Galaxy: {0}".format(", ".join(missing_tools)) # store result @@ -432,6 +797,15 @@ def run_test(self, base_path=None, input_map=None, expected_output_map=None, return test_result def find_missing_tools(self, workflow=None): + """ + Find tools required by the workflow to test and not installed in the configured Galaxy server. + + :type workflow: :class:`bioblend.galaxy.objects.wrappers.Workflow` + :param workflow: an optional instance of :class:`bioblend.galaxy.objects.wrappers.Workflow` + + :rtype: list + :return: the list of missing tools + """ _logger.debug("Checking required tools ...") workflow = self.get_galaxy_workflow() if not workflow else workflow available_tools = self._galaxy_instance.tools.list() @@ -447,7 +821,19 @@ def find_missing_tools(self, workflow=None): _logger.debug("Checking required tools: DONE") return missing_tools - def _check_outputs(self, base_path, outputs, expected_output_map, output_folder): + def _check_outputs(self, base_path, actual_outputs, expected_output_map, output_folder): + """ + Private method responsible for comparing actual to current outputs + + :param base_path: + :param actual_outputs: + :param expected_output_map: + :param output_folder: + + :rtype: tuple + :return: a tuple containing a :class:`WorkflowTestResult` as first element + and a map : as a second. + """ results = {} output_file_map = {} @@ -455,9 +841,9 @@ def _check_outputs(self, base_path, outputs, expected_output_map, output_folder) _os.makedirs(output_folder) _logger.info("Checking test output: ...") - for output in outputs: + for output in actual_outputs: _logger.debug("Checking OUTPUT '%s' ...", output.name) - output_filename = _os.path.join(output_folder, "output_" + str(outputs.index(output))) + output_filename = _os.path.join(output_folder, "output_" + str(actual_outputs.index(output))) with open(output_filename, "w") as out_file: output.download(out_file) output_file_map[output.name] = {"dataset": output, "filename": output_filename} @@ -487,6 +873,10 @@ def cleanup(self): class _WorkflowTestResult(): + """ + Class for representing the result of a workflow test. + """ + def __init__(self, test_id, workflow, inputs, outputs, output_history, expected_outputs, missing_tools, results, output_file_map, output_folder=WorkflowTestConfiguration.DEFAULT_OUTPUT_FOLDER): @@ -516,19 +906,61 @@ def __repr__(self): return self.__str__() def failed(self): + """ + Assert whether the test is failed. + + :rtype: bool + :return: ``True`` if the test is failed; ``False``otherwise. + """ return len(self.failed_outputs) > 0 def passed(self): + """ + Assert whether the test is passed. + + :rtype: bool + :return: ``True`` if the test is passed; ``False``otherwise. + """ return not self.failed() - def check_output(self, output, force=False): + def check_output(self, output): + """ + Assert whether the actual `output` is equal to the expected accordingly + to its associated `comparator` function. + + :type output: str or dict + :param output: output name + + :rtype: bool + :return: ``True`` if the test is passed; ``False``otherwise. + """ return self.results[output if isinstance(output, str) else output.name] - def check_outputs(self, force=False): + def check_outputs(self): + """ + Return a map of pairs :, where is ``True`` + if the actual `OUTPUT_NAME` is equal to the expected accordingly + to its associated `comparator` function. + + :rtype: dict + :return: map of output results + """ return self.results def _get_galaxy_instance(galaxy_url=None, galaxy_api_key=None): + """ + Private utility function to instantiate and configure a :class:`bioblend.GalaxyInstance` + + :type galaxy_url: str + :param galaxy_url: the URL of the Galaxy server + + :type galaxy_api_key: str + :param galaxy_api_key: a registered Galaxy API KEY + + :rtype: :class:`bioblend.GalaxyInstance` + :return: a new :class:`bioblend.GalaxyInstance` instance + """ if not galaxy_url: if ENV_KEY_GALAXY_URL in _os.environ: galaxy_url = _os.environ[ENV_KEY_GALAXY_URL] @@ -563,6 +995,15 @@ def _parse_yaml_list(ylist): def _load_comparator(fully_qualified_comparator_function): + """ + Utility function responsible for dynamically loading a comparator function + given its fully qualified name. + + :type fully_qualified_comparator_function: str + :param fully_qualified_comparator_function: fully qualified name of a comparator function + + :return: a callable reference to the loaded comparator function + """ components = fully_qualified_comparator_function.split('.') mod = __import__(components[0]) for comp in components[1:]: @@ -586,6 +1027,15 @@ def _parse_cli_options(): def run_tests(config=None, debug=None, cleanup=None, assertions=None): + """ + Run a configured test suite. + + :param config: + :param debug: + :param cleanup: + :param assertions: + :return: + """ options, args = _parse_cli_options() config = WorkflowTestConfiguration.load(options.file) if not config else config From 876017e782f6b8c8aeed6f805c7cda14f786a011 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 14 Oct 2016 10:37:08 +0200 Subject: [PATCH 032/588] Added methods to load/store a test suite from/to a configuration file --- workflow_tester.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/workflow_tester.py b/workflow_tester.py index d1561fb..2e61aa3 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -576,6 +576,21 @@ def cleanup(self): for wf in workflows: self._workflow_loader.unload_workflow(wf.id) + def load(self, filename=None): + """ + Load a test suite configuration and set it as default suite to run. + + :type filename: str + :param filename: the path of suite configuration file + """ + self._workflow_test_suite_configuration = WorkflowTestSuite._DEFAULT_SUITE_CONFIGURATION.copy() + self._workflow_test_suite_configuration.update( + WorkflowTestConfiguration.load(filename or WorkflowTestConfiguration.DEFAULT_CONFIG_FILENAME)) + + def dump(self, filename): + WorkflowTestConfiguration.dump(filename or WorkflowTestConfiguration.DEFAULT_CONFIG_FILENAME, + self._workflow_test_suite_configuration) + class WorkflowTestRunner(_unittest.TestCase): """ From 6e6c93c2515ec8b16a1494b6a36cde30845e92c3 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 14 Oct 2016 10:38:21 +0200 Subject: [PATCH 033/588] Added dict which defines the default configuration of a test suite --- workflow_tester.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/workflow_tester.py b/workflow_tester.py index 2e61aa3..37ee333 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -449,7 +449,15 @@ class WorkflowTestSuite: Define a test suite. """ - def __init__(self, galaxy_url, galaxy_api_key): + _DEFAULT_SUITE_CONFIGURATION = { + "enable_logger": True, + "enable_debug": False, + "disable_cleanup": False, + "disable_assertions": False, + "workflows": {} + } + + def __init__(self, galaxy_url=None, galaxy_api_key=None): """ Create an instance of :class:`WorkflowTestSuite`. From 259c6ec1223157ece877e3e9d74acc28382577c0 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 14 Oct 2016 10:39:56 +0200 Subject: [PATCH 034/588] Added new instance property to store the suite configuration --- workflow_tester.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/workflow_tester.py b/workflow_tester.py index 37ee333..f2e068f 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -476,6 +476,8 @@ def __init__(self, galaxy_url=None, galaxy_api_key=None): self._galaxy_instance = _get_galaxy_instance(galaxy_url, galaxy_api_key) # initialize the workflow loader self._workflow_loader = WorkflowLoader(self._galaxy_instance) + # default suite configuration + self._workflow_test_suite_configuration = WorkflowTestSuite._DEFAULT_SUITE_CONFIGURATION.copy() @property def galaxy_instance(self): @@ -485,6 +487,10 @@ def galaxy_instance(self): def workflow_loader(self): return self._workflow_loader + @property + def configuration(self): + return self._workflow_test_suite_configuration + # def get_workflows(self): # """ # From adfaeaf7791e211b9d2a22e82c5643a74505e34d Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 14 Oct 2016 10:47:24 +0200 Subject: [PATCH 035/588] Added methods to manage (add/remove) workflow tests of a test suite instance. --- workflow_tester.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/workflow_tester.py b/workflow_tester.py index f2e068f..6af1c7c 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -513,6 +513,38 @@ def get_workflow_test_results(self, workflow_id=None): return list([w for w in self._workflow_test_results if w.id == workflow_id] if workflow_id else self._workflow_test_results) + @property + def workflow_tests(self): + """ + Return the configurations of the workflows associated to this test suite. + + :rtype: dict + :return: a map : + """ + return self._workflow_test_suite_configuration["workflows"].copy() + + def add_workflow_test(self, workflow_test_configuration): + """ + Add a new workflow test to this suite. + + :type workflow_test_configuration: :class:"WorkflowTestConfiguration" + :param workflow_test_configuration: a workflow test configuration + """ + self._workflow_test_suite_configuration["workflows"][ + workflow_test_configuration.name] = workflow_test_configuration + + def remove_workflow_test(self, workflow_test): + """ + Remove a workflow test from this suite. + + :type workflow_test: str or :class:"WorkflowTestConfiguration" + :param workflow_test: the name of the workflow test to remove or its configuration + """ + if isinstance(workflow_test, WorkflowTestConfiguration): + del self._workflow_test_suite_configuration[workflow_test.name] + elif isinstance(workflow_test, str): + del self._workflow_test_suite_configuration[workflow_test] + def _add_test_result(self, test_result): """ Private method to publish a test result. From b52a43d2619793f5412046cc37809ab8b959e860 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 14 Oct 2016 10:50:11 +0200 Subject: [PATCH 036/588] Fixed configuration of the options (debug mode, logger, assertions, etc...) of test run --- workflow_tester.py | 93 +++++++++++++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 35 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 6af1c7c..f69bb16 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -44,7 +44,7 @@ class WorkflowTestConfiguration: } def __init__(self, base_path=".", filename="workflow.ga", name=None, inputs={}, expected_outputs={}, - cleanup=True, assertions=True): + disable_cleanup=True, disable_assertions=True): """ Create a new class instance and initialize its initial properties. @@ -83,13 +83,13 @@ def compare_outputs(actual_output_filename, expected_output_filename): .... return True | False - :type cleanup: bool - :param cleanup: ``True`` (default) to perform a cleanup (Galaxy workflow, history, datasets) - after the workflow test execution; ``False`` otherwise. + :type disable_cleanup: bool + :param disable_cleanup: ``True`` to skip cleanup (Galaxy workflow, history, datasets) + after the workflow test execution; ``False`` (default) otherwise. - :type assertions: bool - :param assertions: ``True`` (default) to disable assertions during the workflow test execution; - ``False`` otherwise. + :type disable_assertions: bool + :param disable_assertions: ``True`` to disable assertions during the workflow test execution; + ``False`` (default) otherwise. """ # init properties @@ -104,8 +104,8 @@ def compare_outputs(actual_output_filename, expected_output_filename): self.set_filename(filename) self.set_inputs(inputs) self.set_expected_outputs(expected_outputs) - self.disable_cleanup = not cleanup - self.disable_assertions = not assertions + self.disable_cleanup = disable_cleanup + self.disable_assertions = disable_assertions def __str__(self): return "WorkflowTestConfig: name={0}, file={1}, inputs=[{2}], expected_outputs=[{3}]".format( @@ -568,7 +568,21 @@ def _create_test_runner(self, workflow_test_config): self._workflow_runners.append(runner) return runner - def run_tests(self, workflow_tests_config): + def _suite_setup(self, config, enable_logger=None, + enable_debug=None, disable_cleanup=None, disable_assertions=None): + config["enable_logger"] = enable_logger if not enable_logger is None else config.get("enable_logger", True) + config["enable_debug"] = enable_debug if not enable_debug is None else config.get("enable_debug", False) + config["disable_cleanup"] = disable_cleanup \ + if not disable_cleanup is None else config.get("disable_cleanup", False) + config["disable_assertions"] = disable_assertions \ + if not disable_assertions is None else config.get("disable_assertions", False) + # update logger level + if config.get("enable_logger", True): + config["logger_level"] = _logging.DEBUG if config.get("enable_debug", False) else _logging.INFO + _logger.setLevel(config["logger_level"]) + + def run_tests(self, workflow_tests_config=None, enable_logger=None, + enable_debug=None, disable_cleanup=None, disable_assertions=None): """ Execute tests associated to this suite and return the corresponding results. @@ -580,17 +594,19 @@ def run_tests(self, workflow_tests_config): :return: the list of :class:'WorkflowTestResult' instances """ results = [] - for test_config in workflow_tests_config["workflows"].values(): + suite_config = workflow_tests_config or self._workflow_test_suite_configuration + self._suite_setup(suite_config, enable_logger, enable_debug, disable_cleanup, disable_assertions) + for test_config in suite_config["workflows"].values(): runner = self._create_test_runner(test_config) - result = runner.run_test(test_config["inputs"], test_config["outputs"], - workflow_tests_config["output_folder"]) + result = runner.run_test() results.append(result) # cleanup - if not workflow_tests_config["disable_cleanup"]: + if not suite_config["disable_cleanup"]: self.cleanup() return results - def run_test_suite(self, workflow_tests_config): + def run_test_suite(self, workflow_tests_config=None, enable_logger=None, + enable_debug=None, disable_cleanup=None, disable_assertions=None): """ Execute tests associated to this suite using the unittest framework. @@ -599,13 +615,15 @@ def run_test_suite(self, workflow_tests_config): by the `WorkflowTestConfiguration.load(...)` method """ suite = _unittest.TestSuite() - for test_config in workflow_tests_config["workflows"].values(): + suite_config = workflow_tests_config or self._workflow_test_suite_configuration + self._suite_setup(suite_config, enable_logger, enable_debug, disable_cleanup, disable_assertions) + for test_config in suite_config["workflows"].values(): runner = self._create_test_runner(test_config) suite.addTest(runner) _RUNNER = _unittest.TextTestRunner(verbosity=2) _RUNNER.run((suite)) # cleanup - if not workflow_tests_config["disable_cleanup"]: + if not suite_config["disable_cleanup"]: self.cleanup() def cleanup(self): @@ -674,7 +692,7 @@ def new_instance(workflow_test_config, galaxy_url=None, galaxy_api_key=None): :param galaxy_api_key: a registered Galaxy API KEY :rtype: :class:`WorkflowTestRunner` - :return: + :return: a :class:`WorkflowTestRunner` instance """ # initialize the galaxy instance galaxy_instance = _get_galaxy_instance(galaxy_url, galaxy_api_key) @@ -724,7 +742,8 @@ def get_galaxy_workflow(self): return self._galaxy_workflow def run_test(self, base_path=None, inputs=None, expected_outputs=None, - output_folder=WorkflowTestConfiguration.DEFAULT_OUTPUT_FOLDER, assertions=None, cleanup=None): + output_folder=WorkflowTestConfiguration.DEFAULT_OUTPUT_FOLDER, + disable_assertions=None, disable_cleanup=None): """ Run the test with the given inputs and expected_outputs. @@ -760,13 +779,13 @@ def compare_outputs(actual_output_filename, expected_output_filename): :type output_folder: str :param output_folder: the path of folder to temporary store intermediate results - :type cleanup: bool - :param cleanup: ``True`` (default) to perform a cleanup (Galaxy workflow, history, datasets) - after the workflow test execution; ``False`` otherwise. + :type disable_cleanup: bool + :param disable_cleanup: ``True`` to skip cleanup (Galaxy workflow, history, datasets) + after the workflow test execution; ``False`` (default) otherwise. - :type assertions: bool - :param assertions: ``True`` (default) to disable assertions during the workflow test execution; - ``False`` otherwise. + :type disable_assertions: bool + :param disable_assertions: ``True`` to disable assertions during the workflow test execution; + ``False`` (default) otherwise. :rtype: :class:``WorkflowTestResult`` :return: workflow test result @@ -791,8 +810,8 @@ def compare_outputs(actual_output_filename, expected_output_filename): raise ValueError("No output configured !!!") # update config options - disable_cleanup = self._disable_cleanup if not cleanup else not cleanup - disable_assertions = self._disable_assertions if not assertions else not assertions + disable_cleanup = disable_cleanup if not disable_cleanup is None else self._disable_cleanup + disable_assertions = disable_assertions if not disable_assertions is None else self._disable_assertions # uuid of the current test test_uuid = self._get_test_uuid(True) @@ -1087,11 +1106,13 @@ def _parse_cli_options(): return (options, args) -def run_tests(config=None, debug=None, cleanup=None, assertions=None): +def run_tests(config=None, enable_logger=None, enable_debug=None, disable_cleanup=None, disable_assertions=None): """ Run a configured test suite. - :param config: + :type config: dict + :param config: a test suite configuration resulting from YAML configuration file or used defined. + :param debug: :param cleanup: :param assertions: @@ -1110,12 +1131,14 @@ def run_tests(config=None, debug=None, cleanup=None, assertions=None): config["output_folder"] = options.output \ if options.output \ - else config["output_folder"] if "output_folder" in config else WorkflowTestConfiguration.DEFAULT_OUTPUT_FOLDER + else config["output_folder"] if "output_folder" in config \ + else WorkflowTestConfiguration.DEFAULT_OUTPUT_FOLDER - config["enable_logger"] = True if options.enable_logger else config.get("enable_logger", False) - config["debug"] = options.debug if not debug else debug - config["disable_cleanup"] = options.disable_cleanup if not cleanup else cleanup - config["disable_assertions"] = options.disable_assertions if not assertions else assertions + config["enable_logger"] = enable_logger or options.enable_logger or config.get("enable_logger", False) + config["enable_debug"] = enable_debug or options.debug or config.get("enable_debug", False) + config["disable_cleanup"] = disable_cleanup or options.disable_cleanup or config.get("disable_cleanup", False) + config["disable_assertions"] = disable_assertions or options.disable_assertions \ + or config.get("disable_assertions", False) for test_config in config["workflows"].values(): test_config.disable_cleanup = config["disable_cleanup"] @@ -1123,7 +1146,7 @@ def run_tests(config=None, debug=None, cleanup=None, assertions=None): # enable the logger with the proper detail level if config["enable_logger"]: - config["logger_level"] = _logging.DEBUG if debug or options.debug else _logging.INFO + config["logger_level"] = _logging.DEBUG if config["enable_debug"] else _logging.INFO _logger.setLevel(config["logger_level"]) # log the current configuration From 6205b2ecf573e31ddf40bf4796d6ff9d923a5324 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 14 Oct 2016 11:00:36 +0200 Subject: [PATCH 037/588] Fixed doc --- workflow_tester.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index f69bb16..176fdbb 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -1106,20 +1106,29 @@ def _parse_cli_options(): return (options, args) -def run_tests(config=None, enable_logger=None, enable_debug=None, disable_cleanup=None, disable_assertions=None): +def run_tests(enable_logger=None, enable_debug=None, disable_cleanup=None, disable_assertions=None): """ - Run a configured test suite. + Run a workflow test suite defined in a configuration file. - :type config: dict - :param config: a test suite configuration resulting from YAML configuration file or used defined. + :type enable_logger: bool + :param enable_logger: enable logger (disabled by default) - :param debug: - :param cleanup: - :param assertions: - :return: + :type enable_debug: bool + :param enable_debug: enable debug messages (disabled by default) + + :type disable_cleanup: bool + :param disable_cleanup: ``True`` to skip cleanup (Galaxy workflow, history, datasets) + after the workflow test execution; ``False`` (default) otherwise. + + :type disable_assertions: bool + :param disable_assertions: ``True`` to disable assertions during the workflow test execution; + ``False`` (default) otherwise. + + :rtype: tuple + :return: a tuple (test_suite_instance,suite_configuration) """ options, args = _parse_cli_options() - config = WorkflowTestConfiguration.load(options.file) if not config else config + config = WorkflowTestConfiguration.load(options.file) config["galaxy_url"] = options.server \ if options.server \ From cb98136af410a9efa54dd8769afcab0bf9f87460 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 14 Oct 2016 11:53:32 +0200 Subject: [PATCH 038/588] Fixed 'check_outputs' method to skip the check of actual outputs with any configured expected --- workflow_tester.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 176fdbb..03ed398 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -922,24 +922,25 @@ def _check_outputs(self, base_path, actual_outputs, expected_output_map, output_ _logger.info("Checking test output: ...") for output in actual_outputs: - _logger.debug("Checking OUTPUT '%s' ...", output.name) - output_filename = _os.path.join(output_folder, "output_" + str(actual_outputs.index(output))) - with open(output_filename, "w") as out_file: - output.download(out_file) - output_file_map[output.name] = {"dataset": output, "filename": output_filename} + if output.name in expected_output_map: + _logger.debug("Checking OUTPUT '%s' ...", output.name) + output_filename = _os.path.join(output_folder, "output_" + str(actual_outputs.index(output))) + with open(output_filename, "w") as out_file: + output.download(out_file) + output_file_map[output.name] = {"dataset": output, "filename": output_filename} + _logger.debug( + "Downloaded output {0}: dataset_id '{1}', filename '{2}'".format(output.name, output.id, + output_filename)) + config = expected_output_map[output.name] + comparator = _load_comparator(config["comparator"]) + expected_output_filename = _os.path.join(base_path, config["file"]) + result = comparator(output_filename, expected_output_filename) _logger.debug( - "Downloaded output {0}: dataset_id '{1}', filename '{2}'".format(output.name, output.id, - output_filename)) - config = expected_output_map[output.name] - comparator = _load_comparator(config["comparator"]) - expected_output_filename = _os.path.join(base_path, config["file"]) - result = comparator(output_filename, expected_output_filename) - _logger.debug( - "Output '{0}' {1} the expected: dataset '{2}', actual-output '{3}', expected-output '{4}'" - .format(output.name, "is equal to" if result else "differs from", - output.id, output_filename, expected_output_filename)) - results[output.name] = result - _logger.debug("Checking OUTPUT '%s': DONE", output.name) + "Output '{0}' {1} the expected: dataset '{2}', actual-output '{3}', expected-output '{4}'" + .format(output.name, "is equal to" if result else "differs from", + output.id, output_filename, expected_output_filename)) + results[output.name] = result + _logger.debug("Checking OUTPUT '%s': DONE", output.name) _logger.info("Checking test output: DONE") return (results, output_file_map) From 5825728ec66c0892f9700e9b1573c280d3d645a6 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 14 Oct 2016 15:01:32 +0200 Subject: [PATCH 039/588] Updated assertion message --- workflow_tester.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 03ed398..5086ebc 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -850,8 +850,10 @@ def compare_outputs(actual_output_filename, expected_output_filename): expected_outputs, missing_tools, results, output_file_map, output_folder) if test_result.failed(): - error_msg = "The following outputs differ from the expected ones: {0}".format( - ", ".join(test_result.failed_outputs)) + error_msg = "The actual output{0} {2} differ{1} from the expected one{0}." \ + .format("" if len(test_result.failed_outputs) == 1 else "s", + "" if len(test_result.failed_outputs) > 1 else "s", + ", ".join(["'{0}'".format(n) for n in test_result.failed_outputs])) else: # instantiate the result object From 64fde32f29961fcb74ac3a1493407302abaf22d5 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 14 Oct 2016 15:40:55 +0200 Subject: [PATCH 040/588] Added initial setup.py --- setup.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 setup.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..07c187e --- /dev/null +++ b/setup.py @@ -0,0 +1,8 @@ +from distutils.core import setup + +setup(name='workflow_tester', + description='Utility package for testing Galaxy workflows', + author='CRS4', + url='https://bitbucket.org/kikkomep/workflowtester/', + py_modules=['workflow_tester'], + ) From c2261dc5e670f1f5b6fe7f16b1182e798506b71c Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 Oct 2016 09:52:55 +0200 Subject: [PATCH 041/588] Short description for workflow tests --- workflow_tester.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/workflow_tester.py b/workflow_tester.py index 5086ebc..147f8a2 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -680,7 +680,7 @@ def __init__(self, galaxy_instance, workflow_loader, workflow_test_config, test_ @staticmethod def new_instance(workflow_test_config, galaxy_url=None, galaxy_api_key=None): """ - Factory method to create and intialize an instance of :class:`WorkflowTestRunner`. + Factory method to create and initialize an instance of :class:`WorkflowTestRunner`. :type workflow_test_config: :class:`WorkflowTestConfiguration` :param workflow_test_config: the configuration of a workflow test @@ -709,6 +709,9 @@ def worflow_test_name(self): return self._workflow_test_config.name def __str__(self): + return "Workflow Test: '{0}'".format(self._workflow_test_config.name) + + def to_string(self): return "Workflow Test '{0}': testId={1}, workflow='{2}', input=[{3}], output=[{4}]" \ .format(self._workflow_test_config.name, self._get_test_uuid(), From 4a43055dca9bc8d4479ec7a92b7f2b3c0b8805a6 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 Oct 2016 15:21:33 +0200 Subject: [PATCH 042/588] Updated imports --- workflow_tester.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/workflow_tester.py b/workflow_tester.py index 147f8a2..37fe47b 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -7,6 +7,8 @@ import optparse as _optparse from uuid import uuid1 as _uuid1 +from sys import exc_info as _exc_info +from difflib import unified_diff as _unified_diff from yaml import load as _yaml_load, dump as _yaml_dump from json import load as _json_load, dump as _json_dump From 73e1a1b92ec77197a01783c1fb2af50a74143a75 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 Oct 2016 15:23:07 +0200 Subject: [PATCH 043/588] Added new base_comparator which uses the difflib package to compare the actual to the expected output --- workflow_tester.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/workflow_tester.py b/workflow_tester.py index 37fe47b..1a63393 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -1099,6 +1099,16 @@ def _load_comparator(fully_qualified_comparator_function): return mod +def base_comparator(actual_output_filename, expected_output_filename): + _logger.debug("Using default comparator....") + with open(actual_output_filename) as aout, open(expected_output_filename) as eout: + diff = _unified_diff(aout.readlines(), eout.readlines(), actual_output_filename, expected_output_filename) + ldiff = list(diff) + if len(ldiff) > 0: + print "\n{0}\n...\n".format("".join(ldiff[:20])) + return len(ldiff) == 0 + + def _parse_cli_options(): parser = _optparse.OptionParser() parser.add_option('--server', help='Galaxy server URL') From e0542f87c0d377dbeac8a0c2dfc8e695c54051a7 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 Oct 2016 15:24:13 +0200 Subject: [PATCH 044/588] Updated 'comparator_loader' to catch import errors --- workflow_tester.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 1a63393..e59175c 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -1092,10 +1092,18 @@ def _load_comparator(fully_qualified_comparator_function): :return: a callable reference to the loaded comparator function """ - components = fully_qualified_comparator_function.split('.') - mod = __import__(components[0]) - for comp in components[1:]: - mod = getattr(mod, comp) + mod = None + try: + components = fully_qualified_comparator_function.split('.') + mod = __import__(components[0]) + for comp in components[1:]: + mod = getattr(mod, comp) + except ImportError, e: + _logger.error(e) + except AttributeError, e: + _logger.error(e) + except: + _logger.error("Unexpected error:", _exc_info()[0]) return mod From 03e650c1d0a450e7f6da043ee43a94094d2a7422 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 Oct 2016 15:28:21 +0200 Subject: [PATCH 045/588] Updated workflow runner to use the new base_comparator by default --- workflow_tester.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index e59175c..2022d4c 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -211,7 +211,7 @@ def set_expected_outputs(self, expected_outputs): 'name': 'output1'}} """ for name, config in expected_outputs.items(): - self.add_expected_output(name, config["file"], config["comparator"]) + self.add_expected_output(name, config["file"], config.get("comparator", None)) def add_expected_output(self, name, filename, comparator="filecmp.cmp"): """ @@ -939,14 +939,17 @@ def _check_outputs(self, base_path, actual_outputs, expected_output_map, output_ "Downloaded output {0}: dataset_id '{1}', filename '{2}'".format(output.name, output.id, output_filename)) config = expected_output_map[output.name] - comparator = _load_comparator(config["comparator"]) - expected_output_filename = _os.path.join(base_path, config["file"]) - result = comparator(output_filename, expected_output_filename) - _logger.debug( - "Output '{0}' {1} the expected: dataset '{2}', actual-output '{3}', expected-output '{4}'" - .format(output.name, "is equal to" if result else "differs from", - output.id, output_filename, expected_output_filename)) - results[output.name] = result + comparator_fn = config.get("comparator", None) + _logger.debug("Configured comparator function: %s", comparator_fn) + comparator = _load_comparator(comparator_fn) if comparator_fn else base_comparator + if comparator: + expected_output_filename = _os.path.join(base_path, config["file"]) + result = comparator(output_filename, expected_output_filename) + _logger.debug( + "Output '{0}' {1} the expected: dataset '{2}', actual-output '{3}', expected-output '{4}'" + .format(output.name, "is equal to" if result else "differs from", + output.id, output_filename, expected_output_filename)) + results[output.name] = result _logger.debug("Checking OUTPUT '%s': DONE", output.name) _logger.info("Checking test output: DONE") return (results, output_file_map) From c75f11a41f4d43083dd6f2d0bf3e1db0f490ae4e Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 Oct 2016 15:31:31 +0200 Subject: [PATCH 046/588] Enable logger when debug parameter is True --- workflow_tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow_tester.py b/workflow_tester.py index 2022d4c..46a0ca4 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -1183,7 +1183,7 @@ def run_tests(enable_logger=None, enable_debug=None, disable_cleanup=None, disab test_config.disable_assertions = config["disable_assertions"] # enable the logger with the proper detail level - if config["enable_logger"]: + if config["enable_logger"] or config["enable_debug"]: config["logger_level"] = _logging.DEBUG if config["enable_debug"] else _logging.INFO _logger.setLevel(config["logger_level"]) From 309a9a885d53dcb8216107e2b5fb3fc4fede2adb Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 Oct 2016 17:22:47 +0200 Subject: [PATCH 047/588] Updated CLI option description --- workflow_tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow_tester.py b/workflow_tester.py index 46a0ca4..3b4a4f9 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -1128,7 +1128,7 @@ def _parse_cli_options(): parser.add_option('--debug', help='Enable debug mode', action='store_true') parser.add_option('--disable-cleanup', help='Disable cleanup', action='store_false') parser.add_option('--disable-assertions', help='Disable assertions', action='store_false') - parser.add_option('-o', '--output', help='absolute path of the folder to download workflow outputs') + parser.add_option('-o', '--output', help='absolute path of the folder where output is written') parser.add_option('-f', '--file', default=WorkflowTestConfiguration.DEFAULT_CONFIG_FILENAME, help='YAML configuration file of workflow tests') (options, args) = parser.parse_args() From 97719d1a90a24bf5a4d4cbbc1379e8091b5bdd34 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 Oct 2016 17:24:39 +0200 Subject: [PATCH 048/588] Added new 'output_folder' property to configure the folder to store the workflow output --- workflow_tester.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/workflow_tester.py b/workflow_tester.py index 3b4a4f9..6438116 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -46,7 +46,7 @@ class WorkflowTestConfiguration: } def __init__(self, base_path=".", filename="workflow.ga", name=None, inputs={}, expected_outputs={}, - disable_cleanup=True, disable_assertions=True): + output_folder=DEFAULT_OUTPUT_FOLDER, disable_cleanup=True, disable_assertions=True): """ Create a new class instance and initialize its initial properties. @@ -85,6 +85,9 @@ def compare_outputs(actual_output_filename, expected_output_filename): .... return True | False + :type output_folder: str + :param output_folder: absolute path of the folder where output is written + :type disable_cleanup: bool :param disable_cleanup: ``True`` to skip cleanup (Galaxy workflow, history, datasets) after the workflow test execution; ``False`` (default) otherwise. @@ -106,6 +109,7 @@ def compare_outputs(actual_output_filename, expected_output_filename): self.set_filename(filename) self.set_inputs(inputs) self.set_expected_outputs(expected_outputs) + self.output_folder = output_folder self.disable_cleanup = disable_cleanup self.disable_assertions = disable_assertions From f0d2b4b5cc007212b51247fa72d6f1f47901bd2c Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 Oct 2016 17:25:41 +0200 Subject: [PATCH 049/588] Configured a different output folder for every workflow --- workflow_tester.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 6438116..ef5d94e 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -297,14 +297,19 @@ def load(filename=DEFAULT_CONFIG_FILENAME, workflow_test_name=None): base_path = _os.path.dirname(_os.path.abspath(filename)) with open(filename, "r") as config_file: workflows_conf = _yaml_load(config_file) - config["galaxy_url"] = workflows_conf["galaxy_url"] - config["galaxy_api_key"] = workflows_conf["galaxy_api_key"] - config["enable_logger"] = workflows_conf["enable_logger"] + config["galaxy_url"] = workflows_conf.get("galaxy_url", None) + config["galaxy_api_key"] = workflows_conf.get("galaxy_api_key", None) + config["enable_logger"] = workflows_conf.get("enable_logger", False) + config["output_folder"] = workflows_conf.get("output_folder", + WorkflowTestConfiguration.DEFAULT_OUTPUT_FOLDER) config["workflows"] = {} for wf_name, wf_config in workflows_conf.get("workflows").items(): + wf_config["output_folder"] = _os.path.join(config["output_folder"], + wf_config.get("output_folder", wf_name)) # add the workflow w = WorkflowTestConfiguration(base_path=base_path, filename=wf_config["file"], name=wf_name, - inputs=wf_config["inputs"], expected_outputs=wf_config["outputs"]) + inputs=wf_config["inputs"], expected_outputs=wf_config["outputs"], + output_folder=wf_config["output_folder"]) config["workflows"][wf_name] = w # returns the current workflow test config # if its name matches the 'workflow_test_name' param @@ -751,8 +756,7 @@ def get_galaxy_workflow(self): return self._galaxy_workflow def run_test(self, base_path=None, inputs=None, expected_outputs=None, - output_folder=WorkflowTestConfiguration.DEFAULT_OUTPUT_FOLDER, - disable_assertions=None, disable_cleanup=None): + output_folder=None, disable_assertions=None, disable_cleanup=None): """ Run the test with the given inputs and expected_outputs. @@ -805,6 +809,10 @@ def compare_outputs(actual_output_filename, expected_output_filename): # load workflow workflow = self.get_galaxy_workflow() + # output folder + if not output_folder: + output_folder = self._workflow_test_config.output_folder + # check input_map if not inputs: if len(self._workflow_test_config.inputs) > 0: @@ -1183,6 +1191,7 @@ def run_tests(enable_logger=None, enable_debug=None, disable_cleanup=None, disab or config.get("disable_assertions", False) for test_config in config["workflows"].values(): + test_config.output_folder = config["output_folder"] test_config.disable_cleanup = config["disable_cleanup"] test_config.disable_assertions = config["disable_assertions"] From 11a0b15ad70e8e68b815e3606529404ec81ce200 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 Oct 2016 17:26:39 +0200 Subject: [PATCH 050/588] Added new function to delete temp files (e.g., workflow outputs) --- workflow_tester.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/workflow_tester.py b/workflow_tester.py index ef5d94e..23a9861 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -970,10 +970,20 @@ def cleanup(self): for test_uuid, test_result in self._test_cases.items(): if test_result.output_history: self._galaxy_instance.histories.delete(test_result.output_history.id) + self.cleanup_output_folder(test_result) if self._galaxy_workflow: self._workflow_loader.unload_workflow(self._galaxy_workflow.id) self._galaxy_workflow = None + def cleanup_output_folder(self, test_result=None): + test_results = self._test_cases.values() if not test_result else [test_result] + for _test in test_results: + for output_name, output_map in _test.output_file_map.items(): + _logger.debug("Cleaning output folder: %s", output_name) + if _os.path.exists(output_map["filename"]): + _os.remove(output_map["filename"]) + _logger.debug("Deleted output file '%s'.", output_map["filename"]) + class _WorkflowTestResult(): """ From bc7acbf7dc97a7f9a286ded5b65ae3b1aa707d6f Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 Oct 2016 11:36:31 +0200 Subject: [PATCH 051/588] Added a default workflow test name --- workflow_tester.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 23a9861..0bedb2f 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -104,12 +104,13 @@ def compare_outputs(actual_output_filename, expected_output_filename): self._expected_outputs = {} # set parameters - self.name = name + self.name = _uuid1() if not name else name self.set_base_path(base_path) self.set_filename(filename) self.set_inputs(inputs) self.set_expected_outputs(expected_outputs) - self.output_folder = output_folder + self.output_folder = _os.path.join(self.DEFAULT_OUTPUT_FOLDER, self.name) \ + if output_folder is None else output_folder self.disable_cleanup = disable_cleanup self.disable_assertions = disable_assertions From a64995e97f747098ec08e2d98177687784242a1f Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 Oct 2016 11:39:55 +0200 Subject: [PATCH 052/588] Fixed output folder initialization --- workflow_tester.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 0bedb2f..38259ef 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -45,8 +45,8 @@ class WorkflowTestConfiguration: } } - def __init__(self, base_path=".", filename="workflow.ga", name=None, inputs={}, expected_outputs={}, - output_folder=DEFAULT_OUTPUT_FOLDER, disable_cleanup=True, disable_assertions=True): + def __init__(self, name=None, base_path=".", filename="workflow.ga", inputs={}, expected_outputs={}, + output_folder=None, disable_cleanup=True, disable_assertions=True): """ Create a new class instance and initialize its initial properties. @@ -308,7 +308,7 @@ def load(filename=DEFAULT_CONFIG_FILENAME, workflow_test_name=None): wf_config["output_folder"] = _os.path.join(config["output_folder"], wf_config.get("output_folder", wf_name)) # add the workflow - w = WorkflowTestConfiguration(base_path=base_path, filename=wf_config["file"], name=wf_name, + w = WorkflowTestConfiguration(name=wf_name, base_path=base_path, filename=wf_config["file"], inputs=wf_config["inputs"], expected_outputs=wf_config["outputs"], output_folder=wf_config["output_folder"]) config["workflows"][wf_name] = w @@ -341,7 +341,7 @@ def dump(filename, worflow_test_list): workflows = {} config = {"workflows": workflows} worflow_test_list = worflow_test_list.values() if isinstance(worflow_test_list, dict) else worflow_test_list - print worflow_test_list + for worlflow in worflow_test_list: workflows[worlflow.name] = worlflow.to_json() with open(filename, "w") as f: @@ -681,6 +681,7 @@ def __init__(self, galaxy_instance, workflow_loader, workflow_test_config, test_ self._galaxy_history_client = _HistoryClient(galaxy_instance.gi) self._disable_cleanup = workflow_test_config.disable_cleanup self._disable_assertions = workflow_test_config.disable_assertions + self._output_folder = workflow_test_config.output_folder self._base_path = workflow_test_config.base_path self._test_cases = {} self._test_uuid = None @@ -830,6 +831,7 @@ def compare_outputs(actual_output_filename, expected_output_filename): # update config options disable_cleanup = disable_cleanup if not disable_cleanup is None else self._disable_cleanup disable_assertions = disable_assertions if not disable_assertions is None else self._disable_assertions + output_folder = output_folder if not output_folder is None else self._output_folder # uuid of the current test test_uuid = self._get_test_uuid(True) @@ -1202,7 +1204,6 @@ def run_tests(enable_logger=None, enable_debug=None, disable_cleanup=None, disab or config.get("disable_assertions", False) for test_config in config["workflows"].values(): - test_config.output_folder = config["output_folder"] test_config.disable_cleanup = config["disable_cleanup"] test_config.disable_assertions = config["disable_assertions"] From c9d66dd25e4600a3e5a7605dd5cb822edb0994f0 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 Oct 2016 11:42:07 +0200 Subject: [PATCH 053/588] Fixed CLI options initialization --- workflow_tester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 38259ef..04732c8 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -1151,8 +1151,8 @@ def _parse_cli_options(): parser.add_option('--api-key', help='Galaxy server API KEY') parser.add_option('--enable-logger', help='Enable log messages', action='store_true') parser.add_option('--debug', help='Enable debug mode', action='store_true') - parser.add_option('--disable-cleanup', help='Disable cleanup', action='store_false') - parser.add_option('--disable-assertions', help='Disable assertions', action='store_false') + parser.add_option('--disable-cleanup', help='Disable cleanup', action='store_true') + parser.add_option('--disable-assertions', help='Disable assertions', action='store_true') parser.add_option('-o', '--output', help='absolute path of the folder where output is written') parser.add_option('-f', '--file', default=WorkflowTestConfiguration.DEFAULT_CONFIG_FILENAME, help='YAML configuration file of workflow tests') From ceac0b3be6148443219b1a628e93ff2f98f004f7 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 Oct 2016 12:03:04 +0200 Subject: [PATCH 054/588] 'outputs' config field renamed to 'expected' --- workflow_tester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 04732c8..aae7ed5 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -39,7 +39,7 @@ class WorkflowTestConfiguration: "inputs": { "Input Dataset": {"name": "Input Dataset", "file": ["input"]} }, - "outputs": { + "expected": { "output1": {"file": "expected_output", "comparator": "filecmp.cmp", "name": "output1"}, "output2": {"file": "expected_output", "comparator": "filecmp.cmp", "name": "output2"} } @@ -309,7 +309,7 @@ def load(filename=DEFAULT_CONFIG_FILENAME, workflow_test_name=None): wf_config.get("output_folder", wf_name)) # add the workflow w = WorkflowTestConfiguration(name=wf_name, base_path=base_path, filename=wf_config["file"], - inputs=wf_config["inputs"], expected_outputs=wf_config["outputs"], + inputs=wf_config["inputs"], expected_outputs=wf_config["expected"], output_folder=wf_config["output_folder"]) config["workflows"][wf_name] = w # returns the current workflow test config From 8801310ae79dea2eb93ffa0227a915259520b876 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 Oct 2016 12:31:35 +0200 Subject: [PATCH 055/588] Switched to a new configuration loader --- workflow_tester.py | 84 ++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index aae7ed5..10c00fc 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -296,30 +296,29 @@ def load(filename=DEFAULT_CONFIG_FILENAME, workflow_test_name=None): config = {} if _os.path.exists(filename): base_path = _os.path.dirname(_os.path.abspath(filename)) - with open(filename, "r") as config_file: - workflows_conf = _yaml_load(config_file) - config["galaxy_url"] = workflows_conf.get("galaxy_url", None) - config["galaxy_api_key"] = workflows_conf.get("galaxy_api_key", None) - config["enable_logger"] = workflows_conf.get("enable_logger", False) - config["output_folder"] = workflows_conf.get("output_folder", - WorkflowTestConfiguration.DEFAULT_OUTPUT_FOLDER) - config["workflows"] = {} - for wf_name, wf_config in workflows_conf.get("workflows").items(): - wf_config["output_folder"] = _os.path.join(config["output_folder"], - wf_config.get("output_folder", wf_name)) - # add the workflow - w = WorkflowTestConfiguration(name=wf_name, base_path=base_path, filename=wf_config["file"], - inputs=wf_config["inputs"], expected_outputs=wf_config["expected"], - output_folder=wf_config["output_folder"]) - config["workflows"][wf_name] = w - # returns the current workflow test config - # if its name matches the 'workflow_test_name' param - if workflow_test_name and wf_name == workflow_test_name: - return w - # raise an exception if the workflow test we are searching for - # cannot be found within the configuration file. - if workflow_test_name: - raise KeyError("WorkflowTest with name '%s' not found" % workflow_test_name) + workflows_conf = _load_configuration(filename) + config["galaxy_url"] = workflows_conf.get("galaxy_url", None) + config["galaxy_api_key"] = workflows_conf.get("galaxy_api_key", None) + config["enable_logger"] = workflows_conf.get("enable_logger", False) + config["output_folder"] = workflows_conf.get("output_folder", + WorkflowTestConfiguration.DEFAULT_OUTPUT_FOLDER) + config["workflows"] = {} + for wf_name, wf_config in workflows_conf.get("workflows").items(): + wf_config["output_folder"] = _os.path.join(config["output_folder"], + wf_config.get("output_folder", wf_name)) + # add the workflow + w = WorkflowTestConfiguration(name=wf_name, base_path=base_path, filename=wf_config["file"], + inputs=wf_config["inputs"], expected_outputs=wf_config["expected"], + output_folder=wf_config["output_folder"]) + config["workflows"][wf_name] = w + # returns the current workflow test config + # if its name matches the 'workflow_test_name' param + if workflow_test_name and wf_name == workflow_test_name: + return w + # raise an exception if the workflow test we are searching for + # cannot be found within the configuration file. + if workflow_test_name: + raise KeyError("WorkflowTest with name '%s' not found" % workflow_test_name) else: config["workflows"] = {"unknown": WorkflowTestConfiguration.DEFAULT_WORKFLOW_CONFIG.copy()} config["output_folder"] = WorkflowTestConfiguration.DEFAULT_OUTPUT_FOLDER @@ -1092,22 +1091,27 @@ def _get_galaxy_instance(galaxy_url=None, galaxy_api_key=None): return _GalaxyInstance(galaxy_url, galaxy_api_key) -def _parse_yaml_list(ylist): - objs = {} - if isinstance(ylist, list): - for obj in ylist: - obj_data = obj.items() - obj_name = obj_data[0][0] - obj_file = obj_data[0][1] - objs[obj_name] = { - "name": obj_name, - "file": obj_file - } - elif isinstance(ylist, dict): - for obj_name, obj_data in ylist.items(): - obj_data["name"] = obj_name - objs[obj_name] = obj_data - return objs +def _load_configuration(config_filename): + with open(config_filename) as config_file: + workflows_conf = _yaml_load(config_file) + for wf_name, wf in workflows_conf["workflows"].items(): + wf["inputs"] = _parse_dict(wf["inputs"]) + wf["expected"] = _parse_dict(wf["expected"]) + return workflows_conf + + +def _parse_dict(elements): + results = {} + for name, value in elements.items(): + result = value + if isinstance(value, str): + result = {"name": name, "file": value} + elif isinstance(value, dict): + result["name"] = name + else: + raise ValueError("Configuration error: %r", elements) + results[name] = result + return results def _load_comparator(fully_qualified_comparator_function): From 763c0b4626d30b464b8b14f50cab610c7cb79c0a Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 Oct 2016 16:19:41 +0200 Subject: [PATCH 056/588] Updated imports: use 'json.dumps' in place of 'json.dump' --- workflow_tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow_tester.py b/workflow_tester.py index 10c00fc..2e37477 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -10,7 +10,7 @@ from sys import exc_info as _exc_info from difflib import unified_diff as _unified_diff from yaml import load as _yaml_load, dump as _yaml_dump -from json import load as _json_load, dump as _json_dump +from json import load as _json_load, dumps as _json_dumps from bioblend.galaxy.objects import GalaxyInstance as _GalaxyInstance from bioblend.galaxy.workflows import WorkflowClient as _WorkflowClient From 18b6ac58410892a29d97839744aef26ca8acbce4 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 Oct 2016 16:20:26 +0200 Subject: [PATCH 057/588] Fixed export to JSON function of the WorkflowTestConfiguration class --- workflow_tester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 2e37477..bdfd5a7 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -274,8 +274,8 @@ def to_json(self): return dict({ "name": self.name, "file": self.filename, - "inputs": self.inputs, - "outputs": self.expected_outputs + "inputs": {name: input["file"][0] for name, input in self.inputs.items()}, + "expected": self.expected_outputs }) @staticmethod From 4d3d4758e0acae3e5949ef75ea5dbabab2bd68fd Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 Oct 2016 16:21:19 +0200 Subject: [PATCH 058/588] Updated 'parse_element' function to support unicode strings --- workflow_tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow_tester.py b/workflow_tester.py index bdfd5a7..bb1070d 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -1104,7 +1104,7 @@ def _parse_dict(elements): results = {} for name, value in elements.items(): result = value - if isinstance(value, str): + if isinstance(value, str) or isinstance(value, unicode): result = {"name": name, "file": value} elif isinstance(value, dict): result["name"] = name From 934ac2dcf46a50ccc7e3ecb14229222ba7054fe2 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 Oct 2016 16:22:32 +0200 Subject: [PATCH 059/588] Added support for loading and writing JSON and YAML configuration files --- workflow_tester.py | 56 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index bb1070d..fd23159 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -25,6 +25,21 @@ _logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s') +class FILE_FORMATS: + YAML = "YAML" + JSON = "JSON" + + @staticmethod + def is_yaml(file_format): + return file_format and (isinstance(file_format, str) or isinstance(file_format, unicode)) and \ + file_format.upper() == FILE_FORMATS.YAML + + @staticmethod + def is_json(file_format): + return file_format and (isinstance(file_format, str) or isinstance(file_format, unicode)) and \ + file_format.upper() == FILE_FORMATS.JSON + + class WorkflowTestConfiguration: """ Utility class for programmatically handle a workflow test configuration. @@ -325,26 +340,37 @@ def load(filename=DEFAULT_CONFIG_FILENAME, workflow_test_name=None): return config @staticmethod - def dump(filename, worflow_test_list): + def dump(filename, worflow_tests_config, file_format=FILE_FORMATS.YAML): """ Write the configuration of a workflow test suite to a YAML file. :type filename: str :param filename: the absolute path of the YAML file - :type worflow_test_list: dict - :param worflow_test_list: a dictionary which maps a workflow test name + :type worflow_tests_config: dict or list + :param worflow_tests_config: a dictionary which maps a workflow test name to the corresponding configuration (:class:`WorkflowTestConfiguration`) - """ + :type file_format: str + :param file_format: 'YAML' or 'JSON' + """ workflows = {} - config = {"workflows": workflows} - worflow_test_list = worflow_test_list.values() if isinstance(worflow_test_list, dict) else worflow_test_list + config = worflow_tests_config.copy() if isinstance(worflow_tests_config, dict) else {} + config["workflows"] = workflows + + if isinstance(worflow_tests_config, dict): + worflow_tests_config = worflow_tests_config["workflows"].values() + elif not isinstance(worflow_tests_config, list): + raise ValueError( + "'workflow_tests_config' must be a configuration dict " + "or a list of 'WorkflowTestConfiguration' instances") - for worlflow in worflow_test_list: + for worlflow in worflow_tests_config: workflows[worlflow.name] = worlflow.to_json() with open(filename, "w") as f: - _yaml_dump(config, f) + _yaml_dump(config, f) \ + if FILE_FORMATS.is_yaml(file_format) \ + else f.write(_json_dumps(config, indent=2)) return config @@ -1093,10 +1119,16 @@ def _get_galaxy_instance(galaxy_url=None, galaxy_api_key=None): def _load_configuration(config_filename): with open(config_filename) as config_file: - workflows_conf = _yaml_load(config_file) - for wf_name, wf in workflows_conf["workflows"].items(): - wf["inputs"] = _parse_dict(wf["inputs"]) - wf["expected"] = _parse_dict(wf["expected"]) + workflows_conf = None + try: + workflows_conf = _yaml_load(config_file) + except ValueError, e: + _logger.error("Configuration file '%s' is not a valid YAML or JSON file", config_filename) + raise ValueError("Not valid format for the configuration file '%s'.", config_filename) + # update inputs/expected fields + for wf_name, wf in workflows_conf["workflows"].items(): + wf["inputs"] = _parse_dict(wf["inputs"]) + wf["expected"] = _parse_dict(wf["expected"]) return workflows_conf From 1c8af6979d7fe082703d134014c77830728d02cb Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 Oct 2016 16:57:02 +0200 Subject: [PATCH 060/588] Cleaning --- workflow_tester.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index fd23159..657a0e5 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -528,14 +528,6 @@ def workflow_loader(self): def configuration(self): return self._workflow_test_suite_configuration - # def get_workflows(self): - # """ - # - # :rtype: list - # :return: list of :class:`bioblend:Workflow - # """ - # return self.galaxy_instance.workflows.list() - def get_workflow_test_results(self, workflow_id=None): """ Return the list of :class:`WorkflowTestResult` instances resulting by the executed workflow tests. From c833235ccf5dff4df1f8fc8dc735d6b16ce5fe27 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 Oct 2016 16:58:18 +0200 Subject: [PATCH 061/588] Configurable logger for every test run --- workflow_tester.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/workflow_tester.py b/workflow_tester.py index 657a0e5..6c50816 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -775,7 +775,8 @@ def get_galaxy_workflow(self): return self._galaxy_workflow def run_test(self, base_path=None, inputs=None, expected_outputs=None, - output_folder=None, disable_assertions=None, disable_cleanup=None): + output_folder=None, disable_assertions=None, disable_cleanup=None, + enable_logger=None, enable_debug=None): """ Run the test with the given inputs and expected_outputs. @@ -819,9 +820,19 @@ def compare_outputs(actual_output_filename, expected_output_filename): :param disable_assertions: ``True`` to disable assertions during the workflow test execution; ``False`` (default) otherwise. + :type enable_logger: bool + :param enable_logger: 'True' to enable the logger (with INFO level); 'False' otherwise. + + :type enable_debug: bool + :param enable_debug: 'True' to enable the logger with DEBUG level; 'False' otherwise. + :rtype: :class:``WorkflowTestResult`` :return: workflow test result """ + # update logger + if enable_logger or enable_debug: + _logger.setLevel(_logging.DEBUG if enable_debug else _logging.INFO) + # set basepath base_path = self._base_path if not base_path else base_path From fac6f8d8536b5100f3979a9745d9c05a12e602b4 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 Oct 2016 17:08:31 +0200 Subject: [PATCH 062/588] Disabled assertions by default; enabled assertions when tests are launched using the unittest framework --- workflow_tester.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workflow_tester.py b/workflow_tester.py index 6c50816..a84f4e6 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -61,7 +61,7 @@ class WorkflowTestConfiguration: } def __init__(self, name=None, base_path=".", filename="workflow.ga", inputs={}, expected_outputs={}, - output_folder=None, disable_cleanup=True, disable_assertions=True): + output_folder=None, disable_cleanup=False, disable_assertions=True): """ Create a new class instance and initialize its initial properties. @@ -647,6 +647,7 @@ def run_test_suite(self, workflow_tests_config=None, enable_logger=None, suite_config = workflow_tests_config or self._workflow_test_suite_configuration self._suite_setup(suite_config, enable_logger, enable_debug, disable_cleanup, disable_assertions) for test_config in suite_config["workflows"].values(): + test_config.disable_assertions = False runner = self._create_test_runner(test_config) suite.addTest(runner) _RUNNER = _unittest.TextTestRunner(verbosity=2) From 74c0a39f2065f9f99dda1b7d5d276d31a8890803 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 20 Oct 2016 17:03:42 +0200 Subject: [PATCH 063/588] Fixed propagation of the suite configuration options to the runners --- workflow_tester.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index a84f4e6..f79b039 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -583,7 +583,7 @@ def _add_test_result(self, test_result): """ self._workflow_test_results.append(test_result) - def _create_test_runner(self, workflow_test_config): + def _create_test_runner(self, workflow_test_config, suite_config): """ Private method which creates a test runner associated to this suite. @@ -593,6 +593,12 @@ def _create_test_runner(self, workflow_test_config): :rtype: :class:'WorkflowTestRunner' :return: the created :class:'WorkflowTestResult' instance """ + # update test config + workflow_test_config.disable_cleanup = suite_config.get("disable_cleanup", False) + workflow_test_config.disable_assertions = suite_config.get("disable_assertions", False) + workflow_test_config.enable_logger = suite_config.get("enable_logger", False) + workflow_test_config.enable_debug = suite_config.get("enable_debug", False) + # create a new runner instance runner = WorkflowTestRunner(self.galaxy_instance, self.workflow_loader, workflow_test_config, self) self._workflow_runners.append(runner) return runner @@ -606,7 +612,7 @@ def _suite_setup(self, config, enable_logger=None, config["disable_assertions"] = disable_assertions \ if not disable_assertions is None else config.get("disable_assertions", False) # update logger level - if config.get("enable_logger", True): + if config.get("enable_logger", True) or config.get("enable_debug", True): config["logger_level"] = _logging.DEBUG if config.get("enable_debug", False) else _logging.INFO _logger.setLevel(config["logger_level"]) @@ -626,7 +632,7 @@ def run_tests(self, workflow_tests_config=None, enable_logger=None, suite_config = workflow_tests_config or self._workflow_test_suite_configuration self._suite_setup(suite_config, enable_logger, enable_debug, disable_cleanup, disable_assertions) for test_config in suite_config["workflows"].values(): - runner = self._create_test_runner(test_config) + runner = self._create_test_runner(test_config, suite_config) result = runner.run_test() results.append(result) # cleanup @@ -648,7 +654,7 @@ def run_test_suite(self, workflow_tests_config=None, enable_logger=None, self._suite_setup(suite_config, enable_logger, enable_debug, disable_cleanup, disable_assertions) for test_config in suite_config["workflows"].values(): test_config.disable_assertions = False - runner = self._create_test_runner(test_config) + runner = self._create_test_runner(test_config, suite_config) suite.addTest(runner) _RUNNER = _unittest.TextTestRunner(verbosity=2) _RUNNER.run((suite)) From f0a2c4d488152339360f42be048c9145cd6158af Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 21 Oct 2016 15:50:53 +0200 Subject: [PATCH 064/588] Catch runtime errors during workflow execution --- workflow_tester.py | 49 +++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index f79b039..4ac838d 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -874,6 +874,9 @@ def compare_outputs(actual_output_filename, expected_output_filename): # store the current message error_msg = None + # test restul + test_result = None + # check tools missing_tools = self.find_missing_tools() if len(missing_tools) == 0: @@ -892,30 +895,36 @@ def compare_outputs(actual_output_filename, expected_output_filename): for filename in config["file"]: datamap[label].append(history.upload_dataset(_os.path.join(base_path, filename))) - # run the workflow - _logger.info("Workflow '%s' (id: %s) running ...", workflow.name, workflow.id) - outputs, output_history = workflow.run(datamap, history, wait=True, polling_interval=0.5) - _logger.info("Workflow '%s' (id: %s) executed", workflow.name, workflow.id) - - # check outputs - results, output_file_map = self._check_outputs(base_path, outputs, expected_outputs, output_folder) - - # instantiate the result object - test_result = _WorkflowTestResult(test_uuid, workflow, inputs, outputs, output_history, - expected_outputs, missing_tools, results, output_file_map, - output_folder) - if test_result.failed(): - error_msg = "The actual output{0} {2} differ{1} from the expected one{0}." \ - .format("" if len(test_result.failed_outputs) == 1 else "s", - "" if len(test_result.failed_outputs) > 1 else "s", - ", ".join(["'{0}'".format(n) for n in test_result.failed_outputs])) + try: + # run the workflow + _logger.info("Workflow '%s' (id: %s) running ...", workflow.name, workflow.id) + outputs, output_history = workflow.run(datamap, history, wait=True, polling_interval=0.5) + _logger.info("Workflow '%s' (id: %s) executed", workflow.name, workflow.id) + + # check outputs + results, output_file_map = self._check_outputs(base_path, outputs, expected_outputs, output_folder) + + # instantiate the result object + test_result = _WorkflowTestResult(test_uuid, workflow, inputs, outputs, output_history, + expected_outputs, missing_tools, results, output_file_map, + output_folder) + if test_result.failed(): + error_msg = "The actual output{0} {2} differ{1} from the expected one{0}." \ + .format("" if len(test_result.failed_outputs) == 1 else "s", + "" if len(test_result.failed_outputs) > 1 else "s", + ", ".join(["'{0}'".format(n) for n in test_result.failed_outputs])) + except RuntimeError, e: + error_msg = "Runtime error: {0}".format(e.message) + _logger.error(error_msg) else: - # instantiate the result object - test_result = _WorkflowTestResult(test_uuid, workflow, inputs, [], None, - expected_outputs, missing_tools, [], {}, output_folder) error_msg = "Some workflow tools are not available in Galaxy: {0}".format(", ".join(missing_tools)) + # instantiate the result object + if not test_result: + test_result = _WorkflowTestResult(test_uuid, workflow, inputs, [], None, + expected_outputs, missing_tools, {}, {}, output_folder) + # store result self._test_cases[test_uuid] = test_result if self._test_suite: From 35f6678a520b019ab323d8210763f18710b68a9b Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 25 Oct 2016 16:18:02 +0200 Subject: [PATCH 065/588] Added new debug message --- workflow_tester.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/workflow_tester.py b/workflow_tester.py index 4ac838d..9ccd740 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -1014,6 +1014,7 @@ def _check_outputs(self, base_path, actual_outputs, expected_output_map, output_ return (results, output_file_map) def cleanup(self): + _logger.debug("Cleanup of workflow test '%s'...", self._test_uuid) for test_uuid, test_result in self._test_cases.items(): if test_result.output_history: self._galaxy_instance.histories.delete(test_result.output_history.id) @@ -1021,6 +1022,7 @@ def cleanup(self): if self._galaxy_workflow: self._workflow_loader.unload_workflow(self._galaxy_workflow.id) self._galaxy_workflow = None + _logger.debug("Cleanup of workflow test '%s': DONE", self._test_uuid) def cleanup_output_folder(self, test_result=None): test_results = self._test_cases.values() if not test_result else [test_result] From d66a74bb4c02e1a4e8fe749f233acc8a04801e08 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 26 Oct 2016 10:57:58 +0200 Subject: [PATCH 066/588] Added 'params' to the WorkflowTest configuration --- workflow_tester.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 9ccd740..3021785 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -60,8 +60,8 @@ class WorkflowTestConfiguration: } } - def __init__(self, name=None, base_path=".", filename="workflow.ga", inputs={}, expected_outputs={}, - output_folder=None, disable_cleanup=False, disable_assertions=True): + def __init__(self, name=None, base_path=".", filename="workflow.ga", inputs={}, params={}, + expected_outputs={}, output_folder=None, disable_cleanup=False, disable_assertions=True): """ Create a new class instance and initialize its initial properties. @@ -77,6 +77,19 @@ def __init__(self, name=None, base_path=".", filename="workflow.ga", inputs={}, :type inputs: dict :param inputs: a map : (e.g., {"input_name" : {"file": ...}} + :type params: dict + :param params: maps each step with the corresponding dict of params + + :Example: + params = {3: + { + "orthoI": "NA" + "predI": "1" + "respC": "gender" + "testL": "FALSE" + } + } + :type expected_outputs: dict :param expected_outputs: maps actual to expected outputs. Each output requires a dict containing the path of the expected output filename @@ -116,6 +129,7 @@ def compare_outputs(actual_output_filename, expected_output_filename): self._base_path = None self._filename = None self._inputs = {} + self._params = {} self._expected_outputs = {} # set parameters @@ -123,6 +137,7 @@ def compare_outputs(actual_output_filename, expected_output_filename): self.set_base_path(base_path) self.set_filename(filename) self.set_inputs(inputs) + self.set_params(params) self.set_expected_outputs(expected_outputs) self.output_folder = _os.path.join(self.DEFAULT_OUTPUT_FOLDER, self.name) \ if output_folder is None else output_folder From 38622338451c027bfa81d4ae33b5ffa722b3bbc8 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 26 Oct 2016 10:59:53 +0200 Subject: [PATCH 067/588] Updated load & dump methods to support workflow parameters --- workflow_tester.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/workflow_tester.py b/workflow_tester.py index 3021785..61a80bb 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -305,6 +305,7 @@ def to_json(self): "name": self.name, "file": self.filename, "inputs": {name: input["file"][0] for name, input in self.inputs.items()}, + "params": self.params, "expected": self.expected_outputs }) @@ -338,7 +339,8 @@ def load(filename=DEFAULT_CONFIG_FILENAME, workflow_test_name=None): wf_config.get("output_folder", wf_name)) # add the workflow w = WorkflowTestConfiguration(name=wf_name, base_path=base_path, filename=wf_config["file"], - inputs=wf_config["inputs"], expected_outputs=wf_config["expected"], + inputs=wf_config["inputs"], params=wf_config.get("params", {}), + expected_outputs=wf_config["expected"], output_folder=wf_config["output_folder"]) config["workflows"][wf_name] = w # returns the current workflow test config From 95d4ad41f38d9ef688e6f2a8f7c2eaa37b6b4f47 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 26 Oct 2016 11:00:24 +0200 Subject: [PATCH 068/588] Added getters/setters methods to the WorkflowTestConfig class --- workflow_tester.py | 78 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/workflow_tester.py b/workflow_tester.py index 61a80bb..70288f2 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -226,6 +226,84 @@ def get_input(self, name): """ return self._inputs.get(name, None) + @property + def params(self): + """ + Return the dict of parameters + + :rtype: dict + :return: dict of params + """ + return self._params + + def set_params(self, params): + """ + Update the set of parameters of each step + + :type params: dict + :param params: dict of params indexed by step id + """ + for step_id, step_params in params.items(): + for name, value in step_params.items(): + self.add_param(step_id, name, value) + + def add_param(self, step_id, name, value): + """ + Add a new parameter of the step identified by 'step_id'. + + :type step_id: int + :param step_id: step index + + :type name: str + :param name: name of the parameter + + :type value: str + :param value: the value of the parameter + """ + if not self._params.has_key(step_id): + self._params[step_id] = {} + self._params[step_id][name] = value + + def remove_param(self, step_id, name): + """ + Remove a parameter of the step 'step_id'. + + :type step_id: int + :param step_id: step index + + :type name: str + :param name: name of the parameter to be removed + """ + if self._params.has_key(step_id): + del self._params[step_id][name] + + def get_params(self, step_id): + """ + Return the dict of parameters related to the step indexed by 'step_id'. + + :type step_id: int + :param step_id: the step index + + :rtype: dict + :return: the dict of parameters related to the step indexed by 'step_id' + """ + return self._params.get(step_id, None) + + def get_param(self, step_id, name): + """ + Return the value of a specific parameter + + :type step_id: int + :param step_id: the index of the step which the parameter is related to + + :type name: str + :param name: the name of the parameter to be returned + + :return: the value of the requested parameter + """ + step_params = self._params.get(step_id, None) + return step_params.get(name, None) if step_params else None + @property def expected_outputs(self): return self._expected_outputs From 4300491e6c6e6f1b109921d2a04ba2ea4d41b6ff Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Wed, 26 Oct 2016 11:08:03 +0200 Subject: [PATCH 069/588] Updated 'the run_test' method of the WorkflowTestRunner to submit the new 'params' parameter --- workflow_tester.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/workflow_tester.py b/workflow_tester.py index 70288f2..4fb4cc9 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -876,7 +876,7 @@ def get_galaxy_workflow(self): self._galaxy_workflow = self._workflow_loader.load_workflow(self._workflow_test_config) return self._galaxy_workflow - def run_test(self, base_path=None, inputs=None, expected_outputs=None, + def run_test(self, base_path=None, inputs=None, params=None, expected_outputs=None, output_folder=None, disable_assertions=None, disable_cleanup=None, enable_logger=None, enable_debug=None): """ @@ -951,6 +951,12 @@ def compare_outputs(actual_output_filename, expected_output_filename): inputs = self._workflow_test_config.inputs else: raise ValueError("No input configured !!!") + + # check params + if not params: + params = self._workflow_test_config.params + _logger.debug("Using default params") + # check expected_output_map if not expected_outputs: if len(self._workflow_test_config.expected_outputs) > 0: @@ -993,7 +999,7 @@ def compare_outputs(actual_output_filename, expected_output_filename): try: # run the workflow _logger.info("Workflow '%s' (id: %s) running ...", workflow.name, workflow.id) - outputs, output_history = workflow.run(datamap, history, wait=True, polling_interval=0.5) + outputs, output_history = workflow.run(datamap, history, params=params, wait=True, polling_interval=0.5) _logger.info("Workflow '%s' (id: %s) executed", workflow.name, workflow.id) # check outputs From 234efbbb89906534190e6a1808a90d40a7eccfed Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 27 Oct 2016 08:36:32 +0200 Subject: [PATCH 070/588] Changed name of the actual output filename --- workflow_tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow_tester.py b/workflow_tester.py index 4fb4cc9..eaad8f8 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -1091,7 +1091,7 @@ def _check_outputs(self, base_path, actual_outputs, expected_output_map, output_ for output in actual_outputs: if output.name in expected_output_map: _logger.debug("Checking OUTPUT '%s' ...", output.name) - output_filename = _os.path.join(output_folder, "output_" + str(actual_outputs.index(output))) + output_filename = _os.path.join(output_folder, output.name) with open(output_filename, "w") as out_file: output.download(out_file) output_file_map[output.name] = {"dataset": output, "filename": output_filename} From 9a0c7603ee364e32cc6a2832b3c6bb530b179cb5 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 27 Oct 2016 08:37:51 +0200 Subject: [PATCH 071/588] Updated default comparator to write out diff results --- workflow_tester.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/workflow_tester.py b/workflow_tester.py index eaad8f8..54dba96 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -1300,6 +1300,10 @@ def base_comparator(actual_output_filename, expected_output_filename): ldiff = list(diff) if len(ldiff) > 0: print "\n{0}\n...\n".format("".join(ldiff[:20])) + diff_filename = _os.path.join(_os.path.dirname(actual_output_filename), + _os.path.basename(actual_output_filename) + ".diff") + with open(diff_filename, "w") as out_fp: + out_fp.writelines("%s\n" % item for item in ldiff) return len(ldiff) == 0 From f4678256ab3c65c90ba8d5c966172372eb75facf Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Thu, 27 Oct 2016 09:39:34 +0200 Subject: [PATCH 072/588] Fixed content of the diff output file --- workflow_tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow_tester.py b/workflow_tester.py index 54dba96..b68629f 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -1303,7 +1303,7 @@ def base_comparator(actual_output_filename, expected_output_filename): diff_filename = _os.path.join(_os.path.dirname(actual_output_filename), _os.path.basename(actual_output_filename) + ".diff") with open(diff_filename, "w") as out_fp: - out_fp.writelines("%s\n" % item for item in ldiff) + out_fp.writelines("%r\n"% item.rstrip('\n') for item in ldiff) return len(ldiff) == 0 From cd35073aecce79787bef2c4fd91f4590b4155ae6 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 28 Oct 2016 18:05:47 +0200 Subject: [PATCH 073/588] Updated requirements --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 50406b1..a562f69 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -bioblend>=0.8.0 \ No newline at end of file +bioblend>=0.8.0 +ruamel.yaml \ No newline at end of file From 488b2214707d1e51a9e6b1a690a05682ad28f74e Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 28 Oct 2016 18:07:18 +0200 Subject: [PATCH 074/588] Updated imports --- workflow_tester.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/workflow_tester.py b/workflow_tester.py index b68629f..21658a1 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -5,16 +5,20 @@ import logging as _logging import unittest as _unittest import optparse as _optparse +import tarfile as _tarfile +from lxml import etree as _etree from uuid import uuid1 as _uuid1 from sys import exc_info as _exc_info from difflib import unified_diff as _unified_diff from yaml import load as _yaml_load, dump as _yaml_dump -from json import load as _json_load, dumps as _json_dumps +from ruamel.yaml.comments import CommentedMap as _CommentedMap +from json import load as _json_load, loads as _json_loads, dumps as _json_dumps from bioblend.galaxy.objects import GalaxyInstance as _GalaxyInstance from bioblend.galaxy.workflows import WorkflowClient as _WorkflowClient from bioblend.galaxy.histories import HistoryClient as _HistoryClient +from bioblend.galaxy.tools import ToolClient as _ToolClient # Galaxy ENV variable names ENV_KEY_GALAXY_URL = "BIOBLEND_GALAXY_URL" From 3e3e798d013c817d3f655c45150d743a95854867 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Fri, 28 Oct 2016 18:08:33 +0200 Subject: [PATCH 075/588] Added utility function able to parse a workflow definition --- workflow_tester.py | 152 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 151 insertions(+), 1 deletion(-) diff --git a/workflow_tester.py b/workflow_tester.py index 21658a1..5ec3872 100755 --- a/workflow_tester.py +++ b/workflow_tester.py @@ -24,6 +24,9 @@ ENV_KEY_GALAXY_URL = "BIOBLEND_GALAXY_URL" ENV_KEY_GALAXY_API_KEY = "BIOBLEND_GALAXY_API_KEY" +# Default folder where tool configuration is downloaded +DEFAULT_TOOLS_FOLDER = "tools" + # configure module logger _logger = _logging.getLogger("WorkflowTest") _logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s') @@ -1215,6 +1218,153 @@ def check_outputs(self): return self.results +def get_workflow_info(filename, tool_folder=DEFAULT_TOOLS_FOLDER, galaxy_url=None, galaxy_api_key=None): + inputs = [] + params = _CommentedMap() + expected_outputs = {} + + # setup galaxy instance + galaxy_instance = _get_galaxy_instance(galaxy_url, galaxy_api_key) + galaxy_tool_client = _ToolClient(galaxy_instance.gi) + + if not _os.path.exists(DEFAULT_TOOLS_FOLDER): + _os.makedirs(DEFAULT_TOOLS_FOLDER) + + with open(filename) as fp: + wf_config = _json_load(fp) + + for sid, step in wf_config["steps"].items(): + # tool = gi.tools.get() + + _logger.debug("Processing step '%s' -- '%s'", sid, step["name"]) + + # an input step.... + if not step["tool_id"] and step["type"] == "data_input": + for input in step["inputs"]: + _logger.debug("Processing input: '%s' (%s)", input["name"], input["description"]) + inputs.append(input) + + # a processing step (with outputs) ... + if step["tool_id"] and step["type"] == "tool": + + # tool parameters + tool_params = _CommentedMap() + + # process tool info to extract parameters + tool_id = step["tool_id"] + tool = galaxy_instance.tools.get(tool_id) + tool_json = _json_loads(tool.to_json()) + tool_config_xml = _os.path.basename(tool_json["config_file"]) + _logger.debug("Processing step tool '%s'", tool_id) + + try: + _logger.debug("Download TOOL '%s' definition file XML: %s....", tool_id, tool_config_xml) + targz_filename = _os.path.join(DEFAULT_TOOLS_FOLDER, tool_id + ".tar.gz") + targz_content = galaxy_tool_client._get(_os.path.join(tool_id, "download"), json=False) + if targz_content.status_code == 200: + with open(targz_filename, "w") as tfp: + tfp.write(targz_content.content) + + tar = _tarfile.open(targz_filename) + tar.extractall(path=tool_folder) + tar.close() + _logger.debug("Download TOOL '%s' definition file XML: %s....: DONE", tool_id, tool_config_xml) + else: + _logger.debug("Download TOOL '%s' definition file XML: %s....: ERROR %r", + tool_id, tool_config_xml, targz_content.status_code) + + tool_config_xml = _os.path.join(DEFAULT_TOOLS_FOLDER, tool_config_xml) + if _os.path.exists(tool_config_xml): + tree = _etree.parse(tool_config_xml) + root = tree.getroot() + inputs_el = root.find("inputs") + for input_el in inputs_el: + _process_tool_param_element(input_el, tool_params) + if len(tool_params) > 0: + params.insert(int(sid), sid, tool_params) + + except Exception, e: + _logger.debug("Download TOOL '%s' definition file XML: %s....: ERROR", tool_id, tool_config_xml) + _logger.error(e) + + # process + for output in step["workflow_outputs"]: + expected_outputs[output["uuid"]] = output + + return wf_config, inputs, params, expected_outputs + + +def _process_tool_param_element(input_el, tool_params): + """ + Parameter types: + 1) text X + 2) integer and float X + 3) boolean X + 4) data X (no default option) + 5) select ~ (not with OPTIONS) + 6) data_column X (uses the default_value attribute) + 7) data_collection X (no default option) + 8) drill_down X (no default option) + 9) color X + + Tag