diff --git a/README.md b/README.md index 37aa69e..c5be346 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,19 @@ Calrissian is designed to issue tasks in parallel if they are independent, and t When running `calrissian`, you must provide a limit the the number of CPU cores (`--max-cores`) and RAM megabytes (`--max-ram`) to use concurrently. Calrissian will use CWL [ResourceRequirements](https://www.commonwl.org/v1.0/CommandLineTool.html#ResourceRequirement) to track usage and stay within the limits provided. We highly recommend using accurate ResourceRequirements in your workloads, so that they can be scheduled efficiently and are less likely to be terminated or refused by the cluster. +`calrissian` parameters can be provided via a JSON configuration file either stored under `~/.calrissian/default.json` or provided via the `--conf` option. + +Below an example of such a file: + +```json +{ + "max_ram": "16G", + "max_cores": "10", + "outdir": "/calrissian", + "tmpdir_prefix": "/calrissian/tmp" +} +``` + ## CWL Conformance Calrissian leverages [cwltool](https://github.com/common-workflow-language/cwltool) heavily and most conformance tests for CWL v1.0. Please see [conformance](conformance) for further details and processes. diff --git a/calrissian/main.py b/calrissian/main.py index 5f75e02..7134ac9 100644 --- a/calrissian/main.py +++ b/calrissian/main.py @@ -1,3 +1,4 @@ +import contextlib from calrissian.executor import ThreadPoolJobExecutor from calrissian.context import CalrissianLoadingContext, CalrissianRuntimeContext from calrissian.version import version @@ -12,6 +13,7 @@ import subprocess import os import shlex +import json log = logging.getLogger("calrissian.main") @@ -45,13 +47,28 @@ def add_arguments(parser): parser.add_argument('--stdout', type=Text, nargs='?', help='Output file name to tee standard output (CWL output object)') parser.add_argument('--stderr', type=Text, nargs='?', help='Output file name to tee standard error to (includes tool logs)') parser.add_argument('--tool-logs-basepath', type=Text, nargs='?', help='Base path for saving the tool logs') + parser.add_argument('--conf', help='Defines the default values for the CLI arguments', action='append') def print_version(): print(version()) def parse_arguments(parser): + + # read default config from file args = parser.parse_args() + + with contextlib.suppress(KeyError, FileNotFoundError): + with open(os.path.join(os.environ["HOME"], ".calrissian", "default.json"), 'r') as f: + parser.set_defaults(**json.load(f)) + + if args.conf is not None: + + with open(args.conf[0], 'r') as f: + parser.set_defaults(**json.load(f)) + + args = parser.parse_args() + # Check for version arg if args.version: print_version() diff --git a/tests/default-conf.json b/tests/default-conf.json new file mode 100644 index 0000000..73a4e30 --- /dev/null +++ b/tests/default-conf.json @@ -0,0 +1,6 @@ +{ + "max_ram": "16G", + "max_cores": "10", + "outdir": "results", + "tmp_out_dir_prefix": "tmp" +} \ No newline at end of file diff --git a/tests/test_main.py b/tests/test_main.py index cf22168..149c01f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -32,7 +32,7 @@ def test_main_calls_cwlmain_returns_exit_code(self, mock_activate_logging, mock_ mock_install_signal_handler, mock_delete_pods, mock_add_arguments, mock_parse_arguments, mock_version, mock_runtime_context, mock_loading_context, mock_executor, - mock_arg_parser, mock_cwlmain): + mock_arg_parser, mock_cwlmain,): mock_exit_code = Mock() mock_cwlmain.return_value = mock_exit_code # not called before main result = main() @@ -63,26 +63,26 @@ def test_main_calls_cwlmain_returns_exit_code(self, mock_activate_logging, mock_ def test_add_arguments(self): mock_parser = Mock() add_arguments(mock_parser) - self.assertEqual(mock_parser.add_argument.call_count, 10) + self.assertEqual(mock_parser.add_argument.call_count, 11) @patch('calrissian.main.sys') def test_parse_arguments_exits_without_ram_or_cores(self, mock_sys): mock_parser = Mock() - mock_parser.parse_args.return_value = Mock(max_ram=None, max_cores=None, version=None) + mock_parser.parse_args.return_value = Mock(max_ram=None, max_cores=None, version=None, conf=None) parse_arguments(mock_parser) self.assertEqual(mock_sys.exit.call_args, call(1)) @patch('calrissian.main.sys') def test_parse_arguments_exits_with_ram_but_no_cores(self, mock_sys): mock_parser = Mock() - mock_parser.parse_args.return_value = Mock(max_ram=2048, max_cores=None, version=None) + mock_parser.parse_args.return_value = Mock(max_ram=2048, max_cores=None, version=None, conf=None) parse_arguments(mock_parser) self.assertEqual(mock_sys.exit.call_args, call(1)) @patch('calrissian.main.sys') def test_parse_arguments_succeeds_with_ram_and_cores(self, mock_sys): mock_parser = Mock() - mock_parser.parse_args.return_value = Mock(max_ram=2048, max_cores=3, version=None) + mock_parser.parse_args.return_value = Mock(max_ram=2048, max_cores=3, version=None, conf=None) parsed = parse_arguments(mock_parser) self.assertEqual(parsed, mock_parser.parse_args.return_value) self.assertFalse(mock_sys.exit.called) @@ -91,7 +91,7 @@ def test_parse_arguments_succeeds_with_ram_and_cores(self, mock_sys): @patch('calrissian.main.print_version') def test_parse_arguments_exits_with_version(self, mock_print_version, mock_sys): mock_parser = Mock() - mock_parser.parse_args.return_value = Mock(version=True) + mock_parser.parse_args.return_value = Mock(version=True, conf=None) parsed = parse_arguments(mock_parser) self.assertEqual(parsed, mock_parser.parse_args.return_value) self.assertTrue(mock_print_version.called)