diff --git a/src/sophios/api/pythonapi.py b/src/sophios/api/pythonapi.py index a37680c4..a5984c5f 100644 --- a/src/sophios/api/pythonapi.py +++ b/src/sophios/api/pythonapi.py @@ -760,7 +760,8 @@ def run(self) -> None: # `docker run` will NOT query the remote repository for the latest image! # cwltool has a --force-docker-pull option, but this may cause multiple pulls in parallel. if args.container_engine == 'singularity': - cmd = ['cwl-docker-extract', f'-s --dir {args.homedir}', f'autogenerated/{self.process_name}.cwl'] + cmd = ['cwl-docker-extract', '-s', '--dir', + f'{args.singularity_pull_dir}', f'autogenerated/{self.process_name}.cwl'] else: cmd = ['cwl-docker-extract', '--force-download', f'autogenerated/{self.process_name}.cwl'] sub.run(cmd, check=True) diff --git a/src/sophios/cli.py b/src/sophios/cli.py index c892d589..f5089601 100644 --- a/src/sophios/cli.py +++ b/src/sophios/cli.py @@ -28,6 +28,9 @@ parser.add_argument('--homedir', type=str, required=False, default=str(Path().home()), help='''The users home directory. This is necessary because CWL clears environment variables (e.g. HOME)''') +parser.add_argument('--singularity_pull_dir', type=str, required=False, default=str(Path().cwd()), + help='''The user specified pull directory for singularity image pull. + The default is the current working directory i.e. `pwd`''') parser.add_argument('--insert_steps_automatically', default=False, action="store_true", help='''Attempt to fix inference failures by speculatively inserting workflow steps from a curated whitelist.''') diff --git a/src/sophios/main.py b/src/sophios/main.py index 23837057..24a6dbdd 100644 --- a/src/sophios/main.py +++ b/src/sophios/main.py @@ -204,7 +204,8 @@ def main() -> None: # `docker run` will NOT query the remote repository for the latest image! # cwltool has a --force-docker-pull option, but this may cause multiple pulls in parallel. if args.container_engine == 'singularity': - cmd = ['cwl-docker-extract', '-s', '--dir', f'{args.homedir}', f'autogenerated/{yaml_stem}.cwl'] + cmd = ['cwl-docker-extract', '-s', '--dir', + f'{args.singularity_pull_dir}', f'autogenerated/{yaml_stem}.cwl'] else: cmd = ['cwl-docker-extract', '--force-download', f'autogenerated/{yaml_stem}.cwl'] sub.run(cmd, check=True) diff --git a/src/sophios/run_local.py b/src/sophios/run_local.py index 57034123..fe407775 100644 --- a/src/sophios/run_local.py +++ b/src/sophios/run_local.py @@ -63,47 +63,69 @@ def run_local(args: argparse.Namespace, rose_tree: RoseTree, cachedir: Optional[ Returns: retval: The return value """ - + docker_like_engines = ['docker', 'podman'] docker_cmd: str = args.container_engine # Check that docker is installed, so users don't get a nasty runtime error. - cmd = [docker_cmd, 'run', '--rm', 'hello-world'] - output = '' - try: - docker_cmd_exists = True - proc = sub.run(cmd, check=False, stdout=sub.PIPE, stderr=sub.STDOUT) - output = proc.stdout.decode("utf-8") - except FileNotFoundError: - docker_cmd_exists = False - out_d = "Hello from Docker!" - out_p = "Hello Podman World" - permission_denied = 'permission denied while trying to connect to the Docker daemon socket at' - if (not docker_cmd_exists or not (proc.returncode == 0 and out_d in output or out_p in output)) and not args.ignore_docker_install: - if permission_denied in output: - print('Warning! docker appears to be installed, but not configured as a non-root user.') - print('See https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user') - print('TL;DR you probably just need to run the following command (and then restart your machine)') - print('sudo usermod -aG docker $USER') + if docker_cmd in docker_like_engines: + cmd = [docker_cmd, 'run', '--rm', 'hello-world'] + output = '' + try: + docker_cmd_exists = True + proc = sub.run(cmd, check=False, stdout=sub.PIPE, stderr=sub.STDOUT) + output = proc.stdout.decode("utf-8") + except FileNotFoundError: + docker_cmd_exists = False + out_d = "Hello from Docker!" + out_p = "Hello Podman World" + permission_denied = 'permission denied while trying to connect to the Docker daemon socket at' + if ((not docker_cmd_exists + or not (proc.returncode == 0 and out_d in output or out_p in output)) + and not args.ignore_docker_install): + + if permission_denied in output: + print('Warning! docker appears to be installed, but not configured as a non-root user.') + print('See https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user') + print('TL;DR you probably just need to run the following command (and then restart your machine)') + print('sudo usermod -aG docker $USER') + sys.exit(1) + + print(f'Warning! The {docker_cmd} command does not appear to be installed.') + print(f"""Most workflows require docker containers and + will fail at runtime if {docker_cmd} is not installed.""") + print('If you want to try running the workflow anyway, use --ignore_docker_install') + print("""Note that --ignore_docker_install does + NOT change whether or not any step in your workflow uses docker""") sys.exit(1) - print(f'Warning! The {docker_cmd} command does not appear to be installed.') - print(f'Most workflows require docker containers and will fail at runtime if {docker_cmd} is not installed.') - print('If you want to try running the workflow anyway, use --ignore_docker_install') - print('Note that --ignore_docker_install does NOT change whether or not any step in your workflow uses docker') - sys.exit(1) - - # If docker is installed, check for too many running processes. (on linux, macos) - if docker_cmd == 'docker' and docker_cmd_exists and sys.platform != "win32": - cmd = 'pgrep com.docker | wc -l' # type: ignore - proc = sub.run(cmd, check=False, stdout=sub.PIPE, stderr=sub.STDOUT, shell=True) - output = proc.stdout.decode("utf-8") - num_processes = int(output.strip()) - max_processes = 1000 - if num_processes > max_processes and not args.ignore_docker_processes: - print(f'Warning! There are {num_processes} running docker processes.') - print(f'More than {max_processes} may potentially cause intermittent hanging issues.') - print('It is recommended to terminate the processes using the command') - print('`sudo pkill com.docker && sudo pkill Docker`') - print('and then restart Docker.') - print('If you want to run the workflow anyway, use --ignore_docker_processes') + + # If docker is installed, check for too many running processes. (on linux, macos) + if docker_cmd_exists and sys.platform != "win32": + cmd = 'pgrep com.docker | wc -l' # type: ignore + proc = sub.run(cmd, check=False, stdout=sub.PIPE, stderr=sub.STDOUT, shell=True) + output = proc.stdout.decode("utf-8") + num_processes = int(output.strip()) + max_processes = 1000 + if num_processes > max_processes and not args.ignore_docker_processes: + print(f'Warning! There are {num_processes} running docker processes.') + print(f'More than {max_processes} may potentially cause intermittent hanging issues.') + print('It is recommended to terminate the processes using the command') + print('`sudo pkill com.docker && sudo pkill Docker`') + print('and then restart Docker.') + print('If you want to run the workflow anyway, use --ignore_docker_processes') + sys.exit(1) + else: + cmd = [docker_cmd, '--version'] + output = '' + try: + docker_cmd_exists = True + proc = sub.run(cmd, check=False, stdout=sub.PIPE, stderr=sub.STDOUT) + output = proc.stdout.decode("utf-8") + except FileNotFoundError: + docker_cmd_exists = False + if not docker_cmd_exists and not args.ignore_docker_install: + print(f'Warning! The {docker_cmd} command does not appear to be installed.') + print('If you want to try running the workflow anyway, use --ignore_docker_install') + print('Note that --ignore_docker_install does NOT change whether or not') + print('any step in your workflow uses docker or any other containers') sys.exit(1) yaml_path = args.yaml diff --git a/tests/test_examples.py b/tests/test_examples.py index abc60f59..153fdedb 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -218,7 +218,8 @@ def run_workflows(yml_path_str: str, yml_path: Path, cwl_runner: str, args: argp # `docker run` will NOT query the remote repository for the latest image! # cwltool has a --force-docker-pull option, but this may cause multiple pulls in parallel. if args.container_engine == 'singularity': - cmd = ['cwl-docker-extract', f'-s --dir {args.homedir}', f'autogenerated/{Path(yml_path).stem}.cwl'] + cmd = ['cwl-docker-extract', '-s', '--dir', + f'{args.singularity_pull_dir}', f'autogenerated/{Path(yml_path).stem}.cwl'] else: cmd = ['cwl-docker-extract', '--force-download', f'autogenerated/{Path(yml_path).stem}.cwl'] sub.run(cmd, check=True)