Fix for running workflows with singularity (#281)

* singularity flag and cmd fix

* fix checking for container_engine unlike docker on the platform

---------

Co-authored-by: Vasu Jaganath <vasu.jaganath@axleinfo.com>

src/sophios/api/pythonapi.py

@@ -760,7 +760,8 @@ def run(self) -> None:
         # `docker run` will NOT query the remote repository for the latest image!
         # cwltool has a --force-docker-pull option, but this may cause multiple pulls in parallel.
         if args.container_engine == 'singularity':
-            cmd = ['cwl-docker-extract', f'-s --dir {args.homedir}', f'autogenerated/{self.process_name}.cwl']
+            cmd = ['cwl-docker-extract', '-s', '--dir',
+                   f'{args.singularity_pull_dir}', f'autogenerated/{self.process_name}.cwl']
         else:
             cmd = ['cwl-docker-extract', '--force-download', f'autogenerated/{self.process_name}.cwl']
         sub.run(cmd, check=True)

src/sophios/cli.py

@@ -28,6 +28,9 @@
 parser.add_argument('--homedir', type=str, required=False, default=str(Path().home()),
                     help='''The users home directory.
                     This is necessary because CWL clears environment variables (e.g. HOME)''')
+parser.add_argument('--singularity_pull_dir', type=str, required=False, default=str(Path().cwd()),
+                    help='''The user specified pull directory for singularity image pull.
+                    The default is the current working directory i.e. `pwd`''')
 parser.add_argument('--insert_steps_automatically', default=False, action="store_true",
                     help='''Attempt to fix inference failures by speculatively
                     inserting workflow steps from a curated whitelist.''')

src/sophios/main.py

@@ -204,7 +204,8 @@ def main() -> None:
         # `docker run` will NOT query the remote repository for the latest image!
         # cwltool has a --force-docker-pull option, but this may cause multiple pulls in parallel.
         if args.container_engine == 'singularity':
-            cmd = ['cwl-docker-extract', '-s', '--dir', f'{args.homedir}', f'autogenerated/{yaml_stem}.cwl']
+            cmd = ['cwl-docker-extract', '-s', '--dir',
+                   f'{args.singularity_pull_dir}', f'autogenerated/{yaml_stem}.cwl']
         else:
             cmd = ['cwl-docker-extract', '--force-download', f'autogenerated/{yaml_stem}.cwl']
         sub.run(cmd, check=True)

src/sophios/run_local.py

@@ -63,47 +63,69 @@ def run_local(args: argparse.Namespace, rose_tree: RoseTree, cachedir: Optional[
     Returns:
         retval: The return value
     """
-
+    docker_like_engines = ['docker', 'podman']
     docker_cmd: str = args.container_engine
     # Check that docker is installed, so users don't get a nasty runtime error.
-    cmd = [docker_cmd, 'run', '--rm', 'hello-world']
-    output = ''
-    try:
-        docker_cmd_exists = True
-        proc = sub.run(cmd, check=False, stdout=sub.PIPE, stderr=sub.STDOUT)
-        output = proc.stdout.decode("utf-8")
-    except FileNotFoundError:
-        docker_cmd_exists = False
-    out_d = "Hello from Docker!"
-    out_p = "Hello Podman World"
-    permission_denied = 'permission denied while trying to connect to the Docker daemon socket at'
-    if (not docker_cmd_exists or not (proc.returncode == 0 and out_d in output or out_p in output)) and not args.ignore_docker_install:
-        if permission_denied in output:
-            print('Warning! docker appears to be installed, but not configured as a non-root user.')
-            print('See https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user')
-            print('TL;DR you probably just need to run the following command (and then restart your machine)')
-            print('sudo usermod -aG docker $USER')
+    if docker_cmd in docker_like_engines:
+        cmd = [docker_cmd, 'run', '--rm', 'hello-world']
+        output = ''
+        try:
+            docker_cmd_exists = True
+            proc = sub.run(cmd, check=False, stdout=sub.PIPE, stderr=sub.STDOUT)
+            output = proc.stdout.decode("utf-8")
+        except FileNotFoundError:
+            docker_cmd_exists = False
+        out_d = "Hello from Docker!"
+        out_p = "Hello Podman World"
+        permission_denied = 'permission denied while trying to connect to the Docker daemon socket at'
+        if ((not docker_cmd_exists
+            or not (proc.returncode == 0 and out_d in output or out_p in output))
+                and not args.ignore_docker_install):
+
+            if permission_denied in output:
+                print('Warning! docker appears to be installed, but not configured as a non-root user.')
+                print('See https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user')
+                print('TL;DR you probably just need to run the following command (and then restart your machine)')
+                print('sudo usermod -aG docker $USER')
+                sys.exit(1)
+
+            print(f'Warning! The {docker_cmd} command does not appear to be installed.')
+            print(f"""Most workflows require docker containers and
+                  will fail at runtime if {docker_cmd} is not installed.""")
+            print('If you want to try running the workflow anyway, use --ignore_docker_install')
+            print("""Note that --ignore_docker_install does
+                  NOT change whether or not any step in your workflow uses docker""")
             sys.exit(1)
-        print(f'Warning! The {docker_cmd} command does not appear to be installed.')
-        print(f'Most workflows require docker containers and will fail at runtime if {docker_cmd} is not installed.')
-        print('If you want to try running the workflow anyway, use --ignore_docker_install')
-        print('Note that --ignore_docker_install does NOT change whether or not any step in your workflow uses docker')
-        sys.exit(1)
-
-    # If docker is installed, check for too many running processes. (on linux, macos)
-    if docker_cmd == 'docker' and docker_cmd_exists and sys.platform != "win32":
-        cmd = 'pgrep com.docker | wc -l'  # type: ignore
-        proc = sub.run(cmd, check=False, stdout=sub.PIPE, stderr=sub.STDOUT, shell=True)
-        output = proc.stdout.decode("utf-8")
-        num_processes = int(output.strip())
-        max_processes = 1000
-        if num_processes > max_processes and not args.ignore_docker_processes:
-            print(f'Warning! There are {num_processes} running docker processes.')
-            print(f'More than {max_processes} may potentially cause intermittent hanging issues.')
-            print('It is recommended to terminate the processes using the command')
-            print('`sudo pkill com.docker && sudo pkill Docker`')
-            print('and then restart Docker.')
-            print('If you want to run the workflow anyway, use --ignore_docker_processes')
+
+        # If docker is installed, check for too many running processes. (on linux, macos)
+        if docker_cmd_exists and sys.platform != "win32":
+            cmd = 'pgrep com.docker | wc -l'  # type: ignore
+            proc = sub.run(cmd, check=False, stdout=sub.PIPE, stderr=sub.STDOUT, shell=True)
+            output = proc.stdout.decode("utf-8")
+            num_processes = int(output.strip())
+            max_processes = 1000
+            if num_processes > max_processes and not args.ignore_docker_processes:
+                print(f'Warning! There are {num_processes} running docker processes.')
+                print(f'More than {max_processes} may potentially cause intermittent hanging issues.')
+                print('It is recommended to terminate the processes using the command')
+                print('`sudo pkill com.docker && sudo pkill Docker`')
+                print('and then restart Docker.')
+                print('If you want to run the workflow anyway, use --ignore_docker_processes')
+                sys.exit(1)
+    else:
+        cmd = [docker_cmd, '--version']
+        output = ''
+        try:
+            docker_cmd_exists = True
+            proc = sub.run(cmd, check=False, stdout=sub.PIPE, stderr=sub.STDOUT)
+            output = proc.stdout.decode("utf-8")
+        except FileNotFoundError:
+            docker_cmd_exists = False
+        if not docker_cmd_exists and not args.ignore_docker_install:
+            print(f'Warning! The {docker_cmd} command does not appear to be installed.')
+            print('If you want to try running the workflow anyway, use --ignore_docker_install')
+            print('Note that --ignore_docker_install does NOT change whether or not')
+            print('any step in your workflow uses docker or any other containers')
             sys.exit(1)
 
     yaml_path = args.yaml

tests/test_examples.py

@@ -218,7 +218,8 @@ def run_workflows(yml_path_str: str, yml_path: Path, cwl_runner: str, args: argp
         # `docker run` will NOT query the remote repository for the latest image!
         # cwltool has a --force-docker-pull option, but this may cause multiple pulls in parallel.
         if args.container_engine == 'singularity':
-            cmd = ['cwl-docker-extract', f'-s --dir {args.homedir}', f'autogenerated/{Path(yml_path).stem}.cwl']
+            cmd = ['cwl-docker-extract', '-s', '--dir',
+                   f'{args.singularity_pull_dir}', f'autogenerated/{Path(yml_path).stem}.cwl']
         else:
             cmd = ['cwl-docker-extract', '--force-download', f'autogenerated/{Path(yml_path).stem}.cwl']
         sub.run(cmd, check=True)