Add ability to configure which type of shell to use for executing she…

…ll tasks (#45)

- Adds both global and task level options to specify which interpreters the content of a shell task is compatible with, including an option for powershell.
- Removes problematic reliance on the $SHELL env var for choosing a shell, and on WSL to find bash on windows.

README.rst

@@ -207,6 +207,57 @@ scripts *(shell)*, and sequence tasks *(sequence)*.
     pfwd = { "shell" = "ssh -N -L 0.0.0.0:8080:$STAGING:8080 $STAGING & ssh -N -L 0.0.0.0:5432:$STAGINGDB:5432 $STAGINGDB &" }
     pfwdstop = { "shell" = "kill $(pgrep -f "ssh -N -L .*:(8080|5432)")" }
 
+  By default poe attempts to find a posix shell (sh, bash, or zsh in that order) on the system and uses that. When running on windows, this might not always be possible. If bash is not found on the path on windows then poe will explicitly look for `Git bash <https://gitforwindows.org>`_ at the usual location.
+
+  **Using different types of shell/interpreter**
+
+  It is also possible to specify an alternative interpreter (or list of compatible interpreters ordered by preference) to be invoked to execute shell task content. For example if you only expect the task to be executed on windows or other environments with powershell installed then you can specify a powershell based task like so:
+
+  .. code-block:: toml
+
+    [tool.poe.tasks.install-poetry]
+    shell = """
+    (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python -
+    """
+    interpreter = "pwsh"
+
+  If your task content is restricted to syntax that is valid for both posix shells and powershell then you can maximise increase the likelihood of it working on any system by specifying the interpreter as:
+
+  .. code-block:: toml
+
+    interpreter = ["posix", "pwsh"]
+
+  It is also possible to specify python code as the shell task code as in the following example. However it is recommended to use a *script* task rather than writing complex code inline within your pyproject.toml.
+
+  .. code-block:: toml
+
+    [tool.poe.tasks.time]
+    shell = """
+    from datetime import datetime
+
+    print(datetime.now())
+    """
+    interpreter = "python"
+
+  The following interpreter values may be used:
+
+  posix
+      This is the default behavoir, equivalent to ["sh", "bash", "zsh"], meaning that poe will try to find sh, and fallback to bash, then zsh.
+  sh
+      Use the basic posix shell. This is often an alias for bash or dash depending on the operating system.
+  bash
+      Uses whatever version of bash can be found. This is usually the most portable option.
+  zsh
+      Uses whatever version of zsh can be found.
+  fish
+      Uses whatever version of fish can be found.
+  pwsh
+      Uses powershell version 6 or higher.
+  powershell
+      Uses the newest version of powershell that can be found.
+
+  The default value can be changed with the global *shell_interpreter* option as described below.
+
 - **Composite tasks** are defined as a sequence of other tasks as an array.
 
   By default the contents of the array are interpreted as references to other tasks
@@ -664,6 +715,29 @@ parent directory.
 
 See below for more details.
 
+Change the default shell interpreter
+------------------------------------
+
+Normally shell tasks are executed using a posix shell by default (see section for shell tasks above). This default can be overridden to something else by setting the *shell_interpreter* global option. In the following example we configure all shell tasks to use *fish* by default.
+
+.. code-block:: toml
+
+  tool.poe.shell_interpreter = "fish"
+
+  [tool.poe.tasks.fibonacci]
+  help = "Output the fibonacci sequence up to 89"
+  shell = """
+    function fib --argument-names max n0 n1
+      if test $max -ge $n0
+        echo $n0
+        fib $max $n1 (math $n0 + $n1)
+      end
+    end
+
+    fib 89 1 1
+  """
+
+
 Usage without poetry
 ====================
 

poethepoet/config.py

@@ -1,13 +1,23 @@
 from pathlib import Path
 import tomli
-from typing import Any, Dict, Mapping, Optional, Union
+from typing import Any, Dict, Mapping, Optional, Tuple, Union
 from .exceptions import PoeException
 
 
 class PoeConfig:
     _table: Mapping[str, Any]
 
     TOML_NAME = "pyproject.toml"
+    KNOWN_SHELL_INTERPRETERS = (
+        "posix",
+        "sh",
+        "bash",
+        "zsh",
+        "fish",
+        "pwsh",  # powershell >= 6
+        "powershell",  # any version of powershell
+        "python",
+    )
 
     # Options allowed directly under tool.poe in pyproject.toml
     __options__ = {
@@ -17,6 +27,7 @@ class PoeConfig:
         "env": dict,
         "envfile": str,
         "executor": dict,
+        "shell_interpreter": (str, list),
         "verbosity": int,
     }
 
@@ -57,6 +68,13 @@ def global_env(self) -> Dict[str, Union[str, Dict[str, str]]]:
     def global_envfile(self) -> Optional[str]:
         return self._poe.get("envfile")
 
+    @property
+    def shell_interpreter(self) -> Tuple[str, ...]:
+        raw_value = self._poe.get("shell_interpreter", "posix")
+        if isinstance(raw_value, list):
+            return tuple(raw_value)
+        return (raw_value,)
+
     @property
     def verbosity(self) -> int:
         return self._poe.get("verbosity", 0)
@@ -140,6 +158,14 @@ def validate(self):
             if error is None:
                 continue
             raise PoeException(error)
+
+        # validate shell_interpreter type
+        for interpreter in self.shell_interpreter:
+            if interpreter not in self.KNOWN_SHELL_INTERPRETERS:
+                return (
+                    f"Unsupported value {interpreter!r} for option `shell_interpreter`."
+                )
+
         # Validate default verbosity.
         if self.verbosity < -1 or self.verbosity > 1:
             raise PoeException(

poethepoet/task/shell.py

@@ -1,7 +1,18 @@
-import os
-import shutil
-import subprocess
-from typing import Dict, Mapping, Sequence, Tuple, Type, TYPE_CHECKING, Union
+from os import environ
+from shutil import which
+import sys
+from typing import (
+    Any,
+    Dict,
+    List,
+    Mapping,
+    Optional,
+    Sequence,
+    Tuple,
+    Type,
+    TYPE_CHECKING,
+    Union,
+)
 from ..exceptions import PoeException
 from .base import PoeTask
 
@@ -18,7 +29,7 @@ class ShellTask(PoeTask):
     content: str
 
     __key__ = "shell"
-    __options__: Dict[str, Union[Type, Tuple[Type, ...]]] = {}
+    __options__: Dict[str, Union[Type, Tuple[Type, ...]]] = {"interpreter": (str, list)}
 
     def _handle_run(
         self,
@@ -31,42 +42,120 @@ def _handle_run(
         if not has_named_args and any(arg.strip() for arg in extra_args):
             raise PoeException(f"Shell task {self.name!r} does not accept arguments")
 
-        if self._is_windows:
-            shell = self._find_posix_shell_on_windows()
-        else:
-            # Prefer to use configured shell, otherwise look for bash
-            shell = [os.environ.get("SHELL", shutil.which("bash") or "/bin/bash")]
+        interpreter_cmd = self.resolve_interpreter_cmd()
+        if not interpreter_cmd:
+            config_value = self._get_interpreter_config()
+            message = f"Couldn't locate interpreter executable for {config_value!r} to run shell task. "
+            if self._is_windows and config_value in ("posix", "bash"):
+                message += "Installing Git Bash or using WSL should fix this."
+            else:
+                message += "Some dependencies may be missing from your system."
+            raise PoeException(message)
 
         self._print_action(self.content, context.dry)
         return context.get_executor(self.invocation, env, self.options).execute(
-            shell, input=self.content.encode()
+            interpreter_cmd, input=self.content.encode()
         )
 
-    @staticmethod
-    def _find_posix_shell_on_windows():
-        # Try locate a shell from the environment
-        shell_from_env = shutil.which(os.environ.get("SHELL", "bash"))
-        if shell_from_env:
-            return [shell_from_env]
-
-        # Try locate a bash from the environment
-        bash_from_env = shutil.which("bash")
-        if bash_from_env:
-            return [bash_from_env]
-
-        # Or check specifically for git bash
-        bash_from_git = shutil.which("C:\\Program Files\\Git\\bin\\bash.exe")
-        if bash_from_git:
-            return [bash_from_git]
-
-        # or use bash from wsl if it's available
-        wsl = shutil.which("wsl")
-        if wsl and subprocess.run(["wsl", "bash"], capture_output=True).returncode > 0:
-            return [wsl, "bash"]
-
-        # Fail: if there is a bash out there, we don't know how to get to it
-        # > We don't know how to access wsl bash from python installed from python.org
-        raise PoeException(
-            "Couldn't locate bash executable to run shell task. Installing WSL should "
-            "fix this."
+    def _get_interpreter_config(self) -> Tuple[str, ...]:
+        result: Union[str, Tuple[str, ...]] = self.options.get(
+            "interpreter", self._config.shell_interpreter
         )
+        if isinstance(result, str):
+            return (result,)
+        return tuple(result)
+
+    def resolve_interpreter_cmd(self) -> Optional[List[str]]:
+        """
+        Return a formatted command for the first specified interpreter that can be
+        located.
+        """
+        for item in self._get_interpreter_config():
+            executable = self._locate_interpreter(item)
+            if executable is None:
+                return None
+            if item in ("pwsh", "powershell"):
+                return [executable, "-NoLogo", "-Command", "-"]
+            return [executable]
+        return None
+
+    def _locate_interpreter(self, interpreter: str) -> Optional[str]:
+        result = None
+        prog_files = environ.get("PROGRAMFILES", "C:\\Program Files")
+
+        # Try use $SHELL from the environment as a hint
+        shell_var = environ.get("SHELL", "")
+        if shell_var.endswith(f"/{interpreter}") and which(shell_var) == shell_var:
+            result = shell_var
+
+        elif interpreter == "posix":
+            # look for any known posix shell
+            result = (
+                self._locate_interpreter("sh")
+                or self._locate_interpreter("bash")
+                or self._locate_interpreter("zsh")
+            )
+
+        elif interpreter == "sh":
+            result = which("sh") or which("/bin/sh")
+
+        elif interpreter == "bash":
+            result = which("bash") or which("/bin/bash")
+
+            # Specifically look for git bash on windows
+            if result is None and self._is_windows:
+                result = which(f"{prog_files}\\Git\\bin\\bash.exe")
+
+        elif interpreter == "zsh":
+            result = which("zsh") or which("/bin/zsh")
+
+        elif interpreter == "fish":
+            result = which("fish") or which("/bin/fish")
+
+        elif interpreter in ("pwsh", "powershell"):
+            # Look for the pwsh executable and verify the version matches
+            result = (
+                which("pwsh")
+                or which(f"{prog_files}\\PowerShell\\7\\pwsh.exe")
+                or which(f"{prog_files}\\PowerShell\\6\\pwsh.exe")
+            )
+
+            if result is None and interpreter == "powershell" and self._is_windows:
+                # Look for older versions of powershell
+                result = which("powershell") or which(
+                    environ.get("WINDIR", "C:\\Windows")
+                    + "\\System32\\WindowsPowerShell\\v1.0\\powershell.EXE"
+                )
+
+        elif interpreter == "python":
+            result = which("python") or which("python3") or sys.executable
+
+        return result
+
+    @classmethod
+    def _validate_task_def(
+        cls, task_name: str, task_def: Dict[str, Any], config: "PoeConfig"
+    ) -> Optional[str]:
+        interpreter = task_def.get("interpreter")
+        valid_interpreters = config.KNOWN_SHELL_INTERPRETERS
+
+        if isinstance(interpreter, str) and interpreter not in valid_interpreters:
+            return (
+                "Unsupported value for option `interpreter` for task "
+                f"{task_name!r}. Expected one of {valid_interpreters}"
+            )
+
+        if isinstance(interpreter, list):
+            if len(interpreter) == 0:
+                return (
+                    "Unsupported value for option `interpreter` for task "
+                    f"{task_name!r}. Expected at least one item in list."
+                )
+            for item in interpreter:
+                if item not in valid_interpreters:
+                    return (
+                        "Unsupported item {item!r} in option `interpreter` for task "
+                        f"{task_name!r}. Expected one of {valid_interpreters}"
+                    )
+
+        return None

tests/conftest.py

@@ -11,6 +11,7 @@
 from subprocess import PIPE, Popen
 import sys
 from tempfile import TemporaryDirectory
+import time
 import tomli
 from typing import Any, Dict, List, Mapping, Optional
 import venv
@@ -27,7 +28,7 @@ def is_windows():
 
 @pytest.fixture
 def pyproject():
-    with PROJECT_TOML.open("r") as toml_file:
+    with PROJECT_TOML.open("rb") as toml_file:
         return tomli.load(toml_file)
 
 
@@ -121,7 +122,7 @@ def run_poe_subproc(
             with config_path.open("w+") as config_file:
                 toml.dump(config, config_file)
                 config_file.seek(0)
-            config_arg = fr"tomli.load(open(r\"{config_path}\", \"r\"))"
+            config_arg = fr"tomli.load(open(r\"{config_path}\", \"rb\"))"
         else:
             config_arg = "None"
 
@@ -245,7 +246,7 @@ def use_venv(
         yield
         # Only cleanup if we actually created it to avoid this fixture being a bit dangerous
         if not did_exist:
-            shutil.rmtree(location)
+            try_rm_dir(location)
 
     return use_venv
 
@@ -273,11 +274,22 @@ def use_virtualenv(
         yield
         # Only cleanup if we actually created it to avoid this fixture being a bit dangerous
         if not did_exist:
-            shutil.rmtree(location)
+            try_rm_dir(location)
 
     return use_virtualenv
 
 
+def try_rm_dir(location: Path):
+    try:
+        shutil.rmtree(location)
+    except:
+        # The above sometimes files with a Permissions error in CI for Windows
+        # No idea why, but maybe this will help
+        print("Retrying venv cleanup")
+        time.sleep(1)
+        shutil.rmtree(location)
+
+
 @pytest.fixture
 def with_virtualenv_and_venv(use_venv, use_virtualenv):
     def with_virtualenv_and_venv(