Add option to specify that a named argument should access multiple va…

…lues (#70)

Also improve tests for executing on windows

The new `multiple` option can be added to args to make them accept multiple values.

- only the last positional arg may have multiple set
- multiple option defaults to false
- args of type boolean may not have the multiple option set
- if `multiple = true` and `required = false` then the result is `nargs="*"`, or if `required = true` then `nargs="+"`
- an integer value >= 2 may also be given to specify an exact number of values are required for an arg
- for script tasks the values are passed as a list, for command tasks they can be templated into the command (thus
  potentially passed as multiple arguments), shell tasks don't have any special support.
- All tasks also get the list of values as an env var in the form of a space delimited string

.github/workflows/ci.yml

@@ -5,27 +5,18 @@ on: [push, pull_request]
 jobs:
 
   code-quality:
-    runs-on: ubuntu-latest
-
     name: Check coding standards
-
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.7
-      - uses: actions/cache@v2
-        with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-poetry-
+      - uses: actions/checkout@v3
 
       - name: Install poetry
-        shell: bash
-        run: |
-          python -m pip install poetry
-          echo "$HOME/.poetry/bin" >> $GITHUB_PATH
+        run: pipx install poetry
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+          cache: poetry
 
       - name: Install dependencies
         run: poetry install
@@ -43,32 +34,22 @@ jobs:
         run: poetry run poe check-docs
 
   run-tests:
-    runs-on: ${{ matrix.os }}-latest
-
     name: Run tests
-
     strategy:
       matrix:
         os: [Ubuntu, MacOS, Windows]
         python-version: ['3.7', '3.8', '3.9', '3.10']
-
+    runs-on: ${{ matrix.os }}-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - uses: actions/cache@v2
-        with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-poetry-
+      - uses: actions/checkout@v3
 
       - name: Install poetry
-        shell: bash
-        run: |
-          python -m pip install poetry
-          echo "$HOME/.poetry/bin" >> $GITHUB_PATH
+        run: pipx install poetry
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: poetry
 
       - name: Install dependencies
         run: poetry install
@@ -77,20 +58,18 @@ jobs:
         run: poetry run pytest -v
 
   build-release:
+    name: Build and release
     runs-on: ubuntu-latest
     needs: [code-quality, run-tests]
-
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.7
+      - uses: actions/checkout@v3
 
       - name: Install poetry
-        shell: bash
-        run: |
-          python -m pip install poetry
-          echo "$HOME/.poetry/bin" >> $GITHUB_PATH
+        run: pipx install poetry
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
 
       - name: Build package
         run: poetry build

README.rst

@@ -566,6 +566,31 @@ Named arguments support the following configuration options:
    If set to true then the argument becomes a position argument instead of an option
    argument. Note that positional arguments may not have type *bool*.
 
+- **multiple** : Union[bool, int]
+   If the multiple option is set to true on a positional or option argument then that
+   argument will accept multiple values.
+
+   If set to a number, then the argument will accept exactly that number of values.
+
+   For positional aguments, only the last positional argument may have the multiple
+   option set.
+
+   The multiple option is not compatible with arguments with type boolean since
+   these are interpreted as flags. However multiple ones or zeros can be passed to an
+   argument of type "integer" for similar effect.
+
+   The values provided to an argument with the multiple option set are available on
+   the environment as a string of whitespace separated values. For script tasks, the
+   values will be provided to your python function as a list of values. In a cmd task
+   the values can be passed as separate arugments to the task via templating as in the
+   following example.
+
+   .. code-block:: toml
+
+    [tool.poe.tasks.save]
+    cmd  = "echo ${FILE_PATHS}"
+    args = [{ name = "FILE_PATHS", positional = true, multiple = true }]
+
 - **required** : bool
    If true then not providing the argument will result in an error. Arguments are not
    required by default.

poethepoet/task/args.py

@@ -26,6 +26,7 @@
     "positional": (bool, str),
     "required": bool,
     "type": str,
+    "multiple": (bool, int),
 }
 arg_types: Dict[str, Type] = {
     "string": str,
@@ -96,21 +97,22 @@ def get_help_content(
     @classmethod
     def validate_def(cls, task_name: str, args_def: ArgsDef) -> Optional[str]:
         arg_names: Set[str] = set()
+        arg_params = []
+
         if isinstance(args_def, list):
             for item in args_def:
                 # can be a list of strings (just arg name) or ArgConfig dictionaries
                 if isinstance(item, str):
                     arg_name = item
                 elif isinstance(item, dict):
                     arg_name = item.get("name", "")
-                    error = cls._validate_params(item, arg_name, task_name)
-                    if error:
-                        return error
+                    arg_params.append((item, arg_name, task_name))
                 else:
                     return f"Arg {item!r} of task {task_name!r} has invlaid type"
                 error = cls._validate_name(arg_name, task_name, arg_names)
                 if error:
                     return error
+
         elif isinstance(args_def, dict):
             for arg_name, params in args_def.items():
                 error = cls._validate_name(arg_name, task_name, arg_names)
@@ -121,12 +123,26 @@ def validate_def(cls, task_name: str, args_def: ArgsDef) -> Optional[str]:
                         f"Unexpected 'name' option for arg {arg_name!r} of task "
                         f"{task_name!r}"
                     )
-                error = cls._validate_params(params, arg_name, task_name)
-                if error:
-                    return error
                 error = cls._validate_type(params, arg_name, task_name)
                 if error:
                     return error
+                arg_params.append((params, arg_name, task_name))
+
+        positional_multiple = None
+        for params, arg_name, task_name in arg_params:
+            error = cls._validate_params(params, arg_name, task_name)
+            if error:
+                return error
+
+            if params.get("positional", False):
+                if positional_multiple:
+                    return (
+                        f"Only the last positional arg of task {task_name!r} may accept"
+                        f" multiple values ({positional_multiple!r})."
+                    )
+                if params.get("multiple", False):
+                    positional_multiple = arg_name
+
         return None
 
     @classmethod
@@ -182,6 +198,22 @@ def _validate_params(
                     "https://docs.python.org/3/reference/lexical_analysis.html#identifiers"
                 )
 
+        multiple = params.get("multiple", False)
+        if (
+            not isinstance(multiple, bool)
+            and isinstance(multiple, int)
+            and multiple < 2
+        ):
+            return (
+                f"The multiple option for arg {arg_name!r} of {task_name!r}"
+                " must be given a boolean or integer >= 2"
+            )
+        if multiple is not False and params.get("type") == "boolean":
+            return (
+                "Incompatible param 'multiple' for arg {arg_name!r} of {task_name!r} "
+                "with type: 'boolean'"
+            )
+
         return None
 
     @classmethod
@@ -217,10 +249,19 @@ def _get_argument_params(self, arg: ArgParams):
         }
 
         required = arg.get("required", False)
+        multiple = arg.get("multiple", False)
         arg_type = str(arg.get("type"))
 
+        if multiple is True:
+            if required:
+                result["nargs"] = "+"
+            else:
+                result["nargs"] = "*"
+        elif multiple and isinstance(multiple, int):
+            result["nargs"] = multiple
+
         if arg.get("positional", False):
-            if not required:
+            if not multiple and not required:
                 result["nargs"] = "?"
         else:
             result["dest"] = arg["name"]
@@ -235,7 +276,7 @@ def _get_argument_params(self, arg: ArgParams):
 
     def parse(self, extra_args: Sequence[str]):
         parsed_args = vars(self.build_parser().parse_args(extra_args))
-        # Ensure positional args are still exposed by name even if the were parsed with
+        # Ensure positional args are still exposed by name even if they were parsed with
         # alternate identifiers
         for arg in self._args:
             if isinstance(arg.get("positional"), str):

poethepoet/task/base.py

@@ -196,13 +196,18 @@ def has_named_args(self):
         return bool(self.named_args)
 
     def get_named_arg_values(self) -> Mapping[str, str]:
+        result = {}
+
         if not self.named_args:
             return {}
-        return {
-            key: str(value)
-            for key, value in self.named_args.items()
-            if value is not None
-        }
+
+        for key, value in self.named_args.items():
+            if isinstance(value, list):
+                result[key] = " ".join(str(item) for item in value)
+            elif value is not None:
+                result[key] = str(value)
+
+        return result
 
     def run(
         self,

tests/fixtures/cmds_project/pyproject.toml

@@ -1,14 +1,27 @@
-tool.poe.tasks.show_env = "env"
+tool.poe.tasks.show_env = "poe_test_env"
 
 [tool.poe.tasks.echo]
-cmd = "echo POE_ROOT:$POE_ROOT ${BEST_PASSWORD}, task_args:"
+cmd = "poe_test_echo POE_ROOT:$POE_ROOT ${BEST_PASSWORD}, task_args:"
 help = "It says what you say"
 env = { BEST_PASSWORD = "Password1" }
 
 [tool.poe.tasks.greet]
-shell = "echo $formal_greeting $subject"
+shell = "poe_test_echo $formal_greeting $subject"
 args = ["formal-greeting", "subject"]
 
 [tool.poe.tasks.surfin-bird]
-cmd = "echo $WORD is the word"
+cmd = "poe_test_echo $WORD is the word"
 env = { WORD = "${SOME_INPUT_VAR}"}
+
+[tool.poe.tasks.multiple-value-arg]
+cmd = "poe_test_echo \"first: ${first} second: ${second}\""
+
+  [[tool.poe.tasks.multiple-value-arg.args]]
+  name       = "first"
+  positional = true
+
+  [[tool.poe.tasks.multiple-value-arg.args]]
+  name       = "second"
+  positional = true
+  multiple   = true
+  type       = "integer"

tests/fixtures/default_value_project/pyproject.toml

@@ -7,7 +7,7 @@ FIVE.default = "nope"
 SIX.default = "!six!"
 
 [tool.poe.tasks.test]
-cmd = "echo $ONE $TWO $THREE $FOUR $FIVE $SIX"
+cmd = "poe_test_echo $ONE $TWO $THREE $FOUR $FIVE $SIX"
 env.TWO = "!two!"
 env.FIVE = "!five!"
 env.SIX.default = "nope"

tests/fixtures/envfile_project/pyproject.toml

@@ -4,12 +4,12 @@ env = { HOST = "${HOST}:80" }
 
 [tool.poe.tasks.deploy-dev]
 cmd = """
-echo "deploying to ${USER}:${PASSWORD}@${HOST}${PATH_SUFFIX}"
+poe_test_echo "deploying to ${USER}:${PASSWORD}@${HOST}${PATH_SUFFIX}"
 """
 env = { HOST = "${HOST}80" } # reference and override value from envfile
 
 [tool.poe.tasks.deploy-prod]
 cmd = """
-echo "deploying to ${USER}:${PASSWORD}@${HOST}${PATH_SUFFIX}"
+poe_test_echo "deploying to ${USER}:${PASSWORD}@${HOST}${PATH_SUFFIX}"
 """
 envfile = "prod.env"

tests/fixtures/example_project/pyproject.toml

@@ -14,13 +14,13 @@ env.DEST = "MARS"
 
 
 [tool.poe.tasks.echo]
-cmd = "echo POE_ROOT:$POE_ROOT ${BEST_PASSWORD}, task_args:"
+cmd = "poe_test_echo POE_ROOT:$POE_ROOT ${BEST_PASSWORD}, task_args:"
 help = "It says what you say"
 env = { BEST_PASSWORD = "Password1" }
 
 
 [tool.poe.tasks]
-show_env = "env"
+show_env = "poe_test_env"
 greet = { script = "dummy_package:main" }
 greet-shouty = { script = "dummy_package:main(upper=True)" }
 
@@ -37,13 +37,13 @@ echo "my life got flipped;
 echo "bam bam baaam bam"
 """
 
-part1 = "echo 'Hello'"
-_part2.cmd = "echo '${SUBJECT}!'"
+part1 = "poe_test_echo 'Hello'"
+_part2.cmd = "poe_test_echo '${SUBJECT}!'"
 _part2.env = { SUBJECT = "World" }
 composite_task.sequence = [
   ["part1", "_part2"],
   # wrapping in arrays means we can have different types of task in the sequence
-  [{cmd = "echo '${SMILEY}!'"}]
+  [{cmd = "poe_test_echo '${SMILEY}!'"}]
 ]
 # env var is inherited by subtask
 composite_task.env = { SMILEY = ":)" }
@@ -54,7 +54,7 @@ greet-multiple.sequence = ["dummy_package:main('Tom')", "dummy_package:main('Jer
 greet-multiple.default_item_type = "script"
 
 travel = [
-  { cmd = "echo 'from $PLANET to'" },
+  { cmd = "poe_test_echo 'from $PLANET to'" },
   { script = "dummy_package:print_var('DEST')" }
 ]
 

tests/fixtures/high_verbosity/pyproject.toml

@@ -2,4 +2,4 @@
 verbosity = 1
 
 [tool.poe.tasks]
-test = "echo Hello there!"
+test = "poe_test_echo Hello there!"

tests/fixtures/includes_project/greet.toml

@@ -2,12 +2,12 @@
 default_greeting = "Hello"
 
 [tool.poe.tasks]
-echo.cmd = "echo echo echo"
+echo.cmd = "poe_test_echo echo echo"
 echo.help = "This is ignored becuase it's already defined!"
 
-greet = "echo $default_greeting"
+greet = "poe_test_echo $default_greeting"
 
-greet1 = "echo Hoi"
+greet1 = "poe_test_echo Hoi"
 
-greet2.cmd = "echo Hola"
+greet2.cmd = "poe_test_echo Hola"
 greet2.help = "Issue a greeting from the Iberian Peninsula"