From 2d252f3dfd4b2afb2785e1434cfb942c379ef08e Mon Sep 17 00:00:00 2001 From: Nat Noordanus Date: Wed, 25 Dec 2024 22:04:14 +0200 Subject: [PATCH] Make command parsing support operations on param expansions Specifically cmd tasks may now use the following operations like in bash - `:-` fallback value, e.g. ${AWS_REGION:-us-east-1} - `:+` value replacement, e.g. ${AWESOME:+--awesome-mode} This was done by adding AST parser support for Param Expansion operators Also: - Fix bug in param expansion logic for pure whitespace param values - Add env paramater to PoeThePoet class to override os.environ for testing --- .github/workflows/ci.yml | 14 +- docs/tasks/task_types/cmd.rst | 26 +++- poethepoet/app.py | 16 ++- poethepoet/env/template.py | 2 +- poethepoet/exceptions.py | 16 ++- poethepoet/helpers/command/__init__.py | 39 ++++- poethepoet/helpers/command/ast.py | 190 +++++++++++++++++++++---- poethepoet/helpers/command/ast_core.py | 121 +++++++++++++--- poethepoet/task/cmd.py | 6 +- tests/conftest.py | 2 + tests/helpers/command/test_ast.py | 189 ++++++++++++++++++++++++ tests/test_cmd_param_expansion.py | 61 ++++++++ 12 files changed, 613 insertions(+), 69 deletions(-) create mode 100644 tests/test_cmd_param_expansion.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffd5c0d09..a01bcfe75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,14 +118,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Trigger update of homebrew formula - run: > + run: | sleep 10 # some delay seems to be necessary - curl -L -X POST - -H "Accept: application/vnd.github+json" - -H "Authorization: Bearer ${{ secrets.homebrew_pat }}" - -H "X-GitHub-Api-Version: 2022-11-28" - https://api.github.com/repos/nat-n/homebrew-poethepoet/actions/workflows/71211730/dispatches - -d '{"ref":"main", "inputs":{}}' + curl -L -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.homebrew_pat }}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/nat-n/homebrew-poethepoet/actions/workflows/71211730/dispatches \ + -d '{"ref":"main", "inputs":{}}' github-release: name: >- diff --git a/docs/tasks/task_types/cmd.rst b/docs/tasks/task_types/cmd.rst index 0f8d7db76..ea52659e6 100644 --- a/docs/tasks/task_types/cmd.rst +++ b/docs/tasks/task_types/cmd.rst @@ -26,6 +26,7 @@ It is important to understand that ``cmd`` tasks are executed without a shell (t .. _ref_env_vars: + Referencing environment variables ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -50,6 +51,29 @@ Parameter expansion can also can be disabled by escaping the $ with a backslash greet = "echo Hello \\$USER" # the backslash itself needs escaping for the toml parser +Parameter expansion operators +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When referencing an environment variable in a cmd task you can use the ``:-`` operator from bash to specify a *default value*, to be used in case the variable is unset. Similarly the ``:+`` operator can be used to specify an *alternate value* to use in place of the environment variable if it *is* set. + +In the following example, if ``AWS_REGION`` has a value then it will be used, otherwise ``us-east-1`` will be used as a fallback. + +.. code-block:: toml + + [tool.poe.tasks] + tables = "aws dynamodb list-tables --region ${AWS_REGION:-us-east-1}" + +The ``:+`` or *alternate value* operator is especially useful in cases such as the following where you might want to control whether some CLI options are passed to the command. + +.. code-block:: toml + + [tool.poe.tasks.aws-identity] + cmd = "aws sts get-caller-identity ${ARN_ONLY:+ --no-cli-pager --output text --query 'Arn'}" + args = [{ name = "ARN_ONLY", options = ["--arn-only"], type = "boolean" }] + +In this example we declare a boolean argument with no default, so if the ``--arn-only`` flag is provided to the task then three additional CLI options will be included in the task content. + + Glob expansion ~~~~~~~~~~~~~~ @@ -78,7 +102,7 @@ Here's an example of task using a recursive glob pattern: .. seealso:: - Much like in bash, the glob pattern can be escaped by wrapping it in quotes, or preceding it with a backslash. + Just like in bash, the glob pattern can be escaped by wrapping it in quotes, or preceding it with a backslash. .. |glob_link| raw:: html diff --git a/poethepoet/app.py b/poethepoet/app.py index 57cb0e384..ad3ddf399 100644 --- a/poethepoet/app.py +++ b/poethepoet/app.py @@ -20,28 +20,38 @@ class PoeThePoet: this determines where to look for a pyproject.toml file, defaults to ``Path().resolve()`` :type cwd: Path, optional + :param config: Either a dictionary with the same schema as a pyproject.toml file, or a `PoeConfig `_ object to use as an alternative to loading config from a file. :type config: dict | PoeConfig, optional + :param output: A stream for the application to write its own output to, defaults to sys.stdout :type output: IO, optional + :param poetry_env_path: The path to the poetry virtualenv. If provided then it is used by the `PoetryExecutor `_, instead of having to execute poetry in a subprocess to determine this. :type poetry_env_path: str, optional + :param config_name: The name of the file to load tasks and configuration from. If not set then poe will search for config by the following file names: pyproject.toml poe_tasks.toml poe_tasks.yaml poe_tasks.json :type config_name: str, optional + :param program_name: The name of the program that is being run. This is used primarily when outputting help messages, defaults to "poe" :type program_name: str, optional + + :param env: + Optionally provide an alternative base environment for tasks to run with. + If no mapping is provided then ``os.environ`` is used. + :type env: dict, optional """ cwd: Path @@ -58,6 +68,7 @@ def __init__( poetry_env_path: Optional[str] = None, config_name: Optional[str] = None, program_name: str = "poe", + env: Optional[Mapping[str, str]] = None, ): from .config import PoeConfig from .ui import PoeUi @@ -75,6 +86,7 @@ def __init__( ) self.ui = PoeUi(output=output, program_name=program_name) self._poetry_env_path = poetry_env_path + self._env = env if env is not None else os.environ def __call__(self, cli_args: Sequence[str], internal: bool = False) -> int: """ @@ -212,9 +224,9 @@ def get_run_context(self, multistage: bool = False) -> "RunContext": result = RunContext( config=self.config, ui=self.ui, - env=os.environ, + env=self._env, dry=self.ui["dry_run"], - poe_active=os.environ.get("POE_ACTIVE"), + poe_active=self._env.get("POE_ACTIVE"), multistage=multistage, cwd=self.cwd, ) diff --git a/poethepoet/env/template.py b/poethepoet/env/template.py index 627daae52..97a2cc6a3 100644 --- a/poethepoet/env/template.py +++ b/poethepoet/env/template.py @@ -56,7 +56,7 @@ def apply_envvars_to_template( content: str, env: Mapping[str, str], require_braces=False ) -> str: """ - Template in ${environmental} $variables from env as if we were in a shell + Template in ${environment} $variables from env as if we were in a shell Supports escaping of the $ if preceded by an odd number of backslashes, in which case the backslash immediately preceding the $ is removed. This is an diff --git a/poethepoet/exceptions.py b/poethepoet/exceptions.py index 9cbcc8a5a..c914ec3aa 100644 --- a/poethepoet/exceptions.py +++ b/poethepoet/exceptions.py @@ -6,9 +6,19 @@ class PoeException(RuntimeError): cause: Optional[str] def __init__(self, msg, *args): + super().__init__(msg, *args) self.msg = msg - self.cause = args[0].args[0] if args else None - self.args = (msg, *args) + + if args: + cause = args[0] + position_clause = ( + f", near line {cause.line}, position {cause.position}." + if getattr(cause, "has_position", False) + else "." + ) + self.cause = cause.args[0] + position_clause + else: + self.cause = None class CyclicDependencyError(PoeException): @@ -34,7 +44,7 @@ def __init__( task_name: Optional[str] = None, index: Optional[int] = None, global_option: Optional[str] = None, - filename: Optional[str] = None + filename: Optional[str] = None, ): super().__init__(msg, *args) self.context = context diff --git a/poethepoet/helpers/command/__init__.py b/poethepoet/helpers/command/__init__.py index 3a67c6ae0..0925c2d3a 100644 --- a/poethepoet/helpers/command/__init__.py +++ b/poethepoet/helpers/command/__init__.py @@ -31,7 +31,7 @@ def resolve_command_tokens( patterns that are not escaped or quoted. In case there are glob patterns in the token, any escaped glob characters will have been escaped with []. """ - from .ast import Glob, ParamExpansion, ParseConfig, PythonGlob + from .ast import Glob, ParamArgument, ParamExpansion, ParseConfig, PythonGlob if not config: config = ParseConfig(substitute_nodes={Glob: PythonGlob}) @@ -56,6 +56,34 @@ def finalize_token(token_parts): token_parts.clear() return (token, includes_glob) + def resolve_param_argument(argument: ParamArgument, env: Mapping[str, str]): + token_parts = [] + for segment in argument.segments: + for element in segment: + if isinstance(element, ParamExpansion): + token_parts.append(resolve_param_value(element, env)) + else: + token_parts.append(element.content) + + return "".join(token_parts) + + def resolve_param_value(element: ParamExpansion, env: Mapping[str, str]): + param_value = env.get(element.param_name, "") + + if element.operation: + if param_value: + if element.operation.operator == ":+": + # apply 'alternate value' operation + param_value = resolve_param_argument( + element.operation.argument, env + ) + + elif element.operation.operator == ":-": + # apply 'default value' operation + param_value = resolve_param_argument(element.operation.argument, env) + + return param_value + for line in lines: # Ignore line breaks, assuming they're only due to comments for word in line: @@ -63,16 +91,19 @@ def finalize_token(token_parts): # strip out comments continue - # For each token part indicate whether it is a glob token_parts: list[tuple[str, bool]] = [] for segment in word: for element in segment: if isinstance(element, ParamExpansion): - param_value = env.get(element.param_name, "") + param_value = resolve_param_value(element, env) if not param_value: + # Empty param value has no effect continue if segment.is_quoted: - token_parts.append((env.get(element.param_name, ""), False)) + token_parts.append((param_value, False)) + elif param_value.isspace(): + # collapse whitespace value + token_parts.append((" ", False)) else: # If the the param expansion it not quoted then: # - Whitespace inside a substituted param value results in diff --git a/poethepoet/helpers/command/ast.py b/poethepoet/helpers/command/ast.py index ee4b432d6..fc39abb40 100644 --- a/poethepoet/helpers/command/ast.py +++ b/poethepoet/helpers/command/ast.py @@ -1,29 +1,28 @@ -# ruff: noqa: N806 +# ruff: noqa: N806, UP007 r""" -This module implements a hierarchical parser and AST along the lines of the -following grammar which is a subset of bash syntax. - -script : line* -line : word* comment? -word : segment* -segment : UNQUOTED_CONTENT | single_quoted_segment | double_quoted_segment - -unquoted_segment : UNQUOTED_CONTENT | param_expansion | glob -single_quoted_segment : "'" SINGLE_QUOTED_CONTENT "'" -double_quoted_segment : "\"" (DOUBLE_QUOTED_CONTENT | param_expansion) "\"" - -comment : /#[^\n\r\f\v]*/ -glob : "?" | "*" | "[" /(\!?\]([^\s\]\\]|\\.)*|([^\s\]\\]|\\.)+)*/ "]" - -UNQUOTED_CONTENT : /[^\s;#*?[$]+/ -SINGLE_QUOTED_CONTENT : /[^']+/ -DOUBLE_QUOTED_CONTENT : /([^\$"]|\[\$"])+/ +This module implements a hierarchical parser and AST for a subset of bash syntax +including: + +- line breaks and comments +- single or double quotes and character escaping +- basic glob patterns (python style glob patterns also supported) +- basic parameter expansion +- parameter expansion operators: `:+` and `:-` """ +from __future__ import annotations + from collections.abc import Iterable from typing import Literal, Optional, Union, cast -from .ast_core import ContentNode, ParseConfig, ParseCursor, ParseError, SyntaxNode +from .ast_core import ( + AnnotatedContentNode, + ContentNode, + ParseConfig, + ParseCursor, + ParseError, + SyntaxNode, +) PARAM_INIT_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_" PARAM_CHARS = PARAM_INIT_CHARS + "0123456789" @@ -40,7 +39,9 @@ def _parse(self, chars: ParseCursor): return content.append(char) else: - raise ParseError("Unexpected end of input with unmatched single quote") + raise ParseError( + "Unexpected end of input with unmatched single quote", chars + ) class DoubleQuotedText(ContentNode): @@ -63,6 +64,8 @@ def _parse(self, chars: ParseCursor): class UnquotedText(ContentNode): + _break_chars = "'\";#$?*[" + def _parse(self, chars: ParseCursor): content: list[str] = [] for char in chars: @@ -77,7 +80,7 @@ def _parse(self, chars: ParseCursor): content.append(escaped_char) continue - elif char.isspace() or char in "'\";#$?*[": + elif char.isspace() or char in self._break_chars: chars.pushback(char) break @@ -198,11 +201,15 @@ def _parse(self, chars: ParseCursor): self._cancelled = True -class ParamExpansion(ContentNode): +class ParamExpansion(AnnotatedContentNode["ParamOperation"]): @property def param_name(self) -> str: return self._content + @property + def operation(self) -> Optional[ParamOperation]: + return self._annotation + def _parse(self, chars: ParseCursor): assert chars.take() == "$" @@ -218,6 +225,14 @@ def _parse(self, chars: ParseCursor): return if param: + if char == ":": + ParamOperationCls: type = self.get_child_node_cls( + ParamOperation + ) + chars.pushback(char) + self._annotation = ParamOperationCls(chars, self.config) + continue + if char not in PARAM_CHARS: raise ParseError( "Bad substitution: Illegal character in parameter name " @@ -236,7 +251,14 @@ def _parse(self, chars: ParseCursor): f"{char!r}", chars, ) - raise ParseError("Unexpected end of input, expected closing '}' after '${'") + raise ParseError( + "Unexpected end of input, expected closing '}' after '${'", chars + ) + + elif chars.peek() is None: + # End of input means no param expansion + chars.pushback("$") + self._cancelled = True else: for char in chars: @@ -291,17 +313,17 @@ def _parse(self, chars: ParseCursor): self._children = [] if self._quote_char == "'": - return self.__consume_single_quoted(chars) + return self._consume_single_quoted(chars) elif self._quote_char == '"': - return self.__consume_double_quoted(chars) + return self._consume_double_quoted(chars) else: - return self.__consume_unquoted(chars) + return self._consume_unquoted(chars) - def __consume_single_quoted(self, chars): + def _consume_single_quoted(self, chars): SingleQuotedTextCls = self.get_child_node_cls(SingleQuotedText) self._children.append(SingleQuotedTextCls(chars, self.config)) - def __consume_double_quoted(self, chars): + def _consume_double_quoted(self, chars): DoubleQuotedTextCls = self.get_child_node_cls(DoubleQuotedText) ParamExpansionCls = self.get_child_node_cls(ParamExpansion) @@ -324,9 +346,9 @@ def __consume_double_quoted(self, chars): else: self._children.append(DoubleQuotedTextCls(chars, self.config)) - raise ParseError("Unexpected end of input with unmatched double quote") + raise ParseError("Unexpected end of input with unmatched double quote", chars) - def __consume_unquoted(self, chars): + def _consume_unquoted(self, chars): UnquotedTextCls = self.get_child_node_cls(UnquotedText) GlobCls = self.get_child_node_cls(Glob) ParamExpansionCls = self.get_child_node_cls(ParamExpansion) @@ -355,6 +377,108 @@ def __consume_unquoted(self, chars): self._children.append(UnquotedTextCls(chars, self.config)) +class WhitespaceText(ContentNode): + """ + Capture unquoted whitespace strings as a single space + """ + + def _parse(self, chars: ParseCursor): + if chars.peek().isspace(): + self._content = " " + + while char := chars.take(): + if not char.isspace(): + chars.pushback(char) + break + + +class ParamArgumentUnquotedText(UnquotedText): + """ + Just like UnquotedText except that it may include chars in `;#`, but not `}` + """ + + _break_chars = "'\"$}" + + +class ParamArgumentSegment(Segment): + """ + Just like Segment except that : + - it may include unquoted whitespace or chars in `;#`, but not `}` + - glob characters `*?[` are not recognised as special + """ + + def _consume_unquoted(self, chars): + UnquotedTextCls = self.get_child_node_cls(ParamArgumentUnquotedText) + ParamExpansionCls = self.get_child_node_cls(ParamExpansion) + WhitespaceTextCls = self.get_child_node_cls(WhitespaceText) + + while next_char := chars.peek(): + if next_char in "'\"}": + return + + elif next_char.isspace(): + self._children.append(WhitespaceTextCls(chars, self.config)) + + elif next_char == "$": + if param_node := ParamExpansionCls(chars, self.config): + self._children.append(param_node) + else: + # Hack: escape the $ to make it acceptable as unquoted text + chars.pushback("\\") + self._children.append(UnquotedTextCls(chars, self.config)) + + else: + self._children.append(UnquotedTextCls(chars, self.config)) + + +class ParamArgument(SyntaxNode[ParamArgumentSegment]): + @property + def segments(self) -> tuple[ParamArgumentSegment, ...]: + return tuple(self._children) + + def _parse(self, chars: ParseCursor): + SegmentCls = self.get_child_node_cls(ParamArgumentSegment) + + self._children = [] + + while next_char := chars.peek(): + if next_char == "}": + return + self._children.append(SegmentCls(chars, self.config)) + + +class ParamOperation(AnnotatedContentNode[ParamArgument]): + _content: Literal[":-", ":+"] + + @property + def operator(self) -> Literal[":-", ":+"]: + return self._content + + @property + def argument(self) -> ParamArgument: + assert self._annotation + return self._annotation + + def _parse(self, chars: ParseCursor): + ParamArgumentCls = self.get_child_node_cls(ParamArgument) + + op_chars = (chars.take(), chars.take()) + if None in op_chars: + raise ParseError( + "Unexpected end of input in param expansion, expected '}'", + chars, + ) + + self._content = op_chars[0] + op_chars[1] + if self._content not in (":-", ":+"): + raise ParseError( + f"Bad substitution: Unsupported operator {self._content!r}", + chars, + ) + + self._annotation = ParamArgumentCls(chars, self.config) + + class Word(SyntaxNode[Segment]): @property def segments(self) -> tuple[Segment, ...]: @@ -390,6 +514,10 @@ def comment(self) -> str: return self._children[-1].comment return "" + @property + def terminator(self): + return self._terminator + def _parse(self, chars: ParseCursor): WordCls = self.get_child_node_cls(Word) CommentCls = self.get_child_node_cls(Comment) diff --git a/poethepoet/helpers/command/ast_core.py b/poethepoet/helpers/command/ast_core.py index 489cbe6ad..43de63683 100644 --- a/poethepoet/helpers/command/ast_core.py +++ b/poethepoet/helpers/command/ast_core.py @@ -1,5 +1,5 @@ """ -This module core a framework for defining hierarchical parser and ASTs. +This module provides a framework for defining a hierarchical parser and AST. See sibling ast module for an example usage. """ @@ -9,23 +9,26 @@ class ParseCursor: - """ + r""" This makes it easier to parse a text by wrapping it an abstraction like a stack, so the parser can pop off the next character but also push back characters that need - to be reprocessed. + to be reprocessed by a different node. - TODO: keep track of the current line number and position of the read head. + The line and position tracking which may be used for error reporting assumes that + whatever source we're reading the file from encodes new lines as simply '\n'. """ _line: int _position: int + _line_lengths: list[int] _source: Iterator[str] _pushback_stack: list[str] def __init__(self, source: Iterator[str]): self._source = source self._line = 0 - self._position = 0 + self._position = -1 + self._line_lengths = [] self._pushback_stack = [] @classmethod @@ -40,6 +43,10 @@ def iter_chars(): def from_string(cls, string: str): return cls(char for char in string) + @property + def position(self): + return (max(0, self._line), max(0, self._position)) + def peek(self): if not self._pushback_stack: try: @@ -49,21 +56,31 @@ def peek(self): return self._pushback_stack[-1] def take(self): - # TODO: update _line and _position if self._pushback_stack: - return self._pushback_stack.pop() + result = self._pushback_stack.pop() + else: + try: + result = next(self._source) + except StopIteration: + result = None + + if result == "\n": + self._line_lengths.append(self._position) + self._line += 1 + self._position = -1 + else: + self._position += 1 - try: - return next(self._source) - except StopIteration: - return None + return result def pushback(self, *items: str): for item in reversed(items): - # TODO: rewind _line and _position - # HOW to get length of previous line which pushback a line break? - # would need to keep a stack of all previous line lengths to be sure!? self._pushback_stack.append(item) + if item == "\n": + self._position = self._line_lengths.pop(0) + self._line = max(0, self._line - 1) + else: + self._position -= 1 def __iter__(self): return self @@ -78,6 +95,12 @@ def __bool__(self): class ParseConfig: + """ + A ParseConfig is passed to every AstNode in a tree, and may be used to configure + alternative parsing behaviors, thus making to possible to declare small variations + to the parsed syntax without having to duplicate parsing logic. + """ + substitute_nodes: dict[type["AstNode"], type["AstNode"]] line_separators: str @@ -116,10 +139,14 @@ def __len__(self): ... -T = TypeVar("T") +T = TypeVar("T", bound=AstNode) class SyntaxNode(AstNode, Generic[T]): + """ + A SyntaxNode is a branching AST node with no content of its own + """ + _children: list[T] def get_child_node_cls(self, node_type: type[AstNode]) -> type[T]: @@ -138,7 +165,7 @@ def pretty(self, indent: int = 0, increment: int = 4): return "\n".join( [ f"{self.__class__.__name__}:", - *(" " * indent + child.pretty(indent) for child in self), + *(" " * indent + child.pretty(indent, increment) for child in self), ] ) @@ -163,6 +190,10 @@ def __eq__(self, other): class ContentNode(AstNode): + """ + A ContentNode is a terminal AST node with string content + """ + _content: str = "" @property @@ -172,6 +203,13 @@ def content(self) -> str: def pretty(self, indent: int = 0, increment: int = 4): return f"{self.__class__.__name__}: {self._content!r}" + def get_child_node_cls(self, node_type: type[AstNode]) -> type: + """ + Apply Node class substitution for the given node AstNode if specified in + the ParseConfig. + """ + return cast(type, self.config.resolve_node_cls(node_type)) + def __str__(self): return self._content @@ -187,5 +225,54 @@ def __eq__(self, other): return super().__eq__(other) +class AnnotatedContentNode(ContentNode, Generic[T]): + """ + A AnnotatedContentNode is like a ContentNode except that it may also have a + single child node, which is considered an annotation on the content. + """ + + _annotation: Optional[T] = None + + def pretty(self, indent: int = 0, increment: int = 4): + indent += increment + content_line = f"{self.__class__.__name__}: {self._content!r}" + if self._annotation is not None: + return ( + f"{content_line}\n" + f"{' '*indent}{self._annotation.pretty(indent, increment)}" + ) + return content_line + + def __repr__(self): + annotation = f", {self._annotation!r}" if self._annotation else "" + return f"{self.__class__.__name__}({self._content!r}{annotation})" + + def __eq__(self, other): + if isinstance(other, str): + return self._content == other + if isinstance(other, tuple): + if self._annotation is None: + return (self._content,) == other + return (self._content, self._annotation) == other + return super().__eq__(other) + + class ParseError(RuntimeError): - pass + line: Optional[int] + position: Optional[int] + + def __init__(self, message: str, cursor: Optional[ParseCursor] = None): + super().__init__(message) + self.message = message + if cursor is not None: + self.line, self.position = cursor.position + + @property + def has_position(self) -> bool: + return None not in (self.line, self.position) + + def __repr__(self): + details = ( + f", line={self.line}, position={self.position}" if self.has_position else "" + ) + return f'{self.__class__.__name__}("{self.message}"{details})' diff --git a/poethepoet/task/cmd.py b/poethepoet/task/cmd.py index b370193dd..538f9db9f 100644 --- a/poethepoet/task/cmd.py +++ b/poethepoet/task/cmd.py @@ -69,14 +69,14 @@ def _resolve_commandline(self, context: "RunContext", env: "EnvVarsManager"): command_lines = parse_poe_cmd(self.spec.content).command_lines except ParseError as error: raise PoeException( - f"Couldn't parse command line for task {self.name!r}: {error.args[0]}" - ) from error + f"Couldn't parse command line for task {self.name!r}", error + ) if not command_lines: raise PoeException( f"Invalid cmd task {self.name!r} does not include any command lines" ) - if any(line._terminator == ";" for line in command_lines[:-1]): + if any(line.terminator == ";" for line in command_lines[:-1]): # lines terminated by a line break or comment are implicitly joined raise PoeException( f"Invalid cmd task {self.name!r} includes multiple command lines" diff --git a/tests/conftest.py b/tests/conftest.py index bdd31ff46..bcd370e9f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -204,6 +204,7 @@ def run_poe( project: Optional[str] = None, config_name="pyproject.toml", program_name="poe", + env: Optional[Mapping[str, str]] = None, ) -> PoeRunResult: cwd = projects.get(project, cwd) output_capture = StringIO() @@ -213,6 +214,7 @@ def run_poe( output=output_capture, config_name=config_name, program_name=program_name, + env=env, ) result = poe(run_args) output_capture.seek(0) diff --git a/tests/helpers/command/test_ast.py b/tests/helpers/command/test_ast.py index fb12d00b2..99d67705f 100644 --- a/tests/helpers/command/test_ast.py +++ b/tests/helpers/command/test_ast.py @@ -107,6 +107,195 @@ def test_parse_params(): assert tree.lines[14] == ((("a", "xx1", "? a", "yy2", "b a", "$? ", "z"),),) +def test_parse_param_operators(): + tree = parse_poe_cmd( + """ + 0${x}B + 1${x:+foo}B + 2${x:-bar}B + 3${x:+$foo1}B + 4${x:-a${bar2}b}B + 5${x:- a ${bar} b }B + 6${x:- a ${bar:+ $incepted + 1'"'"';#" } b }B + 7${x:++}B + 8${x:-$}B + 9${x:- }B + 10${x:-}B + 11${x:-(#);?[t]*.ok}B + 12${x:-?[t]*.ok}B + 13${x:- + split\\ \\ \ + lines + ' ' + ! + }B + """, + config=ParseConfig(), + ) + print(tree.pretty()) + assert len(tree.lines) == 14 + assert tree.lines[0].words[0].segments[0] == ("0", "x", "B") + assert tree.lines[1].words[0].segments[0] == ( + "1", + ( + "x", + ( + ":+", + (("foo",),), + ), + ), + "B", + ) + assert tree.lines[2].words[0].segments[0] == ( + "2", + ( + "x", + ( + ":-", + (("bar",),), + ), + ), + "B", + ) + assert tree.lines[3].words[0].segments[0] == ( + "3", + ( + "x", + ( + ":+", + ((("foo1",),),), + ), + ), + "B", + ) + assert tree.lines[4].words[0].segments[0] == ( + "4", + ( + "x", + ( + ":-", + (("a", ("bar2",), "b"),), + ), + ), + "B", + ) + assert tree.lines[5].words[0].segments[0] == ( + "5", + ( + "x", + ( + ":-", + ((" ", "a", " ", ("bar",), " ", "b", " "),), + ), + ), + "B", + ) + assert tree.lines[6].words[0].segments[0] == ( + "6", + ( + "x", + ( + ":-", + ( + ( + " ", + "a", + " ", + ( + "bar", + ( + ":+", + ( + (" ", ("incepted",), " ", "+", " ", "1"), + ('"',), + ("';#",), + (" ",), + ), + ), + ), + " ", + "b", + " ", + ), + ), + ), + ), + "B", + ) + assert tree.lines[7].words[0].segments[0] == ( + "7", + ( + "x", + ( + ":+", + (("+",),), + ), + ), + "B", + ) + assert tree.lines[8].words[0].segments[0] == ( + "8", + ( + "x", + ( + ":-", + (("$",),), + ), + ), + "B", + ) + assert tree.lines[9].words[0].segments[0] == ( + "9", + ( + "x", + ( + ":-", + ((" ",),), + ), + ), + "B", + ) + assert tree.lines[10].words[0].segments[0] == ( + "10", + ( + "x", + (":-", tuple()), + ), + "B", + ) + assert tree.lines[11].words[0].segments[0] == ( + "11", + ( + "x", + (":-", (("(#);?[t]*.ok",),)), + ), + "B", + ) + assert tree.lines[12].words[0].segments[0] == ( + "12", + ( + "x", + (":-", (("?[t]*.ok",),)), + ), + "B", + ) + assert tree.lines[13].words[0].segments[0] == ( + "13", + ( + "x", + ( + ":-", + ( + (" ", "split ", " ", " ", " ", "lines", " "), + (" ",), + (" ", "!", " "), + ), + ), + ), + "B", + ) + + def test_invalid_param_expansion(): with pytest.raises(ParseError) as excinfo: parse_poe_cmd("""${}""", config=ParseConfig()) diff --git a/tests/test_cmd_param_expansion.py b/tests/test_cmd_param_expansion.py new file mode 100644 index 000000000..fccd39110 --- /dev/null +++ b/tests/test_cmd_param_expansion.py @@ -0,0 +1,61 @@ +import pytest + + +@pytest.mark.parametrize( + ("expression", "output", "env"), + [ + # basic parameter value expansion + (r"$", "$", {}), + (r"A${FOO}B", "AB", {}), + (r"A${FOO}B", "AB", {"FOO": ""}), + (r"A${FOO}B", "A x B", {"FOO": " x "}), + (r"A${FOO}B", "A B", {"FOO": " "}), + (r"A${FOO}B", "AfooB", {"FOO": "foo"}), + # default value operator + (r"A${FOO:-}B", "AB", {}), + (r"A${FOO:-bar}B", "AbarB", {}), + (r"A${FOO:-bar}B", "AbarB", {"FOO": ""}), + (r"A${FOO:-bar}B", "AfooB", {"FOO": "foo"}), + # alternate value operator + (r"A${FOO:+bar}B", "AB", {}), + (r"A${FOO:+bar}B", "AB", {"FOO": ""}), + (r"A${FOO:+}B", "AB", {"FOO": "foo"}), + (r"A${FOO:+bar}B", "AbarB", {"FOO": "foo"}), + # recursion + (r"A${FOO:->${BAR:+ ${BAZ:- the end }<}}B", "A> the end