From a7f3f942c10d0096daefdcd62b5ee9e906a7e635 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Wed, 15 Mar 2023 22:50:37 +0100 Subject: [PATCH 01/15] Began moving markup code out. --- src/antsibull_docs/jinja2/environment.py | 4 +- src/antsibull_docs/jinja2/filters.py | 42 +++++++++++++++++++ .../{jinja2 => markup}/htmlify.py | 32 ++++---------- .../{jinja2 => markup}/parser.py | 0 .../{jinja2 => markup}/rstify.py | 32 ++++---------- .../{ => markup}/semantic_helper.py | 0 src/sphinx_antsibull_ext/roles.py | 2 +- tests/units/test_jinja2.py | 25 +---------- ...jinja2_parser.py => test_markup_parser.py} | 2 +- 9 files changed, 63 insertions(+), 76 deletions(-) rename src/antsibull_docs/{jinja2 => markup}/htmlify.py (91%) rename src/antsibull_docs/{jinja2 => markup}/parser.py (100%) rename src/antsibull_docs/{jinja2 => markup}/rstify.py (90%) rename src/antsibull_docs/{ => markup}/semantic_helper.py (100%) rename tests/units/{test_jinja2_parser.py => test_markup_parser.py} (98%) diff --git a/src/antsibull_docs/jinja2/environment.py b/src/antsibull_docs/jinja2/environment.py index e4668c9f..b29a864e 100644 --- a/src/antsibull_docs/jinja2/environment.py +++ b/src/antsibull_docs/jinja2/environment.py @@ -10,6 +10,8 @@ from jinja2 import BaseLoader, Environment, FileSystemLoader, PackageLoader +from ..markup.htmlify import html_ify +from ..markup.rstify import rst_code, rst_escape, rst_ify from ..utils.collection_name_transformer import CollectionNameTransformer from .filters import ( do_max, @@ -23,8 +25,6 @@ to_ini_value, to_json, ) -from .htmlify import html_ify -from .rstify import rst_code, rst_escape, rst_ify from .tests import still_relevant, test_list # kludge_ns gives us a kludgey way to set variables inside of loops that need to be visible outside diff --git a/src/antsibull_docs/jinja2/filters.py b/src/antsibull_docs/jinja2/filters.py index ee20ca51..e83b7224 100644 --- a/src/antsibull_docs/jinja2/filters.py +++ b/src/antsibull_docs/jinja2/filters.py @@ -13,6 +13,10 @@ from antsibull_core.logging import log from jinja2.runtime import Context, Undefined +from jinja2.utils import pass_context + +from ..markup.rstify import rst_ify as rst_ify_impl +from ..markup.htmlify import html_ify as html_ify_impl mlog = log.fields(mod=__name__) @@ -127,3 +131,41 @@ def to_ini_value(data: t.Any) -> str: return 'MAPPINGS ARE NOT SUPPORTED' # Handle other values (booleans, integers, floats) as JSON return json.dumps(data) + + +@pass_context +def rst_ify(context: Context, text: str, + *, + plugin_fqcn: t.Optional[str] = None, + plugin_type: t.Optional[str] = None) -> str: + ''' convert symbols like I(this is in italics) to valid restructured text ''' + flog = mlog.fields(func='rst_ify') + flog.fields(text=text).debug('Enter') + + plugin_fqcn, plugin_type = extract_plugin_data( + context, plugin_fqcn=plugin_fqcn, plugin_type=plugin_type) + + text, counts = rst_ify_impl(text, plugin_fqcn=plugin_fqcn, plugin_type=plugin_type) + + flog.fields(counts=counts).info('Number of macros converted to rst equivalents') + flog.debug('Leave') + return text + + +@pass_context +def html_ify(context: Context, text: str, + *, + plugin_fqcn: t.Optional[str] = None, + plugin_type: t.Optional[str] = None) -> str: + ''' convert symbols like I(this is in italics) to valid HTML ''' + flog = mlog.fields(func='html_ify') + flog.fields(text=text).debug('Enter') + + plugin_fqcn, plugin_type = extract_plugin_data( + context, plugin_fqcn=plugin_fqcn, plugin_type=plugin_type) + + text, counts = html_ify_impl(text, plugin_fqcn=plugin_fqcn, plugin_type=plugin_type) + + flog.fields(counts=counts).info('Number of macros converted to html equivalents') + flog.debug('Leave') + return text diff --git a/src/antsibull_docs/jinja2/htmlify.py b/src/antsibull_docs/markup/htmlify.py similarity index 91% rename from src/antsibull_docs/jinja2/htmlify.py rename to src/antsibull_docs/markup/htmlify.py index 91c06df7..46fa67eb 100644 --- a/src/antsibull_docs/jinja2/htmlify.py +++ b/src/antsibull_docs/markup/htmlify.py @@ -11,16 +11,9 @@ from html import escape as html_escape from urllib.parse import quote -from antsibull_core.logging import log -from jinja2.runtime import Context -from jinja2.utils import pass_context - -from ..semantic_helper import parse_option, parse_return_value -from .filters import extract_plugin_data +from .semantic_helper import parse_option, parse_return_value from .parser import Command, CommandSet, convert_text -mlog = log.fields(mod=__name__) - _MODULE = re.compile(r"^([^).]+)\.([^).]+)\.([^)]+)$") _PLUGIN = re.compile(r"^([^).]+)\.([^).]+)\.([^)]+)#([a-z]+)$") @@ -37,15 +30,13 @@ def _create_error(text: str, error: str) -> str: class _Context: - j2_context: Context counts: t.Dict[str, int] plugin_fqcn: t.Optional[str] plugin_type: t.Optional[str] - def __init__(self, j2_context: Context, + def __init__(self, plugin_fqcn: t.Optional[str] = None, plugin_type: t.Optional[str] = None): - self.j2_context = j2_context self.counts = { 'italic': 0, 'bold': 0, @@ -61,8 +52,8 @@ def __init__(self, j2_context: Context, 'return-value': 0, 'ruler': 0, } - self.plugin_fqcn, self.plugin_type = extract_plugin_data( - j2_context, plugin_fqcn=plugin_fqcn, plugin_type=plugin_type) + self.plugin_fqcn = plugin_fqcn + self.plugin_type = plugin_type # In the following, we make heavy use of escaped whitespace ("\ ") being removed from the output. @@ -300,17 +291,12 @@ def handle(self, parameters: t.List[str], context: t.Any) -> str: ]) -@pass_context -def html_ify(context: Context, text: str, +def html_ify(text: str, *, plugin_fqcn: t.Optional[str] = None, - plugin_type: t.Optional[str] = None) -> str: + plugin_type: t.Optional[str] = None) -> t.Tuple[str, t.Mapping[str, int]]: ''' convert symbols like I(this is in italics) to valid HTML ''' - flog = mlog.fields(func='html_ify') - flog.fields(text=text).debug('Enter') - our_context = _Context( - context, plugin_fqcn=plugin_fqcn, plugin_type=plugin_type, ) @@ -318,8 +304,6 @@ def html_ify(context: Context, text: str, try: text = convert_text(text, _COMMAND_SET, html_escape, our_context) except Exception as exc: # pylint:disable=broad-except - return _create_error(text, str(exc)) + text = _create_error(text, str(exc)) - flog.fields(counts=our_context.counts).info('Number of macros converted to html equivalents') - flog.debug('Leave') - return text + return text, our_context.counts diff --git a/src/antsibull_docs/jinja2/parser.py b/src/antsibull_docs/markup/parser.py similarity index 100% rename from src/antsibull_docs/jinja2/parser.py rename to src/antsibull_docs/markup/parser.py diff --git a/src/antsibull_docs/jinja2/rstify.py b/src/antsibull_docs/markup/rstify.py similarity index 90% rename from src/antsibull_docs/jinja2/rstify.py rename to src/antsibull_docs/markup/rstify.py index 865026be..6815a401 100644 --- a/src/antsibull_docs/jinja2/rstify.py +++ b/src/antsibull_docs/markup/rstify.py @@ -10,16 +10,9 @@ import typing as t from urllib.parse import quote -from antsibull_core.logging import log -from jinja2.runtime import Context -from jinja2.utils import pass_context - -from ..semantic_helper import augment_plugin_name_type -from .filters import extract_plugin_data +from .semantic_helper import augment_plugin_name_type from .parser import Command, CommandSet, convert_text -mlog = log.fields(mod=__name__) - _MODULE = re.compile(r"^([^).]+)\.([^).]+)\.([^)]+)$") _PLUGIN = re.compile(r"^([^).]+)\.([^).]+)\.([^)]+)#([a-z]+)$") @@ -65,15 +58,13 @@ def _create_error(text: str, error: str) -> str: class _Context: - j2_context: Context counts: t.Dict[str, int] plugin_fqcn: t.Optional[str] plugin_type: t.Optional[str] - def __init__(self, j2_context: Context, + def __init__(self, plugin_fqcn: t.Optional[str] = None, plugin_type: t.Optional[str] = None): - self.j2_context = j2_context self.counts = { 'italic': 0, 'bold': 0, @@ -89,8 +80,8 @@ def __init__(self, j2_context: Context, 'return-value': 0, 'ruler': 0, } - self.plugin_fqcn, self.plugin_type = extract_plugin_data( - j2_context, plugin_fqcn=plugin_fqcn, plugin_type=plugin_type) + self.plugin_fqcn = plugin_fqcn + self.plugin_type = plugin_type # In the following, we make heavy use of escaped whitespace ("\ ") being removed from the output. @@ -265,17 +256,12 @@ def handle(self, parameters: t.List[str], context: t.Any) -> str: ]) -@pass_context -def rst_ify(context: Context, text: str, +def rst_ify(text: str, *, plugin_fqcn: t.Optional[str] = None, - plugin_type: t.Optional[str] = None) -> str: + plugin_type: t.Optional[str] = None) -> t.Tuple[str, t.Mapping[str, int]]: ''' convert symbols like I(this is in italics) to valid restructured text ''' - flog = mlog.fields(func='rst_ify') - flog.fields(text=text).debug('Enter') - our_context = _Context( - context, plugin_fqcn=plugin_fqcn, plugin_type=plugin_type, ) @@ -283,8 +269,6 @@ def rst_ify(context: Context, text: str, try: text = convert_text(text, _COMMAND_SET, rst_escape, our_context) except Exception as exc: # pylint:disable=broad-except - return _create_error(text, str(exc)) + text = _create_error(text, str(exc)) - flog.fields(counts=our_context.counts).info('Number of macros converted to rst equivalents') - flog.debug('Leave') - return text + return text, our_context.counts diff --git a/src/antsibull_docs/semantic_helper.py b/src/antsibull_docs/markup/semantic_helper.py similarity index 100% rename from src/antsibull_docs/semantic_helper.py rename to src/antsibull_docs/markup/semantic_helper.py diff --git a/src/sphinx_antsibull_ext/roles.py b/src/sphinx_antsibull_ext/roles.py index 49c5b53b..3259a495 100644 --- a/src/sphinx_antsibull_ext/roles.py +++ b/src/sphinx_antsibull_ext/roles.py @@ -12,7 +12,7 @@ from docutils import nodes from sphinx import addnodes -from antsibull_docs.semantic_helper import parse_option, parse_return_value +from antsibull_docs.markup.semantic_helper import parse_option, parse_return_value # pylint:disable-next=unused-argument,dangerous-default-value diff --git a/tests/units/test_jinja2.py b/tests/units/test_jinja2.py index afd0c00b..2dbb434e 100644 --- a/tests/units/test_jinja2.py +++ b/tests/units/test_jinja2.py @@ -4,8 +4,7 @@ import pytest -from antsibull_docs.jinja2.filters import massage_author_name, move_first, to_ini_value, to_json -from antsibull_docs.jinja2.rstify import rst_escape, rst_ify +from antsibull_docs.jinja2.filters import massage_author_name, move_first, to_ini_value, to_json, rst_ify RST_IFY_DATA = { # No substitutions @@ -41,28 +40,6 @@ def test_rst_ify(text, expected): assert rst_ify(context, text) == expected -RST_ESCAPE_DATA = { - '': '', - 'no-op': 'no-op', - None: 'None', - 1: '1', - '*': '\\*', - '_': '\\_', - '<': '\\<', - '>': '\\>', - '`': '\\`', - '\\': '\\\\', - '\\*': '\\\\\\*', - '*\\': '\\*\\\\', - ':role:`test`': ':role:\\`test\\`', -} - - -@pytest.mark.parametrize('value, expected', RST_ESCAPE_DATA.items()) -def test_escape_ify(value, expected): - assert rst_escape(value) == expected - - MOVE_FIRST_DATA = [ ([], [], []), (['a', 'b', 'c'], ['d'], ['a', 'b', 'c']), diff --git a/tests/units/test_jinja2_parser.py b/tests/units/test_markup_parser.py similarity index 98% rename from tests/units/test_jinja2_parser.py rename to tests/units/test_markup_parser.py index 52707037..930fb603 100644 --- a/tests/units/test_jinja2_parser.py +++ b/tests/units/test_markup_parser.py @@ -7,7 +7,7 @@ import pytest -from antsibull_docs.jinja2.parser import Command, CommandSet, ParsingException, convert_text +from antsibull_docs.markup.parser import Command, CommandSet, ParsingException, convert_text class TestCommand0(Command): From f37fe19cdf77db3ee2c145f1432e9c0a83591e76 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Wed, 15 Mar 2023 22:57:27 +0100 Subject: [PATCH 02/15] Forgot __init__.py. --- src/antsibull_docs/markup/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 src/antsibull_docs/markup/__init__.py diff --git a/src/antsibull_docs/markup/__init__.py b/src/antsibull_docs/markup/__init__.py new file mode 100644 index 00000000..7547f6e1 --- /dev/null +++ b/src/antsibull_docs/markup/__init__.py @@ -0,0 +1,4 @@ +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2020, Ansible Project From 98d3cf0ee32641b51d05db01d292e900ca04ee28 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Wed, 15 Mar 2023 23:01:42 +0100 Subject: [PATCH 03/15] Fix imports. --- src/antsibull_docs/jinja2/environment.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/antsibull_docs/jinja2/environment.py b/src/antsibull_docs/jinja2/environment.py index b29a864e..f255c288 100644 --- a/src/antsibull_docs/jinja2/environment.py +++ b/src/antsibull_docs/jinja2/environment.py @@ -10,8 +10,7 @@ from jinja2 import BaseLoader, Environment, FileSystemLoader, PackageLoader -from ..markup.htmlify import html_ify -from ..markup.rstify import rst_code, rst_escape, rst_ify +from ..markup.rstify import rst_code, rst_escape from ..utils.collection_name_transformer import CollectionNameTransformer from .filters import ( do_max, @@ -24,6 +23,8 @@ rst_xline, to_ini_value, to_json, + html_ify, + rst_ify, ) from .tests import still_relevant, test_list From 1c569ca54d51945808d3f2c101eaa235873fafaf Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Thu, 16 Mar 2023 07:27:05 +0100 Subject: [PATCH 04/15] Move parser impl code out. --- src/antsibull_docs/markup/_parser_impl.py | 87 ++++++++++++++++++ src/antsibull_docs/markup/parser.py | 91 ++----------------- tests/units/markup/test_markup.py | 58 ++++++++++++ .../test_parser.py} | 8 +- tests/units/markup/test_parser_impl.py | 65 +++++++++++++ 5 files changed, 224 insertions(+), 85 deletions(-) create mode 100644 src/antsibull_docs/markup/_parser_impl.py create mode 100644 tests/units/markup/test_markup.py rename tests/units/{test_markup_parser.py => markup/test_parser.py} (89%) create mode 100644 tests/units/markup/test_parser_impl.py diff --git a/src/antsibull_docs/markup/_parser_impl.py b/src/antsibull_docs/markup/_parser_impl.py new file mode 100644 index 00000000..21ed5ac4 --- /dev/null +++ b/src/antsibull_docs/markup/_parser_impl.py @@ -0,0 +1,87 @@ +# Author: Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2022, Ansible Project +""" +Internal parsing code. +""" + +import re +import typing as t + +_ESCAPE_OR_COMMA = re.compile(r'\\(.)| *(,) *') +_ESCAPE_OR_CLOSING = re.compile(r'\\(.)|([)])') + + +def parse_parameters_escaped(text: str, index: int, parameter_count: int, + ) -> t.Tuple[t.List[str], int, t.Optional[str]]: + result: t.List[str] = [] + parameters_left = parameter_count + while parameters_left > 1: + parameters_left -= 1 + value: t.List[str] = [] + while True: + match = _ESCAPE_OR_COMMA.search(text, pos=index) + if not match: + result.append(''.join(value)) + return ( + result, + len(text), + 'Cannot find comma separating parameter' + f' {parameter_count - parameters_left} from the next one' + ) + value.append(text[index:match.start(0)]) + index = match.end(0) + if match.group(1): + value.append(match.group(1)) + else: + break + result.append(''.join(value)) + value = [] + while True: + match = _ESCAPE_OR_CLOSING.search(text, pos=index) + if not match: + result.append(''.join(value)) + return result, len(text), 'Cannot find closing ")" after last parameter' + value.append(text[index:match.start(0)]) + index = match.end(0) + if match.group(1): + value.append(match.group(1)) + else: + break + result.append(''.join(value)) + return result, index, None + + +def parse_parameters_unescaped(text: str, index: int, parameter_count: int, + ) -> t.Tuple[t.List[str], int, t.Optional[str]]: + result: t.List[str] = [] + first = True + parameters_left = parameter_count + while parameters_left > 1: + parameters_left -= 1 + next_index = text.find(',', index) + if next_index < 0: + return ( + result, + len(text), + 'Cannot find comma separating parameter' + f' {parameter_count - parameters_left} from the next one' + ) + parameter = text[index:next_index].rstrip(' ') + if not first: + parameter = parameter.lstrip(' ') + else: + first = False + result.append(parameter) + index = next_index + 1 + next_index = text.find(')', index) + if next_index < 0: + return result, len(text), 'Cannot find closing ")" after last parameter' + parameter = text[index:next_index] + if not first: + parameter = parameter.lstrip(' ') + result.append(parameter) + index = next_index + 1 + return result, index, None diff --git a/src/antsibull_docs/markup/parser.py b/src/antsibull_docs/markup/parser.py index 7231592b..fd916abf 100644 --- a/src/antsibull_docs/markup/parser.py +++ b/src/antsibull_docs/markup/parser.py @@ -11,8 +11,7 @@ import re import typing as t -_ESCAPE_OR_COMMA = re.compile(r'\\(.)| *(,) *') -_ESCAPE_OR_CLOSING = re.compile(r'\\(.)|([)])') +from ._parser_impl import parse_parameters_escaped, parse_parameters_unescaped class ParsingException(Exception): @@ -72,83 +71,6 @@ class CommandData(t.NamedTuple): Part = t.Union[str, CommandData] -def parse_parameters_escaped(text: str, index: int, command: Command, - command_start: int) -> t.Tuple[int, t.List[str]]: - result = [] - parameter_count = command.parameter_count - while parameter_count > 1: - parameter_count -= 1 - value = [] - while True: - match = _ESCAPE_OR_COMMA.search(text, pos=index) - if not match: - raise ParsingException( - f'Cannot find comma separating ' - f'parameter {command.parameter_count - parameter_count}' - f' from the next one for command "{command.command}"' - f' starting at index {command_start} in {text!r}' - ) - value.append(text[index:match.start(0)]) - index = match.end(0) - if match.group(1): - value.append(match.group(1)) - else: - break - result.append(''.join(value)) - value = [] - while True: - match = _ESCAPE_OR_CLOSING.search(text, pos=index) - if not match: - raise ParsingException( - f'Cannot find ")" closing after the last parameter for' - f' command "{command.command}" starting at index {command_start} in {text!r}' - ) - value.append(text[index:match.start(0)]) - index = match.end(0) - if match.group(1): - value.append(match.group(1)) - else: - break - result.append(''.join(value)) - return index, result - - -def parse_parameters_unescaped(text: str, index: int, command: Command, - command_start: int) -> t.Tuple[int, t.List[str]]: - result = [] - first = True - parameter_count = command.parameter_count - while parameter_count > 1: - parameter_count -= 1 - next_index = text.find(',', index) - if next_index < 0: - raise ParsingException( - f'Cannot find comma separating ' - f'parameter {command.parameter_count - parameter_count}' - f' from the next one for command "{command.command}"' - f' starting at index {command_start} in {text!r}' - ) - parameter = text[index:next_index].rstrip(' ') - if not first: - parameter = parameter.lstrip(' ') - else: - first = False - result.append(parameter) - index = next_index + 1 - next_index = text.find(')', index) - if next_index < 0: - raise ParsingException( - f'Cannot find ")" closing after the last parameter for' - f' command "{command.command}" starting at index {command_start} in {text!r}' - ) - parameter = text[index:next_index] - if not first: - parameter = parameter.lstrip(' ') - result.append(parameter) - index = next_index + 1 - return index, result - - def parse_text(text: str, commands: CommandSet) -> t.List[Part]: result: t.List[Part] = [] index = 0 @@ -167,9 +89,16 @@ def parse_text(text: str, commands: CommandSet) -> t.List[Part]: continue index += 1 if command.escaped_content: - index, parameters = parse_parameters_escaped(text, index, command, command_start) + parameters, index, error = parse_parameters_escaped( + text, index, command.parameter_count) else: - index, parameters = parse_parameters_unescaped(text, index, command, command_start) + parameters, index, error = parse_parameters_unescaped( + text, index, command.parameter_count) + if error is not None: + raise ParsingException( + error + + f' for command "{command.command}" starting at index {command_start} in {text!r}' + ) result.append(CommandData(command=command, parameters=parameters)) return result diff --git a/tests/units/markup/test_markup.py b/tests/units/markup/test_markup.py new file mode 100644 index 00000000..8bfffc8e --- /dev/null +++ b/tests/units/markup/test_markup.py @@ -0,0 +1,58 @@ +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2020, Ansible Project + +import pytest + +from antsibull_docs.markup.rstify import rst_escape, rst_ify + +RST_IFY_DATA = { + # No substitutions + 'no-op': 'no-op', + 'no-op Z(test)': 'no-op Z(test)', + # Simple cases of all substitutions + 'I(italic)': r'\ :emphasis:`italic`\ ', + 'B(bold)': r'\ :strong:`bold`\ ', + 'M(ansible.builtin.yum)': r'\ :ref:`ansible.builtin.yum' + r' `\ ', + 'U(https://docs.ansible.com)': r'\ https://docs.ansible.com\ ', + 'L(the user guide,https://docs.ansible.com/user-guide.html)': r'\ `the user guide' + r' `__\ ', + 'R(the user guide,user-guide)': r'\ :ref:`the user guide `\ ', + 'C(/usr/bin/file)': r'\ :literal:`/usr/bin/file`\ ', + 'HORIZONTALLINE': '\n\n.. raw:: html\n\n
\n\n', + # Multiple substitutions + 'The M(ansible.builtin.yum) module B(MUST) be given the C(package) parameter. See the R(looping docs,using-loops) for more info': + r'The \ :ref:`ansible.builtin.yum `\ module \ :strong:`MUST`\ be given the \ :literal:`package`\ parameter. See the \ :ref:`looping docs `\ for more info', + # Problem cases + 'IBM(International Business Machines)': 'IBM(International Business Machines)', + 'L(the user guide, https://docs.ansible.com/)': r'\ `the user guide `__\ ', + 'R(the user guide, user-guide)': r'\ :ref:`the user guide `\ ', +} + + +@pytest.mark.parametrize('text, expected', RST_IFY_DATA.items()) +def test_rst_ify(text, expected): + assert rst_ify(text, plugin_fqcn='foo.bar.baz', plugin_type='module')[0] == expected + + +RST_ESCAPE_DATA = { + '': '', + 'no-op': 'no-op', + None: 'None', + 1: '1', + '*': '\\*', + '_': '\\_', + '<': '\\<', + '>': '\\>', + '`': '\\`', + '\\': '\\\\', + '\\*': '\\\\\\*', + '*\\': '\\*\\\\', + ':role:`test`': ':role:\\`test\\`', +} + + +@pytest.mark.parametrize('value, expected', RST_ESCAPE_DATA.items()) +def test_escape_ify(value, expected): + assert rst_escape(value) == expected diff --git a/tests/units/test_markup_parser.py b/tests/units/markup/test_parser.py similarity index 89% rename from tests/units/test_markup_parser.py rename to tests/units/markup/test_parser.py index 930fb603..43ba51cc 100644 --- a/tests/units/test_markup_parser.py +++ b/tests/units/markup/test_parser.py @@ -119,11 +119,11 @@ def test_convert_text(text, expected): CONVERT_TEXT_FAIL_DATA = { - 'command1a(': """Cannot find ")" closing after the last parameter for command "command1a" starting at index 0 in 'command1a('""", - 'command2a(,': """Cannot find ")" closing after the last parameter for command "command2a" starting at index 0 in 'command2a(,'""", + 'command1a(': """Cannot find closing ")" after last parameter for command "command1a" starting at index 0 in 'command1a('""", + 'command2a(,': """Cannot find closing ")" after last parameter for command "command2a" starting at index 0 in 'command2a(,'""", 'command2a(': """Cannot find comma separating parameter 1 from the next one for command "command2a" starting at index 0 in 'command2a('""", - 'command1b(': """Cannot find ")" closing after the last parameter for command "command1b" starting at index 0 in 'command1b('""", - 'command2b(,': """Cannot find ")" closing after the last parameter for command "command2b" starting at index 0 in 'command2b(,'""", + 'command1b(': """Cannot find closing ")" after last parameter for command "command1b" starting at index 0 in 'command1b('""", + 'command2b(,': """Cannot find closing ")" after last parameter for command "command2b" starting at index 0 in 'command2b(,'""", 'command2b(': """Cannot find comma separating parameter 1 from the next one for command "command2b" starting at index 0 in 'command2b('""", r'command2b(\,)': r"""Cannot find comma separating parameter 1 from the next one for command "command2b" starting at index 0 in 'command2b(\\,)'""", } diff --git a/tests/units/markup/test_parser_impl.py b/tests/units/markup/test_parser_impl.py new file mode 100644 index 00000000..ec47b09c --- /dev/null +++ b/tests/units/markup/test_parser_impl.py @@ -0,0 +1,65 @@ +# Author: Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2022, Ansible Project + +import typing as t + +import pytest + +from antsibull_docs.markup._parser_impl import parse_parameters_escaped, parse_parameters_unescaped + + +ESCAPED_TESTS = [ + ['(a)', 1, 1, ['a'], 3, None], + ['(a,b)', 1, 1, ['a,b'], 5, None], + ['(a,b,c)', 1, 1, ['a,b,c'], 7, None], + ['(a,b)', 1, 2, ['a', 'b'], 5, None], + ['(a,b,c)', 1, 2, ['a', 'b,c'], 7, None], + ['(a,b,c)', 1, 3, ['a', 'b', 'c'], 7, None], + ['(a\\,,b\\,\\),c\\))', 1, 3, ['a,', 'b,)', 'c)'], 15, None], + ['(a', 1, 1, [''], 2, 'Cannot find closing ")" after last parameter'], + ['(a', 1, 2, [''], 2, 'Cannot find comma separating parameter 1 from the next one'], + ['(a,b', 1, 2, ['a', ''], 4, 'Cannot find closing ")" after last parameter'], +] + + +@pytest.mark.parametrize( + 'text, index, parameter_count, expected_result, expected_index, expected_error', + ESCAPED_TESTS, +) +def test_parse_parameters_escaped(text: str, index: int, parameter_count: int, + expected_result: t.List[str], expected_index: int, + expected_error: t.Optional[str]) -> None: + result, end_index, error = parse_parameters_escaped(text, index, parameter_count) + print(result, end_index, error) + assert result == expected_result + assert end_index == expected_index + assert error == expected_error + + +UNESCAPED_TESTS = [ + ['(a)', 1, 1, ['a'], 3, None], + ['(a,b)', 1, 1, ['a,b'], 5, None], + ['(a,b,c)', 1, 1, ['a,b,c'], 7, None], + ['(a,b)', 1, 2, ['a', 'b'], 5, None], + ['(a,b,c)', 1, 2, ['a', 'b,c'], 7, None], + ['(a,b,c)', 1, 3, ['a', 'b', 'c'], 7, None], + ['(a', 1, 1, [], 2, 'Cannot find closing ")" after last parameter'], + ['(a', 1, 2, [], 2, 'Cannot find comma separating parameter 1 from the next one'], + ['(a,b', 1, 2, ['a'], 4, 'Cannot find closing ")" after last parameter'], +] + + +@pytest.mark.parametrize( + 'text, index, parameter_count, expected_result, expected_index, expected_error', + UNESCAPED_TESTS, +) +def test_parse_parameters_unescaped(text: str, index: int, parameter_count: int, + expected_result: t.List[str], expected_index: int, + expected_error: t.Optional[str]) -> None: + result, end_index, error = parse_parameters_unescaped(text, index, parameter_count) + print(result, end_index, error) + assert result == expected_result + assert end_index == expected_index + assert error == expected_error From 4ac853267701914fc5554a695fc5fa63a1788a2e Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Thu, 16 Mar 2023 23:20:48 +0100 Subject: [PATCH 05/15] Began with new-style parser. --- src/antsibull_docs/markup/dom.py | 143 ++++++++++++ src/antsibull_docs/markup/parser.py | 341 ++++++++++++++++++++++++++++ 2 files changed, 484 insertions(+) create mode 100644 src/antsibull_docs/markup/dom.py diff --git a/src/antsibull_docs/markup/dom.py b/src/antsibull_docs/markup/dom.py new file mode 100644 index 00000000..6e015958 --- /dev/null +++ b/src/antsibull_docs/markup/dom.py @@ -0,0 +1,143 @@ +# Author: Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2023, Ansible Project +""" +DOM classes used by parser. +""" + +from enum import Enum +from typing import NamedTuple + +import typing as t + + +ErrorType = t.Union[t.Literal['ignore'], t.Literal['message'], t.Literal['exception']] + + +class PluginIdentifier(NamedTuple): + fqcn: str + type: str + + +class PartType(Enum): + ERROR = 0 + BOLD = 1 + CODE = 2 + HORIZONTAL_LINE = 3 + ITALIC = 4 + LINK = 5 + MODULE = 6 + RST_REF = 7 + URL = 8 + TEXT = 9 + ENV_VARIABLE = 10 + OPTION_NAME = 11 + OPTION_VALUE = 12 + PLUGIN = 13 + RETURN_VALUE = 14 + + +class TextPart(NamedTuple): + type: t.Literal[PartType.TEXT] + text: str + + +class ItalicPart(NamedTuple): + type: t.Literal[PartType.ITALIC] + text: str + + +class BoldPart(NamedTuple): + type: t.Literal[PartType.BOLD] + text: str + + +class ModulePart(NamedTuple): + type: t.Literal[PartType.MODULE] + fqcn: str + + +class PluginPart(NamedTuple): + type: t.Literal[PartType.PLUGIN] + plugin: PluginIdentifier + + +class URLPart(NamedTuple): + type: t.Literal[PartType.URL] + url: str + + +class LinkPart(NamedTuple): + type: t.Literal[PartType.LINK] + text: str + url: str + + +class RSTRefPart(NamedTuple): + type: t.Literal[PartType.RST_REF] + text: str + ref: str + + +class CodePart(NamedTuple): + type: t.Literal[PartType.CODE] + text: str + + +class OptionNamePart(NamedTuple): + type: t.Literal[PartType.OPTION_NAME] + plugin: t.Optional[PluginIdentifier] + link: t.List[str] + name: str + value: t.Optional[str] + + +class OptionValuePart(NamedTuple): + type: t.Literal[PartType.OPTION_VALUE] + value: str + + +class EnvVariablePart(NamedTuple): + type: t.Literal[PartType.ENV_VARIABLE] + name: str + + +class ReturnValuePart(NamedTuple): + type: t.Literal[PartType.RETURN_VALUE] + plugin: t.Optional[PluginIdentifier] + link: t.List[str] + name: str + value: t.Optional[str] + + +class HorizontalLinePart(NamedTuple): + type: t.Literal[PartType.HORIZONTAL_LINE] + + +class ErrorPart(NamedTuple): + type: t.Literal[PartType.ERROR] + message: str + + +AnyPart = t.Union[ + TextPart, + ItalicPart, + BoldPart, + ModulePart, + PluginPart, + URLPart, + LinkPart, + RSTRefPart, + CodePart, + OptionNamePart, + OptionValuePart, + EnvVariablePart, + ReturnValuePart, + HorizontalLinePart, + ErrorPart, +] + + +Paragraph = t.List[AnyPart] diff --git a/src/antsibull_docs/markup/parser.py b/src/antsibull_docs/markup/parser.py index fd916abf..fc48005c 100644 --- a/src/antsibull_docs/markup/parser.py +++ b/src/antsibull_docs/markup/parser.py @@ -11,6 +11,7 @@ import re import typing as t +from . import dom from ._parser_impl import parse_parameters_escaped, parse_parameters_unescaped @@ -113,3 +114,343 @@ def convert_text(text: str, commands: CommandSet, process_other_text: t.Callable else: result.append(part.command.handle(part.parameters, context)) return ''.join(result) + + + + + + + + + + + + + + + + + + + + + + + +_IGNORE_MARKER = 'ignore:' +_ARRAY_STUB_RE = re.compile(r'\[([^\]]*)\]') +_FQCN_TYPE_PREFIX_RE = re.compile(r'^([^.]+\.[^.]+\.[^#]+)#([a-z]+):(.*)$') +_FQCN = re.compile(r'^[a-z0-9_]+\.[a-z0-9_]+(?:\.[a-z0-9_]+)+$') +_PLUGIN_TYPE = re.compile(r'^[a-z_]+$') + + +def _is_fqcn(text: str) -> bool: + return _FQCN.match(text) is not None + + +def _is_plugin_type(text: str) -> bool: + # We do not want to hard-code a list of valid plugin types that might be inaccurate, so we + # simply check whether this is a valid kind of Python identifier usually used for plugin + # types. If ansible-core ever adds one with digits, we'll have to update this. + return _PLUGIN_TYPE.match(text) is not None + + +class Context(t.NamedTuple): + current_plugin: t.Optional[dom.PluginIdentifier] + + +class CommandParser(abc.ABC): + command: str + parameters: int + escaped_arguments: bool + + def __init__(self, command: str, parameters: int, escaped_arguments: bool = False): + self.command = command + self.parameters = parameters + self.escaped_arguments = escaped_arguments + + @abc.abstractmethod + def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: + pass + + +class CommandParserEx(CommandParser): + old_markup: bool + + def __init__(self, command: str, parameters: int, + escaped_arguments: bool = False, old_markup: bool = False): + super().__init__(command, parameters, escaped_arguments) + self.old_markup = old_markup + + +# Classic Ansible docs markup: + + +class _Italics(CommandParserEx): + def __init__(self): + super().__init__('I', 1, old_markup=True) + + def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: + return dom.ItalicPart(dom.PartType.ITALIC, text=parameters[0]) + + +class _Bold(CommandParserEx): + def __init__(self): + super().__init__('B', 1, old_markup=True) + + def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: + return dom.BoldPart(dom.PartType.BOLD, text=parameters[0]) + + +class _Module(CommandParserEx): + def __init__(self): + super().__init__('M', 1, old_markup=True) + + def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: + fqcn = parameters[0] + if not _is_fqcn(fqcn): + raise ValueError(f'Module name "{fqcn}" is not a FQCN') + return dom.ModulePart(dom.PartType.MODULE, fqcn=fqcn) + + +class _URL(CommandParserEx): + def __init__(self): + super().__init__('U', 1, old_markup=True) + + def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: + return dom.URLPart(dom.PartType.URL, url=parameters[0]) + + +class _Link(CommandParserEx): + def __init__(self): + super().__init__('L', 2, old_markup=True) + + def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: + text = parameters[0] + url = parameters[1] + return dom.LinkPart(dom.PartType.LINK, text=text, url=url) + + +class _RSTRef(CommandParserEx): + def __init__(self): + super().__init__('R', 2, old_markup=True) + + def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: + text = parameters[0] + ref = parameters[1] + return dom.RSTRefPart(dom.PartType.RST_REF, text=text, ref=ref) + + +class _Code(CommandParserEx): + def __init__(self): + super().__init__('C', 1, old_markup=True) + + def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: + return dom.CodePart(dom.PartType.CODE, text=parameters[0]) + + +class _HorizontalLine(CommandParserEx): + def __init__(self): + super().__init__('HORIZONTALLINE', 0, old_markup=True) + + def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: + return dom.HorizontalLinePart(dom.PartType.HORIZONTAL_LINE) + + +# Semantic Ansible docs markup: + + +def _parse_option_like(text: str, + context: Context, + ) -> t.Tuple[t.Optional[dom.PluginIdentifier], + t.List[str], str, t.Optional[str]]: + value = None + if '=' in text: + text, value = text.split('=', 1) + m = _FQCN_TYPE_PREFIX_RE.match(text) + if m: + plugin_fqcn = m.group(1) + plugin_type = m.group(2) + if not _is_fqcn(plugin_fqcn): + raise ValueError(f'Plugin name "{plugin_fqcn}" is not a FQCN') + if not _is_plugin_type(plugin_type): + raise ValueError(f'Plugin type "{plugin_type}" is not valid') + plugin_identifier = dom.PluginIdentifier(fqcn=plugin_fqcn, type=plugin_type) + text = m.group(3) + elif text.startswith(_IGNORE_MARKER): + plugin_identifier = None + text = text[len(_IGNORE_MARKER):] + else: + plugin_identifier = context.current_plugin + if ':' in text or '#' in text: + raise ValueError(f'Invalid option/return value name "{text}"') + return ( + plugin_identifier, + _ARRAY_STUB_RE.sub('', text).split('.'), + text, + value, + ) + + +class _Plugin(CommandParserEx): + def __init__(self): + super().__init__('P', 1, escaped_arguments=True) + + def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: + name = parameters[0] + if '#' not in name: + raise ValueError(f'Parameter "{name}" is not of the form FQCN#type') + fqcn, ptype = name.split('#', 1) + if not _is_fqcn(fqcn): + raise ValueError(f'Plugin name "{fqcn}" is not a FQCN') + if not _is_plugin_type(ptype): + raise ValueError(f'Plugin type "{type}" is not valid') + return dom.PluginPart( + dom.PartType.PLUGIN, plugin=dom.PluginIdentifier(fqcn=fqcn, type=ptype)) + + +class _EnvVar(CommandParserEx): + def __init__(self): + super().__init__('E', 1, escaped_arguments=True) + + def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: + return dom.EnvVariablePart(dom.PartType.ENV_VARIABLE, name=parameters[0]) + + +class _OptionValue(CommandParserEx): + def __init__(self): + super().__init__('V', 1, escaped_arguments=True) + + def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: + return dom.OptionValuePart(dom.PartType.OPTION_VALUE, value=parameters[0]) + + +class _OptionName(CommandParserEx): + def __init__(self): + super().__init__('O', 1, escaped_arguments=True) + + def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: + plugin, link, name, value = _parse_option_like(parameters[0], context) + return dom.OptionNamePart( + dom.PartType.OPTION_NAME, plugin=plugin, link=link, name=name, value=value) + + +class _ReturnValue(CommandParserEx): + def __init__(self): + super().__init__('RV', 1, escaped_arguments=True) + + def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: + plugin, link, name, value = _parse_option_like(parameters[0], context) + return dom.ReturnValuePart( + dom.PartType.RETURN_VALUE, plugin=plugin, link=link, name=name, value=value) + + +_COMMANDS = [ + _Italics(), + _Bold(), + _Module(), + _URL(), + _Link(), + _RSTRef(), + _Code(), + _HorizontalLine(), + _Plugin(), + _EnvVar(), + _OptionValue(), + _OptionName(), + _ReturnValue(), +] + + +def _command_re(command: CommandParser) -> str: + return r'\b' + re.escape(command.command) + (r'\b' if command.parameters == 0 else r'\(') + + +class Parser: + _group_map: t.Mapping[str, CommandParser] + _re: 're.Pattern' # on Python 3.6 the type is called differently + + def __init__(self, commands: t.Sequence[CommandParser]): + self._group_map = {cmd.command + ('(' if cmd.parameters else ''): cmd for cmd in commands} + if commands: + self._re = re.compile('(' + '|'.join([_command_re(cmd) for cmd in commands]) + ')') + else: + self._re = re.compile('x^') # does not match anything + + @staticmethod + def _parse_command(result: dom.Paragraph, text: str, cmd: CommandParser, index: int, + end_index: int, context: Context, errors: dom.ErrorType, where: str + ) -> int: + args: t.List[str] + error: t.Optional[str] = None + if cmd.parameters == 0: + args = [] + elif cmd.escaped_arguments: + args, end_index, error = parse_parameters_escaped(text, end_index, cmd.parameters) + else: + args, end_index, error = parse_parameters_unescaped( + text, end_index, cmd.parameters) + if error is None: + try: + result.append(cmd.parse(args, context)) + except Exception as exc: # pylint:disable=broad-except + error = f'{exc}' + if error is not None: + error = ( + f'While parsing {cmd.command}{"()" if cmd.parameters else ""}' + f' at index {index + 1}{where}: {error}' + ) + if errors == 'message': + result.append(dom.ErrorPart(dom.PartType.ERROR, message=error)) + elif errors == 'exception': + raise ValueError(error) + return end_index + + def parse_string(self, text: str, context: Context, + errors: dom.ErrorType = 'message', where: str = '') -> dom.Paragraph: + result: dom.Paragraph = [] + length = len(text) + index = 0 + while index < length: + m = self._re.search(text, index) + if m is None: + result.append(dom.TextPart(dom.PartType.TEXT, text=text[index:])) + break + if m.start(1) > index: + result.append(dom.TextPart(dom.PartType.TEXT, text=text[index:m.start(1)])) + index = self._parse_command( + result, + text, + self._group_map[m.group(1)], + m.start(1), + m.end(1), + context, + errors, + where, + ) + return result + + +_CLASSIC = Parser([cmd for cmd in _COMMANDS if cmd.old_markup]) +_SEMANTIC_MARKUP = Parser(_COMMANDS) + + +def parse(text: t.Union[str, t.Sequence[str]], + context: Context, + errors: dom.ErrorType = 'message', + only_classic_markup: bool = False + ) -> t.List[dom.Paragraph]: + has_paragraphs = True + if isinstance(text, str): + has_paragraphs = False + text = [text] + parser = _CLASSIC if only_classic_markup else _SEMANTIC_MARKUP + return [ + parser.parse_string( + par, + context, + errors=errors, + where=f' of paragraph {index + 1}' if has_paragraphs else '', + ) + for index, par in enumerate(text) + ] From 406cacd56f9305e23dd1040f1d58d2b2bab15765 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Fri, 17 Mar 2023 21:12:57 +0100 Subject: [PATCH 06/15] Fix typing for Python 3.6 and 3.7. --- src/antsibull_docs/markup/dom.py | 37 ++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/src/antsibull_docs/markup/dom.py b/src/antsibull_docs/markup/dom.py index 6e015958..1feb1d63 100644 --- a/src/antsibull_docs/markup/dom.py +++ b/src/antsibull_docs/markup/dom.py @@ -9,11 +9,16 @@ from enum import Enum from typing import NamedTuple +import sys import typing as t -ErrorType = t.Union[t.Literal['ignore'], t.Literal['message'], t.Literal['exception']] +if sys.version_info >= (3, 8): + ErrorType = t.Union[t.Literal['ignore'], t.Literal['message'], t.Literal['exception']] +else: + # Python 3.6/3.7 do not have t.Literal + ErrorType = str class PluginIdentifier(NamedTuple): @@ -40,54 +45,54 @@ class PartType(Enum): class TextPart(NamedTuple): - type: t.Literal[PartType.TEXT] + type: 't.Literal[PartType.TEXT]' text: str class ItalicPart(NamedTuple): - type: t.Literal[PartType.ITALIC] + type: 't.Literal[PartType.ITALIC]' text: str class BoldPart(NamedTuple): - type: t.Literal[PartType.BOLD] + type: 't.Literal[PartType.BOLD]' text: str class ModulePart(NamedTuple): - type: t.Literal[PartType.MODULE] + type: 't.Literal[PartType.MODULE]' fqcn: str class PluginPart(NamedTuple): - type: t.Literal[PartType.PLUGIN] + type: 't.Literal[PartType.PLUGIN]' plugin: PluginIdentifier class URLPart(NamedTuple): - type: t.Literal[PartType.URL] + type: 't.Literal[PartType.URL]' url: str class LinkPart(NamedTuple): - type: t.Literal[PartType.LINK] + type: 't.Literal[PartType.LINK]' text: str url: str class RSTRefPart(NamedTuple): - type: t.Literal[PartType.RST_REF] + type: 't.Literal[PartType.RST_REF]' text: str ref: str class CodePart(NamedTuple): - type: t.Literal[PartType.CODE] + type: 't.Literal[PartType.CODE]' text: str class OptionNamePart(NamedTuple): - type: t.Literal[PartType.OPTION_NAME] + type: 't.Literal[PartType.OPTION_NAME]' plugin: t.Optional[PluginIdentifier] link: t.List[str] name: str @@ -95,17 +100,17 @@ class OptionNamePart(NamedTuple): class OptionValuePart(NamedTuple): - type: t.Literal[PartType.OPTION_VALUE] + type: 't.Literal[PartType.OPTION_VALUE]' value: str class EnvVariablePart(NamedTuple): - type: t.Literal[PartType.ENV_VARIABLE] + type: 't.Literal[PartType.ENV_VARIABLE]' name: str class ReturnValuePart(NamedTuple): - type: t.Literal[PartType.RETURN_VALUE] + type: 't.Literal[PartType.RETURN_VALUE]' plugin: t.Optional[PluginIdentifier] link: t.List[str] name: str @@ -113,11 +118,11 @@ class ReturnValuePart(NamedTuple): class HorizontalLinePart(NamedTuple): - type: t.Literal[PartType.HORIZONTAL_LINE] + type: 't.Literal[PartType.HORIZONTAL_LINE]' class ErrorPart(NamedTuple): - type: t.Literal[PartType.ERROR] + type: 't.Literal[PartType.ERROR]' message: str From f6f66fa5e24f9630a8dd6c0d011c3ec16029dfb7 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Fri, 17 Mar 2023 21:48:08 +0100 Subject: [PATCH 07/15] Prepare formatters. --- src/antsibull_docs/markup/dom.py | 147 +++++++++++++++++++++- src/antsibull_docs/markup/format.py | 188 ++++++++++++++++++++++++++++ 2 files changed, 334 insertions(+), 1 deletion(-) create mode 100644 src/antsibull_docs/markup/format.py diff --git a/src/antsibull_docs/markup/dom.py b/src/antsibull_docs/markup/dom.py index 1feb1d63..69d13788 100644 --- a/src/antsibull_docs/markup/dom.py +++ b/src/antsibull_docs/markup/dom.py @@ -7,9 +7,11 @@ DOM classes used by parser. """ +import abc +import sys + from enum import Enum from typing import NamedTuple -import sys import typing as t @@ -146,3 +148,146 @@ class ErrorPart(NamedTuple): Paragraph = t.List[AnyPart] + + +class Walker(abc.ABC): + @abc.abstractmethod + def process_error(self, part: ErrorPart) -> None: + pass + + @abc.abstractmethod + def process_bold(self, part: BoldPart) -> None: + pass + + @abc.abstractmethod + def process_code(self, part: CodePart) -> None: + pass + + @abc.abstractmethod + def process_horizontal_line(self, part: HorizontalLinePart) -> None: + pass + + @abc.abstractmethod + def process_italic(self, part: ItalicPart) -> None: + pass + + @abc.abstractmethod + def process_link(self, part: LinkPart) -> None: + pass + + @abc.abstractmethod + def process_module(self, part: ModulePart) -> None: + pass + + @abc.abstractmethod + def process_rst_ref(self, part: RSTRefPart) -> None: + pass + + @abc.abstractmethod + def process_url(self, part: URLPart) -> None: + pass + + @abc.abstractmethod + def process_text(self, part: TextPart) -> None: + pass + + @abc.abstractmethod + def process_env_variable(self, part: EnvVariablePart) -> None: + pass + + @abc.abstractmethod + def process_option_name(self, part: OptionNamePart) -> None: + pass + + @abc.abstractmethod + def process_option_value(self, part: OptionValuePart) -> None: + pass + + @abc.abstractmethod + def process_plugin(self, part: PluginPart) -> None: + pass + + @abc.abstractmethod + def process_return_value(self, part: ReturnValuePart) -> None: + pass + + +class NoopWalker(Walker): + def process_error(self, part: ErrorPart) -> None: + pass + + def process_bold(self, part: BoldPart) -> None: + pass + + def process_code(self, part: CodePart) -> None: + pass + + def process_horizontal_line(self, part: HorizontalLinePart) -> None: + pass + + def process_italic(self, part: ItalicPart) -> None: + pass + + def process_link(self, part: LinkPart) -> None: + pass + + def process_module(self, part: ModulePart) -> None: + pass + + def process_rst_ref(self, part: RSTRefPart) -> None: + pass + + def process_url(self, part: URLPart) -> None: + pass + + def process_text(self, part: TextPart) -> None: + pass + + def process_env_variable(self, part: EnvVariablePart) -> None: + pass + + def process_option_name(self, part: OptionNamePart) -> None: + pass + + def process_option_value(self, part: OptionValuePart) -> None: + pass + + def process_plugin(self, part: PluginPart) -> None: + pass + + def process_return_value(self, part: ReturnValuePart) -> None: + pass + + +def walk(paragraph: Paragraph, walker: Walker) -> None: # noqa: C901, pylint:disable=too-many-branches + for part in paragraph: + if part.type == PartType.ERROR: + walker.process_error(t.cast(ErrorPart, part)) + elif part.type == PartType.BOLD: + walker.process_bold(t.cast(BoldPart, part)) + elif part.type == PartType.CODE: + walker.process_code(t.cast(CodePart, part)) + elif part.type == PartType.HORIZONTAL_LINE: + walker.process_horizontal_line(t.cast(HorizontalLinePart, part)) + elif part.type == PartType.ITALIC: + walker.process_italic(t.cast(ItalicPart, part)) + elif part.type == PartType.LINK: + walker.process_link(t.cast(LinkPart, part)) + elif part.type == PartType.MODULE: + walker.process_module(t.cast(ModulePart, part)) + elif part.type == PartType.RST_REF: + walker.process_rst_ref(t.cast(RSTRefPart, part)) + elif part.type == PartType.URL: + walker.process_url(t.cast(URLPart, part)) + elif part.type == PartType.TEXT: + walker.process_text(t.cast(TextPart, part)) + elif part.type == PartType.ENV_VARIABLE: + walker.process_env_variable(t.cast(EnvVariablePart, part)) + elif part.type == PartType.OPTION_NAME: + walker.process_option_name(t.cast(OptionNamePart, part)) + elif part.type == PartType.OPTION_VALUE: + walker.process_option_value(t.cast(OptionValuePart, part)) + elif part.type == PartType.PLUGIN: + walker.process_plugin(t.cast(PluginPart, part)) + elif part.type == PartType.RETURN_VALUE: + walker.process_return_value(t.cast(ReturnValuePart, part)) diff --git a/src/antsibull_docs/markup/format.py b/src/antsibull_docs/markup/format.py new file mode 100644 index 00000000..f276ab6c --- /dev/null +++ b/src/antsibull_docs/markup/format.py @@ -0,0 +1,188 @@ +# Author: Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2023, Ansible Project +""" +Flexible formatting of DOM. +""" + +import abc + +import typing as t + +from . import dom + + +class LinkProvider(abc.ABC): + def plugin_link(self, # pylint:disable=no-self-use + plugin: dom.PluginIdentifier, # pylint:disable=unused-argument + ) -> t.Optional[str]: + '''Provides a link to a plugin.''' + return None + + def plugin_option_like_link(self, # pylint:disable=no-self-use + plugin: dom.PluginIdentifier, # pylint:disable=unused-argument + # pylint:disable-next=unused-argument + what: "t.Union[t.Literal['option'], t.Literal['retval']]", + # pylint:disable-next=unused-argument + name: t.List[str], current_plugin: bool) -> t.Optional[str]: + '''Provides a link to a plugin's option or return value.''' + return None + + +class _DefaultLinkProvider(LinkProvider): + pass + + +class Formatter(abc.ABC): + @abc.abstractmethod + def format_error(self, part: dom.ErrorPart) -> str: + pass + + @abc.abstractmethod + def format_bold(self, part: dom.BoldPart) -> str: + pass + + @abc.abstractmethod + def format_code(self, part: dom.CodePart) -> str: + pass + + @abc.abstractmethod + def format_horizontal_line(self, part: dom.HorizontalLinePart) -> str: + pass + + @abc.abstractmethod + def format_italic(self, part: dom.ItalicPart) -> str: + pass + + @abc.abstractmethod + def format_link(self, part: dom.LinkPart) -> str: + pass + + @abc.abstractmethod + def format_module(self, part: dom.ModulePart, url: t.Optional[str]) -> str: + pass + + @abc.abstractmethod + def format_rst_ref(self, part: dom.RSTRefPart) -> str: + pass + + @abc.abstractmethod + def format_url(self, part: dom.URLPart) -> str: + pass + + @abc.abstractmethod + def format_text(self, part: dom.TextPart) -> str: + pass + + @abc.abstractmethod + def format_env_variable(self, part: dom.EnvVariablePart) -> str: + pass + + @abc.abstractmethod + def format_option_name(self, part: dom.OptionNamePart, url: t.Optional[str]) -> str: + pass + + @abc.abstractmethod + def format_option_value(self, part: dom.OptionValuePart) -> str: + pass + + @abc.abstractmethod + def format_plugin(self, part: dom.PluginPart, url: t.Optional[str]) -> str: + pass + + @abc.abstractmethod + def format_return_value(self, part: dom.ReturnValuePart, url: t.Optional[str]) -> str: + pass + + +class _FormatWalker(dom.Walker): + destination: t.List[str] + formatter: Formatter + link_provider: LinkProvider + current_plugin: t.Optional[dom.PluginIdentifier] + + def __init__(self, destination: t.List[str], + formatter: Formatter, + link_provider: LinkProvider, + current_plugin: t.Optional[dom.PluginIdentifier]): + self.destination = destination + self.formatter = formatter + self.link_provider = link_provider + self.current_plugin = current_plugin + + def process_error(self, part: dom.ErrorPart) -> None: + self.destination.append(self.formatter.format_error(part)) + + def process_bold(self, part: dom.BoldPart) -> None: + self.destination.append(self.formatter.format_bold(part)) + + def process_code(self, part: dom.CodePart) -> None: + self.destination.append(self.formatter.format_code(part)) + + def process_horizontal_line(self, part: dom.HorizontalLinePart) -> None: + self.destination.append(self.formatter.format_horizontal_line(part)) + + def process_italic(self, part: dom.ItalicPart) -> None: + self.destination.append(self.formatter.format_italic(part)) + + def process_link(self, part: dom.LinkPart) -> None: + self.destination.append(self.formatter.format_link(part)) + + def process_module(self, part: dom.ModulePart) -> None: + url = self.link_provider.plugin_link(dom.PluginIdentifier(fqcn=part.fqcn, type='module')) + self.destination.append(self.formatter.format_module(part, url)) + + def process_rst_ref(self, part: dom.RSTRefPart) -> None: + self.destination.append(self.formatter.format_rst_ref(part)) + + def process_url(self, part: dom.URLPart) -> None: + self.destination.append(self.formatter.format_url(part)) + + def process_text(self, part: dom.TextPart) -> None: + self.destination.append(self.formatter.format_text(part)) + + def process_env_variable(self, part: dom.EnvVariablePart) -> None: + self.destination.append(self.formatter.format_env_variable(part)) + + def process_option_name(self, part: dom.OptionNamePart) -> None: + url = None + if part.plugin: + url = self.link_provider.plugin_option_like_link( + part.plugin, 'option', part.link, part.plugin == self.current_plugin) + self.destination.append(self.formatter.format_option_name(part, url)) + + def process_option_value(self, part: dom.OptionValuePart) -> None: + self.destination.append(self.formatter.format_option_value(part)) + + def process_plugin(self, part: dom.PluginPart) -> None: + url = self.link_provider.plugin_link(part.plugin) + self.destination.append(self.formatter.format_plugin(part, url)) + + def process_return_value(self, part: dom.ReturnValuePart) -> None: + url = None + if part.plugin: + url = self.link_provider.plugin_option_like_link( + part.plugin, 'retval', part.link, part.plugin == self.current_plugin) + self.destination.append(self.formatter.format_return_value(part, url)) + + +def format_paragraphs(paragraphs: t.Sequence[dom.Paragraph], + formatter: Formatter, + link_provider: t.Optional[LinkProvider] = None, + par_start: str = '', + par_end: str = '', + par_sep: str = '', + current_plugin: t.Optional[dom.PluginIdentifier] = None) -> str: + if link_provider is None: + link_provider = _DefaultLinkProvider() + result: t.List[str] = [] + walker = _FormatWalker(result, formatter, link_provider, current_plugin) + for paragraph in paragraphs: + if result: + result.append(par_sep) + result.append(par_start) + dom.walk(paragraph, walker) + result.append(par_end) + return ''.join(result) From 4b5fc5aa031e631fbe4dbc85629ad25562bf3d95 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sat, 18 Mar 2023 14:32:09 +0100 Subject: [PATCH 08/15] Continue. --- src/antsibull_docs/markup/dom.py | 45 ++-- src/antsibull_docs/markup/format.py | 19 ++ src/antsibull_docs/markup/html.py | 141 +++++++++++ src/antsibull_docs/markup/md.py | 113 +++++++++ src/antsibull_docs/markup/parser.py | 43 ++-- src/antsibull_docs/markup/rst.py | 130 +++++++++++ tests/units/markup/test_dom.py | 121 ++++++++++ tests/units/markup/test_format.py | 65 ++++++ tests/units/markup/test_html.py | 19 ++ tests/units/markup/test_md.py | 18 ++ tests/units/markup/test_parser.py | 348 +++++++++++++++++++++++++++- tests/units/markup/test_rst.py | 19 ++ 12 files changed, 1041 insertions(+), 40 deletions(-) create mode 100644 src/antsibull_docs/markup/html.py create mode 100644 src/antsibull_docs/markup/md.py create mode 100644 src/antsibull_docs/markup/rst.py create mode 100644 tests/units/markup/test_dom.py create mode 100644 tests/units/markup/test_format.py create mode 100644 tests/units/markup/test_html.py create mode 100644 tests/units/markup/test_md.py create mode 100644 tests/units/markup/test_rst.py diff --git a/src/antsibull_docs/markup/dom.py b/src/antsibull_docs/markup/dom.py index 69d13788..7beb05f6 100644 --- a/src/antsibull_docs/markup/dom.py +++ b/src/antsibull_docs/markup/dom.py @@ -47,85 +47,85 @@ class PartType(Enum): class TextPart(NamedTuple): - type: 't.Literal[PartType.TEXT]' text: str + type: 't.Literal[PartType.TEXT]' = PartType.TEXT class ItalicPart(NamedTuple): - type: 't.Literal[PartType.ITALIC]' text: str + type: 't.Literal[PartType.ITALIC]' = PartType.ITALIC class BoldPart(NamedTuple): - type: 't.Literal[PartType.BOLD]' text: str + type: 't.Literal[PartType.BOLD]' = PartType.BOLD class ModulePart(NamedTuple): - type: 't.Literal[PartType.MODULE]' fqcn: str + type: 't.Literal[PartType.MODULE]' = PartType.MODULE class PluginPart(NamedTuple): - type: 't.Literal[PartType.PLUGIN]' plugin: PluginIdentifier + type: 't.Literal[PartType.PLUGIN]' = PartType.PLUGIN class URLPart(NamedTuple): - type: 't.Literal[PartType.URL]' url: str + type: 't.Literal[PartType.URL]' = PartType.URL class LinkPart(NamedTuple): - type: 't.Literal[PartType.LINK]' text: str url: str + type: 't.Literal[PartType.LINK]' = PartType.LINK class RSTRefPart(NamedTuple): - type: 't.Literal[PartType.RST_REF]' text: str ref: str + type: 't.Literal[PartType.RST_REF]' = PartType.RST_REF class CodePart(NamedTuple): - type: 't.Literal[PartType.CODE]' text: str + type: 't.Literal[PartType.CODE]' = PartType.CODE class OptionNamePart(NamedTuple): - type: 't.Literal[PartType.OPTION_NAME]' plugin: t.Optional[PluginIdentifier] link: t.List[str] name: str value: t.Optional[str] + type: 't.Literal[PartType.OPTION_NAME]' = PartType.OPTION_NAME class OptionValuePart(NamedTuple): - type: 't.Literal[PartType.OPTION_VALUE]' value: str + type: 't.Literal[PartType.OPTION_VALUE]' = PartType.OPTION_VALUE class EnvVariablePart(NamedTuple): - type: 't.Literal[PartType.ENV_VARIABLE]' name: str + type: 't.Literal[PartType.ENV_VARIABLE]' = PartType.ENV_VARIABLE class ReturnValuePart(NamedTuple): - type: 't.Literal[PartType.RETURN_VALUE]' plugin: t.Optional[PluginIdentifier] link: t.List[str] name: str value: t.Optional[str] + type: 't.Literal[PartType.RETURN_VALUE]' = PartType.RETURN_VALUE class HorizontalLinePart(NamedTuple): - type: 't.Literal[PartType.HORIZONTAL_LINE]' + type: 't.Literal[PartType.HORIZONTAL_LINE]' = PartType.HORIZONTAL_LINE class ErrorPart(NamedTuple): - type: 't.Literal[PartType.ERROR]' message: str + type: 't.Literal[PartType.ERROR]' = PartType.ERROR AnyPart = t.Union[ @@ -151,6 +151,10 @@ class ErrorPart(NamedTuple): class Walker(abc.ABC): + ''' + Abstract base class for walker whose methods will be called for parts of a paragraph. + ''' + @abc.abstractmethod def process_error(self, part: ErrorPart) -> None: pass @@ -213,6 +217,11 @@ def process_return_value(self, part: ReturnValuePart) -> None: class NoopWalker(Walker): + ''' + Concrete base class for walker whose methods will be called for parts of a paragraph. + The default implementation for every part will not do anything. + ''' + def process_error(self, part: ErrorPart) -> None: pass @@ -259,7 +268,11 @@ def process_return_value(self, part: ReturnValuePart) -> None: pass -def walk(paragraph: Paragraph, walker: Walker) -> None: # noqa: C901, pylint:disable=too-many-branches +# pylint:disable-next=too-many-branches +def walk(paragraph: Paragraph, walker: Walker) -> None: # noqa: C901 + ''' + Call the corresponding methods of a walker object for every part of the paragraph. + ''' for part in paragraph: if part.type == PartType.ERROR: walker.process_error(t.cast(ErrorPart, part)) diff --git a/src/antsibull_docs/markup/format.py b/src/antsibull_docs/markup/format.py index f276ab6c..19483a52 100644 --- a/src/antsibull_docs/markup/format.py +++ b/src/antsibull_docs/markup/format.py @@ -15,6 +15,10 @@ class LinkProvider(abc.ABC): + ''' + Provide URLs for objects, if available. + ''' + def plugin_link(self, # pylint:disable=no-self-use plugin: dom.PluginIdentifier, # pylint:disable=unused-argument ) -> t.Optional[str]: @@ -36,6 +40,10 @@ class _DefaultLinkProvider(LinkProvider): class Formatter(abc.ABC): + ''' + Abstract base class for a formatter whose functions will be called for parts of a paragraph. + ''' + @abc.abstractmethod def format_error(self, part: dom.ErrorPart) -> str: pass @@ -98,6 +106,10 @@ def format_return_value(self, part: dom.ReturnValuePart, url: t.Optional[str]) - class _FormatWalker(dom.Walker): + ''' + Walker which calls a formatter's functions and stores the result in a list. + ''' + destination: t.List[str] formatter: Formatter link_provider: LinkProvider @@ -175,6 +187,13 @@ def format_paragraphs(paragraphs: t.Sequence[dom.Paragraph], par_end: str = '', par_sep: str = '', current_plugin: t.Optional[dom.PluginIdentifier] = None) -> str: + ''' + Apply the formatter to all parts of the given paragraphs, concatenate the results, + and insert start and end sequences for paragraphs and sequences between paragraphs. + + ``link_provider`` and ``current_plugin`` will be used to compute optional URLs + that will be passed to the formatter. + ''' if link_provider is None: link_provider = _DefaultLinkProvider() result: t.List[str] = [] diff --git a/src/antsibull_docs/markup/html.py b/src/antsibull_docs/markup/html.py new file mode 100644 index 00000000..bbf07976 --- /dev/null +++ b/src/antsibull_docs/markup/html.py @@ -0,0 +1,141 @@ +# Author: Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2023, Ansible Project +""" +HTML serialization. +""" + +import typing as t + +from html import escape as _html_escape +from urllib.parse import quote + +from . import dom +from .format import Formatter, LinkProvider, format_paragraphs as _format_paragraphs + + +def html_escape(text: str) -> str: + return _html_escape(text).replace('"', '"') + + +def _url_escape(url: str) -> str: + # We include '<>[]{}' in safe to allow urls such as 'https://:[PORT]/v{version}/' to + # remain unmangled by percent encoding + return quote(url, safe=':/#?%<>[]{}') + + +class HTMLFormatter(Formatter): + @staticmethod + def _format_option_like(part: t.Union[dom.OptionNamePart, dom.ReturnValuePart], + url: t.Optional[str]) -> str: + link_start = '' + link_end = '' + if url: + link_start = ( + f'' + '' + ) + link_end = '' + strong_start = '' + strong_end = '' + if part.type == dom.PartType.OPTION_NAME: + if part.value is None: + cls = 'ansible-option' + strong_start = '' + strong_end = '' + else: + cls = 'ansible-option-value' + else: + cls = 'ansible-return-value' + if part.value is None: + text = part.name + else: + text = f'{part.name}={part.value}' + return ( + f'{strong_start}{link_start}' + f'{html_escape(text)}{link_end}{strong_end}' + ) + + def format_error(self, part: dom.ErrorPart) -> str: + return f'ERROR while parsing: {html_escape(part.message)}' + + def format_bold(self, part: dom.BoldPart) -> str: + return f'{html_escape(part.text)}' + + def format_code(self, part: dom.CodePart) -> str: + return f"{html_escape(part.text)}" + + def format_horizontal_line(self, part: dom.HorizontalLinePart) -> str: + return '
' + + def format_italic(self, part: dom.ItalicPart) -> str: + return f'{html_escape(part.text)}' + + def format_link(self, part: dom.LinkPart) -> str: + return f"{html_escape(part.text)}" + + def format_module(self, part: dom.ModulePart, url: t.Optional[str]) -> str: + if not url: + return f"{html_escape(part.fqcn)}" + return ( + f"" + f'{html_escape(part.fqcn)}' + ) + + def format_rst_ref(self, part: dom.RSTRefPart) -> str: + return f"{html_escape(part.text)}" + + def format_url(self, part: dom.URLPart) -> str: + return ( + f"" + f'{html_escape(_url_escape(part.url))}' + ) + + def format_text(self, part: dom.TextPart) -> str: + return html_escape(part.text) + + def format_env_variable(self, part: dom.EnvVariablePart) -> str: + return ( + '' + f'{html_escape(part.name)}' + ) + + def format_option_name(self, part: dom.OptionNamePart, url: t.Optional[str]) -> str: + return self._format_option_like(part, url) + + def format_option_value(self, part: dom.OptionValuePart) -> str: + return f'{html_escape(part.value)}' + + def format_plugin(self, part: dom.PluginPart, url: t.Optional[str]) -> str: + if not url: + return f"{html_escape(part.plugin.fqcn)}" + return ( + f"" + f'{html_escape(part.plugin.fqcn)}' + ) + + def format_return_value(self, part: dom.ReturnValuePart, url: t.Optional[str]) -> str: + return self._format_option_like(part, url) + + +DEFAULT_FORMATTER = HTMLFormatter() + + +def to_html(paragraphs: t.Sequence[dom.Paragraph], + formatter: Formatter = DEFAULT_FORMATTER, + link_provider: t.Optional[LinkProvider] = None, + par_start: str = '

', + par_end: str = '

', + par_sep: str = '', + current_plugin: t.Optional[dom.PluginIdentifier] = None) -> str: + return _format_paragraphs( + paragraphs, + formatter=formatter, + link_provider=link_provider, + par_start=par_start, + par_end=par_end, + par_sep=par_sep, + current_plugin=current_plugin, + ) diff --git a/src/antsibull_docs/markup/md.py b/src/antsibull_docs/markup/md.py new file mode 100644 index 00000000..6aa5f49c --- /dev/null +++ b/src/antsibull_docs/markup/md.py @@ -0,0 +1,113 @@ +# Author: Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2023, Ansible Project +""" +MarkDown serialization. +""" + +import re +import typing as t + +from . import dom +from .format import Formatter, LinkProvider, format_paragraphs as _format_paragraphs +from .html import _url_escape, html_escape as _html_escape + + +_MD_ESCAPE = re.compile(r'''([!"#$%&'()*+,:;<=>?@[\\\]^_`{|}~-])''') + + +def md_escape(text: str) -> str: + return _MD_ESCAPE.sub(r'\\\1', text) + + +class MDFormatter(Formatter): + @staticmethod + def _format_option_like(part: t.Union[dom.OptionNamePart, dom.ReturnValuePart], + url: t.Optional[str]) -> str: + link_start = '' + link_end = '' + if url: + link_start = f'' + link_end = '' + strong_start = '' + strong_end = '' + if part.type == dom.PartType.OPTION_NAME and part.value is None: + strong_start = '' + strong_end = '' + if part.value is None: + text = part.name + else: + text = f'{part.name}={part.value}' + return f'{strong_start}{link_start}{_html_escape(text)}{link_end}{strong_end}' + + def format_error(self, part: dom.ErrorPart) -> str: + return f'ERROR while parsing: {md_escape(part.message)}' + + def format_bold(self, part: dom.BoldPart) -> str: + return f'{md_escape(part.text)}' + + def format_code(self, part: dom.CodePart) -> str: + return f'{md_escape(part.text)}' + + def format_horizontal_line(self, part: dom.HorizontalLinePart) -> str: + return '
' + + def format_italic(self, part: dom.ItalicPart) -> str: + return f'{md_escape(part.text)}' + + def format_link(self, part: dom.LinkPart) -> str: + return f'[{md_escape(part.text)}]({md_escape(_url_escape(part.url))})' + + def format_module(self, part: dom.ModulePart, url: t.Optional[str]) -> str: + if url: + return f'[{md_escape(part.fqcn)}]({md_escape(_url_escape(url))})' + return md_escape(part.fqcn) + + def format_rst_ref(self, part: dom.RSTRefPart) -> str: + return md_escape(part.text) + + def format_url(self, part: dom.URLPart) -> str: + return f'[{md_escape(_url_escape(part.url))}]({md_escape(_url_escape(part.url))})' + + def format_text(self, part: dom.TextPart) -> str: + return md_escape(part.text) + + def format_env_variable(self, part: dom.EnvVariablePart) -> str: + return f'{md_escape(part.name)}' + + def format_option_name(self, part: dom.OptionNamePart, url: t.Optional[str]) -> str: + return self._format_option_like(part, url) + + def format_option_value(self, part: dom.OptionValuePart) -> str: + return f'{md_escape(part.value)}' + + def format_plugin(self, part: dom.PluginPart, url: t.Optional[str]) -> str: + if url: + return f'[{md_escape(part.plugin.fqcn)}]({md_escape(_url_escape(url))})' + return md_escape(part.plugin.fqcn) + + def format_return_value(self, part: dom.ReturnValuePart, url: t.Optional[str]) -> str: + return self._format_option_like(part, url) + + +DEFAULT_FORMATTER = MDFormatter() + + +def to_md(paragraphs: t.Sequence[dom.Paragraph], + formatter: Formatter = DEFAULT_FORMATTER, + link_provider: t.Optional[LinkProvider] = None, + par_start: str = '', + par_end: str = '', + par_sep: str = '\n\n', + current_plugin: t.Optional[dom.PluginIdentifier] = None) -> str: + return _format_paragraphs( + paragraphs, + formatter=formatter, + link_provider=link_provider, + par_start=par_start, + par_end=par_end, + par_sep=par_sep, + current_plugin=current_plugin, + ) diff --git a/src/antsibull_docs/markup/parser.py b/src/antsibull_docs/markup/parser.py index fc48005c..e8ef7355 100644 --- a/src/antsibull_docs/markup/parser.py +++ b/src/antsibull_docs/markup/parser.py @@ -139,7 +139,7 @@ def convert_text(text: str, commands: CommandSet, process_other_text: t.Callable _IGNORE_MARKER = 'ignore:' _ARRAY_STUB_RE = re.compile(r'\[([^\]]*)\]') -_FQCN_TYPE_PREFIX_RE = re.compile(r'^([^.]+\.[^.]+\.[^#]+)#([a-z]+):(.*)$') +_FQCN_TYPE_PREFIX_RE = re.compile(r'^([^.]+\.[^.]+\.[^#]+)#([^:]+):(.*)$') _FQCN = re.compile(r'^[a-z0-9_]+\.[a-z0-9_]+(?:\.[a-z0-9_]+)+$') _PLUGIN_TYPE = re.compile(r'^[a-z_]+$') @@ -156,7 +156,7 @@ def _is_plugin_type(text: str) -> bool: class Context(t.NamedTuple): - current_plugin: t.Optional[dom.PluginIdentifier] + current_plugin: t.Optional[dom.PluginIdentifier] = None class CommandParser(abc.ABC): @@ -191,7 +191,7 @@ def __init__(self): super().__init__('I', 1, old_markup=True) def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: - return dom.ItalicPart(dom.PartType.ITALIC, text=parameters[0]) + return dom.ItalicPart(text=parameters[0]) class _Bold(CommandParserEx): @@ -199,7 +199,7 @@ def __init__(self): super().__init__('B', 1, old_markup=True) def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: - return dom.BoldPart(dom.PartType.BOLD, text=parameters[0]) + return dom.BoldPart(text=parameters[0]) class _Module(CommandParserEx): @@ -210,7 +210,7 @@ def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: fqcn = parameters[0] if not _is_fqcn(fqcn): raise ValueError(f'Module name "{fqcn}" is not a FQCN') - return dom.ModulePart(dom.PartType.MODULE, fqcn=fqcn) + return dom.ModulePart(fqcn=fqcn) class _URL(CommandParserEx): @@ -218,7 +218,7 @@ def __init__(self): super().__init__('U', 1, old_markup=True) def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: - return dom.URLPart(dom.PartType.URL, url=parameters[0]) + return dom.URLPart(url=parameters[0]) class _Link(CommandParserEx): @@ -228,7 +228,7 @@ def __init__(self): def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: text = parameters[0] url = parameters[1] - return dom.LinkPart(dom.PartType.LINK, text=text, url=url) + return dom.LinkPart(text=text, url=url) class _RSTRef(CommandParserEx): @@ -238,7 +238,7 @@ def __init__(self): def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: text = parameters[0] ref = parameters[1] - return dom.RSTRefPart(dom.PartType.RST_REF, text=text, ref=ref) + return dom.RSTRefPart(text=text, ref=ref) class _Code(CommandParserEx): @@ -246,7 +246,7 @@ def __init__(self): super().__init__('C', 1, old_markup=True) def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: - return dom.CodePart(dom.PartType.CODE, text=parameters[0]) + return dom.CodePart(text=parameters[0]) class _HorizontalLine(CommandParserEx): @@ -254,7 +254,7 @@ def __init__(self): super().__init__('HORIZONTALLINE', 0, old_markup=True) def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: - return dom.HorizontalLinePart(dom.PartType.HORIZONTAL_LINE) + return dom.HorizontalLinePart() # Semantic Ansible docs markup: @@ -304,9 +304,8 @@ def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: if not _is_fqcn(fqcn): raise ValueError(f'Plugin name "{fqcn}" is not a FQCN') if not _is_plugin_type(ptype): - raise ValueError(f'Plugin type "{type}" is not valid') - return dom.PluginPart( - dom.PartType.PLUGIN, plugin=dom.PluginIdentifier(fqcn=fqcn, type=ptype)) + raise ValueError(f'Plugin type "{ptype}" is not valid') + return dom.PluginPart(plugin=dom.PluginIdentifier(fqcn=fqcn, type=ptype)) class _EnvVar(CommandParserEx): @@ -314,7 +313,7 @@ def __init__(self): super().__init__('E', 1, escaped_arguments=True) def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: - return dom.EnvVariablePart(dom.PartType.ENV_VARIABLE, name=parameters[0]) + return dom.EnvVariablePart(name=parameters[0]) class _OptionValue(CommandParserEx): @@ -322,7 +321,7 @@ def __init__(self): super().__init__('V', 1, escaped_arguments=True) def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: - return dom.OptionValuePart(dom.PartType.OPTION_VALUE, value=parameters[0]) + return dom.OptionValuePart(value=parameters[0]) class _OptionName(CommandParserEx): @@ -331,8 +330,7 @@ def __init__(self): def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: plugin, link, name, value = _parse_option_like(parameters[0], context) - return dom.OptionNamePart( - dom.PartType.OPTION_NAME, plugin=plugin, link=link, name=name, value=value) + return dom.OptionNamePart(plugin=plugin, link=link, name=name, value=value) class _ReturnValue(CommandParserEx): @@ -341,8 +339,7 @@ def __init__(self): def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: plugin, link, name, value = _parse_option_like(parameters[0], context) - return dom.ReturnValuePart( - dom.PartType.RETURN_VALUE, plugin=plugin, link=link, name=name, value=value) + return dom.ReturnValuePart(plugin=plugin, link=link, name=name, value=value) _COMMANDS = [ @@ -401,7 +398,7 @@ def _parse_command(result: dom.Paragraph, text: str, cmd: CommandParser, index: f' at index {index + 1}{where}: {error}' ) if errors == 'message': - result.append(dom.ErrorPart(dom.PartType.ERROR, message=error)) + result.append(dom.ErrorPart(message=error)) elif errors == 'exception': raise ValueError(error) return end_index @@ -414,10 +411,10 @@ def parse_string(self, text: str, context: Context, while index < length: m = self._re.search(text, index) if m is None: - result.append(dom.TextPart(dom.PartType.TEXT, text=text[index:])) + result.append(dom.TextPart(text=text[index:])) break if m.start(1) > index: - result.append(dom.TextPart(dom.PartType.TEXT, text=text[index:m.start(1)])) + result.append(dom.TextPart(text=text[index:m.start(1)])) index = self._parse_command( result, text, @@ -443,7 +440,7 @@ def parse(text: t.Union[str, t.Sequence[str]], has_paragraphs = True if isinstance(text, str): has_paragraphs = False - text = [text] + text = [text] if text else [] parser = _CLASSIC if only_classic_markup else _SEMANTIC_MARKUP return [ parser.parse_string( diff --git a/src/antsibull_docs/markup/rst.py b/src/antsibull_docs/markup/rst.py new file mode 100644 index 00000000..3df02248 --- /dev/null +++ b/src/antsibull_docs/markup/rst.py @@ -0,0 +1,130 @@ +# Author: Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2023, Ansible Project +""" +ReStructured Text serialization. +""" + +import typing as t + +from . import dom +from .format import Formatter, LinkProvider, format_paragraphs as _format_paragraphs +from .html import _url_escape + + +def rst_escape(value: t.Any, escape_ending_whitespace=False) -> str: + ''' make sure value is converted to a string, and RST special characters are escaped ''' + + if not isinstance(value, str): + value = str(value) + + value = value.replace('\\', '\\\\') + value = value.replace('<', '\\<') + value = value.replace('>', '\\>') + value = value.replace('_', '\\_') + value = value.replace('*', '\\*') + value = value.replace('`', '\\`') + + if escape_ending_whitespace and value.endswith(' '): + value = value + '\\ ' + if escape_ending_whitespace and value.startswith(' '): + value = '\\ ' + value + + return value + + +def rst_code(value: str) -> str: + ''' Write value as :code:`...` RST construct. ''' + if not isinstance(value, str): + value = str(value) + return f':code:`{rst_escape(value, escape_ending_whitespace=True)}`' + + +class RSTFormatter(Formatter): + @staticmethod + def _format_option_like(part: t.Union[dom.OptionNamePart, dom.ReturnValuePart], + role: str) -> str: + result: t.List[str] = [] + plugin = part.plugin + if plugin: + result.append(plugin.fqcn) + result.append('#') + result.append(plugin.type) + result.append(':') + result.append(part.name) + value = part.value + if value is not None: + result.append('=') + result.append(value) + return f'\\ :{role}:`{rst_escape("".join(result), True)}`\\ ' + + def format_error(self, part: dom.ErrorPart) -> str: + return f'\\ :strong:`ERROR while parsing`\\ : {rst_escape(part.message, True)}\\ ' + + def format_bold(self, part: dom.BoldPart) -> str: + return f'\\ :strong:`{rst_escape(part.text, True)}`\\ ' + + def format_code(self, part: dom.CodePart) -> str: + return f'\\ :literal:`{rst_escape(part.text, True)}`\\ ' + + def format_horizontal_line(self, part: dom.HorizontalLinePart) -> str: + return '\n\n.. raw:: html\n\n
\n\n' + + def format_italic(self, part: dom.ItalicPart) -> str: + return f'\\ :emphasis:`{rst_escape(part.text, True)}`\\ ' + + def format_link(self, part: dom.LinkPart) -> str: + return f'\\ `{rst_escape(part.text)} <{_url_escape(part.url)}>`__\\ ' + + def format_module(self, part: dom.ModulePart, url: t.Optional[str]) -> str: + return f'\\ :ref:`{rst_escape(part.fqcn)} `\\ ' + + def format_rst_ref(self, part: dom.RSTRefPart) -> str: + return f'\\ :ref:`{rst_escape(part.text)} <{part.ref}>`\\ ' + + def format_url(self, part: dom.URLPart) -> str: + return f'\\ {_url_escape(part.url)}\\ ' + + def format_text(self, part: dom.TextPart) -> str: + return rst_escape(part.text) + + def format_env_variable(self, part: dom.EnvVariablePart) -> str: + return f'\\ :envvar:`{rst_escape(part.name, True)}`\\ ' + + def format_option_name(self, part: dom.OptionNamePart, url: t.Optional[str]) -> str: + return self._format_option_like(part, 'ansopt') + + def format_option_value(self, part: dom.OptionValuePart) -> str: + return f'\\ :ansval:`{rst_escape(part.value, True)}`\\ ' + + def format_plugin(self, part: dom.PluginPart, url: t.Optional[str]) -> str: + return ( + f'\\ :ref:`{rst_escape(part.plugin.fqcn)} ' + f'`\\ ' + ) + + def format_return_value(self, part: dom.ReturnValuePart, url: t.Optional[str]) -> str: + return self._format_option_like(part, 'ansretval') + + +DEFAULT_FORMATTER = RSTFormatter() + + +def to_rst(paragraphs: t.Sequence[dom.Paragraph], + formatter: Formatter = DEFAULT_FORMATTER, + link_provider: t.Optional[LinkProvider] = None, + par_start: str = '', + par_end: str = '', + par_sep: str = '\n\n', + current_plugin: t.Optional[dom.PluginIdentifier] = None) -> str: + return _format_paragraphs( + paragraphs, + formatter=formatter, + link_provider=link_provider, + par_start=par_start, + par_end=par_end, + par_sep=par_sep, + current_plugin=current_plugin, + ) diff --git a/tests/units/markup/test_dom.py b/tests/units/markup/test_dom.py new file mode 100644 index 00000000..ec2258b5 --- /dev/null +++ b/tests/units/markup/test_dom.py @@ -0,0 +1,121 @@ +# Author: Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2023, Ansible Project + +import typing as t + +import pytest + +from antsibull_docs.markup import dom + + +class _TestWalker(dom.Walker): + result: t.List[dom.AnyPart] + + def __init__(self): + self.result = [] + + def process_error(self, part: dom.ErrorPart) -> None: + assert part.type == dom.PartType.ERROR + self.result.append(part) + + def process_bold(self, part: dom.BoldPart) -> None: + assert part.type == dom.PartType.BOLD + self.result.append(part) + + def process_code(self, part: dom.CodePart) -> None: + assert part.type == dom.PartType.CODE + self.result.append(part) + + def process_horizontal_line(self, part: dom.HorizontalLinePart) -> None: + assert part.type == dom.PartType.HORIZONTAL_LINE + self.result.append(part) + + def process_italic(self, part: dom.ItalicPart) -> None: + assert part.type == dom.PartType.ITALIC + self.result.append(part) + + def process_link(self, part: dom.LinkPart) -> None: + assert part.type == dom.PartType.LINK + self.result.append(part) + + def process_module(self, part: dom.ModulePart) -> None: + assert part.type == dom.PartType.MODULE + self.result.append(part) + + def process_rst_ref(self, part: dom.RSTRefPart) -> None: + assert part.type == dom.PartType.RST_REF + self.result.append(part) + + def process_url(self, part: dom.URLPart) -> None: + assert part.type == dom.PartType.URL + self.result.append(part) + + def process_text(self, part: dom.TextPart) -> None: + assert part.type == dom.PartType.TEXT + self.result.append(part) + + def process_env_variable(self, part: dom.EnvVariablePart) -> None: + assert part.type == dom.PartType.ENV_VARIABLE + self.result.append(part) + + def process_option_name(self, part: dom.OptionNamePart) -> None: + assert part.type == dom.PartType.OPTION_NAME + self.result.append(part) + + def process_option_value(self, part: dom.OptionValuePart) -> None: + assert part.type == dom.PartType.OPTION_VALUE + self.result.append(part) + + def process_plugin(self, part: dom.PluginPart) -> None: + assert part.type == dom.PartType.PLUGIN + self.result.append(part) + + def process_return_value(self, part: dom.ReturnValuePart) -> None: + assert part.type == dom.PartType.RETURN_VALUE + self.result.append(part) + + +TEST_WALKER = [ + [], + [dom.ErrorPart(message='foo')], + [ + dom.TextPart(text='foo '), + dom.ItalicPart(text='bar'), + dom.TextPart(text=' baz '), + dom.CodePart(text=' bam '), + dom.TextPart(text=' '), + dom.BoldPart(text=' ( boo '), + dom.TextPart(text=' ) '), + dom.URLPart(url='https://example.com/?foo=bar'), + dom.HorizontalLinePart(), + dom.TextPart(text=' '), + dom.LinkPart(text='foo', url='https://bar.com'), + dom.TextPart(text=' '), + dom.RSTRefPart(text=' a', ref='b '), + dom.ModulePart(fqcn='foo.bar.baz'), + dom.TextPart(text='HORIZONTALLINEx '), + dom.ModulePart(fqcn='foo.bar.baz.bam'), + ], + [ + dom.TextPart(text='foo '), + dom.EnvVariablePart(name='a),b'), + dom.TextPart(text=' '), + dom.PluginPart(plugin=dom.PluginIdentifier(fqcn='foo.bar.baz', type='bam')), + dom.TextPart(text=' baz '), + dom.OptionValuePart(value=' b,na)\\m, '), + dom.TextPart(text=' '), + dom.OptionNamePart(plugin=None, link=['foo'], name='foo', value=None), + dom.TextPart(text=' '), + ], +] + + +@pytest.mark.parametrize('data', TEST_WALKER) +def test_walk(data: dom.Paragraph) -> None: + walker = _TestWalker() + dom.walk(data, walker) + assert walker.result == data + diff --git a/tests/units/markup/test_format.py b/tests/units/markup/test_format.py new file mode 100644 index 00000000..d2db7e33 --- /dev/null +++ b/tests/units/markup/test_format.py @@ -0,0 +1,65 @@ +# Author: Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2023, Ansible Project + +import typing as t + +from antsibull_docs.markup import dom +from antsibull_docs.markup.format import Formatter, format_paragraphs + + +class _TestFormatter(Formatter): + def format_error(self, part: dom.ErrorPart) -> str: + return 'format_error' + + def format_bold(self, part: dom.BoldPart) -> str: + return 'format_bold' + + def format_code(self, part: dom.CodePart) -> str: + return 'format_code' + + def format_horizontal_line(self, part: dom.HorizontalLinePart) -> str: + return 'format_horizontal_line' + + def format_italic(self, part: dom.ItalicPart) -> str: + return 'format_italic' + + def format_link(self, part: dom.LinkPart) -> str: + return 'format_link' + + def format_module(self, part: dom.ModulePart, url: t.Optional[str]) -> str: + return 'format_module' + + def format_rst_ref(self, part: dom.RSTRefPart) -> str: + return 'format_rst_ref' + + def format_url(self, part: dom.URLPart) -> str: + return 'format_url' + + def format_text(self, part: dom.TextPart) -> str: + return 'format_text' + + def format_env_variable(self, part: dom.EnvVariablePart) -> str: + return 'format_env_variable' + + def format_option_name(self, part: dom.OptionNamePart, url: t.Optional[str]) -> str: + return 'format_option_name' + + def format_option_value(self, part: dom.OptionValuePart) -> str: + return 'format_option_value' + + def format_plugin(self, part: dom.PluginPart, url: t.Optional[str]) -> str: + return 'format_plugin' + + def format_return_value(self, part: dom.ReturnValuePart, url: t.Optional[str]) -> str: + return 'format_return_value' + + +def test_format_paragraphs(): + assert format_paragraphs([], formatter=_TestFormatter()) == '' + assert format_paragraphs( + [[dom.HorizontalLinePart(), dom.TextPart(text='foo')]], + formatter=_TestFormatter() + ) == 'format_horizontal_lineformat_text' diff --git a/tests/units/markup/test_html.py b/tests/units/markup/test_html.py new file mode 100644 index 00000000..5f9a427b --- /dev/null +++ b/tests/units/markup/test_html.py @@ -0,0 +1,19 @@ +# Author: Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2023, Ansible Project + +from antsibull_docs.markup import dom +from antsibull_docs.markup.html import html_escape, to_html + + +def test_html_escape(): + assert html_escape('') == '' + assert html_escape(' foo ') == ' foo ' + assert html_escape('<&>') == '<a href="a&b">&lt;&amp;&gt;</a>' + +def test_to_html(): + assert to_html([]) == '' + assert to_html([[dom.TextPart(text='test')]]) == '

test

' + assert to_html([[dom.TextPart(text='test')]], par_start='
', par_end='
') == '
test
' diff --git a/tests/units/markup/test_md.py b/tests/units/markup/test_md.py new file mode 100644 index 00000000..e023cbc3 --- /dev/null +++ b/tests/units/markup/test_md.py @@ -0,0 +1,18 @@ +# Author: Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2023, Ansible Project + +from antsibull_docs.markup import dom +from antsibull_docs.markup.md import md_escape, to_md + + +def test_md_escape(): + assert md_escape('') == '' + assert md_escape(' foo ') == ' foo ' + assert md_escape(r'[]!.()-\@<>?[]!.()-\@<>?') == r'\[\]\!.\(\)\-\\\@\<\>\?\[\]\!.\(\)\-\\\@\<\>\?' + +def test_to_rst(): + assert to_md([]) == '' + assert to_md([[dom.TextPart(text='test')]]) == 'test' diff --git a/tests/units/markup/test_parser.py b/tests/units/markup/test_parser.py index 43ba51cc..f1faada3 100644 --- a/tests/units/markup/test_parser.py +++ b/tests/units/markup/test_parser.py @@ -17,7 +17,7 @@ class TestCommand0(Command): def handle(self, parameters: t.List[str], context: t.Any) -> str: assert len(parameters) == 0 - return f'EMPTY' + return 'EMPTY' class TestCommand1A(Command): @@ -135,3 +135,349 @@ def test_convert_text_fail(text, expected_exc): with pytest.raises(ParsingException) as exc: convert_text(text, commands, _escape_regular, None) assert str(exc.value) == expected_exc + + +############################################################################################### + +# import pytest + +from antsibull_docs.markup import dom +from antsibull_docs.markup.parser import parse, CommandParser, Context, Parser + + +TEST_PARSE_DATA: t.List[t.Tuple[t.Union[str, t.List[str]], Context, t.Dict[str, t.Any], t.List[dom.Paragraph]]] = [ + ('', Context(), {}, []), + ([''], Context(), {}, [[]]), + ('test', Context(), {}, [[dom.TextPart(text='test')]]), + ('test', Context(), {}, [[dom.TextPart(text='test')]]), + # classic markup: + ( + 'foo I(bar) baz C( bam ) B( ( boo ) ) U(https://example.com/?foo=bar)HORIZONTALLINE L(foo , https://bar.com) R( a , b )M(foo.bar.baz)HORIZONTALLINEx M(foo.bar.baz.bam)', + Context(), + {}, + [ + [ + dom.TextPart(text='foo '), + dom.ItalicPart(text='bar'), + dom.TextPart(text=' baz '), + dom.CodePart(text=' bam '), + dom.TextPart(text=' '), + dom.BoldPart(text=' ( boo '), + dom.TextPart(text=' ) '), + dom.URLPart(url='https://example.com/?foo=bar'), + dom.HorizontalLinePart(), + dom.TextPart(text=' '), + dom.LinkPart(text='foo', url='https://bar.com'), + dom.TextPart(text=' '), + dom.RSTRefPart(text=' a', ref='b '), + dom.ModulePart(fqcn='foo.bar.baz'), + dom.TextPart(text='HORIZONTALLINEx '), + dom.ModulePart(fqcn='foo.bar.baz.bam'), + ], + ], + ), + ( + 'foo I(bar) baz C( bam ) B( ( boo ) ) U(https://example.com/?foo=bar)HORIZONTALLINE L(foo , https://bar.com) R( a , b )M(foo.bar.baz)HORIZONTALLINEx M(foo.bar.baz.bam)', + Context(), + dict(only_classic_markup=True), + [ + [ + dom.TextPart(text='foo '), + dom.ItalicPart(text='bar'), + dom.TextPart(text=' baz '), + dom.CodePart(text=' bam '), + dom.TextPart(text=' '), + dom.BoldPart(text=' ( boo '), + dom.TextPart(text=' ) '), + dom.URLPart(url='https://example.com/?foo=bar'), + dom.HorizontalLinePart(), + dom.TextPart(text=' '), + dom.LinkPart(text='foo', url='https://bar.com'), + dom.TextPart(text=' '), + dom.RSTRefPart(text=' a', ref='b '), + dom.ModulePart(fqcn='foo.bar.baz'), + dom.TextPart(text='HORIZONTALLINEx '), + dom.ModulePart(fqcn='foo.bar.baz.bam'), + ], + ], + ), + # semantic markup: + ( + 'foo E(a\\),b) P(foo.bar.baz#bam) baz V( b\\,\\na\\)\\\\m\\, ) O(foo) ', + Context(), + {}, + [ + [ + dom.TextPart(text='foo '), + dom.EnvVariablePart(name='a),b'), + dom.TextPart(text=' '), + dom.PluginPart(plugin=dom.PluginIdentifier(fqcn='foo.bar.baz', type='bam')), + dom.TextPart(text=' baz '), + dom.OptionValuePart(value=' b,na)\\m, '), + dom.TextPart(text=' '), + dom.OptionNamePart(plugin=None, link=['foo'], name='foo', value=None), + dom.TextPart(text=' '), + ], + ], + ), + # semantic markup option name: + ('O(foo)', Context(), {}, [ + [ + dom.OptionNamePart(plugin=None, link=['foo'], name='foo', value=None), + ], + ]), + ('O(ignore:foo)', Context(current_plugin=dom.PluginIdentifier('foo.bar.baz', type='bam')), {}, [ + [ + dom.OptionNamePart(plugin=None, link=['foo'], name='foo', value=None), + ], + ]), + ('O(foo)', Context(current_plugin=dom.PluginIdentifier('foo.bar.baz', type='bam')), {}, [ + [ + dom.OptionNamePart(plugin=dom.PluginIdentifier('foo.bar.baz', type='bam'), link=['foo'], name='foo', value=None), + ], + ]), + ('O(foo.bar.baz#bam:foo)', Context(), {}, [ + [ + dom.OptionNamePart(plugin=dom.PluginIdentifier('foo.bar.baz', type='bam'), link=['foo'], name='foo', value=None), + ], + ]), + ('O(foo=bar)', Context(), {}, [ + [ + dom.OptionNamePart(plugin=None, link=['foo'], name='foo', value='bar'), + ], + ]), + ('O(foo.baz=bam)', Context(), {}, [ + [ + dom.OptionNamePart(plugin=None, link=['foo', 'baz'], name='foo.baz', value='bam'), + ], + ]), + ('O(foo[1].baz[bam.bar.boing].boo)', Context(), {}, [ + [ + dom.OptionNamePart(plugin=None, link=['foo', 'baz', 'boo'], name='foo[1].baz[bam.bar.boing].boo', value=None), + ], + ]), + ('O(bar.baz.bam.boo#lookup:foo[1].baz[bam.bar.boing].boo)', Context(), {}, [ + [ + dom.OptionNamePart(plugin=dom.PluginIdentifier('bar.baz.bam.boo', type='lookup'), link=['foo', 'baz', 'boo'], name='foo[1].baz[bam.bar.boing].boo', value=None), + ], + ]), + # semantic markup return value name: + ('RV(foo)', Context(), {}, [ + [ + dom.ReturnValuePart(plugin=None, link=['foo'], name='foo', value=None), + ], + ]), + ('RV(ignore:foo)', Context(current_plugin=dom.PluginIdentifier('foo.bar.baz', type='bam')), {}, [ + [ + dom.ReturnValuePart(plugin=None, link=['foo'], name='foo', value=None), + ], + ]), + ('RV(foo)', Context(current_plugin=dom.PluginIdentifier('foo.bar.baz', type='bam')), {}, [ + [ + dom.ReturnValuePart(plugin=dom.PluginIdentifier('foo.bar.baz', type='bam'), link=['foo'], name='foo', value=None), + ], + ]), + ('RV(foo.bar.baz#bam:foo)', Context(), {}, [ + [ + dom.ReturnValuePart(plugin=dom.PluginIdentifier('foo.bar.baz', type='bam'), link=['foo'], name='foo', value=None), + ], + ]), + ('RV(foo=bar)', Context(), {}, [ + [ + dom.ReturnValuePart(plugin=None, link=['foo'], name='foo', value='bar'), + ], + ]), + ('RV(foo.baz=bam)', Context(), {}, [ + [ + dom.ReturnValuePart(plugin=None, link=['foo', 'baz'], name='foo.baz', value='bam'), + ], + ]), + ('RV(foo[1].baz[bam.bar.boing].boo)', Context(), {}, [ + [ + dom.ReturnValuePart(plugin=None, link=['foo', 'baz', 'boo'], name='foo[1].baz[bam.bar.boing].boo', value=None), + ], + ]), + ('RV(bar.baz.bam.boo#lookup:foo[1].baz[bam.bar.boing].boo)', Context(), {}, [ + [ + dom.ReturnValuePart(plugin=dom.PluginIdentifier('bar.baz.bam.boo', type='lookup'), link=['foo', 'baz', 'boo'], name='foo[1].baz[bam.bar.boing].boo', value=None), + ], + ]), + # bad parameter parsing (no escaping, error message): + ('M(', Context(), {}, [ + [dom.ErrorPart(message='While parsing M() at index 1: Cannot find closing ")" after last parameter')], + ]), + ('M(foo', Context(), dict(errors='message'), [ + [dom.ErrorPart(message='While parsing M() at index 1: Cannot find closing ")" after last parameter')], + ]), + ('L(foo)', Context(), dict(errors='message'), [ + [ + dom.ErrorPart(message='While parsing L() at index 1: Cannot find comma separating parameter 1 from the next one'), + ], + ]), + ('L(foo,bar', Context(), dict(errors='message'), [ + [dom.ErrorPart(message='While parsing L() at index 1: Cannot find closing ")" after last parameter')], + ]), + ('L(foo), bar', Context(), dict(errors='message'), [ + [dom.ErrorPart(message='While parsing L() at index 1: Cannot find closing ")" after last parameter')], + ]), + ('P(', Context(), {}, [ + [dom.ErrorPart(message='While parsing P() at index 1: Cannot find closing ")" after last parameter')], + ]), + ('P(foo', Context(), dict(errors='message'), [ + [dom.ErrorPart(message='While parsing P() at index 1: Cannot find closing ")" after last parameter')], + ]), + # bad module ref (error message): + ('M(foo)', Context(), {}, [ + [dom.ErrorPart(message='While parsing M() at index 1: Module name "foo" is not a FQCN')], + ]), + (' M(foo.bar)', Context(), dict(errors='message'), [ + [ + dom.TextPart(text=' '), + dom.ErrorPart(message='While parsing M() at index 2: Module name "foo.bar" is not a FQCN'), + ], + ]), + (' M(foo. bar.baz)', Context(), dict(errors='message'), [ + [ + dom.TextPart(text=' '), + dom.ErrorPart(message='While parsing M() at index 3: Module name "foo. bar.baz" is not a FQCN'), + ], + ]), + (' M(foo) baz', Context(), dict(errors='message'), [ + [ + dom.TextPart(text=' '), + dom.ErrorPart(message='While parsing M() at index 4: Module name "foo" is not a FQCN'), + dom.TextPart(text=' baz'), + ], + ]), + # bad plugin ref (error message): + ('P(foo)', Context(), {}, [ + [ + dom.ErrorPart(message='While parsing P() at index 1: Parameter "foo" is not of the form FQCN#type'), + ], + ]), + ('P(f o.b r.b z#bar)', Context(), dict(errors='message'), [ + [ + dom.ErrorPart(message='While parsing P() at index 1: Plugin name "f o.b r.b z" is not a FQCN'), + ], + ]), + ('P(foo.bar.baz#b m)', Context(), dict(errors='message'), [ + [ + dom.ErrorPart(message='While parsing P() at index 1: Plugin type "b m" is not valid'), + ], + ]), + # bad option name/return value (error message): + ('O(f o.b r.b z#bam:foobar)', Context(), {}, [ + [ + dom.ErrorPart(message='While parsing O() at index 1: Plugin name "f o.b r.b z" is not a FQCN'), + ], + ]), + ('O(foo.bar.baz#b m:foobar)', Context(), dict(errors='message'), [ + [ + dom.ErrorPart(message='While parsing O() at index 1: Plugin type "b m" is not valid'), + ], + ]), + ('O(foo:bar:baz)', Context(), dict(errors='message'), [ + [ + dom.ErrorPart(message='While parsing O() at index 1: Invalid option/return value name "foo:bar:baz"'), + ], + ]), + # bad parameter parsing (no escaping, ignore error): + ('M(', Context(), dict(errors='ignore'), [[]]), + ('M(foo', Context(), dict(errors='ignore'), [[]]), + ('L(foo)', Context(), dict(errors='ignore'), [[]]), + ('L(foo,bar', Context(), dict(errors='ignore'), [[]]), + ('L(foo), bar', Context(), dict(errors='ignore'), [[]]), + ('P(', Context(), dict(errors='ignore'), [[]]), + ('P(foo', Context(), dict(errors='ignore'), [[]]), + # bad module ref (ignore error): + ('M(foo)', Context(), dict(errors='ignore'), [[]]), + (' M(foo.bar)', Context(), dict(errors='ignore'), [[dom.TextPart(text=' ')]]), + (' M(foo. bar.baz)', Context(), dict(errors='ignore'), [[dom.TextPart(text=' ')]]), + (' M(foo) baz', Context(), dict(errors='ignore'), [ + [ + dom.TextPart(text=' '), + dom.TextPart(text=' baz'), + ], + ]), + # bad plugin ref (ignore error): + ('P(foo#bar)', Context(), dict(errors='ignore'), [[]]), + ('P(f o.b r.b z#bar)', Context(), dict(errors='ignore'), [[]]), + ('P(foo.bar.baz#b m)', Context(), dict(errors='ignore'), [[]]), + # bad option name/return value (ignore error): + ('O(f o.b r.b z#bam:foobar)', Context(), dict(errors='ignore'), [[]]), + ('O(foo.bar.baz#b m:foobar)', Context(), dict(errors='ignore'), [[]]), + ('O(foo:bar:baz)', Context(), dict(errors='ignore'), [[]]), +] + + +@pytest.mark.parametrize('paragraphs, context, kwargs, expected', TEST_PARSE_DATA) +def test_parse(paragraphs: t.Union[str, t.List[str]], context: Context, kwargs: t.Dict[str, t.Any], expected: t.List[dom.Paragraph]) -> None: + result = parse(paragraphs, context, **kwargs) + print(result) + assert result == expected + + +TEST_PARSE_THROW_DATA: t.List[t.Tuple[t.Union[str, t.List[str]], Context, t.Dict[str, t.Any], str]] = [ + # bad parameter parsing (no escaping, throw error): + ('M(', Context(), dict(errors='exception'), + 'While parsing M() at index 1: Cannot find closing ")" after last parameter', + ), + ('M(foo', Context(), dict(errors='exception'), + 'While parsing M() at index 1: Cannot find closing ")" after last parameter', + ), + ('L(foo)', Context(), dict(errors='exception'), + 'While parsing L() at index 1: Cannot find comma separating parameter 1 from the next one', + ), + ('L(foo,bar', Context(), dict(errors='exception'), + 'While parsing L() at index 1: Cannot find closing ")" after last parameter', + ), + ('L(foo), bar', Context(), dict(errors='exception'), + 'While parsing L() at index 1: Cannot find closing ")" after last parameter', + ), + ('P(', Context(), dict(errors='exception'), + 'While parsing P() at index 1: Cannot find closing ")" after last parameter', + ), + ('P(foo', Context(), dict(errors='exception'), + 'While parsing P() at index 1: Cannot find closing ")" after last parameter', + ), + # bad module ref (throw error): + ('M(foo)', Context(), dict(errors='exception'), + 'While parsing M() at index 1: Module name "foo" is not a FQCN', + ), + (' M(foo.bar)', Context(), dict(errors='exception'), + 'While parsing M() at index 2: Module name "foo.bar" is not a FQCN', + ), + (' M(foo. bar.baz)', Context(), dict(errors='exception'), + 'While parsing M() at index 3: Module name "foo. bar.baz" is not a FQCN', + ), + (' M(foo)', Context(), dict(errors='exception'), + 'While parsing M() at index 4: Module name "foo" is not a FQCN', + ), + # bad plugin ref (throw error): + ('P(foo)', Context(), dict(errors='exception'), + 'While parsing P() at index 1: Parameter "foo" is not of the form FQCN#type', + ), + ('P(f o.b r.b z#bar)', Context(), dict(errors='exception'), + 'While parsing P() at index 1: Plugin name "f o.b r.b z" is not a FQCN', + ), + ('P(foo.bar.baz#b m)', Context(), dict(errors='exception'), + 'While parsing P() at index 1: Plugin type "b m" is not valid', + ), + # bad option name/return value (throw error): + ('O(f o.b r.b z#bam:foobar)', Context(), dict(errors='exception'), + 'While parsing O() at index 1: Plugin name "f o.b r.b z" is not a FQCN', + ), + ('O(foo.bar.baz#b m:foobar)', Context(), dict(errors='exception'), + 'While parsing O() at index 1: Plugin type "b m" is not valid', + ), + ('O(foo:bar:baz)', Context(), dict(errors='exception'), + 'While parsing O() at index 1: Invalid option/return value name "foo:bar:baz"', + ), +] + + +@pytest.mark.parametrize('paragraphs, context, kwargs, exc_message', TEST_PARSE_THROW_DATA) +def test_parse_bad(paragraphs: t.Union[str, t.List[str]], context: Context, kwargs: t.Dict[str, t.Any], exc_message: str) -> None: + with pytest.raises(ValueError) as exc: + parse(paragraphs, context, **kwargs) + assert str(exc.value) == exc_message diff --git a/tests/units/markup/test_rst.py b/tests/units/markup/test_rst.py new file mode 100644 index 00000000..39b6e6ea --- /dev/null +++ b/tests/units/markup/test_rst.py @@ -0,0 +1,19 @@ +# Author: Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2023, Ansible Project + +from antsibull_docs.markup import dom +from antsibull_docs.markup.rst import rst_escape, to_rst + + +def test_rst_escape(): + assert rst_escape('') == '' + assert rst_escape(' foo ') == ' foo ' + assert rst_escape(' foo ', True) == '\\ foo \\ ' + assert rst_escape('\\<_>`*<_>*`\\') == '\\\\\\<\\_\\>\\`\\*\\<\\_\\>\\*\\`\\\\' + +def test_to_rst(): + assert to_rst([]) == '' + assert to_rst([[dom.TextPart(text='test')]]) == 'test' From d38de03c4cfe516c7b129d22e828e288918143f7 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sat, 18 Mar 2023 16:30:28 +0100 Subject: [PATCH 09/15] Replace current markup code with new one. --- src/antsibull_docs/markup/htmlify.py | 320 ++---------------- src/antsibull_docs/markup/parser.py | 122 ------- src/antsibull_docs/markup/rst.py | 8 +- src/antsibull_docs/markup/rstify.py | 261 +++----------- .../collections/ns2/col/foo2_module.rst | 4 +- .../collections/ns2/col/foo_become.rst | 4 +- .../collections/ns2/col/foo_module.rst | 10 +- tests/units/markup/test_parser.py | 134 -------- 8 files changed, 93 insertions(+), 770 deletions(-) diff --git a/src/antsibull_docs/markup/htmlify.py b/src/antsibull_docs/markup/htmlify.py index 46fa67eb..a5074a3a 100644 --- a/src/antsibull_docs/markup/htmlify.py +++ b/src/antsibull_docs/markup/htmlify.py @@ -6,289 +6,32 @@ htmlify Jinja2 filter for use in Ansible documentation. """ -import re import typing as t -from html import escape as html_escape -from urllib.parse import quote - -from .semantic_helper import parse_option, parse_return_value -from .parser import Command, CommandSet, convert_text - -_MODULE = re.compile(r"^([^).]+)\.([^).]+)\.([^)]+)$") -_PLUGIN = re.compile(r"^([^).]+)\.([^).]+)\.([^)]+)#([a-z]+)$") - - -def _escape_url(url: str) -> str: - # We include '<>[]{}' in safe to allow urls such as 'https://:[PORT]/v{version}/' to - # remain unmangled by percent encoding - return quote(url, safe=':/#?%<>[]{}') - - -def _create_error(text: str, error: str) -> str: - text = f'{html_escape(text)}' - return f'ERROR while parsing {text}: {html_escape(error)}' - - -class _Context: - counts: t.Dict[str, int] - plugin_fqcn: t.Optional[str] - plugin_type: t.Optional[str] - - def __init__(self, - plugin_fqcn: t.Optional[str] = None, - plugin_type: t.Optional[str] = None): - self.counts = { - 'italic': 0, - 'bold': 0, - 'module': 0, - 'plugin': 0, - 'link': 0, - 'url': 0, - 'ref': 0, - 'const': 0, - 'option-name': 0, - 'option-value': 0, - 'environment-var': 0, - 'return-value': 0, - 'ruler': 0, - } - self.plugin_fqcn = plugin_fqcn - self.plugin_type = plugin_type - - -# In the following, we make heavy use of escaped whitespace ("\ ") being removed from the output. -# See -# https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#character-level-inline-markup-1 -# for further information. - - -class _Italic(Command): - command = 'I' - parameter_count = 1 - escaped_content = False - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['italic'] += 1 - return f"{html_escape(parameters[0])}" - - -class _Bold(Command): - command = 'B' - parameter_count = 1 - escaped_content = False - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['bold'] += 1 - return f"{html_escape(parameters[0])}" - - -class _Module(Command): - command = 'M' - parameter_count = 1 - escaped_content = False - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['module'] += 1 - m = _MODULE.match(parameters[0]) - if m is None: - return _create_error( - f'M({parameters[0]!r})', f'parameter {parameters[0]!r} is not a FQCN') - fqcn = f'{m.group(1)}.{m.group(2)}.{m.group(3)}' - url = html_escape(f'../../{m.group(1)}/{m.group(2)}/{m.group(3)}_module.html') - return f"{html_escape(fqcn)}" - - -class _Plugin(Command): - command = 'P' - parameter_count = 1 - escaped_content = False - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['plugin'] += 1 - m = _PLUGIN.match(parameters[0]) - if m is None: - return _create_error( - f'P({parameters[0]!r})', - f'parameter {parameters[0]!r} is not of the form FQCN#type') - fqcn = f'{m.group(1)}.{m.group(2)}.{m.group(3)}' - plugin_type = m.group(4) - url = html_escape(f'../../{m.group(1)}/{m.group(2)}/{m.group(3)}_{plugin_type}.html') - cssclass = f'plugin-{html_escape(plugin_type)}' - return f"{html_escape(fqcn)}" - - -class _URL(Command): - command = 'U' - parameter_count = 1 - escaped_content = False - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['url'] += 1 - url = parameters[0] - return f"{html_escape(_escape_url(url))}" - - -class _Link(Command): - command = 'L' - parameter_count = 2 - escaped_content = False - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['link'] += 1 - url = parameters[1] - return f"{html_escape(parameters[0])}" - - -class _Ref(Command): - command = 'R' - parameter_count = 2 - escaped_content = False - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['ref'] += 1 - return f"{html_escape(parameters[0])}" - - -class _Const(Command): - command = 'C' - parameter_count = 1 - escaped_content = False - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['const'] += 1 - # Escaping does not work in double backticks, so we use the :literal: role instead - return f"{html_escape(parameters[0])}" - - -class _OptionName(Command): - command = 'O' - parameter_count = 1 - escaped_content = True - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['option-name'] += 1 - if context.plugin_fqcn is None or context.plugin_type is None: - raise Exception('The markup O(...) cannot be used outside a plugin or role') - text = parameters[0] - try: - plugin_fqcn, plugin_type, option_link, option, value = parse_option( - text, context.plugin_fqcn, context.plugin_type, require_plugin=False) - except ValueError as exc: - return _create_error(f'O({text})', str(exc)) - if value is None: - cls = 'ansible-option' - text = f'{option}' - strong_start = '' - strong_end = '' - else: - cls = 'ansible-option-value' - text = f'{option}={value}' - strong_start = '' - strong_end = '' - if plugin_fqcn and plugin_type and plugin_fqcn.count('.') >= 2: - # TODO: handle role arguments (entrypoint!) - namespace, name, plugin = plugin_fqcn.split('.', 2) - url = f'../../{namespace}/{name}/{plugin}_{plugin_type}.html' - fragment = f'parameter-{quote(option_link.replace(".", "/"))}' - link_start = ( - f'' - '' - ) - link_end = '' - else: - link_start = '' - link_end = '' - return ( - f'' - f'{strong_start}{link_start}{html_escape(text)}{link_end}{strong_end}' - ) - - -class _OptionValue(Command): - command = 'V' - parameter_count = 1 - escaped_content = True - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['option-value'] += 1 - text = parameters[0] - return f'{html_escape(text)}' - - -class _EnvVariable(Command): - command = 'E' - parameter_count = 1 - escaped_content = True - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['environment-var'] += 1 - text = parameters[0] - return f'{html_escape(text)}' - - -class _RetValue(Command): - command = 'RV' - parameter_count = 1 - escaped_content = True - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['return-value'] += 1 - if context.plugin_fqcn is None or context.plugin_type is None: - raise Exception('The markup RV(...) cannot be used outside a plugin or role') - text = parameters[0] - try: - plugin_fqcn, plugin_type, rv_link, rv, value = parse_return_value( - text, context.plugin_fqcn, context.plugin_type, require_plugin=False) - except ValueError as exc: - return _create_error(f'RV({text})', str(exc)) - cls = 'ansible-return-value' - if value is None: - text = f'{rv}' - else: - text = f'{rv}={value}' - if plugin_fqcn and plugin_type and plugin_fqcn.count('.') >= 2: - namespace, name, plugin = plugin_fqcn.split('.', 2) - url = f'../../{namespace}/{name}/{plugin}_{plugin_type}.html' - fragment = f'return-{quote(rv_link.replace(".", "/"))}' - link_start = ( - f'' - '' - ) - link_end = '' - else: - link_start = '' - link_end = '' - return ( - f'{link_start}' - f'{html_escape(text)}{link_end}' - ) +from urllib.parse import quote -class _HorizontalLine(Command): - command = 'HORIZONTALLINE' - parameter_count = 0 - escaped_content = False +from . import dom +from .parser import parse, Context +from .html import to_html +from .format import LinkProvider +from .rstify import _count - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['ruler'] += 1 - return '
' +class _HTMLLinkProvider(LinkProvider): + def plugin_link(self, plugin: dom.PluginIdentifier) -> t.Optional[str]: + name = '/'.join(plugin.fqcn.split('.', 2)) + return f'../../{name}_{plugin.type}.html' -_COMMAND_SET = CommandSet([ - _Italic(), - _Bold(), - _Module(), - _Plugin(), - _URL(), - _Link(), - _Ref(), - _Const(), - _OptionName(), - _OptionValue(), - _EnvVariable(), - _RetValue(), - _HorizontalLine(), -]) + def plugin_option_like_link(self, # pylint:disable=no-self-use + plugin: dom.PluginIdentifier, # pylint:disable=unused-argument + # pylint:disable-next=unused-argument + what: "t.Union[t.Literal['option'], t.Literal['retval']]", + # pylint:disable-next=unused-argument + name: t.List[str], current_plugin: bool) -> t.Optional[str]: + base = '' if current_plugin else self.plugin_link(plugin) + w = 'parameter' if what == 'option' else 'return' + slug = quote('/'.join(name)) + return f'{base}#{w}-{slug}' def html_ify(text: str, @@ -296,14 +39,17 @@ def html_ify(text: str, plugin_fqcn: t.Optional[str] = None, plugin_type: t.Optional[str] = None) -> t.Tuple[str, t.Mapping[str, int]]: ''' convert symbols like I(this is in italics) to valid HTML ''' - our_context = _Context( - plugin_fqcn=plugin_fqcn, - plugin_type=plugin_type, + current_plugin: t.Optional[dom.PluginIdentifier] = None + if plugin_fqcn and plugin_type: + current_plugin = dom.PluginIdentifier(fqcn=plugin_fqcn, type=plugin_type) + paragraphs = parse(text, Context(current_plugin=current_plugin), errors='message') + link_provider = _HTMLLinkProvider() + text = to_html( + paragraphs, + link_provider=link_provider, + current_plugin=current_plugin, + par_start='', + par_end='', ) - - try: - text = convert_text(text, _COMMAND_SET, html_escape, our_context) - except Exception as exc: # pylint:disable=broad-except - text = _create_error(text, str(exc)) - - return text, our_context.counts + counts = _count(paragraphs) + return text, counts diff --git a/src/antsibull_docs/markup/parser.py b/src/antsibull_docs/markup/parser.py index e8ef7355..9af78774 100644 --- a/src/antsibull_docs/markup/parser.py +++ b/src/antsibull_docs/markup/parser.py @@ -13,128 +13,6 @@ from . import dom from ._parser_impl import parse_parameters_escaped, parse_parameters_unescaped - - -class ParsingException(Exception): - pass - - -class Command(abc.ABC): - command: str - parameter_count: int - escaped_content: bool - - @abc.abstractmethod - def handle(self, parameters: t.List[str], context: t.Any) -> str: - pass - - -class CommandSet: - _command_map: t.Mapping[str, Command] - _group_map: t.Mapping[str, Command] - _re: 're.Pattern' # on Python 3.6 the type is called differently - - def __init__(self, commands: t.List[Command]): - group_map = {} - command_map = {} - command_names = [] - for command in commands: - if command.command in command_map: - raise Exception( - f'Command "{command.command}" appears multiple times in parameter list') - command_map[command.command] = command - if command.parameter_count == 0: - command_names.append(fr'{re.escape(command.command)}\b') - group_map[command.command] = command - else: - command_names.append(fr'{re.escape(command.command)}\(') - group_map[f'{command.command}('] = command - self._command_map = command_map - self._group_map = group_map - if command_names: - command_re = '|'.join(command_names) - self._re = re.compile(fr'\b({command_re})') - else: - self._re = re.compile(r' ^') # never matches - - def find_next_command(self, text: str, start_index: int) -> t.Optional[t.Tuple[int, Command]]: - match = self._re.search(text, pos=start_index) - if not match: - return None - return match.start(1), self._group_map[match.group(1)] - - -class CommandData(t.NamedTuple): - command: Command - parameters: t.List[str] - - -Part = t.Union[str, CommandData] - - -def parse_text(text: str, commands: CommandSet) -> t.List[Part]: - result: t.List[Part] = [] - index = 0 - length = len(text) - while index < length: - next_command = commands.find_next_command(text, index) - if next_command is None: - result.append(text[index:]) - break - command_start, command = next_command - if command_start > index: - result.append(text[index:command_start]) - index = command_start + len(command.command) - if command.parameter_count == 0: - result.append(CommandData(command=command, parameters=[])) - continue - index += 1 - if command.escaped_content: - parameters, index, error = parse_parameters_escaped( - text, index, command.parameter_count) - else: - parameters, index, error = parse_parameters_unescaped( - text, index, command.parameter_count) - if error is not None: - raise ParsingException( - error + - f' for command "{command.command}" starting at index {command_start} in {text!r}' - ) - result.append(CommandData(command=command, parameters=parameters)) - return result - - -def convert_text(text: str, commands: CommandSet, process_other_text: t.Callable[[str], str], - context: t.Any,) -> str: - parts = parse_text(text, commands) - result = [] - for part in parts: - if isinstance(part, str): - result.append(process_other_text(part)) - else: - result.append(part.command.handle(part.parameters, context)) - return ''.join(result) - - - - - - - - - - - - - - - - - - - - - _IGNORE_MARKER = 'ignore:' diff --git a/src/antsibull_docs/markup/rst.py b/src/antsibull_docs/markup/rst.py index 3df02248..c69e3059 100644 --- a/src/antsibull_docs/markup/rst.py +++ b/src/antsibull_docs/markup/rst.py @@ -14,12 +14,8 @@ from .html import _url_escape -def rst_escape(value: t.Any, escape_ending_whitespace=False) -> str: - ''' make sure value is converted to a string, and RST special characters are escaped ''' - - if not isinstance(value, str): - value = str(value) - +def rst_escape(value: str, escape_ending_whitespace=False) -> str: + '''Escape RST specific constructs.''' value = value.replace('\\', '\\\\') value = value.replace('<', '\\<') value = value.replace('>', '\\>') diff --git a/src/antsibull_docs/markup/rstify.py b/src/antsibull_docs/markup/rstify.py index 6815a401..cc2645b2 100644 --- a/src/antsibull_docs/markup/rstify.py +++ b/src/antsibull_docs/markup/rstify.py @@ -6,65 +6,28 @@ rstify Jinja2 filter for use in Ansible documentation. """ -import re import typing as t -from urllib.parse import quote -from .semantic_helper import augment_plugin_name_type -from .parser import Command, CommandSet, convert_text - -_MODULE = re.compile(r"^([^).]+)\.([^).]+)\.([^)]+)$") -_PLUGIN = re.compile(r"^([^).]+)\.([^).]+)\.([^)]+)#([a-z]+)$") +from . import dom +from .parser import parse, Context +from .rst import rst_escape as _rst_escape, rst_code as _rst_code, to_rst def rst_escape(value: t.Any, escape_ending_whitespace=False) -> str: ''' make sure value is converted to a string, and RST special characters are escaped ''' - - if not isinstance(value, str): - value = str(value) - - value = value.replace('\\', '\\\\') - value = value.replace('<', '\\<') - value = value.replace('>', '\\>') - value = value.replace('_', '\\_') - value = value.replace('*', '\\*') - value = value.replace('`', '\\`') - - if escape_ending_whitespace and value.endswith(' '): - value = value + '\\ ' - if escape_ending_whitespace and value.startswith(' '): - value = '\\ ' + value - - return value - - -def rst_code(value: str) -> str: - ''' Write value as :code:`...` RST construct. ''' if not isinstance(value, str): value = str(value) - return f':code:`{rst_escape(value, escape_ending_whitespace=True)}`' - -def _escape_url(url: str) -> str: - # We include '<>[]{}' in safe to allow urls such as 'https://:[PORT]/v{version}/' to - # remain unmangled by percent encoding - return quote(url, safe=':/#?%<>[]{}') + return _rst_escape(value, escape_ending_whitespace=escape_ending_whitespace) -def _create_error(text: str, error: str) -> str: - text = f':literal:`{rst_escape(text, escape_ending_whitespace=True)}`' - error_msg = f':strong:`{rst_escape(error, escape_ending_whitespace=True)}`' - return f"\\ :strong:`ERROR while parsing` {text}\\ : {error_msg}\\ " +rst_code = _rst_code -class _Context: +class _Counter(dom.Walker): counts: t.Dict[str, int] - plugin_fqcn: t.Optional[str] - plugin_type: t.Optional[str] - def __init__(self, - plugin_fqcn: t.Optional[str] = None, - plugin_type: t.Optional[str] = None): + def __init__(self): self.counts = { 'italic': 0, 'bold': 0, @@ -80,180 +43,58 @@ def __init__(self, 'return-value': 0, 'ruler': 0, } - self.plugin_fqcn = plugin_fqcn - self.plugin_type = plugin_type - - -# In the following, we make heavy use of escaped whitespace ("\ ") being removed from the output. -# See -# https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#character-level-inline-markup-1 -# for further information. - - -class _Italic(Command): - command = 'I' - parameter_count = 1 - escaped_content = False - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['italic'] += 1 - return f"\\ :emphasis:`{rst_escape(parameters[0], escape_ending_whitespace=True)}`\\ " - - -class _Bold(Command): - command = 'B' - parameter_count = 1 - escaped_content = False - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['bold'] += 1 - return f"\\ :strong:`{rst_escape(parameters[0], escape_ending_whitespace=True)}`\\ " - -class _Module(Command): - command = 'M' - parameter_count = 1 - escaped_content = False + def process_error(self, part: dom.ErrorPart) -> None: + pass - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['module'] += 1 - m = _MODULE.match(parameters[0]) - if m is None: - return _create_error( - f'M({parameters[0]!r})', f'parameter {parameters[0]!r} is not a FQCN') - fqcn = f'{m.group(1)}.{m.group(2)}.{m.group(3)}' - return f"\\ :ref:`{rst_escape(fqcn)} `\\ " + def process_bold(self, part: dom.BoldPart) -> None: + self.counts['bold'] += 1 + def process_code(self, part: dom.CodePart) -> None: + self.counts['const'] += 1 -class _Plugin(Command): - command = 'P' - parameter_count = 1 - escaped_content = False + def process_horizontal_line(self, part: dom.HorizontalLinePart) -> None: + self.counts['ruler'] += 1 - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['plugin'] += 1 - m = _PLUGIN.match(parameters[0]) - if m is None: - return _create_error( - f'P({parameters[0]!r})', - f'parameter {parameters[0]!r} is not of the form FQCN#type') - fqcn = f'{m.group(1)}.{m.group(2)}.{m.group(3)}' - plugin_type = m.group(4) - return f"\\ :ref:`{rst_escape(fqcn)} `\\ " + def process_italic(self, part: dom.ItalicPart) -> None: + self.counts['italic'] += 1 + def process_link(self, part: dom.LinkPart) -> None: + self.counts['link'] += 1 -class _URL(Command): - command = 'U' - parameter_count = 1 - escaped_content = False + def process_module(self, part: dom.ModulePart) -> None: + self.counts['module'] += 1 - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['url'] += 1 - return f"\\ {_escape_url(parameters[0])}\\ " + def process_rst_ref(self, part: dom.RSTRefPart) -> None: + self.counts['ref'] += 1 + def process_url(self, part: dom.URLPart) -> None: + self.counts['url'] += 1 -class _Link(Command): - command = 'L' - parameter_count = 2 - escaped_content = False + def process_text(self, part: dom.TextPart) -> None: + pass - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['link'] += 1 - return f"\\ `{rst_escape(parameters[0])} <{_escape_url(parameters[1])}>`__\\ " + def process_env_variable(self, part: dom.EnvVariablePart) -> None: + self.counts['environment-var'] += 1 + def process_option_name(self, part: dom.OptionNamePart) -> None: + self.counts['option-name'] += 1 -class _Ref(Command): - command = 'R' - parameter_count = 2 - escaped_content = False + def process_option_value(self, part: dom.OptionValuePart) -> None: + self.counts['option-value'] += 1 - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['ref'] += 1 - return f"\\ :ref:`{rst_escape(parameters[0])} <{parameters[1]}>`\\ " + def process_plugin(self, part: dom.PluginPart) -> None: + self.counts['plugin'] += 1 + def process_return_value(self, part: dom.ReturnValuePart) -> None: + self.counts['return-value'] += 1 -class _Const(Command): - command = 'C' - parameter_count = 1 - escaped_content = False - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['const'] += 1 - # Escaping does not work in double backticks, so we use the :literal: role instead - return f"\\ :literal:`{rst_escape(parameters[0], escape_ending_whitespace=True)}`\\ " - - -class _OptionName(Command): - command = 'O' - parameter_count = 1 - escaped_content = True - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['option-name'] += 1 - if context.plugin_fqcn is None or context.plugin_type is None: - raise Exception('The markup O(...) cannot be used outside a plugin or role') - text = augment_plugin_name_type(parameters[0], context.plugin_fqcn, context.plugin_type) - return f"\\ :ansopt:`{rst_escape(text, escape_ending_whitespace=True)}`\\ " - - -class _OptionValue(Command): - command = 'V' - parameter_count = 1 - escaped_content = True - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['option-value'] += 1 - return f"\\ :ansval:`{rst_escape(parameters[0], escape_ending_whitespace=True)}`\\ " - - -class _EnvVariable(Command): - command = 'E' - parameter_count = 1 - escaped_content = True - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['environment-var'] += 1 - return f"\\ :envvar:`{rst_escape(parameters[0], escape_ending_whitespace=True)}`\\ " - - -class _RetValue(Command): - command = 'RV' - parameter_count = 1 - escaped_content = True - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['return-value'] += 1 - if context.plugin_fqcn is None or context.plugin_type is None: - raise Exception('The markup RV(...) cannot be used outside a plugin or role') - text = augment_plugin_name_type(parameters[0], context.plugin_fqcn, context.plugin_type) - return f"\\ :ansretval:`{rst_escape(text, escape_ending_whitespace=True)}`\\ " - - -class _HorizontalLine(Command): - command = 'HORIZONTALLINE' - parameter_count = 0 - escaped_content = False - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - context.counts['ruler'] += 1 - return '\n\n.. raw:: html\n\n
\n\n' - - -_COMMAND_SET = CommandSet([ - _Italic(), - _Bold(), - _Module(), - _Plugin(), - _URL(), - _Link(), - _Ref(), - _Const(), - _OptionName(), - _OptionValue(), - _EnvVariable(), - _RetValue(), - _HorizontalLine(), -]) +def _count(paragraphs: t.Sequence[dom.Paragraph]) -> t.Dict[str, int]: + counter = _Counter() + for paragraph in paragraphs: + dom.walk(paragraph, counter) + return counter.counts def rst_ify(text: str, @@ -261,14 +102,10 @@ def rst_ify(text: str, plugin_fqcn: t.Optional[str] = None, plugin_type: t.Optional[str] = None) -> t.Tuple[str, t.Mapping[str, int]]: ''' convert symbols like I(this is in italics) to valid restructured text ''' - our_context = _Context( - plugin_fqcn=plugin_fqcn, - plugin_type=plugin_type, - ) - - try: - text = convert_text(text, _COMMAND_SET, rst_escape, our_context) - except Exception as exc: # pylint:disable=broad-except - text = _create_error(text, str(exc)) - - return text, our_context.counts + current_plugin: t.Optional[dom.PluginIdentifier] = None + if plugin_fqcn and plugin_type: + current_plugin = dom.PluginIdentifier(fqcn=plugin_fqcn, type=plugin_type) + paragraphs = parse(text, Context(current_plugin=current_plugin), errors='message') + text = to_rst(paragraphs, current_plugin=current_plugin) + counts = _count(paragraphs) + return text, counts diff --git a/tests/functional/baseline-use-html-blobs/collections/ns2/col/foo2_module.rst b/tests/functional/baseline-use-html-blobs/collections/ns2/col/foo2_module.rst index 4cdf2b96..f8fbff78 100644 --- a/tests/functional/baseline-use-html-blobs/collections/ns2/col/foo2_module.rst +++ b/tests/functional/baseline-use-html-blobs/collections/ns2/col/foo2_module.rst @@ -323,8 +323,8 @@ Common return values are documented :ref:`here `, the foll

Some bar.

-

Referencing myself as bar.

-

Do not confuse with bar.

+

Referencing myself as bar.

+

Do not confuse with bar.

Returned: success

Sample: "baz"

diff --git a/tests/functional/baseline-use-html-blobs/collections/ns2/col/foo_become.rst b/tests/functional/baseline-use-html-blobs/collections/ns2/col/foo_become.rst index cd2df779..becf20af 100644 --- a/tests/functional/baseline-use-html-blobs/collections/ns2/col/foo_become.rst +++ b/tests/functional/baseline-use-html-blobs/collections/ns2/col/foo_become.rst @@ -109,8 +109,8 @@ Parameters

Bar. BAR!

-

Totally unrelated to become_user. Even with become_user=foo.

-

Might not be compatible when become_user is bar, though.

+

Totally unrelated to become_user. Even with become_user=foo.

+

Might not be compatible when become_user is bar, though.

diff --git a/tests/functional/baseline-use-html-blobs/collections/ns2/col/foo_module.rst b/tests/functional/baseline-use-html-blobs/collections/ns2/col/foo_module.rst index 7fae7e20..cabbfb21 100644 --- a/tests/functional/baseline-use-html-blobs/collections/ns2/col/foo_module.rst +++ b/tests/functional/baseline-use-html-blobs/collections/ns2/col/foo_module.rst @@ -120,8 +120,8 @@ Parameters

A bar.

-

Independent from foo.

-

Do not confuse with bar.

+

Independent from foo.

+

Do not confuse with bar.

@@ -165,7 +165,7 @@ Parameters

A sub foo.

Whatever.

-

Also required when subfoo is specified when foo=bar or baz.

+

Also required when subfoo is specified when foo=bar or baz.

@@ -428,8 +428,8 @@ Common return values are documented :ref:`here `, the foll

Some bar.

-

Referencing myself as bar.

-

Do not confuse with bar.

+

Referencing myself as bar.

+

Do not confuse with bar.

Returned: success

Sample: "baz"

diff --git a/tests/units/markup/test_parser.py b/tests/units/markup/test_parser.py index f1faada3..02182152 100644 --- a/tests/units/markup/test_parser.py +++ b/tests/units/markup/test_parser.py @@ -7,140 +7,6 @@ import pytest -from antsibull_docs.markup.parser import Command, CommandSet, ParsingException, convert_text - - -class TestCommand0(Command): - command = 'command0' - parameter_count = 0 - escaped_content = False - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - assert len(parameters) == 0 - return 'EMPTY' - - -class TestCommand1A(Command): - command = 'command1a' - parameter_count = 1 - escaped_content = False - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - assert len(parameters) == 1 - return f'FOOA{parameters!r}' - - -class TestCommand1B(Command): - command = 'command1b' - parameter_count = 1 - escaped_content = True - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - assert len(parameters) == 1 - return f'FOOB{parameters!r}' - - -class TestCommand2A(Command): - command = 'command2a' - parameter_count = 2 - escaped_content = False - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - assert len(parameters) == 2 - return f'FOOC{parameters!r}' - - -class TestCommand2B(Command): - command = 'command2b' - parameter_count = 2 - escaped_content = True - - def handle(self, parameters: t.List[str], context: t.Any) -> str: - assert len(parameters) == 2 - return f'FOOD{parameters!r}' - - -COMMAND_SET = CommandSet([ - TestCommand0(), - TestCommand1A(), - TestCommand1B(), - TestCommand2A(), - TestCommand2B(), -]) - - -def _escape_regular(text: str) -> str: - return f'TEXT:{text!r}' - - -CONVERT_TEXT_DATA = { - 'nothing': "TEXT:'nothing'", - 'command0': "EMPTY", - 'a command0 b': "TEXT:'a 'EMPTYTEXT:' b'", - 'acommand0b': "TEXT:'acommand0b'", - 'command1a(foo)': "FOOA['foo']", - 'command1a( foo )': "FOOA[' foo ']", - 'command1a(foo, bar)': "FOOA['foo, bar']", - 'command1a(())': "FOOA['(']TEXT:')'", - r'command1a(\(\))': r"FOOA['\\(\\']TEXT:')'", - 'a command1a()b': "TEXT:'a 'FOOA['']TEXT:'b'", - 'a command1a() b': "TEXT:'a 'FOOA['']TEXT:' b'", - 'acommand1a()b': "TEXT:'acommand1a()b'", - 'command1b(foo)': "FOOB['foo']", - 'command1b( foo )': "FOOB[' foo ']", - 'command1b(foo, bar)': "FOOB['foo, bar']", - 'command1b(())': "FOOB['(']TEXT:')'", - r'command1b(\(\))': "FOOB['()']", - 'a command1b()b': "TEXT:'a 'FOOB['']TEXT:'b'", - 'acommand1b()b': "TEXT:'acommand1b()b'", - 'command2a(foo, bar)': "FOOC['foo', 'bar']", - 'command2a(foo,bar)': "FOOC['foo', 'bar']", - 'command2a( foo , bar , baz )': "FOOC[' foo', 'bar , baz ']", - 'command2a((,))': "FOOC['(', '']TEXT:')'", - r'command2a(\(,\))': r"FOOC['\\(', '\\']TEXT:')'", - 'a command2a(,)b': "TEXT:'a 'FOOC['', '']TEXT:'b'", - 'acommand2a(,)b': "TEXT:'acommand2a(,)b'", - 'command2b(foo, bar)': "FOOD['foo', 'bar']", - 'command2b(foo,bar)': "FOOD['foo', 'bar']", - 'command2b( foo , bar , baz )': "FOOD[' foo', 'bar , baz ']", - r'command2b( foo\ , bar , baz )': "FOOD[' foo ', 'bar , baz ']", - 'command2b((,))': "FOOD['(', '']TEXT:')'", - r'command2b(\(,\))': "FOOD['(', ')']", - 'a command2b(,)b': "TEXT:'a 'FOOD['', '']TEXT:'b'", - 'acommand2b(,)b': "TEXT:'acommand2b(,)b'", -} - - -@pytest.mark.parametrize('text, expected', CONVERT_TEXT_DATA.items()) -def test_convert_text(text, expected): - commands = COMMAND_SET - result = convert_text(text, commands, _escape_regular, None) - assert result == expected - - -CONVERT_TEXT_FAIL_DATA = { - 'command1a(': """Cannot find closing ")" after last parameter for command "command1a" starting at index 0 in 'command1a('""", - 'command2a(,': """Cannot find closing ")" after last parameter for command "command2a" starting at index 0 in 'command2a(,'""", - 'command2a(': """Cannot find comma separating parameter 1 from the next one for command "command2a" starting at index 0 in 'command2a('""", - 'command1b(': """Cannot find closing ")" after last parameter for command "command1b" starting at index 0 in 'command1b('""", - 'command2b(,': """Cannot find closing ")" after last parameter for command "command2b" starting at index 0 in 'command2b(,'""", - 'command2b(': """Cannot find comma separating parameter 1 from the next one for command "command2b" starting at index 0 in 'command2b('""", - r'command2b(\,)': r"""Cannot find comma separating parameter 1 from the next one for command "command2b" starting at index 0 in 'command2b(\\,)'""", -} - - -@pytest.mark.parametrize('text, expected_exc', CONVERT_TEXT_FAIL_DATA.items()) -def test_convert_text_fail(text, expected_exc): - commands = COMMAND_SET - with pytest.raises(ParsingException) as exc: - convert_text(text, commands, _escape_regular, None) - assert str(exc.value) == expected_exc - - -############################################################################################### - -# import pytest - from antsibull_docs.markup import dom from antsibull_docs.markup.parser import parse, CommandParser, Context, Parser From 547ca45b7bbbd96dff7814be48d3fc040dc93cd5 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sat, 18 Mar 2023 17:42:31 +0100 Subject: [PATCH 10/15] Make more compatible to TS code, add same test vectors. --- src/antsibull_docs/markup/format.py | 4 + src/antsibull_docs/markup/html.py | 2 + src/antsibull_docs/markup/htmlify.py | 6 +- src/antsibull_docs/markup/md.py | 6 +- src/antsibull_docs/markup/rst.py | 2 + test-vectors.yaml | 700 +++++++++++++++++++++++++++ tests/units/markup/test_vectors.py | 111 +++++ 7 files changed, 825 insertions(+), 6 deletions(-) create mode 100644 test-vectors.yaml create mode 100644 tests/units/markup/test_vectors.py diff --git a/src/antsibull_docs/markup/format.py b/src/antsibull_docs/markup/format.py index 19483a52..91d88e4a 100644 --- a/src/antsibull_docs/markup/format.py +++ b/src/antsibull_docs/markup/format.py @@ -186,6 +186,7 @@ def format_paragraphs(paragraphs: t.Sequence[dom.Paragraph], par_start: str = '', par_end: str = '', par_sep: str = '', + par_empty: str = '', current_plugin: t.Optional[dom.PluginIdentifier] = None) -> str: ''' Apply the formatter to all parts of the given paragraphs, concatenate the results, @@ -202,6 +203,9 @@ def format_paragraphs(paragraphs: t.Sequence[dom.Paragraph], if result: result.append(par_sep) result.append(par_start) + before_len = len(result) dom.walk(paragraph, walker) + if before_len == len(result): + result.append(par_empty) result.append(par_end) return ''.join(result) diff --git a/src/antsibull_docs/markup/html.py b/src/antsibull_docs/markup/html.py index bbf07976..11abbd35 100644 --- a/src/antsibull_docs/markup/html.py +++ b/src/antsibull_docs/markup/html.py @@ -129,6 +129,7 @@ def to_html(paragraphs: t.Sequence[dom.Paragraph], par_start: str = '

', par_end: str = '

', par_sep: str = '', + par_empty: str = '', current_plugin: t.Optional[dom.PluginIdentifier] = None) -> str: return _format_paragraphs( paragraphs, @@ -137,5 +138,6 @@ def to_html(paragraphs: t.Sequence[dom.Paragraph], par_start=par_start, par_end=par_end, par_sep=par_sep, + par_empty=par_empty, current_plugin=current_plugin, ) diff --git a/src/antsibull_docs/markup/htmlify.py b/src/antsibull_docs/markup/htmlify.py index a5074a3a..809d1677 100644 --- a/src/antsibull_docs/markup/htmlify.py +++ b/src/antsibull_docs/markup/htmlify.py @@ -22,11 +22,9 @@ def plugin_link(self, plugin: dom.PluginIdentifier) -> t.Optional[str]: name = '/'.join(plugin.fqcn.split('.', 2)) return f'../../{name}_{plugin.type}.html' - def plugin_option_like_link(self, # pylint:disable=no-self-use - plugin: dom.PluginIdentifier, # pylint:disable=unused-argument - # pylint:disable-next=unused-argument + def plugin_option_like_link(self, + plugin: dom.PluginIdentifier, what: "t.Union[t.Literal['option'], t.Literal['retval']]", - # pylint:disable-next=unused-argument name: t.List[str], current_plugin: bool) -> t.Optional[str]: base = '' if current_plugin else self.plugin_link(plugin) w = 'parameter' if what == 'option' else 'return' diff --git a/src/antsibull_docs/markup/md.py b/src/antsibull_docs/markup/md.py index 6aa5f49c..8b0b6eb7 100644 --- a/src/antsibull_docs/markup/md.py +++ b/src/antsibull_docs/markup/md.py @@ -29,7 +29,7 @@ def _format_option_like(part: t.Union[dom.OptionNamePart, dom.ReturnValuePart], link_start = '' link_end = '' if url: - link_start = f'' + link_start = f'' link_end = '' strong_start = '' strong_end = '' @@ -40,7 +40,7 @@ def _format_option_like(part: t.Union[dom.OptionNamePart, dom.ReturnValuePart], text = part.name else: text = f'{part.name}={part.value}' - return f'{strong_start}{link_start}{_html_escape(text)}{link_end}{strong_end}' + return f'{strong_start}{link_start}{md_escape(text)}{link_end}{strong_end}' def format_error(self, part: dom.ErrorPart) -> str: return f'ERROR while parsing: {md_escape(part.message)}' @@ -101,6 +101,7 @@ def to_md(paragraphs: t.Sequence[dom.Paragraph], par_start: str = '', par_end: str = '', par_sep: str = '\n\n', + par_empty: str = ' ', current_plugin: t.Optional[dom.PluginIdentifier] = None) -> str: return _format_paragraphs( paragraphs, @@ -109,5 +110,6 @@ def to_md(paragraphs: t.Sequence[dom.Paragraph], par_start=par_start, par_end=par_end, par_sep=par_sep, + par_empty=par_empty, current_plugin=current_plugin, ) diff --git a/src/antsibull_docs/markup/rst.py b/src/antsibull_docs/markup/rst.py index c69e3059..363e72df 100644 --- a/src/antsibull_docs/markup/rst.py +++ b/src/antsibull_docs/markup/rst.py @@ -114,6 +114,7 @@ def to_rst(paragraphs: t.Sequence[dom.Paragraph], par_start: str = '', par_end: str = '', par_sep: str = '\n\n', + par_empty: str = r'\ ', current_plugin: t.Optional[dom.PluginIdentifier] = None) -> str: return _format_paragraphs( paragraphs, @@ -122,5 +123,6 @@ def to_rst(paragraphs: t.Sequence[dom.Paragraph], par_start=par_start, par_end=par_end, par_sep=par_sep, + par_empty=par_empty, current_plugin=current_plugin, ) diff --git a/test-vectors.yaml b/test-vectors.yaml new file mode 100644 index 00000000..81727c22 --- /dev/null +++ b/test-vectors.yaml @@ -0,0 +1,700 @@ +--- +# Simplified BSD License (see LICENSES/BSD-2-Clause.txt or https://opensource.org/licenses/BSD-2-Clause) +# SPDX-FileCopyrightText: Ansible Project +# SPDX-License-Identifier: BSD-2-Clause + +test_vectors: + empty: + source: [] + html: '' + html_plain: '' + rst: '' + md: '' + empty_content: + source: [''] + html:

+ html_plain:

+ md: ' ' + rst: '\ ' + simple: + source: |- + This is a C(test) I(module) B(markup). + html: |- +

This is a test module markup.

+ html_plain: |- +

This is a test module markup.

+ md: |- + This is a test module markup. + rst: |- + This is a \ :literal:`test`\ \ :emphasis:`module`\ \ :strong:`markup`\ . + module: + source: |- + The M(a.b.c) module. + html: |- +

The a.b.c module.

+ html_opts: + pluginLink.js: |- + (plugin_data) => `https://docs.ansible.com/ansible/devel/collections/${plugin_data.fqcn.replace(/\./g, '/')}_${plugin_data.type}.html` + pluginLink.py: |- + lambda plugin_data: f"https://docs.ansible.com/ansible/devel/collections/{plugin_data.fqcn.replace('.', '/')}_{plugin_data.type}.html" + html_plain: |- +

The a.b.c module.

+ md: |- + The [a.b.c](https\://docs.ansible.com/ansible/devel/collections/a/b/c\_module.html) module. + md_opts: + pluginLink.js: |- + (plugin_data) => `https://docs.ansible.com/ansible/devel/collections/${plugin_data.fqcn.replace(/\./g, '/')}_${plugin_data.type}.html` + pluginLink.py: |- + lambda plugin_data: f"https://docs.ansible.com/ansible/devel/collections/{plugin_data.fqcn.replace('.', '/')}_{plugin_data.type}.html" + rst: |- + The \ :ref:`a.b.c `\ module. + module_no_link: + source: |- + The M(a.b.c) module. + html: |- +

The a.b.c module.

+ html_plain: |- +

The a.b.c module.

+ md: |- + The a.b.c module. + rst: |- + The \ :ref:`a.b.c `\ module. + plugin: + source: |- + The P(a.b.c#lookup) lookup plugin. + html: |- +

The a.b.c lookup plugin.

+ html_opts: + pluginLink.js: |- + (plugin_data) => `https://docs.ansible.com/ansible/devel/collections/${plugin_data.fqcn.replace(/\./g, '/')}_${plugin_data.type}.html` + pluginLink.py: |- + lambda plugin_data: f"https://docs.ansible.com/ansible/devel/collections/{plugin_data.fqcn.replace('.', '/')}_{plugin_data.type}.html" + html_plain: |- +

The a.b.c lookup plugin.

+ md: |- + The [a.b.c](https\://docs.ansible.com/ansible/devel/collections/a/b/c\_lookup.html) lookup plugin. + md_opts: + pluginLink.js: |- + (plugin_data) => `https://docs.ansible.com/ansible/devel/collections/${plugin_data.fqcn.replace(/\./g, '/')}_${plugin_data.type}.html` + pluginLink.py: |- + lambda plugin_data: f"https://docs.ansible.com/ansible/devel/collections/{plugin_data.fqcn.replace('.', '/')}_{plugin_data.type}.html" + rst: |- + The \ :ref:`a.b.c `\ lookup plugin. + plugin_no_link: + source: |- + The P(a.b.c#lookup) lookup plugin. + html: |- +

The a.b.c lookup plugin.

+ html_plain: |- +

The a.b.c lookup plugin.

+ md: |- + The a.b.c lookup plugin. + rst: |- + The \ :ref:`a.b.c `\ lookup plugin. + link_and_url: + source: |- + An URL U(https://example.com) and L(a link, https://example.org). + html: |- +

An URL https://example.com and a link.

+ html_plain: |- +

An URL https://example.com and a link.

+ md: |- + An URL [https\://example.com](https\://example.com) and [a link](https\://example.org). + rst: |- + An URL \ https://example.com\ and \ `a link `__\ . + rst_ref: + source: |- + A R(RST reference, ansible_collections.community.general.ufw_module). + html: |- +

A RST reference.

+ html_plain: |- +

A RST reference.

+ md: |- + A RST reference. + rst: |- + A \ :ref:`RST reference `\ . + horizontal_line: + source: |- + foo HORIZONTALLINE bar + html: |- +

foo

bar

+ html_plain: |- +

foo

bar

+ md: |- + foo
bar + rst: |- + foo + + .. raw:: html + +
+ + bar + semantic_markup: + source: |- + foo E(FOOBAR) bar P(foo.bar.baz#bam) has value V( foo\),bar\\bam ). + html: |- +

foo FOOBAR bar foo.bar.baz has value foo),bar\bam .

+ html_plain: |- +

foo FOOBAR bar foo.bar.baz has value foo),bar\bam .

+ md: |- + foo FOOBAR bar foo.bar.baz has value foo\)\,bar\\bam . + rst: |- + foo \ :envvar:`FOOBAR`\ bar \ :ref:`foo.bar.baz `\ has value \ :ansval:`\ foo),bar\\bam \ `\ . + option_name_no_current_plugin: + source: + - |- + O(foo) O(bar.baz[123].bam[len(x\) - 1]) + - |- + O(foo=) O(bar.baz[123].bam[len(x\) - 1]=) + - |- + O(foo=bar) O(bar.baz[123].bam[len(x\) - 1]=bar) + - |- + O(bam.baz.foo#lookup:foo) O(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]) + - |- + O(bam.baz.foo#lookup:foo=) O(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]=) + - |- + O(bam.baz.foo#lookup:foo=bar) O(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]=bar) + - |- + O(ignore:foo) O(ignore:bar.baz[123].bam[len(x\) - 1]) + - |- + O(ignore:foo=) O(ignore:bar.baz[123].bam[len(x\) - 1]=) + - |- + O(ignore:foo=bar) O(ignore:bar.baz[123].bam[len(x\) - 1]=bar) + html: |- +

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

+ html_plain: |- +

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

+ md: |- + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + rst: |- + \ :ansopt:`foo`\ \ :ansopt:`bar.baz[123].bam[len(x) - 1]`\ + + \ :ansopt:`foo=`\ \ :ansopt:`bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansopt:`foo=bar`\ \ :ansopt:`bar.baz[123].bam[len(x) - 1]=bar`\ + + \ :ansopt:`bam.baz.foo#lookup:foo`\ \ :ansopt:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]`\ + + \ :ansopt:`bam.baz.foo#lookup:foo=`\ \ :ansopt:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansopt:`bam.baz.foo#lookup:foo=bar`\ \ :ansopt:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]=bar`\ + + \ :ansopt:`foo`\ \ :ansopt:`bar.baz[123].bam[len(x) - 1]`\ + + \ :ansopt:`foo=`\ \ :ansopt:`bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansopt:`foo=bar`\ \ :ansopt:`bar.baz[123].bam[len(x) - 1]=bar`\ + option_name_current_plugin: + source: + - |- + O(foo) O(bar.baz[123].bam[len(x\) - 1]) + - |- + O(foo=) O(bar.baz[123].bam[len(x\) - 1]=) + - |- + O(foo=bar) O(bar.baz[123].bam[len(x\) - 1]=bar) + - |- + O(bam.baz.foo#lookup:foo) O(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]) + - |- + O(bam.baz.foo#lookup:foo=) O(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]=) + - |- + O(bam.baz.foo#lookup:foo=bar) O(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]=bar) + - |- + O(ignore:foo) O(ignore:bar.baz[123].bam[len(x\) - 1]) + - |- + O(ignore:foo=) O(ignore:bar.baz[123].bam[len(x\) - 1]=) + - |- + O(ignore:foo=bar) O(ignore:bar.baz[123].bam[len(x\) - 1]=bar) + html: |- +

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

+ html_plain: |- +

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

+ md: |- + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + rst: |- + \ :ansopt:`foo.bar.baz.bam#boo:foo`\ \ :ansopt:`foo.bar.baz.bam#boo:bar.baz[123].bam[len(x) - 1]`\ + + \ :ansopt:`foo.bar.baz.bam#boo:foo=`\ \ :ansopt:`foo.bar.baz.bam#boo:bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansopt:`foo.bar.baz.bam#boo:foo=bar`\ \ :ansopt:`foo.bar.baz.bam#boo:bar.baz[123].bam[len(x) - 1]=bar`\ + + \ :ansopt:`bam.baz.foo#lookup:foo`\ \ :ansopt:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]`\ + + \ :ansopt:`bam.baz.foo#lookup:foo=`\ \ :ansopt:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansopt:`bam.baz.foo#lookup:foo=bar`\ \ :ansopt:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]=bar`\ + + \ :ansopt:`foo`\ \ :ansopt:`bar.baz[123].bam[len(x) - 1]`\ + + \ :ansopt:`foo=`\ \ :ansopt:`bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansopt:`foo=bar`\ \ :ansopt:`bar.baz[123].bam[len(x) - 1]=bar`\ + parse_opts: + current_plugin: + fqcn: foo.bar.baz.bam + type: boo + option_name_no_current_plugin_w_links: + source: + - |- + O(foo) O(bar.baz[123].bam[len(x\) - 1]) + - |- + O(foo=) O(bar.baz[123].bam[len(x\) - 1]=) + - |- + O(foo=bar) O(bar.baz[123].bam[len(x\) - 1]=bar) + - |- + O(bam.baz.foo#lookup:foo) O(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]) + - |- + O(bam.baz.foo#lookup:foo=) O(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]=) + - |- + O(bam.baz.foo#lookup:foo=bar) O(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]=bar) + - |- + O(ignore:foo) O(ignore:bar.baz[123].bam[len(x\) - 1]) + - |- + O(ignore:foo=) O(ignore:bar.baz[123].bam[len(x\) - 1]=) + - |- + O(ignore:foo=bar) O(ignore:bar.baz[123].bam[len(x\) - 1]=bar) + html: |- +

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

+ html_opts: + pluginOptionLikeLink.js: |- + (plugin, what, name, current_plugin) => `https://docs.ansible.com/ansible/devel/collections/${plugin.fqcn.replace(/\./g, '/')}_${plugin.type}.html#${what}-${name.join('/')}` + pluginOptionLikeLink.py: |- + lambda plugin, what, name, current_plugin: f"https://docs.ansible.com/ansible/devel/collections/{plugin.fqcn.replace('.', '/')}_{plugin.type}.html#{what}-{'/'.join(name)}" + html_plain: |- +

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

+ md: |- + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + md_opts: + pluginOptionLikeLink.js: |- + (plugin, what, name, current_plugin) => `https://docs.ansible.com/ansible/devel/collections/${plugin.fqcn.replace(/\./g, '/')}_${plugin.type}.html#${what}-${name.join('/')}` + pluginOptionLikeLink.py: |- + lambda plugin, what, name, current_plugin: f"https://docs.ansible.com/ansible/devel/collections/{plugin.fqcn.replace('.', '/')}_{plugin.type}.html#{what}-{'/'.join(name)}" + rst: |- + \ :ansopt:`foo`\ \ :ansopt:`bar.baz[123].bam[len(x) - 1]`\ + + \ :ansopt:`foo=`\ \ :ansopt:`bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansopt:`foo=bar`\ \ :ansopt:`bar.baz[123].bam[len(x) - 1]=bar`\ + + \ :ansopt:`bam.baz.foo#lookup:foo`\ \ :ansopt:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]`\ + + \ :ansopt:`bam.baz.foo#lookup:foo=`\ \ :ansopt:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansopt:`bam.baz.foo#lookup:foo=bar`\ \ :ansopt:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]=bar`\ + + \ :ansopt:`foo`\ \ :ansopt:`bar.baz[123].bam[len(x) - 1]`\ + + \ :ansopt:`foo=`\ \ :ansopt:`bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansopt:`foo=bar`\ \ :ansopt:`bar.baz[123].bam[len(x) - 1]=bar`\ + option_name_current_plugin_w_links: + source: + - |- + O(foo) O(bar.baz[123].bam[len(x\) - 1]) + - |- + O(foo=) O(bar.baz[123].bam[len(x\) - 1]=) + - |- + O(foo=bar) O(bar.baz[123].bam[len(x\) - 1]=bar) + - |- + O(bam.baz.foo#lookup:foo) O(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]) + - |- + O(bam.baz.foo#lookup:foo=) O(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]=) + - |- + O(bam.baz.foo#lookup:foo=bar) O(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]=bar) + - |- + O(ignore:foo) O(ignore:bar.baz[123].bam[len(x\) - 1]) + - |- + O(ignore:foo=) O(ignore:bar.baz[123].bam[len(x\) - 1]=) + - |- + O(ignore:foo=bar) O(ignore:bar.baz[123].bam[len(x\) - 1]=bar) + html: |- +

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

+ html_opts: + pluginOptionLikeLink.js: |- + (plugin, what, name, current_plugin) => `https://docs.ansible.com/ansible/devel/collections/${plugin.fqcn.replace(/\./g, '/')}_${plugin.type}.html#${what}-${name.join('/')}` + pluginOptionLikeLink.py: |- + lambda plugin, what, name, current_plugin: f"https://docs.ansible.com/ansible/devel/collections/{plugin.fqcn.replace('.', '/')}_{plugin.type}.html#{what}-{'/'.join(name)}" + html_plain: |- +

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

+ md: |- + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + md_opts: + pluginOptionLikeLink.js: |- + (plugin, what, name, current_plugin) => `https://docs.ansible.com/ansible/devel/collections/${plugin.fqcn.replace(/\./g, '/')}_${plugin.type}.html#${what}-${name.join('/')}` + pluginOptionLikeLink.py: |- + lambda plugin, what, name, current_plugin: f"https://docs.ansible.com/ansible/devel/collections/{plugin.fqcn.replace('.', '/')}_{plugin.type}.html#{what}-{'/'.join(name)}" + rst: |- + \ :ansopt:`foo.bar.baz.bam#boo:foo`\ \ :ansopt:`foo.bar.baz.bam#boo:bar.baz[123].bam[len(x) - 1]`\ + + \ :ansopt:`foo.bar.baz.bam#boo:foo=`\ \ :ansopt:`foo.bar.baz.bam#boo:bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansopt:`foo.bar.baz.bam#boo:foo=bar`\ \ :ansopt:`foo.bar.baz.bam#boo:bar.baz[123].bam[len(x) - 1]=bar`\ + + \ :ansopt:`bam.baz.foo#lookup:foo`\ \ :ansopt:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]`\ + + \ :ansopt:`bam.baz.foo#lookup:foo=`\ \ :ansopt:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansopt:`bam.baz.foo#lookup:foo=bar`\ \ :ansopt:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]=bar`\ + + \ :ansopt:`foo`\ \ :ansopt:`bar.baz[123].bam[len(x) - 1]`\ + + \ :ansopt:`foo=`\ \ :ansopt:`bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansopt:`foo=bar`\ \ :ansopt:`bar.baz[123].bam[len(x) - 1]=bar`\ + parse_opts: + current_plugin: + fqcn: foo.bar.baz.bam + type: boo + return_value_no_current_plugin: + source: + - |- + RV(foo) RV(bar.baz[123].bam[len(x\) - 1]) + - |- + RV(foo=) RV(bar.baz[123].bam[len(x\) - 1]=) + - |- + RV(foo=bar) RV(bar.baz[123].bam[len(x\) - 1]=bar) + - |- + RV(bam.baz.foo#lookup:foo) RV(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]) + - |- + RV(bam.baz.foo#lookup:foo=) RV(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]=) + - |- + RV(bam.baz.foo#lookup:foo=bar) RV(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]=bar) + - |- + RV(ignore:foo) RV(ignore:bar.baz[123].bam[len(x\) - 1]) + - |- + RV(ignore:foo=) RV(ignore:bar.baz[123].bam[len(x\) - 1]=) + - |- + RV(ignore:foo=bar) RV(ignore:bar.baz[123].bam[len(x\) - 1]=bar) + html: |- +

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

+ html_plain: |- +

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

+ md: |- + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + rst: |- + \ :ansretval:`foo`\ \ :ansretval:`bar.baz[123].bam[len(x) - 1]`\ + + \ :ansretval:`foo=`\ \ :ansretval:`bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansretval:`foo=bar`\ \ :ansretval:`bar.baz[123].bam[len(x) - 1]=bar`\ + + \ :ansretval:`bam.baz.foo#lookup:foo`\ \ :ansretval:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]`\ + + \ :ansretval:`bam.baz.foo#lookup:foo=`\ \ :ansretval:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansretval:`bam.baz.foo#lookup:foo=bar`\ \ :ansretval:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]=bar`\ + + \ :ansretval:`foo`\ \ :ansretval:`bar.baz[123].bam[len(x) - 1]`\ + + \ :ansretval:`foo=`\ \ :ansretval:`bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansretval:`foo=bar`\ \ :ansretval:`bar.baz[123].bam[len(x) - 1]=bar`\ + return_value_current_plugin: + source: + - |- + RV(foo) RV(bar.baz[123].bam[len(x\) - 1]) + - |- + RV(foo=) RV(bar.baz[123].bam[len(x\) - 1]=) + - |- + RV(foo=bar) RV(bar.baz[123].bam[len(x\) - 1]=bar) + - |- + RV(bam.baz.foo#lookup:foo) RV(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]) + - |- + RV(bam.baz.foo#lookup:foo=) RV(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]=) + - |- + RV(bam.baz.foo#lookup:foo=bar) RV(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]=bar) + - |- + RV(ignore:foo) RV(ignore:bar.baz[123].bam[len(x\) - 1]) + - |- + RV(ignore:foo=) RV(ignore:bar.baz[123].bam[len(x\) - 1]=) + - |- + RV(ignore:foo=bar) RV(ignore:bar.baz[123].bam[len(x\) - 1]=bar) + html: |- +

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

+ html_plain: |- +

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

+ md: |- + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + rst: |- + \ :ansretval:`foo.bar.baz.bam#boo:foo`\ \ :ansretval:`foo.bar.baz.bam#boo:bar.baz[123].bam[len(x) - 1]`\ + + \ :ansretval:`foo.bar.baz.bam#boo:foo=`\ \ :ansretval:`foo.bar.baz.bam#boo:bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansretval:`foo.bar.baz.bam#boo:foo=bar`\ \ :ansretval:`foo.bar.baz.bam#boo:bar.baz[123].bam[len(x) - 1]=bar`\ + + \ :ansretval:`bam.baz.foo#lookup:foo`\ \ :ansretval:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]`\ + + \ :ansretval:`bam.baz.foo#lookup:foo=`\ \ :ansretval:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansretval:`bam.baz.foo#lookup:foo=bar`\ \ :ansretval:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]=bar`\ + + \ :ansretval:`foo`\ \ :ansretval:`bar.baz[123].bam[len(x) - 1]`\ + + \ :ansretval:`foo=`\ \ :ansretval:`bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansretval:`foo=bar`\ \ :ansretval:`bar.baz[123].bam[len(x) - 1]=bar`\ + parse_opts: + current_plugin: + fqcn: foo.bar.baz.bam + type: boo + return_value_no_current_plugin_w_links: + source: + - |- + RV(foo) RV(bar.baz[123].bam[len(x\) - 1]) + - |- + RV(foo=) RV(bar.baz[123].bam[len(x\) - 1]=) + - |- + RV(foo=bar) RV(bar.baz[123].bam[len(x\) - 1]=bar) + - |- + RV(bam.baz.foo#lookup:foo) RV(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]) + - |- + RV(bam.baz.foo#lookup:foo=) RV(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]=) + - |- + RV(bam.baz.foo#lookup:foo=bar) RV(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]=bar) + - |- + RV(ignore:foo) RV(ignore:bar.baz[123].bam[len(x\) - 1]) + - |- + RV(ignore:foo=) RV(ignore:bar.baz[123].bam[len(x\) - 1]=) + - |- + RV(ignore:foo=bar) RV(ignore:bar.baz[123].bam[len(x\) - 1]=bar) + html: |- +

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

+ html_opts: + pluginOptionLikeLink.js: |- + (plugin, what, name, current_plugin) => `https://docs.ansible.com/ansible/devel/collections/${plugin.fqcn.replace(/\./g, '/')}_${plugin.type}.html#${what}-${name.join('/')}` + pluginOptionLikeLink.py: |- + lambda plugin, what, name, current_plugin: f"https://docs.ansible.com/ansible/devel/collections/{plugin.fqcn.replace('.', '/')}_{plugin.type}.html#{what}-{'/'.join(name)}" + html_plain: |- +

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

+ md: |- + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + md_opts: + pluginOptionLikeLink.js: |- + (plugin, what, name, current_plugin) => `https://docs.ansible.com/ansible/devel/collections/${plugin.fqcn.replace(/\./g, '/')}_${plugin.type}.html#${what}-${name.join('/')}` + pluginOptionLikeLink.py: |- + lambda plugin, what, name, current_plugin: f"https://docs.ansible.com/ansible/devel/collections/{plugin.fqcn.replace('.', '/')}_{plugin.type}.html#{what}-{'/'.join(name)}" + rst: |- + \ :ansretval:`foo`\ \ :ansretval:`bar.baz[123].bam[len(x) - 1]`\ + + \ :ansretval:`foo=`\ \ :ansretval:`bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansretval:`foo=bar`\ \ :ansretval:`bar.baz[123].bam[len(x) - 1]=bar`\ + + \ :ansretval:`bam.baz.foo#lookup:foo`\ \ :ansretval:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]`\ + + \ :ansretval:`bam.baz.foo#lookup:foo=`\ \ :ansretval:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansretval:`bam.baz.foo#lookup:foo=bar`\ \ :ansretval:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]=bar`\ + + \ :ansretval:`foo`\ \ :ansretval:`bar.baz[123].bam[len(x) - 1]`\ + + \ :ansretval:`foo=`\ \ :ansretval:`bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansretval:`foo=bar`\ \ :ansretval:`bar.baz[123].bam[len(x) - 1]=bar`\ + return_value_current_plugin_w_links: + source: + - |- + RV(foo) RV(bar.baz[123].bam[len(x\) - 1]) + - |- + RV(foo=) RV(bar.baz[123].bam[len(x\) - 1]=) + - |- + RV(foo=bar) RV(bar.baz[123].bam[len(x\) - 1]=bar) + - |- + RV(bam.baz.foo#lookup:foo) RV(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]) + - |- + RV(bam.baz.foo#lookup:foo=) RV(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]=) + - |- + RV(bam.baz.foo#lookup:foo=bar) RV(bam.baz.foo#lookup:bar.baz[123].bam[len(x\) - 1]=bar) + - |- + RV(ignore:foo) RV(ignore:bar.baz[123].bam[len(x\) - 1]) + - |- + RV(ignore:foo=) RV(ignore:bar.baz[123].bam[len(x\) - 1]=) + - |- + RV(ignore:foo=bar) RV(ignore:bar.baz[123].bam[len(x\) - 1]=bar) + html: |- +

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

+ html_opts: + pluginOptionLikeLink.js: |- + (plugin, what, name, current_plugin) => `https://docs.ansible.com/ansible/devel/collections/${plugin.fqcn.replace(/\./g, '/')}_${plugin.type}.html#${what}-${name.join('/')}` + pluginOptionLikeLink.py: |- + lambda plugin, what, name, current_plugin: f"https://docs.ansible.com/ansible/devel/collections/{plugin.fqcn.replace('.', '/')}_{plugin.type}.html#{what}-{'/'.join(name)}" + html_plain: |- +

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

foo bar.baz[123].bam[len(x) - 1]

foo= bar.baz[123].bam[len(x) - 1]=

foo=bar bar.baz[123].bam[len(x) - 1]=bar

+ md: |- + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + + foo bar.baz\[123\].bam\[len\(x\) \- 1\] + + foo\= bar.baz\[123\].bam\[len\(x\) \- 1\]\= + + foo\=bar bar.baz\[123\].bam\[len\(x\) \- 1\]\=bar + md_opts: + pluginOptionLikeLink.js: |- + (plugin, what, name, current_plugin) => `https://docs.ansible.com/ansible/devel/collections/${plugin.fqcn.replace(/\./g, '/')}_${plugin.type}.html#${what}-${name.join('/')}` + pluginOptionLikeLink.py: |- + lambda plugin, what, name, current_plugin: f"https://docs.ansible.com/ansible/devel/collections/{plugin.fqcn.replace('.', '/')}_{plugin.type}.html#{what}-{'/'.join(name)}" + rst: |- + \ :ansretval:`foo.bar.baz.bam#boo:foo`\ \ :ansretval:`foo.bar.baz.bam#boo:bar.baz[123].bam[len(x) - 1]`\ + + \ :ansretval:`foo.bar.baz.bam#boo:foo=`\ \ :ansretval:`foo.bar.baz.bam#boo:bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansretval:`foo.bar.baz.bam#boo:foo=bar`\ \ :ansretval:`foo.bar.baz.bam#boo:bar.baz[123].bam[len(x) - 1]=bar`\ + + \ :ansretval:`bam.baz.foo#lookup:foo`\ \ :ansretval:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]`\ + + \ :ansretval:`bam.baz.foo#lookup:foo=`\ \ :ansretval:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansretval:`bam.baz.foo#lookup:foo=bar`\ \ :ansretval:`bam.baz.foo#lookup:bar.baz[123].bam[len(x) - 1]=bar`\ + + \ :ansretval:`foo`\ \ :ansretval:`bar.baz[123].bam[len(x) - 1]`\ + + \ :ansretval:`foo=`\ \ :ansretval:`bar.baz[123].bam[len(x) - 1]=`\ + + \ :ansretval:`foo=bar`\ \ :ansretval:`bar.baz[123].bam[len(x) - 1]=bar`\ + parse_opts: + current_plugin: + fqcn: foo.bar.baz.bam + type: boo + errors: + source: + - P(foo) + - C(foo + - R(foo,bar + html: |- +

ERROR while parsing: While parsing P() at index 1 of paragraph 1: Parameter "foo" is not of the form FQCN#type

ERROR while parsing: While parsing C() at index 1 of paragraph 2: Cannot find closing ")" after last parameter

ERROR while parsing: While parsing R() at index 1 of paragraph 3: Cannot find closing ")" after last parameter

+ html_plain: |- +

ERROR while parsing: While parsing P() at index 1 of paragraph 1: Parameter "foo" is not of the form FQCN#type

ERROR while parsing: While parsing C() at index 1 of paragraph 2: Cannot find closing ")" after last parameter

ERROR while parsing: While parsing R() at index 1 of paragraph 3: Cannot find closing ")" after last parameter

+ md: |- + ERROR while parsing: While parsing P\(\) at index 1 of paragraph 1\: Parameter \"foo\" is not of the form FQCN\#type + + ERROR while parsing: While parsing C\(\) at index 1 of paragraph 2\: Cannot find closing \"\)\" after last parameter + + ERROR while parsing: While parsing R\(\) at index 1 of paragraph 3\: Cannot find closing \"\)\" after last parameter + rst: |- + \ :strong:`ERROR while parsing`\ : While parsing P() at index 1 of paragraph 1: Parameter "foo" is not of the form FQCN#type\ + + \ :strong:`ERROR while parsing`\ : While parsing C() at index 1 of paragraph 2: Cannot find closing ")" after last parameter\ + + \ :strong:`ERROR while parsing`\ : While parsing R() at index 1 of paragraph 3: Cannot find closing ")" after last parameter\ diff --git a/tests/units/markup/test_vectors.py b/tests/units/markup/test_vectors.py new file mode 100644 index 00000000..7e1e350e --- /dev/null +++ b/tests/units/markup/test_vectors.py @@ -0,0 +1,111 @@ +# Author: Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2022, Ansible Project + +import typing as t + +import pytest + +from antsibull_core.yaml import load_yaml_file + +from antsibull_docs.markup import dom +from antsibull_docs.markup.parser import parse, Context +from antsibull_docs.markup.html import to_html +from antsibull_docs.markup.md import to_md +from antsibull_docs.markup.rst import to_rst +from antsibull_docs.markup.format import LinkProvider + + +class _TestLinkProvider(LinkProvider): + _plugin_link = None + _plugin_option_like_link = None + + def plugin_link(self, plugin: dom.PluginIdentifier) -> t.Optional[str]: + if self._plugin_link is not None: + return self._plugin_link(plugin) + return None + + def plugin_option_like_link(self, + plugin: dom.PluginIdentifier, + what: "t.Union[t.Literal['option'], t.Literal['retval']]", + name: t.List[str], current_plugin: bool) -> t.Optional[str]: + if self._plugin_option_like_link is not None: + return self._plugin_option_like_link(plugin, what, name, current_plugin) + return None + + def _update(self, config: t.Mapping[str, t.Any]): + if 'pluginLink.py' in config: + self._plugin_link = eval(config['pluginLink.py']) + if 'pluginOptionLikeLink.py' in config: + self._plugin_option_like_link = eval(config['pluginOptionLikeLink.py']) + + +TEST_DATA = sorted(load_yaml_file('test-vectors.yaml')['test_vectors'].items()) + + +@pytest.mark.parametrize('test_name, test_data', TEST_DATA, ids=[test_name for test_name, test_data in TEST_DATA]) +def test_vectors(test_name: str, test_data: t.Mapping[str, t.Any]) -> None: + parse_opts = {} + context_opts = {} + if test_data.get('parse_opts'): + if 'current_plugin' in test_data['parse_opts']: + context_opts['current_plugin'] = dom.PluginIdentifier( + fqcn=test_data['parse_opts']['current_plugin']['fqcn'], + type=test_data['parse_opts']['current_plugin']['type'], + ) + if 'errors' in test_data['parse_opts']: + context_opts['errors'] = test_data['parse_opts']['errors'] + if 'onlyClassicMarkup' in test_data['parse_opts']: + context_opts['only_classic_markup'] = test_data['parse_opts']['onlyClassicMarkup'] + parsed = parse(test_data['source'], Context(**context_opts), **parse_opts) + + html_opts = {} + html_link_provider = _TestLinkProvider() + if test_data.get('html_opts'): + if 'parStart' in test_data['html_opts']: + html_opts['par_start'] = test_data['html_opts']['parStart'] + if 'parEnd' in test_data['html_opts']: + html_opts['par_end'] = test_data['html_opts']['parEnd'] + if 'current_plugin' in test_data['html_opts']: + rst_opts['current_plugin'] = dom.PluginIdentifier( + fqcn=test_data['html_opts']['current_plugin']['fqcn'], + type=test_data['html_opts']['current_plugin']['type'], + ) + html_link_provider._update(test_data['html_opts']) + + md_opts = {} + md_link_provider = _TestLinkProvider() + if test_data.get('md_opts'): + if 'current_plugin' in test_data['md_opts']: + rst_opts['current_plugin'] = dom.PluginIdentifier( + fqcn=test_data['md_opts']['current_plugin']['fqcn'], + type=test_data['md_opts']['current_plugin']['type'], + ) + md_link_provider._update(test_data['md_opts']) + + rst_opts = {} + if test_data.get('rst_opts'): + if 'current_plugin' in test_data['rst_opts']: + rst_opts['current_plugin'] = dom.PluginIdentifier( + fqcn=test_data['rst_opts']['current_plugin']['fqcn'], + type=test_data['rst_opts']['current_plugin']['type'], + ) + + if 'html' in test_data: + result = to_html(parsed, link_provider=html_link_provider, **html_opts) + assert result == test_data['html'] + + if 'html_plain' in test_data: + ... + # TODO result = to_html(parsed, link_provider=html_link_provider, **html_opts) + # TODO assert result == test_data['html_plain'] + + if 'md' in test_data: + print('XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX') + result = to_md(parsed, link_provider=md_link_provider, **md_opts) + assert result == test_data['md'] + + if 'rst' in test_data: + result = to_rst(parsed, **rst_opts) + assert result == test_data['rst'] From b4104d4cedd804350a4273bd5324df0b61599219 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sat, 18 Mar 2023 18:04:42 +0100 Subject: [PATCH 11/15] Cleanup. --- src/antsibull_docs/markup/_counter.py | 85 ++++++++++++++++++++ src/antsibull_docs/markup/htmlify.py | 2 +- src/antsibull_docs/markup/rst.py | 7 -- src/antsibull_docs/markup/rstify.py | 82 ++----------------- src/antsibull_docs/markup/semantic_helper.py | 52 ++++-------- 5 files changed, 110 insertions(+), 118 deletions(-) create mode 100644 src/antsibull_docs/markup/_counter.py diff --git a/src/antsibull_docs/markup/_counter.py b/src/antsibull_docs/markup/_counter.py new file mode 100644 index 00000000..a2414571 --- /dev/null +++ b/src/antsibull_docs/markup/_counter.py @@ -0,0 +1,85 @@ +# Author: Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2022, Ansible Project +""" +Count markup instructions in parsed markup. +""" + +import typing as t + +from . import dom + + +class _Counter(dom.Walker): + counts: t.Dict[str, int] + + def __init__(self): + self.counts = { + 'italic': 0, + 'bold': 0, + 'module': 0, + 'plugin': 0, + 'link': 0, + 'url': 0, + 'ref': 0, + 'const': 0, + 'option-name': 0, + 'option-value': 0, + 'environment-var': 0, + 'return-value': 0, + 'ruler': 0, + } + + def process_error(self, part: dom.ErrorPart) -> None: + pass + + def process_bold(self, part: dom.BoldPart) -> None: + self.counts['bold'] += 1 + + def process_code(self, part: dom.CodePart) -> None: + self.counts['const'] += 1 + + def process_horizontal_line(self, part: dom.HorizontalLinePart) -> None: + self.counts['ruler'] += 1 + + def process_italic(self, part: dom.ItalicPart) -> None: + self.counts['italic'] += 1 + + def process_link(self, part: dom.LinkPart) -> None: + self.counts['link'] += 1 + + def process_module(self, part: dom.ModulePart) -> None: + self.counts['module'] += 1 + + def process_rst_ref(self, part: dom.RSTRefPart) -> None: + self.counts['ref'] += 1 + + def process_url(self, part: dom.URLPart) -> None: + self.counts['url'] += 1 + + def process_text(self, part: dom.TextPart) -> None: + pass + + def process_env_variable(self, part: dom.EnvVariablePart) -> None: + self.counts['environment-var'] += 1 + + def process_option_name(self, part: dom.OptionNamePart) -> None: + self.counts['option-name'] += 1 + + def process_option_value(self, part: dom.OptionValuePart) -> None: + self.counts['option-value'] += 1 + + def process_plugin(self, part: dom.PluginPart) -> None: + self.counts['plugin'] += 1 + + def process_return_value(self, part: dom.ReturnValuePart) -> None: + self.counts['return-value'] += 1 + + +def count(paragraphs: t.Sequence[dom.Paragraph]) -> t.Dict[str, int]: + counter = _Counter() + for paragraph in paragraphs: + dom.walk(paragraph, counter) + return counter.counts diff --git a/src/antsibull_docs/markup/htmlify.py b/src/antsibull_docs/markup/htmlify.py index 809d1677..02c82baa 100644 --- a/src/antsibull_docs/markup/htmlify.py +++ b/src/antsibull_docs/markup/htmlify.py @@ -11,10 +11,10 @@ from urllib.parse import quote from . import dom +from ._counter import count as _count from .parser import parse, Context from .html import to_html from .format import LinkProvider -from .rstify import _count class _HTMLLinkProvider(LinkProvider): diff --git a/src/antsibull_docs/markup/rst.py b/src/antsibull_docs/markup/rst.py index 363e72df..0d177204 100644 --- a/src/antsibull_docs/markup/rst.py +++ b/src/antsibull_docs/markup/rst.py @@ -31,13 +31,6 @@ def rst_escape(value: str, escape_ending_whitespace=False) -> str: return value -def rst_code(value: str) -> str: - ''' Write value as :code:`...` RST construct. ''' - if not isinstance(value, str): - value = str(value) - return f':code:`{rst_escape(value, escape_ending_whitespace=True)}`' - - class RSTFormatter(Formatter): @staticmethod def _format_option_like(part: t.Union[dom.OptionNamePart, dom.ReturnValuePart], diff --git a/src/antsibull_docs/markup/rstify.py b/src/antsibull_docs/markup/rstify.py index cc2645b2..13f6761c 100644 --- a/src/antsibull_docs/markup/rstify.py +++ b/src/antsibull_docs/markup/rstify.py @@ -9,8 +9,9 @@ import typing as t from . import dom +from ._counter import count as _count from .parser import parse, Context -from .rst import rst_escape as _rst_escape, rst_code as _rst_code, to_rst +from .rst import rst_escape as _rst_escape, to_rst def rst_escape(value: t.Any, escape_ending_whitespace=False) -> str: @@ -21,80 +22,11 @@ def rst_escape(value: t.Any, escape_ending_whitespace=False) -> str: return _rst_escape(value, escape_ending_whitespace=escape_ending_whitespace) -rst_code = _rst_code - - -class _Counter(dom.Walker): - counts: t.Dict[str, int] - - def __init__(self): - self.counts = { - 'italic': 0, - 'bold': 0, - 'module': 0, - 'plugin': 0, - 'link': 0, - 'url': 0, - 'ref': 0, - 'const': 0, - 'option-name': 0, - 'option-value': 0, - 'environment-var': 0, - 'return-value': 0, - 'ruler': 0, - } - - def process_error(self, part: dom.ErrorPart) -> None: - pass - - def process_bold(self, part: dom.BoldPart) -> None: - self.counts['bold'] += 1 - - def process_code(self, part: dom.CodePart) -> None: - self.counts['const'] += 1 - - def process_horizontal_line(self, part: dom.HorizontalLinePart) -> None: - self.counts['ruler'] += 1 - - def process_italic(self, part: dom.ItalicPart) -> None: - self.counts['italic'] += 1 - - def process_link(self, part: dom.LinkPart) -> None: - self.counts['link'] += 1 - - def process_module(self, part: dom.ModulePart) -> None: - self.counts['module'] += 1 - - def process_rst_ref(self, part: dom.RSTRefPart) -> None: - self.counts['ref'] += 1 - - def process_url(self, part: dom.URLPart) -> None: - self.counts['url'] += 1 - - def process_text(self, part: dom.TextPart) -> None: - pass - - def process_env_variable(self, part: dom.EnvVariablePart) -> None: - self.counts['environment-var'] += 1 - - def process_option_name(self, part: dom.OptionNamePart) -> None: - self.counts['option-name'] += 1 - - def process_option_value(self, part: dom.OptionValuePart) -> None: - self.counts['option-value'] += 1 - - def process_plugin(self, part: dom.PluginPart) -> None: - self.counts['plugin'] += 1 - - def process_return_value(self, part: dom.ReturnValuePart) -> None: - self.counts['return-value'] += 1 - - -def _count(paragraphs: t.Sequence[dom.Paragraph]) -> t.Dict[str, int]: - counter = _Counter() - for paragraph in paragraphs: - dom.walk(paragraph, counter) - return counter.counts +def rst_code(value: str) -> str: + ''' Write value as :code:`...` RST construct. ''' + if not isinstance(value, str): + value = str(value) + return f':code:`{rst_escape(value, escape_ending_whitespace=True)}`' def rst_ify(text: str, diff --git a/src/antsibull_docs/markup/semantic_helper.py b/src/antsibull_docs/markup/semantic_helper.py index bdbe102f..08e9e803 100644 --- a/src/antsibull_docs/markup/semantic_helper.py +++ b/src/antsibull_docs/markup/semantic_helper.py @@ -10,6 +10,7 @@ import re import typing as t + _ARRAY_STUB_RE = re.compile(r'\[([^\]]*)\]') _FQCN_TYPE_PREFIX_RE = re.compile(r'^([^.]+\.[^.]+\.[^#]+)#([a-z]+):(.*)$') _IGNORE_MARKER = 'ignore:' @@ -19,9 +20,9 @@ def _remove_array_stubs(text: str) -> str: return _ARRAY_STUB_RE.sub('', text) -def parse_option(text: str, plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str], - require_plugin=False - ) -> t.Tuple[t.Optional[str], t.Optional[str], str, str, t.Optional[str]]: +def _parse(text: str, plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str], what: str, + require_plugin=False + ) -> t.Tuple[t.Optional[str], t.Optional[str], str, str, t.Optional[str]]: """ Given the contents of O(...) / :ansopt:`...` with potential escaping removed, split it into plugin FQCN, plugin type, option link name, option name, and option value. @@ -41,45 +42,26 @@ def parse_option(text: str, plugin_fqcn: t.Optional[str], plugin_type: t.Optiona plugin_type = '' text = text[len(_IGNORE_MARKER):] if ':' in text or '#' in text: - raise ValueError(f'Invalid option name "{text}"') + raise ValueError(f'Invalid {what} "{text}"') return plugin_fqcn, plugin_type, _remove_array_stubs(text), text, value -def parse_return_value(text: str, plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str], - require_plugin=False - ) -> t.Tuple[t.Optional[str], t.Optional[str], str, str, t.Optional[str]]: +def parse_option(text: str, plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str], + require_plugin=False + ) -> t.Tuple[t.Optional[str], t.Optional[str], str, str, t.Optional[str]]: """ - Given the contents of RV(...) / :ansretval:`...` with potential escaping removed, + Given the contents of O(...) / :ansopt:`...` with potential escaping removed, split it into plugin FQCN, plugin type, option link name, option name, and option value. """ - value = None - if '=' in text: - text, value = text.split('=', 1) - m = _FQCN_TYPE_PREFIX_RE.match(text) - if m: - plugin_fqcn = m.group(1) - plugin_type = m.group(2) - text = m.group(3) - elif require_plugin: - raise ValueError('Cannot extract plugin name and type') - elif text.startswith(_IGNORE_MARKER): - plugin_fqcn = '' - plugin_type = '' - text = text[len(_IGNORE_MARKER):] - if ':' in text or '#' in text: - raise ValueError(f'Invalid return value name "{text}"') - return plugin_fqcn, plugin_type, _remove_array_stubs(text), text, value + return _parse(text, plugin_fqcn, plugin_type, 'option name', require_plugin=require_plugin) -def augment_plugin_name_type(text: str, plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str] - ) -> str: +def parse_return_value(text: str, plugin_fqcn: t.Optional[str], plugin_type: t.Optional[str], + require_plugin=False + ) -> t.Tuple[t.Optional[str], t.Optional[str], str, str, t.Optional[str]]: """ - Given the text contents of O(...) or RV(...) and a plugin's FQCN and type, insert - the FQCN and type if they are not already present. + Given the contents of RV(...) / :ansretval:`...` with potential escaping removed, + split it into plugin FQCN, plugin type, option link name, option name, and option value. """ - value = None - if '=' in text: - text, value = text.split('=', 1) - if ':' not in text and plugin_fqcn and plugin_type: - text = f'{plugin_fqcn}#{plugin_type}:{text}' - return text if value is None else f'{text}={value}' + return _parse( + text, plugin_fqcn, plugin_type, 'return value name', require_plugin=require_plugin) From 20241ca0c0029d8aefd541f21dccbf3e2146c762 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sat, 18 Mar 2023 22:02:30 +0100 Subject: [PATCH 12/15] Add more tests. --- src/antsibull_docs/markup/dom.py | 34 ++++---- src/antsibull_docs/markup/format.py | 30 +++---- src/antsibull_docs/markup/parser.py | 2 +- tests/units/markup/test_counter.py | 122 ++++++++++++++++++++++++++++ tests/units/markup/test_dom.py | 13 +++ tests/units/markup/test_parser.py | 15 ++++ 6 files changed, 184 insertions(+), 32 deletions(-) create mode 100644 tests/units/markup/test_counter.py diff --git a/src/antsibull_docs/markup/dom.py b/src/antsibull_docs/markup/dom.py index 7beb05f6..52403a7e 100644 --- a/src/antsibull_docs/markup/dom.py +++ b/src/antsibull_docs/markup/dom.py @@ -20,7 +20,7 @@ ErrorType = t.Union[t.Literal['ignore'], t.Literal['message'], t.Literal['exception']] else: # Python 3.6/3.7 do not have t.Literal - ErrorType = str + ErrorType = str # pragma: no cover class PluginIdentifier(NamedTuple): @@ -157,63 +157,63 @@ class Walker(abc.ABC): @abc.abstractmethod def process_error(self, part: ErrorPart) -> None: - pass + pass # pragma: no cover @abc.abstractmethod def process_bold(self, part: BoldPart) -> None: - pass + pass # pragma: no cover @abc.abstractmethod def process_code(self, part: CodePart) -> None: - pass + pass # pragma: no cover @abc.abstractmethod def process_horizontal_line(self, part: HorizontalLinePart) -> None: - pass + pass # pragma: no cover @abc.abstractmethod def process_italic(self, part: ItalicPart) -> None: - pass + pass # pragma: no cover @abc.abstractmethod def process_link(self, part: LinkPart) -> None: - pass + pass # pragma: no cover @abc.abstractmethod def process_module(self, part: ModulePart) -> None: - pass + pass # pragma: no cover @abc.abstractmethod def process_rst_ref(self, part: RSTRefPart) -> None: - pass + pass # pragma: no cover @abc.abstractmethod def process_url(self, part: URLPart) -> None: - pass + pass # pragma: no cover @abc.abstractmethod def process_text(self, part: TextPart) -> None: - pass + pass # pragma: no cover @abc.abstractmethod def process_env_variable(self, part: EnvVariablePart) -> None: - pass + pass # pragma: no cover @abc.abstractmethod def process_option_name(self, part: OptionNamePart) -> None: - pass + pass # pragma: no cover @abc.abstractmethod def process_option_value(self, part: OptionValuePart) -> None: - pass + pass # pragma: no cover @abc.abstractmethod def process_plugin(self, part: PluginPart) -> None: - pass + pass # pragma: no cover @abc.abstractmethod def process_return_value(self, part: ReturnValuePart) -> None: - pass + pass # pragma: no cover class NoopWalker(Walker): @@ -304,3 +304,5 @@ def walk(paragraph: Paragraph, walker: Walker) -> None: # noqa: C901 walker.process_plugin(t.cast(PluginPart, part)) elif part.type == PartType.RETURN_VALUE: walker.process_return_value(t.cast(ReturnValuePart, part)) + else: + raise RuntimeError(f'Internal error: unknown type {part.type!r}') diff --git a/src/antsibull_docs/markup/format.py b/src/antsibull_docs/markup/format.py index 91d88e4a..70da5d50 100644 --- a/src/antsibull_docs/markup/format.py +++ b/src/antsibull_docs/markup/format.py @@ -46,63 +46,63 @@ class Formatter(abc.ABC): @abc.abstractmethod def format_error(self, part: dom.ErrorPart) -> str: - pass + pass # pragma: no cover @abc.abstractmethod def format_bold(self, part: dom.BoldPart) -> str: - pass + pass # pragma: no cover @abc.abstractmethod def format_code(self, part: dom.CodePart) -> str: - pass + pass # pragma: no cover @abc.abstractmethod def format_horizontal_line(self, part: dom.HorizontalLinePart) -> str: - pass + pass # pragma: no cover @abc.abstractmethod def format_italic(self, part: dom.ItalicPart) -> str: - pass + pass # pragma: no cover @abc.abstractmethod def format_link(self, part: dom.LinkPart) -> str: - pass + pass # pragma: no cover @abc.abstractmethod def format_module(self, part: dom.ModulePart, url: t.Optional[str]) -> str: - pass + pass # pragma: no cover @abc.abstractmethod def format_rst_ref(self, part: dom.RSTRefPart) -> str: - pass + pass # pragma: no cover @abc.abstractmethod def format_url(self, part: dom.URLPart) -> str: - pass + pass # pragma: no cover @abc.abstractmethod def format_text(self, part: dom.TextPart) -> str: - pass + pass # pragma: no cover @abc.abstractmethod def format_env_variable(self, part: dom.EnvVariablePart) -> str: - pass + pass # pragma: no cover @abc.abstractmethod def format_option_name(self, part: dom.OptionNamePart, url: t.Optional[str]) -> str: - pass + pass # pragma: no cover @abc.abstractmethod def format_option_value(self, part: dom.OptionValuePart) -> str: - pass + pass # pragma: no cover @abc.abstractmethod def format_plugin(self, part: dom.PluginPart, url: t.Optional[str]) -> str: - pass + pass # pragma: no cover @abc.abstractmethod def format_return_value(self, part: dom.ReturnValuePart, url: t.Optional[str]) -> str: - pass + pass # pragma: no cover class _FormatWalker(dom.Walker): diff --git a/src/antsibull_docs/markup/parser.py b/src/antsibull_docs/markup/parser.py index 9af78774..90f5817b 100644 --- a/src/antsibull_docs/markup/parser.py +++ b/src/antsibull_docs/markup/parser.py @@ -49,7 +49,7 @@ def __init__(self, command: str, parameters: int, escaped_arguments: bool = Fals @abc.abstractmethod def parse(self, parameters: t.List[str], context: Context) -> dom.AnyPart: - pass + pass # pragma: no cover class CommandParserEx(CommandParser): diff --git a/tests/units/markup/test_counter.py b/tests/units/markup/test_counter.py new file mode 100644 index 00000000..2df1a156 --- /dev/null +++ b/tests/units/markup/test_counter.py @@ -0,0 +1,122 @@ +# Author: Felix Fontein +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2023, Ansible Project + +import typing as t + +import pytest + +from antsibull_docs.markup import dom +from antsibull_docs.markup._counter import count + + +TEST_COUNTER = [ + ( + [], + { + 'italic': 0, + 'bold': 0, + 'module': 0, + 'plugin': 0, + 'link': 0, + 'url': 0, + 'ref': 0, + 'const': 0, + 'option-name': 0, + 'option-value': 0, + 'environment-var': 0, + 'return-value': 0, + 'ruler': 0, + }, + ), + ( + [dom.ErrorPart(message='foo')], + { + 'italic': 0, + 'bold': 0, + 'module': 0, + 'plugin': 0, + 'link': 0, + 'url': 0, + 'ref': 0, + 'const': 0, + 'option-name': 0, + 'option-value': 0, + 'environment-var': 0, + 'return-value': 0, + 'ruler': 0, + }, + ), + ( + [ + dom.TextPart(text='foo '), + dom.ItalicPart(text='bar'), + dom.TextPart(text=' baz '), + dom.CodePart(text=' bam '), + dom.TextPart(text=' '), + dom.BoldPart(text=' ( boo '), + dom.TextPart(text=' ) '), + dom.URLPart(url='https://example.com/?foo=bar'), + dom.HorizontalLinePart(), + dom.TextPart(text=' '), + dom.LinkPart(text='foo', url='https://bar.com'), + dom.TextPart(text=' '), + dom.RSTRefPart(text=' a', ref='b '), + dom.ModulePart(fqcn='foo.bar.baz'), + dom.TextPart(text='HORIZONTALLINEx '), + dom.ModulePart(fqcn='foo.bar.baz.bam'), + ], + { + 'italic': 1, + 'bold': 1, + 'module': 2, + 'plugin': 0, + 'link': 1, + 'url': 1, + 'ref': 1, + 'const': 1, + 'option-name': 0, + 'option-value': 0, + 'environment-var': 0, + 'return-value': 0, + 'ruler': 1, + }, + ), + ( + [ + dom.TextPart(text='foo '), + dom.EnvVariablePart(name='a),b'), + dom.TextPart(text=' '), + dom.PluginPart(plugin=dom.PluginIdentifier(fqcn='foo.bar.baz', type='bam')), + dom.TextPart(text=' baz '), + dom.OptionValuePart(value=' b,na)\\m, '), + dom.TextPart(text=' '), + dom.OptionNamePart(plugin=None, link=['foo'], name='foo', value=None), + dom.TextPart(text=' '), + dom.ReturnValuePart(plugin=None, link=['bar', 'baz'], name='bar.baz[1]', value=None), + ], + { + 'italic': 0, + 'bold': 0, + 'module': 0, + 'plugin': 1, + 'link': 0, + 'url': 0, + 'ref': 0, + 'const': 0, + 'option-name': 1, + 'option-value': 1, + 'environment-var': 1, + 'return-value': 1, + 'ruler': 0, + }, + ), +] + + +@pytest.mark.parametrize('paragraph, counter', TEST_COUNTER) +def test_count(paragraph: dom.Paragraph, counter: t.Dict[str, int]) -> None: + result = count([paragraph]) + assert result == counter diff --git a/tests/units/markup/test_dom.py b/tests/units/markup/test_dom.py index ec2258b5..c25a3394 100644 --- a/tests/units/markup/test_dom.py +++ b/tests/units/markup/test_dom.py @@ -109,6 +109,7 @@ def process_return_value(self, part: dom.ReturnValuePart) -> None: dom.TextPart(text=' '), dom.OptionNamePart(plugin=None, link=['foo'], name='foo', value=None), dom.TextPart(text=' '), + dom.ReturnValuePart(plugin=None, link=['bar', 'baz'], name='bar.baz[1]', value=None), ], ] @@ -119,3 +120,15 @@ def test_walk(data: dom.Paragraph) -> None: dom.walk(data, walker) assert walker.result == data + # The following has no side-effect (except increasing line coverage) + walker = dom.NoopWalker() + dom.walk(data, walker) + + +def test_internal_error() -> None: + class FakePart(t.NamedTuple): + type: int = 23 + + with pytest.raises(RuntimeError) as exc: + dom.walk([FakePart()], dom.NoopWalker()) + assert str(exc.value) == 'Internal error: unknown type 23' diff --git a/tests/units/markup/test_parser.py b/tests/units/markup/test_parser.py index 02182152..23b4ac41 100644 --- a/tests/units/markup/test_parser.py +++ b/tests/units/markup/test_parser.py @@ -347,3 +347,18 @@ def test_parse_bad(paragraphs: t.Union[str, t.List[str]], context: Context, kwar with pytest.raises(ValueError) as exc: parse(paragraphs, context, **kwargs) assert str(exc.value) == exc_message + + +TEST_TRIVIAL_PARSER = [ + '', + 'foo', + 'I(foo) B(bar) HORIZONTALLINE C(baz)' +] + + +@pytest.mark.parametrize('input', TEST_TRIVIAL_PARSER) +def test_trivial_parser(input: str) -> None: + parser = Parser([]) + result = parser.parse_string(input, Context()) + expected = [dom.TextPart(text=input)] if input else [] + assert result == expected From 4722f470b0833410305064591d7e2ba95ffe0b7d Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sun, 19 Mar 2023 15:17:15 +0100 Subject: [PATCH 13/15] Add changelog fragment. --- changelogs/fragments/108-markup.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelogs/fragments/108-markup.yml diff --git a/changelogs/fragments/108-markup.yml b/changelogs/fragments/108-markup.yml new file mode 100644 index 00000000..43e65174 --- /dev/null +++ b/changelogs/fragments/108-markup.yml @@ -0,0 +1,2 @@ +minor_change: + - Internal refactoring of markup code (https://github.com/ansible-community/antsibull-docs/pull/108). From 65d256706e654dc0c6ed5406e4ee8b9aeeab7e58 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sun, 19 Mar 2023 15:23:18 +0100 Subject: [PATCH 14/15] Add plain HTML output. --- src/antsibull_docs/markup/html.py | 100 ++++++++++++++++++++++++++++- tests/units/markup/test_html.py | 9 ++- tests/units/markup/test_vectors.py | 8 +-- 3 files changed, 108 insertions(+), 9 deletions(-) diff --git a/src/antsibull_docs/markup/html.py b/src/antsibull_docs/markup/html.py index 11abbd35..57bb4a43 100644 --- a/src/antsibull_docs/markup/html.py +++ b/src/antsibull_docs/markup/html.py @@ -26,7 +26,7 @@ def _url_escape(url: str) -> str: return quote(url, safe=':/#?%<>[]{}') -class HTMLFormatter(Formatter): +class AntsibullHTMLFormatter(Formatter): @staticmethod def _format_option_like(part: t.Union[dom.OptionNamePart, dom.ReturnValuePart], url: t.Optional[str]) -> str: @@ -120,11 +120,85 @@ def format_return_value(self, part: dom.ReturnValuePart, url: t.Optional[str]) - return self._format_option_like(part, url) -DEFAULT_FORMATTER = HTMLFormatter() +class PlainHTMLFormatter(Formatter): + @staticmethod + def _format_option_like(part: t.Union[dom.OptionNamePart, dom.ReturnValuePart], + url: t.Optional[str]) -> str: + link_start = '' + link_end = '' + if url: + link_start = f'' + link_end = '' + strong_start = '' + strong_end = '' + if part.type == dom.PartType.OPTION_NAME and part.value is None: + strong_start = '' + strong_end = '' + if part.value is None: + text = part.name + else: + text = f'{part.name}={part.value}' + return f'{strong_start}{link_start}{html_escape(text)}{link_end}{strong_end}' + + def format_error(self, part: dom.ErrorPart) -> str: + return f'ERROR while parsing: {html_escape(part.message)}' + + def format_bold(self, part: dom.BoldPart) -> str: + return f'{html_escape(part.text)}' + + def format_code(self, part: dom.CodePart) -> str: + return f"{html_escape(part.text)}" + + def format_horizontal_line(self, part: dom.HorizontalLinePart) -> str: + return '
' + + def format_italic(self, part: dom.ItalicPart) -> str: + return f'{html_escape(part.text)}' + + def format_link(self, part: dom.LinkPart) -> str: + return f"{html_escape(part.text)}" + + def format_module(self, part: dom.ModulePart, url: t.Optional[str]) -> str: + if not url: + return f"{html_escape(part.fqcn)}" + return f"{html_escape(part.fqcn)}" + + def format_rst_ref(self, part: dom.RSTRefPart) -> str: + return f"{html_escape(part.text)}" + + def format_url(self, part: dom.URLPart) -> str: + return ( + f"" + f'{html_escape(_url_escape(part.url))}' + ) + + def format_text(self, part: dom.TextPart) -> str: + return html_escape(part.text) + + def format_env_variable(self, part: dom.EnvVariablePart) -> str: + return f'{html_escape(part.name)}' + + def format_option_name(self, part: dom.OptionNamePart, url: t.Optional[str]) -> str: + return self._format_option_like(part, url) + + def format_option_value(self, part: dom.OptionValuePart) -> str: + return f'{html_escape(part.value)}' + + def format_plugin(self, part: dom.PluginPart, url: t.Optional[str]) -> str: + if not url: + return f"{html_escape(part.plugin.fqcn)}" + return f"{html_escape(part.plugin.fqcn)}" + + def format_return_value(self, part: dom.ReturnValuePart, url: t.Optional[str]) -> str: + return self._format_option_like(part, url) + + +DEFAULT_ANTSIBULL_FORMATTER = AntsibullHTMLFormatter() +DEFAULT_PLAIN_FORMATTER = PlainHTMLFormatter() def to_html(paragraphs: t.Sequence[dom.Paragraph], - formatter: Formatter = DEFAULT_FORMATTER, + formatter: Formatter = DEFAULT_ANTSIBULL_FORMATTER, link_provider: t.Optional[LinkProvider] = None, par_start: str = '

', par_end: str = '

', @@ -141,3 +215,23 @@ def to_html(paragraphs: t.Sequence[dom.Paragraph], par_empty=par_empty, current_plugin=current_plugin, ) + + +def to_html_plain(paragraphs: t.Sequence[dom.Paragraph], + formatter: Formatter = DEFAULT_PLAIN_FORMATTER, + link_provider: t.Optional[LinkProvider] = None, + par_start: str = '

', + par_end: str = '

', + par_sep: str = '', + par_empty: str = '', + current_plugin: t.Optional[dom.PluginIdentifier] = None) -> str: + return _format_paragraphs( + paragraphs, + formatter=formatter, + link_provider=link_provider, + par_start=par_start, + par_end=par_end, + par_sep=par_sep, + par_empty=par_empty, + current_plugin=current_plugin, + ) diff --git a/tests/units/markup/test_html.py b/tests/units/markup/test_html.py index 5f9a427b..de1f8f60 100644 --- a/tests/units/markup/test_html.py +++ b/tests/units/markup/test_html.py @@ -5,7 +5,7 @@ # SPDX-FileCopyrightText: 2023, Ansible Project from antsibull_docs.markup import dom -from antsibull_docs.markup.html import html_escape, to_html +from antsibull_docs.markup.html import html_escape, to_html, to_html_plain def test_html_escape(): @@ -17,3 +17,10 @@ def test_to_html(): assert to_html([]) == '' assert to_html([[dom.TextPart(text='test')]]) == '

test

' assert to_html([[dom.TextPart(text='test')]], par_start='
', par_end='
') == '
test
' + assert to_html([[dom.CodePart(text='test')]]) == "

test

" + +def test_to_html_plain(): + assert to_html_plain([]) == '' + assert to_html_plain([[dom.TextPart(text='test')]]) == '

test

' + assert to_html_plain([[dom.TextPart(text='test')]], par_start='
', par_end='
') == '
test
' + assert to_html_plain([[dom.CodePart(text='test')]]) == '

test

' diff --git a/tests/units/markup/test_vectors.py b/tests/units/markup/test_vectors.py index 7e1e350e..52598b17 100644 --- a/tests/units/markup/test_vectors.py +++ b/tests/units/markup/test_vectors.py @@ -11,7 +11,7 @@ from antsibull_docs.markup import dom from antsibull_docs.markup.parser import parse, Context -from antsibull_docs.markup.html import to_html +from antsibull_docs.markup.html import to_html, to_html_plain from antsibull_docs.markup.md import to_md from antsibull_docs.markup.rst import to_rst from antsibull_docs.markup.format import LinkProvider @@ -97,12 +97,10 @@ def test_vectors(test_name: str, test_data: t.Mapping[str, t.Any]) -> None: assert result == test_data['html'] if 'html_plain' in test_data: - ... - # TODO result = to_html(parsed, link_provider=html_link_provider, **html_opts) - # TODO assert result == test_data['html_plain'] + result = to_html_plain(parsed, link_provider=html_link_provider, **html_opts) + assert result == test_data['html_plain'] if 'md' in test_data: - print('XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX') result = to_md(parsed, link_provider=md_link_provider, **md_opts) assert result == test_data['md'] From 08be426222055ed086b5f9fe49dfb625d2726fa8 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sun, 19 Mar 2023 15:23:55 +0100 Subject: [PATCH 15/15] Rename default RST formatter. --- src/antsibull_docs/markup/rst.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/antsibull_docs/markup/rst.py b/src/antsibull_docs/markup/rst.py index 0d177204..30d3255c 100644 --- a/src/antsibull_docs/markup/rst.py +++ b/src/antsibull_docs/markup/rst.py @@ -31,7 +31,7 @@ def rst_escape(value: str, escape_ending_whitespace=False) -> str: return value -class RSTFormatter(Formatter): +class AntsibullRSTFormatter(Formatter): @staticmethod def _format_option_like(part: t.Union[dom.OptionNamePart, dom.ReturnValuePart], role: str) -> str: @@ -98,11 +98,11 @@ def format_return_value(self, part: dom.ReturnValuePart, url: t.Optional[str]) - return self._format_option_like(part, 'ansretval') -DEFAULT_FORMATTER = RSTFormatter() +DEFAULT_ANTSIBULL_FORMATTER = AntsibullRSTFormatter() def to_rst(paragraphs: t.Sequence[dom.Paragraph], - formatter: Formatter = DEFAULT_FORMATTER, + formatter: Formatter = DEFAULT_ANTSIBULL_FORMATTER, link_provider: t.Optional[LinkProvider] = None, par_start: str = '', par_end: str = '',