Merge pull request #6 from felixfontein/tests-filters

Support test and filter plugins

changelogs/fragments/6-tests-filters.yml

@@ -0,0 +1,6 @@
+minor_changes:
+  - "Support test and filter plugins when ansible-core 2.14+ is used. This works with the current ``devel`` branch of ansible-core (https://github.com/ansible-community/antsibull-docs/pull/6)."
+  - "Support parameter type ``any``, and show ``raw`` as ``any`` (https://github.com/ansible-community/antsibull-docs/pull/6)."
+  - "More robust handling of parsing errors when ansible-doc was unable to extract documentation (https://github.com/ansible-community/antsibull-docs/pull/6)."
+  - "If lookup plugins have an option called ``_terms``, it is now shown in its own section ``Terms``, and not in the regular ``Parameters`` section (https://github.com/ansible-community/antsibull-docs/pull/6)."
+  - "If lookup plugins have a single return value starting with ``_``, that return value is now labelled ``Return value`` (https://github.com/ansible-community/antsibull-docs/pull/6)."

src/antsibull_docs/cli/doc_commands/stable.py

@@ -136,6 +136,9 @@ def normalize_plugin_info(plugin_type: str,
     # If you wonder why this code isn't showing up in code coverage: that's because it's executed
     # in a subprocess. See normalize_all_plugin_info below.
 
+    if 'error' in plugin_info:
+        return ({}, [plugin_info['error']])
+
     errors = []
     if plugin_type == 'role':
         try:
@@ -251,10 +254,12 @@ def get_plugin_contents(plugin_info: t.Mapping[str, t.Mapping[str, t.Any]],
             namespace, collection, short_name = get_fqcn_parts(plugin_name)
             if plugin_type == 'role':
                 desc = ''
-                if 'main' in plugin_desc['entry_points']:
-                    desc = plugin_desc['entry_points']['main']['short_description']
+                if 'entry_points' in plugin_desc and 'main' in plugin_desc['entry_points']:
+                    desc = plugin_desc['entry_points']['main'].get('short_description') or ''
+            elif 'doc' in plugin_desc:
+                desc = plugin_desc['doc'].get('short_description') or ''
             else:
-                desc = plugin_desc['doc']['short_description']
+                desc = ''
             plugin_contents[plugin_type]['.'.join((namespace, collection))][short_name] = desc
 
     return plugin_contents

src/antsibull_docs/constants.py

@@ -16,9 +16,11 @@
 DOCUMENTABLE_PLUGINS: FrozenSet[str] = frozenset(('become', 'cache', 'callback', 'cliconf',
                                                   'connection', 'httpapi', 'inventory', 'lookup',
                                                   'netconf', 'shell', 'vars', 'module',
-                                                  'strategy', 'role',))
+                                                  'strategy', 'role', 'filter', 'test'))
 
 
 DOCUMENTABLE_PLUGINS_MIN_VERSION: Dict[str, str] = {
+    'filter': '2.14.0',
     'role': '2.11.0',
+    'test': '2.14.0',
 }

src/antsibull_docs/data/docsite/macros/parameters.rst.j2

@@ -11,7 +11,7 @@
 
   * - Parameter
     - Comments
-{% for key, value in elements | dictsort recursive %}
+{% for key, value in elements recursive %}
 {#   parameter name with required and/or introduced label #}
 
   * - .. raw:: html
@@ -186,7 +186,7 @@
   </thead>
   <tbody>
 {% set row_class = cycler('even', 'odd') %}
-{% for key, value in elements | dictsort recursive %}
+{% for key, value in elements recursive %}
 {#   parameter name with required and/or introduced label #}
   <tr class="row-@{ row_class.next() }@">
     <td>{% for i in range(1, loop.depth) %}<div class="ansible-option-indent"></div>{% endfor %}<div class="ansible-option-cell">

src/antsibull_docs/data/docsite/macros/returnvalues.rst.j2

@@ -8,7 +8,7 @@
 
   * - Key
     - Description
-{% for key, value in elements | dictsort recursive %}
+{% for key, value in elements recursive %}
 {#   return value name #}
 
   * - .. raw:: html
@@ -114,7 +114,7 @@
   </thead>
   <tbody>
 {% set row_class = cycler('even', 'odd') %}
-{% for key, value in elements | dictsort recursive %}
+{% for key, value in elements recursive %}
 {#   return value name #}
   <tr class="row-@{ row_class.next() }@">
     <td>{% for i in range(1, loop.depth) %}<div class="ansible-option-indent"></div>{% endfor %}<div class="ansible-option-cell">

src/antsibull_docs/data/docsite/plugin.rst.j2

@@ -162,17 +162,90 @@ The below requirements are needed on the local controller node that executes thi
 
 {% endif %}
 
+{% set options_to_skip = [] %}
+
+{% if doc['options']['_terms'] and plugin_type in ['lookup'] %}
+{%   set options_to_skip = options_to_skip + ['_terms'] %}
+
+.. Terms
+
+Terms
+-----
+
+{%   if use_html_blobs %}
+@{     parameters_html([['Terms', doc['options']['_terms']]], suboption_key='suboptions') }@
+{%   else %}
+@{     parameters_rst([['Terms', doc['options']['_terms']]], suboption_key='suboptions') }@
+{%   endif %}
+
+{% endif %}
+
+{% if doc['options']['_input'] and plugin_type in ['filter', 'test'] %}
+{%   set options_to_skip = options_to_skip + ['_input'] %}
+
+.. Input
+
+Input
+-----
+
+{%   if plugin_type == 'filter' %}
+This describes the input of the filter, the value before ``| @{plugin_name}@``.
+{%   else %}
+This describes the input of the test, the value before ``is @{plugin_name}@`` or ``is not @{plugin_name}@``.
+{%   endif %}
+
+{%   if use_html_blobs %}
+@{     parameters_html([['Input', doc['options']['_input']]], suboption_key='suboptions') }@
+{%   else %}
+@{     parameters_rst([['Input', doc['options']['_input']]], suboption_key='suboptions') }@
+{%   endif %}
+
+{% endif %}
+
+{% if doc['positional'] and plugin_type in ['lookup', 'filter', 'test'] %}
+{%   set options_to_skip = options_to_skip + doc['positional'] %}
+
+.. Positional
+
+Positional parameters
+---------------------
+
+{%   if plugin_type == 'filter' %}
+This describes positional parameters of the filter. These are the values ``positional1``, ``positional2`` and so on in the following example: ``input | @{plugin_name}@(positional1, positional2, ...)``.
+{%   elif plugin_type == 'test' %}
+This describes positional parameters of the test. These are the values ``positional1``, ``positional2`` and so on in the following examples: ``input is @{plugin_name}@(positional1, positional2, ...)`` and ``input is not @{plugin_name}@(positional1, positional2, ...)``.
+{%   endif %}
+
+{%   if use_html_blobs %}
+@{     parameters_html(doc['options'] | extract_options_from_list(doc['positional']), suboption_key='suboptions') }@
+{%   else %}
+@{     parameters_rst(doc['options'] | extract_options_from_list(doc['positional']), suboption_key='suboptions') }@
+{%   endif %}
+
+{% endif %}
+
 .. Options
 
-{% if doc['options'] -%}
+{% if doc['options'] | remove_options_from_list(options_to_skip) -%}
 
+{%   if plugin_type in ['filter', 'test'] %}
+Keyword parameters
+------------------
+{%   else %}
 Parameters
 ----------
+{%   endif %}
+
+{%   if plugin_type == 'filter' %}
+This describes keyword parameters of the filter. These are the values ``key1=value1``, ``key2=value2`` and so on in the following example: ``input | @{plugin_name}@(key1=value1, key2=value2, ...)``.
+{%   elif plugin_type == 'test' %}
+This describes keyword parameters of the test. These are the values ``key1=value1``, ``key2=value2`` and so on in the following examples: ``input is @{plugin_name}@(key1=value1, key2=value2, ...)`` and ``input is not @{plugin_name}@(key1=value1, key2=value2, ...)``.
+{%   endif %}
 
 {%   if use_html_blobs %}
-@{     parameters_html(doc['options'], suboption_key='suboptions') }@
+@{     parameters_html(doc['options'] | remove_options_from_list(options_to_skip) | dictsort, suboption_key='suboptions') }@
 {%   else %}
-@{     parameters_rst(doc['options'], suboption_key='suboptions') }@
+@{     parameters_rst(doc['options'] | remove_options_from_list(options_to_skip) | dictsort, suboption_key='suboptions') }@
 {%   endif %}
 {% endif %}
 
@@ -235,7 +308,7 @@ Examples
 
 {% endif %}
 
-{% if 'ansible_facts' in returndocs %}
+{% if 'ansible_facts' in returndocs and plugin_type == 'module' %}
 {%   set returnfacts = returndocs['ansible_facts']['contains'] %}
 {%   set _x = returndocs.pop('ansible_facts', None) %}
 {% endif %}
@@ -248,23 +321,37 @@ Returned Facts
 Facts returned by this module are added/updated in the ``hostvars`` host facts and can be referenced by name just like any other host fact. They do not need to be registered in order to use them.
 
 {%   if use_html_blobs %}
-@{     return_docs_html(returnfacts) }@
+@{     return_docs_html(returnfacts | dictsort) }@
 {%   else %}
-@{     return_docs_rst(returnfacts) }@
+@{     return_docs_rst(returnfacts | dictsort) }@
 {%   endif %}
 {% endif %}
 
 .. Return values
 
 {% if returndocs -%}
+{%   if plugin_type not in ['lookup', 'filter', 'test'] %}
+{#     This only makes sense for plugins which can return more than one value. #}
 Return Values
 -------------
 Common return values are documented :ref:`here <common_return_values>`, the following are the fields unique to this @{ plugin_type }@:
+{%   else %}
+Return Value
+------------
+{%   endif %}
 
-{%   if use_html_blobs %}
-@{     return_docs_html(returndocs) }@
+{%   if plugin_type in ['lookup', 'filter', 'test'] and returndocs | length == 1 and (returndocs | first).startswith('_') %}
+{%     if use_html_blobs %}
+@{       return_docs_html([['Return value', returndocs.values() | first]]) }@
+{%     else %}
+@{       return_docs_rst([['Return value', returndocs.values() | first]]) }@
+{%     endif %}
 {%   else %}
-@{     return_docs_rst(returndocs) }@
+{%     if use_html_blobs %}
+@{       return_docs_html(returndocs | dictsort) }@
+{%     else %}
+@{       return_docs_rst(returndocs | dictsort) }@
+{%     endif %}
 {%   endif %}
 {% endif %}
 

src/antsibull_docs/data/docsite/role.rst.j2

@@ -128,9 +128,9 @@ Parameters
 ^^^^^^^^^^
 
 {%     if use_html_blobs %}
-@{       parameters_html(ep_doc['options'], suboption_key='options', parameter_html_prefix=entry_point ~ '--', parameter_rst_prefix=entry_point ~ '__') }@
+@{       parameters_html(ep_doc['options'] | dictsort, suboption_key='options', parameter_html_prefix=entry_point ~ '--', parameter_rst_prefix=entry_point ~ '__') }@
 {%     else %}
-@{       parameters_rst(ep_doc['options'], suboption_key='options', parameter_html_prefix=entry_point ~ '--', parameter_rst_prefix=entry_point ~ '__') }@
+@{       parameters_rst(ep_doc['options'] | dictsort, suboption_key='options', parameter_html_prefix=entry_point ~ '--', parameter_rst_prefix=entry_point ~ '__') }@
 {%     endif %}
 {%   endif %}
 

src/antsibull_docs/jinja2/environment.py

@@ -9,7 +9,7 @@
 
 from .filters import (
     do_max, documented_type, html_ify, rst_ify, rst_escape, rst_fmt, rst_xline, move_first,
-    massage_author_name,
+    massage_author_name, extract_options_from_list, remove_options_from_list,
 )
 from .tests import still_relevant, test_list
 
@@ -70,6 +70,8 @@ def doc_environment(template_location):
     env.filters['documented_type'] = documented_type
     env.filters['move_first'] = move_first
     env.filters['massage_author_name'] = massage_author_name
+    env.filters['extract_options_from_list'] = extract_options_from_list
+    env.filters['remove_options_from_list'] = remove_options_from_list
     env.tests['list'] = test_list
     env.tests['still_relevant'] = still_relevant
 

src/antsibull_docs/jinja2/filters.py

@@ -194,3 +194,18 @@ def massage_author_name(value):
     value = _EMAIL_ADDRESS.sub('', value)
     value = value.replace('(!UNKNOWN)', '')
     return value
+
+
+def extract_options_from_list(options: t.Dict[str, t.Any],
+                              options_to_extract: t.List[str]) -> t.List[t.Tuple[str, t.Any]]:
+    ''' return list of tuples (option, option_data) with option from options_to_extract '''
+    return [(option, options[option]) for option in options_to_extract if option in options]
+
+
+def remove_options_from_list(options: t.Dict[str, t.Any],
+                             options_to_remove: t.List[str]) -> t.Dict[str, t.Any]:
+    ''' return copy of dictionary with the options from options_to_remove removed '''
+    result = options.copy()
+    for option in options_to_remove:
+        result.pop(option, None)
+    return result

src/antsibull_docs/schemas/ansible_doc.py

@@ -13,5 +13,5 @@
 
 
 warnings.warn('antsibull.schemas.ansible_doc is deprecated.'
-              ' Use antsibull.schemas.docs.ansible_doc instead.',
+              ' Use antsibull_docs.schemas.docs.ansible_doc instead.',
               DeprecationWarning, stacklevel=2)

src/antsibull_docs/schemas/docs/__init__.py

@@ -10,6 +10,7 @@
 """
 
 from .callback import CallbackDocSchema, CallbackSchema
+from .positional import PositionalDocSchema, PositionalSchema
 from .module import ModuleDocSchema, ModuleSchema
 from .plugin import (PluginDocSchema, PluginExamplesSchema,
                      PluginMetadataSchema, PluginReturnSchema, PluginSchema)
@@ -37,6 +38,14 @@
     'return': PluginReturnSchema,
 }
 
+_POSITIONAL_PLUGIN_SCHEMA_RECORD = {
+    'top': PositionalSchema,
+    'doc': PositionalDocSchema,
+    'examples': PluginExamplesSchema,
+    'metadata': PluginMetadataSchema,
+    'return': PluginReturnSchema,
+}
+
 
 #: Mapping of plugin_types to the schemas which validate and normalize their documentation.
 #: The structure of this mapping is a two level nested dict.  The outer key is the plugin_type.
@@ -54,9 +63,10 @@
     },
     'cliconf': _PLUGIN_SCHEMA_RECORD,
     'connection': _PLUGIN_SCHEMA_RECORD,
+    'filter': _POSITIONAL_PLUGIN_SCHEMA_RECORD,
     'httpapi': _PLUGIN_SCHEMA_RECORD,
     'inventory': _PLUGIN_SCHEMA_RECORD,
-    'lookup': _PLUGIN_SCHEMA_RECORD,
+    'lookup': _POSITIONAL_PLUGIN_SCHEMA_RECORD,
     'module': {
         'top': ModuleSchema,
         'doc': ModuleDocSchema,
@@ -67,6 +77,7 @@
     'netconf': _PLUGIN_SCHEMA_RECORD,
     'shell': _PLUGIN_SCHEMA_RECORD,
     'strategy': _PLUGIN_SCHEMA_RECORD,
+    'test': _POSITIONAL_PLUGIN_SCHEMA_RECORD,
     'vars': _PLUGIN_SCHEMA_RECORD,
     'role': RoleSchema,
 }