Improve environment variable handling

changelogs/fragments/73-env-vars.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - "Use correct markup (``envvar`` role) for environment variables. Compile an index of all environment variables used by plugins (https://github.com/ansible-community/antsibull-docs/pull/73)."

src/antsibull_docs/cli/doc_commands/stable.py

@@ -37,6 +37,7 @@
     load_all_collection_routing,
     remove_redirect_duplicates,
 )
+from ...env_variables import load_ansible_config, collect_referenced_environment_variables
 from ...schemas.docs import DOCS_SCHEMAS
 from ...utils.collection_name_transformer import CollectionNameTransformer
 from ...write_docs import (
@@ -47,6 +48,7 @@
     output_indexes,
     output_plugin_indexes,
     output_extra_docs,
+    output_environment_variables,
 )
 
 if t.TYPE_CHECKING:
@@ -337,12 +339,16 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
     app_ctx = app_context.app_ctx.get()
 
     # Get the info from the plugins
-    plugin_info, collection_metadata = asyncio_run(get_ansible_plugin_info(
+    plugin_info, full_collection_metadata = asyncio_run(get_ansible_plugin_info(
         venv, collection_dir, collection_names=collection_names))
     flog.notice('Finished parsing info from plugins and collections')
     # flog.fields(plugin_info=plugin_info).debug('Plugin data')
     # flog.fields(
-    #     collection_metadata=collection_metadata).debug('Collection metadata')
+    #     collection_metadata=full_collection_metadata).debug('Collection metadata')
+
+    collection_metadata = dict(full_collection_metadata)
+    if collection_names is not None and 'ansible.builtin' not in collection_names:
+        del collection_metadata['ansible.builtin']
 
     # Load collection routing information
     collection_routing = asyncio_run(load_all_collection_routing(collection_metadata))
@@ -363,7 +369,7 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
         {name: data.path for name, data in collection_metadata.items()}))
     flog.debug('Finished getting collection extra docs data')
 
-    # Load collection extra docs data
+    # Load collection links data
     link_data = asyncio_run(load_collections_links(
         {name: data.path for name, data in collection_metadata.items()}))
     flog.debug('Finished getting collection link data')
@@ -384,6 +390,10 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
                     print(f"{plugin_name} {plugin_type}: {textwrap.indent(error, '    ').lstrip()}")
         return 1
 
+    # Handle environment variables
+    ansible_config = load_ansible_config(full_collection_metadata['ansible.builtin'])
+    referenced_env_vars = collect_referenced_environment_variables(new_plugin_info, ansible_config)
+
     collection_namespaces = get_collection_namespaces(collection_to_plugin_info.keys())
 
     collection_url = CollectionNameTransformer(
@@ -444,7 +454,14 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
 
     asyncio_run(output_extra_docs(dest_dir, extra_docs_data,
                                   squash_hierarchy=squash_hierarchy))
-    flog.debug('Finished writing extra extra docs docs')
+    flog.debug('Finished writing extra docs')
+
+    if referenced_env_vars:
+        asyncio_run(output_environment_variables(dest_dir, referenced_env_vars,
+                                                 squash_hierarchy=squash_hierarchy))
+        flog.debug('Finished writing environment variables')
+    else:
+        flog.debug('Skipping environment variables (as there are none)')
     return 0
 
 

src/antsibull_docs/data/collection-enum.py

@@ -249,11 +249,10 @@ def main(args):
         collection_name = f'{meta["namespace"]}.{meta["name"]}'
         if match_filter(collection_name, coll_filter):
             result['collections'][collection_name] = meta
-    if match_filter('ansible.builtin', coll_filter):
-        result['collections']['ansible.builtin'] = {
-            'path': os.path.dirname(ansible_release.__file__),
-            'version': ansible_release.__version__,
-        }
+    result['collections']['ansible.builtin'] = {
+        'path': os.path.dirname(ansible_release.__file__),
+        'version': ansible_release.__version__,
+    }
 
     print(json.dumps(
         result, cls=AnsibleJSONEncoder, sort_keys=True, indent=4 if arguments.pretty else None))

src/antsibull_docs/data/docsite/list_of_env_variables.rst.j2

@@ -0,0 +1,39 @@
+{#
+  Copyright (c) Ansible Project
+  GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+  SPDX-License-Identifier: GPL-3.0-or-later
+#}
+
+:orphan:
+
+.. _list_of_collection_env_vars:
+
+Index of all Collection Environment Variables
+=============================================
+
+The following index documents all environment variables declared by plugins in collections.
+Environment variables used by the ansible-core configuation are documented in :ref:`ansible_configuration_settings`.
+{# TODO: use label `ansible_configuration_env_vars` once the ansible-core PR is merged #}
+
+{% for _, env_var in env_variables | dictsort %}
+.. envvar:: @{ env_var.name }@
+
+{%   for paragraph in env_var.description or [] %}
+    @{ paragraph | replace('\n', '\n    ') | rst_ify | indent(4) }@
+
+{%   endfor %}
+    *Used by:*
+{% set plugins_ = [] %}
+{%   for plugin_type, plugins in env_var.plugins.items() %}
+{%     for plugin_name in plugins %}
+{%       set _ = plugins_.append((plugin_name, plugin_type)) %}
+{%     endfor %}
+{%   endfor %}
+{%   for plugin_name, plugin_type in plugins_ | unique | sort %}
+    :ref:`@{ plugin_name | rst_escape }@ {% if plugin_type == 'module' %}module{% else %}@{ plugin_type }@ plugin{% endif %} <ansible_collections.@{ plugin_name }@_@{ plugin_type }@>`
+{%-    if not loop.last -%}
+,
+{%     endif -%}
+{%-  endfor %}
+
+{% endfor %}

src/antsibull_docs/data/docsite/macros/parameters.rst.j2

@@ -125,7 +125,7 @@
 {%       endfor %}
 {%     endif %}
 {%     for env in value['env'] %}
-      - Environment variable: @{ env['name'] | rst_escape }@
+      - Environment variable: :envvar:`@{ env['name'] | rst_escape(escape_ending_whitespace=true) }@`
 {%       if env['version_added'] is still_relevant(collection=env['version_added_collection'] or collection) %}
 
         :ansible-option-versionadded:`added in @{ version_added_rst(env['version_added'], env['version_added_collection'] or collection) }@`
@@ -250,7 +250,7 @@
 {%     endif %}
 {%     for env in value['env'] %}
       <li>
-        <p>Environment variable: @{ env['name'] | escape }@</p>
+        <p>Environment variable: <code class="xref std std-envvar literal notranslate">@{ env['name'] | escape }@</code></p>
 {%       if env['version_added'] is still_relevant(collection=env['version_added_collection'] or collection) %}
         <p><span class="ansible-option-versionadded">added in @{ version_added_html(env['version_added'], env['version_added_collection'] or collection) }@</span></p>
 {%       endif %}

src/antsibull_docs/docs_parsing/ansible_doc.py

@@ -217,12 +217,11 @@ def get_collection_metadata(venv: t.Union['VenvRunner', 'FakeVenvRunner'],
                             ) -> t.Dict[str, AnsibleCollectionMetadata]:
     collection_metadata = {}
 
-    # Obtain ansible.builtin version
-    if collection_names is None or 'ansible.builtin' in collection_names:
-        venv_ansible = venv.get_command('ansible')
-        ansible_version_cmd = venv_ansible('--version', _env=env)
-        raw_result = ansible_version_cmd.stdout.decode('utf-8', errors='surrogateescape')
-        collection_metadata['ansible.builtin'] = _extract_ansible_builtin_metadata(raw_result)
+    # Obtain ansible.builtin version and path
+    venv_ansible = venv.get_command('ansible')
+    ansible_version_cmd = venv_ansible('--version', _env=env)
+    raw_result = ansible_version_cmd.stdout.decode('utf-8', errors='surrogateescape')
+    collection_metadata['ansible.builtin'] = _extract_ansible_builtin_metadata(raw_result)
 
     # Obtain collection versions
     venv_ansible_galaxy = venv.get_command('ansible-galaxy')

src/antsibull_docs/docs_parsing/parsing.py

@@ -53,8 +53,9 @@ async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'],
                     {information from ansible-doc --json.  See the ansible-doc documentation
                      for more info.}
 
-        The second component is a Mapping of collection names to metadata.
-
+        The second component is a Mapping of collection names to metadata. The second mapping
+        always includes the metadata for ansible.builtin, even if it was not explicitly
+        mentioned in ``collection_names``.
     """
     flog = mlog.fields(func='get_ansible_plugin_info')
 

src/antsibull_docs/env_variables.py

@@ -0,0 +1,124 @@
+# Author: Felix Fontein <felix@fontein.de>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or
+# https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2022, Ansible Project
+"""Environment variable handling."""
+
+import os
+import os.path
+import typing as t
+
+from antsibull_core import yaml
+
+from .docs_parsing import AnsibleCollectionMetadata
+
+
+class EnvironmentVariableInfo:
+    name: str
+    description: t.Optional[t.List[str]]
+    plugins: t.Dict[str, t.List[str]]  # maps plugin_type to lists of plugin FQCNs
+
+    def __init__(self,
+                 name: str,
+                 description: t.Optional[t.List[str]] = None,
+                 plugins: t.Optional[t.Dict[str, t.List[str]]] = None):
+        self.name = name
+        self.description = description
+        self.plugins = plugins or {}
+
+    def __repr__(self):
+        return f'E({self.name}, description={repr(self.description)}, plugins={self.plugins})'
+
+
+def load_ansible_config(ansible_builtin_metadata: AnsibleCollectionMetadata
+                        ) -> t.Mapping[str, t.Mapping[str, t.Any]]:
+    """
+    Load Ansible base configuration (``lib/ansible/config/base.yml``).
+
+    :arg ansible_builtin_metadata: Metadata for the ansible.builtin collection.
+    :returns: A Mapping of configuration options to information on these options.
+    """
+    return yaml.load_yaml_file(os.path.join(ansible_builtin_metadata.path, 'config', 'base.yml'))
+
+
+def _find_env_vars(options: t.Mapping[str, t.Mapping[str, t.Any]]
+                   ) -> t.Generator[t.Tuple[str, t.Optional[t.List[str]]], None, None]:
+    for _, option_data in options.items():
+        if isinstance(option_data.get('env'), list):
+            description = option_data.get('description')
+            if isinstance(description, str):
+                description = [description]
+            if isinstance(description, list):
+                description = [str(desc) for desc in description]
+            else:
+                description = None
+            for env_var in option_data['env']:
+                if isinstance(env_var.get('name'), str):
+                    yield (env_var['name'], description)
+        if isinstance(option_data.get('suboptions'), dict):
+            yield from _find_env_vars(option_data['suboptions'])
+
+
+def _collect_env_vars_and_descriptions(plugin_info: t.Mapping[str, t.Mapping[str, t.Any]],
+                                       core_envs: t.Set[str],
+                                       ) -> t.Tuple[t.Mapping[str, EnvironmentVariableInfo],
+                                                    t.Mapping[str, t.List[t.List[str]]]]:
+    other_variables: t.Dict[str, EnvironmentVariableInfo] = {}
+    other_variable_description: t.Dict[str, t.List[t.List[str]]] = {}
+    for plugin_type, plugins in plugin_info.items():
+        for plugin_name, plugin_data in plugins.items():
+            plugin_options: t.Mapping[str, t.Mapping[str, t.Any]] = (
+                (plugin_data.get('doc') or {}).get('options') or {}
+            )
+            for env_var, env_var_description in _find_env_vars(plugin_options):
+                if env_var in core_envs:
+                    continue
+                if env_var not in other_variables:
+                    other_variables[env_var] = EnvironmentVariableInfo(env_var)
+                    other_variable_description[env_var] = []
+                if plugin_type not in other_variables[env_var].plugins:
+                    other_variables[env_var].plugins[plugin_type] = []
+                other_variables[env_var].plugins[plugin_type].append(plugin_name)
+                if env_var_description is not None:
+                    other_variable_description[env_var].append(env_var_description)
+    return other_variables, other_variable_description
+
+
+def _augment_env_var_descriptions(other_variables: t.Mapping[str, EnvironmentVariableInfo],
+                                  other_variable_description: t.Mapping[str, t.List[t.List[str]]],
+                                  ) -> None:
+    for variable, variable_info in other_variables.items():
+        if other_variable_description[variable]:
+            value: t.Optional[t.List[str]] = other_variable_description[variable][0]
+            for other_value in other_variable_description[variable]:
+                if value != other_value:
+                    value = [
+                        'See the documentations for the options where this environment variable'
+                        ' is used.'
+                    ]
+                    break
+            variable_info.description = value
+
+
+def collect_referenced_environment_variables(plugin_info: t.Mapping[str, t.Mapping[str, t.Any]],
+                                             ansible_config: t.Mapping[str, t.Mapping[str, t.Any]],
+                                             ) -> t.Mapping[str, EnvironmentVariableInfo]:
+    """
+    Collect referenced environment variables that are not defined in the ansible-core
+    configuration.
+
+    :arg plugin_info: Mapping of plugin type to a mapping of plugin name to plugin record.
+    :arg ansible_config: The Ansible base configuration (``lib/ansible/config/base.yml``).
+    :returns: A Mapping of environment variable name to an environment variable infomation object.
+    """
+    core_envs = {'ANSIBLE_CONFIG'}
+    for config in ansible_config.values():
+        if config.get('env'):
+            for env in config['env']:
+                core_envs.add(env['name'])
+
+    other_variables, other_variable_description = _collect_env_vars_and_descriptions(
+        plugin_info, core_envs)
+    _augment_env_var_descriptions(other_variables, other_variable_description)
+    return other_variables

src/antsibull_docs/write_docs.py

@@ -21,8 +21,9 @@
 
 from .jinja2.environment import doc_environment
 from .collection_links import CollectionLinks
-from .extra_docs import CollectionExtraDocsInfoT
 from .docs_parsing import AnsibleCollectionMetadata
+from .env_variables import EnvironmentVariableInfo
+from .extra_docs import CollectionExtraDocsInfoT
 from .utils.collection_name_transformer import CollectionNameTransformer
 
 
@@ -907,7 +908,7 @@ async def output_extra_docs(dest_dir: str,
                             extra_docs_data: t.Mapping[str, CollectionExtraDocsInfoT],
                             squash_hierarchy: bool = False) -> None:
     """
-    Generate collection-level index pages for the collections.
+    Write extra docs pages for the collections.
 
     :arg dest_dir: The directory to place the documentation in.
     :arg extra_docs_data: Dictionary mapping collection names to CollectionExtraDocsInfoT.
@@ -940,3 +941,46 @@ async def output_extra_docs(dest_dir: str,
         await asyncio.gather(*writers)
 
     flog.debug('Leave')
+
+
+async def output_environment_variables(dest_dir: str,
+                                       env_variables: t.Mapping[str, EnvironmentVariableInfo],
+                                       squash_hierarchy: bool = False
+                                       ) -> None:
+    """
+    Write environment variable Generate collection-level index pages for the collections.
+
+    :arg dest_dir: The directory to place the documentation in.
+    :arg env_variables: Mapping of environment variable names to environment variable information.
+    :arg squash_hierarchy: If set to ``True``, no directory hierarchy will be used.
+                           Undefined behavior if documentation for multiple collections are
+                           created.
+    """
+    flog = mlog.fields(func='write_environment_variables')
+    flog.debug('Enter')
+
+    if not squash_hierarchy:
+        collection_toplevel = os.path.join(dest_dir, 'collections')
+    else:
+        collection_toplevel = dest_dir
+
+    env = doc_environment(('antsibull_docs.data', 'docsite'))
+    # Get the templates
+    env_var_list_tmpl = env.get_template('list_of_env_variables.rst.j2')
+
+    flog.fields(toplevel=collection_toplevel, exists=os.path.isdir(collection_toplevel)).debug(
+        'collection_toplevel exists?')
+    # This is only safe because we made sure that the top of the directory tree we're writing to
+    # (docs/docsite/rst) is only writable by us.
+    os.makedirs(collection_toplevel, mode=0o755, exist_ok=True)
+
+    index_file = os.path.join(collection_toplevel, 'environment_variables.rst')
+    index_contents = _render_template(
+        env_var_list_tmpl,
+        index_file,
+        env_variables=env_variables,
+    )
+
+    await write_file(index_file, index_contents)
+
+    flog.debug('Leave')