Change how we embed docsite links in help text. (#12266)

Cherry-pick of #12261

[ci skip-build-wheels]

build-support/bin/generate_docs.py

@@ -33,17 +33,28 @@
 from common import die
 
 from pants.help.help_info_extracter import to_help_str
+from pants.util.docutil import DocUrlMatcher, DocUrlRewriter, get_titles
 from pants.version import MAJOR_MINOR
 
 logger = logging.getLogger(__name__)
 
 
 def main() -> None:
     logging.basicConfig(format="[%(levelname)s]: %(message)s", level=logging.INFO)
-    version = determine_pants_version()
     args = create_parser().parse_args()
+    version = determine_pants_version()
     help_info = run_pants_help_all()
-    generator = ReferenceGenerator(args, version, help_info)
+    doc_urls = DocUrlMatcher().find_doc_urls(value_strs_iter(help_info))
+    logger.info("Found the following docsite URLs:")
+    for url in sorted(doc_urls):
+        logger.info(f"  {url}")
+    logger.info("Fetching titles...")
+    slug_to_title = get_titles(doc_urls)
+    logger.info("Found the following titles:")
+    for slug, title in sorted(slug_to_title.items()):
+        logger.info(f"  {slug}: {title}")
+    rewritten_help_info = rewrite_value_strs(help_info, slug_to_title)
+    generator = ReferenceGenerator(args, version, rewritten_help_info)
     if args.sync:
         generator.sync()
     else:
@@ -142,8 +153,41 @@ def run_pants_help_all() -> Dict:
     return cast(Dict, json.loads(run.stdout))
 
 
+def value_strs_iter(help_info: dict) -> Iterable[str]:
+    def _recurse(val: Any) -> Iterable[str]:
+        if isinstance(val, str):
+            yield val
+        if isinstance(val, dict):
+            for v in val.values():
+                for x in _recurse(v):
+                    yield x
+        if isinstance(val, list):
+            for v in val:
+                for x in _recurse(v):
+                    yield x
+
+    for x in _recurse(help_info):
+        yield x
+
+
+def rewrite_value_strs(help_info: dict, slug_to_title: dict[str, str]) -> dict:
+    """Return a copy of the argument with rewritten docsite URLs."""
+    rewriter = DocUrlRewriter(slug_to_title)
+
+    def _recurse(val: Any) -> Any:
+        if isinstance(val, str):
+            return rewriter.rewrite(val)
+        if isinstance(val, dict):
+            return {k: _recurse(v) for k, v in val.items()}
+        if isinstance(val, list):
+            return [_recurse(x) for x in val]
+        return val
+
+    return cast(dict, _recurse(help_info))
+
+
 class ReferenceGenerator:
-    def __init__(self, args: argparse.Namespace, version: str, help_info: Dict) -> None:
+    def __init__(self, args: argparse.Namespace, version: str, help_info: dict) -> None:
         self._args = args
         self._version = version
 
@@ -177,7 +221,7 @@ def _link(scope: str, *, sync: bool) -> str:
         return f"reference-{url_safe_scope}" if sync else f"{url_safe_scope}.md"
 
     @classmethod
-    def process_options_input(cls, help_info: Dict, *, sync: bool) -> Dict:
+    def process_options_input(cls, help_info: dict, *, sync: bool) -> Dict:
         scope_to_help_info = help_info["scope_to_help_info"]
 
         # Process the list of consumed_scopes into a comma-separated list, and add it to the option

build-support/bin/generate_docs_test.py

@@ -1,7 +1,7 @@
 # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-from generate_docs import html_safe, markdown_safe
+from generate_docs import html_safe, markdown_safe, value_strs_iter
 
 
 def test_markdown_safe():
@@ -10,3 +10,14 @@ def test_markdown_safe():
 
 def test_html_safe():
     assert "foo <code>bar==&#x27;baz&#x27;</code> qux" == html_safe("foo `bar=='baz'` qux")
+
+
+def test_gather_value_strs():
+    help_info = {
+        "a": "foo",
+        "b": ["bar", 5, "baz"],
+        "c": 42,
+        "d": True,
+        "e": {"f": 5, "g": "qux", "h": {"i": "quux"}},
+    }
+    assert set(value_strs_iter(help_info)) == {"foo", "bar", "baz", "qux", "quux"}

src/python/pants/backend/awslambda/python/rules.py

@@ -40,7 +40,7 @@
     targets_with_sources_types,
 )
 from pants.engine.unions import UnionMembership, UnionRule
-from pants.util.docutil import bracketed_docs_url
+from pants.util.docutil import doc_url
 from pants.util.logging import LogLevel
 
 logger = logging.getLogger(__name__)
@@ -119,7 +119,7 @@ async def package_python_awslambda(
             "files targets, but Pants will not include them in the built Lambda. Filesystem APIs "
             "like `open()` are not able to load files within the binary itself; instead, they "
             "read from the current working directory."
-            f"\n\nInstead, use `resources` targets. See {bracketed_docs_url('resources')}."
+            f"\n\nInstead, use `resources` targets. See {doc_url('resources')}."
             f"\n\nFiles targets dependencies: {files_addresses}"
         )
 

src/python/pants/backend/awslambda/python/target_types.py

@@ -30,7 +30,7 @@
 from pants.engine.unions import UnionRule
 from pants.source.filespec import Filespec
 from pants.source.source_root import SourceRoot, SourceRootRequest
-from pants.util.docutil import bracketed_docs_url
+from pants.util.docutil import doc_url
 
 
 class DeprecatedAwsLambdaInterpreterConstraints(InterpreterConstraintsField):
@@ -205,8 +205,8 @@ class PythonAWSLambda(Target):
         PythonAwsLambdaRuntime,
     )
     help = (
-        "A self-contained Python function suitable for uploading to AWS Lambda.\n\nSee "
-        f"{bracketed_docs_url('awslambda-python')}."
+        "A self-contained Python function suitable for uploading to AWS Lambda.\n\n"
+        f"See {doc_url('awslambda-python')}."
     )
 
 

src/python/pants/backend/codegen/protobuf/python/python_protobuf_subsystem.py

@@ -11,14 +11,12 @@
 from pants.engine.unions import UnionRule
 from pants.option.custom_types import target_option
 from pants.option.subsystem import Subsystem
-from pants.util.docutil import bracketed_docs_url
+from pants.util.docutil import doc_url
 
 
 class PythonProtobufSubsystem(Subsystem):
     options_scope = "python-protobuf"
-    help = (
-        f"Options related to the Protobuf Python backend.\n\nSee {bracketed_docs_url('protobuf')}."
-    )
+    help = f"Options related to the Protobuf Python backend.\n\nSee {doc_url('protobuf')}."
 
     @classmethod
     def register_options(cls, register):

src/python/pants/backend/codegen/protobuf/target_types.py

@@ -2,11 +2,12 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 from pants.engine.target import COMMON_TARGET_FIELDS, BoolField, Dependencies, Sources, Target
-from pants.util.docutil import bracketed_docs_url
-
 
 # NB: We subclass Dependencies so that specific backends can add dependency injection rules to
 # Protobuf targets.
+from pants.util.docutil import doc_url
+
+
 class ProtobufDependencies(Dependencies):
     pass
 
@@ -25,4 +26,4 @@ class ProtobufGrpcToggle(BoolField):
 class ProtobufLibrary(Target):
     alias = "protobuf_library"
     core_fields = (*COMMON_TARGET_FIELDS, ProtobufDependencies, ProtobufSources, ProtobufGrpcToggle)
-    help = f"Protobuf files used to generate various languages.\n\nSee {bracketed_docs_url('protobuf')}."
+    help = f"Protobuf files used to generate various languages.\n\nSee f{doc_url('protobuf')}."

src/python/pants/backend/python/goals/package_pex_binary.py

@@ -44,7 +44,7 @@
     targets_with_sources_types,
 )
 from pants.engine.unions import UnionMembership, UnionRule
-from pants.util.docutil import bracketed_docs_url
+from pants.util.docutil import doc_url
 from pants.util.logging import LogLevel
 
 logger = logging.getLogger(__name__)
@@ -120,8 +120,8 @@ async def package_pex_binary(
             "targets, but Pants will not include them in the PEX. Filesystem APIs like `open()` "
             "are not able to load files within the binary itself; instead, they read from the "
             "current working directory."
-            "\n\nInstead, use `resources` targets or wrap this `pex_binary` in an `archive`. See "
-            f"{bracketed_docs_url('resources')}."
+            "\n\nInstead, use `resources` targets or wrap this `pex_binary` in an `archive`. "
+            f"See {doc_url('resources')}."
             f"\n\nFiles targets dependencies: {files_addresses}"
         )
 

src/python/pants/backend/python/goals/setup_py.py

@@ -61,7 +61,7 @@
 from pants.engine.unions import UnionMembership, UnionRule, union
 from pants.option.subsystem import Subsystem
 from pants.python.python_setup import PythonSetup
-from pants.util.docutil import bracketed_docs_url
+from pants.util.docutil import doc_url
 from pants.util.logging import LogLevel
 from pants.util.memo import memoized_property
 from pants.util.meta import frozen_after_init
@@ -88,7 +88,7 @@ class OwnershipError(Exception):
 
     def __init__(self, msg: str):
         super().__init__(
-            f"{msg} See {bracketed_docs_url('python-distributions')} for "
+            f"{msg} See {doc_url('python-distributions')} for "
             f"how python_library targets are mapped to distributions."
         )
 

src/python/pants/backend/python/lint/pylint/subsystem.py

@@ -11,7 +11,7 @@
 from pants.core.util_rules.config_files import ConfigFilesRequest
 from pants.engine.addresses import UnparsedAddressInputs
 from pants.option.custom_types import file_option, shell_str, target_option
-from pants.util.docutil import bracketed_docs_url
+from pants.util.docutil import doc_url
 
 
 class Pylint(PythonToolBase):
@@ -74,7 +74,7 @@ def register_options(cls, register):
                 "example, if your plugin is at `build-support/pylint/custom_plugin.py`, add "
                 "'build-support/pylint' to `[source].root_patterns` in `pants.toml`. This is "
                 "necessary for Pants to know how to tell Pylint to discover your plugin. See "
-                f"{bracketed_docs_url('source-roots')}\n\nYou must also set `load-plugins=$module_name` in "
+                f"{doc_url('source-roots')}\n\nYou must also set `load-plugins=$module_name` in "
                 "your Pylint config file, and set the `[pylint].config` option in `pants.toml`."
                 "\n\nWhile your plugin's code can depend on other first-party code and third-party "
                 "requirements, all first-party dependencies of the plugin must live in the same "

src/python/pants/backend/python/target_types.py

@@ -46,7 +46,7 @@
 from pants.option.subsystem import Subsystem
 from pants.python.python_setup import PythonSetup
 from pants.source.filespec import Filespec
-from pants.util.docutil import bracketed_docs_url
+from pants.util.docutil import doc_url
 from pants.util.frozendict import FrozenDict
 
 logger = logging.getLogger(__name__)
@@ -69,9 +69,9 @@ class InterpreterConstraintsField(StringSequenceField):
         "`CPython` as a shorthand, e.g. '>=2.7' will be expanded to 'CPython>=2.7'.\n\nSpecify "
         "more than one element to OR the constraints, e.g. `['PyPy==3.7.*', 'CPython==3.7.*']` "
         "means either PyPy 3.7 _or_ CPython 3.7.\n\nIf the field is not set, it will default to "
-        "the option `[python-setup].interpreter_constraints`.\n\nSee "
-        f"{bracketed_docs_url('python-interpreter-compatibility')} for how these interpreter "
-        f"constraints are merged with the constraints of dependencies."
+        "the option `[python-setup].interpreter_constraints`.\n\n"
+        f"See {doc_url('python-interpreter-compatibility')} for how these interpreter "
+        "constraints are merged with the constraints of dependencies."
     )
 
     def value_or_global_default(self, python_setup: PythonSetup) -> Tuple[str, ...]:
@@ -390,7 +390,7 @@ class PexBinary(Target):
     help = (
         "A Python target that can be converted into an executable PEX file.\n\nPEX files are "
         "self-contained executable files that contain a complete Python environment capable of "
-        f"running the target. For more information, see {bracketed_docs_url('pex-files')}."
+        f"running the target. For more information, see {doc_url('pex-files')}."
     )
 
 
@@ -471,7 +471,7 @@ class PythonTests(Target):
     help = (
         "Python tests, written in either Pytest style or unittest style.\n\nAll test util code, "
         "other than `conftest.py`, should go into a dedicated `python_library()` target and then "
-        f"be included in the `dependencies` field.\n\nSee {bracketed_docs_url('python-test-goal')}."
+        f"be included in the `dependencies` field.\n\nSee {doc_url('python-test-goal')}."
     )
 
 
@@ -650,7 +650,7 @@ class PythonRequirementLibrary(Target):
         "Python requirements inline in a BUILD file. If you have a `requirements.txt` file "
         "already, you can instead use the macro `python_requirements()` to convert each "
         "requirement into a `python_requirement_library()` target automatically.\n\nSee "
-        f"{bracketed_docs_url('python-third-party-dependencies')}."
+        f"{doc_url('python-third-party-dependencies')}."
     )
 
 
@@ -710,8 +710,8 @@ class PythonProvidesField(ScalarField, ProvidesField):
         "`name`. You can also set almost any keyword argument accepted by setup.py in the "
         "`setup()` function: "
         "(https://packaging.python.org/guides/distributing-packages-using-setuptools/#setup-args)."
-        f"\n\nSee {bracketed_docs_url('plugins-setup-py')} for how to write a plugin to "
-        f"dynamically generate kwargs."
+        f"\n\nSee {doc_url('plugins-setup-py')} for how to write a plugin to "
+        "dynamically generate kwargs."
     )
 
     @classmethod
@@ -742,5 +742,5 @@ class PythonDistribution(Target):
     )
     help = (
         "A publishable Python setuptools distribution (e.g. an sdist or wheel).\n\nSee "
-        f"{bracketed_docs_url('python-distributions')}."
+        f"{doc_url('python-distributions')}."
     )

src/python/pants/backend/python/typecheck/mypy/rules.py

@@ -34,7 +34,7 @@
 from pants.engine.target import FieldSet, Target, TransitiveTargets, TransitiveTargetsRequest
 from pants.engine.unions import UnionRule
 from pants.python.python_setup import PythonSetup
-from pants.util.docutil import bracketed_docs_url
+from pants.util.docutil import doc_url
 from pants.util.logging import LogLevel
 from pants.util.ordered_set import FrozenOrderedSet, OrderedSet
 from pants.util.strutil import pluralize
@@ -99,7 +99,7 @@ def check_and_warn_if_python_version_configured(
         logger.warning(
             f"You set {formatted_configured}. Normally, Pants would automatically set this for you "
             "based on your code's interpreter constraints "
-            f"({bracketed_docs_url('python-interpreter-compatibility')}). Instead, it will "
+            f"({doc_url('python-interpreter-compatibility')}). Instead, it will "
             "use what you set.\n\n(Automatically setting the option allows Pants to partition your "
             "targets by their constraints, so that, for example, you can run MyPy on Python 2-only "
             "code and Python 3-only code at the same time. This feature may no longer work.)"

src/python/pants/core/goals/tailor.py

@@ -35,6 +35,7 @@
     UnexpandedTargets,
 )
 from pants.engine.unions import UnionMembership, union
+from pants.util.docutil import doc_url
 from pants.util.frozendict import FrozenDict
 from pants.util.logging import LogLevel
 from pants.util.memo import memoized
@@ -227,7 +228,7 @@ def register_options(cls, register):
             type=dict,
             help="A mapping from standard target type to custom type to use instead. The custom "
             "type can be a custom target type or a macro that offers compatible functionality "
-            "to the one it replaces (see https://www.pantsbuild.org/docs/macros).",
+            f"to the one it replaces (see {doc_url('macros')}).",
         )
 
     @property

src/python/pants/engine/internals/graph.py

@@ -76,7 +76,7 @@
 from pants.engine.unions import UnionMembership
 from pants.option.global_options import GlobalOptions, OwnersNotFoundBehavior
 from pants.source.filespec import matches_filespec
-from pants.util.docutil import bracketed_docs_url
+from pants.util.docutil import doc_url
 from pants.util.logging import LogLevel
 from pants.util.ordered_set import FrozenOrderedSet, OrderedSet
 
@@ -418,9 +418,9 @@ def _log_or_raise_unmatched_owners(
             f"target whose `sources` field includes the file."
         )
     msg = (
-        f"{prefix} See {bracketed_docs_url('targets')} for more information on target definitions."
+        f"{prefix} See {doc_url('targets')} for more information on target definitions."
         f"\n\nYou may want to run `./pants tailor` to autogenerate your BUILD files. See "
-        f"{bracketed_docs_url('create-initial-build-files')}.{option_msg}"
+        f"{doc_url('create-initial-build-files')}.{option_msg}"
     )
 
     if owners_not_found_behavior == OwnersNotFoundBehavior.warn:

src/python/pants/engine/internals/parser.py

@@ -12,7 +12,7 @@
 from pants.base.parse_context import ParseContext
 from pants.build_graph.build_file_aliases import BuildFileAliases
 from pants.engine.internals.target_adaptor import TargetAdaptor
-from pants.util.docutil import bracketed_docs_url
+from pants.util.docutil import doc_url
 from pants.util.frozendict import FrozenDict
 
 
@@ -105,7 +105,7 @@ def parse(
             original = e.args[0].capitalize()
             help_str = (
                 "If you expect to see more symbols activated in the below list,"
-                f" refer to {bracketed_docs_url('enabling-backends')} for all available"
+                f" refer to {doc_url('enabling-backends')} for all available"
                 " backends to activate."
             )
 
@@ -142,6 +142,6 @@ def error_on_imports(build_file_content: str, filepath: str) -> None:
         raise ParseError(
             f"Import used in {filepath} at line {lineno}. Import statements are banned in "
             "BUILD files because they can easily break Pants caching and lead to stale results. "
-            f"\n\nInstead, consider writing a macro ({bracketed_docs_url('macros')}) or "
-            f"writing a plugin ({bracketed_docs_url('plugins-overview')}."
+            f"\n\nInstead, consider writing a macro ({doc_url('macros')}) or "
+            f"writing a plugin ({doc_url('plugins-overview')}."
         )