Merge pull request #8021 from keewis/fix-type-preprocessor

Fix type preprocessor

sphinx/ext/napoleon/docstring.py

@@ -37,18 +37,23 @@
     r'((?::(?:[a-zA-Z0-9]+[\-_+:.])*[a-zA-Z0-9]+:`.+?`)|'
     r'(?:``.+``))')
 _xref_regex = re.compile(
-    r'(?::(?:[a-zA-Z0-9]+[\-_+:.])*[a-zA-Z0-9]+:`.+?`)'
+    r'(?:(?::(?:[a-zA-Z0-9]+[\-_+:.])*[a-zA-Z0-9]+:)?`.+?`)'
 )
 _bullet_list_regex = re.compile(r'^(\*|\+|\-)(\s+\S|\s*$)')
 _enumerated_list_regex = re.compile(
     r'^(?P<paren>\()?'
     r'(\d+|#|[ivxlcdm]+|[IVXLCDM]+|[a-zA-Z])'
     r'(?(paren)\)|\.)(\s+\S|\s*$)')
 _token_regex = re.compile(
-    r"(\sor\s|\sof\s|:\s|,\s|[{]|[}]"
+    r"(,\sor\s|\sor\s|\sof\s|:\s|\sto\s|,\sand\s|\sand\s|,\s"
+    r"|[{]|[}]"
     r'|"(?:\\"|[^"])*"'
     r"|'(?:\\'|[^'])*')"
 )
+_default_regex = re.compile(
+    r"^default[^_0-9A-Za-z].*$",
+)
+_SINGLETONS = ("None", "True", "False", "Ellipsis")
 
 
 class GoogleDocstring:
@@ -808,6 +813,9 @@ def takewhile_set(tokens):
                 previous_token = token
                 continue
 
+            if not token.strip():
+                continue
+
             if token in keywords:
                 tokens.appendleft(token)
                 if previous_token is not None:
@@ -846,8 +854,13 @@ def combine_set(tokens):
 
 def _tokenize_type_spec(spec: str) -> List[str]:
     def postprocess(item):
-        if item.startswith("default"):
-            return [item[:7], item[7:]]
+        if _default_regex.match(item):
+            default = item[:7]
+            # can't be separated by anything other than a single space
+            # for now
+            other = item[8:]
+
+            return [default, " ", other]
         else:
             return [item]
 
@@ -861,10 +874,19 @@ def postprocess(item):
 
 
 def _token_type(token: str, location: str = None) -> str:
+    def is_numeric(token):
+        try:
+            # use complex to make sure every numeric value is detected as literal
+            complex(token)
+        except ValueError:
+            return False
+        else:
+            return True
+
     if token.startswith(" ") or token.endswith(" "):
         type_ = "delimiter"
     elif (
-            token.isnumeric() or
+            is_numeric(token) or
             (token.startswith("{") and token.endswith("}")) or
             (token.startswith('"') and token.endswith('"')) or
             (token.startswith("'") and token.endswith("'"))
@@ -914,9 +936,12 @@ def _convert_numpy_type_spec(_type: str, location: str = None, translations: dic
     def convert_obj(obj, translations, default_translation):
         translation = translations.get(obj, obj)
 
-        # use :class: (the default) only if obj is not a standard singleton (None, True, False)
-        if translation in ("None", "True", "False") and default_translation == ":class:`%s`":
+        # use :class: (the default) only if obj is not a standard singleton
+        if translation in _SINGLETONS and default_translation == ":class:`%s`":
             default_translation = ":obj:`%s`"
+        elif translation == "..." and default_translation == ":class:`%s`":
+            # allow referencing the builtin ...
+            default_translation = ":obj:`%s <Ellipsis>`"
 
         if _xref_regex.match(translation) is None:
             translation = default_translation % translation

tests/test_ext_napoleon_docstring.py

@@ -1078,6 +1078,22 @@ def test_noindex(self):
                                      options={'noindex': True}))
         self.assertEqual(expected, actual)
 
+    def test_keywords_with_types(self):
+        docstring = """\
+Do as you please
+
+Keyword Args:
+    gotham_is_yours (None): shall interfere.
+"""
+        actual = str(GoogleDocstring(docstring))
+        expected = """\
+Do as you please
+
+:keyword gotham_is_yours: shall interfere.
+:kwtype gotham_is_yours: None
+"""
+        self.assertEqual(expected, actual)
+
 
 class NumpyDocstringTest(BaseDocstringTest):
     docstrings = [(
@@ -1987,6 +2003,8 @@ def test_list_in_parameter_description(self):
     def test_token_type(self):
         tokens = (
             ("1", "literal"),
+            ("-4.6", "literal"),
+            ("2j", "literal"),
             ("'string'", "literal"),
             ('"another_string"', "literal"),
             ("{1, 2}", "literal"),
@@ -2010,20 +2028,32 @@ def test_token_type(self):
     def test_tokenize_type_spec(self):
         specs = (
             "str",
+            "defaultdict",
+            "int, float, or complex",
             "int or float or None, optional",
             '{"F", "C", "N"}',
             "{'F', 'C', 'N'}, default: 'F'",
             "{'F', 'C', 'N or C'}, default 'F'",
+            "str, default: 'F or C'",
+            "int, default: None",
+            "int, default None",
+            "int, default :obj:`None`",
             '"ma{icious"',
             r"'with \'quotes\''",
         )
 
         tokens = (
             ["str"],
+            ["defaultdict"],
+            ["int", ", ", "float", ", or ", "complex"],
             ["int", " or ", "float", " or ", "None", ", ", "optional"],
             ["{", '"F"', ", ", '"C"', ", ", '"N"', "}"],
             ["{", "'F'", ", ", "'C'", ", ", "'N'", "}", ", ", "default", ": ", "'F'"],
             ["{", "'F'", ", ", "'C'", ", ", "'N or C'", "}", ", ", "default", " ", "'F'"],
+            ["str", ", ", "default", ": ", "'F or C'"],
+            ["int", ", ", "default", ": ", "None"],
+            ["int", ", " , "default", " ", "None"],
+            ["int", ", ", "default", " ", ":obj:`None`"],
             ['"ma{icious"'],
             [r"'with \'quotes\''"],
         )
@@ -2037,12 +2067,14 @@ def test_recombine_set_tokens(self):
             ["{", "1", ", ", "2", "}"],
             ["{", '"F"', ", ", '"C"', ", ", '"N"', "}", ", ", "optional"],
             ["{", "'F'", ", ", "'C'", ", ", "'N'", "}", ", ", "default", ": ", "None"],
+            ["{", "'F'", ", ", "'C'", ", ", "'N'", "}", ", ", "default", " ", "None"],
         )
 
         combined_tokens = (
             ["{1, 2}"],
             ['{"F", "C", "N"}', ", ", "optional"],
             ["{'F', 'C', 'N'}", ", ", "default", ": ", "None"],
+            ["{'F', 'C', 'N'}", ", ", "default", " ", "None"],
         )
 
         for tokens_, expected in zip(tokens, combined_tokens):
@@ -2075,8 +2107,10 @@ def test_convert_numpy_type_spec(self):
             "optional",
             "str, optional",
             "int or float or None, default: None",
+            "int, default None",
             '{"F", "C", "N"}',
             "{'F', 'C', 'N'}, default: 'N'",
+            "{'F', 'C', 'N'}, default 'N'",
             "DataFrame, optional",
         )
 
@@ -2085,8 +2119,10 @@ def test_convert_numpy_type_spec(self):
             "*optional*",
             ":class:`str`, *optional*",
             ":class:`int` or :class:`float` or :obj:`None`, *default*: :obj:`None`",
+            ":class:`int`, *default* :obj:`None`",
             '``{"F", "C", "N"}``',
             "``{'F', 'C', 'N'}``, *default*: ``'N'``",
+            "``{'F', 'C', 'N'}``, *default* ``'N'``",
             ":class:`pandas.DataFrame`, *optional*",
         )
 
@@ -2100,29 +2136,43 @@ def test_parameter_types(self):
             ----------
             param1 : DataFrame
                 the data to work on
-            param2 : int or float or None
+            param2 : int or float or None, optional
                 a parameter with different types
             param3 : dict-like, optional
                 a optional mapping
             param4 : int or float or None, optional
                 a optional parameter with different types
             param5 : {"F", "C", "N"}, optional
                 a optional parameter with fixed values
+            param6 : int, default None
+                different default format
+            param7 : mapping of hashable to str, optional
+                a optional mapping
+            param8 : ... or Ellipsis
+                ellipsis
         """)
         expected = dedent("""\
             :param param1: the data to work on
             :type param1: DataFrame
             :param param2: a parameter with different types
-            :type param2: :class:`int` or :class:`float` or :obj:`None`
+            :type param2: :class:`int` or :class:`float` or :obj:`None`, *optional*
             :param param3: a optional mapping
             :type param3: :term:`dict-like <mapping>`, *optional*
             :param param4: a optional parameter with different types
             :type param4: :class:`int` or :class:`float` or :obj:`None`, *optional*
             :param param5: a optional parameter with fixed values
             :type param5: ``{"F", "C", "N"}``, *optional*
+            :param param6: different default format
+            :type param6: :class:`int`, *default* :obj:`None`
+            :param param7: a optional mapping
+            :type param7: :term:`mapping` of :term:`hashable` to :class:`str`, *optional*
+            :param param8: ellipsis
+            :type param8: :obj:`... <Ellipsis>` or :obj:`Ellipsis`
         """)
         translations = {
             "dict-like": ":term:`dict-like <mapping>`",
+            "mapping": ":term:`mapping`",
+            "hashable": ":term:`hashable`",
         }
         config = Config(
             napoleon_use_param=True,
@@ -2132,22 +2182,6 @@ def test_parameter_types(self):
         actual = str(NumpyDocstring(docstring, config))
         self.assertEqual(expected, actual)
 
-    def test_keywords_with_types(self):
-        docstring = """\
-Do as you please
-
-Keyword Args:
-    gotham_is_yours (None): shall interfere.
-"""
-        actual = str(GoogleDocstring(docstring))
-        expected = """\
-Do as you please
-
-:keyword gotham_is_yours: shall interfere.
-:kwtype gotham_is_yours: None
-"""
-        self.assertEqual(expected, actual)
-
 @contextmanager
 def warns(warning, match):
     match_re = re.compile(match)