⬆️ Drop Sphinx 3, add Sphinx 5 (#579)

.github/workflows/tests.yml

@@ -26,16 +26,15 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.7", "3.8", "3.9", "3.10"]
-        sphinx: [">=4,<5"]
+        sphinx: [">=5,<6"]
         os: [ubuntu-latest]
         include:
-          # fails because of: https://github.com/sphinx-doc/sphinx/issues/10291
-          # - os: ubuntu-latest
-          #   python-version: "3.8"
-          #   sphinx: ">=3,<4"
-          - os: windows-latest
-            python-version: "3.8"
-            sphinx: ">=4,<5"
+        - os: ubuntu-latest
+          python-version: "3.8"
+          sphinx: ">=4,<5"
+        - os: windows-latest
+          python-version: "3.8"
+          sphinx: ">=4,<5"
 
     runs-on: ${{ matrix.os }}
 

.pre-commit-config.yaml

@@ -50,7 +50,7 @@ repos:
     - id: mypy
       args: [--config-file=pyproject.toml]
       additional_dependencies:
-      - sphinx~=4.1
+      - sphinx~=5.0
       - markdown-it-py>=1.0.0,<3.0.0
       - mdit-py-plugins~=0.3.0
       files: >

myst_parser/_compat.py

@@ -0,0 +1,11 @@
+"""Helpers for cross compatibility across dependency versions."""
+from typing import Callable, Iterable
+
+from docutils.nodes import Element
+
+
+def findall(node: Element) -> Callable[..., Iterable[Element]]:
+    """Iterate through"""
+    # findall replaces traverse in docutils v0.18
+    # note a difference is that findall is an iterator
+    return getattr(node, "findall", node.traverse)

myst_parser/mdit_to_docutils/base.py

@@ -32,6 +32,7 @@
 from markdown_it.token import Token
 from markdown_it.tree import SyntaxTreeNode
 
+from myst_parser._compat import findall
 from myst_parser.config.main import MdParserConfig
 from myst_parser.mocking import (
     MockIncludeDirective,
@@ -41,7 +42,7 @@
     MockState,
     MockStateMachine,
 )
-from ..parsers.directives import DirectiveParsingError, parse_directive_text
+from myst_parser.parsers.directives import DirectiveParsingError, parse_directive_text
 from .html_to_nodes import html_to_nodes
 from .utils import is_external_url
 
@@ -260,7 +261,7 @@ def _render_finalise(self) -> None:
         # those from the initial markdown parse
         # instead we gather them from a walk of the created document
         foot_refs = OrderedDict()
-        for refnode in self.document.traverse(nodes.footnote_reference):
+        for refnode in findall(self.document)(nodes.footnote_reference):
             if refnode["refname"] not in foot_refs:
                 foot_refs[refnode["refname"]] = True
 
@@ -447,7 +448,7 @@ def render_inline(self, token: SyntaxTreeNode) -> None:
         self.render_children(token)
 
     def render_text(self, token: SyntaxTreeNode) -> None:
-        self.current_node.append(nodes.Text(token.content, token.content))
+        self.current_node.append(nodes.Text(token.content))
 
     def render_bullet_list(self, token: SyntaxTreeNode) -> None:
         list_node = nodes.bullet_list()
@@ -888,7 +889,7 @@ def dict_to_fm_field_list(
 
             field_node = nodes.field()
             field_node.source = value
-            field_node += nodes.field_name(key, "", nodes.Text(key, key))
+            field_node += nodes.field_name(key, "", nodes.Text(key))
             field_node += nodes.field_body(value, *[body])
             field_list += field_node
 

myst_parser/sphinx_ext/myst_refs.py

@@ -16,6 +16,8 @@
 from sphinx.util import docname_join, logging
 from sphinx.util.nodes import clean_astext, make_refnode
 
+from myst_parser._compat import findall
+
 try:
     from sphinx.errors import NoUri
 except ImportError:
@@ -35,7 +37,7 @@ class MystReferenceResolver(ReferencesResolver):
 
     def run(self, **kwargs: Any) -> None:
         self.document: document
-        for node in self.document.traverse(addnodes.pending_xref):
+        for node in findall(self.document)(addnodes.pending_xref):
             if node["reftype"] != "myst":
                 continue
 

pyproject.toml

@@ -34,12 +34,12 @@ keywords = [
 ]
 requires-python = ">=3.7"
 dependencies = [
-    "docutils>=0.15,<0.18",
+    "docutils>=0.15,<0.19",
     "jinja2", # required for substitutions, but let sphinx choose version
     "markdown-it-py>=1.0.0,<3.0.0",
     "mdit-py-plugins~=0.3.0",
     "pyyaml",
-    "sphinx>=3.1,<5",
+    "sphinx>=4,<6",
     "typing-extensions",
 ]
 
@@ -63,7 +63,6 @@ rtd = [
 testing = [
     "beautifulsoup4",
     "coverage[toml]",
-    "docutils~=0.17.0", # this version changes some HTML tags
     "pytest>=6,<7",
     "pytest-cov",
     "pytest-regressions",

tests/test_renderers/fixtures/sphinx_directives.md

@@ -14,7 +14,7 @@ default-domain (`sphinx.directives.DefaultDomain`):
 <document source="<src>/index.md">
 .
 
-SPHINX4 object (`sphinx.directives.ObjectDescription`):
+object (`sphinx.directives.ObjectDescription`):
 .
 ```{object} something
 ```
@@ -161,7 +161,7 @@ acks (`sphinx.directives.other.Acks`):
                     name
 .
 
-SPHINX4 hlist (`sphinx.directives.other.HList`):
+hlist (`sphinx.directives.other.HList`):
 .
 ```{hlist}
 
@@ -386,18 +386,18 @@ term 2 : B
                         Definition of both terms.
 .
 
-SPHINX3 productionlist (`sphinx.domains.std.ProductionList`):
+SPHINX4-SKIP productionlist (`sphinx.domains.std.ProductionList`):
 .
 ```{productionlist} try_stmt: try1_stmt | try2_stmt
 ```
 .
 <document source="<src>/index.md">
     <productionlist>
-        <production ids="grammar-token-try_stmt grammar-token-try-stmt" tokenname="try_stmt" xml:space="preserve">
+        <production ids="grammar-token-try_stmt" tokenname="try_stmt" xml:space="preserve">
              try1_stmt | try2_stmt
 .
 
-SPHINX4 cmdoption (`sphinx.domains.std.Cmdoption`):
+cmdoption (`sphinx.domains.std.Cmdoption`):
 .
 ```{cmdoption} a
 ```
@@ -412,7 +412,7 @@ SPHINX4 cmdoption (`sphinx.domains.std.Cmdoption`):
         <desc_content>
 .
 
-SPHINX4 rst:directive (`sphinx.domains.rst.ReSTDirective`):
+rst:directive (`sphinx.domains.rst.ReSTDirective`):
 .
 ```{rst:directive} a
 ```
@@ -426,15 +426,15 @@ SPHINX4 rst:directive (`sphinx.domains.rst.ReSTDirective`):
         <desc_content>
 .
 
-SPHINX4 rst:directive:option (`sphinx.domains.rst.ReSTDirectiveOption`):
+SPHINX4-SKIP rst:directive:option (`sphinx.domains.rst.ReSTDirectiveOption`):
 .
 ```{rst:directive:option} a
 ```
 .
 <document source="<src>/index.md">
     <index entries="('single',\ ':a:\ (directive\ option)',\ 'directive-option-a',\ '',\ 'A')">
     <desc classes="rst directive:option" desctype="directive:option" domain="rst" noindex="False" objtype="directive:option">
-        <desc_signature classes="sig sig-object" ids="directive-option-a directive:option--a">
+        <desc_signature classes="sig sig-object" ids="directive-option-a">
             <desc_name classes="sig-name descname" xml:space="preserve">
                 :a:
         <desc_content>

tests/test_renderers/test_fixtures_sphinx.py

@@ -9,7 +9,6 @@
 from pathlib import Path
 
 import pytest
-import sphinx
 from sphinx_pytest.plugin import CreateDoctree
 
 from myst_parser.mdit_to_docutils.sphinx_ import SphinxRenderer
@@ -42,11 +41,9 @@ def test_directive_options(file_params, sphinx_doctree_no_tr: CreateDoctree):
 def test_sphinx_directives(file_params, sphinx_doctree_no_tr: CreateDoctree):
     # TODO fix skipped directives
     # TODO test domain directives
-    if file_params.title.startswith("SKIP"):
-        pytest.skip(file_params.title)
-    elif file_params.title.startswith("SPHINX3") and sphinx.version_info[0] < 3:
-        pytest.skip(file_params.title)
-    elif file_params.title.startswith("SPHINX4") and sphinx.version_info[0] < 4:
+    if file_params.title.startswith("SKIP") or file_params.title.startswith(
+        "SPHINX4-SKIP"
+    ):
         pytest.skip(file_params.title)
 
     sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]})
@@ -63,8 +60,6 @@ def test_sphinx_directives(file_params, sphinx_doctree_no_tr: CreateDoctree):
 def test_sphinx_roles(file_params, sphinx_doctree_no_tr: CreateDoctree):
     if file_params.title.startswith("SKIP"):
         pytest.skip(file_params.title)
-    elif file_params.title.startswith("SPHINX4") and sphinx.version_info[0] < 4:
-        pytest.skip(file_params.title)
 
     sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]})
     pformat = sphinx_doctree_no_tr(file_params.content, "index.md").pformat("index")

tests/test_sphinx/conftest.py

@@ -39,6 +39,8 @@ def test_basic(app, status, warning, get_sphinx_app_output):
 from bs4 import BeautifulSoup
 from sphinx.testing.path import path
 
+from myst_parser._compat import findall
+
 SOURCE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "sourcedirs"))
 
 
@@ -109,7 +111,7 @@ def read(
             extension = regress_ext
 
         # convert absolute filenames
-        for node in doctree.traverse(
+        for node in findall(doctree)(
             lambda n: "source" in n and not isinstance(n, str)
         ):
             node["source"] = pathlib.Path(node["source"]).name

tests/test_sphinx/test_sphinx_builds.py

@@ -37,19 +37,19 @@ def test_basic(
     warnings = warning.getvalue().strip()
     assert warnings == ""
 
-    get_sphinx_app_doctree(
-        app,
-        docname="content",
-        regress=True,
-        regress_ext=f".sphinx{sphinx.version_info[0]}.xml",
-    )
-    get_sphinx_app_doctree(
-        app,
-        docname="content",
-        resolve=True,
-        regress=True,
-        regress_ext=f".sphinx{sphinx.version_info[0]}.xml",
-    )
+    try:
+        get_sphinx_app_doctree(
+            app,
+            docname="content",
+            regress=True,
+        )
+    finally:
+        get_sphinx_app_doctree(
+            app,
+            docname="content",
+            resolve=True,
+            regress=True,
+        )
     get_sphinx_app_output(
         app,
         filename="content.html",
@@ -104,7 +104,7 @@ def test_references(
                 app,
                 filename="index.html",
                 regress_html=True,
-                regress_ext=f".sphinx{sphinx.version_info[0]}.html",
+                replace={"Permalink to this headline": "Permalink to this heading"},
             )
 
 
@@ -154,7 +154,7 @@ def test_references_singlehtml(
         filename="index.html",
         buildername="singlehtml",
         regress_html=True,
-        regress_ext=f".sphinx{sphinx.version_info[0]}.html",
+        replace={"Permalink to this headline": "Permalink to this heading"},
     )
 
 
@@ -185,7 +185,7 @@ def test_heading_slug_func(
         app,
         filename="index.html",
         regress_html=True,
-        regress_ext=f".sphinx{sphinx.version_info[0]}.html",
+        replace={"Permalink to this headline": "Permalink to this heading"},
     )
 
 
@@ -216,14 +216,13 @@ def test_extended_syntaxes(
             app,
             docname="index",
             regress=True,
-            regress_ext=f".sphinx{sphinx.version_info[0]}.xml",
         )
     finally:
         get_sphinx_app_output(
             app,
             filename="index.html",
             regress_html=True,
-            regress_ext=f".sphinx{sphinx.version_info[0]}.html",
+            replace={"Permalink to this headline": "Permalink to this heading"},
         )
 
 
@@ -249,7 +248,6 @@ def test_includes(
             app,
             docname="index",
             regress=True,
-            regress_ext=f".sphinx{sphinx.version_info[0]}.xml",
             # fix for Windows CI
             replace={
                 r"subfolder\example2.jpg": "subfolder/example2.jpg",
@@ -262,8 +260,8 @@ def test_includes(
             app,
             filename="index.html",
             regress_html=True,
-            regress_ext=f".sphinx{sphinx.version_info[0]}.html",
             replace={
+                "Permalink to this headline": "Permalink to this heading",
                 r"'subfolder\\example2'": "'subfolder/example2'",
                 r'uri="subfolder\\example2"': 'uri="subfolder/example2"',
                 "_images/example21.jpg": "_images/example2.jpg",
@@ -354,7 +352,7 @@ def test_commonmark_only(
             app,
             filename="index.html",
             regress_html=True,
-            regress_ext=f".sphinx{sphinx.version_info[0]}.html",
+            replace={"Permalink to this headline": "Permalink to this heading"},
         )
 
 
@@ -407,7 +405,7 @@ def test_gettext(
     output = re.sub(r"POT-Creation-Date: [0-9: +-]+", "POT-Creation-Date: ", output)
     output = re.sub(r"Copyright \(C\) [0-9]{4}", "Copyright (C) XXXX", output)
 
-    file_regression.check(output, extension=f".sphinx{sphinx.version_info[0]}.pot")
+    file_regression.check(output, extension=".pot")
 
 
 @pytest.mark.sphinx(
@@ -434,15 +432,13 @@ def test_gettext_html(
             app,
             docname="index",
             regress=True,
-            regress_ext=f".sphinx{sphinx.version_info[0]}.xml",
         )
     finally:
         get_sphinx_app_doctree(
             app,
             docname="index",
             resolve=True,
             regress=True,
-            regress_ext=f".sphinx{sphinx.version_info[0]}.xml",
         )
     get_sphinx_app_output(
         app,
@@ -483,7 +479,7 @@ def test_gettext_additional_targets(
     output = re.sub(r"POT-Creation-Date: [0-9: +-]+", "POT-Creation-Date: ", output)
     output = re.sub(r"Copyright \(C\) [0-9]{4}", "Copyright (C) XXXX", output)
 
-    file_regression.check(output, extension=f".sphinx{sphinx.version_info[0]}.pot")
+    file_regression.check(output, extension=".pot")
 
 
 @pytest.mark.sphinx(
@@ -530,7 +526,6 @@ def test_fieldlist_extension(
             app,
             docname="index",
             regress=True,
-            regress_ext=f".sphinx{sphinx.version_info[0]}.xml",
             # changed in:
             # https://www.sphinx-doc.org/en/master/changes.html#release-4-4-0-released-jan-17-2022
             replace={