Trac #33546: Use pytest to run doctests

As suggested in https://trac.sagemath.org/ticket/33232#comment:1,
`./sage --pytest` now uses pytest to discover and run doctests.

To test, run `./sage --pytest src/sage/manifolds/`. This results in
{{{
48 failed, 1416 passed, 7 warnings
}}}
With many of the failed tests being due to some issues with assumptions
and/or symbolic  variables and/or deprecation warnings.
To see examples of these failures, run pytest on
{{{
src/sage/manifolds/differentiable/examples/sphere.py
src/sage/manifolds/utilities.py
src/sage/manifolds/chart.py
}}}

We also add tests that verify that `sage --pytest` (and `sage -t`)
correctly complain about failing doctests.

Follow-ups:
- #33826 Fix these issues
- #33825 Use `pytest-parallel` or `pytest-xdist` to run pytest in
parallel
- #33827 Also run doctests defined in cython files using
https://github.com/lgpage/pytest-cython

URL: https://trac.sagemath.org/33546
Reported by: mkoeppe
Ticket author(s): Tobias Diez
Reviewer(s): Matthias Koeppe

.vscode/launch.json

@@ -1,6 +1,17 @@
 {
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": "Sage: Pytest",
+            "type": "python",
+            "request": "launch",
+            "module": "pytest",
+            "args": [
+                "${file}"
+            ],
+            "console": "integratedTerminal",
+            "justMyCode": false
+        },
         {
             "name": "Python: Current File",
             "type": "python",
@@ -22,4 +33,4 @@
             "justMyCode": false
         }
     ],
-}
+}

.vscode/settings.json

@@ -16,7 +16,9 @@
     },
     "python.testing.pytestEnabled": true,
     "python.testing.pytestArgs": [
-        "src"
+        "--rootdir=src/sage",
+        "-c=src/tox.ini",
+        "--doctest-modules",
     ],
     "python.testing.unittestEnabled": false,
 }

src/bin/sage

@@ -960,10 +960,10 @@ if [ "$1" = '-pytest' -o "$1" = '--pytest' ]; then
             for a in $*; do
                 case $a in
                     -*) ;;
-                    *)  exec pytest --rootdir="$SAGE_SRC" "$@"
+                    *)  exec pytest --rootdir="$SAGE_SRC" --doctest-modules "$@"
                 esac
             done
-            exec pytest --rootdir="$SAGE_SRC" "$@" "$SAGE_SRC"
+            exec pytest --rootdir="$SAGE_SRC" --doctest-modules "$@" "$SAGE_SRC"
         else
             echo "Run 'sage -i pytest' to install"
         fi

src/conftest.py

@@ -7,11 +7,114 @@
 """
 
 from __future__ import annotations
+
+import inspect
 from pathlib import Path
-from typing import Any
+from typing import Any, Iterable
+
 import pytest
+from _pytest.doctest import (
+    DoctestItem,
+    DoctestModule,
+    _get_continue_on_failure,
+    _get_runner,
+    _is_mocked,
+    _patch_unwrap_mock_aware,
+    get_optionflags,
+)
+from _pytest.pathlib import import_path, ImportMode
+
+# Import sage.all is necessary to:
+# - avoid cyclic import errors, see Trac #33580
+# - inject it into globals namespace for doctests
+import sage.all
+from sage.doctest.parsing import SageDocTestParser, SageOutputChecker
+
+
+class SageDoctestModule(DoctestModule):
+    """
+    This is essentially a copy of `DoctestModule` from
+    https://github.com/pytest-dev/pytest/blob/main/src/_pytest/doctest.py.
+    The only change is that we use `SageDocTestParser` to extract the doctests
+    and `SageOutputChecker` to verify the output.
+    """
+
+    def collect(self) -> Iterable[DoctestItem]:
+        import doctest
+
+        class MockAwareDocTestFinder(doctest.DocTestFinder):
+            """A hackish doctest finder that overrides stdlib internals to fix a stdlib bug.
+            https://github.com/pytest-dev/pytest/issues/3456
+            https://bugs.python.org/issue25532
+            """
+
+            def __init__(self) -> None:
+                super().__init__(parser=SageDocTestParser(set(["sage"])))
 
-import sage.all  # type: ignore  # to avoid cyclic import errors, see Trac #33580
+            def _find_lineno(self, obj, source_lines):
+                """Doctest code does not take into account `@property`, this
+                is a hackish way to fix it. https://bugs.python.org/issue17446
+                Wrapped Doctests will need to be unwrapped so the correct
+                line number is returned. This will be reported upstream. #8796
+                """
+                if isinstance(obj, property):
+                    obj = getattr(obj, "fget", obj)
+
+                if hasattr(obj, "__wrapped__"):
+                    # Get the main obj in case of it being wrapped
+                    obj = inspect.unwrap(obj)
+
+                # Type ignored because this is a private function.
+                return super()._find_lineno(  # type:ignore[misc]
+                    obj,
+                    source_lines,
+                )
+
+            def _find(
+                self, tests, obj, name, module, source_lines, globs, seen
+            ) -> None:
+                if _is_mocked(obj):
+                    return
+                with _patch_unwrap_mock_aware():
+
+                    # Type ignored because this is a private function.
+                    super()._find(  # type:ignore[misc]
+                        tests, obj, name, module, source_lines, globs, seen
+                    )
+
+        if self.path.name == "conftest.py":
+            module = self.config.pluginmanager._importconftest(
+                self.path,
+                self.config.getoption("importmode"),
+                rootpath=self.config.rootpath,
+            )
+        else:
+            try:
+                module = import_path(
+                    self.path,
+                    mode=ImportMode.importlib,
+                    root=self.config.rootpath,
+                )
+            except ImportError:
+                if self.config.getvalue("doctest_ignore_import_errors"):
+                    pytest.skip("unable to import module %r" % self.path)
+                else:
+                    raise
+        # Uses internal doctest module parsing mechanism.
+        finder = MockAwareDocTestFinder()
+        optionflags = get_optionflags(self)
+        runner = _get_runner(
+            verbose=False,
+            optionflags=optionflags,
+            checker=SageOutputChecker(),
+            continue_on_failure=_get_continue_on_failure(self.config),
+        )
+
+        for test in finder.find(module, module.__name__):
+            if test.examples:  # skip empty doctests
+                yield DoctestItem.from_parent(
+                    self, name=test.name, runner=runner, dtest=test
+                )
 
 
 def pytest_collect_file(
@@ -30,6 +133,9 @@ def pytest_collect_file(
         # Normally, Cython files are filtered out already by pytest and we only
         # hit this here if someone explicitly runs `pytest some_file.pyx`.
         return pytest.skip("Skipping Cython file")
+    elif file_path.suffix == ".py":
+        if parent.config.option.doctestmodules:
+            return SageDoctestModule.from_parent(parent, path=file_path)
 
 
 @pytest.fixture(autouse=True)
@@ -39,5 +145,14 @@ def add_imports(doctest_namespace: dict[str, Any]):
 
     See `pytest documentation <https://docs.pytest.org/en/stable/doctest.html#doctest-namespace-fixture>`.
     """
-    import sage.all  # type: ignore # implicitly used below by calling locals()
-    doctest_namespace.update(**locals())
+    # Inject sage.all into each doctest
+    dict_all = sage.all.__dict__
+
+    # Remove '__package__' item from the globals since it is not
+    # always in the globals in an actual Sage session.
+    dict_all.pop("__package__", None)
+
+    sage_namespace = dict(dict_all)
+    sage_namespace["__name__"] = "__main__"
+
+    doctest_namespace.update(**sage_namespace)

src/conftest_inputtest.py

@@ -0,0 +1,16 @@
+def something():
+    """ a doctest in a docstring
+    
+    EXAMPLES::
+
+        sage: something()
+        42
+        sage: something() + 1
+        43
+
+    TESTS::
+
+        sage: something()
+        44
+    """
+    return 42

src/conftest_test.py

@@ -0,0 +1,51 @@
+import subprocess
+
+
+class TestOldDoctestSageScript:
+    """Run `sage --t`."""
+
+    def test_invoke_on_inputtest_file(self):
+        result = subprocess.run(
+            ["sage", "-t", "./src/conftest_inputtest.py"],
+            capture_output=True,
+            text=True,
+        )
+        assert result.returncode == 1  # There are failures in the input test
+        assert (
+            "Failed example:\n"
+            "    something()\n"
+            "Expected:\n"
+            "    44\n"
+            "Got:\n"
+            "    42\n"
+            "" in result.stdout
+        )
+
+
+class TestPytestSageScript:
+    """Run `sage --pytest`."""
+
+    def test_invoke_on_inputtest_file(self):
+        result = subprocess.run(
+            ["sage", "--pytest", "./src/conftest_inputtest.py"],
+            capture_output=True,
+            text=True,
+        )
+        assert result.returncode == 1  # There are failures in the input test
+        assert (
+            "004     EXAMPLES::\n"
+            "005 \n"
+            "006         sage: something()\n"
+            "007         42\n"
+            "008         sage: something() + 1\n"
+            "009         43\n"
+            "010 \n"
+            "011     TESTS::\n"
+            "012 \n"
+            "013         sage: something()\n"
+            "Expected:\n"
+            "    44\n"
+            "Got:\n"
+            "    42\n"
+            "" in result.stdout
+        )

src/tox.ini

@@ -173,7 +173,9 @@ commands = codespell \
 
 [pytest]
 python_files = *_test.py
+norecursedirs = local prefix venv build pkgs .git src/doc src/bin
 addopts = --import-mode importlib
+doctest_optionflags = NORMALIZE_WHITESPACE ELLIPSIS
 
 [coverage:run]
 source = sage