diff --git a/doc/user_guide/usage/run.rst b/doc/user_guide/usage/run.rst index 5cb02e557a..081517a39b 100644 --- a/doc/user_guide/usage/run.rst +++ b/doc/user_guide/usage/run.rst @@ -160,11 +160,6 @@ This will spawn 4 parallel Pylint sub-process, where each provided module will be checked in parallel. Discovered problems by checkers are not displayed immediately. They are shown just after checking a module is complete. -There is one known limitation with running checks in parallel as currently -implemented. Since the division of files into worker processes is indeterminate, -checkers that depend on comparing multiple files (e.g. ``cyclic-import`` -and ``duplicate-code``) can produce indeterminate results. - Exit codes ---------- diff --git a/doc/whatsnew/fragments/4171.bugfix b/doc/whatsnew/fragments/4171.bugfix new file mode 100644 index 0000000000..6e9500bf5b --- /dev/null +++ b/doc/whatsnew/fragments/4171.bugfix @@ -0,0 +1,3 @@ +Support ``cyclic-import`` message when parallelizing with ``--jobs``. + +Closes #4171 diff --git a/pylint/checkers/imports.py b/pylint/checkers/imports.py index 42649f3d97..9935de895e 100644 --- a/pylint/checkers/imports.py +++ b/pylint/checkers/imports.py @@ -455,6 +455,7 @@ def __init__(self, linter: PyLinter) -> None: ("RP0401", "External dependencies", self._report_external_dependencies), ("RP0402", "Modules dependencies graph", self._report_dependencies_graph), ) + self._excluded_edges: defaultdict[str, set[str]] = defaultdict(set) def open(self) -> None: """Called before visiting project (i.e set of modules).""" @@ -463,7 +464,6 @@ def open(self) -> None: self.import_graph = defaultdict(set) self._module_pkg = {} # mapping of modules to the pkg they belong in self._current_module_package = False - self._excluded_edges: defaultdict[str, set[str]] = defaultdict(set) self._ignored_modules: Sequence[str] = self.linter.config.ignored_modules # Build a mapping {'module': 'preferred-module'} self.preferred_modules = dict( @@ -488,6 +488,17 @@ def close(self) -> None: for cycle in get_cycles(graph, vertices=vertices): self.add_message("cyclic-import", args=" -> ".join(cycle)) + def get_map_data(self) -> defaultdict[str, set[str]]: + return self.import_graph + + def reduce_map_data( + self, linter: PyLinter, data: list[defaultdict[str, set[str]]] + ) -> None: + self.import_graph = defaultdict(set) + for graph in data: + self.import_graph.update(graph) + self.close() + def deprecated_modules(self) -> set[str]: """Callback returning the deprecated modules.""" # First get the modules the user indicated diff --git a/tests/test_check_parallel.py b/tests/test_check_parallel.py index d56502eaf7..eadfaea6f6 100644 --- a/tests/test_check_parallel.py +++ b/tests/test_check_parallel.py @@ -13,6 +13,7 @@ import sys from concurrent.futures import ProcessPoolExecutor from concurrent.futures.process import BrokenProcessPool +from pathlib import Path from pickle import PickleError from typing import TYPE_CHECKING from unittest.mock import patch @@ -23,11 +24,13 @@ import pylint.interfaces import pylint.lint.parallel from pylint.checkers import BaseRawFileChecker +from pylint.checkers.imports import ImportsChecker from pylint.lint import PyLinter, augmented_sys_path from pylint.lint.parallel import _worker_check_single_file as worker_check_single_file from pylint.lint.parallel import _worker_initialize as worker_initialize from pylint.lint.parallel import check_parallel from pylint.testutils import GenericTestReporter as Reporter +from pylint.testutils.utils import _test_cwd from pylint.typing import FileItem from pylint.utils import LinterStats, ModuleStats @@ -625,3 +628,30 @@ def test_no_deadlock_due_to_initializer_error(self) -> None: # because arguments has to be an Iterable. extra_packages_paths=1, # type: ignore[arg-type] ) + + @pytest.mark.needs_two_cores + def test_cyclic_import_parallel(self) -> None: + tests_dir = Path("tests") + package_path = Path("input") / "func_w0401_package" + linter = PyLinter(reporter=Reporter()) + linter.register_checker(ImportsChecker(linter)) + + with _test_cwd(tests_dir): + check_parallel( + linter, + jobs=2, + files=[ + FileItem( + name="input.func_w0401_package.all_the_things", + filepath=str(package_path / "all_the_things.py"), + modpath="input.func_w0401_package", + ), + FileItem( + name="input.func_w0401_package.thing2", + filepath=str(package_path / "thing2.py"), + modpath="input.func_w0401_package", + ), + ], + ) + + assert "cyclic-import" in linter.stats.by_msg