From 7be05066eb9c9a678bbeaaaf0bb313cf1768fc21 Mon Sep 17 00:00:00 2001 From: taschini Date: Wed, 25 May 2016 13:13:24 +0200 Subject: [PATCH 1/6] Ensure that a module within a namespace package can be found by --pyargs. --- _pytest/main.py | 55 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/_pytest/main.py b/_pytest/main.py index 8654d7af627..9808b9481a5 100644 --- a/_pytest/main.py +++ b/_pytest/main.py @@ -648,8 +648,34 @@ def _recurse(self, path): ihook.pytest_collect_directory(path=path, parent=self) return True + def _locate_module(self, modulename, searchpath): + """Find the locations of a module or package in the filesytem. + + In case of a presumptive namespace package return all of its possible + locations. + + """ + try: + fd, pathname, type_ = imp.find_module(modulename, searchpath) + except ImportError: + return [] + else: + if fd is not None: + fd.close() + if type_[2] != imp.PKG_DIRECTORY: + return [pathname] + else: + return [pathname] + self._locate_module( + modulename, + searchpath[searchpath.index(os.path.dirname(pathname)) + 1:]) + def _tryconvertpyarg(self, x): - mod = None + """Convert a dotted module name to a list of file-system locations. + + Always return a list. If the module cannot be found, the list contains + just the given argument. + + """ path = [os.path.abspath('.')] + sys.path for name in x.split('.'): # ignore anything that's not a proper name here @@ -658,27 +684,20 @@ def _tryconvertpyarg(self, x): # but it's supposed to be considered a filesystem path # not a package if name_re.match(name) is None: - return x - try: - fd, mod, type_ = imp.find_module(name, path) - except ImportError: - return x - else: - if fd is not None: - fd.close() - - if type_[2] != imp.PKG_DIRECTORY: - path = [os.path.dirname(mod)] - else: - path = [mod] - return mod + return [x] + path = self._locate_module(name, path) + if len(path) == 0: + return [x] + return path def _parsearg(self, arg): """ return (fspath, names) tuple after checking the file exists. """ - arg = str(arg) - if self.config.option.pyargs: - arg = self._tryconvertpyarg(arg) parts = str(arg).split("::") + if self.config.option.pyargs: + paths = self._tryconvertpyarg(parts[0]) + if len(paths) != 1: + raise pytest.UsageError("Cannot uniquely resolve package directory: " + arg) + parts[0] = paths[0] relpath = parts[0].replace("/", os.sep) path = self.config.invocation_dir.join(relpath, abs=True) if not path.check(): From 36b99a4b14f8cf6759267e22c7dadd14198ab876 Mon Sep 17 00:00:00 2001 From: taschini Date: Wed, 25 May 2016 15:44:37 +0200 Subject: [PATCH 2/6] Testing --pyargs behaviour with namespace packages. --- testing/acceptance_test.py | 63 +++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 4e964503756..4959870462a 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import sys import _pytest._code @@ -557,6 +558,67 @@ def join_pythonpath(what): "*not*found*test_hello*", ]) + def test_cmdline_python_namespace_package(self, testdir, monkeypatch): + monkeypatch.delenv('PYTHONDONTWRITEBYTECODE', False) + + search_path = [] + for dirname in "hello", "world": + d = testdir.mkdir(dirname) + search_path.append(d) + ns = d.mkdir("ns_pkg") + ns.join("__init__.py").write( + "__import__('pkg_resources').declare_namespace(__name__)") + lib = ns.mkdir(dirname) + lib.ensure("__init__.py") + lib.join("test_{0}.py".format(dirname)). \ + write("def test_{0}(): pass\n" + "def test_other():pass".format(dirname)) + + # The structure of the test directory is now: + # . + # ├── hello + # │   └── ns_pkg + # │   ├── __init__.py + # │   └── hello + # │   ├── __init__.py + # │   └── test_hello.py + # └── world + # └── ns_pkg + # ├── __init__.py + # └── world + # ├── __init__.py + # └── test_world.py + + def join_pythonpath(*dirs): + cur = py.std.os.environ.get('PYTHONPATH') + if cur: + dirs = dir + (cur,) + return ':'.join(str(p) for p in dirs) + monkeypatch.setenv('PYTHONPATH', join_pythonpath(*search_path)) + for p in search_path: + monkeypatch.syspath_prepend(p) + + # mixed module and filenames: + result = testdir.runpytest("--pyargs", "ns_pkg.hello", "world/ns_pkg") + assert result.ret == 0 + result.stdout.fnmatch_lines([ + "*4 passed*" + ]) + + # specify tests within a module + result = testdir.runpytest("--pyargs", "ns_pkg.world.test_world::test_other") + assert result.ret == 0 + result.stdout.fnmatch_lines([ + "*1 passed*" + ]) + + # namespace package + result = testdir.runpytest("--pyargs", "ns_pkg") + assert result.ret + result.stderr.fnmatch_lines([ + "ERROR:*Cannot*uniquely*resolve*package*directory:*ns_pkg", + ]) + def test_cmdline_python_package_not_exists(self, testdir): result = testdir.runpytest("--pyargs", "tpkgwhatv") assert result.ret @@ -697,4 +759,3 @@ def test_setup_function(self, testdir): * setup *test_1* * call *test_1* """) - From 19e16f7f4655f9ae48b99bc901165dbd767dc53b Mon Sep 17 00:00:00 2001 From: taschini Date: Wed, 25 May 2016 16:14:28 +0200 Subject: [PATCH 3/6] Noted the change the author in the logs. --- AUTHORS | 1 + CHANGELOG.rst | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/AUTHORS b/AUTHORS index f4a21b22dd4..f504ef77832 100644 --- a/AUTHORS +++ b/AUTHORS @@ -78,6 +78,7 @@ Ronny Pfannschmidt Ross Lawley Ryan Wooden Samuele Pedroni +Stefano Taschini Tom Viner Trevor Bekolay Wouter van Ackooy diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ab5da9d01c2..61e1187620b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,10 @@ * +* Ensure that a module within a namespace package can be found when it + is specified on the command line together with the``--pyargs`` + option. + * Fix win32 path issue when puttinging custom config file with absolute path in ``pytest.main("-c your_absolute_path")``. From 09b249259e8424dc77ed5972a9a1e8791e62f213 Mon Sep 17 00:00:00 2001 From: taschini Date: Wed, 25 May 2016 16:31:33 +0200 Subject: [PATCH 4/6] Fixed typo. --- testing/acceptance_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 4959870462a..1dd7250485e 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -592,7 +592,7 @@ def test_cmdline_python_namespace_package(self, testdir, monkeypatch): def join_pythonpath(*dirs): cur = py.std.os.environ.get('PYTHONPATH') if cur: - dirs = dir + (cur,) + dirs += (cur,) return ':'.join(str(p) for p in dirs) monkeypatch.setenv('PYTHONPATH', join_pythonpath(*search_path)) for p in search_path: From 11b04e0df47381555659c1797dd72d805af677e9 Mon Sep 17 00:00:00 2001 From: taschini Date: Wed, 25 May 2016 23:40:37 +0200 Subject: [PATCH 5/6] Improved the heuristics to detect a namespace package without importing it. --- _pytest/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/_pytest/main.py b/_pytest/main.py index 9808b9481a5..88c4b89d16c 100644 --- a/_pytest/main.py +++ b/_pytest/main.py @@ -665,6 +665,9 @@ def _locate_module(self, modulename, searchpath): if type_[2] != imp.PKG_DIRECTORY: return [pathname] else: + init_file = os.path.join(pathname, '__init__.py') + if os.path.getsize(init_file) == 0: + return [pathname] return [pathname] + self._locate_module( modulename, searchpath[searchpath.index(os.path.dirname(pathname)) + 1:]) From 811f837300c1ee5ce6467d5904acdd323b0f6cb2 Mon Sep 17 00:00:00 2001 From: taschini Date: Mon, 30 May 2016 09:43:51 +0200 Subject: [PATCH 6/6] Added comments and docstrings in accordance to feedback from @nicoddemus. --- _pytest/main.py | 11 +++++++++++ testing/acceptance_test.py | 14 +++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/_pytest/main.py b/_pytest/main.py index 88c4b89d16c..964d7fd0487 100644 --- a/_pytest/main.py +++ b/_pytest/main.py @@ -654,6 +654,13 @@ def _locate_module(self, modulename, searchpath): In case of a presumptive namespace package return all of its possible locations. + Note: The only reliable way to determine whether a package is a + namespace package, i.e., whether its ``__path__`` has more than one + element, is to import it. This method does not do that and hence we + are talking of a *presumptive* namespace package. The ``_parsearg`` + method is aware of this and, quite conservatively, tends to raise an + exception in case of doubt. + """ try: fd, pathname, type_ = imp.find_module(modulename, searchpath) @@ -666,6 +673,10 @@ def _locate_module(self, modulename, searchpath): return [pathname] else: init_file = os.path.join(pathname, '__init__.py') + # The following check is a little heuristic do determine whether a + # package is a namespace package. If its '__init__.py' is empty + # then it should be treated as a regular package (see #1568 for + # further discussion): if os.path.getsize(init_file) == 0: return [pathname] return [pathname] + self._locate_module( diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 1dd7250485e..95278245d78 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -559,7 +559,10 @@ def join_pythonpath(what): ]) def test_cmdline_python_namespace_package(self, testdir, monkeypatch): - monkeypatch.delenv('PYTHONDONTWRITEBYTECODE', False) + """ + test --pyargs option with namespace packages (#1567) + """ + monkeypatch.delenv('PYTHONDONTWRITEBYTECODE', raising=False) search_path = [] for dirname in "hello", "world": @@ -599,16 +602,21 @@ def join_pythonpath(*dirs): monkeypatch.syspath_prepend(p) # mixed module and filenames: - result = testdir.runpytest("--pyargs", "ns_pkg.hello", "world/ns_pkg") + result = testdir.runpytest("--pyargs", "-v", "ns_pkg.hello", "world/ns_pkg") assert result.ret == 0 result.stdout.fnmatch_lines([ + "*test_hello.py::test_hello*PASSED", + "*test_hello.py::test_other*PASSED", + "*test_world.py::test_world*PASSED", + "*test_world.py::test_other*PASSED", "*4 passed*" ]) # specify tests within a module - result = testdir.runpytest("--pyargs", "ns_pkg.world.test_world::test_other") + result = testdir.runpytest("--pyargs", "-v", "ns_pkg.world.test_world::test_other") assert result.ret == 0 result.stdout.fnmatch_lines([ + "*test_world.py::test_other*PASSED", "*1 passed*" ])