From 80aee58db5eea18b8d19126a178ea1ad719a01ab Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Mon, 9 Apr 2018 08:34:49 -0700 Subject: [PATCH] Implement PEP 561 (#4693) This makes it possible to use inline types from installed packages and use installed stubs according to PEP 561. Adds `--python-executable` and `--no-site-packages` command-line options. PEP 561: https://www.python.org/dev/peps/pep-0561/ Fixes #2625, #1190, #965. --- docs/source/command_line.rst | 20 ++- docs/source/index.rst | 1 + docs/source/installed_packages.rst | 123 ++++++++++++++++++ mypy/build.py | 110 +++++++++++----- mypy/fscache.py | 38 +++--- mypy/main.py | 96 +++++++++++++- mypy/options.py | 3 + mypy/sitepkgs.py | 28 ++++ mypy/stubgen.py | 3 +- mypy/test/config.py | 1 + mypy/test/helpers.py | 1 + mypy/test/testargs.py | 51 +++++++- mypy/test/testcheck.py | 4 +- mypy/test/testcmdline.py | 1 + mypy/test/testpep561.py | 118 +++++++++++++++++ mypy/test/testpythoneval.py | 2 +- runtests.py | 2 + test-data/packages/typedpkg-stubs/setup.py | 13 ++ .../typedpkg-stubs/__init__.pyi | 0 .../typedpkg-stubs/typedpkg-stubs/sample.pyi | 2 + test-data/packages/typedpkg/setup.py | 14 ++ .../packages/typedpkg/typedpkg/__init__.py | 0 test-data/packages/typedpkg/typedpkg/py.typed | 0 .../packages/typedpkg/typedpkg/sample.py | 7 + 24 files changed, 573 insertions(+), 65 deletions(-) create mode 100644 docs/source/installed_packages.rst create mode 100644 mypy/sitepkgs.py create mode 100644 mypy/test/testpep561.py create mode 100644 test-data/packages/typedpkg-stubs/setup.py create mode 100644 test-data/packages/typedpkg-stubs/typedpkg-stubs/__init__.pyi create mode 100644 test-data/packages/typedpkg-stubs/typedpkg-stubs/sample.pyi create mode 100644 test-data/packages/typedpkg/setup.py create mode 100644 test-data/packages/typedpkg/typedpkg/__init__.py create mode 100644 test-data/packages/typedpkg/typedpkg/py.typed create mode 100644 test-data/packages/typedpkg/typedpkg/sample.py diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 48fd22c0bd55..3de2beca83f8 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -366,11 +366,27 @@ Here are some more useful flags: updates the cache, but regular incremental mode ignores cache files written by quick mode. +- ``--python-executable EXECUTABLE`` will have mypy collect type information + from `PEP 561`_ compliant packages installed for the Python executable + ``EXECUTABLE``. If not provided, mypy will use PEP 561 compliant packages + installed for the Python executable running mypy. See + :ref:`installed-packages` for more on making PEP 561 compliant packages. + This flag will attempt to set ``--python-version`` if not already set. + - ``--python-version X.Y`` will make mypy typecheck your code as if it were run under Python version X.Y. Without this option, mypy will default to using whatever version of Python is running mypy. Note that the ``-2`` and ``--py2`` flags are aliases for ``--python-version 2.7``. See - :ref:`version_and_platform_checks` for more about this feature. + :ref:`version_and_platform_checks` for more about this feature. This flag + will attempt to find a Python executable of the corresponding version to + search for `PEP 561`_ compliant packages. If you'd like to disable this, + see ``--no-site-packages`` below. + +- ``--no-site-packages`` will disable searching for `PEP 561`_ compliant + packages. This will also disable searching for a usable Python executable. + Use this flag if mypy cannot find a Python executable for the version of + Python being checked, and you don't need to use PEP 561 typed packages. + Otherwise, use ``--python-executable``. - ``--platform PLATFORM`` will make mypy typecheck your code as if it were run under the the given operating system. Without this option, mypy will @@ -459,6 +475,8 @@ For the remaining flags you can read the full ``mypy -h`` output. Command line flags are liable to change between releases. +.. _PEP 561: https://www.python.org/dev/peps/pep-0561/ + .. _integrating-mypy: Integrating mypy into another Python application diff --git a/docs/source/index.rst b/docs/source/index.rst index 90cc74941da8..582c1c4ee1b8 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -30,6 +30,7 @@ Mypy is a static type checker for Python. command_line config_file python36 + installed_packages faq cheat_sheet cheat_sheet_py3 diff --git a/docs/source/installed_packages.rst b/docs/source/installed_packages.rst new file mode 100644 index 000000000000..a4e8c9e6abb6 --- /dev/null +++ b/docs/source/installed_packages.rst @@ -0,0 +1,123 @@ +.. _installed-packages: + +Using Installed Packages +======================== + +`PEP 561 `_ specifies how to mark +a package as supporting type checking. Below is a summary of how to create +PEP 561 compatible packages and have mypy use them in type checking. + +Using PEP 561 compatible packages with mypy +******************************************* + +Generally, you do not need to do anything to use installed packages that +support typing for the Python executable used to run mypy. Note that most +packages do not support typing. Packages that do support typing should be +automatically picked up by mypy and used for type checking. + +By default, mypy searches for packages installed for the Python executable +running mypy. It is highly unlikely you want this situation if you have +installed typed packages in another Python's package directory. + +Generally, you can use the ``--python-version`` flag and mypy will try to find +the correct package directory. If that fails, you can use the +``--python-executable`` flag to point to the exact executable, and mypy will +find packages installed for that Python executable. + +Note that mypy does not support some more advanced import features, such as zip +imports, namespace packages, and custom import hooks. + +If you do not want to use typed packages, use the ``--no-site-packages`` flag +to disable searching. + +Making PEP 561 compatible packages +********************************** + +PEP 561 notes three main ways to distribute type information. The first is a +package that has only inline type annotations in the code itself. The second is +a package that ships stub files with type information alongside the runtime +code. The third method, also known as a "stub only package" is a package that +ships type information for a package separately as stub files. + +If you would like to publish a library package to a package repository (e.g. +PyPI) for either internal or external use in type checking, packages that +supply type information via type comments or annotations in the code should put +a ``py.typed`` in their package directory. For example, with a directory +structure as follows: + +.. code-block:: text + + setup.py + package_a/ + __init__.py + lib.py + py.typed + +the setup.py might look like: + +.. code-block:: python + + from distutils.core import setup + + setup( + name="SuperPackageA", + author="Me", + version="0.1", + package_data={"package_a": ["py.typed"]}, + packages=["package_a"] + ) + +Some packages have a mix of stub files and runtime files. These packages also +require a ``py.typed`` file. An example can be seen below: + +.. code-block:: text + + setup.py + package_b/ + __init__.py + lib.py + lib.pyi + py.typed + +the setup.py might look like: + +.. code-block:: python + + from distutils.core import setup + + setup( + name="SuperPackageB", + author="Me", + version="0.1", + package_data={"package_b": ["py.typed", "lib.pyi"]}, + packages=["package_b"] + ) + +In this example, both ``lib.py`` and ``lib.pyi`` exist. At runtime, the Python +interpeter will use ``lib.py``, but mypy will use ``lib.pyi`` instead. + +If the package is stub-only (not imported at runtime), the package should have +a prefix of the runtime package name and a suffix of ``-stubs``. +A ``py.typed`` file is not needed for stub-only packages. For example, if we +had stubs for ``package_c``, we might do the following: + +.. code-block:: text + + setup.py + package_c-stubs/ + __init__.pyi + lib.pyi + +the setup.py might look like: + +.. code-block:: python + + from distutils.core import setup + + setup( + name="SuperPackageC", + author="Me", + version="0.1", + package_data={"package_c-stubs": ["__init__.pyi", "lib.pyi"]}, + packages=["package_c-stubs"] + ) diff --git a/mypy/build.py b/mypy/build.py index 096aabdb010e..cdd955defadc 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -10,10 +10,12 @@ """ # TODO: More consistent terminology, e.g. path/fnam, module/id, state/file +import ast import binascii import collections import contextlib from distutils.sysconfig import get_python_lib +import functools import gc import hashlib import json @@ -21,6 +23,7 @@ import re import site import stat +import subprocess import sys import time from os.path import dirname, basename @@ -33,6 +36,7 @@ if MYPY: from typing import Deque +from mypy import sitepkgs from mypy.nodes import (MODULE_REF, MypyFile, Node, ImportBase, Import, ImportFrom, ImportAll) from mypy.semanal_pass1 import SemanticAnalyzerPass1 from mypy.semanal import SemanticAnalyzerPass2, apply_semantic_analyzer_patches @@ -698,7 +702,8 @@ def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str: def is_module(self, id: str) -> bool: """Is there a file in the file system corresponding to module id?""" - return self.find_module_cache.find_module(id, self.lib_path) is not None + return self.find_module_cache.find_module(id, self.lib_path, + self.options.python_executable) is not None def parse_file(self, id: str, path: str, source: str, ignore_errors: bool) -> MypyFile: """Parse the source of a file with the given name. @@ -789,6 +794,24 @@ def remove_cwd_prefix_from_path(fscache: FileSystemCache, p: str) -> str: return p +@functools.lru_cache(maxsize=None) +def _get_site_packages_dirs(python_executable: Optional[str]) -> List[str]: + """Find package directories for given python. + + This runs a subprocess call, which generates a list of the site package directories. + To avoid repeatedly calling a subprocess (which can be slow!) we lru_cache the results.""" + if python_executable is None: + return [] + if python_executable == sys.executable: + # Use running Python's package dirs + return sitepkgs.getsitepackages() + else: + # Use subprocess to get the package directory of given Python + # executable + return ast.literal_eval(subprocess.check_output([python_executable, sitepkgs.__file__], + stderr=subprocess.PIPE).decode()) + + class FindModuleCache: """Module finder with integrated cache. @@ -802,20 +825,29 @@ class FindModuleCache: def __init__(self, fscache: Optional[FileSystemMetaCache] = None) -> None: self.fscache = fscache or FileSystemMetaCache() - # Cache find_module: (id, lib_path) -> result. - self.results = {} # type: Dict[Tuple[str, Tuple[str, ...]], Optional[str]] + self.find_lib_path_dirs = functools.lru_cache(maxsize=None)(self._find_lib_path_dirs) + self.find_module = functools.lru_cache(maxsize=None)(self._find_module) + + def clear(self) -> None: + self.find_module.cache_clear() + self.find_lib_path_dirs.cache_clear() + def _find_lib_path_dirs(self, dir_chain: str, lib_path: Tuple[str, ...]) -> List[str]: # Cache some repeated work within distinct find_module calls: finding which # elements of lib_path have even the subdirectory they'd need for the module # to exist. This is shared among different module ids when they differ only # in the last component. - self.dirs = {} # type: Dict[Tuple[str, Tuple[str, ...]], List[str]] - - def clear(self) -> None: - self.results.clear() - self.dirs.clear() - - def _find_module(self, id: str, lib_path: Tuple[str, ...]) -> Optional[str]: + dirs = [] + for pathitem in lib_path: + # e.g., '/usr/lib/python3.4/foo/bar' + dir = os.path.normpath(os.path.join(pathitem, dir_chain)) + if self.fscache.isdir(dir): + dirs.append(dir) + return dirs + + def _find_module(self, id: str, lib_path: Tuple[str, ...], + python_executable: Optional[str]) -> Optional[str]: + """Return the path of the module source file, or None if not found.""" fscache = self.fscache # If we're looking for a module like 'foo.bar.baz', it's likely that most of the @@ -824,15 +856,23 @@ def _find_module(self, id: str, lib_path: Tuple[str, ...]) -> Optional[str]: # that will require the same subdirectory. components = id.split('.') dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' - if (dir_chain, lib_path) not in self.dirs: - dirs = [] - for pathitem in lib_path: - # e.g., '/usr/lib/python3.4/foo/bar' - dir = os.path.normpath(os.path.join(pathitem, dir_chain)) - if fscache.isdir(dir): - dirs.append(dir) - self.dirs[dir_chain, lib_path] = dirs - candidate_base_dirs = self.dirs[dir_chain, lib_path] + # TODO (ethanhs): refactor each path search to its own method with lru_cache + + third_party_dirs = [] + # Third-party stub/typed packages + for pkg_dir in _get_site_packages_dirs(python_executable): + stub_name = components[0] + '-stubs' + typed_file = os.path.join(pkg_dir, components[0], 'py.typed') + stub_dir = os.path.join(pkg_dir, stub_name) + if fscache.isdir(stub_dir): + stub_components = [stub_name] + components[1:] + path = os.path.join(pkg_dir, *stub_components[:-1]) + if fscache.isdir(path): + third_party_dirs.append(path) + elif fscache.isfile(typed_file): + path = os.path.join(pkg_dir, dir_chain) + third_party_dirs.append(path) + candidate_base_dirs = self.find_lib_path_dirs(dir_chain, lib_path) + third_party_dirs # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now # contains just the subdirectories 'foo/bar' that actually exist under the @@ -845,8 +885,11 @@ def _find_module(self, id: str, lib_path: Tuple[str, ...]) -> Optional[str]: # Prefer package over module, i.e. baz/__init__.py* over baz.py*. for extension in PYTHON_EXTENSIONS: path = base_path + sepinit + extension + path_stubs = base_path + '-stubs' + sepinit + extension if fscache.isfile_case(path) and verify_module(fscache, id, path): return path + elif fscache.isfile_case(path_stubs) and verify_module(fscache, id, path_stubs): + return path_stubs # No package, look for module. for extension in PYTHON_EXTENSIONS: path = base_path + extension @@ -854,17 +897,9 @@ def _find_module(self, id: str, lib_path: Tuple[str, ...]) -> Optional[str]: return path return None - def find_module(self, id: str, lib_path_arg: Iterable[str]) -> Optional[str]: - """Return the path of the module source file, or None if not found.""" - lib_path = tuple(lib_path_arg) - - key = (id, lib_path) - if key not in self.results: - self.results[key] = self._find_module(id, lib_path) - return self.results[key] - - def find_modules_recursive(self, module: str, lib_path: List[str]) -> List[BuildSource]: - module_path = self.find_module(module, lib_path) + def find_modules_recursive(self, module: str, lib_path: Tuple[str, ...], + python_executable: Optional[str]) -> List[BuildSource]: + module_path = self.find_module(module, lib_path, python_executable) if not module_path: return [] result = [BuildSource(module_path, module, None)] @@ -884,13 +919,15 @@ def find_modules_recursive(self, module: str, lib_path: List[str]) -> List[Build (os.path.isfile(os.path.join(abs_path, '__init__.py')) or os.path.isfile(os.path.join(abs_path, '__init__.pyi'))): hits.add(item) - result += self.find_modules_recursive(module + '.' + item, lib_path) + result += self.find_modules_recursive(module + '.' + item, lib_path, + python_executable) elif item != '__init__.py' and item != '__init__.pyi' and \ item.endswith(('.py', '.pyi')): mod = item.split('.')[0] if mod not in hits: hits.add(mod) - result += self.find_modules_recursive(module + '.' + mod, lib_path) + result += self.find_modules_recursive(module + '.' + mod, lib_path, + python_executable) return result @@ -2001,7 +2038,8 @@ def find_module_and_diagnose(manager: BuildManager, # difference and just assume 'builtins' everywhere, # which simplifies code. file_id = '__builtin__' - path = manager.find_module_cache.find_module(file_id, manager.lib_path) + path = manager.find_module_cache.find_module(file_id, manager.lib_path, + manager.options.python_executable) if path: # For non-stubs, look at options.follow_imports: # - normal (default) -> fully analyze @@ -2125,12 +2163,14 @@ def dispatch(sources: List[BuildSource], manager: BuildManager) -> Graph: graph = load_graph(sources, manager) t1 = time.time() + fm_cache_size = manager.find_module_cache.find_module.cache_info().currsize + fm_dir_cache_size = manager.find_module_cache.find_lib_path_dirs.cache_info().currsize manager.add_stats(graph_size=len(graph), stubs_found=sum(g.path is not None and g.path.endswith('.pyi') for g in graph.values()), graph_load_time=(t1 - t0), - fm_cache_size=len(manager.find_module_cache.results), - fm_dir_cache_size=len(manager.find_module_cache.dirs), + fm_cache_size=fm_cache_size, + fm_dir_cache_size=fm_dir_cache_size, ) if not graph: print("Nothing to do?!") diff --git a/mypy/fscache.py b/mypy/fscache.py index 75600dba2951..9075b41fbe77 100644 --- a/mypy/fscache.py +++ b/mypy/fscache.py @@ -28,49 +28,45 @@ advantage of the benefits. """ +import functools import os import stat -from typing import Tuple, Dict, List, Optional +from typing import Dict, List, Optional, Tuple, TypeVar from mypy.util import read_with_python_encoding class FileSystemMetaCache: def __init__(self) -> None: - self.flush() + self.stat = functools.lru_cache(maxsize=None)(self._stat) + self.listdir = functools.lru_cache(maxsize=None)(self._listdir) + # lru_cache doesn't handle exceptions, so we need special caches for them. + self.stat_error_cache = {} # type: Dict[str, Exception] + self.listdir_error_cache = {} # type: Dict[str, Exception] def flush(self) -> None: """Start another transaction and empty all caches.""" - self.stat_cache = {} # type: Dict[str, os.stat_result] - self.stat_error_cache = {} # type: Dict[str, Exception] - self.listdir_cache = {} # type: Dict[str, List[str]] - self.listdir_error_cache = {} # type: Dict[str, Exception] - self.isfile_case_cache = {} # type: Dict[str, bool] + self.stat.cache_clear() + self.listdir.cache_clear() + self.stat_error_cache.clear() + self.listdir_error_cache.clear() - def stat(self, path: str) -> os.stat_result: - if path in self.stat_cache: - return self.stat_cache[path] + def _stat(self, path: str) -> os.stat_result: if path in self.stat_error_cache: raise self.stat_error_cache[path] try: - st = os.stat(path) + return os.stat(path) except Exception as err: self.stat_error_cache[path] = err raise - self.stat_cache[path] = st - return st - def listdir(self, path: str) -> List[str]: - if path in self.listdir_cache: - return self.listdir_cache[path] + def _listdir(self, path: str) -> List[str]: if path in self.listdir_error_cache: raise self.listdir_error_cache[path] try: - results = os.listdir(path) + return os.listdir(path) except Exception as err: self.listdir_error_cache[path] = err raise err - self.listdir_cache[path] = results - return results def isfile(self, path: str) -> bool: try: @@ -88,8 +84,6 @@ def isfile_case(self, path: str) -> bool: TODO: We should maybe check the case for some directory components also, to avoid permitting wrongly-cased *packages*. """ - if path in self.isfile_case_cache: - return self.isfile_case_cache[path] head, tail = os.path.split(path) if not tail: res = False @@ -99,7 +93,6 @@ def isfile_case(self, path: str) -> bool: res = tail in names and self.isfile(path) except OSError: res = False - self.isfile_case_cache[path] = res return res def isdir(self, path: str) -> bool: @@ -119,6 +112,7 @@ def exists(self, path: str) -> bool: class FileSystemCache(FileSystemMetaCache): def __init__(self, pyversion: Tuple[int, int]) -> None: + super().__init__() self.pyversion = pyversion self.flush() diff --git a/mypy/main.py b/mypy/main.py index 270e45f22a29..eccf6ba67432 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1,10 +1,12 @@ """Mypy type checker command line tool.""" import argparse +import ast import configparser import fnmatch import os import re +import subprocess import sys import time @@ -208,6 +210,80 @@ def invert_flag_name(flag: str) -> str: return '--no-{}'.format(flag[2:]) +class PythonExecutableInferenceError(Exception): + """Represents a failure to infer the version or executable while searching.""" + + +def python_executable_prefix(v: str) -> List[str]: + if sys.platform == 'win32': + # on Windows, all Python executables are named `python`. To handle this, there + # is the `py` launcher, which can be passed a version e.g. `py -3.5`, and it will + # execute an installed Python 3.5 interpreter. See also: + # https://docs.python.org/3/using/windows.html#python-launcher-for-windows + return ['py', '-{}'.format(v)] + else: + return ['python{}'.format(v)] + + +def _python_version_from_executable(python_executable: str) -> Tuple[int, int]: + try: + check = subprocess.check_output([python_executable, '-c', + 'import sys; print(repr(sys.version_info[:2]))'], + stderr=subprocess.STDOUT).decode() + return ast.literal_eval(check) + except (subprocess.CalledProcessError, FileNotFoundError): + raise PythonExecutableInferenceError( + 'Error: invalid Python executable {}'.format(python_executable)) + + +def _python_executable_from_version(python_version: Tuple[int, int]) -> str: + if sys.version_info[:2] == python_version: + return sys.executable + str_ver = '.'.join(map(str, python_version)) + try: + sys_exe = subprocess.check_output(python_executable_prefix(str_ver) + + ['-c', 'import sys; print(sys.executable)'], + stderr=subprocess.STDOUT).decode().strip() + return sys_exe + except (subprocess.CalledProcessError, FileNotFoundError): + raise PythonExecutableInferenceError( + 'Error: failed to find a Python executable matching version {},' + ' perhaps try --python-executable, or --no-site-packages?'.format(python_version)) + + +def infer_python_version_and_executable(options: Options, + special_opts: argparse.Namespace) -> None: + """Infer the Python version or executable from each other. Check they are consistent. + + This function mutates options based on special_opts to infer the correct Python version and + executable to use. + """ + # Infer Python version and/or executable if one is not given + + # TODO: (ethanhs) Look at folding these checks and the site packages subprocess calls into + # one subprocess call for speed. + if special_opts.python_executable is not None and special_opts.python_version is not None: + py_exe_ver = _python_version_from_executable(special_opts.python_executable) + if py_exe_ver != special_opts.python_version: + raise PythonExecutableInferenceError( + 'Python version {} did not match executable {}, got version {}.'.format( + special_opts.python_version, special_opts.python_executable, py_exe_ver + )) + else: + options.python_version = special_opts.python_version + options.python_executable = special_opts.python_executable + elif special_opts.python_executable is None and special_opts.python_version is not None: + options.python_version = special_opts.python_version + py_exe = None + if not special_opts.no_executable: + py_exe = _python_executable_from_version(special_opts.python_version) + options.python_executable = py_exe + elif special_opts.python_version is None and special_opts.python_executable is not None: + options.python_version = _python_version_from_executable( + special_opts.python_executable) + options.python_executable = special_opts.python_executable + + def process_options(args: List[str], require_targets: bool = True, server_options: bool = False, @@ -258,10 +334,16 @@ def add_invertible_flag(flag: str, parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__) parser.add_argument('--python-version', type=parse_version, metavar='x.y', - help='use Python x.y') + help='use Python x.y', dest='special-opts:python_version') + parser.add_argument('--python-executable', action='store', metavar='EXECUTABLE', + help="Python executable used for finding PEP 561 compliant installed" + " packages and stubs", dest='special-opts:python_executable') + parser.add_argument('--no-site-packages', action='store_true', + dest='special-opts:no_executable', + help="Do not search for installed PEP 561 compliant packages") parser.add_argument('--platform', action='store', metavar='PLATFORM', help="typecheck special-cased code for the given OS platform " - "(defaults to sys.platform).") + "(defaults to sys.platform)") parser.add_argument('-2', '--py2', dest='python_version', action='store_const', const=defaults.PYTHON2_VERSION, help="use Python 2 mode") parser.add_argument('--ignore-missing-imports', action='store_true', @@ -490,6 +572,14 @@ def add_invertible_flag(flag: str, print("Warning: --no-fast-parser no longer has any effect. The fast parser " "is now mypy's default and only parser.") + try: + infer_python_version_and_executable(options, special_opts) + except PythonExecutableInferenceError as e: + parser.error(str(e)) + + if special_opts.no_executable: + options.python_executable = None + # Check for invalid argument combinations. if require_targets: code_methods = sum(bool(c) for c in [special_opts.modules + special_opts.packages, @@ -539,7 +629,7 @@ def add_invertible_flag(flag: str, for p in special_opts.packages: if os.sep in p or os.altsep and os.altsep in p: fail("Package name '{}' cannot have a slash in it.".format(p)) - p_targets = cache.find_modules_recursive(p, lib_path) + p_targets = cache.find_modules_recursive(p, tuple(lib_path), options.python_executable) if not p_targets: fail("Can't find package '{}'".format(p)) targets.extend(p_targets) diff --git a/mypy/options.py b/mypy/options.py index 71ab15c5beaf..bde93ca42e52 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -56,6 +56,9 @@ def __init__(self) -> None: # -- build options -- self.build_type = BuildType.STANDARD self.python_version = sys.version_info[:2] # type: Tuple[int, int] + # The executable used to search for PEP 561 packages. If this is None, + # then mypy does not search for PEP 561 packages. + self.python_executable = sys.executable # type: Optional[str] self.platform = sys.platform self.custom_typing_module = None # type: Optional[str] self.custom_typeshed_dir = None # type: Optional[str] diff --git a/mypy/sitepkgs.py b/mypy/sitepkgs.py new file mode 100644 index 000000000000..8a36bffb98c9 --- /dev/null +++ b/mypy/sitepkgs.py @@ -0,0 +1,28 @@ +from __future__ import print_function +"""This file is used to find the site packages of a Python executable, which may be Python 2. + +This file MUST remain compatible with Python 2. Since we cannot make any assumptions about the +Python being executed, this module should not use *any* dependencies outside of the standard +library found in Python 2. This file is run each mypy run, so it should be kept as fast as +possible. +""" + + +from distutils.sysconfig import get_python_lib +import site +MYPY = False +if MYPY: + from typing import List + + +def getsitepackages(): + # type: () -> List[str] + if hasattr(site, 'getusersitepackages') and hasattr(site, 'getsitepackages'): + user_dir = site.getusersitepackages() + return site.getsitepackages() + [user_dir] + else: + return [get_python_lib()] + + +if __name__ == '__main__': + print(repr(getsitepackages())) diff --git a/mypy/stubgen.py b/mypy/stubgen.py index 386d012fe7b8..b95efefeb119 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -160,7 +160,8 @@ def find_module_path_and_all(module: str, pyversion: Tuple[int, int], module_all = getattr(mod, '__all__', None) else: # Find module by going through search path. - module_path = mypy.build.FindModuleCache().find_module(module, ['.'] + search_path) + module_path = mypy.build.FindModuleCache().find_module(module, ['.'] + search_path, + interpreter) if not module_path: raise SystemExit( "Can't find module '{}' (consider using --search-path)".format(module)) diff --git a/mypy/test/config.py b/mypy/test/config.py index 5dbe791e593a..5e6792fb0d57 100644 --- a/mypy/test/config.py +++ b/mypy/test/config.py @@ -5,6 +5,7 @@ # Location of test data files such as test case descriptions. test_data_prefix = os.path.join(PREFIX, 'test-data', 'unit') +package_path = os.path.join(PREFIX, 'test-data', 'packages') assert os.path.isdir(test_data_prefix), \ 'Test data prefix ({}) not set correctly'.format(test_data_prefix) diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index 4b2fa3a4f131..96d806efe503 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -332,6 +332,7 @@ def parse_options(program_text: str, testcase: DataDrivenTestCase, flag_list = None if flags: flag_list = flags.group(1).split() + flag_list.append('--no-site-packages') # the tests shouldn't need an installed Python targets, options = process_options(flag_list, require_targets=False) if targets: # TODO: support specifying targets via the flags pragma diff --git a/mypy/test/testargs.py b/mypy/test/testargs.py index 20db610cda18..ecfe4143d7b4 100644 --- a/mypy/test/testargs.py +++ b/mypy/test/testargs.py @@ -4,10 +4,15 @@ defaults, and that argparse doesn't assign any new members to the Options object it creates. """ +import argparse +import sys + +import pytest # type: ignore from mypy.test.helpers import Suite, assert_equal from mypy.options import Options -from mypy.main import process_options +from mypy.main import (process_options, PythonExecutableInferenceError, + infer_python_version_and_executable) class ArgSuite(Suite): @@ -17,3 +22,47 @@ def test_coherence(self) -> None: # FIX: test this too. Requires changing working dir to avoid finding 'setup.cfg' options.config_file = parsed_options.config_file assert_equal(options, parsed_options) + + def test_executable_inference(self) -> None: + """Test the --python-executable flag with --python-version""" + sys_ver_str = '{ver.major}.{ver.minor}'.format(ver=sys.version_info) + + base = ['file.py'] # dummy file + + # test inference given one (infer the other) + matching_version = base + ['--python-version={}'.format(sys_ver_str)] + _, options = process_options(matching_version) + assert options.python_version == sys.version_info[:2] + assert options.python_executable == sys.executable + + matching_version = base + ['--python-executable={}'.format(sys.executable)] + _, options = process_options(matching_version) + assert options.python_version == sys.version_info[:2] + assert options.python_executable == sys.executable + + # test inference given both + matching_version = base + ['--python-version={}'.format(sys_ver_str), + '--python-executable={}'.format(sys.executable)] + _, options = process_options(matching_version) + assert options.python_version == sys.version_info[:2] + assert options.python_executable == sys.executable + + # test that we error if the version mismatch + # argparse sys.exits on a parser.error, we need to check the raw inference function + options = Options() + + special_opts = argparse.Namespace() + special_opts.python_executable = sys.executable + special_opts.python_version = (2, 10) # obviously wrong + special_opts.no_executable = None + with pytest.raises(PythonExecutableInferenceError) as e: + infer_python_version_and_executable(options, special_opts) + assert str(e.value) == 'Python version (2, 10) did not match executable {}, got' \ + ' version {}.'.format(sys.executable, sys.version_info[:2]) + + # test that --no-site-packages will disable executable inference + matching_version = base + ['--python-version={}'.format(sys_ver_str), + '--no-site-packages'] + _, options = process_options(matching_version) + assert options.python_version == sys.version_info[:2] + assert options.python_executable is None diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 997a795a4723..bc3feab1a90a 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -3,6 +3,7 @@ import os import re import shutil +import sys from typing import Dict, List, Optional, Set, Tuple @@ -276,7 +277,8 @@ def parse_module(self, module_names = m.group(1) out = [] for module_name in module_names.split(' '): - path = build.FindModuleCache().find_module(module_name, [test_temp_dir]) + path = build.FindModuleCache().find_module(module_name, (test_temp_dir,), + sys.executable) assert path is not None, "Can't find ad hoc case file" with open(path) as f: program_text = f.read() diff --git a/mypy/test/testcmdline.py b/mypy/test/testcmdline.py index 57910c1a1dc0..88ad272fab0d 100644 --- a/mypy/test/testcmdline.py +++ b/mypy/test/testcmdline.py @@ -47,6 +47,7 @@ def test_python_cmdline(testcase: DataDrivenTestCase) -> None: file.write('{}\n'.format(s)) args = parse_args(testcase.input[0]) args.append('--show-traceback') + args.append('--no-site-packages') # Type check the program. fixed = [python3_path, os.path.join(testcase.old_cwd, 'scripts', 'mypy')] diff --git a/mypy/test/testpep561.py b/mypy/test/testpep561.py new file mode 100644 index 000000000000..5d2e3a5963e3 --- /dev/null +++ b/mypy/test/testpep561.py @@ -0,0 +1,118 @@ +from contextlib import contextmanager +import os +import shutil +import sys +from typing import Iterator, List +from unittest import TestCase, main + +import mypy.api +from mypy.build import FindModuleCache, _get_site_packages_dirs +from mypy.test.config import package_path +from mypy.test.helpers import run_command +from mypy.util import try_find_python2_interpreter + +SIMPLE_PROGRAM = """ +from typedpkg.sample import ex +a = ex(['']) +reveal_type(a) +""" + + +def check_mypy_run(cmd_line: List[str], + expected_out: str, + expected_err: str = '', + expected_returncode: int = 1) -> None: + """Helper to run mypy and check the output.""" + out, err, returncode = mypy.api.run(cmd_line) + assert out == expected_out, err + assert err == expected_err, out + assert returncode == expected_returncode, returncode + + +class TestPEP561(TestCase): + @contextmanager + def install_package(self, pkg: str, + python_executable: str = sys.executable) -> Iterator[None]: + """Context manager to temporarily install a package from test-data/packages/pkg/""" + working_dir = os.path.join(package_path, pkg) + install_cmd = [python_executable, '-m', 'pip', 'install', '.'] + # if we aren't in a virtualenv, install in the + # user package directory so we don't need sudo + # In a virtualenv, real_prefix is patched onto + # sys + if not hasattr(sys, 'real_prefix') or python_executable != sys.executable: + install_cmd.append('--user') + returncode, lines = run_command(install_cmd, cwd=working_dir) + if returncode != 0: + self.fail('\n'.join(lines)) + try: + yield + finally: + run_command([python_executable, '-m', 'pip', 'uninstall', '-y', pkg], cwd=package_path) + + def test_get_pkg_dirs(self) -> None: + """Check that get_package_dirs works.""" + dirs = _get_site_packages_dirs(sys.executable) + assert dirs + + def test_typed_pkg(self) -> None: + """Tests type checking based on installed packages. + + This test CANNOT be split up, concurrency means that simultaneously + installing/uninstalling will break tests. + """ + test_file = 'simple.py' + if not os.path.isdir('test-packages-data'): + os.mkdir('test-packages-data') + old_cwd = os.getcwd() + os.chdir('test-packages-data') + with open(test_file, 'w') as f: + f.write(SIMPLE_PROGRAM) + try: + with self.install_package('typedpkg-stubs'): + check_mypy_run( + [test_file], + "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + ) + + # The Python 2 tests are intentionally placed after a Python 3 test to check + # the package_dir_cache is behaving correctly. + python2 = try_find_python2_interpreter() + if python2: + with self.install_package('typedpkg-stubs', python2): + check_mypy_run( + ['--python-executable={}'.format(python2), test_file], + "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + ) + with self.install_package('typedpkg', python2): + check_mypy_run( + ['--python-executable={}'.format(python2), 'simple.py'], + "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" + ) + + with self.install_package('typedpkg', python2): + with self.install_package('typedpkg-stubs', python2): + check_mypy_run( + ['--python-executable={}'.format(python2), test_file], + "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + ) + + with self.install_package('typedpkg'): + check_mypy_run( + [test_file], + "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" + ) + + with self.install_package('typedpkg'): + with self.install_package('typedpkg-stubs'): + check_mypy_run( + [test_file], + "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + ) + finally: + os.chdir(old_cwd) + shutil.rmtree('test-packages-data') + + +if __name__ == '__main__': + main() diff --git a/mypy/test/testpythoneval.py b/mypy/test/testpythoneval.py index 0634442f172a..7b914322c7ce 100644 --- a/mypy/test/testpythoneval.py +++ b/mypy/test/testpythoneval.py @@ -49,7 +49,7 @@ def test_python_evaluation(testcase: DataDrivenTestCase) -> None: version. """ assert testcase.old_cwd is not None, "test was not properly set up" - mypy_cmdline = ['--show-traceback'] + mypy_cmdline = ['--show-traceback', '--no-site-packages'] py2 = testcase.name.lower().endswith('python2') if py2: mypy_cmdline.append('--py2') diff --git a/runtests.py b/runtests.py index b09e7c676856..e82b1c207cb3 100755 --- a/runtests.py +++ b/runtests.py @@ -73,6 +73,7 @@ def add_mypy_cmd(self, name: str, mypy_args: List[str], cwd: Optional[str] = Non return args = [sys.executable, self.mypy] + mypy_args args.append('--show-traceback') + args.append('--no-site-packages') self.waiter.add(LazySubprocess(full_name, args, cwd=cwd, env=self.env)) def add_mypy(self, name: str, *args: str, cwd: Optional[str] = None) -> None: @@ -228,6 +229,7 @@ def test_path(*names: str): ) SLOW_FILES = test_path( + 'testpep561', 'testpythoneval', 'testcmdline', 'teststubgen', diff --git a/test-data/packages/typedpkg-stubs/setup.py b/test-data/packages/typedpkg-stubs/setup.py new file mode 100644 index 000000000000..b90e3a011f23 --- /dev/null +++ b/test-data/packages/typedpkg-stubs/setup.py @@ -0,0 +1,13 @@ +""" +This setup file installs packages to test mypy's PEP 561 implementation +""" + +from distutils.core import setup + +setup( + name='typedpkg-stubs', + author="The mypy team", + version='0.1', + package_data={'typedpkg-stubs': ['sample.pyi', '__init__.pyi']}, + packages=['typedpkg-stubs'], +) diff --git a/test-data/packages/typedpkg-stubs/typedpkg-stubs/__init__.pyi b/test-data/packages/typedpkg-stubs/typedpkg-stubs/__init__.pyi new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/typedpkg-stubs/typedpkg-stubs/sample.pyi b/test-data/packages/typedpkg-stubs/typedpkg-stubs/sample.pyi new file mode 100644 index 000000000000..355deefd6a2d --- /dev/null +++ b/test-data/packages/typedpkg-stubs/typedpkg-stubs/sample.pyi @@ -0,0 +1,2 @@ +from typing import Iterable, List +def ex(a: Iterable[str]) -> List[str]: ... diff --git a/test-data/packages/typedpkg/setup.py b/test-data/packages/typedpkg/setup.py new file mode 100644 index 000000000000..6da37d2f6629 --- /dev/null +++ b/test-data/packages/typedpkg/setup.py @@ -0,0 +1,14 @@ +""" +This setup file installs packages to test mypy's PEP 561 implementation +""" + +from distutils.core import setup + +setup( + name='typedpkg', + author="The mypy team", + version='0.1', + package_data={'typedpkg': ['py.typed']}, + packages=['typedpkg'], + include_package_data=True, +) diff --git a/test-data/packages/typedpkg/typedpkg/__init__.py b/test-data/packages/typedpkg/typedpkg/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/typedpkg/typedpkg/py.typed b/test-data/packages/typedpkg/typedpkg/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/typedpkg/typedpkg/sample.py b/test-data/packages/typedpkg/typedpkg/sample.py new file mode 100644 index 000000000000..6a5f88a0f372 --- /dev/null +++ b/test-data/packages/typedpkg/typedpkg/sample.py @@ -0,0 +1,7 @@ +from typing import Iterable, Tuple + + +def ex(a): + # type: (Iterable[str]) -> Tuple[str, ...] + """Example typed package.""" + return tuple(a)