Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Restore dict based typesafe caching #4898

Merged
merged 2 commits into from
Apr 13, 2018
Merged

Restore dict based typesafe caching #4898

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4dbd7cd
Restore dict based typesafe caching
ethanhs Apr 13, 2018
964af59
Merge branch 'master' into dictcache
ethanhs Apr 13, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 23 additions & 10 deletions mypy/build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -825,18 +825,26 @@ class FindModuleCache:

def __init__(self, fscache: Optional[FileSystemMetaCache] = None) -> None:
self.fscache = fscache or FileSystemMetaCache()
self.find_lib_path_dirs = functools.lru_cache(maxsize=None)(self._find_lib_path_dirs)
self.find_module = functools.lru_cache(maxsize=None)(self._find_module)
# Cache find_lib_path_dirs: (dir_chain, lib_path)
self.dirs = {} # type: Dict[Tuple[str, Tuple[str, ...]], List[str]]
# Cache find_module: (id, lib_path, python_version) -> result.
self.results = {} # type: Dict[Tuple[str, Tuple[str, ...], Optional[str]], Optional[str]]

def clear(self) -> None:
self.find_module.cache_clear()
self.find_lib_path_dirs.cache_clear()
self.results.clear()
self.dirs.clear()

def _find_lib_path_dirs(self, dir_chain: str, lib_path: Tuple[str, ...]) -> List[str]:
def find_lib_path_dirs(self, dir_chain: str, lib_path: Tuple[str, ...]) -> List[str]:
# Cache some repeated work within distinct find_module calls: finding which
# elements of lib_path have even the subdirectory they'd need for the module
# to exist. This is shared among different module ids when they differ only
# in the last component.
key = (dir_chain, lib_path)
if key not in self.dirs:
self.dirs[key] = self._find_lib_path_dirs(dir_chain, lib_path)
return self.dirs[key]

def _find_lib_path_dirs(self, dir_chain: str, lib_path: Tuple[str, ...]) -> List[str]:
dirs = []
for pathitem in lib_path:
# e.g., '/usr/lib/python3.4/foo/bar'
Expand All @@ -845,9 +853,16 @@ def _find_lib_path_dirs(self, dir_chain: str, lib_path: Tuple[str, ...]) -> List
dirs.append(dir)
return dirs

def find_module(self, id: str, lib_path: Tuple[str, ...],
python_executable: Optional[str]) -> Optional[str]:
"""Return the path of the module source file, or None if not found."""
key = (id, lib_path, python_executable)
if key not in self.results:
self.results[key] = self._find_module(id, lib_path, python_executable)
return self.results[key]

def _find_module(self, id: str, lib_path: Tuple[str, ...],
python_executable: Optional[str]) -> Optional[str]:
"""Return the path of the module source file, or None if not found."""
fscache = self.fscache

# If we're looking for a module like 'foo.bar.baz', it's likely that most of the
Expand Down Expand Up @@ -2167,14 +2182,12 @@ def dispatch(sources: List[BuildSource], manager: BuildManager) -> Graph:
graph = load_graph(sources, manager)

t1 = time.time()
fm_cache_size = manager.find_module_cache.find_module.cache_info().currsize
fm_dir_cache_size = manager.find_module_cache.find_lib_path_dirs.cache_info().currsize
manager.add_stats(graph_size=len(graph),
stubs_found=sum(g.path is not None and g.path.endswith('.pyi')
for g in graph.values()),
graph_load_time=(t1 - t0),
fm_cache_size=fm_cache_size,
fm_dir_cache_size=fm_dir_cache_size,
fm_cache_size=len(manager.find_module_cache.results),
fm_dir_cache_size=len(manager.find_module_cache.dirs),
)
if not graph:
print("Nothing to do?!")
Expand Down
34 changes: 21 additions & 13 deletions mypy/fscache.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,36 +37,41 @@

class FileSystemMetaCache:
def __init__(self) -> None:
self.stat = functools.lru_cache(maxsize=None)(self._stat)
self.listdir = functools.lru_cache(maxsize=None)(self._listdir)
# lru_cache doesn't handle exceptions, so we need special caches for them.
self.stat_error_cache = {} # type: Dict[str, Exception]
self.listdir_error_cache = {} # type: Dict[str, Exception]
self.flush()

def flush(self) -> None:
"""Start another transaction and empty all caches."""
self.stat.cache_clear()
self.listdir.cache_clear()
self.stat_error_cache.clear()
self.listdir_error_cache.clear()
self.stat_cache = {} # type: Dict[str, os.stat_result]
self.stat_error_cache = {} # type: Dict[str, Exception]
self.listdir_cache = {} # type: Dict[str, List[str]]
self.listdir_error_cache = {} # type: Dict[str, Exception]
self.isfile_case_cache = {} # type: Dict[str, bool]

def _stat(self, path: str) -> os.stat_result:
def stat(self, path: str) -> os.stat_result:
if path in self.stat_cache:
return self.stat_cache[path]
if path in self.stat_error_cache:
raise self.stat_error_cache[path]
try:
return os.stat(path)
st = os.stat(path)
except Exception as err:
self.stat_error_cache[path] = err
raise
self.stat_cache[path] = st
return st

def _listdir(self, path: str) -> List[str]:
def listdir(self, path: str) -> List[str]:
if path in self.listdir_cache:
return self.listdir_cache[path]
if path in self.listdir_error_cache:
raise self.listdir_error_cache[path]
try:
return os.listdir(path)
results = os.listdir(path)
except Exception as err:
self.listdir_error_cache[path] = err
raise err
self.listdir_cache[path] = results
return results

def isfile(self, path: str) -> bool:
try:
Expand All @@ -84,6 +89,8 @@ def isfile_case(self, path: str) -> bool:
TODO: We should maybe check the case for some directory components also,
to avoid permitting wrongly-cased *packages*.
"""
if path in self.isfile_case_cache:
return self.isfile_case_cache[path]
head, tail = os.path.split(path)
if not tail:
res = False
Expand All @@ -93,6 +100,7 @@ def isfile_case(self, path: str) -> bool:
res = tail in names and self.isfile(path)
except OSError:
res = False
self.isfile_case_cache[path] = res
return res

def isdir(self, path: str) -> bool:
Expand Down
3 changes: 2 additions & 1 deletion mypy/stubgen.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,8 @@ def find_module_path_and_all(module: str, pyversion: Tuple[int, int],
module_all = getattr(mod, '__all__', None)
else:
# Find module by going through search path.
module_path = mypy.build.FindModuleCache().find_module(module, ['.'] + search_path,
module_path = mypy.build.FindModuleCache().find_module(module,
('.',) + tuple(search_path),
Copy link
Collaborator Author

@ethanhs ethanhs Apr 13, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@gvanrossum this change (just this!) is a bug on master. lib_path should be a tuple, and as you can see to the left, it is passed as a list. I can split this fix out if you don't want the rest. (This could cause issues with lru_cache).

(The rest is related to file system consistency)

EDIT: added more info.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the concrete effect here? Could this cause problems to users?

Copy link
Collaborator Author

@ethanhs ethanhs Apr 13, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Essentially, lru_cache won't be able to hash the passed list.

$ python -m mypy.stubgen --no-import mypy
Traceback (most recent call last):
  File "C:\Python36\lib\runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "C:\Python36\lib\runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "C:\Users\ethanhs\Documents\mypy\mypy\stubgen.py", line 1022, in <module>
    main()
  File "C:\Users\ethanhs\Documents\mypy\mypy\stubgen.py", line 901, in main
    raise e
  File "C:\Users\ethanhs\Documents\mypy\mypy\stubgen.py", line 898, in main
    include_private=options.include_private)
  File "C:\Users\ethanhs\Documents\mypy\mypy\stubgen.py", line 106, in generate_stub_for_module
    interpreter=interpreter)
  File "C:\Users\ethanhs\Documents\mypy\mypy\stubgen.py", line 164, in find_module_path_and_all
    interpreter)
TypeError: unhashable type: 'list'

So yes, at least changing the list to a tuple should go into 0.590.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Then maybe make a separate PR for that change?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, please split that out -- I can cherry-pick a 3-line fix to stubgen with a lot less risk and effort than this whole PR.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@gvanrossum #4899

interpreter)
if not module_path:
raise SystemExit(
Expand Down