diff --git a/mypy/build.py b/mypy/build.py index 5c6ed710ceb9..a164f5fd8d84 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -389,7 +389,6 @@ def default_lib_path(data_dir: str, CacheMeta = NamedTuple('CacheMeta', [('id', str), ('path', str), - ('memory_only', bool), # no corresponding json files (fine-grained only) ('mtime', int), ('size', int), ('hash', str), @@ -415,7 +414,6 @@ def cache_meta_from_dict(meta: Dict[str, Any], data_json: str) -> CacheMeta: return CacheMeta( meta.get('id', sentinel), meta.get('path', sentinel), - meta.get('memory_only', False), int(meta['mtime']) if 'mtime' in meta else sentinel, meta.get('size', sentinel), meta.get('hash', sentinel), @@ -1121,12 +1119,6 @@ def validate_meta(meta: Optional[CacheMeta], id: str, path: Optional[str], manager.log('Metadata abandoned for {}: errors were previously ignored'.format(id)) return None - if meta.memory_only: - # Special case for fine-grained incremental mode when the JSON file is missing but - # we want to cache the module anyway. - manager.log('Memory-only metadata for {}'.format(id)) - return meta - assert path is not None, "Internal error: meta was provided without a path" # Check data_json; assume if its mtime matches it's good. # TODO: stat() errors diff --git a/mypy/server/update.py b/mypy/server/update.py index e09174b91059..d76c462771b0 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -115,15 +115,14 @@ - Fully support multiple type checking passes - Use mypy.fscache to access file system -- Don't use load_graph() and update the import graph incrementally """ import os.path from typing import Dict, List, Set, Tuple, Iterable, Union, Optional, Mapping, NamedTuple from mypy.build import ( - BuildManager, State, BuildSource, Graph, load_graph, SavedCache, CacheMeta, - cache_meta_from_dict, find_module_clear_caches, DEBUG_FINE_GRAINED + BuildManager, State, BuildSource, Graph, load_graph, find_module_clear_caches, + DEBUG_FINE_GRAINED, ) from mypy.checker import DeferredNode from mypy.errors import Errors, CompileError @@ -172,7 +171,6 @@ def __init__(self, # this directly reflected in load_graph's interface. self.options.cache_dir = os.devnull manager.saved_cache = {} - self.type_maps = extract_type_maps(graph) # Active triggers during the last update self.triggered = [] # type: List[str] @@ -253,6 +251,7 @@ def update_single(self, module: str, path: str) -> Tuple[List[str], # TODO: If new module brings in other modules, we parse some files multiple times. manager = self.manager previous_modules = self.previous_modules + graph = self.graph # Record symbol table snaphot of old version the changed module. old_snapshots = {} # type: Dict[str, Dict[str, SnapshotItem]] @@ -261,14 +260,14 @@ def update_single(self, module: str, path: str) -> Tuple[List[str], old_snapshots[module] = snapshot manager.errors.reset() - result = update_single_isolated(module, path, manager, previous_modules, self.graph) + result = update_single_isolated(module, path, manager, previous_modules, graph) if isinstance(result, BlockedUpdate): # Blocking error -- just give up module, path, remaining, errors = result self.previous_modules = get_module_to_path_map(manager) return errors, remaining, (module, path), True assert isinstance(result, NormalUpdate) # Work around #4124 - module, path, remaining, tree, graph = result + module, path, remaining, tree = result # TODO: What to do with stale dependencies? triggered = calculate_active_triggers(manager, old_snapshots, {module: tree}) @@ -285,20 +284,7 @@ def update_single(self, module: str, path: str) -> Tuple[List[str], # Preserve state needed for the next update. self.previous_targets_with_errors = manager.errors.targets() - # If deleted, module won't be in the graph. - if module in graph: - # Generate metadata so that we can reuse the AST in the next run. - graph[module].write_cache() - for id, state in graph.items(): - # Look up missing ASTs from saved cache. - if state.tree is None and id in manager.saved_cache: - meta, tree, type_map = manager.saved_cache[id] - state.tree = tree self.previous_modules = get_module_to_path_map(manager) - self.type_maps = extract_type_maps(graph) - - # XXX: I want us to not need this - self.graph = graph return manager.errors.new_messages(), remaining, (module, path), False @@ -317,15 +303,13 @@ def get_all_dependencies(manager: BuildManager, graph: Dict[str, State], # - Id of the changed module (can be different from the module argument) # - Path of the changed module # - New AST for the changed module (None if module was deleted) -# - The entire updated build graph # - Remaining changed modules that are not processed yet as (module id, path) # tuples (non-empty if the original changed module imported other new # modules) NormalUpdate = NamedTuple('NormalUpdate', [('module', str), ('path', str), ('remaining', List[Tuple[str, str]]), - ('tree', Optional[MypyFile]), - ('graph', Graph)]) + ('tree', Optional[MypyFile])]) # The result of update_single_isolated when there is a blocking error. Items # are similar to NormalUpdate (but there are fewer). @@ -362,10 +346,11 @@ def update_single_isolated(module: str, old_modules = dict(manager.modules) sources = get_sources(previous_modules, [(module, path)]) - invalidate_stale_cache_entries(manager.saved_cache, graph, [(module, path)]) manager.missing_modules.clear() try: + if module in graph: + del graph[module] load_graph(sources, manager, graph) except CompileError as err: # Parse error somewhere in the program -- a blocker @@ -383,8 +368,8 @@ def update_single_isolated(module: str, return BlockedUpdate(err.module_with_blocker, path, remaining_modules, err.messages) if not os.path.isfile(path): - graph = delete_module(module, graph, manager) - return NormalUpdate(module, path, [], None, graph) + delete_module(module, graph, manager) + return NormalUpdate(module, path, [], None) # Find any other modules brought in by imports. changed_modules = get_all_changed_modules(module, path, previous_modules, graph) @@ -438,7 +423,7 @@ def update_single_isolated(module: str, graph[module] = state - return NormalUpdate(module, path, remaining_modules, state.tree, graph) + return NormalUpdate(module, path, remaining_modules, state.tree) def find_relative_leaf_module(modules: List[Tuple[str, str]], graph: Graph) -> Tuple[str, str]: @@ -475,14 +460,13 @@ def assert_equivalent_paths(path1: str, path2: str) -> None: def delete_module(module_id: str, - graph: Dict[str, State], - manager: BuildManager) -> Dict[str, State]: + graph: Graph, + manager: BuildManager) -> None: manager.log_fine_grained('delete module %r' % module_id) # TODO: Deletion of a package # TODO: Remove deps for the module (this only affects memory use, not correctness) - new_graph = graph.copy() - if module_id in new_graph: - del new_graph[module_id] + if module_id in graph: + del graph[module_id] if module_id in manager.modules: del manager.modules[module_id] if module_id in manager.saved_cache: @@ -496,7 +480,6 @@ def delete_module(module_id: str, parent = manager.modules[parent_id] if components[-1] in parent.names: del parent.names[components[-1]] - return new_graph def dedupe_modules(modules: List[Tuple[str, str]]) -> List[Tuple[str, str]]: @@ -518,15 +501,10 @@ def get_sources(modules: Dict[str, str], changed_modules: List[Tuple[str, str]]) -> List[BuildSource]: # TODO: Race condition when reading from the file system; we should only read each # bit of external state once during a build to have a consistent view of the world - items = sorted(modules.items(), key=lambda x: x[0]) - sources = [BuildSource(path, id, None) - for id, path in items - if os.path.isfile(path)] sources = [] for id, path in changed_modules: - if os.path.isfile(path):# and id not in modules: + if os.path.isfile(path): sources.append(BuildSource(path, id, None)) - # print(changed_modules, sources) return sources @@ -544,16 +522,6 @@ def get_all_changed_modules(root_module: str, return changed_modules -def invalidate_stale_cache_entries(cache: SavedCache, - graph: Graph, - changed_modules: List[Tuple[str, str]]) -> None: - for name, _ in changed_modules: - if name in cache: - del cache[name] - if name in graph: - del graph[name] - - def verify_dependencies(state: State, manager: BuildManager) -> None: """Report errors for import targets in module that don't exist.""" for dep in state.dependencies + state.suppressed: # TODO: ancestors? @@ -907,11 +875,5 @@ def lookup_target(modules: Dict[str, MypyFile], target: str) -> List[DeferredNod return [DeferredNode(node, active_class_name, active_class)] -def extract_type_maps(graph: Graph) -> Dict[str, Dict[Expression, Type]]: - # This is used to export information used only by the testmerge harness. - return {id: state.type_map() for id, state in graph.items() - if state.tree} - - def is_verbose(manager: BuildManager) -> bool: return manager.options.verbosity >= 1 or DEBUG_FINE_GRAINED diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 4cca36ef3503..0b1f1573760e 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -149,7 +149,6 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int = 0) options = parse_options(original_program_text, testcase, incremental_step) options.use_builtins_fixtures = True options.show_traceback = True - options.verbosity = 1 if 'optional' in testcase.file: options.strict_optional = True if incremental_step: diff --git a/mypy/test/testmerge.py b/mypy/test/testmerge.py index cc79958d5c15..20dd7f9d6980 100644 --- a/mypy/test/testmerge.py +++ b/mypy/test/testmerge.py @@ -5,7 +5,7 @@ from typing import List, Tuple, Dict, Optional from mypy import build -from mypy.build import BuildManager, BuildSource, State +from mypy.build import BuildManager, BuildSource, State, Graph from mypy.errors import Errors, CompileError from mypy.nodes import ( Node, MypyFile, SymbolTable, SymbolTableNode, TypeInfo, Expression, Var, UNBOUND_IMPORTED @@ -77,13 +77,13 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: target_path = os.path.join(test_temp_dir, 'target.py') shutil.copy(os.path.join(test_temp_dir, 'target.py.next'), target_path) - a.extend(self.dump(manager, kind)) + a.extend(self.dump(fine_grained_manager, kind)) old_subexpr = get_subexpressions(manager.modules['target']) a.append('==>') new_file, new_types = self.build_increment(fine_grained_manager, 'target', target_path) - a.extend(self.dump(manager, kind)) + a.extend(self.dump(fine_grained_manager, kind)) for expr in old_subexpr: # Verify that old AST nodes are removed from the expression type map. @@ -119,13 +119,13 @@ def build_increment(self, manager: FineGrainedBuildManager, Dict[Expression, Type]]: manager.update([(module_id, path)]) module = manager.manager.modules[module_id] - type_map = manager.type_maps[module_id] + type_map = manager.graph[module_id].type_map() return module, type_map def dump(self, - manager: BuildManager, + manager: FineGrainedBuildManager, kind: str) -> List[str]: - modules = manager.modules + modules = manager.manager.modules if kind == AST: return self.dump_asts(modules) elif kind == TYPEINFO: @@ -203,14 +203,14 @@ def dump_typeinfo(self, info: TypeInfo) -> List[str]: type_str_conv=self.type_str_conv) return s.splitlines() - def dump_types(self, manager: BuildManager) -> List[str]: + def dump_types(self, manager: FineGrainedBuildManager) -> List[str]: a = [] # To make the results repeatable, we try to generate unique and # deterministic sort keys. - for module_id in sorted(manager.modules): + for module_id in sorted(manager.manager.modules): if not is_dumped_module(module_id): continue - type_map = manager.saved_cache[module_id][2] + type_map = manager.graph[module_id].type_map() if type_map: a.append('## {}'.format(module_id)) for expr in sorted(type_map, key=lambda n: (n.line, short_type(n),