From 46d4ab47e1cb6b1cccb250e37e69aea969b2f9b4 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Fri, 26 Jan 2024 00:09:59 -0800 Subject: [PATCH 1/6] stubtest: adjust symtable logic --- mypy/stubtest.py | 51 ++++++++++++++++++--------------------- mypy/test/teststubtest.py | 18 ++++++++++++++ 2 files changed, 41 insertions(+), 28 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 6061e98bd7cd..d221c486dc0d 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -10,6 +10,7 @@ import collections.abc import copy import enum +import functools import importlib import importlib.machinery import inspect @@ -310,35 +311,23 @@ def _verify_exported_names( ) -def _get_imported_symbol_names(runtime: types.ModuleType) -> frozenset[str] | None: - """Retrieve the names in the global namespace which are known to be imported. +@functools.lru_cache +def _module_symbol_table(runtime: types.ModuleType) -> symtable.SymbolTable | None: + """Retrieve the symbol table for the module (or None on failure). - 1). Use inspect to retrieve the source code of the module - 2). Use symtable to parse the source and retrieve names that are known to be imported - from other modules. - - If either of the above steps fails, return `None`. - - Note that if a set of names is returned, - it won't include names imported via `from foo import *` imports. + 1) Use inspect to retrieve the source code of the module + 2) Use symtable to parse the source (and use what symtable knows for its purposes) """ try: source = inspect.getsource(runtime) except (OSError, TypeError, SyntaxError): return None - if not source.strip(): - # The source code for the module was an empty file, - # no point in parsing it with symtable - return frozenset() - try: - module_symtable = symtable.symtable(source, runtime.__name__, "exec") + return symtable.symtable(source, runtime.__name__, "exec") except SyntaxError: return None - return frozenset(sym.get_name() for sym in module_symtable.get_symbols() if sym.is_imported()) - @verify.register(nodes.MypyFile) def verify_mypyfile( @@ -369,25 +358,31 @@ def verify_mypyfile( if not o.module_hidden and (not is_probably_private(m) or hasattr(runtime, m)) } - imported_symbols = _get_imported_symbol_names(runtime) - def _belongs_to_runtime(r: types.ModuleType, attr: str) -> bool: """Heuristics to determine whether a name originates from another module.""" obj = getattr(r, attr) if isinstance(obj, types.ModuleType): return False - if callable(obj): - # It's highly likely to be a class or a function if it's callable, - # so the __module__ attribute will give a good indication of which module it comes from + + symbols = _module_symbol_table(r) + if symbols is not None: + if attr in {sym.get_name() for sym in symbols.get_symbols() if sym.is_imported()}: + # symtable says we got this from another module + return False + + # but we can't just return True here, because symtable doesn't know about star imports + if attr in {sym.get_name() for sym in symbols.get_symbols() if sym.is_assigned()}: + # symtable knows we assigned this symbol in the module + return True + + if hasattr(obj, "__module__"): try: obj_mod = obj.__module__ - except Exception: - pass - else: if isinstance(obj_mod, str): return bool(obj_mod == r.__name__) - if imported_symbols is not None: - return attr not in imported_symbols + except Exception: + pass + return True runtime_public_contents = ( diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index c0bb2eb6f9da..1e7d6a8bae4a 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -1266,6 +1266,24 @@ def test_missing_no_runtime_all(self) -> Iterator[Case]: yield Case(stub="", runtime="from json.scanner import NUMBER_RE", error=None) yield Case(stub="", runtime="from string import ascii_letters", error=None) + @collect_cases + def test_missing_no_runtime_terrible(self) -> Iterator[Case]: + yield Case( + stub="", + runtime=""" +import sys +import types +import __future__ +_m = types.SimpleNamespace() +_m.annotations = __future__.annotations +sys.modules["_terrible_stubtest_test_module"] = _m + +from _terrible_stubtest_test_module import * +assert annotations +""", + error=None, + ) + @collect_cases def test_non_public_1(self) -> Iterator[Case]: yield Case( From 673a9e8b1cbb0fab74b52f2602c035236744e4a5 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Fri, 26 Jan 2024 00:17:52 -0800 Subject: [PATCH 2/6] remove change --- mypy/stubtest.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index d221c486dc0d..a62ef3f2f530 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -374,15 +374,13 @@ def _belongs_to_runtime(r: types.ModuleType, attr: str) -> bool: if attr in {sym.get_name() for sym in symbols.get_symbols() if sym.is_assigned()}: # symtable knows we assigned this symbol in the module return True - - if hasattr(obj, "__module__"): - try: - obj_mod = obj.__module__ - if isinstance(obj_mod, str): - return bool(obj_mod == r.__name__) - except Exception: - pass - + try: + obj_mod = obj.__module__ + except Exception: + pass + else: + if isinstance(obj_mod, str): + return bool(obj_mod == r.__name__) return True runtime_public_contents = ( From 3806b8c954ed9271959d041b399691cce2ef5953 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Fri, 26 Jan 2024 00:18:46 -0800 Subject: [PATCH 3/6] . --- mypy/test/teststubtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index 1e7d6a8bae4a..707be1f0e359 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -1267,7 +1267,7 @@ def test_missing_no_runtime_all(self) -> Iterator[Case]: yield Case(stub="", runtime="from string import ascii_letters", error=None) @collect_cases - def test_missing_no_runtime_terrible(self) -> Iterator[Case]: + def test_missing_no_runtime_all_terrible(self) -> Iterator[Case]: yield Case( stub="", runtime=""" From 0bda3e0e5daf8249678c6e3ce97bfc889a7fe7d8 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Fri, 26 Jan 2024 12:07:37 -0800 Subject: [PATCH 4/6] some feedback --- mypy/stubtest.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index a62ef3f2f530..4cd01f6a4a53 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -366,14 +366,17 @@ def _belongs_to_runtime(r: types.ModuleType, attr: str) -> bool: symbols = _module_symbol_table(r) if symbols is not None: - if attr in {sym.get_name() for sym in symbols.get_symbols() if sym.is_imported()}: + if any(attr == sym.get_name() for sym in symbols.get_symbols() if sym.is_imported()): # symtable says we got this from another module return False - # but we can't just return True here, because symtable doesn't know about star imports + # But we can't just return True here, because symtable doesn't know about star imports if attr in {sym.get_name() for sym in symbols.get_symbols() if sym.is_assigned()}: # symtable knows we assigned this symbol in the module return True + + # The __module__ attribute is unreliable for anything except functions and classes, + # but it's our best guess at this point try: obj_mod = obj.__module__ except Exception: From 62682e5b4766be8d77bb0cd8fc9a7ae94ec2d61f Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Sat, 27 Jan 2024 00:03:12 -0800 Subject: [PATCH 5/6] use symtable lookup --- mypy/stubtest.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 4cd01f6a4a53..e967bf8d8a2a 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -364,16 +364,21 @@ def _belongs_to_runtime(r: types.ModuleType, attr: str) -> bool: if isinstance(obj, types.ModuleType): return False - symbols = _module_symbol_table(r) - if symbols is not None: - if any(attr == sym.get_name() for sym in symbols.get_symbols() if sym.is_imported()): - # symtable says we got this from another module - return False - - # But we can't just return True here, because symtable doesn't know about star imports - if attr in {sym.get_name() for sym in symbols.get_symbols() if sym.is_assigned()}: - # symtable knows we assigned this symbol in the module - return True + symbol_table = _module_symbol_table(r) + if symbol_table is not None: + try: + symbol = symbol_table.lookup(attr) + except KeyError: + pass + else: + if symbol.is_imported(): + # symtable says we got this from another module + return False + # But we can't just return True here, because symtable doesn't know about symbols + # that come from `from module import *` + if symbol is not None and symbol.is_assigned(): + # symtable knows we assigned this symbol in the module + return True # The __module__ attribute is unreliable for anything except functions and classes, # but it's our best guess at this point From 73aaf87a5fec9337b60a28fd53766a8f0a7c8452 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Sat, 27 Jan 2024 00:05:44 -0800 Subject: [PATCH 6/6] . --- mypy/stubtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index e967bf8d8a2a..32fe46f76d5e 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -376,7 +376,7 @@ def _belongs_to_runtime(r: types.ModuleType, attr: str) -> bool: return False # But we can't just return True here, because symtable doesn't know about symbols # that come from `from module import *` - if symbol is not None and symbol.is_assigned(): + if symbol.is_assigned(): # symtable knows we assigned this symbol in the module return True