From 210cb18ff21080fe6a85d42f93cdaeba7696fc8e Mon Sep 17 00:00:00 2001 From: beauxq Date: Fri, 6 Sep 2024 13:24:46 -0700 Subject: [PATCH 1/5] stdlib included --- .../src/analyzer/typeEvaluator.ts | 232 ++++++++++++++++++ 1 file changed, 232 insertions(+) diff --git a/packages/pyright-internal/src/analyzer/typeEvaluator.ts b/packages/pyright-internal/src/analyzer/typeEvaluator.ts index 329d6bd383..d1a13836cc 100644 --- a/packages/pyright-internal/src/analyzer/typeEvaluator.ts +++ b/packages/pyright-internal/src/analyzer/typeEvaluator.ts @@ -998,6 +998,238 @@ export function createTypeEvaluator( supportsKeysAndGetItemClass = mappingClass; } + // This makes it so auto-importer can find these stdlib modules. + // This is everything in typeshed stdlib that doesn't start with "_". + const stdlib = [ + "abc", + "aifc", + "antigravity", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "builtins", + "bz2", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "codecs", + "codeop", + "code", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "cProfile", + "crypt", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "dummy_threading", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "formatter", + "fractions", + "ftplib", + "functools", + "gc", + "genericpath", + "getopt", + "getpass", + "gettext", + "glob", + "graphlib", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "imaplib", + "imghdr", + "importlib", + "imp", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "ntpath", + "nt", + "nturl2path", + "numbers", + "opcode", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posixpath", + "posix", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "pyclbr", + "py_compile", + "pydoc_data", + "pydoc", + "pyexpat", + "queue", + "quopri", + "random", + "readline", + "reprlib", + "re", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "selectors", + "select", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "statistics", + "stat", + "stringprep", + "string", + "struct", + "subprocess", + "sunau", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "textwrap", + "this", + "threading", + "timeit", + "time", + "tkinter", + "tokenize", + "token", + "tomllib", + "traceback", + "tracemalloc", + "trace", + "tty", + "turtle", + "types", + "typing_extensions", + "typing", + "unicodedata", + "unittest", + "urllib", + "uuid", + "uu", + "venv", + "VERSIONS", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "xxlimited", + "zipapp", + "zipfile", + "zipimport", + "zlib", + "zoneinfo", + ]; + const fileInfo = AnalyzerNodeInfo.getFileInfo(node); + stdlib.forEach((moduleName) => { + importLookup({ nameParts: [moduleName], importingFileUri: fileInfo.fileUri }); + }); + // Wire up the `Any` class to the special-form version of our internal AnyType. if (objectClass && isInstantiableClass(objectClass) && typeClass && isInstantiableClass(typeClass)) { const anyClass = ClassType.createInstantiable( From 8be4f6a3cbde065efc5614ed578b2bacc3916d85 Mon Sep 17 00:00:00 2001 From: detachhead Date: Sun, 8 Sep 2024 11:19:24 +1000 Subject: [PATCH 2/5] rework how the stdlib modules are loaded to avoid having to define them all in an array and only do it in the language server but not the cli --- .../src/analyzer/importResolver.ts | 10 +- .../pyright-internal/src/analyzer/program.ts | 11 + .../src/analyzer/typeEvaluator.ts | 232 ------------------ .../src/common/extensibility.ts | 3 + .../src/languageService/completionProvider.ts | 1 + 5 files changed, 20 insertions(+), 237 deletions(-) diff --git a/packages/pyright-internal/src/analyzer/importResolver.ts b/packages/pyright-internal/src/analyzer/importResolver.ts index 845efb9e12..22d87ecae4 100644 --- a/packages/pyright-internal/src/analyzer/importResolver.ts +++ b/packages/pyright-internal/src/analyzer/importResolver.ts @@ -108,7 +108,7 @@ export class ImportResolver { private _cachedEntriesForPath = new Map(); private _cachedFilesForPath = new Map(); private _cachedDirExistenceForRoot = new Map(); - private _stdlibModules: Set | undefined; + stdlibModules: Set | undefined; protected readonly cachedParentImportResults: ParentDirectoryCache; @@ -142,7 +142,7 @@ export class ImportResolver { this._cachedImportResults = new Map(); this._cachedModuleNameResults = new Map>(); this.cachedParentImportResults.reset(); - this._stdlibModules = undefined; + this.stdlibModules = undefined; this._invalidateFileSystemCache(); @@ -338,11 +338,11 @@ export class ImportResolver { } isStdlibModule(module: ImportedModuleDescriptor, execEnv: ExecutionEnvironment): boolean { - if (!this._stdlibModules) { - this._stdlibModules = this._buildStdlibCache(this.getTypeshedStdLibPath(execEnv), execEnv); + if (!this.stdlibModules) { + this.stdlibModules = this._buildStdlibCache(this.getTypeshedStdLibPath(execEnv), execEnv); } - return this._stdlibModules.has(module.nameParts.join('.')); + return this.stdlibModules.has(module.nameParts.join('.')); } getImportRoots(execEnv: ExecutionEnvironment, forLogging = false) { diff --git a/packages/pyright-internal/src/analyzer/program.ts b/packages/pyright-internal/src/analyzer/program.ts index f1e169914b..2a62ecd14b 100644 --- a/packages/pyright-internal/src/analyzer/program.ts +++ b/packages/pyright-internal/src/analyzer/program.ts @@ -982,6 +982,17 @@ export class Program { return program; } + loadStdlibModules = (sourceFileUri: Uri) => { + this._importResolver?.stdlibModules?.forEach((module) => { + if (!module.includes('.')) { + this._lookUpImport({ + importingFileUri: sourceFileUri, + nameParts: [module], + }); + } + }); + }; + // Returns a value from 0 to 1 (or more) indicating how "full" the cache is // relative to some predetermined high-water mark. We'll compute this value // based on two easy-to-compute metrics: the number of entries in the type diff --git a/packages/pyright-internal/src/analyzer/typeEvaluator.ts b/packages/pyright-internal/src/analyzer/typeEvaluator.ts index d1a13836cc..329d6bd383 100644 --- a/packages/pyright-internal/src/analyzer/typeEvaluator.ts +++ b/packages/pyright-internal/src/analyzer/typeEvaluator.ts @@ -998,238 +998,6 @@ export function createTypeEvaluator( supportsKeysAndGetItemClass = mappingClass; } - // This makes it so auto-importer can find these stdlib modules. - // This is everything in typeshed stdlib that doesn't start with "_". - const stdlib = [ - "abc", - "aifc", - "antigravity", - "argparse", - "array", - "ast", - "asynchat", - "asyncio", - "asyncore", - "atexit", - "audioop", - "base64", - "bdb", - "binascii", - "binhex", - "bisect", - "builtins", - "bz2", - "calendar", - "cgi", - "cgitb", - "chunk", - "cmath", - "cmd", - "codecs", - "codeop", - "code", - "collections", - "colorsys", - "compileall", - "concurrent", - "configparser", - "contextlib", - "contextvars", - "copy", - "copyreg", - "cProfile", - "crypt", - "csv", - "ctypes", - "curses", - "dataclasses", - "datetime", - "dbm", - "decimal", - "difflib", - "dis", - "distutils", - "doctest", - "dummy_threading", - "email", - "encodings", - "ensurepip", - "enum", - "errno", - "faulthandler", - "fcntl", - "filecmp", - "fileinput", - "fnmatch", - "formatter", - "fractions", - "ftplib", - "functools", - "gc", - "genericpath", - "getopt", - "getpass", - "gettext", - "glob", - "graphlib", - "grp", - "gzip", - "hashlib", - "heapq", - "hmac", - "html", - "http", - "imaplib", - "imghdr", - "importlib", - "imp", - "inspect", - "io", - "ipaddress", - "itertools", - "json", - "keyword", - "lib2to3", - "linecache", - "locale", - "logging", - "lzma", - "mailbox", - "mailcap", - "marshal", - "math", - "mimetypes", - "mmap", - "modulefinder", - "msilib", - "msvcrt", - "multiprocessing", - "netrc", - "nis", - "nntplib", - "ntpath", - "nt", - "nturl2path", - "numbers", - "opcode", - "operator", - "optparse", - "os", - "ossaudiodev", - "parser", - "pathlib", - "pdb", - "pickle", - "pickletools", - "pipes", - "pkgutil", - "platform", - "plistlib", - "poplib", - "posixpath", - "posix", - "pprint", - "profile", - "pstats", - "pty", - "pwd", - "pyclbr", - "py_compile", - "pydoc_data", - "pydoc", - "pyexpat", - "queue", - "quopri", - "random", - "readline", - "reprlib", - "re", - "resource", - "rlcompleter", - "runpy", - "sched", - "secrets", - "selectors", - "select", - "shelve", - "shlex", - "shutil", - "signal", - "site", - "smtpd", - "smtplib", - "sndhdr", - "socket", - "socketserver", - "spwd", - "sqlite3", - "sre_compile", - "sre_constants", - "sre_parse", - "ssl", - "statistics", - "stat", - "stringprep", - "string", - "struct", - "subprocess", - "sunau", - "symbol", - "symtable", - "sys", - "sysconfig", - "syslog", - "tabnanny", - "tarfile", - "telnetlib", - "tempfile", - "termios", - "textwrap", - "this", - "threading", - "timeit", - "time", - "tkinter", - "tokenize", - "token", - "tomllib", - "traceback", - "tracemalloc", - "trace", - "tty", - "turtle", - "types", - "typing_extensions", - "typing", - "unicodedata", - "unittest", - "urllib", - "uuid", - "uu", - "venv", - "VERSIONS", - "warnings", - "wave", - "weakref", - "webbrowser", - "winreg", - "winsound", - "wsgiref", - "xdrlib", - "xml", - "xmlrpc", - "xxlimited", - "zipapp", - "zipfile", - "zipimport", - "zlib", - "zoneinfo", - ]; - const fileInfo = AnalyzerNodeInfo.getFileInfo(node); - stdlib.forEach((moduleName) => { - importLookup({ nameParts: [moduleName], importingFileUri: fileInfo.fileUri }); - }); - // Wire up the `Any` class to the special-form version of our internal AnyType. if (objectClass && isInstantiableClass(objectClass) && typeClass && isInstantiableClass(typeClass)) { const anyClass = ClassType.createInstantiable( diff --git a/packages/pyright-internal/src/common/extensibility.ts b/packages/pyright-internal/src/common/extensibility.ts index 1167b5a68a..c9b24aafd8 100644 --- a/packages/pyright-internal/src/common/extensibility.ts +++ b/packages/pyright-internal/src/common/extensibility.ts @@ -90,6 +90,9 @@ export interface ProgramView { // See whether we can get rid of these methods handleMemoryHighUsage(): void; clone(): prog.Program; + + // TODO: does this "mutate the program"? if so it should be moved somewhere else + loadStdlibModules: (sourceFileUri: Uri) => void; } // This exposes some APIs to mutate program. Unlike ProgramMutator, this will only mutate this program diff --git a/packages/pyright-internal/src/languageService/completionProvider.ts b/packages/pyright-internal/src/languageService/completionProvider.ts index cfa8196a3e..a2b6aa2cf9 100644 --- a/packages/pyright-internal/src/languageService/completionProvider.ts +++ b/packages/pyright-internal/src/languageService/completionProvider.ts @@ -309,6 +309,7 @@ export class CompletionProvider { return null; } + this.program.loadStdlibModules(this.fileUri); const completionMap = this._getCompletions(); return CompletionList.create(completionMap?.toArray()); } From 606cdf55ffd8c24f55e7dcb1696ac852f36192f2 Mon Sep 17 00:00:00 2001 From: detachhead Date: Sun, 8 Sep 2024 12:18:31 +1000 Subject: [PATCH 3/5] load stdlib modules in `Program.loadStdlibModules` even if they haven't yet been cached --- .../src/analyzer/importResolver.ts | 17 ++++++++++++----- .../pyright-internal/src/analyzer/program.ts | 18 ++++++++++-------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/packages/pyright-internal/src/analyzer/importResolver.ts b/packages/pyright-internal/src/analyzer/importResolver.ts index 22d87ecae4..aaa34890fe 100644 --- a/packages/pyright-internal/src/analyzer/importResolver.ts +++ b/packages/pyright-internal/src/analyzer/importResolver.ts @@ -108,7 +108,7 @@ export class ImportResolver { private _cachedEntriesForPath = new Map(); private _cachedFilesForPath = new Map(); private _cachedDirExistenceForRoot = new Map(); - stdlibModules: Set | undefined; + private _stdlibModules: Set | undefined; protected readonly cachedParentImportResults: ParentDirectoryCache; @@ -142,7 +142,7 @@ export class ImportResolver { this._cachedImportResults = new Map(); this._cachedModuleNameResults = new Map>(); this.cachedParentImportResults.reset(); - this.stdlibModules = undefined; + this._stdlibModules = undefined; this._invalidateFileSystemCache(); @@ -337,12 +337,19 @@ export class ImportResolver { return this._getThirdPartyTypeshedPath(this._configOptions.typeshedPath, unused); } + getStdlibModules = (execEnv: ExecutionEnvironment) => { + if (!this._stdlibModules) { + this._stdlibModules = this._buildStdlibCache(this.getTypeshedStdLibPath(execEnv), execEnv); + } + return this._stdlibModules; + }; + isStdlibModule(module: ImportedModuleDescriptor, execEnv: ExecutionEnvironment): boolean { - if (!this.stdlibModules) { - this.stdlibModules = this._buildStdlibCache(this.getTypeshedStdLibPath(execEnv), execEnv); + if (!this._stdlibModules) { + this._stdlibModules = this._buildStdlibCache(this.getTypeshedStdLibPath(execEnv), execEnv); } - return this.stdlibModules.has(module.nameParts.join('.')); + return this.getStdlibModules(execEnv).has(module.nameParts.join('.')); } getImportRoots(execEnv: ExecutionEnvironment, forLogging = false) { diff --git a/packages/pyright-internal/src/analyzer/program.ts b/packages/pyright-internal/src/analyzer/program.ts index 2a62ecd14b..e626e98e02 100644 --- a/packages/pyright-internal/src/analyzer/program.ts +++ b/packages/pyright-internal/src/analyzer/program.ts @@ -983,14 +983,16 @@ export class Program { } loadStdlibModules = (sourceFileUri: Uri) => { - this._importResolver?.stdlibModules?.forEach((module) => { - if (!module.includes('.')) { - this._lookUpImport({ - importingFileUri: sourceFileUri, - nameParts: [module], - }); - } - }); + this._importResolver + ?.getStdlibModules(this._configOptions.findExecEnvironment(sourceFileUri)) + .forEach((module) => { + if (!module.includes('.')) { + this._lookUpImport({ + importingFileUri: sourceFileUri, + nameParts: [module], + }); + } + }); }; // Returns a value from 0 to 1 (or more) indicating how "full" the cache is From 45f5f7a6be8d63795743944c3f03d8fa56599031 Mon Sep 17 00:00:00 2001 From: beauxq Date: Sun, 8 Sep 2024 08:50:52 -0700 Subject: [PATCH 4/5] adjustments to unit tests to test what they're supposed to be testing more specifically --- .../src/tests/completions.test.ts | 30 +++++++++---------- .../completions.builtinOverride.fourslash.ts | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/pyright-internal/src/tests/completions.test.ts b/packages/pyright-internal/src/tests/completions.test.ts index f42b78a1fe..fb53f484ea 100644 --- a/packages/pyright-internal/src/tests/completions.test.ts +++ b/packages/pyright-internal/src/tests/completions.test.ts @@ -369,12 +369,12 @@ test('include literals in expression completion', async () => { //// from typing import TypedDict //// //// class TestType(TypedDict): -//// A: str -//// B: int +//// key_a: str +//// key_b: int //// //// var: TestType = {} //// -//// var[[|A/*marker*/|]] +//// var[[|key_a/*marker*/|]] `; const state = parseAndGetTestState(code).state; @@ -386,8 +386,8 @@ test('include literals in expression completion', async () => { completions: [ { kind: CompletionItemKind.Constant, - label: "'A'", - textEdit: { range: state.getPositionRange('marker'), newText: "'A'" }, + label: "'key_a'", + textEdit: { range: state.getPositionRange('marker'), newText: "'key_a'" }, }, ], }, @@ -400,10 +400,10 @@ test('include literals in set key', async () => { //// from typing import TypedDict //// //// class TestType(TypedDict): -//// A: str -//// B: int +//// key_a: str +//// key_b: int //// -//// var: TestType = { [|A/*marker*/|] } +//// var: TestType = { [|key_a/*marker*/|] } `; const state = parseAndGetTestState(code).state; @@ -415,8 +415,8 @@ test('include literals in set key', async () => { completions: [ { kind: CompletionItemKind.Constant, - label: "'A'", - textEdit: { range: state.getPositionRange('marker'), newText: "'A'" }, + label: "'key_a'", + textEdit: { range: state.getPositionRange('marker'), newText: "'key_a'" }, }, ], }, @@ -429,10 +429,10 @@ test('include literals in dict key', async () => { //// from typing import TypedDict //// //// class TestType(TypedDict): -//// A: str -//// B: int +//// key_a: str +//// key_b: int //// -//// var: TestType = { [|A/*marker*/|] : "hello" } +//// var: TestType = { [|key_a/*marker*/|] : "hello" } `; const state = parseAndGetTestState(code).state; @@ -444,8 +444,8 @@ test('include literals in dict key', async () => { completions: [ { kind: CompletionItemKind.Constant, - label: '"A"', - textEdit: { range: state.getPositionRange('marker'), newText: '"A"' }, + label: '"key_a"', + textEdit: { range: state.getPositionRange('marker'), newText: '"key_a"' }, }, ], }, diff --git a/packages/pyright-internal/src/tests/fourslash/completions.builtinOverride.fourslash.ts b/packages/pyright-internal/src/tests/fourslash/completions.builtinOverride.fourslash.ts index 32cf9e8145..bece44c86c 100644 --- a/packages/pyright-internal/src/tests/fourslash/completions.builtinOverride.fourslash.ts +++ b/packages/pyright-internal/src/tests/fourslash/completions.builtinOverride.fourslash.ts @@ -1,7 +1,7 @@ /// // @filename: test.py -//// Cust[|/*marker1*/|] +//// Custo[|/*marker1*/|] //// my_v[|/*marker2*/|] // @filename: __builtins__.pyi From 51fd34803e8e5c4a15a3a0749215eaf8913c226f Mon Sep 17 00:00:00 2001 From: detachhead Date: Tue, 10 Sep 2024 21:31:19 +1000 Subject: [PATCH 5/5] fix completions test (see https://github.com/DetachHead/basedpyright/issues/663) --- .../tests/fourslash/completions.declNames.method.fourslash.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pyright-internal/src/tests/fourslash/completions.declNames.method.fourslash.ts b/packages/pyright-internal/src/tests/fourslash/completions.declNames.method.fourslash.ts index e45b9ebe93..7c9f6d1d18 100644 --- a/packages/pyright-internal/src/tests/fourslash/completions.declNames.method.fourslash.ts +++ b/packages/pyright-internal/src/tests/fourslash/completions.declNames.method.fourslash.ts @@ -20,7 +20,7 @@ //// def method(p:[|/*marker8*/|]): //// pass //// -//// def method(p, p2[|/*marker9*/|]): +//// def method(p, p21234[|/*marker9*/|]): //// pass //// def method(p, p2:[|/*marker10*/|]): //// pass @@ -37,7 +37,7 @@ //// def method(p:[|/*marker14*/|]): //// pass //// -//// def method(p, p2[|/*marker15*/|]): +//// def method(p, p21234[|/*marker15*/|]): //// pass //// def method(p, p2:[|/*marker16*/|]): //// pass