diff --git a/pythonFiles/interpreterInfo.py b/pythonFiles/interpreterInfo.py index bb284e729d71..601959b7c2d5 100644 --- a/pythonFiles/interpreterInfo.py +++ b/pythonFiles/interpreterInfo.py @@ -7,7 +7,7 @@ obj = {} obj["versionInfo"] = tuple(sys.version_info) obj["sysPrefix"] = sys.prefix -obj["version"] = sys.version +obj["sysVersion"] = sys.version obj["is64Bit"] = sys.maxsize > 2 ** 32 print(json.dumps(obj)) diff --git a/src/client/common/process/internal/scripts/index.ts b/src/client/common/process/internal/scripts/index.ts index 74408e81e1e6..30bb3d913d49 100644 --- a/src/client/common/process/internal/scripts/index.ts +++ b/src/client/common/process/internal/scripts/index.ts @@ -41,19 +41,19 @@ export * as vscode_datascience_helpers from './vscode_datascience_helpers'; type ReleaseLevel = 'alpha' | 'beta' | 'candidate' | 'final'; type PythonVersionInfo = [number, number, number, ReleaseLevel, number]; -export type PythonEnvInfo = { +export type InterpreterInfoJson = { versionInfo: PythonVersionInfo; sysPrefix: string; sysVersion: string; is64Bit: boolean; }; -export function interpreterInfo(): [string[], (out: string) => PythonEnvInfo] { +export function interpreterInfo(): [string[], (out: string) => InterpreterInfoJson] { const script = path.join(SCRIPTS_DIR, 'interpreterInfo.py'); const args = [ISOLATED, script]; - function parse(out: string): PythonEnvInfo { - let json: PythonEnvInfo; + function parse(out: string): InterpreterInfoJson { + let json: InterpreterInfoJson; try { json = JSON.parse(out); } catch (ex) { diff --git a/src/client/common/process/pythonDaemon.ts b/src/client/common/process/pythonDaemon.ts index b588af7f8053..225d58804609 100644 --- a/src/client/common/process/pythonDaemon.ts +++ b/src/client/common/process/pythonDaemon.ts @@ -11,7 +11,7 @@ import { extractInterpreterInfo } from '../../pythonEnvironments/info/interprete import { traceWarning } from '../logger'; import { IPlatformService } from '../platform/types'; import { BasePythonDaemon } from './baseDaemon'; -import { PythonEnvInfo } from './internal/scripts'; +import { InterpreterInfoJson } from './internal/scripts'; import { IPythonDaemonExecutionService, IPythonExecutionService, @@ -45,7 +45,9 @@ export class PythonDaemonExecutionService extends BasePythonDaemon implements IP public async getInterpreterInformation(): Promise { try { this.throwIfRPCConnectionIsDead(); - const request = new RequestType0('get_interpreter_information'); + const request = new RequestType0( + 'get_interpreter_information' + ); const response = await this.sendRequestWithoutArgs(request); if (response.error) { throw Error(response.error); diff --git a/src/client/pythonEnvironments/base/info/interpreter.ts b/src/client/pythonEnvironments/base/info/interpreter.ts new file mode 100644 index 000000000000..16d82d3280b5 --- /dev/null +++ b/src/client/pythonEnvironments/base/info/interpreter.ts @@ -0,0 +1,90 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { PythonExecutableInfo, PythonVersion } from '.'; +import { interpreterInfo as getInterpreterInfoCommand, InterpreterInfoJson } from '../../../common/process/internal/scripts'; +import { Architecture } from '../../../common/utils/platform'; +import { copyPythonExecInfo, PythonExecInfo } from '../../exec'; +import { parseVersion } from './pythonVersion'; + +export type InterpreterInformation = { + arch: Architecture; + executable: PythonExecutableInfo; + version: PythonVersion; +}; + +/** + * Compose full interpreter information based on the given data. + * + * The data format corresponds to the output of the `interpreterInfo.py` script. + * + * @param python - the path to the Python executable + * @param raw - the information returned by the `interpreterInfo.py` script + */ +function extractInterpreterInfo(python: string, raw: InterpreterInfoJson): InterpreterInformation { + const rawVersion = `${raw.versionInfo.slice(0, 3).join('.')}-${raw.versionInfo[3]}`; + return { + arch: raw.is64Bit ? Architecture.x64 : Architecture.x86, + executable: { + filename: python, + sysPrefix: raw.sysPrefix, + mtime: -1, + ctime: -1, + }, + version: { + ...parseVersion(rawVersion), + sysVersion: raw.sysVersion, + }, + }; +} + +type ShellExecResult = { + stdout: string; + stderr?: string; +}; +type ShellExecFunc = (command: string, timeout: number) => Promise; + +type Logger = { + info(msg: string): void; + + error(msg: string): void; +}; + +/** + * Collect full interpreter information from the given Python executable. + * + * @param python - the information to use when running Python + * @param shellExec - the function to use to exec Python + * @param logger - if provided, used to log failures or other info + */ +export async function getInterpreterInfo( + python: PythonExecInfo, + shellExec: ShellExecFunc, + logger?: Logger, +): Promise { + const [args, parse] = getInterpreterInfoCommand(); + const info = copyPythonExecInfo(python, args); + const argv = [info.command, ...info.args]; + + // Concat these together to make a set of quoted strings + const quoted = argv.reduce((p, c) => (p ? `${p} "${c}"` : `"${c.replace('\\', '\\\\')}"`), ''); + + // Try shell execing the command, followed by the arguments. This will make node kill the process if it + // takes too long. + // Sometimes the python path isn't valid, timeout if that's the case. + // See these two bugs: + // https://github.com/microsoft/vscode-python/issues/7569 + // https://github.com/microsoft/vscode-python/issues/7760 + const result = await shellExec(quoted, 15000); + if (result.stderr) { + if (logger) { + logger.error(`Failed to parse interpreter information for ${argv} stderr: ${result.stderr}`); + } + return undefined; + } + const json = parse(result.stdout); + if (logger) { + logger.info(`Found interpreter for ${argv}`); + } + return extractInterpreterInfo(python.pythonExecutable, json); +} diff --git a/src/client/pythonEnvironments/base/info/pythonVersion.ts b/src/client/pythonEnvironments/base/info/pythonVersion.ts new file mode 100644 index 000000000000..58248f2c1789 --- /dev/null +++ b/src/client/pythonEnvironments/base/info/pythonVersion.ts @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { PythonReleaseLevel, PythonVersion } from '.'; +import { EMPTY_VERSION, parseBasicVersionInfo } from '../../../common/utils/version'; + +export function parseVersion(versionStr: string): PythonVersion { + const parsed = parseBasicVersionInfo(versionStr); + if (!parsed) { + if (versionStr === '') { + return EMPTY_VERSION as PythonVersion; + } + throw Error(`invalid version ${versionStr}`); + } + const { version, after } = parsed; + const match = after.match(/^(a|b|rc)(\d+)$/); + if (match) { + const [, levelStr, serialStr] = match; + let level: PythonReleaseLevel; + if (levelStr === 'a') { + level = PythonReleaseLevel.Alpha; + } else if (levelStr === 'b') { + level = PythonReleaseLevel.Beta; + } else if (levelStr === 'rc') { + level = PythonReleaseLevel.Candidate; + } else { + throw Error('unreachable!'); + } + version.release = { + level, + serial: parseInt(serialStr, 10), + }; + } + return version; +} diff --git a/src/client/pythonEnvironments/info/environmentInfoService.ts b/src/client/pythonEnvironments/info/environmentInfoService.ts index 12a84f017692..6ab761c86278 100644 --- a/src/client/pythonEnvironments/info/environmentInfoService.ts +++ b/src/client/pythonEnvironments/info/environmentInfoService.ts @@ -2,11 +2,11 @@ // Licensed under the MIT License. import { injectable } from 'inversify'; -import { EnvironmentType, PythonEnvironment } from '.'; +import { createDeferred, Deferred } from '../../common/utils/async'; import { createWorkerPool, IWorkerPool, QueuePosition } from '../../common/utils/workerPool'; +import { getInterpreterInfo, InterpreterInformation } from '../base/info/interpreter'; import { shellExecute } from '../common/externalDependencies'; import { buildPythonExecInfo } from '../exec'; -import { getInterpreterInfo } from './interpreter'; export enum EnvironmentInfoServiceQueuePriority { Default, @@ -18,37 +18,17 @@ export interface IEnvironmentInfoService { getEnvironmentInfo( interpreterPath: string, priority?: EnvironmentInfoServiceQueuePriority - ): Promise; + ): Promise; } -async function buildEnvironmentInfo(interpreterPath: string): Promise { - const interpreterInfo = await getInterpreterInfo(buildPythonExecInfo(interpreterPath), shellExecute); +async function buildEnvironmentInfo(interpreterPath: string): Promise { + const interpreterInfo = await getInterpreterInfo(buildPythonExecInfo(interpreterPath), shellExecute).catch( + () => undefined, + ); if (interpreterInfo === undefined || interpreterInfo.version === undefined) { return undefined; } - return { - path: interpreterInfo.path, - // Have to do this because the type returned by getInterpreterInfo is SemVer - // But we expect this to be PythonVersion - version: { - raw: interpreterInfo.version.raw, - major: interpreterInfo.version.major, - minor: interpreterInfo.version.minor, - patch: interpreterInfo.version.patch, - build: interpreterInfo.version.build, - prerelease: interpreterInfo.version.prerelease, - }, - sysVersion: interpreterInfo.sysVersion, - architecture: interpreterInfo.architecture, - sysPrefix: interpreterInfo.sysPrefix, - pipEnvWorkspaceFolder: interpreterInfo.pipEnvWorkspaceFolder, - companyDisplayName: '', - displayName: '', - envType: EnvironmentType.Unknown, // Code to handle This will be added later. - envName: '', - envPath: '', - cachedEntry: false, - }; + return interpreterInfo; } @injectable() @@ -57,29 +37,35 @@ export class EnvironmentInfoService implements IEnvironmentInfoService { // path again and again in a given session. This information will likely not change in a given // session. There are definitely cases where this will change. But a simple reload should address // those. - private readonly cache: Map = new Map(); + private readonly cache: Map> = new Map< + string, + Deferred + >(); - private readonly workerPool: IWorkerPool; + private readonly workerPool: IWorkerPool; public constructor() { - this.workerPool = createWorkerPool(buildEnvironmentInfo); + this.workerPool = createWorkerPool(buildEnvironmentInfo); } public async getEnvironmentInfo( interpreterPath: string, priority?: EnvironmentInfoServiceQueuePriority, - ): Promise { + ): Promise { const result = this.cache.get(interpreterPath); if (result !== undefined) { - return result; + // Another call for this environment has already been made, return its result + return result.promise; } - + const deferred = createDeferred(); + this.cache.set(interpreterPath, deferred); return (priority === EnvironmentInfoServiceQueuePriority.High ? this.workerPool.addToQueue(interpreterPath, QueuePosition.Front) : this.workerPool.addToQueue(interpreterPath, QueuePosition.Back) ).then((r) => { - if (r !== undefined) { - this.cache.set(interpreterPath, r); + deferred.resolve(r); + if (r === undefined) { + this.cache.delete(interpreterPath); } return r; }); diff --git a/src/client/pythonEnvironments/info/interpreter.ts b/src/client/pythonEnvironments/info/interpreter.ts index 5082c38ea727..fb0b11f0506a 100644 --- a/src/client/pythonEnvironments/info/interpreter.ts +++ b/src/client/pythonEnvironments/info/interpreter.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. import { InterpreterInformation } from '.'; -import { interpreterInfo as getInterpreterInfoCommand, PythonEnvInfo } from '../../common/process/internal/scripts'; +import { interpreterInfo as getInterpreterInfoCommand, InterpreterInfoJson } from '../../common/process/internal/scripts'; import { Architecture } from '../../common/utils/platform'; import { copyPythonExecInfo, PythonExecInfo } from '../exec'; import { parsePythonVersion } from './pythonVersion'; @@ -15,7 +15,7 @@ import { parsePythonVersion } from './pythonVersion'; * @param python - the path to the Python executable * @param raw - the information returned by the `interpreterInfo.py` script */ -export function extractInterpreterInfo(python: string, raw: PythonEnvInfo): InterpreterInformation { +export function extractInterpreterInfo(python: string, raw: InterpreterInfoJson): InterpreterInformation { const rawVersion = `${raw.versionInfo.slice(0, 3).join('.')}-${raw.versionInfo[3]}`; return { architecture: raw.is64Bit ? Architecture.x64 : Architecture.x86, diff --git a/src/test/pythonEnvironments/base/common.ts b/src/test/pythonEnvironments/base/common.ts index e55e0a64391b..6f56011f1f03 100644 --- a/src/test/pythonEnvironments/base/common.ts +++ b/src/test/pythonEnvironments/base/common.ts @@ -3,13 +3,11 @@ import { createDeferred, flattenIterator, iterable, mapToIterator } from '../../../client/common/utils/async'; import { Architecture } from '../../../client/common/utils/platform'; -import { EMPTY_VERSION, parseBasicVersionInfo } from '../../../client/common/utils/version'; import { PythonEnvInfo, PythonEnvKind, - PythonReleaseLevel, - PythonVersion } from '../../../client/pythonEnvironments/base/info'; +import { parseVersion } from '../../../client/pythonEnvironments/base/info/pythonVersion'; import { IPythonEnvsIterator, Locator, PythonLocatorQuery } from '../../../client/pythonEnvironments/base/locator'; import { PythonEnvsChangedEvent } from '../../../client/pythonEnvironments/base/watcher'; @@ -45,36 +43,6 @@ export function createEnv( }; } -function parseVersion(versionStr: string): PythonVersion { - const parsed = parseBasicVersionInfo(versionStr); - if (!parsed) { - if (versionStr === '') { - return EMPTY_VERSION as PythonVersion; - } - throw Error(`invalid version ${versionStr}`); - } - const { version, after } = parsed; - const match = after.match(/^(a|b|rc)(\d+)$/); - if (match) { - const [, levelStr, serialStr ] = match; - let level: PythonReleaseLevel; - if (levelStr === 'a') { - level = PythonReleaseLevel.Alpha; - } else if (levelStr === 'b') { - level = PythonReleaseLevel.Beta; - } else if (levelStr === 'rc') { - level = PythonReleaseLevel.Candidate; - } else { - throw Error('unreachable!'); - } - version.release = { - level, - serial: parseInt(serialStr, 10) - }; - } - return version; -} - export function createLocatedEnv( location: string, versionStr: string, diff --git a/src/test/pythonEnvironments/info/environmentInfoService.functional.test.ts b/src/test/pythonEnvironments/info/environmentInfoService.functional.test.ts index 9d810f96f2d9..e1d64c353305 100644 --- a/src/test/pythonEnvironments/info/environmentInfoService.functional.test.ts +++ b/src/test/pythonEnvironments/info/environmentInfoService.functional.test.ts @@ -8,8 +8,9 @@ import * as sinon from 'sinon'; import { ImportMock } from 'ts-mock-imports'; import { ExecutionResult } from '../../../client/common/process/types'; import { Architecture } from '../../../client/common/utils/platform'; +import { InterpreterInformation } from '../../../client/pythonEnvironments/base/info/interpreter'; +import { parseVersion } from '../../../client/pythonEnvironments/base/info/pythonVersion'; import * as ExternalDep from '../../../client/pythonEnvironments/common/externalDependencies'; -import { EnvironmentType, PythonEnvironment } from '../../../client/pythonEnvironments/info'; import { EnvironmentInfoService, EnvironmentInfoServiceQueuePriority, @@ -18,27 +19,16 @@ import { suite('Environment Info Service', () => { let stubShellExec: sinon.SinonStub; - function createExpectedEnvInfo(path: string): PythonEnvironment { + function createExpectedEnvInfo(executable: string): InterpreterInformation { return { - path, - architecture: Architecture.x64, - sysVersion: undefined, - sysPrefix: 'path', - pipEnvWorkspaceFolder: undefined, - version: { - build: [], - major: 3, - minor: 8, - patch: 3, - prerelease: ['final'], - raw: '3.8.3-final', + version: { ...parseVersion('3.8.3-final'), sysVersion: '3.8.3 (tags/v3.8.3:6f8c832, May 13 2020, 22:37:02) [MSC v.1924 64 bit (AMD64)]' }, + arch: Architecture.x64, + executable: { + filename: executable, + sysPrefix: 'path', + mtime: -1, + ctime: -1, }, - companyDisplayName: '', - displayName: '', - envType: EnvironmentType.Unknown, - envName: '', - envPath: '', - cachedEntry: false, }; } @@ -49,7 +39,7 @@ suite('Environment Info Service', () => { new Promise>((resolve) => { resolve({ stdout: - '{"versionInfo": [3, 8, 3, "final", 0], "sysPrefix": "path", "version": "3.8.3 (tags/v3.8.3:6f8c832, May 13 2020, 22:37:02) [MSC v.1924 64 bit (AMD64)]", "is64Bit": true}', + '{"versionInfo": [3, 8, 3, "final", 0], "sysPrefix": "path", "sysVersion": "3.8.3 (tags/v3.8.3:6f8c832, May 13 2020, 22:37:02) [MSC v.1924 64 bit (AMD64)]", "is64Bit": true}', }); }), ); @@ -59,8 +49,8 @@ suite('Environment Info Service', () => { }); test('Add items to queue and get results', async () => { const envService = new EnvironmentInfoService(); - const promises: Promise[] = []; - const expected: PythonEnvironment[] = []; + const promises: Promise[] = []; + const expected: InterpreterInformation[] = []; for (let i = 0; i < 10; i = i + 1) { const path = `any-path${i}`; if (i < 5) { @@ -82,8 +72,8 @@ suite('Environment Info Service', () => { test('Add same item to queue', async () => { const envService = new EnvironmentInfoService(); - const promises: Promise[] = []; - const expected: PythonEnvironment[] = []; + const promises: Promise[] = []; + const expected: InterpreterInformation[] = []; const path = 'any-path'; // Clear call counts