From 43c059ee9f453a8f5e584a2fdd5ee8461b51410d Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Fri, 28 Oct 2022 11:07:20 -0700 Subject: [PATCH] Resolve conda environments without relying on the updated list of conda envs (#20112) Closes https://github.com/microsoft/vscode-python/issues/20069 Closes https://github.com/microsoft/vscode-python/issues/20110 Closes https://github.com/microsoft/vscode-python/issues/20070 --- src/client/proposedApi.ts | 19 ++-- src/client/proposedApiTypes.ts | 32 ++++--- .../base/locators/composite/envsResolver.ts | 4 +- .../base/locators/composite/resolverUtils.ts | 88 ++++++++----------- .../base/locators/lowLevel/condaLocator.ts | 8 +- .../common/environmentManagers/conda.ts | 65 +++++++------- .../composite/resolverUtils.unit.test.ts | 52 ++++++----- 7 files changed, 128 insertions(+), 140 deletions(-) diff --git a/src/client/proposedApi.ts b/src/client/proposedApi.ts index 2bae98edef11..46b726304261 100644 --- a/src/client/proposedApi.ts +++ b/src/client/proposedApi.ts @@ -271,19 +271,19 @@ export function convertCompleteEnvInfo(env: PythonEnvInfo): ResolvedEnvironment path, id: getEnvID(path), executable: { - uri: Uri.file(env.executable.filename), + uri: env.executable.filename === 'python' ? undefined : Uri.file(env.executable.filename), bitness: convertBitness(env.arch), sysPrefix: env.executable.sysPrefix, }, environment: env.type ? { type: convertEnvType(env.type), - name: env.name, + name: env.name === '' ? undefined : env.name, folderUri: Uri.file(env.location), workspaceFolder: env.searchLocation, } : undefined, - version: version as ResolvedEnvironment['version'], + version: env.executable.filename === 'python' ? undefined : (version as ResolvedEnvironment['version']), tools: tool ? [tool] : [], }; return resolvedEnv; @@ -325,19 +325,16 @@ export function convertEnvInfo(env: PythonEnvInfo): Environment { if (convertedEnv.executable.sysPrefix === '') { convertedEnv.executable.sysPrefix = undefined; } - if (convertedEnv.executable.uri?.fsPath === 'python') { - convertedEnv.executable.uri = undefined; + if (convertedEnv.version?.sysVersion === '') { + convertedEnv.version.sysVersion = undefined; } - if (convertedEnv.environment?.name === '') { - convertedEnv.environment.name = undefined; - } - if (convertedEnv.version.major === -1) { + if (convertedEnv.version?.major === -1) { convertedEnv.version.major = undefined; } - if (convertedEnv.version.micro === -1) { + if (convertedEnv.version?.micro === -1) { convertedEnv.version.micro = undefined; } - if (convertedEnv.version.minor === -1) { + if (convertedEnv.version?.minor === -1) { convertedEnv.version.minor = undefined; } return convertedEnv as Environment; diff --git a/src/client/proposedApiTypes.ts b/src/client/proposedApiTypes.ts index b2a2d3d80b97..4c7b49c6257f 100644 --- a/src/client/proposedApiTypes.ts +++ b/src/client/proposedApiTypes.ts @@ -129,14 +129,16 @@ export type Environment = EnvironmentPath & { } | undefined; /** - * Carries Python version information known at this moment. + * Carries Python version information known at this moment, carries `undefined` for envs without python. */ - readonly version: VersionInfo & { - /** - * Value of `sys.version` in sys module if known at this moment. - */ - readonly sysVersion: string | undefined; - }; + readonly version: + | (VersionInfo & { + /** + * Value of `sys.version` in sys module if known at this moment. + */ + readonly sysVersion: string | undefined; + }) + | undefined; /** * Tools/plugins which created the environment or where it came from. First value in array corresponds * to the primary tool which manages the environment, which never changes over time. @@ -171,14 +173,16 @@ export type ResolvedEnvironment = Environment & { readonly sysPrefix: string; }; /** - * Carries complete Python version information. + * Carries complete Python version information, carries `undefined` for envs without python. */ - readonly version: ResolvedVersionInfo & { - /** - * Value of `sys.version` in sys module if known at this moment. - */ - readonly sysVersion: string; - }; + readonly version: + | (ResolvedVersionInfo & { + /** + * Value of `sys.version` in sys module if known at this moment. + */ + readonly sysVersion: string; + }) + | undefined; }; export type EnvironmentsChangeEvent = { diff --git a/src/client/pythonEnvironments/base/locators/composite/envsResolver.ts b/src/client/pythonEnvironments/base/locators/composite/envsResolver.ts index 9b12230b8499..c3d159d2ff84 100644 --- a/src/client/pythonEnvironments/base/locators/composite/envsResolver.ts +++ b/src/client/pythonEnvironments/base/locators/composite/envsResolver.ts @@ -95,7 +95,7 @@ export class PythonEnvsResolver implements IResolvingLocator { } else if (seen[event.index] !== undefined) { const old = seen[event.index]; await setKind(event.update, environmentKinds); - seen[event.index] = await resolveBasicEnv(event.update, true); + seen[event.index] = await resolveBasicEnv(event.update); didUpdate.fire({ old, index: event.index, update: seen[event.index] }); this.resolveInBackground(event.index, state, didUpdate, seen).ignoreErrors(); } else { @@ -113,7 +113,7 @@ export class PythonEnvsResolver implements IResolvingLocator { while (!result.done) { // Use cache from the current refresh where possible. await setKind(result.value, environmentKinds); - const currEnv = await resolveBasicEnv(result.value, true); + const currEnv = await resolveBasicEnv(result.value); seen.push(currEnv); yield currEnv; this.resolveInBackground(seen.indexOf(currEnv), state, didUpdate, seen).ignoreErrors(); diff --git a/src/client/pythonEnvironments/base/locators/composite/resolverUtils.ts b/src/client/pythonEnvironments/base/locators/composite/resolverUtils.ts index 44a69019601c..c0506d4a06ba 100644 --- a/src/client/pythonEnvironments/base/locators/composite/resolverUtils.ts +++ b/src/client/pythonEnvironments/base/locators/composite/resolverUtils.ts @@ -12,18 +12,8 @@ import { UNKNOWN_PYTHON_VERSION, virtualEnvKinds, } from '../../info'; -import { - buildEnvInfo, - comparePythonVersionSpecificity, - setEnvDisplayString, - areSameEnv, - getEnvID, -} from '../../info/env'; -import { - getEnvironmentDirFromPath, - getInterpreterPathFromDir, - getPythonVersionFromPath, -} from '../../../common/commonUtils'; +import { buildEnvInfo, comparePythonVersionSpecificity, setEnvDisplayString, getEnvID } from '../../info/env'; +import { getEnvironmentDirFromPath, getPythonVersionFromPath } from '../../../common/commonUtils'; import { arePathsSame, getFileInfo, isParentPath } from '../../../common/externalDependencies'; import { AnacondaCompanyName, Conda, isCondaEnvironment } from '../../../common/environmentManagers/conda'; import { getPyenvVersionsDir, parsePyenvVersion } from '../../../common/environmentManagers/pyenv'; @@ -36,8 +26,8 @@ import { traceError, traceWarn } from '../../../../logging'; import { isVirtualEnvironment } from '../../../common/environmentManagers/simplevirtualenvs'; import { getWorkspaceFolderPaths } from '../../../../common/vscodeApis/workspaceApis'; -function getResolvers(): Map Promise> { - const resolvers = new Map Promise>(); +function getResolvers(): Map Promise> { + const resolvers = new Map Promise>(); Object.values(PythonEnvKind).forEach((k) => { resolvers.set(k, resolveGloballyInstalledEnv); }); @@ -55,11 +45,11 @@ function getResolvers(): Map { +export async function resolveBasicEnv(env: BasicEnvInfo): Promise { const { kind, source } = env; const resolvers = getResolvers(); const resolverForKind = resolvers.get(kind)!; - const resolvedEnv = await resolverForKind(env, useCache); + const resolvedEnv = await resolverForKind(env); resolvedEnv.searchLocation = getSearchLocation(resolvedEnv); resolvedEnv.source = uniq(resolvedEnv.source.concat(source ?? [])); if (getOSType() === OSType.Windows && resolvedEnv.source?.includes(PythonEnvSource.WindowsRegistry)) { @@ -165,47 +155,41 @@ async function resolveSimpleEnv(env: BasicEnvInfo): Promise { return envInfo; } -async function resolveCondaEnv(env: BasicEnvInfo, useCache?: boolean): Promise { +async function resolveCondaEnv(env: BasicEnvInfo): Promise { const { executablePath } = env; const conda = await Conda.getConda(); if (conda === undefined) { - traceWarn(`${executablePath} identified as Conda environment even though Conda is not installed`); + traceWarn(`${executablePath} identified as Conda environment even though Conda is not found`); + // Environment could still be valid, resolve as a simple env. + env.kind = PythonEnvKind.Unknown; + const envInfo = await resolveSimpleEnv(env); + envInfo.type = PythonEnvType.Conda; + // Assume it's a prefixed env by default because prefixed CLIs work even for named environments. + envInfo.name = ''; + return envInfo; } - const envs = (await conda?.getEnvList(useCache)) ?? []; - for (const { name, prefix } of envs) { - let executable = await getInterpreterPathFromDir(prefix); - const currEnv: BasicEnvInfo = { executablePath: executable ?? '', kind: PythonEnvKind.Conda, envPath: prefix }; - if (areSameEnv(env, currEnv)) { - if (env.executablePath.length > 0) { - executable = env.executablePath; - } else { - executable = await conda?.getInterpreterPathForEnvironment({ name, prefix }); - } - const info = buildEnvInfo({ - executable, - kind: PythonEnvKind.Conda, - org: AnacondaCompanyName, - location: prefix, - source: [], - version: executable ? await getPythonVersionFromPath(executable) : undefined, - type: PythonEnvType.Conda, - }); - if (name) { - info.name = name; - } - return info; - } + + const envPath = env.envPath ?? getEnvironmentDirFromPath(env.executablePath); + let executable: string; + if (env.executablePath.length > 0) { + executable = env.executablePath; + } else { + executable = await conda.getInterpreterPathForEnvironment({ prefix: envPath }); } - traceError( - `${env.envPath ?? env.executablePath} identified as a Conda environment but is not returned via '${ - conda?.command - } info' command`, - ); - // Environment could still be valid, resolve as a simple env. - env.kind = PythonEnvKind.Unknown; - const envInfo = await resolveSimpleEnv(env); - envInfo.type = PythonEnvType.Conda; - return envInfo; + const info = buildEnvInfo({ + executable, + kind: PythonEnvKind.Conda, + org: AnacondaCompanyName, + location: envPath, + source: [], + version: executable ? await getPythonVersionFromPath(executable) : undefined, + type: PythonEnvType.Conda, + }); + const name = await conda?.getName(envPath); + if (name) { + info.name = name; + } + return info; } async function resolvePyenvEnv(env: BasicEnvInfo): Promise { diff --git a/src/client/pythonEnvironments/base/locators/lowLevel/condaLocator.ts b/src/client/pythonEnvironments/base/locators/lowLevel/condaLocator.ts index 2ef07fef0555..a355a72d5336 100644 --- a/src/client/pythonEnvironments/base/locators/lowLevel/condaLocator.ts +++ b/src/client/pythonEnvironments/base/locators/lowLevel/condaLocator.ts @@ -32,12 +32,8 @@ export class CondaEnvironmentLocator extends FSWatchingLocator { try { traceVerbose(`Looking into conda env for executable: ${JSON.stringify(env)}`); const executablePath = await conda.getInterpreterPathForEnvironment(env); - if (executablePath !== undefined) { - traceVerbose(`Found conda executable: ${executablePath}`); - yield { kind: PythonEnvKind.Conda, executablePath, envPath: env.prefix }; - } else { - traceError(`Executable for conda env not found: ${JSON.stringify(env)}`); - } + traceVerbose(`Found conda executable: ${executablePath}`); + yield { kind: PythonEnvKind.Conda, executablePath, envPath: env.prefix }; } catch (ex) { traceError(`Failed to process conda env: ${JSON.stringify(env)}`, ex); } diff --git a/src/client/pythonEnvironments/common/environmentManagers/conda.ts b/src/client/pythonEnvironments/common/environmentManagers/conda.ts index 03201ae0ac08..7325c4cd69de 100644 --- a/src/client/pythonEnvironments/common/environmentManagers/conda.ts +++ b/src/client/pythonEnvironments/common/environmentManagers/conda.ts @@ -444,34 +444,34 @@ export class Conda { * Corresponds to "conda env list --json", but also computes environment names. */ @cache(30_000, true, 10_000) - public async getEnvList(useCache?: boolean): Promise { - const info = await this.getInfo(useCache); + public async getEnvList(): Promise { + const info = await this.getInfo(); const { envs } = info; if (envs === undefined) { return []; } + return Promise.all( + envs.map(async (prefix) => ({ + prefix, + name: await this.getName(prefix, info), + })), + ); + } - function getName(prefix: string) { - if (info.root_prefix && arePathsSame(prefix, info.root_prefix)) { - return 'base'; - } - - const parentDir = path.dirname(prefix); - if (info.envs_dirs !== undefined) { - for (const envsDir of info.envs_dirs) { - if (arePathsSame(parentDir, envsDir)) { - return path.basename(prefix); - } + public async getName(prefix: string, info?: CondaInfo): Promise { + info = info ?? (await this.getInfo(true)); + if (info.root_prefix && arePathsSame(prefix, info.root_prefix)) { + return 'base'; + } + const parentDir = path.dirname(prefix); + if (info.envs_dirs !== undefined) { + for (const envsDir of info.envs_dirs) { + if (arePathsSame(parentDir, envsDir)) { + return path.basename(prefix); } } - - return undefined; } - - return envs.map((prefix) => ({ - prefix, - name: getName(prefix), - })); + return undefined; } /** @@ -493,22 +493,17 @@ export class Conda { * Returns executable associated with the conda env, swallows exceptions. */ // eslint-disable-next-line class-methods-use-this - public async getInterpreterPathForEnvironment(condaEnv: CondaEnvInfo): Promise { - try { - const executablePath = await getInterpreterPath(condaEnv.prefix); - if (executablePath) { - traceVerbose('Found executable within conda env', JSON.stringify(condaEnv)); - return executablePath; - } - traceVerbose( - 'Executable does not exist within conda env, assume the executable to be `python`', - JSON.stringify(condaEnv), - ); - return 'python'; - } catch (ex) { - traceError(`Failed to get executable for conda env: ${JSON.stringify(condaEnv)}`, ex); - return undefined; + public async getInterpreterPathForEnvironment(condaEnv: CondaEnvInfo | { prefix: string }): Promise { + const executablePath = await getInterpreterPath(condaEnv.prefix); + if (executablePath) { + traceVerbose('Found executable within conda env', JSON.stringify(condaEnv)); + return executablePath; } + traceVerbose( + 'Executable does not exist within conda env, assume the executable to be `python`', + JSON.stringify(condaEnv), + ); + return 'python'; } public async getRunPythonArgs(env: CondaEnvInfo, forShellExecution?: boolean): Promise { diff --git a/src/test/pythonEnvironments/base/locators/composite/resolverUtils.unit.test.ts b/src/test/pythonEnvironments/base/locators/composite/resolverUtils.unit.test.ts index 2310f6dc942f..dbd41715db0f 100644 --- a/src/test/pythonEnvironments/base/locators/composite/resolverUtils.unit.test.ts +++ b/src/test/pythonEnvironments/base/locators/composite/resolverUtils.unit.test.ts @@ -191,18 +191,17 @@ suite('Resolver Utils', () => { suite('Conda', () => { const condaPrefixNonWindows = path.join(TEST_LAYOUT_ROOT, 'conda2'); const condaPrefixWindows = path.join(TEST_LAYOUT_ROOT, 'conda1'); - function condaInfo(condaPrefix: string): CondaInfo { - return { - conda_version: '4.8.0', - python_version: '3.9.0', - 'sys.version': '3.9.0', - 'sys.prefix': '/some/env', - root_prefix: condaPrefix, - envs: [condaPrefix], - }; - } + const condaInfo: CondaInfo = { + conda_version: '4.8.0', + python_version: '3.9.0', + 'sys.version': '3.9.0', + 'sys.prefix': '/some/env', + root_prefix: path.dirname(TEST_LAYOUT_ROOT), + envs: [], + envs_dirs: [TEST_LAYOUT_ROOT], + }; - function expectedEnvInfo(executable: string, location: string) { + function expectedEnvInfo(executable: string, location: string, name: string) { const info = buildEnvInfo({ executable, kind: PythonEnvKind.Conda, @@ -211,7 +210,7 @@ suite('Resolver Utils', () => { source: [], version: UNKNOWN_PYTHON_VERSION, fileInfo: undefined, - name: 'base', + name, type: PythonEnvType.Conda, }); setEnvDisplayString(info); @@ -254,32 +253,45 @@ suite('Resolver Utils', () => { sinon.stub(platformApis, 'getOSType').callsFake(() => platformApis.OSType.Windows); sinon.stub(externalDependencies, 'exec').callsFake(async (command: string, args: string[]) => { if (command === 'conda' && args[0] === 'info' && args[1] === '--json') { - return { stdout: JSON.stringify(condaInfo(condaPrefixWindows)) }; + return { stdout: JSON.stringify(condaInfo) }; } throw new Error(`${command} is missing or is not executable`); }); const actual = await resolveBasicEnv({ - executablePath: path.join(TEST_LAYOUT_ROOT, 'conda1', 'python.exe'), + executablePath: path.join(condaPrefixWindows, 'python.exe'), + envPath: condaPrefixWindows, kind: PythonEnvKind.Conda, }); - assertEnvEqual(actual, expectedEnvInfo(path.join(condaPrefixWindows, 'python.exe'), condaPrefixWindows)); + assertEnvEqual( + actual, + expectedEnvInfo( + path.join(condaPrefixWindows, 'python.exe'), + condaPrefixWindows, + path.basename(condaPrefixWindows), + ), + ); }); test('resolveEnv (non-Windows)', async () => { sinon.stub(platformApis, 'getOSType').callsFake(() => platformApis.OSType.Linux); sinon.stub(externalDependencies, 'exec').callsFake(async (command: string, args: string[]) => { if (command === 'conda' && args[0] === 'info' && args[1] === '--json') { - return { stdout: JSON.stringify(condaInfo(condaPrefixNonWindows)) }; + return { stdout: JSON.stringify(condaInfo) }; } throw new Error(`${command} is missing or is not executable`); }); const actual = await resolveBasicEnv({ - executablePath: path.join(TEST_LAYOUT_ROOT, 'conda2', 'bin', 'python'), + executablePath: path.join(condaPrefixNonWindows, 'bin', 'python'), kind: PythonEnvKind.Conda, + envPath: condaPrefixNonWindows, }); assertEnvEqual( actual, - expectedEnvInfo(path.join(condaPrefixNonWindows, 'bin', 'python'), condaPrefixNonWindows), + expectedEnvInfo( + path.join(condaPrefixNonWindows, 'bin', 'python'), + condaPrefixNonWindows, + path.basename(condaPrefixNonWindows), + ), ); }); @@ -298,7 +310,7 @@ suite('Resolver Utils', () => { path.join(TEST_LAYOUT_ROOT, 'conda1', 'python.exe'), PythonEnvKind.Unknown, undefined, - 'conda1', + '', path.join(TEST_LAYOUT_ROOT, 'conda1'), ), ); @@ -627,7 +639,7 @@ suite('Resolver Utils', () => { version: parseVersion('3.8.5'), arch: Architecture.x64, // Provided by registry org: 'ContinuumAnalytics', // Provided by registry - name: 'conda3', + name: '', source: [PythonEnvSource.WindowsRegistry], type: PythonEnvType.Conda, });