diff --git a/src/client/pythonEnvironments/base/locator.ts b/src/client/pythonEnvironments/base/locator.ts index c6a77c473d05..03eb206445bd 100644 --- a/src/client/pythonEnvironments/base/locator.ts +++ b/src/client/pythonEnvironments/base/locator.ts @@ -92,7 +92,7 @@ export type PythonLocatorQuery = BasicPythonLocatorQuery & { searchLocations?: Uri[]; }; -export type QueryForEvent = E extends PythonEnvsChangedEvent ? PythonLocatorQuery : BasicPythonLocatorQuery; +type QueryForEvent = E extends PythonEnvsChangedEvent ? PythonLocatorQuery : BasicPythonLocatorQuery; /** * A single Python environment locator. diff --git a/src/client/pythonEnvironments/collection/environmentsReducer.ts b/src/client/pythonEnvironments/base/locators/composite/environmentsReducer.ts similarity index 91% rename from src/client/pythonEnvironments/collection/environmentsReducer.ts rename to src/client/pythonEnvironments/base/locators/composite/environmentsReducer.ts index b4c80254a3f9..ba00eac2a546 100644 --- a/src/client/pythonEnvironments/collection/environmentsReducer.ts +++ b/src/client/pythonEnvironments/base/locators/composite/environmentsReducer.ts @@ -3,13 +3,13 @@ import { cloneDeep, isEqual } from 'lodash'; import { Event, EventEmitter } from 'vscode'; -import { traceVerbose } from '../../common/logger'; -import { createDeferred } from '../../common/utils/async'; -import { areSameEnvironment, PythonEnvInfo, PythonEnvKind } from '../base/info'; +import { traceVerbose } from '../../../../common/logger'; +import { createDeferred } from '../../../../common/utils/async'; +import { areSameEnvironment, PythonEnvInfo, PythonEnvKind } from '../../info'; import { - ILocator, IPythonEnvsIterator, PythonEnvUpdatedEvent, QueryForEvent, -} from '../base/locator'; -import { PythonEnvsChangedEvent } from '../base/watcher'; + ILocator, IPythonEnvsIterator, PythonEnvUpdatedEvent, PythonLocatorQuery, +} from '../../locator'; +import { PythonEnvsChangedEvent } from '../../watcher'; /** * Combines duplicate environments received from the incoming locator into one and passes on unique environments @@ -47,7 +47,7 @@ export class PythonEnvsReducer implements ILocator { return this.parentLocator.resolveEnv(environment); } - public iterEnvs(query?: QueryForEvent): IPythonEnvsIterator { + public iterEnvs(query?: PythonLocatorQuery): IPythonEnvsIterator { const didUpdate = new EventEmitter(); const incomingIterator = this.parentLocator.iterEnvs(query); const iterator: IPythonEnvsIterator = iterEnvsIterator(incomingIterator, didUpdate); diff --git a/src/client/pythonEnvironments/base/locators/composite/environmentsResolver.ts b/src/client/pythonEnvironments/base/locators/composite/environmentsResolver.ts new file mode 100644 index 000000000000..347785f73660 --- /dev/null +++ b/src/client/pythonEnvironments/base/locators/composite/environmentsResolver.ts @@ -0,0 +1,136 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { cloneDeep } from 'lodash'; +import { Event, EventEmitter } from 'vscode'; +import { traceVerbose } from '../../../../common/logger'; +import { IEnvironmentInfoService } from '../../../info/environmentInfoService'; +import { areSameEnvironment, PythonEnvInfo } from '../../info'; +import { InterpreterInformation } from '../../info/interpreter'; +import { + ILocator, IPythonEnvsIterator, PythonEnvUpdatedEvent, PythonLocatorQuery, +} from '../../locator'; +import { PythonEnvsChangedEvent } from '../../watcher'; + +/** + * Calls environment info service which runs `interpreterInfo.py` script on environments received + * from the parent locator. Uses information received to populate environments further and pass it on. + */ +export class PythonEnvsResolver implements ILocator { + public get onChanged(): Event { + return this.parentLocator.onChanged; + } + + constructor( + private readonly parentLocator: ILocator, + private readonly environmentInfoService: IEnvironmentInfoService, + ) {} + + public async resolveEnv(env: string | PythonEnvInfo): Promise { + const environment = await this.parentLocator.resolveEnv(env); + if (!environment) { + return undefined; + } + const interpreterInfo = await this.environmentInfoService.getEnvironmentInfo(environment.executable.filename); + if (!interpreterInfo) { + return undefined; + } + return getResolvedEnv(interpreterInfo, environment); + } + + public iterEnvs(query?: PythonLocatorQuery): IPythonEnvsIterator { + const didUpdate = new EventEmitter(); + const incomingIterator = this.parentLocator.iterEnvs(query); + const iterator: IPythonEnvsIterator = this.iterEnvsIterator(incomingIterator, didUpdate); + iterator.onUpdated = didUpdate.event; + return iterator; + } + + private async* iterEnvsIterator( + iterator: IPythonEnvsIterator, + didUpdate: EventEmitter, + ): AsyncIterator { + const state = { + done: false, + pending: 0, + }; + const seen: PythonEnvInfo[] = []; + + if (iterator.onUpdated !== undefined) { + iterator.onUpdated((event) => { + if (event === null) { + state.done = true; + checkIfFinishedAndNotify(state, didUpdate); + } else { + const oldIndex = seen.findIndex((s) => areSameEnvironment(s, event.old)); + if (oldIndex !== -1) { + seen[oldIndex] = event.new; + state.pending += 1; + this.resolveInBackground(oldIndex, state, didUpdate, seen).ignoreErrors(); + } else { + // This implies a problem in a downstream locator + traceVerbose(`Expected already iterated env in resolver, got ${event.old}`); + } + } + }); + } + + let result = await iterator.next(); + while (!result.done) { + const currEnv = result.value; + seen.push(currEnv); + yield currEnv; + state.pending += 1; + this.resolveInBackground(seen.indexOf(currEnv), state, didUpdate, seen).ignoreErrors(); + // eslint-disable-next-line no-await-in-loop + result = await iterator.next(); + } + if (iterator.onUpdated === undefined) { + state.done = true; + checkIfFinishedAndNotify(state, didUpdate); + } + } + + private async resolveInBackground( + envIndex: number, + state: { done: boolean; pending: number }, + didUpdate: EventEmitter, + seen: PythonEnvInfo[], + ) { + const interpreterInfo = await this.environmentInfoService.getEnvironmentInfo( + seen[envIndex].executable.filename, + ); + if (interpreterInfo) { + const resolvedEnv = getResolvedEnv(interpreterInfo, seen[envIndex]); + didUpdate.fire({ old: seen[envIndex], new: resolvedEnv }); + seen[envIndex] = resolvedEnv; + } + state.pending -= 1; + checkIfFinishedAndNotify(state, didUpdate); + } +} + +/** + * When all info from incoming iterator has been received and all background calls finishes, notify that we're done + * @param state Carries the current state of progress + * @param didUpdate Used to notify when finished + */ +function checkIfFinishedAndNotify( + state: { done: boolean; pending: number }, + didUpdate: EventEmitter, +) { + if (state.done && state.pending === 0) { + didUpdate.fire(null); + didUpdate.dispose(); + } +} + +function getResolvedEnv(interpreterInfo: InterpreterInformation, environment: PythonEnvInfo) { + // Deep copy into a new object + const resolvedEnv = cloneDeep(environment); + resolvedEnv.version = interpreterInfo.version; + resolvedEnv.executable.filename = interpreterInfo.executable.filename; + resolvedEnv.executable.sysPrefix = interpreterInfo.executable.sysPrefix; + resolvedEnv.arch = interpreterInfo.arch; + return resolvedEnv; +} diff --git a/src/test/pythonEnvironments/collection/environmentsReducer.unit.test.ts b/src/test/pythonEnvironments/base/locators/composite/environmentsReducer.unit.test.ts similarity index 93% rename from src/test/pythonEnvironments/collection/environmentsReducer.unit.test.ts rename to src/test/pythonEnvironments/base/locators/composite/environmentsReducer.unit.test.ts index a05ebc772063..9bb67f672768 100644 --- a/src/test/pythonEnvironments/collection/environmentsReducer.unit.test.ts +++ b/src/test/pythonEnvironments/base/locators/composite/environmentsReducer.unit.test.ts @@ -5,15 +5,15 @@ import { assert, expect } from 'chai'; import { isEqual } from 'lodash'; import * as path from 'path'; import { EventEmitter } from 'vscode'; -import { PythonEnvInfo, PythonEnvKind } from '../../../client/pythonEnvironments/base/info'; -import { PythonEnvUpdatedEvent } from '../../../client/pythonEnvironments/base/locator'; -import { PythonEnvsChangedEvent } from '../../../client/pythonEnvironments/base/watcher'; +import { PythonEnvInfo, PythonEnvKind } from '../../../../../client/pythonEnvironments/base/info'; +import { PythonEnvUpdatedEvent } from '../../../../../client/pythonEnvironments/base/locator'; import { mergeEnvironments, PythonEnvsReducer, -} from '../../../client/pythonEnvironments/collection/environmentsReducer'; -import { sleep } from '../../core'; -import { createEnv, getEnvs, SimpleLocator } from '../base/common'; +} from '../../../../../client/pythonEnvironments/base/locators/composite/environmentsReducer'; +import { PythonEnvsChangedEvent } from '../../../../../client/pythonEnvironments/base/watcher'; +import { sleep } from '../../../../core'; +import { createEnv, getEnvs, SimpleLocator } from '../../common'; suite('Environments Reducer', () => { suite('iterEnvs()', () => { diff --git a/src/test/pythonEnvironments/base/locators/composite/environmentsResolver.unit.test.ts b/src/test/pythonEnvironments/base/locators/composite/environmentsResolver.unit.test.ts new file mode 100644 index 000000000000..b0a7d5bbf17b --- /dev/null +++ b/src/test/pythonEnvironments/base/locators/composite/environmentsResolver.unit.test.ts @@ -0,0 +1,244 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { assert, expect } from 'chai'; +import { cloneDeep } from 'lodash'; +import * as path from 'path'; +import { ImportMock } from 'ts-mock-imports'; +import { EventEmitter } from 'vscode'; +import { ExecutionResult } from '../../../../../client/common/process/types'; +import { Architecture } from '../../../../../client/common/utils/platform'; +import { PythonEnvInfo, PythonEnvKind } from '../../../../../client/pythonEnvironments/base/info'; +import { parseVersion } from '../../../../../client/pythonEnvironments/base/info/pythonVersion'; +import { PythonEnvUpdatedEvent } from '../../../../../client/pythonEnvironments/base/locator'; +import { PythonEnvsResolver } from '../../../../../client/pythonEnvironments/base/locators/composite/environmentsResolver'; +import { PythonEnvsChangedEvent } from '../../../../../client/pythonEnvironments/base/watcher'; +import * as ExternalDep from '../../../../../client/pythonEnvironments/common/externalDependencies'; +import { EnvironmentInfoService } from '../../../../../client/pythonEnvironments/info/environmentInfoService'; +import { sleep } from '../../../../core'; +import { createEnv, getEnvs, SimpleLocator } from '../../common'; + +suite('Environments Resolver', () => { + /** + * Returns the expected environment to be returned by Environment info service + */ + function createExpectedEnvInfo(env: PythonEnvInfo): PythonEnvInfo { + const updatedEnv = cloneDeep(env); + updatedEnv.version = { + ...parseVersion('3.8.3-final'), + sysVersion: '3.8.3 (tags/v3.8.3:6f8c832, May 13 2020, 22:37:02) [MSC v.1924 64 bit (AMD64)]', + }; + updatedEnv.executable.filename = env.executable.filename; + updatedEnv.executable.sysPrefix = 'path'; + updatedEnv.arch = Architecture.x64; + return updatedEnv; + } + suite('iterEnvs()', () => { + let stubShellExec: sinon.SinonStub; + setup(() => { + stubShellExec = ImportMock.mockFunction( + ExternalDep, + 'shellExecute', + new Promise>((resolve) => { + resolve({ + stdout: + '{"versionInfo": [3, 8, 3, "final", 0], "sysPrefix": "path", "sysVersion": "3.8.3 (tags/v3.8.3:6f8c832, May 13 2020, 22:37:02) [MSC v.1924 64 bit (AMD64)]", "is64Bit": true}', + }); + }), + ); + }); + + teardown(() => { + stubShellExec.restore(); + }); + + test('Iterator yields environments as-is', async () => { + const env1 = createEnv('env1', '3.5.12b1', PythonEnvKind.Venv, path.join('path', 'to', 'exec1')); + const env2 = createEnv('env2', '3.8.1', PythonEnvKind.Conda, path.join('path', 'to', 'exec2')); + const env3 = createEnv('env3', '2.7', PythonEnvKind.System, path.join('path', 'to', 'exec3')); + const env4 = createEnv('env4', '3.9.0rc2', PythonEnvKind.Unknown, path.join('path', 'to', 'exec2')); + const environmentsToBeIterated = [env1, env2, env3, env4]; + const parentLocator = new SimpleLocator(environmentsToBeIterated); + const resolver = new PythonEnvsResolver(parentLocator, new EnvironmentInfoService()); + + const iterator = resolver.iterEnvs(); + const envs = await getEnvs(iterator); + + assert.deepEqual(envs, environmentsToBeIterated); + }); + + test('Updates for environments are sent correctly followed by the null event', async () => { + // Arrange + const env1 = createEnv('env1', '3.5.12b1', PythonEnvKind.Unknown, path.join('path', 'to', 'exec1')); + const env2 = createEnv('env2', '3.8.1', PythonEnvKind.Unknown, path.join('path', 'to', 'exec2')); + const environmentsToBeIterated = [env1, env2]; + const parentLocator = new SimpleLocator(environmentsToBeIterated); + const onUpdatedEvents: (PythonEnvUpdatedEvent | null)[] = []; + const resolver = new PythonEnvsResolver(parentLocator, new EnvironmentInfoService()); + + const iterator = resolver.iterEnvs(); // Act + + // Assert + let { onUpdated } = iterator; + expect(onUpdated).to.not.equal(undefined, ''); + + // Arrange + onUpdated = onUpdated!; + onUpdated((e) => { + onUpdatedEvents.push(e); + }); + + // Act + await getEnvs(iterator); + await sleep(1); // Resolve pending calls in the background + + // Assert + const expectedUpdates = [ + { old: env1, new: createExpectedEnvInfo(env1) }, + { old: env2, new: createExpectedEnvInfo(env2) }, + null, + ]; + assert.deepEqual(expectedUpdates, onUpdatedEvents); + }); + + test('Updates to environments from the incoming iterator are sent correctly followed by the null event', async () => { + // Arrange + const env = createEnv('env1', '3.8', PythonEnvKind.Unknown, path.join('path', 'to', 'exec')); + const updatedEnv = createEnv('env1', '3.8.1', PythonEnvKind.System, path.join('path', 'to', 'exec')); + const environmentsToBeIterated = [env]; + const didUpdate = new EventEmitter(); + const parentLocator = new SimpleLocator(environmentsToBeIterated, { onUpdated: didUpdate.event }); + const onUpdatedEvents: (PythonEnvUpdatedEvent | null)[] = []; + const resolver = new PythonEnvsResolver(parentLocator, new EnvironmentInfoService()); + + const iterator = resolver.iterEnvs(); // Act + + // Assert + let { onUpdated } = iterator; + expect(onUpdated).to.not.equal(undefined, ''); + + // Arrange + onUpdated = onUpdated!; + onUpdated((e) => { + onUpdatedEvents.push(e); + }); + + // Act + await getEnvs(iterator); + await sleep(1); + didUpdate.fire({ old: env, new: updatedEnv }); + didUpdate.fire(null); // It is essential for the incoming iterator to fire "null" event signifying it's done + await sleep(1); + + // Assert + // The updates can be anything, even the number of updates, but they should lead to the same final state + const { length } = onUpdatedEvents; + assert.deepEqual( + onUpdatedEvents[length - 2]?.new, + createExpectedEnvInfo(updatedEnv), + 'The final update to environment is incorrect', + ); + assert.equal(onUpdatedEvents[length - 1], null, 'Last update should be null'); + didUpdate.dispose(); + }); + }); + + test('onChanged fires iff onChanged from resolver fires', () => { + const parentLocator = new SimpleLocator([]); + const event1: PythonEnvsChangedEvent = {}; + const event2: PythonEnvsChangedEvent = { kind: PythonEnvKind.Unknown }; + const expected = [event1, event2]; + const resolver = new PythonEnvsResolver(parentLocator, new EnvironmentInfoService()); + + const events: PythonEnvsChangedEvent[] = []; + resolver.onChanged((e) => events.push(e)); + + parentLocator.fire(event1); + parentLocator.fire(event2); + + assert.deepEqual(events, expected); + }); + + suite('resolveEnv()', () => { + let stubShellExec: sinon.SinonStub; + setup(() => { + stubShellExec = ImportMock.mockFunction( + ExternalDep, + 'shellExecute', + new Promise>((resolve) => { + resolve({ + stdout: + '{"versionInfo": [3, 8, 3, "final", 0], "sysPrefix": "path", "sysVersion": "3.8.3 (tags/v3.8.3:6f8c832, May 13 2020, 22:37:02) [MSC v.1924 64 bit (AMD64)]", "is64Bit": true}', + }); + }), + ); + }); + + teardown(() => { + stubShellExec.restore(); + }); + + test('Calls into parent locator to get resolved environment, then calls environnment service to resolve environment further and return it', async () => { + const env = createEnv('env1', '3.8', PythonEnvKind.Unknown, path.join('path', 'to', 'exec')); + const resolvedEnvReturnedByReducer = createEnv( + 'env1', + '3.8.1', + PythonEnvKind.Conda, + 'resolved/path/to/exec', + ); + const parentLocator = new SimpleLocator([], { + resolve: async (e: PythonEnvInfo) => { + if (e === env) { + return resolvedEnvReturnedByReducer; + } + throw new Error('Incorrect environment sent to the resolver'); + }, + }); + const resolver = new PythonEnvsResolver(parentLocator, new EnvironmentInfoService()); + + const expected = await resolver.resolveEnv(env); + + assert.deepEqual(expected, createExpectedEnvInfo(resolvedEnvReturnedByReducer)); + }); + + test('If the parent locator resolves environment, but fetching interpreter info returns undefined, return undefined', async () => { + stubShellExec.returns( + new Promise>((_resolve, reject) => { + reject(); + }), + ); + const env = createEnv('env1', '3.8', PythonEnvKind.Unknown, path.join('path', 'to', 'exec')); + const resolvedEnvReturnedByReducer = createEnv( + 'env1', + '3.8.1', + PythonEnvKind.Conda, + 'resolved/path/to/exec', + ); + const parentLocator = new SimpleLocator([], { + resolve: async (e: PythonEnvInfo) => { + if (e === env) { + return resolvedEnvReturnedByReducer; + } + throw new Error('Incorrect environment sent to the resolver'); + }, + }); + const resolver = new PythonEnvsResolver(parentLocator, new EnvironmentInfoService()); + + const expected = await resolver.resolveEnv(env); + + assert.deepEqual(expected, undefined); + }); + + test("If the parent locator isn't able to resolve environment, return undefined", async () => { + const env = createEnv('env', '3.8', PythonEnvKind.Unknown, path.join('path', 'to', 'exec')); + const parentLocator = new SimpleLocator([], { + resolve: async () => undefined, + }); + const resolver = new PythonEnvsResolver(parentLocator, new EnvironmentInfoService()); + + const expected = await resolver.resolveEnv(env); + + assert.deepEqual(expected, undefined); + }); + }); +});