Skip to content

Commit

Permalink
PTVS engine update + handling of the interpreter change (#1613)
Browse files Browse the repository at this point in the history 
* Undo changes

* Test fixes

* Increase timeout

* Remove double event listening

* Remove test

* Revert "Remove test"

This reverts commit e240c3f.

* Revert "Remove double event listening"

This reverts commit af573be.

* #1096 The if statement is automatically formatted incorrectly

* Merge fix

* Add more tests

* More tests

* Typo

* Test

* Also better handle multiline arguments

* Add a couple missing periods

[skip ci]

* Undo changes

* Test fixes

* Increase timeout

* Remove double event listening

* Remove test

* Revert "Remove test"

This reverts commit e240c3f.

* Revert "Remove double event listening"

This reverts commit af573be.

* Merge fix

* #1257 On type formatting errors for args and kwargs

* Handle f-strings

* Stop importing from test code

* #1308 Single line statements leading to an indentation on the next line

* #726 editing python after inline if statement invalid indent

* Undo change

* Move constant

* Harden LS startup error checks

* #1364 Intellisense doesn't work after specific const string

* Telemetry for the analysis enging

* PR feedback

* Fix typo

* Test baseline update

* Jedi 0.12

* Priority to goto_defition

* News

* Replace unzip

* Linux flavors + test

* Grammar check

* Grammar test

* Test baselines

* Add news

* Pin dependency

[skip ci]

* Specify markdown as preferable format

* Improve function argument detection

* Specify markdown

* Pythia setting

* Baseline updates

* Baseline update

* Improve startup

* Handle missing interpreter better

* Handle interpreter change

* Delete old file

* Fix LS startup time reporting

* Remove Async suffix from IFileSystem
  • Loading branch information
Mikhail Arkhipov authored May 9, 2018
1 parent 2324754 commit f18a5ee
Show file tree
Hide file tree
Showing 40 changed files with 316 additions and 198 deletions.
12 changes: 12 additions & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1232,6 +1232,18 @@
"description": "Whether to install Python modules globally when not using an environment.",
"scope": "resource"
},
"python.pythiaEnabled": {
"type": "boolean",
"default": true,
"description": "Enables AI-driven additions to the completion list. Does not apply to Jedi.",
"scope": "resource"
},
"python.jediEnabled": {
"type": "boolean",
"default": true,
"description": "Enables Jedi as IntelliSense engine instead of Microsoft Python Analysis Engine.",
"scope": "resource"
},
"python.jediMemoryLimit": {
"type": "number",
"default": 0,
Expand Down
75 changes: 58 additions & 17 deletions src/client/activation/analysis.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import { IProcessServiceFactory } from '../common/process/types';
import { StopWatch } from '../common/stopWatch';
import { IConfigurationService, IOutputChannel, IPythonSettings } from '../common/types';
import { IEnvironmentVariablesProvider } from '../common/variables/types';
import { IInterpreterService } from '../interpreter/contracts';
import { IServiceContainer } from '../ioc/types';
import {
PYTHON_ANALYSIS_ENGINE_DOWNLOADED,
Expand All @@ -35,11 +36,11 @@ class LanguageServerStartupErrorHandler implements ErrorHandler {
constructor(private readonly deferred: Deferred<void>) { }
public error(error: Error, message: Message, count: number): ErrorAction {
this.deferred.reject(error);
return ErrorAction.Shutdown;
return ErrorAction.Continue;
}
public closed(): CloseAction {
this.deferred.reject();
return CloseAction.DoNotRestart;
return CloseAction.Restart;
}
}

Expand All @@ -50,39 +51,66 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
private readonly fs: IFileSystem;
private readonly sw = new StopWatch();
private readonly platformData: PlatformData;
private readonly interpreterService: IInterpreterService;
private readonly disposables: Disposable[] = [];
private languageClient: LanguageClient | undefined;
private context: ExtensionContext | undefined;
private interpreterHash: string = '';

constructor(private readonly services: IServiceContainer, pythonSettings: IPythonSettings) {
this.configuration = this.services.get<IConfigurationService>(IConfigurationService);
this.appShell = this.services.get<IApplicationShell>(IApplicationShell);
this.output = this.services.get<OutputChannel>(IOutputChannel, STANDARD_OUTPUT_CHANNEL);
this.fs = this.services.get<IFileSystem>(IFileSystem);
this.platformData = new PlatformData(services.get<IPlatformService>(IPlatformService), this.fs);
this.interpreterService = this.services.get<IInterpreterService>(IInterpreterService);
}

public async activate(context: ExtensionContext): Promise<boolean> {
this.sw.reset();
this.context = context;
const clientOptions = await this.getAnalysisOptions(context);
if (!clientOptions) {
return false;
}
this.disposables.push(this.interpreterService.onDidChangeInterpreter(() => this.restartLanguageServer()));
return this.startLanguageServer(context, clientOptions);
}

public async deactivate(): Promise<void> {
if (this.languageClient) {
await this.languageClient.stop();
}
for (const d of this.disposables) {
d.dispose();
}
}

private async restartLanguageServer(): Promise<void> {
if (!this.context) {
return;
}
const ids = new InterpreterDataService(this.context, this.services);
const idata = await ids.getInterpreterData();
if (!idata || idata.hash !== this.interpreterHash) {
this.interpreterHash = idata ? idata.hash : '';
await this.deactivate();
await this.activate(this.context);
}
}

private async startLanguageServer(context: ExtensionContext, clientOptions: LanguageClientOptions): Promise<boolean> {
// Determine if we are running MSIL/Universal via dotnet or self-contained app.
const mscorlib = path.join(context.extensionPath, analysisEngineFolder, 'mscorlib.dll');
const downloader = new AnalysisEngineDownloader(this.services, analysisEngineFolder);
let downloadPackage = false;

const reporter = getTelemetryReporter();
reporter.sendTelemetryEvent(PYTHON_ANALYSIS_ENGINE_ENABLED);

if (!await this.fs.fileExistsAsync(mscorlib)) {
await this.checkPythiaModel(context, downloader);

if (!await this.fs.fileExists(mscorlib)) {
// Depends on .NET Runtime or SDK
this.languageClient = this.createSimpleLanguageClient(context, clientOptions);
try {
Expand All @@ -100,7 +128,7 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
}

if (downloadPackage) {
const downloader = new AnalysisEngineDownloader(this.services, analysisEngineFolder);
this.appShell.showWarningMessage('.NET Runtime is not found, platform-specific Python Analysis Engine will be downloaded.');
await downloader.downloadAnalysisEngine(context);
reporter.sendTelemetryEvent(PYTHON_ANALYSIS_ENGINE_DOWNLOADED);
}
Expand Down Expand Up @@ -128,7 +156,9 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
disposable = lc.start();
lc.onReady()
.then(() => deferred.resolve())
.catch(deferred.reject);
.catch((reason) => {
deferred.reject(reason);
});
await deferred.promise;

this.output.appendLine(`Language server ready: ${this.sw.elapsedTime} ms`);
Expand Down Expand Up @@ -172,20 +202,19 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
const interpreterData = await interpreterDataService.getInterpreterData();
if (!interpreterData) {
const appShell = this.services.get<IApplicationShell>(IApplicationShell);
appShell.showErrorMessage('Unable to determine path to Python interpreter.');
return;
appShell.showWarningMessage('Unable to determine path to Python interpreter. IntelliSense will be limited.');
}

// tslint:disable-next-line:no-string-literal
properties['InterpreterPath'] = interpreterData.path;
// tslint:disable-next-line:no-string-literal
properties['Version'] = interpreterData.version;
// tslint:disable-next-line:no-string-literal
properties['PrefixPath'] = interpreterData.prefix;
// tslint:disable-next-line:no-string-literal
properties['DatabasePath'] = path.join(context.extensionPath, analysisEngineFolder);
if (interpreterData) {
// tslint:disable-next-line:no-string-literal
properties['InterpreterPath'] = interpreterData.path;
// tslint:disable-next-line:no-string-literal
properties['Version'] = interpreterData.version;
// tslint:disable-next-line:no-string-literal
properties['PrefixPath'] = interpreterData.prefix;
}

let searchPaths = interpreterData.searchPaths;
let searchPaths = interpreterData ? interpreterData.searchPaths : '';
const settings = this.configuration.getSettings();
if (settings.autoComplete) {
const extraPaths = settings.autoComplete.extraPaths;
Expand All @@ -194,12 +223,15 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
}
}

// tslint:disable-next-line:no-string-literal
properties['DatabasePath'] = path.join(context.extensionPath, analysisEngineFolder);

const envProvider = this.services.get<IEnvironmentVariablesProvider>(IEnvironmentVariablesProvider);
const pythonPath = (await envProvider.getEnvironmentVariables()).PYTHONPATH;
this.interpreterHash = interpreterData ? interpreterData.hash : '';

// tslint:disable-next-line:no-string-literal
properties['SearchPaths'] = `${searchPaths};${pythonPath ? pythonPath : ''}`;

const selector: string[] = [PYTHON];

// Options to control the language client
Expand All @@ -215,12 +247,14 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
properties
},
displayOptions: {
preferredFormat: 1, // Markdown
trimDocumentationLines: false,
maxDocumentationLineLength: 0,
trimDocumentationText: false,
maxDocumentationTextLength: 0
},
asyncStartup: true,
pythiaEnabled: settings.pythiaEnabled,
testEnvironment: isTestExecution()
}
};
Expand All @@ -231,4 +265,11 @@ export class AnalysisExtensionActivator implements IExtensionActivator {
const result = await ps.exec('dotnet', ['--version']).catch(() => { return { stdout: '' }; });
return result.stdout.trim().startsWith('2.');
}

private async checkPythiaModel(context: ExtensionContext, downloader: AnalysisEngineDownloader): Promise<void> {
const settings = this.configuration.getSettings();
if (settings.pythiaEnabled) {
await downloader.downloadPythiaModel(context);
}
}
}
64 changes: 46 additions & 18 deletions src/client/activation/downloader.ts
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

import * as fs from 'fs';
import * as fileSystem from 'fs';
import * as path from 'path';
import * as request from 'request';
import * as requestProgress from 'request-progress';
import { ExtensionContext, OutputChannel, ProgressLocation, window } from 'vscode';
import { STANDARD_OUTPUT_CHANNEL } from '../common/constants';
import { noop } from '../common/core.utils';
import { createDeferred, createTemporaryFile } from '../common/helpers';
import { IFileSystem, IPlatformService } from '../common/platform/types';
import { IOutputChannel } from '../common/types';
Expand All @@ -22,49 +21,78 @@ const downloadUriPrefix = 'https://pvsc.blob.core.windows.net/python-analysis';
const downloadBaseFileName = 'python-analysis-vscode';
const downloadVersion = '0.1.0';
const downloadFileExtension = '.nupkg';
const pythiaModelName = 'model-sequence.json.gz';

export class AnalysisEngineDownloader {
private readonly output: OutputChannel;
private readonly platform: IPlatformService;
private readonly platformData: PlatformData;
private readonly fs: IFileSystem;

constructor(private readonly services: IServiceContainer, private engineFolder: string) {
this.output = this.services.get<OutputChannel>(IOutputChannel, STANDARD_OUTPUT_CHANNEL);
this.fs = this.services.get<IFileSystem>(IFileSystem);
this.platform = this.services.get<IPlatformService>(IPlatformService);
this.platformData = new PlatformData(this.platform, this.services.get<IFileSystem>(IFileSystem));
this.platformData = new PlatformData(this.platform, this.fs);
}

public async downloadAnalysisEngine(context: ExtensionContext): Promise<void> {
const localTempFilePath = await this.downloadFile();
const platformString = await this.platformData.getPlatformName();
const enginePackageFileName = `${downloadBaseFileName}-${platformString}.${downloadVersion}${downloadFileExtension}`;

let localTempFilePath = '';
try {
await this.verifyDownload(localTempFilePath);
localTempFilePath = await this.downloadFile(downloadUriPrefix, enginePackageFileName, 'Downloading Python Analysis Engine... ');
await this.verifyDownload(localTempFilePath, platformString);
await this.unpackArchive(context.extensionPath, localTempFilePath);
} catch (err) {
this.output.appendLine('failed.');
this.output.appendLine(err);
throw new Error(err);
} finally {
fs.unlink(localTempFilePath, noop);
if (localTempFilePath.length > 0) {
await this.fs.deleteFile(localTempFilePath);
}
}
}

private async downloadFile(): Promise<string> {
const platformString = await this.platformData.getPlatformName();
const remoteFileName = `${downloadBaseFileName}-${platformString}.${downloadVersion}${downloadFileExtension}`;
const uri = `${downloadUriPrefix}/${remoteFileName}`;
public async downloadPythiaModel(context: ExtensionContext): Promise<void> {
const modelFolder = path.join(context.extensionPath, 'analysis', 'Pythia', 'model');
const localPath = path.join(modelFolder, pythiaModelName);
if (await this.fs.fileExists(localPath)) {
return;
}

let localTempFilePath = '';
try {
localTempFilePath = await this.downloadFile(downloadUriPrefix, pythiaModelName, 'Downloading IntelliSense Model File... ');
await this.fs.createDirectory(modelFolder);
await this.fs.copyFile(localTempFilePath, localPath);
} catch (err) {
this.output.appendLine('failed.');
this.output.appendLine(err);
throw new Error(err);
} finally {
if (localTempFilePath.length > 0) {
await this.fs.deleteFile(localTempFilePath);
}
}
}

private async downloadFile(location: string, fileName: string, title: string): Promise<string> {
const uri = `${location}/${fileName}`;
this.output.append(`Downloading ${uri}... `);
const tempFile = await createTemporaryFile(downloadFileExtension);

const deferred = createDeferred();
const fileStream = fs.createWriteStream(tempFile.filePath);
const fileStream = fileSystem.createWriteStream(tempFile.filePath);
fileStream.on('finish', () => {
fileStream.close();
}).on('error', (err) => {
tempFile.cleanupCallback();
deferred.reject(err);
});

const title = 'Downloading Python Analysis Engine... ';
await window.withProgress({
location: ProgressLocation.Window,
title
Expand Down Expand Up @@ -94,11 +122,11 @@ export class AnalysisEngineDownloader {
return tempFile.filePath;
}

private async verifyDownload(filePath: string): Promise<void> {
private async verifyDownload(filePath: string, platformString: string): Promise<void> {
this.output.appendLine('');
this.output.append('Verifying download... ');
const verifier = new HashVerifier();
if (!await verifier.verifyHash(filePath, await this.platformData.getExpectedHash())) {
if (!await verifier.verifyHash(filePath, platformString, await this.platformData.getExpectedHash())) {
throw new Error('Hash of the downloaded file does not match.');
}
this.output.append('valid.');
Expand All @@ -123,10 +151,10 @@ export class AnalysisEngineDownloader {

let totalFiles = 0;
let extractedFiles = 0;
zip.on('ready', () => {
zip.on('ready', async () => {
totalFiles = zip.entriesCount;
if (!fs.existsSync(installFolder)) {
fs.mkdirSync(installFolder);
if (!await this.fs.directoryExists(installFolder)) {
await this.fs.createDirectory(installFolder);
}
zip.extract(null, installFolder, (err, count) => {
if (err) {
Expand All @@ -147,7 +175,7 @@ export class AnalysisEngineDownloader {
// Set file to executable
if (!this.platform.isWindows) {
const executablePath = path.join(installFolder, this.platformData.getEngineExecutableName());
fs.chmodSync(executablePath, '0764'); // -rwxrw-r--
fileSystem.chmodSync(executablePath, '0764'); // -rwxrw-r--
}
}
}
4 changes: 2 additions & 2 deletions src/client/activation/hashVerifier.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import * as fs from 'fs';
import { createDeferred } from '../common/helpers';

export class HashVerifier {
public async verifyHash(filePath: string, expectedDigest: string): Promise<boolean> {
public async verifyHash(filePath: string, platformString: string, expectedDigest: string): Promise<boolean> {
const readStream = fs.createReadStream(filePath);
const deferred = createDeferred();
const hash = createHash('sha512');
Expand All @@ -23,6 +23,6 @@ export class HashVerifier {
readStream.pipe(hash);
await deferred.promise;
const actual = hash.read();
return expectedDigest === '' ? true : actual === expectedDigest;
return expectedDigest === platformString ? true : actual === expectedDigest;
}
}
Loading

0 comments on commit f18a5ee

Please sign in to comment.