NB Convert 6.0 support for export

news/2 Fixes/14169.md

@@ -0,0 +1 @@
+Support nbconvert version 6+ for exporting notebooks to python code.

src/client/datascience/common.ts

@@ -3,6 +3,7 @@
 'use strict';
 import type { nbformat } from '@jupyterlab/coreutils';
 import * as os from 'os';
+import { parse, SemVer } from 'semver';
 import { Memento, Uri } from 'vscode';
 import { splitMultilineString } from '../../datascience-ui/common';
 import { traceError, traceInfo } from '../common/logger';
@@ -188,3 +189,14 @@ export async function getRealPath(
         }
     }
 }
+
+// For the given string parse it out to a SemVer or return undefined
+export function parseSemVer(versionString: string): SemVer | undefined {
+    const versionMatch = /^\s*(\d+)\.(\d+)\.(.+)\s*$/.exec(versionString);
+    if (versionMatch && versionMatch.length > 2) {
+        const major = parseInt(versionMatch[1], 10);
+        const minor = parseInt(versionMatch[2], 10);
+        const build = parseInt(versionMatch[3], 10);
+        return parse(`${major}.${minor}.${build}`, true) ?? undefined;
+    }
+}

src/client/datascience/data-viewing/dataViewerDependencyService.ts

@@ -4,7 +4,7 @@
 'use strict';
 
 import { inject, injectable } from 'inversify';
-import { parse, SemVer } from 'semver';
+import { SemVer } from 'semver';
 import { CancellationToken } from 'vscode';
 import { IApplicationShell } from '../../common/application/types';
 import { Cancellation, createPromiseFromCancellation, wrapCancellationTokens } from '../../common/cancellation';
@@ -14,6 +14,7 @@ import { IInstaller, InstallerResponse, Product } from '../../common/types';
 import { Common, DataScience } from '../../common/utils/localize';
 import { PythonEnvironment } from '../../pythonEnvironments/info';
 import { sendTelemetryEvent } from '../../telemetry';
+import { parseSemVer } from '../common';
 import { Telemetry } from '../constants';
 
 const minimumSupportedPandaVersion = '0.20.0';
@@ -104,13 +105,8 @@ export class DataViewerDependencyService {
                 throwOnStdErr: true,
                 token
             });
-            const versionMatch = /^\s*(\d+)\.(\d+)\.(.+)\s*$/.exec(result.stdout);
-            if (versionMatch && versionMatch.length > 2) {
-                const major = parseInt(versionMatch[1], 10);
-                const minor = parseInt(versionMatch[2], 10);
-                const build = parseInt(versionMatch[3], 10);
-                return parse(`${major}.${minor}.${build}`, true) ?? undefined;
-            }
+
+            return parseSemVer(result.stdout);
         } catch (ex) {
             traceWarning('Failed to get version of Pandas to use Data Viewer', ex);
             return;

src/client/datascience/jupyter/interpreter/jupyterInterpreterDependencyService.ts

@@ -4,6 +4,7 @@
 'use strict';
 
 import { inject, injectable } from 'inversify';
+import { SemVer } from 'semver';
 import { CancellationToken } from 'vscode';
 import { IApplicationShell } from '../../../common/application/types';
 import { Cancellation, createPromiseFromCancellation, wrapCancellationTokens } from '../../../common/cancellation';
@@ -14,6 +15,7 @@ import { Common, DataScience } from '../../../common/utils/localize';
 import { noop } from '../../../common/utils/misc';
 import { PythonEnvironment } from '../../../pythonEnvironments/info';
 import { sendTelemetryEvent } from '../../../telemetry';
+import { parseSemVer } from '../../common';
 import { HelpLinks, JupyterCommands, Telemetry } from '../../constants';
 import { reportAction } from '../../progress/decorator';
 import { ReportableAction } from '../../progress/types';
@@ -241,6 +243,23 @@ export class JupyterInterpreterDependencyService {
         return installed;
     }
 
+    public async getNbConvertVersion(
+        interpreter: PythonEnvironment,
+        _token?: CancellationToken
+    ): Promise<SemVer | undefined> {
+        const command = this.commandFactory.createInterpreterCommand(
+            JupyterCommands.ConvertCommand,
+            'jupyter',
+            ['-m', 'jupyter', 'nbconvert'],
+            interpreter,
+            false
+        );
+
+        const result = await command.exec(['--version'], { throwOnStdErr: true });
+
+        return parseSemVer(result.stdout);
+    }
+
     /**
      * Gets a list of the dependencies not installed, dependencies that are required to launch the jupyter notebook server.
      *

src/client/datascience/jupyter/interpreter/jupyterInterpreterSubCommandExecutionService.ts

@@ -5,6 +5,7 @@
 
 import { inject, injectable, named } from 'inversify';
 import * as path from 'path';
+import { SemVer } from 'semver';
 import { CancellationToken, Uri } from 'vscode';
 import { Cancellation } from '../../../common/cancellation';
 import { traceError, traceInfo, traceWarning } from '../../../common/logger';
@@ -76,12 +77,16 @@ export class JupyterInterpreterSubCommandExecutionService
         }
         return this.jupyterDependencyService.areDependenciesInstalled(interpreter, token);
     }
-    public async isExportSupported(token?: CancellationToken): Promise<boolean> {
+    public async isExportSupported(token?: CancellationToken): Promise<SemVer | undefined> {
         const interpreter = await this.jupyterInterpreter.getSelectedInterpreter(token);
         if (!interpreter) {
-            return false;
+            return;
+        }
+
+        // If nbconvert is there check and return the version
+        if (await this.jupyterDependencyService.isExportSupported(interpreter, token)) {
+            return this.jupyterDependencyService.getNbConvertVersion(interpreter, token);
         }
-        return this.jupyterDependencyService.isExportSupported(interpreter, token);
     }
     public async getReasonForJupyterNotebookNotBeingSupported(token?: CancellationToken): Promise<string> {
         let interpreter = await this.jupyterInterpreter.getSelectedInterpreter(token);
@@ -176,6 +181,7 @@ export class JupyterInterpreterSubCommandExecutionService
         const args = template
             ? [file.fsPath, '--to', 'python', '--stdout', '--template', template]
             : [file.fsPath, '--to', 'python', '--stdout'];
+
         // Ignore stderr, as nbconvert writes conversion result to stderr.
         // stdout contains the generated python code.
         return daemon

src/client/datascience/jupyter/jupyterExecution.ts

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 'use strict';
 import * as path from 'path';
+import { SemVer } from 'semver';
 import * as uuid from 'uuid/v4';
 import { CancellationToken, CancellationTokenSource, Event, EventEmitter, Uri } from 'vscode';
 
@@ -123,7 +124,7 @@ export class JupyterExecutionBase implements IJupyterExecution {
     }
 
     @reportAction(ReportableAction.CheckingIfImportIsSupported)
-    public async isImportSupported(cancelToken?: CancellationToken): Promise<boolean> {
+    public async isImportSupported(cancelToken?: CancellationToken): Promise<SemVer | undefined> {
         // See if we can find the command nbconvert
         return this.jupyterInterpreterService.isExportSupported(cancelToken);
     }

src/client/datascience/jupyter/jupyterExecutionFactory.ts

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 'use strict';
 import { inject, injectable, named } from 'inversify';
+import { SemVer } from 'semver';
 import { CancellationToken, Event, EventEmitter, Uri } from 'vscode';
 
 import { IApplicationShell, ILiveShareApi, IWorkspaceService } from '../../common/application/types';
@@ -117,7 +118,7 @@ export class JupyterExecutionFactory implements IJupyterExecution, IAsyncDisposa
         return execution.getNotebookError();
     }
 
-    public async isImportSupported(cancelToken?: CancellationToken): Promise<boolean> {
+    public async isImportSupported(cancelToken?: CancellationToken): Promise<SemVer | undefined> {
         const execution = await this.executionFactory.get();
         return execution.isImportSupported(cancelToken);
     }

src/client/datascience/jupyter/jupyterImporter.ts

@@ -26,20 +26,22 @@ import {
 export class JupyterImporter implements INotebookImporter {
     public isDisposed: boolean = false;
     // Template that changes markdown cells to have # %% [markdown] in the comments
-    private readonly nbconvertTemplateFormat =
+    private readonly nbconvertBaseTemplateFormat =
         // tslint:disable-next-line:no-multiline-string
-        `{%- extends 'null.tpl' -%}
+        `{%- extends '{0}' -%}
 {% block codecell %}
-{0}
+{1}
 {{ super() }}
 {% endblock codecell %}
 {% block in_prompt %}{% endblock in_prompt %}
 {% block input %}{{ cell.source | ipython2python }}{% endblock input %}
 {% block markdowncell scoped %}{0} [markdown]
 {{ cell.source | comment_lines }}
 {% endblock markdowncell %}`;
-
-    private templatePromise: Promise<string | undefined>;
+    private readonly nbconvert5Null = 'null.tpl';
+    private readonly nbconvert6Null = 'base/null.j2';
+    private template5Promise?: Promise<string | undefined>;
+    private template6Promise?: Promise<string | undefined>;
 
     constructor(
         @inject(IDataScienceFileSystem) private fs: IDataScienceFileSystem,
@@ -50,13 +52,9 @@ export class JupyterImporter implements INotebookImporter {
         @inject(IPlatformService) private readonly platform: IPlatformService,
         @inject(IJupyterInterpreterDependencyManager)
         private readonly dependencyManager: IJupyterInterpreterDependencyManager
-    ) {
-        this.templatePromise = this.createTemplateFile();
-    }
+    ) {}
 
     public async importFromFile(sourceFile: Uri): Promise<string> {
-        const template = await this.templatePromise;
-
         // If the user has requested it, add a cd command to the imported file so that relative paths still work
         const settings = this.configuration.getSettings();
         let directoryChange: string | undefined;
@@ -69,8 +67,26 @@ export class JupyterImporter implements INotebookImporter {
             await this.dependencyManager.installMissingDependencies();
         }
 
+        const nbConvertVersion = await this.jupyterExecution.isImportSupported();
         // Use the jupyter nbconvert functionality to turn the notebook into a python file
-        if (await this.jupyterExecution.isImportSupported()) {
+        if (nbConvertVersion) {
+            // nbconvert 5 and 6 use a different base template file
+            // Create and select the correct one
+            let template: string | undefined;
+            if (nbConvertVersion.major >= 6) {
+                if (!this.template6Promise) {
+                    this.template6Promise = this.createTemplateFile(true);
+                }
+
+                template = await this.template6Promise;
+            } else {
+                if (!this.template5Promise) {
+                    this.template5Promise = this.createTemplateFile(false);
+                }
+
+                template = await this.template5Promise;
+            }
+
             let fileOutput: string = await this.jupyterExecution.importNotebook(sourceFile, template);
             if (fileOutput.includes('get_ipython()')) {
                 fileOutput = this.addIPythonImport(fileOutput);
@@ -153,7 +169,7 @@ export class JupyterImporter implements INotebookImporter {
         }
     }
 
-    private async createTemplateFile(): Promise<string | undefined> {
+    private async createTemplateFile(nbconvert6: boolean): Promise<string | undefined> {
         // Create a temp file on disk
         const file = await this.fs.createTemporaryLocalFile('.tpl');
 
@@ -164,7 +180,10 @@ export class JupyterImporter implements INotebookImporter {
                 this.disposableRegistry.push(file);
                 await this.fs.appendLocalFile(
                     file.filePath,
-                    this.nbconvertTemplateFormat.format(this.defaultCellMarker)
+                    this.nbconvertBaseTemplateFormat.format(
+                        nbconvert6 ? this.nbconvert6Null : this.nbconvert5Null,
+                        this.defaultCellMarker
+                    )
                 );
 
                 // Now we should have a template that will convert

src/client/datascience/jupyter/liveshare/guestJupyterExecution.ts

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 'use strict';
 import { injectable } from 'inversify';
+import { SemVer } from 'semver';
 import * as uuid from 'uuid/v4';
 import { CancellationToken } from 'vscode';
 
@@ -72,10 +73,27 @@ export class GuestJupyterExecution extends LiveShareParticipantGuest(
     }
 
     public async isNotebookSupported(cancelToken?: CancellationToken): Promise<boolean> {
-        return this.checkSupported(LiveShareCommands.isNotebookSupported, cancelToken);
+        const service = await this.waitForService();
+
+        // Make a remote call on the proxy
+        if (service) {
+            const result = await service.request(LiveShareCommands.isNotebookSupported, [], cancelToken);
+            return result as boolean;
+        }
+
+        return false;
     }
-    public isImportSupported(cancelToken?: CancellationToken): Promise<boolean> {
-        return this.checkSupported(LiveShareCommands.isImportSupported, cancelToken);
+    public async isImportSupported(cancelToken?: CancellationToken): Promise<SemVer | undefined> {
+        const service = await this.waitForService();
+
+        // Make a remote call on the proxy
+        if (service) {
+            const result = await service.request(LiveShareCommands.isImportSupported, [], cancelToken);
+
+            if (result) {
+                return result as SemVer;
+            }
+        }
     }
     public isSpawnSupported(_cancelToken?: CancellationToken): Promise<boolean> {
         return Promise.resolve(false);
@@ -144,16 +162,4 @@ export class GuestJupyterExecution extends LiveShareParticipantGuest(
     public async getServer(options?: INotebookServerOptions): Promise<INotebookServer | undefined> {
         return this.serverCache.get(options);
     }
-
-    private async checkSupported(command: string, cancelToken?: CancellationToken): Promise<boolean> {
-        const service = await this.waitForService();
-
-        // Make a remote call on the proxy
-        if (service) {
-            const result = await service.request(command, [], cancelToken);
-            return result as boolean;
-        }
-
-        return false;
-    }
 }

src/client/datascience/types.ts

@@ -7,6 +7,7 @@ import type { Kernel, KernelMessage } from '@jupyterlab/services/lib/kernel';
 import type { JSONObject } from '@phosphor/coreutils';
 import { WriteStream } from 'fs-extra';
 import { Observable } from 'rxjs/Observable';
+import { SemVer } from 'semver';
 import {
     CancellationToken,
     CodeLens,
@@ -282,7 +283,7 @@ export const IJupyterExecution = Symbol('IJupyterExecution');
 export interface IJupyterExecution extends IAsyncDisposable {
     serverStarted: Event<INotebookServerOptions | undefined>;
     isNotebookSupported(cancelToken?: CancellationToken): Promise<boolean>;
-    isImportSupported(cancelToken?: CancellationToken): Promise<boolean>;
+    isImportSupported(cancelToken?: CancellationToken): Promise<SemVer | undefined>;
     isSpawnSupported(cancelToken?: CancellationToken): Promise<boolean>;
     connectToNotebookServer(
         options?: INotebookServerOptions,
@@ -994,12 +995,10 @@ export interface IJupyterSubCommandExecutionService {
     isNotebookSupported(cancelToken?: CancellationToken): Promise<boolean>;
     /**
      * Checks whether exporting of ipynb is supported.
-     *
-     * @param {CancellationToken} [cancelToken]
-     * @returns {Promise<boolean>}
-     * @memberof IJupyterSubCommandExecutionService
+     * If exporting is supported return the version of nbconvert available
+     * otherwise undefined.
      */
-    isExportSupported(cancelToken?: CancellationToken): Promise<boolean>;
+    isExportSupported(cancelToken?: CancellationToken): Promise<SemVer | undefined>;
     /**
      * Error message indicating why jupyter notebook isn't supported.
      *

src/test/datascience/execution.unit.test.ts

@@ -1000,6 +1000,14 @@ suite('Jupyter Execution', async () => {
                 return [];
             }
         );
+        when(dependencyService.getNbConvertVersion(anything(), anything())).thenCall(
+            async (interpreter: PythonEnvironment) => {
+                if (interpreter === missingNotebookPython) {
+                    return undefined;
+                }
+                return new SemVer('1.1.1');
+            }
+        );
         const oldStore = mock(JupyterInterpreterOldCacheStateStore);
         when(oldStore.getCachedInterpreterPath()).thenReturn();
         const jupyterInterpreterService = mock(JupyterInterpreterService);
@@ -1047,7 +1055,8 @@ suite('Jupyter Execution', async () => {
         const jupyterExecutionFactory = createExecution(workingPython);
 
         await assert.eventually.equal(jupyterExecutionFactory.isNotebookSupported(), true, 'Notebook not supported');
-        await assert.eventually.equal(jupyterExecutionFactory.isImportSupported(), true, 'Import not supported');
+        const nbConvertVer = await jupyterExecutionFactory.isImportSupported();
+        assert.isTrue(nbConvertVer?.compare('1.1.1') === 0);
         const usableInterpreter = await jupyterExecutionFactory.getUsableJupyterPython();
         assert.isOk(usableInterpreter, 'Usable interpreter not found');
         await assert.isFulfilled(jupyterExecutionFactory.connectToNotebookServer(), 'Should be able to start a server');
@@ -1062,7 +1071,8 @@ suite('Jupyter Execution', async () => {
         );
 
         await assert.eventually.equal(jupyterExecutionFactory.isNotebookSupported(), true, 'Notebook not supported');
-        await assert.eventually.equal(jupyterExecutionFactory.isImportSupported(), true, 'Import not supported');
+        const nbConvertVer = await jupyterExecutionFactory.isImportSupported();
+        assert.isTrue(nbConvertVer?.compare('1.1.1') === 0);
         const usableInterpreter = await jupyterExecutionFactory.getUsableJupyterPython();
         assert.isOk(usableInterpreter, 'Usable interpreter not found');
         await assert.isFulfilled(jupyterExecutionFactory.connectToNotebookServer(), 'Should be able to start a server');

src/test/datascience/jupyter/interpreter/jupyterInterpreterSubCommandExecutionService.unit.test.ts

@@ -102,7 +102,7 @@ suite('DataScience - Jupyter InterpreterSubCommandExecutionService', () => {
         });
         test('Export is not supported', async () => {
             const isSupported = await jupyterInterpreterExecutionService.isExportSupported(undefined);
-            assert.isFalse(isSupported);
+            assert.isUndefined(isSupported);
         });
         test('Jupyter cannot be started because no interpreter has been selected', async () => {
             when(interperterService.getActiveInterpreter(undefined)).thenResolve(undefined);