Skip to content

Commit

Permalink
More of the functional tests are failing (microsoft#14346)
Browse files Browse the repository at this point in the history
* Splitting test log

* Fix problem with kernels ports being reused

* Make kernel launcher port round robin only for testing

* Make formatters change only apply during testing

* Add news entry

* Apply black formatting

* Code review feedback and skip flakey remote password test

* Another flakey test

* More CR feedback

* Missed a spot

* Some more log parser changes and try to get interrupt to be less flakey

* Fix interrupt killing kernel and add more logging for export

* More logging

* See if updating fixes the problem

* Dont delete temp files

* Upload webview output to figure out trust failure

* Add name to step

* Try another way to upload

* Upload doesn't seem to work

* Try a different way to upload

* Try without webview logging as this makes the test pass

* Try fixing test another way. Any logging is making the test pass

* Compile error
  • Loading branch information
rchiodo authored and luabud committed Oct 26, 2020
1 parent c21a114 commit c2e5239
Show file tree
Hide file tree
Showing 10 changed files with 52 additions and 32 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/pr_datascience.yml
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ jobs:
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
report_paths: ${{ env.TEST_RESULTS_GLOB }}
check_name: Functional Test Report
check_name: Functional Test Report ${{matrix.test-suite}}
if: steps.test_functional_group.outcome == 'failure' && failure()

testsInVSCode:
Expand Down
4 changes: 3 additions & 1 deletion .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,9 @@
// Remove `X` prefix and update path to test with real python interpreter (for DS functional tests).
"XCI_PYTHON_PATH": "<Python Path>",
// Remove 'X' prefix to dump output for debugger. Directory has to exist prior to launch
"XDEBUGPY_LOG_DIR": "${workspaceRoot}/tmp/Debug_Output"
"XDEBUGPY_LOG_DIR": "${workspaceRoot}/tmp/Debug_Output",
// Remove 'X' prefix to dump webview redux action log
"XVSC_PYTHON_WEBVIEW_LOG_FILE": "${workspaceRoot}/test-webview.log"
},
"outFiles": ["${workspaceFolder}/out/**/*.js", "!${workspaceFolder}/**/node_modules**/*"],
"preLaunchTask": "Compile",
Expand Down
5 changes: 3 additions & 2 deletions build/ci/scripts/runFunctionalTests.js
Original file line number Diff line number Diff line change
Expand Up @@ -68,12 +68,13 @@ async function generateGroups(files) {

async function runIndividualTest(extraArgs, file, index) {
var subMochaFile = `${mochaBaseFile}_${index}_${path.basename(file)}${mochaFileExt}`;
process.env['MOCHA_FILE'] = subMochaFile;
var args = gatherArgs(extraArgs, file);
console.log(`Running functional test for file ${file} ...`);
var exitCode = await new Promise((resolve) => {
// Spawn the sub node process
var proc = child_process.fork('./node_modules/mocha/bin/_mocha', args);
var proc = child_process.fork('./node_modules/mocha/bin/_mocha', args, {
env: { ...process.env, MOCHA_FILE: subMochaFile }
});
proc.on('exit', resolve);
});

Expand Down
10 changes: 7 additions & 3 deletions pythonFiles/vscode_datascience_helpers/jupyter_daemon.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,11 +140,15 @@ def _print_kernel_list_json(self):
sys.stdout.flush()

def _convert(self, args):
self.log.info("nbconvert")
self.log.info("Starting nbconvert wirth args %s", args)
from nbconvert import nbconvertapp as app

sys.argv = [""] + args
app.main()
try:
sys.argv = [""] + args
app.main()
except Exception as e:
self.log.info("Nbconvert error: %s", e)
raise

def _start_notebook(self, args, cwd, env):
from notebook import notebookapp as app
Expand Down
22 changes: 17 additions & 5 deletions pythonFiles/vscode_datascience_helpers/tests/logParser.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from io import TextIOWrapper
import sys
import argparse
import os
Expand All @@ -18,13 +19,25 @@
)
ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
pid_regex = re.compile(r"(\d+).*")
timestamp_regex = re.compile(r"\d{4}-\d{2}-\d{2}T.*\dZ")


def stripTimestamp(line: str):
match = timestamp_regex.match(line)
if match:
return line[match.end() :]
return line


def readStripLines(f: TextIOWrapper):
return map(stripTimestamp, f.readlines())


def printTestOutput(testlog):
# Find all the lines that don't have a PID in them. These are the test output
p = Path(testlog[0])
with p.open() as f:
for line in f.readlines():
for line in readStripLines(f):
stripped = line.strip()
if len(stripped) > 2 and stripped[0] == "\x1B" and stripped[1] == "[":
print(line.rstrip()) # Should be a test line as it has color encoding
Expand All @@ -38,15 +51,14 @@ def splitByPid(testlog):
logs = {}
pid = None
with p.open() as f:
for line in f.readlines():
for line in readStripLines(f):
stripped = ansi_escape.sub("", line.strip())
# See if starts with a pid
if len(stripped) > 0 and stripped[0] <= "9" and stripped[0] >= "0":
if len(stripped) > 0:
# Pull out the pid
match = pid_regex.match(stripped)

# Pids are at least two digits
if match != None and len(match.group(1)) > 2:
if match and len(match.group(1)) > 2:
# Pid is found
pid = int(match.group(1))

Expand Down
3 changes: 2 additions & 1 deletion src/client/datascience/export/exportBase.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,8 @@ export class ExportBase implements IExport {
'--output',
path.basename(tempTarget.filePath),
'--output-dir',
path.dirname(tempTarget.filePath)
path.dirname(tempTarget.filePath),
'--debug'
];
const result = await service.execModule('jupyter', ['nbconvert'].concat(args), {
throwOnStdErr: false,
Expand Down
15 changes: 0 additions & 15 deletions src/test/datascience/dataScienceIocContainer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@
//tslint:disable:trailing-comma no-any
import * as child_process from 'child_process';
import { ReactWrapper } from 'enzyme';
import * as fs from 'fs-extra';
import * as glob from 'glob';
import { interfaces } from 'inversify';
import * as os from 'os';
import * as path from 'path';
Expand All @@ -27,7 +25,6 @@ import {
import * as vsls from 'vsls/vscode';
import { KernelDaemonPool } from '../../client/datascience/kernel-launcher/kernelDaemonPool';

import { promisify } from 'util';
import { LanguageServerExtensionActivationService } from '../../client/activation/activationService';
import { LanguageServerDownloader } from '../../client/activation/common/downloader';
import { JediExtensionActivator } from '../../client/activation/jedi';
Expand Down Expand Up @@ -500,18 +497,6 @@ export class DataScienceIocContainer extends UnitTestIocContainer {
// Make sure to disable all command handling during dispose. Don't want
// anything to startup again.
this.commandManager.dispose();
try {
// Make sure to delete any temp files written by native editor storage
const globPr = promisify(glob);
const tempLocation = os.tmpdir;
const tempFiles = await globPr(`${tempLocation}/*.ipynb`);
if (tempFiles && tempFiles.length) {
await Promise.all(tempFiles.map((t) => fs.remove(t)));
}
} catch (exc) {
// tslint:disable-next-line: no-console
console.log(`Exception on cleanup: ${exc}`);
}
await this.asyncRegistry.dispose();
await super.dispose();
this.disposed = true;
Expand Down
4 changes: 2 additions & 2 deletions src/test/datascience/notebook.functional.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -680,8 +680,8 @@ suite('DataScience notebook tests', () => {
try {
importer.dispose();
temp.dispose();
} catch {
// Don't care if they don't delete
} catch (exc) {
console.log(exc);
}
}
});
Expand Down
12 changes: 11 additions & 1 deletion src/test/datascience/raw-kernel/rawKernel.functional.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -120,13 +120,23 @@ suite('DataScience raw kernel tests', () => {
// Hence timeout is a test failure.
const longCellExecutionRequest = requestExecute(
rawKernel,
'import time\nfor i in range(300):\n time.sleep(1)',
`
import time
import sys
for i in range(3000):
sys.stdout.write('.')
sys.stdout.flush()
time.sleep(0.1)
sys.stdout.write('\\\r')`,
executionStarted
);

// Wait until the execution has started (cuz we cannot interrupt until exec has started).
await executionStarted.promise;

// Give it a bit to start running
await sleep(300);

// Then throw the interrupt
await rawKernel.interrupt();

Expand Down
7 changes: 6 additions & 1 deletion src/test/datascience/trustedNotebooks.functional.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import * as chaiAsPromised from 'chai-as-promised';
import { ReactWrapper } from 'enzyme';
import * as fs from 'fs-extra';
import { Disposable } from 'vscode';
import { sleep } from '../../client/common/utils/async';
import { noop } from '../../client/common/utils/misc';
import { InteractiveWindowMessages } from '../../client/datascience/interactive-common/interactiveWindowTypes';
import { INotebookEditor, INotebookEditorProvider, ITrustService } from '../../client/datascience/types';
Expand Down Expand Up @@ -440,10 +441,14 @@ suite('DataScience Notebook trust', () => {
// Reopen
const newNativeEditor = await openEditor(ioc, baseFile, notebookFile.filePath);
const newWrapper = newNativeEditor.mount.wrapper;
assert.ok(newNativeEditor.editor.model.isTrusted, 'Editor did not open as trusted');

// Wait a bit. UI doesn't seem to update right away
await sleep(500);

// Verify notebook is now trusted
const after = newWrapper.find(TrustMessage);
assert.equal(after.text(), 'Trusted');
assert.equal(after.text(), 'Trusted', 'Notebook UI not reflecting trust state');
});
});
});

0 comments on commit c2e5239

Please sign in to comment.