Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[v16.x backport] test: add WPTRunner support for variants and generating WPT reports #46768

Closed
wants to merge 3 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -573,6 +573,12 @@ test-message: test-build
test-wpt: all
$(PYTHON) tools/test.py $(PARALLEL_ARGS) wpt

.PHONY: test-wpt-report
test-wpt-report:
$(RM) -r out/wpt
mkdir -p out/wpt
WPT_REPORT=1 $(PYTHON) tools/test.py --shell $(NODE) $(PARALLEL_ARGS) wpt

.PHONY: test-simple
test-simple: | cctest # Depends on 'all'.
$(PYTHON) tools/test.py $(PARALLEL_ARGS) parallel sequential
Expand Down
261 changes: 204 additions & 57 deletions test/common/wpt.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,119 @@ const fs = require('fs');
const fsPromises = fs.promises;
const path = require('path');
const events = require('events');
const os = require('os');
const { inspect } = require('util');
const { Worker } = require('worker_threads');

function getBrowserProperties() {
const { node: version } = process.versions; // e.g. 18.13.0, 20.0.0-nightly202302078e6e215481
const release = /^\d+\.\d+\.\d+$/.test(version);
const browser = {
browser_channel: release ? 'stable' : 'experimental',
browser_version: version,
};

return browser;
}

/**
* Return one of three expected values
* https://github.com/web-platform-tests/wpt/blob/1c6ff12/tools/wptrunner/wptrunner/tests/test_update.py#L953-L958
*/
function getOs() {
switch (os.type()) {
case 'Linux':
return 'linux';
case 'Darwin':
return 'mac';
case 'Windows_NT':
return 'win';
default:
throw new Error('Unsupported os.type()');
}
}

// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3705
function sanitizeUnpairedSurrogates(str) {
return str.replace(
/([\ud800-\udbff]+)(?![\udc00-\udfff])|(^|[^\ud800-\udbff])([\udc00-\udfff]+)/g,
function(_, low, prefix, high) {
let output = prefix || ''; // Prefix may be undefined
const string = low || high; // Only one of these alternates can match
for (let i = 0; i < string.length; i++) {
output += codeUnitStr(string[i]);
}
return output;
});
}

function codeUnitStr(char) {
return 'U+' + char.charCodeAt(0).toString(16);
}

class WPTReport {
constructor() {
this.results = [];
this.time_start = Date.now();
}

addResult(name, status) {
const result = {
test: name,
status,
subtests: [],
addSubtest(name, status, message) {
const subtest = {
status,
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3722
name: sanitizeUnpairedSurrogates(name),
};
if (message) {
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L4506
subtest.message = sanitizeUnpairedSurrogates(message);
}
this.subtests.push(subtest);
return subtest;
},
};
this.results.push(result);
return result;
}

write() {
this.time_end = Date.now();
this.results = this.results.filter((result) => {
return result.status === 'SKIP' || result.subtests.length !== 0;
}).map((result) => {
const url = new URL(result.test, 'http://wpt');
url.pathname = url.pathname.replace(/\.js$/, '.html');
result.test = url.href.slice(url.origin.length);
return result;
});

if (fs.existsSync('out/wpt/wptreport.json')) {
const prev = JSON.parse(fs.readFileSync('out/wpt/wptreport.json'));
this.results = [...prev.results, ...this.results];
this.time_start = prev.time_start;
this.time_end = Math.max(this.time_end, prev.time_end);
this.run_info = prev.run_info;
} else {
/**
* Return required and some optional properties
* https://github.com/web-platform-tests/wpt.fyi/blob/60da175/api/README.md?plain=1#L331-L335
*/
this.run_info = {
product: 'node.js',
...getBrowserProperties(),
revision: process.env.WPT_REVISION || 'unknown',
os: getOs(),
};
}

fs.writeFileSync('out/wpt/wptreport.json', JSON.stringify(this));
}
}

// https://github.com/web-platform-tests/wpt/blob/HEAD/resources/testharness.js
// TODO: get rid of this half-baked harness in favor of the one
// pulled from WPT
Expand Down Expand Up @@ -313,6 +423,10 @@ class WPTRunner {
this.unexpectedFailures = [];

this.scriptsModifier = null;

if (process.env.WPT_REPORT != null) {
this.report = new WPTReport();
}
}

/**
Expand All @@ -339,18 +453,27 @@ class WPTRunner {
this.scriptsModifier = modifier;
}

get fullInitScript() {
if (this.initScript === null && this.dummyGlobalThisScript === null) {
fullInitScript(hasSubsetScript, locationSearchString) {
let { initScript } = this;
if (hasSubsetScript || locationSearchString) {
initScript = `${initScript}\n\n//===\nglobalThis.location ||= {};`;
}

if (locationSearchString) {
initScript = `${initScript}\n\n//===\nglobalThis.location.search = "${locationSearchString}";`;
}

if (initScript === null && this.dummyGlobalThisScript === null) {
return null;
}

if (this.initScript === null) {
if (initScript === null) {
return this.dummyGlobalThisScript;
} else if (this.dummyGlobalThisScript === null) {
return this.initScript;
return initScript;
}

return `${this.dummyGlobalThisScript}\n\n//===\n${this.initScript}`;
return `${this.dummyGlobalThisScript}\n\n//===\n${initScript}`;
}

/**
Expand Down Expand Up @@ -398,15 +521,20 @@ class WPTRunner {
for (const spec of queue) {
const testFileName = spec.filename;
const content = spec.getContent();
const meta = spec.title = this.getMeta(content);
const meta = spec.meta = this.getMeta(content);

const absolutePath = spec.getAbsolutePath();
const relativePath = spec.getRelativePath();
const harnessPath = fixtures.path('wpt', 'resources', 'testharness.js');
const scriptsToRun = [];
let hasSubsetScript = false;

// Scripts specified with the `// META: script=` header
if (meta.script) {
for (const script of meta.script) {
if (script === '/common/subset-tests.js' || script === '/common/subset-tests-by-key.js') {
hasSubsetScript = true;
}
const obj = {
filename: this.resource.toRealFilePath(relativePath, script),
code: this.resource.read(relativePath, script, false)
Expand All @@ -423,54 +551,65 @@ class WPTRunner {
this.scriptsModifier?.(obj);
scriptsToRun.push(obj);

const workerPath = path.join(__dirname, 'wpt/worker.js');
const worker = new Worker(workerPath, {
execArgv: this.flags,
workerData: {
testRelativePath: relativePath,
wptRunner: __filename,
wptPath: this.path,
initScript: this.fullInitScript,
harness: {
code: fs.readFileSync(harnessPath, 'utf8'),
filename: harnessPath,
/**
* Example test with no META variant
* https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/sign_verify/hmac.https.any.js#L1-L4
*
* Example test with multiple META variants
* https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/generateKey/successes_RSASSA-PKCS1-v1_5.https.any.js#L1-L9
*/
for (const variant of meta.variant || ['']) {
const workerPath = path.join(__dirname, 'wpt/worker.js');
const worker = new Worker(workerPath, {
execArgv: this.flags,
workerData: {
testRelativePath: relativePath,
wptRunner: __filename,
wptPath: this.path,
initScript: this.fullInitScript(hasSubsetScript, variant),
harness: {
code: fs.readFileSync(harnessPath, 'utf8'),
filename: harnessPath,
},
scriptsToRun,
},
scriptsToRun,
},
});
this.workers.set(testFileName, worker);

worker.on('message', (message) => {
switch (message.type) {
case 'result':
return this.resultCallback(testFileName, message.result);
case 'completion':
return this.completionCallback(testFileName, message.status);
default:
throw new Error(`Unexpected message from worker: ${message.type}`);
}
});
});
this.workers.set(testFileName, worker);

let reportResult;
worker.on('message', (message) => {
switch (message.type) {
case 'result':
reportResult ||= this.report?.addResult(`/${relativePath}${variant}`, 'OK');
return this.resultCallback(testFileName, message.result, reportResult);
case 'completion':
return this.completionCallback(testFileName, message.status);
default:
throw new Error(`Unexpected message from worker: ${message.type}`);
}
});

worker.on('error', (err) => {
if (!this.inProgress.has(testFileName)) {
worker.on('error', (err) => {
if (!this.inProgress.has(testFileName)) {
// The test is already finished. Ignore errors that occur after it.
// This can happen normally, for example in timers tests.
return;
}
this.fail(
testFileName,
{
status: NODE_UNCAUGHT,
name: 'evaluation in WPTRunner.runJsTests()',
message: err.message,
stack: inspect(err)
},
kUncaught
);
this.inProgress.delete(testFileName);
});
return;
}
this.fail(
testFileName,
{
status: NODE_UNCAUGHT,
name: 'evaluation in WPTRunner.runJsTests()',
message: err.message,
stack: inspect(err)
},
kUncaught
);
this.inProgress.delete(testFileName);
});

await events.once(worker, 'exit').catch(() => {});
await events.once(worker, 'exit').catch(() => {});
}
}

process.on('exit', () => {
Expand Down Expand Up @@ -529,6 +668,8 @@ class WPTRunner {
}
}

this.report?.write();

const ran = total - skipped;
const passed = ran - expectedFailures - failures.length;
console.log(`Ran ${ran}/${total} tests, ${skipped} skipped,`,
Expand All @@ -552,8 +693,7 @@ class WPTRunner {

getTestTitle(filename) {
const spec = this.specMap.get(filename);
const title = spec.meta && spec.meta.title;
return title ? `${filename} : ${title}` : filename;
return spec.meta?.title || filename.split('.')[0];
}

// Map WPT test status to strings
Expand All @@ -579,14 +719,17 @@ class WPTRunner {
* @param {string} filename
* @param {Test} test The Test object returned by WPT harness
*/
resultCallback(filename, test) {
resultCallback(filename, test, reportResult) {
const status = this.getTestStatus(test.status);
const title = this.getTestTitle(filename);
if (/^Untitled( \d+)?$/.test(test.name)) {
test.name = `${title}${test.name.slice(8)}`;
}
console.log(`---- ${title} ----`);
if (status !== kPass) {
this.fail(filename, test, status);
this.fail(filename, test, status, reportResult);
} else {
this.succeed(filename, test, status);
this.succeed(filename, test, status, reportResult);
}
}

Expand Down Expand Up @@ -634,11 +777,12 @@ class WPTRunner {
}
}

succeed(filename, test, status) {
succeed(filename, test, status, reportResult) {
console.log(`[${status.toUpperCase()}] ${test.name}`);
reportResult?.addSubtest(test.name, 'PASS');
}

fail(filename, test, status) {
fail(filename, test, status, reportResult) {
const spec = this.specMap.get(filename);
const expected = spec.failedTests.includes(test.name);
if (expected) {
Expand All @@ -654,6 +798,9 @@ class WPTRunner {
const command = `${process.execPath} ${process.execArgv}` +
` ${require.main.filename} ${filename}`;
console.log(`Command: ${command}\n`);

reportResult?.addSubtest(test.name, 'FAIL', test.message);

this.addTestResult(filename, {
name: test.name,
expected,
Expand Down Expand Up @@ -683,7 +830,7 @@ class WPTRunner {
const parts = match.match(/\/\/ META: ([^=]+?)=(.+)/);
const key = parts[1];
const value = parts[2];
if (key === 'script') {
if (key === 'script' || key === 'variant') {
if (result[key]) {
result[key].push(value);
} else {
Expand Down
4 changes: 2 additions & 2 deletions test/wpt/status/dom/events.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
"Event-constructors.any.js": {
"fail": {
"expected": [
"Untitled 2",
"Untitled 3"
"Event constructors 2",
"Event constructors 3"
]
}
},
Expand Down