Skip to content

Commit

Permalink
benchmark: add test and all options and improve errors
Browse files Browse the repository at this point in the history
This adds a new `test` option. Using it automatically uses a single
minimal option matrix to verify the benchmark works as expected.

Using the new `all` option makes sure all test suites are run.

On top of that the benchmarks will from now on report properly
what category might have a typo, if any.

The http duration was also refactored to use a option instead of
relying on a configuration setting.

The fixture folder is ignored as test suite from now on.

PR-URL: nodejs#31396
Fixes: nodejs#31083
Reviewed-By: Sam Roberts <vieuxtech@gmail.com>
Reviewed-By: Rich Trott <rtrott@gmail.com>
  • Loading branch information
BridgeAR committed Feb 9, 2020
1 parent b70741e commit dac5795
Show file tree
Hide file tree
Showing 37 changed files with 292 additions and 189 deletions.
22 changes: 16 additions & 6 deletions benchmark/_cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,24 @@ const path = require('path');
// Create an object of all benchmark scripts
const benchmarks = {};
fs.readdirSync(__dirname)
.filter((name) => fs.statSync(path.resolve(__dirname, name)).isDirectory())
.filter((name) => {
return name !== 'fixtures' &&
fs.statSync(path.resolve(__dirname, name)).isDirectory();
})
.forEach((category) => {
benchmarks[category] = fs.readdirSync(path.resolve(__dirname, category))
.filter((filename) => filename[0] !== '.' && filename[0] !== '_');
});

function CLI(usage, settings) {
if (!(this instanceof CLI)) return new CLI(usage, settings);

if (process.argv.length < 3) {
this.abort(usage); // Abort will exit the process
}

this.usage = usage;
this.optional = {};
this.items = [];
this.test = false;

for (const argName of settings.arrayArgs) {
this.optional[argName] = [];
Expand All @@ -34,7 +36,7 @@ function CLI(usage, settings) {
if (arg === '--') {
// Only items can follow --
mode = 'item';
} else if ('both' === mode && arg[0] === '-') {
} else if (mode === 'both' && arg[0] === '-') {
// Optional arguments declaration

if (arg[1] === '-') {
Expand All @@ -61,6 +63,8 @@ function CLI(usage, settings) {

// The next value can be either an option or an item
mode = 'both';
} else if (arg === 'test') {
this.test = true;
} else if (['both', 'item'].includes(mode)) {
// item arguments
this.items.push(arg);
Expand All @@ -83,9 +87,15 @@ CLI.prototype.abort = function(msg) {
CLI.prototype.benchmarks = function() {
const paths = [];

if (this.items.includes('all')) {
this.items = Object.keys(benchmarks);
}

for (const category of this.items) {
if (benchmarks[category] === undefined)
continue;
if (benchmarks[category] === undefined) {
console.error(`The "${category}" category does not exist.`);
process.exit(1);
}
for (const scripts of benchmarks[category]) {
if (this.shouldSkip(scripts)) continue;

Expand Down
23 changes: 13 additions & 10 deletions benchmark/_http-benchmarkers.js
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,8 @@ class AutocannonBenchmarker {
}
if (!result || !result.requests || !result.requests.average) {
return undefined;
} else {
return result.requests.average;
}
return result.requests.average;
}
}

Expand Down Expand Up @@ -77,9 +76,8 @@ class WrkBenchmarker {
const throughput = match && +match[1];
if (!isFinite(throughput)) {
return undefined;
} else {
return throughput;
}
return throughput;
}
}

Expand All @@ -89,18 +87,20 @@ class WrkBenchmarker {
*/
class TestDoubleBenchmarker {
constructor(type) {
// `type` is the type ofbenchmarker. Possible values are 'http' and 'http2'.
// `type` is the type of benchmarker. Possible values are 'http' and
// 'http2'.
this.name = `test-double-${type}`;
this.executable = path.resolve(__dirname, '_test-double-benchmarker.js');
this.present = fs.existsSync(this.executable);
this.type = type;
}

create(options) {
const env = Object.assign({
const env = {
duration: options.duration,
test_url: `http://127.0.0.1:${options.port}${options.path}`,
}, process.env);
...process.env
};

const child = child_process.fork(this.executable,
[this.type],
Expand Down Expand Up @@ -189,13 +189,14 @@ http_benchmarkers.forEach((benchmarker) => {
});

exports.run = function(options, callback) {
options = Object.assign({
options = {
port: exports.PORT,
path: '/',
connections: 100,
duration: 5,
benchmarker: exports.default_http_benchmarker,
}, options);
...options
};
if (!options.benchmarker) {
callback(new Error('Could not locate required http benchmarker. See ' +
`${requirementsURL} for further instructions.`));
Expand All @@ -212,6 +213,7 @@ exports.run = function(options, callback) {
'is not installed'));
return;
}
process.env.duration = process.env.duration || options.duration || 5;

const benchmarker_start = process.hrtime();

Expand All @@ -220,7 +222,8 @@ exports.run = function(options, callback) {
child.stderr.pipe(process.stderr);

let stdout = '';
child.stdout.on('data', (chunk) => stdout += chunk.toString());
child.stdout.setEncoding('utf8');
child.stdout.on('data', (chunk) => stdout += chunk);

child.once('close', (code) => {
const elapsed = process.hrtime(benchmarker_start);
Expand Down
10 changes: 6 additions & 4 deletions benchmark/_test-double-benchmarker.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ if (!['http', 'http2'].includes(myModule)) {

const http = require(myModule);

const duration = process.env.duration || 0;
const duration = +process.env.duration;
const url = process.env.test_url;

const start = process.hrtime();
Expand All @@ -18,13 +18,15 @@ function request(res, client) {
res.on('error', () => {});
res.on('end', () => {
throughput++;
const diff = process.hrtime(start);
if (duration > 0 && diff[0] < duration) {
const [sec, nanosec] = process.hrtime(start);
const ms = sec * 1000 + nanosec / 1e6;
if (ms < duration * 1000) {
run();
} else {
console.log(JSON.stringify({ throughput }));
if (client) {
client.destroy();
process.exit(0);
}
}
});
Expand All @@ -33,7 +35,7 @@ function request(res, client) {
function run() {
if (http.get) { // HTTP
http.get(url, request);
} else { // HTTP/2
} else { // HTTP/2
const client = http.connect(url);
client.on('error', (e) => { throw e; });
request(client.request(), client);
Expand Down
6 changes: 4 additions & 2 deletions benchmark/async_hooks/http-server.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@ const common = require('../common.js');

const bench = common.createBenchmark(main, {
asyncHooks: ['init', 'before', 'after', 'all', 'disabled', 'none'],
connections: [50, 500]
connections: [50, 500],
duration: 5
});

function main({ asyncHooks, connections }) {
function main({ asyncHooks, connections, duration }) {
if (asyncHooks !== 'none') {
let hooks = {
init() {},
Expand All @@ -33,6 +34,7 @@ function main({ asyncHooks, connections }) {
bench.http({
connections,
path,
duration
}, () => {
server.close();
});
Expand Down
2 changes: 2 additions & 0 deletions benchmark/buffers/buffer-base64-encode.js
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ const common = require('../common.js');
const bench = common.createBenchmark(main, {
len: [64 * 1024 * 1024],
n: [32]
}, {
test: { len: 256 }
});

function main({ n, len }) {
Expand Down
2 changes: 2 additions & 0 deletions benchmark/buffers/buffer-swap.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ const bench = common.createBenchmark(main, {
method: ['swap16', 'swap32', 'swap64'/* , 'htons', 'htonl', 'htonll' */],
len: [64, 256, 768, 1024, 2056, 8192],
n: [1e6]
}, {
test: { len: 16 }
});

// The htons and htonl methods below are used to benchmark the
Expand Down
Loading

0 comments on commit dac5795

Please sign in to comment.