diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c60e81dd02b..ab546e491337 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -160,7 +160,8 @@ - `[*]` Standardize file names ([#7316](https://github.com/facebook/jest/pull/7316), [#7266](https://github.com/facebook/jest/pull/7266), [#7238](https://github.com/facebook/jest/pull/7238), [#7314](https://github.com/facebook/jest/pull/7314), [#7467](https://github.com/facebook/jest/pull/7467), [#7464](https://github.com/facebook/jest/pull/7464)), [#7471](https://github.com/facebook/jest/pull/7471)) - `[docs]` Add `testPathIgnorePatterns` in CLI documentation ([#7440](https://github.com/facebook/jest/pull/7440)) - `[docs]` Removed misleading text about `describe()` grouping together tests into a test suite ([#7434](https://github.com/facebook/jest/pull/7434)) -- `[*]` Replace as many `Object.assign` with object spread as possible +- `[diff-sequences]` Add performance benchmark to package ([#7603](https://github.com/facebook/jest/pull/7603)) +- `[*]` Replace as many `Object.assign` with object spread as possible ([#7627](https://github.com/facebook/jest/pull/7627)) - `[ci]` Initial support for Azure Pipelines ([#7556](https://github.com/facebook/jest/pull/7556)) ### Performance diff --git a/packages/diff-sequences/package.json b/packages/diff-sequences/package.json index d499f0de5684..19370970cb78 100644 --- a/packages/diff-sequences/package.json +++ b/packages/diff-sequences/package.json @@ -18,5 +18,12 @@ "engines": { "node": ">= 6" }, - "main": "build/index.js" + "main": "build/index.js", + "scripts": { + "perf": "node --expose-gc perf/index.js" + }, + "devDependencies": { + "benchmark": "^2.1.4", + "diff": "^4.0.1" + } } diff --git a/packages/diff-sequences/perf/example.md b/packages/diff-sequences/perf/example.md new file mode 100644 index 000000000000..663f25de078f --- /dev/null +++ b/packages/diff-sequences/perf/example.md @@ -0,0 +1,48 @@ +## Benchmark time for `diff-sequences` versus `diff` + +A ratio less than 1.0 means `diff-sequences` is faster. + +### n = 20 + +| name | % | ratio | improved | rme | baseline | rme | +| :----- | ---: | :----- | :-------- | ----: | :-------- | ----: | +| delete | 20% | 0.1824 | 3.0639e-6 | 1.11% | 1.6795e-5 | 0.78% | +| insert | 20% | 0.1367 | 2.5786e-6 | 0.75% | 1.8866e-5 | 0.85% | +| delete | 40% | 0.1015 | 3.0534e-6 | 1.00% | 3.0090e-5 | 0.70% | +| insert | 40% | 0.0791 | 2.6722e-6 | 0.61% | 3.3791e-5 | 0.56% | +| delete | 80% | 0.0410 | 2.8870e-6 | 0.93% | 7.0411e-5 | 0.72% | +| insert | 80% | 0.0371 | 2.5786e-6 | 0.83% | 6.9431e-5 | 0.62% | +| change | 10% | 0.1504 | 2.8703e-6 | 0.71% | 1.9087e-5 | 0.83% | +| change | 20% | 0.1706 | 3.2637e-6 | 0.78% | 1.9127e-5 | 0.62% | +| change | 50% | 0.0944 | 1.2012e-5 | 0.55% | 1.2724e-4 | 0.76% | +| change | 100% | 0.0522 | 1.5422e-5 | 0.61% | 2.9566e-4 | 0.66% | + +### n = 200 + +| name | % | ratio | improved | rme | baseline | rme | +| :----- | ---: | :----- | :-------- | ----: | :-------- | ----: | +| delete | 20% | 0.1524 | 7.2866e-5 | 0.75% | 4.7797e-4 | 0.80% | +| insert | 20% | 0.1226 | 6.1561e-5 | 0.58% | 5.0198e-4 | 0.66% | +| delete | 40% | 0.1118 | 1.5674e-4 | 0.67% | 1.4020e-3 | 0.58% | +| insert | 40% | 0.0894 | 1.2906e-4 | 0.64% | 1.4435e-3 | 0.53% | +| delete | 80% | 0.0796 | 3.0119e-4 | 0.58% | 3.7852e-3 | 0.52% | +| insert | 80% | 0.0734 | 2.4713e-4 | 0.67% | 3.3653e-3 | 0.54% | +| change | 10% | 0.1572 | 7.2965e-5 | 0.48% | 4.6426e-4 | 0.73% | +| change | 20% | 0.1446 | 7.0056e-5 | 0.69% | 4.8456e-4 | 0.53% | +| change | 50% | 0.0764 | 6.5638e-4 | 0.67% | 8.5946e-3 | 0.70% | +| change | 100% | 0.0525 | 1.1160e-3 | 0.51% | 2.1249e-2 | 0.63% | + +### n = 2000 + +| name | % | ratio | improved | rme | baseline | rme | +| :----- | ---: | :----- | :-------- | ----: | :-------- | ----: | +| delete | 20% | 0.0669 | 3.4073e-3 | 0.54% | 5.0922e-2 | 0.54% | +| insert | 20% | 0.0588 | 2.8273e-3 | 0.51% | 4.8111e-2 | 0.46% | +| delete | 40% | 0.0517 | 1.1048e-2 | 0.52% | 2.1367e-1 | 0.47% | +| insert | 40% | 0.0460 | 9.1469e-3 | 0.37% | 1.9878e-1 | 0.26% | +| delete | 80% | 0.0563 | 2.7426e-2 | 0.56% | 4.8674e-1 | 0.36% | +| insert | 80% | 0.0506 | 2.2208e-2 | 0.35% | 4.3888e-1 | 0.47% | +| change | 10% | 0.0716 | 3.1267e-3 | 1.21% | 4.3652e-2 | 0.56% | +| change | 20% | 0.0621 | 3.0197e-3 | 0.72% | 4.8652e-2 | 0.45% | +| change | 50% | 0.0083 | 5.4250e-2 | 0.62% | 6.5595e+0 | 3.60% | +| change | 100% | 0.0493 | 1.0534e-1 | 0.71% | 2.1362e+0 | 0.21% | diff --git a/packages/diff-sequences/perf/index.js b/packages/diff-sequences/perf/index.js new file mode 100644 index 000000000000..bd0af5584b46 --- /dev/null +++ b/packages/diff-sequences/perf/index.js @@ -0,0 +1,155 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +// Make sure to run node with --expose-gc option! + +// The times are reliable if about 1% relative mean error if you run it: + +// * immediately after restart +// * with 100% battery charge +// * not connected to network + +/* eslint import/no-extraneous-dependencies: "off" */ + +const Benchmark = require('benchmark'); + +const diffBaseline = require('diff').diffLines; +const diffImproved = require('../build/index.js').default; + +const testBaseline = (a, b) => { + const benchmark = new Benchmark({ + fn() { + diffBaseline(a, b); + }, + name: 'baseline', + onCycle() { + global.gc(); // after run cycle + }, + onStart() { + global.gc(); // when benchmark starts + }, + }); + + benchmark.run({async: false}); + + return benchmark.stats; +}; + +const testImproved = function(a, b) { + const benchmark = new Benchmark({ + fn() { + // Split string arguments to make fair comparison with baseline. + const aItems = a.split('\n'); + const bItems = b.split('\n'); + + const isCommon = (aIndex, bIndex) => aItems[aIndex] === bItems[bIndex]; + + // This callback obviously does less than baseline `diff` package, + // but avoiding double work and memory churn is the goal. + // For example, `jest-diff` has had to split strings that `diff` joins. + const foundSubsequence = () => {}; + + diffImproved(aItems.length, bItems.length, isCommon, foundSubsequence); + }, + name: 'improved', + onCycle() { + global.gc(); // after run cycle + }, + onStart() { + global.gc(); // when benchmark starts + }, + }); + + benchmark.run({async: false}); + + return benchmark.stats; +}; + +const writeHeading2 = () => { + console.log('## Benchmark time for `diff-sequences` versus `diff`\n'); + console.log('A ratio less than 1.0 means `diff-sequences` is faster.'); +}; + +const writeHeading3 = n => { + console.log(`\n### n = ${n}\n`); + console.log('| name | % | ratio | improved | rme | baseline | rme |'); + console.log('| :--- | ---: | :--- | :--- | ---: | :--- | ---: |'); +}; + +const writeRow = (name, percent, statsImproved, statsBaseline) => { + const {mean: meanImproved, rme: rmeImproved} = statsImproved; + const {mean: meanBaseline, rme: rmeBaseline} = statsBaseline; + const ratio = meanImproved / meanBaseline; + + console.log( + `| ${name} | ${percent}% | ${ratio.toFixed( + 4, + )} | ${meanImproved.toExponential(4)} | ${rmeImproved.toFixed( + 2, + )}% | ${meanBaseline.toExponential(4)} | ${rmeBaseline.toFixed(2)}% |`, + ); +}; + +const testDeleteInsert = (tenths, more, less) => { + // For improved `diff-sequences` package, delete is often slower than insert. + const statsDeleteImproved = testImproved(more, less); + const statsDeleteBaseline = testBaseline(more, less); + writeRow('delete', tenths * 10, statsDeleteImproved, statsDeleteBaseline); + + // For baseline `diff` package, many insertions is serious perf problem. + // However, the benchmark package cannot accurately measure for large n. + const statsInsertBaseline = testBaseline(less, more); + const statsInsertImproved = testImproved(less, more); + writeRow('insert', tenths * 10, statsInsertImproved, statsInsertBaseline); +}; + +const testChange = (tenths, expected, received) => { + const statsImproved = testImproved(expected, received); + const statsBaseline = testBaseline(expected, received); + writeRow('change', tenths * 10, statsImproved, statsBaseline); +}; + +const getItems = (n, callback) => { + const items = []; + + for (let i = 0; i !== n; i += 1) { + const item = callback(i); + if (typeof item === 'string') { + items.push(item); + } + } + + return items.join('\n'); +}; + +// Simulate change of property name which is usually not same line. +// Expected: 0 1 2 3 4 5 6 7 8 9 and so on +// Received: 1 2 3 4 x0 5 6 7 8 9 and so on +const change2 = i => { + const j = i % 10; + return j === 4 ? `x${i - 4}` : j < 4 ? `${i + 1}` : `${i}`; +}; + +const testLength = n => { + const all = getItems(n, i => `${i}`); + + writeHeading3(n); + + [2, 4, 8].forEach(tenth => { + testDeleteInsert(tenth, all, getItems(n, i => i % 10 >= tenth && `${i}`)); + }); + testChange(1, all, getItems(n, i => (i % 10 === 0 ? `x${i}` : `${i}`))); + testChange(2, all, getItems(n, change2)); + testChange(5, all, getItems(n, i => (i % 2 === 0 ? `x${i}` : `${i}`))); + testChange(10, all, getItems(n, i => `x${i}`)); // simulate TDD +}; + +writeHeading2(); + +testLength(20); +testLength(200); +testLength(2000); diff --git a/yarn.lock b/yarn.lock index 8a4e188212b9..7a59dbb0cdf8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2176,6 +2176,14 @@ beeper@^1.0.0: resolved "https://registry.yarnpkg.com/beeper/-/beeper-1.1.1.tgz#e6d5ea8c5dad001304a70b22638447f69cb2f809" integrity sha1-5tXqjF2tABMEpwsiY4RH9pyy+Ak= +benchmark@^2.1.4: + version "2.1.4" + resolved "https://registry.yarnpkg.com/benchmark/-/benchmark-2.1.4.tgz#09f3de31c916425d498cc2ee565a0ebf3c2a5629" + integrity sha1-CfPeMckWQl1JjMLuVloOvzwqVik= + dependencies: + lodash "^4.17.4" + platform "^1.3.3" + better-assert@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/better-assert/-/better-assert-1.0.2.tgz#40866b9e1b9e0b55b481894311e68faffaebc522" @@ -4091,6 +4099,11 @@ diff@3.5.0: resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12" integrity sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA== +diff@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.1.tgz#0c667cb467ebbb5cea7f14f135cc2dba7780a8ff" + integrity sha512-s2+XdvhPCOF01LRQBC8hf4vhbVmI2CGS5aZnxLJlT5FtdhPCDFq80q++zK2KlrVorVDdL5BOGZ/VfLrVtYNF+Q== + diffie-hellman@^5.0.0: version "5.0.3" resolved "https://registry.yarnpkg.com/diffie-hellman/-/diffie-hellman-5.0.3.tgz#40e8ee98f55a2149607146921c63e1ae5f3d2875" @@ -9945,6 +9958,11 @@ pkg-dir@^3.0.0: dependencies: find-up "^3.0.0" +platform@^1.3.3: + version "1.3.5" + resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.5.tgz#fb6958c696e07e2918d2eeda0f0bc9448d733444" + integrity sha512-TuvHS8AOIZNAlE77WUDiR4rySV/VMptyMfcfeoMgs4P8apaZM3JrnbzBiixKUv+XR6i+BXrQh8WAnjaSPFO65Q== + plist@2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/plist/-/plist-2.0.1.tgz#0a32ca9481b1c364e92e18dc55c876de9d01da8b"