diff --git a/.gitignore b/.gitignore index 0860eef..70a8619 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ node_modules/ package-lock.json .history .yarn/cache -.DS_Store \ No newline at end of file +.yarn/install-state.gz +.DS_Store diff --git a/README.md b/README.md index 84afb5f..dfb9af4 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ 1. [Before and After](#before-and-after) 2. [BeforeAll and AfterAll](#beforeall-and-afterall) 4. [Data tables](#data-tables) + 5. [Step reporting](#step-reporting) 8. [Using typescript and ESnext features](#using-typescript-and-esnext-features) 9. [Contributing](#contributing) 1. [Commits](#commits) @@ -375,7 +376,92 @@ When steps have a data table, they are passed an object with methods that can be - `raw`: returns the table as a 2-D array - `rowsHash`: returns an object where each row corresponds to an entry (first column is the key, second column is the value) -see examples section for an example +See the examples section for an example + +### Step reporting + +By default, the reporter used by TestCafé is `spec`. +TestCafé has no reason to handle the concept of "step" because it's a notion that is specific to gherkin. + +To work around that: +- The metadata of a `test` now contains the full list of steps that compose the `scenario` it's based on. +In case of failure of a step, its index is also added to the metadata. +- A custom reporter (`gtc-reporter-spec`) has been added to the project. +It is automatically used instead of spec as the default reporter for `gherkin-testcafe`. +Note that `spec` remains usable by simply using the `reporter` option provided by TestCafé: + ```bash + gherkin-testcafe chrome ./tests/* --reporter spec + ``` + If you use the API, + ```js + runner.reporter("spec") + ``` +- Custom internal reporters have also been created based on `list` and `minimal`. +The gtc reporters behave in exactly the same way as their TestCafé counterparts, except that the steps are part of the +output, with highlighing indicating which ones succeeded, which ones failed, and which ones didn't run. + + > ✓ Given some step that succeeded + > + > ✖ When some step that failed + > + > - Then some step that didn't run + + To use one of this package's internal reporters, use its name in the reporter option: + ```bash + gherkin-testcafe chrome ./tests/* --reporter gtc-reporter-list + gherkin-testcafe chrome ./tests/* --reporter gtc-reporter-minimal + gherkin-testcafe chrome ./tests/* --reporter gtc-reporter-spec # unnecessary as it is the default behavior + ``` + +Note that other official reporters could be adapted in the future. +#### Implement / Adapt a custom reporter + +If you are using a [custom reporter](https://testcafe.io/documentation/402810/guides/extend-testcafe/reporter-plugin) +and want to use or display the step information, all you need to do is access the metadata from your reporter's methods. + +Fortunately, accessing metadata is [built-in behavior](https://testcafe.io/documentation/402810/guides/extend-testcafe/reporter-plugin#implement-the-reporter) for normal TestCafé reporters: + +TestCafé will pass the metadata object to your test reporting function as the third argument. + +The properties that are dedicated to this feature are `steps` and `failIndex`. +Each step in the `steps` array has two properties: `type` and `text`. + +Representation: +```json +{ + "failIndex": 1, + "steps": [ + { "type": "Context", "text": "some step that succeeded"}, + { "type": "Action", "text": "some step that failed"}, + { "type": "Outcome", "text": "some step that didn't run"} + ] +} +``` + +Usage example: +```js +const reportTestDone = function (name, testRunInfo, meta) { + const keywords = { Context: 'Given ', Action: 'When ', Outcome: 'Then ' }; + meta.steps + .map((step) => keywords[step.type].concat(step.text)) + .forEach((phrase, index) => { + let color; + let symbol; + if (index < meta.failIndex) { + color = 'green'; + symbol = this.symbols.ok; + } else if (index === meta.failIndex) { + color = 'red'; + symbol = this.symbols.err; + } else { + color = 'grey'; + symbol = '-'; + } + this.write(this.chalk[color](symbol, phrase)); + this.newline(); + }); +}, +``` ## Using Typescript and ESnext features diff --git a/src/cli.js b/src/cli.js index ccfb09c..29813e4 100644 --- a/src/cli.js +++ b/src/cli.js @@ -1,3 +1,5 @@ require('./rewire-compiler'); require('./rewire-argument-parser'); +require('./rewire-reporter'); + require('testcafe/lib/cli/cli'); diff --git a/src/compiler.js b/src/compiler.js index 8c82c50..7d76e93 100644 --- a/src/compiler.js +++ b/src/compiler.js @@ -182,15 +182,19 @@ module.exports = class GherkinTestcafeCompiler { return; } + const setFailIndex = (test, index) => test.meta({ failIndex: index }); const test = new Test(testFile)(`Scenario: ${scenario.name}`, async (t) => { let error; + let index = 0; try { for (const step of scenario.steps) { await this._resolveAndRunStepDefinition(t, step); + index += 1; } } catch (e) { error = e; + setFailIndex(test, index); } if (error) { @@ -200,10 +204,16 @@ module.exports = class GherkinTestcafeCompiler { .page('about:blank') .before((t) => this._runHooks(t, this._findHook(scenario, this.beforeHooks))) .after((t) => this._runHooks(t, this._findHook(scenario, this.afterHooks))) - .meta( - 'tags', - scenario.tags.length > 0 ? scenario.tags.map((tag) => tag.name).reduce((acc, cur) => `${acc},${cur}`) : '' - ); + .meta({ + tags: + scenario.tags.length > 0 + ? scenario.tags.map((tag) => tag.name).reduce((acc, cur) => `${acc},${cur}`) + : '', + steps: scenario.steps.map(({ type, text }) => ({ + type, + text, + })), + }); if (foundCredentialFiles[0]) { test.httpAuth(require(foundCredentialFiles[0])); diff --git a/src/index.js b/src/index.js index e5fb05c..763e321 100644 --- a/src/index.js +++ b/src/index.js @@ -1,4 +1,5 @@ require('./rewire-compiler'); require('./rewire-runner'); +require('./rewire-reporter'); module.exports = require('testcafe'); diff --git a/src/reporter.js b/src/reporter.js new file mode 100644 index 0000000..a900758 --- /dev/null +++ b/src/reporter.js @@ -0,0 +1,60 @@ +const { existsSync } = require('fs'); +const { join } = require('path'); +const { GeneralError } = require('testcafe/lib/errors/runtime'); +const { RUNTIME_ERRORS } = require('testcafe/lib/errors/types'); +const TestcafeReporter = require('testcafe/lib/reporter'); + +const isReporterPluginFactory = (value) => { + return typeof value === 'function'; +}; + +const requireReporterPluginFactory = (reporterName) => { + try { + const gherkinReporterPath = join(__dirname, 'reporters', reporterName); + return reporterName.includes('gtc-reporter') && existsSync(gherkinReporterPath.concat('.js')) + ? require(gherkinReporterPath) + : require('testcafe-reporter-' + reporterName); + } catch (err) { + throw new GeneralError(RUNTIME_ERRORS.cannotFindReporterForAlias, reporterName); + } +}; + +const getPluginFactory = (reporterFactorySource) => { + if (!isReporterPluginFactory(reporterFactorySource)) { + return requireReporterPluginFactory(reporterFactorySource); + } + return reporterFactorySource; +}; + +const processReporterName = (value) => { + if (isReporterPluginFactory(value)) return value.name || 'function () {}'; + + return value; +}; + +TestcafeReporter._addDefaultReporter = function (reporters) { + reporters.push({ + name: 'gtc-reporter-spec', + output: process.stdout, + }); +}; + +TestcafeReporter.getReporterPlugins = function (reporters = []) { + if (!reporters.length) { + TestcafeReporter._addDefaultReporter(reporters); + } + return Promise.all( + reporters.map(async ({ name, output, options }) => { + const pluginFactory = getPluginFactory(name); + const processedName = processReporterName(name); + const outStream = output ? await TestcafeReporter._ensureOutStream(output) : void 0; + return { + plugin: pluginFactory(options), + name: processedName, + outStream, + }; + }) + ); +}; + +module.exports = TestcafeReporter; diff --git a/src/reporters/gtc-reporter-list.js b/src/reporters/gtc-reporter-list.js new file mode 100644 index 0000000..0a2a258 --- /dev/null +++ b/src/reporters/gtc-reporter-list.js @@ -0,0 +1,138 @@ +const _renderErrors = function (errs) { + this.setIndent(3).newline(); + + errs.forEach((err, idx) => { + var prefix = this.chalk.red(`${idx + 1}) `); + + this.newline().write(this.formatError(err, prefix)).newline().newline(); + }); +}; + +const _renderWarnings = function (warnings) { + this.newline() + .setIndent(1) + .write(this.chalk.bold.yellow(`Warnings (${warnings.length}):`)) + .newline(); + + warnings.forEach((msg) => { + this.setIndent(1).write(this.chalk.bold.yellow(`--`)).newline().setIndent(2).write(msg).newline(); + }); +}; + +const reportTaskStart = function (startTime, userAgents, testCount) { + this.startTime = startTime; + this.testCount = testCount; + + this.setIndent(1).useWordWrap(true).write(this.chalk.bold('Running tests in:')).newline(); + + userAgents.forEach((ua) => { + this.write(`- ${this.chalk.blue(ua)}`).newline(); + }); + + this.newline(); +}; + +const reportFixtureStart = function (name) { + this.currentFixtureName = name; +}; + +const reportTestDone = function (name, testRunInfo, meta) { + const hasErr = !!testRunInfo.errs.length; + let symbol = null; + let nameStyle = null; + + if (testRunInfo.skipped) { + this.skipped++; + + symbol = this.chalk.cyan('-'); + nameStyle = this.chalk.cyan; + } else if (hasErr) { + symbol = this.chalk.red.bold(this.symbols.err); + nameStyle = this.chalk.red.bold; + } else { + symbol = this.chalk.green(this.symbols.ok); + nameStyle = this.chalk.grey; + } + + name = `${this.currentFixtureName} - ${name}`; + + let title = `${symbol} ${nameStyle(name)}`; + + if (testRunInfo.unstable) { + title = title.concat(this.chalk.yellow(' (unstable)')); + } + + if (testRunInfo.screenshotPath) { + title = title.concat(` (screenshots: ${this.chalk.grey.underline(testRunInfo.screenshotPath)})`); + } + + this.setIndent(1).useWordWrap(true).write(title); + + if (hasErr) { + this.setIndent(2).useWordWrap(true); + const keywords = { Context: 'Given ', Action: 'When ', Outcome: 'Then ' }; + meta.steps + .map((step) => keywords[step.type].concat(step.text)) + .forEach((phrase, index) => { + let color; + let symbol; + if (index < meta.failIndex) { + color = 'green'; + symbol = this.symbols.ok; + } else if (index === meta.failIndex) { + color = 'red'; + symbol = this.symbols.err; + } else { + color = 'grey'; + symbol = '-'; + } + this.write(this.chalk[color](symbol, phrase)); + this.newline(); + }); + this._renderErrors(testRunInfo.errs); + } + + this.afterErrList = hasErr; + + this.newline(); +}; + +const reportTaskDone = function (endTime, passed, warnings) { + var durationMs = endTime - this.startTime; + var durationStr = this.moment.duration(durationMs).format('h[h] mm[m] ss[s]'); + var footer = + passed === this.testCount + ? this.chalk.bold.green(`${this.testCount} passed`) + : this.chalk.bold.red(`${this.testCount - passed}/${this.testCount} failed`); + + footer += this.chalk.gray(` (${durationStr})`); + + this.setIndent(1).useWordWrap(true); + + if (!this.afterErrList) this.newline(); + + this.newline().write(footer).newline(); + + if (this.skipped > 0) { + this.write(this.chalk.cyan(`${this.skipped} skipped`)).newline(); + } + + if (warnings.length) this._renderWarnings(warnings); +}; + +module.exports = () => { + return { + noColors: false, + startTime: null, + afterErrList: false, + currentFixtureName: null, + testCount: 0, + skipped: 0, + reportTaskStart, + reportFixtureStart, + reportTestDone, + reportTaskDone, + _renderErrors, + _renderWarnings, + }; +}; diff --git a/src/reporters/gtc-reporter-minimal.js b/src/reporters/gtc-reporter-minimal.js new file mode 100644 index 0000000..03283a1 --- /dev/null +++ b/src/reporters/gtc-reporter-minimal.js @@ -0,0 +1,137 @@ +const NEW_LINE = '\n '; + +const _renderErrors = function () { + this.newline(); + + this.errDescriptors.forEach((errDescriptor) => { + const title = `${this.chalk.bold.red(this.symbols.err)} ${errDescriptor.fixtureName} - ${errDescriptor.testName}`; + + this.setIndent(1).useWordWrap(true).newline().write(title); + + this.setIndent(2); + errDescriptor.steps.forEach((step) => this.newline().write(step)); + + this.newline().newline().setIndent(3).write(this.formatError(errDescriptor.err)).newline().newline(); + }); +}; + +const _renderWarnings = function (warnings) { + this.newline() + .setIndent(1) + .write(this.chalk.bold.yellow(`Warnings (${warnings.length}):`)) + .newline(); + + warnings.forEach((msg) => { + this.setIndent(1).write(this.chalk.bold.yellow(`--`)).newline().setIndent(2).write(msg).newline(); + }); +}; + +const reportTaskStart = function (_, userAgents, testCount) { + this.testCount = testCount; + + this.setIndent(1).useWordWrap(true).write(this.chalk.bold('Running tests in:')).newline(); + + userAgents.forEach((ua) => { + this.write(`- ${this.chalk.blue(ua)}`).newline(); + }); +}; + +const reportFixtureStart = function (name) { + this.currentFixtureName = name; +}; + +const reportTestDone = function (name, testRunInfo, meta) { + const hasErr = !!testRunInfo.errs.length; + let symbol = null; + + if (testRunInfo.skipped) { + this.skipped++; + symbol = this.chalk.cyan('-'); + } else if (hasErr) { + symbol = this.chalk.red('!'); + } else { + symbol = '.'; + } + + if (this.spaceLeft - 1 < 0) { + this.spaceLeft = this.viewportWidth - NEW_LINE.length - 1; + this.write(NEW_LINE); + } else { + this.spaceLeft--; + } + + this.write(symbol); + + if (hasErr) { + this.setIndent(2).useWordWrap(true); + const keywords = { Context: 'Given ', Action: 'When ', Outcome: 'Then ' }; + const formattedSteps = meta.steps + .map((step) => keywords[step.type].concat(step.text)) + .map((phrase, index) => { + let symbol; + let color; + if (index < meta.failIndex) { + color = 'green'; + symbol = this.symbols.ok; + } else if (index === meta.failIndex) { + color = 'red'; + symbol = this.symbols.err; + } else { + color = 'grey'; + symbol = '-'; + } + return this.chalk[color](`${symbol} ${phrase}`); + }); + + this.errDescriptors = this.errDescriptors.concat( + testRunInfo.errs.map((err) => { + return { + steps: formattedSteps, + err: err, + testName: name, + fixtureName: this.currentFixtureName, + }; + }) + ); + } +}; + +const reportTaskDone = function (_, passed, warnings) { + const allPassed = !this.errDescriptors.length; + const footer = allPassed + ? this.chalk.bold.green(`${this.testCount} passed`) + : this.chalk.bold.red(`${this.testCount - passed}/${this.testCount} failed`); + + if (!allPassed) { + this._renderErrors(); + } else { + this.newline(); + } + + this.setIndent(1).newline().write(footer).newline(); + + if (this.skipped > 0) { + this.write(this.chalk.cyan(`${this.skipped} skipped`)).newline(); + } + + if (warnings.length) { + this._renderWarnings(warnings); + } +}; + +module.exports = () => { + return { + noColors: false, + spaceLeft: 0, + errDescriptors: [], + currentFixtureName: null, + testCount: 0, + skipped: 0, + reportTaskStart, + reportFixtureStart, + reportTestDone, + reportTaskDone, + _renderErrors, + _renderWarnings, + }; +}; diff --git a/src/reporters/gtc-reporter-spec.js b/src/reporters/gtc-reporter-spec.js new file mode 100644 index 0000000..19b1bd7 --- /dev/null +++ b/src/reporters/gtc-reporter-spec.js @@ -0,0 +1,141 @@ +const _renderWarnings = function (warnings) { + this.newline() + .setIndent(1) + .write(this.chalk.bold.yellow(`Warnings (${warnings.length}):`)) + .newline(); + + warnings.forEach((msg) => { + this.setIndent(1).write(this.chalk.bold.yellow(`--`)).newline().setIndent(2).write(msg).newline(); + }); +}; + +const _renderErrors = function (errs) { + this.setIndent(3).newline(); + + errs.forEach((err, idx) => { + var prefix = this.chalk.red(`${idx + 1}) `); + this.newline().write(this.formatError(err, prefix)).newline().newline(); + }); +}; + +const reportTaskStart = function (startTime, userAgents, testCount) { + this.startTime = startTime; + this.testCount = testCount; + + this.setIndent(1).useWordWrap(true).write(this.chalk.bold('Running tests in:')).newline(); + + userAgents.forEach((ua) => { + this.write(`- ${this.chalk.blue(ua)}`).newline(); + }); +}; + +const reportFixtureStart = function (name) { + this.setIndent(1).useWordWrap(true); + + if (this.afterErrorList) { + this.afterErrorList = false; + } else { + this.newline(); + } + + this.write(name).newline(); +}; + +const reportTestDone = function (name, testRunInfo, meta) { + var hasErr = !!testRunInfo.errs.length; + var symbol = null; + var nameStyle = null; + + if (testRunInfo.skipped) { + this.skipped++; + + symbol = this.chalk.cyan('-'); + nameStyle = this.chalk.cyan; + } else if (hasErr) { + symbol = this.chalk.red.bold(this.symbols.err); + nameStyle = this.chalk.red.bold; + } else { + symbol = this.chalk.green(this.symbols.ok); + nameStyle = this.chalk.grey; + } + + const title = `${symbol} ${nameStyle(name)}`; + + this.setIndent(1).useWordWrap(true); + + if (testRunInfo.unstable) title += this.chalk.yellow(' (unstable)'); + + if (testRunInfo.screenshotPath) title += ` (screenshots: ${this.chalk.underline.grey(testRunInfo.screenshotPath)})`; + + this.write(title); + this.newline(); + + if (hasErr) { + this.setIndent(2).useWordWrap(true); + const keywords = { Context: 'Given ', Action: 'When ', Outcome: 'Then ' }; + meta.steps + .map((step) => keywords[step.type].concat(step.text)) + .forEach((phrase, index) => { + let color; + let symbol; + if (index < meta.failIndex) { + color = 'green'; + symbol = this.symbols.ok; + } else if (index === meta.failIndex) { + color = 'red'; + symbol = this.symbols.err; + } else { + color = 'grey'; + symbol = '-'; + } + this.write(this.chalk[color](symbol, phrase)); + this.newline(); + }); + this._renderErrors(testRunInfo.errs); + } + + this.afterErrorList = hasErr; +}; + +const reportTaskDone = function (endTime, passed, warnings) { + const durationMs = endTime - this.startTime; + const durationStr = this.moment.duration(durationMs).format('h[h] mm[m] ss[s]'); + let footer = + passed === this.testCount + ? this.chalk.bold.green(`${this.testCount} passed`) + : this.chalk.bold.red(`${this.testCount - passed}/${this.testCount} failed`); + + footer = footer.concat(this.chalk.grey(` (${durationStr})`)); + + if (!this.afterErrorList) { + this.newline(); + } + + this.setIndent(1).useWordWrap(true); + + this.newline().write(footer).newline(); + + if (this.skipped > 0) { + this.write(this.chalk.cyan(`${this.skipped} skipped`)).newline(); + } + + if (warnings.length) { + this._renderWarnings(warnings); + } +}; + +module.exports = () => { + return { + noColors: false, + startTime: null, + afterErrorList: false, + testCount: 0, + skipped: 0, + _renderErrors, + _renderWarnings, + reportTaskStart, + reportFixtureStart, + reportTestDone, + reportTaskDone, + }; +}; diff --git a/src/rewire-reporter.js b/src/rewire-reporter.js new file mode 100644 index 0000000..333bff3 --- /dev/null +++ b/src/rewire-reporter.js @@ -0,0 +1,2 @@ +require('./reporter'); +require.cache[require.resolve('testcafe/lib/reporter')] = require.cache[require.resolve('./reporter')]; diff --git a/src/runner.js b/src/runner.js index f21f893..098ef9b 100644 --- a/src/runner.js +++ b/src/runner.js @@ -14,7 +14,7 @@ module.exports = class GherkinTestcafeRunner extends TestcafeRunner { if (this.apiMethodWasCalled.tags) throw new GeneralError(RUNTIME_ERRORS.multipleAPIMethodCallForbidden, 'tags'); // remove tags from previous runs (if starting multiple runs in a row in the same process) - const tagsIndex = process.argv.findIndex(val => val === '--tags'); + const tagsIndex = process.argv.findIndex((val) => val === '--tags'); if (tagsIndex !== -1) { process.argv.splice(tagsIndex, 2); }