feat(reporting): output steps as metadata

steps and failure index are added to the metadata of the step, and the reporter from testcafe has
been rewired to use a custom version of spec by default

re #48

.gitignore

@@ -5,4 +5,5 @@ node_modules/
 package-lock.json
 .history
 .yarn/cache
-.DS_Store
+.yarn/install-state.gz
+.DS_Store

README.md

@@ -20,6 +20,7 @@
         1. [Before and After](#before-and-after)
         2. [BeforeAll and AfterAll](#beforeall-and-afterall)
     4. [Data tables](#data-tables)
+    5. [Step reporting](#step-reporting)
 8. [Using typescript and ESnext features](#using-typescript-and-esnext-features)
 9. [Contributing](#contributing)
     1. [Commits](#commits)
@@ -375,7 +376,92 @@ When steps have a data table, they are passed an object with methods that can be
   - `raw`: returns the table as a 2-D array
   - `rowsHash`: returns an object where each row corresponds to an entry (first column is the key, second column is the value)
 
-see examples section for an example
+See the examples section for an example
+
+### Step reporting
+
+By default, the reporter used by TestCafé is `spec`. 
+TestCafé has no reason to handle the concept of "step" because it's a notion that is specific to gherkin.
+
+To work around that:
+- The metadata of a `test` now contains the full list of steps that compose the `scenario` it's based on.
+In case of failure of a step, its index is also added to the metadata.
+- A custom reporter (`gtc-reporter-spec`) has been added to the project.
+It is automatically used instead of spec as the default reporter for `gherkin-testcafe`. 
+Note that `spec` remains usable by simply using the `reporter` option provided by TestCafé:
+    ```bash
+    gherkin-testcafe chrome ./tests/* --reporter spec
+    ```
+    If you use the API,
+    ```js
+    runner.reporter("spec")
+    ```
+- Custom internal reporters have also been created based on `list` and `minimal`. 
+The gtc reporters behave in exactly the same way as their TestCafé counterparts, except that the steps are part of the 
+output, with highlighing indicating which ones succeeded, which ones failed, and which ones didn't run.
+
+    ><span style="color:green"> ✓ Given some step that succeeded</span>
+    >
+    ><span style="color:red"> ✖ When some step that failed</span>
+    >
+    ><span style="color:grey"> - Then some step that didn't run</span>
+
+    To use one of this package's internal reporters, use its name in the reporter option:
+    ```bash
+    gherkin-testcafe chrome ./tests/* --reporter gtc-reporter-list
+    gherkin-testcafe chrome ./tests/* --reporter gtc-reporter-minimal
+    gherkin-testcafe chrome ./tests/* --reporter gtc-reporter-spec # unnecessary as it is the default behavior
+    ```
+
+Note that other official reporters could be adapted in the future.
+#### Implement / Adapt a custom reporter
+
+If you are using a [custom reporter](https://testcafe.io/documentation/402810/guides/extend-testcafe/reporter-plugin) 
+and want to use or display the step information, all you need to do is access the metadata from your reporter's methods.
+
+Fortunately, accessing metadata is [built-in behavior](https://testcafe.io/documentation/402810/guides/extend-testcafe/reporter-plugin#implement-the-reporter) for normal TestCafé reporters:
+
+TestCafé will pass the metadata object to your test reporting function as the third argument.
+
+The properties that are dedicated to this feature are `steps` and `failIndex`. 
+Each step in the `steps` array has two properties: `type` and `text`.
+
+Representation:
+```json
+{
+    "failIndex": 1,
+    "steps": [
+        { "type": "Context", "text": "some step that succeeded"},
+        { "type": "Action", "text": "some step that failed"},
+        { "type": "Outcome", "text": "some step that didn't run"}
+    ] 
+}
+```
+
+Usage example:
+```js
+const reportTestDone = function (name, testRunInfo, meta) { 
+    const keywords = { Context: 'Given ', Action: 'When ', Outcome: 'Then ' };
+    meta.steps
+      .map((step) => keywords[step.type].concat(step.text))
+      .forEach((phrase, index) => {
+        let color;
+        let symbol;
+        if (index < meta.failIndex) {
+          color = 'green';
+          symbol = this.symbols.ok;
+        } else if (index === meta.failIndex) {
+          color = 'red';
+          symbol = this.symbols.err;
+        } else {
+          color = 'grey';
+          symbol = '-';
+        }
+        this.write(this.chalk[color](symbol, phrase));
+        this.newline();
+      });
+},
+```
 
 ## Using Typescript and ESnext features
 

src/cli.js

@@ -1,3 +1,5 @@
 require('./rewire-compiler');
 require('./rewire-argument-parser');
+require('./rewire-reporter');
+
 require('testcafe/lib/cli/cli');

src/compiler.js

@@ -182,15 +182,19 @@ module.exports = class GherkinTestcafeCompiler {
             return;
           }
 
+          const setFailIndex = (test, index) => test.meta({ failIndex: index });
           const test = new Test(testFile)(`Scenario: ${scenario.name}`, async (t) => {
             let error;
+            let index = 0;
 
             try {
               for (const step of scenario.steps) {
                 await this._resolveAndRunStepDefinition(t, step);
+                index += 1;
               }
             } catch (e) {
               error = e;
+              setFailIndex(test, index);
             }
 
             if (error) {
@@ -200,10 +204,16 @@ module.exports = class GherkinTestcafeCompiler {
             .page('about:blank')
             .before((t) => this._runHooks(t, this._findHook(scenario, this.beforeHooks)))
             .after((t) => this._runHooks(t, this._findHook(scenario, this.afterHooks)))
-            .meta(
-              'tags',
-              scenario.tags.length > 0 ? scenario.tags.map((tag) => tag.name).reduce((acc, cur) => `${acc},${cur}`) : ''
-            );
+            .meta({
+              tags:
+                scenario.tags.length > 0
+                  ? scenario.tags.map((tag) => tag.name).reduce((acc, cur) => `${acc},${cur}`)
+                  : '',
+              steps: scenario.steps.map(({ type, text }) => ({
+                type,
+                text,
+              })),
+            });
 
           if (foundCredentialFiles[0]) {
             test.httpAuth(require(foundCredentialFiles[0]));

src/index.js

@@ -1,4 +1,5 @@
 require('./rewire-compiler');
 require('./rewire-runner');
+require('./rewire-reporter');
 
 module.exports = require('testcafe');

src/reporter.js

@@ -0,0 +1,60 @@
+const { existsSync } = require('fs');
+const { join } = require('path');
+const { GeneralError } = require('testcafe/lib/errors/runtime');
+const { RUNTIME_ERRORS } = require('testcafe/lib/errors/types');
+const TestcafeReporter = require('testcafe/lib/reporter');
+
+const isReporterPluginFactory = (value) => {
+  return typeof value === 'function';
+};
+
+const requireReporterPluginFactory = (reporterName) => {
+  try {
+    const gherkinReporterPath = join(__dirname, 'reporters', reporterName);
+    return reporterName.includes('gtc-reporter') && existsSync(gherkinReporterPath.concat('.js'))
+      ? require(gherkinReporterPath)
+      : require('testcafe-reporter-' + reporterName);
+  } catch (err) {
+    throw new GeneralError(RUNTIME_ERRORS.cannotFindReporterForAlias, reporterName);
+  }
+};
+
+const getPluginFactory = (reporterFactorySource) => {
+  if (!isReporterPluginFactory(reporterFactorySource)) {
+    return requireReporterPluginFactory(reporterFactorySource);
+  }
+  return reporterFactorySource;
+};
+
+const processReporterName = (value) => {
+  if (isReporterPluginFactory(value)) return value.name || 'function () {}';
+
+  return value;
+};
+
+TestcafeReporter._addDefaultReporter = function (reporters) {
+  reporters.push({
+    name: 'gtc-reporter-spec',
+    output: process.stdout,
+  });
+};
+
+TestcafeReporter.getReporterPlugins = function (reporters = []) {
+  if (!reporters.length) {
+    TestcafeReporter._addDefaultReporter(reporters);
+  }
+  return Promise.all(
+    reporters.map(async ({ name, output, options }) => {
+      const pluginFactory = getPluginFactory(name);
+      const processedName = processReporterName(name);
+      const outStream = output ? await TestcafeReporter._ensureOutStream(output) : void 0;
+      return {
+        plugin: pluginFactory(options),
+        name: processedName,
+        outStream,
+      };
+    })
+  );
+};
+
+module.exports = TestcafeReporter;

src/reporters/gtc-reporter-list.js

@@ -0,0 +1,138 @@
+const _renderErrors = function (errs) {
+  this.setIndent(3).newline();
+
+  errs.forEach((err, idx) => {
+    var prefix = this.chalk.red(`${idx + 1}) `);
+
+    this.newline().write(this.formatError(err, prefix)).newline().newline();
+  });
+};
+
+const _renderWarnings = function (warnings) {
+  this.newline()
+    .setIndent(1)
+    .write(this.chalk.bold.yellow(`Warnings (${warnings.length}):`))
+    .newline();
+
+  warnings.forEach((msg) => {
+    this.setIndent(1).write(this.chalk.bold.yellow(`--`)).newline().setIndent(2).write(msg).newline();
+  });
+};
+
+const reportTaskStart = function (startTime, userAgents, testCount) {
+  this.startTime = startTime;
+  this.testCount = testCount;
+
+  this.setIndent(1).useWordWrap(true).write(this.chalk.bold('Running tests in:')).newline();
+
+  userAgents.forEach((ua) => {
+    this.write(`- ${this.chalk.blue(ua)}`).newline();
+  });
+
+  this.newline();
+};
+
+const reportFixtureStart = function (name) {
+  this.currentFixtureName = name;
+};
+
+const reportTestDone = function (name, testRunInfo, meta) {
+  const hasErr = !!testRunInfo.errs.length;
+  let symbol = null;
+  let nameStyle = null;
+
+  if (testRunInfo.skipped) {
+    this.skipped++;
+
+    symbol = this.chalk.cyan('-');
+    nameStyle = this.chalk.cyan;
+  } else if (hasErr) {
+    symbol = this.chalk.red.bold(this.symbols.err);
+    nameStyle = this.chalk.red.bold;
+  } else {
+    symbol = this.chalk.green(this.symbols.ok);
+    nameStyle = this.chalk.grey;
+  }
+
+  name = `${this.currentFixtureName} - ${name}`;
+
+  let title = `${symbol} ${nameStyle(name)}`;
+
+  if (testRunInfo.unstable) {
+    title = title.concat(this.chalk.yellow(' (unstable)'));
+  }
+
+  if (testRunInfo.screenshotPath) {
+    title = title.concat(` (screenshots: ${this.chalk.grey.underline(testRunInfo.screenshotPath)})`);
+  }
+
+  this.setIndent(1).useWordWrap(true).write(title);
+
+  if (hasErr) {
+    this.setIndent(2).useWordWrap(true);
+    const keywords = { Context: 'Given ', Action: 'When ', Outcome: 'Then ' };
+    meta.steps
+      .map((step) => keywords[step.type].concat(step.text))
+      .forEach((phrase, index) => {
+        let color;
+        let symbol;
+        if (index < meta.failIndex) {
+          color = 'green';
+          symbol = this.symbols.ok;
+        } else if (index === meta.failIndex) {
+          color = 'red';
+          symbol = this.symbols.err;
+        } else {
+          color = 'grey';
+          symbol = '-';
+        }
+        this.write(this.chalk[color](symbol, phrase));
+        this.newline();
+      });
+    this._renderErrors(testRunInfo.errs);
+  }
+
+  this.afterErrList = hasErr;
+
+  this.newline();
+};
+
+const reportTaskDone = function (endTime, passed, warnings) {
+  var durationMs = endTime - this.startTime;
+  var durationStr = this.moment.duration(durationMs).format('h[h] mm[m] ss[s]');
+  var footer =
+    passed === this.testCount
+      ? this.chalk.bold.green(`${this.testCount} passed`)
+      : this.chalk.bold.red(`${this.testCount - passed}/${this.testCount} failed`);
+
+  footer += this.chalk.gray(` (${durationStr})`);
+
+  this.setIndent(1).useWordWrap(true);
+
+  if (!this.afterErrList) this.newline();
+
+  this.newline().write(footer).newline();
+
+  if (this.skipped > 0) {
+    this.write(this.chalk.cyan(`${this.skipped} skipped`)).newline();
+  }
+
+  if (warnings.length) this._renderWarnings(warnings);
+};
+
+module.exports = () => {
+  return {
+    noColors: false,
+    startTime: null,
+    afterErrList: false,
+    currentFixtureName: null,
+    testCount: 0,
+    skipped: 0,
+    reportTaskStart,
+    reportFixtureStart,
+    reportTestDone,
+    reportTaskDone,
+    _renderErrors,
+    _renderWarnings,
+  };
+};