feat(formatters): add markdown formatter (#2662)

* feat(formatters): add markdown formatter

- Format results as a markdown table
- New utility function getDocumentationUrl.ts that builds the documentation's url
from the rule or the ruleset

* feat(cli): add markdown output to the cli

* chore(deps): use custom markdown table generator

markdown-table-ts is not compatible with node12

---------

Co-authored-by: jb.muscat <jb.muscat@criteo.com>
Co-authored-by: Nauman <mnaumanali94@gmail.com>

docs/guides/2-cli.md

@@ -33,7 +33,7 @@ Other options include:
           [string] [choices: "utf8", "ascii", "utf-8", "utf16le", "ucs2", "ucs-2", "base64", "latin1"] [default: "utf8"]
   -f, --format                   formatters to use for outputting results, more than one can be provided by using
                                  multiple flags
-         [string] [choices: "json", "stylish", "junit", "html", "text", "teamcity", "pretty", "github-actions", "sarif"]
+         [string] [choices: "json", "stylish", "junit", "html", "text", "teamcity", "pretty", "github-actions", "sarif", "markdown"]
                                                                                                     [default: "stylish"]
   -o, --output                   where to output results, can be a single file name, multiple "output.<format>" or
                                  missing to print to stdout                                                     [string]

packages/cli/src/services/config.ts

@@ -13,6 +13,7 @@ export enum OutputFormat {
   PRETTY = 'pretty',
   GITHUB_ACTIONS = 'github-actions',
   SARIF = 'sarif',
+  MARKDOWN = 'markdown',
 }
 
 export interface ILintConfig {

packages/cli/src/services/output.ts

@@ -11,6 +11,7 @@ import {
   pretty,
   githubActions,
   sarif,
+  markdown,
 } from '@stoplight/spectral-formatters';
 import type { Formatter, FormatterOptions } from '@stoplight/spectral-formatters';
 import type { OutputFormat } from './config';
@@ -26,6 +27,7 @@ const formatters: Record<OutputFormat, Formatter> = {
   teamcity,
   'github-actions': githubActions,
   sarif,
+  markdown,
 };
 
 export function formatOutput(

packages/formatters/README.md

@@ -28,6 +28,7 @@ console.error(output);
 - html
 - text
 - teamcity
+- markdown (example: [markdown_example.md](markdown_example.md))
 
 ### Node.js only
 

packages/formatters/markdown_example.md

@@ -0,0 +1,5 @@
+| Code                                                                   | Path                         | Message                                      | Severity | Start | End  | Source                                              |
+| ---------------------------------------------------------------------- | ---------------------------- | -------------------------------------------- | -------- | ----- | ---- | --------------------------------------------------- |
+| [operation-description](https://rule-documentation-url.com)            | paths.\/pets.get.description | paths.\/pets.get.description is not truthy   | Error    | 1:0   | 10:1 | .\/src\/\_\_tests\_\_\/fixtures\/petstore.oas2.yaml |
+| [operation-tags](https://ruleset-documentation-url.com#operation-tags) | paths.\/pets.get.tags        | paths.\/pets.get.tags is not truthy          | Warning  | 11:0  | 20:1 | .\/src\/\_\_tests\_\_\/fixtures\/petstore.oas2.yaml |
+| rule-from-other-ruleset                                                | paths                        | i should not have any documentation url link | Warning  | 21:0  | 30:1 | .\/src\/\_\_tests\_\_\/fixtures\/petstore.oas2.yaml |

packages/formatters/package.json

@@ -38,9 +38,11 @@
     "@stoplight/spectral-core": "^1.15.1",
     "@stoplight/spectral-runtime": "^1.1.0",
     "@stoplight/types": "^13.15.0",
+    "@types/markdown-escape": "^1.1.3",
     "chalk": "4.1.2",
     "cliui": "7.0.4",
     "lodash": "^4.17.21",
+    "markdown-escape": "^2.0.0",
     "node-sarif-builder": "^2.0.3",
     "strip-ansi": "6.0",
     "text-table": "^0.2.0",

packages/formatters/src/__tests__/markdown.test.ts

@@ -0,0 +1,111 @@
+import { DiagnosticSeverity } from '@stoplight/types';
+import type { IRuleResult } from '@stoplight/spectral-core';
+import { FormatterContext } from '../types';
+import { markdown } from '../markdown';
+
+const results: IRuleResult[] = [
+  {
+    code: 'operation-description',
+    message: 'paths./pets.get.description is not truthy',
+    path: ['paths', '/pets', 'get', 'description'],
+    severity: DiagnosticSeverity.Error,
+    source: './src/__tests__/fixtures/petstore.oas2.yaml',
+    range: {
+      start: {
+        line: 1,
+        character: 0,
+      },
+      end: {
+        line: 10,
+        character: 1,
+      },
+    },
+  },
+  {
+    code: 'operation-tags',
+    message: 'paths./pets.get.tags is not truthy',
+    path: ['paths', '/pets', 'get', 'tags'],
+    severity: DiagnosticSeverity.Warning,
+    source: './src/__tests__/fixtures/petstore.oas2.yaml',
+    range: {
+      start: {
+        line: 11,
+        character: 0,
+      },
+      end: {
+        line: 20,
+        character: 1,
+      },
+    },
+  },
+  {
+    code: 'rule-from-other-ruleset',
+    message: 'i should not have any documentation url link',
+    path: ['paths'],
+    severity: DiagnosticSeverity.Warning,
+    source: './src/__tests__/fixtures/petstore.oas2.yaml',
+    range: {
+      start: {
+        line: 21,
+        character: 0,
+      },
+      end: {
+        line: 30,
+        character: 1,
+      },
+    },
+  },
+];
+
+const context = {
+  ruleset: {
+    rules: {
+      'operation-description': {
+        documentationUrl: 'https://rule-documentation-url.com',
+        owner: {
+          definition: {
+            documentationUrl: 'https://ruleset-documentation-url.com',
+          },
+        },
+      },
+      'operation-tags': {
+        documentationUrl: '', //nothing
+        owner: {
+          definition: {
+            documentationUrl: 'https://ruleset-documentation-url.com',
+          },
+        },
+      },
+      'rule-from-other-ruleset': {
+        documentationUrl: '', //nothing
+        owner: {
+          definition: {
+            documentationUrl: '', //nothing
+          },
+        },
+      },
+    },
+  },
+} as unknown as FormatterContext;
+
+const expectedMd = String.raw`
+| Code                                                                   | Path                         | Message                                      | Severity | Start | End  | Source                                              |
+| ---------------------------------------------------------------------- | ---------------------------- | -------------------------------------------- | -------- | ----- | ---- | --------------------------------------------------- |
+| [operation-description](https://rule-documentation-url.com)            | paths.\/pets.get.description | paths.\/pets.get.description is not truthy   | Error    | 1:0   | 10:1 | .\/src\/\_\_tests\_\_\/fixtures\/petstore.oas2.yaml |
+| [operation-tags](https://ruleset-documentation-url.com#operation-tags) | paths.\/pets.get.tags        | paths.\/pets.get.tags is not truthy          | Warning  | 11:0  | 20:1 | .\/src\/\_\_tests\_\_\/fixtures\/petstore.oas2.yaml |
+| rule-from-other-ruleset                                                | paths                        | i should not have any documentation url link | Warning  | 21:0  | 30:1 | .\/src\/\_\_tests\_\_\/fixtures\/petstore.oas2.yaml |
+`;
+
+describe('Markdown formatter', () => {
+  test('should format as markdown table', () => {
+    const CRLF = '\r\n';
+    const md = markdown(results, { failSeverity: DiagnosticSeverity.Warning }, context);
+
+    // We normalize the line-breaks and trailing whitespaces because the expected markdown file is can be created on a Windows machine
+    // and prettier instert a line break automatically
+    const normalizedMd = md.replace(new RegExp(CRLF, 'g'), '\n').trim();
+    const normalizedExpectedMd = expectedMd.replace(new RegExp(CRLF, 'g'), '\n').trim();
+
+    expect(normalizedMd).toEqual(normalizedExpectedMd);
+  });
+});

packages/formatters/src/index.node.ts

@@ -1,4 +1,4 @@
-export { html, json, junit, text, stylish, teamcity } from './index';
+export { html, json, junit, text, stylish, teamcity, markdown } from './index';
 export type { Formatter, FormatterOptions } from './index';
 export { pretty } from './pretty';
 export { githubActions } from './github-actions';

packages/formatters/src/index.ts

@@ -4,6 +4,7 @@ export * from './junit';
 export * from './html';
 export * from './text';
 export * from './teamcity';
+export * from './markdown';
 import type { Formatter } from './types';
 export type { Formatter, FormatterOptions } from './types';
 

packages/formatters/src/markdown.ts

@@ -0,0 +1,71 @@
+import { printPath, PrintStyle } from '@stoplight/spectral-runtime';
+import { Formatter, FormatterContext } from './types';
+import { groupBySource } from './utils';
+import { DiagnosticSeverity } from '@stoplight/types';
+import markdownEscape from 'markdown-escape';
+import { getRuleDocumentationUrl } from './utils/getDocumentationUrl';
+
+export const markdown: Formatter = (results, { failSeverity }, ctx?: FormatterContext) => {
+  const groupedResults = groupBySource(results);
+
+  const lines: string[][] = [];
+  for (const [source, validationResults] of Object.entries(groupedResults)) {
+    validationResults.sort((a, b) => a.range.start.line - b.range.start.line);
+
+    if (validationResults.length > 0) {
+      const filteredValidationResults = validationResults.filter(result => result.severity <= failSeverity);
+
+      for (const result of filteredValidationResults) {
+        const ruleDocumentationUrl = getRuleDocumentationUrl(result.code, ctx);
+        const codeWithOptionalLink =
+          ruleDocumentationUrl != null
+            ? `[${result.code.toString()}](${ruleDocumentationUrl})`
+            : result.code.toString();
+        const escapedPath = markdownEscape(printPath(result.path, PrintStyle.Dot));
+        const escapedMessage = markdownEscape(result.message);
+        const severityString = DiagnosticSeverity[result.severity];
+        const start = `${result.range.start.line}:${result.range.start.character}`;
+        const end = `${result.range.end.line}:${result.range.end.character}`;
+        const escapedSource = markdownEscape(source);
+        lines.push([codeWithOptionalLink, escapedPath, escapedMessage, severityString, start, end, escapedSource]);
+      }
+    }
+  }
+
+  const headers = ['Code', 'Path', 'Message', 'Severity', 'Start', 'End', 'Source'];
+  return createMdTable(headers, lines);
+};
+
+function createMdTable(headers: string[], lines: string[][]): string {
+  //find lenght of each column
+  const columnLengths = headers.map((_, i) => Math.max(...lines.map(line => line[i].length), headers[i].length));
+
+  let string = '';
+  //create markdown table header
+  string += '|';
+  for (const header of headers) {
+    string += ` ${header}`;
+    string += ' '.repeat(columnLengths[headers.indexOf(header)] - header.length);
+    string += ' |';
+  }
+
+  //create markdown table rows delimiter
+  string += '\n|';
+  for (const _ of headers) {
+    string += ' ';
+    string += '-'.repeat(columnLengths[headers.indexOf(_)]);
+    string += ' |';
+  }
+
+  //create markdown table rows
+  for (const line of lines) {
+    string += '\n|';
+    for (const cell of line) {
+      string += ` ${cell}`;
+      string += ' '.repeat(columnLengths[line.indexOf(cell)] - cell.length);
+      string += ' |';
+    }
+  }
+
+  return string;
+}

packages/formatters/src/utils/getDocumentationUrl.ts

@@ -0,0 +1,22 @@
+import { FormatterContext } from '../types';
+
+/// Returns the documentation URL, either directly from the rule or by combining the ruleset documentation URL with the rule code.
+export function getRuleDocumentationUrl(ruleCode: string | number, ctx?: FormatterContext): string | undefined {
+  if (!ctx?.ruleset) {
+    return undefined;
+  }
+
+  const rule = ctx.ruleset.rules[ruleCode.toString()];
+  //if rule.documentationUrl is not null and not empty and not undefined, return it
+  if (rule.documentationUrl != null && rule.documentationUrl) {
+    return rule.documentationUrl;
+  }
+
+  //otherwise use the ruleset documentationUrl and append the rulecode as an anchor
+  const rulesetDocumentationUrl = rule.owner?.definition.documentationUrl;
+  if (rulesetDocumentationUrl != null && rulesetDocumentationUrl) {
+    return `${rulesetDocumentationUrl}#${ruleCode}`;
+  }
+
+  return undefined;
+}

test-harness/scenarios/help-no-document.scenario

@@ -20,7 +20,7 @@ Options:
       --version                  Show version number  [boolean]
       --help                     Show help  [boolean]
   -e, --encoding                 text encoding to use  [string] [choices: "utf8", "ascii", "utf-8", "utf16le", "ucs2", "ucs-2", "base64", "latin1"] [default: "utf8"]
-  -f, --format                   formatters to use for outputting results, more than one can be provided by using multiple flags  [string] [choices: "json", "stylish", "junit", "html", "text", "teamcity", "pretty", "github-actions", "sarif"] [default: "stylish"]
+  -f, --format                   formatters to use for outputting results, more than one can be provided by using multiple flags  [string] [choices: "json", "stylish", "junit", "html", "text", "teamcity", "pretty", "github-actions", "sarif", "markdown"] [default: "stylish"]
   -o, --output                   where to output results, can be a single file name, multiple "output.<format>" or missing to print to stdout  [string]
       --stdin-filepath           path to a file to pretend that stdin comes from  [string]
       --resolver                 path to custom json-ref-resolver instance  [string]

test-harness/scenarios/strict-options.scenario

@@ -20,7 +20,7 @@ Options:
       --version                  Show version number  [boolean]
       --help                     Show help  [boolean]
   -e, --encoding                 text encoding to use  [string] [choices: "utf8", "ascii", "utf-8", "utf16le", "ucs2", "ucs-2", "base64", "latin1"] [default: "utf8"]
-  -f, --format                   formatters to use for outputting results, more than one can be provided by using multiple flags  [string] [choices: "json", "stylish", "junit", "html", "text", "teamcity", "pretty", "github-actions", "sarif"] [default: "stylish"]
+  -f, --format                   formatters to use for outputting results, more than one can be provided by using multiple flags  [string] [choices: "json", "stylish", "junit", "html", "text", "teamcity", "pretty", "github-actions", "sarif", "markdown"] [default: "stylish"]
   -o, --output                   where to output results, can be a single file name, multiple "output.<format>" or missing to print to stdout  [string]
       --stdin-filepath           path to a file to pretend that stdin comes from  [string]
       --resolver                 path to custom json-ref-resolver instance  [string]

yarn.lock

@@ -2746,12 +2746,14 @@ __metadata:
     "@stoplight/spectral-core": ^1.15.1
     "@stoplight/spectral-runtime": ^1.1.0
     "@stoplight/types": ^13.15.0
+    "@types/markdown-escape": ^1.1.3
     ast-types: ^0.14.2
     astring: ^1.8.4
     chalk: 4.1.2
     cliui: 7.0.4
     eol: 0.9.1
     lodash: ^4.17.21
+    markdown-escape: ^2.0.0
     node-html-parser: ^4.1.5
     node-sarif-builder: ^2.0.3
     strip-ansi: 6.0
@@ -3323,6 +3325,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/markdown-escape@npm:^1.1.3":
+  version: 1.1.3
+  resolution: "@types/markdown-escape@npm:1.1.3"
+  checksum: cb2e410993271f0ccc526190391a08344f4f602be69e06fee989d36d5886866ba9ba2184054895d0ad2a12d57b02f3ccf86d7a1fe8904be48bcc1ee61b98e32f
+  languageName: node
+  linkType: hard
+
 "@types/minimatch@npm:*, @types/minimatch@npm:^3.0.5":
   version: 3.0.5
   resolution: "@types/minimatch@npm:3.0.5"
@@ -9508,6 +9517,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"markdown-escape@npm:^2.0.0":
+  version: 2.0.0
+  resolution: "markdown-escape@npm:2.0.0"
+  checksum: 74c66d817636ac5f6a275fdc79ecb1e208d907ca85289d660b515256fbc3e380eb18d29b6bbbd6a77968ee4fb5872d40ecf31e52bc9f17855bb01bb723569fa0
+  languageName: node
+  linkType: hard
+
 "marked-terminal@npm:^5.0.0":
   version: 5.2.0
   resolution: "marked-terminal@npm:5.2.0"