Fetch spec info from W3C API, Specref or the spec

This update adds the logic needed to complete specification metadata with the
spec's title (w3c#10) and the TR/ED URLs (w3c#11).

This additional information gets fetched from the W3C API when possible, from
Specref when the W3C API does not know anything about the spec, or from the
spec itself when neither the W3C API nor Specref knows anything about the spec.

Fetching logic is in `src/fetch-info.js`. By definition, fetching this
information requires sending network requests and, as such, cannot be done in
realtime on hundreds of specs. Instead, the commit adds a GitHub action set to
run every 6 hours to fetch the information and update the `specs-info.json`
file.

In turn, this file is imported by the `index.js` script, which now also returns
the information.

New properties returned for each specification are:
- `title`: the title of the spec
- `trUrl`: the URL of the TR document for specs published in TR space
- `edUrl`: the URL of the latest Editor's Draft when known
- `source`: one of `w3c`, `specref`, or `spec` to give the source of the info

Other updates:
- Refactored JSON schemas in a schema folder with a common `dfns.json` file to
avoid duplicating type definitions.
- Extended the filtering logic in `index.js` to also look for titles, source,
and URLs.

Note that the code and the tests now expect to find a `config.json` file in the
root folder that contains an object with a `w3cApiKey` property set to a valid
W3C API key (this file is *not* part of the commit).

Also note that tests current fail if one tries to add a new spec to `specs.json`
and does not run `npm run fetch-info`, because the new spec won't have a `title`
property, which is defined as `required` in the JSON schema. This means that
updates to `specs-info.json` need to be part of pull requests on the repo.
Depending on the envisioned workflow for updates (and packaging), we might want
to relax that rule later on.

.github/workflows/fetch-info.yml

@@ -0,0 +1,30 @@
+on:
+  schedule:
+    - cron: '10 */6 * * *'
+  push:
+    branches:
+    - master
+name: Fetch additional spec info from external sources
+jobs:
+  fetch:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v1
+    - uses: actions/setup-node@v1
+      with:
+        node-version: 12.x
+    - name: Setup environment
+      run: |
+        echo "${{ secrets.CONFIG_JSON }}" | base64 --decode > config.json
+        npm ci
+    - name: Fetch info
+      run: npm run fetch-info
+    - name: Commit updates
+      run: |
+        git config user.name "fetch-info bot"
+        git config user.email "<>"
+        git commit -m "[data] Update spec info" -a
+    - name: Push changes
+      uses: ad-m/github-push-action@v0.5.0
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/lint.yml

@@ -8,6 +8,7 @@ jobs:
     - uses: actions/setup-node@v1
       with:
         node-version: 12.x
+    - run: echo "${{ secrets.CONFIG_JSON }}" | base64 --decode > config.json
     - run: npm ci
     - run: npm run test
     - run: npm run lint

.gitignore

@@ -1 +1,2 @@
 node_modules/
+config.json

index.js

@@ -4,6 +4,16 @@ const computeShortname = require("./src/compute-shortname.js");
 const computePrevNext = require("./src/compute-prevnext.js");
 const computeCurrentLevel = require("./src/compute-currentlevel.js");
 
+// Retrieve generated spec info (if file was properly generated)
+const specInfo = (function () {
+  try {
+    return require("./specs-info.json");
+  }
+  catch (err) {
+    return {};
+  }
+})();
+
 const specs = require("./specs.json")
   // Turn all specs into objects
   // (and handle syntactic sugar notation for delta/current flags)
@@ -37,7 +47,10 @@ const specs = require("./specs.json")
   .map(spec => { delete spec.forceCurrent; return spec; })
 
   // Complete information with previous/next level links
-  .map((spec, _, list) => Object.assign(spec, computePrevNext(spec, list)));
+  .map((spec, _, list) => Object.assign(spec, computePrevNext(spec, list)))
+
+  // Complete information with title and link to TR/ED URLs, when known
+  .map(spec => Object.assign(spec, specInfo[spec.name]));
 
 
 if (require.main === module) {
@@ -55,7 +68,11 @@ if (require.main === module) {
         s.url === id ||
         s.name === id ||
         s.shortname === id ||
-        s.levelComposition === id);
+        s.levelComposition === id ||
+        s.title === id ||
+        s.trUrl === id ||
+        s.edUrl === id ||
+        s.source === id);
     console.log(JSON.stringify(res.length === 1 ? res[0] : res, null, 2));
   }
   else {

lint.js

@@ -1,17 +1,28 @@
 "use strict";
 
 const fs = require("fs").promises;
-const schema = require("./specs-schema.json");
 const computeShortname = require("./src/compute-shortname.js");
 const computePrevNext = require("./src/compute-prevnext.js");
+
+const schema = require("./schema/specs.json");
+const dfnsSchema = require("./schema/dfns.json");
 const Ajv = require("ajv");
 const ajv = new Ajv();
+const validate = ajv.addSchema(dfnsSchema).compile(schema);
 
 // When an entry is invalid, the schema validator returns one error for each
 // "oneOf" option and one error on overall "oneOf" problem. This is confusing
 // for humans. The following function improves the error being returned.
 const clarifyErrors = errors => {
-  if (!errors || errors.length < 2) {
+  if (!errors) {
+    return errors;
+  }
+
+  // Update dataPath to drop misleading "[object Object]"
+  errors.forEach(err =>
+    err.dataPath = err.dataPath.replace(/^\[object Object\]/, ''));
+
+  if (errors.length < 2) {
     return errors;
   }
 
@@ -96,12 +107,12 @@ function lintStr(specsStr) {
 
   const isSchemaValid = ajv.validateSchema(schema);
   if (!isSchemaValid) {
-    throw "The specs-schema.json file must be a valid JSON Schema file";
+    throw "The schemas/specs.json file must be a valid JSON Schema file";
   }
 
-  var isValid = ajv.validate(schema, specs, { format: "full" });
+  const isValid = validate(specs, { format: "full" });
   if (!isValid) {
-    throw ajv.errorsText(clarifyErrors(ajv.errors), {
+    throw ajv.errorsText(clarifyErrors(validate.errors), {
       dataVar: "specs", separator: "\n"
     });
   }

package.json

@@ -6,6 +6,7 @@
   },
   "main": "index.js",
   "scripts": {
+    "fetch-info": "node src/fetch-info.js > specs-info.json",
     "lint": "node lint.js",
     "lint-fix": "node lint.js --fix",
     "test": "mocha"

schema/dfns.json

@@ -0,0 +1,44 @@
+{
+  "$schema": "http://json-schema.org/schema#",
+  "$id": "https://github.com/w3c/browser-specs/tree/master/schema/dfns.json",
+
+  "dfns": {
+    "url": {
+      "type": "string",
+      "format": "uri"
+    },
+
+    "name": {
+      "type": "string",
+      "pattern": "^[\\w\\-]+((?<=\\-\\d+)\\.\\d+)?$"
+    },
+
+    "shortname": {
+      "type": "string",
+      "pattern": "^[\\w\\-]+$"
+    },
+
+    "level": {
+      "type": "number",
+      "minimum": 1
+    },
+
+    "levelComposition": {
+      "type": "string",
+      "enum": ["full", "delta"]
+    },
+
+    "forceCurrent": {
+      "type": "boolean"
+    },
+
+    "title": {
+      "type": "string"
+    },
+
+    "source": {
+      "type": "string",
+      "enum": ["w3c", "specref", "spec"]
+    }
+  }
+}

schema/index.json

@@ -0,0 +1,26 @@
+{
+  "$schema": "http://json-schema.org/schema#",
+  "$id": "https://github.com/w3c/browser-specs/tree/master/schema/index.json",
+
+  "type": "array",
+  "items": {
+    "type": "object",
+    "properties": {
+      "url": { "$ref": "dfns.json#/dfns/url" },
+      "name": { "$ref": "dfns.json#/dfns/name" },
+      "shortname": { "$ref": "dfns.json#/dfns/shortname" },
+      "level": { "$ref": "dfns.json#/dfns/level" },
+      "levelComposition": { "$ref": "dfns.json#/dfns/levelComposition" },
+      "currentLevel": { "$ref": "dfns.json#/dfns/name" },
+      "previousLevel": { "$ref": "dfns.json#/dfns/name" },
+      "nextLevel": { "$ref": "dfns.json#/dfns/name" },
+      "edUrl": { "$ref": "dfns.json#/dfns/url" },
+      "trUrl": { "$ref": "dfns.json#/dfns/url" },
+      "title": { "$ref": "dfns.json#/dfns/title" },
+      "source": { "$ref": "dfns.json#/dfns/source" }
+    },
+    "required": ["url", "name", "shortname", "currentLevel", "title", "source"],
+    "additionalProperties": false
+  },
+  "minItems": 1
+}

schema/specs-info.json

@@ -0,0 +1,21 @@
+{
+  "$schema": "http://json-schema.org/schema#",
+  "$id": "https://github.com/w3c/browser-specs/tree/master/schema/specs-info.json",
+
+  "type": "object",
+  "propertyNames": {
+    "pattern": "^[\\w\\-]+((?<=\\-\\d+)\\.\\d+)?$"
+  },
+  "additionalProperties": {
+    "type": "object",
+    "properties": {
+      "edUrl": { "$ref": "dfns.json#/dfns/url" },
+      "trUrl": { "$ref": "dfns.json#/dfns/url" },
+      "title": { "$ref": "dfns.json#/dfns/title" },
+      "source": { "$ref": "dfns.json#/dfns/source" }
+    },
+    "required": ["title", "source"],
+    "additionalProperties": false
+  },
+  "minItems": 1
+}

schema/specs.json

@@ -0,0 +1,28 @@
+{
+  "$schema": "http://json-schema.org/schema#",
+  "$id": "https://github.com/w3c/browser-specs/tree/master/schema/specs.json",
+
+  "type": "array",
+  "items": {
+    "oneOf": [
+      {
+        "type": "string",
+        "pattern": "^https://[^\\s]+(\\s(delta|current))?$"
+      },
+      {
+        "type": "object",
+        "properties": {
+          "url": { "$ref": "dfns.json#/dfns/url" },
+          "name": { "$ref": "dfns.json#/dfns/name" },
+          "shortname": { "$ref": "dfns.json#/dfns/shortname" },
+          "level": { "$ref": "dfns.json#/dfns/level" },
+          "levelComposition": { "$ref": "dfns.json#/dfns/levelComposition" },
+          "forceCurrent": { "type": "boolean" }
+        },
+        "required": ["url"],
+        "additionalProperties": false
+      }
+    ]
+  },
+  "minItems": 1
+}

specs-info.json

@@ -0,0 +1,7 @@
+{
+  "compat": {
+    "edUrl": "https://compat.spec.whatwg.org/",
+    "title": "Compatibility Standard",
+    "source": "specref"
+  }
+}