Add ECMA 262 and TC39 Stage 3 proposals to the list of specs (#196)

* Monitor ECMA Stage 3 proposals for addition
* Add support for indexing ecma262 and TC39 Stage 3 proposals
* Add ECMA 262 and TC39 Stage 3 proposals to the list of specs

close #139

index.json

@@ -754,6 +754,159 @@
     "source": "spec",
     "shortTitle": "SVG Animations 2"
   },
+  {
+    "url": "https://tc39.es/ecma262/",
+    "seriesComposition": "full",
+    "shortname": "ecmascript",
+    "series": {
+      "shortname": "ecmascript",
+      "currentSpecification": "ecmascript"
+    },
+    "shortTitle": "ECMAScript",
+    "nightly": {
+      "url": "https://tc39.es/ecma262/",
+      "repository": "https://github.com/tc39/ecma262",
+      "filename": "index.html"
+    },
+    "title": "ECMAScript Language Specification",
+    "source": "specref"
+  },
+  {
+    "url": "https://tc39.es/proposal-atomics-wait-async/",
+    "seriesComposition": "full",
+    "shortname": "tc39-atomics-wait-async",
+    "series": {
+      "shortname": "tc39-atomics-wait-async",
+      "currentSpecification": "tc39-atomics-wait-async"
+    },
+    "nightly": {
+      "url": "https://tc39.es/proposal-atomics-wait-async/",
+      "repository": "https://github.com/tc39/proposal-atomics-wait-async",
+      "filename": "index.html"
+    },
+    "title": "Atomics.waitAsync",
+    "source": "spec",
+    "shortTitle": "Atomics.waitAsync"
+  },
+  {
+    "url": "https://tc39.es/proposal-class-fields/",
+    "seriesComposition": "full",
+    "shortname": "tc39-class-fields",
+    "series": {
+      "shortname": "tc39-class-fields",
+      "currentSpecification": "tc39-class-fields"
+    },
+    "nightly": {
+      "url": "https://tc39.es/proposal-class-fields/",
+      "repository": "https://github.com/tc39/proposal-class-fields",
+      "filename": "index.html"
+    },
+    "title": "Public and private instance fields proposal",
+    "source": "spec",
+    "shortTitle": "Public and private instance fields proposal"
+  },
+  {
+    "url": "https://tc39.es/proposal-import-assertions/",
+    "seriesComposition": "full",
+    "shortname": "tc39-import-assertions",
+    "series": {
+      "shortname": "tc39-import-assertions",
+      "currentSpecification": "tc39-import-assertions"
+    },
+    "nightly": {
+      "url": "https://tc39.es/proposal-import-assertions/",
+      "repository": "https://github.com/tc39/proposal-import-assertions",
+      "filename": "index.html"
+    },
+    "title": "import assertions",
+    "source": "spec",
+    "shortTitle": "import assertions"
+  },
+  {
+    "url": "https://tc39.es/proposal-private-methods/",
+    "seriesComposition": "full",
+    "shortname": "tc39-private-methods",
+    "series": {
+      "shortname": "tc39-private-methods",
+      "currentSpecification": "tc39-private-methods"
+    },
+    "nightly": {
+      "url": "https://tc39.es/proposal-private-methods/",
+      "repository": "https://github.com/tc39/proposal-private-methods",
+      "filename": "index.html"
+    },
+    "title": "Private Methods and Accessors Proposal",
+    "source": "spec",
+    "shortTitle": "Private Methods and Accessors Proposal"
+  },
+  {
+    "url": "https://tc39.es/proposal-regexp-match-indices/",
+    "seriesComposition": "full",
+    "shortname": "tc39-regexp-match-indices",
+    "series": {
+      "shortname": "tc39-regexp-match-indices",
+      "currentSpecification": "tc39-regexp-match-indices"
+    },
+    "nightly": {
+      "url": "https://tc39.es/proposal-regexp-match-indices/",
+      "repository": "https://github.com/tc39/proposal-regexp-match-indices",
+      "filename": "index.html"
+    },
+    "title": "RegExp Match Indices",
+    "source": "spec",
+    "shortTitle": "RegExp Match Indices"
+  },
+  {
+    "url": "https://tc39.es/proposal-relative-indexing-method/",
+    "seriesComposition": "full",
+    "shortname": "tc39-relative-indexing-method",
+    "series": {
+      "shortname": "tc39-relative-indexing-method",
+      "currentSpecification": "tc39-relative-indexing-method"
+    },
+    "nightly": {
+      "url": "https://tc39.es/proposal-relative-indexing-method/",
+      "repository": "https://github.com/tc39/proposal-relative-indexing-method",
+      "filename": "index.html"
+    },
+    "title": "Relative Indexing Method",
+    "source": "spec",
+    "shortTitle": "Relative Indexing Method"
+  },
+  {
+    "url": "https://tc39.es/proposal-static-class-features/",
+    "seriesComposition": "full",
+    "shortname": "tc39-static-class-features",
+    "series": {
+      "shortname": "tc39-static-class-features",
+      "currentSpecification": "tc39-static-class-features"
+    },
+    "nightly": {
+      "url": "https://tc39.es/proposal-static-class-features/",
+      "repository": "https://github.com/tc39/proposal-static-class-features",
+      "filename": "index.html"
+    },
+    "title": "Static class features",
+    "source": "spec",
+    "shortTitle": "Static class features"
+  },
+  {
+    "url": "https://tc39.es/proposal-top-level-await/",
+    "seriesComposition": "full",
+    "shortname": "tc39-top-level-await",
+    "series": {
+      "shortname": "tc39-top-level-await",
+      "currentSpecification": "tc39-top-level-await"
+    },
+    "nightly": {
+      "url": "https://tc39.es/proposal-top-level-await/",
+      "repository": "https://github.com/tc39/proposal-top-level-await",
+      "filename": "index.html"
+    },
+    "title": "Top-Level Await",
+    "source": "spec",
+    "shortTitle": "Top-Level Await"
+  },
   {
     "url": "https://url.spec.whatwg.org/",
     "seriesComposition": "full",

specs.json

@@ -55,6 +55,19 @@
   "https://storage.spec.whatwg.org/",
   "https://streams.spec.whatwg.org/",
   "https://svgwg.org/specs/animations/",
+  {
+    "url": "https://tc39.es/ecma262/",
+    "shortname": "ecmascript",
+    "shortTitle": "ECMAScript"
+  },
+  "https://tc39.es/proposal-atomics-wait-async/",
+  "https://tc39.es/proposal-class-fields/",
+  "https://tc39.es/proposal-import-assertions/",
+  "https://tc39.es/proposal-private-methods/",
+  "https://tc39.es/proposal-regexp-match-indices/",
+  "https://tc39.es/proposal-relative-indexing-method/",
+  "https://tc39.es/proposal-static-class-features/",
+  "https://tc39.es/proposal-top-level-await/",
   "https://url.spec.whatwg.org/",
   "https://w3c.github.io/badging/",
   "https://w3c.github.io/contentEditable/",

src/compute-repository.js

@@ -29,6 +29,11 @@ function urlToGitHubRepository(url) {
     return { owner: "whatwg", name: whatwg[1] };
   }
 
+  const tc39 = url.match(/^https:\/\/tc39.es\/([^\/]*)\//);
+  if (tc39) {
+    return { owner: "tc39", name: tc39[1] };
+  }
+
   const csswg = url.match(/^https?:\/\/drafts.csswg.org\/([^\/]*)\/?/);
   if (csswg) {
     return { owner: "w3c", name: "csswg-drafts" };
@@ -120,4 +125,4 @@ module.exports = async function (specs, options) {
   });
 
   return specs;
-};
+};

src/compute-shortname.js

@@ -56,6 +56,13 @@ function computeShortname(url) {
         return whatwg[1];
     }
 
+    // Handle TC39 Proposals
+    const tc39 = url.match(/\/\/tc39\.es\/proposal-([^\/]+)\/$/);
+    if (tc39) {
+        return "tc39-" + tc39[1];
+    }
+
+
     // Handle Khronos extensions
     const khronos = url.match(/https:\/\/www\.khronos\.org\/registry\/webgl\/extensions\/([^\/]+)\/$/);
     if (khronos) {

src/data/ignore.json

@@ -1,5 +1,11 @@
 {
   "repos": {
+    "tc39/proposal-regexp-legacy-features": {
+      "comment": "no proper spec, legacy"
+    },
+    "tc39/proposal-hashbang/": {
+      "comment": "not meant for browsers environments"
+    },
     "w3c/adpt": {
       "comment": "not targeted at browsers"
     },

src/fetch-info.js

@@ -42,6 +42,7 @@
  */
 
 const https = require("https");
+const {JSDOM} = require("jsdom");
 
 async function fetchInfoFromW3CApi(specs, options) {
   // Cannot query the W3C API if API key was not given
@@ -254,40 +255,33 @@ async function fetchInfoFromSpecref(specs, options) {
 
 async function fetchInfoFromSpecs(specs, options) {
   const info = await Promise.all(specs.map(async spec => {
-    const html = await new Promise((resolve, reject) => {
-      const request = https.get(spec.url, options, res => {
-        if (res.statusCode !== 200) {
-          reject(`Could not fetch URL ${spec.url} for spec "${spec.shortname}", ` +
-            `status code is ${res.statusCode}`);
-          return;
-        }
-        res.setEncoding("utf8");
-        let data = "";
-        res.on("data", chunk => data += chunk);
-        res.on("end", () => {
-          resolve(data);
-        });
-      });
-      request.on("error", err => reject(err));
-      request.end();
-    });
-
+    const dom = await JSDOM.fromURL(spec.url);
+
+    if (spec.url.startsWith("https://tc39.es/")) {
+      const h1ecma = [...dom.window.document.querySelectorAll("h1")][1];
+      if (h1ecma) {
+        return {
+          nightly: { url: spec.url },
+          title: h1ecma.textContent.replace(/\n/g, '').trim()
+        };
+      }
+    }
     // Extract first heading
-    const h1Match = html.match(/<h1[^>]*?>(.*?)<\/h1>/mis);
-    if (h1Match) {
+    const h1 = dom.window.document.querySelector("h1");
+    if (h1) {
       return {
         nightly: { url: spec.url },
-        title: h1Match[1].replace(/\n/g, '').trim()
+        title: h1.textContent.replace(/\n/g, '').trim()
       };
     }
 
     // Use the document's title if first heading could not be found
     // (that typically happens in Respec specs)
-    const titleMatch = html.match(/<title[^>]*?>(.*?)<\/title>/mis);
-    if (titleMatch) {
+    const title = dom.window.document.querySelector("title");
+    if (title) {
       return {
         nightly: { url: spec.url },
-        title: titleMatch[1].replace(/\n/g, '').trim()
+        title: title.textContent.replace(/\n/g, '').trim()
       };
     }
 
@@ -339,4 +333,4 @@ async function fetchInfo(specs, options) {
 }
 
 
-module.exports = fetchInfo;
+module.exports = fetchInfo;

src/find-specs.js

@@ -5,6 +5,8 @@ const core = require('@actions/core');
 
 const fetch = require("node-fetch");
 
+const {JSDOM} = require("jsdom");
+
 const computeShortname = require("./compute-shortname");
 
 const specs = require("../index.json");
@@ -98,6 +100,11 @@ const hasPublishedContent = (candidate) => fetch(candidate.spec).then(({ok, url}
   const fxtfSpecs = await fetch("https://api.github.com/repos/w3c/fxtf-drafts/contents/").then(r => r.json()).then(data => data.filter(p => p.type === "dir" && !fxtfMetaDir.includes(p.path)).map(p => p.path));
   const houdiniSpecs = await fetch("https://api.github.com/repos/w3c/css-houdini-drafts/contents/").then(r => r.json()).then(data => data.filter(p => p.type === "dir" && !houdiniMetaDir.includes(p.path)).map(p => p.path));
 
+  const ecmaProposals = await JSDOM.fromURL("https://github.com/tc39/proposals/blob/master/README.md")
+  // we only watch stage 3 proposals, which are in the first table on the page above
+    .then(dom => [...dom.window.document.querySelector("table").querySelectorAll("tr td:first-child a")].map(a => a.href));
+
+
   const chromeFeatures = await fetch("https://www.chromestatus.com/features.json").then(r => r.json());
 
   const wgs = Object.values(groups).filter(g => g.type === "working group" && !nonBrowserSpecWgs.includes(g.name));
@@ -173,6 +180,7 @@ const hasPublishedContent = (candidate) => fetch(candidate.spec).then(({ok, url}
                                  .filter(hasUntrackedURL)
                                  .filter(isInScope));
 
+
   // Check for new CSS specs
   candidates = candidates.concat(cssSpecs.map(s => { return {repo: "w3c/csswg-drafts", spec: `https://drafts.csswg.org/${s}/`};})
                                  .filter(hasUntrackedURL)
@@ -193,6 +201,12 @@ const hasPublishedContent = (candidate) => fetch(candidate.spec).then(({ok, url}
                                  .filter(hasUntrackedURL)
                                  .filter(isInScope));
 
+  // Check for new TC39 Stage 3 proposals
+  candidates = candidates.concat(ecmaProposals.map(s => { return {repo: s.replace('https://github.com/', ''), spec: s.replace('https://github.com/tc39/', 'https://tc39.github.io/') + '/'};})
+                                 .filter(hasUntrackedURL)
+                                 .filter(isInScope));
+
+
   // Add information from Chrome Feature status
   candidates = candidates.map(c => { return {...c, impl: { chrome: (chromeFeatures.find(f => f.standards.spec && f.standards.spec.startsWith(c.spec)) || {}).id}};});
 

test/compute-repository.js

@@ -26,6 +26,12 @@ describe("compute-repository module", async () => {
       "https://github.com/whatwg/specname");
   });
 
+  it("handles TC39 URLs", async () => {
+    assert.equal(
+      await computeSingleRepo("https://tc39.es/js-ftw/"),
+      "https://github.com/tc39/js-ftw");
+  });
+
   it("handles CSS WG URLs", async () => {
     assert.equal(
       await computeSingleRepo("https://drafts.csswg.org/css-everything-42/"),

test/compute-shortname.js

@@ -16,6 +16,10 @@ describe("compute-shortname module", () => {
       assertName("https://myspec.spec.whatwg.org/whatever/", "myspec");
     });
 
+    it("handles ECMAScript proposal URLs", () => {
+      assertName("https://tc39.es/proposal-smartidea/", "tc39-smartidea");
+    });
+
     it("handles URLs of drafts on GitHub", () => {
       assertName("https://wicg.github.io/whataspec/", "whataspec");
     });