Add links back to spec in idlparsed extracts

With this update, the idlparsed postprocessing module completes the parsed
structures with an `href` property that links back to the definition of the
IDL construct in the spec wherever possible.

That is not always possible. That should signal a problem with the spec. Main
reasons why linking back to definitions may not be possible:
- The spec simply does not have the definitions. That happens for a number
of enumeration values for instance.
- The spec does not use the appropriate dfn convention for IDL members. That's
the case for WebGL specs but also still a problem for some attributes and
methods in the HTML spec.
- The IDL in Webref is patched (for good reasons), and no longer matches what's
defined in the spec.
- The spec has an informative/normative hiccup: it defines the IDL as normative
but the actual dfns are in an informative section.

schemas/postprocessing/idlparsed.json

@@ -32,7 +32,8 @@
             "required": ["fragment", "type"],
             "properties": {
               "fragment": { "type": "string" },
-              "type": { "$ref": "../common.json#/$defs/interfacetype" }
+              "type": { "$ref": "../common.json#/$defs/interfacetype" },
+              "href": { "$ref": "../common.json#/$defs/url" }
             }
           }
         },

src/postprocessing/idlparsed.js

@@ -8,17 +8,170 @@
 const webidlParser = require('../cli/parse-webidl');
 
 module.exports = {
-  dependsOn: ['idl'],
+  dependsOn: ['dfns', 'idl'],
   input: 'spec',
   property: 'idlparsed',
 
   run: async function(spec, options) {
+    function getHref(idl, member) {
+      let dfnType;
+      let dfnFor;
+      let dfnOverload = 0;
+      let dfnName;
+      if (member) {
+        if (['iterable', 'maplike', 'setlike'].includes(member.type) ||
+            ['getter', 'setter', 'stringifier', 'deleter'].includes(member.special)) {
+          // No dfns of these types in any spec as of Feb 2024, or at least no
+          // no dfns that we can easily map to (for example, the HTML spec
+          // tends to use generic "dfn" for these).
+          return null;
+        }
+        if (member.type === 'operation') {
+          dfnType = 'method';
+          dfnOverload = idl.members
+            .filter(m => m.type === member.type && m.name === member.name)
+            .findIndex(m => m === member);
+        }
+        else if (member.type === 'field') {
+          dfnType = 'dict-member';
+        }
+        else {
+          dfnType = member.type;
+        }
+        if (!['constructor', 'method', 'attribute', 'enum-value', 'dict-member', 'const'].includes(dfnType)) {
+          console.error(`[error] Found unexpected IDL member type "${dfnType}" in ${spec.shortname}`);
+        }
+        dfnName = member.name ?? member.value;
+        dfnFor = idl.name;
+      }
+      else {
+        // The type of the dfn to look for is the same as the IDL type, except
+        // that composed IDL types ("interface mixin", "callback interface")
+        // only have the basic type in definitions.
+        dfnType = idl.type.split(' ')[0];
+        dfnName = idl.name;
+      }
+
+      const dfnNames = [];
+      if (dfnType === 'enum-value') {
+        // Bikeshed keeps wrapping quotes in the dfn linking text, not ReSpec.
+        dfnNames.push(dfnName);
+        dfnNames.push(`"${dfnName}"`);
+      }
+      else if (dfnType === 'method') {
+        // Bikeshed adds "..." for variadic arguments, not ReSpec. Let's try
+        // both variants. For overloads, Bikeshed essentially expects arguments
+        // to have different names, while ReSpec adds "!overload-x" to
+        // overloaded methods. We'll test all possibilities in order. If the
+        // spec only has a dfn for the most basic method, it's possible that we
+        // end up linking to that dfn from the overloaded methods too, but that
+        // seems good enough in practice.
+        // Last, method definitions sometimes appear without arguments (notably
+        // in the HTML spec).
+        const argsVariadic = member.arguments.map(arg => (arg.variadic ? '...' : '') + arg.name);
+        const args = member.arguments.map(arg => arg.name);
+        dfnNames.push(`${dfnName}!overload-${dfnOverload}(${args.join(', ')})`);
+        dfnNames.push(`${dfnName}(${argsVariadic.join(', ')})`);
+        dfnNames.push(`${dfnName}(${args.join(', ')})`);
+        dfnNames.push(`${dfnName}()`);
+      }
+      else if (dfnType === 'constructor') {
+        // Same as for methods
+        const argsVariadic = member.arguments.map(arg => (arg.variadic ? '...' : '') + arg.name);
+        const args = member.arguments.map(arg => arg.name);
+        dfnNames.push(`constructor!overload-${dfnOverload}(${args.join(', ')})`);
+        dfnNames.push(`constructor(${argsVariadic.join(', ')})`);
+        dfnNames.push(`constructor(${args.join(', ')})`);
+        dfnNames.push(`constructor()`);
+      }
+      else {
+        dfnNames.push(dfnName);
+      }
+
+      // Look for definitions that look like good initial candidates
+      const candidateDfns = spec.dfns
+        .filter(dfn => dfn.type === dfnType && !dfn.informative &&
+          (dfnFor ? dfn.for.includes(dfnFor) : true));
+
+      // Look for names in turn in that list of candidates.
+      for (const name of dfnNames) {
+        const dfns = candidateDfns.filter(dfn => dfn.linkingText.includes(name));
+        if (dfns.length > 0) {
+          if (dfns.length > 1) {
+            const forLabel = dfnFor ? ` for \`${dfnFor}\`` : '';
+            console.warn(`[warn] More than one dfn for ${dfnType} \`${dfnName}\`${forLabel} in [${spec.shortname}](${spec.crawled}).`);
+            return null;
+          }
+          else {
+            return dfns[0].href;
+          }
+        }
+      }
+
+      // Report missing dfns except for specs that we know already lack them
+      if (!['webgl1', 'webgl2', 'svg-animations', 'SVG2'].includes(spec.shortname)) {
+        const forLabel = dfnFor ? ` for \`${dfnFor}\`` : '';
+        console.warn(`[warn] No dfn for ${dfnType} \`${dfnName}\`${forLabel} in [${spec.shortname}](${spec.crawled})`);
+      }
+      return null;
+    }
+
     if (!spec?.idl) {
       return spec;
     }
     try {
       spec.idlparsed = await webidlParser.parse(spec.idl);
       spec.idlparsed.hasObsoleteIdl = webidlParser.hasObsoleteIdl(spec.idl);
+
+      if (spec.dfns) {
+        for (const idl of Object.values(spec.idlparsed.idlNames)) {
+          const href = getHref(idl);
+          if (href) {
+            idl.href = href;
+          }
+
+          if (idl.values) {
+            for (const value of idl.values) {
+              const href = getHref(idl, value);
+              if (href) {
+                value.href = href;
+              }
+            }
+          }
+
+          if (idl.members) {
+            for (const member of idl.members) {
+              const href = getHref(idl, member);
+              if (href) {
+                member.href = href;
+              }
+            }
+          }
+        }
+
+        for (const extendedIdl of Object.values(spec.idlparsed.idlExtendedNames)) {
+          for (const idl of extendedIdl) {
+            // No dfn for the extension, we can only link specific members
+            if (idl.values) {
+              for (const value of idl.values) {
+                const href = getHref(idl, value);
+                if (href) {
+                  value.href = href;
+                }
+              }
+            }
+
+            if (idl.members) {
+              for (const member of idl.members) {
+                const href = getHref(idl, member);
+                if (href) {
+                  member.href = href;
+                }
+              }
+            }
+          }
+        }
+      }
     }
     catch (err) {
       // IDL content is invalid and cannot be parsed.

tests/generate-idlparsed.js

@@ -33,4 +33,44 @@ describe('The parsed IDL generator', function () {
 intraface foo {};
 ^ Unrecognised tokens`);
   });
+
+
+  function getIdlSpecWithDfn(type) {
+    return {
+      dfns: [{
+        href: 'about:blank/#foo',
+        linkingText: ['foo'],
+        localLinkingText: [],
+        type: type.split(' ')[0],
+        for: [],
+        access: 'public',
+        informative: false
+      }],
+      idl: `${type} foo {};`
+    };
+  }
+
+  // Note: we could also test "enum", "typedef" and "callback" IDL types, but
+  // the IDL syntax would need to be different (e.g., "enum foo {}" is invalid)
+  for (const type of [
+    'dictionary', 'interface', 'interface mixin',
+    'callback interface', 'namespace'
+  ]) {
+    it(`links back to the definition in the spec when available (${type})`, async () => {
+      const spec = getIdlSpecWithDfn(type);
+      const result = await run(spec);
+      assert.deepEqual(result?.idlparsed?.idlNames, {
+        foo: {
+          extAttrs: [],
+          fragment: `${type} foo {};`,
+          inheritance: null,
+          members: [],
+          name: 'foo',
+          partial: false,
+          type: type,
+          href: 'about:blank/#foo'
+        }
+      });
+    });
+  }
 });