refactor(route/caniuse): merge sidvishnmoi/respec-caniuse-route to main

package.json

@@ -17,7 +17,6 @@
     "helmet": "^4.1.0",
     "morgan": "^1.10.0",
     "node-fetch": "^2.6.1",
-    "respec-caniuse-route": "^3.1.1",
     "respec-github-apis": "^2.0.0",
     "respec-xref-route": "^9.0.4",
     "split2": "^3.2.2",

routes/caniuse/index.js

@@ -5,7 +5,7 @@ import cors from "cors";
 import authGithubWebhook from "../../utils/auth-github-webhook.js";
 import { env, seconds } from "../../utils/misc.js";
 
-import { createResponseBody } from "respec-caniuse-route";
+import { createResponseBody } from "./lib/index.js";
 import updateRoute from "./update.js";
 
 const caniuse = Router({ mergeParams: true });

routes/caniuse/lib/README.md

@@ -0,0 +1 @@
+This directory was originally maintained in a separate repository at https://github.com/sidvishnoi/respec-caniuse-route.

routes/caniuse/lib/constants.ts

@@ -0,0 +1,28 @@
+export const BROWSERS = new Map([
+  ['and_chr', 'Chrome (Android)'],
+  ['and_ff', 'Firefox (Android)'],
+  ['and_uc', 'UC Browser (Android)'],
+  ['android', 'Android'],
+  ['bb', 'Blackberry'],
+  ['chrome', 'Chrome'],
+  ['edge', 'Edge'],
+  ['firefox', 'Firefox'],
+  ['ie', 'IE'],
+  ['ios_saf', 'Safari (iOS)'],
+  ['op_mini', 'Opera Mini'],
+  ['op_mob', 'Opera Mobile'],
+  ['opera', 'Opera'],
+  ['safari', 'Safari'],
+  ['samsung', 'Samsung Internet'],
+]);
+
+// Keys from https://github.com/Fyrd/caniuse/blob/master/CONTRIBUTING.md
+export const SUPPORT_TITLES = new Map([
+  ['y', 'Supported.'],
+  ['a', 'Almost supported (aka Partial support).'],
+  ['n', 'No support, or disabled by default.'],
+  ['p', 'No support, but has Polyfill.'],
+  ['u', 'Support unknown.'],
+  ['x', 'Requires prefix to work.'],
+  ['d', 'Disabled by default (needs to enabled).'],
+]);

routes/caniuse/lib/index.ts

@@ -0,0 +1,159 @@
+import * as path from "path";
+import { promises as fs } from "fs";
+
+import { html } from "ucontent";
+
+import { BROWSERS, SUPPORT_TITLES } from "./constants.js";
+import { env } from "../../../utils/misc.js";
+import { MemCache } from "../../../utils/mem-cache.js";
+
+const DATA_DIR = env("DATA_DIR");
+
+interface Options {
+  feature: string;
+  browsers?: string[];
+  versions?: number;
+  format?: "html" | "json";
+}
+type NormalizedOptions = Required<Options>;
+
+type SupportKeys = ("y" | "n" | "a" | string)[];
+// [ version, ['y', 'n'] ]
+type BrowserVersionData = [string, SupportKeys];
+
+interface Data {
+  [browserName: string]: BrowserVersionData[];
+}
+
+const defaultOptions = {
+  browsers: ["chrome", "firefox", "safari", "edge"],
+  versions: 4,
+};
+
+// Content in this cache is invalidated through `POST /caniuse/update`.
+export const cache = new MemCache<Data>(Infinity);
+
+export async function createResponseBody(options: Options) {
+  const opts = normalizeOptions(options);
+
+  switch (opts.format) {
+    case "json":
+      return await createResponseBodyJSON(opts);
+    case "html":
+    default:
+      return await createResponseBodyHTML(opts);
+  }
+}
+
+export async function createResponseBodyJSON(options: NormalizedOptions) {
+  const { feature, browsers, versions } = options;
+  const data = await getData(feature);
+  if (!data) {
+    return null;
+  }
+
+  if (!browsers.length) {
+    browsers.push(...Object.keys(data));
+  }
+
+  const response: Data = Object.create(null);
+  for (const browser of browsers) {
+    const browserData = data[browser] || [];
+    response[browser] = browserData.slice(0, versions);
+  }
+  return response;
+}
+
+export async function createResponseBodyHTML(options: NormalizedOptions) {
+  const data = await createResponseBodyJSON(options);
+  return data === null ? null : formatAsHTML(options, data);
+}
+
+function normalizeOptions(options: Options): NormalizedOptions {
+  const feature = options.feature;
+  const versions = options.versions || defaultOptions.versions;
+  const browsers = sanitizeBrowsersList(options.browsers);
+  const format = options.format === "html" ? "html" : "json";
+  return { feature, versions, browsers, format };
+}
+
+function sanitizeBrowsersList(browsers?: string | string[]) {
+  if (!Array.isArray(browsers)) {
+    if (browsers === "all") return [];
+    return defaultOptions.browsers;
+  }
+  const filtered = browsers.filter(browser => BROWSERS.has(browser));
+  return filtered.length ? filtered : defaultOptions.browsers;
+}
+
+async function getData(feature: string) {
+  if (cache.has(feature)) {
+    return cache.get(feature) as Data;
+  }
+  const file = path.format({
+    dir: path.join(DATA_DIR, "caniuse"),
+    name: `${feature}.json`,
+  });
+
+  try {
+    const str = await fs.readFile(file, "utf8");
+    const data: Data = JSON.parse(str);
+    cache.set(feature, data);
+    return data;
+  } catch (error) {
+    console.error(error);
+    return null;
+  }
+}
+
+function formatAsHTML(options: NormalizedOptions, data: Data) {
+  const getSupportTitle = (keys: SupportKeys) => {
+    return keys
+      .filter(key => SUPPORT_TITLES.has(key))
+      .map(key => SUPPORT_TITLES.get(key)!)
+      .join(" ");
+  };
+
+  const getClassName = (keys: SupportKeys) => `caniuse-cell ${keys.join(" ")}`;
+
+  const renderLatestVersion = (
+    browserName: string,
+    [version, supportKeys]: BrowserVersionData,
+  ) => {
+    const text = `${BROWSERS.get(browserName) || browserName} ${version}`;
+    const className = getClassName(supportKeys);
+    const title = getSupportTitle(supportKeys);
+    return html`<button class="${className}" title="${title}">${text}</button>`;
+  };
+
+  const renderOlderVersion = ([version, supportKeys]: BrowserVersionData) => {
+    const text = version;
+    const className = getClassName(supportKeys);
+    const title = getSupportTitle(supportKeys);
+    return html`<li class="${className}" title="${title}">${text}</li>`;
+  };
+
+  const renderBrowser = (
+    browser: string,
+    browserData: BrowserVersionData[],
+  ) => {
+    const [latestVersion, ...olderVersions] = browserData;
+    return html`
+      <div class="caniuse-browser">
+        ${renderLatestVersion(browser, latestVersion)}
+        <ul>
+          ${olderVersions.map(renderOlderVersion)}
+        </ul>
+      </div>
+    `;
+  };
+
+  const browsers = html`${Object.entries(data).map(([browser, browserData]) =>
+    renderBrowser(browser, browserData),
+  )}`;
+
+  const featureURL = new URL(options.feature, "https://caniuse.com/").href;
+  const moreInfo = html`<a href="${featureURL}">More info</a>`;
+
+  return html`${browsers} ${moreInfo}`.toString();
+}

routes/caniuse/lib/scraper.ts

@@ -0,0 +1,122 @@
+// Reads features-json/*.json files from caniuse repository
+// and writes each file in a "respec friendly way"
+//  - Keep only the `stats` from features-json data
+//  - Sort browser versions (latest first)
+//  - Remove footnotes and other unnecessary data
+
+import * as path from "path";
+import { existsSync } from "fs";
+import { readFile, writeFile, readdir, mkdir } from "fs/promises";
+
+import sh from "../../../utils/sh.js";
+import { env } from "../../../utils/misc.js";
+
+interface Input {
+  stats: {
+    [browserName: string]: { [version: string]: string };
+  };
+}
+
+interface Output {
+  [browserName: string]: [string, ReturnType<typeof formatStatus>][];
+}
+
+const DATA_DIR = env("DATA_DIR");
+const INPUT_REPO_SRC = "https://github.com/Fyrd/caniuse.git";
+const INPUT_REPO_NAME = "caniuse-raw";
+const INPUT_DIR = path.join(DATA_DIR, INPUT_REPO_NAME, "features-json");
+const OUTPUT_DIR = path.join(DATA_DIR, "caniuse");
+
+const defaultOptions = { forceUpdate: false };
+type Options = typeof defaultOptions;
+
+export default async function main(options: Partial<Options> = {}) {
+  const opts = { ...defaultOptions, ...options };
+  const hasUpdated = await updateInputSource();
+  if (!hasUpdated && !opts.forceUpdate) {
+    console.log("Nothing to update");
+    return false;
+  }
+
+  console.log("INPUT_DIR:", INPUT_DIR);
+  console.log("OUTPUT_DIR:", OUTPUT_DIR);
+  if (!existsSync(OUTPUT_DIR)) {
+    await mkdir(OUTPUT_DIR, { recursive: true });
+  }
+
+  const fileNames = await readdir(INPUT_DIR);
+  console.log(`Processing ${fileNames.length} files...`);
+  const promisesToProcess = fileNames.map(processFile);
+  await Promise.all(promisesToProcess);
+  console.log(`Processed ${fileNames.length} files.`);
+  return true;
+}
+
+async function updateInputSource() {
+  const dataDir = path.join(DATA_DIR, INPUT_REPO_NAME);
+  const shouldClone = !existsSync(dataDir);
+
+  const command = shouldClone
+    ? `git clone ${INPUT_REPO_SRC} ${INPUT_REPO_NAME} --filter=blob:none`
+    : `git pull --depth=1`;
+  const cwd = shouldClone ? path.resolve(DATA_DIR) : dataDir;
+
+  const stdout = await sh(command, { cwd, output: "stream" });
+  const hasUpdated = !stdout.toString().includes("Already up to date");
+  return hasUpdated;
+}
+
+async function processFile(fileName: string) {
+  const inputFile = path.join(INPUT_DIR, fileName);
+  const outputFile = path.join(OUTPUT_DIR, fileName);
+
+  const json = await readJSON(inputFile);
+
+  const output: Output = {};
+  for (const [browserName, browserData] of Object.entries(json.stats)) {
+    const stats = Object.entries(browserData)
+      .sort(semverCompare)
+      .map(([version, status]) => [version, formatStatus(status)])
+      .reverse() as [string, string[]][];
+    output[browserName] = stats;
+  }
+
+  await writeJSON(outputFile, output);
+}
+
+type BrowserDataEntry = [string, string];
+/**
+ * semverCompare
+ * https://github.com/substack/semver-compare
+ */
+function semverCompare(a: BrowserDataEntry, b: BrowserDataEntry) {
+  const pa = a[0].split(".");
+  const pb = b[0].split(".");
+  for (let i = 0; i < 3; i++) {
+    const na = Number(pa[i]);
+    const nb = Number(pb[i]);
+    if (na > nb) return 1;
+    if (nb > na) return -1;
+    if (!isNaN(na) && isNaN(nb)) return 1;
+    if (isNaN(na) && !isNaN(nb)) return -1;
+  }
+  return 0;
+}
+
+/**  @example "n d #6" => ["n", "d"] */
+function formatStatus(status: string) {
+  return status
+    .split("#", 1)[0] // don't care about footnotes.
+    .split(" ")
+    .filter(item => item);
+}
+
+async function readJSON(file: string) {
+  const str = await readFile(file, "utf8");
+  return JSON.parse(str) as Input;
+}
+
+async function writeJSON(file: string, json: Output) {
+  const str = JSON.stringify(json);
+  await writeFile(file, str);
+}

routes/caniuse/update.js

@@ -1,8 +1,8 @@
 // @ts-check
 import { queue } from "../../utils/background-task-queue.js";
 
-import { cache } from "respec-caniuse-route";
-import { main as scraper } from "respec-caniuse-route/scraper.js";
+import { cache } from "./lib/index.js";
+import scraper from "./lib/scraper.js";
 
 export default function route(req, res) {
   if (req.body.ref !== "refs/heads/master") {

scripts/update-data-sources.js

@@ -1,5 +1,6 @@
+import "../build/utils/dotenv.js";
+import caniuse from "../build/routes/caniuse/lib/scraper.js";
 import { main as xref } from "respec-xref-route/scraper.js";
-import { main as caniuse } from "respec-caniuse-route/scraper.js";
 
 async function update() {
   console.group("caniuse");