feat(evasive-transform): isolate source transform system into its own…

… package

This extracts everything from `@endo/bundle-source/src/transform.js` into a new package (formerly `@endo/transforms`).

LavaMoat has need to consume the SES-censorship-evasion functionality, and it would be cumbersome to try to use `@endo/bundle-source` for this.

packages/evasive-transform/README.md

@@ -0,0 +1,39 @@
+# @endo/evasive-transform
+
+> Source transforms for evading censorship in [SES](https://github.com/endojs/endo/tree/master/packages/ses)-enabled applications
+
+This package provides a function which transforms comments contained in source code which would otherwise be rejected outright by SES.
+
+## Example
+
+```js
+// ESM example
+import { evadeCensor } from '@endo/evasive-transform';
+import fs from 'node:fs/promises';
+
+/**
+ * Imagine this file contains a comment like `@property {import('foo').Bar} bar`. SES will refuse to run this code.
+ */
+const source = await fs.readFile('./dist/index.js', 'utf8');
+const sourceMap = await fs.readFile('./dist/index.js.map', 'utf8');
+const sourceUrl = 'index.js'; // assuming the source map references index.js
+const sourceType = 'script';
+
+const { code, map } = await evadeCensor(source, {
+  sourceMap,
+  sourceUrl,
+  sourceType,
+});
+
+/**
+ * The resulting file will now contain `@property {ІᛖРΟᏒТ('foo').Bar} bar`, which SES will allow (and TypeScript no longer understands, but that should be fine for the use-case). 
+ * 
+ * Note that this could be avoided entirely by stripping comments during, say, a bundling phase.
+ */
+await fs.writeFile('./dist/index.ses.js', code);
+await fs.writeFile('./dist/index.ses.js.map', JSON.stringify(map));
+```
+
+## License
+
+Apache-2.0

packages/evasive-transform/index.js

@@ -0,0 +1 @@
+export * from './src/index.js';

packages/transforms/package.json → packages/evasive-transform/package.json

@@ -1,12 +1,11 @@
 {
-  "name": "@endo/transforms",
+  "name": "@endo/evasive-transform",
   "version": "0.1.3",
-  "private": true,
-  "description": "Automatic code transformations for Endo JavaScript applications",
-  "keywords": [],
+  "description": "Source transforms to evade SES censorship",
+  "keywords": ["ses", "transform"],
   "author": "Endo contributors",
   "license": "Apache-2.0",
-  "homepage": "https://github.com/endojs/endo/tree/master/packages/transforms#readme",
+  "homepage": "https://github.com/endojs/endo/tree/master/packages/evasive-transform#readme",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/endojs/endo.git"
@@ -33,12 +32,19 @@
     "lint-check": "yarn lint",
     "lint": "yarn lint:types && yarn lint:eslint",
     "lint:types": "tsc",
-    "lint:eslint": "eslint '**/*.js'"
+    "lint:eslint": "eslint '**/*.js'",
+    "test:rebuild-fixtures": "node test/rebuild-fixtures.js"
   },
   "devDependencies": {
+    "@babel/types": "7.23.0",    
     "@endo/ses-ava": "^0.2.44",
+    "@rollup/plugin-commonjs": "^19.0.0",
+    "@rollup/plugin-node-resolve": "^13.0.0",
+    "@types/babel__generator": "7.6.5",
+    "@types/babel__traverse": "^7.20.2",
     "ava": "^5.3.0",
     "c8": "^7.14.0",
+    "rollup": "^2.79.1",
     "tsd": "^0.28.1"
   },
   "files": [
@@ -62,5 +68,14 @@
       "test/**/test-*.js"
     ],
     "timeout": "2m"
+  },
+  "dependencies": {
+    "@agoric/babel-generator": "7.17.6",
+    "@babel/parser": "7.20.3",
+    "@babel/traverse": "7.20.1",
+    "source-map": "0.7.4"
+  },
+  "resolutions": {
+    "@babel/types": "7.23.0"
   }
 }

packages/evasive-transform/src/generate.js

@@ -0,0 +1,80 @@
+/**
+ * Provides {@link generate}, which is the final step of the transformation
+ *
+ * @module
+ */
+
+import babelGenerate from '@agoric/babel-generator';
+
+/**
+ * It works; don't ask.
+ * @type {typeof import('@babel/generator')}
+ */
+const { default: generator } = /** @type {any} */ (babelGenerate);
+
+/**
+ * Options for {@link generateCode} with source map
+ *
+ * @typedef GenerateAstOptionsWithSourceMap
+ * @property {string} sourceUrl - If present, we will generate a source map
+ * @internal
+ */
+
+/**
+ * Options for {@link generateCode} (no source map generated)
+ *
+ * @typedef GenerateAstOptionsWithoutSourceMap
+ * @property {undefined} sourceUrl - This should be undefined or otherwise not provided
+ * @internal
+ */
+
+/**
+ * The result of {@link generate}; depends on whether a `sourceUrl` was
+ * provided to the options.
+ *
+ * @template {string|undefined} [SourceUrl=undefined]
+ * @typedef {{code: string, map: SourceUrl extends string ? import('source-map').RawSourceMap : never}} TransformedResult
+ * @internal
+ */
+
+/**
+ * Generates new code from a Babel AST; returns code and source map
+ *
+ * @callback GenerateAstWithSourceMap
+ * @param {import('@babel/types').File} ast - Babel "File" AST
+ * @param {GenerateAstOptionsWithSourceMap} options - Options for the transform
+ * @returns {TransformedResult<string>}
+ * @internal
+ */
+
+/**
+ * Generates new code from a Babel AST; returns code only
+ *
+ * @callback GenerateAstWithoutSourceMap
+ * @param {import('@babel/types').File} ast - Babel "File" AST
+ * @param {GenerateAstOptionsWithoutSourceMap} [options] - Options for the transform
+ * @returns {TransformedResult<undefined>}
+ * @internal
+ */
+export const generate =
+  /** @type {GenerateAstWithSourceMap & GenerateAstWithoutSourceMap} */ (
+    (ast, options) => {
+      const sourceUrl = options?.sourceUrl;
+      const result = generator(ast, {
+        sourceFileName: sourceUrl,
+        sourceMaps: Boolean(sourceUrl),
+        retainLines: true,
+        compact: true,
+      });
+
+      if (sourceUrl) {
+        return {
+          code: result.code,
+          map: result.map,
+        };
+      }
+      return {
+        code: result.code,
+      };
+    }
+  );

packages/evasive-transform/src/index.js

@@ -0,0 +1,62 @@
+/**
+ * Entry point for this package.  Provides public API and types.
+ *
+ * @module
+ */
+
+import { transformAst } from './transform-ast.js';
+import { parseAst } from './parse-ast.js';
+import { generate } from './generate.js';
+
+/**
+ * Options for {@link evadeCensor}
+ *
+ * @typedef EvadeCensorOptions
+ * @property {string|import('source-map').RawSourceMap} [sourceMap] - Original source map in JSON string or object form
+ * @property {string} [sourceUrl] - URL or filepath of the original source in `code`
+ * @property {boolean} [useLocationUnmap] - Enable location unmapping. Only applies if `sourceMap` was provided
+ * @property {import('./parse-ast.js').SourceType} [sourceType] - Module source type
+ * @public
+ */
+
+/**
+ * Apply SES censorship evasion transforms on the given code `source`
+ *
+ * If the `sourceUrl` option is provided, the `map` property of the fulfillment
+ * value will be a source map object; otherwise it will be `undefined`.
+ *
+ * If the `sourceMap` option is _not_ provided, the `useLocationUnmap` option
+ * will have no effect.
+ *
+ * @template {EvadeCensorOptions} T
+ * @param {string} source - Source code to transform
+ * @param {T} [options] - Options for the transform
+ * @returns {Promise<import('./generate.js').TransformedResult<T['sourceUrl']>>} Object containing new code and optionally source map object (ready for stringification)
+ * @public
+ */
+export async function evadeCensor(source, options) {
+  const { sourceMap, sourceUrl, useLocationUnmap, sourceType } = options ?? {};
+
+  // See "Chesterton's Fence"
+  await null;
+
+  // Parse the rolled-up chunk with Babel.
+  // We are prepared for different module systems.
+  const ast = parseAst(source, {
+    sourceType,
+  });
+
+  const sourceMapJson =
+    typeof sourceMap === 'string' ? sourceMap : JSON.stringify(sourceMap);
+
+  if (sourceMap && useLocationUnmap) {
+    await transformAst(ast, { sourceMap: sourceMapJson, useLocationUnmap });
+  } else {
+    await transformAst(ast);
+  }
+
+  if (sourceUrl) {
+    return generate(ast, { sourceUrl });
+  }
+  return generate(ast);
+}

packages/evasive-transform/src/location-unmapper.js

@@ -0,0 +1,81 @@
+/**
+ * Provides {@link makeLocationUnmapper}
+ *
+ * @module
+ */
+
+import { SourceMapConsumer } from 'source-map';
+
+/**
+ * A function which modifies an AST Node's source location
+ *
+ * @callback LocationUnmapper
+ * @param {import('@babel/types').SourceLocation|null} [loc]
+ * @returns {void}
+ * @internal
+ */
+
+/**
+ * Creates a {@link LocationUnmapper} function
+ *
+ * @internal
+ * @param {string} sourceMap - Source map
+ * @param {import('@babel/types').File} ast - AST as created by Babel
+ * @returns {Promise<LocationUnmapper>}
+ */
+export async function makeLocationUnmapper(sourceMap, ast) {
+  if (!sourceMap) {
+    throw new TypeError('Invalid arguments; expected sourceMap');
+  }
+  if (!ast || typeof ast !== 'object') {
+    throw new TypeError('Invalid arguments; expected AST ast');
+  }
+  try {
+    // We rearrange the rolled-up chunk according to its sourcemap to move
+    // its source lines back to the right place.
+    return await SourceMapConsumer.with(sourceMap, null, async consumer => {
+      if (!ast.loc) {
+        throw new TypeError('No SourceLocation found in AST');
+      }
+      const unmapped = new WeakSet();
+      /**
+       * Change this type to `import('@babel/types').Position` if we assign the
+       * `index` prop below
+       * @type {any}
+       */
+      let lastPos = {
+        ...ast.loc.start,
+      };
+      return loc => {
+        if (!loc || unmapped.has(loc)) {
+          return;
+        }
+        // Make sure things start at least at the right place.
+        loc.end = { ...loc.start };
+        for (const pos of /** @type {const} */ (['start', 'end'])) {
+          if (loc[pos]) {
+            const newPos = consumer.originalPositionFor(loc[pos]);
+            if (newPos.source !== null) {
+              // This assumes that if source is non-null, then line and column are
+              // also non-null
+              lastPos = {
+                line: /** @type {number} */ (newPos.line),
+                column: /** @type {number} */ (newPos.column),
+                // XXX: what of the `index` prop?
+              };
+            }
+            loc[pos] = lastPos;
+          }
+        }
+        unmapped.add(loc);
+      };
+    });
+  } catch (err) {
+    // A source map string should be valid JSON, and if `JSON.parse()` fails, a
+    // SyntaxError is thrown
+    if (err instanceof SyntaxError) {
+      throw new TypeError(`Invalid source map: ${err}`);
+    }
+    throw err;
+  }
+}

packages/evasive-transform/src/parse-ast.js

@@ -0,0 +1,39 @@
+/**
+ * Provides {@link parseAst} adapter
+ *
+ * @module
+ */
+
+import babelParser from '@babel/parser';
+
+const { parse: parseBabel } = babelParser;
+
+/**
+ * This is the same type as `@babel/parser`'s `ParserOptions['sourceType']`, but
+ * re-implemented here for decoupling purposes.
+ *
+ * Still, this is likely Babel-specific.
+ *
+ * @typedef {'module'|'script'|'unambiguous'} SourceType
+ * @public
+ */
+
+/**
+ * Options for {@link parseAst}.
+ *
+ * @typedef ParseAstOptions
+ * @property {SourceType} [sourceType]
+ * @internal
+ */
+
+/**
+ * Adapter for parsing an AST.
+ *
+ * @param {string} source - Source code
+ * @param {ParseAstOptions} [opts] - Options for underlying parser
+ * @internal
+ */
+export function parseAst(source, opts) {
+  // Might not want to pass `opts` verbatim, but also might not matter!
+  return parseBabel(source, opts);
+}