Compile with WASI

.github/workflows/javascript-bindings.yml

@@ -0,0 +1,48 @@
+name: JavaScript Bindings
+
+on:
+  push:
+    paths:
+      - ".github/workflows/javascript-bindings.yml"
+      - "include/"
+      - "src/"
+      - "*akefile*"
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: head
+          bundler-cache: true
+
+      - name: rake templates
+        run: bundle exec rake templates
+
+      - name: Set up WASI-SDK
+        run: |
+          wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-linux.tar.gz
+          tar xvf wasi-sdk-20.0-linux.tar.gz
+
+      - name: Build the project
+        run: make wasm WASI_SDK_PATH=$(pwd)/wasi-sdk-20.0
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20.x
+
+      - name: Run the tests
+        run: npm test
+        working-directory: javascript
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: prism.wasm
+          path: javascript/src/prism.wasm

.gitignore

@@ -25,6 +25,10 @@ a.out
 /ext/prism/api_node.c
 /fuzz/output/
 /include/prism/ast.h
+/javascript/src/deserialize.js
+/javascript/src/nodes.js
+/javascript/src/prism.wasm
+/javascript/types/
 /java/org/prism/AbstractNodeVisitor.java
 /java/org/prism/Loader.java
 /java/org/prism/Nodes.java

Makefile

@@ -13,6 +13,7 @@ SOEXT := $(shell ruby -e 'puts RbConfig::CONFIG["SOEXT"]')
 CPPFLAGS := -Iinclude
 CFLAGS := -g -O2 -std=c99 -Wall -Werror -Wextra -Wpedantic -Wundef -Wconversion -fPIC -fvisibility=hidden
 CC := cc
+WASI_SDK_PATH := /opt/wasi-sdk
 
 HEADERS := $(shell find include -name '*.h')
 SOURCES := $(shell find src -name '*.c')
@@ -23,6 +24,7 @@ all: shared static
 
 shared: build/librubyparser.$(SOEXT)
 static: build/librubyparser.a
+wasm: javascript/src/prism.wasm
 
 build/librubyparser.$(SOEXT): $(SHARED_OBJECTS)
 	$(ECHO) "linking $@"
@@ -32,6 +34,10 @@ build/librubyparser.a: $(STATIC_OBJECTS)
 	$(ECHO) "building $@"
 	$(Q) $(AR) $(ARFLAGS) $@ $(STATIC_OBJECTS) $(Q1:0=>/dev/null)
 
+javascript/src/prism.wasm: Makefile $(SOURCES) $(HEADERS)
+	$(ECHO) "building $@"
+	$(Q) $(WASI_SDK_PATH)/bin/clang --sysroot=$(WASI_SDK_PATH)/share/wasi-sysroot/ $(DEBUG_FLAGS) -DPRISM_EXPORT_SYMBOLS -D_WASI_EMULATED_MMAN -lwasi-emulated-mman $(CPPFLAGS) $(CFLAGS) -Wl,--export-all -Wl,--no-entry -mexec-model=reactor -o $@ $(SOURCES)
+
 build/shared/%.o: src/%.c Makefile $(HEADERS)
 	$(ECHO) "compiling $@"
 	$(Q) mkdir -p $(@D)

docs/configuration.md

@@ -4,6 +4,8 @@ A lot of code in prism's repository is templated from a single configuration fil
 
 * `ext/prism/api_node.c` - for defining how to build Ruby objects for the nodes out of C structs
 * `include/prism/ast.h` - for defining the C structs that represent the nodes
+* `javascript/src/deserialize.js` - for defining how to deserialize the nodes in JavaScript
+* `javascript/src/nodes.js` - for defining the nodes in JavaScript
 * `java/org/prism/AbstractNodeVisitor.java` - for defining the visitor interface for the nodes in Java
 * `java/org/prism/Loader.java` - for defining how to deserialize the nodes in Java
 * `java/org/prism/Nodes.java` - for defining the nodes in Java

javascript/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "ruby-prism",
+  "version": "0.15.1",
+  "description": "Prism Ruby parser",
+  "type": "module",
+  "main": "src/index.js",
+  "scripts": {
+    "test": "node test.js",
+    "type": "tsc --allowJs -d --emitDeclarationOnly --outDir types src/index.js"
+  },
+  "author": "Shopify <ruby@shopify.com>",
+  "license": "MIT"
+}

javascript/src/index.js

@@ -0,0 +1,57 @@
+import { readFile } from "node:fs/promises";
+import url from "node:url";
+import { WASI } from "node:wasi";
+import { ParseResult, deserialize } from "./deserialize.js";
+
+/**
+ * The exports of the Prism wasm module.
+ *
+ * @type {WebAssembly.Exports | null}
+ */
+let prism = null;
+
+/**
+ * Load the Prism wasm module.
+ *
+ * @returns {Promise<WebAssembly.Exports>}
+ */
+async function loadPrism() {
+  const bytes = await readFile(url.fileURLToPath(new URL("prism.wasm", import.meta.url)));
+  const wasm = await WebAssembly.compile(bytes);
+
+  const wasi = new WASI({ version: "preview1" });
+  const instance = await WebAssembly.instantiate(wasm, wasi.getImportObject());
+  wasi.initialize(instance);
+
+  return instance.exports;
+}
+
+/**
+ * Parse the given source code.
+ *
+ * @param {string} source
+ * @returns {Promise<ParseResult>}
+ */
+export async function parse(source) {
+  if (prism === null) {
+    prism = await loadPrism();
+  }
+
+  const sourceArray = new TextEncoder().encode(source);
+  const sourcePointer = prism.calloc(1, sourceArray.length);
+
+  const bufferPointer = prism.calloc(prism.pm_buffer_sizeof(), 1);
+  prism.pm_buffer_init(bufferPointer);
+
+  const sourceView = new Uint8Array(prism.memory.buffer, sourcePointer, sourceArray.length);
+  sourceView.set(sourceArray);
+
+  prism.pm_parse_serialize(sourcePointer, sourceArray.length, bufferPointer);
+  const serializedView = new Uint8Array(prism.memory.buffer, prism.pm_buffer_value(bufferPointer), prism.pm_buffer_length(bufferPointer));
+  const result = deserialize(sourceArray, serializedView);
+
+  prism.pm_buffer_free(bufferPointer);
+  prism.free(sourcePointer);
+  prism.free(bufferPointer);
+  return result;
+}

javascript/test.js

@@ -0,0 +1,78 @@
+import test from "node:test";
+import assert from "node:assert";
+import { parse } from "./src/index.js";
+import * as nodes from "./src/nodes.js";
+
+test("node", async () => {
+  const result = await parse("foo");
+  assert(result.value instanceof nodes.ProgramNode);
+});
+
+test("node? present", async () => {
+  const result = await parse("foo.bar");
+  assert(result.value.statements.body[0].receiver instanceof nodes.CallNode);
+});
+
+test("node? absent", async () => {
+  const result = await parse("foo");
+  assert(result.value.statements.body[0].receiver === null);
+});
+
+test("node[]", async () => {
+  const result = await parse("foo.bar");
+  assert(result.value.statements.body instanceof Array);
+});
+
+test("string", async () => {
+  const result = await parse('"foo"');
+  assert(result.value.statements.body[0].unescaped === "foo");
+});
+
+test("constant", async () => {
+  const result = await parse("foo = 1");
+  assert(result.value.locals[0] === "foo");
+});
+
+test("constant? present", async () => {
+  const result = await parse("def foo(*bar); end");
+  assert(result.value.statements.body[0].parameters.rest.name === "bar");
+});
+
+test("constant? absent", async () => {
+  const result = await parse("def foo(*); end");
+  assert(result.value.statements.body[0].parameters.rest.name === null);
+});
+
+test("constant[]", async() => {
+  const result = await parse("foo = 1");
+  assert(result.value.locals instanceof Array);
+});
+
+test("location", async () => {
+  const result = await parse("foo = 1");
+  assert(typeof result.value.location.startOffset === "number");
+});
+
+test("location? present", async () => {
+  const result = await parse("def foo = bar");
+  assert(result.value.statements.body[0].equalLoc !== null);
+});
+
+test("location? absent", async () => {
+  const result = await parse("def foo; bar; end");
+  assert(result.value.statements.body[0].equalLoc === null);
+});
+
+test("uint32", async () => {
+  const result = await parse("foo = 1");
+  assert(result.value.statements.body[0].depth === 0);
+});
+
+test("flags", async () => {
+  const result = await parse("/foo/mi");
+  const regexp = result.value.statements.body[0];
+
+  assert(regexp.isIgnoreCase());
+  assert(regexp.isMultiLine());
+  assert(!regexp.isExtended());
+});

rakelib/check_manifest.rake

@@ -15,6 +15,7 @@ task :check_manifest => [:templates] do
     build
     doc
     fuzz
+    javascript
     java
     pkg
     rakelib