js: semgrep js pattern testing

.github/workflows/build-test-javascript.jsonnet

@@ -87,8 +87,7 @@ local build_job =
         with: {
           'retention-days': 1,
           path: |||
-            _build/default/js/engine/*.bc.js
-            _build/default/js/languages/*/*.bc.js
+            _build/default/js/**/*.bc.js
           |||,
           name: build_artifact_name,
         },
@@ -130,7 +129,18 @@ local test_job = {
       name: 'Build JS artifacts',
       run: |||
         make -C js -j $(nproc) build
-
+      |||
+    },
+    {
+      name: 'Test JS artifacts',
+      run: |||
+        make -C js -j $(nproc) test
+        make -C js/tests
+      |||
+    },
+    {
+      name: 'Package JS artifacts',
+      run:|||
         tar cvzf semgrep-js-artifacts.tar.gz \
           js/engine/dist/index.cjs \
           js/engine/dist/index.mjs \
@@ -148,11 +158,7 @@ local test_job = {
         'retention-days': 2,
         name: artifact_name,
       },
-    },
-    {
-      name: 'Run semgrep js e2e tests',
-      run: 'make -C js test',
-    },
+    }
   ],
 };
 

.github/workflows/build-test-javascript.yml

@@ -37,8 +37,7 @@ jobs:
     - uses: actions/upload-artifact@v3
       with:
         retention-days: 1
-        path: "\n            _build/default/js/engine/*.bc.js\n            _build/default/js/languages/*/*.bc.js\n
-          \         "
+        path: "\n            _build/default/js/**/*.bc.js\n          "
         name: semgrep-js-ocaml-build-${{ github.sha }}
     container: returntocorp/ocaml:alpine-2023-10-12
     env:
@@ -75,19 +74,22 @@ jobs:
       with:
         node-version: "18"
     - name: Build JS artifacts
-      run: "\n        make -C js -j $(nproc) build\n\n        tar cvzf semgrep-js-artifacts.tar.gz
-        \\\n          js/engine/dist/index.cjs \\\n          js/engine/dist/index.mjs
-        \\\n          js/engine/dist/semgrep-engine.wasm \\\n          js/languages/*/dist/index.cjs
-        \\\n          js/languages/*/dist/index.mjs \\\n          js/languages/*/dist/semgrep-parser.wasm\n
-        \     "
+      run: "\n        make -C js -j $(nproc) build\n      "
+    - name: Test JS artifacts
+      run: "\n        make -C js -j $(nproc) test\n      "
+    - name: Run JS e2e tests
+      run: "\n        make -C js/tests\n      "
+    - name: Package JS artifacts
+      run: "\n        tar cvzf semgrep-js-artifacts.tar.gz \\\n          js/engine/dist/index.cjs
+        \\\n          js/engine/dist/index.mjs \\\n          js/engine/dist/semgrep-engine.wasm
+        \\\n          js/languages/*/dist/index.cjs \\\n          js/languages/*/dist/index.mjs
+        \\\n          js/languages/*/dist/semgrep-parser.wasm\n      "
     - name: Upload JS artifacts
       uses: actions/upload-artifact@v3
       with:
         path: semgrep-js-artifacts.tar.gz
         retention-days: 2
         name: semgrep-js-artifacts-${{ github.sha }}
-    - name: Run semgrep js e2e tests
-      run: make -C js test
   upload:
     needs:
     - test

js/Makefile

@@ -13,8 +13,8 @@ test:
 	cd libpcre; $(MAKE) test
 	cd libyaml; $(MAKE) test
 	cd engine; $(MAKE) test
-	cd languages; $(MAKE) test # These are easily parallelizable
-	#cd tests; $(MAKE) # This is really slow for some reason :(
+	cd languages; $(MAKE) test
+	cd tests; $(MAKE)
 
 .PHONY: e2e
 e2e: engine languages

js/engine/Main.ml

@@ -1,46 +1,16 @@
 open Js_of_ocaml
-open Semgrep_js_shared
 
 let _ =
   Js.export_all
     (object%js
        (*
           The following methods are used internally by js/engine/src/index.js.
         *)
-       method init = init_jsoo
-       method getMountpoints = get_jsoo_mountpoint ()
-
-       method setParsePattern (func : jbool -> jstring -> jstring -> 'a) =
-         Parse_pattern.parse_pattern_ref :=
-           fun print_error lang pattern ->
-             match lang with
-             (* The Yaml and JSON parsers are embedded in the engine because it's a
-                core component needed to parse rules *)
-             | Lang.Yaml -> Yaml_to_generic.any pattern
-             | _ ->
-                 func (Js.bool print_error)
-                   (Js.string (Lang.to_lowercase_alnum lang))
-                   (Js.string pattern)
-
-       method setJustParseWithLang
-           (func : jstring -> jstring -> Parsing_result2.t) =
-         Parse_target.just_parse_with_lang_ref :=
-           fun lang filename ->
-             match lang with
-             (* The Yaml and JSON parsers are embedded in the engine because it's a
-                core component needed to parse rules *)
-             | Lang.Yaml ->
-                 {
-                   ast = Yaml_to_generic.program filename;
-                   errors = [];
-                   skipped_tokens = [];
-                   inserted_tokens = [];
-                   stat = Parsing_stat.default_stat filename;
-                 }
-             | _ ->
-                 func
-                   (Js.string (Lang.to_lowercase_alnum lang))
-                   (Js.string filename)
+       method init = Semgrep_js_shared.init_jsoo
+       method getMountpoints = Semgrep_js_shared.get_jsoo_mountpoint ()
+       method setParsePattern = Semgrep_js_shared.setParsePattern
+       method setJustParseWithLang = Semgrep_js_shared.setJustParseWithLang
+       method setJsonnetParser = Semgrep_js_shared.setJsonnetParser
 
        (*
           The following methods are part of the engine's public API.
@@ -98,5 +68,5 @@ let _ =
            in
            Semgrep_output_v1_j.string_of_core_output res
          in
-         wrap_with_js_error execute
+         Semgrep_js_shared.wrap_with_js_error execute
     end)

js/engine/Makefile

@@ -1,10 +1,4 @@
-EMCC_DEFAULTS = \
-   -sALLOW_MEMORY_GROWTH=1 \
-   -sEXPORTED_RUNTIME_METHODS=UTF8ToString,stringToUTF8,stringToAscii,lengthBytesUTF8,getValue,setValue \
-   -sMODULARIZE
 
-include ../libyaml/Makefile.include
-include ../libpcre/Makefile.include
 
 ESBUILD_DEFAULTS = \
 	--bundle \
@@ -15,6 +9,8 @@ ESBUILD_DEFAULTS = \
 .PHONY: default
 default: build
 
+include Makefile.include
+
 .PHONY: build
 build: dist/index.cjs dist/index.mjs dist/index.d.ts
 
@@ -34,21 +30,6 @@ clean:
 distclean:
 	rm -rf dist node_modules
 
-../libyaml/dist/libyaml.o:
-	cd ../libyaml; $(MAKE) dist/libyaml.o
-
-../libpcre/dist/libpcre.a:
-	cd ../libpcre; $(MAKE) dist/libpcre.a
-
-dist/semgrep-engine.js dist/semgrep-engine.wasm: ../libyaml/dist/libyaml.o ../libpcre/dist/libpcre.a
-	mkdir -p dist
-	emcc \
-		-O3 \
-		$^ \
-		$(EMCC_DEFAULTS) \
-		-sEXPORTED_FUNCTIONS=_malloc,_free,$(YAML_EXPORTED_METHODS),$(PCRE_EXPORTED_METHODS) \
-		-o $(word 1,$@)
-
 package-lock.json:
 	npm install
 

js/engine/Makefile.include

@@ -0,0 +1,22 @@
+EMCC_DEFAULTS = \
+   -sALLOW_MEMORY_GROWTH=1 \
+   -sEXPORTED_RUNTIME_METHODS=UTF8ToString,stringToUTF8,stringToAscii,lengthBytesUTF8,getValue,setValue,intArrayFromString,writeArrayToMemory \
+   -sMODULARIZE
+
+include ../libyaml/Makefile.include
+include ../libpcre/Makefile.include
+
+../libyaml/dist/libyaml.o:
+	$(MAKE) -C ../libyaml dist/libyaml.o
+
+../libpcre/dist/libpcre.a:
+	$(MAKE) -C ../libpcre dist/libpcre.a
+
+dist/semgrep-engine.js dist/semgrep-engine.wasm: ../libyaml/dist/libyaml.o ../libpcre/dist/libpcre.a
+	mkdir -p dist
+	emcc \
+		-O3 \
+		$^ \
+		$(EMCC_DEFAULTS) \
+		-sEXPORTED_FUNCTIONS=_malloc,_free,$(YAML_EXPORTED_METHODS),$(PCRE_EXPORTED_METHODS) \
+		-o $(word 1,$@)

js/engine/src/index.js

@@ -25,6 +25,7 @@ export const EngineFactory = async (wasmUri) => {
     getMountpoints,
     setParsePattern,
     setJustParseWithLang,
+    setJsonnetParser,
     execute,
     lookupLang,
     writeFile,
@@ -65,6 +66,14 @@ export const EngineFactory = async (wasmUri) => {
         languages.set(lang, parser);
         missingLanguages.delete(lang);
       });
+      // The Semgrep core engine eagerly uses the Jsonnet parser
+      // because we support it as a format to write rules in.
+      // Unlike other languages, there is no pure-OCaml (e.g. Pfff)
+      // option that we can use instead, so we need to explicitly
+      // pass a reference of the Jsonnet parser to the engine.
+      if (parser.getLangs().includes("jsonnet")) {
+        setJsonnetParser((file) => parser.parseTargetTsOnly(file));
+      }
     },
     hasParser: (lang) => languages.has(lang),
     isMissingLanguages: () => missingLanguages.size > 0,

js/engine/tests/index.test.js

@@ -23,7 +23,7 @@ describe("engine", () => {
       "python",
       `${__dirname}/test-rule-python.json`,
       `${__dirname}`,
-      [`dirname}/../../languages/python/tests/test.py`]
+      [`${__dirname}/test.py`]
     );
     expect(engine.isMissingLanguages()).toBe(true);
     expect(engine.getMissingLanguages()).toEqual(["python"]);

js/languages/bash/tests/index.test.js

@@ -1,23 +1,10 @@
-const { ParserFactory } = require("../dist/index.cjs");
-
-const parserPromise = ParserFactory();
+const { createParser, testParser } = require("../../shared/parser");
 
 const LANG = "bash";
 const EXPECTED_LANGS = [LANG];
 
-test("getLangs", async () => {
-  const parser = await parserPromise;
-  expect(parser.getLangs()).toEqual(EXPECTED_LANGS);
-});
-
-test("it parses a pattern", async () => {
-  const parser = await parserPromise;
-  const pattern = parser.parsePattern(false, LANG, "echo $X");
-  expect(typeof pattern).toEqual("object");
-});
+describe(`${LANG} parser`, () => {
+  const parserPromise = createParser(`${__dirname}/../dist/index.cjs`);
 
-test("it parses a file", async () => {
-  const parser = await parserPromise;
-  const target = parser.parseTarget(LANG, "tests/test.sh");
-  expect(typeof target).toEqual("object");
+  testParser(LANG, parserPromise);
 });

js/languages/c/Parser.ml

@@ -1,6 +1,11 @@
 let parse_target _ file =
   Pfff_or_tree_sitter.run file
-    [ TreeSitter Parse_c_tree_sitter.parse ]
+    [
+      Pfff
+        (Pfff_or_tree_sitter.throw_tokens (fun file ->
+             Parse_c.parse (Fpath.v file)));
+      TreeSitter Parse_c_tree_sitter.parse;
+    ]
     C_to_generic.program
 
 let parse_pattern print_errors _ str =

js/languages/c/tests/index.test.js

@@ -1,23 +1,9 @@
-const { ParserFactory } = require("../dist/index.cjs");
-
-const parserPromise = ParserFactory();
+const { createParser, testParser } = require("../../shared/parser");
 
 const LANG = "c";
-const EXPECTED_LANGS = [LANG];
-
-test("getLangs", async () => {
-  const parser = await parserPromise;
-  expect(parser.getLangs()).toEqual(EXPECTED_LANGS);
-});
 
-test("it parses a pattern", async () => {
-  const parser = await parserPromise;
-  const pattern = parser.parsePattern(false, LANG, "printf(...)");
-  expect(typeof pattern).toEqual("object");
-});
+describe(`${LANG} parser`, () => {
+  const parserPromise = createParser(`${__dirname}/../dist/index.cjs`);
 
-test("it parses a file", async () => {
-  const parser = await parserPromise;
-  const target = parser.parseTarget(LANG, "tests/test.c");
-  expect(typeof target).toEqual("object");
+  testParser(LANG, parserPromise);
 });

js/languages/cairo/tests/index.test.js

@@ -1,24 +1,9 @@
-const { ParserFactory } = require("../dist/index.cjs");
-
-const parserPromise = ParserFactory();
+const { createParser, testParser } = require("../../shared/parser");
 
 const LANG = "cairo";
-const EXPECTED_LANGS = [LANG];
-
-test("getLangs", async () => {
-  const parser = await parserPromise;
-  expect(parser.getLangs()).toEqual(EXPECTED_LANGS);
-});
 
-// TODO
-// test("it parses a pattern", async () => {
-//   const parser = await parserPromise;
-//   const pattern = parser.parsePattern(false, LANG, "sum by (foo) $X");
-//   expect(typeof pattern).toEqual("object");
-// });
+describe(`${LANG} parser`, () => {
+  const parserPromise = createParser(`${__dirname}/../dist/index.cjs`);
 
-test("it parses a file", async () => {
-  const parser = await parserPromise;
-  const target = parser.parseTarget(LANG, "tests/test.cairo");
-  expect(typeof target).toEqual("object");
+  testParser(LANG, parserPromise);
 });

js/languages/cpp/tests/index.test.js

@@ -1,23 +1,9 @@
-const { ParserFactory } = require("../dist/index.cjs");
-
-const parserPromise = ParserFactory();
+const { createParser, testParser } = require("../../shared/parser");
 
 const LANG = "cpp";
-const EXPECTED_LANGS = [LANG];
-
-test("getLangs", async () => {
-  const parser = await parserPromise;
-  expect(parser.getLangs()).toEqual(EXPECTED_LANGS);
-});
 
-test("it parses a pattern", async () => {
-  const parser = await parserPromise;
-  const pattern = parser.parsePattern(false, LANG, "std::cout << $X");
-  expect(typeof pattern).toEqual("object");
-});
+describe(`${LANG} parser`, () => {
+  const parserPromise = createParser(`${__dirname}/../dist/index.cjs`);
 
-test("it parses a file", async () => {
-  const parser = await parserPromise;
-  const target = parser.parseTarget(LANG, "tests/test.cpp");
-  expect(typeof target).toEqual("object");
+  testParser(LANG, parserPromise);
 });