Test node n web

node-usfm-parser/src/usfmGenerator.js

@@ -14,7 +14,6 @@ class USFMGenerator {
         this.usfmString += "+";
       }
       this.usfmString += `${usjObj.marker} `;
-
     }
     ["code", "number", "caller"].forEach((key) => {
       if (usjObj[key]) {
@@ -38,9 +37,6 @@ class USFMGenerator {
         this.usfmString += `\\vp ${usjObj.pubnumber} \\vp* `
       }
     }
-    if (usjObj.category) {
-      this.usfmString += `\\cat ${usjObj.category}\\cat*\n`;
-    }
     if (Array.isArray(usjObj.content)) {
       usjObj.content.forEach((item) => {
         if (typeof item === "string") {

web-usfm-parser/package.json

@@ -11,7 +11,8 @@
   },
   "source": "src/index.js",
   "scripts": {
-    "build": "parcel build src/index.js"
+    "build": "parcel build src/index.js",
+    "test": "mocha --parallel --timeout 40000"
   },
   "files": [
     "dist/",
@@ -40,6 +41,9 @@
     "parcel": "latest",
     "path-browserify": "^1.0.1",
     "process": "^0.11.10",
-    "web-tree-sitter": "^0.22.6"
+    "web-tree-sitter": "^0.22.6",
+    "glob": "^11.0.0",
+    "mocha": "^10.7.3",
+    "xml2js": "^0.6.2"
   }
 }

web-usfm-parser/src/usfmGenerator.js

@@ -5,7 +5,9 @@ class USFMGenerator {
   }
 
   usjToUsfm(usjObj, nested = false) {
-
+    if (usjObj.type === "ref") {
+        usjObj.marker = "ref";
+    }
     if (!NO_USFM_USJ_TYPES.includes(usjObj.type)) {
       this.usfmString += "\\";
       if (nested && usjObj.type === "char") {
@@ -21,6 +23,20 @@ class USFMGenerator {
     if (usjObj.category) {
       this.usfmString += `\\cat ${usjObj.category}\\cat*\n`;
     }
+    if (usjObj.altnumber) {
+      if (usjObj.marker === "c") {
+        this.usfmString += `\\ca ${usjObj.altnumber} \\ca*\n`
+      }else if (usjObj.marker === "v") {
+        this.usfmString += `\\va ${usjObj.altnumber} \\va* `
+      }
+    }
+    if (usjObj.pubnumber) {
+      if (usjObj.marker === "c") {
+        this.usfmString += `\\cp ${usjObj.pubnumber}\n`
+      }else if (usjObj.marker === "v") {
+        this.usfmString += `\\vp ${usjObj.pubnumber} \\vp* `
+      }
+    }
     if (Array.isArray(usjObj.content)) {
       usjObj.content.forEach((item) => {
         if (typeof item === "string") {

web-usfm-parser/src/usjGenerator.js

@@ -8,7 +8,7 @@ class USJGenerator {
     this.usfm = usfmString;
     this.jsonRootObj = usjRootObj || {
       type: "USJ",
-      version: "0.3.0",
+      version: "3.1",
       content: [],
     };
   }
@@ -89,15 +89,15 @@ class USJGenerator {
       sid: chapRef,
     };
 
-    chapCap.forEach((tuple) => {
-      if (tuple[1] === "alt-num") {
+    chapCap.forEach((cap) => {
+      if (cap.name === "alt-num") {
         chapJsonObj.altnumber = this.usfm
-          .substring(tuple[0].startIndex, tuple[0].endIndex)
+          .substring(cap.node.startIndex, cap.node.endIndex)
           .trim();
       }
-      if (tuple[1] === "pub-num") {
+      if (cap.name === "pub-num") {
         chapJsonObj.pubnumber = this.usfm
-          .substring(tuple[0].startIndex, tuple[0].endIndex)
+          .substring(cap.node.startIndex, cap.node.endIndex)
           .trim();
       }
     });
@@ -206,7 +206,7 @@ class USJGenerator {
       const paraMarker = paraTagCap.node.type;
 
       if (paraMarker === "b") {
-        this.nodeToUSJSpecial(paraTagCap, parentJsonObj);
+        parentJsonObj.content.push( { type: "para", marker: paraMarker} );
       } else if (!paraMarker.endsWith("Block")) {
         const paraJsonObj = { type: "para", marker: paraMarker, content: [] };
         paraTagCap.node.children.forEach((child) => {
@@ -394,7 +394,7 @@ class USJGenerator {
         .query("((category) @category)")
         .captures(node)[0];
       const category = this.usfm
-        .substring(catCap[0].startIndex, catCap[0].endIndex)
+        .substring(catCap.node.startIndex, catCap.node.endIndex)
         .trim();
       parentJsonObj.category = category;
     } else if (node.type === "fig") {
@@ -403,17 +403,12 @@ class USJGenerator {
         this.nodeToUSJ(child, figJsonObj);
       });
       parentJsonObj.content.push(figJsonObj);
-    } else if (node.type === "b") {
-      const bJsonObj = { type: "optbreak", marker: "b" };
-      parentJsonObj.content.push(bJsonObj);
-    } else if (node.type === "usfm") {
-      const verJsonObj = { type: "para", marker: "usfm", content: [] };
-      const version = this.usfm
-        .substring(node.startIndex, node.endIndex)
-        .replace("\\usfm", "")
-        .trim();
-      verJsonObj.content.push(version);
-      parentJsonObj.content.push(verJsonObj);
+    } else if (node.type === "ref") {
+      const refJsonObj = { type: "ref", content: [] };
+      node.children.slice(1, -1).forEach((child) => {
+        this.nodeToUSJ(child, refJsonObj);
+      });
+      parentJsonObj.content.push(refJsonObj);
     }
   }
   nodeToUSJGeneric(node, parentJsonObj) {
@@ -496,9 +491,10 @@ class USJGenerator {
         this.nodeToUSJPara(node, parentJsonObj);
         break;
       case "text":
-        const textVal = this.usfm
+        let textVal = this.usfm
           .substring(node.startIndex, node.endIndex)
           .trim();
+        textVal = textVal.replace("~", " ")
         if (textVal !== "") {
           parentJsonObj.content.push(textVal);
         }
@@ -514,17 +510,24 @@ class USJGenerator {
       case "esb":
       case "cat":
       case "fig":
-      case "usfm":
+      case "ref":
         this.nodeToUSJSpecial(node, parentJsonObj);
         break;
+      case "usfm":
+        break;
       default:
-        if (
+        if (NOTE_MARKERS.includes(node.type)) {
+          this.nodeToUSJNotes(node, parentJsonObj)
+        }
+        else if (
           CHAR_STYLE_MARKERS.includes(node.type) ||
           NESTED_CHAR_STYLE_MARKERS.includes(node.type) ||
           ["xt_standalone"].includes(node.type)
         ) {
           this.nodeToUSJChar(node, parentJsonObj);
-        } else if (node.type.endsWith("Attribute")) {
+        } else if (TABLE_CELL_MARKERS.includes(node.type)) {
+          this.nodeToUSJTable(node, parentJsonObj)
+        }else if (node.type.endsWith("Attribute")) {
           this.nodeToUSJAttrib(node, parentJsonObj);
         } else if (
           PARA_STYLE_MARKERS.includes(node.type) ||

web-usfm-parser/src/utils/markers.js

@@ -112,9 +112,13 @@ export const NESTED_CHAR_STYLE_MARKERS = CHAR_STYLE_MARKERS.map(
 export const DEFAULT_ATTRIB_MAP = {
   w: "lemma",
   rb: "gloss",
-  xt: "link-href",
+  xt: "href",
   fig: "alt",
-  xt_standalone: "link-href",
+  xt_standalone: "href",
+  xtNested: "href",
+  ref: "loc",
+  "milestone": "who",
+  "k":"key"
 };
 export const TABLE_CELL_MARKERS = ["tc", "th", "tcr", "thr"];
 export const MISC_MARKERS = ["fig", "cat", "esb", "b", "ph", "pi"];

web-usfm-parser/test/basic.js

@@ -0,0 +1,110 @@
+// const assert = require('assert');
+// const {USFMParser} = require("../src/index");
+import assert from 'assert'
+import {USFMParser} from '../src/index.js';
+
+const simpleUSFM = '\\id GEN\n\\c 1\n\\p\n\\v 1 In the begining..\\v 2';
+const simpleUSJ = {
+  type: 'USJ',
+  version: '0.3.0',
+  content: [
+    { type: 'book', marker: 'id', code: 'GEN', content: [] },
+    { type: 'chapter', marker: 'c', number: '1', sid: 'GEN 1' },
+    { type: 'para', marker: 'p', content: [
+        {type: 'verse', marker: 'v', number: 1 },
+        "In the begining..",
+        {type: 'verse', marker: 'v', number: 2 }
+      ] }
+  ]
+}
+describe("Sanity Check for the testing pipeline", () => {
+
+  it("Parse, toUSJ and back toUSFM", async () => {
+    await USFMParser.init("./tree-sitter-usfm.wasm", "./tree-sitter.wasm");
+    const usfmParser = new USFMParser(simpleUSFM);
+    const output = usfmParser.toUSJ()
+    assert.notStrictEqual(output, null, 'The result should not be null and no errors during conversion');
+
+    const usfm = usfmParser.usjToUsfm(output)
+    assert.notStrictEqual(usfm, null, 'The result should not be null and no errors during conversion');
+
+
+  });
+});
+
+describe("USFMParser Object initialization", () => {
+
+  it("with USFM", async () => {
+    await USFMParser.init("./tree-sitter-usfm.wasm", "./tree-sitter.wasm");
+    const usfmParser = new USFMParser(simpleUSFM)
+    assert.strictEqual(usfmParser.usfm, simpleUSFM) 
+
+  });
+
+  it("with USJ", async () => {
+    const usfmParser = new USFMParser(null, simpleUSJ)
+    assert.strictEqual(usfmParser.usj, simpleUSJ) 
+
+  });
+
+  it("with nothing", async () => {
+    await USFMParser.init("./tree-sitter-usfm.wasm", "./tree-sitter.wasm");
+    let usfmParser = null;
+    try {
+      const usfmParser = new USFMParser()
+
+    } catch(err) {
+      assert.strictEqual(err.message, "Missing input! Either USFM, USJ or USX is to be provided.")
+    }
+    assert.strictEqual(usfmParser, null);
+  });
+
+  it("with usfm and usj", async () => {
+    await USFMParser.init("./tree-sitter-usfm.wasm", "./tree-sitter.wasm");
+    let usfmParser = null;
+    try {
+      const usfmParser = new USFMParser(simpleUSFM, simpleUSJ)
+
+    } catch(err) {
+      assert.strictEqual(err.message, `Found more than one input!
+Only one of USFM, USJ or USX is supported in one object.` )
+   }
+    assert.strictEqual(usfmParser, null);
+  });
+
+  it("with usj in place of USFM", async () => {
+    await USFMParser.init("./tree-sitter-usfm.wasm", "./tree-sitter.wasm");
+    let usfmParser = null;
+    try {
+      const usfmParser = new USFMParser(simpleUSJ)
+
+    } catch(err) {
+      assert.strictEqual(err.message, "Invalid input for USFM. Expected a string.")
+    }
+    assert.strictEqual(usfmParser, null);
+  });
+
+  it("with usfm in place of USJ", async () => {
+    await USFMParser.init("./tree-sitter-usfm.wasm", "./tree-sitter.wasm");
+    let usfmParser = null;
+    try {
+      const usfmParser = new USFMParser(null, simpleUSJ)
+
+    } catch(err) {
+      assert.strictEqual(err.message, "Invalid input for USJ. Expected an object.")
+    }
+    assert.strictEqual(usfmParser, null);
+  });
+
+  it("with usj as default", async () => {
+    await USFMParser.init("./tree-sitter-usfm.wasm", "./tree-sitter.wasm");
+    let usfmParser = null;
+    try {
+      const usfmParser = new USFMParser(simpleUSJ)
+
+    } catch(err) {
+      assert.strictEqual(err.message, "Invalid input for USFM. Expected a string.")
+    }
+    assert.strictEqual(usfmParser, null);
+  });
+});