Add tests for CLI mode

.github/workflows/check-on-push.yml

@@ -26,6 +26,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Setup node and npm
         uses: actions/setup-node@v1
+        with:
+          node-version: 13.12.0
       - name: install packages
         run: npm install .
       # Runs the Linter action

.github/workflows/npm-publish.yml

@@ -14,7 +14,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v1
         with:
-          node-version: 12
+          node-version: 13.12.0
       - run: npm ci
       - run: npm run test-on-github
 
@@ -25,7 +25,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v1
         with:
-          node-version: 12
+          node-version: 13.12.0
           registry-url: https://registry.npmjs.org/
       - run: npm ci
       - run: npm publish

cli.js

@@ -13,8 +13,6 @@ const { argv } = require('yargs')
   .choices('filter', ['scripture'])
   .describe('format', 'specifies the output file format')
   .choices('format', ['csv', 'tsv', 'usfm', 'json'])
-  .alias('o', 'output')
-  .describe('o', 'specify the fully qualified file path for output.')
   .alias('h', 'help')
   .alias('v', 'version')
   .help('help');
@@ -73,18 +71,4 @@ if (argv.format === 'usfm' || isJson) {
   }
 }
 
-if (Object.prototype.hasOwnProperty.call(argv, 'o') || Object.prototype.hasOwnProperty.call(argv, 'output')) {
-  let filePath = argv.o;
-  if (!filePath) {
-    filePath = argv.output;
-  }
-  try {
-    fs.writeFileSync(filePath, output);
-  } catch (e) {
-    console.error('Error writting output file');
-    console.error(e.message);
-    process.exit(1);
-  }
-} else {
-  console.log(output);
-}
+console.log(output);

js/USFMparser.js

@@ -137,13 +137,21 @@ class USFMParser extends Parser {
 
   toCSV() {
     const jsonOutput = this.toJSON();
+    if (Object.keys(jsonOutput).includes('_messages')
+      && Object.keys(jsonOutput._messages).includes('_error')) {
+      return jsonOutput;
+    }
     const myJsonParser = new JSONParser(jsonOutput);
     const csvOutput = myJsonParser.toCSV();
     return csvOutput;
   }
 
   toTSV() {
     const jsonOutput = this.toJSON();
+    if (Object.keys(jsonOutput).includes('_messages')
+      && Object.keys(jsonOutput._messages).includes('_error')) {
+      return jsonOutput;
+    }
     const myJsonParser = new JSONParser(jsonOutput);
     const csvOutput = myJsonParser.toTSV();
     return csvOutput;

js/grammarOperations-relaxed.js

@@ -1,5 +1,6 @@
 const ohm = require('ohm-js');
 const { contents: grammar } = require('../grammar/usfm-relaxed.ohm.js');
+const { buildVerseText } = require('./grammarOperations.js');
 
 const { usfmRelaxed: bib } = ohm.grammars(grammar);
 const sem = bib.createSemantics();
@@ -11,15 +12,15 @@ const sem = bib.createSemantics();
 // We need to know which all marker's contents should be considered as verseText
 // while composing the .verseText property of each verse element.
 // This list is consulted for that
-const verseCarryingMarkers = ['li', 'li1', 'li2', 'li3', 'lh', 'lf', 'lim', 'litl',
-  'lik', 'liv', 'liv1', 'liv2', 'liv3', 'th', 'th1', 'th2', 'th3',
-  'thr', 'thr1', 'thr2', 'thr3', 'tc', 'tc1', 'tc2', 'tc3', 'tcr',
-  'tcr1', 'tcr2', 'tcr3', 'add', 'bk', 'dc', 'k', 'lit', 'nd', 'ord',
-  'pn', 'png', 'addpn', 'qt', 'sig', 'sls', 'tl', 'wj', 'em', 'bd',
-  'it', 'bdit', 'no', 'sc', 'sup', 'w', 'rb', 'wa', 'wg', 'wh', 'pro',
-  '+add', '+bk', '+dc', '+k', '+lit', '+nd', '+ord', '+pn', '+png',
-  '+addpn', '+qt', '+sig', '+sls', '+tl', '+wj', '+em', '+bd', '+it',
-  '+bdit', '+no', '+sc', '+sup', '+w', '+rb', '+wa', '+wg', '+wh', '+pro'];
+// const verseCarryingMarkers = ['li', 'li1', 'li2', 'li3', 'lh', 'lf', 'lim', 'litl',
+//   'lik', 'liv', 'liv1', 'liv2', 'liv3', 'th', 'th1', 'th2', 'th3',
+//   'thr', 'thr1', 'thr2', 'thr3', 'tc', 'tc1', 'tc2', 'tc3', 'tcr',
+//   'tcr1', 'tcr2', 'tcr3', 'add', 'bk', 'dc', 'k', 'lit', 'nd', 'ord',
+//   'pn', 'png', 'addpn', 'qt', 'sig', 'sls', 'tl', 'wj', 'em', 'bd',
+//   'it', 'bdit', 'no', 'sc', 'sup', 'w', 'rb', 'wa', 'wg', 'wh', 'pro',
+//   '+add', '+bk', '+dc', '+k', '+lit', '+nd', '+ord', '+pn', '+png',
+//   '+addpn', '+qt', '+sig', '+sls', '+tl', '+wj', '+em', '+bd', '+it',
+//   '+bdit', '+no', '+sc', '+sup', '+w', '+rb', '+wa', '+wg', '+wh', '+pro'];
 
 // In normal grammar these markers are implemented as not containing text or other contents.
 // The relaxed grammar doesnot implement makers separately but have general rules for all.
@@ -30,7 +31,7 @@ const paraMarkers = ['p', 'm', 'po', 'pr', 'cls', 'pmo', 'pm', 'pmc',
   'ph3', 'b', 'q', 'q1', 'q2', 'q3', 'qr', 'qc', 'qs', 'qa', 'qac', 'qm',
   'qm1', 'qm2', 'qm3'];
 
-const punctPattern = new RegExp('^[,./;:\'"`~!@#$%^&*(){}[}|]');
+// const punctPattern = new RegExp('^[,./;:\'"`~!@#$%^&*(){}[}|]');
 
 sem.addOperation('buildJson', {
   File(bookhead, chapters) {
@@ -105,42 +106,7 @@ sem.addOperation('buildJson', {
       verseNumber: num.sourceString.trim(),
       contents: contents.buildJson(),
     };
-    res.verseText = '';
-    for (let i = 0; i < res.contents.length; i += 1) {
-      if (typeof res.contents[i] === 'string') {
-        if (punctPattern.test(res.contents[i])) {
-          res.verseText = res.verseText.trim();
-          res.verseText += res.contents[i];
-        } else {
-          res.verseText += ` ${res.contents[i]}`;
-        }
-      } else {
-        // console.log(res.contents[i].keys());
-        const key = Object.keys(res.contents[i])[0];
-        if (verseCarryingMarkers.includes(key)) {
-          if (punctPattern.test(res.contents[i][key])) {
-            res.verseText = res.verseText.trim();
-            res.verseText += res.contents[i][key];
-          } else {
-            res.verseText += ` ${res.contents[i][key]}`;
-          }
-        } else if (paraMarkers.includes(key)) {
-          const text = res.contents[i][key];
-          if (punctPattern.test(text)) {
-            res.verseText = res.verseText.trim();
-            res.verseText += text;
-          } else {
-            res.verseText += ` ${text}`;
-          }
-          res.contents[i][key] = null;
-          if (text !== '') {
-            res.contents.splice(i + 1, 0, text);
-            i += 1;
-          }
-        }
-      }
-    }
-    res.verseText = res.verseText.replace(/ +/g, ' ').trim();
+    res.verseText = buildVerseText(res.contents).replace(/ +/g, ' ').trim();
     return res;
   },
 

js/grammarOperations.js

@@ -25,17 +25,27 @@ const verseCarryingMarkers = ['li', 'li1', 'li2', 'li3', 'lh', 'lf', 'lim', 'lit
   '+addpn', '+qt', '+sig', '+sls', '+tl', '+wj', '+em', '+bd', '+it',
   '+bdit', '+no', '+sc', '+sup', '+w', '+rb', '+wa', '+wg', '+wh', '+pro'];
 
-// const paraMarkers = ['p', 'm', 'po', 'pr', 'cls', 'pmo', 'pm', 'pmc',
-//   'pmr', 'pi', 'pi1', 'pi2', 'pi3', 'mi', 'nb', 'pc', 'ph', 'ph1', 'ph2',
-//   'ph3', 'b', 'q', 'q1', 'q2', 'q3', 'qr', 'qc', 'qs', 'qa', 'qac', 'qm',
-//   'qm1', 'qm2', 'qm3'];
+const paraMarkers = ['p', 'm', 'po', 'pr', 'cls', 'pmo', 'pm', 'pmc',
+  'pmr', 'pi', 'pi1', 'pi2', 'pi3', 'mi', 'nb', 'pc', 'ph', 'ph1', 'ph2',
+  'ph3', 'b', 'q', 'q1', 'q2', 'q3', 'qr', 'qc', 'qs', 'qa', 'qac', 'qm',
+  'qm1', 'qm2', 'qm3'];
 
 const punctPattern = new RegExp('^[,./;:\'"`~!@#$%^&*(){}[}|]');
 
 function buildVerseText(elmts) {
   let verseTextPartial = '';
   if (typeof elmts === 'string') {
     verseTextPartial = elmts;
+  } else if (Array.isArray(elmts)) {
+    for (let i = 0; i < elmts.length; i += 1) {
+      const text = buildVerseText(elmts[i]);
+      if (punctPattern.test(text)) {
+        verseTextPartial = verseTextPartial.trim();
+        verseTextPartial += text;
+      } else {
+        verseTextPartial += ` ${text}`;
+      }
+    }
   } else {
     const key = Object.keys(elmts)[0];
     if (verseCarryingMarkers.includes(key) && typeof elmts[key] === 'string') {
@@ -70,6 +80,8 @@ function buildVerseText(elmts) {
       if (elmts.text !== null) {
         verseTextPartial = elmts.text;
       }
+    } else if (paraMarkers.includes(key) && elmts[key] !== null) {
+      verseTextPartial = buildVerseText(elmts[key]);
     }
   }
   return verseTextPartial;
@@ -210,26 +222,15 @@ sem.addOperation('composeJson', {
     verse.contents = [];
     if (verseMeta.sourceString !== '') {
       const metadata = verseMeta.composeJson();
-      verse.contents.concat(metadata);
+      verse.contents = verse.contents.concat(metadata);
     }
     const elmts = verseContent.composeJson();
     if (verseContent.sourceString === '') {
       emitter.emit('warning', new Error(`Verse text is empty, at \\v ${verseNumber.sourceString}. `));
+    } else {
+      verse.contents = verse.contents.concat(elmts);
     }
-    for (let i = 0; i < elmts.length; i += 1) {
-      const text = buildVerseText(elmts[i]);
-      if (punctPattern.test(text)) {
-        verse.verseText = verse.verseText.trim();
-        verse.verseText += text;
-      } else {
-        verse.verseText += ` ${text}`;
-      }
-      if (Object.prototype.hasOwnProperty.call(elmts[i], 'table')) {
-        delete elmts[i].text;
-      }
-      verse.contents.push(elmts[i]);
-    }
-    verse.verseText = verse.verseText.trim().replace(/ +/g, ' ');
+    verse.verseText = buildVerseText(elmts).trim().replace(/ +/g, ' ');
     return verse;
   },
 
@@ -1219,3 +1220,4 @@ function match(str) {
 }
 
 exports.match = match;
+exports.buildVerseText = buildVerseText;

package.json

@@ -7,7 +7,7 @@
         "test": "mocha --expose-gc --timeout 40000",
         "start": "node ../server.js",
         "lint": "eslint .",
-        "test-on-github": "mocha --expose-gc --timeout 40000 --ignore test/test_filesFromWild.js"
+        "test-on-github": "mocha --expose-gc --timeout 40000 --ignore test/test_filesFromWild.js --ignore test/test_cli.js"
     },
     "dependencies": {
         "jsonschema": "^1.4.0",
@@ -34,6 +34,7 @@
     },
     "homepage": "https://github.com/Bridgeconn/usfm-grammar",
     "devDependencies": {
+        "concat-stream": "^2.0.0",
         "eslint": "^7.7.0",
         "eslint-config-airbnb-base": "^14.2.0",
         "eslint-plugin-import": "^2.22.0",

test/resources/small.json

@@ -0,0 +1,40 @@
+{
+  "book": {
+    "bookCode": "GEN",
+    "description": "A small sample usfm file"
+  },
+  "chapters": [
+    {
+      "chapterNumber": "1",
+      "contents": [
+        {
+          "p": null
+        },
+        {
+          "verseNumber": "1",
+          "verseText": "one verse",
+          "contents": [
+            "one verse"
+          ]
+        },
+        {
+          "verseNumber": "2",
+          "verseText": "second verse",
+          "contents": [
+            "second verse"
+          ]
+        },
+        {
+          "verseNumber": "3",
+          "verseText": "final verse",
+          "contents": [
+            "final verse"
+          ]
+        }
+      ]
+    }
+  ],
+  "_messages": {
+    "_warnings": []
+  }
+}