feat: 🎸 add [TARDump]

packages/basics/package.json

@@ -5,7 +5,6 @@
   "author": "Nicolas Thouvenin <nthouvenin@gmail.com>",
   "bugs": "https://github.com/Inist-CNRS/ezs/issues",
   "dependencies": {
-    "JSONStream": "1.3.5",
     "async-retry": "1.3.3",
     "better-https-proxy-agent": "1.0.9",
     "csv-string": "3.2.0",
@@ -16,10 +15,12 @@
     "get-stream": "6.0.1",
     "higher-path": "1.0.0",
     "inflection": "2.0.1",
+    "JSONStream": "1.3.5",
     "lodash.escaperegexp": "4.1.2",
     "lodash.get": "4.4.2",
     "lodash.mapkeys": "4.6.0",
     "lodash.mapvalues": "4.6.0",
+    "lodash.merge": "4.6.2",
     "lodash.set": "4.3.2",
     "lodash.zipobject": "4.1.3",
     "make-dir": "4.0.0",
@@ -28,7 +29,7 @@
     "parse-headers": "2.0.4",
     "path-exists": "4.0.0",
     "stream-write": "1.0.1",
-    "tar-stream": "3.1.4",
+    "tar-stream": "3.1.6",
     "tmp-filepath": "2.0.0",
     "unzipper": "0.10.11",
     "xml-mapping": "1.7.2",

packages/basics/src/index.js

@@ -26,6 +26,7 @@ import URLConnect from './url-connect';
 import TXTZip from './txt-zip';
 import ZIPExtract from './zip-extract';
 import TARExtract from './tar-extract';
+import TARDump from './tar-dump';
 import INIString from './ini-string';
 import FILESave from './file-save';
 import FILELoad from './file-load';
@@ -59,6 +60,7 @@ const funcs = {
     TXTZip,
     ZIPExtract,
     TARExtract,
+    TARDump,
     INIString,
     FILESave,
     FILELoad,

packages/basics/src/tar-dump.js

@@ -0,0 +1,40 @@
+import tar from 'tar-stream';
+import { createGzip } from 'zlib';
+import merge from 'lodash.merge';
+
+const eol = '\n';
+
+/**
+ * Take all recevied objects and build a tar file
+ *
+ * ```json
+ * {
+ * }
+ * ```
+ *
+ * @name TARDump
+ * @param {String} [manifest] Location path to store files in the tarball
+ * @param {String} [location=data] Location path to store files in the tarball
+ * @param {String} [json=true] Parse as JSON the content of each file
+ * @param {Boolean} [compress=false] Enable gzip compression
+ */
+export default function TARDump(data, feed) {
+    if (!this.pack) {
+        this.pack = tar.pack();
+        const compress = this.getParam('compress', false);
+        const d = new Date();
+        const manifest = merge({ created: d.toUTCString() }, [].concat(this.getParam('manifest', [])).filter(Boolean));
+        this.pack.entry({ name: 'manifest.json' }, JSON.stringify(manifest, null, '  '));
+        const stream = compress ? this.pack.pipe(createGzip()) : this.pack;
+        feed.flow(stream).finally(() => feed.close());
+    }
+    if (this.isLast()) {
+        this.pack.finalize();
+        return;
+    }
+    const id = this.getIndex().toString().padStart(10, '0');
+    const value = JSON.stringify(data).concat(eol);
+    const location = this.getParam('location', 'data');
+    this.pack.entry({ name: `${location}/f${id}.json` }, value);
+    feed.end();
+}

packages/basics/test/tar-dump.js

@@ -0,0 +1,116 @@
+const assert = require('assert');
+const from = require('from');
+const ezs = require('../../core/src');
+
+ezs.use(require('../src'));
+
+describe('TARDump', () => {
+    const input = [
+        { a: 1, b: true, c: 'un' },
+        { a: 2, b: true, c: 'deux' },
+        { a: 3, b: true, c: 'trois' },
+    ];
+    it('should dump and extract few objects (tmpfile)', (done) => {
+        const result = [];
+        const script = `
+        [TARDump]
+
+        [FILESave]
+        identifier = test-a.tar
+        location = /tmp
+
+        [exchange]
+        value = get('filename')
+
+        [FILELoad]
+        location = /tmp
+
+        [TARExtract]
+        path = data/*.json
+        `;
+        from(input)
+            .pipe(ezs('delegate', { script }))
+            .pipe(ezs.catch())
+            .on('error', done)
+            .on('data', (obj) => {
+                assert.equal(obj.b, true);
+                result.push(obj);
+            })
+            .on('end', () => {
+                assert.equal(result.length, 3);
+                done();
+            });
+    });
+
+    it('should dump and extract few objects (stream)', (done) => {
+        const result = [];
+        const script = `
+        [TARDump]
+
+        [TARExtract]
+        path = data/*.json
+        `;
+        from(input)
+            .pipe(ezs('delegate', { script }))
+            .pipe(ezs.catch())
+            .on('error', done)
+            .on('data', (obj) => {
+                assert.equal(obj.b, true);
+                result.push(obj);
+            })
+            .on('end', () => {
+                assert.equal(result.length, 3);
+                done();
+            });
+    });
+
+    it('should dump and extract few objects (stream & custom path)', (done) => {
+        const result = [];
+        const script = `
+        [TARDump]
+        location = another/pathname
+
+        [TARExtract]
+        path = another/pathname/*.json
+        `;
+        from(input)
+            .pipe(ezs('delegate', { script }))
+            .pipe(ezs.catch())
+            .on('error', done)
+            .on('data', (obj) => {
+                assert.equal(obj.b, true);
+                result.push(obj);
+            })
+            .on('end', () => {
+                assert.equal(result.length, 3);
+                done();
+            });
+    });
+
+    it('should dump and extract few objects (stream & compress)', (done) => {
+        const result = [];
+        const script = `
+        [TARDump]
+        compress = true
+
+        [TARExtract]
+        compress = true
+        path = data/*.json
+        `;
+        from(input)
+            .pipe(ezs('delegate', { script }))
+            .pipe(ezs.catch())
+            .on('error', done)
+            .on('data', (obj) => {
+                assert.equal(obj.b, true);
+                result.push(obj);
+            })
+            .on('end', () => {
+                assert.equal(result.length, 3);
+                done();
+            });
+    });
+
+
+
+});