fix: 🐛 missing string decoder

Inist-CNRS · Sep 29, 2023 · b50f1a2 · b50f1a2
1 parent e9461e1
commit b50f1a2
Show file tree

Hide file tree

Showing 4 changed files with 60 additions and 6 deletions.
diff --git a/packages/basics/src/csv-parse.js b/packages/basics/src/csv-parse.js
@@ -11,9 +11,13 @@ function CSVParse(data, feed) {
         this.whenFinish = feed.flow(this.input);
     }
     if (this.isLast()) {
-        this.decoder.end();
+        writeTo(
+            this.input,
+            this.decoder.end(),
+            () => this.input.end(),
+        );
         this.whenFinish.finally(() => feed.close());
-        return this.input.end();
+        return ;
     }
     writeTo(
         this.input,

diff --git a/packages/basics/src/txt-parse.js b/packages/basics/src/txt-parse.js
@@ -3,14 +3,16 @@ import { StringDecoder } from 'string_decoder';
 function TXTParse(data, feed) {
     if (!this.decoder) {
         this.decoder = new StringDecoder('utf8');
+        this.remainder = '';
+        this.counter = 0;
     }
     if (this.isLast()) {
-        this.decoder.end();
+        this.remainder += this.decoder.end();
+        if (this.remainder && this.counter > 1) {
+            feed.write(this.remainder);
+        }
         return feed.end();
     }
-
-    this.remainder = this.remainder || '';
-
     let separator;
     try {
         const val = '"'.concat(this.getParam('separator', '\n')).concat('"');
@@ -32,6 +34,7 @@ function TXTParse(data, feed) {
     lines.forEach((line) => {
         feed.write(line);
     });
+    this.counter += lines.length;
     feed.end();
 }
 

diff --git a/packages/basics/test/basics.js b/packages/basics/test/basics.js
@@ -38,6 +38,31 @@ describe('test', () => {
                 done();
             });
     });
+    it('CSVParse #2', (done) => {
+        const res = [];
+        from([
+            'a,b,c\nd,',
+            'e,f\ng,',
+            Buffer.from([0xE2]),
+            Buffer.from([0x82]),
+            Buffer.from([0xAC]),
+            Buffer.from(','),
+            Buffer.from([0xC2]),
+            Buffer.from([0xA2]),
+
+        ])
+            .pipe(ezs('CSVParse'))
+            .on('data', (chunk) => {
+                assert(typeof chunk === 'object');
+                res.push(chunk);
+            })
+            .on('end', () => {
+                assert.equal(res.length, 3);
+                assert.equal(res[2][1], '€');
+                assert.equal(res[2][2], '¢');
+                done();
+            });
+    });
 
     it('CSVString#1', (done) => {
         const res = [];

diff --git a/packages/basics/test/txt-parse.js b/packages/basics/test/txt-parse.js
@@ -81,6 +81,28 @@ describe('TXTParse', () => {
                 done();
             });
     });
+    it('should generate with a separator parameter and multi bytes char', (done) => {
+        let res = [];
+        from([
+            'a*b*',
+            Buffer.from([0xE2]),
+            Buffer.from([0x82]),
+            Buffer.from([0xAC]),
+            '*',
+            Buffer.from([0xC2]),
+            Buffer.from([0xA2]),
+        ])
+            .pipe(ezs('TXTParse', { separator: '*' }))
+            .on('data', (data) => {
+                res = [...res, data];
+            })
+            .on('end', () => {
+                expect(res).toStrictEqual(['a', 'b', '€', '¢']);
+                done();
+            });
+    });
+
+
 
     it('should not generate with a tab separator', (done) => {
         let res = [];