diff --git a/packages/basics/src/csv-parse.js b/packages/basics/src/csv-parse.js index c45cc6a7..6bb5dc79 100644 --- a/packages/basics/src/csv-parse.js +++ b/packages/basics/src/csv-parse.js @@ -11,9 +11,13 @@ function CSVParse(data, feed) { this.whenFinish = feed.flow(this.input); } if (this.isLast()) { - this.decoder.end(); + writeTo( + this.input, + this.decoder.end(), + () => this.input.end(), + ); this.whenFinish.finally(() => feed.close()); - return this.input.end(); + return ; } writeTo( this.input, diff --git a/packages/basics/src/txt-parse.js b/packages/basics/src/txt-parse.js index f9bf79f9..465f1997 100644 --- a/packages/basics/src/txt-parse.js +++ b/packages/basics/src/txt-parse.js @@ -3,14 +3,16 @@ import { StringDecoder } from 'string_decoder'; function TXTParse(data, feed) { if (!this.decoder) { this.decoder = new StringDecoder('utf8'); + this.remainder = ''; + this.counter = 0; } if (this.isLast()) { - this.decoder.end(); + this.remainder += this.decoder.end(); + if (this.remainder && this.counter > 1) { + feed.write(this.remainder); + } return feed.end(); } - - this.remainder = this.remainder || ''; - let separator; try { const val = '"'.concat(this.getParam('separator', '\n')).concat('"'); @@ -32,6 +34,7 @@ function TXTParse(data, feed) { lines.forEach((line) => { feed.write(line); }); + this.counter += lines.length; feed.end(); } diff --git a/packages/basics/test/basics.js b/packages/basics/test/basics.js index fc45df88..f63bc9e3 100644 --- a/packages/basics/test/basics.js +++ b/packages/basics/test/basics.js @@ -38,6 +38,31 @@ describe('test', () => { done(); }); }); + it('CSVParse #2', (done) => { + const res = []; + from([ + 'a,b,c\nd,', + 'e,f\ng,', + Buffer.from([0xE2]), + Buffer.from([0x82]), + Buffer.from([0xAC]), + Buffer.from(','), + Buffer.from([0xC2]), + Buffer.from([0xA2]), + + ]) + .pipe(ezs('CSVParse')) + .on('data', (chunk) => { + assert(typeof chunk === 'object'); + res.push(chunk); + }) + .on('end', () => { + assert.equal(res.length, 3); + assert.equal(res[2][1], '€'); + assert.equal(res[2][2], '¢'); + done(); + }); + }); it('CSVString#1', (done) => { const res = []; diff --git a/packages/basics/test/txt-parse.js b/packages/basics/test/txt-parse.js index 9c92cb2b..59899bfd 100644 --- a/packages/basics/test/txt-parse.js +++ b/packages/basics/test/txt-parse.js @@ -81,6 +81,28 @@ describe('TXTParse', () => { done(); }); }); + it('should generate with a separator parameter and multi bytes char', (done) => { + let res = []; + from([ + 'a*b*', + Buffer.from([0xE2]), + Buffer.from([0x82]), + Buffer.from([0xAC]), + '*', + Buffer.from([0xC2]), + Buffer.from([0xA2]), + ]) + .pipe(ezs('TXTParse', { separator: '*' })) + .on('data', (data) => { + res = [...res, data]; + }) + .on('end', () => { + expect(res).toStrictEqual(['a', 'b', '€', '¢']); + done(); + }); + }); + + it('should not generate with a tab separator', (done) => { let res = [];