From 52165854b72aee41f7472f7a314bff3a396956bb Mon Sep 17 00:00:00 2001 From: KaKa Date: Fri, 29 Mar 2024 18:35:16 +0800 Subject: [PATCH 01/14] feat: replace into-stream to Readable.from --- index.js | 11 ++++++----- lib/utils.js | 10 +++++++++- package.json | 4 ++-- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/index.js b/index.js index 7343f98..b52285d 100644 --- a/index.js +++ b/index.js @@ -7,12 +7,13 @@ const fp = require('fastify-plugin') const encodingNegotiator = require('@fastify/accept-negotiator') const pump = require('pump') const mimedb = require('mime-db') -const intoStream = require('into-stream') const peek = require('peek-stream') const { Minipass } = require('minipass') const pumpify = require('pumpify') +const { Readable } = require('readable-stream') -const { isStream, isGzip, isDeflate } = require('./lib/utils') + +const { isStream, isGzip, isDeflate, intoAsyncIterator } = require('./lib/utils') const InvalidRequestEncodingError = createError('FST_CP_ERR_INVALID_CONTENT_ENCODING', 'Unsupported Content-Encoding: %s', 415) const InvalidRequestCompressedPayloadError = createError('FST_CP_ERR_INVALID_CONTENT', 'Could not decompress the request payload using the provided encoding', 400) @@ -276,7 +277,7 @@ function buildRouteCompress (fastify, params, routeOptions, decorateOnly) { if (Buffer.byteLength(payload) < params.threshold) { return next() } - payload = intoStream(payload) + payload = Readable.from(intoAsyncIterator(payload)) } setVaryHeader(reply) @@ -400,7 +401,7 @@ function compress (params) { if (Buffer.byteLength(payload) < params.threshold) { return this.send(payload) } - payload = intoStream(payload) + payload = Readable.from(intoAsyncIterator(payload)) } setVaryHeader(this) @@ -509,7 +510,7 @@ function maybeUnzip (payload, serialize) { // handle case where serialize doesn't return a string or Buffer if (!Buffer.isBuffer(buf)) return result if (isCompressed(buf) === 0) return result - return intoStream(result) + return Readable.from(intoAsyncIterator(result)) } function zipStream (deflate, encoding) { diff --git a/lib/utils.js b/lib/utils.js index 599d4c8..2c8a3a6 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -35,4 +35,12 @@ function isStream (stream) { return stream !== null && typeof stream === 'object' && typeof stream.pipe === 'function' } -module.exports = { isGzip, isDeflate, isStream } +// It is a helper used to provide a async iteratable for +// Readable.from +async function *intoAsyncIterator (payload) { + // string | Buffer + yield payload +} + + +module.exports = { isGzip, isDeflate, isStream, intoAsyncIterator } diff --git a/package.json b/package.json index cd02d9c..f4be429 100644 --- a/package.json +++ b/package.json @@ -8,12 +8,12 @@ "dependencies": { "@fastify/accept-negotiator": "^1.1.0", "fastify-plugin": "^4.5.0", - "into-stream": "^6.0.0", "mime-db": "^1.52.0", "minipass": "^7.0.2", "peek-stream": "^1.1.3", "pump": "^3.0.0", - "pumpify": "^2.0.1" + "pumpify": "^2.0.1", + "readable-stream": "^4.5.2" }, "devDependencies": { "@fastify/pre-commit": "^2.0.2", From 58e1260a38be1de43a36ab404bd0b678c946edee Mon Sep 17 00:00:00 2001 From: KaKa Date: Fri, 29 Mar 2024 18:38:44 +0800 Subject: [PATCH 02/14] test: regression test of issue 288 --- package.json | 3 ++- test/issue-288.test.js | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 test/issue-288.test.js diff --git a/package.json b/package.json index f4be429..1cfb423 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "peek-stream": "^1.1.3", "pump": "^3.0.0", "pumpify": "^2.0.1", - "readable-stream": "^4.5.2" + "readable-stream": "^4.5.2", + "undici": "^5.28.3" }, "devDependencies": { "@fastify/pre-commit": "^2.0.2", diff --git a/test/issue-288.test.js b/test/issue-288.test.js new file mode 100644 index 0000000..a53cc4c --- /dev/null +++ b/test/issue-288.test.js @@ -0,0 +1,42 @@ +'use strict' + +const { test } = require('tap') +const Fastify = require('fastify') +const fastifyCompress = require('..') +const { fetch } = require('undici') + +test('should not corrupt the file content', async (t) => { + // provide 2 byte unicode content + const twoByteUnicodeContent = new Array(5_000) + .fill('0') + .map(() => { + const random = new Array(10).fill('A').join('🍃') + return random + '- FASTIFY COMPRESS,🍃 FASTIFY COMPRESS' + }) + .join('\n') + const fastify = new Fastify() + t.teardown(() => fastify.close()) + + fastify.register(async (instance, opts) => { + await fastify.register(fastifyCompress) + // compression + instance.get('/issue', async (req, reply) => { + return twoByteUnicodeContent + }) + }) + + // no compression + fastify.get('/good', async (req, reply) => { + return twoByteUnicodeContent + }) + + await fastify.listen({ port: 0 }) + + const { port } = fastify.server.address() + const url = `http://localhost:${port}` + const response = await fetch(`${url}/issue`) + const response2 = await fetch(`${url}/good`) + const body = await response.text() + const body2 = await response2.text() + t.equal(body, body2) +}) \ No newline at end of file From 2e5810a735e5a4aa150586fea78d463ed13777f7 Mon Sep 17 00:00:00 2001 From: KaKa Date: Fri, 29 Mar 2024 18:40:52 +0800 Subject: [PATCH 03/14] fixup --- index.js | 1 - lib/utils.js | 3 +-- test/issue-288.test.js | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index b52285d..cd17201 100644 --- a/index.js +++ b/index.js @@ -12,7 +12,6 @@ const { Minipass } = require('minipass') const pumpify = require('pumpify') const { Readable } = require('readable-stream') - const { isStream, isGzip, isDeflate, intoAsyncIterator } = require('./lib/utils') const InvalidRequestEncodingError = createError('FST_CP_ERR_INVALID_CONTENT_ENCODING', 'Unsupported Content-Encoding: %s', 415) diff --git a/lib/utils.js b/lib/utils.js index 2c8a3a6..bede437 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -37,10 +37,9 @@ function isStream (stream) { // It is a helper used to provide a async iteratable for // Readable.from -async function *intoAsyncIterator (payload) { +async function * intoAsyncIterator (payload) { // string | Buffer yield payload } - module.exports = { isGzip, isDeflate, isStream, intoAsyncIterator } diff --git a/test/issue-288.test.js b/test/issue-288.test.js index a53cc4c..b4b6898 100644 --- a/test/issue-288.test.js +++ b/test/issue-288.test.js @@ -39,4 +39,4 @@ test('should not corrupt the file content', async (t) => { const body = await response.text() const body2 = await response2.text() t.equal(body, body2) -}) \ No newline at end of file +}) From 9b4a0e471df383b5b716abd34fa1398b1a65b61e Mon Sep 17 00:00:00 2001 From: KaKa Date: Fri, 29 Mar 2024 18:58:10 +0800 Subject: [PATCH 04/14] feat: support more types --- lib/utils.js | 31 +++++++++++++++++++++++++++++++ test/utils.test.js | 39 ++++++++++++++++++++++++++++++++++++++- types/index.d.ts | 21 ++++++++++++++------- 3 files changed, 83 insertions(+), 8 deletions(-) diff --git a/lib/utils.js b/lib/utils.js index bede437..852ee97 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -38,6 +38,37 @@ function isStream (stream) { // It is a helper used to provide a async iteratable for // Readable.from async function * intoAsyncIterator (payload) { + const isBuffer = Buffer.isBuffer(payload) + + if ( + ( + // ArrayBuffer + payload instanceof ArrayBuffer || + // NodeJS.TypedArray + ArrayBuffer.isView(payload) + ) && + // Exclude Buffer to prevent double cast + !isBuffer + ) { + payload = Buffer.from(payload) + } + + // Iterator + if (typeof payload === 'object' && typeof payload[Symbol.iterator] === 'function' && !isBuffer) { + for (const chunk of payload) { + yield chunk + } + return + } + + // Async Iterator + if (typeof payload === 'object' && typeof payload[Symbol.asyncIterator] === 'function' && !isBuffer) { + for await (const chunk of payload) { + yield chunk + } + return + } + // string | Buffer yield payload } diff --git a/test/utils.test.js b/test/utils.test.js index 6507c2e..627ea5b 100644 --- a/test/utils.test.js +++ b/test/utils.test.js @@ -4,7 +4,7 @@ const { createReadStream } = require('node:fs') const { Socket } = require('node:net') const { Duplex, PassThrough, Readable, Stream, Transform, Writable } = require('node:stream') const { test } = require('tap') -const { isStream, isDeflate, isGzip } = require('../lib/utils') +const { isStream, isDeflate, isGzip, intoAsyncIterator } = require('../lib/utils') test('isStream() utility should be able to detect Streams', async (t) => { t.plan(12) @@ -61,3 +61,40 @@ test('isGzip() utility should be able to detect gzip compressed Buffer', async ( t.equal(isGzip(undefined), false) t.equal(isGzip(''), false) }) + +test('intoAsyncIterator() utility should handle different data', async (t) => { + t.plan(15) + + const buf = Buffer.from('foo') + const str = 'foo' + const arr = [str, str] + const arrayBuffer = new ArrayBuffer(8) + const typedArray = new Int32Array(arrayBuffer) + const asyncIterator = (async function* () { + yield str; + })(); + + for await (const buffer of intoAsyncIterator(buf)) { + t.equal(buffer, buf) + } + + for await (const string of intoAsyncIterator(str)) { + t.equal(string, str) + } + + for await (const chunk of intoAsyncIterator(arr)) { + t.equal(chunk, str) + } + + for await (const chunk of intoAsyncIterator(arrayBuffer)) { + t.equal(chunk, 0) + } + + for await (const chunk of intoAsyncIterator(typedArray)) { + t.equal(chunk, 0) + } + + for await (const chunk of intoAsyncIterator(asyncIterator)) { + t.equal(chunk, str) + } +}) diff --git a/types/index.d.ts b/types/index.d.ts index fd1fe6b..24bcbe0 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -2,13 +2,12 @@ import { FastifyPluginCallback, FastifyReply, FastifyRequest, + RouteOptions as FastifyRouteOptions, RawServerBase, - RawServerDefault, - RouteOptions as FastifyRouteOptions -} from 'fastify' -import { Input, InputObject } from 'into-stream' -import { Stream } from 'stream' -import { BrotliOptions, ZlibOptions } from 'zlib' + RawServerDefault +} from 'fastify'; +import { Stream } from 'stream'; +import { BrotliOptions, ZlibOptions } from 'zlib'; declare module 'fastify' { export interface FastifyContextConfig { @@ -26,7 +25,7 @@ declare module 'fastify' { } interface FastifyReply { - compress(input: Stream | Input | InputObject): void; + compress(input: Stream | Input): void; } export interface RouteOptions { @@ -61,6 +60,14 @@ type EncodingToken = 'br' | 'deflate' | 'gzip' | 'identity'; type CompressibleContentTypeFunction = (contentType: string) => boolean; +type Input = + | Buffer + | NodeJS.TypedArray + | ArrayBuffer + | string + | Iterable + | AsyncIterable; + declare namespace fastifyCompress { export interface FastifyCompressOptions { From db10fb43b5afa745244887f55ed9b6ddc951e598 Mon Sep 17 00:00:00 2001 From: KaKa Date: Fri, 29 Mar 2024 18:59:09 +0800 Subject: [PATCH 05/14] chore: fix deps --- package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 1cfb423..6726127 100644 --- a/package.json +++ b/package.json @@ -13,8 +13,7 @@ "peek-stream": "^1.1.3", "pump": "^3.0.0", "pumpify": "^2.0.1", - "readable-stream": "^4.5.2", - "undici": "^5.28.3" + "readable-stream": "^4.5.2" }, "devDependencies": { "@fastify/pre-commit": "^2.0.2", @@ -27,7 +26,8 @@ "standard": "^17.1.0", "tap": "^16.3.7", "tsd": "^0.30.0", - "typescript": "^5.1.6" + "typescript": "^5.1.6", + "undici": "^6.10.2" }, "scripts": { "coverage": "npm run test:unit -- --coverage-report=html", From a610972887c5922d33b1e5abda0de6cc29c7bda7 Mon Sep 17 00:00:00 2001 From: KaKa Date: Fri, 29 Mar 2024 18:59:38 +0800 Subject: [PATCH 06/14] fixup --- test/utils.test.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/utils.test.js b/test/utils.test.js index 627ea5b..44b124a 100644 --- a/test/utils.test.js +++ b/test/utils.test.js @@ -64,15 +64,15 @@ test('isGzip() utility should be able to detect gzip compressed Buffer', async ( test('intoAsyncIterator() utility should handle different data', async (t) => { t.plan(15) - + const buf = Buffer.from('foo') const str = 'foo' const arr = [str, str] const arrayBuffer = new ArrayBuffer(8) const typedArray = new Int32Array(arrayBuffer) - const asyncIterator = (async function* () { - yield str; - })(); + const asyncIterator = (async function * () { + yield str + })() for await (const buffer of intoAsyncIterator(buf)) { t.equal(buffer, buf) From fbacb29f4eff48d5704a30a6c5a35d2f4016ddce Mon Sep 17 00:00:00 2001 From: KaKa Date: Fri, 29 Mar 2024 19:01:44 +0800 Subject: [PATCH 07/14] fixup --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6726127..7d78ba4 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "tap": "^16.3.7", "tsd": "^0.30.0", "typescript": "^5.1.6", - "undici": "^6.10.2" + "undici": "^5.28.3" }, "scripts": { "coverage": "npm run test:unit -- --coverage-report=html", From 9d2685b77fc93d8b9ccac1c932c598757626d458 Mon Sep 17 00:00:00 2001 From: KaKa Date: Fri, 29 Mar 2024 19:07:30 +0800 Subject: [PATCH 08/14] fixup --- test/issue-288.test.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/issue-288.test.js b/test/issue-288.test.js index b4b6898..c3d4462 100644 --- a/test/issue-288.test.js +++ b/test/issue-288.test.js @@ -3,7 +3,12 @@ const { test } = require('tap') const Fastify = require('fastify') const fastifyCompress = require('..') -const { fetch } = require('undici') +const { fetch, setGlobalDispatcher, Agent } = require('undici') + +setGlobalDispatcher(new Agent({ + keepAliveTimeout: 10, + keepAliveMaxTimeout: 10 +})) test('should not corrupt the file content', async (t) => { // provide 2 byte unicode content From 6091169262b572cad12427a44ab788edafdf37de Mon Sep 17 00:00:00 2001 From: KaKa Date: Fri, 29 Mar 2024 19:11:11 +0800 Subject: [PATCH 09/14] fixup --- test/issue-288.test.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/issue-288.test.js b/test/issue-288.test.js index c3d4462..8c53ea5 100644 --- a/test/issue-288.test.js +++ b/test/issue-288.test.js @@ -3,7 +3,7 @@ const { test } = require('tap') const Fastify = require('fastify') const fastifyCompress = require('..') -const { fetch, setGlobalDispatcher, Agent } = require('undici') +const { request, setGlobalDispatcher, Agent } = require('undici') setGlobalDispatcher(new Agent({ keepAliveTimeout: 10, @@ -39,9 +39,10 @@ test('should not corrupt the file content', async (t) => { const { port } = fastify.server.address() const url = `http://localhost:${port}` - const response = await fetch(`${url}/issue`) - const response2 = await fetch(`${url}/good`) - const body = await response.text() - const body2 = await response2.text() + + const response = await request(`${url}/issue`) + const response2 = await request(`${url}/good`) + const body = await response.body.text() + const body2 = await response2.body.text() t.equal(body, body2) }) From 6d63b9b918e6cf9cd8f0f0f58d0900ecc13b98bc Mon Sep 17 00:00:00 2001 From: KaKa Date: Fri, 29 Mar 2024 21:59:48 +0800 Subject: [PATCH 10/14] refactor: update checking condition --- lib/utils.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/utils.js b/lib/utils.js index 852ee97..8ee2955 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -41,20 +41,20 @@ async function * intoAsyncIterator (payload) { const isBuffer = Buffer.isBuffer(payload) if ( - ( + !isBuffer && ( // ArrayBuffer payload instanceof ArrayBuffer || // NodeJS.TypedArray ArrayBuffer.isView(payload) - ) && - // Exclude Buffer to prevent double cast - !isBuffer + ) ) { payload = Buffer.from(payload) } + const isObject = typeof payload === 'object' + // Iterator - if (typeof payload === 'object' && typeof payload[Symbol.iterator] === 'function' && !isBuffer) { + if (!isBuffer && isObject && Symbol.iterator in payload) { for (const chunk of payload) { yield chunk } @@ -62,7 +62,7 @@ async function * intoAsyncIterator (payload) { } // Async Iterator - if (typeof payload === 'object' && typeof payload[Symbol.asyncIterator] === 'function' && !isBuffer) { + if (!isBuffer && isObject && Symbol.asyncIterator in payload) { for await (const chunk of payload) { yield chunk } From 81bb72ce17fca983d659a4895a3a285b7a509647 Mon Sep 17 00:00:00 2001 From: KaKa Date: Fri, 29 Mar 2024 22:05:41 +0800 Subject: [PATCH 11/14] fixup --- lib/utils.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/utils.js b/lib/utils.js index 8ee2955..a277957 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -41,11 +41,12 @@ async function * intoAsyncIterator (payload) { const isBuffer = Buffer.isBuffer(payload) if ( - !isBuffer && ( - // ArrayBuffer + !isBuffer && + ( + // ArrayBuffer payload instanceof ArrayBuffer || - // NodeJS.TypedArray - ArrayBuffer.isView(payload) + // NodeJS.TypedArray + ArrayBuffer.isView(payload) ) ) { payload = Buffer.from(payload) From ce46b6732cb9eef03068c04496cbb733aa8a1b50 Mon Sep 17 00:00:00 2001 From: KaKa <23028015+climba03003@users.noreply.github.com> Date: Fri, 29 Mar 2024 22:27:35 +0800 Subject: [PATCH 12/14] chore: apply suggestion Co-authored-by: Aras Abbasi Signed-off-by: KaKa <23028015+climba03003@users.noreply.github.com> --- lib/utils.js | 52 +++++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/lib/utils.js b/lib/utils.js index a277957..37e5fb1 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -35,42 +35,44 @@ function isStream (stream) { return stream !== null && typeof stream === 'object' && typeof stream.pipe === 'function' } -// It is a helper used to provide a async iteratable for -// Readable.from +/** + * Provide a async iteratable for Readable.from + */ async function * intoAsyncIterator (payload) { - const isBuffer = Buffer.isBuffer(payload) - - if ( - !isBuffer && - ( + if (typeof payload === 'object') { + if (Buffer.isBuffer(payload)) { + yield payload + return + } + + if ( // ArrayBuffer payload instanceof ArrayBuffer || // NodeJS.TypedArray ArrayBuffer.isView(payload) - ) - ) { - payload = Buffer.from(payload) - } - - const isObject = typeof payload === 'object' + ) { + yield Buffer.from(payload) + return + } - // Iterator - if (!isBuffer && isObject && Symbol.iterator in payload) { - for (const chunk of payload) { - yield chunk + // Iterator + if (Symbol.iterator in payload) { + for (const chunk of payload) { + yield chunk + } + return } - return - } - // Async Iterator - if (!isBuffer && isObject && Symbol.asyncIterator in payload) { - for await (const chunk of payload) { - yield chunk + // Async Iterator + if (Symbol.asyncIterator in payload) { + for await (const chunk of payload) { + yield chunk + } + return } - return } - // string | Buffer + // string yield payload } From ea1951b6474f3cb1a916a828dc2f82c285a56917 Mon Sep 17 00:00:00 2001 From: KaKa Date: Fri, 29 Mar 2024 22:28:34 +0800 Subject: [PATCH 13/14] fixup --- lib/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/utils.js b/lib/utils.js index 37e5fb1..2f265fe 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -44,7 +44,7 @@ async function * intoAsyncIterator (payload) { yield payload return } - + if ( // ArrayBuffer payload instanceof ArrayBuffer || From 1664f002b5925474d330b06c3d821cdde7ca707a Mon Sep 17 00:00:00 2001 From: KaKa Date: Fri, 29 Mar 2024 22:55:30 +0800 Subject: [PATCH 14/14] fixup --- test/utils.test.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/utils.test.js b/test/utils.test.js index 44b124a..6c37465 100644 --- a/test/utils.test.js +++ b/test/utils.test.js @@ -63,7 +63,7 @@ test('isGzip() utility should be able to detect gzip compressed Buffer', async ( }) test('intoAsyncIterator() utility should handle different data', async (t) => { - t.plan(15) + t.plan(8) const buf = Buffer.from('foo') const str = 'foo' @@ -73,6 +73,7 @@ test('intoAsyncIterator() utility should handle different data', async (t) => { const asyncIterator = (async function * () { yield str })() + const obj = {} for await (const buffer of intoAsyncIterator(buf)) { t.equal(buffer, buf) @@ -87,14 +88,18 @@ test('intoAsyncIterator() utility should handle different data', async (t) => { } for await (const chunk of intoAsyncIterator(arrayBuffer)) { - t.equal(chunk, 0) + t.equal(chunk.toString(), Buffer.from(arrayBuffer).toString()) } for await (const chunk of intoAsyncIterator(typedArray)) { - t.equal(chunk, 0) + t.equal(chunk.toString(), Buffer.from(typedArray).toString()) } for await (const chunk of intoAsyncIterator(asyncIterator)) { t.equal(chunk, str) } + + for await (const chunk of intoAsyncIterator(obj)) { + t.equal(chunk, obj) + } })