From 0691692de464ae049c6fa8e55039142d3e9e637e Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Thu, 2 Feb 2023 14:32:25 +0800 Subject: [PATCH 01/63] =?UTF-8?q?fix:=20=F0=9F=90=9B=20fix=20stuck=20in=20?= =?UTF-8?q?http=20request?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/misc.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/misc.ts b/src/misc.ts index 4157f45..5e5c70d 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -3,6 +3,8 @@ import https from 'https' import nodeUrl from 'url' import type stream from 'stream' +const HTTP_TIMEOUT = Number(process.env['FILEBOX_HTTP_TIMEOUT']) || 5000 + export function dataUrlToBase64 (dataUrl: string): string { const dataList = dataUrl.split(',') return dataList[dataList.length - 1]! @@ -57,8 +59,11 @@ export async function httpHeadHeader (url: string): Promise((resolve, reject) => { - request(options, resolve) + const req = request(options, resolve) .on('error', reject) + .setTimeout(HTTP_TIMEOUT,()=>{ + req.destroy(new Error('Http request timeout!')) + }) .end() }) } @@ -120,8 +125,11 @@ export async function httpStream ( } const res = await new Promise((resolve, reject) => { - get(options, resolve) + const req = get(options, resolve) .on('error', reject) + .setTimeout(HTTP_TIMEOUT,()=>{ + req.destroy(new Error('Http request timeout!')) + }) .end() }) return res From f2c4bcea01ee87792ef07ad041c6f238bc32a594 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Thu, 2 Feb 2023 14:35:55 +0800 Subject: [PATCH 02/63] =?UTF-8?q?refactor:=20=F0=9F=94=87=20remove=20log?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/file-box.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/file-box.ts b/src/file-box.ts index 7d3dfbb..b778fa5 100644 --- a/src/file-box.ts +++ b/src/file-box.ts @@ -1019,8 +1019,6 @@ class FileBox implements Pipeable, FileBoxInterface { ): T { this.toStream().then(stream => { stream.on('error', e => { - console.info('error:', e) - destination.emit('error', e) }) return stream.pipe(destination) From 57876a1ceba09080e10850ff2eace810a8c61f00 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Thu, 2 Feb 2023 15:24:29 +0800 Subject: [PATCH 03/63] =?UTF-8?q?docs:=20=F0=9F=93=9D=20update=20README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index e04a65c..6ec556b 100644 --- a/README.md +++ b/README.md @@ -474,6 +474,12 @@ console.log(fileBox.remoteSize) 1. Serializable 1. Can be Transfered from server to server, server to browser. +## Environments + +Environment variables can be used to control some behavior. + +- `FILEBOX_HTTP_TIMEOUT` [default=5000] Socket idle timeout when FileBox downloads data from URL. For example, when the network is temporarily interrupted, the request will be considered as failed after waiting for a specified time. + ## SCHEMAS ### Url @@ -515,6 +521,10 @@ console.log(fileBox.remoteSize) ## History +### main v1.5.6 (Feb 18, 2023) + +1. Environment variables `FILEBOX_HTTP_TIMEOUT` can be set by user. see [Environments](#Environments). + ### main v1.5 (Jan 18, 2022) 1. `fileBox.md5` can be set by user. This filed is for the receiver of the filebox to check, and it is not computed from the file. From b8361f14972a732801d11c03b4ba09b6dda592af Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Thu, 2 Feb 2023 15:24:42 +0800 Subject: [PATCH 04/63] 1.5.6 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2d81fd2..40794d0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "file-box", - "version": "1.5.5", + "version": "1.5.6", "description": "Pack a File into Box for easy move/transfer between servers no matter of where it is.(local path, remote url, or cloud storage)", "type": "module", "exports": { From a8264bd6868e27e01771ecb3f889455e6bddfaaf Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Fri, 3 Feb 2023 01:44:59 +0800 Subject: [PATCH 05/63] =?UTF-8?q?fix:=20=F0=9F=90=9B=20fix=20throw=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/file-box.ts | 10 +++++----- src/misc.ts | 31 +++++++++++++++++++++---------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/file-box.ts b/src/file-box.ts index b778fa5..a35ea4c 100644 --- a/src/file-box.ts +++ b/src/file-box.ts @@ -1017,12 +1017,12 @@ class FileBox implements Pipeable, FileBoxInterface { pipe ( destination: T, ): T { - this.toStream().then(stream => { - stream.on('error', e => { - destination.emit('error', e) + this.toStream() + .then(stream => { + stream.once('error', e => destination?.destroy(e)) + return stream.pipe(destination) }) - return stream.pipe(destination) - }).catch(e => destination.emit('error', e)) + .catch(e => destination?.destroy(e)) return destination } diff --git a/src/misc.ts b/src/misc.ts index 5e5c70d..f7082b9 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -59,10 +59,16 @@ export async function httpHeadHeader (url: string): Promise((resolve, reject) => { - const req = request(options, resolve) - .on('error', reject) - .setTimeout(HTTP_TIMEOUT,()=>{ - req.destroy(new Error('Http request timeout!')) + let res: http.IncomingMessage + const req = request(options, (response) => { + res = response + resolve(res) + }) + .once('error', reject) + .setTimeout(HTTP_TIMEOUT, () => { + const e = new Error('Http request timeout!') + res?.destroy(e) + req.destroy(e) }) .end() }) @@ -124,15 +130,20 @@ export async function httpStream ( ...headers, } - const res = await new Promise((resolve, reject) => { - const req = get(options, resolve) - .on('error', reject) - .setTimeout(HTTP_TIMEOUT,()=>{ - req.destroy(new Error('Http request timeout!')) + return new Promise((resolve, reject) => { + let res: http.IncomingMessage + const req = get(options, (response) => { + res = response + resolve(res) + }) + .once('error', reject) + .setTimeout(HTTP_TIMEOUT, () => { + const e = new Error('Http request timeout!') + res?.destroy(e) + req.destroy(e) }) .end() }) - return res } export async function streamToBuffer ( From 689dd11caf737a2c3c30400d579d867ed3ea9261 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Fri, 3 Feb 2023 10:45:00 +0800 Subject: [PATCH 06/63] =?UTF-8?q?refactor:=20=E2=99=BB=EF=B8=8F=20use=20ur?= =?UTF-8?q?l.URL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes: Get rid of deprecated Node.js API: url.parse #20 --- src/misc.ts | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/misc.ts b/src/misc.ts index f7082b9..5b77ff3 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -1,6 +1,6 @@ import http from 'http' import https from 'https' -import nodeUrl from 'url' +import { URL } from 'url' import type stream from 'stream' const HTTP_TIMEOUT = Number(process.env['FILEBOX_HTTP_TIMEOUT']) || 5000 @@ -41,9 +41,8 @@ export async function httpHeadHeader (url: string): Promise { - const parsedUrl = nodeUrl.parse(destUrl) + const parsedUrl = new URL(destUrl) const options = { - ...parsedUrl, method : 'HEAD', // method : 'GET', } @@ -60,7 +59,7 @@ export async function httpHeadHeader (url: string): Promise((resolve, reject) => { let res: http.IncomingMessage - const req = request(options, (response) => { + const req = request(parsedUrl, options, (response) => { res = response resolve(res) }) @@ -98,14 +97,11 @@ export async function httpStream ( url : string, headers : http.OutgoingHttpHeaders = {}, ): Promise { - - /* eslint node/no-deprecated-api: off */ - // FIXME: - const parsedUrl = nodeUrl.parse(url) + const parsedUrl = new URL(url) const protocol = parsedUrl.protocol - let options: http.RequestOptions + const options: http.RequestOptions = {} let get: typeof https.get @@ -115,24 +111,21 @@ export async function httpStream ( if (protocol.match(/^https:/i)) { get = https.get - options = parsedUrl options.agent = https.globalAgent } else if (protocol.match(/^http:/i)) { get = http.get - options = parsedUrl options.agent = http.globalAgent } else { throw new Error('protocol unknown: ' + protocol) } options.headers = { - ...options.headers, ...headers, } return new Promise((resolve, reject) => { let res: http.IncomingMessage - const req = get(options, (response) => { + const req = get(parsedUrl, options, (response) => { res = response resolve(res) }) From 00fd0ff47c1d101922fbf1dbcaae787380e796bb Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Fri, 3 Feb 2023 10:45:06 +0800 Subject: [PATCH 07/63] =?UTF-8?q?refactor:=20=F0=9F=9A=A8=20fix=20lint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/file-box.ts | 4 ++-- src/misc.ts | 12 ++++++++---- src/pure-functions/sized-chunk-transformer.ts | 2 +- src/urn-registry/uniform-resource-name-registry.ts | 4 ++-- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/file-box.ts b/src/file-box.ts index a35ea4c..430daa1 100644 --- a/src/file-box.ts +++ b/src/file-box.ts @@ -1019,10 +1019,10 @@ class FileBox implements Pipeable, FileBoxInterface { ): T { this.toStream() .then(stream => { - stream.once('error', e => destination?.destroy(e)) + stream.once('error', e => destination.destroy(e)) return stream.pipe(destination) }) - .catch(e => destination?.destroy(e)) + .catch(e => destination.destroy(e)) return destination } diff --git a/src/misc.ts b/src/misc.ts index 5b77ff3..c2ec735 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -58,7 +58,7 @@ export async function httpHeadHeader (url: string): Promise((resolve, reject) => { - let res: http.IncomingMessage + let res: http.IncomingMessage | null = null const req = request(parsedUrl, options, (response) => { res = response resolve(res) @@ -66,7 +66,9 @@ export async function httpHeadHeader (url: string): Promise { const e = new Error('Http request timeout!') - res?.destroy(e) + if (res) { + res.destroy(e) + } req.destroy(e) }) .end() @@ -124,7 +126,7 @@ export async function httpStream ( } return new Promise((resolve, reject) => { - let res: http.IncomingMessage + let res: http.IncomingMessage | null = null const req = get(parsedUrl, options, (response) => { res = response resolve(res) @@ -132,7 +134,9 @@ export async function httpStream ( .once('error', reject) .setTimeout(HTTP_TIMEOUT, () => { const e = new Error('Http request timeout!') - res?.destroy(e) + if (res) { + res.destroy(e) + } req.destroy(e) }) .end() diff --git a/src/pure-functions/sized-chunk-transformer.ts b/src/pure-functions/sized-chunk-transformer.ts index 6463328..b221ded 100644 --- a/src/pure-functions/sized-chunk-transformer.ts +++ b/src/pure-functions/sized-chunk-transformer.ts @@ -18,7 +18,7 @@ function sizedChunkTransformer (chunkByte = DEFAULT_CHUNK_BYTE) { let buffer = Buffer.from([]) const transform: stream.TransformOptions['transform'] = function (chunk, _, done) { - buffer = Buffer.concat([buffer, chunk]) + buffer = Buffer.concat([ buffer, chunk ]) while (buffer.length >= chunkByte) { this.push(buffer.slice(0, chunkByte)) diff --git a/src/urn-registry/uniform-resource-name-registry.ts b/src/urn-registry/uniform-resource-name-registry.ts index 5e9af72..b436272 100644 --- a/src/urn-registry/uniform-resource-name-registry.ts +++ b/src/urn-registry/uniform-resource-name-registry.ts @@ -142,7 +142,7 @@ class UniformResourceNameRegistry { protected purgeExpiredUuid () { log.verbose('UniformResourceNameRegistry', 'purgeExpiredUuid()') - const expireTimeList = [...this.uuidExpiringTable.keys()] + const expireTimeList = [ ...this.uuidExpiringTable.keys() ] .sort((a, b) => Number(a) - Number(b)) for (const expireTime of expireTimeList) { @@ -291,7 +291,7 @@ class UniformResourceNameRegistry { */ destroy () { log.verbose('UniformResourceNameRegistry', 'destroy() %s UUIDs left', - [...this.uuidExpiringTable.values()].flat().length, + [ ...this.uuidExpiringTable.values() ].flat().length, ) if (this.purgerTimer) { From 2a5e53485eaee2c55f8f2fc70b70648e27783af4 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Fri, 3 Feb 2023 10:46:41 +0800 Subject: [PATCH 08/63] 1.5.7 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 40794d0..9b8c340 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "file-box", - "version": "1.5.6", + "version": "1.5.7", "description": "Pack a File into Box for easy move/transfer between servers no matter of where it is.(local path, remote url, or cloud storage)", "type": "module", "exports": { From e5104a6f7225af92983e0ca6d6e56f7673a87a0e Mon Sep 17 00:00:00 2001 From: Huan Date: Sun, 5 Feb 2023 20:10:20 -0800 Subject: [PATCH 09/63] code clean, use emit error instead of destroying destination --- README.md | 2 +- src/file-box.ts | 7 +++++-- src/misc.ts | 11 ++++++----- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 6ec556b..b353147 100644 --- a/README.md +++ b/README.md @@ -478,7 +478,7 @@ console.log(fileBox.remoteSize) Environment variables can be used to control some behavior. -- `FILEBOX_HTTP_TIMEOUT` [default=5000] Socket idle timeout when FileBox downloads data from URL. For example, when the network is temporarily interrupted, the request will be considered as failed after waiting for a specified time. +- `FILEBOX_HTTP_TIMEOUT` [default=60000] Socket idle timeout when FileBox downloads data from URL. For example, when the network is temporarily interrupted, the request will be considered as failed after waiting for a specified time. ## SCHEMAS diff --git a/src/file-box.ts b/src/file-box.ts index 430daa1..35ee05f 100644 --- a/src/file-box.ts +++ b/src/file-box.ts @@ -1019,10 +1019,13 @@ class FileBox implements Pipeable, FileBoxInterface { ): T { this.toStream() .then(stream => { - stream.once('error', e => destination.destroy(e)) + stream.on('error', e => { + console.error(e) + destination.emit('error', e) + }) return stream.pipe(destination) }) - .catch(e => destination.destroy(e)) + .catch(e => destination.emit('error', e)) return destination } diff --git a/src/misc.ts b/src/misc.ts index c2ec735..2eaa4fb 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -3,7 +3,7 @@ import https from 'https' import { URL } from 'url' import type stream from 'stream' -const HTTP_TIMEOUT = Number(process.env['FILEBOX_HTTP_TIMEOUT']) || 5000 +const HTTP_TIMEOUT = Number(process.env['FILEBOX_HTTP_TIMEOUT']) || 60 * 1000 export function dataUrlToBase64 (dataUrl: string): string { const dataList = dataUrl.split(',') @@ -58,18 +58,19 @@ export async function httpHeadHeader (url: string): Promise((resolve, reject) => { - let res: http.IncomingMessage | null = null + let res: undefined | http.IncomingMessage const req = request(parsedUrl, options, (response) => { res = response resolve(res) }) .once('error', reject) .setTimeout(HTTP_TIMEOUT, () => { - const e = new Error('Http request timeout!') + const e = new Error(`Http request timeout (${HTTP_TIMEOUT})!`) if (res) { - res.destroy(e) + res.emit('error', e) + } else { + req.emit('error', e) } - req.destroy(e) }) .end() }) From 545baae669a3c570d6920c176bddd44262afbaeb Mon Sep 17 00:00:00 2001 From: Huan Date: Sun, 5 Feb 2023 20:12:55 -0800 Subject: [PATCH 10/63] v1.7 for FILEBOX_HTTP_TIMEOUT --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9b8c340..4ddb607 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "file-box", - "version": "1.5.7", + "version": "1.7.0", "description": "Pack a File into Box for easy move/transfer between servers no matter of where it is.(local path, remote url, or cloud storage)", "type": "module", "exports": { From f382c3d9ccec4205520c9af8a6d8db03d54527ff Mon Sep 17 00:00:00 2001 From: Huan Date: Sun, 5 Feb 2023 20:14:36 -0800 Subject: [PATCH 11/63] add link to issue --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b353147..f5cd14a 100644 --- a/README.md +++ b/README.md @@ -521,9 +521,9 @@ Environment variables can be used to control some behavior. ## History -### main v1.5.6 (Feb 18, 2023) +### main v1.7 (Feb 18, 2023) -1. Environment variables `FILEBOX_HTTP_TIMEOUT` can be set by user. see [Environments](#Environments). +1. Environment variables `FILEBOX_HTTP_TIMEOUT` can be set by user. see [Environments](#Environments). ([#80](https://github.com/huan/file-box/issues/80), by @[binsee](https://github.com/binsee)) ### main v1.5 (Jan 18, 2022) From db0c1c59f7aa32322e841573cc8e7e05f470bd1d Mon Sep 17 00:00:00 2001 From: Huan Date: Sun, 5 Feb 2023 20:16:20 -0800 Subject: [PATCH 12/63] better log --- src/misc.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/misc.ts b/src/misc.ts index 2eaa4fb..f05f9f0 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -134,7 +134,7 @@ export async function httpStream ( }) .once('error', reject) .setTimeout(HTTP_TIMEOUT, () => { - const e = new Error('Http request timeout!') + const e = new Error(`Http request timeout (${HTTP_TIMEOUT})!`) if (res) { res.destroy(e) } From 77a9e26adc040d3e265da6cf32a217071c59b7d6 Mon Sep 17 00:00:00 2001 From: Huan Date: Sun, 5 Feb 2023 20:17:57 -0800 Subject: [PATCH 13/63] better log --- src/misc.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/misc.ts b/src/misc.ts index f05f9f0..576871f 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -65,7 +65,7 @@ export async function httpHeadHeader (url: string): Promise { - const e = new Error(`Http request timeout (${HTTP_TIMEOUT})!`) + const e = new Error(`FileBox: Http request timeout (${HTTP_TIMEOUT})!`) if (res) { res.emit('error', e) } else { @@ -134,7 +134,7 @@ export async function httpStream ( }) .once('error', reject) .setTimeout(HTTP_TIMEOUT, () => { - const e = new Error(`Http request timeout (${HTTP_TIMEOUT})!`) + const e = new Error(`FileBox: Http request timeout (${HTTP_TIMEOUT})!`) if (res) { res.destroy(e) } From 56ea2a55db71965b4045107a9155d6b9409972f2 Mon Sep 17 00:00:00 2001 From: Huan Date: Sun, 5 Feb 2023 20:40:42 -0800 Subject: [PATCH 14/63] add unit test for network timeout --- src/config.ts | 3 +++ src/misc.ts | 2 +- tests/network-timeout.spec.ts | 30 ++++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100755 tests/network-timeout.spec.ts diff --git a/src/config.ts b/src/config.ts index 7e129bc..eb3a833 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,2 +1,5 @@ /// export { VERSION } from './version.js' + +export const HTTP_TIMEOUT = Number(process.env['FILEBOX_HTTP_TIMEOUT']) + || 60 * 1000 diff --git a/src/misc.ts b/src/misc.ts index 576871f..54e6785 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -3,7 +3,7 @@ import https from 'https' import { URL } from 'url' import type stream from 'stream' -const HTTP_TIMEOUT = Number(process.env['FILEBOX_HTTP_TIMEOUT']) || 60 * 1000 +import { HTTP_TIMEOUT } from './config.js' export function dataUrlToBase64 (dataUrl: string): string { const dataList = dataUrl.split(',') diff --git a/tests/network-timeout.spec.ts b/tests/network-timeout.spec.ts new file mode 100755 index 0000000..396eb20 --- /dev/null +++ b/tests/network-timeout.spec.ts @@ -0,0 +1,30 @@ +#!/usr/bin/env -S node --no-warnings --loader ts-node/esm + +import { sinon, test } from 'tstest' + +import { FileBox } from '../src/mod.js' + +import { HTTP_TIMEOUT } from '../src/config.js' + +test('slow network stall HTTP_TIMEOUT', async t => { + const sandbox = sinon.createSandbox() + sandbox.useFakeTimers(Date.now()) + + const spy = sandbox.spy() + + const stream = await FileBox + .fromUrl('https://www.google.com') + .toStream() + + stream.on('error', spy) + + const start = Date.now() + + await sandbox.clock.tickAsync(HTTP_TIMEOUT - 1) + t.ok(spy.notCalled, `should not get error after TIMEOUT ${HTTP_TIMEOUT} - 1 (${Date.now() - start} passed)`) + + await sandbox.clock.tickAsync(10) + t.ok(spy.calledOnce, `should get error after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) + + sandbox.restore() +}) From c047822796b3ad7840a5970011e692de90d19b8e Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Mon, 6 Feb 2023 14:54:34 +0800 Subject: [PATCH 15/63] =?UTF-8?q?refactor:=20=F0=9F=94=87=20remove=20log?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/file-box.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/file-box.ts b/src/file-box.ts index 35ee05f..a411a21 100644 --- a/src/file-box.ts +++ b/src/file-box.ts @@ -1020,7 +1020,6 @@ class FileBox implements Pipeable, FileBoxInterface { this.toStream() .then(stream => { stream.on('error', e => { - console.error(e) destination.emit('error', e) }) return stream.pipe(destination) From 0eb3759024cfa9f4d39b894594f023228fc81386 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Tue, 7 Feb 2023 12:55:54 +0800 Subject: [PATCH 16/63] update unit test for network timeout --- tests/network-timeout.spec.ts | 86 ++++++++++++++++++++++++++++++----- 1 file changed, 74 insertions(+), 12 deletions(-) diff --git a/tests/network-timeout.spec.ts b/tests/network-timeout.spec.ts index 396eb20..cc14bda 100755 --- a/tests/network-timeout.spec.ts +++ b/tests/network-timeout.spec.ts @@ -1,30 +1,92 @@ #!/usr/bin/env -S node --no-warnings --loader ts-node/esm -import { sinon, test } from 'tstest' +import { createServer } from 'http' +import type { AddressInfo } from 'net' +import { setTimeout } from 'timers/promises' +import { sinon, test } from 'tstest' import { FileBox } from '../src/mod.js' import { HTTP_TIMEOUT } from '../src/config.js' -test('slow network stall HTTP_TIMEOUT', async t => { +test('slow network stall HTTP_TIMEOUT', async (t) => { const sandbox = sinon.createSandbox() sandbox.useFakeTimers(Date.now()) + const port = Math.floor(Math.random() * (65535 - 49152 + 1)) + 49152 + const URL = { + NOT_TIMEOUT: '/not_timeout', + TIMEOUT: '/timeout', + } - const spy = sandbox.spy() + const server = createServer(async (req, res) => { + res.write(Buffer.from('This is the first chunk of data.')) - const stream = await FileBox - .fromUrl('https://www.google.com') - .toStream() + if (req.url === URL.TIMEOUT) { + await setTimeout(HTTP_TIMEOUT + 1) + } else { + await setTimeout(HTTP_TIMEOUT - 1) + } - stream.on('error', spy) + // console.debug(`${new Date().toLocaleTimeString()} call res.end`) + res.end(Buffer.from('This is the second chunk of data after 10 seconds.')) + }) - const start = Date.now() + const host = await new Promise((resolve) => { + server.listen(port, '127.0.0.1', () => { + const addr = server.address() as AddressInfo + // console.debug(`Server is listening on port ${JSON.stringify(addr)}`) + resolve(`http://127.0.0.1:${addr.port}`) + }) + }) - await sandbox.clock.tickAsync(HTTP_TIMEOUT - 1) - t.ok(spy.notCalled, `should not get error after TIMEOUT ${HTTP_TIMEOUT} - 1 (${Date.now() - start} passed)`) + await t.test('should not timeout', async (t) => { + const url = `${host}${URL.NOT_TIMEOUT}` + const dataSpy = sandbox.spy() + const errorSpy = sandbox.spy() - await sandbox.clock.tickAsync(10) - t.ok(spy.calledOnce, `should get error after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) + // console.debug(`${new Date().toLocaleTimeString()} Start request ...`) + const stream = await FileBox.fromUrl(url).toStream() + + stream.once('error', errorSpy).on('data', dataSpy) + + const start = Date.now() + + await sandbox.clock.tickAsync(1) + t.ok(dataSpy.calledOnce, `should get chunk 1 (${Date.now() - start} passed)`) + t.ok(errorSpy.notCalled, `should not get error (${Date.now() - start} passed)`) + + // FIXME: tickAsync does not work on socket timeout + await setTimeout(HTTP_TIMEOUT) + // await sandbox.clock.tickAsync(HTTP_TIMEOUT) + + t.ok(dataSpy.calledTwice, `should get chunk 2 after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) + t.ok(errorSpy.notCalled, `should not get error after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) + }) + + await t.test('should timeout', async (t) => { + const url = `${host}${URL.TIMEOUT}` + const dataSpy = sandbox.spy() + const errorSpy = sandbox.spy() + + // console.debug(`${new Date().toLocaleTimeString()} Start request ...`) + const stream = await FileBox.fromUrl(url).toStream() + + stream.once('error', errorSpy).on('data', dataSpy) + + const start = Date.now() + + await sandbox.clock.tickAsync(1) + t.ok(dataSpy.calledOnce, `should get chunk 1 (${Date.now() - start} passed)`) + t.ok(errorSpy.notCalled, `should not get error (${Date.now() - start} passed)`) + + // FIXME: tickAsync does not work on socket timeout + await setTimeout(HTTP_TIMEOUT) + // await sandbox.clock.tickAsync(HTTP_TIMEOUT) + + t.ok(dataSpy.calledOnce, `should not get chunk 2 after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) + t.ok(errorSpy.calledOnce, `should get error after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) + }) sandbox.restore() + server.close() }) From eb0f363fb21df1a04227beeaee6f3b48da6a1578 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Tue, 7 Feb 2023 13:14:35 +0800 Subject: [PATCH 17/63] =?UTF-8?q?test:=20=F0=9F=9A=A8=20fix=20lint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/network-timeout.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/network-timeout.spec.ts b/tests/network-timeout.spec.ts index cc14bda..05864dd 100755 --- a/tests/network-timeout.spec.ts +++ b/tests/network-timeout.spec.ts @@ -18,6 +18,7 @@ test('slow network stall HTTP_TIMEOUT', async (t) => { TIMEOUT: '/timeout', } + /* eslint @typescript-eslint/no-misused-promises:off */ const server = createServer(async (req, res) => { res.write(Buffer.from('This is the first chunk of data.')) From a9b4c2e719316fde94c5078673aaec48a9869c68 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Tue, 7 Feb 2023 14:12:35 +0800 Subject: [PATCH 18/63] test: update unit test for network timeout --- tests/network-timeout.spec.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/network-timeout.spec.ts b/tests/network-timeout.spec.ts index 05864dd..1351a80 100755 --- a/tests/network-timeout.spec.ts +++ b/tests/network-timeout.spec.ts @@ -88,6 +88,19 @@ test('slow network stall HTTP_TIMEOUT', async (t) => { t.ok(errorSpy.calledOnce, `should get error after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) }) + await t.test('ready should timeout', async (t) => { + const url = `${host}${URL.TIMEOUT}` + const errorSpy = sandbox.spy() + + // console.debug(`${new Date().toLocaleTimeString()} Start request ...`) + const start = Date.now() + const fileBox = FileBox.fromUrl(url) + await fileBox.ready().catch(errorSpy) + + await sandbox.clock.tickAsync(1) + t.ok(errorSpy.calledOnce, `should not get error (${Date.now() - start} passed)`) + }) + sandbox.restore() server.close() }) From ee9cdd399dd6135c4f760229d34e5a2574a0fcdf Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Wed, 8 Feb 2023 15:19:45 +0800 Subject: [PATCH 19/63] test: fix unit test for network timeout --- tests/network-timeout.spec.ts | 71 ++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 22 deletions(-) diff --git a/tests/network-timeout.spec.ts b/tests/network-timeout.spec.ts index 1351a80..45dfd29 100755 --- a/tests/network-timeout.spec.ts +++ b/tests/network-timeout.spec.ts @@ -11,11 +11,18 @@ import { HTTP_TIMEOUT } from '../src/config.js' test('slow network stall HTTP_TIMEOUT', async (t) => { const sandbox = sinon.createSandbox() - sandbox.useFakeTimers(Date.now()) + sandbox.useFakeTimers({ + now: Date.now(), + shouldAdvanceTime: true, + shouldClearNativeTimers: true, + toFake: ['setTimeout', 'clearTimeout', 'setInterval', 'clearInterval', 'nextTick'], + }) + t.jobs = 3 const port = Math.floor(Math.random() * (65535 - 49152 + 1)) + 49152 const URL = { NOT_TIMEOUT: '/not_timeout', TIMEOUT: '/timeout', + READY: '/ready', } /* eslint @typescript-eslint/no-misused-promises:off */ @@ -23,12 +30,12 @@ test('slow network stall HTTP_TIMEOUT', async (t) => { res.write(Buffer.from('This is the first chunk of data.')) if (req.url === URL.TIMEOUT) { - await setTimeout(HTTP_TIMEOUT + 1) + await setTimeout(HTTP_TIMEOUT + 100) } else { - await setTimeout(HTTP_TIMEOUT - 1) + await setTimeout(HTTP_TIMEOUT - 100) } - // console.debug(`${new Date().toLocaleTimeString()} call res.end`) + // console.debug(`${new Date().toLocaleTimeString()} call res.end "${req.url}"`) res.end(Buffer.from('This is the second chunk of data after 10 seconds.')) }) @@ -40,67 +47,87 @@ test('slow network stall HTTP_TIMEOUT', async (t) => { }) }) - await t.test('should not timeout', async (t) => { + t.teardown(() => { + // console.debug('teardown') + server.close() + sandbox.restore() + }) + + t.test('should not timeout', async (t) => { const url = `${host}${URL.NOT_TIMEOUT}` const dataSpy = sandbox.spy() const errorSpy = sandbox.spy() - // console.debug(`${new Date().toLocaleTimeString()} Start request ...`) + // console.debug(`${new Date().toLocaleTimeString()} Start request "${url}" ...`) + const start = Date.now() const stream = await FileBox.fromUrl(url).toStream() stream.once('error', errorSpy).on('data', dataSpy) - const start = Date.now() - await sandbox.clock.tickAsync(1) t.ok(dataSpy.calledOnce, `should get chunk 1 (${Date.now() - start} passed)`) t.ok(errorSpy.notCalled, `should not get error (${Date.now() - start} passed)`) // FIXME: tickAsync does not work on socket timeout - await setTimeout(HTTP_TIMEOUT) + await new Promise((resolve) => { + stream.once('error', resolve).on('close', resolve) + resolve(setTimeout(HTTP_TIMEOUT)) + }) + await sandbox.clock.tickAsync(1) // await sandbox.clock.tickAsync(HTTP_TIMEOUT) t.ok(dataSpy.calledTwice, `should get chunk 2 after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) t.ok(errorSpy.notCalled, `should not get error after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) + t.end() }) - await t.test('should timeout', async (t) => { + t.test('should timeout', async (t) => { const url = `${host}${URL.TIMEOUT}` const dataSpy = sandbox.spy() const errorSpy = sandbox.spy() - // console.debug(`${new Date().toLocaleTimeString()} Start request ...`) + // console.debug(`${new Date().toLocaleTimeString()} Start request "${url}" ...`) + const start = Date.now() const stream = await FileBox.fromUrl(url).toStream() - stream.once('error', errorSpy).on('data', dataSpy) - - const start = Date.now() + stream.once('error', errorSpy).once('data', dataSpy) + // .once('error', (e) => { + // console.error('on error:', e.stack) + // errorSpy(e) + // }) + // .on('data', (d: Buffer) => { + // console.error('on data:', d.toString()) + // dataSpy(d) + // }) await sandbox.clock.tickAsync(1) t.ok(dataSpy.calledOnce, `should get chunk 1 (${Date.now() - start} passed)`) t.ok(errorSpy.notCalled, `should not get error (${Date.now() - start} passed)`) // FIXME: tickAsync does not work on socket timeout - await setTimeout(HTTP_TIMEOUT) + await new Promise((resolve) => { + stream.once('error', resolve).on('close', resolve) + resolve(setTimeout(HTTP_TIMEOUT)) + }) + await sandbox.clock.tickAsync(1) // await sandbox.clock.tickAsync(HTTP_TIMEOUT) t.ok(dataSpy.calledOnce, `should not get chunk 2 after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) t.ok(errorSpy.calledOnce, `should get error after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) + t.end() }) - await t.test('ready should timeout', async (t) => { - const url = `${host}${URL.TIMEOUT}` + t.test('ready should timeout', async (t) => { + const url = `${host}${URL.READY}` const errorSpy = sandbox.spy() - // console.debug(`${new Date().toLocaleTimeString()} Start request ...`) + // console.debug(`${new Date().toLocaleTimeString()} Start request "${url}" ...`) const start = Date.now() const fileBox = FileBox.fromUrl(url) await fileBox.ready().catch(errorSpy) await sandbox.clock.tickAsync(1) - t.ok(errorSpy.calledOnce, `should not get error (${Date.now() - start} passed)`) + t.ok(errorSpy.notCalled, `should not get error (${Date.now() - start} passed)`) + t.end() }) - - sandbox.restore() - server.close() }) From 7ecf628e550e2cf2d1f85b094b441c4c69a477e6 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Wed, 8 Feb 2023 15:26:06 +0800 Subject: [PATCH 20/63] test: fix unit test for network timeout --- tests/network-timeout.spec.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/network-timeout.spec.ts b/tests/network-timeout.spec.ts index 45dfd29..17613ed 100755 --- a/tests/network-timeout.spec.ts +++ b/tests/network-timeout.spec.ts @@ -15,14 +15,14 @@ test('slow network stall HTTP_TIMEOUT', async (t) => { now: Date.now(), shouldAdvanceTime: true, shouldClearNativeTimers: true, - toFake: ['setTimeout', 'clearTimeout', 'setInterval', 'clearInterval', 'nextTick'], + toFake: [ 'setTimeout', 'clearTimeout', 'setInterval', 'clearInterval', 'nextTick' ], }) t.jobs = 3 const port = Math.floor(Math.random() * (65535 - 49152 + 1)) + 49152 const URL = { NOT_TIMEOUT: '/not_timeout', - TIMEOUT: '/timeout', READY: '/ready', + TIMEOUT: '/timeout', } /* eslint @typescript-eslint/no-misused-promises:off */ @@ -53,6 +53,7 @@ test('slow network stall HTTP_TIMEOUT', async (t) => { sandbox.restore() }) + /** eslint @typescript-eslint/no-floating-promises:off */ t.test('should not timeout', async (t) => { const url = `${host}${URL.NOT_TIMEOUT}` const dataSpy = sandbox.spy() @@ -79,8 +80,9 @@ test('slow network stall HTTP_TIMEOUT', async (t) => { t.ok(dataSpy.calledTwice, `should get chunk 2 after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) t.ok(errorSpy.notCalled, `should not get error after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) t.end() - }) + }).catch(t.threw) + /** eslint @typescript-eslint/no-floating-promises:off */ t.test('should timeout', async (t) => { const url = `${host}${URL.TIMEOUT}` const dataSpy = sandbox.spy() @@ -115,8 +117,9 @@ test('slow network stall HTTP_TIMEOUT', async (t) => { t.ok(dataSpy.calledOnce, `should not get chunk 2 after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) t.ok(errorSpy.calledOnce, `should get error after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) t.end() - }) + }).catch(t.threw) + /** eslint @typescript-eslint/no-floating-promises:off */ t.test('ready should timeout', async (t) => { const url = `${host}${URL.READY}` const errorSpy = sandbox.spy() @@ -129,5 +132,5 @@ test('slow network stall HTTP_TIMEOUT', async (t) => { await sandbox.clock.tickAsync(1) t.ok(errorSpy.notCalled, `should not get error (${Date.now() - start} passed)`) t.end() - }) + }).catch(t.threw) }) From 5d78dd960d983253d86768530edaa285f252cfde Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Wed, 8 Feb 2023 15:53:08 +0800 Subject: [PATCH 21/63] ci: set timeout for tap to 90 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4ddb607..e742ff0 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "lint": "npm-run-all lint:es lint:ts", "lint:ts": "tsc --isolatedModules --noEmit", "test": "npm-run-all lint test:unit", - "test:unit": "cross-env NODE_OPTIONS=\"--no-warnings --loader=ts-node/esm\" tap \"src/**/*.spec.ts\" \"tests/*.spec.ts\"", + "test:unit": "cross-env NODE_OPTIONS=\"--no-warnings --loader=ts-node/esm\" tap --timeout=90 \"src/**/*.spec.ts\" \"tests/*.spec.ts\"", "test:pack": "bash -x scripts/npm-pack-testing.sh", "lint:es": "eslint --ignore-pattern fixtures/ \"src/**/*.ts\" \"tests/**/*.ts\"" }, From 95c4e03a33fe52849434b876d18522415b4f593e Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Wed, 8 Feb 2023 17:24:23 +0800 Subject: [PATCH 22/63] =?UTF-8?q?fix:=20=F0=9F=90=9B=20fix=20ERR=5FSTREAM?= =?UTF-8?q?=5FPREMATURE=5FCLOSE=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/misc.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/misc.ts b/src/misc.ts index 54e6785..916c038 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -136,9 +136,10 @@ export async function httpStream ( .setTimeout(HTTP_TIMEOUT, () => { const e = new Error(`FileBox: Http request timeout (${HTTP_TIMEOUT})!`) if (res) { - res.destroy(e) + res.emit('error', e) } - req.destroy(e) + req.emit('error', e) + req.destroy() }) .end() }) From 7e01fa7f28df4ea8e82e8993b4af456ff60e74c3 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Wed, 8 Feb 2023 18:18:37 +0800 Subject: [PATCH 23/63] test: update unit test for network timeout --- tests/network-timeout.spec.ts | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/tests/network-timeout.spec.ts b/tests/network-timeout.spec.ts index 17613ed..31731d0 100755 --- a/tests/network-timeout.spec.ts +++ b/tests/network-timeout.spec.ts @@ -29,14 +29,16 @@ test('slow network stall HTTP_TIMEOUT', async (t) => { const server = createServer(async (req, res) => { res.write(Buffer.from('This is the first chunk of data.')) - if (req.url === URL.TIMEOUT) { - await setTimeout(HTTP_TIMEOUT + 100) + if (req.url === URL.NOT_TIMEOUT) { + await setTimeout(HTTP_TIMEOUT * 0.5) + res.write(Buffer.from('This is the second chunk of data.')) + await setTimeout(HTTP_TIMEOUT * 0.9) } else { - await setTimeout(HTTP_TIMEOUT - 100) + await setTimeout(HTTP_TIMEOUT + 100) } // console.debug(`${new Date().toLocaleTimeString()} call res.end "${req.url}"`) - res.end(Buffer.from('This is the second chunk of data after 10 seconds.')) + res.end(Buffer.from('All data end.')) }) const host = await new Promise((resolve) => { @@ -72,12 +74,14 @@ test('slow network stall HTTP_TIMEOUT', async (t) => { // FIXME: tickAsync does not work on socket timeout await new Promise((resolve) => { stream.once('error', resolve).on('close', resolve) - resolve(setTimeout(HTTP_TIMEOUT)) + // resolve(setTimeout(HTTP_TIMEOUT)) }) await sandbox.clock.tickAsync(1) // await sandbox.clock.tickAsync(HTTP_TIMEOUT) - t.ok(dataSpy.calledTwice, `should get chunk 2 after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) + // t.comment('recv data count:', dataSpy.callCount) + // t.comment('recv error count:', errorSpy.callCount) + t.ok(dataSpy.calledThrice, `should get chunk 3 after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) t.ok(errorSpy.notCalled, `should not get error after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) t.end() }).catch(t.threw) @@ -93,28 +97,30 @@ test('slow network stall HTTP_TIMEOUT', async (t) => { const stream = await FileBox.fromUrl(url).toStream() stream.once('error', errorSpy).once('data', dataSpy) - // .once('error', (e) => { - // console.error('on error:', e.stack) - // errorSpy(e) + // .on('error', (e) => { + // console.error(`on error for req "${url}":`, e.stack) // }) // .on('data', (d: Buffer) => { - // console.error('on data:', d.toString()) - // dataSpy(d) + // console.error(`on data for req "${url}":`, d.toString()) // }) await sandbox.clock.tickAsync(1) + + // t.comment('recv data count:', dataSpy.callCount) + // t.comment('recv error count:', errorSpy.callCount) t.ok(dataSpy.calledOnce, `should get chunk 1 (${Date.now() - start} passed)`) t.ok(errorSpy.notCalled, `should not get error (${Date.now() - start} passed)`) // FIXME: tickAsync does not work on socket timeout await new Promise((resolve) => { stream.once('error', resolve).on('close', resolve) - resolve(setTimeout(HTTP_TIMEOUT)) + // resolve(setTimeout(HTTP_TIMEOUT)) }) await sandbox.clock.tickAsync(1) // await sandbox.clock.tickAsync(HTTP_TIMEOUT) - t.ok(dataSpy.calledOnce, `should not get chunk 2 after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) + // t.comment('recv data count:', dataSpy.callCount) + // t.comment('recv error count:', errorSpy.callCount) t.ok(errorSpy.calledOnce, `should get error after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) t.end() }).catch(t.threw) @@ -130,7 +136,8 @@ test('slow network stall HTTP_TIMEOUT', async (t) => { await fileBox.ready().catch(errorSpy) await sandbox.clock.tickAsync(1) - t.ok(errorSpy.notCalled, `should not get error (${Date.now() - start} passed)`) + // t.comment('recv error count:', errorSpy.callCount) + t.ok(errorSpy.calledOnce, `should get error after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) t.end() }).catch(t.threw) }) From 6980bd069f5c28c66467901b11f0b70ad6183fcf Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Wed, 8 Feb 2023 18:51:12 +0800 Subject: [PATCH 24/63] =?UTF-8?q?fix:=20=F0=9F=90=9B=20fix=20catch=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/misc.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/misc.ts b/src/misc.ts index 916c038..cdffecc 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -63,14 +63,11 @@ export async function httpHeadHeader (url: string): Promise { const e = new Error(`FileBox: Http request timeout (${HTTP_TIMEOUT})!`) - if (res) { - res.emit('error', e) - } else { - req.emit('error', e) - } + req.emit('error', e) + req.destroy() }) .end() }) @@ -132,7 +129,7 @@ export async function httpStream ( res = response resolve(res) }) - .once('error', reject) + .on('error', reject) .setTimeout(HTTP_TIMEOUT, () => { const e = new Error(`FileBox: Http request timeout (${HTTP_TIMEOUT})!`) if (res) { From 51cf34d01b132477049941d6d2ca6108cb027bc9 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Thu, 16 Feb 2023 10:46:28 +0800 Subject: [PATCH 25/63] build: update scope of the package --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2d81fd2..5058a21 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "name": "file-box", + "name": "@juzi/file-box", "version": "1.5.5", "description": "Pack a File into Box for easy move/transfer between servers no matter of where it is.(local path, remote url, or cloud storage)", "type": "module", From 2f1ac652e0f0153340b369ee23d6e95963c881f1 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Thu, 16 Feb 2023 12:04:22 +0800 Subject: [PATCH 26/63] =?UTF-8?q?build:=20=F0=9F=93=8C=20lock=20esquery=20?= =?UTF-8?q?to=201.4.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 2d81fd2..d9ae588 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "gts": "^3.1.0", "pkg-jq": "^0.2.11", "read-pkg-up": "^8.0.0", + "esquery": "1.4.0", "reflect-metadata": "^0.1.13" }, "dependencies": { From 00a8238649cd5b658f362f908b5a896cf73fe204 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Thu, 16 Feb 2023 12:35:24 +0800 Subject: [PATCH 27/63] =?UTF-8?q?fix:=20=F0=9F=90=9B=20fix=20import=20deps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/fixtures/smoke-testing.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fixtures/smoke-testing.ts b/tests/fixtures/smoke-testing.ts index 54ffe41..ecb0afe 100644 --- a/tests/fixtures/smoke-testing.ts +++ b/tests/fixtures/smoke-testing.ts @@ -3,7 +3,7 @@ import assert from 'assert' import { FileBox, VERSION, -} from 'file-box' +} from '@juzi/file-box' async function main () { const box = FileBox.fromUrl('https://raw.githubusercontent.com/huan/file-box/main/docs/images/file-box-logo.jpg') From 05edb185d85ef0943107303fc115866635237117 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Thu, 16 Feb 2023 14:31:26 +0800 Subject: [PATCH 28/63] =?UTF-8?q?ci:=20=F0=9F=91=B7=20update=20npm=20token?= =?UTF-8?q?=20env?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/npm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/npm.yml b/.github/workflows/npm.yml index 6ae217a..39c673c 100644 --- a/.github/workflows/npm.yml +++ b/.github/workflows/npm.yml @@ -93,7 +93,7 @@ jobs: else npm publish fi env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + NODE_AUTH_TOKEN: ${{ secrets.JUZI_ADMIN_SCOPE_NPM_TOKEN }} - name: Is Not A Publish Branch if: steps.check-branch.outputs.match != 'true' run: echo 'Not A Publish Branch' From c23d9d70664c8d38b135aff13d74502fa467516a Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Fri, 17 Feb 2023 01:12:29 +0800 Subject: [PATCH 29/63] =?UTF-8?q?Revert=20"build:=20=F0=9F=93=8C=20lock=20?= =?UTF-8?q?esquery=20to=201.4.0"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 2f1ac652e0f0153340b369ee23d6e95963c881f1. --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index 776ad39..57f12fe 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,6 @@ "gts": "^3.1.0", "pkg-jq": "^0.2.11", "read-pkg-up": "^8.0.0", - "esquery": "1.4.0", "reflect-metadata": "^0.1.13" }, "dependencies": { From d727b9ed520c635cc0c15575a6d5181004241715 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Fri, 17 Feb 2023 01:13:10 +0800 Subject: [PATCH 30/63] 1.7.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 57f12fe..db11819 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@juzi/file-box", - "version": "1.7.0", + "version": "1.7.1", "description": "Pack a File into Box for easy move/transfer between servers no matter of where it is.(local path, remote url, or cloud storage)", "type": "module", "exports": { From 56275474c63ffbe6e6850df61ef3f609249570b7 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Thu, 2 Mar 2023 20:43:10 +0800 Subject: [PATCH 31/63] =?UTF-8?q?feat:=20=E2=9C=A8=20support=20for=20chunk?= =?UTF-8?q?ed=20downloads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/config.ts | 3 + src/misc.ts | 176 +++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 169 insertions(+), 10 deletions(-) diff --git a/src/config.ts b/src/config.ts index eb3a833..e8b1c8c 100644 --- a/src/config.ts +++ b/src/config.ts @@ -3,3 +3,6 @@ export { VERSION } from './version.js' export const HTTP_TIMEOUT = Number(process.env['FILEBOX_HTTP_TIMEOUT']) || 60 * 1000 + +export const HTTP_CHUNK_SIZE = Number(process.env['FILEBOX_HTTP_CHUNK_SIZE']) + || 1024 * 512 diff --git a/src/misc.ts b/src/misc.ts index cdffecc..5f0791d 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -1,9 +1,10 @@ import http from 'http' import https from 'https' import { URL } from 'url' -import type stream from 'stream' -import { HTTP_TIMEOUT } from './config.js' +import { PassThrough, pipeline, Readable } from 'stream' + +import { HTTP_CHUNK_SIZE, HTTP_TIMEOUT } from './config.js' export function dataUrlToBase64 (dataUrl: string): string { const dataList = dataUrl.split(',') @@ -96,10 +97,10 @@ export function httpHeaderToFileName ( export async function httpStream ( url : string, headers : http.OutgoingHttpHeaders = {}, -): Promise { +): Promise { const parsedUrl = new URL(url) - const protocol = parsedUrl.protocol + const protocol = parsedUrl.protocol const options: http.RequestOptions = {} @@ -123,12 +124,29 @@ export async function httpStream ( ...headers, } - return new Promise((resolve, reject) => { + const headHeaders = await httpHeadHeader(url) + const fileSize = Number(headHeaders['content-length']) + + if (headHeaders['accept-ranges'] === 'bytes' && fileSize > HTTP_CHUNK_SIZE) { + return await downloadFileInChunks(get, url, options, fileSize, HTTP_CHUNK_SIZE) + } else { + return await downloadFile(get, url, options) + } +} + +async function downloadFile ( + get: typeof https.get, + url: string, + options: http.RequestOptions, +): Promise { + return new Promise((resolve, reject) => { let res: http.IncomingMessage | null = null - const req = get(parsedUrl, options, (response) => { + const req = get(url, options, (response) => { res = response resolve(res) }) + + req .on('error', reject) .setTimeout(HTTP_TIMEOUT, () => { const e = new Error(`FileBox: Http request timeout (${HTTP_TIMEOUT})!`) @@ -142,9 +160,147 @@ export async function httpStream ( }) } -export async function streamToBuffer ( - stream: stream.Readable, -): Promise { +async function downloadFileInChunks ( + get: typeof https.get, + url: string, + options: http.RequestOptions, + fileSize: number, + chunkSize = HTTP_CHUNK_SIZE, +): Promise { + const ac = new AbortController() + const stream = new PassThrough() + + const abortAc = () => { + if (!ac.signal.aborted) { + ac.abort() + } + } + + stream + .once('close', abortAc) + .once('error', abortAc) + + const chunksCount = Math.ceil(fileSize / chunkSize) + let chunksDownloaded = 0 + let dataTotalSize = 0 + + const doDownloadChunk = async function (i: number, retries: number) { + const start = i * chunkSize + const end = Math.min((i + 1) * chunkSize - 1, fileSize - 1) + const range = `bytes=${start}-${end}` + + // console.info('doDownloadChunk() range:', range) + + if (ac.signal.aborted) { + stream.destroy(new Error('Signal aborted.')) + return + } + + const requestOptions: http.RequestOptions = { + ...options, + signal: ac.signal, + timeout: HTTP_TIMEOUT, + } + if (!requestOptions.headers) { + requestOptions.headers = {} + } + requestOptions.headers['Range'] = range + + try { + const chunk = await downloadChunk(get, url, requestOptions, retries) + if (chunk.errored) { + throw new Error('chunk stream error') + } + if (chunk.closed) { + throw new Error('chunk stream closed') + } + if (chunk.destroyed) { + throw new Error('chunk stream destroyed') + } + const buf = await streamToBuffer(chunk) + stream.push(buf) + chunksDownloaded++ + dataTotalSize += buf.length + + if (chunksDownloaded === chunksCount || dataTotalSize >= fileSize) { + stream.push(null) + } + } catch (err) { + if (retries === 0) { + stream.emit('error', err) + } else { + await doDownloadChunk(i, retries - 1) + } + } + } + + const doDownloadAllChunks = async function () { + for (let i = 0; i < chunksCount; i++) { + if (ac.signal.aborted) { + return + } + await doDownloadChunk(i, 3) + } + } + + void doDownloadAllChunks().catch((e) => { + stream.emit('error', e) + }) + + return stream +} + +async function downloadChunk ( + get: typeof https.get, + url: string, + requestOptions: http.RequestOptions, + retries: number, +): Promise { + return new Promise((resolve, reject) => { + const doRequest = (attempt: number) => { + let resolved = false + const req = get(url, requestOptions, (res) => { + const statusCode = res.statusCode ?? 0 + // console.info('downloadChunk(%d) statusCode: %d rsp.headers: %o', attempt, statusCode, res.headers) + + if (statusCode < 200 || statusCode >= 300) { + if (attempt < retries) { + void doRequest(attempt + 1) + } else { + reject(new Error(`Request failed with status code ${res.statusCode}`)) + } + return + } + + const stream = pipeline(res, new PassThrough(), () => {}) + resolve(stream) + resolved = true + }) + + req + .once('error', (err) => { + if (resolved) { + return + } + // console.error('downloadChunk(%d) req error:', attempt, err) + if (attempt < retries) { + void doRequest(attempt + 1) + } else { + reject(err) + } + }) + .setTimeout(HTTP_TIMEOUT, () => { + const e = new Error(`FileBox: Http request timeout (${HTTP_TIMEOUT})!`) + req.emit('error', e) + req.destroy() + }) + .end() + } + void doRequest(0) + }) +} + +export async function streamToBuffer (stream: Readable): Promise { return new Promise((resolve, reject) => { const bufferList: Buffer[] = [] stream.once('error', reject) @@ -152,6 +308,6 @@ export async function streamToBuffer ( const fullBuffer = Buffer.concat(bufferList) resolve(fullBuffer) }) - stream.on('data', buffer => bufferList.push(buffer)) + stream.on('data', (buffer) => bufferList.push(buffer)) }) } From 58160b1f8f85be4902c0db217875e1f59eeb91f4 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Thu, 2 Mar 2023 20:46:48 +0800 Subject: [PATCH 32/63] =?UTF-8?q?test:=20=E2=9C=85=20add=20test=20download?= =?UTF-8?q?=20in=20chunks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/misc.spec.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/misc.spec.ts b/src/misc.spec.ts index ed04509..28e5a7b 100755 --- a/src/misc.spec.ts +++ b/src/misc.spec.ts @@ -68,3 +68,13 @@ test('httpStream', async t => { const obj = JSON.parse(buffer.toString()) t.equal(obj.headers[MOL_KEY], MOL_VAL, 'should send the header right') }) + +test('httpStream in chunks', async (t) => { + const URL = 'https://media.w3.org/2010/05/sintel/trailer.mp4' + const res = await httpStream(URL) + let length = 0 + for await (const chunk of res) { + length += chunk.length + } + t.equal(length, 4372373, 'should get data in chunks right') +}) From 130541e4516886599fa308f023f022a2b829e6a3 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Thu, 2 Mar 2023 23:49:59 +0800 Subject: [PATCH 33/63] =?UTF-8?q?test:=20=E2=9C=85=20fix=20timeout=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/network-timeout.spec.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/network-timeout.spec.ts b/tests/network-timeout.spec.ts index 31731d0..fbacf77 100755 --- a/tests/network-timeout.spec.ts +++ b/tests/network-timeout.spec.ts @@ -32,9 +32,12 @@ test('slow network stall HTTP_TIMEOUT', async (t) => { if (req.url === URL.NOT_TIMEOUT) { await setTimeout(HTTP_TIMEOUT * 0.5) res.write(Buffer.from('This is the second chunk of data.')) - await setTimeout(HTTP_TIMEOUT * 0.9) - } else { + } else if (req.url === URL.READY) { await setTimeout(HTTP_TIMEOUT + 100) + } else if (req.url === URL.TIMEOUT) { + if (req.method === 'GET') { + await setTimeout(HTTP_TIMEOUT + 100) + } } // console.debug(`${new Date().toLocaleTimeString()} call res.end "${req.url}"`) From 2b86c84bdfc6940e25b1b2e8bc0824bad9233c0f Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Thu, 2 Mar 2023 22:12:03 +0800 Subject: [PATCH 34/63] 1.7.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index db11819..878152b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@juzi/file-box", - "version": "1.7.1", + "version": "1.7.2", "description": "Pack a File into Box for easy move/transfer between servers no matter of where it is.(local path, remote url, or cloud storage)", "type": "module", "exports": { From 305be464cb74e8df65a6d4bdba4aa053814abab1 Mon Sep 17 00:00:00 2001 From: suchang Date: Fri, 14 Jul 2023 14:15:15 +0800 Subject: [PATCH 35/63] fix: :bug: should close stream when receive error event --- src/file-box.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/file-box.ts b/src/file-box.ts index a411a21..54c247b 100644 --- a/src/file-box.ts +++ b/src/file-box.ts @@ -921,7 +921,10 @@ class FileBox implements Pipeable, FileBoxInterface { await new Promise((resolve, reject) => { writeStream .once('close', resolve) - .once('error', reject) + .once('error', () => { + writeStream.close() + reject() + }) this.pipe(writeStream) }) From 8f22c97136c7df5f9d0ce6408562d8c70bfbaa5b Mon Sep 17 00:00:00 2001 From: suchang Date: Fri, 14 Jul 2023 14:21:55 +0800 Subject: [PATCH 36/63] 1.7.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 878152b..1a693a8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@juzi/file-box", - "version": "1.7.2", + "version": "1.7.3", "description": "Pack a File into Box for easy move/transfer between servers no matter of where it is.(local path, remote url, or cloud storage)", "type": "module", "exports": { From 2f44b3da5374e09f2448179c00a6fa55d20b9d1c Mon Sep 17 00:00:00 2001 From: suchang Date: Fri, 14 Jul 2023 14:24:37 +0800 Subject: [PATCH 37/63] fix: :bug: reject error --- src/file-box.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/file-box.ts b/src/file-box.ts index 54c247b..da2cb84 100644 --- a/src/file-box.ts +++ b/src/file-box.ts @@ -921,9 +921,9 @@ class FileBox implements Pipeable, FileBoxInterface { await new Promise((resolve, reject) => { writeStream .once('close', resolve) - .once('error', () => { + .once('error', (error) => { writeStream.close() - reject() + reject(error) }) this.pipe(writeStream) From cef273848dd3e6c735c875798873834cfb586797 Mon Sep 17 00:00:00 2001 From: suchang Date: Fri, 14 Jul 2023 14:27:17 +0800 Subject: [PATCH 38/63] fix: remove useless blank line --- src/version.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/version.ts b/src/version.ts index 2d26f07..c36f2c5 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,4 +1,3 @@ - /** * This file will be overwrite when we publish NPM module * by scripts/generate_version.ts From c23f1ba8c32049eca40a7bbe14c775694bed94b4 Mon Sep 17 00:00:00 2001 From: suchang Date: Fri, 14 Jul 2023 14:32:29 +0800 Subject: [PATCH 39/63] fix: tscongif ci --- tsconfig.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tsconfig.json b/tsconfig.json index cd74e88..9e7f33d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,6 +2,9 @@ "extends": "@chatie/tsconfig", "compilerOptions": { "outDir": "dist/esm", + "verbatimModuleSyntax": false, + // See: https://github.com/wechaty/wechaty/issues/2551 + "ignoreDeprecations": "5.0" }, "exclude": [ "node_modules/", From 22350c1622e0de85edcb94995e69d9d3bd60db01 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Fri, 27 Oct 2023 12:23:49 +0800 Subject: [PATCH 40/63] =?UTF-8?q?fix:=20=F0=9F=90=9B=20fix=20redirect?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/misc.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/misc.ts b/src/misc.ts index 5f0791d..6681972 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -19,6 +19,7 @@ export function dataUrlToBase64 (dataUrl: string): string { */ export async function httpHeadHeader (url: string): Promise { + const originUrl = url let REDIRECT_TTL = 7 while (true) { @@ -29,6 +30,9 @@ export async function httpHeadHeader (url: string): Promise { + const headHeaders = await httpHeadHeader(url) + if (headHeaders.location) { + url = headHeaders.location + } + const parsedUrl = new URL(url) const protocol = parsedUrl.protocol @@ -124,7 +133,6 @@ export async function httpStream ( ...headers, } - const headHeaders = await httpHeadHeader(url) const fileSize = Number(headHeaders['content-length']) if (headHeaders['accept-ranges'] === 'bytes' && fileSize > HTTP_CHUNK_SIZE) { From 4da443b8f163f30e1cf2392aa27eabe478786565 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Fri, 27 Oct 2023 13:46:07 +0800 Subject: [PATCH 41/63] 1.7.4 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1a693a8..58e495c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@juzi/file-box", - "version": "1.7.3", + "version": "1.7.4", "description": "Pack a File into Box for easy move/transfer between servers no matter of where it is.(local path, remote url, or cloud storage)", "type": "module", "exports": { From 5b9ff334bf62279f4fafdec34dbc257f8e9d4051 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Fri, 27 Oct 2023 17:05:19 +0800 Subject: [PATCH 42/63] =?UTF-8?q?fix:=20=F0=9F=90=9B=20fix=20download=20wi?= =?UTF-8?q?th=20chunk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/config.ts | 2 + src/misc.ts | 282 +++++++++++++++++--------------------------------- 2 files changed, 96 insertions(+), 188 deletions(-) diff --git a/src/config.ts b/src/config.ts index e8b1c8c..78260c4 100644 --- a/src/config.ts +++ b/src/config.ts @@ -4,5 +4,7 @@ export { VERSION } from './version.js' export const HTTP_TIMEOUT = Number(process.env['FILEBOX_HTTP_TIMEOUT']) || 60 * 1000 +export const NO_SLICE_DOWN = process.env['FILEBOX_NO_SLICE_DOWN'] === 'true' + export const HTTP_CHUNK_SIZE = Number(process.env['FILEBOX_HTTP_CHUNK_SIZE']) || 1024 * 512 diff --git a/src/misc.ts b/src/misc.ts index 6681972..2a0589a 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -1,12 +1,30 @@ -import http from 'http' -import https from 'https' +import assert from 'assert' +import { randomUUID } from 'crypto' +import { once } from 'events' +import { createReadStream, createWriteStream } from 'fs' +import { rm } from 'fs/promises' +import http from 'http' +import https from 'https' +import { tmpdir } from 'os' +import { join } from 'path' +import type { Readable } from 'stream' import { URL } from 'url' -import { PassThrough, pipeline, Readable } from 'stream' +import { HTTP_CHUNK_SIZE, HTTP_TIMEOUT, NO_SLICE_DOWN } from './config.js' -import { HTTP_CHUNK_SIZE, HTTP_TIMEOUT } from './config.js' +const protocolMap: { + [key: string]: { request: typeof http.request; agent: http.Agent } +} = { + 'http:': { request: http.request, agent: http.globalAgent }, + 'https:': { request: https.request, agent: https.globalAgent }, +} + +function getProtocol(protocol: string) { + assert(protocolMap[protocol], new Error('unknown protocol: ' + protocol)) + return protocolMap[protocol]! +} -export function dataUrlToBase64 (dataUrl: string): string { +export function dataUrlToBase64(dataUrl: string): string { const dataList = dataUrl.split(',') return dataList[dataList.length - 1]! } @@ -17,8 +35,7 @@ export function dataUrlToBase64 (dataUrl: string): string { * * @credit https://stackoverflow.com/a/43632171/1123955 */ -export async function httpHeadHeader (url: string): Promise { - +export async function httpHeadHeader(url: string): Promise { const originUrl = url let REDIRECT_TTL = 7 @@ -45,10 +62,10 @@ export async function httpHeadHeader (url: string): Promise { + async function _headHeader(destUrl: string): Promise { const parsedUrl = new URL(destUrl) const options = { - method : 'HEAD', + method: 'HEAD', // method : 'GET', } @@ -79,9 +96,7 @@ export async function httpHeadHeader (url: string): Promise { +export async function httpStream(url: string, headers: http.OutgoingHttpHeaders = {}): Promise { const headHeaders = await httpHeadHeader(url) if (headHeaders.location) { url = headHeaders.location @@ -109,206 +121,100 @@ export async function httpStream ( const parsedUrl = new URL(url) - const protocol = parsedUrl.protocol - - const options: http.RequestOptions = {} - - let get: typeof https.get - - if (!protocol) { - throw new Error('protocol is empty') + const { request, agent } = getProtocol(parsedUrl.protocol) + const options: http.RequestOptions = { + method: 'GET', + agent, + headers: { ...headers }, } - if (protocol.match(/^https:/i)) { - get = https.get - options.agent = https.globalAgent - } else if (protocol.match(/^http:/i)) { - get = http.get - options.agent = http.globalAgent - } else { - throw new Error('protocol unknown: ' + protocol) - } - - options.headers = { - ...headers, - } + const fileSize = Number(headHeaders['content-length']) - const fileSize = Number(headHeaders['content-length']) - - if (headHeaders['accept-ranges'] === 'bytes' && fileSize > HTTP_CHUNK_SIZE) { - return await downloadFileInChunks(get, url, options, fileSize, HTTP_CHUNK_SIZE) + if (!NO_SLICE_DOWN && headHeaders['accept-ranges'] === 'bytes' && fileSize > HTTP_CHUNK_SIZE) { + return await downloadFileInChunks(request, url, options, fileSize, HTTP_CHUNK_SIZE) } else { - return await downloadFile(get, url, options) + return await downloadFile(request, url, options) } } -async function downloadFile ( - get: typeof https.get, +async function downloadFile( + request: typeof https.request, url: string, - options: http.RequestOptions, -): Promise { - return new Promise((resolve, reject) => { - let res: http.IncomingMessage | null = null - const req = get(url, options, (response) => { - res = response - resolve(res) + options: http.RequestOptions +): Promise { + const req = request(url, options) + .setTimeout(HTTP_TIMEOUT) + .once('timeout', () => { + req.destroy(new Error(`FileBox: Http request timeout (${HTTP_TIMEOUT})!`)) }) - - req - .on('error', reject) - .setTimeout(HTTP_TIMEOUT, () => { - const e = new Error(`FileBox: Http request timeout (${HTTP_TIMEOUT})!`) - if (res) { - res.emit('error', e) - } - req.emit('error', e) - req.destroy() - }) - .end() - }) + .end() + const [res] = (await once(req, 'response')) as [http.IncomingMessage] + return res } -async function downloadFileInChunks ( - get: typeof https.get, +async function downloadFileInChunks( + request: typeof https.request, url: string, options: http.RequestOptions, fileSize: number, - chunkSize = HTTP_CHUNK_SIZE, + chunkSize = HTTP_CHUNK_SIZE ): Promise { + const tmpFile = join(tmpdir(), `filebox-${randomUUID()}`) + const writeStream = createWriteStream(tmpFile) + const allowStatusCode = [200, 206] const ac = new AbortController() - const stream = new PassThrough() - - const abortAc = () => { - if (!ac.signal.aborted) { - ac.abort() - } + const requestBaseOptions: http.RequestOptions = { + headers: {}, + ...options, + signal: ac.signal, + timeout: HTTP_TIMEOUT, } - - stream - .once('close', abortAc) - .once('error', abortAc) - - const chunksCount = Math.ceil(fileSize / chunkSize) - let chunksDownloaded = 0 - let dataTotalSize = 0 - - const doDownloadChunk = async function (i: number, retries: number) { - const start = i * chunkSize - const end = Math.min((i + 1) * chunkSize - 1, fileSize - 1) + let chunkSeq = 0 + let start = 0 + let end = 0 + let downSize = 0 + let retries = 3 + + while (downSize < fileSize) { + end = Math.min(start + chunkSize, fileSize - 1) const range = `bytes=${start}-${end}` - - // console.info('doDownloadChunk() range:', range) - - if (ac.signal.aborted) { - stream.destroy(new Error('Signal aborted.')) - return - } - - const requestOptions: http.RequestOptions = { - ...options, - signal: ac.signal, - timeout: HTTP_TIMEOUT, - } - if (!requestOptions.headers) { - requestOptions.headers = {} - } + const requestOptions = Object.assign({}, requestBaseOptions) + assert(requestOptions.headers, 'Errors that should not happen: Invalid headers') requestOptions.headers['Range'] = range try { - const chunk = await downloadChunk(get, url, requestOptions, retries) - if (chunk.errored) { - throw new Error('chunk stream error') + const res = await downloadFile(request, url, options) + assert(allowStatusCode.includes(res.statusCode ?? 0), `Request failed with status code ${res.statusCode}`) + assert(Number(res.headers['content-length']) > 0, 'Server returned 0 bytes of data') + for await (const chunk of res) { + assert(Buffer.isBuffer(chunk)) + downSize += chunk.length + writeStream.write(chunk) } - if (chunk.closed) { - throw new Error('chunk stream closed') + res.destroy() + } catch (error) { + const err = error as Error + if (--retries <= 0) { + void rm(tmpFile, { force: true, maxRetries: 5 }) + writeStream.close() + throw new Error(`Download file with chunk failed! ${err.message}`, { cause: err }) } - if (chunk.destroyed) { - throw new Error('chunk stream destroyed') - } - const buf = await streamToBuffer(chunk) - stream.push(buf) - chunksDownloaded++ - dataTotalSize += buf.length - - if (chunksDownloaded === chunksCount || dataTotalSize >= fileSize) { - stream.push(null) - } - } catch (err) { - if (retries === 0) { - stream.emit('error', err) - } else { - await doDownloadChunk(i, retries - 1) - } - } - } - - const doDownloadAllChunks = async function () { - for (let i = 0; i < chunksCount; i++) { - if (ac.signal.aborted) { - return - } - await doDownloadChunk(i, 3) } + chunkSeq++ + start = downSize } + writeStream.close() - void doDownloadAllChunks().catch((e) => { - stream.emit('error', e) - }) - - return stream -} - -async function downloadChunk ( - get: typeof https.get, - url: string, - requestOptions: http.RequestOptions, - retries: number, -): Promise { - return new Promise((resolve, reject) => { - const doRequest = (attempt: number) => { - let resolved = false - const req = get(url, requestOptions, (res) => { - const statusCode = res.statusCode ?? 0 - // console.info('downloadChunk(%d) statusCode: %d rsp.headers: %o', attempt, statusCode, res.headers) - - if (statusCode < 200 || statusCode >= 300) { - if (attempt < retries) { - void doRequest(attempt + 1) - } else { - reject(new Error(`Request failed with status code ${res.statusCode}`)) - } - return - } - - const stream = pipeline(res, new PassThrough(), () => {}) - resolve(stream) - resolved = true - }) - - req - .once('error', (err) => { - if (resolved) { - return - } - // console.error('downloadChunk(%d) req error:', attempt, err) - if (attempt < retries) { - void doRequest(attempt + 1) - } else { - reject(err) - } - }) - .setTimeout(HTTP_TIMEOUT, () => { - const e = new Error(`FileBox: Http request timeout (${HTTP_TIMEOUT})!`) - req.emit('error', e) - req.destroy() - }) - .end() - } - void doRequest(0) - }) + const readStream = createReadStream(tmpFile) + readStream + .once('end', () => readStream.close()) + .once('close', () => { + void rm(tmpFile, { force: true, maxRetries: 5 }) + }) + return readStream } -export async function streamToBuffer (stream: Readable): Promise { +export async function streamToBuffer(stream: Readable): Promise { return new Promise((resolve, reject) => { const bufferList: Buffer[] = [] stream.once('error', reject) From d59e2eb0fed10f670eb7d4e39819aec41934057c Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Fri, 27 Oct 2023 17:35:47 +0800 Subject: [PATCH 43/63] =?UTF-8?q?docs:=20=F0=9F=93=9D=20update=20README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f5cd14a..06ab007 100644 --- a/README.md +++ b/README.md @@ -479,6 +479,8 @@ console.log(fileBox.remoteSize) Environment variables can be used to control some behavior. - `FILEBOX_HTTP_TIMEOUT` [default=60000] Socket idle timeout when FileBox downloads data from URL. For example, when the network is temporarily interrupted, the request will be considered as failed after waiting for a specified time. +- `NO_SLICE_DOWN` [default=false] Whether to turn off slice downloading. If the network is unstable, an error while downloading the file will cause the file download to fail. If it is not closed, the file will be divided into multiple fragments for downloading, and breakpoint re-downloading is supported. +- `HTTP_CHUNK_SIZE` [default=524288] When downloading a file using slicing, the number of bytes in each slice. ## SCHEMAS From 6a0e10eb586bfbe3f7a37178acd6cc069a1122dd Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Fri, 27 Oct 2023 18:11:02 +0800 Subject: [PATCH 44/63] refactor: refactor streamToBuffer --- src/misc.ts | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/misc.ts b/src/misc.ts index 2a0589a..d2590df 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -215,13 +215,9 @@ async function downloadFileInChunks( } export async function streamToBuffer(stream: Readable): Promise { - return new Promise((resolve, reject) => { - const bufferList: Buffer[] = [] - stream.once('error', reject) - stream.once('end', () => { - const fullBuffer = Buffer.concat(bufferList) - resolve(fullBuffer) - }) - stream.on('data', (buffer) => bufferList.push(buffer)) - }) + const chunks: Buffer[] = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + return Buffer.concat(chunks) } From 4034632cb1a114f7f5b119382e6d8451e916e754 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Fri, 27 Oct 2023 18:31:48 +0800 Subject: [PATCH 45/63] =?UTF-8?q?fix:=20=F0=9F=90=9B=20fix=20res=20not=20c?= =?UTF-8?q?lose=20in=20httpHeadHeader?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/misc.ts | 71 ++++++++++++++--------------------------------------- 1 file changed, 19 insertions(+), 52 deletions(-) diff --git a/src/misc.ts b/src/misc.ts index d2590df..7f316f2 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -44,7 +44,10 @@ export async function httpHeadHeader(url: string): Promise${REDIRECT_TTL}) 302 redirection.`) } - const res = await _headHeader(url) + const res = await fetch(url, { + method: 'HEAD', + }) + res.destroy() if (!/^3/.test(String(res.statusCode))) { if (originUrl !== url) { @@ -61,39 +64,6 @@ export async function httpHeadHeader(url: string): Promise { - const parsedUrl = new URL(destUrl) - const options = { - method: 'HEAD', - // method : 'GET', - } - - let request: typeof http.request - - if (parsedUrl.protocol === 'https:') { - request = https.request - } else if (parsedUrl.protocol === 'http:') { - request = http.request - } else { - throw new Error('unknown protocol: ' + parsedUrl.protocol) - } - - return new Promise((resolve, reject) => { - let res: undefined | http.IncomingMessage - const req = request(parsedUrl, options, (response) => { - res = response - resolve(res) - }) - .on('error', reject) - .setTimeout(HTTP_TIMEOUT, () => { - const e = new Error(`FileBox: Http request timeout (${HTTP_TIMEOUT})!`) - req.emit('error', e) - req.destroy() - }) - .end() - }) - } } export function httpHeaderToFileName(headers: http.IncomingHttpHeaders): null | string { @@ -117,32 +87,32 @@ export async function httpStream(url: string, headers: http.OutgoingHttpHeaders const headHeaders = await httpHeadHeader(url) if (headHeaders.location) { url = headHeaders.location + const { protocol } = new URL(url) + getProtocol(protocol) } - const parsedUrl = new URL(url) - - const { request, agent } = getProtocol(parsedUrl.protocol) const options: http.RequestOptions = { method: 'GET', - agent, headers: { ...headers }, } const fileSize = Number(headHeaders['content-length']) if (!NO_SLICE_DOWN && headHeaders['accept-ranges'] === 'bytes' && fileSize > HTTP_CHUNK_SIZE) { - return await downloadFileInChunks(request, url, options, fileSize, HTTP_CHUNK_SIZE) + return await downloadFileInChunks(url, options, fileSize, HTTP_CHUNK_SIZE) } else { - return await downloadFile(request, url, options) + return await fetch(url, options) } } -async function downloadFile( - request: typeof https.request, - url: string, - options: http.RequestOptions -): Promise { - const req = request(url, options) +async function fetch(url: string, options: http.RequestOptions): Promise { + const { protocol } = new URL(url) + const { request, agent } = getProtocol(protocol) + const opts = { + agent, + ...options, + } + const req = request(url, opts) .setTimeout(HTTP_TIMEOUT) .once('timeout', () => { req.destroy(new Error(`FileBox: Http request timeout (${HTTP_TIMEOUT})!`)) @@ -153,7 +123,6 @@ async function downloadFile( } async function downloadFileInChunks( - request: typeof https.request, url: string, options: http.RequestOptions, fileSize: number, @@ -162,11 +131,9 @@ async function downloadFileInChunks( const tmpFile = join(tmpdir(), `filebox-${randomUUID()}`) const writeStream = createWriteStream(tmpFile) const allowStatusCode = [200, 206] - const ac = new AbortController() const requestBaseOptions: http.RequestOptions = { headers: {}, ...options, - signal: ac.signal, timeout: HTTP_TIMEOUT, } let chunkSeq = 0 @@ -183,7 +150,7 @@ async function downloadFileInChunks( requestOptions.headers['Range'] = range try { - const res = await downloadFile(request, url, options) + const res = await fetch(url, options) assert(allowStatusCode.includes(res.statusCode ?? 0), `Request failed with status code ${res.statusCode}`) assert(Number(res.headers['content-length']) > 0, 'Server returned 0 bytes of data') for await (const chunk of res) { @@ -195,7 +162,7 @@ async function downloadFileInChunks( } catch (error) { const err = error as Error if (--retries <= 0) { - void rm(tmpFile, { force: true, maxRetries: 5 }) + void rm(tmpFile, { force: true }) writeStream.close() throw new Error(`Download file with chunk failed! ${err.message}`, { cause: err }) } @@ -209,7 +176,7 @@ async function downloadFileInChunks( readStream .once('end', () => readStream.close()) .once('close', () => { - void rm(tmpFile, { force: true, maxRetries: 5 }) + void rm(tmpFile, { force: true }) }) return readStream } From f3feb79ec2949f788ad4c21d505d221beb932075 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Fri, 27 Oct 2023 18:39:46 +0800 Subject: [PATCH 46/63] =?UTF-8?q?style:=20=F0=9F=9A=A8=20fix=20lint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/misc.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/misc.ts b/src/misc.ts index 7f316f2..39b4bc3 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -13,10 +13,10 @@ import { URL } from 'url' import { HTTP_CHUNK_SIZE, HTTP_TIMEOUT, NO_SLICE_DOWN } from './config.js' const protocolMap: { - [key: string]: { request: typeof http.request; agent: http.Agent } + [key: string]: { agent: http.Agent; request: typeof http.request } } = { - 'http:': { request: http.request, agent: http.globalAgent }, - 'https:': { request: https.request, agent: https.globalAgent }, + 'http:': { agent: http.globalAgent, request: http.request }, + 'https:': { agent: https.globalAgent, request: https.request }, } function getProtocol(protocol: string) { @@ -92,8 +92,8 @@ export async function httpStream(url: string, headers: http.OutgoingHttpHeaders } const options: http.RequestOptions = { - method: 'GET', headers: { ...headers }, + method: 'GET', } const fileSize = Number(headHeaders['content-length']) @@ -118,7 +118,7 @@ async function fetch(url: string, options: http.RequestOptions): Promise Date: Fri, 27 Oct 2023 18:40:54 +0800 Subject: [PATCH 47/63] =?UTF-8?q?style:=20=F0=9F=9A=A8=20fix=20lint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/misc.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/misc.ts b/src/misc.ts index 39b4bc3..d12615d 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -19,12 +19,12 @@ const protocolMap: { 'https:': { agent: https.globalAgent, request: https.request }, } -function getProtocol(protocol: string) { +function getProtocol (protocol: string) { assert(protocolMap[protocol], new Error('unknown protocol: ' + protocol)) return protocolMap[protocol]! } -export function dataUrlToBase64(dataUrl: string): string { +export function dataUrlToBase64 (dataUrl: string): string { const dataList = dataUrl.split(',') return dataList[dataList.length - 1]! } @@ -35,7 +35,7 @@ export function dataUrlToBase64(dataUrl: string): string { * * @credit https://stackoverflow.com/a/43632171/1123955 */ -export async function httpHeadHeader(url: string): Promise { +export async function httpHeadHeader (url: string): Promise { const originUrl = url let REDIRECT_TTL = 7 @@ -66,7 +66,7 @@ export async function httpHeadHeader(url: string): Promise { +export async function httpStream (url: string, headers: http.OutgoingHttpHeaders = {}): Promise { const headHeaders = await httpHeadHeader(url) if (headHeaders.location) { url = headHeaders.location @@ -105,7 +105,7 @@ export async function httpStream(url: string, headers: http.OutgoingHttpHeaders } } -async function fetch(url: string, options: http.RequestOptions): Promise { +async function fetch (url: string, options: http.RequestOptions): Promise { const { protocol } = new URL(url) const { request, agent } = getProtocol(protocol) const opts = { @@ -122,7 +122,7 @@ async function fetch(url: string, options: http.RequestOptions): Promise { +export async function streamToBuffer (stream: Readable): Promise { const chunks: Buffer[] = [] for await (const chunk of stream) { chunks.push(chunk) From 67761236209106c750a6d11e56f8b83420792472 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Fri, 27 Oct 2023 18:42:33 +0800 Subject: [PATCH 48/63] =?UTF-8?q?style:=20=F0=9F=9A=A8=20fix=20lint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/misc.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/misc.ts b/src/misc.ts index d12615d..c4cb5e8 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -126,11 +126,11 @@ async function downloadFileInChunks ( url: string, options: http.RequestOptions, fileSize: number, - chunkSize = HTTP_CHUNK_SIZE + chunkSize = HTTP_CHUNK_SIZE, ): Promise { const tmpFile = join(tmpdir(), `filebox-${randomUUID()}`) const writeStream = createWriteStream(tmpFile) - const allowStatusCode = [200, 206] + const allowStatusCode = [ 200, 206 ] const requestBaseOptions: http.RequestOptions = { headers: {}, ...options, From ff024ccd7f7b2e68432084bf1dfe4ebd343d556d Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Mon, 30 Oct 2023 15:29:50 +0800 Subject: [PATCH 49/63] =?UTF-8?q?fix:=20=F0=9F=90=9B=20fix=20params?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/misc.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/misc.ts b/src/misc.ts index c4cb5e8..172505c 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -150,7 +150,7 @@ async function downloadFileInChunks ( requestOptions.headers['Range'] = range try { - const res = await fetch(url, options) + const res = await fetch(url, requestOptions) assert(allowStatusCode.includes(res.statusCode ?? 0), `Request failed with status code ${res.statusCode}`) assert(Number(res.headers['content-length']) > 0, 'Server returned 0 bytes of data') for await (const chunk of res) { From 41a7b948305f8c9dc2d76780659e7d6af7133284 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Mon, 30 Oct 2023 15:31:01 +0800 Subject: [PATCH 50/63] =?UTF-8?q?feat:=20=F0=9F=92=A5=20split=20env=20for?= =?UTF-8?q?=20http=20timeout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/config.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/config.ts b/src/config.ts index 78260c4..4e42ea5 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,7 +1,10 @@ /// export { VERSION } from './version.js' -export const HTTP_TIMEOUT = Number(process.env['FILEBOX_HTTP_TIMEOUT']) +export const HTTP_REQUEST_TIMEOUT = Number(process.env['FILEBOX_HTTP_REQUEST_TIMEOUT']) + || 10 * 1000 + +export const HTTP_RESPONSE_TIMEOUT = Number(process.env['FILEBOX_HTTP_RESPONSE_TIMEOUT']) || 60 * 1000 export const NO_SLICE_DOWN = process.env['FILEBOX_NO_SLICE_DOWN'] === 'true' From 6e39f4e971c607d3777623033abf3fc0ee21a490 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Mon, 30 Oct 2023 15:31:46 +0800 Subject: [PATCH 51/63] =?UTF-8?q?fix:=20=F0=9F=90=9B=20fix=20request=20tim?= =?UTF-8?q?eout=20and=20catch=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/misc.ts | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/misc.ts b/src/misc.ts index 172505c..4dfdffb 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -10,7 +10,7 @@ import { join } from 'path' import type { Readable } from 'stream' import { URL } from 'url' -import { HTTP_CHUNK_SIZE, HTTP_TIMEOUT, NO_SLICE_DOWN } from './config.js' +import { HTTP_CHUNK_SIZE, HTTP_REQUEST_TIMEOUT, HTTP_RESPONSE_TIMEOUT, NO_SLICE_DOWN } from './config.js' const protocolMap: { [key: string]: { agent: http.Agent; request: typeof http.request } @@ -108,17 +108,27 @@ export async function httpStream (url: string, headers: http.OutgoingHttpHeaders async function fetch (url: string, options: http.RequestOptions): Promise { const { protocol } = new URL(url) const { request, agent } = getProtocol(protocol) - const opts = { + const opts: http.RequestOptions = { agent, ...options, } - const req = request(url, opts) - .setTimeout(HTTP_TIMEOUT) - .once('timeout', () => { - req.destroy(new Error(`FileBox: Http request timeout (${HTTP_TIMEOUT})!`)) + const req = request(url, opts).end() + req + .on('error', () => { + req.destroy() + }) + .setTimeout(HTTP_REQUEST_TIMEOUT, () => { + req.emit('error', new Error(`FileBox: Http request timeout (${HTTP_REQUEST_TIMEOUT})!`)) + }) + const responseEvents = await once(req, 'response') + const res = responseEvents[0] as http.IncomingMessage + res + .on('error', () => { + res.destroy() + }) + .setTimeout(HTTP_RESPONSE_TIMEOUT, () => { + res.emit('error', new Error(`FileBox: Http response timeout (${HTTP_RESPONSE_TIMEOUT})!`)) }) - .end() - const [ res ] = (await once(req, 'response')) as [ http.IncomingMessage ] return res } @@ -134,7 +144,6 @@ async function downloadFileInChunks ( const requestBaseOptions: http.RequestOptions = { headers: {}, ...options, - timeout: HTTP_TIMEOUT, } let chunkSeq = 0 let start = 0 From a084436ac856196f322add9f0ee640cb384f041f Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Mon, 30 Oct 2023 15:39:50 +0800 Subject: [PATCH 52/63] =?UTF-8?q?test:=20=E2=9C=85=20fix=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/network-timeout.spec.ts | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/tests/network-timeout.spec.ts b/tests/network-timeout.spec.ts index fbacf77..e4db9c7 100755 --- a/tests/network-timeout.spec.ts +++ b/tests/network-timeout.spec.ts @@ -5,10 +5,9 @@ import type { AddressInfo } from 'net' import { setTimeout } from 'timers/promises' import { sinon, test } from 'tstest' +import { HTTP_REQUEST_TIMEOUT, HTTP_RESPONSE_TIMEOUT } from '../src/config.js' import { FileBox } from '../src/mod.js' -import { HTTP_TIMEOUT } from '../src/config.js' - test('slow network stall HTTP_TIMEOUT', async (t) => { const sandbox = sinon.createSandbox() sandbox.useFakeTimers({ @@ -30,13 +29,13 @@ test('slow network stall HTTP_TIMEOUT', async (t) => { res.write(Buffer.from('This is the first chunk of data.')) if (req.url === URL.NOT_TIMEOUT) { - await setTimeout(HTTP_TIMEOUT * 0.5) + await setTimeout(HTTP_REQUEST_TIMEOUT * 0.5) res.write(Buffer.from('This is the second chunk of data.')) } else if (req.url === URL.READY) { - await setTimeout(HTTP_TIMEOUT + 100) + await setTimeout(HTTP_REQUEST_TIMEOUT + 100) } else if (req.url === URL.TIMEOUT) { if (req.method === 'GET') { - await setTimeout(HTTP_TIMEOUT + 100) + await setTimeout(HTTP_RESPONSE_TIMEOUT + 100) } } @@ -77,15 +76,15 @@ test('slow network stall HTTP_TIMEOUT', async (t) => { // FIXME: tickAsync does not work on socket timeout await new Promise((resolve) => { stream.once('error', resolve).on('close', resolve) - // resolve(setTimeout(HTTP_TIMEOUT)) + // resolve(setTimeout(HTTP_REQUEST_TIMEOUT)) }) await sandbox.clock.tickAsync(1) - // await sandbox.clock.tickAsync(HTTP_TIMEOUT) + // await sandbox.clock.tickAsync(HTTP_RESPONSE_TIMEOUT) - // t.comment('recv data count:', dataSpy.callCount) - // t.comment('recv error count:', errorSpy.callCount) - t.ok(dataSpy.calledThrice, `should get chunk 3 after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) - t.ok(errorSpy.notCalled, `should not get error after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) + t.comment('recv data count:', dataSpy.callCount) + t.comment('recv error count:', errorSpy.callCount) + t.ok(dataSpy.calledThrice, `should get chunk 3 after TIMEOUT ${HTTP_REQUEST_TIMEOUT} (${Date.now() - start} passed)`) + t.ok(errorSpy.notCalled, `should not get error after TIMEOUT ${HTTP_REQUEST_TIMEOUT} (${Date.now() - start} passed)`) t.end() }).catch(t.threw) @@ -117,14 +116,14 @@ test('slow network stall HTTP_TIMEOUT', async (t) => { // FIXME: tickAsync does not work on socket timeout await new Promise((resolve) => { stream.once('error', resolve).on('close', resolve) - // resolve(setTimeout(HTTP_TIMEOUT)) + // resolve(setTimeout(HTTP_RESPONSE_TIMEOUT)) }) await sandbox.clock.tickAsync(1) - // await sandbox.clock.tickAsync(HTTP_TIMEOUT) + // await sandbox.clock.tickAsync(HTTP_RESPONSE_TIMEOUT) // t.comment('recv data count:', dataSpy.callCount) // t.comment('recv error count:', errorSpy.callCount) - t.ok(errorSpy.calledOnce, `should get error after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) + t.ok(errorSpy.calledOnce, `should get error after TIMEOUT ${HTTP_RESPONSE_TIMEOUT} (${Date.now() - start} passed)`) t.end() }).catch(t.threw) @@ -140,7 +139,7 @@ test('slow network stall HTTP_TIMEOUT', async (t) => { await sandbox.clock.tickAsync(1) // t.comment('recv error count:', errorSpy.callCount) - t.ok(errorSpy.calledOnce, `should get error after TIMEOUT ${HTTP_TIMEOUT} (${Date.now() - start} passed)`) + t.ok(errorSpy.calledOnce, `should get error after TIMEOUT ${HTTP_REQUEST_TIMEOUT} (${Date.now() - start} passed)`) t.end() }).catch(t.threw) }) From 286fc6f0bef07233996e49dc0749b9933c598119 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Mon, 30 Oct 2023 15:46:21 +0800 Subject: [PATCH 53/63] =?UTF-8?q?chore:=20=E2=9E=95=20add=20some=20dev=20d?= =?UTF-8?q?eps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 58e495c..7ee6501 100644 --- a/package.json +++ b/package.json @@ -52,12 +52,15 @@ "@chatie/tsconfig": "^4.6.2", "@types/isomorphic-fetch": "0.0.35", "@types/mime": "^2.0.3", + "@types/node": "^18.18.7", "@types/qrcode": "^1.4.1", "@types/uuid": "^8.3.3", + "eslint-plugin-n": "^16.2.0", "gts": "^3.1.0", "pkg-jq": "^0.2.11", "read-pkg-up": "^8.0.0", - "reflect-metadata": "^0.1.13" + "reflect-metadata": "^0.1.13", + "tap": "^16.3.9" }, "dependencies": { "brolog": "^1.14.2", From 5cdab18d22bd8ab7e0ca6116aad24f6577ba4016 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Mon, 30 Oct 2023 15:52:03 +0800 Subject: [PATCH 54/63] feat: Compatible with old env --- src/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.ts b/src/config.ts index 4e42ea5..c98c184 100644 --- a/src/config.ts +++ b/src/config.ts @@ -4,7 +4,7 @@ export { VERSION } from './version.js' export const HTTP_REQUEST_TIMEOUT = Number(process.env['FILEBOX_HTTP_REQUEST_TIMEOUT']) || 10 * 1000 -export const HTTP_RESPONSE_TIMEOUT = Number(process.env['FILEBOX_HTTP_RESPONSE_TIMEOUT']) +export const HTTP_RESPONSE_TIMEOUT = Number(process.env['FILEBOX_HTTP_RESPONSE_TIMEOUT'] ?? process.env['FILEBOX_HTTP_TIMEOUT']) || 60 * 1000 export const NO_SLICE_DOWN = process.env['FILEBOX_NO_SLICE_DOWN'] === 'true' From 69169f0dbedeaad2779216f097b1b2ab268b15cf Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Mon, 30 Oct 2023 15:54:13 +0800 Subject: [PATCH 55/63] =?UTF-8?q?docs:=20=F0=9F=93=9D=20update=20README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 06ab007..d13a7ea 100644 --- a/README.md +++ b/README.md @@ -478,7 +478,9 @@ console.log(fileBox.remoteSize) Environment variables can be used to control some behavior. -- `FILEBOX_HTTP_TIMEOUT` [default=60000] Socket idle timeout when FileBox downloads data from URL. For example, when the network is temporarily interrupted, the request will be considered as failed after waiting for a specified time. +- `FILEBOX_HTTP_REQUEST_TIMEOUT` [default=10000] The timeout period for establishing a communication request with the server. For example, if the network is unavailable or the delay is too high, communication cannot be successfully established. +- `FILEBOX_HTTP_RESPONSE_TIMEOUT` [default=60000] Socket idle timeout when FileBox downloads data from URL. For example, when the network is temporarily interrupted, the request will be considered as failed after waiting for a specified time. +- ~~`FILEBOX_HTTP_TIMEOUT` [default=60000]~~ **Deprecated!** Please use `FILEBOX_HTTP_RESPONSE_TIMEOUT`. - `NO_SLICE_DOWN` [default=false] Whether to turn off slice downloading. If the network is unstable, an error while downloading the file will cause the file download to fail. If it is not closed, the file will be divided into multiple fragments for downloading, and breakpoint re-downloading is supported. - `HTTP_CHUNK_SIZE` [default=524288] When downloading a file using slicing, the number of bytes in each slice. From 03e685f902a6b0d8008ba94071f57354470a2ddb Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Mon, 30 Oct 2023 15:55:23 +0800 Subject: [PATCH 56/63] 1.7.5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7ee6501..b90ce31 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@juzi/file-box", - "version": "1.7.4", + "version": "1.7.5", "description": "Pack a File into Box for easy move/transfer between servers no matter of where it is.(local path, remote url, or cloud storage)", "type": "module", "exports": { From 03203e7e83156077dd805b26797680acea30c6ee Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Mon, 30 Oct 2023 16:06:05 +0800 Subject: [PATCH 57/63] 1.7.6 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b90ce31..ad8c639 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@juzi/file-box", - "version": "1.7.5", + "version": "1.7.6", "description": "Pack a File into Box for easy move/transfer between servers no matter of where it is.(local path, remote url, or cloud storage)", "type": "module", "exports": { From 0a603a5de077acd3402fe5884e8e0546ac44c497 Mon Sep 17 00:00:00 2001 From: NickWang Date: Wed, 21 Feb 2024 18:13:21 +0800 Subject: [PATCH 58/63] =?UTF-8?q?feat:=20=F0=9F=8E=B8=20export=20misc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/file-box.ts | 1 + src/mod.ts | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/file-box.ts b/src/file-box.ts index da2cb84..0a1932d 100644 --- a/src/file-box.ts +++ b/src/file-box.ts @@ -503,6 +503,7 @@ class FileBox implements Pipeable, FileBoxInterface { */ private readonly base64? : string private readonly remoteUrl? : string + public url? : string private readonly qrCode? : string private readonly uuid? : string diff --git a/src/mod.ts b/src/mod.ts index 9d50c67..cd362e4 100644 --- a/src/mod.ts +++ b/src/mod.ts @@ -31,3 +31,5 @@ export { UniformResourceNameRegistry, VERSION, } + +export * from './misc.js' From 164d85d137e64970d9ed4886c53cfc1da3cff9c2 Mon Sep 17 00:00:00 2001 From: NickWang Date: Wed, 21 Feb 2024 18:13:28 +0800 Subject: [PATCH 59/63] 1.7.7 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ad8c639..ed6bb9a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@juzi/file-box", - "version": "1.7.6", + "version": "1.7.7", "description": "Pack a File into Box for easy move/transfer between servers no matter of where it is.(local path, remote url, or cloud storage)", "type": "module", "exports": { From 98bcfb8d2c01e6ad9a6f1d07731e5eb7c80d6fcf Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Mon, 3 Jun 2024 10:26:37 +0800 Subject: [PATCH 60/63] =?UTF-8?q?feat:=20=E2=9C=A8=20support=20http=20prox?= =?UTF-8?q?y?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 1 + src/config.ts | 6 ++++++ src/misc.spec.ts | 4 ++-- src/misc.ts | 36 ++++++++++++++++++++++++++++++++++-- 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index ad8c639..026ba56 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "gts": "^3.1.0", "pkg-jq": "^0.2.11", "read-pkg-up": "^8.0.0", + "https-proxy-agent": "^5.0.1", "reflect-metadata": "^0.1.13", "tap": "^16.3.9" }, diff --git a/src/config.ts b/src/config.ts index c98c184..1b26e56 100644 --- a/src/config.ts +++ b/src/config.ts @@ -11,3 +11,9 @@ export const NO_SLICE_DOWN = process.env['FILEBOX_NO_SLICE_DOWN'] === 'true' export const HTTP_CHUNK_SIZE = Number(process.env['FILEBOX_HTTP_CHUNK_SIZE']) || 1024 * 512 + +export const PROXY_TYPE = process.env['PROXY_TYPE'] +export const PROXY_HOST = process.env['PROXY_HOST'] || '' +export const PROXY_PORT = Number(process.env['PROXY_PORT']) || 0 +export const PROXY_USERNAME = process.env['PROXY_USERNAME'] || '' +export const PROXY_PASSWORD = process.env['PROXY_PASSWORD'] || '' diff --git a/src/misc.spec.ts b/src/misc.spec.ts index 28e5a7b..4563d1b 100755 --- a/src/misc.spec.ts +++ b/src/misc.spec.ts @@ -1,7 +1,7 @@ #!/usr/bin/env -S node --no-warnings --loader ts-node/esm // tslint:disable:no-shadowed-variable -import { test } from 'tstest' +import { test } from 'tstest' import { dataUrlToBase64, @@ -9,7 +9,7 @@ import { httpHeadHeader, httpStream, streamToBuffer, -} from './misc.js' +} from './misc.js' test('dataUrl to base64', async t => { const base64 = [ diff --git a/src/misc.ts b/src/misc.ts index 4dfdffb..da110a6 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -3,14 +3,25 @@ import { randomUUID } from 'crypto' import { once } from 'events' import { createReadStream, createWriteStream } from 'fs' import { rm } from 'fs/promises' -import http from 'http' +import http, { RequestOptions } from 'http' import https from 'https' +import { HttpsProxyAgent } from 'https-proxy-agent' import { tmpdir } from 'os' import { join } from 'path' import type { Readable } from 'stream' import { URL } from 'url' -import { HTTP_CHUNK_SIZE, HTTP_REQUEST_TIMEOUT, HTTP_RESPONSE_TIMEOUT, NO_SLICE_DOWN } from './config.js' +import { + HTTP_CHUNK_SIZE, + HTTP_REQUEST_TIMEOUT, + HTTP_RESPONSE_TIMEOUT, + NO_SLICE_DOWN, + PROXY_HOST, + PROXY_PASSWORD, + PROXY_PORT, + PROXY_TYPE, + PROXY_USERNAME, +} from './config.js' const protocolMap: { [key: string]: { agent: http.Agent; request: typeof http.request } @@ -112,6 +123,7 @@ async function fetch (url: string, options: http.RequestOptions): Promise { @@ -197,3 +209,23 @@ export async function streamToBuffer (stream: Readable): Promise { } return Buffer.concat(chunks) } + +function getProxyUrl () { + const proxyType = PROXY_TYPE + const proxyHost = PROXY_HOST + const proxyPort = PROXY_PORT + const proxyUsername = PROXY_USERNAME + const proxyPassword = PROXY_PASSWORD + if (proxyType === 'http') { + return `http://${proxyUsername}:${proxyPassword}@${proxyHost}:${proxyPort}` + } + return '' +} + +function setProxy (options: RequestOptions): void { + const url = getProxyUrl() + if (url) { + const agent = new HttpsProxyAgent(url) + options.agent = agent + } +} From 13870f440971e554be37c2c8e82d907b8b01dc77 Mon Sep 17 00:00:00 2001 From: binsee <5285894+binsee@users.noreply.github.com> Date: Mon, 3 Jun 2024 10:28:51 +0800 Subject: [PATCH 61/63] 1.7.7 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 026ba56..50f53cc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@juzi/file-box", - "version": "1.7.6", + "version": "1.7.7", "description": "Pack a File into Box for easy move/transfer between servers no matter of where it is.(local path, remote url, or cloud storage)", "type": "module", "exports": { From 02303a4baf61af7d87a7e7480ba410d37b3729ce Mon Sep 17 00:00:00 2001 From: NickWang Date: Mon, 3 Jun 2024 10:45:37 +0800 Subject: [PATCH 62/63] =?UTF-8?q?fix:=20=F0=9F=90=9B=20env?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/config.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/config.ts b/src/config.ts index 1b26e56..37ff45b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -12,8 +12,8 @@ export const NO_SLICE_DOWN = process.env['FILEBOX_NO_SLICE_DOWN'] === 'true' export const HTTP_CHUNK_SIZE = Number(process.env['FILEBOX_HTTP_CHUNK_SIZE']) || 1024 * 512 -export const PROXY_TYPE = process.env['PROXY_TYPE'] -export const PROXY_HOST = process.env['PROXY_HOST'] || '' -export const PROXY_PORT = Number(process.env['PROXY_PORT']) || 0 -export const PROXY_USERNAME = process.env['PROXY_USERNAME'] || '' -export const PROXY_PASSWORD = process.env['PROXY_PASSWORD'] || '' +export const PROXY_TYPE = process.env['FILEBOX_PROXY_TYPE'] +export const PROXY_HOST = process.env['FILEBOX_PROXY_HOST'] || '' +export const PROXY_PORT = Number(process.env['FILEBOX_PROXY_PORT']) || 0 +export const PROXY_USERNAME = process.env['FILEBOX_PROXY_USERNAME'] || '' +export const PROXY_PASSWORD = process.env['FILEBOX_PROXY_PASSWORD'] || '' From 5bdb6022ef793b11b9ecac6d3087d42329d3b46d Mon Sep 17 00:00:00 2001 From: NickWang Date: Mon, 3 Jun 2024 10:45:40 +0800 Subject: [PATCH 63/63] 1.7.8 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 50f53cc..3677ea3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@juzi/file-box", - "version": "1.7.7", + "version": "1.7.8", "description": "Pack a File into Box for easy move/transfer between servers no matter of where it is.(local path, remote url, or cloud storage)", "type": "module", "exports": {