From fded0703221a8216548f27c4d93ec0affc1a3c4b Mon Sep 17 00:00:00 2001 From: lino-levan <11367844+lino-levan@users.noreply.github.com> Date: Sat, 9 Sep 2023 15:22:49 +1200 Subject: [PATCH 1/3] feat(streams): arrayBuffer/blob/json/textFromReadableStream --- streams/array_buffer_from_readable_stream.ts | 23 +++++++++++++ .../array_buffer_from_readable_stream_test.ts | 18 ++++++++++ streams/blob_from_readable_stream.ts | 21 ++++++++++++ streams/blob_from_readable_stream_test.ts | 23 +++++++++++++ streams/json_from_readable_stream.ts | 13 +++++++ streams/json_from_readable_stream_test.ts | 34 +++++++++++++++++++ streams/mod.ts | 4 +++ streams/text_from_readable_stream.ts | 23 +++++++++++++ streams/text_from_readable_stream_test.ts | 30 ++++++++++++++++ 9 files changed, 189 insertions(+) create mode 100644 streams/array_buffer_from_readable_stream.ts create mode 100644 streams/array_buffer_from_readable_stream_test.ts create mode 100644 streams/blob_from_readable_stream.ts create mode 100644 streams/blob_from_readable_stream_test.ts create mode 100644 streams/json_from_readable_stream.ts create mode 100644 streams/json_from_readable_stream_test.ts create mode 100644 streams/text_from_readable_stream.ts create mode 100644 streams/text_from_readable_stream_test.ts diff --git a/streams/array_buffer_from_readable_stream.ts b/streams/array_buffer_from_readable_stream.ts new file mode 100644 index 000000000000..2ebb2f669c92 --- /dev/null +++ b/streams/array_buffer_from_readable_stream.ts @@ -0,0 +1,23 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. +// This module is browser compatible. + +import { concat } from "../bytes/concat.ts"; + +export async function arrayBufferFromReadableStream( + readableStream: ReadableStream, +): Promise { + const reader = readableStream.getReader(); + const chunks: Uint8Array[] = []; + + while (true) { + const { done, value } = await reader.read(); + + if (done) { + break; + } + + chunks.push(value); + } + + return concat(...chunks).buffer; +} diff --git a/streams/array_buffer_from_readable_stream_test.ts b/streams/array_buffer_from_readable_stream_test.ts new file mode 100644 index 000000000000..2fb513649187 --- /dev/null +++ b/streams/array_buffer_from_readable_stream_test.ts @@ -0,0 +1,18 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +import { assertEquals } from "../assert/assert_equals.ts"; +import { arrayBufferFromReadableStream } from "./array_buffer_from_readable_stream.ts"; + +Deno.test("[streams] arrayBufferFromReadableStream", async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(Uint8Array.of(1, 2, 3, 4, 5)); + controller.enqueue(Uint8Array.of(6, 7)); + controller.enqueue(Uint8Array.of(8, 9)); + controller.close(); + }, + }); + + const buf = await arrayBufferFromReadableStream(stream); + assertEquals(buf, Uint8Array.of(1, 2, 3, 4, 5, 6, 7, 8, 9).buffer); +}); diff --git a/streams/blob_from_readable_stream.ts b/streams/blob_from_readable_stream.ts new file mode 100644 index 000000000000..e7027652b54f --- /dev/null +++ b/streams/blob_from_readable_stream.ts @@ -0,0 +1,21 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. +// This module is browser compatible. + +export async function blobFromReadableStream( + readableStream: ReadableStream, +): Promise { + const reader = readableStream.getReader(); + const chunks: Uint8Array[] = []; + + while (true) { + const { done, value } = await reader.read(); + + if (done) { + break; + } + + chunks.push(value); + } + + return new Blob(chunks); +} diff --git a/streams/blob_from_readable_stream_test.ts b/streams/blob_from_readable_stream_test.ts new file mode 100644 index 000000000000..9f773e21cbe8 --- /dev/null +++ b/streams/blob_from_readable_stream_test.ts @@ -0,0 +1,23 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +import { assert } from "../assert/assert.ts"; +import { assertEquals } from "../assert/assert_equals.ts"; +import { blobFromReadableStream } from "./blob_from_readable_stream.ts"; + +Deno.test("[streams] blobFromReadableStream", async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(Uint8Array.of(1, 2, 3, 4, 5)); + controller.enqueue(Uint8Array.of(6, 7)); + controller.enqueue(Uint8Array.of(8, 9)); + controller.close(); + }, + }); + + const blob = await blobFromReadableStream(stream); + assert(blob instanceof Blob); + assertEquals( + await blob.arrayBuffer(), + Uint8Array.of(1, 2, 3, 4, 5, 6, 7, 8, 9).buffer, + ); +}); diff --git a/streams/json_from_readable_stream.ts b/streams/json_from_readable_stream.ts new file mode 100644 index 000000000000..68ce158627d4 --- /dev/null +++ b/streams/json_from_readable_stream.ts @@ -0,0 +1,13 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. +// This module is browser compatible. + +import { textFromReadableStream } from "./text_from_readable_stream.ts"; + +export function jsonFromReadableStream( + readableStream: ReadableStream, +): Promise { + return textFromReadableStream(readableStream).then((res) => { + console.log(res); + return JSON.parse(res); + }); +} diff --git a/streams/json_from_readable_stream_test.ts b/streams/json_from_readable_stream_test.ts new file mode 100644 index 000000000000..60e4bc322492 --- /dev/null +++ b/streams/json_from_readable_stream_test.ts @@ -0,0 +1,34 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +import { assertEquals } from "../assert/assert_equals.ts"; +import { jsonFromReadableStream } from "./json_from_readable_stream.ts"; + +const textEncoder = new TextEncoder(); + +Deno.test("[streams] textFromReadableStream", async () => { + const byteStream = new ReadableStream({ + start(controller) { + controller.enqueue(textEncoder.encode("[")); + controller.enqueue(textEncoder.encode("1, 2, 3, 4")); + controller.enqueue(textEncoder.encode("]")); + controller.close(); + }, + }); + + assertEquals(await jsonFromReadableStream(byteStream), [1, 2, 3, 4]); + + const stringStream = new ReadableStream({ + start(controller) { + controller.enqueue('{ "a": 2,'); + controller.enqueue(' "b": 3,'); + controller.enqueue(' "c": 4 }'); + controller.close(); + }, + }); + + assertEquals(await jsonFromReadableStream(stringStream), { + a: 2, + b: 3, + c: 4, + }); +}); diff --git a/streams/mod.ts b/streams/mod.ts index 7889bc7312cb..1c9cf5279c5b 100644 --- a/streams/mod.ts +++ b/streams/mod.ts @@ -8,12 +8,15 @@ * @module */ +export * from "./array_buffer_from_readable_stream.ts"; +export * from "./blob_from_readable_stream.ts"; export * from "./buffer.ts"; export * from "./byte_slice_stream.ts"; export * from "./copy.ts"; export * from "./delimiter_stream.ts"; export * from "./early_zip_readable_streams.ts"; export * from "./iterate_reader.ts"; +export * from "./json_from_readable_stream.ts"; export * from "./limited_bytes_transform_stream.ts"; export * from "./limited_transform_stream.ts"; export * from "./merge_readable_streams.ts"; @@ -22,6 +25,7 @@ export * from "./readable_stream_from_reader.ts"; export * from "./reader_from_iterable.ts"; export * from "./reader_from_stream_reader.ts"; export * from "./text_delimiter_stream.ts"; +export * from "./text_from_readable_stream.ts"; export * from "./text_line_stream.ts"; export * from "./to_transform_stream.ts"; export * from "./writable_stream_from_writer.ts"; diff --git a/streams/text_from_readable_stream.ts b/streams/text_from_readable_stream.ts new file mode 100644 index 000000000000..376980050ebc --- /dev/null +++ b/streams/text_from_readable_stream.ts @@ -0,0 +1,23 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. +// This module is browser compatible. + +const textDecoder = new TextDecoder(); + +export async function textFromReadableStream( + readableStream: ReadableStream, +): Promise { + const reader = readableStream.getReader(); + let result = ""; + + while (true) { + const { done, value } = await reader.read(); + + if (done) { + break; + } + + result += typeof value === "string" ? value : textDecoder.decode(value); + } + + return result; +} diff --git a/streams/text_from_readable_stream_test.ts b/streams/text_from_readable_stream_test.ts new file mode 100644 index 000000000000..c9dd6576c1ae --- /dev/null +++ b/streams/text_from_readable_stream_test.ts @@ -0,0 +1,30 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +import { assertEquals } from "../assert/assert_equals.ts"; +import { textFromReadableStream } from "./text_from_readable_stream.ts"; + +const textEncoder = new TextEncoder(); + +Deno.test("[streams] textFromReadableStream", async () => { + const byteStream = new ReadableStream({ + start(controller) { + controller.enqueue(textEncoder.encode("hello")); + controller.enqueue(textEncoder.encode(" js ")); + controller.enqueue(textEncoder.encode("fans")); + controller.close(); + }, + }); + + assertEquals(await textFromReadableStream(byteStream), "hello js fans"); + + const stringStream = new ReadableStream({ + start(controller) { + controller.enqueue("hello"); + controller.enqueue(" deno "); + controller.enqueue("world"); + controller.close(); + }, + }); + + assertEquals(await textFromReadableStream(stringStream), "hello deno world"); +}); From 66aef57f0faaff5cee3ecb727e62b1264ff9210b Mon Sep 17 00:00:00 2001 From: lino-levan <11367844+lino-levan@users.noreply.github.com> Date: Sat, 9 Sep 2023 16:07:08 +1200 Subject: [PATCH 2/3] fix: remove debugging console log --- streams/json_from_readable_stream.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/streams/json_from_readable_stream.ts b/streams/json_from_readable_stream.ts index 68ce158627d4..740f6a6f5fcf 100644 --- a/streams/json_from_readable_stream.ts +++ b/streams/json_from_readable_stream.ts @@ -6,8 +6,5 @@ import { textFromReadableStream } from "./text_from_readable_stream.ts"; export function jsonFromReadableStream( readableStream: ReadableStream, ): Promise { - return textFromReadableStream(readableStream).then((res) => { - console.log(res); - return JSON.parse(res); - }); + return textFromReadableStream(readableStream).then(JSON.parse); } From 3de053db64b4173a67e67461722112a8f8b0a5cc Mon Sep 17 00:00:00 2001 From: lino-levan <11367844+lino-levan@users.noreply.github.com> Date: Wed, 13 Sep 2023 09:02:37 +1200 Subject: [PATCH 3/3] fix: address kt3k's comments --- streams/json_from_readable_stream.ts | 10 ---------- streams/mod.ts | 8 ++++---- ...ffer_from_readable_stream.ts => to_array_buffer.ts} | 4 ++-- ...readable_stream_test.ts => to_array_buffer_test.ts} | 6 +++--- streams/{blob_from_readable_stream.ts => to_blob.ts} | 2 +- ...ob_from_readable_stream_test.ts => to_blob_test.ts} | 6 +++--- streams/to_json.ts | 10 ++++++++++ ...on_from_readable_stream_test.ts => to_json_test.ts} | 8 ++++---- streams/{text_from_readable_stream.ts => to_text.ts} | 2 +- ...xt_from_readable_stream_test.ts => to_text_test.ts} | 8 ++++---- 10 files changed, 32 insertions(+), 32 deletions(-) delete mode 100644 streams/json_from_readable_stream.ts rename streams/{array_buffer_from_readable_stream.ts => to_array_buffer.ts} (83%) rename streams/{array_buffer_from_readable_stream_test.ts => to_array_buffer_test.ts} (68%) rename streams/{blob_from_readable_stream.ts => to_blob.ts} (89%) rename streams/{blob_from_readable_stream_test.ts => to_blob_test.ts} (75%) create mode 100644 streams/to_json.ts rename streams/{json_from_readable_stream_test.ts => to_json_test.ts} (73%) rename streams/{text_from_readable_stream.ts => to_text.ts} (91%) rename streams/{text_from_readable_stream_test.ts => to_text_test.ts} (70%) diff --git a/streams/json_from_readable_stream.ts b/streams/json_from_readable_stream.ts deleted file mode 100644 index 740f6a6f5fcf..000000000000 --- a/streams/json_from_readable_stream.ts +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. -// This module is browser compatible. - -import { textFromReadableStream } from "./text_from_readable_stream.ts"; - -export function jsonFromReadableStream( - readableStream: ReadableStream, -): Promise { - return textFromReadableStream(readableStream).then(JSON.parse); -} diff --git a/streams/mod.ts b/streams/mod.ts index 1c9cf5279c5b..de9789583f39 100644 --- a/streams/mod.ts +++ b/streams/mod.ts @@ -8,15 +8,12 @@ * @module */ -export * from "./array_buffer_from_readable_stream.ts"; -export * from "./blob_from_readable_stream.ts"; export * from "./buffer.ts"; export * from "./byte_slice_stream.ts"; export * from "./copy.ts"; export * from "./delimiter_stream.ts"; export * from "./early_zip_readable_streams.ts"; export * from "./iterate_reader.ts"; -export * from "./json_from_readable_stream.ts"; export * from "./limited_bytes_transform_stream.ts"; export * from "./limited_transform_stream.ts"; export * from "./merge_readable_streams.ts"; @@ -25,8 +22,11 @@ export * from "./readable_stream_from_reader.ts"; export * from "./reader_from_iterable.ts"; export * from "./reader_from_stream_reader.ts"; export * from "./text_delimiter_stream.ts"; -export * from "./text_from_readable_stream.ts"; export * from "./text_line_stream.ts"; +export * from "./to_array_buffer.ts"; +export * from "./to_blob.ts"; +export * from "./to_json.ts"; +export * from "./to_text.ts"; export * from "./to_transform_stream.ts"; export * from "./writable_stream_from_writer.ts"; export * from "./write_all.ts"; diff --git a/streams/array_buffer_from_readable_stream.ts b/streams/to_array_buffer.ts similarity index 83% rename from streams/array_buffer_from_readable_stream.ts rename to streams/to_array_buffer.ts index 2ebb2f669c92..46bfab1dd621 100644 --- a/streams/array_buffer_from_readable_stream.ts +++ b/streams/to_array_buffer.ts @@ -3,8 +3,8 @@ import { concat } from "../bytes/concat.ts"; -export async function arrayBufferFromReadableStream( - readableStream: ReadableStream, +export async function toArrayBuffer( + readableStream: ReadableStream, ): Promise { const reader = readableStream.getReader(); const chunks: Uint8Array[] = []; diff --git a/streams/array_buffer_from_readable_stream_test.ts b/streams/to_array_buffer_test.ts similarity index 68% rename from streams/array_buffer_from_readable_stream_test.ts rename to streams/to_array_buffer_test.ts index 2fb513649187..a1cccf139946 100644 --- a/streams/array_buffer_from_readable_stream_test.ts +++ b/streams/to_array_buffer_test.ts @@ -1,9 +1,9 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. import { assertEquals } from "../assert/assert_equals.ts"; -import { arrayBufferFromReadableStream } from "./array_buffer_from_readable_stream.ts"; +import { toArrayBuffer } from "./to_array_buffer.ts"; -Deno.test("[streams] arrayBufferFromReadableStream", async () => { +Deno.test("[streams] toArrayBuffer", async () => { const stream = new ReadableStream({ start(controller) { controller.enqueue(Uint8Array.of(1, 2, 3, 4, 5)); @@ -13,6 +13,6 @@ Deno.test("[streams] arrayBufferFromReadableStream", async () => { }, }); - const buf = await arrayBufferFromReadableStream(stream); + const buf = await toArrayBuffer(stream); assertEquals(buf, Uint8Array.of(1, 2, 3, 4, 5, 6, 7, 8, 9).buffer); }); diff --git a/streams/blob_from_readable_stream.ts b/streams/to_blob.ts similarity index 89% rename from streams/blob_from_readable_stream.ts rename to streams/to_blob.ts index e7027652b54f..33a9f996a550 100644 --- a/streams/blob_from_readable_stream.ts +++ b/streams/to_blob.ts @@ -1,7 +1,7 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. // This module is browser compatible. -export async function blobFromReadableStream( +export async function toBlob( readableStream: ReadableStream, ): Promise { const reader = readableStream.getReader(); diff --git a/streams/blob_from_readable_stream_test.ts b/streams/to_blob_test.ts similarity index 75% rename from streams/blob_from_readable_stream_test.ts rename to streams/to_blob_test.ts index 9f773e21cbe8..adf02d32f081 100644 --- a/streams/blob_from_readable_stream_test.ts +++ b/streams/to_blob_test.ts @@ -2,9 +2,9 @@ import { assert } from "../assert/assert.ts"; import { assertEquals } from "../assert/assert_equals.ts"; -import { blobFromReadableStream } from "./blob_from_readable_stream.ts"; +import { toBlob } from "./to_blob.ts"; -Deno.test("[streams] blobFromReadableStream", async () => { +Deno.test("[streams] toBlob", async () => { const stream = new ReadableStream({ start(controller) { controller.enqueue(Uint8Array.of(1, 2, 3, 4, 5)); @@ -14,7 +14,7 @@ Deno.test("[streams] blobFromReadableStream", async () => { }, }); - const blob = await blobFromReadableStream(stream); + const blob = await toBlob(stream); assert(blob instanceof Blob); assertEquals( await blob.arrayBuffer(), diff --git a/streams/to_json.ts b/streams/to_json.ts new file mode 100644 index 000000000000..27d83146d6bc --- /dev/null +++ b/streams/to_json.ts @@ -0,0 +1,10 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. +// This module is browser compatible. + +import { toText } from "./to_text.ts"; + +export function toJson( + readableStream: ReadableStream, +): Promise { + return toText(readableStream).then(JSON.parse); +} diff --git a/streams/json_from_readable_stream_test.ts b/streams/to_json_test.ts similarity index 73% rename from streams/json_from_readable_stream_test.ts rename to streams/to_json_test.ts index 60e4bc322492..c97cbd84af9c 100644 --- a/streams/json_from_readable_stream_test.ts +++ b/streams/to_json_test.ts @@ -1,11 +1,11 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. import { assertEquals } from "../assert/assert_equals.ts"; -import { jsonFromReadableStream } from "./json_from_readable_stream.ts"; +import { toJson } from "./to_json.ts"; const textEncoder = new TextEncoder(); -Deno.test("[streams] textFromReadableStream", async () => { +Deno.test("[streams] toJson", async () => { const byteStream = new ReadableStream({ start(controller) { controller.enqueue(textEncoder.encode("[")); @@ -15,7 +15,7 @@ Deno.test("[streams] textFromReadableStream", async () => { }, }); - assertEquals(await jsonFromReadableStream(byteStream), [1, 2, 3, 4]); + assertEquals(await toJson(byteStream), [1, 2, 3, 4]); const stringStream = new ReadableStream({ start(controller) { @@ -26,7 +26,7 @@ Deno.test("[streams] textFromReadableStream", async () => { }, }); - assertEquals(await jsonFromReadableStream(stringStream), { + assertEquals(await toJson(stringStream), { a: 2, b: 3, c: 4, diff --git a/streams/text_from_readable_stream.ts b/streams/to_text.ts similarity index 91% rename from streams/text_from_readable_stream.ts rename to streams/to_text.ts index 376980050ebc..52cf0c7c92bc 100644 --- a/streams/text_from_readable_stream.ts +++ b/streams/to_text.ts @@ -3,7 +3,7 @@ const textDecoder = new TextDecoder(); -export async function textFromReadableStream( +export async function toText( readableStream: ReadableStream, ): Promise { const reader = readableStream.getReader(); diff --git a/streams/text_from_readable_stream_test.ts b/streams/to_text_test.ts similarity index 70% rename from streams/text_from_readable_stream_test.ts rename to streams/to_text_test.ts index c9dd6576c1ae..9c931d650a32 100644 --- a/streams/text_from_readable_stream_test.ts +++ b/streams/to_text_test.ts @@ -1,11 +1,11 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. import { assertEquals } from "../assert/assert_equals.ts"; -import { textFromReadableStream } from "./text_from_readable_stream.ts"; +import { toText } from "./to_text.ts"; const textEncoder = new TextEncoder(); -Deno.test("[streams] textFromReadableStream", async () => { +Deno.test("[streams] toText", async () => { const byteStream = new ReadableStream({ start(controller) { controller.enqueue(textEncoder.encode("hello")); @@ -15,7 +15,7 @@ Deno.test("[streams] textFromReadableStream", async () => { }, }); - assertEquals(await textFromReadableStream(byteStream), "hello js fans"); + assertEquals(await toText(byteStream), "hello js fans"); const stringStream = new ReadableStream({ start(controller) { @@ -26,5 +26,5 @@ Deno.test("[streams] textFromReadableStream", async () => { }, }); - assertEquals(await textFromReadableStream(stringStream), "hello deno world"); + assertEquals(await toText(stringStream), "hello deno world"); });