From 05126e0729d3dfd98cf17cd19ad322db2f8ebbc4 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Thu, 4 Apr 2024 21:56:00 +1100 Subject: [PATCH 01/82] refactor(archive): An implementation of Tar as streams --- archive/tar_stream.ts | 269 ++++++++++++++++++++++++++++++ archive/tar_stream_test.ts | 31 ++++ archive/untar_stream.ts | 314 +++++++++++++++++++++++++++++++++++ archive/untar_stream_test.ts | 61 +++++++ 4 files changed, 675 insertions(+) create mode 100644 archive/tar_stream.ts create mode 100644 archive/tar_stream_test.ts create mode 100644 archive/untar_stream.ts create mode 100644 archive/untar_stream_test.ts diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts new file mode 100644 index 000000000000..29f2fb389d08 --- /dev/null +++ b/archive/tar_stream.ts @@ -0,0 +1,269 @@ +/** + * The type required to provide a file. + */ +export type TarFile = { + pathname: string; + size: number; + sizeExtension?: boolean; + iterable: Iterable | AsyncIterable; + options?: Partial; +}; + +/** + * The type required to provide a directory. + */ +export type TarDir = { + pathname: string; + options?: Partial; +}; + +/** + * The options that can go along with a file or directory. + * @param mode An octal number in ASCII. + * @param uid An octal number in ASCII. + * @param gid An octal number in ASCII. + * @param mtime A number of seconds since the start of epoch. Avoid negative + * values. + * @param uname An ASCII string. Should be used in preference of uid. + * @param gname An ASCII string. Should be used in preference of gid. + * @param devmajor The major number for character device. + * @param devminor The minor number for block device entry. + */ +export type TarOptions = { + mode: string; + uid: string; + gid: string; + mtime: number; + uname: string; + gname: string; + devmajor: string; + devminor: string; +}; + +/** + * ### Overview + * A TransformStream to create a tar archive. Tar archives allow for storing + * multiple files in a single file (called an archive, or sometimes a tarball). + * These archives typically have a singe '.tar' extension. + * + * ### File Format & Limitations + * The ustar file format is used for creating the tar archive. While this + * format is compatible with most tar readers, the format has several + * limitations, including: + * - Pathnames must be at most 256 characters. + * - Files must be at most 8 GiBs in size, or 64 GiBs if `sizeExtension` is set + * to true. + * - Sparse files are not supported. + * + * ### Usage + * TarStream may throw an error for several reasons. A few of those are: + * - The pathname is invalid. + * - The size provided does not match that of the iterable's length. + * + * @example + * ```ts + * import { TarStream } from '@std/archive/tar' + * + * await ReadableStream.from([ + * { + * pathname: 'potato/' + * }, + * { + * pathname: 'deno.json', + * size: (await Deno.stat('deno.json')).size, + * iterable: (await Deno.open('deno.json')).readable + * }, + * { + * pathname: 'deno.lock', + * size: (await Deno.stat('deno.lock')).size, + * iterable: (await Deno.open('deno.lock')).readable + * } + * ]) + * .pipeThrough(new TarStream()) + * .pipeTo((await Deno.create('./out.tar)).writable) + * ``` + * + * ### Compression + * Tar archives are not compressed by default. If you'd like to compress the + * archive, you may do so by piping it through a compression stream. + * + * @example + * ```ts + * import { TarStream } from '@std/archive/tar' + * + * await ReadableStream.from([ + * { + * pathname: 'potato/' + * }, + * { + * pathname: 'deno.json', + * size: (await Deno.stat('deno.json')).size, + * iterable: (await Deno.open('deno.json')).readable + * }, + * { + * pathname: 'deno.lock', + * size: (await Deno.stat('deno.lock')).size, + * iterable: (await Deno.open('deno.lock')).readable + * } + * ]) + * .pipeThrough(new TarStream()) + * .pipeThrough(new CompressionStream('gzip')) + * .pipeTo((await Deno.create('./out.tar.gz)).writable) + * ``` + */ +export class TarStream { + #readable: ReadableStream; + #writable: WritableStream; + /** + * Constructs a new instance. + */ + constructor() { + const { readable, writable } = new TransformStream< + TarFile | TarDir, + TarFile | TarDir + >(); + const gen = (async function* () { + const paths: string[] = []; + for await (const chunk of readable) { + if ( + "size" in chunk && + ( + chunk.size < 0 || + Math.pow(8, chunk.sizeExtension ? 12 : 11) < chunk.size || + chunk.size.toString() === "NaN" + ) + ) { + throw new Error( + "Invalid Size Provided! Size cannot exceed 8 GiBs by default or 64 GiBs with sizeExtension set to true.", + ); + } + + chunk.pathname = chunk.pathname.split("/").filter((x) => x).join("/"); + if (chunk.pathname.startsWith("./")) { + chunk.pathname = chunk.pathname.slice(2); + } + if (!("size" in chunk)) { + chunk.pathname += "/"; + } + + const pathname = new TextEncoder().encode(chunk.pathname); + if (pathname.length > 256) { + throw new Error( + "Invalid Pathname! Pathname cannot exceed 256 bytes.", + ); + } + + let i = Math.max(0, pathname.lastIndexOf(47)); + if (pathname.slice(i + 1).length > 100) { + throw new Error( + "Invalid Filename! Filename cannot exceed 100 bytes.", + ); + } + + if (pathname.length <= 100) { + i = 0; + } else { + for (; i > 0; --i) { + i = pathname.lastIndexOf(47, i); + if (pathname.slice(i + 1).length > 100) { + i = Math.max(0, pathname.indexOf(47, i + 1)); + break; + } + } + } + + const prefix = pathname.slice(0, i); + if (prefix.length > 155) { + throw new Error( + "Invalid Pathname! Pathname needs to be split-able on a forward slash separator into [155, 100] bytes respectively.", + ); + } + if (paths.includes(chunk.pathname)) { + continue; + } + paths.push(chunk.pathname); + const typeflag = "size" in chunk ? "0" : "5"; + const sizeExtension = "size" in chunk && chunk.sizeExtension || false; + const encoder = new TextEncoder(); + const header = new Uint8Array(512); + + header.set(prefix.length ? pathname.slice(i + 1) : pathname); // name + header.set( + encoder.encode( + (chunk.options?.mode ?? (typeflag === "5" ? "755" : "644")) + .padStart(6, "0") + + " \0" + // mode + (chunk.options?.uid ?? "").padStart(6, "0") + " \0" + // uid + (chunk.options?.gid ?? "").padStart(6, "0") + " \0" + // gid + ("size" in chunk ? chunk.size.toString(8) : "").padStart( + sizeExtension ? 12 : 11, + "0", + ) + (sizeExtension ? "" : " ") + // size + (chunk.options?.mtime?.toString(8) ?? "").padStart(11, "0") + + " " + // mtime + " ".repeat(8) + // checksum | Needs to be updated + typeflag + // typeflag + "\0".repeat(100) + // linkname + "ustar\0" + // magic + "00" + // version + (chunk.options?.uname ?? "").padEnd(32, "\0") + // uname + (chunk.options?.gname ?? "").padEnd(32, "\0") + // gname + (chunk.options?.devmajor ?? "").padEnd(8, "\0") + // devmajor + (chunk.options?.devminor ?? "").padEnd(8, "\0"), // devminor + ), + 100, + ); + header.set(prefix, 345); // prefix + + header.set( + encoder.encode( + header.reduce((x, y) => x + y).toString(8).padStart(6, "0") + "\0", + ), + 148, + ); // update checksum + yield header; + + if ("size" in chunk) { + let size = 0; + for await (const x of chunk.iterable) { + size += x.length; + yield x; + } + if (chunk.size !== size) { + throw new Error( + "Invalid Tarball! Provided size did not match bytes read from iterable.", + ); + } + yield new Uint8Array(new Array(512 - chunk.size % 512).fill(0)); + } + } + yield new Uint8Array(new Array(1024).fill(0)); + })(); + + this.#readable = new ReadableStream({ + async pull(controller) { + const { done, value } = await gen.next(); + if (done) { + controller.close(); + } else { + controller.enqueue(value); + } + }, + }); + this.#writable = writable; + } + + /** + * The ReadableStream + */ + get readable(): ReadableStream { + return this.#readable; + } + + /** + * The WritableStream + */ + get writable(): WritableStream { + return this.#writable; + } +} diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts new file mode 100644 index 000000000000..3267fcc13667 --- /dev/null +++ b/archive/tar_stream_test.ts @@ -0,0 +1,31 @@ +import { TarStream } from "https://deno.land/std@$STD_VERSION/archive/tar_stream.ts" +import { assertEquals } from "https://deno.land/std@$STD_VERSION/assert/assert_equals.ts" + +Deno.test('createTarArchiveViaStream', async function () { + const text = new TextEncoder().encode('Hello World!') + + const size = (await reduce(ReadableStream.from([ + { + pathname: './potato' + }, + { + pathname: './text.txt', + size: text.length, + iterable: [text] + } + ]) + .pipeThrough(new TarStream()))).length + + assertEquals(size, 512 + 512 + Math.ceil(text.length / 512) * 512 + 1024) +}) + +async function reduce(readable: ReadableStream) { + let y = new Uint8Array(0) + for await (const x of readable) { + const z = new Uint8Array(x.length + y.length) + z.set(y) + z.set(x, y.length) + y = z + } + return y +} diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts new file mode 100644 index 000000000000..c4f4c5940ffa --- /dev/null +++ b/archive/untar_stream.ts @@ -0,0 +1,314 @@ +/** + * The type extracted from the archive. + */ +export type TarEntry = { + pathname: string; + header: TarHeader; + readable?: ReadableStream; +}; + +/** + * The header of an entry in the archive. + */ +export type TarHeader = { + name: string; + mode: string; + uid: string; + gid: string; + size: number; + mtime: number; + checksum: string; + typeflag: string; + linkname: string; + pad: Uint8Array; +} | { + name: string; + mode: string; + uid: string; + gid: string; + size: number; + mtime: number; + checksum: string; + typeflag: string; + linkname: string; + magic: string; + version: string; + uname: string; + gname: string; + devmajor: string; + devminor: string; + prefix: string; + pad: Uint8Array; +}; + +/** + * ### Overview + * A TransformStream to expand a tar archive. Tar archives allow for storing + * multiple files in a single file (called an archive, or sometimes a tarball). + * These archives typically have a single '.tar' extension. + * + * ### Supported File Formats + * Only the ustar file format is supported. This is the most common format. + * Additionally the numeric extension for file size. + * + * ### Usage + * When expanding the archive, as demonstrated in the example, one must decide + * to either consume the Readable Stream, if present, or cancel it. The next + * entry won't be resolved until the previous ReadableStream is either consumed + * or cancelled. + * + * @example + * ```ts + * import { UnTarStream } from '@std/archive/tar' + * + * for await ( + * const entry of (await Deno.open('./out.tar')) + * .readable + * .pipeThrough(new UnTarStream()) + * ) { + * console.log(entry.pathname) + * await entry + * .readable? + * .pipeTo((await Deno.create(entry.pathname)).writable) + * } + * ``` + * + * ### Understanding Compressed + * A tar archive may be compressed, often identified by an additional file + * extension, such as '.tar.gz' for gzip. This TransformStream does not support + * decompression which must be done before expanding the archive. + * + * @example + * ```ts + * import { UnTarStream } from '@std/archive/tar' + * + * for await ( + * const entry of (await Deno.open('./out.tar.gz')) + * .readable + * .pipeThrough(new DecompressionStream('gzip')) + * .pipeThrough(new UnTarStream()) + * ) { + * console.log(entry.pathname) + * entry + * .readable? + * .pipeTo((await Deno.create(entry.pathname)).writable) + * } + * ``` + */ +export class UnTarStream { + #readable: ReadableStream; + #writable: WritableStream; + /** + * Constructs a new instance. + */ + constructor() { + const { readable, writable } = new TransformStream< + Uint8Array, + Uint8Array + >(); + const reader = readable + .pipeThrough( + new TransformStream( + { // Slices ReadableStream's Uint8Array into 512 byte chunks. + x: new Uint8Array(0), + transform(chunk, controller) { + const y = new Uint8Array(this.x.length + chunk.length); + y.set(this.x); + y.set(chunk, this.x.length); + + for (let i = 512; i <= y.length; i += 512) { + controller.enqueue(y.slice(i - 512, i)); + } + this.x = y.length % 512 + ? y.slice(-y.length % 512) + : new Uint8Array(0); + }, + flush(controller) { + if (this.x.length) { + controller.error( + "Tarball has an unexpected number of bytes.!!", + ); + } + }, + } as Transformer & { x: Uint8Array }, + ), + ) + .pipeThrough( + new TransformStream( + { // Trims the last Uint8Array chunks off. + x: [], + transform(chunk, controller) { + this.x.push(chunk); + if (this.x.length === 3) { + controller.enqueue(this.x.shift()!); + } + }, + flush(controller) { + if (this.x.length < 2) { + controller.error("Tarball was too small to be valid."); + } else if (!this.x.every((x) => x.every((x) => x === 0))) { + controller.error("Tarball has invalid ending."); + } + }, + } as Transformer & { x: Uint8Array[] }, + ), + ) + .getReader(); + let header: TarHeader | undefined; + this.#readable = new ReadableStream( + { + cancelled: false, + async pull(controller) { + while (header != undefined) { + await new Promise((a) => setTimeout(a, 0)); + } + + const { done, value } = await reader.read(); + if (done) { + return controller.close(); + } + + const decoder = new TextDecoder(); + { // Validate checksum + const checksum = value.slice(); + checksum.set(new Uint8Array(8).fill(32), 148); + if ( + checksum.reduce((x, y) => x + y) !== + parseInt(decoder.decode(value.slice(148, 156 - 2)), 8) + ) { + return controller.error( + "Invalid Tarball. Header failed to pass checksum.", + ); + } + } + header = { + name: decoder.decode(value.slice(0, 100)).replaceAll("\0", ""), + mode: decoder.decode(value.slice(100, 108 - 2)), + uid: decoder.decode(value.slice(108, 116 - 2)), + gid: decoder.decode(value.slice(116, 124 - 2)), + size: parseInt(decoder.decode(value.slice(124, 136)).trimEnd(), 8), + mtime: parseInt(decoder.decode(value.slice(136, 148 - 1)), 8), + checksum: decoder.decode(value.slice(148, 156 - 2)), + typeflag: decoder.decode(value.slice(156, 157)), + linkname: decoder.decode(value.slice(157, 257)).replaceAll( + "\0", + "", + ), + pad: value.slice(257), + }; + if (header.typeflag === "\0") { + header.typeflag = "0"; + } + // Check if header is POSIX ustar | new TextEncoder().encode('ustar\0' + '00') + if ( + [117, 115, 116, 97, 114, 0, 48, 48].every((byte, i) => + value[i + 257] === byte + ) + ) { + header = { + ...header, + magic: decoder.decode(value.slice(257, 263)), + version: decoder.decode(value.slice(263, 265)), + uname: decoder.decode(value.slice(265, 297)).replaceAll("\0", ""), + gname: decoder.decode(value.slice(297, 329)).replaceAll("\0", ""), + devmajor: decoder.decode(value.slice(329, 337)).replaceAll( + "\0", + "", + ), + devminor: decoder.decode(value.slice(337, 345)).replaceAll( + "\0", + "", + ), + prefix: decoder.decode(value.slice(345, 500)).replaceAll( + "\0", + "", + ), + pad: value.slice(500), + }; + } + + if (header.typeflag === "0") { + const size = header.size; + let i = Math.ceil(size / 512); + const isCancelled = () => this.cancelled; + let lock = false; + controller.enqueue({ + pathname: ("prefix" in header && header.prefix.length + ? header.prefix + "/" + : "") + header.name, + header, + readable: new ReadableStream({ + async pull(controller) { + if (i > 0) { + lock = true; + const { done, value } = await reader.read(); + if (done) { + header = undefined; + controller.error("Tarball ended unexpectedly."); + } else { + // Pull is unlocked before enqueue is called because if pull is in the middle of processing a chunk when cancel is called, nothing after enqueue will run. + lock = false; + controller.enqueue( + i-- === 1 ? value.slice(0, size % 512) : value, + ); + } + } else { + header = undefined; + if (isCancelled()) { + reader.cancel(); + } + controller.close(); + } + }, + async cancel() { + while (lock) { + await new Promise((a) => + setTimeout(a, 0) + ); + } + try { + while (i-- > 0) { + if ((await reader.read()).done) { + throw new Error("Tarball ended unexpectedly."); + } + } + } catch (error) { + throw error; + } finally { + header = undefined; + } + }, + }), + }); + } else { + controller.enqueue({ + pathname: ("prefix" in header && header.prefix.length + ? header.prefix + "/" + : "") + header.name, + header, + }); + header = undefined; + } + }, + cancel() { + this.cancelled = true; + }, + } as UnderlyingSource & { cancelled: boolean }, + ); + this.#writable = writable; + } + + /** + * The ReadableStream + */ + get readable(): ReadableStream { + return this.#readable; + } + + /** + * The WritableStream + */ + get writable(): WritableStream { + return this.#writable; + } +} diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts new file mode 100644 index 000000000000..e1cdd7ba3dbc --- /dev/null +++ b/archive/untar_stream_test.ts @@ -0,0 +1,61 @@ +import { TarStream } from "https://deno.land/std@$STD_VERSION/archive/tar_stream.ts" +import { assertEquals } from "https://deno.land/std@$STD_VERSION/assert/assert_equals.ts" +import { UnTarStream } from "https://deno.land/std@$STD_VERSION/archive/untar_stream.ts" + +Deno.test('unTarStreamCheckingHeaders', async function () { + const text = new TextEncoder().encode('Hello World!') + + const pathnames: string[] = [] + for await ( + const item of ReadableStream.from([ + { + pathname: './potato' + }, + { + pathname: './text.txt', + size: text.length, + iterable: [text] + } + ]) + .pipeThrough(new TarStream()) + .pipeThrough(new UnTarStream()) + ) { + pathnames.push(item.pathname) + item.readable?.cancel() + } + + assertEquals(pathnames, ['potato/', 'text.txt']) +}) + +Deno.test('unTarStreamValidatingBodies', async function () { + const text = new TextEncoder().encode('Hello World!') + + for await ( + const item of ReadableStream.from([ + { + pathname: './potato' + }, + { + pathname: './text.txt', + size: text.length, + iterable: [text] + } + ]) + .pipeThrough(new TarStream()) + .pipeThrough(new UnTarStream()) + ) { + if (item.readable) + assertEquals(await reduce(item.readable), text) + } +}) + +async function reduce(readable: ReadableStream) { + let y = new Uint8Array(0) + for await (const x of readable) { + const z = new Uint8Array(x.length + y.length) + z.set(y) + z.set(x, y.length) + y = z + } + return y +} From 0f28c168664facc1b6bd72bd217a65338038abf1 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Thu, 4 Apr 2024 22:04:43 +1100 Subject: [PATCH 02/82] fmt(archive): Ran `deno fmt` --- archive/tar_stream_test.ts | 48 +++++++++++++++-------------- archive/untar_stream_test.ts | 59 ++++++++++++++++++------------------ 2 files changed, 55 insertions(+), 52 deletions(-) diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index 3267fcc13667..4b987a95a22b 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -1,31 +1,33 @@ -import { TarStream } from "https://deno.land/std@$STD_VERSION/archive/tar_stream.ts" -import { assertEquals } from "https://deno.land/std@$STD_VERSION/assert/assert_equals.ts" +import { TarStream } from "https://deno.land/std@$STD_VERSION/archive/tar_stream.ts"; +import { assertEquals } from "https://deno.land/std@$STD_VERSION/assert/assert_equals.ts"; -Deno.test('createTarArchiveViaStream', async function () { - const text = new TextEncoder().encode('Hello World!') +Deno.test("createTarArchiveViaStream", async function () { + const text = new TextEncoder().encode("Hello World!"); - const size = (await reduce(ReadableStream.from([ - { - pathname: './potato' - }, - { - pathname: './text.txt', - size: text.length, - iterable: [text] - } - ]) - .pipeThrough(new TarStream()))).length + const size = (await reduce( + ReadableStream.from([ + { + pathname: "./potato", + }, + { + pathname: "./text.txt", + size: text.length, + iterable: [text], + }, + ]) + .pipeThrough(new TarStream()), + )).length; - assertEquals(size, 512 + 512 + Math.ceil(text.length / 512) * 512 + 1024) -}) + assertEquals(size, 512 + 512 + Math.ceil(text.length / 512) * 512 + 1024); +}); async function reduce(readable: ReadableStream) { - let y = new Uint8Array(0) + let y = new Uint8Array(0); for await (const x of readable) { - const z = new Uint8Array(x.length + y.length) - z.set(y) - z.set(x, y.length) - y = z + const z = new Uint8Array(x.length + y.length); + z.set(y); + z.set(x, y.length); + y = z; } - return y + return y; } diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index e1cdd7ba3dbc..f2b86a9a36ea 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -1,61 +1,62 @@ -import { TarStream } from "https://deno.land/std@$STD_VERSION/archive/tar_stream.ts" -import { assertEquals } from "https://deno.land/std@$STD_VERSION/assert/assert_equals.ts" -import { UnTarStream } from "https://deno.land/std@$STD_VERSION/archive/untar_stream.ts" +import { TarStream } from "https://deno.land/std@$STD_VERSION/archive/tar_stream.ts"; +import { assertEquals } from "https://deno.land/std@$STD_VERSION/assert/assert_equals.ts"; +import { UnTarStream } from "https://deno.land/std@$STD_VERSION/archive/untar_stream.ts"; -Deno.test('unTarStreamCheckingHeaders', async function () { - const text = new TextEncoder().encode('Hello World!') +Deno.test("unTarStreamCheckingHeaders", async function () { + const text = new TextEncoder().encode("Hello World!"); - const pathnames: string[] = [] + const pathnames: string[] = []; for await ( const item of ReadableStream.from([ { - pathname: './potato' + pathname: "./potato", }, { - pathname: './text.txt', + pathname: "./text.txt", size: text.length, - iterable: [text] - } + iterable: [text], + }, ]) .pipeThrough(new TarStream()) .pipeThrough(new UnTarStream()) ) { - pathnames.push(item.pathname) - item.readable?.cancel() + pathnames.push(item.pathname); + item.readable?.cancel(); } - assertEquals(pathnames, ['potato/', 'text.txt']) -}) + assertEquals(pathnames, ["potato/", "text.txt"]); +}); -Deno.test('unTarStreamValidatingBodies', async function () { - const text = new TextEncoder().encode('Hello World!') +Deno.test("unTarStreamValidatingBodies", async function () { + const text = new TextEncoder().encode("Hello World!"); for await ( const item of ReadableStream.from([ { - pathname: './potato' + pathname: "./potato", }, { - pathname: './text.txt', + pathname: "./text.txt", size: text.length, - iterable: [text] - } + iterable: [text], + }, ]) .pipeThrough(new TarStream()) .pipeThrough(new UnTarStream()) ) { - if (item.readable) - assertEquals(await reduce(item.readable), text) + if (item.readable) { + assertEquals(await reduce(item.readable), text); + } } -}) +}); async function reduce(readable: ReadableStream) { - let y = new Uint8Array(0) + let y = new Uint8Array(0); for await (const x of readable) { - const z = new Uint8Array(x.length + y.length) - z.set(y) - z.set(x, y.length) - y = z + const z = new Uint8Array(x.length + y.length); + z.set(y); + z.set(x, y.length); + y = z; } - return y + return y; } From e7b0b1c20cef3a43ee0d72d8c237d75ca62a5ebd Mon Sep 17 00:00:00 2001 From: Doctor <44320105+BlackAsLight@users.noreply.github.com> Date: Sat, 6 Apr 2024 15:04:40 +1100 Subject: [PATCH 03/82] fix(archive): fixed JSDoc examples in tar_streams.ts --- archive/tar_stream.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 29f2fb389d08..03dd0d184ea6 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -80,7 +80,7 @@ export type TarOptions = { * } * ]) * .pipeThrough(new TarStream()) - * .pipeTo((await Deno.create('./out.tar)).writable) + * .pipeTo((await Deno.create('./out.tar')).writable) * ``` * * ### Compression @@ -108,7 +108,7 @@ export type TarOptions = { * ]) * .pipeThrough(new TarStream()) * .pipeThrough(new CompressionStream('gzip')) - * .pipeTo((await Deno.create('./out.tar.gz)).writable) + * .pipeTo((await Deno.create('./out.tar.gz')).writable) * ``` */ export class TarStream { From 970a2375e9d4e30c43c45723fa4de8250eed0c96 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sat, 6 Apr 2024 22:25:34 +1100 Subject: [PATCH 04/82] fix(archive): fixed JSDoc examples so `deno task test` doesn't complain --- archive/tar_stream.ts | 4 ++-- archive/untar_stream.ts | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 03dd0d184ea6..aa1b0bd9c5f0 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -62,7 +62,7 @@ export type TarOptions = { * * @example * ```ts - * import { TarStream } from '@std/archive/tar' + * import { TarStream } from 'https://deno.land/std@$STD_VERSION/archive/tar_stream.ts' * * await ReadableStream.from([ * { @@ -89,7 +89,7 @@ export type TarOptions = { * * @example * ```ts - * import { TarStream } from '@std/archive/tar' + * import { TarStream } from 'https://deno.land/std@$STD_VERSION/archive/tar_stream.ts' * * await ReadableStream.from([ * { diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index c4f4c5940ffa..1f00c344c1da 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -59,7 +59,7 @@ export type TarHeader = { * * @example * ```ts - * import { UnTarStream } from '@std/archive/tar' + * import { UnTarStream } from 'https://deno.land/std@$STD_VERSION/archive/untar_stream.ts' * * for await ( * const entry of (await Deno.open('./out.tar')) @@ -68,8 +68,8 @@ export type TarHeader = { * ) { * console.log(entry.pathname) * await entry - * .readable? - * .pipeTo((await Deno.create(entry.pathname)).writable) + * .readable + * ?.pipeTo((await Deno.create(entry.pathname)).writable) * } * ``` * @@ -80,7 +80,7 @@ export type TarHeader = { * * @example * ```ts - * import { UnTarStream } from '@std/archive/tar' + * import { UnTarStream } from 'https://deno.land/std@$STD_VERSION/archive/untar_stream.ts' * * for await ( * const entry of (await Deno.open('./out.tar.gz')) @@ -90,8 +90,8 @@ export type TarHeader = { * ) { * console.log(entry.pathname) * entry - * .readable? - * .pipeTo((await Deno.create(entry.pathname)).writable) + * .readable + * ?.pipeTo((await Deno.create(entry.pathname)).writable) * } * ``` */ From a7aa4e11ef0ecb765388a1ff9a361bc0777cf5a6 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Mon, 8 Apr 2024 17:20:03 +1000 Subject: [PATCH 05/82] fix(archive): lint license error --- archive/tar_stream.ts | 1 + archive/tar_stream_test.ts | 5 +++-- archive/untar_stream.ts | 1 + archive/untar_stream_test.ts | 7 ++++--- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index aa1b0bd9c5f0..dd99048ae446 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -1,3 +1,4 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. /** * The type required to provide a file. */ diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index 4b987a95a22b..eb85c53476a7 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -1,5 +1,6 @@ -import { TarStream } from "https://deno.land/std@$STD_VERSION/archive/tar_stream.ts"; -import { assertEquals } from "https://deno.land/std@$STD_VERSION/assert/assert_equals.ts"; +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +import { TarStream } from "./tar_stream.ts"; +import { assertEquals } from "../assert/mod.ts"; Deno.test("createTarArchiveViaStream", async function () { const text = new TextEncoder().encode("Hello World!"); diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 1f00c344c1da..f7ff1a4bf282 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -1,3 +1,4 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. /** * The type extracted from the archive. */ diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index f2b86a9a36ea..c5b279490262 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -1,6 +1,7 @@ -import { TarStream } from "https://deno.land/std@$STD_VERSION/archive/tar_stream.ts"; -import { assertEquals } from "https://deno.land/std@$STD_VERSION/assert/assert_equals.ts"; -import { UnTarStream } from "https://deno.land/std@$STD_VERSION/archive/untar_stream.ts"; +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +import { TarStream } from "./tar_stream.ts"; +import { UnTarStream } from "./untar_stream.ts"; +import { assertEquals } from "../assert/mod.ts"; Deno.test("unTarStreamCheckingHeaders", async function () { const text = new TextEncoder().encode("Hello World!"); From ad3326a087d337d6bc9e3f6f40d7cbf87bd48a9f Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Mon, 8 Apr 2024 17:42:18 +1000 Subject: [PATCH 06/82] fix(archive): lint error files not exported --- archive/mod.ts | 2 ++ archive/tar_stream.ts | 18 +++++++++--------- archive/untar_stream.ts | 16 ++++++++-------- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/archive/mod.ts b/archive/mod.ts index 15c3566e7e59..f7d376ec00f2 100644 --- a/archive/mod.ts +++ b/archive/mod.ts @@ -66,3 +66,5 @@ */ export * from "./tar.ts"; export * from "./untar.ts"; +export * from "./tar_stream.ts"; +export * from "./untar_stream.ts"; diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index dd99048ae446..4ba0685ef87f 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -2,20 +2,20 @@ /** * The type required to provide a file. */ -export type TarFile = { +export type TarStreamFile = { pathname: string; size: number; sizeExtension?: boolean; iterable: Iterable | AsyncIterable; - options?: Partial; + options?: Partial; }; /** * The type required to provide a directory. */ -export type TarDir = { +export type TarStreamDir = { pathname: string; - options?: Partial; + options?: Partial; }; /** @@ -30,7 +30,7 @@ export type TarDir = { * @param devmajor The major number for character device. * @param devminor The minor number for block device entry. */ -export type TarOptions = { +export type TarStreamOptions = { mode: string; uid: string; gid: string; @@ -114,14 +114,14 @@ export type TarOptions = { */ export class TarStream { #readable: ReadableStream; - #writable: WritableStream; + #writable: WritableStream; /** * Constructs a new instance. */ constructor() { const { readable, writable } = new TransformStream< - TarFile | TarDir, - TarFile | TarDir + TarStreamFile | TarStreamDir, + TarStreamFile | TarStreamDir >(); const gen = (async function* () { const paths: string[] = []; @@ -264,7 +264,7 @@ export class TarStream { /** * The WritableStream */ - get writable(): WritableStream { + get writable(): WritableStream { return this.#writable; } } diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index f7ff1a4bf282..9ff6682a7209 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -2,16 +2,16 @@ /** * The type extracted from the archive. */ -export type TarEntry = { +export type TarStreamEntry = { pathname: string; - header: TarHeader; + header: TarStreamHeader; readable?: ReadableStream; }; /** * The header of an entry in the archive. */ -export type TarHeader = { +export type TarStreamHeader = { name: string; mode: string; uid: string; @@ -97,7 +97,7 @@ export type TarHeader = { * ``` */ export class UnTarStream { - #readable: ReadableStream; + #readable: ReadableStream; #writable: WritableStream; /** * Constructs a new instance. @@ -155,8 +155,8 @@ export class UnTarStream { ), ) .getReader(); - let header: TarHeader | undefined; - this.#readable = new ReadableStream( + let header: TarStreamHeader | undefined; + this.#readable = new ReadableStream( { cancelled: false, async pull(controller) { @@ -294,7 +294,7 @@ export class UnTarStream { cancel() { this.cancelled = true; }, - } as UnderlyingSource & { cancelled: boolean }, + } as UnderlyingSource & { cancelled: boolean }, ); this.#writable = writable; } @@ -302,7 +302,7 @@ export class UnTarStream { /** * The ReadableStream */ - get readable(): ReadableStream { + get readable(): ReadableStream { return this.#readable; } From c5d3ff50633837b8a7326be50cd78de7d3db0c34 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Tue, 9 Apr 2024 15:49:35 +1000 Subject: [PATCH 07/82] set(archive): Set current time as mtime for default --- archive/tar_stream.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 4ba0685ef87f..70d53de4e4cf 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -200,7 +200,8 @@ export class TarStream { sizeExtension ? 12 : 11, "0", ) + (sizeExtension ? "" : " ") + // size - (chunk.options?.mtime?.toString(8) ?? "").padStart(11, "0") + + (chunk.options?.mtime?.toString(8) ?? + (new Date().getTime() / 1000).toFixed(0)).padStart(11, "0") + " " + // mtime " ".repeat(8) + // checksum | Needs to be updated typeflag + // typeflag From 20ecf4a6974ab257cfdb0c5b7a15fa0f009e4872 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Mon, 15 Apr 2024 16:48:59 +1000 Subject: [PATCH 08/82] resolve(archive): resolves comments made --- archive/tar_stream.ts | 39 ++++++++------------------------------- archive/untar_stream.ts | 40 +++++++++++++++++----------------------- 2 files changed, 25 insertions(+), 54 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 70d53de4e4cf..a6dbe9af6928 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -1,22 +1,22 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. /** - * The type required to provide a file. + * The interface required to provide a file. */ -export type TarStreamFile = { +export interface TarStreamFile { pathname: string; size: number; sizeExtension?: boolean; iterable: Iterable | AsyncIterable; options?: Partial; -}; +} /** - * The type required to provide a directory. + * The interface required to provide a directory. */ -export type TarStreamDir = { +export interface TarStreamDir { pathname: string; options?: Partial; -}; +} /** * The options that can go along with a file or directory. @@ -30,7 +30,7 @@ export type TarStreamDir = { * @param devmajor The major number for character device. * @param devminor The minor number for block device entry. */ -export type TarStreamOptions = { +export interface TarStreamOptions { mode: string; uid: string; gid: string; @@ -39,7 +39,7 @@ export type TarStreamOptions = { gname: string; devmajor: string; devminor: string; -}; +} /** * ### Overview @@ -61,29 +61,6 @@ export type TarStreamOptions = { * - The pathname is invalid. * - The size provided does not match that of the iterable's length. * - * @example - * ```ts - * import { TarStream } from 'https://deno.land/std@$STD_VERSION/archive/tar_stream.ts' - * - * await ReadableStream.from([ - * { - * pathname: 'potato/' - * }, - * { - * pathname: 'deno.json', - * size: (await Deno.stat('deno.json')).size, - * iterable: (await Deno.open('deno.json')).readable - * }, - * { - * pathname: 'deno.lock', - * size: (await Deno.stat('deno.lock')).size, - * iterable: (await Deno.open('deno.lock')).readable - * } - * ]) - * .pipeThrough(new TarStream()) - * .pipeTo((await Deno.create('./out.tar')).writable) - * ``` - * * ### Compression * Tar archives are not compressed by default. If you'd like to compress the * archive, you may do so by piping it through a compression stream. diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 9ff6682a7209..ddd0432d0d4e 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -1,17 +1,17 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. /** - * The type extracted from the archive. + * The interface extracted from the archive. */ -export type TarStreamEntry = { +export interface TarStreamEntry { pathname: string; header: TarStreamHeader; readable?: ReadableStream; -}; +} /** - * The header of an entry in the archive. + * The original tar archive header format. */ -export type TarStreamHeader = { +export interface OldStyleFormat { name: string; mode: string; uid: string; @@ -22,7 +22,12 @@ export type TarStreamHeader = { typeflag: string; linkname: string; pad: Uint8Array; -} | { +} + +/** + * The POSIX ustar archive header format. + */ +export interface PosixUstarFormat { name: string; mode: string; uid: string; @@ -40,7 +45,12 @@ export type TarStreamHeader = { devminor: string; prefix: string; pad: Uint8Array; -}; +} + +/** + * The header of an entry in the archive. + */ +export type TarStreamHeader = OldStyleFormat | PosixUstarFormat; /** * ### Overview @@ -58,22 +68,6 @@ export type TarStreamHeader = { * entry won't be resolved until the previous ReadableStream is either consumed * or cancelled. * - * @example - * ```ts - * import { UnTarStream } from 'https://deno.land/std@$STD_VERSION/archive/untar_stream.ts' - * - * for await ( - * const entry of (await Deno.open('./out.tar')) - * .readable - * .pipeThrough(new UnTarStream()) - * ) { - * console.log(entry.pathname) - * await entry - * .readable - * ?.pipeTo((await Deno.create(entry.pathname)).writable) - * } - * ``` - * * ### Understanding Compressed * A tar archive may be compressed, often identified by an additional file * extension, such as '.tar.gz' for gzip. This TransformStream does not support From e4f28dadbb4970d7b72040ecf84314884a1f6d0f Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sat, 20 Apr 2024 23:13:28 +1000 Subject: [PATCH 09/82] add(archive): `{ mode: 'byob' }` support for TarStream --- archive/tar_stream.ts | 44 ++++++++++++++++++++++++++++++++---- archive/tar_stream_test.ts | 2 +- archive/untar_stream_test.ts | 2 +- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index a6dbe9af6928..b4c622d52c34 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -219,13 +219,47 @@ export class TarStream { yield new Uint8Array(new Array(1024).fill(0)); })(); - this.#readable = new ReadableStream({ + this.#readable = new ReadableStream({ + type: "bytes", async pull(controller) { - const { done, value } = await gen.next(); - if (done) { - controller.close(); + if (controller.byobRequest?.view) { + const buffer = new Uint8Array( + controller.byobRequest.view.buffer, + controller.byobRequest.view.byteOffset, + controller.byobRequest.view.byteLength, + ); + let offset = 0; + + while (offset < buffer.length) { + const { done, value } = await gen.next(); + if (done) { + if (offset) { + controller.byobRequest.respond(offset); + controller.close(); + } else { + controller.close(); + controller.byobRequest.respond(0); + } + return; + } + if (value.length <= buffer.length - offset) { + buffer.set(value, offset); + offset += value.length; + } else { + buffer.set(value.slice(0, buffer.length - offset), offset); + offset = buffer.length - offset; + controller.byobRequest.respond(buffer.length); + return controller.enqueue(value.slice(offset)); + } + } + controller.byobRequest.respond(buffer.length); } else { - controller.enqueue(value); + const { done, value } = await gen.next(); + if (done) { + controller.close(); + } else { + controller.enqueue(value); + } } }, }); diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index eb85c53476a7..981b80170575 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -13,7 +13,7 @@ Deno.test("createTarArchiveViaStream", async function () { { pathname: "./text.txt", size: text.length, - iterable: [text], + iterable: [text.slice()], }, ]) .pipeThrough(new TarStream()), diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index c5b279490262..8ce6ac8bb340 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -39,7 +39,7 @@ Deno.test("unTarStreamValidatingBodies", async function () { { pathname: "./text.txt", size: text.length, - iterable: [text], + iterable: [text.slice()], }, ]) .pipeThrough(new TarStream()) From ecf365d8a3f7ca3eb9d9acd3d4e6e2a8649332e8 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sun, 21 Apr 2024 18:35:50 +1000 Subject: [PATCH 10/82] add(archive): `{ mode: 'byob' }` support for UnTarStream --- archive/untar_stream.ts | 66 ++++++++++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 10 deletions(-) diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index ddd0432d0d4e..f6dd24b055e7 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -232,20 +232,66 @@ export class UnTarStream { ? header.prefix + "/" : "") + header.name, header, - readable: new ReadableStream({ + readable: new ReadableStream({ + type: "bytes", async pull(controller) { if (i > 0) { lock = true; - const { done, value } = await reader.read(); - if (done) { - header = undefined; - controller.error("Tarball ended unexpectedly."); - } else { - // Pull is unlocked before enqueue is called because if pull is in the middle of processing a chunk when cancel is called, nothing after enqueue will run. - lock = false; - controller.enqueue( - i-- === 1 ? value.slice(0, size % 512) : value, + if (controller.byobRequest?.view) { + const buffer = new Uint8Array( + controller.byobRequest.view.buffer, + controller.byobRequest.view.byteOffset, + controller.byobRequest.view.byteLength, ); + let offset = 0; + while (offset < buffer.length) { + const { done, value } = await (async function () { + const x = await reader.read(); + if (!x.done && i-- === 1) { + x.value = x.value.slice(0, size % 512); + } + return x; + })(); + if (done) { + header = undefined; + lock = false; + if (offset) { + controller.byobRequest.respond(offset); + controller.close(); + } else { + controller.close(); + controller.byobRequest.respond(0); + } + return; + } + if (value.length <= buffer.length - offset) { + buffer.set(value, offset); + offset += value.length; + } else { + buffer.set( + value.slice(0, buffer.length - offset), + offset, + ); + offset = buffer.length - offset; + lock = false; + controller.byobRequest.respond(buffer.length); + return controller.enqueue(value.slice(offset)); + } + } + lock = false; + controller.byobRequest.respond(buffer.length); + } else { + const { done, value } = await reader.read(); + if (done) { + header = undefined; + controller.error("Tarball ended unexpectedly."); + } else { + // Pull is unlocked before enqueue is called because if pull is in the middle of processing a chunk when cancel is called, nothing after enqueue will run. + lock = false; + controller.enqueue( + i-- === 1 ? value.slice(0, size % 512) : value, + ); + } } } else { header = undefined; From 730fb7433bb073a2cbc070189b57709e82912d4c Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sun, 21 Apr 2024 18:47:55 +1000 Subject: [PATCH 11/82] adjust(archive): The logical flow of a few if statements --- archive/tar_stream.ts | 34 ++++++++++++++------------------ archive/untar_stream.ts | 43 +++++++++++++++++++---------------------- 2 files changed, 35 insertions(+), 42 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index b4c622d52c34..efeecc00c70a 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -222,45 +222,41 @@ export class TarStream { this.#readable = new ReadableStream({ type: "bytes", async pull(controller) { + // If Byte Stream if (controller.byobRequest?.view) { const buffer = new Uint8Array( controller.byobRequest.view.buffer, - controller.byobRequest.view.byteOffset, + controller.byobRequest.view.byteOffset, // Will this ever be anything but zero? controller.byobRequest.view.byteLength, ); let offset = 0; - while (offset < buffer.length) { const { done, value } = await gen.next(); if (done) { if (offset) { controller.byobRequest.respond(offset); - controller.close(); - } else { - controller.close(); - controller.byobRequest.respond(0); + return controller.close(); } - return; + controller.close(); + return controller.byobRequest.respond(0); } - if (value.length <= buffer.length - offset) { - buffer.set(value, offset); - offset += value.length; - } else { + if (value.length > buffer.length - offset) { buffer.set(value.slice(0, buffer.length - offset), offset); offset = buffer.length - offset; controller.byobRequest.respond(buffer.length); return controller.enqueue(value.slice(offset)); } + buffer.set(value, offset); + offset += value.length; } - controller.byobRequest.respond(buffer.length); - } else { - const { done, value } = await gen.next(); - if (done) { - controller.close(); - } else { - controller.enqueue(value); - } + return controller.byobRequest.respond(buffer.length); + } + // Else Default Stream + const { done, value } = await gen.next(); + if (done) { + return controller.close(); } + controller.enqueue(value); }, }); this.#writable = writable; diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index f6dd24b055e7..0941706f8566 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -237,10 +237,11 @@ export class UnTarStream { async pull(controller) { if (i > 0) { lock = true; + // If Byte Stream if (controller.byobRequest?.view) { const buffer = new Uint8Array( controller.byobRequest.view.buffer, - controller.byobRequest.view.byteOffset, + controller.byobRequest.view.byteOffset, // Will this ever be anything but zero? controller.byobRequest.view.byteLength, ); let offset = 0; @@ -257,17 +258,12 @@ export class UnTarStream { lock = false; if (offset) { controller.byobRequest.respond(offset); - controller.close(); - } else { - controller.close(); - controller.byobRequest.respond(0); + return controller.close(); } - return; + controller.close(); + return controller.byobRequest.respond(0); } - if (value.length <= buffer.length - offset) { - buffer.set(value, offset); - offset += value.length; - } else { + if (value.length > buffer.length - offset) { buffer.set( value.slice(0, buffer.length - offset), offset, @@ -277,22 +273,23 @@ export class UnTarStream { controller.byobRequest.respond(buffer.length); return controller.enqueue(value.slice(offset)); } + buffer.set(value, offset); + offset += value.length; } lock = false; - controller.byobRequest.respond(buffer.length); - } else { - const { done, value } = await reader.read(); - if (done) { - header = undefined; - controller.error("Tarball ended unexpectedly."); - } else { - // Pull is unlocked before enqueue is called because if pull is in the middle of processing a chunk when cancel is called, nothing after enqueue will run. - lock = false; - controller.enqueue( - i-- === 1 ? value.slice(0, size % 512) : value, - ); - } + return controller.byobRequest.respond(buffer.length); + } + // Else Default Stream + const { done, value } = await reader.read(); + if (done) { + header = undefined; + return controller.error("Tarball ended unexpectedly."); } + // Pull is unlocked before enqueue is called because if pull is in the middle of processing a chunk when cancel is called, nothing after enqueue will run. + lock = false; + controller.enqueue( + i-- === 1 ? value.slice(0, size % 512) : value, + ); } else { header = undefined; if (isCancelled()) { From 35715123796d895480917beee0eedd5f74a2f8dc Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sun, 21 Apr 2024 19:08:00 +1000 Subject: [PATCH 12/82] tests(archive): Updated Tests for Un/TarStream --- archive/tar_stream_test.ts | 71 ++++++++++++++------- archive/untar_stream_test.ts | 117 +++++++++++++++++++++++------------ 2 files changed, 124 insertions(+), 64 deletions(-) diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index 981b80170575..dff8ca55ce54 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -2,33 +2,58 @@ import { TarStream } from "./tar_stream.ts"; import { assertEquals } from "../assert/mod.ts"; -Deno.test("createTarArchiveViaStream", async function () { +Deno.test("createTarArchiveDefaultStream", async function () { const text = new TextEncoder().encode("Hello World!"); - const size = (await reduce( - ReadableStream.from([ - { - pathname: "./potato", - }, - { - pathname: "./text.txt", - size: text.length, - iterable: [text.slice()], - }, - ]) - .pipeThrough(new TarStream()), - )).length; + const reader = ReadableStream.from([ + { + pathname: "./potato", + }, + { + pathname: "./text.txt", + size: text.length, + iterable: [text.slice()], + }, + ]) + .pipeThrough(new TarStream()) + .getReader(); + let size = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + size += value.length; + } assertEquals(size, 512 + 512 + Math.ceil(text.length / 512) * 512 + 1024); }); -async function reduce(readable: ReadableStream) { - let y = new Uint8Array(0); - for await (const x of readable) { - const z = new Uint8Array(x.length + y.length); - z.set(y); - z.set(x, y.length); - y = z; +Deno.test("createTarArchiveByteStream", async function () { + const text = new TextEncoder().encode("Hello World!"); + + const reader = ReadableStream.from([ + { + pathname: "./potato", + }, + { + pathname: "./text.txt", + size: text.length, + iterable: [text.slice()], + }, + ]) + .pipeThrough(new TarStream()) + .getReader({ mode: "byob" }); + + let size = 0; + while (true) { + const { done, value } = await reader.read( + new Uint8Array(Math.floor(Math.random() * 1024)), + ); + if (done) { + break; + } + size += value.length; } - return y; -} + assertEquals(size, 512 + 512 + Math.ceil(text.length / 512) * 512 + 1024); +}); diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index 8ce6ac8bb340..9b368cdcb974 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -3,61 +3,96 @@ import { TarStream } from "./tar_stream.ts"; import { UnTarStream } from "./untar_stream.ts"; import { assertEquals } from "../assert/mod.ts"; -Deno.test("unTarStreamCheckingHeaders", async function () { +Deno.test("expandTarArchiveCheckingHeaders", async function () { const text = new TextEncoder().encode("Hello World!"); + const readable = ReadableStream.from([ + { + pathname: "./potato", + }, + { + pathname: "./text.txt", + size: text.length, + iterable: [text], + }, + ]) + .pipeThrough(new TarStream()) + .pipeThrough(new UnTarStream()); + const pathnames: string[] = []; - for await ( - const item of ReadableStream.from([ - { - pathname: "./potato", - }, - { - pathname: "./text.txt", - size: text.length, - iterable: [text], - }, - ]) - .pipeThrough(new TarStream()) - .pipeThrough(new UnTarStream()) - ) { + for await (const item of readable) { pathnames.push(item.pathname); item.readable?.cancel(); } - assertEquals(pathnames, ["potato/", "text.txt"]); }); -Deno.test("unTarStreamValidatingBodies", async function () { +Deno.test("expandTarArchiveCheckingBodiesDefaultStream", async function () { const text = new TextEncoder().encode("Hello World!"); - for await ( - const item of ReadableStream.from([ - { - pathname: "./potato", - }, - { - pathname: "./text.txt", - size: text.length, - iterable: [text.slice()], - }, - ]) - .pipeThrough(new TarStream()) - .pipeThrough(new UnTarStream()) - ) { + const readable = ReadableStream.from([ + { + pathname: "./potato", + }, + { + pathname: "./text.txt", + size: text.length, + iterable: [text.slice()], + }, + ]) + .pipeThrough(new TarStream()) + .pipeThrough(new UnTarStream()); + + for await (const item of readable) { if (item.readable) { - assertEquals(await reduce(item.readable), text); + const buffer = new Uint8Array(text.length); + let offset = 0; + const reader = item.readable.getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + buffer.set(value, offset); + offset += value.length; + } + assertEquals(buffer, text); } } }); -async function reduce(readable: ReadableStream) { - let y = new Uint8Array(0); - for await (const x of readable) { - const z = new Uint8Array(x.length + y.length); - z.set(y); - z.set(x, y.length); - y = z; +Deno.test("expandTarArchiveCheckingBodiesByteStream", async function () { + const text = new TextEncoder().encode("Hello World!"); + + const readable = ReadableStream.from([ + { + pathname: "./potato", + }, + { + pathname: "./text.txt", + size: text.length, + iterable: [text.slice()], + }, + ]) + .pipeThrough(new TarStream()) + .pipeThrough(new UnTarStream()); + + for await (const item of readable) { + if (item.readable) { + const buffer = new Uint8Array(text.length); + let offset = 0; + const reader = item.readable.getReader({ mode: "byob" }); + while (true) { + const { done, value } = await reader.read( + new Uint8Array(Math.floor(Math.random() * 1024)), + ); + if (done) { + break; + } + buffer.set(value, offset); + offset += value.length; + } + assertEquals(buffer, text); + } } - return y; -} +}); From 479a89045d6cc5e586bebeb96b6fb03cab22f9c0 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Mon, 22 Apr 2024 18:54:58 +1000 Subject: [PATCH 13/82] fix(archive): TarStream mtime wasn't an octal --- archive/tar_stream.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index efeecc00c70a..217d08d4642a 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -178,7 +178,7 @@ export class TarStream { "0", ) + (sizeExtension ? "" : " ") + // size (chunk.options?.mtime?.toString(8) ?? - (new Date().getTime() / 1000).toFixed(0)).padStart(11, "0") + + Math.floor(new Date().getTime() / 1000).toString(8)).padStart(11, "0") + " " + // mtime " ".repeat(8) + // checksum | Needs to be updated typeflag + // typeflag From eeb9e35ca74fbb341376bbfcb0675e7d30436d3b Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Tue, 23 Apr 2024 12:22:59 +1000 Subject: [PATCH 14/82] fix(archive): TarStream tests --- archive/tar_stream.ts | 5 ++++- archive/tar_stream_test.ts | 2 +- archive/untar_stream_test.ts | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 217d08d4642a..2a5fc2b09f27 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -178,7 +178,10 @@ export class TarStream { "0", ) + (sizeExtension ? "" : " ") + // size (chunk.options?.mtime?.toString(8) ?? - Math.floor(new Date().getTime() / 1000).toString(8)).padStart(11, "0") + + Math.floor(new Date().getTime() / 1000).toString(8)).padStart( + 11, + "0", + ) + " " + // mtime " ".repeat(8) + // checksum | Needs to be updated typeflag + // typeflag diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index dff8ca55ce54..f9205d1e964c 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -48,7 +48,7 @@ Deno.test("createTarArchiveByteStream", async function () { let size = 0; while (true) { const { done, value } = await reader.read( - new Uint8Array(Math.floor(Math.random() * 1024)), + new Uint8Array(Math.ceil(Math.random() * 1024)), ); if (done) { break; diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index 9b368cdcb974..e864bf638646 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -84,7 +84,7 @@ Deno.test("expandTarArchiveCheckingBodiesByteStream", async function () { const reader = item.readable.getReader({ mode: "byob" }); while (true) { const { done, value } = await reader.read( - new Uint8Array(Math.floor(Math.random() * 1024)), + new Uint8Array(Math.ceil(Math.random() * 1024)), ); if (done) { break; From 620c6b6c960a40c16cb636ed07137ebc13d29c3c Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sun, 28 Apr 2024 13:36:22 +1000 Subject: [PATCH 15/82] add(archive): Added parsePathname function Added parsePathname function abstracting the logic out of TarStream allowing the developer to validate pathnames before providing them to TarStream hoping it doesn't throw an error and require the archive creation to start all over again. --- archive/tar_stream.ts | 116 ++++++++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 44 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 2a5fc2b09f27..545d9c569ce3 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -3,7 +3,7 @@ * The interface required to provide a file. */ export interface TarStreamFile { - pathname: string; + pathname: string | [Uint8Array, Uint8Array]; size: number; sizeExtension?: boolean; iterable: Iterable | AsyncIterable; @@ -14,7 +14,7 @@ export interface TarStreamFile { * The interface required to provide a directory. */ export interface TarStreamDir { - pathname: string; + pathname: string | [Uint8Array, Uint8Array]; options?: Partial; } @@ -116,56 +116,34 @@ export class TarStream { ); } - chunk.pathname = chunk.pathname.split("/").filter((x) => x).join("/"); - if (chunk.pathname.startsWith("./")) { - chunk.pathname = chunk.pathname.slice(2); - } - if (!("size" in chunk)) { - chunk.pathname += "/"; - } - - const pathname = new TextEncoder().encode(chunk.pathname); - if (pathname.length > 256) { - throw new Error( - "Invalid Pathname! Pathname cannot exceed 256 bytes.", - ); - } - - let i = Math.max(0, pathname.lastIndexOf(47)); - if (pathname.slice(i + 1).length > 100) { - throw new Error( - "Invalid Filename! Filename cannot exceed 100 bytes.", - ); - } - - if (pathname.length <= 100) { - i = 0; - } else { - for (; i > 0; --i) { - i = pathname.lastIndexOf(47, i); - if (pathname.slice(i + 1).length > 100) { - i = Math.max(0, pathname.indexOf(47, i + 1)); - break; + const [prefix, name] = typeof chunk.pathname === "string" + ? parsePathname(chunk.pathname, !("size" in chunk)) + : function () { + if ("size" in chunk === (chunk.pathname[1].slice(-1)[0] === 47)) { + throw new Error( + `Pre-parsed pathname for ${ + "size" in chunk ? "directory" : "file" + } is not suffixed correctly. Directories should end in a forward slash, while files shouldn't.`, + ); } + return chunk.pathname; + }(); + { + const decoder = new TextDecoder(); + const pathname = prefix.length + ? decoder.decode(prefix) + "/" + decoder.decode(name) + : decoder.decode(name); + if (paths.includes(pathname)) { + continue; } + paths.push(pathname); } - - const prefix = pathname.slice(0, i); - if (prefix.length > 155) { - throw new Error( - "Invalid Pathname! Pathname needs to be split-able on a forward slash separator into [155, 100] bytes respectively.", - ); - } - if (paths.includes(chunk.pathname)) { - continue; - } - paths.push(chunk.pathname); const typeflag = "size" in chunk ? "0" : "5"; const sizeExtension = "size" in chunk && chunk.sizeExtension || false; const encoder = new TextEncoder(); const header = new Uint8Array(512); - header.set(prefix.length ? pathname.slice(i + 1) : pathname); // name + header.set(name); // name header.set( encoder.encode( (chunk.options?.mode ?? (typeflag === "5" ? "755" : "644")) @@ -279,3 +257,53 @@ export class TarStream { return this.#writable; } } + +/** + * parsePathname is a function that validates the correctness of the pathname + * being provided. + * Function will throw if invalid pathname is provided. + * The result can be provided instead of the string version to TarStream, + * or can just be used to check in advance of creating the Tar archive. + */ +export function parsePathname( + pathname: string, + isDirectory = false, +): [Uint8Array, Uint8Array] { + pathname = pathname.split("/").filter((x) => x).join("/"); + if (pathname.startsWith("./")) { + pathname = pathname.slice(2); + } + if (isDirectory) { + pathname += "/"; + } + + const name = new TextEncoder().encode(pathname); + if (name.length <= 100) { + return [new Uint8Array(0), name]; + } + + if (name.length > 256) { + throw new Error("Invalid Pathname! Pathname cannot exceed 256 bytes."); + } + + let i = Math.max(0, name.lastIndexOf(47)); + if (pathname.slice(i + 1).length > 100) { + throw new Error("Invalid Filename! Filename cannot exceed 100 bytes."); + } + + for (; i > 0; --i) { + i = name.lastIndexOf(47, i) + 1; + if (name.slice(i + 1).length > 100) { + i = Math.max(0, name.indexOf(47, i + 1)); + break; + } + } + + const prefix = name.slice(0, i); + if (prefix.length > 155) { + throw new Error( + "Invalid Pathname! Pathname needs to be split-able on a forward slash separator into [155, 100] bytes respectively.", + ); + } + return [prefix, name.slice(i + 1)]; +} From 2ac96b997f5dce96f1c8c9a50ab56741cdfe5df2 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Fri, 3 May 2024 17:02:53 +1000 Subject: [PATCH 16/82] fix(archive): extra bytes incorrectly appending at the end of files When the appending file was exactly divisible by 512 bytes, an extra 512 bytes was being appending instead of zero to fill in the gap, causing the next header to be read at the wrong place. --- archive/tar_stream.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 545d9c569ce3..a80a514b3fdd 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -194,7 +194,9 @@ export class TarStream { "Invalid Tarball! Provided size did not match bytes read from iterable.", ); } - yield new Uint8Array(new Array(512 - chunk.size % 512).fill(0)); + if (chunk.size % 512) { + yield new Uint8Array(new Array(512 - chunk.size % 512).fill(0)); + } } } yield new Uint8Array(new Array(1024).fill(0)); From 043a0a9e704307a96222fde29624a6c80eb0aa38 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Fri, 3 May 2024 17:44:21 +1000 Subject: [PATCH 17/82] adjust(archive): to always return the amount of bytes requested Instead of using enqueue, the leftover bytes are saved for later for the next buffer provided. --- archive/tar_stream.ts | 85 ++++++++++++--------- archive/untar_stream.ts | 161 +++++++++++++++++++++------------------- 2 files changed, 134 insertions(+), 112 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index a80a514b3fdd..23f9b32f7e54 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -202,46 +202,57 @@ export class TarStream { yield new Uint8Array(new Array(1024).fill(0)); })(); - this.#readable = new ReadableStream({ - type: "bytes", - async pull(controller) { - // If Byte Stream - if (controller.byobRequest?.view) { - const buffer = new Uint8Array( - controller.byobRequest.view.buffer, - controller.byobRequest.view.byteOffset, // Will this ever be anything but zero? - controller.byobRequest.view.byteLength, - ); - let offset = 0; - while (offset < buffer.length) { - const { done, value } = await gen.next(); - if (done) { - if (offset) { - controller.byobRequest.respond(offset); - return controller.close(); - } - controller.close(); - return controller.byobRequest.respond(0); + this.#readable = new ReadableStream( + { + leftover: new Uint8Array(0), + type: "bytes", + async pull(controller) { + // If Byte Stream + if (controller.byobRequest?.view) { + const buffer = new Uint8Array( + controller.byobRequest.view.buffer, + ); + if (buffer.length < this.leftover.length) { + buffer.set(this.leftover.slice(0, buffer.length)); + this.leftover = this.leftover.slice(buffer.length); + return controller.byobRequest.respond(buffer.length); } - if (value.length > buffer.length - offset) { - buffer.set(value.slice(0, buffer.length - offset), offset); - offset = buffer.length - offset; - controller.byobRequest.respond(buffer.length); - return controller.enqueue(value.slice(offset)); + buffer.set(this.leftover); + let offset = this.leftover.length; + while (offset < buffer.length) { + const { done, value } = await gen.next(); + if (done) { + try { + controller.byobRequest.respond(offset); // Will throw if zero + controller.close(); + } catch { + controller.close(); + controller.byobRequest.respond(0); // But still needs to be resolved. + } + return; + } + if (value.length > buffer.length - offset) { + buffer.set(value.slice(0, buffer.length - offset), offset); + offset = buffer.length - offset; + controller.byobRequest.respond(buffer.length); + this.leftover = value.slice(offset); + return; + } + buffer.set(value, offset); + offset += value.length; } - buffer.set(value, offset); - offset += value.length; + this.leftover = new Uint8Array(0); + return controller.byobRequest.respond(buffer.length); } - return controller.byobRequest.respond(buffer.length); - } - // Else Default Stream - const { done, value } = await gen.next(); - if (done) { - return controller.close(); - } - controller.enqueue(value); - }, - }); + // Else Default Stream + const { done, value } = await gen.next(); + if (done) { + return controller.close(); + } + controller.enqueue(value); + }, + } as UnderlyingByteSource & { leftover: Uint8Array }, + ); this.#writable = writable; } diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 0941706f8566..db56ab69b3de 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -232,91 +232,102 @@ export class UnTarStream { ? header.prefix + "/" : "") + header.name, header, - readable: new ReadableStream({ - type: "bytes", - async pull(controller) { - if (i > 0) { - lock = true; - // If Byte Stream - if (controller.byobRequest?.view) { - const buffer = new Uint8Array( - controller.byobRequest.view.buffer, - controller.byobRequest.view.byteOffset, // Will this ever be anything but zero? - controller.byobRequest.view.byteLength, - ); - let offset = 0; - while (offset < buffer.length) { - const { done, value } = await (async function () { - const x = await reader.read(); - if (!x.done && i-- === 1) { - x.value = x.value.slice(0, size % 512); + readable: new ReadableStream( + { + leftover: new Uint8Array(0), + type: "bytes", + async pull(controller) { + if (i > 0) { + lock = true; + // If Byte Stream + if (controller.byobRequest?.view) { + const buffer = new Uint8Array( + controller.byobRequest.view.buffer, + ); + if (buffer.length < this.leftover.length) { + buffer.set(this.leftover.slice(0, buffer.length)); + this.leftover = this.leftover.slice(buffer.length); + return controller.byobRequest.respond(buffer.length); + } + buffer.set(this.leftover); + let offset = this.leftover.length; + while (offset < buffer.length) { + const { done, value } = await (async function () { + const x = await reader.read(); + if (!x.done && i-- === 1) { + x.value = x.value.slice(0, size % 512); + } + return x; + })(); + if (done) { + header = undefined; + lock = false; + try { + controller.byobRequest.respond(offset); // Will throw if zero. + controller.close(); + } catch { + controller.close(); + controller.byobRequest.respond(0); // But still needs to be resolved. + } + return; } - return x; - })(); - if (done) { - header = undefined; - lock = false; - if (offset) { - controller.byobRequest.respond(offset); - return controller.close(); + if (value.length > buffer.length - offset) { + buffer.set( + value.slice(0, buffer.length - offset), + offset, + ); + offset = buffer.length - offset; + lock = false; + controller.byobRequest.respond(buffer.length); + this.leftover = value.slice(offset); + return; } - controller.close(); - return controller.byobRequest.respond(0); - } - if (value.length > buffer.length - offset) { - buffer.set( - value.slice(0, buffer.length - offset), - offset, - ); - offset = buffer.length - offset; - lock = false; - controller.byobRequest.respond(buffer.length); - return controller.enqueue(value.slice(offset)); + buffer.set(value, offset); + offset += value.length; } - buffer.set(value, offset); - offset += value.length; + lock = false; + this.leftover = new Uint8Array(0); + return controller.byobRequest.respond(buffer.length); } + // Else Default Stream + const { done, value } = await reader.read(); + if (done) { + header = undefined; + return controller.error("Tarball ended unexpectedly."); + } + // Pull is unlocked before enqueue is called because if pull is in the middle of processing a chunk when cancel is called, nothing after enqueue will run. lock = false; - return controller.byobRequest.respond(buffer.length); - } - // Else Default Stream - const { done, value } = await reader.read(); - if (done) { + controller.enqueue( + i-- === 1 ? value.slice(0, size % 512) : value, + ); + } else { header = undefined; - return controller.error("Tarball ended unexpectedly."); + if (isCancelled()) { + reader.cancel(); + } + controller.close(); } - // Pull is unlocked before enqueue is called because if pull is in the middle of processing a chunk when cancel is called, nothing after enqueue will run. - lock = false; - controller.enqueue( - i-- === 1 ? value.slice(0, size % 512) : value, - ); - } else { - header = undefined; - if (isCancelled()) { - reader.cancel(); + }, + async cancel() { + while (lock) { + await new Promise((a) => + setTimeout(a, 0) + ); } - controller.close(); - } - }, - async cancel() { - while (lock) { - await new Promise((a) => - setTimeout(a, 0) - ); - } - try { - while (i-- > 0) { - if ((await reader.read()).done) { - throw new Error("Tarball ended unexpectedly."); + try { + while (i-- > 0) { + if ((await reader.read()).done) { + throw new Error("Tarball ended unexpectedly."); + } } + } catch (error) { + throw error; + } finally { + header = undefined; } - } catch (error) { - throw error; - } finally { - header = undefined; - } - }, - }), + }, + } as UnderlyingByteSource & { leftover: Uint8Array }, + ), }); } else { controller.enqueue({ From 22e2ef4449a15e714caade527ae0c99fb990182c Mon Sep 17 00:00:00 2001 From: Asher Gomez Date: Mon, 13 May 2024 16:28:27 +1000 Subject: [PATCH 18/82] tweaks --- archive/deno.json | 2 ++ archive/tar_stream.ts | 2 +- archive/untar_stream.ts | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/archive/deno.json b/archive/deno.json index e5134bef12f6..172ebf52ca3b 100644 --- a/archive/deno.json +++ b/archive/deno.json @@ -3,7 +3,9 @@ "version": "0.224.0", "exports": { ".": "./mod.ts", + "./tar-stream": "./tar-stream.ts", "./tar": "./tar.ts", + "./untar-stream": "./untar-stream.ts", "./untar": "./untar.ts" } } diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 23f9b32f7e54..74f193f39fdc 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -67,7 +67,7 @@ export interface TarStreamOptions { * * @example * ```ts - * import { TarStream } from 'https://deno.land/std@$STD_VERSION/archive/tar_stream.ts' + * import { TarStream } from "@std/archive/tar-stream"; * * await ReadableStream.from([ * { diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index db56ab69b3de..628446cee7cc 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -75,7 +75,7 @@ export type TarStreamHeader = OldStyleFormat | PosixUstarFormat; * * @example * ```ts - * import { UnTarStream } from 'https://deno.land/std@$STD_VERSION/archive/untar_stream.ts' + * import { UnTarStream } from "@std/archive/untar-stream"; * * for await ( * const entry of (await Deno.open('./out.tar.gz')) From dacaea5b10a5bc21abe96a2759ee3ead8a58bf11 Mon Sep 17 00:00:00 2001 From: Asher Gomez Date: Mon, 13 May 2024 17:33:58 +1000 Subject: [PATCH 19/82] fix --- archive/deno.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archive/deno.json b/archive/deno.json index 172ebf52ca3b..dbe48201534d 100644 --- a/archive/deno.json +++ b/archive/deno.json @@ -3,9 +3,9 @@ "version": "0.224.0", "exports": { ".": "./mod.ts", - "./tar-stream": "./tar-stream.ts", + "./tar-stream": "./tar_stream.ts", "./tar": "./tar.ts", - "./untar-stream": "./untar-stream.ts", + "./untar-stream": "./untar_stream.ts", "./untar": "./untar.ts" } } From f3d93b61794ba6737413d56c26fda4f2d6eaf1c9 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Fri, 24 May 2024 15:23:53 +1000 Subject: [PATCH 20/82] docs(archive): Link to the spec that they're following --- archive/tar_stream.ts | 3 ++- archive/untar_stream.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 74f193f39fdc..770892e94c0e 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -45,7 +45,8 @@ export interface TarStreamOptions { * ### Overview * A TransformStream to create a tar archive. Tar archives allow for storing * multiple files in a single file (called an archive, or sometimes a tarball). - * These archives typically have a singe '.tar' extension. + * These archives typically have a singe '.tar' extension. This + * implementation follows the [FreeBSD 15.0](https://man.freebsd.org/cgi/man.cgi?query=tar&sektion=5&apropos=0&manpath=FreeBSD+15.0-CURRENT) spec. * * ### File Format & Limitations * The ustar file format is used for creating the tar archive. While this diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 628446cee7cc..1024a3e3cafd 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -56,7 +56,8 @@ export type TarStreamHeader = OldStyleFormat | PosixUstarFormat; * ### Overview * A TransformStream to expand a tar archive. Tar archives allow for storing * multiple files in a single file (called an archive, or sometimes a tarball). - * These archives typically have a single '.tar' extension. + * These archives typically have a single '.tar' extension. This + * implementation follows the [FreeBSD 15.0](https://man.freebsd.org/cgi/man.cgi?query=tar&sektion=5&apropos=0&manpath=FreeBSD+15.0-CURRENT) spec. * * ### Supported File Formats * Only the ustar file format is supported. This is the most common format. From da8206a8b970972270526abb6eb4cc6ffbeeecda Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Fri, 24 May 2024 15:25:02 +1000 Subject: [PATCH 21/82] docs(archive): fix spelling --- archive/tar_stream.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 770892e94c0e..43a34767fa61 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -45,7 +45,7 @@ export interface TarStreamOptions { * ### Overview * A TransformStream to create a tar archive. Tar archives allow for storing * multiple files in a single file (called an archive, or sometimes a tarball). - * These archives typically have a singe '.tar' extension. This + * These archives typically have a single '.tar' extension. This * implementation follows the [FreeBSD 15.0](https://man.freebsd.org/cgi/man.cgi?query=tar&sektion=5&apropos=0&manpath=FreeBSD+15.0-CURRENT) spec. * * ### File Format & Limitations From 68f8400b177babd66be4f1a34a76bbbaaaadd540 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Fri, 24 May 2024 16:02:08 +1000 Subject: [PATCH 22/82] add(archive): function validTarSteamOptions - To make sure, if TarStreamOptions are being provided, that they are in the correct format so as to not create bad tarballs. --- archive/tar_stream.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 43a34767fa61..1f99b5e31520 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -104,6 +104,10 @@ export class TarStream { const gen = (async function* () { const paths: string[] = []; for await (const chunk of readable) { + if (chunk.options && !validTarStreamOptions(chunk.options)) { + throw new Error("Invalid Options Provided!"); + } + if ( "size" in chunk && ( @@ -321,3 +325,21 @@ export function parsePathname( } return [prefix, name.slice(i + 1)]; } +/** + * validTarStreamOptions is a function that returns a true if all of the options + * provided are in the correct format, otherwise returns false. + */ +export function validTarStreamOptions( + options: Partial, +): boolean { + return !!(options.mode && !/^[0-7+$]/.test(options.mode) || + options.uid && !/^[0-7+$]/.test(options.uid) || + options.gid && !/^[0-7+$]/.test(options.gid) || + options.mtime && options.mtime.toString() === "NaN" || + // deno-lint-ignore no-control-regex + options.uname && /^[\x00-\x7F]*$/.test(options.uname) || + // deno-lint-ignore no-control-regex + options.gname && /^[\x00-\x7F]*$/.test(options.gname) || + options.devmajor && !/^ [0 - 7 + $] /.test(options.devmajor) || + options.devminor && !/^[0-7+$]/.test(options.devminor)); +} From 33799963e05cfe00a936de5b851363b669260a3b Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Fri, 24 May 2024 18:38:03 +1000 Subject: [PATCH 23/82] add(archive): more tests --- archive/tar_stream.ts | 8 ++- archive/tar_stream_test.ts | 115 +++++++++++++++++++++++++++++++++++-- 2 files changed, 116 insertions(+), 7 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 1f99b5e31520..87117dbfd0e2 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -18,6 +18,12 @@ export interface TarStreamDir { options?: Partial; } +/** + * A union type merging all the TarStream interfaces that can be piped into the + * TarStream class. + */ +export type TarStreamInput = TarStreamFile | TarStreamDir; + /** * The options that can go along with a file or directory. * @param mode An octal number in ASCII. @@ -92,7 +98,7 @@ export interface TarStreamOptions { */ export class TarStream { #readable: ReadableStream; - #writable: WritableStream; + #writable: WritableStream; /** * Constructs a new instance. */ diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index f9205d1e964c..6b4a24d68307 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -1,11 +1,11 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -import { TarStream } from "./tar_stream.ts"; -import { assertEquals } from "../assert/mod.ts"; +import { TarStream, type TarStreamInput } from "./tar_stream.ts"; +import { assertEquals, assertRejects } from "../assert/mod.ts"; -Deno.test("createTarArchiveDefaultStream", async function () { +Deno.test("TarStream() with default stream", async () => { const text = new TextEncoder().encode("Hello World!"); - const reader = ReadableStream.from([ + const reader = ReadableStream.from([ { pathname: "./potato", }, @@ -29,10 +29,10 @@ Deno.test("createTarArchiveDefaultStream", async function () { assertEquals(size, 512 + 512 + Math.ceil(text.length / 512) * 512 + 1024); }); -Deno.test("createTarArchiveByteStream", async function () { +Deno.test("TarStream() with byte stream", async () => { const text = new TextEncoder().encode("Hello World!"); - const reader = ReadableStream.from([ + const reader = ReadableStream.from([ { pathname: "./potato", }, @@ -57,3 +57,106 @@ Deno.test("createTarArchiveByteStream", async function () { } assertEquals(size, 512 + 512 + Math.ceil(text.length / 512) * 512 + 1024); }); + +Deno.test("TarStream() with negative size", async () => { + const text = new TextEncoder().encode("Hello World"); + + const readable = ReadableStream.from([ + { + pathname: "name", + size: -text.length, + iterable: [text.slice()], + }, + ]) + .pipeThrough(new TarStream()); + + await assertRejects( + async function () { + await Array.fromAsync(readable); + }, + Error, + "Invalid Size Provided! Size cannot exceed 8 GiBs by default or 64 GiBs with sizeExtension set to true.", + ); +}); + +Deno.test("TarStream() with 9 GiB size", async () => { + const size = 1024 ** 3 * 9; + const step = 1024; // Size must equally be divisible by step + const iterable = function* () { + for (let i = 0; i < size; i += step) { + yield new Uint8Array(step).map(() => Math.floor(Math.random() * 256)); + } + }(); + + const readable = ReadableStream.from([ + { + pathname: "name", + size, + iterable, + }, + ]) + .pipeThrough(new TarStream()); + + await assertRejects( + async function () { + await Array.fromAsync(readable); + }, + Error, + "Invalid Size Provided! Size cannot exceed 8 GiBs by default or 64 GiBs with sizeExtension set to true.", + ); +}); + +Deno.test("TarStream() with 65 GiB size", async () => { + const size = 1024 ** 3 * 65; + const step = 1024; // Size must equally be divisible by step + const iterable = function* () { + for (let i = 0; i < size; i += step) { + yield new Uint8Array(step).map(() => Math.floor(Math.random() * 256)); + } + }(); + + const readable = ReadableStream.from([ + { + pathname: "name", + size, + iterable, + sizeExtension: true, + }, + ]) + .pipeThrough(new TarStream()); + + await assertRejects( + async function () { + await Array.fromAsync(readable); + }, + Error, + "Invalid Size Provided! Size cannot exceed 8 GiBs by default or 64 GiBs with sizeExtension set to true.", + ); +}); + +Deno.test("TarStream() with NaN size", async () => { + const size = NaN; + const step = 1024; // Size must equally be divisible by step + const iterable = function* () { + for (let i = 0; i < size; i += step) { + yield new Uint8Array(step).map(() => Math.floor(Math.random() * 256)); + } + }(); + + const readable = ReadableStream.from([ + { + pathname: "name", + size, + iterable, + }, + ]) + .pipeThrough(new TarStream()); + + await assertRejects( + async function () { + await Array.fromAsync(readable); + }, + Error, + "Invalid Size Provided! Size cannot exceed 8 GiBs by default or 64 GiBs with sizeExtension set to true.", + ); +}); From 6f8b662cab9505e217ff09878444f9600d223fd2 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Mon, 3 Jun 2024 19:03:58 +1000 Subject: [PATCH 24/82] fix(archive): validTarStreamOptions --- archive/tar_stream.ts | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 87117dbfd0e2..c2b263ee52d7 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -338,14 +338,37 @@ export function parsePathname( export function validTarStreamOptions( options: Partial, ): boolean { - return !!(options.mode && !/^[0-7+$]/.test(options.mode) || - options.uid && !/^[0-7+$]/.test(options.uid) || - options.gid && !/^[0-7+$]/.test(options.gid) || - options.mtime && options.mtime.toString() === "NaN" || + if ( + options.mode && (options.mode.length > 6 || !/^[0-7]*$/.test(options.mode)) + ) return false; + if ( + options.uid && (options.uid.length > 6 || !/^[0-7]*$/.test(options.uid)) + ) return false; + if ( + options.gid && (options.gid.length > 6 || !/^[0-7]*$/.test(options.gid)) + ) return false; + if ( + options.mtime != undefined && + (options.mtime.toString(8).length > 11 || + options.mtime.toString() === "NaN") + ) return false; + if ( + options.uname && // deno-lint-ignore no-control-regex - options.uname && /^[\x00-\x7F]*$/.test(options.uname) || + (options.uname.length > 32 || !/^[\x00-\x7F]*$/.test(options.uname)) + ) return false; + if ( + options.gname && // deno-lint-ignore no-control-regex - options.gname && /^[\x00-\x7F]*$/.test(options.gname) || - options.devmajor && !/^ [0 - 7 + $] /.test(options.devmajor) || - options.devminor && !/^[0-7+$]/.test(options.devminor)); + (options.gname.length > 32 || !/^[\x00-\x7F]*$/.test(options.gname)) + ) return false; + if ( + options.devmajor && + (options.devmajor.length > 8 || !/^[0-7]*$/.test(options.devmajor)) + ) return false; + if ( + options.devminor && + (options.devminor.length > 8 || !/^[0-7]*$/.test(options.devminor)) + ) return false; + return true; } From de3d67f4434038dcc789f994b01bfc92a220d75d Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Mon, 3 Jun 2024 19:04:30 +1000 Subject: [PATCH 25/82] add(archive): tests for validTarStreamOptions --- archive/tar_stream_test.ts | 53 +++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index 6b4a24d68307..85a78b7cea6b 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -1,5 +1,9 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -import { TarStream, type TarStreamInput } from "./tar_stream.ts"; +import { + TarStream, + type TarStreamInput, + validTarStreamOptions, +} from "./tar_stream.ts"; import { assertEquals, assertRejects } from "../assert/mod.ts"; Deno.test("TarStream() with default stream", async () => { @@ -160,3 +164,50 @@ Deno.test("TarStream() with NaN size", async () => { "Invalid Size Provided! Size cannot exceed 8 GiBs by default or 64 GiBs with sizeExtension set to true.", ); }); + +Deno.test("validTarStreamOptions()", () => { + assertEquals(validTarStreamOptions({}), true); + + assertEquals(validTarStreamOptions({ mode: "" }), true); + assertEquals(validTarStreamOptions({ mode: "000" }), true); + assertEquals(validTarStreamOptions({ mode: "008" }), false); + assertEquals(validTarStreamOptions({ mode: "0000000" }), false); + + assertEquals(validTarStreamOptions({ uid: "" }), true); + assertEquals(validTarStreamOptions({ uid: "000" }), true); + assertEquals(validTarStreamOptions({ uid: "008" }), false); + assertEquals(validTarStreamOptions({ uid: "0000000" }), false); + + assertEquals(validTarStreamOptions({ gid: "" }), true); + assertEquals(validTarStreamOptions({ gid: "000" }), true); + assertEquals(validTarStreamOptions({ gid: "008" }), false); + assertEquals(validTarStreamOptions({ gid: "0000000" }), false); + + assertEquals(validTarStreamOptions({ mtime: 0 }), true); + assertEquals(validTarStreamOptions({ mtime: NaN }), false); + assertEquals( + validTarStreamOptions({ mtime: Math.floor(new Date().getTime() / 1000) }), + true, + ); + assertEquals(validTarStreamOptions({ mtime: new Date().getTime() }), false); + + assertEquals(validTarStreamOptions({ uname: "" }), true); + assertEquals(validTarStreamOptions({ uname: "abcdef" }), true); + assertEquals(validTarStreamOptions({ uname: "å-abcdef" }), false); + assertEquals(validTarStreamOptions({ uname: "a".repeat(100) }), false); + + assertEquals(validTarStreamOptions({ gname: "" }), true); + assertEquals(validTarStreamOptions({ gname: "abcdef" }), true); + assertEquals(validTarStreamOptions({ gname: "å-abcdef" }), false); + assertEquals(validTarStreamOptions({ gname: "a".repeat(100) }), false); + + assertEquals(validTarStreamOptions({ devmajor: "" }), true); + assertEquals(validTarStreamOptions({ devmajor: "000" }), true); + assertEquals(validTarStreamOptions({ devmajor: "008" }), false); + assertEquals(validTarStreamOptions({ devmajor: "000000000" }), false); + + assertEquals(validTarStreamOptions({ devminor: "" }), true); + assertEquals(validTarStreamOptions({ devminor: "000" }), true); + assertEquals(validTarStreamOptions({ devminor: "008" }), false); + assertEquals(validTarStreamOptions({ devminor: "000000000" }), false); +}); From 46624209f05330e5e07c595046c3af93082a06a4 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Mon, 5 Aug 2024 19:14:06 +1000 Subject: [PATCH 26/82] refactor(archive): code to copy the changes made in the @doctor/tar-stream version --- archive/tar_stream.ts | 212 +++++++++---------- archive/tar_stream_test.ts | 37 +--- archive/untar_stream.ts | 423 +++++++++++++++++-------------------- 3 files changed, 293 insertions(+), 379 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index c2b263ee52d7..92d37c9ba15c 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -5,7 +5,6 @@ export interface TarStreamFile { pathname: string | [Uint8Array, Uint8Array]; size: number; - sizeExtension?: boolean; iterable: Iterable | AsyncIterable; options?: Partial; } @@ -47,6 +46,8 @@ export interface TarStreamOptions { devminor: string; } +const SLASH_CODE_POINT = "/".charCodeAt(0); + /** * ### Overview * A TransformStream to create a tar archive. Tar archives allow for storing @@ -104,43 +105,55 @@ export class TarStream { */ constructor() { const { readable, writable } = new TransformStream< - TarStreamFile | TarStreamDir, - TarStreamFile | TarStreamDir - >(); - const gen = (async function* () { - const paths: string[] = []; - for await (const chunk of readable) { + TarStreamInput, + TarStreamInput & { pathname: [Uint8Array, Uint8Array] } + >({ + transform(chunk, controller) { if (chunk.options && !validTarStreamOptions(chunk.options)) { - throw new Error("Invalid Options Provided!"); + return controller.error("Invalid Options Provided!"); } if ( "size" in chunk && - ( - chunk.size < 0 || - Math.pow(8, chunk.sizeExtension ? 12 : 11) < chunk.size || - chunk.size.toString() === "NaN" - ) + (chunk.size < 0 || 8 ** 12 < chunk.size || + chunk.size.toString() === "NaN") ) { - throw new Error( - "Invalid Size Provided! Size cannot exceed 8 GiBs by default or 64 GiBs with sizeExtension set to true.", + return controller.error( + "Invalid Size Provided! Size cannot exceed 64 Gibs.", ); } - const [prefix, name] = typeof chunk.pathname === "string" + const pathname = typeof chunk.pathname === "string" ? parsePathname(chunk.pathname, !("size" in chunk)) : function () { - if ("size" in chunk === (chunk.pathname[1].slice(-1)[0] === 47)) { - throw new Error( + if ( + "size" in chunk === + (chunk.pathname[1].slice(-1)[0] === SLASH_CODE_POINT) + ) { + controller.error( `Pre-parsed pathname for ${ "size" in chunk ? "directory" : "file" - } is not suffixed correctly. Directories should end in a forward slash, while files shouldn't.`, + } is not suffixed correctly. ${ + "size" in chunk ? "Directories" : "Files" + } should${ + "size" in chunk ? "" : "n't" + } end in a forward slash.`, ); } return chunk.pathname; }(); + + controller.enqueue({ ...chunk, pathname }); + }, + }); + this.#writable = writable; + const gen = async function* () { + const paths: string[] = []; + const encoder = new TextEncoder(); + const decoder = new TextDecoder(); + for await (const chunk of readable) { + const [prefix, name] = chunk.pathname; { - const decoder = new TextDecoder(); const pathname = prefix.length ? decoder.decode(prefix) + "/" + decoder.decode(name) : decoder.decode(name); @@ -150,121 +163,94 @@ export class TarStream { paths.push(pathname); } const typeflag = "size" in chunk ? "0" : "5"; - const sizeExtension = "size" in chunk && chunk.sizeExtension || false; - const encoder = new TextEncoder(); const header = new Uint8Array(512); + const size = "size" in chunk ? chunk.size : 0; + const options: TarStreamOptions = { + mode: typeflag === "5" ? "755" : "644", + uid: "", + gid: "", + mtime: Math.floor(new Date().getTime() / 1000), + uname: "", + gname: "", + devmajor: "", + devminor: "", + ...chunk.options, + }; header.set(name); // name header.set( encoder.encode( - (chunk.options?.mode ?? (typeflag === "5" ? "755" : "644")) - .padStart(6, "0") + - " \0" + // mode - (chunk.options?.uid ?? "").padStart(6, "0") + " \0" + // uid - (chunk.options?.gid ?? "").padStart(6, "0") + " \0" + // gid - ("size" in chunk ? chunk.size.toString(8) : "").padStart( - sizeExtension ? 12 : 11, - "0", - ) + (sizeExtension ? "" : " ") + // size - (chunk.options?.mtime?.toString(8) ?? - Math.floor(new Date().getTime() / 1000).toString(8)).padStart( - 11, - "0", - ) + - " " + // mtime - " ".repeat(8) + // checksum | Needs to be updated + options.mode.padStart(6, "0") + " \0" + // mode + options.uid.padStart(6, "0") + " \0" + //uid + options.gid.padStart(6, "0") + " \0" + // gid + size.toString(8).padStart(size < 8 ** 11 ? 11 : 12, "0") + + (size < 8 ** 11 ? " " : "") + // size + options.mtime.toString(8).padStart(11, "0") + " " + // mtime + " ".repeat(8) + // checksum | To be updated later typeflag + // typeflag "\0".repeat(100) + // linkname "ustar\0" + // magic "00" + // version - (chunk.options?.uname ?? "").padEnd(32, "\0") + // uname - (chunk.options?.gname ?? "").padEnd(32, "\0") + // gname - (chunk.options?.devmajor ?? "").padEnd(8, "\0") + // devmajor - (chunk.options?.devminor ?? "").padEnd(8, "\0"), // devminor + options.uname.padStart(32, "\0") + // uname + options.gname.padStart(32, "\0") + // gname + options.devmajor.padStart(8, "\0") + // devmajor + options.devminor.padStart(8, "\0"), // devminor ), 100, ); header.set(prefix, 345); // prefix - + // Update Checksum header.set( encoder.encode( header.reduce((x, y) => x + y).toString(8).padStart(6, "0") + "\0", ), 148, - ); // update checksum + ); yield header; if ("size" in chunk) { let size = 0; - for await (const x of chunk.iterable) { - size += x.length; - yield x; + for await (const value of chunk.iterable) { + size += value.length; + yield value; } if (chunk.size !== size) { throw new Error( - "Invalid Tarball! Provided size did not match bytes read from iterable.", + "Invalid Tarball! Provided size did not match bytes read from provided iterable.", ); } if (chunk.size % 512) { - yield new Uint8Array(new Array(512 - chunk.size % 512).fill(0)); + yield new Uint8Array(512 - size % 512); } } } - yield new Uint8Array(new Array(1024).fill(0)); - })(); + yield new Uint8Array(1024); + }(); + this.#readable = new ReadableStream({ + type: "bytes", + async pull(controller) { + const { done, value } = await gen.next(); + if (done) { + controller.close(); + return controller.byobRequest?.respond(0); + } + if (controller.byobRequest?.view) { + const buffer = new Uint8Array(controller.byobRequest.view.buffer); - this.#readable = new ReadableStream( - { - leftover: new Uint8Array(0), - type: "bytes", - async pull(controller) { - // If Byte Stream - if (controller.byobRequest?.view) { - const buffer = new Uint8Array( - controller.byobRequest.view.buffer, - ); - if (buffer.length < this.leftover.length) { - buffer.set(this.leftover.slice(0, buffer.length)); - this.leftover = this.leftover.slice(buffer.length); - return controller.byobRequest.respond(buffer.length); - } - buffer.set(this.leftover); - let offset = this.leftover.length; - while (offset < buffer.length) { - const { done, value } = await gen.next(); - if (done) { - try { - controller.byobRequest.respond(offset); // Will throw if zero - controller.close(); - } catch { - controller.close(); - controller.byobRequest.respond(0); // But still needs to be resolved. - } - return; - } - if (value.length > buffer.length - offset) { - buffer.set(value.slice(0, buffer.length - offset), offset); - offset = buffer.length - offset; - controller.byobRequest.respond(buffer.length); - this.leftover = value.slice(offset); - return; - } - buffer.set(value, offset); - offset += value.length; - } - this.leftover = new Uint8Array(0); - return controller.byobRequest.respond(buffer.length); - } - // Else Default Stream - const { done, value } = await gen.next(); - if (done) { - return controller.close(); + const size = buffer.length; + if (size < value.length) { + buffer.set(value.slice(0, size)); + controller.byobRequest.respond(size); + controller.enqueue(value.slice(size)); + } else { + buffer.set(value); + controller.byobRequest.respond(value.length); } + } else { controller.enqueue(value); - }, - } as UnderlyingByteSource & { leftover: Uint8Array }, - ); - this.#writable = writable; + } + }, + }); } /** @@ -277,7 +263,7 @@ export class TarStream { /** * The WritableStream */ - get writable(): WritableStream { + get writable(): WritableStream { return this.#writable; } } @@ -310,26 +296,32 @@ export function parsePathname( throw new Error("Invalid Pathname! Pathname cannot exceed 256 bytes."); } - let i = Math.max(0, name.lastIndexOf(47)); - if (pathname.slice(i + 1).length > 100) { + // If length of last part is > 100, then there's no possible answer to split the path + let suitableSlashPos = Math.max(0, name.lastIndexOf(SLASH_CODE_POINT)); // always holds position of '/' + if (name.length - suitableSlashPos > 100) { throw new Error("Invalid Filename! Filename cannot exceed 100 bytes."); } - for (; i > 0; --i) { - i = name.lastIndexOf(47, i) + 1; - if (name.slice(i + 1).length > 100) { - i = Math.max(0, name.indexOf(47, i + 1)); + for ( + let nextPos = suitableSlashPos; + nextPos > 0; + suitableSlashPos = nextPos + ) { + // disclaimer: '/' won't appear at pos 0, so nextPos always be > 0 or = -1 + nextPos = name.lastIndexOf(SLASH_CODE_POINT, suitableSlashPos - 1); + // disclaimer: since name.length > 100 in this case, if nextPos = -1, name.length - nextPos will also > 100 + if (name.length - nextPos > 100) { break; } } - const prefix = name.slice(0, i); + const prefix = name.slice(0, suitableSlashPos); if (prefix.length > 155) { throw new Error( "Invalid Pathname! Pathname needs to be split-able on a forward slash separator into [155, 100] bytes respectively.", ); } - return [prefix, name.slice(i + 1)]; + return [prefix, name.slice(suitableSlashPos + 1)]; } /** * validTarStreamOptions is a function that returns a true if all of the options diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index 85a78b7cea6b..b89b5f5691b8 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -78,35 +78,7 @@ Deno.test("TarStream() with negative size", async () => { async function () { await Array.fromAsync(readable); }, - Error, - "Invalid Size Provided! Size cannot exceed 8 GiBs by default or 64 GiBs with sizeExtension set to true.", - ); -}); - -Deno.test("TarStream() with 9 GiB size", async () => { - const size = 1024 ** 3 * 9; - const step = 1024; // Size must equally be divisible by step - const iterable = function* () { - for (let i = 0; i < size; i += step) { - yield new Uint8Array(step).map(() => Math.floor(Math.random() * 256)); - } - }(); - - const readable = ReadableStream.from([ - { - pathname: "name", - size, - iterable, - }, - ]) - .pipeThrough(new TarStream()); - - await assertRejects( - async function () { - await Array.fromAsync(readable); - }, - Error, - "Invalid Size Provided! Size cannot exceed 8 GiBs by default or 64 GiBs with sizeExtension set to true.", + "Invalid Size Provided! Size cannot exceed 64 Gibs.", ); }); @@ -124,7 +96,6 @@ Deno.test("TarStream() with 65 GiB size", async () => { pathname: "name", size, iterable, - sizeExtension: true, }, ]) .pipeThrough(new TarStream()); @@ -133,8 +104,7 @@ Deno.test("TarStream() with 65 GiB size", async () => { async function () { await Array.fromAsync(readable); }, - Error, - "Invalid Size Provided! Size cannot exceed 8 GiBs by default or 64 GiBs with sizeExtension set to true.", + "Invalid Size Provided! Size cannot exceed 64 Gibs.", ); }); @@ -160,8 +130,7 @@ Deno.test("TarStream() with NaN size", async () => { async function () { await Array.fromAsync(readable); }, - Error, - "Invalid Size Provided! Size cannot exceed 8 GiBs by default or 64 GiBs with sizeExtension set to true.", + "Invalid Size Provided! Size cannot exceed 64 Gibs.", ); }); diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 1024a3e3cafd..785326183495 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -98,254 +98,207 @@ export class UnTarStream { * Constructs a new instance. */ constructor() { - const { readable, writable } = new TransformStream< - Uint8Array, - Uint8Array - >(); - const reader = readable - .pipeThrough( - new TransformStream( - { // Slices ReadableStream's Uint8Array into 512 byte chunks. - x: new Uint8Array(0), - transform(chunk, controller) { - const y = new Uint8Array(this.x.length + chunk.length); - y.set(this.x); - y.set(chunk, this.x.length); - - for (let i = 512; i <= y.length; i += 512) { - controller.enqueue(y.slice(i - 512, i)); - } - this.x = y.length % 512 - ? y.slice(-y.length % 512) - : new Uint8Array(0); - }, - flush(controller) { - if (this.x.length) { - controller.error( - "Tarball has an unexpected number of bytes.!!", - ); - } - }, - } as Transformer & { x: Uint8Array }, - ), - ) - .pipeThrough( - new TransformStream( - { // Trims the last Uint8Array chunks off. - x: [], - transform(chunk, controller) { - this.x.push(chunk); - if (this.x.length === 3) { - controller.enqueue(this.x.shift()!); - } - }, - flush(controller) { - if (this.x.length < 2) { - controller.error("Tarball was too small to be valid."); - } else if (!this.x.every((x) => x.every((x) => x === 0))) { - controller.error("Tarball has invalid ending."); - } - }, - } as Transformer & { x: Uint8Array[] }, - ), - ) - .getReader(); - let header: TarStreamHeader | undefined; - this.#readable = new ReadableStream( - { - cancelled: false, - async pull(controller) { - while (header != undefined) { - await new Promise((a) => setTimeout(a, 0)); + const { readable, writable } = function () { + let push: Uint8Array | undefined; + const array: Uint8Array[] = []; + return new TransformStream({ + transform(chunk, controller) { + if (push) { + const concat = new Uint8Array(push.length + chunk.length); + concat.set(push); + concat.set(chunk, push.length); + chunk = concat; } + push = chunk.length % 512 + ? chunk.slice(-chunk.length % 512) + : undefined; - const { done, value } = await reader.read(); - if (done) { - return controller.close(); + for (let i = 512; i <= chunk.length; i += 512) { + array.push(chunk.slice(i - 512, i)); } - - const decoder = new TextDecoder(); - { // Validate checksum - const checksum = value.slice(); - checksum.set(new Uint8Array(8).fill(32), 148); - if ( - checksum.reduce((x, y) => x + y) !== - parseInt(decoder.decode(value.slice(148, 156 - 2)), 8) - ) { - return controller.error( - "Invalid Tarball. Header failed to pass checksum.", - ); - } + while (array.length > 2) { + controller.enqueue(array.shift()!); + } + }, + flush(controller) { + if (push) { + return controller.error( + "Tarball has an unexpected number of bytes.", + ); } + if (array.length < 2) { + return controller.error("Tarball was too small to be valid."); + } + if (!array.every((x) => x.every((x) => x === 0))) { + controller.error("Tarball has invalid ending."); + } + }, + }); + }(); + this.#writable = writable; + const decoder = new TextDecoder(); + const reader = readable.getReader(); + let header: OldStyleFormat | PosixUstarFormat | undefined; + let cancelled = false; + let reason: unknown; + this.#readable = new ReadableStream({ + async pull(controller) { + while (header != undefined) { + await new Promise((a) => setTimeout(a, 0)); + } + + const { done, value } = await reader.read(); + if (done) { + return controller.close(); + } + + // Validate Checksum + const checksum = value.slice(); + checksum.set(new Uint8Array(8).fill(32), 148); + if ( + checksum.reduce((x, y) => x + y) !== + parseInt(decoder.decode(value.slice(148, 156 - 2)), 8) + ) { + return controller.error( + "Invalid Tarball. Header failed to pass checksum.", + ); + } + + // Decode Header + header = { + name: decoder.decode(value.slice(0, 100)).replaceAll("\0", ""), + mode: decoder.decode(value.slice(100, 108 - 2)), + uid: decoder.decode(value.slice(108, 116 - 2)), + gid: decoder.decode(value.slice(116, 124 - 2)), + size: parseInt(decoder.decode(value.slice(124, 136)).trimEnd(), 8), + mtime: parseInt(decoder.decode(value.slice(136, 148 - 1)), 8), + checksum: decoder.decode(value.slice(148, 156 - 2)), + typeflag: decoder.decode(value.slice(156, 157)), + linkname: decoder.decode(value.slice(157, 257)).replaceAll( + "\0", + "", + ), + pad: value.slice(257), + }; + if (header.typeflag === "\0") { + header.typeflag = "0"; + } + if ( + [117, 115, 116, 97, 114, 0, 48, 48].every((byte, i) => + value[i + 257] === byte + ) + ) { header = { - name: decoder.decode(value.slice(0, 100)).replaceAll("\0", ""), - mode: decoder.decode(value.slice(100, 108 - 2)), - uid: decoder.decode(value.slice(108, 116 - 2)), - gid: decoder.decode(value.slice(116, 124 - 2)), - size: parseInt(decoder.decode(value.slice(124, 136)).trimEnd(), 8), - mtime: parseInt(decoder.decode(value.slice(136, 148 - 1)), 8), - checksum: decoder.decode(value.slice(148, 156 - 2)), - typeflag: decoder.decode(value.slice(156, 157)), - linkname: decoder.decode(value.slice(157, 257)).replaceAll( + ...header, + magic: decoder.decode(value.slice(257, 263)), + version: decoder.decode(value.slice(263, 265)), + uname: decoder.decode(value.slice(265, 297)).replaceAll("\0", ""), + gname: decoder.decode(value.slice(297, 329)).replaceAll("\0", ""), + devmajor: decoder.decode(value.slice(329, 337)).replaceAll( "\0", "", ), - pad: value.slice(257), + devminor: decoder.decode(value.slice(337, 345)).replaceAll( + "\0", + "", + ), + prefix: decoder.decode(value.slice(345, 500)).replaceAll( + "\0", + "", + ), + pad: value.slice(500), }; - if (header.typeflag === "\0") { - header.typeflag = "0"; - } - // Check if header is POSIX ustar | new TextEncoder().encode('ustar\0' + '00') - if ( - [117, 115, 116, 97, 114, 0, 48, 48].every((byte, i) => - value[i + 257] === byte - ) - ) { - header = { - ...header, - magic: decoder.decode(value.slice(257, 263)), - version: decoder.decode(value.slice(263, 265)), - uname: decoder.decode(value.slice(265, 297)).replaceAll("\0", ""), - gname: decoder.decode(value.slice(297, 329)).replaceAll("\0", ""), - devmajor: decoder.decode(value.slice(329, 337)).replaceAll( - "\0", - "", - ), - devminor: decoder.decode(value.slice(337, 345)).replaceAll( - "\0", - "", - ), - prefix: decoder.decode(value.slice(345, 500)).replaceAll( - "\0", - "", - ), - pad: value.slice(500), - }; - } + } - if (header.typeflag === "0") { - const size = header.size; - let i = Math.ceil(size / 512); - const isCancelled = () => this.cancelled; - let lock = false; - controller.enqueue({ - pathname: ("prefix" in header && header.prefix.length - ? header.prefix + "/" - : "") + header.name, - header, - readable: new ReadableStream( - { - leftover: new Uint8Array(0), - type: "bytes", - async pull(controller) { - if (i > 0) { - lock = true; - // If Byte Stream - if (controller.byobRequest?.view) { - const buffer = new Uint8Array( - controller.byobRequest.view.buffer, - ); - if (buffer.length < this.leftover.length) { - buffer.set(this.leftover.slice(0, buffer.length)); - this.leftover = this.leftover.slice(buffer.length); - return controller.byobRequest.respond(buffer.length); - } - buffer.set(this.leftover); - let offset = this.leftover.length; - while (offset < buffer.length) { - const { done, value } = await (async function () { - const x = await reader.read(); - if (!x.done && i-- === 1) { - x.value = x.value.slice(0, size % 512); - } - return x; - })(); - if (done) { - header = undefined; - lock = false; - try { - controller.byobRequest.respond(offset); // Will throw if zero. - controller.close(); - } catch { - controller.close(); - controller.byobRequest.respond(0); // But still needs to be resolved. - } - return; - } - if (value.length > buffer.length - offset) { - buffer.set( - value.slice(0, buffer.length - offset), - offset, - ); - offset = buffer.length - offset; - lock = false; - controller.byobRequest.respond(buffer.length); - this.leftover = value.slice(offset); - return; - } - buffer.set(value, offset); - offset += value.length; - } - lock = false; - this.leftover = new Uint8Array(0); - return controller.byobRequest.respond(buffer.length); - } - // Else Default Stream - const { done, value } = await reader.read(); - if (done) { - header = undefined; - return controller.error("Tarball ended unexpectedly."); - } - // Pull is unlocked before enqueue is called because if pull is in the middle of processing a chunk when cancel is called, nothing after enqueue will run. - lock = false; - controller.enqueue( - i-- === 1 ? value.slice(0, size % 512) : value, - ); - } else { - header = undefined; - if (isCancelled()) { - reader.cancel(); - } - controller.close(); + if (header.typeflag === "0") { + const size = header.size; + let i = Math.ceil(size / 512); + let lock = false; + controller.enqueue({ + pathname: ("prefix" in header && header.prefix.length + ? header.prefix + "/" + : "") + header.name, + header, + readable: new ReadableStream({ + type: "bytes", + async pull(controller) { + if (i > 0) { + lock = true; + const { done, value } = await async function () { + const x = await reader.read(); + if (!x.done && i-- === 1 && size % 512) { + x.value = x.value.slice(0, size % 512); // Slice off suffix padding. } - }, - async cancel() { - while (lock) { - await new Promise((a) => - setTimeout(a, 0) - ); + return x; + }(); + if (done) { + header = undefined; + lock = false; + controller.error("Tarball ended unexpectedly."); + return; + } + if (controller.byobRequest?.view) { + const buffer = new Uint8Array( + controller.byobRequest.view.buffer, + ); + const size = buffer.length; + if (size < value.length) { + buffer.set(value.slice(0, size)); + controller.byobRequest.respond(size); + controller.enqueue(value.slice(size)); + } else { + buffer.set(value); + controller.byobRequest.respond(value.length); } - try { - while (i-- > 0) { - if ((await reader.read()).done) { - throw new Error("Tarball ended unexpectedly."); - } - } - } catch (error) { - throw error; - } finally { - header = undefined; + } else { + controller.enqueue(value); + } + lock = false; + } else { + header = undefined; + if (cancelled) { + reader.cancel(reason); + } + controller.close(); + controller.byobRequest?.respond(0); + } + }, + async cancel(r) { + reason = r; + while (lock) { + await new Promise((a) => + setTimeout(a, 0) + ); + } + try { + while (i-- > 0) { + if ((await reader.read()).done) { + throw new Error("Tarball ended unexpectedly."); } - }, - } as UnderlyingByteSource & { leftover: Uint8Array }, - ), - }); - } else { - controller.enqueue({ - pathname: ("prefix" in header && header.prefix.length - ? header.prefix + "/" - : "") + header.name, - header, - }); - header = undefined; - } - }, - cancel() { - this.cancelled = true; - }, - } as UnderlyingSource & { cancelled: boolean }, - ); - this.#writable = writable; + } + } catch (error) { + throw error; + } finally { + header = undefined; + } + }, + }), + }); + } else { + controller.enqueue({ + pathname: ("prefix" in header && header.prefix.length + ? header.prefix + "/" + : "") + header.name, + header, + }); + header = undefined; + } + }, + cancel(r) { + reason = r; + cancelled = true; + }, + }); } /** From 4acf28443f044af512d98ed5ba4278687f8d4de6 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Mon, 5 Aug 2024 19:16:57 +1000 Subject: [PATCH 27/82] test(archive): added from @doctor/tar-stream --- archive/tar_stream_test.ts | 44 ++++++++++++++++++++++++++++++++++++ archive/untar_stream_test.ts | 31 ++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index b89b5f5691b8..e9d68bb691e3 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. import { + parsePathname, TarStream, type TarStreamInput, validTarStreamOptions, @@ -134,6 +135,49 @@ Deno.test("TarStream() with NaN size", async () => { ); }); +Deno.test("parsePathname()", () => { + const encoder = new TextEncoder(); + + assertEquals( + parsePathname( + "./Veeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeery/LongPath", + true, + ), + [ + encoder.encode( + "Veeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeery", + ), + encoder.encode("LongPath/"), + ], + ); + + assertEquals( + parsePathname( + "./some random path/with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/path", + true, + ), + [ + encoder.encode("some random path"), + encoder.encode( + "with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/path/", + ), + ], + ); + + assertEquals( + parsePathname( + "./some random path/with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/file", + false, + ), + [ + encoder.encode("some random path"), + encoder.encode( + "with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/file", + ), + ], + ); +}); + Deno.test("validTarStreamOptions()", () => { assertEquals(validTarStreamOptions({}), true); diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index e864bf638646..8df8dcfa3534 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -1,5 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -import { TarStream } from "./tar_stream.ts"; +import { TarStream, type TarStreamInput } from "./tar_stream.ts"; import { UnTarStream } from "./untar_stream.ts"; import { assertEquals } from "../assert/mod.ts"; @@ -96,3 +96,32 @@ Deno.test("expandTarArchiveCheckingBodiesByteStream", async function () { } } }); + +Deno.test("UnTarStream() with size equals to multiple of 512", async () => { + const size = 512 * 3; + const data = Uint8Array.from( + { length: size }, + () => Math.floor(Math.random() * 256), + ); + + const readable = ReadableStream.from([ + { + pathname: "name", + size, + iterable: [data.slice()], + }, + ]) + .pipeThrough(new TarStream()) + .pipeThrough(new UnTarStream()); + + for await (const item of readable) { + if (item.readable) { + assertEquals( + Uint8Array.from( + (await Array.fromAsync(item.readable)).map((x) => [...x]).flat(), + ), + data, + ); + } + } +}); From 666609c2bafc23b63d86e01abe78930f2d19b1e4 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Tue, 6 Aug 2024 13:37:43 +1000 Subject: [PATCH 28/82] chore: nit on anonymous function --- archive/untar_stream_test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index 8df8dcfa3534..24b6a5fe9eeb 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -3,7 +3,7 @@ import { TarStream, type TarStreamInput } from "./tar_stream.ts"; import { UnTarStream } from "./untar_stream.ts"; import { assertEquals } from "../assert/mod.ts"; -Deno.test("expandTarArchiveCheckingHeaders", async function () { +Deno.test("expandTarArchiveCheckingHeaders", async () => { const text = new TextEncoder().encode("Hello World!"); const readable = ReadableStream.from([ @@ -27,7 +27,7 @@ Deno.test("expandTarArchiveCheckingHeaders", async function () { assertEquals(pathnames, ["potato/", "text.txt"]); }); -Deno.test("expandTarArchiveCheckingBodiesDefaultStream", async function () { +Deno.test("expandTarArchiveCheckingBodiesDefaultStream", async () => { const text = new TextEncoder().encode("Hello World!"); const readable = ReadableStream.from([ @@ -61,7 +61,7 @@ Deno.test("expandTarArchiveCheckingBodiesDefaultStream", async function () { } }); -Deno.test("expandTarArchiveCheckingBodiesByteStream", async function () { +Deno.test("expandTarArchiveCheckingBodiesByteStream", async () => { const text = new TextEncoder().encode("Hello World!"); const readable = ReadableStream.from([ From da387a1670b803d8b1aced2f5f1451fedcb301ba Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Tue, 6 Aug 2024 20:44:56 +1000 Subject: [PATCH 29/82] refactor(archive): UnTarStream that fixes unexplainable memory leak - The second newest test introduced here '... with invalid ending' seems to detect a memory leak due to an invalid tarball. I couldn't figure out why the memory leak was happening but I know this restructure of the code doesn't have that same memory leak. --- archive/untar_stream.ts | 329 +++++++++++++++-------------------- archive/untar_stream_test.ts | 69 +++++++- 2 files changed, 209 insertions(+), 189 deletions(-) diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 785326183495..3337fd1c0b1f 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -92,215 +92,168 @@ export type TarStreamHeader = OldStyleFormat | PosixUstarFormat; * ``` */ export class UnTarStream { + #lock = false; #readable: ReadableStream; #writable: WritableStream; - /** - * Constructs a new instance. - */ + #gen: AsyncGenerator; constructor() { - const { readable, writable } = function () { - let push: Uint8Array | undefined; - const array: Uint8Array[] = []; - return new TransformStream({ - transform(chunk, controller) { - if (push) { - const concat = new Uint8Array(push.length + chunk.length); - concat.set(push); - concat.set(chunk, push.length); - chunk = concat; - } - push = chunk.length % 512 - ? chunk.slice(-chunk.length % 512) - : undefined; - - for (let i = 512; i <= chunk.length; i += 512) { - array.push(chunk.slice(i - 512, i)); - } - while (array.length > 2) { - controller.enqueue(array.shift()!); - } - }, - flush(controller) { - if (push) { - return controller.error( - "Tarball has an unexpected number of bytes.", - ); - } - if (array.length < 2) { - return controller.error("Tarball was too small to be valid."); - } - if (!array.every((x) => x.every((x) => x === 0))) { - controller.error("Tarball has invalid ending."); - } - }, - }); - }(); + const { readable, writable } = new TransformStream< + Uint8Array, + Uint8Array + >(); + this.#readable = ReadableStream.from(this.#untar()); this.#writable = writable; - const decoder = new TextDecoder(); - const reader = readable.getReader(); - let header: OldStyleFormat | PosixUstarFormat | undefined; - let cancelled = false; - let reason: unknown; - this.#readable = new ReadableStream({ - async pull(controller) { - while (header != undefined) { - await new Promise((a) => setTimeout(a, 0)); + + this.#gen = async function* () { + let push: Uint8Array | undefined; + const buffer: Uint8Array[] = []; + for await (let chunk of readable) { + if (push) { + const concat = new Uint8Array(push.length + chunk.length); + concat.set(push); + concat.set(chunk, push.length); + chunk = concat; } - const { done, value } = await reader.read(); - if (done) { - return controller.close(); + for (let i = 512; i <= chunk.length; i += 512) { + buffer.push(chunk.slice(i - 512, i)); } - // Validate Checksum - const checksum = value.slice(); - checksum.set(new Uint8Array(8).fill(32), 148); - if ( - checksum.reduce((x, y) => x + y) !== - parseInt(decoder.decode(value.slice(148, 156 - 2)), 8) - ) { - return controller.error( - "Invalid Tarball. Header failed to pass checksum.", - ); + const remainder = -chunk.length % 512; + push = remainder ? chunk.slice(remainder) : undefined; + + while (buffer.length > 2) { + yield buffer.shift()!; } + } + if (push) throw new Error("Tarball has an unexpected number of bytes."); + if (buffer.length < 2) { + throw new Error("Tarball was too small to be valid."); + } + if (!buffer.every((value) => value.every((x) => x === 0))) { + throw new Error("Tarball has invalid ending."); + } + }(); + } - // Decode Header + async *#untar(): AsyncGenerator { + const decoder = new TextDecoder(); + while (true) { + while (this.#lock) { + await new Promise((a) => setTimeout(a, 0)); + } + + const { done, value } = await this.#gen.next(); + if (done) break; + + // Validate Checksum + const checksum = value.slice(); + checksum.set(new Uint8Array(8).fill(32), 148); + if ( + checksum.reduce((x, y) => x + y) !== + parseInt(decoder.decode(value.slice(148, 156 - 2)), 8) + ) throw new Error("Invalid Tarball. Header failed to pass checksum."); + + // Decode Header + let header: OldStyleFormat | PosixUstarFormat = { + name: decoder.decode(value.slice(0, 100)).replaceAll("\0", ""), + mode: decoder.decode(value.slice(100, 108 - 2)), + uid: decoder.decode(value.slice(108, 116 - 2)), + gid: decoder.decode(value.slice(116, 124 - 2)), + size: parseInt(decoder.decode(value.slice(124, 136)).trimEnd(), 8), + mtime: parseInt(decoder.decode(value.slice(136, 148 - 1)), 8), + checksum: decoder.decode(value.slice(148, 156 - 2)), + typeflag: decoder.decode(value.slice(156, 157)), + linkname: decoder.decode(value.slice(157, 257)).replaceAll( + "\0", + "", + ), + pad: value.slice(257), + }; + if (header.typeflag === "\0") header.typeflag = "0"; + // "ustar\u000000" + if ( + [117, 115, 116, 97, 114, 0, 48, 48].every((byte, i) => + value[i + 257] === byte + ) + ) { header = { - name: decoder.decode(value.slice(0, 100)).replaceAll("\0", ""), - mode: decoder.decode(value.slice(100, 108 - 2)), - uid: decoder.decode(value.slice(108, 116 - 2)), - gid: decoder.decode(value.slice(116, 124 - 2)), - size: parseInt(decoder.decode(value.slice(124, 136)).trimEnd(), 8), - mtime: parseInt(decoder.decode(value.slice(136, 148 - 1)), 8), - checksum: decoder.decode(value.slice(148, 156 - 2)), - typeflag: decoder.decode(value.slice(156, 157)), - linkname: decoder.decode(value.slice(157, 257)).replaceAll( + ...header, + magic: decoder.decode(value.slice(257, 263)), + version: decoder.decode(value.slice(263, 265)), + uname: decoder.decode(value.slice(265, 297)).replaceAll("\0", ""), + gname: decoder.decode(value.slice(297, 329)).replaceAll("\0", ""), + devmajor: decoder.decode(value.slice(329, 337)).replaceAll( + "\0", + "", + ), + devminor: decoder.decode(value.slice(337, 345)).replaceAll( "\0", "", ), - pad: value.slice(257), + prefix: decoder.decode(value.slice(345, 500)).replaceAll( + "\0", + "", + ), + pad: value.slice(500), }; - if (header.typeflag === "\0") { - header.typeflag = "0"; - } - if ( - [117, 115, 116, 97, 114, 0, 48, 48].every((byte, i) => - value[i + 257] === byte - ) - ) { - header = { - ...header, - magic: decoder.decode(value.slice(257, 263)), - version: decoder.decode(value.slice(263, 265)), - uname: decoder.decode(value.slice(265, 297)).replaceAll("\0", ""), - gname: decoder.decode(value.slice(297, 329)).replaceAll("\0", ""), - devmajor: decoder.decode(value.slice(329, 337)).replaceAll( - "\0", - "", - ), - devminor: decoder.decode(value.slice(337, 345)).replaceAll( - "\0", - "", - ), - prefix: decoder.decode(value.slice(345, 500)).replaceAll( - "\0", - "", - ), - pad: value.slice(500), - }; - } + } - if (header.typeflag === "0") { - const size = header.size; - let i = Math.ceil(size / 512); - let lock = false; - controller.enqueue({ - pathname: ("prefix" in header && header.prefix.length - ? header.prefix + "/" - : "") + header.name, - header, - readable: new ReadableStream({ - type: "bytes", - async pull(controller) { - if (i > 0) { - lock = true; - const { done, value } = await async function () { - const x = await reader.read(); - if (!x.done && i-- === 1 && size % 512) { - x.value = x.value.slice(0, size % 512); // Slice off suffix padding. - } - return x; - }(); - if (done) { - header = undefined; - lock = false; - controller.error("Tarball ended unexpectedly."); - return; - } - if (controller.byobRequest?.view) { - const buffer = new Uint8Array( - controller.byobRequest.view.buffer, - ); - const size = buffer.length; - if (size < value.length) { - buffer.set(value.slice(0, size)); - controller.byobRequest.respond(size); - controller.enqueue(value.slice(size)); - } else { - buffer.set(value); - controller.byobRequest.respond(value.length); - } - } else { - controller.enqueue(value); - } - lock = false; - } else { - header = undefined; - if (cancelled) { - reader.cancel(reason); - } - controller.close(); - controller.byobRequest?.respond(0); - } - }, - async cancel(r) { - reason = r; - while (lock) { - await new Promise((a) => - setTimeout(a, 0) - ); - } - try { - while (i-- > 0) { - if ((await reader.read()).done) { - throw new Error("Tarball ended unexpectedly."); - } - } - } catch (error) { - throw error; - } finally { - header = undefined; - } - }, - }), - }); - } else { - controller.enqueue({ - pathname: ("prefix" in header && header.prefix.length - ? header.prefix + "/" - : "") + header.name, - header, - }); - header = undefined; - } + yield { + pathname: ("prefix" in header && header.prefix.length + ? header.prefix + "/" + : "") + header.name, + header, + readable: header.typeflag === "0" + ? this.#readableFile(header.size) + : undefined, + }; + } + } + + #readableFile(size: number): ReadableStream { + const gen = this.#genFile(size); + return new ReadableStream({ + type: "bytes", + async pull(controller) { + const { done, value } = await gen.next(); + if (done) { + controller.close(); + controller.byobRequest?.respond(0); + } else if (controller.byobRequest?.view) { + const buffer = new Uint8Array(controller.byobRequest.view.buffer); + const size = buffer.length; + if (value.length > size) { + buffer.set(value.slice(0, size)); + controller.byobRequest.respond(size); + controller.enqueue(value.slice(size)); + } else { + buffer.set(value); + controller.byobRequest.respond(value.length); + } + } else controller.enqueue(value); }, - cancel(r) { - reason = r; - cancelled = true; + async cancel() { + // deno-lint-ignore no-empty + for await (const _ of gen) {} }, }); } + async *#genFile(size: number): AsyncGenerator { + this.#lock = true; + for (let i = Math.ceil(size / 512); i > 0; --i) { + const { done, value } = await this.#gen.next(); + if (done) { + this.#lock = false; + throw new Error("Unexpected end of Tarball."); + } + if (i === 1 && size % 512) yield value.slice(0, size % 512); + else yield value; + } + this.#lock = false; + } + /** * The ReadableStream */ diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index 24b6a5fe9eeb..5b30f6338295 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -1,7 +1,8 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +import { concat } from "../bytes/mod.ts"; import { TarStream, type TarStreamInput } from "./tar_stream.ts"; import { UnTarStream } from "./untar_stream.ts"; -import { assertEquals } from "../assert/mod.ts"; +import { assert, assertEquals } from "../assert/mod.ts"; Deno.test("expandTarArchiveCheckingHeaders", async () => { const text = new TextEncoder().encode("Hello World!"); @@ -125,3 +126,69 @@ Deno.test("UnTarStream() with size equals to multiple of 512", async () => { } } }); + +Deno.test("UnTarStream() with invalid size", async () => { + const readable = ReadableStream.from([ + { + pathname: "newFile.txt", + size: 512, + iterable: [new Uint8Array(512).fill(97)], + }, + ]) + .pipeThrough(new TarStream()) + .pipeThrough( + new TransformStream({ + flush(controller) { + controller.enqueue(new Uint8Array(100)); + }, + }), + ) + .pipeThrough(new UnTarStream()); + + let threw = false; + try { + for await (const entry of readable) { + await entry.readable?.cancel(); + } + } catch (error) { + threw = true; + assert(error instanceof Error); + assertEquals(error.message, "Tarball has an unexpected number of bytes."); + } + assertEquals(threw, true); +}); + +Deno.test("UnTarStream() with invalid ending", async () => { + const tarBytes = concat( + await Array.fromAsync( + ReadableStream.from([ + { + pathname: "newFile.txt", + size: 512, + iterable: [new Uint8Array(512).fill(97)], + }, + ]) + .pipeThrough(new TarStream()), + ), + ); + tarBytes[tarBytes.length - 1] = 1; + + const readable = ReadableStream.from([tarBytes]) + .pipeThrough(new UnTarStream()); + + let threw = false; + try { + for await (const entry of readable) { + await entry.readable?.cancel(); + } + } catch (error) { + threw = true; + assert(error instanceof Error); + assertEquals( + error.message, + "Tarball has invalid ending.", + ); + } + assertEquals(threw, true); +}); + From 583fa04bffbd46ef5b51b7ca1842890529a97c63 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Tue, 6 Aug 2024 20:49:14 +1000 Subject: [PATCH 30/82] chore: fmt --- archive/untar_stream_test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index 5b30f6338295..fcd1a05ae843 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -191,4 +191,3 @@ Deno.test("UnTarStream() with invalid ending", async () => { } assertEquals(threw, true); }); - From f6875635b3d0b424b7677a5ce8da3407efab1a9a Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Tue, 6 Aug 2024 21:06:21 +1000 Subject: [PATCH 31/82] tests(archive): added remaining tests to cover many lines as possible --- archive/untar_stream_test.ts | 51 ++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index fcd1a05ae843..1a0d252bb90e 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -191,3 +191,54 @@ Deno.test("UnTarStream() with invalid ending", async () => { } assertEquals(threw, true); }); + +Deno.test("UnTarStream() with too small size", async () => { + const readable = ReadableStream.from([new Uint8Array(512)]) + .pipeThrough(new UnTarStream()); + + let threw = false; + try { + for await (const entry of readable) { + await entry.readable?.cancel(); + } + } catch (error) { + threw = true; + assert(error instanceof Error); + assertEquals(error.message, "Tarball was too small to be valid."); + } + assertEquals(threw, true); +}); + +Deno.test("UnTarStream() with invalid checksum", async () => { + const tarBytes = concat( + await Array.fromAsync( + ReadableStream.from([ + { + pathname: "newFile.txt", + size: 512, + iterable: [new Uint8Array(512).fill(97)], + }, + ]) + .pipeThrough(new TarStream()), + ), + ); + tarBytes[148] = 97; + + const readable = ReadableStream.from([tarBytes]) + .pipeThrough(new UnTarStream()); + + let threw = false; + try { + for await (const entry of readable) { + await entry.readable?.cancel(); + } + } catch (error) { + threw = true; + assert(error instanceof Error); + assertEquals( + error.message, + "Invalid Tarball. Header failed to pass checksum.", + ); + } + assertEquals(threw, true); +}); From 1e81120d8be601779f4b53d5ee12388d09ef824a Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Wed, 7 Aug 2024 15:36:49 +1000 Subject: [PATCH 32/82] adjust(archive): remove simplify pathname code --- archive/tar_stream.ts | 22 +++------------------- archive/tar_stream_test.ts | 13 +++++-------- archive/untar_stream_test.ts | 2 +- 3 files changed, 9 insertions(+), 28 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 92d37c9ba15c..6ba46f9bd2de 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -124,20 +124,13 @@ export class TarStream { } const pathname = typeof chunk.pathname === "string" - ? parsePathname(chunk.pathname, !("size" in chunk)) + ? parsePathname(chunk.pathname) : function () { if ( - "size" in chunk === - (chunk.pathname[1].slice(-1)[0] === SLASH_CODE_POINT) + chunk.pathname[0].length > 155 || chunk.pathname[1].length > 100 ) { controller.error( - `Pre-parsed pathname for ${ - "size" in chunk ? "directory" : "file" - } is not suffixed correctly. ${ - "size" in chunk ? "Directories" : "Files" - } should${ - "size" in chunk ? "" : "n't" - } end in a forward slash.`, + "Invalid Pathname. Pathnames, when provided as a Uint8Array, need to be no more than [155, 100] bytes respectively.", ); } return chunk.pathname; @@ -277,16 +270,7 @@ export class TarStream { */ export function parsePathname( pathname: string, - isDirectory = false, ): [Uint8Array, Uint8Array] { - pathname = pathname.split("/").filter((x) => x).join("/"); - if (pathname.startsWith("./")) { - pathname = pathname.slice(2); - } - if (isDirectory) { - pathname += "/"; - } - const name = new TextEncoder().encode(pathname); if (name.length <= 100) { return [new Uint8Array(0), name]; diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index e9d68bb691e3..6611f9a28b2d 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -141,25 +141,23 @@ Deno.test("parsePathname()", () => { assertEquals( parsePathname( "./Veeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeery/LongPath", - true, ), [ encoder.encode( - "Veeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeery", + "./Veeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeery", ), - encoder.encode("LongPath/"), + encoder.encode("LongPath"), ], ); assertEquals( parsePathname( "./some random path/with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/path", - true, ), [ - encoder.encode("some random path"), + encoder.encode("./some random path"), encoder.encode( - "with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/path/", + "with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/path", ), ], ); @@ -167,10 +165,9 @@ Deno.test("parsePathname()", () => { assertEquals( parsePathname( "./some random path/with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/file", - false, ), [ - encoder.encode("some random path"), + encoder.encode("./some random path"), encoder.encode( "with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/file", ), diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index 1a0d252bb90e..946dbf9c9224 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -25,7 +25,7 @@ Deno.test("expandTarArchiveCheckingHeaders", async () => { pathnames.push(item.pathname); item.readable?.cancel(); } - assertEquals(pathnames, ["potato/", "text.txt"]); + assertEquals(pathnames, ["./potato", "./text.txt"]); }); Deno.test("expandTarArchiveCheckingBodiesDefaultStream", async () => { From b803f5f87bb31d7d6230438b4baaf5fdf1c89b0b Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Wed, 7 Aug 2024 15:39:30 +1000 Subject: [PATCH 33/82] adjust(archive): remove checking for duplicate pathnames in taring process --- archive/tar_stream.ts | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 6ba46f9bd2de..19293b33ca4e 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -141,20 +141,9 @@ export class TarStream { }); this.#writable = writable; const gen = async function* () { - const paths: string[] = []; const encoder = new TextEncoder(); - const decoder = new TextDecoder(); for await (const chunk of readable) { const [prefix, name] = chunk.pathname; - { - const pathname = prefix.length - ? decoder.decode(prefix) + "/" + decoder.decode(name) - : decoder.decode(name); - if (paths.includes(pathname)) { - continue; - } - paths.push(pathname); - } const typeflag = "size" in chunk ? "0" : "5"; const header = new Uint8Array(512); const size = "size" in chunk ? chunk.size : 0; From 84094415ab0e6cdaf26bb65335a722114b586058 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Wed, 7 Aug 2024 15:43:38 +1000 Subject: [PATCH 34/82] adjust(archive): A readable will exist on TarEntry unless string values 1-6 --- archive/untar_stream.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 3337fd1c0b1f..e89cce634647 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -204,7 +204,7 @@ export class UnTarStream { ? header.prefix + "/" : "") + header.name, header, - readable: header.typeflag === "0" + readable: !["1", "2", "3", "4", "5", "6"].includes(header.typeflag) ? this.#readableFile(header.size) : undefined, }; From 46a914382ef3f7c7018c3462196a720be8283dc3 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Wed, 7 Aug 2024 15:58:29 +1000 Subject: [PATCH 35/82] tests(archive): added more tests for higher coverage --- archive/tar_stream_test.ts | 93 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index 6611f9a28b2d..9866714fcc8b 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -6,6 +6,7 @@ import { validTarStreamOptions, } from "./tar_stream.ts"; import { assertEquals, assertRejects } from "../assert/mod.ts"; +import { assert } from "../assert/assert.ts"; Deno.test("TarStream() with default stream", async () => { const text = new TextEncoder().encode("Hello World!"); @@ -221,3 +222,95 @@ Deno.test("validTarStreamOptions()", () => { assertEquals(validTarStreamOptions({ devminor: "008" }), false); assertEquals(validTarStreamOptions({ devminor: "000000000" }), false); }); + +Deno.test("TarStream() with invalid options", async () => { + const readable = ReadableStream.from([ + { pathname: "potato", options: { mode: "009" } }, + ]).pipeThrough(new TarStream()); + + let threw = false; + try { + // deno-lint-ignore no-empty + for await (const _ of readable) {} + } catch (error) { + threw = true; + assert(typeof error === "string"); + assertEquals(error, "Invalid Options Provided!"); + } + assertEquals(threw, true); +}); + +Deno.test("TarStream() with invalid pathname", async () => { + const readable = ReadableStream.from([ + { pathname: [new Uint8Array(156), new Uint8Array(0)] }, + ]).pipeThrough(new TarStream()); + + let threw = false; + try { + // deno-lint-ignore no-empty + for await (const _ of readable) {} + } catch (error) { + threw = true; + assert(typeof error === "string"); + assertEquals( + error, + "Invalid Pathname. Pathnames, when provided as a Uint8Array, need to be no more than [155, 100] bytes respectively.", + ); + } + assertEquals(threw, true); +}); + +Deno.test("TarStream() with mismatching sizes", async () => { + const text = new TextEncoder().encode("Hello World!"); + const readable = ReadableStream.from([ + { + pathname: "potato", + size: text.length + 1, + iterable: [text.slice()], + }, + ]).pipeThrough(new TarStream()); + + let threw = false; + try { + // deno-lint-ignore no-empty + for await (const _ of readable) {} + } catch (error) { + threw = true; + assert(error instanceof Error); + assertEquals( + error.message, + "Invalid Tarball! Provided size did not match bytes read from provided iterable.", + ); + } + assertEquals(threw, true); +}); + +Deno.test("parsePathname() with too long path", () => { + let threw = false; + try { + parsePathname("0".repeat(300)); + } catch (error) { + threw = true; + assert(error instanceof Error); + assertEquals( + error.message, + "Invalid Pathname! Pathname cannot exceed 256 bytes.", + ); + } + assertEquals(threw, true); +}); + +Deno.test("parsePathname() with too long path", () => { + let threw = false; + try { + parsePathname("0".repeat(160) + "/"); + } catch (error) { + threw = true; + assert(error instanceof Error); + assertEquals( + error.message, + "Invalid Pathname! Pathname needs to be split-able on a forward slash separator into [155, 100] bytes respectively.", + ); + } + assertEquals(threw, true); +}); From 8c15203e51211b749bf93d3a5cc73f810132d553 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Wed, 7 Aug 2024 20:07:51 +1000 Subject: [PATCH 36/82] adjust(archives): TarStream and UnTarStream to implement TransformStream --- archive/tar_stream.ts | 2 +- archive/untar_stream.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 19293b33ca4e..fe5d60b60c06 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -97,7 +97,7 @@ const SLASH_CODE_POINT = "/".charCodeAt(0); * .pipeTo((await Deno.create('./out.tar.gz')).writable) * ``` */ -export class TarStream { +export class TarStream implements TransformStream { #readable: ReadableStream; #writable: WritableStream; /** diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index e89cce634647..46e895462a56 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -91,7 +91,8 @@ export type TarStreamHeader = OldStyleFormat | PosixUstarFormat; * } * ``` */ -export class UnTarStream { +export class UnTarStream + implements TransformStream { #lock = false; #readable: ReadableStream; #writable: WritableStream; From c8e61b9f734818cd08ae30d963ab740356fc124f Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sat, 10 Aug 2024 17:55:22 +1000 Subject: [PATCH 37/82] docs(archive): moved TarStreamOptions docs into properties. --- archive/tar_stream.ts | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index fe5d60b60c06..9ae298ca19b2 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -25,24 +25,39 @@ export type TarStreamInput = TarStreamFile | TarStreamDir; /** * The options that can go along with a file or directory. - * @param mode An octal number in ASCII. - * @param uid An octal number in ASCII. - * @param gid An octal number in ASCII. - * @param mtime A number of seconds since the start of epoch. Avoid negative - * values. - * @param uname An ASCII string. Should be used in preference of uid. - * @param gname An ASCII string. Should be used in preference of gid. - * @param devmajor The major number for character device. - * @param devminor The minor number for block device entry. */ export interface TarStreamOptions { + /** + * An octal number in ASCII + */ mode: string; + /** + * An octal number in ASCII. + */ uid: string; + /** + * An octal number in ASCII. + */ gid: string; + /** + * A number of seconds since the start of epoch. Avoid negative values. + */ mtime: number; + /** + * An ASCII string. Should be used in preference of uid. + */ uname: string; + /** + * An ASCII string. Should be used in preference of gid. + */ gname: string; + /** + * The major number for character device. + */ devmajor: string; + /** + * The minor number for block device entry. + */ devminor: string; } From 275b67844ab4ec405901a5cc85c3a8a5715b311f Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sat, 10 Aug 2024 18:03:32 +1000 Subject: [PATCH 38/82] adjust(archive): TarStreamFile to take a ReadableSteam instead of an Iterable | AsyncIterable --- archive/tar_stream.ts | 4 ++-- archive/tar_stream_test.ts | 12 ++++++------ archive/untar_stream_test.ts | 20 ++++++++++---------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 9ae298ca19b2..d0e10470d32c 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -5,7 +5,7 @@ export interface TarStreamFile { pathname: string | [Uint8Array, Uint8Array]; size: number; - iterable: Iterable | AsyncIterable; + readable: ReadableStream; options?: Partial; } @@ -207,7 +207,7 @@ export class TarStream implements TransformStream { if ("size" in chunk) { let size = 0; - for await (const value of chunk.iterable) { + for await (const value of chunk.readable) { size += value.length; yield value; } diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index 9866714fcc8b..6c3826cec074 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -18,7 +18,7 @@ Deno.test("TarStream() with default stream", async () => { { pathname: "./text.txt", size: text.length, - iterable: [text.slice()], + readable: ReadableStream.from([text.slice()]), }, ]) .pipeThrough(new TarStream()) @@ -45,7 +45,7 @@ Deno.test("TarStream() with byte stream", async () => { { pathname: "./text.txt", size: text.length, - iterable: [text.slice()], + readable: ReadableStream.from([text.slice()]), }, ]) .pipeThrough(new TarStream()) @@ -71,7 +71,7 @@ Deno.test("TarStream() with negative size", async () => { { pathname: "name", size: -text.length, - iterable: [text.slice()], + readable: ReadableStream.from([text.slice()]), }, ]) .pipeThrough(new TarStream()); @@ -97,7 +97,7 @@ Deno.test("TarStream() with 65 GiB size", async () => { { pathname: "name", size, - iterable, + readable: ReadableStream.from(iterable), }, ]) .pipeThrough(new TarStream()); @@ -123,7 +123,7 @@ Deno.test("TarStream() with NaN size", async () => { { pathname: "name", size, - iterable, + readable: ReadableStream.from(iterable), }, ]) .pipeThrough(new TarStream()); @@ -266,7 +266,7 @@ Deno.test("TarStream() with mismatching sizes", async () => { { pathname: "potato", size: text.length + 1, - iterable: [text.slice()], + readable: ReadableStream.from([text.slice()]), }, ]).pipeThrough(new TarStream()); diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index 946dbf9c9224..c7fa9b932a0f 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -7,14 +7,14 @@ import { assert, assertEquals } from "../assert/mod.ts"; Deno.test("expandTarArchiveCheckingHeaders", async () => { const text = new TextEncoder().encode("Hello World!"); - const readable = ReadableStream.from([ + const readable = ReadableStream.from([ { pathname: "./potato", }, { pathname: "./text.txt", size: text.length, - iterable: [text], + readable: ReadableStream.from([text]), }, ]) .pipeThrough(new TarStream()) @@ -31,14 +31,14 @@ Deno.test("expandTarArchiveCheckingHeaders", async () => { Deno.test("expandTarArchiveCheckingBodiesDefaultStream", async () => { const text = new TextEncoder().encode("Hello World!"); - const readable = ReadableStream.from([ + const readable = ReadableStream.from([ { pathname: "./potato", }, { pathname: "./text.txt", size: text.length, - iterable: [text.slice()], + readable: ReadableStream.from([text.slice()]), }, ]) .pipeThrough(new TarStream()) @@ -65,14 +65,14 @@ Deno.test("expandTarArchiveCheckingBodiesDefaultStream", async () => { Deno.test("expandTarArchiveCheckingBodiesByteStream", async () => { const text = new TextEncoder().encode("Hello World!"); - const readable = ReadableStream.from([ + const readable = ReadableStream.from([ { pathname: "./potato", }, { pathname: "./text.txt", size: text.length, - iterable: [text.slice()], + readable: ReadableStream.from([text.slice()]), }, ]) .pipeThrough(new TarStream()) @@ -109,7 +109,7 @@ Deno.test("UnTarStream() with size equals to multiple of 512", async () => { { pathname: "name", size, - iterable: [data.slice()], + readable: ReadableStream.from([data.slice()]), }, ]) .pipeThrough(new TarStream()) @@ -132,7 +132,7 @@ Deno.test("UnTarStream() with invalid size", async () => { { pathname: "newFile.txt", size: 512, - iterable: [new Uint8Array(512).fill(97)], + readable: ReadableStream.from([new Uint8Array(512).fill(97)]), }, ]) .pipeThrough(new TarStream()) @@ -165,7 +165,7 @@ Deno.test("UnTarStream() with invalid ending", async () => { { pathname: "newFile.txt", size: 512, - iterable: [new Uint8Array(512).fill(97)], + readable: ReadableStream.from([new Uint8Array(512).fill(97)]), }, ]) .pipeThrough(new TarStream()), @@ -216,7 +216,7 @@ Deno.test("UnTarStream() with invalid checksum", async () => { { pathname: "newFile.txt", size: 512, - iterable: [new Uint8Array(512).fill(97)], + readable: ReadableStream.from([new Uint8Array(512).fill(97)]), }, ]) .pipeThrough(new TarStream()), From 8389f379990fad5a239dd82fcac8b95d8b8cc4a1 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sat, 10 Aug 2024 18:11:03 +1000 Subject: [PATCH 39/82] adjust(archive): to use FixedChunkStream instead of rolling it's own implementation --- archive/untar_stream.ts | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 46e895462a56..1537b133df5c 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -1,4 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +import { FixedChunkStream } from "@std/streams"; + /** * The interface extracted from the archive. */ @@ -106,28 +108,15 @@ export class UnTarStream this.#writable = writable; this.#gen = async function* () { - let push: Uint8Array | undefined; const buffer: Uint8Array[] = []; - for await (let chunk of readable) { - if (push) { - const concat = new Uint8Array(push.length + chunk.length); - concat.set(push); - concat.set(chunk, push.length); - chunk = concat; - } - - for (let i = 512; i <= chunk.length; i += 512) { - buffer.push(chunk.slice(i - 512, i)); + for await (let chunk of readable.pipeThrough(new FixedChunkStream(512))) { + if (chunk.length !== 512) { + throw new Error("Tarball has an unexpected number of bytes."); } - const remainder = -chunk.length % 512; - push = remainder ? chunk.slice(remainder) : undefined; - - while (buffer.length > 2) { - yield buffer.shift()!; - } + buffer.push(chunk); + if (buffer.length > 2) yield buffer.shift()!; } - if (push) throw new Error("Tarball has an unexpected number of bytes."); if (buffer.length < 2) { throw new Error("Tarball was too small to be valid."); } From eb42cd945137e8e655b28abfb4522b7a007c7a63 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sat, 10 Aug 2024 18:13:43 +1000 Subject: [PATCH 40/82] fix(archive): lint error --- archive/untar_stream.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 1537b133df5c..be1de3562421 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -109,7 +109,7 @@ export class UnTarStream this.#gen = async function* () { const buffer: Uint8Array[] = []; - for await (let chunk of readable.pipeThrough(new FixedChunkStream(512))) { + for await (const chunk of readable.pipeThrough(new FixedChunkStream(512))) { if (chunk.length !== 512) { throw new Error("Tarball has an unexpected number of bytes."); } From 27b25855aae2770a5aefe63322e7e14c10db5498 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sat, 10 Aug 2024 18:18:40 +1000 Subject: [PATCH 41/82] adjust(archive): Error types and messages --- archive/tar_stream.ts | 8 ++++---- archive/tar_stream_test.ts | 8 ++++---- archive/untar_stream.ts | 6 ++++-- archive/untar_stream_test.ts | 4 ++-- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index d0e10470d32c..8768440d0030 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -212,8 +212,8 @@ export class TarStream implements TransformStream { yield value; } if (chunk.size !== size) { - throw new Error( - "Invalid Tarball! Provided size did not match bytes read from provided iterable.", + throw new RangeError( + "Provided size did not match bytes read from provided iterable", ); } if (chunk.size % 512) { @@ -281,13 +281,13 @@ export function parsePathname( } if (name.length > 256) { - throw new Error("Invalid Pathname! Pathname cannot exceed 256 bytes."); + throw new RangeError("Pathname cannot exceed 256 bytes"); } // If length of last part is > 100, then there's no possible answer to split the path let suitableSlashPos = Math.max(0, name.lastIndexOf(SLASH_CODE_POINT)); // always holds position of '/' if (name.length - suitableSlashPos > 100) { - throw new Error("Invalid Filename! Filename cannot exceed 100 bytes."); + throw new RangeError("Filename cannot exceed 100 bytes"); } for ( diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index 6c3826cec074..4c95efaa8219 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -276,10 +276,10 @@ Deno.test("TarStream() with mismatching sizes", async () => { for await (const _ of readable) {} } catch (error) { threw = true; - assert(error instanceof Error); + assert(error instanceof RangeError); assertEquals( error.message, - "Invalid Tarball! Provided size did not match bytes read from provided iterable.", + "Provided size did not match bytes read from provided iterable", ); } assertEquals(threw, true); @@ -291,10 +291,10 @@ Deno.test("parsePathname() with too long path", () => { parsePathname("0".repeat(300)); } catch (error) { threw = true; - assert(error instanceof Error); + assert(error instanceof RangeError); assertEquals( error.message, - "Invalid Pathname! Pathname cannot exceed 256 bytes.", + "Pathname cannot exceed 256 bytes", ); } assertEquals(threw, true); diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index be1de3562421..9a06b999fd09 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -109,7 +109,9 @@ export class UnTarStream this.#gen = async function* () { const buffer: Uint8Array[] = []; - for await (const chunk of readable.pipeThrough(new FixedChunkStream(512))) { + for await ( + const chunk of readable.pipeThrough(new FixedChunkStream(512)) + ) { if (chunk.length !== 512) { throw new Error("Tarball has an unexpected number of bytes."); } @@ -142,7 +144,7 @@ export class UnTarStream if ( checksum.reduce((x, y) => x + y) !== parseInt(decoder.decode(value.slice(148, 156 - 2)), 8) - ) throw new Error("Invalid Tarball. Header failed to pass checksum."); + ) throw new SyntaxError("Tarball header failed to pass checksum"); // Decode Header let header: OldStyleFormat | PosixUstarFormat = { diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index c7fa9b932a0f..74ae2e03fe9f 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -234,10 +234,10 @@ Deno.test("UnTarStream() with invalid checksum", async () => { } } catch (error) { threw = true; - assert(error instanceof Error); + assert(error instanceof SyntaxError); assertEquals( error.message, - "Invalid Tarball. Header failed to pass checksum.", + "Tarball header failed to pass checksum", ); } assertEquals(threw, true); From 80d1b546a8a767237a3875a3c32f64e28e129c79 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sat, 10 Aug 2024 18:28:23 +1000 Subject: [PATCH 42/82] adjust(archive): more Error messages / improve tests --- archive/tar_stream.ts | 10 +++---- archive/tar_stream_test.ts | 55 +++++++++++++++++++++--------------- archive/untar_stream.ts | 8 +++--- archive/untar_stream_test.ts | 6 ++-- 4 files changed, 44 insertions(+), 35 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 8768440d0030..973962191f31 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -125,7 +125,7 @@ export class TarStream implements TransformStream { >({ transform(chunk, controller) { if (chunk.options && !validTarStreamOptions(chunk.options)) { - return controller.error("Invalid Options Provided!"); + return controller.error("Invalid TarStreamOptions Provided"); } if ( @@ -134,7 +134,7 @@ export class TarStream implements TransformStream { chunk.size.toString() === "NaN") ) { return controller.error( - "Invalid Size Provided! Size cannot exceed 64 Gibs.", + "Size cannot exceed 64 Gibs", ); } @@ -145,7 +145,7 @@ export class TarStream implements TransformStream { chunk.pathname[0].length > 155 || chunk.pathname[1].length > 100 ) { controller.error( - "Invalid Pathname. Pathnames, when provided as a Uint8Array, need to be no more than [155, 100] bytes respectively.", + "Pathnames, when provided as a Uint8Array, need to be no more than [155, 100] bytes respectively", ); } return chunk.pathname; @@ -213,7 +213,7 @@ export class TarStream implements TransformStream { } if (chunk.size !== size) { throw new RangeError( - "Provided size did not match bytes read from provided iterable", + "Provided size did not match bytes read from provided readable", ); } if (chunk.size % 512) { @@ -306,7 +306,7 @@ export function parsePathname( const prefix = name.slice(0, suitableSlashPos); if (prefix.length > 155) { throw new Error( - "Invalid Pathname! Pathname needs to be split-able on a forward slash separator into [155, 100] bytes respectively.", + "Pathname needs to be split-able on a forward slash separator into [155, 100] bytes respectively", ); } return [prefix, name.slice(suitableSlashPos + 1)]; diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index 4c95efaa8219..0727b1270544 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -5,7 +5,7 @@ import { type TarStreamInput, validTarStreamOptions, } from "./tar_stream.ts"; -import { assertEquals, assertRejects } from "../assert/mod.ts"; +import { assertEquals } from "../assert/mod.ts"; import { assert } from "../assert/assert.ts"; Deno.test("TarStream() with default stream", async () => { @@ -76,12 +76,15 @@ Deno.test("TarStream() with negative size", async () => { ]) .pipeThrough(new TarStream()); - await assertRejects( - async function () { - await Array.fromAsync(readable); - }, - "Invalid Size Provided! Size cannot exceed 64 Gibs.", - ); + let threw = false; + try { + await Array.fromAsync(readable); + } catch (error) { + threw = true; + assert(typeof error === "string"); + assertEquals(error, "Size cannot exceed 64 Gibs"); + } + assertEquals(threw, true); }); Deno.test("TarStream() with 65 GiB size", async () => { @@ -102,12 +105,15 @@ Deno.test("TarStream() with 65 GiB size", async () => { ]) .pipeThrough(new TarStream()); - await assertRejects( - async function () { - await Array.fromAsync(readable); - }, - "Invalid Size Provided! Size cannot exceed 64 Gibs.", - ); + let threw = false; + try { + await Array.fromAsync(readable); + } catch (error) { + threw = true; + assert(typeof error === "string"); + assertEquals(error, "Size cannot exceed 64 Gibs"); + } + assertEquals(threw, true); }); Deno.test("TarStream() with NaN size", async () => { @@ -128,12 +134,15 @@ Deno.test("TarStream() with NaN size", async () => { ]) .pipeThrough(new TarStream()); - await assertRejects( - async function () { - await Array.fromAsync(readable); - }, - "Invalid Size Provided! Size cannot exceed 64 Gibs.", - ); + let threw = false; + try { + await Array.fromAsync(readable); + } catch (error) { + threw = true; + assert(typeof error === "string"); + assertEquals(error, "Size cannot exceed 64 Gibs"); + } + assertEquals(threw, true); }); Deno.test("parsePathname()", () => { @@ -235,7 +244,7 @@ Deno.test("TarStream() with invalid options", async () => { } catch (error) { threw = true; assert(typeof error === "string"); - assertEquals(error, "Invalid Options Provided!"); + assertEquals(error, "Invalid TarStreamOptions Provided"); } assertEquals(threw, true); }); @@ -254,7 +263,7 @@ Deno.test("TarStream() with invalid pathname", async () => { assert(typeof error === "string"); assertEquals( error, - "Invalid Pathname. Pathnames, when provided as a Uint8Array, need to be no more than [155, 100] bytes respectively.", + "Pathnames, when provided as a Uint8Array, need to be no more than [155, 100] bytes respectively", ); } assertEquals(threw, true); @@ -279,7 +288,7 @@ Deno.test("TarStream() with mismatching sizes", async () => { assert(error instanceof RangeError); assertEquals( error.message, - "Provided size did not match bytes read from provided iterable", + "Provided size did not match bytes read from provided readable", ); } assertEquals(threw, true); @@ -309,7 +318,7 @@ Deno.test("parsePathname() with too long path", () => { assert(error instanceof Error); assertEquals( error.message, - "Invalid Pathname! Pathname needs to be split-able on a forward slash separator into [155, 100] bytes respectively.", + "Pathname needs to be split-able on a forward slash separator into [155, 100] bytes respectively", ); } assertEquals(threw, true); diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 9a06b999fd09..e551585e79fa 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -113,17 +113,17 @@ export class UnTarStream const chunk of readable.pipeThrough(new FixedChunkStream(512)) ) { if (chunk.length !== 512) { - throw new Error("Tarball has an unexpected number of bytes."); + throw new Error("Tarball has an unexpected number of bytes"); } buffer.push(chunk); if (buffer.length > 2) yield buffer.shift()!; } if (buffer.length < 2) { - throw new Error("Tarball was too small to be valid."); + throw new Error("Tarball was too small to be valid"); } if (!buffer.every((value) => value.every((x) => x === 0))) { - throw new Error("Tarball has invalid ending."); + throw new Error("Tarball has invalid ending"); } }(); } @@ -238,7 +238,7 @@ export class UnTarStream const { done, value } = await this.#gen.next(); if (done) { this.#lock = false; - throw new Error("Unexpected end of Tarball."); + throw new Error("Unexpected end of Tarball"); } if (i === 1 && size % 512) yield value.slice(0, size % 512); else yield value; diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index 74ae2e03fe9f..6cc8c63f6f1b 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -153,7 +153,7 @@ Deno.test("UnTarStream() with invalid size", async () => { } catch (error) { threw = true; assert(error instanceof Error); - assertEquals(error.message, "Tarball has an unexpected number of bytes."); + assertEquals(error.message, "Tarball has an unexpected number of bytes"); } assertEquals(threw, true); }); @@ -186,7 +186,7 @@ Deno.test("UnTarStream() with invalid ending", async () => { assert(error instanceof Error); assertEquals( error.message, - "Tarball has invalid ending.", + "Tarball has invalid ending", ); } assertEquals(threw, true); @@ -204,7 +204,7 @@ Deno.test("UnTarStream() with too small size", async () => { } catch (error) { threw = true; assert(error instanceof Error); - assertEquals(error.message, "Tarball was too small to be valid."); + assertEquals(error.message, "Tarball was too small to be valid"); } assertEquals(threw, true); }); From a84c5fbede4e7efc1c87f1862f1ff594ad8acb79 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sat, 10 Aug 2024 18:44:17 +1000 Subject: [PATCH 43/82] refactor(archive): UnTarStream to return TarStreamChunk instead of TarStreamEntry --- archive/untar_stream.ts | 86 +++++++++++-------------------- archive/untar_stream_test.ts | 98 +++++++++--------------------------- 2 files changed, 53 insertions(+), 131 deletions(-) diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index e551585e79fa..423224d35d56 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -1,15 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. import { FixedChunkStream } from "@std/streams"; -/** - * The interface extracted from the archive. - */ -export interface TarStreamEntry { - pathname: string; - header: TarStreamHeader; - readable?: ReadableStream; -} - /** * The original tar archive header format. */ @@ -52,7 +43,24 @@ export interface PosixUstarFormat { /** * The header of an entry in the archive. */ -export type TarStreamHeader = OldStyleFormat | PosixUstarFormat; +export interface TarStreamHeader { + type: "header"; + pathname: string; + header: OldStyleFormat | PosixUstarFormat; +} + +/** + * The data belonging to the last entry returned. + */ +export interface TarStreamData { + type: "data"; + data: Uint8Array; +} + +/** + * The type extracted from the archive. + */ +export type TarStreamChunk = TarStreamHeader | TarStreamData; /** * ### Overview @@ -94,9 +102,8 @@ export type TarStreamHeader = OldStyleFormat | PosixUstarFormat; * ``` */ export class UnTarStream - implements TransformStream { - #lock = false; - #readable: ReadableStream; + implements TransformStream { + #readable: ReadableStream; #writable: WritableStream; #gen: AsyncGenerator; constructor() { @@ -128,13 +135,9 @@ export class UnTarStream }(); } - async *#untar(): AsyncGenerator { + async *#untar(): AsyncGenerator { const decoder = new TextDecoder(); while (true) { - while (this.#lock) { - await new Promise((a) => setTimeout(a, 0)); - } - const { done, value } = await this.#gen.next(); if (done) break; @@ -192,64 +195,33 @@ export class UnTarStream } yield { + type: "header", pathname: ("prefix" in header && header.prefix.length ? header.prefix + "/" : "") + header.name, header, - readable: !["1", "2", "3", "4", "5", "6"].includes(header.typeflag) - ? this.#readableFile(header.size) - : undefined, }; + if (!["1", "2", "3", "4", "5", "6"].includes(header.typeflag)) { + for await (const data of this.#genFile(header.size)) { + yield { type: "data", data }; + } + } } } - #readableFile(size: number): ReadableStream { - const gen = this.#genFile(size); - return new ReadableStream({ - type: "bytes", - async pull(controller) { - const { done, value } = await gen.next(); - if (done) { - controller.close(); - controller.byobRequest?.respond(0); - } else if (controller.byobRequest?.view) { - const buffer = new Uint8Array(controller.byobRequest.view.buffer); - const size = buffer.length; - if (value.length > size) { - buffer.set(value.slice(0, size)); - controller.byobRequest.respond(size); - controller.enqueue(value.slice(size)); - } else { - buffer.set(value); - controller.byobRequest.respond(value.length); - } - } else controller.enqueue(value); - }, - async cancel() { - // deno-lint-ignore no-empty - for await (const _ of gen) {} - }, - }); - } - async *#genFile(size: number): AsyncGenerator { - this.#lock = true; for (let i = Math.ceil(size / 512); i > 0; --i) { const { done, value } = await this.#gen.next(); - if (done) { - this.#lock = false; - throw new Error("Unexpected end of Tarball"); - } + if (done) throw new Error("Unexpected end of Tarball"); if (i === 1 && size % 512) yield value.slice(0, size % 512); else yield value; } - this.#lock = false; } /** * The ReadableStream */ - get readable(): ReadableStream { + get readable(): ReadableStream { return this.#readable; } diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index 6cc8c63f6f1b..9966eb36cc9d 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -22,13 +22,12 @@ Deno.test("expandTarArchiveCheckingHeaders", async () => { const pathnames: string[] = []; for await (const item of readable) { - pathnames.push(item.pathname); - item.readable?.cancel(); + if (item.type === "header") pathnames.push(item.pathname); } assertEquals(pathnames, ["./potato", "./text.txt"]); }); -Deno.test("expandTarArchiveCheckingBodiesDefaultStream", async () => { +Deno.test("expandTarArchiveCheckingBodies", async () => { const text = new TextEncoder().encode("Hello World!"); const readable = ReadableStream.from([ @@ -44,58 +43,15 @@ Deno.test("expandTarArchiveCheckingBodiesDefaultStream", async () => { .pipeThrough(new TarStream()) .pipeThrough(new UnTarStream()); + const buffer = new Uint8Array(text.length); + let offset = 0; for await (const item of readable) { - if (item.readable) { - const buffer = new Uint8Array(text.length); - let offset = 0; - const reader = item.readable.getReader(); - while (true) { - const { done, value } = await reader.read(); - if (done) { - break; - } - buffer.set(value, offset); - offset += value.length; - } - assertEquals(buffer, text); - } - } -}); - -Deno.test("expandTarArchiveCheckingBodiesByteStream", async () => { - const text = new TextEncoder().encode("Hello World!"); - - const readable = ReadableStream.from([ - { - pathname: "./potato", - }, - { - pathname: "./text.txt", - size: text.length, - readable: ReadableStream.from([text.slice()]), - }, - ]) - .pipeThrough(new TarStream()) - .pipeThrough(new UnTarStream()); - - for await (const item of readable) { - if (item.readable) { - const buffer = new Uint8Array(text.length); - let offset = 0; - const reader = item.readable.getReader({ mode: "byob" }); - while (true) { - const { done, value } = await reader.read( - new Uint8Array(Math.ceil(Math.random() * 1024)), - ); - if (done) { - break; - } - buffer.set(value, offset); - offset += value.length; - } - assertEquals(buffer, text); + if (item.type === "data") { + buffer.set(item.data, offset); + offset += item.data.length; } } + assertEquals(buffer, text); }); Deno.test("UnTarStream() with size equals to multiple of 512", async () => { @@ -115,16 +71,14 @@ Deno.test("UnTarStream() with size equals to multiple of 512", async () => { .pipeThrough(new TarStream()) .pipeThrough(new UnTarStream()); - for await (const item of readable) { - if (item.readable) { - assertEquals( - Uint8Array.from( - (await Array.fromAsync(item.readable)).map((x) => [...x]).flat(), - ), - data, - ); - } - } + assertEquals( + concat( + (await Array.fromAsync(readable)).filter((x) => x.type === "data").map( + (x) => x.data, + ), + ), + data, + ); }); Deno.test("UnTarStream() with invalid size", async () => { @@ -147,9 +101,8 @@ Deno.test("UnTarStream() with invalid size", async () => { let threw = false; try { - for await (const entry of readable) { - await entry.readable?.cancel(); - } + // deno-lint-ignore no-empty + for await (const _ of readable) {} } catch (error) { threw = true; assert(error instanceof Error); @@ -178,9 +131,8 @@ Deno.test("UnTarStream() with invalid ending", async () => { let threw = false; try { - for await (const entry of readable) { - await entry.readable?.cancel(); - } + // deno-lint-ignore no-empty + for await (const _ of readable) {} } catch (error) { threw = true; assert(error instanceof Error); @@ -198,9 +150,8 @@ Deno.test("UnTarStream() with too small size", async () => { let threw = false; try { - for await (const entry of readable) { - await entry.readable?.cancel(); - } + // deno-lint-ignore no-empty + for await (const _ of readable) {} } catch (error) { threw = true; assert(error instanceof Error); @@ -229,9 +180,8 @@ Deno.test("UnTarStream() with invalid checksum", async () => { let threw = false; try { - for await (const entry of readable) { - await entry.readable?.cancel(); - } + // deno-lint-ignore no-empty + for await (const _ of readable) {} } catch (error) { threw = true; assert(error instanceof SyntaxError); From c201e4c8ac50e5d8def7862b8d485b822110f29f Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sat, 10 Aug 2024 18:54:48 +1000 Subject: [PATCH 44/82] fix(archive): JSDoc example --- archive/untar_stream.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 423224d35d56..1935f4ec07e6 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -88,16 +88,17 @@ export type TarStreamChunk = TarStreamHeader | TarStreamData; * ```ts * import { UnTarStream } from "@std/archive/untar-stream"; * + * let fileWriter: WritableStreamDefaultWriter | undefined; * for await ( * const entry of (await Deno.open('./out.tar.gz')) * .readable * .pipeThrough(new DecompressionStream('gzip')) * .pipeThrough(new UnTarStream()) * ) { - * console.log(entry.pathname) - * entry - * .readable - * ?.pipeTo((await Deno.create(entry.pathname)).writable) + * if (entry.type === "header") { + * fileWriter?.close(); + * fileWriter = (await Deno.create(entry.pathname)).writable.getWriter(); + * } else await fileWriter!.write(entry.data); * } * ``` */ From 8059edca9eac241fb13890d2a975b6f83c5bd2c9 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sat, 10 Aug 2024 19:02:58 +1000 Subject: [PATCH 45/82] adjust(archive): mode, uid, gid options to be provided as numbers instead of strings. --- archive/tar_stream.ts | 30 ++++++++++++++++++------------ archive/tar_stream_test.ts | 23 ++++++++++------------- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 973962191f31..3e2ff0565ecb 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -30,15 +30,15 @@ export interface TarStreamOptions { /** * An octal number in ASCII */ - mode: string; + mode: number; /** * An octal number in ASCII. */ - uid: string; + uid: number; /** * An octal number in ASCII. */ - gid: string; + gid: number; /** * A number of seconds since the start of epoch. Avoid negative values. */ @@ -163,9 +163,9 @@ export class TarStream implements TransformStream { const header = new Uint8Array(512); const size = "size" in chunk ? chunk.size : 0; const options: TarStreamOptions = { - mode: typeflag === "5" ? "755" : "644", - uid: "", - gid: "", + mode: typeflag === "5" ? 755 : 644, + uid: 0, + gid: 0, mtime: Math.floor(new Date().getTime() / 1000), uname: "", gname: "", @@ -177,9 +177,9 @@ export class TarStream implements TransformStream { header.set(name); // name header.set( encoder.encode( - options.mode.padStart(6, "0") + " \0" + // mode - options.uid.padStart(6, "0") + " \0" + //uid - options.gid.padStart(6, "0") + " \0" + // gid + options.mode.toString().padStart(6, "0") + " \0" + // mode + options.uid.toString().padStart(6, "0") + " \0" + //uid + options.gid.toString().padStart(6, "0") + " \0" + // gid size.toString(8).padStart(size < 8 ** 11 ? 11 : 12, "0") + (size < 8 ** 11 ? " " : "") + // size options.mtime.toString(8).padStart(11, "0") + " " + // mtime @@ -319,13 +319,19 @@ export function validTarStreamOptions( options: Partial, ): boolean { if ( - options.mode && (options.mode.length > 6 || !/^[0-7]*$/.test(options.mode)) + options.mode && + (options.mode.toString().length > 6 || + !/^[0-7]*$/.test(options.mode.toString())) ) return false; if ( - options.uid && (options.uid.length > 6 || !/^[0-7]*$/.test(options.uid)) + options.uid && + (options.uid.toString().length > 6 || + !/^[0-7]*$/.test(options.uid.toString())) ) return false; if ( - options.gid && (options.gid.length > 6 || !/^[0-7]*$/.test(options.gid)) + options.gid && + (options.gid.toString().length > 6 || + !/^[0-7]*$/.test(options.gid.toString())) ) return false; if ( options.mtime != undefined && diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index 0727b1270544..43d501dcf70d 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -188,20 +188,17 @@ Deno.test("parsePathname()", () => { Deno.test("validTarStreamOptions()", () => { assertEquals(validTarStreamOptions({}), true); - assertEquals(validTarStreamOptions({ mode: "" }), true); - assertEquals(validTarStreamOptions({ mode: "000" }), true); - assertEquals(validTarStreamOptions({ mode: "008" }), false); - assertEquals(validTarStreamOptions({ mode: "0000000" }), false); + assertEquals(validTarStreamOptions({ mode: 0 }), true); + assertEquals(validTarStreamOptions({ mode: 8 }), false); + assertEquals(validTarStreamOptions({ mode: 1111111 }), false); - assertEquals(validTarStreamOptions({ uid: "" }), true); - assertEquals(validTarStreamOptions({ uid: "000" }), true); - assertEquals(validTarStreamOptions({ uid: "008" }), false); - assertEquals(validTarStreamOptions({ uid: "0000000" }), false); + assertEquals(validTarStreamOptions({ uid: 0 }), true); + assertEquals(validTarStreamOptions({ uid: 8 }), false); + assertEquals(validTarStreamOptions({ uid: 1111111 }), false); - assertEquals(validTarStreamOptions({ gid: "" }), true); - assertEquals(validTarStreamOptions({ gid: "000" }), true); - assertEquals(validTarStreamOptions({ gid: "008" }), false); - assertEquals(validTarStreamOptions({ gid: "0000000" }), false); + assertEquals(validTarStreamOptions({ gid: 0 }), true); + assertEquals(validTarStreamOptions({ gid: 8 }), false); + assertEquals(validTarStreamOptions({ gid: 1111111 }), false); assertEquals(validTarStreamOptions({ mtime: 0 }), true); assertEquals(validTarStreamOptions({ mtime: NaN }), false); @@ -234,7 +231,7 @@ Deno.test("validTarStreamOptions()", () => { Deno.test("TarStream() with invalid options", async () => { const readable = ReadableStream.from([ - { pathname: "potato", options: { mode: "009" } }, + { pathname: "potato", options: { mode: 9 } }, ]).pipeThrough(new TarStream()); let threw = false; From 42ca5062a1f197dc63de4e343a80148f3841fc70 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sat, 10 Aug 2024 19:16:31 +1000 Subject: [PATCH 46/82] adjust(archive): TarStream's pathname to be only of type string --- archive/tar_stream.ts | 23 ++++++++--------------- archive/tar_stream_test.ts | 20 -------------------- 2 files changed, 8 insertions(+), 35 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 3e2ff0565ecb..fbdfc16b4d18 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -3,7 +3,7 @@ * The interface required to provide a file. */ export interface TarStreamFile { - pathname: string | [Uint8Array, Uint8Array]; + pathname: string; size: number; readable: ReadableStream; options?: Partial; @@ -13,7 +13,7 @@ export interface TarStreamFile { * The interface required to provide a directory. */ export interface TarStreamDir { - pathname: string | [Uint8Array, Uint8Array]; + pathname: string; options?: Partial; } @@ -22,6 +22,9 @@ export interface TarStreamDir { * TarStream class. */ export type TarStreamInput = TarStreamFile | TarStreamDir; +type TarStreamInputInternal = + & (Omit | Omit) + & { pathname: [Uint8Array, Uint8Array] }; /** * The options that can go along with a file or directory. @@ -112,6 +115,7 @@ const SLASH_CODE_POINT = "/".charCodeAt(0); * .pipeTo((await Deno.create('./out.tar.gz')).writable) * ``` */ + export class TarStream implements TransformStream { #readable: ReadableStream; #writable: WritableStream; @@ -121,7 +125,7 @@ export class TarStream implements TransformStream { constructor() { const { readable, writable } = new TransformStream< TarStreamInput, - TarStreamInput & { pathname: [Uint8Array, Uint8Array] } + TarStreamInputInternal >({ transform(chunk, controller) { if (chunk.options && !validTarStreamOptions(chunk.options)) { @@ -138,18 +142,7 @@ export class TarStream implements TransformStream { ); } - const pathname = typeof chunk.pathname === "string" - ? parsePathname(chunk.pathname) - : function () { - if ( - chunk.pathname[0].length > 155 || chunk.pathname[1].length > 100 - ) { - controller.error( - "Pathnames, when provided as a Uint8Array, need to be no more than [155, 100] bytes respectively", - ); - } - return chunk.pathname; - }(); + const pathname = parsePathname(chunk.pathname); controller.enqueue({ ...chunk, pathname }); }, diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index 43d501dcf70d..9365849ca8a1 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -246,26 +246,6 @@ Deno.test("TarStream() with invalid options", async () => { assertEquals(threw, true); }); -Deno.test("TarStream() with invalid pathname", async () => { - const readable = ReadableStream.from([ - { pathname: [new Uint8Array(156), new Uint8Array(0)] }, - ]).pipeThrough(new TarStream()); - - let threw = false; - try { - // deno-lint-ignore no-empty - for await (const _ of readable) {} - } catch (error) { - threw = true; - assert(typeof error === "string"); - assertEquals( - error, - "Pathnames, when provided as a Uint8Array, need to be no more than [155, 100] bytes respectively", - ); - } - assertEquals(threw, true); -}); - Deno.test("TarStream() with mismatching sizes", async () => { const text = new TextEncoder().encode("Hello World!"); const readable = ReadableStream.from([ From 2e5caf3799bf18ebc5bf5cc1fe57b92b84c2d327 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sun, 11 Aug 2024 18:00:14 +1000 Subject: [PATCH 47/82] fix(archive): prefix/name to ignore everything past the first NULL --- archive/untar_stream.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 1935f4ec07e6..5bf5ef8ca1e9 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -152,7 +152,7 @@ export class UnTarStream // Decode Header let header: OldStyleFormat | PosixUstarFormat = { - name: decoder.decode(value.slice(0, 100)).replaceAll("\0", ""), + name: decoder.decode(value.slice(0, 100)).split("\0")[0]!, mode: decoder.decode(value.slice(100, 108 - 2)), uid: decoder.decode(value.slice(108, 116 - 2)), gid: decoder.decode(value.slice(116, 124 - 2)), @@ -187,10 +187,7 @@ export class UnTarStream "\0", "", ), - prefix: decoder.decode(value.slice(345, 500)).replaceAll( - "\0", - "", - ), + prefix: decoder.decode(value.slice(345, 500)).split("\0")[0]!, pad: value.slice(500), }; } From 38e23448bd8ea38d8d51895f1fb3d0e6ba29441a Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sun, 11 Aug 2024 18:03:28 +1000 Subject: [PATCH 48/82] adjust(archive): `checksum` and `pad` to not be exposed from UnTarStream --- archive/untar_stream.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 5bf5ef8ca1e9..76e2f5a425ad 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -11,10 +11,8 @@ export interface OldStyleFormat { gid: string; size: number; mtime: number; - checksum: string; typeflag: string; linkname: string; - pad: Uint8Array; } /** @@ -27,7 +25,6 @@ export interface PosixUstarFormat { gid: string; size: number; mtime: number; - checksum: string; typeflag: string; linkname: string; magic: string; @@ -37,7 +34,6 @@ export interface PosixUstarFormat { devmajor: string; devminor: string; prefix: string; - pad: Uint8Array; } /** @@ -158,13 +154,11 @@ export class UnTarStream gid: decoder.decode(value.slice(116, 124 - 2)), size: parseInt(decoder.decode(value.slice(124, 136)).trimEnd(), 8), mtime: parseInt(decoder.decode(value.slice(136, 148 - 1)), 8), - checksum: decoder.decode(value.slice(148, 156 - 2)), typeflag: decoder.decode(value.slice(156, 157)), linkname: decoder.decode(value.slice(157, 257)).replaceAll( "\0", "", ), - pad: value.slice(257), }; if (header.typeflag === "\0") header.typeflag = "0"; // "ustar\u000000" @@ -188,7 +182,6 @@ export class UnTarStream "", ), prefix: decoder.decode(value.slice(345, 500)).split("\0")[0]!, - pad: value.slice(500), }; } From 86b4d943791972cf63ece4508b5c77cad7fc1c8b Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sun, 11 Aug 2024 18:07:41 +1000 Subject: [PATCH 49/82] adjust(archive): checksum calculation --- archive/untar_stream.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 76e2f5a425ad..6deb3d67edd9 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -139,12 +139,14 @@ export class UnTarStream if (done) break; // Validate Checksum - const checksum = value.slice(); - checksum.set(new Uint8Array(8).fill(32), 148); - if ( - checksum.reduce((x, y) => x + y) !== - parseInt(decoder.decode(value.slice(148, 156 - 2)), 8) - ) throw new SyntaxError("Tarball header failed to pass checksum"); + const checksum = parseInt( + decoder.decode(value.subarray(148, 156 - 2)), + 8, + ); + value.fill(32, 148, 156); + if (value.reduce((x, y) => x + y) !== checksum) { + throw new SyntaxError("Tarball header failed to pass checksum"); + } // Decode Header let header: OldStyleFormat | PosixUstarFormat = { From 5b5a246c924cffb5f6f71ea96133260744bffc15 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sun, 11 Aug 2024 18:10:09 +1000 Subject: [PATCH 50/82] change(archive): `.slice` to `.subarray` --- archive/untar_stream.ts | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 6deb3d67edd9..04e7160710b8 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -150,14 +150,14 @@ export class UnTarStream // Decode Header let header: OldStyleFormat | PosixUstarFormat = { - name: decoder.decode(value.slice(0, 100)).split("\0")[0]!, - mode: decoder.decode(value.slice(100, 108 - 2)), - uid: decoder.decode(value.slice(108, 116 - 2)), - gid: decoder.decode(value.slice(116, 124 - 2)), - size: parseInt(decoder.decode(value.slice(124, 136)).trimEnd(), 8), - mtime: parseInt(decoder.decode(value.slice(136, 148 - 1)), 8), - typeflag: decoder.decode(value.slice(156, 157)), - linkname: decoder.decode(value.slice(157, 257)).replaceAll( + name: decoder.decode(value.subarray(0, 100)).split("\0")[0]!, + mode: decoder.decode(value.subarray(100, 108 - 2)), + uid: decoder.decode(value.subarray(108, 116 - 2)), + gid: decoder.decode(value.subarray(116, 124 - 2)), + size: parseInt(decoder.decode(value.subarray(124, 136)).trimEnd(), 8), + mtime: parseInt(decoder.decode(value.subarray(136, 148 - 1)), 8), + typeflag: decoder.decode(value.subarray(156, 157)), + linkname: decoder.decode(value.subarray(157, 257)).replaceAll( "\0", "", ), @@ -171,19 +171,19 @@ export class UnTarStream ) { header = { ...header, - magic: decoder.decode(value.slice(257, 263)), - version: decoder.decode(value.slice(263, 265)), - uname: decoder.decode(value.slice(265, 297)).replaceAll("\0", ""), - gname: decoder.decode(value.slice(297, 329)).replaceAll("\0", ""), - devmajor: decoder.decode(value.slice(329, 337)).replaceAll( + magic: decoder.decode(value.subarray(257, 263)), + version: decoder.decode(value.subarray(263, 265)), + uname: decoder.decode(value.subarray(265, 297)).replaceAll("\0", ""), + gname: decoder.decode(value.subarray(297, 329)).replaceAll("\0", ""), + devmajor: decoder.decode(value.subarray(329, 337)).replaceAll( "\0", "", ), - devminor: decoder.decode(value.slice(337, 345)).replaceAll( + devminor: decoder.decode(value.subarray(337, 345)).replaceAll( "\0", "", ), - prefix: decoder.decode(value.slice(345, 500)).split("\0")[0]!, + prefix: decoder.decode(value.subarray(345, 500)).split("\0")[0]!, }; } @@ -206,7 +206,7 @@ export class UnTarStream for (let i = Math.ceil(size / 512); i > 0; --i) { const { done, value } = await this.#gen.next(); if (done) throw new Error("Unexpected end of Tarball"); - if (i === 1 && size % 512) yield value.slice(0, size % 512); + if (i === 1 && size % 512) yield value.subarray(0, size % 512); else yield value; } } From 93bfe5ba25fc715bec6f2a71045d9acd2165665b Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sun, 11 Aug 2024 18:14:23 +1000 Subject: [PATCH 51/82] doc(archive): "octal number" to "octal literal" --- archive/tar_stream.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index fbdfc16b4d18..03a9b6dca3fe 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -31,15 +31,15 @@ type TarStreamInputInternal = */ export interface TarStreamOptions { /** - * An octal number in ASCII + * An octal literal. */ mode: number; /** - * An octal number in ASCII. + * An octal literal. */ uid: number; /** - * An octal number in ASCII. + * An octal literal. */ gid: number; /** From 7b86bab147467d5dbce6186d363534528815ada4 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sun, 11 Aug 2024 18:27:52 +1000 Subject: [PATCH 52/82] adjust(archive): TarStreamOptions to be optional with defaults --- archive/tar_stream.ts | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 03a9b6dca3fe..2eceaa608136 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -6,7 +6,7 @@ export interface TarStreamFile { pathname: string; size: number; readable: ReadableStream; - options?: Partial; + options?: TarStreamOptions; } /** @@ -14,7 +14,7 @@ export interface TarStreamFile { */ export interface TarStreamDir { pathname: string; - options?: Partial; + options?: TarStreamOptions; } /** @@ -33,35 +33,41 @@ export interface TarStreamOptions { /** * An octal literal. */ - mode: number; + mode?: number; /** * An octal literal. + * @default {0} */ - uid: number; + uid?: number; /** * An octal literal. + * @default {0} */ - gid: number; + gid?: number; /** * A number of seconds since the start of epoch. Avoid negative values. */ - mtime: number; + mtime?: number; /** * An ASCII string. Should be used in preference of uid. + * @default {''} */ - uname: string; + uname?: string; /** * An ASCII string. Should be used in preference of gid. + * @default {''} */ - gname: string; + gname?: string; /** * The major number for character device. + * @default {''} */ - devmajor: string; + devmajor?: string; /** * The minor number for block device entry. + * @default {''} */ - devminor: string; + devminor?: string; } const SLASH_CODE_POINT = "/".charCodeAt(0); @@ -155,7 +161,7 @@ export class TarStream implements TransformStream { const typeflag = "size" in chunk ? "0" : "5"; const header = new Uint8Array(512); const size = "size" in chunk ? chunk.size : 0; - const options: TarStreamOptions = { + const options: Required = { mode: typeflag === "5" ? 755 : 644, uid: 0, gid: 0, @@ -309,7 +315,7 @@ export function parsePathname( * provided are in the correct format, otherwise returns false. */ export function validTarStreamOptions( - options: Partial, + options: TarStreamOptions, ): boolean { if ( options.mode && From b3d5e0039af5648bf740391a778563015c81a230 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Sun, 11 Aug 2024 18:37:42 +1000 Subject: [PATCH 53/82] doc(archive): added more docs for the interfaces --- archive/tar_stream.ts | 18 +++++++++ archive/untar_stream.ts | 85 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 2eceaa608136..931b13113fc2 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -3,9 +3,21 @@ * The interface required to provide a file. */ export interface TarStreamFile { + /** + * The name of the file. + */ pathname: string; + /** + * The size of the file. + */ size: number; + /** + * The contents of the file. + */ readable: ReadableStream; + /** + * The metadata of the file. + */ options?: TarStreamOptions; } @@ -13,7 +25,13 @@ export interface TarStreamFile { * The interface required to provide a directory. */ export interface TarStreamDir { + /** + * The name of the directory. + */ pathname: string; + /** + * The metadata of the directory. + */ options?: TarStreamOptions; } diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 04e7160710b8..9be3cd605654 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -5,13 +5,37 @@ import { FixedChunkStream } from "@std/streams"; * The original tar archive header format. */ export interface OldStyleFormat { + /** + * The name of the entry. + */ name: string; + /** + * The mode of the entry. + */ mode: string; + /** + * The uid of the entry. + */ uid: string; + /** + * The gid of the entry. + */ gid: string; + /** + * The size of the entry. + */ size: number; + /** + * The mtime of the entry. + */ mtime: number; + /** + * The typeflag of the entry. + */ typeflag: string; + /** + * The linkname of the entry. + */ linkname: string; } @@ -19,20 +43,65 @@ export interface OldStyleFormat { * The POSIX ustar archive header format. */ export interface PosixUstarFormat { + /** + * The latter half of the name of the entry. + */ name: string; + /** + * The mode of the entry. + */ mode: string; + /** + * The uid of the entry. + */ uid: string; + /** + * The gid of the entry. + */ gid: string; + /** + * The size of the entry. + */ size: number; + /** + * The mtime of the entry. + */ mtime: number; + /** + * The typeflag of the entry. + */ typeflag: string; + /** + * The linkname of the entry. + */ linkname: string; + /** + * The magic number of the entry. + */ magic: string; + /** + * The version number of the entry. + */ version: string; + /** + * The uname of the entry. + */ uname: string; + /** + * The gname of the entry. + */ gname: string; + /** + * The devmajor of the entry. + */ devmajor: string; + /** + * The devminor of the entry. + */ devminor: string; + /** + * The former half of the name of the entry. + */ prefix: string; } @@ -40,8 +109,17 @@ export interface PosixUstarFormat { * The header of an entry in the archive. */ export interface TarStreamHeader { + /** + * The type 'header' indicating the start of a new entry. + */ type: "header"; + /** + * The pathname of the entry. + */ pathname: string; + /** + * The header of the entry. + */ header: OldStyleFormat | PosixUstarFormat; } @@ -49,7 +127,14 @@ export interface TarStreamHeader { * The data belonging to the last entry returned. */ export interface TarStreamData { + /** + * The type 'data' indicating a chunk of content from the last 'header' + * resolved. + */ type: "data"; + /** + * A chunk of content of from the entry. + */ data: Uint8Array; } From 5c56d2aa59b12d773c39a031fe002c4cd52bbdb3 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:49:40 +1000 Subject: [PATCH 54/82] docs(archive): denoting defaults --- archive/tar_stream.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 931b13113fc2..d01541aac5cc 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -50,6 +50,7 @@ type TarStreamInputInternal = export interface TarStreamOptions { /** * An octal literal. + * Defaults to 755 for directories and 644 for files. */ mode?: number; /** @@ -64,6 +65,7 @@ export interface TarStreamOptions { gid?: number; /** * A number of seconds since the start of epoch. Avoid negative values. + * Defaults to the current time in seconds. */ mtime?: number; /** From 9ccca0191961850cfee8e54313c111679da966e5 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Wed, 21 Aug 2024 17:36:54 +1000 Subject: [PATCH 55/82] docs(archive): updated for new lint rules --- archive/tar_stream.ts | 71 ++++++++++++++++++++++++++++++++++++++--- archive/untar_stream.ts | 44 +++++++++++++++++++++++-- 2 files changed, 108 insertions(+), 7 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index d01541aac5cc..a3fda54945aa 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -1,4 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + /** * The interface required to provide a file. */ @@ -117,8 +118,8 @@ const SLASH_CODE_POINT = "/".charCodeAt(0); * Tar archives are not compressed by default. If you'd like to compress the * archive, you may do so by piping it through a compression stream. * - * @example - * ```ts + * @example Usage + * ```ts no-eval * import { TarStream } from "@std/archive/tar-stream"; * * await ReadableStream.from([ @@ -145,9 +146,6 @@ const SLASH_CODE_POINT = "/".charCodeAt(0); export class TarStream implements TransformStream { #readable: ReadableStream; #writable: WritableStream; - /** - * Constructs a new instance. - */ constructor() { const { readable, writable } = new TransformStream< TarStreamInput, @@ -271,6 +269,32 @@ export class TarStream implements TransformStream { /** * The ReadableStream + * + * @return ReadableStream + * + * @example Usage + * ```ts no-eval + * import { TarStream } from "@std/archive/tar-stream"; + * + * await ReadableStream.from([ + * { + * pathname: 'potato/' + * }, + * { + * pathname: 'deno.json', + * size: (await Deno.stat('deno.json')).size, + * iterable: (await Deno.open('deno.json')).readable + * }, + * { + * pathname: 'deno.lock', + * size: (await Deno.stat('deno.lock')).size, + * iterable: (await Deno.open('deno.lock')).readable + * } + * ]) + * .pipeThrough(new TarStream()) + * .pipeThrough(new CompressionStream('gzip')) + * .pipeTo((await Deno.create('./out.tar.gz')).writable) + * ``` */ get readable(): ReadableStream { return this.#readable; @@ -278,6 +302,32 @@ export class TarStream implements TransformStream { /** * The WritableStream + * + * @return WritableStream + * + * @example Usage + * ```ts no-eval + * import { TarStream } from "@std/archive/tar-stream"; + * + * await ReadableStream.from([ + * { + * pathname: 'potato/' + * }, + * { + * pathname: 'deno.json', + * size: (await Deno.stat('deno.json')).size, + * iterable: (await Deno.open('deno.json')).readable + * }, + * { + * pathname: 'deno.lock', + * size: (await Deno.stat('deno.lock')).size, + * iterable: (await Deno.open('deno.lock')).readable + * } + * ]) + * .pipeThrough(new TarStream()) + * .pipeThrough(new CompressionStream('gzip')) + * .pipeTo((await Deno.create('./out.tar.gz')).writable) + * ``` */ get writable(): WritableStream { return this.#writable; @@ -333,6 +383,17 @@ export function parsePathname( /** * validTarStreamOptions is a function that returns a true if all of the options * provided are in the correct format, otherwise returns false. + * + * @param options The TarStreamOptions + * @return boolean + * + * @example Usage + * ```ts + * import { validTarStreamOptions } from "@std/archive"; + * import { assertEquals } from "@std/assert"; + * + * assertEquals(validTarStreamOptions({ mode: 755 }), true); + * ``` */ export function validTarStreamOptions( options: TarStreamOptions, diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 9be3cd605654..eeffda12665c 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -165,8 +165,8 @@ export type TarStreamChunk = TarStreamHeader | TarStreamData; * extension, such as '.tar.gz' for gzip. This TransformStream does not support * decompression which must be done before expanding the archive. * - * @example - * ```ts + * @example Usage + * ```ts no-eval * import { UnTarStream } from "@std/archive/untar-stream"; * * let fileWriter: WritableStreamDefaultWriter | undefined; @@ -298,6 +298,26 @@ export class UnTarStream /** * The ReadableStream + * + * @return ReadableStream + * + * @example Usage + * ```ts no-eval + * import { UnTarStream } from "@std/archive/untar-stream"; + * + * let fileWriter: WritableStreamDefaultWriter | undefined; + * for await ( + * const entry of (await Deno.open('./out.tar.gz')) + * .readable + * .pipeThrough(new DecompressionStream('gzip')) + * .pipeThrough(new UnTarStream()) + * ) { + * if (entry.type === "header") { + * fileWriter?.close(); + * fileWriter = (await Deno.create(entry.pathname)).writable.getWriter(); + * } else await fileWriter!.write(entry.data); + * } + * ``` */ get readable(): ReadableStream { return this.#readable; @@ -305,6 +325,26 @@ export class UnTarStream /** * The WritableStream + * + * @return WritableStream + * + * @example Usage + * ```ts no-eval + * import { UnTarStream } from "@std/archive/untar-stream"; + * + * let fileWriter: WritableStreamDefaultWriter | undefined; + * for await ( + * const entry of (await Deno.open('./out.tar.gz')) + * .readable + * .pipeThrough(new DecompressionStream('gzip')) + * .pipeThrough(new UnTarStream()) + * ) { + * if (entry.type === "header") { + * fileWriter?.close(); + * fileWriter = (await Deno.create(entry.pathname)).writable.getWriter(); + * } else await fileWriter!.write(entry.data); + * } + * ``` */ get writable(): WritableStream { return this.#writable; From 78721cac97a2ec0054d0cee5f0152fc0de5e0c5c Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Wed, 21 Aug 2024 17:41:11 +1000 Subject: [PATCH 56/82] adjust(archive): Tests to use assertRejects where appropriate & add `validPathname` function - The `validPathname` is meant to be a nicer exposed function for users of this lib to validate that their pathnames are valid before pipping it through the TarStream, over exposing parsePathname where the user may be confused about what to do with the result. --- archive/tar_stream.ts | 50 ++++++--- archive/tar_stream_test.ts | 198 ++++++++++++++--------------------- archive/untar_stream_test.ts | 68 ++++-------- 3 files changed, 140 insertions(+), 176 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index a3fda54945aa..dd6a36d23403 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -153,7 +153,9 @@ export class TarStream implements TransformStream { >({ transform(chunk, controller) { if (chunk.options && !validTarStreamOptions(chunk.options)) { - return controller.error("Invalid TarStreamOptions Provided"); + return controller.error( + new Error("Invalid TarStreamOptions Provided"), + ); } if ( @@ -161,9 +163,7 @@ export class TarStream implements TransformStream { (chunk.size < 0 || 8 ** 12 < chunk.size || chunk.size.toString() === "NaN") ) { - return controller.error( - "Size cannot exceed 64 Gibs", - ); + return controller.error(new Error("Size cannot exceed 64 Gibs")); } const pathname = parsePathname(chunk.pathname); @@ -334,14 +334,7 @@ export class TarStream implements TransformStream { } } -/** - * parsePathname is a function that validates the correctness of the pathname - * being provided. - * Function will throw if invalid pathname is provided. - * The result can be provided instead of the string version to TarStream, - * or can just be used to check in advance of creating the Tar archive. - */ -export function parsePathname( +function parsePathname( pathname: string, ): [Uint8Array, Uint8Array] { const name = new TextEncoder().encode(pathname); @@ -380,6 +373,39 @@ export function parsePathname( } return [prefix, name.slice(suitableSlashPos + 1)]; } + +/** + * The type that may be returned from the `validPathname` function. + */ +export type PathnameResult = { ok: true } | { ok: false; error: string }; + +/** + * validPathname is a function that validates the correctness of a pathname that + * may be piped to a `TarStream`. It provides a means to check that a pathname is + * valid before pipping it through the `TarStream`, where if invalid will throw an + * error. Ruining any progress made when archiving. + * + * @param pathname The pathname as a string + * @return { ok: true } | { ok: false, error: string } + * + * @example Usage + * ```ts + * import { validPathname, type PathnameResult } from "@std/archive"; + * import { assertEquals } from "@std/assert"; + * + * assertEquals(validPathname('MyFile.txt'), { ok: true }); + * ``` + */ +export function validPathname(pathname: string): PathnameResult { + try { + parsePathname(pathname); + return { ok: true }; + } catch (error) { + if (!(error instanceof Error)) throw error; + return { ok: false, error: error.message }; + } +} + /** * validTarStreamOptions is a function that returns a true if all of the options * provided are in the correct format, otherwise returns false. diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index 9365849ca8a1..8176fb35af34 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -1,12 +1,11 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. import { - parsePathname, TarStream, type TarStreamInput, validTarStreamOptions, } from "./tar_stream.ts"; -import { assertEquals } from "../assert/mod.ts"; -import { assert } from "../assert/assert.ts"; +import { assert, assertEquals, assertRejects } from "../assert/mod.ts"; +import { UnTarStream } from "./untar_stream.ts"; Deno.test("TarStream() with default stream", async () => { const text = new TextEncoder().encode("Hello World!"); @@ -76,15 +75,11 @@ Deno.test("TarStream() with negative size", async () => { ]) .pipeThrough(new TarStream()); - let threw = false; - try { - await Array.fromAsync(readable); - } catch (error) { - threw = true; - assert(typeof error === "string"); - assertEquals(error, "Size cannot exceed 64 Gibs"); - } - assertEquals(threw, true); + await assertRejects( + () => Array.fromAsync(readable), + Error, + "Size cannot exceed 64 Gibs", + ); }); Deno.test("TarStream() with 65 GiB size", async () => { @@ -105,15 +100,11 @@ Deno.test("TarStream() with 65 GiB size", async () => { ]) .pipeThrough(new TarStream()); - let threw = false; - try { - await Array.fromAsync(readable); - } catch (error) { - threw = true; - assert(typeof error === "string"); - assertEquals(error, "Size cannot exceed 64 Gibs"); - } - assertEquals(threw, true); + await assertRejects( + () => Array.fromAsync(readable), + Error, + "Size cannot exceed 64 Gibs", + ); }); Deno.test("TarStream() with NaN size", async () => { @@ -134,55 +125,45 @@ Deno.test("TarStream() with NaN size", async () => { ]) .pipeThrough(new TarStream()); - let threw = false; - try { - await Array.fromAsync(readable); - } catch (error) { - threw = true; - assert(typeof error === "string"); - assertEquals(error, "Size cannot exceed 64 Gibs"); - } - assertEquals(threw, true); -}); - -Deno.test("parsePathname()", () => { - const encoder = new TextEncoder(); - - assertEquals( - parsePathname( - "./Veeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeery/LongPath", - ), - [ - encoder.encode( - "./Veeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeery", - ), - encoder.encode("LongPath"), - ], - ); - - assertEquals( - parsePathname( - "./some random path/with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/path", - ), - [ - encoder.encode("./some random path"), - encoder.encode( - "with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/path", - ), - ], + await assertRejects( + () => Array.fromAsync(readable), + Error, + "Size cannot exceed 64 Gibs", ); +}); - assertEquals( - parsePathname( - "./some random path/with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/file", - ), - [ - encoder.encode("./some random path"), - encoder.encode( - "with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/file", - ), - ], - ); +Deno.test("parsePathname()", async () => { + const readable = ReadableStream.from([ + { + pathname: + "./Veeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeery/LongPath", + }, + { + pathname: + "./some random path/with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/path", + }, + { + pathname: + "./some random path/with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/file", + }, + { + pathname: + "./some random path/with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/file", + }, + ]) + .pipeThrough(new TarStream()) + .pipeThrough(new UnTarStream()); + + const output = [ + "./Veeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeery/LongPath", + "./some random path/with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/path", + "./some random path/with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/file", + "./some random path/with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/file", + ]; + for await (const tarChunk of readable) { + assert(tarChunk.type === "header"); + assertEquals(tarChunk.pathname, output.shift()); + } }); Deno.test("validTarStreamOptions()", () => { @@ -234,16 +215,11 @@ Deno.test("TarStream() with invalid options", async () => { { pathname: "potato", options: { mode: 9 } }, ]).pipeThrough(new TarStream()); - let threw = false; - try { - // deno-lint-ignore no-empty - for await (const _ of readable) {} - } catch (error) { - threw = true; - assert(typeof error === "string"); - assertEquals(error, "Invalid TarStreamOptions Provided"); - } - assertEquals(threw, true); + await assertRejects( + () => Array.fromAsync(readable), + Error, + "Invalid TarStreamOptions Provided", + ); }); Deno.test("TarStream() with mismatching sizes", async () => { @@ -256,47 +232,35 @@ Deno.test("TarStream() with mismatching sizes", async () => { }, ]).pipeThrough(new TarStream()); - let threw = false; - try { - // deno-lint-ignore no-empty - for await (const _ of readable) {} - } catch (error) { - threw = true; - assert(error instanceof RangeError); - assertEquals( - error.message, - "Provided size did not match bytes read from provided readable", - ); - } - assertEquals(threw, true); + await assertRejects( + () => Array.fromAsync(readable), + RangeError, + "Provided size did not match bytes read from provided readable", + ); }); -Deno.test("parsePathname() with too long path", () => { - let threw = false; - try { - parsePathname("0".repeat(300)); - } catch (error) { - threw = true; - assert(error instanceof RangeError); - assertEquals( - error.message, - "Pathname cannot exceed 256 bytes", - ); - } - assertEquals(threw, true); +Deno.test("parsePathname() with too long path", async () => { + const readable = ReadableStream.from([{ + pathname: "0".repeat(300), + }]) + .pipeThrough(new TarStream()); + + await assertRejects( + () => Array.fromAsync(readable), + RangeError, + "Pathname cannot exceed 256 bytes", + ); }); -Deno.test("parsePathname() with too long path", () => { - let threw = false; - try { - parsePathname("0".repeat(160) + "/"); - } catch (error) { - threw = true; - assert(error instanceof Error); - assertEquals( - error.message, - "Pathname needs to be split-able on a forward slash separator into [155, 100] bytes respectively", - ); - } - assertEquals(threw, true); +Deno.test("parsePathname() with too long path", async () => { + const readable = ReadableStream.from([{ + pathname: "0".repeat(160) + "/", + }]) + .pipeThrough(new TarStream()); + + await assertRejects( + () => Array.fromAsync(readable), + Error, + "Pathname needs to be split-able on a forward slash separator into [155, 100] bytes respectively", + ); }); diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index 9966eb36cc9d..8f75382eaaef 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -2,7 +2,7 @@ import { concat } from "../bytes/mod.ts"; import { TarStream, type TarStreamInput } from "./tar_stream.ts"; import { UnTarStream } from "./untar_stream.ts"; -import { assert, assertEquals } from "../assert/mod.ts"; +import { assertEquals, assertRejects } from "../assert/mod.ts"; Deno.test("expandTarArchiveCheckingHeaders", async () => { const text = new TextEncoder().encode("Hello World!"); @@ -99,16 +99,11 @@ Deno.test("UnTarStream() with invalid size", async () => { ) .pipeThrough(new UnTarStream()); - let threw = false; - try { - // deno-lint-ignore no-empty - for await (const _ of readable) {} - } catch (error) { - threw = true; - assert(error instanceof Error); - assertEquals(error.message, "Tarball has an unexpected number of bytes"); - } - assertEquals(threw, true); + await assertRejects( + () => Array.fromAsync(readable), + Error, + "Tarball has an unexpected number of bytes", + ); }); Deno.test("UnTarStream() with invalid ending", async () => { @@ -129,35 +124,22 @@ Deno.test("UnTarStream() with invalid ending", async () => { const readable = ReadableStream.from([tarBytes]) .pipeThrough(new UnTarStream()); - let threw = false; - try { - // deno-lint-ignore no-empty - for await (const _ of readable) {} - } catch (error) { - threw = true; - assert(error instanceof Error); - assertEquals( - error.message, - "Tarball has invalid ending", - ); - } - assertEquals(threw, true); + await assertRejects( + () => Array.fromAsync(readable), + Error, + "Tarball has invalid ending", + ); }); Deno.test("UnTarStream() with too small size", async () => { const readable = ReadableStream.from([new Uint8Array(512)]) .pipeThrough(new UnTarStream()); - let threw = false; - try { - // deno-lint-ignore no-empty - for await (const _ of readable) {} - } catch (error) { - threw = true; - assert(error instanceof Error); - assertEquals(error.message, "Tarball was too small to be valid"); - } - assertEquals(threw, true); + await assertRejects( + () => Array.fromAsync(readable), + Error, + "Tarball was too small to be valid", + ); }); Deno.test("UnTarStream() with invalid checksum", async () => { @@ -178,17 +160,9 @@ Deno.test("UnTarStream() with invalid checksum", async () => { const readable = ReadableStream.from([tarBytes]) .pipeThrough(new UnTarStream()); - let threw = false; - try { - // deno-lint-ignore no-empty - for await (const _ of readable) {} - } catch (error) { - threw = true; - assert(error instanceof SyntaxError); - assertEquals( - error.message, - "Tarball header failed to pass checksum", - ); - } - assertEquals(threw, true); + await assertRejects( + () => Array.fromAsync(readable), + Error, + "Tarball header failed to pass checksum", + ); }); From a163b456a063a2f7e19840a085046d5d6f27e9cc Mon Sep 17 00:00:00 2001 From: Doctor <44320105+BlackAsLight@users.noreply.github.com> Date: Thu, 22 Aug 2024 19:06:03 +1000 Subject: [PATCH 57/82] adjust(archive): to use `Date.now()` instead of `new Date().getTime()` Co-authored-by: ud2 --- archive/tar_stream.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index dd6a36d23403..114f7ccda17f 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -183,7 +183,7 @@ export class TarStream implements TransformStream { mode: typeflag === "5" ? 755 : 644, uid: 0, gid: 0, - mtime: Math.floor(new Date().getTime() / 1000), + mtime: Math.floor(Date.now() / 1000), uname: "", gname: "", devmajor: "", From d926d69b459685bc5bfc67a4216055ea670bec1b Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Mon, 26 Aug 2024 12:44:31 +1000 Subject: [PATCH 58/82] adjust(archive): mode, uid, and gid to be numbers instead of strings when Untaring --- archive/untar_stream.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index eeffda12665c..d7e04dd69728 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -12,15 +12,15 @@ export interface OldStyleFormat { /** * The mode of the entry. */ - mode: string; + mode: number; /** * The uid of the entry. */ - uid: string; + uid: number; /** * The gid of the entry. */ - gid: string; + gid: number; /** * The size of the entry. */ @@ -236,9 +236,9 @@ export class UnTarStream // Decode Header let header: OldStyleFormat | PosixUstarFormat = { name: decoder.decode(value.subarray(0, 100)).split("\0")[0]!, - mode: decoder.decode(value.subarray(100, 108 - 2)), - uid: decoder.decode(value.subarray(108, 116 - 2)), - gid: decoder.decode(value.subarray(116, 124 - 2)), + mode: parseInt(decoder.decode(value.subarray(100, 108 - 2))), + uid: parseInt(decoder.decode(value.subarray(108, 116 - 2))), + gid: parseInt(decoder.decode(value.subarray(116, 124 - 2))), size: parseInt(decoder.decode(value.subarray(124, 136)).trimEnd(), 8), mtime: parseInt(decoder.decode(value.subarray(136, 148 - 1)), 8), typeflag: decoder.decode(value.subarray(156, 157)), From e8e862cee302f65b257bf2e8c80e7df4298966b9 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Mon, 26 Aug 2024 12:53:54 +1000 Subject: [PATCH 59/82] tests(archive): adjust two tests to also validate the contents of the files are valid --- archive/tar_stream_test.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index 8176fb35af34..ce0171dad72a 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -6,6 +6,7 @@ import { } from "./tar_stream.ts"; import { assert, assertEquals, assertRejects } from "../assert/mod.ts"; import { UnTarStream } from "./untar_stream.ts"; +import { concat } from "../bytes/mod.ts"; Deno.test("TarStream() with default stream", async () => { const text = new TextEncoder().encode("Hello World!"); @@ -24,14 +25,25 @@ Deno.test("TarStream() with default stream", async () => { .getReader(); let size = 0; + const data: Uint8Array[] = []; while (true) { const { done, value } = await reader.read(); if (done) { break; } size += value.length; + data.push(value); } assertEquals(size, 512 + 512 + Math.ceil(text.length / 512) * 512 + 1024); + assertEquals( + text, + concat(data).slice( + 512 + // Slicing off ./potato header + 512, // Slicing off ./text.txt header + -1024, // Slicing off 1024 bytes of end padding + ) + .slice(0, text.length), // Slice off padding added to text to make it divisible by 512 + ); }); Deno.test("TarStream() with byte stream", async () => { @@ -51,6 +63,7 @@ Deno.test("TarStream() with byte stream", async () => { .getReader({ mode: "byob" }); let size = 0; + const data: Uint8Array[] = []; while (true) { const { done, value } = await reader.read( new Uint8Array(Math.ceil(Math.random() * 1024)), @@ -59,8 +72,18 @@ Deno.test("TarStream() with byte stream", async () => { break; } size += value.length; + data.push(value); } assertEquals(size, 512 + 512 + Math.ceil(text.length / 512) * 512 + 1024); + assertEquals( + text, + concat(data).slice( + 512 + // Slicing off ./potato header + 512, // Slicing off ./text.txt header + -1024, // Slicing off 1024 bytes of end padding + ) + .slice(0, text.length), // Slice off padding added to text to make it divisible by 512 + ); }); Deno.test("TarStream() with negative size", async () => { From 32577cfcc008c334f5147a8ac83ad7148403970b Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:00:45 +1000 Subject: [PATCH 60/82] adjust(archive): linkname, uname, and gname to follow the same decoding rules as name and prefix --- archive/untar_stream.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index d7e04dd69728..6355b9d70d8c 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -242,10 +242,7 @@ export class UnTarStream size: parseInt(decoder.decode(value.subarray(124, 136)).trimEnd(), 8), mtime: parseInt(decoder.decode(value.subarray(136, 148 - 1)), 8), typeflag: decoder.decode(value.subarray(156, 157)), - linkname: decoder.decode(value.subarray(157, 257)).replaceAll( - "\0", - "", - ), + linkname: decoder.decode(value.subarray(157, 257)).split("\0")[0]!, }; if (header.typeflag === "\0") header.typeflag = "0"; // "ustar\u000000" @@ -258,8 +255,8 @@ export class UnTarStream ...header, magic: decoder.decode(value.subarray(257, 263)), version: decoder.decode(value.subarray(263, 265)), - uname: decoder.decode(value.subarray(265, 297)).replaceAll("\0", ""), - gname: decoder.decode(value.subarray(297, 329)).replaceAll("\0", ""), + uname: decoder.decode(value.subarray(265, 297)).split("\0")[0]!, + gname: decoder.decode(value.subarray(297, 329)).split("\0")[0]!, devmajor: decoder.decode(value.subarray(329, 337)).replaceAll( "\0", "", From 49a4a2d59827d81891ef4adb7d9001c270f420ce Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:04:43 +1000 Subject: [PATCH 61/82] rename(archive): UnTarStream to UntarStream --- archive/tar_stream_test.ts | 4 ++-- archive/untar_stream.ts | 14 +++++++------- archive/untar_stream_test.ts | 26 +++++++++++++------------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index ce0171dad72a..98d3d691478c 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -5,7 +5,7 @@ import { validTarStreamOptions, } from "./tar_stream.ts"; import { assert, assertEquals, assertRejects } from "../assert/mod.ts"; -import { UnTarStream } from "./untar_stream.ts"; +import { UntarStream } from "./untar_stream.ts"; import { concat } from "../bytes/mod.ts"; Deno.test("TarStream() with default stream", async () => { @@ -175,7 +175,7 @@ Deno.test("parsePathname()", async () => { }, ]) .pipeThrough(new TarStream()) - .pipeThrough(new UnTarStream()); + .pipeThrough(new UntarStream()); const output = [ "./Veeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeery/LongPath", diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 6355b9d70d8c..fccefba561c9 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -167,14 +167,14 @@ export type TarStreamChunk = TarStreamHeader | TarStreamData; * * @example Usage * ```ts no-eval - * import { UnTarStream } from "@std/archive/untar-stream"; + * import { UntarStream } from "@std/archive/untar-stream"; * * let fileWriter: WritableStreamDefaultWriter | undefined; * for await ( * const entry of (await Deno.open('./out.tar.gz')) * .readable * .pipeThrough(new DecompressionStream('gzip')) - * .pipeThrough(new UnTarStream()) + * .pipeThrough(new UntarStream()) * ) { * if (entry.type === "header") { * fileWriter?.close(); @@ -183,7 +183,7 @@ export type TarStreamChunk = TarStreamHeader | TarStreamData; * } * ``` */ -export class UnTarStream +export class UntarStream implements TransformStream { #readable: ReadableStream; #writable: WritableStream; @@ -300,14 +300,14 @@ export class UnTarStream * * @example Usage * ```ts no-eval - * import { UnTarStream } from "@std/archive/untar-stream"; + * import { UntarStream } from "@std/archive/untar-stream"; * * let fileWriter: WritableStreamDefaultWriter | undefined; * for await ( * const entry of (await Deno.open('./out.tar.gz')) * .readable * .pipeThrough(new DecompressionStream('gzip')) - * .pipeThrough(new UnTarStream()) + * .pipeThrough(new UntarStream()) * ) { * if (entry.type === "header") { * fileWriter?.close(); @@ -327,14 +327,14 @@ export class UnTarStream * * @example Usage * ```ts no-eval - * import { UnTarStream } from "@std/archive/untar-stream"; + * import { UntarStream } from "@std/archive/untar-stream"; * * let fileWriter: WritableStreamDefaultWriter | undefined; * for await ( * const entry of (await Deno.open('./out.tar.gz')) * .readable * .pipeThrough(new DecompressionStream('gzip')) - * .pipeThrough(new UnTarStream()) + * .pipeThrough(new UntarStream()) * ) { * if (entry.type === "header") { * fileWriter?.close(); diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index 8f75382eaaef..0dc597d8c183 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -1,7 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. import { concat } from "../bytes/mod.ts"; import { TarStream, type TarStreamInput } from "./tar_stream.ts"; -import { UnTarStream } from "./untar_stream.ts"; +import { UntarStream } from "./untar_stream.ts"; import { assertEquals, assertRejects } from "../assert/mod.ts"; Deno.test("expandTarArchiveCheckingHeaders", async () => { @@ -18,7 +18,7 @@ Deno.test("expandTarArchiveCheckingHeaders", async () => { }, ]) .pipeThrough(new TarStream()) - .pipeThrough(new UnTarStream()); + .pipeThrough(new UntarStream()); const pathnames: string[] = []; for await (const item of readable) { @@ -41,7 +41,7 @@ Deno.test("expandTarArchiveCheckingBodies", async () => { }, ]) .pipeThrough(new TarStream()) - .pipeThrough(new UnTarStream()); + .pipeThrough(new UntarStream()); const buffer = new Uint8Array(text.length); let offset = 0; @@ -54,7 +54,7 @@ Deno.test("expandTarArchiveCheckingBodies", async () => { assertEquals(buffer, text); }); -Deno.test("UnTarStream() with size equals to multiple of 512", async () => { +Deno.test("UntarStream() with size equals to multiple of 512", async () => { const size = 512 * 3; const data = Uint8Array.from( { length: size }, @@ -69,7 +69,7 @@ Deno.test("UnTarStream() with size equals to multiple of 512", async () => { }, ]) .pipeThrough(new TarStream()) - .pipeThrough(new UnTarStream()); + .pipeThrough(new UntarStream()); assertEquals( concat( @@ -81,7 +81,7 @@ Deno.test("UnTarStream() with size equals to multiple of 512", async () => { ); }); -Deno.test("UnTarStream() with invalid size", async () => { +Deno.test("UntarStream() with invalid size", async () => { const readable = ReadableStream.from([ { pathname: "newFile.txt", @@ -97,7 +97,7 @@ Deno.test("UnTarStream() with invalid size", async () => { }, }), ) - .pipeThrough(new UnTarStream()); + .pipeThrough(new UntarStream()); await assertRejects( () => Array.fromAsync(readable), @@ -106,7 +106,7 @@ Deno.test("UnTarStream() with invalid size", async () => { ); }); -Deno.test("UnTarStream() with invalid ending", async () => { +Deno.test("UntarStream() with invalid ending", async () => { const tarBytes = concat( await Array.fromAsync( ReadableStream.from([ @@ -122,7 +122,7 @@ Deno.test("UnTarStream() with invalid ending", async () => { tarBytes[tarBytes.length - 1] = 1; const readable = ReadableStream.from([tarBytes]) - .pipeThrough(new UnTarStream()); + .pipeThrough(new UntarStream()); await assertRejects( () => Array.fromAsync(readable), @@ -131,9 +131,9 @@ Deno.test("UnTarStream() with invalid ending", async () => { ); }); -Deno.test("UnTarStream() with too small size", async () => { +Deno.test("UntarStream() with too small size", async () => { const readable = ReadableStream.from([new Uint8Array(512)]) - .pipeThrough(new UnTarStream()); + .pipeThrough(new UntarStream()); await assertRejects( () => Array.fromAsync(readable), @@ -142,7 +142,7 @@ Deno.test("UnTarStream() with too small size", async () => { ); }); -Deno.test("UnTarStream() with invalid checksum", async () => { +Deno.test("UntarStream() with invalid checksum", async () => { const tarBytes = concat( await Array.fromAsync( ReadableStream.from([ @@ -158,7 +158,7 @@ Deno.test("UnTarStream() with invalid checksum", async () => { tarBytes[148] = 97; const readable = ReadableStream.from([tarBytes]) - .pipeThrough(new UnTarStream()); + .pipeThrough(new UntarStream()); await assertRejects( () => Array.fromAsync(readable), From 89a833ad7970b70c6ef866cd03446aaae3cc84a0 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:41:51 +1000 Subject: [PATCH 62/82] fix(archive): type that was missed getting updated --- archive/untar_stream.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index fccefba561c9..41a9da6eeb51 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -50,15 +50,15 @@ export interface PosixUstarFormat { /** * The mode of the entry. */ - mode: string; + mode: number; /** * The uid of the entry. */ - uid: string; + uid: number; /** * The gid of the entry. */ - gid: string; + gid: number; /** * The size of the entry. */ From 3f113c3c3508401b3233659111626684a70ba273 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:42:46 +1000 Subject: [PATCH 63/82] tests(archive): adjust check headers test to validate all header properties instead of just pathnames --- archive/tar_stream.ts | 12 ++++---- archive/tar_stream_test.ts | 10 +++---- archive/untar_stream_test.ts | 58 ++++++++++++++++++++++++++++++++---- 3 files changed, 63 insertions(+), 17 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 114f7ccda17f..b0d40473682f 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -205,8 +205,8 @@ export class TarStream implements TransformStream { "\0".repeat(100) + // linkname "ustar\0" + // magic "00" + // version - options.uname.padStart(32, "\0") + // uname - options.gname.padStart(32, "\0") + // gname + options.uname.padEnd(32, "\0") + // uname + options.gname.padEnd(32, "\0") + // gname options.devmajor.padStart(8, "\0") + // devmajor options.devminor.padStart(8, "\0"), // devminor ), @@ -447,20 +447,20 @@ export function validTarStreamOptions( if ( options.uname && // deno-lint-ignore no-control-regex - (options.uname.length > 32 || !/^[\x00-\x7F]*$/.test(options.uname)) + (options.uname.length > 32 - 1 || !/^[\x00-\x7F]*$/.test(options.uname)) ) return false; if ( options.gname && // deno-lint-ignore no-control-regex - (options.gname.length > 32 || !/^[\x00-\x7F]*$/.test(options.gname)) + (options.gname.length > 32 - 1 || !/^[\x00-\x7F]*$/.test(options.gname)) ) return false; if ( options.devmajor && - (options.devmajor.length > 8 || !/^[0-7]*$/.test(options.devmajor)) + (options.devmajor.length > 8) ) return false; if ( options.devminor && - (options.devminor.length > 8 || !/^[0-7]*$/.test(options.devminor)) + (options.devminor.length > 8) ) return false; return true; } diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index 98d3d691478c..fe8999469a74 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -223,14 +223,12 @@ Deno.test("validTarStreamOptions()", () => { assertEquals(validTarStreamOptions({ gname: "a".repeat(100) }), false); assertEquals(validTarStreamOptions({ devmajor: "" }), true); - assertEquals(validTarStreamOptions({ devmajor: "000" }), true); - assertEquals(validTarStreamOptions({ devmajor: "008" }), false); - assertEquals(validTarStreamOptions({ devmajor: "000000000" }), false); + assertEquals(validTarStreamOptions({ devmajor: "1234" }), true); + assertEquals(validTarStreamOptions({ devmajor: "123456789" }), false); assertEquals(validTarStreamOptions({ devminor: "" }), true); - assertEquals(validTarStreamOptions({ devminor: "000" }), true); - assertEquals(validTarStreamOptions({ devminor: "008" }), false); - assertEquals(validTarStreamOptions({ devminor: "000000000" }), false); + assertEquals(validTarStreamOptions({ devminor: "1234" }), true); + assertEquals(validTarStreamOptions({ devminor: "123456789" }), false); }); Deno.test("TarStream() with invalid options", async () => { diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index 0dc597d8c183..37bddb29aa3a 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -1,30 +1,78 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. import { concat } from "../bytes/mod.ts"; import { TarStream, type TarStreamInput } from "./tar_stream.ts"; -import { UntarStream } from "./untar_stream.ts"; +import { + type OldStyleFormat, + type PosixUstarFormat, + UntarStream, +} from "./untar_stream.ts"; import { assertEquals, assertRejects } from "../assert/mod.ts"; Deno.test("expandTarArchiveCheckingHeaders", async () => { const text = new TextEncoder().encode("Hello World!"); + const seconds = Math.floor(Date.now() / 1000); const readable = ReadableStream.from([ { pathname: "./potato", + options: { + mode: 111111, + uid: 12, + gid: 21, + mtime: seconds, + uname: "potato", + gname: "cake", + devmajor: "ice", + devminor: "scream", + }, }, { pathname: "./text.txt", size: text.length, - readable: ReadableStream.from([text]), + readable: ReadableStream.from([text.slice()]), + options: { mtime: seconds }, }, ]) .pipeThrough(new TarStream()) .pipeThrough(new UntarStream()); - const pathnames: string[] = []; + const headers: (OldStyleFormat | PosixUstarFormat)[] = []; for await (const item of readable) { - if (item.type === "header") pathnames.push(item.pathname); + if (item.type === "header") headers.push(item.header); } - assertEquals(pathnames, ["./potato", "./text.txt"]); + assertEquals(headers, [{ + name: "./potato", + mode: 111111, + uid: 12, + gid: 21, + mtime: seconds, + uname: "potato", + gname: "cake", + devmajor: "ice", + devminor: "scream", + size: 0, + typeflag: "5", + linkname: "", + magic: "ustar\0", + version: "00", + prefix: "", + }, { + name: "./text.txt", + mode: 644, + uid: 0, + gid: 0, + mtime: seconds, + uname: "", + gname: "", + devmajor: "", + devminor: "", + size: text.length, + typeflag: "0", + linkname: "", + magic: "ustar\0", + version: "00", + prefix: "", + }]); }); Deno.test("expandTarArchiveCheckingBodies", async () => { From a575800c989b4618fc5786576c92544ea769fbfe Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:52:46 +1000 Subject: [PATCH 64/82] rename(archive): `pathname` properties to `path` --- archive/tar_stream.ts | 64 ++++++++++++++++++------------------ archive/tar_stream_test.ts | 42 +++++++++++------------ archive/untar_stream.ts | 12 +++---- archive/untar_stream_test.ts | 16 ++++----- 4 files changed, 67 insertions(+), 67 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index b0d40473682f..7a5ba829cdde 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -7,7 +7,7 @@ export interface TarStreamFile { /** * The name of the file. */ - pathname: string; + path: string; /** * The size of the file. */ @@ -29,7 +29,7 @@ export interface TarStreamDir { /** * The name of the directory. */ - pathname: string; + path: string; /** * The metadata of the directory. */ @@ -42,8 +42,8 @@ export interface TarStreamDir { */ export type TarStreamInput = TarStreamFile | TarStreamDir; type TarStreamInputInternal = - & (Omit | Omit) - & { pathname: [Uint8Array, Uint8Array] }; + & (Omit | Omit) + & { path: [Uint8Array, Uint8Array] }; /** * The options that can go along with a file or directory. @@ -104,14 +104,14 @@ const SLASH_CODE_POINT = "/".charCodeAt(0); * The ustar file format is used for creating the tar archive. While this * format is compatible with most tar readers, the format has several * limitations, including: - * - Pathnames must be at most 256 characters. + * - Paths must be at most 256 characters. * - Files must be at most 8 GiBs in size, or 64 GiBs if `sizeExtension` is set * to true. * - Sparse files are not supported. * * ### Usage * TarStream may throw an error for several reasons. A few of those are: - * - The pathname is invalid. + * - The path is invalid. * - The size provided does not match that of the iterable's length. * * ### Compression @@ -124,15 +124,15 @@ const SLASH_CODE_POINT = "/".charCodeAt(0); * * await ReadableStream.from([ * { - * pathname: 'potato/' + * path: 'potato/' * }, * { - * pathname: 'deno.json', + * path: 'deno.json', * size: (await Deno.stat('deno.json')).size, * iterable: (await Deno.open('deno.json')).readable * }, * { - * pathname: 'deno.lock', + * path: 'deno.lock', * size: (await Deno.stat('deno.lock')).size, * iterable: (await Deno.open('deno.lock')).readable * } @@ -166,16 +166,16 @@ export class TarStream implements TransformStream { return controller.error(new Error("Size cannot exceed 64 Gibs")); } - const pathname = parsePathname(chunk.pathname); + const path = parsePath(chunk.path); - controller.enqueue({ ...chunk, pathname }); + controller.enqueue({ ...chunk, path }); }, }); this.#writable = writable; const gen = async function* () { const encoder = new TextEncoder(); for await (const chunk of readable) { - const [prefix, name] = chunk.pathname; + const [prefix, name] = chunk.path; const typeflag = "size" in chunk ? "0" : "5"; const header = new Uint8Array(512); const size = "size" in chunk ? chunk.size : 0; @@ -278,15 +278,15 @@ export class TarStream implements TransformStream { * * await ReadableStream.from([ * { - * pathname: 'potato/' + * path: 'potato/' * }, * { - * pathname: 'deno.json', + * path: 'deno.json', * size: (await Deno.stat('deno.json')).size, * iterable: (await Deno.open('deno.json')).readable * }, * { - * pathname: 'deno.lock', + * path: 'deno.lock', * size: (await Deno.stat('deno.lock')).size, * iterable: (await Deno.open('deno.lock')).readable * } @@ -311,15 +311,15 @@ export class TarStream implements TransformStream { * * await ReadableStream.from([ * { - * pathname: 'potato/' + * path: 'potato/' * }, * { - * pathname: 'deno.json', + * path: 'deno.json', * size: (await Deno.stat('deno.json')).size, * iterable: (await Deno.open('deno.json')).readable * }, * { - * pathname: 'deno.lock', + * path: 'deno.lock', * size: (await Deno.stat('deno.lock')).size, * iterable: (await Deno.open('deno.lock')).readable * } @@ -334,16 +334,16 @@ export class TarStream implements TransformStream { } } -function parsePathname( - pathname: string, +function parsePath( + path: string, ): [Uint8Array, Uint8Array] { - const name = new TextEncoder().encode(pathname); + const name = new TextEncoder().encode(path); if (name.length <= 100) { return [new Uint8Array(0), name]; } if (name.length > 256) { - throw new RangeError("Pathname cannot exceed 256 bytes"); + throw new RangeError("Path cannot exceed 256 bytes"); } // If length of last part is > 100, then there's no possible answer to split the path @@ -368,37 +368,37 @@ function parsePathname( const prefix = name.slice(0, suitableSlashPos); if (prefix.length > 155) { throw new Error( - "Pathname needs to be split-able on a forward slash separator into [155, 100] bytes respectively", + "Path needs to be split-able on a forward slash separator into [155, 100] bytes respectively", ); } return [prefix, name.slice(suitableSlashPos + 1)]; } /** - * The type that may be returned from the `validPathname` function. + * The type that may be returned from the `validPath` function. */ -export type PathnameResult = { ok: true } | { ok: false; error: string }; +export type PathResult = { ok: true } | { ok: false; error: string }; /** - * validPathname is a function that validates the correctness of a pathname that - * may be piped to a `TarStream`. It provides a means to check that a pathname is + * validPath is a function that validates the correctness of a path that + * may be piped to a `TarStream`. It provides a means to check that a path is * valid before pipping it through the `TarStream`, where if invalid will throw an * error. Ruining any progress made when archiving. * - * @param pathname The pathname as a string + * @param path The path as a string * @return { ok: true } | { ok: false, error: string } * * @example Usage * ```ts - * import { validPathname, type PathnameResult } from "@std/archive"; + * import { validPath, type PathResult } from "@std/archive"; * import { assertEquals } from "@std/assert"; * - * assertEquals(validPathname('MyFile.txt'), { ok: true }); + * assertEquals(validPath('MyFile.txt'), { ok: true }); * ``` */ -export function validPathname(pathname: string): PathnameResult { +export function validPath(path: string): PathResult { try { - parsePathname(pathname); + parsePath(path); return { ok: true }; } catch (error) { if (!(error instanceof Error)) throw error; diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index fe8999469a74..6e8e8e5fc00b 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -13,10 +13,10 @@ Deno.test("TarStream() with default stream", async () => { const reader = ReadableStream.from([ { - pathname: "./potato", + path: "./potato", }, { - pathname: "./text.txt", + path: "./text.txt", size: text.length, readable: ReadableStream.from([text.slice()]), }, @@ -51,10 +51,10 @@ Deno.test("TarStream() with byte stream", async () => { const reader = ReadableStream.from([ { - pathname: "./potato", + path: "./potato", }, { - pathname: "./text.txt", + path: "./text.txt", size: text.length, readable: ReadableStream.from([text.slice()]), }, @@ -91,7 +91,7 @@ Deno.test("TarStream() with negative size", async () => { const readable = ReadableStream.from([ { - pathname: "name", + path: "name", size: -text.length, readable: ReadableStream.from([text.slice()]), }, @@ -116,7 +116,7 @@ Deno.test("TarStream() with 65 GiB size", async () => { const readable = ReadableStream.from([ { - pathname: "name", + path: "name", size, readable: ReadableStream.from(iterable), }, @@ -141,7 +141,7 @@ Deno.test("TarStream() with NaN size", async () => { const readable = ReadableStream.from([ { - pathname: "name", + path: "name", size, readable: ReadableStream.from(iterable), }, @@ -155,22 +155,22 @@ Deno.test("TarStream() with NaN size", async () => { ); }); -Deno.test("parsePathname()", async () => { +Deno.test("parsePath()", async () => { const readable = ReadableStream.from([ { - pathname: + path: "./Veeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeery/LongPath", }, { - pathname: + path: "./some random path/with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/path", }, { - pathname: + path: "./some random path/with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/file", }, { - pathname: + path: "./some random path/with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/file", }, ]) @@ -185,7 +185,7 @@ Deno.test("parsePathname()", async () => { ]; for await (const tarChunk of readable) { assert(tarChunk.type === "header"); - assertEquals(tarChunk.pathname, output.shift()); + assertEquals(tarChunk.path, output.shift()); } }); @@ -233,7 +233,7 @@ Deno.test("validTarStreamOptions()", () => { Deno.test("TarStream() with invalid options", async () => { const readable = ReadableStream.from([ - { pathname: "potato", options: { mode: 9 } }, + { path: "potato", options: { mode: 9 } }, ]).pipeThrough(new TarStream()); await assertRejects( @@ -247,7 +247,7 @@ Deno.test("TarStream() with mismatching sizes", async () => { const text = new TextEncoder().encode("Hello World!"); const readable = ReadableStream.from([ { - pathname: "potato", + path: "potato", size: text.length + 1, readable: ReadableStream.from([text.slice()]), }, @@ -260,28 +260,28 @@ Deno.test("TarStream() with mismatching sizes", async () => { ); }); -Deno.test("parsePathname() with too long path", async () => { +Deno.test("parsePath() with too long path", async () => { const readable = ReadableStream.from([{ - pathname: "0".repeat(300), + path: "0".repeat(300), }]) .pipeThrough(new TarStream()); await assertRejects( () => Array.fromAsync(readable), RangeError, - "Pathname cannot exceed 256 bytes", + "Path cannot exceed 256 bytes", ); }); -Deno.test("parsePathname() with too long path", async () => { +Deno.test("parsePath() with too long path", async () => { const readable = ReadableStream.from([{ - pathname: "0".repeat(160) + "/", + path: "0".repeat(160) + "/", }]) .pipeThrough(new TarStream()); await assertRejects( () => Array.fromAsync(readable), Error, - "Pathname needs to be split-able on a forward slash separator into [155, 100] bytes respectively", + "Path needs to be split-able on a forward slash separator into [155, 100] bytes respectively", ); }); diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 41a9da6eeb51..410e7bfbadda 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -114,9 +114,9 @@ export interface TarStreamHeader { */ type: "header"; /** - * The pathname of the entry. + * The path of the entry. */ - pathname: string; + path: string; /** * The header of the entry. */ @@ -178,7 +178,7 @@ export type TarStreamChunk = TarStreamHeader | TarStreamData; * ) { * if (entry.type === "header") { * fileWriter?.close(); - * fileWriter = (await Deno.create(entry.pathname)).writable.getWriter(); + * fileWriter = (await Deno.create(entry.path)).writable.getWriter(); * } else await fileWriter!.write(entry.data); * } * ``` @@ -271,7 +271,7 @@ export class UntarStream yield { type: "header", - pathname: ("prefix" in header && header.prefix.length + path: ("prefix" in header && header.prefix.length ? header.prefix + "/" : "") + header.name, header, @@ -311,7 +311,7 @@ export class UntarStream * ) { * if (entry.type === "header") { * fileWriter?.close(); - * fileWriter = (await Deno.create(entry.pathname)).writable.getWriter(); + * fileWriter = (await Deno.create(entry.path)).writable.getWriter(); * } else await fileWriter!.write(entry.data); * } * ``` @@ -338,7 +338,7 @@ export class UntarStream * ) { * if (entry.type === "header") { * fileWriter?.close(); - * fileWriter = (await Deno.create(entry.pathname)).writable.getWriter(); + * fileWriter = (await Deno.create(entry.path)).writable.getWriter(); * } else await fileWriter!.write(entry.data); * } * ``` diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index 37bddb29aa3a..d210d6143c7d 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -14,7 +14,7 @@ Deno.test("expandTarArchiveCheckingHeaders", async () => { const readable = ReadableStream.from([ { - pathname: "./potato", + path: "./potato", options: { mode: 111111, uid: 12, @@ -27,7 +27,7 @@ Deno.test("expandTarArchiveCheckingHeaders", async () => { }, }, { - pathname: "./text.txt", + path: "./text.txt", size: text.length, readable: ReadableStream.from([text.slice()]), options: { mtime: seconds }, @@ -80,10 +80,10 @@ Deno.test("expandTarArchiveCheckingBodies", async () => { const readable = ReadableStream.from([ { - pathname: "./potato", + path: "./potato", }, { - pathname: "./text.txt", + path: "./text.txt", size: text.length, readable: ReadableStream.from([text.slice()]), }, @@ -111,7 +111,7 @@ Deno.test("UntarStream() with size equals to multiple of 512", async () => { const readable = ReadableStream.from([ { - pathname: "name", + path: "name", size, readable: ReadableStream.from([data.slice()]), }, @@ -132,7 +132,7 @@ Deno.test("UntarStream() with size equals to multiple of 512", async () => { Deno.test("UntarStream() with invalid size", async () => { const readable = ReadableStream.from([ { - pathname: "newFile.txt", + path: "newFile.txt", size: 512, readable: ReadableStream.from([new Uint8Array(512).fill(97)]), }, @@ -159,7 +159,7 @@ Deno.test("UntarStream() with invalid ending", async () => { await Array.fromAsync( ReadableStream.from([ { - pathname: "newFile.txt", + path: "newFile.txt", size: 512, readable: ReadableStream.from([new Uint8Array(512).fill(97)]), }, @@ -195,7 +195,7 @@ Deno.test("UntarStream() with invalid checksum", async () => { await Array.fromAsync( ReadableStream.from([ { - pathname: "newFile.txt", + path: "newFile.txt", size: 512, readable: ReadableStream.from([new Uint8Array(512).fill(97)]), }, From 3b203634be5dffc30c2c26f6c9739b8afd81d9c9 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:59:11 +1000 Subject: [PATCH 65/82] docs(archive): updated to be more descriptive --- archive/tar_stream.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 7a5ba829cdde..378f285ad634 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -5,11 +5,11 @@ */ export interface TarStreamFile { /** - * The name of the file. + * The path to the file, relative to the archive's root directory. */ path: string; /** - * The size of the file. + * The size of the file in bytes. */ size: number; /** @@ -27,7 +27,7 @@ export interface TarStreamFile { */ export interface TarStreamDir { /** - * The name of the directory. + * The path of the directory, relative to the archive's root directory. */ path: string; /** @@ -132,9 +132,9 @@ const SLASH_CODE_POINT = "/".charCodeAt(0); * iterable: (await Deno.open('deno.json')).readable * }, * { - * path: 'deno.lock', - * size: (await Deno.stat('deno.lock')).size, - * iterable: (await Deno.open('deno.lock')).readable + * path: '.vscode/settings.json', + * size: (await Deno.stat('.vscode/settings.json')).size, + * iterable: (await Deno.open('.vscode/settings.json')).readable * } * ]) * .pipeThrough(new TarStream()) @@ -286,9 +286,9 @@ export class TarStream implements TransformStream { * iterable: (await Deno.open('deno.json')).readable * }, * { - * path: 'deno.lock', - * size: (await Deno.stat('deno.lock')).size, - * iterable: (await Deno.open('deno.lock')).readable + * path: '.vscode/settings.json', + * size: (await Deno.stat('.vscode/settings.json')).size, + * iterable: (await Deno.open('.vscode/settings.json')).readable * } * ]) * .pipeThrough(new TarStream()) @@ -319,9 +319,9 @@ export class TarStream implements TransformStream { * iterable: (await Deno.open('deno.json')).readable * }, * { - * path: 'deno.lock', - * size: (await Deno.stat('deno.lock')).size, - * iterable: (await Deno.open('deno.lock')).readable + * path: '.vscode/settings.json', + * size: (await Deno.stat('.vscode/settings.json')).size, + * iterable: (await Deno.open('.vscode/settings.json')).readable * } * ]) * .pipeThrough(new TarStream()) From 51071747026e3a382afc1ae51ea4b21cb4966663 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Thu, 29 Aug 2024 18:57:18 +1000 Subject: [PATCH 66/82] docs(archive): Updated error types --- archive/tar_stream.ts | 4 ++-- archive/tar_stream_test.ts | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 378f285ad634..848d9451984d 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -154,7 +154,7 @@ export class TarStream implements TransformStream { transform(chunk, controller) { if (chunk.options && !validTarStreamOptions(chunk.options)) { return controller.error( - new Error("Invalid TarStreamOptions Provided"), + new TypeError("Invalid TarStreamOptions Provided"), ); } @@ -163,7 +163,7 @@ export class TarStream implements TransformStream { (chunk.size < 0 || 8 ** 12 < chunk.size || chunk.size.toString() === "NaN") ) { - return controller.error(new Error("Size cannot exceed 64 Gibs")); + return controller.error(new RangeError("Size cannot exceed 64 Gibs")); } const path = parsePath(chunk.path); diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index 6e8e8e5fc00b..f3620c2cb28a 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -100,7 +100,7 @@ Deno.test("TarStream() with negative size", async () => { await assertRejects( () => Array.fromAsync(readable), - Error, + RangeError, "Size cannot exceed 64 Gibs", ); }); @@ -125,7 +125,7 @@ Deno.test("TarStream() with 65 GiB size", async () => { await assertRejects( () => Array.fromAsync(readable), - Error, + RangeError, "Size cannot exceed 64 Gibs", ); }); @@ -150,7 +150,7 @@ Deno.test("TarStream() with NaN size", async () => { await assertRejects( () => Array.fromAsync(readable), - Error, + RangeError, "Size cannot exceed 64 Gibs", ); }); @@ -238,7 +238,7 @@ Deno.test("TarStream() with invalid options", async () => { await assertRejects( () => Array.fromAsync(readable), - Error, + TypeError, "Invalid TarStreamOptions Provided", ); }); From e6e426ed9611ccf1a6df710160a4488c53a426c3 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Thu, 29 Aug 2024 19:02:42 +1000 Subject: [PATCH 67/82] adjust(archive): `validPath` to `assertValidPath` --- archive/tar_stream.ts | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 848d9451984d..ccce68275238 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -375,35 +375,23 @@ function parsePath( } /** - * The type that may be returned from the `validPath` function. - */ -export type PathResult = { ok: true } | { ok: false; error: string }; - -/** - * validPath is a function that validates the correctness of a path that + * assertValidPath is a function that validates the correctness of a path that * may be piped to a `TarStream`. It provides a means to check that a path is * valid before pipping it through the `TarStream`, where if invalid will throw an * error. Ruining any progress made when archiving. * * @param path The path as a string - * @return { ok: true } | { ok: false, error: string } + * @return void * * @example Usage * ```ts - * import { validPath, type PathResult } from "@std/archive"; - * import { assertEquals } from "@std/assert"; + * import { assertValidPath } from "@std/archive"; * - * assertEquals(validPath('MyFile.txt'), { ok: true }); + * assertValidPath('MyFile.txt'); * ``` */ -export function validPath(path: string): PathResult { - try { - parsePath(path); - return { ok: true }; - } catch (error) { - if (!(error instanceof Error)) throw error; - return { ok: false, error: error.message }; - } +export function assertValidPath(path: string): void { + parsePath(path); } /** From 76d2f390b72517226f2d6e943f1b85bf2745d3e8 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Thu, 29 Aug 2024 19:21:25 +1000 Subject: [PATCH 68/82] adjust(archive): `validTarStreamOptions` to `assertValidTarStreamOptions` --- archive/tar_stream.ts | 42 ++++++------- archive/tar_stream_test.ts | 126 +++++++++++++++++++++++++++---------- 2 files changed, 114 insertions(+), 54 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index ccce68275238..8e20a506138e 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -152,10 +152,12 @@ export class TarStream implements TransformStream { TarStreamInputInternal >({ transform(chunk, controller) { - if (chunk.options && !validTarStreamOptions(chunk.options)) { - return controller.error( - new TypeError("Invalid TarStreamOptions Provided"), - ); + if (chunk.options) { + try { + assertValidTarStreamOptions(chunk.options); + } catch (e) { + return controller.error(e); + } } if ( @@ -384,7 +386,7 @@ function parsePath( * @return void * * @example Usage - * ```ts + * ```ts no-assert * import { assertValidPath } from "@std/archive"; * * assertValidPath('MyFile.txt'); @@ -395,60 +397,58 @@ export function assertValidPath(path: string): void { } /** - * validTarStreamOptions is a function that returns a true if all of the options + * assertValidTarStreamOptions is a function that returns a true if all of the options * provided are in the correct format, otherwise returns false. * * @param options The TarStreamOptions * @return boolean * * @example Usage - * ```ts - * import { validTarStreamOptions } from "@std/archive"; - * import { assertEquals } from "@std/assert"; + * ```ts no-assert + * import { assertValidTarStreamOptions } from "@std/archive"; * - * assertEquals(validTarStreamOptions({ mode: 755 }), true); + * assertValidTarStreamOptions({ mode: 755 }) * ``` */ -export function validTarStreamOptions( +export function assertValidTarStreamOptions( options: TarStreamOptions, -): boolean { +): void { if ( options.mode && (options.mode.toString().length > 6 || !/^[0-7]*$/.test(options.mode.toString())) - ) return false; + ) throw new TypeError("Invalid Mode Provided"); if ( options.uid && (options.uid.toString().length > 6 || !/^[0-7]*$/.test(options.uid.toString())) - ) return false; + ) throw new TypeError("Invalid UID Provided"); if ( options.gid && (options.gid.toString().length > 6 || !/^[0-7]*$/.test(options.gid.toString())) - ) return false; + ) throw new TypeError("Invalid GID Provided"); if ( options.mtime != undefined && (options.mtime.toString(8).length > 11 || options.mtime.toString() === "NaN") - ) return false; + ) throw new TypeError("Invalid MTime Provided"); if ( options.uname && // deno-lint-ignore no-control-regex (options.uname.length > 32 - 1 || !/^[\x00-\x7F]*$/.test(options.uname)) - ) return false; + ) throw new TypeError("Invalid UName Provided"); if ( options.gname && // deno-lint-ignore no-control-regex (options.gname.length > 32 - 1 || !/^[\x00-\x7F]*$/.test(options.gname)) - ) return false; + ) throw new TypeError("Invalid GName Provided"); if ( options.devmajor && (options.devmajor.length > 8) - ) return false; + ) throw new TypeError("Invalid DevMajor Provided"); if ( options.devminor && (options.devminor.length > 8) - ) return false; - return true; + ) throw new TypeError("Invalid DevMinor Provided"); } diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index f3620c2cb28a..578aa1f01054 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -1,10 +1,15 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. import { + assertValidTarStreamOptions, TarStream, type TarStreamInput, - validTarStreamOptions, } from "./tar_stream.ts"; -import { assert, assertEquals, assertRejects } from "../assert/mod.ts"; +import { + assert, + assertEquals, + assertRejects, + assertThrows, +} from "../assert/mod.ts"; import { UntarStream } from "./untar_stream.ts"; import { concat } from "../bytes/mod.ts"; @@ -190,45 +195,100 @@ Deno.test("parsePath()", async () => { }); Deno.test("validTarStreamOptions()", () => { - assertEquals(validTarStreamOptions({}), true); + assertValidTarStreamOptions({}); - assertEquals(validTarStreamOptions({ mode: 0 }), true); - assertEquals(validTarStreamOptions({ mode: 8 }), false); - assertEquals(validTarStreamOptions({ mode: 1111111 }), false); + assertValidTarStreamOptions({ mode: 0 }); + assertThrows( + () => assertValidTarStreamOptions({ mode: 8 }), + TypeError, + "Invalid Mode Provided", + ); + assertThrows( + () => assertValidTarStreamOptions({ mode: 1111111 }), + TypeError, + "Invalid Mode Provided", + ); - assertEquals(validTarStreamOptions({ uid: 0 }), true); - assertEquals(validTarStreamOptions({ uid: 8 }), false); - assertEquals(validTarStreamOptions({ uid: 1111111 }), false); + assertValidTarStreamOptions({ uid: 0 }); + assertThrows( + () => assertValidTarStreamOptions({ uid: 8 }), + TypeError, + "Invalid UID Provided", + ); + assertThrows( + () => assertValidTarStreamOptions({ uid: 1111111 }), + TypeError, + "Invalid UID Provided", + ); - assertEquals(validTarStreamOptions({ gid: 0 }), true); - assertEquals(validTarStreamOptions({ gid: 8 }), false); - assertEquals(validTarStreamOptions({ gid: 1111111 }), false); + assertValidTarStreamOptions({ gid: 0 }); + assertThrows( + () => assertValidTarStreamOptions({ gid: 8 }), + TypeError, + "Invalid GID Provided", + ); + assertThrows( + () => assertValidTarStreamOptions({ gid: 1111111 }), + TypeError, + "Invalid GID Provided", + ); - assertEquals(validTarStreamOptions({ mtime: 0 }), true); - assertEquals(validTarStreamOptions({ mtime: NaN }), false); - assertEquals( - validTarStreamOptions({ mtime: Math.floor(new Date().getTime() / 1000) }), - true, + assertValidTarStreamOptions({ mtime: 0 }); + assertThrows( + () => assertValidTarStreamOptions({ mtime: NaN }), + TypeError, + "Invalid MTime Provided", + ); + assertValidTarStreamOptions({ + mtime: Math.floor(new Date().getTime() / 1000), + }); + assertThrows( + () => assertValidTarStreamOptions({ mtime: new Date().getTime() }), + TypeError, + "Invalid MTime Provided", ); - assertEquals(validTarStreamOptions({ mtime: new Date().getTime() }), false); - assertEquals(validTarStreamOptions({ uname: "" }), true); - assertEquals(validTarStreamOptions({ uname: "abcdef" }), true); - assertEquals(validTarStreamOptions({ uname: "å-abcdef" }), false); - assertEquals(validTarStreamOptions({ uname: "a".repeat(100) }), false); + assertValidTarStreamOptions({ uname: "" }); + assertValidTarStreamOptions({ uname: "abcdef" }); + assertThrows( + () => assertValidTarStreamOptions({ uname: "å-abcdef" }), + TypeError, + "Invalid UName Provided", + ); + assertThrows( + () => assertValidTarStreamOptions({ uname: "a".repeat(100) }), + TypeError, + "Invalid UName Provided", + ); - assertEquals(validTarStreamOptions({ gname: "" }), true); - assertEquals(validTarStreamOptions({ gname: "abcdef" }), true); - assertEquals(validTarStreamOptions({ gname: "å-abcdef" }), false); - assertEquals(validTarStreamOptions({ gname: "a".repeat(100) }), false); + assertValidTarStreamOptions({ gname: "" }); + assertValidTarStreamOptions({ gname: "abcdef" }); + assertThrows( + () => assertValidTarStreamOptions({ gname: "å-abcdef" }), + TypeError, + "Invalid GName Provided", + ); + assertThrows( + () => assertValidTarStreamOptions({ gname: "a".repeat(100) }), + TypeError, + "Invalid GName Provided", + ); - assertEquals(validTarStreamOptions({ devmajor: "" }), true); - assertEquals(validTarStreamOptions({ devmajor: "1234" }), true); - assertEquals(validTarStreamOptions({ devmajor: "123456789" }), false); + assertValidTarStreamOptions({ devmajor: "" }); + assertValidTarStreamOptions({ devmajor: "1234" }); + assertThrows( + () => assertValidTarStreamOptions({ devmajor: "123456789" }), + TypeError, + "Invalid DevMajor Provided", + ); - assertEquals(validTarStreamOptions({ devminor: "" }), true); - assertEquals(validTarStreamOptions({ devminor: "1234" }), true); - assertEquals(validTarStreamOptions({ devminor: "123456789" }), false); + assertValidTarStreamOptions({ devminor: "" }); + assertValidTarStreamOptions({ devminor: "1234" }); + assertThrows( + () => assertValidTarStreamOptions({ devminor: "123456789" }), + TypeError, + "Invalid DevMinor Provided", + ); }); Deno.test("TarStream() with invalid options", async () => { @@ -239,7 +299,7 @@ Deno.test("TarStream() with invalid options", async () => { await assertRejects( () => Array.fromAsync(readable), TypeError, - "Invalid TarStreamOptions Provided", + "Invalid Mode Provided", ); }); From fd564264fbf4bed7975a251b99d20fd53698ada8 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Thu, 29 Aug 2024 20:23:00 +1000 Subject: [PATCH 69/82] revert(archive): UntarStream to produce TarStreamEntry instead of TarStreamChunk --- archive/tar_stream_test.ts | 13 +--- archive/untar_stream.ts | 139 ++++++++++++++++++++--------------- archive/untar_stream_test.ts | 62 +++++++++++----- 3 files changed, 126 insertions(+), 88 deletions(-) diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index 578aa1f01054..70b7e94c2e7f 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -4,12 +4,7 @@ import { TarStream, type TarStreamInput, } from "./tar_stream.ts"; -import { - assert, - assertEquals, - assertRejects, - assertThrows, -} from "../assert/mod.ts"; +import { assertEquals, assertRejects, assertThrows } from "../assert/mod.ts"; import { UntarStream } from "./untar_stream.ts"; import { concat } from "../bytes/mod.ts"; @@ -188,9 +183,9 @@ Deno.test("parsePath()", async () => { "./some random path/with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/file", "./some random path/with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/file", ]; - for await (const tarChunk of readable) { - assert(tarChunk.type === "header"); - assertEquals(tarChunk.path, output.shift()); + for await (const tarEntry of readable) { + assertEquals(tarEntry.path, output.shift()); + tarEntry.readable?.cancel(); } }); diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 410e7bfbadda..13dfa6b20bbe 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -106,43 +106,24 @@ export interface PosixUstarFormat { } /** - * The header of an entry in the archive. + * The structure of an entry extracted from a Tar archive. */ -export interface TarStreamHeader { +export interface TarStreamEntry { /** - * The type 'header' indicating the start of a new entry. - */ - type: "header"; - /** - * The path of the entry. - */ - path: string; - /** - * The header of the entry. + * The header information attributed to the entry, presented in one of two + * valid forms. */ header: OldStyleFormat | PosixUstarFormat; -} - -/** - * The data belonging to the last entry returned. - */ -export interface TarStreamData { /** - * The type 'data' indicating a chunk of content from the last 'header' - * resolved. + * The path of the entry as stated in the archive. */ - type: "data"; + path: string; /** - * A chunk of content of from the entry. + * If present, the content of the entry. e.g. a file's content. */ - data: Uint8Array; + readable?: ReadableStream; } -/** - * The type extracted from the archive. - */ -export type TarStreamChunk = TarStreamHeader | TarStreamData; - /** * ### Overview * A TransformStream to expand a tar archive. Tar archives allow for storing @@ -156,9 +137,9 @@ export type TarStreamChunk = TarStreamHeader | TarStreamData; * * ### Usage * When expanding the archive, as demonstrated in the example, one must decide - * to either consume the Readable Stream, if present, or cancel it. The next - * entry won't be resolved until the previous ReadableStream is either consumed - * or cancelled. + * to either consume the ReadableStream property, if present, or cancel it. The + * next entry won't be resolved until the previous ReadableStream is either + * consumed or cancelled. * * ### Understanding Compressed * A tar archive may be compressed, often identified by an additional file @@ -168,26 +149,26 @@ export type TarStreamChunk = TarStreamHeader | TarStreamData; * @example Usage * ```ts no-eval * import { UntarStream } from "@std/archive/untar-stream"; + * import { dirname, normalize } from "@std/path"; * - * let fileWriter: WritableStreamDefaultWriter | undefined; * for await ( - * const entry of (await Deno.open('./out.tar.gz')) + * const entry of (await Deno.open("./out.tar.gz")) * .readable - * .pipeThrough(new DecompressionStream('gzip')) + * .pipeThrough(new DecompressionStream("gzip")) * .pipeThrough(new UntarStream()) * ) { - * if (entry.type === "header") { - * fileWriter?.close(); - * fileWriter = (await Deno.create(entry.path)).writable.getWriter(); - * } else await fileWriter!.write(entry.data); + * const path = normalize(entry.path); + * await Deno.mkdir(dirname(path)); + * await entry.readable?.pipeTo((await Deno.create(path)).writable); * } * ``` */ export class UntarStream - implements TransformStream { - #readable: ReadableStream; + implements TransformStream { + #readable: ReadableStream; #writable: WritableStream; #gen: AsyncGenerator; + #lock = false; constructor() { const { readable, writable } = new TransformStream< Uint8Array, @@ -217,9 +198,13 @@ export class UntarStream }(); } - async *#untar(): AsyncGenerator { + async *#untar(): AsyncGenerator { const decoder = new TextDecoder(); while (true) { + while (this.#lock) { + await new Promise((resolve) => setTimeout(resolve, 0)); + } + const { done, value } = await this.#gen.next(); if (done) break; @@ -270,17 +255,14 @@ export class UntarStream } yield { - type: "header", path: ("prefix" in header && header.prefix.length ? header.prefix + "/" : "") + header.name, header, + readable: ["1", "2", "3", "4", "5", "6"].includes(header.typeflag) + ? undefined + : this.#readableFile(header.size), }; - if (!["1", "2", "3", "4", "5", "6"].includes(header.typeflag)) { - for await (const data of this.#genFile(header.size)) { - yield { type: "data", data }; - } - } } } @@ -293,6 +275,43 @@ export class UntarStream } } + #readableFile(size: number): ReadableStream { + this.#lock = true; + const releaseLock = () => this.#lock = false; + const gen = this.#genFile(size); + return new ReadableStream({ + type: "bytes", + async pull(controller) { + const { done, value } = await gen.next(); + if (done) { + releaseLock(); + controller.close(); + return controller.byobRequest?.respond(0); + } + if (controller.byobRequest?.view) { + const buffer = new Uint8Array(controller.byobRequest.view.buffer); + + const size = buffer.length; + if (size < value.length) { + buffer.set(value.slice(0, size)); + controller.byobRequest.respond(size); + controller.enqueue(value.slice(size)); + } else { + buffer.set(value); + controller.byobRequest.respond(value.length); + } + } else { + controller.enqueue(value); + } + }, + async cancel() { + // deno-lint-ignore no-empty + for await (const _ of gen) {} + releaseLock(); + }, + }); + } + /** * The ReadableStream * @@ -301,22 +320,21 @@ export class UntarStream * @example Usage * ```ts no-eval * import { UntarStream } from "@std/archive/untar-stream"; + * import { dirname, normalize } from "@std/path"; * - * let fileWriter: WritableStreamDefaultWriter | undefined; * for await ( - * const entry of (await Deno.open('./out.tar.gz')) + * const entry of (await Deno.open("./out.tar.gz")) * .readable - * .pipeThrough(new DecompressionStream('gzip')) + * .pipeThrough(new DecompressionStream("gzip")) * .pipeThrough(new UntarStream()) * ) { - * if (entry.type === "header") { - * fileWriter?.close(); - * fileWriter = (await Deno.create(entry.path)).writable.getWriter(); - * } else await fileWriter!.write(entry.data); + * const path = normalize(entry.path); + * await Deno.mkdir(dirname(path)); + * await entry.readable?.pipeTo((await Deno.create(path)).writable); * } * ``` */ - get readable(): ReadableStream { + get readable(): ReadableStream { return this.#readable; } @@ -328,18 +346,17 @@ export class UntarStream * @example Usage * ```ts no-eval * import { UntarStream } from "@std/archive/untar-stream"; + * import { dirname, normalize } from "@std/path"; * - * let fileWriter: WritableStreamDefaultWriter | undefined; * for await ( - * const entry of (await Deno.open('./out.tar.gz')) + * const entry of (await Deno.open("./out.tar.gz")) * .readable - * .pipeThrough(new DecompressionStream('gzip')) + * .pipeThrough(new DecompressionStream("gzip")) * .pipeThrough(new UntarStream()) * ) { - * if (entry.type === "header") { - * fileWriter?.close(); - * fileWriter = (await Deno.create(entry.path)).writable.getWriter(); - * } else await fileWriter!.write(entry.data); + * const path = normalize(entry.path); + * await Deno.mkdir(dirname(path)); + * await entry.readable?.pipeTo((await Deno.create(path)).writable); * } * ``` */ diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index d210d6143c7d..bfb9dd3abeeb 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -38,7 +38,8 @@ Deno.test("expandTarArchiveCheckingHeaders", async () => { const headers: (OldStyleFormat | PosixUstarFormat)[] = []; for await (const item of readable) { - if (item.type === "header") headers.push(item.header); + headers.push(item.header); + await item.readable?.cancel(); } assertEquals(headers, [{ name: "./potato", @@ -91,12 +92,10 @@ Deno.test("expandTarArchiveCheckingBodies", async () => { .pipeThrough(new TarStream()) .pipeThrough(new UntarStream()); - const buffer = new Uint8Array(text.length); - let offset = 0; + let buffer = new Uint8Array(); for await (const item of readable) { - if (item.type === "data") { - buffer.set(item.data, offset); - offset += item.data.length; + if (item.readable) { + buffer = concat(await Array.fromAsync(item.readable)); } } assertEquals(buffer, text); @@ -119,14 +118,13 @@ Deno.test("UntarStream() with size equals to multiple of 512", async () => { .pipeThrough(new TarStream()) .pipeThrough(new UntarStream()); - assertEquals( - concat( - (await Array.fromAsync(readable)).filter((x) => x.type === "data").map( - (x) => x.data, - ), - ), - data, - ); + let buffer = new Uint8Array(); + for await (const entry of readable) { + if (entry.readable) { + buffer = concat(await Array.fromAsync(entry.readable)); + } + } + assertEquals(buffer, data); }); Deno.test("UntarStream() with invalid size", async () => { @@ -148,7 +146,14 @@ Deno.test("UntarStream() with invalid size", async () => { .pipeThrough(new UntarStream()); await assertRejects( - () => Array.fromAsync(readable), + async () => { + for await (const entry of readable) { + if (entry.readable) { + // deno-lint-ignore no-empty + for await (const _ of entry.readable) {} + } + } + }, Error, "Tarball has an unexpected number of bytes", ); @@ -173,7 +178,14 @@ Deno.test("UntarStream() with invalid ending", async () => { .pipeThrough(new UntarStream()); await assertRejects( - () => Array.fromAsync(readable), + async () => { + for await (const entry of readable) { + if (entry.readable) { + // deno-lint-ignore no-empty + for await (const _ of entry.readable) {} + } + } + }, Error, "Tarball has invalid ending", ); @@ -184,7 +196,14 @@ Deno.test("UntarStream() with too small size", async () => { .pipeThrough(new UntarStream()); await assertRejects( - () => Array.fromAsync(readable), + async () => { + for await (const entry of readable) { + if (entry.readable) { + // deno-lint-ignore no-empty + for await (const _ of entry.readable) {} + } + } + }, Error, "Tarball was too small to be valid", ); @@ -209,7 +228,14 @@ Deno.test("UntarStream() with invalid checksum", async () => { .pipeThrough(new UntarStream()); await assertRejects( - () => Array.fromAsync(readable), + async () => { + for await (const entry of readable) { + if (entry.readable) { + // deno-lint-ignore no-empty + for await (const _ of entry.readable) {} + } + } + }, Error, "Tarball header failed to pass checksum", ); From 931109b8617119a7c129274f08554263217b0ae8 Mon Sep 17 00:00:00 2001 From: Asher Gomez Date: Fri, 30 Aug 2024 09:12:29 +1000 Subject: [PATCH 70/82] refactor: remove redundant `void` return types --- archive/tar_stream.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 8e20a506138e..44f596dbb31d 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -383,7 +383,6 @@ function parsePath( * error. Ruining any progress made when archiving. * * @param path The path as a string - * @return void * * @example Usage * ```ts no-assert @@ -392,7 +391,7 @@ function parsePath( * assertValidPath('MyFile.txt'); * ``` */ -export function assertValidPath(path: string): void { +export function assertValidPath(path: string) { parsePath(path); } @@ -401,7 +400,6 @@ export function assertValidPath(path: string): void { * provided are in the correct format, otherwise returns false. * * @param options The TarStreamOptions - * @return boolean * * @example Usage * ```ts no-assert @@ -412,7 +410,7 @@ export function assertValidPath(path: string): void { */ export function assertValidTarStreamOptions( options: TarStreamOptions, -): void { +) { if ( options.mode && (options.mode.toString().length > 6 || From f9f1933b7f23637f8c4444d248ad902fb7008b06 Mon Sep 17 00:00:00 2001 From: Asher Gomez Date: Fri, 30 Aug 2024 09:15:03 +1000 Subject: [PATCH 71/82] docs: cleanup assertion function docs --- archive/tar_stream.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 44f596dbb31d..268906255fb7 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -377,10 +377,11 @@ function parsePath( } /** - * assertValidPath is a function that validates the correctness of a path that - * may be piped to a `TarStream`. It provides a means to check that a path is - * valid before pipping it through the `TarStream`, where if invalid will throw an - * error. Ruining any progress made when archiving. + * Asserts that the path provided is valid for a {@linkcode TarStream}. + * + * It provides a means to check that a path is valid before pipping it through + * the `TarStream`, where if invalid will throw an error. Ruining any progress + * made when archiving. * * @param path The path as a string * @@ -396,8 +397,7 @@ export function assertValidPath(path: string) { } /** - * assertValidTarStreamOptions is a function that returns a true if all of the options - * provided are in the correct format, otherwise returns false. + * Asserts that the options provided are valid for a {@linkcode TarStream}. * * @param options The TarStreamOptions * From 390208559940a87bea0440af0a93e59ef4a85784 Mon Sep 17 00:00:00 2001 From: Asher Gomez Date: Fri, 30 Aug 2024 09:18:44 +1000 Subject: [PATCH 72/82] docs: correct `TarStream` example --- archive/tar_stream.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 268906255fb7..cf9b8baee809 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -120,21 +120,21 @@ const SLASH_CODE_POINT = "/".charCodeAt(0); * * @example Usage * ```ts no-eval - * import { TarStream } from "@std/archive/tar-stream"; + * import { TarStream, type TarStreamInput } from "@std/archive/tar-stream"; * - * await ReadableStream.from([ + * await ReadableStream.from([ * { * path: 'potato/' * }, * { * path: 'deno.json', * size: (await Deno.stat('deno.json')).size, - * iterable: (await Deno.open('deno.json')).readable + * readable: (await Deno.open('deno.json')).readable * }, * { * path: '.vscode/settings.json', * size: (await Deno.stat('.vscode/settings.json')).size, - * iterable: (await Deno.open('.vscode/settings.json')).readable + * readable: (await Deno.open('.vscode/settings.json')).readable * } * ]) * .pipeThrough(new TarStream()) @@ -408,9 +408,7 @@ export function assertValidPath(path: string) { * assertValidTarStreamOptions({ mode: 755 }) * ``` */ -export function assertValidTarStreamOptions( - options: TarStreamOptions, -) { +export function assertValidTarStreamOptions(options: TarStreamOptions) { if ( options.mode && (options.mode.toString().length > 6 || From c5c663eb5f353bfe6da2f8a766a5dd9a2363d1fe Mon Sep 17 00:00:00 2001 From: Asher Gomez Date: Fri, 30 Aug 2024 09:21:08 +1000 Subject: [PATCH 73/82] docs: minor docs cleanups --- archive/untar_stream.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 13dfa6b20bbe..54707e9f9ebe 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -119,7 +119,7 @@ export interface TarStreamEntry { */ path: string; /** - * If present, the content of the entry. e.g. a file's content. + * The content of the entry, if the entry is a file. */ readable?: ReadableStream; } @@ -128,12 +128,13 @@ export interface TarStreamEntry { * ### Overview * A TransformStream to expand a tar archive. Tar archives allow for storing * multiple files in a single file (called an archive, or sometimes a tarball). - * These archives typically have a single '.tar' extension. This + * + * These archives typically have a single '.tar' extension. This * implementation follows the [FreeBSD 15.0](https://man.freebsd.org/cgi/man.cgi?query=tar&sektion=5&apropos=0&manpath=FreeBSD+15.0-CURRENT) spec. * * ### Supported File Formats * Only the ustar file format is supported. This is the most common format. - * Additionally the numeric extension for file size. + * Additionally the numeric extension for file size. * * ### Usage * When expanding the archive, as demonstrated in the example, one must decide From ae9621e99d4c8db464ab6d28357151b8fae86e32 Mon Sep 17 00:00:00 2001 From: Asher Gomez Date: Fri, 30 Aug 2024 09:25:01 +1000 Subject: [PATCH 74/82] refactor: improve error class specificity --- archive/tar_stream.ts | 2 +- archive/tar_stream_test.ts | 2 +- archive/untar_stream.ts | 8 ++++---- archive/untar_stream_test.ts | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index cf9b8baee809..d52a2b422f34 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -369,7 +369,7 @@ function parsePath( const prefix = name.slice(0, suitableSlashPos); if (prefix.length > 155) { - throw new Error( + throw new TypeError( "Path needs to be split-able on a forward slash separator into [155, 100] bytes respectively", ); } diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index 70b7e94c2e7f..1e56c3c0d9b1 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -336,7 +336,7 @@ Deno.test("parsePath() with too long path", async () => { await assertRejects( () => Array.fromAsync(readable), - Error, + TypeError, "Path needs to be split-able on a forward slash separator into [155, 100] bytes respectively", ); }); diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 54707e9f9ebe..bc98ed5c9c13 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -184,17 +184,17 @@ export class UntarStream const chunk of readable.pipeThrough(new FixedChunkStream(512)) ) { if (chunk.length !== 512) { - throw new Error("Tarball has an unexpected number of bytes"); + throw new RangeError("Tarball has an unexpected number of bytes"); } buffer.push(chunk); if (buffer.length > 2) yield buffer.shift()!; } if (buffer.length < 2) { - throw new Error("Tarball was too small to be valid"); + throw new RangeError("Tarball was too small to be valid"); } if (!buffer.every((value) => value.every((x) => x === 0))) { - throw new Error("Tarball has invalid ending"); + throw new TypeError("Tarball has invalid ending"); } }(); } @@ -270,7 +270,7 @@ export class UntarStream async *#genFile(size: number): AsyncGenerator { for (let i = Math.ceil(size / 512); i > 0; --i) { const { done, value } = await this.#gen.next(); - if (done) throw new Error("Unexpected end of Tarball"); + if (done) throw new SyntaxError("Unexpected end of Tarball"); if (i === 1 && size % 512) yield value.subarray(0, size % 512); else yield value; } diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index bfb9dd3abeeb..c65b850fcea9 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -154,7 +154,7 @@ Deno.test("UntarStream() with invalid size", async () => { } } }, - Error, + RangeError, "Tarball has an unexpected number of bytes", ); }); @@ -186,7 +186,7 @@ Deno.test("UntarStream() with invalid ending", async () => { } } }, - Error, + TypeError, "Tarball has invalid ending", ); }); @@ -204,7 +204,7 @@ Deno.test("UntarStream() with too small size", async () => { } } }, - Error, + RangeError, "Tarball was too small to be valid", ); }); From 1f2d2b69dc33cd40d0fc5f274933a633f8c3999c Mon Sep 17 00:00:00 2001 From: Asher Gomez Date: Fri, 30 Aug 2024 14:01:38 +1000 Subject: [PATCH 75/82] docs: add `@experimental` JSDoc tags --- archive/tar_stream.ts | 16 +++++++++++++++- archive/untar_stream.ts | 8 ++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index d52a2b422f34..bdd3bd90b29d 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -2,6 +2,8 @@ /** * The interface required to provide a file. + * + * @experimental **UNSTABLE**: New API, yet to be vetted. */ export interface TarStreamFile { /** @@ -24,6 +26,8 @@ export interface TarStreamFile { /** * The interface required to provide a directory. + * + * @experimental **UNSTABLE**: New API, yet to be vetted. */ export interface TarStreamDir { /** @@ -39,6 +43,8 @@ export interface TarStreamDir { /** * A union type merging all the TarStream interfaces that can be piped into the * TarStream class. + * + * @experimental **UNSTABLE**: New API, yet to be vetted. */ export type TarStreamInput = TarStreamFile | TarStreamDir; type TarStreamInputInternal = @@ -47,6 +53,8 @@ type TarStreamInputInternal = /** * The options that can go along with a file or directory. + * + * @experimental **UNSTABLE**: New API, yet to be vetted. */ export interface TarStreamOptions { /** @@ -95,7 +103,7 @@ const SLASH_CODE_POINT = "/".charCodeAt(0); /** * ### Overview - * A TransformStream to create a tar archive. Tar archives allow for storing + * A TransformStream to create a tar archive. Tar archives allow for storing * multiple files in a single file (called an archive, or sometimes a tarball). * These archives typically have a single '.tar' extension. This * implementation follows the [FreeBSD 15.0](https://man.freebsd.org/cgi/man.cgi?query=tar&sektion=5&apropos=0&manpath=FreeBSD+15.0-CURRENT) spec. @@ -118,6 +126,8 @@ const SLASH_CODE_POINT = "/".charCodeAt(0); * Tar archives are not compressed by default. If you'd like to compress the * archive, you may do so by piping it through a compression stream. * + * @experimental **UNSTABLE**: New API, yet to be vetted. + * * @example Usage * ```ts no-eval * import { TarStream, type TarStreamInput } from "@std/archive/tar-stream"; @@ -379,6 +389,8 @@ function parsePath( /** * Asserts that the path provided is valid for a {@linkcode TarStream}. * + * @experimental **UNSTABLE**: New API, yet to be vetted. + * * It provides a means to check that a path is valid before pipping it through * the `TarStream`, where if invalid will throw an error. Ruining any progress * made when archiving. @@ -399,6 +411,8 @@ export function assertValidPath(path: string) { /** * Asserts that the options provided are valid for a {@linkcode TarStream}. * + * @experimental **UNSTABLE**: New API, yet to be vetted. + * * @param options The TarStreamOptions * * @example Usage diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index bc98ed5c9c13..f44ab7bd37aa 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -3,6 +3,8 @@ import { FixedChunkStream } from "@std/streams"; /** * The original tar archive header format. + * + * @experimental **UNSTABLE**: New API, yet to be vetted. */ export interface OldStyleFormat { /** @@ -41,6 +43,8 @@ export interface OldStyleFormat { /** * The POSIX ustar archive header format. + * + * @experimental **UNSTABLE**: New API, yet to be vetted. */ export interface PosixUstarFormat { /** @@ -107,6 +111,8 @@ export interface PosixUstarFormat { /** * The structure of an entry extracted from a Tar archive. + * + * @experimental **UNSTABLE**: New API, yet to be vetted. */ export interface TarStreamEntry { /** @@ -147,6 +153,8 @@ export interface TarStreamEntry { * extension, such as '.tar.gz' for gzip. This TransformStream does not support * decompression which must be done before expanding the archive. * + * @experimental **UNSTABLE**: New API, yet to be vetted. + * * @example Usage * ```ts no-eval * import { UntarStream } from "@std/archive/untar-stream"; From 55bbfa3c15d1dedc1e4ac3d8c44044310b92aff9 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Fri, 30 Aug 2024 16:35:41 +1000 Subject: [PATCH 76/82] docs(archive): Updated examples for `assertValidPath` and `assertValidTarStreamOptions``` --- archive/tar_stream.ts | 61 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index bdd3bd90b29d..814d9816882e 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -399,9 +399,37 @@ function parsePath( * * @example Usage * ```ts no-assert - * import { assertValidPath } from "@std/archive"; + * import { assertValidPath, TarStream, type TarStreamInput } from "@std/archive"; * - * assertValidPath('MyFile.txt'); + * const paths = (await Array.fromAsync(Deno.readDir("./"))) + * .filter(entry => entry.isFile) + * .map((entry) => entry.name) + * // Filter out any paths that are invalid as they are to be placed inside a Tar. + * .filter(path => { + * try { + * assertValidPath(path); + * return true; + * } catch (error) { + * console.error(error); + * return false; + * } + * }); + * + * await ReadableStream.from(paths) + * .pipeThrough( + * new TransformStream({ + * async transform(path, controller) { + * controller.enqueue({ + * path, + * size: (await Deno.stat(path)).size, + * readable: (await Deno.open(path)).readable, + * }); + * }, + * }), + * ) + * .pipeThrough(new TarStream()) + * .pipeThrough(new CompressionStream('gzip')) + * .pipeTo((await Deno.create('./archive.tar.gz')).writable); * ``` */ export function assertValidPath(path: string) { @@ -417,9 +445,34 @@ export function assertValidPath(path: string) { * * @example Usage * ```ts no-assert - * import { assertValidTarStreamOptions } from "@std/archive"; + * import { assertValidTarStreamOptions, TarStream, type TarStreamInput } from "@std/archive"; * - * assertValidTarStreamOptions({ mode: 755 }) + * const paths = (await Array.fromAsync(Deno.readDir('./'))) + * .filter(entry => entry.isFile) + * .map(entry => entry.name); + * + * await ReadableStream.from(paths) + * .pipeThrough(new TransformStream({ + * async transform(path, controller) { + * const stats = await Deno.stat(path); + * const options = { mtime: stats.mtime?.getTime()! / 1000 }; + * try { + * // Filter out any paths that would have an invalid options provided. + * assertValidTarStreamOptions(options); + * controller.enqueue({ + * path, + * size: stats.size, + * readable: (await Deno.open(path)).readable, + * options, + * }); + * } catch (error) { + * console.error(error); + * } + * }, + * })) + * .pipeThrough(new TarStream()) + * .pipeThrough(new CompressionStream('gzip')) + * .pipeTo((await Deno.create('./archive.tar.gz')).writable); * ``` */ export function assertValidTarStreamOptions(options: TarStreamOptions) { From 426de970f5c6e562a5eae22e61504132f9662400 Mon Sep 17 00:00:00 2001 From: BlackAsLight <44320105+BlackAsLight@users.noreply.github.com> Date: Fri, 30 Aug 2024 16:46:52 +1000 Subject: [PATCH 77/82] fix(archive): problem with tests - I suspect the problem is that a file that was read by `Deno.readDir` changed size between being read at `Deno.stat` and when `Deno.open` finished pulling it all in. --- archive/tar_stream.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 814d9816882e..35e790c24ccd 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -415,6 +415,7 @@ function parsePath( * } * }); * + * await Deno.mkdir('./out/', { recursive: true }) * await ReadableStream.from(paths) * .pipeThrough( * new TransformStream({ @@ -429,7 +430,7 @@ function parsePath( * ) * .pipeThrough(new TarStream()) * .pipeThrough(new CompressionStream('gzip')) - * .pipeTo((await Deno.create('./archive.tar.gz')).writable); + * .pipeTo((await Deno.create('./out/archive.tar.gz')).writable); * ``` */ export function assertValidPath(path: string) { @@ -451,6 +452,7 @@ export function assertValidPath(path: string) { * .filter(entry => entry.isFile) * .map(entry => entry.name); * + * await Deno.mkdir('./out/', { recursive: true }) * await ReadableStream.from(paths) * .pipeThrough(new TransformStream({ * async transform(path, controller) { @@ -472,7 +474,7 @@ export function assertValidPath(path: string) { * })) * .pipeThrough(new TarStream()) * .pipeThrough(new CompressionStream('gzip')) - * .pipeTo((await Deno.create('./archive.tar.gz')).writable); + * .pipeTo((await Deno.create('./out/archive.tar.gz')).writable); * ``` */ export function assertValidTarStreamOptions(options: TarStreamOptions) { From 05e5e19983901f0a7209ae148896797b8c001b5e Mon Sep 17 00:00:00 2001 From: Yoshiya Hinosawa Date: Mon, 2 Sep 2024 13:47:25 +0900 Subject: [PATCH 78/82] update error messages --- archive/tar_stream.ts | 56 +++++++++++++++++++++++++++++--------- archive/tar_stream_test.ts | 12 ++++---- 2 files changed, 49 insertions(+), 19 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 35e790c24ccd..c437c7944651 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -175,7 +175,11 @@ export class TarStream implements TransformStream { (chunk.size < 0 || 8 ** 12 < chunk.size || chunk.size.toString() === "NaN") ) { - return controller.error(new RangeError("Size cannot exceed 64 Gibs")); + return controller.error( + new RangeError( + "Cannot add to the tar archive: The size cannot exceed 64 Gibs", + ), + ); } const path = parsePath(chunk.path); @@ -242,7 +246,7 @@ export class TarStream implements TransformStream { } if (chunk.size !== size) { throw new RangeError( - "Provided size did not match bytes read from provided readable", + `Cannot add to the tar archive: The provided size (${chunk.size}) did not match bytes read from provided readable (${size})`, ); } if (chunk.size % 512) { @@ -355,13 +359,19 @@ function parsePath( } if (name.length > 256) { - throw new RangeError("Path cannot exceed 256 bytes"); + throw new RangeError( + `Cannot parse the path as the path lenth cannot exceed 256 bytes: The path length is ${name.length}`, + ); } // If length of last part is > 100, then there's no possible answer to split the path let suitableSlashPos = Math.max(0, name.lastIndexOf(SLASH_CODE_POINT)); // always holds position of '/' if (name.length - suitableSlashPos > 100) { - throw new RangeError("Filename cannot exceed 100 bytes"); + throw new RangeError( + `Cannot parse the path as the filename cannot exceed 100 bytes: The filename length is ${ + name.length - suitableSlashPos + }`, + ); } for ( @@ -380,7 +390,7 @@ function parsePath( const prefix = name.slice(0, suitableSlashPos); if (prefix.length > 155) { throw new TypeError( - "Path needs to be split-able on a forward slash separator into [155, 100] bytes respectively", + "Cannot parse the path as the path needs to be split-able on a forward slash separator into [155, 100] bytes respectively", ); } return [prefix, name.slice(suitableSlashPos + 1)]; @@ -482,38 +492,58 @@ export function assertValidTarStreamOptions(options: TarStreamOptions) { options.mode && (options.mode.toString().length > 6 || !/^[0-7]*$/.test(options.mode.toString())) - ) throw new TypeError("Invalid Mode Provided"); + ) throw new TypeError("Cannot add to the tar archive: Invalid Mode Provided"); if ( options.uid && (options.uid.toString().length > 6 || !/^[0-7]*$/.test(options.uid.toString())) - ) throw new TypeError("Invalid UID Provided"); + ) throw new TypeError("Cannot add to the tar archive: Invalid UID Provided"); if ( options.gid && (options.gid.toString().length > 6 || !/^[0-7]*$/.test(options.gid.toString())) - ) throw new TypeError("Invalid GID Provided"); + ) throw new TypeError("Cannot add to the tar archive: Invalid GID Provided"); if ( options.mtime != undefined && (options.mtime.toString(8).length > 11 || options.mtime.toString() === "NaN") - ) throw new TypeError("Invalid MTime Provided"); + ) { + throw new TypeError( + "Cannot add to the tar archive: Invalid MTime Provided", + ); + } if ( options.uname && // deno-lint-ignore no-control-regex (options.uname.length > 32 - 1 || !/^[\x00-\x7F]*$/.test(options.uname)) - ) throw new TypeError("Invalid UName Provided"); + ) { + throw new TypeError( + "Cannot add to the tar archive: Invalid UName Provided", + ); + } if ( options.gname && // deno-lint-ignore no-control-regex (options.gname.length > 32 - 1 || !/^[\x00-\x7F]*$/.test(options.gname)) - ) throw new TypeError("Invalid GName Provided"); + ) { + throw new TypeError( + "Cannot add to the tar archive: Invalid GName Provided", + ); + } if ( options.devmajor && (options.devmajor.length > 8) - ) throw new TypeError("Invalid DevMajor Provided"); + ) { + throw new TypeError( + "Cannot add to the tar archive: Invalid DevMajor Provided", + ); + } if ( options.devminor && (options.devminor.length > 8) - ) throw new TypeError("Invalid DevMinor Provided"); + ) { + throw new TypeError( + "Cannot add to the tar archive: Invalid DevMinor Provided", + ); + } } diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index 1e56c3c0d9b1..fef9a2ae37d5 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -101,7 +101,7 @@ Deno.test("TarStream() with negative size", async () => { await assertRejects( () => Array.fromAsync(readable), RangeError, - "Size cannot exceed 64 Gibs", + "Cannot add to the tar archive: The size cannot exceed 64 Gibs", ); }); @@ -126,7 +126,7 @@ Deno.test("TarStream() with 65 GiB size", async () => { await assertRejects( () => Array.fromAsync(readable), RangeError, - "Size cannot exceed 64 Gibs", + "Cannot add to the tar archive: The size cannot exceed 64 Gibs", ); }); @@ -151,7 +151,7 @@ Deno.test("TarStream() with NaN size", async () => { await assertRejects( () => Array.fromAsync(readable), RangeError, - "Size cannot exceed 64 Gibs", + "Cannot add to the tar archive: The size cannot exceed 64 Gibs", ); }); @@ -311,7 +311,7 @@ Deno.test("TarStream() with mismatching sizes", async () => { await assertRejects( () => Array.fromAsync(readable), RangeError, - "Provided size did not match bytes read from provided readable", + "Cannot add to the tar archive: The provided size (13) did not match bytes read from provided readable (12)", ); }); @@ -324,7 +324,7 @@ Deno.test("parsePath() with too long path", async () => { await assertRejects( () => Array.fromAsync(readable), RangeError, - "Path cannot exceed 256 bytes", + "Cannot parse the path as the path lenth cannot exceed 256 bytes: The path length is 300", ); }); @@ -337,6 +337,6 @@ Deno.test("parsePath() with too long path", async () => { await assertRejects( () => Array.fromAsync(readable), TypeError, - "Path needs to be split-able on a forward slash separator into [155, 100] bytes respectively", + "Cannot parse the path as the path needs to be split-able on a forward slash separator into [155, 100] bytes respectively", ); }); From cbd50e47bc17e4462e02be6066f2e3e8c853895e Mon Sep 17 00:00:00 2001 From: Yoshiya Hinosawa Date: Mon, 2 Sep 2024 14:00:49 +0900 Subject: [PATCH 79/82] update error messages --- archive/untar_stream.ts | 22 +++++++++++++++++----- archive/untar_stream_test.ts | 8 ++++---- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index f44ab7bd37aa..36c8702d3444 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -192,17 +192,23 @@ export class UntarStream const chunk of readable.pipeThrough(new FixedChunkStream(512)) ) { if (chunk.length !== 512) { - throw new RangeError("Tarball has an unexpected number of bytes"); + throw new RangeError( + `Cannot extract the tar archive: The tarball chunk has an unexpected number of bytes (${chunk.length})`, + ); } buffer.push(chunk); if (buffer.length > 2) yield buffer.shift()!; } if (buffer.length < 2) { - throw new RangeError("Tarball was too small to be valid"); + throw new RangeError( + "Cannot extract the tar achive: The tarball is too small to be valid", + ); } if (!buffer.every((value) => value.every((x) => x === 0))) { - throw new TypeError("Tarball has invalid ending"); + throw new TypeError( + "Cannot extract the tar archive: The tarball has invalid ending", + ); } }(); } @@ -224,7 +230,9 @@ export class UntarStream ); value.fill(32, 148, 156); if (value.reduce((x, y) => x + y) !== checksum) { - throw new SyntaxError("Tarball header failed to pass checksum"); + throw new SyntaxError( + "Cannot extract the tar archive: An archive entry has invalid header checksum", + ); } // Decode Header @@ -278,7 +286,11 @@ export class UntarStream async *#genFile(size: number): AsyncGenerator { for (let i = Math.ceil(size / 512); i > 0; --i) { const { done, value } = await this.#gen.next(); - if (done) throw new SyntaxError("Unexpected end of Tarball"); + if (done) { + throw new SyntaxError( + "Cannot extract the tar archive: Unexpected end of Tarball", + ); + } if (i === 1 && size % 512) yield value.subarray(0, size % 512); else yield value; } diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index c65b850fcea9..1a8ec870758f 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -155,7 +155,7 @@ Deno.test("UntarStream() with invalid size", async () => { } }, RangeError, - "Tarball has an unexpected number of bytes", + "Cannot extract the tar archive: The tarball chunk has an unexpected number of bytes (100)", ); }); @@ -187,7 +187,7 @@ Deno.test("UntarStream() with invalid ending", async () => { } }, TypeError, - "Tarball has invalid ending", + "Cannot extract the tar archive: The tarball has invalid ending", ); }); @@ -205,7 +205,7 @@ Deno.test("UntarStream() with too small size", async () => { } }, RangeError, - "Tarball was too small to be valid", + "Cannot extract the tar achive: The tarball is too small to be valid", ); }); @@ -237,6 +237,6 @@ Deno.test("UntarStream() with invalid checksum", async () => { } }, Error, - "Tarball header failed to pass checksum", + "Cannot extract the tar archive: An archive entry has invalid header checksum", ); }); From eb3e21143fda81c6bbec6cba50ac38e47902c242 Mon Sep 17 00:00:00 2001 From: Yoshiya Hinosawa Date: Mon, 2 Sep 2024 14:15:56 +0900 Subject: [PATCH 80/82] fix typos --- archive/tar_stream.ts | 2 +- archive/tar_stream_test.ts | 2 +- archive/untar_stream.ts | 2 +- archive/untar_stream_test.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index c437c7944651..843f6a2426ca 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -360,7 +360,7 @@ function parsePath( if (name.length > 256) { throw new RangeError( - `Cannot parse the path as the path lenth cannot exceed 256 bytes: The path length is ${name.length}`, + `Cannot parse the path as the path length cannot exceed 256 bytes: The path length is ${name.length}`, ); } diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index fef9a2ae37d5..356f00f47e2f 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -324,7 +324,7 @@ Deno.test("parsePath() with too long path", async () => { await assertRejects( () => Array.fromAsync(readable), RangeError, - "Cannot parse the path as the path lenth cannot exceed 256 bytes: The path length is 300", + "Cannot parse the path as the path length cannot exceed 256 bytes: The path length is 300", ); }); diff --git a/archive/untar_stream.ts b/archive/untar_stream.ts index 36c8702d3444..1b8acffd68fa 100644 --- a/archive/untar_stream.ts +++ b/archive/untar_stream.ts @@ -202,7 +202,7 @@ export class UntarStream } if (buffer.length < 2) { throw new RangeError( - "Cannot extract the tar achive: The tarball is too small to be valid", + "Cannot extract the tar archive: The tarball is too small to be valid", ); } if (!buffer.every((value) => value.every((x) => x === 0))) { diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index 1a8ec870758f..a63c2948e7aa 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -205,7 +205,7 @@ Deno.test("UntarStream() with too small size", async () => { } }, RangeError, - "Cannot extract the tar achive: The tarball is too small to be valid", + "Cannot extract the tar archive: The tarball is too small to be valid", ); }); From 16caed689578f37a63e3ba0dea3c42d32455de26 Mon Sep 17 00:00:00 2001 From: Asher Gomez Date: Mon, 2 Sep 2024 17:23:30 +1000 Subject: [PATCH 81/82] refactor: tweak error messages --- archive/tar_stream.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 843f6a2426ca..2989ef4f1a3d 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -492,24 +492,24 @@ export function assertValidTarStreamOptions(options: TarStreamOptions) { options.mode && (options.mode.toString().length > 6 || !/^[0-7]*$/.test(options.mode.toString())) - ) throw new TypeError("Cannot add to the tar archive: Invalid Mode Provided"); + ) throw new TypeError("Cannot add to the tar archive: Invalid Mode provided"); if ( options.uid && (options.uid.toString().length > 6 || !/^[0-7]*$/.test(options.uid.toString())) - ) throw new TypeError("Cannot add to the tar archive: Invalid UID Provided"); + ) throw new TypeError("Cannot add to the tar archive: Invalid UID provided"); if ( options.gid && (options.gid.toString().length > 6 || !/^[0-7]*$/.test(options.gid.toString())) - ) throw new TypeError("Cannot add to the tar archive: Invalid GID Provided"); + ) throw new TypeError("Cannot add to the tar archive: Invalid GID provided"); if ( options.mtime != undefined && (options.mtime.toString(8).length > 11 || options.mtime.toString() === "NaN") ) { throw new TypeError( - "Cannot add to the tar archive: Invalid MTime Provided", + "Cannot add to the tar archive: Invalid MTime provided", ); } if ( @@ -518,7 +518,7 @@ export function assertValidTarStreamOptions(options: TarStreamOptions) { (options.uname.length > 32 - 1 || !/^[\x00-\x7F]*$/.test(options.uname)) ) { throw new TypeError( - "Cannot add to the tar archive: Invalid UName Provided", + "Cannot add to the tar archive: Invalid UName provided", ); } if ( @@ -527,7 +527,7 @@ export function assertValidTarStreamOptions(options: TarStreamOptions) { (options.gname.length > 32 - 1 || !/^[\x00-\x7F]*$/.test(options.gname)) ) { throw new TypeError( - "Cannot add to the tar archive: Invalid GName Provided", + "Cannot add to the tar archive: Invalid GName provided", ); } if ( @@ -535,7 +535,7 @@ export function assertValidTarStreamOptions(options: TarStreamOptions) { (options.devmajor.length > 8) ) { throw new TypeError( - "Cannot add to the tar archive: Invalid DevMajor Provided", + "Cannot add to the tar archive: Invalid DevMajor provided", ); } if ( @@ -543,7 +543,7 @@ export function assertValidTarStreamOptions(options: TarStreamOptions) { (options.devminor.length > 8) ) { throw new TypeError( - "Cannot add to the tar archive: Invalid DevMinor Provided", + "Cannot add to the tar archive: Invalid DevMinor provided", ); } } From 456ee81744852530b11156365efd2d285a162a61 Mon Sep 17 00:00:00 2001 From: Asher Gomez Date: Mon, 2 Sep 2024 17:38:53 +1000 Subject: [PATCH 82/82] refactor: tweaks and add type field --- archive/tar_stream.ts | 32 ++++++++++++++++++++----- archive/tar_stream_test.ts | 46 +++++++++++++++++++++++------------- archive/untar_stream_test.ts | 8 +++++++ 3 files changed, 64 insertions(+), 22 deletions(-) diff --git a/archive/tar_stream.ts b/archive/tar_stream.ts index 2989ef4f1a3d..92211fd06c9a 100644 --- a/archive/tar_stream.ts +++ b/archive/tar_stream.ts @@ -6,6 +6,10 @@ * @experimental **UNSTABLE**: New API, yet to be vetted. */ export interface TarStreamFile { + /** + * The type of the input. + */ + type: "file"; /** * The path to the file, relative to the archive's root directory. */ @@ -30,6 +34,10 @@ export interface TarStreamFile { * @experimental **UNSTABLE**: New API, yet to be vetted. */ export interface TarStreamDir { + /** + * The type of the input. + */ + type: "directory"; /** * The path of the directory, relative to the archive's root directory. */ @@ -47,6 +55,7 @@ export interface TarStreamDir { * @experimental **UNSTABLE**: New API, yet to be vetted. */ export type TarStreamInput = TarStreamFile | TarStreamDir; + type TarStreamInputInternal = & (Omit | Omit) & { path: [Uint8Array, Uint8Array] }; @@ -134,14 +143,17 @@ const SLASH_CODE_POINT = "/".charCodeAt(0); * * await ReadableStream.from([ * { + * type: "directory", * path: 'potato/' * }, * { + * type: "file", * path: 'deno.json', * size: (await Deno.stat('deno.json')).size, * readable: (await Deno.open('deno.json')).readable * }, * { + * type: "file", * path: '.vscode/settings.json', * size: (await Deno.stat('.vscode/settings.json')).size, * readable: (await Deno.open('.vscode/settings.json')).readable @@ -294,17 +306,20 @@ export class TarStream implements TransformStream { * * await ReadableStream.from([ * { + * type: "directory", * path: 'potato/' * }, * { + * type: "file", * path: 'deno.json', * size: (await Deno.stat('deno.json')).size, - * iterable: (await Deno.open('deno.json')).readable + * readable: (await Deno.open('deno.json')).readable * }, * { + * type: "file", * path: '.vscode/settings.json', * size: (await Deno.stat('.vscode/settings.json')).size, - * iterable: (await Deno.open('.vscode/settings.json')).readable + * readable: (await Deno.open('.vscode/settings.json')).readable * } * ]) * .pipeThrough(new TarStream()) @@ -327,17 +342,20 @@ export class TarStream implements TransformStream { * * await ReadableStream.from([ * { + * type: "directory", * path: 'potato/' * }, * { + * type: "file", * path: 'deno.json', * size: (await Deno.stat('deno.json')).size, - * iterable: (await Deno.open('deno.json')).readable + * readable: (await Deno.open('deno.json')).readable * }, * { + * type: "file", * path: '.vscode/settings.json', * size: (await Deno.stat('.vscode/settings.json')).size, - * iterable: (await Deno.open('.vscode/settings.json')).readable + * readable: (await Deno.open('.vscode/settings.json')).readable * } * ]) * .pipeThrough(new TarStream()) @@ -408,7 +426,7 @@ function parsePath( * @param path The path as a string * * @example Usage - * ```ts no-assert + * ```ts no-assert no-eval * import { assertValidPath, TarStream, type TarStreamInput } from "@std/archive"; * * const paths = (await Array.fromAsync(Deno.readDir("./"))) @@ -431,6 +449,7 @@ function parsePath( * new TransformStream({ * async transform(path, controller) { * controller.enqueue({ + * type: "file", * path, * size: (await Deno.stat(path)).size, * readable: (await Deno.open(path)).readable, @@ -455,7 +474,7 @@ export function assertValidPath(path: string) { * @param options The TarStreamOptions * * @example Usage - * ```ts no-assert + * ```ts no-assert no-eval * import { assertValidTarStreamOptions, TarStream, type TarStreamInput } from "@std/archive"; * * const paths = (await Array.fromAsync(Deno.readDir('./'))) @@ -472,6 +491,7 @@ export function assertValidPath(path: string) { * // Filter out any paths that would have an invalid options provided. * assertValidTarStreamOptions(options); * controller.enqueue({ + * type: "file", * path, * size: stats.size, * readable: (await Deno.open(path)).readable, diff --git a/archive/tar_stream_test.ts b/archive/tar_stream_test.ts index 356f00f47e2f..adf872956251 100644 --- a/archive/tar_stream_test.ts +++ b/archive/tar_stream_test.ts @@ -13,9 +13,11 @@ Deno.test("TarStream() with default stream", async () => { const reader = ReadableStream.from([ { + type: "directory", path: "./potato", }, { + type: "file", path: "./text.txt", size: text.length, readable: ReadableStream.from([text.slice()]), @@ -51,9 +53,11 @@ Deno.test("TarStream() with byte stream", async () => { const reader = ReadableStream.from([ { + type: "directory", path: "./potato", }, { + type: "file", path: "./text.txt", size: text.length, readable: ReadableStream.from([text.slice()]), @@ -91,6 +95,7 @@ Deno.test("TarStream() with negative size", async () => { const readable = ReadableStream.from([ { + type: "file", path: "name", size: -text.length, readable: ReadableStream.from([text.slice()]), @@ -116,6 +121,7 @@ Deno.test("TarStream() with 65 GiB size", async () => { const readable = ReadableStream.from([ { + type: "file", path: "name", size, readable: ReadableStream.from(iterable), @@ -141,6 +147,7 @@ Deno.test("TarStream() with NaN size", async () => { const readable = ReadableStream.from([ { + type: "file", path: "name", size, readable: ReadableStream.from(iterable), @@ -158,18 +165,22 @@ Deno.test("TarStream() with NaN size", async () => { Deno.test("parsePath()", async () => { const readable = ReadableStream.from([ { + type: "directory", path: "./Veeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeery/LongPath", }, { + type: "directory", path: "./some random path/with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/path", }, { + type: "directory", path: "./some random path/with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/file", }, { + type: "directory", path: "./some random path/with/loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong/file", }, @@ -196,43 +207,43 @@ Deno.test("validTarStreamOptions()", () => { assertThrows( () => assertValidTarStreamOptions({ mode: 8 }), TypeError, - "Invalid Mode Provided", + "Invalid Mode provided", ); assertThrows( () => assertValidTarStreamOptions({ mode: 1111111 }), TypeError, - "Invalid Mode Provided", + "Invalid Mode provided", ); assertValidTarStreamOptions({ uid: 0 }); assertThrows( () => assertValidTarStreamOptions({ uid: 8 }), TypeError, - "Invalid UID Provided", + "Invalid UID provided", ); assertThrows( () => assertValidTarStreamOptions({ uid: 1111111 }), TypeError, - "Invalid UID Provided", + "Invalid UID provided", ); assertValidTarStreamOptions({ gid: 0 }); assertThrows( () => assertValidTarStreamOptions({ gid: 8 }), TypeError, - "Invalid GID Provided", + "Invalid GID provided", ); assertThrows( () => assertValidTarStreamOptions({ gid: 1111111 }), TypeError, - "Invalid GID Provided", + "Invalid GID provided", ); assertValidTarStreamOptions({ mtime: 0 }); assertThrows( () => assertValidTarStreamOptions({ mtime: NaN }), TypeError, - "Invalid MTime Provided", + "Invalid MTime provided", ); assertValidTarStreamOptions({ mtime: Math.floor(new Date().getTime() / 1000), @@ -240,7 +251,7 @@ Deno.test("validTarStreamOptions()", () => { assertThrows( () => assertValidTarStreamOptions({ mtime: new Date().getTime() }), TypeError, - "Invalid MTime Provided", + "Invalid MTime provided", ); assertValidTarStreamOptions({ uname: "" }); @@ -248,12 +259,12 @@ Deno.test("validTarStreamOptions()", () => { assertThrows( () => assertValidTarStreamOptions({ uname: "å-abcdef" }), TypeError, - "Invalid UName Provided", + "Invalid UName provided", ); assertThrows( () => assertValidTarStreamOptions({ uname: "a".repeat(100) }), TypeError, - "Invalid UName Provided", + "Invalid UName provided", ); assertValidTarStreamOptions({ gname: "" }); @@ -261,12 +272,12 @@ Deno.test("validTarStreamOptions()", () => { assertThrows( () => assertValidTarStreamOptions({ gname: "å-abcdef" }), TypeError, - "Invalid GName Provided", + "Invalid GName provided", ); assertThrows( () => assertValidTarStreamOptions({ gname: "a".repeat(100) }), TypeError, - "Invalid GName Provided", + "Invalid GName provided", ); assertValidTarStreamOptions({ devmajor: "" }); @@ -274,7 +285,7 @@ Deno.test("validTarStreamOptions()", () => { assertThrows( () => assertValidTarStreamOptions({ devmajor: "123456789" }), TypeError, - "Invalid DevMajor Provided", + "Invalid DevMajor provided", ); assertValidTarStreamOptions({ devminor: "" }); @@ -282,19 +293,19 @@ Deno.test("validTarStreamOptions()", () => { assertThrows( () => assertValidTarStreamOptions({ devminor: "123456789" }), TypeError, - "Invalid DevMinor Provided", + "Invalid DevMinor provided", ); }); Deno.test("TarStream() with invalid options", async () => { const readable = ReadableStream.from([ - { path: "potato", options: { mode: 9 } }, + { type: "directory", path: "potato", options: { mode: 9 } }, ]).pipeThrough(new TarStream()); await assertRejects( () => Array.fromAsync(readable), TypeError, - "Invalid Mode Provided", + "Invalid Mode provided", ); }); @@ -302,6 +313,7 @@ Deno.test("TarStream() with mismatching sizes", async () => { const text = new TextEncoder().encode("Hello World!"); const readable = ReadableStream.from([ { + type: "file", path: "potato", size: text.length + 1, readable: ReadableStream.from([text.slice()]), @@ -317,6 +329,7 @@ Deno.test("TarStream() with mismatching sizes", async () => { Deno.test("parsePath() with too long path", async () => { const readable = ReadableStream.from([{ + type: "directory", path: "0".repeat(300), }]) .pipeThrough(new TarStream()); @@ -330,6 +343,7 @@ Deno.test("parsePath() with too long path", async () => { Deno.test("parsePath() with too long path", async () => { const readable = ReadableStream.from([{ + type: "directory", path: "0".repeat(160) + "/", }]) .pipeThrough(new TarStream()); diff --git a/archive/untar_stream_test.ts b/archive/untar_stream_test.ts index a63c2948e7aa..a89f04c4a2b1 100644 --- a/archive/untar_stream_test.ts +++ b/archive/untar_stream_test.ts @@ -14,6 +14,7 @@ Deno.test("expandTarArchiveCheckingHeaders", async () => { const readable = ReadableStream.from([ { + type: "directory", path: "./potato", options: { mode: 111111, @@ -27,6 +28,7 @@ Deno.test("expandTarArchiveCheckingHeaders", async () => { }, }, { + type: "file", path: "./text.txt", size: text.length, readable: ReadableStream.from([text.slice()]), @@ -81,9 +83,11 @@ Deno.test("expandTarArchiveCheckingBodies", async () => { const readable = ReadableStream.from([ { + type: "directory", path: "./potato", }, { + type: "file", path: "./text.txt", size: text.length, readable: ReadableStream.from([text.slice()]), @@ -110,6 +114,7 @@ Deno.test("UntarStream() with size equals to multiple of 512", async () => { const readable = ReadableStream.from([ { + type: "file", path: "name", size, readable: ReadableStream.from([data.slice()]), @@ -130,6 +135,7 @@ Deno.test("UntarStream() with size equals to multiple of 512", async () => { Deno.test("UntarStream() with invalid size", async () => { const readable = ReadableStream.from([ { + type: "file", path: "newFile.txt", size: 512, readable: ReadableStream.from([new Uint8Array(512).fill(97)]), @@ -164,6 +170,7 @@ Deno.test("UntarStream() with invalid ending", async () => { await Array.fromAsync( ReadableStream.from([ { + type: "file", path: "newFile.txt", size: 512, readable: ReadableStream.from([new Uint8Array(512).fill(97)]), @@ -214,6 +221,7 @@ Deno.test("UntarStream() with invalid checksum", async () => { await Array.fromAsync( ReadableStream.from([ { + type: "file", path: "newFile.txt", size: 512, readable: ReadableStream.from([new Uint8Array(512).fill(97)]),