Skip to content

Commit

Permalink
@uppy/aws-s3-multipart: refactor to TS (#4902)
Browse files Browse the repository at this point in the history
  • Loading branch information
aduh95 authored Mar 19, 2024
1 parent 96dc197 commit 7b0533b
Show file tree
Hide file tree
Showing 12 changed files with 2,014 additions and 1,263 deletions.
427 changes: 427 additions & 0 deletions packages/@uppy/aws-s3-multipart/src/HTTPCommunicationQueue.ts

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -1,24 +1,53 @@
import type { Uppy } from '@uppy/core'
import { AbortController } from '@uppy/utils/lib/AbortController'
import type { Meta, UppyFile } from '@uppy/utils/lib/UppyFile'
import type { HTTPCommunicationQueue } from './HTTPCommunicationQueue'
import type { Body } from './utils'

const MB = 1024 * 1024

interface MultipartUploaderOptions<M extends Meta, B extends Body> {
getChunkSize?: (file: { size: number }) => number
onProgress?: (bytesUploaded: number, bytesTotal: number) => void
onPartComplete?: (part: { PartNumber: number; ETag: string }) => void
shouldUseMultipart?: boolean | ((file: UppyFile<M, B>) => boolean)
onSuccess?: (result: B) => void
onError?: (err: unknown) => void
companionComm: HTTPCommunicationQueue<M, B>
file: UppyFile<M, B>
log: Uppy<M, B>['log']

uploadId: string
key: string
}

const defaultOptions = {
getChunkSize (file) {
getChunkSize(file: { size: number }) {
return Math.ceil(file.size / 10000)
},
onProgress () {},
onPartComplete () {},
onSuccess () {},
onError (err) {
onProgress() {},
onPartComplete() {},
onSuccess() {},
onError(err: unknown) {
throw err
},
} satisfies Partial<MultipartUploaderOptions<any, any>>

export interface Chunk {
getData: () => Blob
onProgress: (ev: ProgressEvent) => void
onComplete: (etag: string) => void
shouldUseMultipart: boolean
setAsUploaded?: () => void
}

function ensureInt (value) {
function ensureInt<T>(value: T): T extends number | string ? number : never {
if (typeof value === 'string') {
// @ts-expect-error TS is not able to recognize it's fine.
return parseInt(value, 10)
}
if (typeof value === 'number') {
// @ts-expect-error TS is not able to recognize it's fine.
return value
}
throw new TypeError('Expected a number')
Expand All @@ -32,47 +61,41 @@ export const pausingUploadReason = Symbol('pausing upload, not an actual error')
* (based on the user-provided `shouldUseMultipart` option value) and to manage
* the chunk splitting.
*/
class MultipartUploader {
class MultipartUploader<M extends Meta, B extends Body> {
options: MultipartUploaderOptions<M, B> &
Required<Pick<MultipartUploaderOptions<M, B>, keyof typeof defaultOptions>>

#abortController = new AbortController()

/** @type {import("../types/chunk").Chunk[]} */
#chunks
#chunks: Array<Chunk | null>

/** @type {{ uploaded: number, etag?: string, done?: boolean }[]} */
#chunkState
#chunkState: { uploaded: number; etag?: string; done?: boolean }[]

/**
* The (un-chunked) data to upload.
*
* @type {Blob}
*/
#data
#data: Blob

/** @type {import("@uppy/core").UppyFile} */
#file
#file: UppyFile<M, B>

/** @type {boolean} */
#uploadHasStarted = false

/** @type {(err?: Error | any) => void} */
#onError
#onError: (err: unknown) => void

/** @type {() => void} */
#onSuccess
#onSuccess: (result: B) => void

/** @type {import('../types/index').AwsS3MultipartOptions["shouldUseMultipart"]} */
#shouldUseMultipart
#shouldUseMultipart: MultipartUploaderOptions<M, B>['shouldUseMultipart']

/** @type {boolean} */
#isRestoring
#isRestoring: boolean

#onReject = (err) => (err?.cause === pausingUploadReason ? null : this.#onError(err))
#onReject = (err: unknown) =>
(err as any)?.cause === pausingUploadReason ? null : this.#onError(err)

#maxMultipartParts = 10_000

#minPartSize = 5 * MB

constructor (data, options) {
constructor(data: Blob, options: MultipartUploaderOptions<M, B>) {
this.options = {
...defaultOptions,
...options,
Expand All @@ -89,7 +112,7 @@ class MultipartUploader {
// When we are restoring an upload, we already have an UploadId and a Key. Otherwise
// we need to call `createMultipartUpload` to get an `uploadId` and a `key`.
// Non-multipart uploads are not restorable.
this.#isRestoring = options.uploadId && options.key
this.#isRestoring = (options.uploadId && options.key) as any as boolean

this.#initChunks()
}
Expand All @@ -98,15 +121,19 @@ class MultipartUploader {
// and calculates the optimal part size. When using multipart part uploads every part except for the last has
// to be at least 5 MB and there can be no more than 10K parts.
// This means we sometimes need to change the preferred part size from the user in order to meet these requirements.
#initChunks () {
#initChunks() {
const fileSize = this.#data.size
const shouldUseMultipart = typeof this.#shouldUseMultipart === 'function'
? this.#shouldUseMultipart(this.#file)
const shouldUseMultipart =
typeof this.#shouldUseMultipart === 'function' ?
this.#shouldUseMultipart(this.#file)
: Boolean(this.#shouldUseMultipart)

if (shouldUseMultipart && fileSize > this.#minPartSize) {
// At least 5MB per request:
let chunkSize = Math.max(this.options.getChunkSize(this.#data), this.#minPartSize)
let chunkSize = Math.max(
this.options.getChunkSize(this.#data),
this.#minPartSize,
)
let arraySize = Math.floor(fileSize / chunkSize)

// At most 10k requests per file:
Expand All @@ -132,41 +159,48 @@ class MultipartUploader {
shouldUseMultipart,
}
if (this.#isRestoring) {
const size = offset + chunkSize > fileSize ? fileSize - offset : chunkSize
const size =
offset + chunkSize > fileSize ? fileSize - offset : chunkSize
// setAsUploaded is called by listPart, to keep up-to-date the
// quantity of data that is left to actually upload.
this.#chunks[j].setAsUploaded = () => {
this.#chunks[j]!.setAsUploaded = () => {
this.#chunks[j] = null
this.#chunkState[j].uploaded = size
}
}
}
} else {
this.#chunks = [{
getData: () => this.#data,
onProgress: this.#onPartProgress(0),
onComplete: this.#onPartComplete(0),
shouldUseMultipart,
}]
this.#chunks = [
{
getData: () => this.#data,
onProgress: this.#onPartProgress(0),
onComplete: this.#onPartComplete(0),
shouldUseMultipart,
},
]
}

this.#chunkState = this.#chunks.map(() => ({ uploaded: 0 }))
}

#createUpload () {
this
.options.companionComm.uploadFile(this.#file, this.#chunks, this.#abortController.signal)
#createUpload() {
this.options.companionComm
.uploadFile(
this.#file,
this.#chunks as Chunk[],
this.#abortController.signal,
)
.then(this.#onSuccess, this.#onReject)
this.#uploadHasStarted = true
}

#resumeUpload () {
this
.options.companionComm.resumeUploadFile(this.#file, this.#chunks, this.#abortController.signal)
#resumeUpload() {
this.options.companionComm
.resumeUploadFile(this.#file, this.#chunks, this.#abortController.signal)
.then(this.#onSuccess, this.#onReject)
}

#onPartProgress = (index) => (ev) => {
#onPartProgress = (index: number) => (ev: ProgressEvent) => {
if (!ev.lengthComputable) return

this.#chunkState[index].uploaded = ensureInt(ev.loaded)
Expand All @@ -175,7 +209,7 @@ class MultipartUploader {
this.options.onProgress(totalUploaded, this.#data.size)
}

#onPartComplete = (index) => (etag) => {
#onPartComplete = (index: number) => (etag: string) => {
// This avoids the net::ERR_OUT_OF_MEMORY in Chromium Browsers.
this.#chunks[index] = null
this.#chunkState[index].etag = etag
Expand All @@ -188,37 +222,44 @@ class MultipartUploader {
this.options.onPartComplete(part)
}

#abortUpload () {
#abortUpload() {
this.#abortController.abort()
this.options.companionComm.abortFileUpload(this.#file).catch((err) => this.options.log(err))
this.options.companionComm
.abortFileUpload(this.#file)
.catch((err: unknown) => this.options.log(err as Error))
}

start () {
start(): void {
if (this.#uploadHasStarted) {
if (!this.#abortController.signal.aborted) this.#abortController.abort(pausingUploadReason)
if (!this.#abortController.signal.aborted)
this.#abortController.abort(pausingUploadReason)
this.#abortController = new AbortController()
this.#resumeUpload()
} else if (this.#isRestoring) {
this.options.companionComm.restoreUploadFile(this.#file, { uploadId: this.options.uploadId, key: this.options.key })
this.options.companionComm.restoreUploadFile(this.#file, {
uploadId: this.options.uploadId,
key: this.options.key,
})
this.#resumeUpload()
} else {
this.#createUpload()
}
}

pause () {
pause(): void {
this.#abortController.abort(pausingUploadReason)
// Swap it out for a new controller, because this instance may be resumed later.
this.#abortController = new AbortController()
}

abort (opts = undefined) {
abort(opts?: { really?: boolean }): void {
if (opts?.really) this.#abortUpload()
else this.pause()
}

// TODO: remove this in the next major
get chunkState () {
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
get chunkState() {
return this.#chunkState
}
}
Expand Down
113 changes: 0 additions & 113 deletions packages/@uppy/aws-s3-multipart/src/createSignedURL.test.js

This file was deleted.

Loading

0 comments on commit 7b0533b

Please sign in to comment.