From 7d2cff9af92251d9bcc29f5a7f202ca6a43d4184 Mon Sep 17 00:00:00 2001 From: Donny Wu Date: Mon, 16 Sep 2024 12:04:30 -0700 Subject: [PATCH 01/10] feat: opt in checksum --- .../apis/uploadData/multipartHandlers.test.ts | 51 ++++-- .../s3/apis/uploadData/putObjectJob.test.ts | 153 +++++++++++------- .../providers/s3/utils/crc32.native.test.ts | 13 -- .../uploadData/multipart/initialUpload.ts | 23 ++- .../apis/uploadData/multipart/uploadCache.ts | 4 +- .../uploadData/multipart/uploadHandlers.ts | 8 + .../s3/apis/uploadData/putObjectJob.ts | 7 +- .../storage/src/providers/s3/types/options.ts | 4 + .../src/providers/s3/utils/crc32.native.ts | 11 -- .../storage/src/providers/s3/utils/crc32.ts | 2 +- .../s3/utils/resolveS3ConfigAndInput.ts | 2 +- 11 files changed, 165 insertions(+), 113 deletions(-) delete mode 100644 packages/storage/__tests__/providers/s3/utils/crc32.native.test.ts delete mode 100644 packages/storage/src/providers/s3/utils/crc32.native.ts diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts index 695822f57c8..e804ee4bf39 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts @@ -47,9 +47,15 @@ const bucket = 'bucket'; const region = 'region'; const defaultKey = 'key'; const defaultContentType = 'application/octet-stream'; -const defaultCacheKey = '8388608_application/octet-stream_bucket_public_key'; +const defaultCacheKey = + '/twwTw==_8388608_application/octet-stream_bucket_public_key'; const testPath = 'testPath/object'; -const testPathCacheKey = `8388608_${defaultContentType}_${bucket}_custom_${testPath}`; +const testPathCacheKey = `/twwTw==_8388608_${defaultContentType}_${bucket}_custom_${testPath}`; + +const generateTestPathCacheKey = (optionsHash: string) => + `${optionsHash}_8388608_${defaultContentType}_${bucket}_custom_${testPath}`; +const generateDefaultCacheKey = (optionsHash: string) => + `${optionsHash}_8388608_application/octet-stream_bucket_public_key`; const mockCreateMultipartUpload = jest.mocked(createMultipartUpload); const mockUploadPart = jest.mocked(uploadPart); @@ -83,10 +89,6 @@ const mockCalculateContentCRC32Mock = () => { seed: 0, }); }; -const mockCalculateContentCRC32Undefined = () => { - mockCalculateContentCRC32.mockReset(); - mockCalculateContentCRC32.mockResolvedValue(undefined); -}; const mockCalculateContentCRC32Reset = () => { mockCalculateContentCRC32.mockReset(); mockCalculateContentCRC32.mockImplementation( @@ -291,6 +293,9 @@ describe('getMultipartUploadHandlers with key', () => { const { multipartUploadJob } = getMultipartUploadHandlers({ key: defaultKey, data: twoPartsPayload, + options: { + checksumAlgorithm: 'crc-32', + }, }); await multipartUploadJob(); @@ -301,9 +306,11 @@ describe('getMultipartUploadHandlers with key', () => { * * uploading each part calls calculateContentCRC32 1 time each * - * these steps results in 5 calls in total + * 1 time for optionsHash + * + * these steps results in 6 calls in total */ - expect(calculateContentCRC32).toHaveBeenCalledTimes(5); + expect(calculateContentCRC32).toHaveBeenCalledTimes(6); expect(calculateContentMd5).not.toHaveBeenCalled(); expect(mockUploadPart).toHaveBeenCalledTimes(2); expect(mockUploadPart).toHaveBeenCalledWith( @@ -317,8 +324,7 @@ describe('getMultipartUploadHandlers with key', () => { }, ); - it('should use md5 if crc32 is returning undefined', async () => { - mockCalculateContentCRC32Undefined(); + it('should use md5 if no using crc32', async () => { mockMultipartUploadSuccess(); Amplify.libraryOptions = { Storage: { @@ -372,6 +378,9 @@ describe('getMultipartUploadHandlers with key', () => { { key: defaultKey, data: file, + options: { + checksumAlgorithm: 'crc-32', + }, }, file.size, ); @@ -589,7 +598,7 @@ describe('getMultipartUploadHandlers with key', () => { expect(Object.keys(cacheValue)).toEqual([ expect.stringMatching( // \d{13} is the file lastModified property of a file - /someName_\d{13}_8388608_application\/octet-stream_bucket_public_key/, + /someName_\d{13}_\/twwTw==_8388608_application\/octet-stream_bucket_public_key/, ), ]); }); @@ -800,7 +809,7 @@ describe('getMultipartUploadHandlers with key', () => { >; mockDefaultStorage.getItem.mockResolvedValue( JSON.stringify({ - [defaultCacheKey]: { + [generateDefaultCacheKey('o6a/Qw==')]: { uploadId: 'uploadId', bucket, key: defaultKey, @@ -942,6 +951,9 @@ describe('getMultipartUploadHandlers with path', () => { const { multipartUploadJob } = getMultipartUploadHandlers({ path: testPath, data: twoPartsPayload, + options: { + checksumAlgorithm: 'crc-32', + }, }); await multipartUploadJob(); @@ -952,9 +964,11 @@ describe('getMultipartUploadHandlers with path', () => { * * uploading each part calls calculateContentCRC32 1 time each * - * these steps results in 5 calls in total + * 1 time for optionsHash + * + * these steps results in 6 calls in total */ - expect(calculateContentCRC32).toHaveBeenCalledTimes(5); + expect(calculateContentCRC32).toHaveBeenCalledTimes(6); expect(calculateContentMd5).not.toHaveBeenCalled(); expect(mockUploadPart).toHaveBeenCalledTimes(2); expect(mockUploadPart).toHaveBeenCalledWith( @@ -968,8 +982,7 @@ describe('getMultipartUploadHandlers with path', () => { }, ); - it('should use md5 if crc32 is returning undefined', async () => { - mockCalculateContentCRC32Undefined(); + it('should use md5 if no using crc32', async () => { mockMultipartUploadSuccess(); Amplify.libraryOptions = { Storage: { @@ -1023,6 +1036,9 @@ describe('getMultipartUploadHandlers with path', () => { { path: testPath, data: file, + options: { + checksumAlgorithm: 'crc-32', + }, }, file.size, ); @@ -1533,9 +1549,10 @@ describe('getMultipartUploadHandlers with path', () => { const mockDefaultStorage = defaultStorage as jest.Mocked< typeof defaultStorage >; + mockDefaultStorage.getItem.mockResolvedValue( JSON.stringify({ - [testPathCacheKey]: { + [generateTestPathCacheKey('o6a/Qw==')]: { uploadId: 'uploadId', bucket, key: testPath, diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts index a76871e2435..062b22f0888 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts @@ -15,6 +15,7 @@ import { calculateContentMd5 } from '../../../../../src/providers/s3/utils'; import * as CRC32 from '../../../../../src/providers/s3/utils/crc32'; import { putObjectJob } from '../../../../../src/providers/s3/apis/uploadData/putObjectJob'; import '../testUtils'; +import { UploadDataChecksumAlgorithm } from '../../../../../src/providers/s3/types/options'; global.Blob = BlobPolyfill as any; global.File = FilePolyfill as any; @@ -75,66 +76,75 @@ mockPutObject.mockResolvedValue({ /* TODO Remove suite when `key` parameter is removed */ describe('putObjectJob with key', () => { beforeEach(() => { + mockPutObject.mockClear(); jest.spyOn(CRC32, 'calculateContentCRC32').mockRestore(); }); - it('should supply the correct parameters to putObject API handler', async () => { - const abortController = new AbortController(); - const inputKey = 'key'; - const data = 'data'; - const mockContentType = 'contentType'; - const contentDisposition = 'contentDisposition'; - const contentEncoding = 'contentEncoding'; - const mockMetadata = { key: 'value' }; - const onProgress = jest.fn(); - const useAccelerateEndpoint = true; + it.each<{ checksumAlgorithm: UploadDataChecksumAlgorithm | undefined }>([ + { checksumAlgorithm: 'crc-32' }, + { checksumAlgorithm: undefined }, + ])( + 'should supply the correct parameters to putObject API handler with checksumAlgorithm as $checksumAlgorithm', + async ({ checksumAlgorithm }) => { + const abortController = new AbortController(); + const inputKey = 'key'; + const data = 'data'; + const mockContentType = 'contentType'; + const contentDisposition = 'contentDisposition'; + const contentEncoding = 'contentEncoding'; + const mockMetadata = { key: 'value' }; + const onProgress = jest.fn(); + const useAccelerateEndpoint = true; - const job = putObjectJob( - { + const job = putObjectJob( + { + key: inputKey, + data, + options: { + contentDisposition, + contentEncoding, + contentType: mockContentType, + metadata: mockMetadata, + onProgress, + useAccelerateEndpoint, + checksumAlgorithm, + }, + }, + abortController.signal, + ); + const result = await job(); + expect(result).toEqual({ key: inputKey, - data, - options: { - contentDisposition, - contentEncoding, - contentType: mockContentType, - metadata: mockMetadata, - onProgress, - useAccelerateEndpoint, + eTag: 'eTag', + versionId: 'versionId', + contentType: 'contentType', + metadata: { key: 'value' }, + size: undefined, + }); + expect(mockPutObject).toHaveBeenCalledTimes(1); + await expect(mockPutObject).toBeLastCalledWithConfigAndInput( + { + credentials, + region, + abortSignal: abortController.signal, + onUploadProgress: expect.any(Function), + useAccelerateEndpoint: true, + userAgentValue: expect.any(String), }, - }, - abortController.signal, - ); - const result = await job(); - expect(result).toEqual({ - key: inputKey, - eTag: 'eTag', - versionId: 'versionId', - contentType: 'contentType', - metadata: { key: 'value' }, - size: undefined, - }); - expect(mockPutObject).toHaveBeenCalledTimes(1); - await expect(mockPutObject).toBeLastCalledWithConfigAndInput( - { - credentials, - region, - abortSignal: abortController.signal, - onUploadProgress: expect.any(Function), - useAccelerateEndpoint: true, - userAgentValue: expect.any(String), - }, - { - Bucket: bucket, - Key: `public/${inputKey}`, - Body: data, - ContentType: mockContentType, - ContentDisposition: contentDisposition, - ContentEncoding: contentEncoding, - Metadata: mockMetadata, - ChecksumCRC32: 'rfPzYw==', - }, - ); - }); + { + Bucket: bucket, + Key: `public/${inputKey}`, + Body: data, + ContentType: mockContentType, + ContentDisposition: contentDisposition, + ContentEncoding: contentEncoding, + Metadata: mockMetadata, + ChecksumCRC32: + checksumAlgorithm === 'crc-32' ? 'rfPzYw==' : undefined, + }, + ); + }, + ); it('should set ContentMD5 if object lock is enabled', async () => { jest @@ -193,7 +203,6 @@ describe('putObjectJob with key', () => { Key: 'public/key', Body: data, ContentType: 'application/octet-stream', - ChecksumCRC32: 'rfPzYw==', }, ); }); @@ -225,7 +234,6 @@ describe('putObjectJob with key', () => { Key: 'public/key', Body: data, ContentType: 'application/octet-stream', - ChecksumCRC32: 'rfPzYw==', }, ); }); @@ -238,18 +246,39 @@ describe('putObjectJob with path', () => { jest.spyOn(CRC32, 'calculateContentCRC32').mockRestore(); }); - test.each([ + it.each<{ checksumAlgorithm: UploadDataChecksumAlgorithm | undefined }>([ + { checksumAlgorithm: 'crc-32' }, + { checksumAlgorithm: undefined }, + ]); + + test.each<{ + path: string | (() => string); + expectedKey: string; + checksumAlgorithm: UploadDataChecksumAlgorithm | undefined; + }>([ + { + path: testPath, + expectedKey: testPath, + checksumAlgorithm: 'crc-32', + }, + { + path: () => testPath, + expectedKey: testPath, + checksumAlgorithm: 'crc-32', + }, { path: testPath, expectedKey: testPath, + checksumAlgorithm: undefined, }, { path: () => testPath, expectedKey: testPath, + checksumAlgorithm: undefined, }, ])( - 'should supply the correct parameters to putObject API handler when path is $path', - async ({ path: inputPath, expectedKey }) => { + 'should supply the correct parameters to putObject API handler when path is $path and checksumAlgorithm is $checksumAlgorithm', + async ({ path: inputPath, expectedKey, checksumAlgorithm }) => { const abortController = new AbortController(); const data = 'data'; const mockContentType = 'contentType'; @@ -270,6 +299,7 @@ describe('putObjectJob with path', () => { metadata: mockMetadata, onProgress, useAccelerateEndpoint, + checksumAlgorithm, }, }, abortController.signal, @@ -301,7 +331,8 @@ describe('putObjectJob with path', () => { ContentDisposition: contentDisposition, ContentEncoding: contentEncoding, Metadata: mockMetadata, - ChecksumCRC32: 'rfPzYw==', + ChecksumCRC32: + checksumAlgorithm === 'crc-32' ? 'rfPzYw==' : undefined, }, ); }, @@ -439,7 +470,6 @@ describe('putObjectJob with path', () => { Key: 'path/', Body: data, ContentType: 'application/octet-stream', - ChecksumCRC32: 'rfPzYw==', }, ); }); @@ -471,7 +501,6 @@ describe('putObjectJob with path', () => { Key: 'path/', Body: data, ContentType: 'application/octet-stream', - ChecksumCRC32: 'rfPzYw==', }, ); }); diff --git a/packages/storage/__tests__/providers/s3/utils/crc32.native.test.ts b/packages/storage/__tests__/providers/s3/utils/crc32.native.test.ts deleted file mode 100644 index 0f4c1adce27..00000000000 --- a/packages/storage/__tests__/providers/s3/utils/crc32.native.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { calculateContentCRC32 } from '../../../../src/providers/s3/utils/crc32.native'; - -const MB = 1024 * 1024; -const getBlob = (size: number) => new Blob(['1'.repeat(size)]); - -describe('calculate crc32 native', () => { - it('should return undefined', async () => { - expect(await calculateContentCRC32(getBlob(8 * MB))).toEqual(undefined); - }); -}); diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts index 32462a83545..e54b04dcab7 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts @@ -3,7 +3,11 @@ import { StorageAccessLevel } from '@aws-amplify/core'; -import { ContentDisposition, ResolvedS3Config } from '../../../types/options'; +import { + ContentDisposition, + ResolvedS3Config, + UploadDataChecksumAlgorithm, +} from '../../../types/options'; import { StorageUploadDataPayload } from '../../../../../types'; import { Part, createMultipartUpload } from '../../../utils/client/s3data'; import { logger } from '../../../../../utils'; @@ -30,6 +34,8 @@ interface LoadOrCreateMultipartUploadOptions { metadata?: Record; size?: number; abortSignal?: AbortSignal; + checksumAlgorithm?: UploadDataChecksumAlgorithm; + optionsHash: string; } interface LoadOrCreateMultipartUploadResult { @@ -57,6 +63,8 @@ export const loadOrCreateMultipartUpload = async ({ contentEncoding, metadata, abortSignal, + checksumAlgorithm, + optionsHash, }: LoadOrCreateMultipartUploadOptions): Promise => { const finalKey = keyPrefix !== undefined ? keyPrefix + key : key; @@ -79,6 +87,7 @@ export const loadOrCreateMultipartUpload = async ({ bucket, accessLevel, key, + optionsHash, }); const cachedUploadParts = await findCachedUploadParts({ @@ -99,7 +108,10 @@ export const loadOrCreateMultipartUpload = async ({ finalCrc32: cachedUpload.finalCrc32, }; } else { - const finalCrc32 = await getCombinedCrc32(data, size); + const finalCrc32 = + checksumAlgorithm === 'crc-32' + ? await getCombinedCrc32(data, size) + : undefined; const { UploadId } = await createMultipartUpload( { @@ -133,6 +145,7 @@ export const loadOrCreateMultipartUpload = async ({ bucket, accessLevel, key, + optionsHash, }); await cacheMultipartUpload(uploadCacheKey, { uploadId: UploadId!, @@ -157,12 +170,10 @@ const getCombinedCrc32 = async ( const crc32List: ArrayBuffer[] = []; const dataChunker = getDataChunker(data, size); for (const { data: checkData } of dataChunker) { - const checksumArrayBuffer = (await calculateContentCRC32(checkData)) - ?.checksumArrayBuffer; - if (checksumArrayBuffer === undefined) return undefined; + const { checksumArrayBuffer } = await calculateContentCRC32(checkData); crc32List.push(checksumArrayBuffer); } - return `${(await calculateContentCRC32(new Blob(crc32List)))?.checksum}-${crc32List.length}`; + return `${(await calculateContentCRC32(new Blob(crc32List))).checksum}-${crc32List.length}`; }; diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadCache.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadCache.ts index 9761ee85732..10d7c0ddd5a 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadCache.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadCache.ts @@ -100,6 +100,7 @@ interface UploadsCacheKeyOptions { accessLevel?: StorageAccessLevel; key: string; file?: File; + optionsHash: string; } /** @@ -114,6 +115,7 @@ export const getUploadsCacheKey = ({ bucket, accessLevel, key, + optionsHash, }: UploadsCacheKeyOptions) => { let levelStr; const resolvedContentType = @@ -126,7 +128,7 @@ export const getUploadsCacheKey = ({ levelStr = accessLevel === 'guest' ? 'public' : accessLevel; } - const baseId = `${size}_${resolvedContentType}_${bucket}_${levelStr}_${key}`; + const baseId = `${optionsHash}_${size}_${resolvedContentType}_${bucket}_${levelStr}_${key}`; if (file) { return `${file.name}_${file.lastModified}_${baseId}`; diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts index 0c3379d04d2..ad5db1ad6fc 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts @@ -30,6 +30,7 @@ import { import { getStorageUserAgentValue } from '../../../utils/userAgent'; import { logger } from '../../../../../utils'; import { validateObjectNotExists } from '../validateObjectNotExists'; +import { calculateContentCRC32 } from '../../../utils/crc32'; import { uploadPartExecutor } from './uploadPartExecutor'; import { getUploadsCacheKey, removeCachedUpload } from './uploadCache'; @@ -110,6 +111,10 @@ export const getMultipartUploadHandlers = ( resolvedAccessLevel = resolveAccessLevel(accessLevel); } + const optionsHash = ( + await calculateContentCRC32(JSON.stringify(uploadDataOptions)) + ).checksum; + if (!inProgressUpload) { const { uploadId, cachedParts, finalCrc32 } = await loadOrCreateMultipartUpload({ @@ -125,6 +130,8 @@ export const getMultipartUploadHandlers = ( data, size, abortSignal: abortController.signal, + checksumAlgorithm: uploadDataOptions?.checksumAlgorithm, + optionsHash, }); inProgressUpload = { uploadId, @@ -141,6 +148,7 @@ export const getMultipartUploadHandlers = ( bucket: resolvedBucket!, size, key: objectKey, + optionsHash, }) : undefined; diff --git a/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts b/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts index cbd602580a0..6fbb37d1853 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts @@ -47,10 +47,15 @@ export const putObjectJob = contentType = 'application/octet-stream', preventOverwrite, metadata, + checksumAlgorithm, onProgress, } = uploadDataOptions ?? {}; - const checksumCRC32 = await calculateContentCRC32(data); + const checksumCRC32 = + checksumAlgorithm === 'crc-32' + ? await calculateContentCRC32(data) + : undefined; + const contentMD5 = // check if checksum exists. ex: should not exist in react native !checksumCRC32 && isObjectLockEnabled diff --git a/packages/storage/src/providers/s3/types/options.ts b/packages/storage/src/providers/s3/types/options.ts index d19baf69476..cd7b35af97d 100644 --- a/packages/storage/src/providers/s3/types/options.ts +++ b/packages/storage/src/providers/s3/types/options.ts @@ -194,6 +194,8 @@ export type DownloadDataOptions = CommonOptions & export type DownloadDataOptionsWithKey = ReadOptions & DownloadDataOptions; export type DownloadDataOptionsWithPath = DownloadDataOptions; +export type UploadDataChecksumAlgorithm = 'crc-32'; + export type UploadDataOptions = CommonOptions & TransferOptions & { /** @@ -224,6 +226,8 @@ export type UploadDataOptions = CommonOptions & * @default false */ preventOverwrite?: boolean; + + checksumAlgorithm?: UploadDataChecksumAlgorithm; }; /** @deprecated Use {@link UploadDataOptionsWithPath} instead. */ diff --git a/packages/storage/src/providers/s3/utils/crc32.native.ts b/packages/storage/src/providers/s3/utils/crc32.native.ts deleted file mode 100644 index 389cb5fc87b..00000000000 --- a/packages/storage/src/providers/s3/utils/crc32.native.ts +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { CRC32Checksum } from './crc32'; - -export const calculateContentCRC32 = async ( - content: Blob | string | ArrayBuffer | ArrayBufferView, - _seed = 0, -): Promise => { - return undefined; -}; diff --git a/packages/storage/src/providers/s3/utils/crc32.ts b/packages/storage/src/providers/s3/utils/crc32.ts index 6d9e194c3af..3a876716489 100644 --- a/packages/storage/src/providers/s3/utils/crc32.ts +++ b/packages/storage/src/providers/s3/utils/crc32.ts @@ -12,7 +12,7 @@ export interface CRC32Checksum { export const calculateContentCRC32 = async ( content: Blob | string | ArrayBuffer | ArrayBufferView, seed = 0, -): Promise => { +): Promise => { let internalSeed = seed; let blob: Blob; diff --git a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts index f19be5ddca7..c07888871f6 100644 --- a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts +++ b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts @@ -123,7 +123,7 @@ export const resolveS3ConfigAndInput = async ( apiOptions?.accessLevel ?? defaultAccessLevel ?? DEFAULT_ACCESS_LEVEL; const targetIdentityId = accessLevel === 'protected' - ? (apiOptions?.targetIdentityId ?? identityId) + ? apiOptions?.targetIdentityId ?? identityId : identityId; const keyPrefix = await prefixResolver({ accessLevel, targetIdentityId }); From 8689812370382177125beaf611d235f8b8db0c6e Mon Sep 17 00:00:00 2001 From: Donny Wu Date: Mon, 16 Sep 2024 13:50:35 -0700 Subject: [PATCH 02/10] fix: revert local prettier suggestion --- .../storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts index c07888871f6..f19be5ddca7 100644 --- a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts +++ b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts @@ -123,7 +123,7 @@ export const resolveS3ConfigAndInput = async ( apiOptions?.accessLevel ?? defaultAccessLevel ?? DEFAULT_ACCESS_LEVEL; const targetIdentityId = accessLevel === 'protected' - ? apiOptions?.targetIdentityId ?? identityId + ? (apiOptions?.targetIdentityId ?? identityId) : identityId; const keyPrefix = await prefixResolver({ accessLevel, targetIdentityId }); From 01b3bd5237a82ff8c90362d4fa9506ab469e09e1 Mon Sep 17 00:00:00 2001 From: Donny Wu Date: Mon, 16 Sep 2024 14:23:30 -0700 Subject: [PATCH 03/10] fix: up size limit for storage upload data --- packages/aws-amplify/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index 914bb57b867..5eda692c109 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -497,7 +497,7 @@ "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "21.83 kB" + "limit": "21.88 kB" } ] } From 92bbe33c6085bc512cb9e3ed3a39cc6730a0e40b Mon Sep 17 00:00:00 2001 From: Donny Wu Date: Fri, 20 Sep 2024 14:56:17 -0700 Subject: [PATCH 04/10] feat: react native crc32 --- .../providers/s3/utils/crc32.native.test.ts | 131 ++++++++++++++++++ .../s3/utils/getCombinedCrc32.native.test.ts | 113 +++++++++++++++ .../s3/utils/getCombinedCrc32.test.ts | 113 +++++++++++++++ .../uploadData/multipart/initialUpload.ts | 18 +-- .../src/providers/s3/utils/crc32.native.ts | 77 ++++++++++ .../storage/src/providers/s3/utils/crc32.ts | 4 +- .../s3/utils/getCombinedCrc32.native..ts | 34 +++++ .../providers/s3/utils/getCombinedCrc32.ts | 22 +++ 8 files changed, 493 insertions(+), 19 deletions(-) create mode 100644 packages/storage/__tests__/providers/s3/utils/crc32.native.test.ts create mode 100644 packages/storage/__tests__/providers/s3/utils/getCombinedCrc32.native.test.ts create mode 100644 packages/storage/__tests__/providers/s3/utils/getCombinedCrc32.test.ts create mode 100644 packages/storage/src/providers/s3/utils/crc32.native.ts create mode 100644 packages/storage/src/providers/s3/utils/getCombinedCrc32.native..ts create mode 100644 packages/storage/src/providers/s3/utils/getCombinedCrc32.ts diff --git a/packages/storage/__tests__/providers/s3/utils/crc32.native.test.ts b/packages/storage/__tests__/providers/s3/utils/crc32.native.test.ts new file mode 100644 index 00000000000..e82080345d7 --- /dev/null +++ b/packages/storage/__tests__/providers/s3/utils/crc32.native.test.ts @@ -0,0 +1,131 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + TextDecoder as TextDecoderPolyfill, + TextEncoder as TextEncoderPolyfill, +} from 'node:util'; + +import { calculateContentCRC32 } from '../../../../src/providers/s3/utils/crc32.native'; + +global.TextEncoder = TextEncoderPolyfill as any; +global.TextDecoder = TextDecoderPolyfill as any; + +const MB = 1024 * 1024; +const getBlob = (size: number) => new Blob(['1'.repeat(size)]); +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +describe('calculate crc32', () => { + describe.each([ + { + type: 'file', + size: '4B', + data: new File(['data'], 'someName'), + expected: { + checksum: 'rfPzYw==', + checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer, + seed: 2918445923, + }, + }, + { + type: 'blob', + size: '4B', + data: new Blob(['data']), + expected: { + checksum: 'rfPzYw==', + checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer, + seed: 2918445923, + }, + }, + { + type: 'string', + size: '4B', + data: 'data', + expected: { + checksum: 'rfPzYw==', + checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer, + seed: 2918445923, + }, + }, + { + type: 'arrayBuffer', + size: '4B', + data: new Uint8Array(encoder.encode('data')).buffer, + expected: { + checksum: 'rfPzYw==', + checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer, + seed: 2918445923, + }, + }, + { + type: 'arrayBufferView', + size: '4B', + data: new DataView(encoder.encode('1234 data 5678').buffer, 5, 4), + expected: { + checksum: 'rfPzYw==', + checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer, + seed: 2918445923, + }, + }, + { + type: 'file', + size: '8MB', + data: new File([getBlob(8 * MB)], 'someName'), + expected: { + checksum: '/YBlgg==', + checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer, + seed: 4253050242, + }, + }, + { + type: 'blob', + size: '8MB', + data: getBlob(8 * MB), + expected: { + checksum: '/YBlgg==', + checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer, + seed: 4253050242, + }, + }, + { + type: 'string', + size: '8MB', + data: '1'.repeat(8 * MB), + expected: { + checksum: '/YBlgg==', + checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer, + seed: 4253050242, + }, + }, + { + type: 'arrayBuffer', + size: '8MB', + data: new Uint8Array(encoder.encode('1'.repeat(8 * MB))).buffer, + expected: { + checksum: '/YBlgg==', + checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer, + seed: 4253050242, + }, + }, + { + type: 'arrayBufferView', + size: '8MB', + data: encoder.encode('1'.repeat(8 * MB)), + expected: { + checksum: '/YBlgg==', + checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer, + seed: 4253050242, + }, + }, + ])('output for data type of $type with size $size', ({ data, expected }) => { + it('should match expected checksum results', async () => { + const result = (await calculateContentCRC32(data))!; + expect(result.checksum).toEqual(expected.checksum); + expect(result.seed).toEqual(expected.seed); + expect(decoder.decode(result.checksumArrayBuffer)).toEqual( + decoder.decode(expected.checksumArrayBuffer), + ); + }); + }); +}); diff --git a/packages/storage/__tests__/providers/s3/utils/getCombinedCrc32.native.test.ts b/packages/storage/__tests__/providers/s3/utils/getCombinedCrc32.native.test.ts new file mode 100644 index 00000000000..e6bc4745968 --- /dev/null +++ b/packages/storage/__tests__/providers/s3/utils/getCombinedCrc32.native.test.ts @@ -0,0 +1,113 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Blob as BlobPolyfill, File as FilePolyfill } from 'node:buffer'; +import { WritableStream as WritableStreamPolyfill } from 'node:stream/web'; +import { + TextDecoder as TextDecoderPolyfill, + TextEncoder as TextEncoderPolyfill, +} from 'node:util'; + +import { getCombinedCrc32 } from '../../../../src/providers/s3/utils/getCombinedCrc32.native.'; +import { byteLength } from '../../../../src/providers/s3/apis/uploadData/byteLength'; + +global.Blob = BlobPolyfill as any; +global.File = FilePolyfill as any; +global.WritableStream = WritableStreamPolyfill as any; +global.TextEncoder = TextEncoderPolyfill as any; +global.TextDecoder = TextDecoderPolyfill as any; + +const MB = 1024 * 1024; +const getBlob = (size: number) => new Blob(['1'.repeat(size)]); +const encoder = new TextEncoder(); + +describe('calculate crc32', () => { + describe.each([ + { + type: 'file', + size: '4B', + data: new File(['data'], 'someName'), + expected: { + checksum: 'wu1R0Q==-1', + }, + }, + { + type: 'blob', + size: '4B', + data: new Blob(['data']), + expected: { + checksum: 'wu1R0Q==-1', + }, + }, + { + type: 'string', + size: '4B', + data: 'data', + expected: { + checksum: 'wu1R0Q==-1', + }, + }, + { + type: 'arrayBuffer', + size: '4B', + data: new Uint8Array(encoder.encode('data')).buffer, + expected: { + checksum: 'wu1R0Q==-1', + }, + }, + { + type: 'arrayBufferView', + size: '4B', + data: new DataView(encoder.encode('1234 data 5678').buffer, 5, 4), + expected: { + checksum: 'wu1R0Q==-1', + }, + }, + { + type: 'file', + size: '8MB', + data: new File([getBlob(8 * MB)], 'someName'), + expected: { + checksum: 'hwOICA==-2', + }, + }, + { + type: 'blob', + size: '8MB', + data: getBlob(8 * MB), + expected: { + checksum: 'hwOICA==-2', + }, + }, + { + type: 'string', + size: '8MB', + data: '1'.repeat(8 * MB), + expected: { + checksum: 'hwOICA==-2', + }, + }, + { + type: 'arrayBuffer', + size: '8MB', + data: new Uint8Array(encoder.encode('1'.repeat(8 * MB))).buffer, + expected: { + checksum: 'hwOICA==-2', + }, + }, + { + type: 'arrayBufferView', + size: '8MB', + data: encoder.encode('1'.repeat(8 * MB)), + expected: { + checksum: 'hwOICA==-2', + }, + }, + ])('output for data type of $type with size $size', ({ data, expected }) => { + it('should match expected checksum results', async () => { + expect((await getCombinedCrc32(data, byteLength(data)))!).toEqual( + expected.checksum, + ); + }); + }); +}); diff --git a/packages/storage/__tests__/providers/s3/utils/getCombinedCrc32.test.ts b/packages/storage/__tests__/providers/s3/utils/getCombinedCrc32.test.ts new file mode 100644 index 00000000000..13ee357cd4a --- /dev/null +++ b/packages/storage/__tests__/providers/s3/utils/getCombinedCrc32.test.ts @@ -0,0 +1,113 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Blob as BlobPolyfill, File as FilePolyfill } from 'node:buffer'; +import { WritableStream as WritableStreamPolyfill } from 'node:stream/web'; +import { + TextDecoder as TextDecoderPolyfill, + TextEncoder as TextEncoderPolyfill, +} from 'node:util'; + +import { getCombinedCrc32 } from '../../../../src/providers/s3/utils/getCombinedCrc32'; +import { byteLength } from '../../../../src/providers/s3/apis/uploadData/byteLength'; + +global.Blob = BlobPolyfill as any; +global.File = FilePolyfill as any; +global.WritableStream = WritableStreamPolyfill as any; +global.TextEncoder = TextEncoderPolyfill as any; +global.TextDecoder = TextDecoderPolyfill as any; + +const MB = 1024 * 1024; +const getBlob = (size: number) => new Blob(['1'.repeat(size)]); +const encoder = new TextEncoder(); + +describe('calculate crc32', () => { + describe.each([ + { + type: 'file', + size: '4B', + data: new File(['data'], 'someName'), + expected: { + checksum: 'wu1R0Q==-1', + }, + }, + { + type: 'blob', + size: '4B', + data: new Blob(['data']), + expected: { + checksum: 'wu1R0Q==-1', + }, + }, + { + type: 'string', + size: '4B', + data: 'data', + expected: { + checksum: 'wu1R0Q==-1', + }, + }, + { + type: 'arrayBuffer', + size: '4B', + data: new Uint8Array(encoder.encode('data')).buffer, + expected: { + checksum: 'wu1R0Q==-1', + }, + }, + { + type: 'arrayBufferView', + size: '4B', + data: new DataView(encoder.encode('1234 data 5678').buffer, 5, 4), + expected: { + checksum: 'wu1R0Q==-1', + }, + }, + { + type: 'file', + size: '8MB', + data: new File([getBlob(8 * MB)], 'someName'), + expected: { + checksum: 'hwOICA==-2', + }, + }, + { + type: 'blob', + size: '8MB', + data: getBlob(8 * MB), + expected: { + checksum: 'hwOICA==-2', + }, + }, + { + type: 'string', + size: '8MB', + data: '1'.repeat(8 * MB), + expected: { + checksum: 'hwOICA==-2', + }, + }, + { + type: 'arrayBuffer', + size: '8MB', + data: new Uint8Array(encoder.encode('1'.repeat(8 * MB))).buffer, + expected: { + checksum: 'hwOICA==-2', + }, + }, + { + type: 'arrayBufferView', + size: '8MB', + data: encoder.encode('1'.repeat(8 * MB)), + expected: { + checksum: 'hwOICA==-2', + }, + }, + ])('output for data type of $type with size $size', ({ data, expected }) => { + it('should match expected checksum results', async () => { + expect((await getCombinedCrc32(data, byteLength(data)))!).toEqual( + expected.checksum, + ); + }); + }); +}); diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts index e54b04dcab7..9b174206589 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts @@ -11,15 +11,14 @@ import { import { StorageUploadDataPayload } from '../../../../../types'; import { Part, createMultipartUpload } from '../../../utils/client/s3data'; import { logger } from '../../../../../utils'; -import { calculateContentCRC32 } from '../../../utils/crc32'; import { constructContentDisposition } from '../../../utils/constructContentDisposition'; +import { getCombinedCrc32 } from '../../../utils/getCombinedCrc32'; import { cacheMultipartUpload, findCachedUploadParts, getUploadsCacheKey, } from './uploadCache'; -import { getDataChunker } from './getDataChunker'; interface LoadOrCreateMultipartUploadOptions { s3Config: ResolvedS3Config; @@ -162,18 +161,3 @@ export const loadOrCreateMultipartUpload = async ({ }; } }; - -const getCombinedCrc32 = async ( - data: StorageUploadDataPayload, - size: number | undefined, -) => { - const crc32List: ArrayBuffer[] = []; - const dataChunker = getDataChunker(data, size); - for (const { data: checkData } of dataChunker) { - const { checksumArrayBuffer } = await calculateContentCRC32(checkData); - - crc32List.push(checksumArrayBuffer); - } - - return `${(await calculateContentCRC32(new Blob(crc32List))).checksum}-${crc32List.length}`; -}; diff --git a/packages/storage/src/providers/s3/utils/crc32.native.ts b/packages/storage/src/providers/s3/utils/crc32.native.ts new file mode 100644 index 00000000000..0b898995c28 --- /dev/null +++ b/packages/storage/src/providers/s3/utils/crc32.native.ts @@ -0,0 +1,77 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import crc32 from 'crc-32'; + +import { hexToArrayBuffer, hexToBase64 } from './crc32'; + +const CHUNK_SIZE = 1024 * 1024; // 1MB chunks + +export interface CRC32Checksum { + checksumArrayBuffer: ArrayBuffer; + checksum: string; + seed: number; +} + +export const calculateContentCRC32 = async ( + content: Blob | string | ArrayBuffer | ArrayBufferView, + seed = 0, +): Promise => { + let internalSeed = seed; + + if (content instanceof ArrayBuffer || ArrayBuffer.isView(content)) { + let uint8Array: Uint8Array; + + if (content instanceof ArrayBuffer) { + uint8Array = new Uint8Array(content); + } else { + uint8Array = new Uint8Array( + content.buffer, + content.byteOffset, + content.byteLength, + ); + } + + let offset = 0; + while (offset < uint8Array.length) { + const end = Math.min(offset + CHUNK_SIZE, uint8Array.length); + const chunk = uint8Array.slice(offset, end); + internalSeed = crc32.buf(chunk, internalSeed) >>> 0; + offset = end; + } + } else { + let blob: Blob; + + if (content instanceof Blob) { + blob = content; + } else { + blob = new Blob([content]); + } + + let offset = 0; + while (offset < blob.size) { + const end = Math.min(offset + CHUNK_SIZE, blob.size); + const chunk = blob.slice(offset, end); + const arrayBuffer = await new Promise(resolve => { + const reader = new FileReader(); + reader.onload = () => { + resolve(reader.result as ArrayBuffer); + }; + reader.readAsArrayBuffer(chunk); + }); + const uint8Array = new Uint8Array(arrayBuffer); + + internalSeed = crc32.buf(uint8Array, internalSeed) >>> 0; + + offset = end; + } + } + + const hex = internalSeed.toString(16).padStart(8, '0'); + + return { + checksumArrayBuffer: hexToArrayBuffer(hex), + checksum: hexToBase64(hex), + seed: internalSeed, + }; +}; diff --git a/packages/storage/src/providers/s3/utils/crc32.ts b/packages/storage/src/providers/s3/utils/crc32.ts index 3a876716489..f55fef00d63 100644 --- a/packages/storage/src/providers/s3/utils/crc32.ts +++ b/packages/storage/src/providers/s3/utils/crc32.ts @@ -38,11 +38,11 @@ export const calculateContentCRC32 = async ( }; }; -const hexToArrayBuffer = (hexString: string) => +export const hexToArrayBuffer = (hexString: string) => new Uint8Array((hexString.match(/\w{2}/g)! ?? []).map(h => parseInt(h, 16))) .buffer; -const hexToBase64 = (hexString: string) => +export const hexToBase64 = (hexString: string) => btoa( hexString .match(/\w{2}/g)! diff --git a/packages/storage/src/providers/s3/utils/getCombinedCrc32.native..ts b/packages/storage/src/providers/s3/utils/getCombinedCrc32.native..ts new file mode 100644 index 00000000000..5eae1c28172 --- /dev/null +++ b/packages/storage/src/providers/s3/utils/getCombinedCrc32.native..ts @@ -0,0 +1,34 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { StorageUploadDataPayload } from '../../../types'; +import { getDataChunker } from '../apis/uploadData/multipart/getDataChunker'; + +import { calculateContentCRC32 } from './crc32'; + +export const getCombinedCrc32 = async ( + data: StorageUploadDataPayload, + size: number | undefined, +) => { + const crc32List: Uint8Array[] = []; + const dataChunker = getDataChunker(data, size); + + let totalLength = 0; + for (const { data: checkData } of dataChunker) { + const checksum = new Uint8Array( + (await calculateContentCRC32(checkData)).checksumArrayBuffer, + ); + totalLength += checksum.length; + crc32List.push(checksum); + } + + // Combine all Uint8Arrays into a single Uint8Array + const combinedArray = new Uint8Array(totalLength); + let offset = 0; + for (const crc32Hash of crc32List) { + combinedArray.set(crc32Hash, offset); + offset += crc32Hash.length; + } + + return `${(await calculateContentCRC32(combinedArray.buffer)).checksum}-${crc32List.length}`; +}; diff --git a/packages/storage/src/providers/s3/utils/getCombinedCrc32.ts b/packages/storage/src/providers/s3/utils/getCombinedCrc32.ts new file mode 100644 index 00000000000..a12d189f14e --- /dev/null +++ b/packages/storage/src/providers/s3/utils/getCombinedCrc32.ts @@ -0,0 +1,22 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { StorageUploadDataPayload } from '../../../types'; +import { getDataChunker } from '../apis/uploadData/multipart/getDataChunker'; + +import { calculateContentCRC32 } from './crc32'; + +export const getCombinedCrc32 = async ( + data: StorageUploadDataPayload, + size: number | undefined, +) => { + const crc32List: ArrayBuffer[] = []; + const dataChunker = getDataChunker(data, size); + for (const { data: checkData } of dataChunker) { + const { checksumArrayBuffer } = await calculateContentCRC32(checkData); + + crc32List.push(checksumArrayBuffer); + } + + return `${(await calculateContentCRC32(new Blob(crc32List))).checksum}-${crc32List.length}`; +}; From bc6ce6b6f8d6e595d0664050007439c79a3b3ba2 Mon Sep 17 00:00:00 2001 From: Donny Wu Date: Mon, 23 Sep 2024 16:09:24 -0700 Subject: [PATCH 05/10] fix: up bundle size limit and fix typo --- packages/aws-amplify/package.json | 2 +- .../providers/s3/utils/getCombinedCrc32.native.test.ts | 2 +- .../{getCombinedCrc32.native..ts => getCombinedCrc32.native.ts} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename packages/storage/src/providers/s3/utils/{getCombinedCrc32.native..ts => getCombinedCrc32.native.ts} (100%) diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index 5eda692c109..0282c421b3b 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -497,7 +497,7 @@ "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "21.88 kB" + "limit": "21.89 kB" } ] } diff --git a/packages/storage/__tests__/providers/s3/utils/getCombinedCrc32.native.test.ts b/packages/storage/__tests__/providers/s3/utils/getCombinedCrc32.native.test.ts index e6bc4745968..875c1f90466 100644 --- a/packages/storage/__tests__/providers/s3/utils/getCombinedCrc32.native.test.ts +++ b/packages/storage/__tests__/providers/s3/utils/getCombinedCrc32.native.test.ts @@ -8,7 +8,7 @@ import { TextEncoder as TextEncoderPolyfill, } from 'node:util'; -import { getCombinedCrc32 } from '../../../../src/providers/s3/utils/getCombinedCrc32.native.'; +import { getCombinedCrc32 } from '../../../../src/providers/s3/utils/getCombinedCrc32.native'; import { byteLength } from '../../../../src/providers/s3/apis/uploadData/byteLength'; global.Blob = BlobPolyfill as any; diff --git a/packages/storage/src/providers/s3/utils/getCombinedCrc32.native..ts b/packages/storage/src/providers/s3/utils/getCombinedCrc32.native.ts similarity index 100% rename from packages/storage/src/providers/s3/utils/getCombinedCrc32.native..ts rename to packages/storage/src/providers/s3/utils/getCombinedCrc32.native.ts From 3cae42cd10072e4d942280a3137fa29060804ca2 Mon Sep 17 00:00:00 2001 From: Donny Wu Date: Tue, 24 Sep 2024 15:02:05 -0700 Subject: [PATCH 06/10] feat: add documentation for checksumAlgorithm --- packages/storage/src/providers/s3/types/options.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/storage/src/providers/s3/types/options.ts b/packages/storage/src/providers/s3/types/options.ts index cd7b35af97d..7b196f5a15e 100644 --- a/packages/storage/src/providers/s3/types/options.ts +++ b/packages/storage/src/providers/s3/types/options.ts @@ -226,7 +226,11 @@ export type UploadDataOptions = CommonOptions & * @default false */ preventOverwrite?: boolean; - + /** + * Indicates the algorithm used to create the checksum for the object. + * This checksum can be used as a data integrity check to verify that the data received is the same data that was originally sent. + * @default undefined + */ checksumAlgorithm?: UploadDataChecksumAlgorithm; }; From ebd61096456ce8d09bed9127bcee44d9d95c3ec2 Mon Sep 17 00:00:00 2001 From: Donny Wu Date: Tue, 24 Sep 2024 15:46:55 -0700 Subject: [PATCH 07/10] fix: update bundle size limit --- packages/aws-amplify/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index 41e506e36f0..088f8effc69 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -497,7 +497,7 @@ "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "22.01 kB" + "limit": "22.06 kB" } ] } From 8d721e2f14692b9699153bad01fdce2603069f98 Mon Sep 17 00:00:00 2001 From: Donny Wu Date: Tue, 24 Sep 2024 17:04:34 -0700 Subject: [PATCH 08/10] fix: update bundle size limit --- packages/aws-amplify/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index 088f8effc69..3e1e875cd87 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -497,7 +497,7 @@ "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "22.06 kB" + "limit": "22.07 kB" } ] } From f951902e59f2ebcbf51cca04b78722812a320959 Mon Sep 17 00:00:00 2001 From: Donny Wu Date: Wed, 2 Oct 2024 17:37:31 -0700 Subject: [PATCH 09/10] fix: address pr feedbacks --- .../apis/uploadData/multipartHandlers.test.ts | 13 ++++++++----- .../s3/apis/uploadData/putObjectJob.test.ts | 17 +++++++++++------ .../apis/uploadData/multipart/initialUpload.ts | 3 ++- .../s3/apis/uploadData/putObjectJob.ts | 7 +++++-- .../storage/src/providers/s3/types/options.ts | 4 ++-- .../storage/src/providers/s3/utils/constants.ts | 2 ++ .../src/providers/s3/utils/crc32.native.ts | 2 +- .../storage/src/providers/s3/utils/crc32.ts | 14 ++------------ .../s3/utils/getCombinedCrc32.native.ts | 12 ++++++++++++ .../src/providers/s3/utils/getCombinedCrc32.ts | 12 ++++++++++++ .../storage/src/providers/s3/utils/hexUtils.ts | 14 ++++++++++++++ 11 files changed, 71 insertions(+), 29 deletions(-) create mode 100644 packages/storage/src/providers/s3/utils/hexUtils.ts diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts index e804ee4bf39..0796df5540e 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts @@ -20,7 +20,10 @@ import { StorageValidationErrorCode, validationErrorMap, } from '../../../../../src/errors/types/validation'; -import { UPLOADS_STORAGE_KEY } from '../../../../../src/providers/s3/utils/constants'; +import { + CHECKSUM_ALGORITHM_CRC32, + UPLOADS_STORAGE_KEY, +} from '../../../../../src/providers/s3/utils/constants'; import { byteLength } from '../../../../../src/providers/s3/apis/uploadData/byteLength'; import { CanceledError } from '../../../../../src/errors/CanceledError'; import { StorageOptions } from '../../../../../src/types'; @@ -294,7 +297,7 @@ describe('getMultipartUploadHandlers with key', () => { key: defaultKey, data: twoPartsPayload, options: { - checksumAlgorithm: 'crc-32', + checksumAlgorithm: CHECKSUM_ALGORITHM_CRC32, }, }); await multipartUploadJob(); @@ -379,7 +382,7 @@ describe('getMultipartUploadHandlers with key', () => { key: defaultKey, data: file, options: { - checksumAlgorithm: 'crc-32', + checksumAlgorithm: CHECKSUM_ALGORITHM_CRC32, }, }, file.size, @@ -952,7 +955,7 @@ describe('getMultipartUploadHandlers with path', () => { path: testPath, data: twoPartsPayload, options: { - checksumAlgorithm: 'crc-32', + checksumAlgorithm: CHECKSUM_ALGORITHM_CRC32, }, }); await multipartUploadJob(); @@ -1037,7 +1040,7 @@ describe('getMultipartUploadHandlers with path', () => { path: testPath, data: file, options: { - checksumAlgorithm: 'crc-32', + checksumAlgorithm: CHECKSUM_ALGORITHM_CRC32, }, }, file.size, diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts index 062b22f0888..bc99e69712d 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts @@ -16,6 +16,7 @@ import * as CRC32 from '../../../../../src/providers/s3/utils/crc32'; import { putObjectJob } from '../../../../../src/providers/s3/apis/uploadData/putObjectJob'; import '../testUtils'; import { UploadDataChecksumAlgorithm } from '../../../../../src/providers/s3/types/options'; +import { CHECKSUM_ALGORITHM_CRC32 } from '../../../../../src/providers/s3/utils/constants'; global.Blob = BlobPolyfill as any; global.File = FilePolyfill as any; @@ -81,7 +82,7 @@ describe('putObjectJob with key', () => { }); it.each<{ checksumAlgorithm: UploadDataChecksumAlgorithm | undefined }>([ - { checksumAlgorithm: 'crc-32' }, + { checksumAlgorithm: CHECKSUM_ALGORITHM_CRC32 }, { checksumAlgorithm: undefined }, ])( 'should supply the correct parameters to putObject API handler with checksumAlgorithm as $checksumAlgorithm', @@ -140,7 +141,9 @@ describe('putObjectJob with key', () => { ContentEncoding: contentEncoding, Metadata: mockMetadata, ChecksumCRC32: - checksumAlgorithm === 'crc-32' ? 'rfPzYw==' : undefined, + checksumAlgorithm === CHECKSUM_ALGORITHM_CRC32 + ? 'rfPzYw==' + : undefined, }, ); }, @@ -247,7 +250,7 @@ describe('putObjectJob with path', () => { }); it.each<{ checksumAlgorithm: UploadDataChecksumAlgorithm | undefined }>([ - { checksumAlgorithm: 'crc-32' }, + { checksumAlgorithm: CHECKSUM_ALGORITHM_CRC32 }, { checksumAlgorithm: undefined }, ]); @@ -259,12 +262,12 @@ describe('putObjectJob with path', () => { { path: testPath, expectedKey: testPath, - checksumAlgorithm: 'crc-32', + checksumAlgorithm: CHECKSUM_ALGORITHM_CRC32, }, { path: () => testPath, expectedKey: testPath, - checksumAlgorithm: 'crc-32', + checksumAlgorithm: CHECKSUM_ALGORITHM_CRC32, }, { path: testPath, @@ -332,7 +335,9 @@ describe('putObjectJob with path', () => { ContentEncoding: contentEncoding, Metadata: mockMetadata, ChecksumCRC32: - checksumAlgorithm === 'crc-32' ? 'rfPzYw==' : undefined, + checksumAlgorithm === CHECKSUM_ALGORITHM_CRC32 + ? 'rfPzYw==' + : undefined, }, ); }, diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts index 9b174206589..8a35940abb6 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts @@ -13,6 +13,7 @@ import { Part, createMultipartUpload } from '../../../utils/client/s3data'; import { logger } from '../../../../../utils'; import { constructContentDisposition } from '../../../utils/constructContentDisposition'; import { getCombinedCrc32 } from '../../../utils/getCombinedCrc32'; +import { CHECKSUM_ALGORITHM_CRC32 } from '../../../utils/constants'; import { cacheMultipartUpload, @@ -108,7 +109,7 @@ export const loadOrCreateMultipartUpload = async ({ }; } else { const finalCrc32 = - checksumAlgorithm === 'crc-32' + checksumAlgorithm === CHECKSUM_ALGORITHM_CRC32 ? await getCombinedCrc32(data, size) : undefined; diff --git a/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts b/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts index 6fbb37d1853..f29ccf12ec6 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts @@ -13,7 +13,10 @@ import { import { ItemWithKey, ItemWithPath } from '../../types/outputs'; import { putObject } from '../../utils/client/s3data'; import { getStorageUserAgentValue } from '../../utils/userAgent'; -import { STORAGE_INPUT_KEY } from '../../utils/constants'; +import { + CHECKSUM_ALGORITHM_CRC32, + STORAGE_INPUT_KEY, +} from '../../utils/constants'; import { calculateContentCRC32 } from '../../utils/crc32'; import { constructContentDisposition } from '../../utils/constructContentDisposition'; @@ -52,7 +55,7 @@ export const putObjectJob = } = uploadDataOptions ?? {}; const checksumCRC32 = - checksumAlgorithm === 'crc-32' + checksumAlgorithm === CHECKSUM_ALGORITHM_CRC32 ? await calculateContentCRC32(data) : undefined; diff --git a/packages/storage/src/providers/s3/types/options.ts b/packages/storage/src/providers/s3/types/options.ts index 32017524ac3..57322f34bc0 100644 --- a/packages/storage/src/providers/s3/types/options.ts +++ b/packages/storage/src/providers/s3/types/options.ts @@ -227,8 +227,8 @@ export type UploadDataOptions = CommonOptions & */ preventOverwrite?: boolean; /** - * Indicates the algorithm used to create the checksum for the object. - * This checksum can be used as a data integrity check to verify that the data received is the same data that was originally sent. + * The algorithm used to compute a checksum for the object. Used to verify that the data received by S3 + * matches what was originally sent. Disabled by default. * @default undefined */ checksumAlgorithm?: UploadDataChecksumAlgorithm; diff --git a/packages/storage/src/providers/s3/utils/constants.ts b/packages/storage/src/providers/s3/utils/constants.ts index e96c83c8f3c..dbbb03da72a 100644 --- a/packages/storage/src/providers/s3/utils/constants.ts +++ b/packages/storage/src/providers/s3/utils/constants.ts @@ -25,3 +25,5 @@ export const STORAGE_INPUT_KEY = 'key'; export const STORAGE_INPUT_PATH = 'path'; export const DEFAULT_DELIMITER = '/'; + +export const CHECKSUM_ALGORITHM_CRC32 = 'crc-32'; diff --git a/packages/storage/src/providers/s3/utils/crc32.native.ts b/packages/storage/src/providers/s3/utils/crc32.native.ts index 0b898995c28..3f906eda0c8 100644 --- a/packages/storage/src/providers/s3/utils/crc32.native.ts +++ b/packages/storage/src/providers/s3/utils/crc32.native.ts @@ -3,7 +3,7 @@ import crc32 from 'crc-32'; -import { hexToArrayBuffer, hexToBase64 } from './crc32'; +import { hexToArrayBuffer, hexToBase64 } from './hexUtils'; const CHUNK_SIZE = 1024 * 1024; // 1MB chunks diff --git a/packages/storage/src/providers/s3/utils/crc32.ts b/packages/storage/src/providers/s3/utils/crc32.ts index f55fef00d63..20e7247f593 100644 --- a/packages/storage/src/providers/s3/utils/crc32.ts +++ b/packages/storage/src/providers/s3/utils/crc32.ts @@ -3,6 +3,8 @@ import crc32 from 'crc-32'; +import { hexToArrayBuffer, hexToBase64 } from './hexUtils'; + export interface CRC32Checksum { checksumArrayBuffer: ArrayBuffer; checksum: string; @@ -37,15 +39,3 @@ export const calculateContentCRC32 = async ( seed: internalSeed, }; }; - -export const hexToArrayBuffer = (hexString: string) => - new Uint8Array((hexString.match(/\w{2}/g)! ?? []).map(h => parseInt(h, 16))) - .buffer; - -export const hexToBase64 = (hexString: string) => - btoa( - hexString - .match(/\w{2}/g)! - .map((a: string) => String.fromCharCode(parseInt(a, 16))) - .join(''), - ); diff --git a/packages/storage/src/providers/s3/utils/getCombinedCrc32.native.ts b/packages/storage/src/providers/s3/utils/getCombinedCrc32.native.ts index 5eae1c28172..f33a8689f4a 100644 --- a/packages/storage/src/providers/s3/utils/getCombinedCrc32.native.ts +++ b/packages/storage/src/providers/s3/utils/getCombinedCrc32.native.ts @@ -6,6 +6,18 @@ import { getDataChunker } from '../apis/uploadData/multipart/getDataChunker'; import { calculateContentCRC32 } from './crc32'; +/** + * Calculates a combined CRC32 checksum for the given data. + * + * This function chunks the input data, calculates CRC32 for each chunk, + * and then combines these checksums into a single value. + * + * @async + * @param {StorageUploadDataPayload} data - The data to calculate the checksum for. + * @param {number | undefined} size - The size of each chunk. If undefined, a default chunk size will be used. + * @returns {Promise} A promise that resolves to a string containing the combined CRC32 checksum + * and the number of chunks, separated by a hyphen. + */ export const getCombinedCrc32 = async ( data: StorageUploadDataPayload, size: number | undefined, diff --git a/packages/storage/src/providers/s3/utils/getCombinedCrc32.ts b/packages/storage/src/providers/s3/utils/getCombinedCrc32.ts index a12d189f14e..c7e927b381d 100644 --- a/packages/storage/src/providers/s3/utils/getCombinedCrc32.ts +++ b/packages/storage/src/providers/s3/utils/getCombinedCrc32.ts @@ -6,6 +6,18 @@ import { getDataChunker } from '../apis/uploadData/multipart/getDataChunker'; import { calculateContentCRC32 } from './crc32'; +/** + * Calculates a combined CRC32 checksum for the given data. + * + * This function chunks the input data, calculates CRC32 for each chunk, + * and then combines these checksums into a single value. + * + * @async + * @param {StorageUploadDataPayload} data - The data to calculate the checksum for. + * @param {number | undefined} size - The size of each chunk. If undefined, a default chunk size will be used. + * @returns {Promise} A promise that resolves to a string containing the combined CRC32 checksum + * and the number of chunks, separated by a hyphen. + */ export const getCombinedCrc32 = async ( data: StorageUploadDataPayload, size: number | undefined, diff --git a/packages/storage/src/providers/s3/utils/hexUtils.ts b/packages/storage/src/providers/s3/utils/hexUtils.ts new file mode 100644 index 00000000000..a71f94e9f98 --- /dev/null +++ b/packages/storage/src/providers/s3/utils/hexUtils.ts @@ -0,0 +1,14 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const hexToArrayBuffer = (hexString: string) => + new Uint8Array((hexString.match(/\w{2}/g)! ?? []).map(h => parseInt(h, 16))) + .buffer; + +export const hexToBase64 = (hexString: string) => + btoa( + hexString + .match(/\w{2}/g)! + .map((a: string) => String.fromCharCode(parseInt(a, 16))) + .join(''), + ); From b51a94df3520fd61fa7eeef3c50062b77afac05c Mon Sep 17 00:00:00 2001 From: Donny Wu Date: Wed, 2 Oct 2024 18:11:58 -0700 Subject: [PATCH 10/10] fix: bundle-size limit --- packages/aws-amplify/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index 3e1e875cd87..3b425757b9e 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -497,7 +497,7 @@ "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "22.07 kB" + "limit": "22.14 kB" } ] }