From 366c32e2d87d73210bbd01ca1da55a5899f5a503 Mon Sep 17 00:00:00 2001 From: Venkata Ramyasri Kota <34170013+kvramyasri7@users.noreply.github.com> Date: Thu, 28 Jul 2022 07:23:33 -0700 Subject: [PATCH] feat(@aws-amplify/storage): Access all files from S3 with List API (#10095) Co-authored-by: Aaron S <94858815+stocaaro@users.noreply.github.com> --- .../providers/AWSS3Provider-unit-test.ts | 139 ++++++++++++------ .../storage/src/providers/AWSS3Provider.ts | 80 +++++++--- packages/storage/src/types/AWSS3Provider.ts | 12 +- 3 files changed, 161 insertions(+), 70 deletions(-) diff --git a/packages/storage/__tests__/providers/AWSS3Provider-unit-test.ts b/packages/storage/__tests__/providers/AWSS3Provider-unit-test.ts index ae502413b39..9a004e3c438 100644 --- a/packages/storage/__tests__/providers/AWSS3Provider-unit-test.ts +++ b/packages/storage/__tests__/providers/AWSS3Provider-unit-test.ts @@ -15,7 +15,7 @@ import { Logger, Hub, Credentials, ICredentials } from '@aws-amplify/core'; import * as formatURL from '@aws-sdk/util-format-url'; import { S3Client, - ListObjectsCommand, + ListObjectsV2Command, CreateMultipartUploadCommand, UploadPartCommand, } from '@aws-sdk/client-s3'; @@ -40,26 +40,25 @@ const mockEventEmitter = { removeAllListeners: mockRemoveAllListeners, }; -jest.mock('events', function() { +jest.mock('events', function () { return { EventEmitter: jest.fn().mockImplementation(() => mockEventEmitter), }; }); S3Client.prototype.send = jest.fn(async command => { - if (command instanceof ListObjectsCommand) { + if (command instanceof ListObjectsV2Command) { + const resultObj = { + Key: 'public/path/itemsKey', + ETag: 'etag', + LastModified: 'lastmodified', + Size: 'size', + }; if (command.input.Prefix === 'public/emptyListResultsPath') { return {}; } return { - Contents: [ - { - Key: 'public/path/itemsKey', - ETag: 'etag', - LastModified: 'lastmodified', - Size: 'size', - }, - ], + Contents: [resultObj], }; } return 'data'; @@ -137,7 +136,8 @@ describe('StorageProvider test', () => { const aws_options = { aws_user_files_s3_bucket: 'bucket', aws_user_files_s3_bucket_region: 'region', - aws_user_files_s3_dangerously_connect_to_http_endpoint_for_testing: true, + aws_user_files_s3_dangerously_connect_to_http_endpoint_for_testing: + true, }; const config = storage.configure(aws_options); @@ -311,7 +311,8 @@ describe('StorageProvider test', () => { }); await storage.get('key', { download: true, - progressCallback: ('this is not a function' as unknown) as S3ProviderGetConfig['progressCallback'], // this is intentional + progressCallback: + 'this is not a function' as unknown as S3ProviderGetConfig['progressCallback'], // this is intentional }); expect(loggerSpy).toHaveBeenCalledWith( 'WARN', @@ -729,7 +730,8 @@ describe('StorageProvider test', () => { const storage = new StorageProvider(); storage.configure(options); await storage.put('key', 'object', { - progressCallback: ('hello' as unknown) as S3ProviderGetConfig['progressCallback'], // this is intentional + progressCallback: + 'hello' as unknown as S3ProviderGetConfig['progressCallback'], // this is intentional }); expect(loggerSpy).toHaveBeenCalledWith( 'WARN', @@ -947,6 +949,12 @@ describe('StorageProvider test', () => { }); describe('list test', () => { + const resultObj = { + eTag: 'etag', + key: 'path/itemsKey', + lastModified: 'lastmodified', + size: 'size', + }; test('list object successfully', async () => { jest.spyOn(Credentials, 'get').mockImplementationOnce(() => { return new Promise((res, rej) => { @@ -960,15 +968,11 @@ describe('StorageProvider test', () => { expect.assertions(2); expect(await storage.list('path', { level: 'public' })).toEqual([ - { - eTag: 'etag', - key: 'path/itemsKey', - lastModified: 'lastmodified', - size: 'size', - }, + resultObj, ]); expect(spyon.mock.calls[0][0].input).toEqual({ Bucket: 'bucket', + MaxKeys: 1000, Prefix: 'public/path', }); spyon.mockClear(); @@ -991,6 +995,7 @@ describe('StorageProvider test', () => { ).toEqual([]); expect(spyon.mock.calls[0][0].input).toEqual({ Bucket: 'bucket', + MaxKeys: 1000, Prefix: 'public/emptyListResultsPath', }); spyon.mockClear(); @@ -1011,16 +1016,10 @@ describe('StorageProvider test', () => { expect.assertions(3); expect( await storage.list('path', { level: 'public', track: true }) - ).toEqual([ - { - eTag: 'etag', - key: 'path/itemsKey', - lastModified: 'lastmodified', - size: 'size', - }, - ]); + ).toEqual([resultObj]); expect(spyon.mock.calls[0][0].input).toEqual({ Bucket: 'bucket', + MaxKeys: 1000, Prefix: 'public/path', }); expect(spyon2).toBeCalledWith( @@ -1052,14 +1051,7 @@ describe('StorageProvider test', () => { expect.assertions(2); expect( await storage.list('path', { level: 'public', maxKeys: 1 }) - ).toEqual([ - { - eTag: 'etag', - key: 'path/itemsKey', - lastModified: 'lastmodified', - size: 'size', - }, - ]); + ).toEqual([resultObj]); expect(spyon.mock.calls[0][0].input).toEqual({ Bucket: 'bucket', Prefix: 'public/path', @@ -1070,6 +1062,66 @@ describe('StorageProvider test', () => { curCredSpyOn.mockClear(); }); + test('list object with maxKeys with ALL having 3 pages', async () => { + const curCredSpyOn = jest + .spyOn(Credentials, 'get') + .mockImplementationOnce(() => { + return new Promise((res, rej) => { + res({}); + }); + }); + const storage = new StorageProvider(); + storage.configure(options); + const listResultObj = { + Key: 'public/path/itemsKey', + ETag: 'etag', + LastModified: 'lastmodified', + Size: 'size', + }; + let methodCalls = 0; + let continuationToken = 'TEST_TOKEN'; + let listResult = []; + const listAllFunction = async command => { + if (command instanceof ListObjectsV2Command) { + let token = undefined; + methodCalls++; + if (command.input.ContinuationToken === undefined || methodCalls < 3) + token = continuationToken; + + if (command.input.Prefix === 'public/listALLResultsPath') { + return { + Contents: [listResultObj], + NextContinuationToken: token, + }; + } + } + return 'data'; + }; + S3Client.prototype.send = jest.fn(listAllFunction); + const spyon = jest.spyOn(S3Client.prototype, 'send'); + expect.assertions(5); + for (let i = 0; i < 3; i++) listResult.push(resultObj); + expect( + await storage.list('listALLResultsPath', { + level: 'public', + maxKeys: 'ALL', + }) + ).toEqual(listResult); + expect(spyon).toHaveBeenCalledTimes(3); + const inputResult = { + Bucket: 'bucket', + Prefix: 'public/listALLResultsPath', + MaxKeys: 1000, + ContinuationToken: undefined, + }; + for (let i = 0; i < 3; i++) { + expect(spyon.mock.calls[i][0].input).toEqual(inputResult); + inputResult.ContinuationToken = continuationToken; + } + spyon.mockClear(); + curCredSpyOn.mockClear(); + }); + test('list object failed', async () => { jest.spyOn(Credentials, 'get').mockImplementationOnce(() => { return new Promise((res, rej) => { @@ -1158,13 +1210,10 @@ describe('StorageProvider test', () => { // wrong key type await expect( - storage.copy( - ({ level: 'public', key: 123 } as unknown) as S3CopySource, - { - key: 'dest', - level: 'public', - } - ) + storage.copy({ level: 'public', key: 123 } as unknown as S3CopySource, { + key: 'dest', + level: 'public', + }) ).rejects.toThrowError( 'source param should be an object with the property "key" with value of type string' ); @@ -1188,10 +1237,10 @@ describe('StorageProvider test', () => { // wrong key type await expect( - storage.copy({ key: 'src', level: 'public' }, ({ + storage.copy({ key: 'src', level: 'public' }, { key: 123, level: 'public', - } as unknown) as S3CopyDestination) + } as unknown as S3CopyDestination) ).rejects.toThrowError( 'destination param should be an object with the property "key" with value of type string' ); diff --git a/packages/storage/src/providers/AWSS3Provider.ts b/packages/storage/src/providers/AWSS3Provider.ts index a9d59334eac..53057f436fd 100644 --- a/packages/storage/src/providers/AWSS3Provider.ts +++ b/packages/storage/src/providers/AWSS3Provider.ts @@ -22,13 +22,14 @@ import { S3Client, GetObjectCommand, DeleteObjectCommand, - ListObjectsCommand, + ListObjectsV2Command, GetObjectCommandOutput, DeleteObjectCommandInput, CopyObjectCommandInput, CopyObjectCommand, PutObjectCommandInput, GetObjectCommandInput, + ListObjectsV2Request, } from '@aws-sdk/client-s3'; import { formatUrl } from '@aws-sdk/util-format-url'; import { createRequest } from '@aws-sdk/util-create-request'; @@ -56,6 +57,7 @@ import { S3ProviderPutOutput, ResumableUploadConfig, UploadTask, + S3ClientOptions, } from '../types'; import { StorageErrorStrings } from '../common/StorageErrorStrings'; import { dispatchStorageEvent } from '../common/StorageUtils'; @@ -67,6 +69,7 @@ import { autoAdjustClockskewMiddlewareOptions, createS3Client, } from '../common/S3ClientUtils'; +import { S3ProviderListOutputWithToken } from '.././types/AWSS3Provider'; import { AWSS3ProviderManagedUpload } from './AWSS3ProviderManagedUpload'; import { AWSS3UploadTask, TaskEvents } from './AWSS3UploadTask'; import { UPLOADS_STORAGE_KEY } from '../common/StorageConstants'; @@ -678,6 +681,31 @@ export class AWSS3Provider implements StorageProvider { throw error; } } + private async _list( + params: ListObjectsV2Request, + opt: S3ClientOptions, + prefix: string + ): Promise { + const result: S3ProviderListOutputWithToken = { + contents: [], + nextToken: '', + }; + const s3 = this._createNewS3Client(opt); + const listObjectsV2Command = new ListObjectsV2Command({ ...params }); + const response = await s3.send(listObjectsV2Command); + if (response && response.Contents) { + result.contents = response.Contents.map(item => { + return { + key: item.Key.substr(prefix.length), + eTag: item.ETag, + lastModified: item.LastModified, + size: item.Size, + }; + }); + result.nextToken = response.NextContinuationToken; + } + return result; + } /** * List bucket objects relative to the level and prefix specified @@ -694,34 +722,38 @@ export class AWSS3Provider implements StorageProvider { if (!credentialsOK || !this._isWithCredentials(this._config)) { throw new Error(StorageErrorStrings.NO_CREDENTIALS); } - const opt = Object.assign({}, this._config, config); + const opt: S3ClientOptions = Object.assign({}, this._config, config); const { bucket, track, maxKeys } = opt; - const prefix = this._prefix(opt); const final_path = prefix + path; - const s3 = this._createNewS3Client(opt); logger.debug('list ' + path + ' from ' + final_path); - - const params = { - Bucket: bucket, - Prefix: final_path, - MaxKeys: maxKeys, - }; - - const listObjectsCommand = new ListObjectsCommand(params); - try { - const response = await s3.send(listObjectsCommand); - let list: S3ProviderListOutput = []; - if (response && response.Contents) { - list = response.Contents.map(item => { - return { - key: item.Key.substr(prefix.length), - eTag: item.ETag, - lastModified: item.LastModified, - size: item.Size, - }; - }); + const list: S3ProviderListOutput = []; + let token: string; + let listResult: S3ProviderListOutputWithToken; + const params: ListObjectsV2Request = { + Bucket: bucket, + Prefix: final_path, + MaxKeys: 1000, + }; + if (maxKeys === 'ALL') { + do { + params.ContinuationToken = token; + params.MaxKeys = 1000; + listResult = await this._list(params, opt, prefix); + list.push(...listResult.contents); + if (listResult.nextToken) token = listResult.nextToken; + } while (listResult.nextToken); + } else { + maxKeys < 1000 || typeof maxKeys === 'string' + ? (params.MaxKeys = maxKeys) + : (params.MaxKeys = 1000); + listResult = await this._list(params, opt, prefix); + list.push(...listResult.contents); + if (maxKeys > 1000) + logger.warn( + "maxkeys can be from 0 - 1000 or 'ALL'. To list all files you can set maxKeys to 'ALL'." + ); } dispatchStorageEvent( track, diff --git a/packages/storage/src/types/AWSS3Provider.ts b/packages/storage/src/types/AWSS3Provider.ts index 4ed84827e6f..b8ce6c7592e 100644 --- a/packages/storage/src/types/AWSS3Provider.ts +++ b/packages/storage/src/types/AWSS3Provider.ts @@ -12,6 +12,7 @@ import { UploadTaskProgressEvent, } from '../providers/AWSS3UploadTask'; import { UploadTask } from './Provider'; +import { ICredentials } from '@aws-amplify/core'; type ListObjectsCommandOutputContent = _Object; @@ -94,15 +95,24 @@ export type S3ProviderRemoveConfig = CommonStorageOptions & { provider?: 'AWSS3'; }; +export type S3ProviderListOutputWithToken = { + contents: S3ProviderListOutputItem[]; + nextToken: string; +}; + export type S3ProviderRemoveOutput = DeleteObjectCommandOutput; export type S3ProviderListConfig = CommonStorageOptions & { bucket?: string; - maxKeys?: number; + maxKeys?: number | 'ALL'; provider?: 'AWSS3'; identityId?: string; }; +export type S3ClientOptions = StorageOptions & { + credentials: ICredentials; +} & S3ProviderListConfig; + export interface S3ProviderListOutputItem { key: ListObjectsCommandOutputContent['Key']; eTag: ListObjectsCommandOutputContent['ETag'];