Skip to content

Commit

Permalink
feat(@aws-amplify/storage): Access all files from S3 with List API (a…
Browse files Browse the repository at this point in the history 
…ws-amplify#10095)

Co-authored-by: Aaron S <94858815+stocaaro@users.noreply.github.com>
  • Loading branch information
@stocaaro
Venkata Ramyasri Kota and stocaaro authored Jul 28, 2022
1 parent 0729e68 commit 366c32e
Show file tree
Hide file tree
Showing 3 changed files with 161 additions and 70 deletions.
139 changes: 94 additions & 45 deletions packages/storage/__tests__/providers/AWSS3Provider-unit-test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import { Logger, Hub, Credentials, ICredentials } from '@aws-amplify/core';
import * as formatURL from '@aws-sdk/util-format-url';
import {
S3Client,
ListObjectsCommand,
ListObjectsV2Command,
CreateMultipartUploadCommand,
UploadPartCommand,
} from '@aws-sdk/client-s3';
Expand All @@ -40,26 +40,25 @@ const mockEventEmitter = {
removeAllListeners: mockRemoveAllListeners,
};

jest.mock('events', function() {
jest.mock('events', function () {
return {
EventEmitter: jest.fn().mockImplementation(() => mockEventEmitter),
};
});

S3Client.prototype.send = jest.fn(async command => {
if (command instanceof ListObjectsCommand) {
if (command instanceof ListObjectsV2Command) {
const resultObj = {
Key: 'public/path/itemsKey',
ETag: 'etag',
LastModified: 'lastmodified',
Size: 'size',
};
if (command.input.Prefix === 'public/emptyListResultsPath') {
return {};
}
return {
Contents: [
{
Key: 'public/path/itemsKey',
ETag: 'etag',
LastModified: 'lastmodified',
Size: 'size',
},
],
Contents: [resultObj],
};
}
return 'data';
Expand Down Expand Up @@ -137,7 +136,8 @@ describe('StorageProvider test', () => {
const aws_options = {
aws_user_files_s3_bucket: 'bucket',
aws_user_files_s3_bucket_region: 'region',
aws_user_files_s3_dangerously_connect_to_http_endpoint_for_testing: true,
aws_user_files_s3_dangerously_connect_to_http_endpoint_for_testing:
true,
};

const config = storage.configure(aws_options);
Expand Down Expand Up @@ -311,7 +311,8 @@ describe('StorageProvider test', () => {
});
await storage.get('key', {
download: true,
progressCallback: ('this is not a function' as unknown) as S3ProviderGetConfig['progressCallback'], // this is intentional
progressCallback:
'this is not a function' as unknown as S3ProviderGetConfig['progressCallback'], // this is intentional
});
expect(loggerSpy).toHaveBeenCalledWith(
'WARN',
Expand Down Expand Up @@ -729,7 +730,8 @@ describe('StorageProvider test', () => {
const storage = new StorageProvider();
storage.configure(options);
await storage.put('key', 'object', {
progressCallback: ('hello' as unknown) as S3ProviderGetConfig['progressCallback'], // this is intentional
progressCallback:
'hello' as unknown as S3ProviderGetConfig['progressCallback'], // this is intentional
});
expect(loggerSpy).toHaveBeenCalledWith(
'WARN',
Expand Down Expand Up @@ -947,6 +949,12 @@ describe('StorageProvider test', () => {
});

describe('list test', () => {
const resultObj = {
eTag: 'etag',
key: 'path/itemsKey',
lastModified: 'lastmodified',
size: 'size',
};
test('list object successfully', async () => {
jest.spyOn(Credentials, 'get').mockImplementationOnce(() => {
return new Promise((res, rej) => {
Expand All @@ -960,15 +968,11 @@ describe('StorageProvider test', () => {

expect.assertions(2);
expect(await storage.list('path', { level: 'public' })).toEqual([
{
eTag: 'etag',
key: 'path/itemsKey',
lastModified: 'lastmodified',
size: 'size',
},
resultObj,
]);
expect(spyon.mock.calls[0][0].input).toEqual({
Bucket: 'bucket',
MaxKeys: 1000,
Prefix: 'public/path',
});
spyon.mockClear();
Expand All @@ -991,6 +995,7 @@ describe('StorageProvider test', () => {
).toEqual([]);
expect(spyon.mock.calls[0][0].input).toEqual({
Bucket: 'bucket',
MaxKeys: 1000,
Prefix: 'public/emptyListResultsPath',
});
spyon.mockClear();
Expand All @@ -1011,16 +1016,10 @@ describe('StorageProvider test', () => {
expect.assertions(3);
expect(
await storage.list('path', { level: 'public', track: true })
).toEqual([
{
eTag: 'etag',
key: 'path/itemsKey',
lastModified: 'lastmodified',
size: 'size',
},
]);
).toEqual([resultObj]);
expect(spyon.mock.calls[0][0].input).toEqual({
Bucket: 'bucket',
MaxKeys: 1000,
Prefix: 'public/path',
});
expect(spyon2).toBeCalledWith(
Expand Down Expand Up @@ -1052,14 +1051,7 @@ describe('StorageProvider test', () => {
expect.assertions(2);
expect(
await storage.list('path', { level: 'public', maxKeys: 1 })
).toEqual([
{
eTag: 'etag',
key: 'path/itemsKey',
lastModified: 'lastmodified',
size: 'size',
},
]);
).toEqual([resultObj]);
expect(spyon.mock.calls[0][0].input).toEqual({
Bucket: 'bucket',
Prefix: 'public/path',
Expand All @@ -1070,6 +1062,66 @@ describe('StorageProvider test', () => {
curCredSpyOn.mockClear();
});

test('list object with maxKeys with ALL having 3 pages', async () => {
const curCredSpyOn = jest
.spyOn(Credentials, 'get')
.mockImplementationOnce(() => {
return new Promise((res, rej) => {
res({});
});
});
const storage = new StorageProvider();
storage.configure(options);
const listResultObj = {
Key: 'public/path/itemsKey',
ETag: 'etag',
LastModified: 'lastmodified',
Size: 'size',
};
let methodCalls = 0;
let continuationToken = 'TEST_TOKEN';
let listResult = [];
const listAllFunction = async command => {
if (command instanceof ListObjectsV2Command) {
let token = undefined;
methodCalls++;
if (command.input.ContinuationToken === undefined || methodCalls < 3)
token = continuationToken;

if (command.input.Prefix === 'public/listALLResultsPath') {
return {
Contents: [listResultObj],
NextContinuationToken: token,
};
}
}
return 'data';
};
S3Client.prototype.send = jest.fn(listAllFunction);
const spyon = jest.spyOn(S3Client.prototype, 'send');
expect.assertions(5);
for (let i = 0; i < 3; i++) listResult.push(resultObj);
expect(
await storage.list('listALLResultsPath', {
level: 'public',
maxKeys: 'ALL',
})
).toEqual(listResult);
expect(spyon).toHaveBeenCalledTimes(3);
const inputResult = {
Bucket: 'bucket',
Prefix: 'public/listALLResultsPath',
MaxKeys: 1000,
ContinuationToken: undefined,
};
for (let i = 0; i < 3; i++) {
expect(spyon.mock.calls[i][0].input).toEqual(inputResult);
inputResult.ContinuationToken = continuationToken;
}
spyon.mockClear();
curCredSpyOn.mockClear();
});

test('list object failed', async () => {
jest.spyOn(Credentials, 'get').mockImplementationOnce(() => {
return new Promise((res, rej) => {
Expand Down Expand Up @@ -1158,13 +1210,10 @@ describe('StorageProvider test', () => {

// wrong key type
await expect(
storage.copy(
({ level: 'public', key: 123 } as unknown) as S3CopySource,
{
key: 'dest',
level: 'public',
}
)
storage.copy({ level: 'public', key: 123 } as unknown as S3CopySource, {
key: 'dest',
level: 'public',
})
).rejects.toThrowError(
'source param should be an object with the property "key" with value of type string'
);
Expand All @@ -1188,10 +1237,10 @@ describe('StorageProvider test', () => {

// wrong key type
await expect(
storage.copy({ key: 'src', level: 'public' }, ({
storage.copy({ key: 'src', level: 'public' }, {
key: 123,
level: 'public',
} as unknown) as S3CopyDestination)
} as unknown as S3CopyDestination)
).rejects.toThrowError(
'destination param should be an object with the property "key" with value of type string'
);
Expand Down
80 changes: 56 additions & 24 deletions packages/storage/src/providers/AWSS3Provider.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,14 @@ import {
S3Client,
GetObjectCommand,
DeleteObjectCommand,
ListObjectsCommand,
ListObjectsV2Command,
GetObjectCommandOutput,
DeleteObjectCommandInput,
CopyObjectCommandInput,
CopyObjectCommand,
PutObjectCommandInput,
GetObjectCommandInput,
ListObjectsV2Request,
} from '@aws-sdk/client-s3';
import { formatUrl } from '@aws-sdk/util-format-url';
import { createRequest } from '@aws-sdk/util-create-request';
Expand Down Expand Up @@ -56,6 +57,7 @@ import {
S3ProviderPutOutput,
ResumableUploadConfig,
UploadTask,
S3ClientOptions,
} from '../types';
import { StorageErrorStrings } from '../common/StorageErrorStrings';
import { dispatchStorageEvent } from '../common/StorageUtils';
Expand All @@ -67,6 +69,7 @@ import {
autoAdjustClockskewMiddlewareOptions,
createS3Client,
} from '../common/S3ClientUtils';
import { S3ProviderListOutputWithToken } from '.././types/AWSS3Provider';
import { AWSS3ProviderManagedUpload } from './AWSS3ProviderManagedUpload';
import { AWSS3UploadTask, TaskEvents } from './AWSS3UploadTask';
import { UPLOADS_STORAGE_KEY } from '../common/StorageConstants';
Expand Down Expand Up @@ -678,6 +681,31 @@ export class AWSS3Provider implements StorageProvider {
throw error;
}
}
private async _list(
params: ListObjectsV2Request,
opt: S3ClientOptions,
prefix: string
): Promise<S3ProviderListOutputWithToken> {
const result: S3ProviderListOutputWithToken = {
contents: [],
nextToken: '',
};
const s3 = this._createNewS3Client(opt);
const listObjectsV2Command = new ListObjectsV2Command({ ...params });
const response = await s3.send(listObjectsV2Command);
if (response && response.Contents) {
result.contents = response.Contents.map(item => {
return {
key: item.Key.substr(prefix.length),
eTag: item.ETag,
lastModified: item.LastModified,
size: item.Size,
};
});
result.nextToken = response.NextContinuationToken;
}
return result;
}

/**
* List bucket objects relative to the level and prefix specified
Expand All @@ -694,34 +722,38 @@ export class AWSS3Provider implements StorageProvider {
if (!credentialsOK || !this._isWithCredentials(this._config)) {
throw new Error(StorageErrorStrings.NO_CREDENTIALS);
}
const opt = Object.assign({}, this._config, config);
const opt: S3ClientOptions = Object.assign({}, this._config, config);
const { bucket, track, maxKeys } = opt;

const prefix = this._prefix(opt);
const final_path = prefix + path;
const s3 = this._createNewS3Client(opt);
logger.debug('list ' + path + ' from ' + final_path);

const params = {
Bucket: bucket,
Prefix: final_path,
MaxKeys: maxKeys,
};

const listObjectsCommand = new ListObjectsCommand(params);

try {
const response = await s3.send(listObjectsCommand);
let list: S3ProviderListOutput = [];
if (response && response.Contents) {
list = response.Contents.map(item => {
return {
key: item.Key.substr(prefix.length),
eTag: item.ETag,
lastModified: item.LastModified,
size: item.Size,
};
});
const list: S3ProviderListOutput = [];
let token: string;
let listResult: S3ProviderListOutputWithToken;
const params: ListObjectsV2Request = {
Bucket: bucket,
Prefix: final_path,
MaxKeys: 1000,
};
if (maxKeys === 'ALL') {
do {
params.ContinuationToken = token;
params.MaxKeys = 1000;
listResult = await this._list(params, opt, prefix);
list.push(...listResult.contents);
if (listResult.nextToken) token = listResult.nextToken;
} while (listResult.nextToken);
} else {
maxKeys < 1000 || typeof maxKeys === 'string'
? (params.MaxKeys = maxKeys)
: (params.MaxKeys = 1000);
listResult = await this._list(params, opt, prefix);
list.push(...listResult.contents);
if (maxKeys > 1000)
logger.warn(
"maxkeys can be from 0 - 1000 or 'ALL'. To list all files you can set maxKeys to 'ALL'."
);
}
dispatchStorageEvent(
track,
Expand Down
Loading

0 comments on commit 366c32e

Please sign in to comment.