feat(storage): add delimiter support (#13480)

* feat: add delimiter input/output types * feat: pass Delimiter parameter to ListObjectsV2 API * chore: add unit tests * chore: bump bunde size * chore: address feedback * chore: fix build * chore: address feedback * chore: address feedback * chore: address feedback
aws-amplify · Jun 10, 2024 · 92e6347 · 92e6347
1 parent 092e25c
commit 92e6347
Show file tree

Hide file tree

Showing 5 changed files with 161 additions and 14 deletions.
diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json
@@ -485,7 +485,7 @@
 			"name": "[Storage] list (S3)",
 			"path": "./dist/esm/storage/index.mjs",
 			"import": "{ list }",
-			"limit": "14.94 kB"
+			"limit": "15.04 kB"
 		},
 		{
 			"name": "[Storage] remove (S3)",

diff --git a/packages/storage/__tests__/providers/s3/apis/list.test.ts b/packages/storage/__tests__/providers/s3/apis/list.test.ts
@@ -512,4 +512,116 @@ describe('list API', () => {
 			}
 		});
 	});
+
+	describe('with delimiter', () => {
+		const mockedContents = [
+			{
+				Key: 'photos/',
+				...listObjectClientBaseResultItem,
+			},
+			{
+				Key: 'photos/2023.png',
+				...listObjectClientBaseResultItem,
+			},
+			{
+				Key: 'photos/2024.png',
+				...listObjectClientBaseResultItem,
+			},
+		];
+		const mockedCommonPrefixes = [
+			{ Prefix: 'photos/2023/' },
+			{ Prefix: 'photos/2024/' },
+			{ Prefix: 'photos/2025/' },
+		];
+
+		const mockedPath = 'photos/';
+
+		beforeEach(() => {
+			mockListObject.mockResolvedValueOnce({
+				Contents: mockedContents,
+				CommonPrefixes: mockedCommonPrefixes,
+			});
+		});
+		afterEach(() => {
+			jest.clearAllMocks();
+			mockListObject.mockClear();
+		});
+
+		it('should return subpaths when delimiter is passed in the request', async () => {
+			const { items, subpaths } = await list({
+				path: mockedPath,
+				options: {
+					delimiter: '/',
+				},
+			});
+			expect(items).toHaveLength(3);
+			expect(subpaths).toEqual([
+				'photos/2023/',
+				'photos/2024/',
+				'photos/2025/',
+			]);
+			expect(listObjectsV2).toHaveBeenCalledTimes(1);
+			await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
+				listObjectClientConfig,
+				{
+					Bucket: bucket,
+					MaxKeys: 1000,
+					Prefix: mockedPath,
+					Delimiter: '/',
+				},
+			);
+		});
+
+		it('should return subpaths when delimiter and listAll are passed in the request', async () => {
+			const { items, subpaths } = await list({
+				path: mockedPath,
+				options: {
+					delimiter: '/',
+					listAll: true,
+				},
+			});
+			expect(items).toHaveLength(3);
+			expect(subpaths).toEqual([
+				'photos/2023/',
+				'photos/2024/',
+				'photos/2025/',
+			]);
+			expect(listObjectsV2).toHaveBeenCalledTimes(1);
+			await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
+				listObjectClientConfig,
+				{
+					Bucket: bucket,
+					MaxKeys: 1000,
+					Prefix: mockedPath,
+					Delimiter: '/',
+				},
+			);
+		});
+
+		it('should return subpaths when delimiter is pageSize are passed in the request', async () => {
+			const { items, subpaths } = await list({
+				path: mockedPath,
+				options: {
+					delimiter: '/',
+					pageSize: 3,
+				},
+			});
+			expect(items).toHaveLength(3);
+			expect(subpaths).toEqual([
+				'photos/2023/',
+				'photos/2024/',
+				'photos/2025/',
+			]);
+			expect(listObjectsV2).toHaveBeenCalledTimes(1);
+			await expect(listObjectsV2).toBeLastCalledWithConfigAndInput(
+				listObjectClientConfig,
+				{
+					Bucket: bucket,
+					MaxKeys: 3,
+					Prefix: mockedPath,
+					Delimiter: '/',
+				},
+			);
+		});
+	});
 });
diff --git a/packages/storage/src/providers/s3/apis/internal/list.ts b/packages/storage/src/providers/s3/apis/internal/list.ts
@@ -29,6 +29,7 @@ import {
 import { getStorageUserAgentValue } from '../../utils/userAgent';
 import { logger } from '../../../../utils';
 import { STORAGE_INPUT_PREFIX } from '../../utils/constants';
+import { CommonPrefix } from '../../utils/client/types';
 
 const MAX_PAGE_SIZE = 1000;
 
@@ -79,6 +80,7 @@ export const list = async (
 		Prefix: isInputWithPrefix ? `${generatedPrefix}${objectKey}` : objectKey,
 		MaxKeys: options?.listAll ? undefined : options?.pageSize,
 		ContinuationToken: options?.listAll ? undefined : options?.nextToken,
+		Delimiter: options?.delimiter,
 	};
 	logger.debug(`listing items from "${listParams.Prefix}"`);
 
@@ -176,23 +178,29 @@ const _listAllWithPath = async ({
 	listParams,
 }: ListInputArgs): Promise<ListAllWithPathOutput> => {
 	const listResult: ListOutputItemWithPath[] = [];
+	const subpaths: string[] = [];
 	let continuationToken = listParams.ContinuationToken;
 	do {
-		const { items: pageResults, nextToken: pageNextToken } =
-			await _listWithPath({
-				s3Config,
-				listParams: {
-					...listParams,
-					ContinuationToken: continuationToken,
-					MaxKeys: MAX_PAGE_SIZE,
-				},
-			});
+		const {
+			items: pageResults,
+			subpaths: pageSubpaths,
+			nextToken: pageNextToken,
+		} = await _listWithPath({
+			s3Config,
+			listParams: {
+				...listParams,
+				ContinuationToken: continuationToken,
+				MaxKeys: MAX_PAGE_SIZE,
+			},
+		});
 		listResult.push(...pageResults);
+		subpaths.push(...(pageSubpaths ?? []));
 		continuationToken = pageNextToken;
 	} while (continuationToken);
 
 	return {
 		items: listResult,
+		...parseSubpaths(subpaths),
 	};
 };
 
@@ -206,27 +214,47 @@ const _listWithPath = async ({
 		listParamsClone.MaxKeys = MAX_PAGE_SIZE;
 	}
 
-	const response: ListObjectsV2Output = await listObjectsV2(
+	const {
+		Contents: contents,
+		NextContinuationToken: nextContinuationToken,
+		CommonPrefixes: commonPrefixes,
+	}: ListObjectsV2Output = await listObjectsV2(
 		{
 			...s3Config,
 			userAgentValue: getStorageUserAgentValue(StorageAction.List),
 		},
 		listParamsClone,
 	);
 
-	if (!response?.Contents) {
+	const subpaths = mapCommonPrefixesToSubpaths(commonPrefixes);
+
+	if (!contents) {
 		return {
 			items: [],
+			...parseSubpaths(subpaths),
 		};
 	}
 
 	return {
-		items: response.Contents.map(item => ({
+		items: contents.map(item => ({
 			path: item.Key!,
 			eTag: item.ETag,
 			lastModified: item.LastModified,
 			size: item.Size,
 		})),
-		nextToken: response.NextContinuationToken,
+		nextToken: nextContinuationToken,
+		...parseSubpaths(subpaths),
 	};
 };
+
+function mapCommonPrefixesToSubpaths(
+	commonPrefixes?: CommonPrefix[],
+): string[] | undefined {
+	const mappedSubpaths = commonPrefixes?.map(({ Prefix }) => Prefix);
+
+	return mappedSubpaths?.filter((subpath): subpath is string => !!subpath);
+}
+
+function parseSubpaths(subpaths?: string[]) {
+	return subpaths && subpaths.length > 0 ? { subpaths } : {};
+}
diff --git a/packages/storage/src/types/options.ts b/packages/storage/src/types/options.ts
@@ -10,12 +10,14 @@ export interface StorageOptions {
 
 export type StorageListAllOptions = StorageOptions & {
 	listAll: true;
+	delimiter?: string;
 };
 
 export type StorageListPaginateOptions = StorageOptions & {
 	listAll?: false;
 	pageSize?: number;
 	nextToken?: string;
+	delimiter?: string;
 };
 
 export type StorageRemoveOptions = StorageOptions;
diff --git a/packages/storage/src/types/outputs.ts b/packages/storage/src/types/outputs.ts
@@ -70,4 +70,9 @@ export interface StorageListOutput<Item extends StorageItem> {
 	 * List of items returned by the list API.
 	 */
 	items: Item[];
+	/**
+	 * List of subpaths returned by the list API when a delimiter option is passed
+	 * in the request of the list API.
+	 */
+	subpaths?: string[];
 }