Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(storage): Add API support for Expected Bucket Owner #13914

Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions packages/storage/src/errors/types/validation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ export enum StorageValidationErrorCode {
InvalidCopyOperationStorageBucket = 'InvalidCopyOperationStorageBucket',
InvalidStorageOperationPrefixInput = 'InvalidStorageOperationPrefixInput',
InvalidStorageOperationInput = 'InvalidStorageOperationInput',
InvalidAWSAccountID = 'InvalidAWSAccountID',
InvalidStoragePathInput = 'InvalidStoragePathInput',
InvalidUploadSource = 'InvalidUploadSource',
ObjectIsTooLarge = 'ObjectIsTooLarge',
Expand Down Expand Up @@ -69,6 +70,9 @@ export const validationErrorMap: AmplifyErrorMap<StorageValidationErrorCode> = {
message:
'Path or key parameter must be specified in the input. Both can not be specified at the same time.',
},
[StorageValidationErrorCode.InvalidAWSAccountID]: {
message: 'Invalid AWS account ID was provided.',
},
[StorageValidationErrorCode.InvalidStorageOperationPrefixInput]: {
message: 'Both path and prefix can not be specified at the same time.',
},
Expand Down
1 change: 1 addition & 0 deletions packages/storage/src/internals/apis/getProperties.ts
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Missing other internals APIs?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With current plan, Jim will work on internals I think next week

Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ export const getProperties = (
useAccelerateEndpoint: input?.options?.useAccelerateEndpoint,
bucket: input?.options?.bucket,
locationCredentialsProvider: input?.options?.locationCredentialsProvider,
expectedBucketOwner: input?.options?.expectedBucketOwner,
},
// Type casting is necessary because `getPropertiesInternal` supports both Gen1 and Gen2 signatures, but here
// given in input can only be Gen2 signature, the return can only ben Gen2 signature.
Expand Down
16 changes: 15 additions & 1 deletion packages/storage/src/providers/s3/apis/internal/copy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { ResolvedS3Config, StorageBucket } from '../../types/options';
import {
isInputWithPath,
resolveS3ConfigAndInput,
validateBucketOwnerID,
validateStorageOperationInput,
} from '../../utils';
import { StorageValidationErrorCode } from '../../../../errors/types/validation';
Expand Down Expand Up @@ -97,7 +98,8 @@ const copyWithPath = async (
destination,
identityId,
);

validateBucketOwnerID(source.expectedBucketOwner);
validateBucketOwnerID(destination.expectedBucketOwner);
const finalCopySource = `${sourceBucket}/${sourcePath}`;
const finalCopyDestination = destinationPath;
logger.debug(`copying "${finalCopySource}" to "${finalCopyDestination}".`);
Expand All @@ -109,6 +111,8 @@ const copyWithPath = async (
s3Config,
notModifiedSince: input.source.notModifiedSince,
eTag: input.source.eTag,
expectedSourceBucketOwner: input.source?.expectedBucketOwner,
expectedBucketOwner: input.destination?.expectedBucketOwner,
});

return { path: finalCopyDestination };
Expand All @@ -128,6 +132,8 @@ export const copyWithKey = async (
!!destination.key,
StorageValidationErrorCode.NoDestinationKey,
);
validateBucketOwnerID(source.expectedBucketOwner);
validateBucketOwnerID(destination.expectedBucketOwner);

const { bucket: sourceBucket, keyPrefix: sourceKeyPrefix } =
await resolveS3ConfigAndInput(amplify, {
Expand Down Expand Up @@ -168,6 +174,8 @@ export const copyWithKey = async (
s3Config,
notModifiedSince: input.source.notModifiedSince,
eTag: input.source.eTag,
expectedSourceBucketOwner: input.source?.expectedBucketOwner,
expectedBucketOwner: input.destination?.expectedBucketOwner,
});

return {
Expand All @@ -182,13 +190,17 @@ const serviceCopy = async ({
s3Config,
notModifiedSince,
eTag,
expectedSourceBucketOwner,
expectedBucketOwner,
}: {
source: string;
destination: string;
bucket: string;
s3Config: ResolvedS3Config;
notModifiedSince?: Date;
eTag?: string;
expectedSourceBucketOwner?: string;
expectedBucketOwner?: string;
}) => {
await copyObject(
{
Expand All @@ -202,6 +214,8 @@ const serviceCopy = async ({
MetadataDirective: 'COPY', // Copies over metadata like contentType as well
CopySourceIfMatch: eTag,
CopySourceIfUnmodifiedSince: notModifiedSince,
ExpectedSourceBucketOwner: expectedSourceBucketOwner,
ExpectedBucketOwner: expectedBucketOwner,
},
);
};
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,11 @@ import { Amplify } from '@aws-amplify/core';
import { StorageAction } from '@aws-amplify/core/internals/utils';

import { resolveS3ConfigAndInput } from '../../utils/resolveS3ConfigAndInput';
import { createDownloadTask, validateStorageOperationInput } from '../../utils';
import {
createDownloadTask,
validateBucketOwnerID,
validateStorageOperationInput,
} from '../../utils';
import { getObject } from '../../utils/client/s3data';
import { getStorageUserAgentValue } from '../../utils/userAgent';
import { logger } from '../../../../utils';
Expand Down Expand Up @@ -48,6 +52,7 @@ const downloadDataJob =
downloadDataInput,
identityId,
);
validateBucketOwnerID(downloadDataOptions?.expectedBucketOwner);
const finalKey =
inputType === STORAGE_INPUT_KEY ? keyPrefix + objectKey : objectKey;
logger.debug(`download ${objectKey} from ${finalKey}.`);
Expand All @@ -72,6 +77,7 @@ const downloadDataJob =
...(downloadDataOptions?.bytesRange && {
Range: `bytes=${downloadDataOptions.bytesRange.start}-${downloadDataOptions.bytesRange.end}`,
}),
ExpectedBucketOwner: downloadDataOptions?.expectedBucketOwner,
},
);
const result = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
} from '../../types';
import {
resolveS3ConfigAndInput,
validateBucketOwnerID,
validateStorageOperationInput,
} from '../../utils';
import { headObject } from '../../utils/client/s3data';
Expand All @@ -31,6 +32,9 @@ export const getProperties = async (
input,
identityId,
);

validateBucketOwnerID(input.options?.expectedBucketOwner);

const finalKey =
inputType === STORAGE_INPUT_KEY ? keyPrefix + objectKey : objectKey;

Expand All @@ -45,6 +49,7 @@ export const getProperties = async (
{
Bucket: bucket,
Key: finalKey,
ExpectedBucketOwner: input.options?.expectedBucketOwner,
},
);

Expand Down
3 changes: 3 additions & 0 deletions packages/storage/src/providers/s3/apis/internal/getUrl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { StorageValidationErrorCode } from '../../../../errors/types/validation'
import { getPresignedGetObjectUrl } from '../../utils/client/s3data';
import {
resolveS3ConfigAndInput,
validateBucketOwnerID,
validateStorageOperationInput,
} from '../../utils';
import { assertValidationError } from '../../../../errors/utils/assertValidationError';
Expand All @@ -34,6 +35,7 @@ export const getUrl = async (
input,
identityId,
);
validateBucketOwnerID(getUrlOptions?.expectedBucketOwner);

const finalKey =
inputType === STORAGE_INPUT_KEY ? keyPrefix + objectKey : objectKey;
Expand Down Expand Up @@ -80,6 +82,7 @@ export const getUrl = async (
...(getUrlOptions?.contentType && {
ResponseContentType: getUrlOptions.contentType,
}),
ExpectedBucketOwner: getUrlOptions?.expectedBucketOwner,
},
),
expiresAt: new Date(Date.now() + urlExpirationInSec * 1000),
Expand Down
3 changes: 3 additions & 0 deletions packages/storage/src/providers/s3/apis/internal/list.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
} from '../../types';
import {
resolveS3ConfigAndInput,
validateBucketOwnerID,
validateStorageOperationInputWithPrefix,
} from '../../utils';
import {
Expand Down Expand Up @@ -64,6 +65,7 @@ export const list = async (
input,
identityId,
);
validateBucketOwnerID(options.expectedBucketOwner);
const isInputWithPrefix = inputType === STORAGE_INPUT_PREFIX;

// @ts-expect-error pageSize and nextToken should not coexist with listAll
Expand All @@ -82,6 +84,7 @@ export const list = async (
MaxKeys: options?.listAll ? undefined : options?.pageSize,
ContinuationToken: options?.listAll ? undefined : options?.nextToken,
Delimiter: getDelimiter(options),
expectedBucketOwner: options?.expectedBucketOwner,
joon-won marked this conversation as resolved.
Show resolved Hide resolved
};
logger.debug(`listing items from "${listParams.Prefix}"`);

Expand Down
3 changes: 3 additions & 0 deletions packages/storage/src/providers/s3/apis/internal/remove.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { StorageAction } from '@aws-amplify/core/internals/utils';
import { RemoveInput, RemoveOutput, RemoveWithPathOutput } from '../../types';
import {
resolveS3ConfigAndInput,
validateBucketOwnerID,
validateStorageOperationInput,
} from '../../utils';
import { deleteObject } from '../../utils/client/s3data';
Expand All @@ -27,6 +28,7 @@ export const remove = async (
input,
identityId,
);
validateBucketOwnerID(input.options?.expectedBucketOwner);

let finalKey;
if (inputType === STORAGE_INPUT_KEY) {
Expand All @@ -45,6 +47,7 @@ export const remove = async (
{
Bucket: bucket,
Key: finalKey,
ExpectedBucketOwner: input.options?.expectedBucketOwner,
},
);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ interface LoadOrCreateMultipartUploadOptions {
metadata?: Record<string, string>;
size?: number;
abortSignal?: AbortSignal;
expectedBucketOwner?: string;
}

interface LoadOrCreateMultipartUploadResult {
Expand Down Expand Up @@ -60,6 +61,7 @@ export const loadOrCreateMultipartUpload = async ({
contentEncoding,
metadata,
abortSignal,
expectedBucketOwner,
}: LoadOrCreateMultipartUploadOptions): Promise<LoadOrCreateMultipartUploadResult> => {
const finalKey = keyPrefix !== undefined ? keyPrefix + key : key;

Expand Down Expand Up @@ -117,6 +119,7 @@ export const loadOrCreateMultipartUpload = async ({
ContentEncoding: contentEncoding,
Metadata: metadata,
ChecksumAlgorithm: finalCrc32 ? 'CRC32' : undefined,
ExpectedBucketOwner: expectedBucketOwner,
},
);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ export const getMultipartUploadHandlers = (
metadata,
preventOverwrite,
onProgress,
expectedBucketOwner,
} = uploadDataOptions ?? {};

finalKey = objectKey;
Expand Down Expand Up @@ -125,6 +126,7 @@ export const getMultipartUploadHandlers = (
data,
size,
abortSignal: abortController.signal,
expectedBucketOwner,
});
inProgressUpload = {
uploadId,
Expand Down Expand Up @@ -181,6 +183,7 @@ export const getMultipartUploadHandlers = (
onProgress: concurrentUploadsProgressTracker.getOnProgressListener(),
isObjectLockEnabled: resolvedS3Options.isObjectLockEnabled,
useCRC32Checksum: Boolean(inProgressUpload.finalCrc32),
expectedBucketOwner,
}),
);
}
Expand Down Expand Up @@ -210,6 +213,7 @@ export const getMultipartUploadHandlers = (
(partA, partB) => partA.PartNumber! - partB.PartNumber!,
),
},
ExpectedBucketOwner: expectedBucketOwner,
},
);

Expand All @@ -219,6 +223,7 @@ export const getMultipartUploadHandlers = (
{
Bucket: resolvedBucket,
Key: finalKey,
ExpectedBucketOwner: expectedBucketOwner,
},
);
if (uploadedObjectSize && uploadedObjectSize !== size) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ interface UploadPartExecutorOptions {
crc32: string | undefined,
): void;
onProgress?(event: TransferProgressEvent): void;
expectedBucketOwner?: string;
}

export const uploadPartExecutor = async ({
Expand All @@ -40,6 +41,7 @@ export const uploadPartExecutor = async ({
onProgress,
isObjectLockEnabled,
useCRC32Checksum,
expectedBucketOwner,
}: UploadPartExecutorOptions) => {
let transferredBytes = 0;
for (const { data, partNumber, size } of dataChunkerGenerator) {
Expand Down Expand Up @@ -85,6 +87,7 @@ export const uploadPartExecutor = async ({
PartNumber: partNumber,
ChecksumCRC32: checksumCRC32?.checksum,
ContentMD5: contentMD5,
ExpectedBucketOwner: expectedBucketOwner,
},
);
transferredBytes += size;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { UploadDataInput, UploadDataWithPathInput } from '../../../types';
import {
calculateContentMd5,
resolveS3ConfigAndInput,
validateBucketOwnerID,
validateStorageOperationInput,
} from '../../../utils';
import { ItemWithKey, ItemWithPath } from '../../../types/outputs';
Expand Down Expand Up @@ -38,6 +39,7 @@ export const putObjectJob =
uploadDataInput,
identityId,
);
validateBucketOwnerID(uploadDataOptions?.expectedBucketOwner);

const finalKey =
inputType === STORAGE_INPUT_KEY ? keyPrefix + objectKey : objectKey;
Expand All @@ -48,6 +50,7 @@ export const putObjectJob =
preventOverwrite,
metadata,
onProgress,
expectedBucketOwner,
} = uploadDataOptions ?? {};

const checksumCRC32 = await calculateContentCRC32(data);
Expand Down Expand Up @@ -81,6 +84,7 @@ export const putObjectJob =
Metadata: metadata,
ContentMD5: contentMD5,
ChecksumCRC32: checksumCRC32?.checksum,
ExpectedBucketOwner: expectedBucketOwner,
},
);

Expand Down
9 changes: 9 additions & 0 deletions packages/storage/src/providers/s3/types/options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,11 @@ interface CommonOptions {
useAccelerateEndpoint?: boolean;

bucket?: StorageBucket;

/**
* The expected owner (i.e. account) of the requested object.
joon-won marked this conversation as resolved.
Show resolved Hide resolved
joon-won marked this conversation as resolved.
Show resolved Hide resolved
*/
expectedBucketOwner?: string;
}

/**
Expand Down Expand Up @@ -235,23 +240,27 @@ export type CopySourceWithKeyOptions = ReadOptions & {
bucket?: StorageBucket;
notModifiedSince?: Date;
eTag?: string;
expectedBucketOwner?: string;
};

/** @deprecated This may be removed in the next major version. */
export type CopyDestinationWithKeyOptions = WriteOptions & {
/** @deprecated This may be removed in the next major version. */
key: string;
bucket?: StorageBucket;
expectedBucketOwner?: string;
};

export interface CopyWithPathSourceOptions {
bucket?: StorageBucket;
notModifiedSince?: Date;
eTag?: string;
expectedBucketOwner?: string;
}

export interface CopyWithPathDestinationOptions {
bucket?: StorageBucket;
expectedBucketOwner?: string;
}

/**
Expand Down
Loading
Loading