Skip to content

Commit

Permalink
feat(asset-server-plugin): Update s3 asset storage strategy to use AW…
Browse files Browse the repository at this point in the history 
…S sdk v3 (#2102)

v2 is deprecated and v3 is re-architected to be modular.

BREAKING CHANGE: If you are using the s3 storage strategy of the AssetServerPlugin, it has been updated to use v3 of the AWS SDKs. This update introduces [an improved modular architecture to the AWS sdk](https://aws.amazon.com/blogs/developer/modular-packages-in-aws-sdk-for-javascript/), resulting in smaller bundle sizes. You need to install the `@aws-sdk/client-s3` & `@aws-sdk/lib-storage` packages, and can remove the `aws-sdk` package.
  • Loading branch information
@asonnleitner
asonnleitner authored Apr 28, 2023
1 parent 7091a12 commit d628659
Show file tree
Hide file tree
Showing 3 changed files with 1,115 additions and 138 deletions.
2 changes: 2 additions & 0 deletions packages/asset-server-plugin/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
"access": "public"
},
"devDependencies": {
"@aws-sdk/client-s3": "^3.312.0",
"@aws-sdk/lib-storage": "^3.312.0",
"@types/express": "^4.17.8",
"@types/fs-extra": "^9.0.8",
"@types/node-fetch": "^2.5.8",
Expand Down
257 changes: 123 additions & 134 deletions packages/asset-server-plugin/src/s3-asset-storage-strategy.ts
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,14 @@
import { PutObjectRequest, S3ClientConfig } from '@aws-sdk/client-s3';
import { AwsCredentialIdentity, AwsCredentialIdentityProvider } from '@aws-sdk/types';
import { AssetStorageStrategy, Logger } from '@vendure/core';
import { Request } from 'express';
import * as path from 'path';
import { Readable, Stream } from 'stream';
import * as path from 'node:path';
import { Readable } from 'node:stream';

import { getAssetUrlPrefixFn } from './common';
import { loggerCtx } from './constants';
import { AssetServerOptions } from './types';

export type S3Credentials = {
accessKeyId: string;
secretAccessKey: string;
};

export type S3CredentialsProfile = {
profile: string;
};

/**
* @description
* Configuration for connecting to AWS S3.
Expand All @@ -26,12 +19,12 @@ export type S3CredentialsProfile = {
export interface S3Config {
/**
* @description
* The credentials used to access your s3 account. You can supply either the access key ID & secret,
* or you can make use of a
* The credentials used to access your s3 account. You can supply either the access key ID & secret, or you can make use of a
* [shared credentials file](https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-shared.html)
* and supply the profile name (e.g. `'default'`).
* To use a shared credentials file, import the `fromIni()` function from the "@aws-sdk/credential-provider-ini" or "@aws-sdk/credential-providers" package and supply
* the profile name (e.g. `{ profile: 'default' }`) as its argument.
*/
credentials?: S3Credentials | S3CredentialsProfile;
credentials: AwsCredentialIdentity | AwsCredentialIdentityProvider;
/**
* @description
* The S3 bucket in which to store the assets. If it does not exist, it will be created on startup.
Expand All @@ -58,16 +51,17 @@ export interface S3Config {
* Returns a configured instance of the {@link S3AssetStorageStrategy} which can then be passed to the {@link AssetServerOptions}
* `storageStrategyFactory` property.
*
* Before using this strategy, make sure you have the `aws-sdk` package installed:
* Before using this strategy, make sure you have the `@aws-sdk/client-s3` and `@aws-sdk/lib-storage` package installed:
*
* ```sh
* npm install aws-sdk
* npm install @aws-sdk/client-s3 @aws-sdk/lib-storage
* ```
*
* @example
* ```TypeScript
* import { AssetServerPlugin, configureS3AssetStorage } from '\@vendure/asset-server-plugin';
* import { DefaultAssetNamingStrategy } from '\@vendure/core';
* import { fromEnv } from '\@aws-sdk/credential-providers';
*
* // ...
*
Expand All @@ -78,10 +72,7 @@ export interface S3Config {
* namingStrategy: new DefaultAssetNamingStrategy(),
* storageStrategyFactory: configureS3AssetStorage({
* bucket: 'my-s3-bucket',
* credentials: {
* accessKeyId: process.env.AWS_ACCESS_KEY_ID,
* secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
* },
* credentials: fromEnv(), // or any other credential provider
* }),
* }),
* ```
Expand Down Expand Up @@ -121,7 +112,6 @@ export interface S3Config {
*/
export function configureS3AssetStorage(s3Config: S3Config) {
return (options: AssetServerOptions) => {
const { assetUrlPrefix, route } = options;
const prefixFn = getAssetUrlPrefixFn(options);
const toAbsoluteUrlFn = (request: Request, identifier: string): string => {
if (!identifier) {
Expand All @@ -138,12 +128,12 @@ export function configureS3AssetStorage(s3Config: S3Config) {
* @description
* An {@link AssetStorageStrategy} which uses [Amazon S3](https://aws.amazon.com/s3/) object storage service.
* To us this strategy you must first have access to an AWS account.
* See their [getting started guide](https://aws.amazon.com/s3/getting-started/?nc=sn&loc=5) for how to get set up.
* See their [getting started guide](https://aws.amazon.com/s3/getting-started/) for how to get set up.
*
* Before using this strategy, make sure you have the `aws-sdk` package installed:
* Before using this strategy, make sure you have the `@aws-sdk/client-s3` and `@aws-sdk/lib-storage` package installed:
*
* ```sh
* npm install aws-sdk
* npm install @aws-sdk/client-s3 @aws-sdk/lib-storage
* ```
*
* **Note:** Rather than instantiating this manually, use the {@link configureS3AssetStorage} function.
Expand All @@ -157,113 +147,113 @@ export function configureS3AssetStorage(s3Config: S3Config) {
* @docsWeight 0
*/
export class S3AssetStorageStrategy implements AssetStorageStrategy {
private AWS: typeof import('aws-sdk');
private s3: import('aws-sdk').S3;
constructor(
private s3Config: S3Config,
public readonly toAbsoluteUrl: (request: Request, identifier: string) => string,
) {}
private AWS: typeof import('@aws-sdk/client-s3');
private libStorage: typeof import('@aws-sdk/lib-storage');
private s3Client: import('@aws-sdk/client-s3').S3Client;

constructor(private s3Config: S3Config, public readonly toAbsoluteUrl: (request: Request, identifier: string) => string) {}

async init() {
try {
this.AWS = await import('aws-sdk');
} catch (e: any) {
Logger.error(
'Could not find the "aws-sdk" package. Make sure it is installed',
loggerCtx,
e.stack,
);
this.AWS = await import('@aws-sdk/client-s3');
} catch (err: any) {
Logger.error('Could not find the "@aws-sdk/client-s3" package. Make sure it is installed', loggerCtx, err.stack);
}

try {
this.libStorage = await import('@aws-sdk/lib-storage');
} catch (err: any) {
Logger.error('Could not find the "@aws-sdk/lib-storage" package. Make sure it is installed', loggerCtx, err.stack);
}

const config = {
credentials: this.getS3Credentials(),
...this.s3Config.nativeS3Configuration,
};
this.s3 = new this.AWS.S3(config);
await this.ensureBucket(this.s3Config.bucket);
credentials: await this.getCredentials() // Avoid credentials overriden by nativeS3Configuration
} satisfies S3ClientConfig

this.s3Client = new this.AWS.S3Client(config);

await this.ensureBucket();
}

destroy?: (() => void | Promise<void>) | undefined;

async writeFileFromBuffer(fileName: string, data: Buffer): Promise<string> {
const result = await this.s3
.upload(
{
Bucket: this.s3Config.bucket,
Key: fileName,
Body: data,
},
this.s3Config.nativeS3UploadConfiguration,
)
.promise();
return result.Key;
async writeFileFromBuffer(fileName: string, data: Buffer) {
return this.writeFile(fileName, data);
}

async writeFileFromStream(fileName: string, data: Stream): Promise<string> {
const result = await this.s3
.upload(
{
Bucket: this.s3Config.bucket,
Key: fileName,
Body: data,
},
this.s3Config.nativeS3UploadConfiguration,
)
.promise();
return result.Key;
async writeFileFromStream(fileName: string, data: Readable) {
return this.writeFile(fileName, data);
}

async readFileToBuffer(identifier: string): Promise<Buffer> {
const result = await this.s3.getObject(this.getObjectParams(identifier)).promise();
const body = result.Body;
async readFileToBuffer(identifier: string) {
const body = await this.readFile(identifier);

if (!body) {
Logger.error(`Got undefined Body for ${identifier}`, loggerCtx);
return Buffer.from('');
}
if (body instanceof Buffer) {
return body;
}
if (body instanceof Uint8Array || typeof body === 'string') {
return Buffer.from(body);
}
if (body instanceof Readable) {
return new Promise((resolve, reject) => {
const buf: any[] = [];
body.on('data', data => buf.push(data));
body.on('error', err => reject(err));
body.on('end', () => {
const intArray = Uint8Array.from(buf);
resolve(Buffer.concat([intArray]));
});
});

const chunks: Buffer[] = [];
for await (const chunk of body) {
chunks.push(chunk);
}
return Buffer.from(body as any);

return Buffer.concat(chunks);
}

async readFileToStream(identifier: string): Promise<Stream> {
const result = await this.s3.getObject(this.getObjectParams(identifier)).promise();
const body = result.Body;
if (!(body instanceof Stream)) {
const readable = new Readable();
readable._read = () => {
/* noop */
};
readable.push(body);
readable.push(null);
return readable;
async readFileToStream(identifier: string) {
const body = await this.readFile(identifier);

if (!body) {
return new Readable({ read() { this.push(null); } });
}

return body;
}

async deleteFile(identifier: string): Promise<void> {
await this.s3.deleteObject(this.getObjectParams(identifier)).promise();
private async readFile(identifier: string) {
const { GetObjectCommand } = this.AWS;

const result = await this.s3Client.send(new GetObjectCommand(this.getObjectParams(identifier)));
return result.Body as Readable | undefined;
}

async fileExists(fileName: string): Promise<boolean> {
private async writeFile(fileName: string, data: PutObjectRequest['Body'] | string | Uint8Array | Buffer) {
const { Upload } = this.libStorage

const upload = new Upload({
client: this.s3Client,
params: {
...this.s3Config.nativeS3UploadConfiguration,
Bucket: this.s3Config.bucket,
Key: fileName,
Body: data,
},
});

return upload.done().then((result) => {
if (!('Key' in result) || !result.Key) {
Logger.error(`Got undefined Key for ${fileName}`, loggerCtx);
throw new Error(`Got undefined Key for ${fileName}`);
}

return result.Key;
});
}

async deleteFile(identifier: string) {
const { DeleteObjectCommand } = this.AWS
await this.s3Client.send(new DeleteObjectCommand(this.getObjectParams(identifier)));
}

async fileExists(fileName: string) {
const { HeadObjectCommand } = this.AWS

try {
await this.s3.headObject(this.getObjectParams(fileName)).promise();
await this.s3Client.send(new HeadObjectCommand(this.getObjectParams(fileName)));
return true;
} catch (e: any) {
} catch (err: any) {
return false;
}
}
Expand All @@ -275,44 +265,43 @@ export class S3AssetStorageStrategy implements AssetStorageStrategy {
};
}

private getS3Credentials() {
const { credentials } = this.s3Config;
if (credentials == null) {
return null;
} else if (this.isCredentialsProfile(credentials)) {
return new this.AWS.SharedIniFileCredentials(credentials);
}
return new this.AWS.Credentials(credentials);
}
private async ensureBucket(bucket = this.s3Config.bucket) {
const { HeadBucketCommand, CreateBucketCommand } = this.AWS

private async ensureBucket(bucket: string) {
let bucketExists = false;
try {
await this.s3.headBucket({ Bucket: this.s3Config.bucket }).promise();
bucketExists = true;
await this.s3Client.send(new HeadBucketCommand({ Bucket: bucket }));
Logger.verbose(`Found S3 bucket "${bucket}"`, loggerCtx);
} catch (e: any) {
Logger.verbose(
`Could not find bucket "${bucket}: ${JSON.stringify(e.message)}". Attempting to create...`,
);
return
} catch (err: any) {
Logger.verbose(`Could not find bucket "${bucket}: ${JSON.stringify(err.message)}". Attempting to create...`);
}
if (!bucketExists) {
try {
await this.s3.createBucket({ Bucket: bucket, ACL: 'private' }).promise();
Logger.verbose(`Created S3 bucket "${bucket}"`, loggerCtx);
} catch (e: any) {
Logger.error(
`Could not find nor create the S3 bucket "${bucket}: ${JSON.stringify(e.message)}"`,
loggerCtx,
e.stack,
);
}

try {
await this.s3Client.send(new CreateBucketCommand({Bucket: bucket, ACL: 'private'}));
Logger.verbose(`Created S3 bucket "${bucket}"`, loggerCtx);
} catch (err: any) {
Logger.error(`Could not find nor create the S3 bucket "${bucket}: ${JSON.stringify(err.message)}"`, loggerCtx, err.stack);
}
}

private async getCredentials() {
if (this.s3Config.credentials == null) {
return undefined;
}

if (this.isCredentialsProfile(this.s3Config.credentials)) {
Logger.warn(
'The "profile" property of the "s3Config.credentials" is deprecated. ' +
'Please use the "fromIni()" function from the "@aws-sdk/credential-provider-ini" or "@aws-sdk/credential-providers" package instead.',
loggerCtx
);
return (await import('@aws-sdk/credential-provider-ini')).fromIni({ profile: this.s3Config.credentials.profile });
}

return this.s3Config.credentials
}

private isCredentialsProfile(
credentials: S3Credentials | S3CredentialsProfile,
): credentials is S3CredentialsProfile {
return credentials.hasOwnProperty('profile');
private isCredentialsProfile(credentials: AwsCredentialIdentity | AwsCredentialIdentityProvider,): credentials is AwsCredentialIdentity & { profile: string } {
return credentials !== null && typeof credentials === 'object' && 'profile' in credentials && Object.keys(credentials).length === 1;
}
}
Loading

0 comments on commit d628659

Please sign in to comment.