From 0bf2e0901aa09587051316acdaa0e7a7acb6736b Mon Sep 17 00:00:00 2001 From: Ruslan Konviser Date: Wed, 27 Mar 2024 13:39:57 +0100 Subject: [PATCH 01/10] feat: improve storage providers to support forcePathStyle setting --- packages/contracts/src/file-provider.ts | 11 +++++--- .../file-storage/providers/s3.provider.ts | 21 ++++++++++++-- .../providers/wasabi-s3.provider.ts | 25 +++++++++++++++-- .../dto/aws-s3-provider-config.dto.ts | 26 ++++++++++------- .../dto/wasabi-s3-provider-config.dto.ts | 28 +++++++++++-------- .../tenant-setting/tenant-setting.service.ts | 7 ++++- 6 files changed, 87 insertions(+), 31 deletions(-) diff --git a/packages/contracts/src/file-provider.ts b/packages/contracts/src/file-provider.ts index 61f79a77daf..5126f719be7 100644 --- a/packages/contracts/src/file-provider.ts +++ b/packages/contracts/src/file-provider.ts @@ -11,10 +11,11 @@ export interface FileSystem { } export enum FileStorageProviderEnum { - LOCAL = "LOCAL", - S3 = "S3", - WASABI = "WASABI", - CLOUDINARY = "CLOUDINARY", + LOCAL = 'LOCAL', + S3 = 'S3', + WASABI = 'WASABI', + CLOUDINARY = 'CLOUDINARY', + DIGITALOCEAN = 'DIGITALOCEAN' } export interface UploadedFile { @@ -34,6 +35,7 @@ export interface IS3FileStorageProviderConfig { aws_secret_access_key?: string; aws_default_region?: string; aws_bucket?: string; + aws_force_path_style?: boolean; } export interface IWasabiFileStorageProviderConfig { @@ -42,6 +44,7 @@ export interface IWasabiFileStorageProviderConfig { wasabi_aws_default_region?: string; wasabi_aws_service_url?: string; wasabi_aws_bucket?: string; + wasabi_aws_force_path_style?: boolean; } export interface ICloudinaryFileStorageProviderConfig { diff --git a/packages/core/src/core/file-storage/providers/s3.provider.ts b/packages/core/src/core/file-storage/providers/s3.provider.ts index a51c3f9dc53..9e62d95ba78 100644 --- a/packages/core/src/core/file-storage/providers/s3.provider.ts +++ b/packages/core/src/core/file-storage/providers/s3.provider.ts @@ -37,6 +37,9 @@ export interface IS3ProviderConfig { // The name of the AWS S3 bucket. aws_bucket: string; + + // Whether to force path style URLs for S3 objects + aws_force_path_style: boolean; } export class S3Provider extends Provider { @@ -54,7 +57,8 @@ export class S3Provider extends Provider { aws_access_key_id: environment.awsConfig.accessKeyId, aws_secret_access_key: environment.awsConfig.secretAccessKey, aws_default_region: environment.awsConfig.region, - aws_bucket: environment.awsConfig.s3.bucket + aws_bucket: environment.awsConfig.s3.bucket, + aws_force_path_style: environment.awsConfig.s3.forcePathStyle }; } @@ -103,6 +107,14 @@ export class S3Provider extends Provider { if (trimAndGetValue(settings.aws_bucket)) this.config.aws_bucket = trimAndGetValue(settings.aws_bucket); + + const forcePathStyle = trimAndGetValue(settings.aws_force_path_style); + + if (forcePathStyle) { + this.config.aws_force_path_style = forcePathStyle === 'true' || forcePathStyle === '1'; + } else { + this.config.aws_force_path_style = false; + } } } } catch (error) { @@ -184,7 +196,10 @@ export class S3Provider extends Provider { if (filename) { fileName = typeof filename === 'string' ? filename : filename(file, extension); } else { - fileName = `${prefix}-${moment().unix()}-${parseInt('' + Math.random() * 1000, 10)}.${extension}`; + fileName = `${prefix}-${moment().unix()}-${parseInt( + '' + Math.random() * 1000, + 10 + )}.${extension}`; } // Replace double backslashes with single forward slashes @@ -337,7 +352,7 @@ export class S3Provider extends Provider { * Whether to force path style URLs for S3 objects * (e.g., https://s3.amazonaws.com// instead of https://.s3.amazonaws.com/ */ - forcePathStyle: true + forcePathStyle: this.config.aws_force_path_style }); return s3Client; diff --git a/packages/core/src/core/file-storage/providers/wasabi-s3.provider.ts b/packages/core/src/core/file-storage/providers/wasabi-s3.provider.ts index d18d7699371..f3122d582d5 100644 --- a/packages/core/src/core/file-storage/providers/wasabi-s3.provider.ts +++ b/packages/core/src/core/file-storage/providers/wasabi-s3.provider.ts @@ -45,6 +45,9 @@ export interface IWasabiProviderConfig { // AWS bucket name for Wasabi wasabi_aws_bucket: string; + + // Whether to force path style URLs for Wasabi objects + wasabi_aws_force_path_style: boolean; } /** @@ -115,6 +118,7 @@ export class WasabiS3Provider extends Provider { wasabi_aws_access_key_id: wasabi.accessKeyId, wasabi_aws_secret_access_key: wasabi.secretAccessKey, wasabi_aws_bucket: wasabi.s3.bucket, + wasabi_aws_force_path_style: wasabi.s3.forcePathStyle, ...this._mapDefaultWasabiServiceUrl(wasabi.region, addHttpsPrefix(wasabi.serviceUrl)) }; } @@ -207,6 +211,20 @@ export class WasabiS3Provider extends Provider { this.config.wasabi_aws_bucket ); } + + const forcePathStyle = trimAndGetValue(settings.wasabi_aws_force_path_style); + + if (forcePathStyle) { + this.config.wasabi_aws_force_path_style = forcePathStyle === 'true' || forcePathStyle === '1'; + } else { + this.config.wasabi_aws_force_path_style = false; + } + + if (this._detailedloggingEnabled) + console.log( + 'setWasabiConfiguration this.config.wasabi_aws_force_path_style value: ', + this.config.wasabi_aws_force_path_style + ); } } } catch (error) { @@ -295,7 +313,10 @@ export class WasabiS3Provider extends Provider { if (filename) { fileName = typeof filename === 'string' ? filename : filename(file, extension); } else { - fileName = `${prefix}-${moment().unix()}-${parseInt('' + Math.random() * 1000, 10)}.${extension}`; + fileName = `${prefix}-${moment().unix()}-${parseInt( + '' + Math.random() * 1000, + 10 + )}.${extension}`; } // Replace double backslashes with single forward slashes @@ -467,7 +488,7 @@ export class WasabiS3Provider extends Provider { * https://s3.wasabisys.com * (e.g., https://s3.wasabisys.com// instead of https://.s3.wasabisys.com/ */ - forcePathStyle: true + forcePathStyle: this.config.wasabi_aws_force_path_style }); return s3Client; diff --git a/packages/core/src/tenant/tenant-setting/dto/aws-s3-provider-config.dto.ts b/packages/core/src/tenant/tenant-setting/dto/aws-s3-provider-config.dto.ts index 1393954d96c..138cea3dbf0 100644 --- a/packages/core/src/tenant/tenant-setting/dto/aws-s3-provider-config.dto.ts +++ b/packages/core/src/tenant/tenant-setting/dto/aws-s3-provider-config.dto.ts @@ -1,16 +1,15 @@ -import { ApiProperty } from "@nestjs/swagger"; -import { IsOptional, IsString, ValidateIf } from "class-validator"; -import { FileStorageProviderEnum, IS3FileStorageProviderConfig } from "@gauzy/contracts"; -import { Transform, TransformFnParams } from "class-transformer"; -import { IsSecret } from "./../../../core/decorators"; +import { ApiProperty } from '@nestjs/swagger'; +import { IsOptional, IsBoolean, IsString, ValidateIf } from 'class-validator'; +import { FileStorageProviderEnum, IS3FileStorageProviderConfig } from '@gauzy/contracts'; +import { Transform, TransformFnParams } from 'class-transformer'; +import { IsSecret } from './../../../core/decorators'; /** * Aws S3 FileStorage Provider Configuration DTO validation */ export class AwsS3ProviderConfigDTO implements IS3FileStorageProviderConfig { - @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => params.value ? params.value.trim() : null) + @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.S3) @IsOptional() @IsString() @@ -18,7 +17,7 @@ export class AwsS3ProviderConfigDTO implements IS3FileStorageProviderConfig { readonly aws_access_key_id: string; @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => params.value ? params.value.trim() : null) + @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.S3) @IsOptional() @IsString() @@ -26,16 +25,23 @@ export class AwsS3ProviderConfigDTO implements IS3FileStorageProviderConfig { readonly aws_secret_access_key: string; @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => params.value ? params.value.trim() : null) + @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.S3) @IsOptional() @IsString() readonly aws_default_region: string; @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => params.value ? params.value.trim() : null) + @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.S3) @IsOptional() @IsString() readonly aws_bucket: string; + + @ApiProperty({ type: () => Boolean }) + @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) + @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.S3) + @IsOptional() + @IsBoolean() + readonly aws_force_path_style: boolean; } diff --git a/packages/core/src/tenant/tenant-setting/dto/wasabi-s3-provider-config.dto.ts b/packages/core/src/tenant/tenant-setting/dto/wasabi-s3-provider-config.dto.ts index 852c9cc9788..b4ee1136daf 100644 --- a/packages/core/src/tenant/tenant-setting/dto/wasabi-s3-provider-config.dto.ts +++ b/packages/core/src/tenant/tenant-setting/dto/wasabi-s3-provider-config.dto.ts @@ -1,44 +1,50 @@ -import { ApiProperty } from "@nestjs/swagger"; -import { IsString, IsUrl, ValidateIf } from "class-validator"; -import { FileStorageProviderEnum, IWasabiFileStorageProviderConfig } from "@gauzy/contracts"; -import { Transform, TransformFnParams } from "class-transformer"; -import { IsSecret } from "./../../../core/decorators"; +import { ApiProperty } from '@nestjs/swagger'; +import { IsString, IsOptional, IsBoolean, IsUrl, ValidateIf } from 'class-validator'; +import { FileStorageProviderEnum, IWasabiFileStorageProviderConfig } from '@gauzy/contracts'; +import { Transform, TransformFnParams } from 'class-transformer'; +import { IsSecret } from './../../../core/decorators'; /** * Wasabi S3 FileStorage Provider Configuration DTO validation */ export class WasabiS3ProviderConfigDTO implements IWasabiFileStorageProviderConfig { - @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => params.value ? params.value.trim() : null) + @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.WASABI) @IsString() @IsSecret() readonly wasabi_aws_access_key_id: string; @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => params.value ? params.value.trim() : null) + @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.WASABI) @IsString() @IsSecret() readonly wasabi_aws_secret_access_key: string; @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => params.value ? params.value.trim() : null) + @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.WASABI) @IsString() readonly wasabi_aws_bucket: string; @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => params.value ? params.value.trim() : null) + @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.WASABI) @IsString() readonly wasabi_aws_default_region: string; @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => params.value ? params.value.trim() : null) + @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.WASABI) @IsString() @IsUrl() readonly wasabi_aws_service_url: string; + + @ApiProperty({ type: () => Boolean }) + @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) + @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.S3) + @IsOptional() + @IsBoolean() + readonly wasabi_aws_force_path_style: boolean; } diff --git a/packages/core/src/tenant/tenant-setting/tenant-setting.service.ts b/packages/core/src/tenant/tenant-setting/tenant-setting.service.ts index e625d2203fe..33f3c537917 100644 --- a/packages/core/src/tenant/tenant-setting/tenant-setting.service.ts +++ b/packages/core/src/tenant/tenant-setting/tenant-setting.service.ts @@ -98,7 +98,12 @@ export class TenantSettingService extends TenantAwareCrudService secretAccessKey: entity.wasabi_aws_secret_access_key }, region, - endpoint + endpoint, + /** + * Whether to force path style URLs for S3 objects + * (e.g., https://s3.amazonaws.com// instead of https://.s3.amazonaws.com/ + */ + forcePathStyle: entity.wasabi_aws_force_path_style }); // Create the parameters for calling createBucket From 167d305bbdffb3f1ca3343da489dd61257954db0 Mon Sep 17 00:00:00 2001 From: Ruslan Konviser Date: Wed, 27 Mar 2024 14:50:15 +0100 Subject: [PATCH 02/10] chore: tiny fix --- packages/common/src/interfaces/IAwsConfig.ts | 1 + packages/common/src/interfaces/IWasabiConfig.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/common/src/interfaces/IAwsConfig.ts b/packages/common/src/interfaces/IAwsConfig.ts index daebe1a7f06..77f1cdc9ef8 100644 --- a/packages/common/src/interfaces/IAwsConfig.ts +++ b/packages/common/src/interfaces/IAwsConfig.ts @@ -4,5 +4,6 @@ export interface IAwsConfig { region: string; s3: { bucket: string; + forcePathStyle: boolean; }; } diff --git a/packages/common/src/interfaces/IWasabiConfig.ts b/packages/common/src/interfaces/IWasabiConfig.ts index 80babee765e..bb7d7e07e76 100644 --- a/packages/common/src/interfaces/IWasabiConfig.ts +++ b/packages/common/src/interfaces/IWasabiConfig.ts @@ -18,5 +18,6 @@ export interface IWasabiConfig { readonly s3: { /** S3 Bucket Name */ readonly bucket: string; + readonly forcePathStyle: boolean; }; } From 7d0988f7c49630199e508765947b4987923a7f69 Mon Sep 17 00:00:00 2001 From: Ruslan Konviser Date: Wed, 27 Mar 2024 14:56:14 +0100 Subject: [PATCH 03/10] fix: more fixes --- packages/config/src/environments/environment.prod.ts | 6 ++++-- packages/config/src/environments/environment.ts | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/config/src/environments/environment.prod.ts b/packages/config/src/environments/environment.prod.ts index 1374042ea45..d45434d8175 100644 --- a/packages/config/src/environments/environment.prod.ts +++ b/packages/config/src/environments/environment.prod.ts @@ -79,7 +79,8 @@ export const environment: IEnvironment = { secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, region: process.env.AWS_REGION || 'us-east-1', s3: { - bucket: process.env.AWS_S3_BUCKET || 'gauzy' + bucket: process.env.AWS_S3_BUCKET || 'gauzy', + forcePathStyle: process.env.AWS_S3_FORCE_PATH_STYLE === 'true' ? true : false } }, @@ -92,7 +93,8 @@ export const environment: IEnvironment = { region: process.env.WASABI_REGION || 'us-east-1', serviceUrl: process.env.WASABI_SERVICE_URL || 'https://s3.wasabisys.com', s3: { - bucket: process.env.WASABI_S3_BUCKET || 'gauzy' + bucket: process.env.WASABI_S3_BUCKET || 'gauzy', + forcePathStyle: process.env.WASABI_S3_FORCE_PATH_STYLE === 'true' ? true : false } }, diff --git a/packages/config/src/environments/environment.ts b/packages/config/src/environments/environment.ts index 0d8851c191d..5740d2fc39b 100644 --- a/packages/config/src/environments/environment.ts +++ b/packages/config/src/environments/environment.ts @@ -80,7 +80,8 @@ export const environment: IEnvironment = { secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, region: process.env.AWS_REGION || 'us-east-1', s3: { - bucket: process.env.AWS_S3_BUCKET || 'gauzy' + bucket: process.env.AWS_S3_BUCKET || 'gauzy', + forcePathStyle: process.env.AWS_S3_FORCE_PATH_STYLE === 'true' ? true : false } }, @@ -93,7 +94,8 @@ export const environment: IEnvironment = { region: process.env.WASABI_REGION || 'us-east-1', serviceUrl: process.env.WASABI_SERVICE_URL || 'https://s3.wasabisys.com', s3: { - bucket: process.env.WASABI_S3_BUCKET || 'gauzy' + bucket: process.env.WASABI_S3_BUCKET || 'gauzy', + forcePathStyle: process.env.WASABI_S3_FORCE_PATH_STYLE === 'true' ? true : false } }, From 5efa412977fd76142f33a4714353bec486cb9ae7 Mon Sep 17 00:00:00 2001 From: RAHUL RATHORE <41804588+rahul-rocket@users.noreply.github.com> Date: Wed, 27 Mar 2024 22:27:39 +0530 Subject: [PATCH 04/10] feat: support for digital ocean spaces with AWS s3 --- .../src/interfaces/IDigitalOceanConfig.ts | 23 + packages/common/src/interfaces/index.ts | 1 + .../src/environments/environment.prod.ts | 14 + .../config/src/environments/environment.ts | 14 + .../config/src/environments/ienvironment.ts | 4 +- .../providers/digitalocean-s3.provider.ts | 441 ++++++++++++++++++ .../src/core/file-storage/providers/index.ts | 1 + 7 files changed, 497 insertions(+), 1 deletion(-) create mode 100644 packages/common/src/interfaces/IDigitalOceanConfig.ts create mode 100644 packages/core/src/core/file-storage/providers/digitalocean-s3.provider.ts diff --git a/packages/common/src/interfaces/IDigitalOceanConfig.ts b/packages/common/src/interfaces/IDigitalOceanConfig.ts new file mode 100644 index 00000000000..539507dbe14 --- /dev/null +++ b/packages/common/src/interfaces/IDigitalOceanConfig.ts @@ -0,0 +1,23 @@ +/** + * DigitalOcean Spaces Configuration + */ +export interface IDigitalOceanConfig { + /** DigitalOcean Access Key ID */ + readonly accessKeyId: string; + + /** DigitalOcean Secret Access Key */ + readonly secretAccessKey: string; + + /** DigitalOcean Region */ + readonly region: string; + + /** DigitalOcean Service URL */ + readonly serviceUrl: string; + + /** S3 Bucket Configuration */ + readonly s3: { + /** S3 Bucket Name */ + readonly bucket: string; + readonly forcePathStyle: boolean; + }; +} diff --git a/packages/common/src/interfaces/index.ts b/packages/common/src/interfaces/index.ts index a3f526fc17a..219eeb910c4 100644 --- a/packages/common/src/interfaces/index.ts +++ b/packages/common/src/interfaces/index.ts @@ -18,6 +18,7 @@ export * from './ITwitterConfig'; export * from './IUnleashConfig'; export * from './IUpworkConfig'; export * from './IWasabiConfig'; +export * from './IDigitalOceanConfig'; export * from './IHubstaffConfig'; export * from './IJitsuConfig'; export * from './IJiraIntegrationConfig'; diff --git a/packages/config/src/environments/environment.prod.ts b/packages/config/src/environments/environment.prod.ts index d45434d8175..5650941de5d 100644 --- a/packages/config/src/environments/environment.prod.ts +++ b/packages/config/src/environments/environment.prod.ts @@ -98,6 +98,20 @@ export const environment: IEnvironment = { } }, + /** + * DigitalOcean Spaces Configuration + */ + digitalOcean: { + accessKeyId: process.env.DIGITALOCEAN_ACCESS_KEY_ID, + secretAccessKey: process.env.DIGITALOCEAN_SECRET_ACCESS_KEY, + region: process.env.DIGITALOCEAN_REGION || 'us-east-1', + serviceUrl: process.env.DIGITALOCEAN_SERVICE_URL || 'https://gauzy.sfo2.digitaloceanspaces.com', // Find your endpoint in the control panel, under Settings. Prepend "https://". + s3: { + bucket: process.env.DIGITALOCEAN_S3_BUCKET || 'gauzy', + forcePathStyle: process.env.DIGITALOCEAN_S3_FORCE_PATH_STYLE === 'true' || false // Configures to use subdomain/virtual calling format. + } + }, + /** * Cloudinary FileSystem Storage Configuration */ diff --git a/packages/config/src/environments/environment.ts b/packages/config/src/environments/environment.ts index 5740d2fc39b..0657df3f889 100644 --- a/packages/config/src/environments/environment.ts +++ b/packages/config/src/environments/environment.ts @@ -99,6 +99,20 @@ export const environment: IEnvironment = { } }, + /** + * DigitalOcean Spaces Configuration + */ + digitalOcean: { + accessKeyId: process.env.DIGITALOCEAN_ACCESS_KEY_ID, + secretAccessKey: process.env.DIGITALOCEAN_SECRET_ACCESS_KEY, + region: process.env.DIGITALOCEAN_REGION || 'us-east-1', + serviceUrl: process.env.DIGITALOCEAN_SERVICE_URL || 'https://gauzy.sfo2.digitaloceanspaces.com', // Find your endpoint in the control panel, under Settings. Prepend "https://". + s3: { + bucket: process.env.DIGITALOCEAN_S3_BUCKET || 'gauzy', + forcePathStyle: process.env.DIGITALOCEAN_S3_FORCE_PATH_STYLE === 'true' || false // Configures to use subdomain/virtual calling format. + } + }, + /** * Cloudinary FileSystem Storage Configuration */ diff --git a/packages/config/src/environments/ienvironment.ts b/packages/config/src/environments/ienvironment.ts index b99424bb61a..2cacbfe64fe 100644 --- a/packages/config/src/environments/ienvironment.ts +++ b/packages/config/src/environments/ienvironment.ts @@ -17,7 +17,8 @@ import { IUnleashConfig, IUpworkConfig, IWasabiConfig, - IJiraIntegrationConfig + IJiraIntegrationConfig, + IDigitalOceanConfig } from '@gauzy/common'; export type LogLevel = 'debug' | 'info' | 'warn' | 'error'; @@ -98,6 +99,7 @@ export interface IEnvironment { awsConfig?: IAwsConfig; wasabi?: IWasabiConfig; cloudinary?: ICloudinaryConfig; + digitalOcean: IDigitalOceanConfig; github: IGithubIntegrationConfig /** Github Configuration */; jira: IJiraIntegrationConfig /** Jira Configuration */; fiverrConfig: IFiverrConfig; diff --git a/packages/core/src/core/file-storage/providers/digitalocean-s3.provider.ts b/packages/core/src/core/file-storage/providers/digitalocean-s3.provider.ts new file mode 100644 index 00000000000..c8f81b5e465 --- /dev/null +++ b/packages/core/src/core/file-storage/providers/digitalocean-s3.provider.ts @@ -0,0 +1,441 @@ +import { HttpException, HttpStatus } from '@nestjs/common'; +import * as multerS3 from 'multer-s3'; +import { basename, join } from 'path'; +import * as moment from 'moment'; +import { + S3Client, + DeleteObjectCommand, + DeleteObjectCommandOutput, + GetObjectCommand, + GetObjectCommandOutput, + PutObjectCommand, + HeadObjectCommand +} from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { StorageEngine } from 'multer'; +import { environment } from '@gauzy/config'; +import { FileStorageOption, FileStorageProviderEnum, UploadedFile } from '@gauzy/contracts'; +import { addHttpsPrefix, trimAndGetValue } from '@gauzy/common'; +import { Provider } from './provider'; +import { RequestContext } from '../../context'; + +/** + * Digital Ocean Configuration + */ +const { digitalOcean } = environment; + +/** + * Configuration interface for DigitalOcean storage. + */ +export interface IDigitalOceanProviderConfig { + rootPath: string; // Root path for DigitalOcean storage + digitalocean_access_key_id: string; // AWS access key ID for DigitalOcean + digitalocean_secret_access_key: string; // AWS secret access key for DigitalOcean + digitalocean_default_region: string; // AWS default region for DigitalOcean + digitalocean_service_url: string; // AWS service URL for DigitalOcean + digitalocean_s3_bucket: string; // AWS bucket name for DigitalOcean + digitalocean_s3_force_path_style: boolean; // Whether to force path style URLs for DigitalOcean objects +} + +export class DigitalOceanS3Provider extends Provider { + + public readonly name = FileStorageProviderEnum.DIGITALOCEAN; + public instance: DigitalOceanS3Provider; + public config: IDigitalOceanProviderConfig; + public defaultConfig: IDigitalOceanProviderConfig; + + private readonly _detailedloggingEnabled = false; + + constructor() { + super(); + this.config = this.defaultConfig = { + rootPath: '', + digitalocean_access_key_id: digitalOcean.accessKeyId, + digitalocean_secret_access_key: digitalOcean.secretAccessKey, + digitalocean_default_region: digitalOcean.region, + digitalocean_service_url: digitalOcean.serviceUrl, + digitalocean_s3_bucket: digitalOcean.s3.bucket, + digitalocean_s3_force_path_style: digitalOcean.s3.forcePathStyle, + }; + } + + /** + * Get the singleton instance of DigitalOceanS3Provider + * + * @returns {DigitalOceanS3Provider} The singleton instance + */ + getProviderInstance(): DigitalOceanS3Provider { + if (!this.instance) { + this.instance = new DigitalOceanS3Provider(); + } + + this.setDigitalOceanConfiguration(); + return this.instance; + } + + /** + * Set DigitalOcean details based on the current request's tenantSettings. + * If such settings does not have any DigitalOcean details, use the default configuration. + * If they have DigitalOcean details, use them to override the default configuration. + */ + private setDigitalOceanConfiguration() { + // Use the default configuration as a starting point + this.config = { + ...this.defaultConfig + }; + + if (this._detailedloggingEnabled) { + console.log(`setDigitalOceanConfiguration this config value: ${JSON.stringify(this.config)}`); + } + + try { + const request = RequestContext.currentRequest(); + if (request) { + const settings = request['tenantSettings']; + + if (settings) { + if (this._detailedloggingEnabled) { + console.log(`setDigitalOceanConfiguration Tenant Settings Value: ${JSON.stringify(settings)}`); + } + + if (trimAndGetValue(settings.digitalocean_access_key_id)) { + this.config.digitalocean_access_key_id = trimAndGetValue(settings.digitalocean_access_key_id); + + if (this._detailedloggingEnabled) { + console.log(`setDigitalOceanConfiguration this.config.digitalocean_access_key_id value: ${this.config.digitalocean_access_key_id}`); + } + } + + if (trimAndGetValue(settings.digitalocean_secret_access_key)) { + this.config.digitalocean_secret_access_key = trimAndGetValue(settings.digitalocean_secret_access_key); + + if (this._detailedloggingEnabled) { + console.log(`setDigitalOceanConfiguration this.config.digitalocean_secret_access_key value: ${this.config.digitalocean_secret_access_key}`); + } + } + + if (trimAndGetValue(settings.digitalocean_service_url)) { + this.config.digitalocean_service_url = addHttpsPrefix(trimAndGetValue(settings.digitalocean_service_url)); + + if (this._detailedloggingEnabled) { + console.log('setDigitalOceanConfiguration this.config.digitalocean_service_url value: ', this.config.digitalocean_service_url); + } + } + + if (trimAndGetValue(settings.digitalocean_default_region)) { + this.config.digitalocean_default_region = trimAndGetValue(settings.digitalocean_default_region); + + if (this._detailedloggingEnabled) { + console.log('setDigitalOceanConfiguration this.config.digitalocean_default_region value: ', this.config.digitalocean_default_region); + } + } + + if (trimAndGetValue(settings.digitalocean_s3_bucket)) { + this.config.digitalocean_s3_bucket = trimAndGetValue(settings.digitalocean_s3_bucket); + + if (this._detailedloggingEnabled) { + console.log('setDigitalOceanConfiguration this.config.digitalocean_s3_bucket value: ', this.config.digitalocean_s3_bucket); + } + } + + const forcePathStyle = trimAndGetValue(settings.digitalocean_s3_force_path_style); + + if (forcePathStyle) { + this.config.digitalocean_s3_force_path_style = forcePathStyle === 'true' || forcePathStyle === '1'; + } else { + this.config.digitalocean_s3_force_path_style = false; + } + + if (this._detailedloggingEnabled) { + console.log('setDigitalOceanConfiguration this.config.digitalocean_s3_force_path_style value: ', this.config.digitalocean_s3_force_path_style); + } + } + } + } catch (error) { + console.error('Error while setting DigitalOcean configuration. Default configuration will be used', error); + } + } + + /** + * Get a pre-signed URL for a given file URL. + * + * @param fileURL - The file URL for which to generate a pre-signed URL + * @returns Pre-signed URL or null if the input is invalid + */ + public async url(fileURL: string): Promise { + if (!fileURL || fileURL.startsWith('http')) { + return fileURL; + } + + try { + const s3Client = this.getDigitalOceanInstance(); + + if (s3Client) { + const signedUrl = await getSignedUrl( + s3Client, + new GetObjectCommand({ + Bucket: this.getDigitalOceanBucket(), + Key: fileURL + }), + { + expiresIn: 3600 + } + ); + + return signedUrl; + } else { + console.error('Error while retrieving signed URL: s3Client is null'); + return null; + } + } catch (error) { + console.error('Error while retrieving signed URL:', error); + return null; + } + } + + /** + * Get the full path of the file in the storage. + * + * @param filePath - The file path to join with the root path + * @returns Full path or null if filePath is falsy + */ + public path(filePath: string): string | null { + return filePath ? join(this.config.rootPath, filePath) : null; + } + + /** + * Create a Multer storage engine configured for AWS S3 (DigitalOcean). + * + * @param options - Configuration options for the storage engine + * @returns A Multer storage engine + */ + public handler(options: FileStorageOption): StorageEngine { + const { dest, filename, prefix = 'file' } = options; + + try { + const s3Client = this.getDigitalOceanInstance(); + + if (s3Client) { + return multerS3({ + s3: s3Client, + bucket: (_req, _file, callback) => { + callback(null, this.getDigitalOceanBucket()); + }, + metadata: function (_req, _file, callback) { + callback(null, { fieldName: _file.fieldname }); + }, + key: (_req, file, callback) => { + // A string or function that determines the destination path for uploaded + const destination = dest instanceof Function ? dest(file) : dest; + + // A file extension, or filename extension, is a suffix at the end of a file. + const extension = file.originalname.split('.').pop(); + + // A function that determines the name of the uploaded file. + let fileName: string; + + if (filename) { + fileName = typeof filename === 'string' ? filename : filename(file, extension); + } else { + fileName = `${prefix}-${moment().unix()}-${parseInt('' + Math.random() * 1000, 10)}.${extension}`; + } + + // Replace double backslashes with single forward slashes + const fullPath = join(destination, fileName).replace(/\\/g, '/'); + + callback(null, fullPath); + } + }); + } else { + console.error('Error while retrieving Multer for DigitalOcean: s3Client is null'); + return null; + } + } catch (error) { + console.error('Error while retrieving Multer for DigitalOcean:', error); + return null; + } + } + + /** + * Get a file from DigitalOcean storage. + * + * @param key - The key of the file to retrieve + * @returns A Promise resolving to a Buffer containing the file data + */ + async getFile(key: string): Promise { + try { + const s3Client = this.getDigitalOceanInstance(); + + if (s3Client) { + // Input parameters when using the GetObjectCommand to retrieve an object from DigitalOcean storage. + const command = new GetObjectCommand({ + Bucket: this.getDigitalOceanBucket(), // The name of the bucket from which to retrieve the object. + Key: key // The key (path) of the object to retrieve from the bucket. + }); + + /** + * Send a GetObjectCommand to DigitalOcean to retrieve an object + */ + const data: GetObjectCommandOutput = await s3Client.send(command); + return data.Body; + } else { + console.error('Error while retrieving signed URL: s3Client is null'); + } + } catch (error) { + console.error(`Error while fetching file with key '${key}':`, error); + } + } + + /** + * Upload a file to DigitalOcean storage. + * + * @param fileContent - The content of the file to upload + * @param key - The key under which to store the file + * @returns A Promise resolving to an UploadedFile, or undefined on error + */ + async putFile(fileContent: string, key: string = ''): Promise { + try { + // Replace double backslashes with single forward slashes + key = key.replace(/\\/g, '/'); + + const s3Client = this.getDigitalOceanInstance(); + + if (s3Client) { + const filename = basename(key); + + // Input parameters for the PutObjectCommand when uploading a file to DigitalOcean storage. + const putObjectCommand = new PutObjectCommand({ + Bucket: this.getDigitalOceanBucket(), // The name of the bucket to which the file should be uploaded. + Body: fileContent, // The content of the file to be uploaded. + Key: key, // The key (path) under which to store the file in the bucket. + ContentDisposition: `inline; ${filename}`, // Additional headers for the object. + ContentType: 'image' + }); + + /** + * Send a PutObjectCommand to DigitalOcean to upload the object + */ + await s3Client.send(putObjectCommand); + + // Input parameters for the HeadObjectCommand when retrieving metadata about an object in DigitalOcean storage. + const headObjectCommand = new HeadObjectCommand({ + Key: key, // The key (path) of the object for which to retrieve metadata. + Bucket: this.getDigitalOceanBucket() // The name of the bucket where the object is stored. + }); + + // Send a HeadObjectCommand to DigitalOcean to retrieve ContentLength property metadata + const { ContentLength } = await s3Client.send(headObjectCommand); + + const file: Partial = { + originalname: filename, // original file name + size: ContentLength, // files in bytes + filename: filename, + path: key, // Full path of the file + key: key // Full path of the file + }; + + return await this.mapUploadedFile(file); + } else { + console.warn('Error while retrieving signed URL: s3Client is null'); + } + } catch (error) { + console.error('Error while put file for DigitalOcean provider', error); + } + } + + /** + * Delete a file from DigitalOcean storage. + * + * @param key - The key of the file to delete + * @returns A Promise that resolves when the file is deleted successfully, or rejects with an error + */ + async deleteFile(key: string): Promise { + try { + const s3Client = this.getDigitalOceanInstance(); + + if (s3Client) { + // Input parameters when using the DeleteObjectCommand to delete an object from DigitalOcean storage. + const command = new DeleteObjectCommand({ + Bucket: this.getDigitalOceanBucket(), // The name of the bucket from which to delete the object. + Key: key // The key (path) of the object to delete from the bucket. + }); + + /** + * Send a DeleteObjectCommand to DigitalOcean to delete an object + */ + const data: DeleteObjectCommandOutput = await s3Client.send(command); + return new Object({ + status: HttpStatus.OK, + message: `file with key: ${key} is successfully deleted`, + data + }); + } else { + console.warn('Error while retrieving signed URL: s3Client is null'); + } + } catch (error) { + console.error(`Error while deleting file with key '${key}':`, error); + throw new HttpException(error, HttpStatus.BAD_REQUEST, { + description: `Error while deleting file with key: '${key}'` + }); + } + } + + /** + * Get an AWS S3 instance configured with DigitalOcean details. + * + * @returns An AWS S3 instance or null in case of an error + */ + private getDigitalOceanInstance(): S3Client | null { + try { + this.setDigitalOceanConfiguration(); + + if (this.config && this.config.digitalocean_access_key_id && this.config.digitalocean_secret_access_key) { + const endpoint = addHttpsPrefix(this.config.digitalocean_service_url); + + const s3Client = new S3Client({ + /** + * Whether to force path style URLs for S3 objects + * (e.g., https://s3.amazonaws.com// instead of https://.s3.amazonaws.com/ + */ + forcePathStyle: this.config.digitalocean_s3_force_path_style, // Configures to use subdomain/virtual calling format. + endpoint, + region: this.config.digitalocean_default_region || 'us-east-1', + credentials: { + accessKeyId: this.config.digitalocean_access_key_id, + secretAccessKey: this.config.digitalocean_secret_access_key + }, + }); + + return s3Client; + } else { + console.warn(`Can't retrieve ${FileStorageProviderEnum.DIGITALOCEAN} instance for tenant: this.config.digitalocean_service_url, digitalocean_access_key_id or digitalocean_secret_access_key undefined in that tenant settings`); + return null; + } + } catch (error) { + console.error(`Error while retrieving ${FileStorageProviderEnum.DIGITALOCEAN} instance:`, error); + return null; + } + } + + /** + * Get the DigitalOcean bucket from the configuration. + * + * @returns The DigitalOcean bucket name or null if not configured + */ + public getDigitalOceanBucket(): string | null { + this.setDigitalOceanConfiguration(); + return this.config.digitalocean_s3_bucket || null; + } + + /** + * Map a partial UploadedFile object to include filename and URL. + * + * @param file - The partial UploadedFile object to map + * @returns The mapped file object + */ + public async mapUploadedFile(file: any): Promise { + file.filename = file.originalname; + file.url = await this.url(file.key); // file.location; + return file; + } +} diff --git a/packages/core/src/core/file-storage/providers/index.ts b/packages/core/src/core/file-storage/providers/index.ts index 0a9cbea2ebf..335755d0829 100644 --- a/packages/core/src/core/file-storage/providers/index.ts +++ b/packages/core/src/core/file-storage/providers/index.ts @@ -2,3 +2,4 @@ export * from './local.provider'; export * from './s3.provider'; export * from './wasabi-s3.provider'; export * from './cloudinary.provider'; +export * from './digitalocean-s3.provider'; From 1fdcbf1abba6674746e459af5bb950276d020593 Mon Sep 17 00:00:00 2001 From: RAHUL RATHORE <41804588+rahul-rocket@users.noreply.github.com> Date: Wed, 27 Mar 2024 22:35:32 +0530 Subject: [PATCH 05/10] fix: missing CDN for digital ocean storage provider --- packages/common/src/interfaces/IDigitalOceanConfig.ts | 6 ++---- packages/config/src/environments/environment.prod.ts | 1 + packages/config/src/environments/environment.ts | 1 + .../core/file-storage/providers/digitalocean-s3.provider.ts | 2 ++ 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/common/src/interfaces/IDigitalOceanConfig.ts b/packages/common/src/interfaces/IDigitalOceanConfig.ts index 539507dbe14..4c8d1a305a6 100644 --- a/packages/common/src/interfaces/IDigitalOceanConfig.ts +++ b/packages/common/src/interfaces/IDigitalOceanConfig.ts @@ -4,16 +4,14 @@ export interface IDigitalOceanConfig { /** DigitalOcean Access Key ID */ readonly accessKeyId: string; - /** DigitalOcean Secret Access Key */ readonly secretAccessKey: string; - /** DigitalOcean Region */ readonly region: string; - /** DigitalOcean Service URL */ readonly serviceUrl: string; - + /** The CDN (Content Delivery Network) DigitalOcean configuration. */ + readonly cdn?: string; /** S3 Bucket Configuration */ readonly s3: { /** S3 Bucket Name */ diff --git a/packages/config/src/environments/environment.prod.ts b/packages/config/src/environments/environment.prod.ts index 5650941de5d..50ac91c3098 100644 --- a/packages/config/src/environments/environment.prod.ts +++ b/packages/config/src/environments/environment.prod.ts @@ -106,6 +106,7 @@ export const environment: IEnvironment = { secretAccessKey: process.env.DIGITALOCEAN_SECRET_ACCESS_KEY, region: process.env.DIGITALOCEAN_REGION || 'us-east-1', serviceUrl: process.env.DIGITALOCEAN_SERVICE_URL || 'https://gauzy.sfo2.digitaloceanspaces.com', // Find your endpoint in the control panel, under Settings. Prepend "https://". + cdn: process.env.DIGITALOCEAN_CDN_URL, s3: { bucket: process.env.DIGITALOCEAN_S3_BUCKET || 'gauzy', forcePathStyle: process.env.DIGITALOCEAN_S3_FORCE_PATH_STYLE === 'true' || false // Configures to use subdomain/virtual calling format. diff --git a/packages/config/src/environments/environment.ts b/packages/config/src/environments/environment.ts index 0657df3f889..f86d548e5ea 100644 --- a/packages/config/src/environments/environment.ts +++ b/packages/config/src/environments/environment.ts @@ -107,6 +107,7 @@ export const environment: IEnvironment = { secretAccessKey: process.env.DIGITALOCEAN_SECRET_ACCESS_KEY, region: process.env.DIGITALOCEAN_REGION || 'us-east-1', serviceUrl: process.env.DIGITALOCEAN_SERVICE_URL || 'https://gauzy.sfo2.digitaloceanspaces.com', // Find your endpoint in the control panel, under Settings. Prepend "https://". + cdn: process.env.DIGITALOCEAN_CDN_URL, s3: { bucket: process.env.DIGITALOCEAN_S3_BUCKET || 'gauzy', forcePathStyle: process.env.DIGITALOCEAN_S3_FORCE_PATH_STYLE === 'true' || false // Configures to use subdomain/virtual calling format. diff --git a/packages/core/src/core/file-storage/providers/digitalocean-s3.provider.ts b/packages/core/src/core/file-storage/providers/digitalocean-s3.provider.ts index c8f81b5e465..6f4257a605d 100644 --- a/packages/core/src/core/file-storage/providers/digitalocean-s3.provider.ts +++ b/packages/core/src/core/file-storage/providers/digitalocean-s3.provider.ts @@ -33,6 +33,7 @@ export interface IDigitalOceanProviderConfig { digitalocean_secret_access_key: string; // AWS secret access key for DigitalOcean digitalocean_default_region: string; // AWS default region for DigitalOcean digitalocean_service_url: string; // AWS service URL for DigitalOcean + digitalocean_cdn_url: string; // AWS service CDN for DigitalOcean digitalocean_s3_bucket: string; // AWS bucket name for DigitalOcean digitalocean_s3_force_path_style: boolean; // Whether to force path style URLs for DigitalOcean objects } @@ -54,6 +55,7 @@ export class DigitalOceanS3Provider extends Provider { digitalocean_secret_access_key: digitalOcean.secretAccessKey, digitalocean_default_region: digitalOcean.region, digitalocean_service_url: digitalOcean.serviceUrl, + digitalocean_cdn_url: digitalOcean.cdn, digitalocean_s3_bucket: digitalOcean.s3.bucket, digitalocean_s3_force_path_style: digitalOcean.s3.forcePathStyle, }; From b95d773e26ab6f4e460d6adb90705aeaa46400f4 Mon Sep 17 00:00:00 2001 From: RAHUL RATHORE <41804588+rahul-rocket@users.noreply.github.com> Date: Wed, 27 Mar 2024 23:09:54 +0530 Subject: [PATCH 06/10] fix: requested changes in PR --- packages/common/src/interfaces/IDigitalOceanConfig.ts | 2 +- packages/config/src/environments/environment.prod.ts | 6 +++--- packages/config/src/environments/environment.ts | 6 +++--- packages/config/src/environments/ienvironment.ts | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/common/src/interfaces/IDigitalOceanConfig.ts b/packages/common/src/interfaces/IDigitalOceanConfig.ts index 4c8d1a305a6..e0a9b88cbeb 100644 --- a/packages/common/src/interfaces/IDigitalOceanConfig.ts +++ b/packages/common/src/interfaces/IDigitalOceanConfig.ts @@ -11,7 +11,7 @@ export interface IDigitalOceanConfig { /** DigitalOcean Service URL */ readonly serviceUrl: string; /** The CDN (Content Delivery Network) DigitalOcean configuration. */ - readonly cdn?: string; + readonly cdnUrl?: string; /** S3 Bucket Configuration */ readonly s3: { /** S3 Bucket Name */ diff --git a/packages/config/src/environments/environment.prod.ts b/packages/config/src/environments/environment.prod.ts index 50ac91c3098..8aa56f9279d 100644 --- a/packages/config/src/environments/environment.prod.ts +++ b/packages/config/src/environments/environment.prod.ts @@ -80,7 +80,7 @@ export const environment: IEnvironment = { region: process.env.AWS_REGION || 'us-east-1', s3: { bucket: process.env.AWS_S3_BUCKET || 'gauzy', - forcePathStyle: process.env.AWS_S3_FORCE_PATH_STYLE === 'true' ? true : false + forcePathStyle: process.env.AWS_S3_FORCE_PATH_STYLE === 'true' || false } }, @@ -94,7 +94,7 @@ export const environment: IEnvironment = { serviceUrl: process.env.WASABI_SERVICE_URL || 'https://s3.wasabisys.com', s3: { bucket: process.env.WASABI_S3_BUCKET || 'gauzy', - forcePathStyle: process.env.WASABI_S3_FORCE_PATH_STYLE === 'true' ? true : false + forcePathStyle: process.env.WASABI_S3_FORCE_PATH_STYLE === 'true' || false } }, @@ -106,7 +106,7 @@ export const environment: IEnvironment = { secretAccessKey: process.env.DIGITALOCEAN_SECRET_ACCESS_KEY, region: process.env.DIGITALOCEAN_REGION || 'us-east-1', serviceUrl: process.env.DIGITALOCEAN_SERVICE_URL || 'https://gauzy.sfo2.digitaloceanspaces.com', // Find your endpoint in the control panel, under Settings. Prepend "https://". - cdn: process.env.DIGITALOCEAN_CDN_URL, + cdnUrl: process.env.DIGITALOCEAN_CDN_URL, s3: { bucket: process.env.DIGITALOCEAN_S3_BUCKET || 'gauzy', forcePathStyle: process.env.DIGITALOCEAN_S3_FORCE_PATH_STYLE === 'true' || false // Configures to use subdomain/virtual calling format. diff --git a/packages/config/src/environments/environment.ts b/packages/config/src/environments/environment.ts index f86d548e5ea..490d52c0777 100644 --- a/packages/config/src/environments/environment.ts +++ b/packages/config/src/environments/environment.ts @@ -81,7 +81,7 @@ export const environment: IEnvironment = { region: process.env.AWS_REGION || 'us-east-1', s3: { bucket: process.env.AWS_S3_BUCKET || 'gauzy', - forcePathStyle: process.env.AWS_S3_FORCE_PATH_STYLE === 'true' ? true : false + forcePathStyle: process.env.AWS_S3_FORCE_PATH_STYLE === 'true' || false } }, @@ -95,7 +95,7 @@ export const environment: IEnvironment = { serviceUrl: process.env.WASABI_SERVICE_URL || 'https://s3.wasabisys.com', s3: { bucket: process.env.WASABI_S3_BUCKET || 'gauzy', - forcePathStyle: process.env.WASABI_S3_FORCE_PATH_STYLE === 'true' ? true : false + forcePathStyle: process.env.WASABI_S3_FORCE_PATH_STYLE === 'true' || false } }, @@ -107,7 +107,7 @@ export const environment: IEnvironment = { secretAccessKey: process.env.DIGITALOCEAN_SECRET_ACCESS_KEY, region: process.env.DIGITALOCEAN_REGION || 'us-east-1', serviceUrl: process.env.DIGITALOCEAN_SERVICE_URL || 'https://gauzy.sfo2.digitaloceanspaces.com', // Find your endpoint in the control panel, under Settings. Prepend "https://". - cdn: process.env.DIGITALOCEAN_CDN_URL, + cdnUrl: process.env.DIGITALOCEAN_CDN_URL, s3: { bucket: process.env.DIGITALOCEAN_S3_BUCKET || 'gauzy', forcePathStyle: process.env.DIGITALOCEAN_S3_FORCE_PATH_STYLE === 'true' || false // Configures to use subdomain/virtual calling format. diff --git a/packages/config/src/environments/ienvironment.ts b/packages/config/src/environments/ienvironment.ts index 2cacbfe64fe..8f28f563dd6 100644 --- a/packages/config/src/environments/ienvironment.ts +++ b/packages/config/src/environments/ienvironment.ts @@ -99,7 +99,7 @@ export interface IEnvironment { awsConfig?: IAwsConfig; wasabi?: IWasabiConfig; cloudinary?: ICloudinaryConfig; - digitalOcean: IDigitalOceanConfig; + digitalOcean?: IDigitalOceanConfig; github: IGithubIntegrationConfig /** Github Configuration */; jira: IJiraIntegrationConfig /** Jira Configuration */; fiverrConfig: IFiverrConfig; From e1fe7ca43fd26d04464d18ae22e1070a8349c8cc Mon Sep 17 00:00:00 2001 From: RAHUL RATHORE <41804588+rahul-rocket@users.noreply.github.com> Date: Wed, 27 Mar 2024 23:53:24 +0530 Subject: [PATCH 07/10] fix: improved trim decorator --- .../dto/aws-s3-provider-config.dto.ts | 10 ++-- .../dto/cloudinary-provider-config.dto.ts | 8 +-- .../digitalocean-s3.provider-config.dto.ts | 57 +++++++++++++++++++ .../tenant-setting/dto/trim.decorator.ts | 5 ++ .../dto/wasabi-s3-provider-config.dto.ts | 19 +++---- 5 files changed, 79 insertions(+), 20 deletions(-) create mode 100644 packages/core/src/tenant/tenant-setting/dto/digitalocean-s3.provider-config.dto.ts create mode 100644 packages/core/src/tenant/tenant-setting/dto/trim.decorator.ts diff --git a/packages/core/src/tenant/tenant-setting/dto/aws-s3-provider-config.dto.ts b/packages/core/src/tenant/tenant-setting/dto/aws-s3-provider-config.dto.ts index 138cea3dbf0..b1c20da3af1 100644 --- a/packages/core/src/tenant/tenant-setting/dto/aws-s3-provider-config.dto.ts +++ b/packages/core/src/tenant/tenant-setting/dto/aws-s3-provider-config.dto.ts @@ -1,45 +1,43 @@ import { ApiProperty } from '@nestjs/swagger'; import { IsOptional, IsBoolean, IsString, ValidateIf } from 'class-validator'; import { FileStorageProviderEnum, IS3FileStorageProviderConfig } from '@gauzy/contracts'; -import { Transform, TransformFnParams } from 'class-transformer'; import { IsSecret } from './../../../core/decorators'; +import { Trimmed } from './trim.decorator'; /** * Aws S3 FileStorage Provider Configuration DTO validation */ export class AwsS3ProviderConfigDTO implements IS3FileStorageProviderConfig { @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.S3) @IsOptional() @IsString() @IsSecret() + @Trimmed() readonly aws_access_key_id: string; @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.S3) @IsOptional() @IsString() @IsSecret() + @Trimmed() readonly aws_secret_access_key: string; @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.S3) @IsOptional() @IsString() readonly aws_default_region: string; @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.S3) @IsOptional() @IsString() + @Trimmed() readonly aws_bucket: string; @ApiProperty({ type: () => Boolean }) - @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.S3) @IsOptional() @IsBoolean() diff --git a/packages/core/src/tenant/tenant-setting/dto/cloudinary-provider-config.dto.ts b/packages/core/src/tenant/tenant-setting/dto/cloudinary-provider-config.dto.ts index a78dd748801..2dc1be3753f 100644 --- a/packages/core/src/tenant/tenant-setting/dto/cloudinary-provider-config.dto.ts +++ b/packages/core/src/tenant/tenant-setting/dto/cloudinary-provider-config.dto.ts @@ -1,8 +1,8 @@ import { ApiProperty } from "@nestjs/swagger"; import { IsNotEmpty, IsOptional, ValidateIf } from "class-validator"; import { FileStorageProviderEnum, ICloudinaryFileStorageProviderConfig } from "@gauzy/contracts"; -import { Transform, TransformFnParams } from "class-transformer"; import { IsSecret } from "./../../../core/decorators"; +import { Trimmed } from "./trim.decorator"; /** * Cloudinary FileStorage Provider Configuration DTO validation @@ -10,23 +10,23 @@ import { IsSecret } from "./../../../core/decorators"; export class CloudinaryProviderConfigDTO implements ICloudinaryFileStorageProviderConfig { @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => params.value ? params.value.trim() : null) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.CLOUDINARY) @IsNotEmpty() + @Trimmed() readonly cloudinary_cloud_name: string; @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => params.value ? params.value.trim() : null) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.CLOUDINARY) @IsNotEmpty() @IsSecret() + @Trimmed() readonly cloudinary_api_key: string; @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => params.value ? params.value.trim() : null) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.CLOUDINARY) @IsNotEmpty() @IsSecret() + @Trimmed() readonly cloudinary_api_secret: string; @ApiProperty({ type: () => String }) diff --git a/packages/core/src/tenant/tenant-setting/dto/digitalocean-s3.provider-config.dto.ts b/packages/core/src/tenant/tenant-setting/dto/digitalocean-s3.provider-config.dto.ts new file mode 100644 index 00000000000..d6a8fafc3a6 --- /dev/null +++ b/packages/core/src/tenant/tenant-setting/dto/digitalocean-s3.provider-config.dto.ts @@ -0,0 +1,57 @@ +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { IsString, IsOptional, IsBoolean, IsUrl, ValidateIf } from 'class-validator'; +import { FileStorageProviderEnum, IDigitalOceanFileStorageProviderConfig } from '@gauzy/contracts'; +import { IsSecret } from '../../../core/decorators'; +import { Trimmed } from './trim.decorator'; + +/** + * DigitalOcean S3 FileStorage Provider Configuration DTO validation + */ +export class DigitalOceanS3ProviderConfigDTO implements IDigitalOceanFileStorageProviderConfig { + @ApiProperty({ type: () => String }) + @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.DIGITALOCEAN) + @IsString() + @IsSecret() + @Trimmed() + readonly digitalocean_access_key_id: string; + + @ApiProperty({ type: () => String }) + @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.DIGITALOCEAN) + @IsString() + @IsSecret() + @Trimmed() + readonly digitalocean_secret_access_key: string; + + @ApiProperty({ type: () => String }) + @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.DIGITALOCEAN) + @IsString() + @Trimmed() + readonly digitalocean_s3_bucket: string; + + @ApiProperty({ type: () => String }) + @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.DIGITALOCEAN) + @IsString() + @IsUrl() + @Trimmed() + readonly digitalocean_service_url: string; + + @ApiProperty({ type: () => String }) + @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.DIGITALOCEAN) + @IsOptional() + @IsString() + @IsUrl() + @Trimmed() + readonly digitalocean_cdn_url: string; + + @ApiPropertyOptional({ type: () => String }) + @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.DIGITALOCEAN) + @IsOptional() + @IsString() + readonly digitalocean_default_region: string; + + @ApiPropertyOptional({ type: () => Boolean }) + @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.DIGITALOCEAN) + @IsOptional() + @IsBoolean() + readonly digitalocean_s3_force_path_style: boolean; +} diff --git a/packages/core/src/tenant/tenant-setting/dto/trim.decorator.ts b/packages/core/src/tenant/tenant-setting/dto/trim.decorator.ts new file mode 100644 index 00000000000..d567abbbfa8 --- /dev/null +++ b/packages/core/src/tenant/tenant-setting/dto/trim.decorator.ts @@ -0,0 +1,5 @@ +import { Transform, TransformFnParams } from "class-transformer"; + +export function Trimmed(): PropertyDecorator { + return Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)); +} diff --git a/packages/core/src/tenant/tenant-setting/dto/wasabi-s3-provider-config.dto.ts b/packages/core/src/tenant/tenant-setting/dto/wasabi-s3-provider-config.dto.ts index b4ee1136daf..23ff0e54342 100644 --- a/packages/core/src/tenant/tenant-setting/dto/wasabi-s3-provider-config.dto.ts +++ b/packages/core/src/tenant/tenant-setting/dto/wasabi-s3-provider-config.dto.ts @@ -1,49 +1,48 @@ -import { ApiProperty } from '@nestjs/swagger'; +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; import { IsString, IsOptional, IsBoolean, IsUrl, ValidateIf } from 'class-validator'; import { FileStorageProviderEnum, IWasabiFileStorageProviderConfig } from '@gauzy/contracts'; -import { Transform, TransformFnParams } from 'class-transformer'; import { IsSecret } from './../../../core/decorators'; +import { Trimmed } from './trim.decorator'; /** * Wasabi S3 FileStorage Provider Configuration DTO validation */ export class WasabiS3ProviderConfigDTO implements IWasabiFileStorageProviderConfig { @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.WASABI) @IsString() @IsSecret() + @Trimmed() readonly wasabi_aws_access_key_id: string; @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.WASABI) @IsString() @IsSecret() + @Trimmed() readonly wasabi_aws_secret_access_key: string; @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.WASABI) @IsString() + @Trimmed() readonly wasabi_aws_bucket: string; @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.WASABI) @IsString() + @Trimmed() readonly wasabi_aws_default_region: string; @ApiProperty({ type: () => String }) - @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.WASABI) @IsString() @IsUrl() + @Trimmed() readonly wasabi_aws_service_url: string; - @ApiProperty({ type: () => Boolean }) - @Transform((params: TransformFnParams) => (params.value ? params.value.trim() : null)) - @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.S3) + @ApiPropertyOptional({ type: () => Boolean }) + @ValidateIf((it) => it.fileStorageProvider === FileStorageProviderEnum.WASABI) @IsOptional() @IsBoolean() readonly wasabi_aws_force_path_style: boolean; From 19e44da68f0f5745c34d48d33ef0efc96488ac5a Mon Sep 17 00:00:00 2001 From: RAHUL RATHORE <41804588+rahul-rocket@users.noreply.github.com> Date: Wed, 27 Mar 2024 23:59:42 +0530 Subject: [PATCH 08/10] feat: digital ocean storage inputs --- .../file-storage/file-storage.component.html | 133 ++++++++++++++++++ .../file-storage/file-storage.component.ts | 27 +++- .../file-storage/file-storage.module.ts | 15 +- apps/gauzy/src/assets/i18n/en.json | 23 ++- 4 files changed, 185 insertions(+), 13 deletions(-) diff --git a/apps/gauzy/src/app/pages/settings/file-storage/file-storage.component.html b/apps/gauzy/src/app/pages/settings/file-storage/file-storage.component.html index 57ca75ca392..09aae7d8fc0 100644 --- a/apps/gauzy/src/app/pages/settings/file-storage/file-storage.component.html +++ b/apps/gauzy/src/app/pages/settings/file-storage/file-storage.component.html @@ -36,6 +36,11 @@

*ngTemplateOutlet="cloudinaryStorageFormTemplate; context: { group: form }"> + + + + @@ -244,6 +249,17 @@
/> +
+ +
+ +
+
@@ -358,3 +374,120 @@
+ + + + + +
{{ 'SETTINGS_FILE_STORAGE.DIGITALOCEAN.HEADER' | translate }}
+
+ +
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+
+
diff --git a/apps/gauzy/src/app/pages/settings/file-storage/file-storage.component.ts b/apps/gauzy/src/app/pages/settings/file-storage/file-storage.component.ts index b8f83c72925..75611a3a618 100644 --- a/apps/gauzy/src/app/pages/settings/file-storage/file-storage.component.ts +++ b/apps/gauzy/src/app/pages/settings/file-storage/file-storage.component.ts @@ -31,12 +31,18 @@ export class FileStorageComponent extends TranslationBaseComponent loading: boolean = false; public readonly form: UntypedFormGroup = FileStorageComponent.buildForm(this.fb); + + /** + * + * @param fb + * @returns + */ static buildForm(fb: UntypedFormBuilder): UntypedFormGroup { + const defaultFileStorageProvider = environment.FILE_PROVIDER.toUpperCase() as FileStorageProviderEnum || FileStorageProviderEnum.LOCAL; + + // const form = fb.group({ - fileStorageProvider: [ - (environment.FILE_PROVIDER).toUpperCase() as FileStorageProviderEnum || FileStorageProviderEnum.LOCAL, - Validators.required - ], + fileStorageProvider: [defaultFileStorageProvider, Validators.required], // Aws Configuration S3: fb.group({ aws_access_key_id: [], @@ -50,7 +56,8 @@ export class FileStorageComponent extends TranslationBaseComponent wasabi_aws_secret_access_key: [], wasabi_aws_default_region: ['us-east-1'], wasabi_aws_service_url: ['https://s3.wasabisys.com'], - wasabi_aws_bucket: ['gauzy'] + wasabi_aws_bucket: ['gauzy'], + wasabi_aws_force_path_style: [false] }), // Cloudinary Configuration CLOUDINARY: fb.group({ @@ -60,6 +67,16 @@ export class FileStorageComponent extends TranslationBaseComponent cloudinary_api_secure: ['true'], cloudinary_delivery_url: ['https://res.cloudinary.com'] }), + // DigitalOcean Configuration + DIGITALOCEAN: fb.group({ + digitalocean_access_key_id: [], + digitalocean_secret_access_key: [], + digitalocean_default_region: [{ value: 'us-east-1', disabled: true }], + digitalocean_service_url: [], + digitalocean_cdn_url: [], + digitalocean_s3_bucket: ['gauzy'], + digitalocean_s3_force_path_style: [false] + }), }); return form; } diff --git a/apps/gauzy/src/app/pages/settings/file-storage/file-storage.module.ts b/apps/gauzy/src/app/pages/settings/file-storage/file-storage.module.ts index 9f2e0c1d142..d18fe47d0f5 100644 --- a/apps/gauzy/src/app/pages/settings/file-storage/file-storage.module.ts +++ b/apps/gauzy/src/app/pages/settings/file-storage/file-storage.module.ts @@ -1,6 +1,6 @@ import { NgModule } from '@angular/core'; import { FormsModule, ReactiveFormsModule } from '@angular/forms'; -import { NbButtonModule, NbCardModule, NbInputModule, NbSelectModule, NbSpinnerModule } from '@nebular/theme'; +import { NbButtonModule, NbCardModule, NbInputModule, NbSelectModule, NbSpinnerModule, NbToggleModule } from '@nebular/theme'; import { NgxPermissionsModule } from 'ngx-permissions'; import { FileProviderModule } from '../../../@shared/selectors/file-provider/file-provider.module'; import { SharedModule } from '../../../@shared/shared.module'; @@ -14,20 +14,21 @@ import { FileStorageComponent } from './file-storage.component'; FormsModule, ReactiveFormsModule, FileStorageRoutingModule, - ThemeModule, - TranslateModule, - SharedModule, - NgxPermissionsModule.forChild(), NbButtonModule, NbCardModule, NbInputModule, NbSelectModule, + NbSpinnerModule, + NbToggleModule, + NgxPermissionsModule.forChild(), + ThemeModule, + TranslateModule, + SharedModule, FileProviderModule, - NbSpinnerModule ], declarations: [ FileStorageComponent ], providers: [] }) -export class FileStorageModule {} +export class FileStorageModule { } diff --git a/apps/gauzy/src/assets/i18n/en.json b/apps/gauzy/src/assets/i18n/en.json index 39e7a1a6193..d69d7008839 100644 --- a/apps/gauzy/src/assets/i18n/en.json +++ b/apps/gauzy/src/assets/i18n/en.json @@ -4445,7 +4445,8 @@ "SECRET_ACCESS_KEY": "Secret access key", "REGION": "Region", "BUCKET": "Bucket", - "SERVICE_URL": "Service URL" + "SERVICE_URL": "Service URL", + "FORCE_PATH_STYLE": "Force path style URLs" }, "PLACEHOLDERS": { "ACCESS_KEY_ID": "Access key id", @@ -4471,6 +4472,26 @@ "DELIVERY_URL": "Delivery URL", "SECURE": "Secure" } + }, + "DIGITALOCEAN": { + "HEADER": "DigitalOcean Configuration", + "LABELS": { + "ACCESS_KEY_ID": "Access key id", + "SECRET_ACCESS_KEY": "Secret access key", + "REGION": "Region", + "BUCKET": "Bucket", + "SERVICE_URL": "Service URL", + "CDN_URL": "CDN URL", + "FORCE_PATH_STYLE": "Force path style URLs" + }, + "PLACEHOLDERS": { + "ACCESS_KEY_ID": "Access key id", + "SECRET_ACCESS_KEY": "Secret access key", + "REGION": "Region", + "BUCKET": "Bucket", + "SERVICE_URL": "Service URL", + "CDN_URL": "CDN URL" + } } }, "CUSTOM_SMTP_PAGE": { From 416c95933b15d468298960c2076d76c96837a171 Mon Sep 17 00:00:00 2001 From: RAHUL RATHORE <41804588+rahul-rocket@users.noreply.github.com> Date: Thu, 28 Mar 2024 00:01:31 +0530 Subject: [PATCH 09/10] feat: digital ocean s3 provider improvment --- packages/contracts/src/file-provider.ts | 10 +++ .../providers/digitalocean-s3.provider.ts | 10 +-- .../providers/wasabi-s3.provider.ts | 74 +++++++------------ .../handlers/tenant-setting.get.handler.ts | 10 ++- .../dto/create-tenant-setting.dto.ts | 4 +- .../src/tenant/tenant-setting/dto/index.ts | 1 + .../tenant-setting.controller.ts | 2 +- 7 files changed, 52 insertions(+), 59 deletions(-) diff --git a/packages/contracts/src/file-provider.ts b/packages/contracts/src/file-provider.ts index 5126f719be7..0c2dd5401d2 100644 --- a/packages/contracts/src/file-provider.ts +++ b/packages/contracts/src/file-provider.ts @@ -52,3 +52,13 @@ export interface ICloudinaryFileStorageProviderConfig { cloudinary_api_key?: string; cloudinary_api_secret?: string; } + +export interface IDigitalOceanFileStorageProviderConfig { + digitalocean_access_key_id?: string; + digitalocean_secret_access_key?: string; + digitalocean_default_region?: string; + digitalocean_service_url?: string; + digitalocean_cdn_url?: string; + digitalocean_s3_bucket?: string; + digitalocean_s3_force_path_style?: boolean; +} diff --git a/packages/core/src/core/file-storage/providers/digitalocean-s3.provider.ts b/packages/core/src/core/file-storage/providers/digitalocean-s3.provider.ts index 6f4257a605d..4454df016d0 100644 --- a/packages/core/src/core/file-storage/providers/digitalocean-s3.provider.ts +++ b/packages/core/src/core/file-storage/providers/digitalocean-s3.provider.ts @@ -55,7 +55,7 @@ export class DigitalOceanS3Provider extends Provider { digitalocean_secret_access_key: digitalOcean.secretAccessKey, digitalocean_default_region: digitalOcean.region, digitalocean_service_url: digitalOcean.serviceUrl, - digitalocean_cdn_url: digitalOcean.cdn, + digitalocean_cdn_url: digitalOcean.cdnUrl, digitalocean_s3_bucket: digitalOcean.s3.bucket, digitalocean_s3_force_path_style: digitalOcean.s3.forcePathStyle, }; @@ -140,13 +140,9 @@ export class DigitalOceanS3Provider extends Provider { } } + // Assuming trimAndGetValue() function trims and retrieves the value from settings const forcePathStyle = trimAndGetValue(settings.digitalocean_s3_force_path_style); - - if (forcePathStyle) { - this.config.digitalocean_s3_force_path_style = forcePathStyle === 'true' || forcePathStyle === '1'; - } else { - this.config.digitalocean_s3_force_path_style = false; - } + this.config.digitalocean_s3_force_path_style = forcePathStyle === 'true' || forcePathStyle === '1'; if (this._detailedloggingEnabled) { console.log('setDigitalOceanConfiguration this.config.digitalocean_s3_force_path_style value: ', this.config.digitalocean_s3_force_path_style); diff --git a/packages/core/src/core/file-storage/providers/wasabi-s3.provider.ts b/packages/core/src/core/file-storage/providers/wasabi-s3.provider.ts index f3122d582d5..e1d14c4b50d 100644 --- a/packages/core/src/core/file-storage/providers/wasabi-s3.provider.ts +++ b/packages/core/src/core/file-storage/providers/wasabi-s3.provider.ts @@ -157,74 +157,57 @@ export class WasabiS3Provider extends Provider { const settings = request['tenantSettings']; if (settings) { - if (this._detailedloggingEnabled) - console.log(`setWasabiConfiguration Tenant Settings value: ${JSON.stringify(settings)}`); + if (this._detailedloggingEnabled) { + console.log(`setWasabiConfiguration Tenant Settings Value: ${JSON.stringify(settings)}`); + } if (trimAndGetValue(settings.wasabi_aws_access_key_id)) { this.config.wasabi_aws_access_key_id = trimAndGetValue(settings.wasabi_aws_access_key_id); - if (this._detailedloggingEnabled) - console.log( - `setWasabiConfiguration this.config.wasabi_aws_access_key_id value: ${this.config.wasabi_aws_access_key_id}` - ); + if (this._detailedloggingEnabled) { + console.log(`setWasabiConfiguration this.config.wasabi_aws_access_key_id value: ${this.config.wasabi_aws_access_key_id}`); + } } if (trimAndGetValue(settings.wasabi_aws_secret_access_key)) { - this.config.wasabi_aws_secret_access_key = trimAndGetValue( - settings.wasabi_aws_secret_access_key - ); - - if (this._detailedloggingEnabled) - console.log( - `setWasabiConfiguration this.config.wasabi_aws_secret_access_key value: ${this.config.wasabi_aws_secret_access_key}` - ); + this.config.wasabi_aws_secret_access_key = trimAndGetValue(settings.wasabi_aws_secret_access_key); + + if (this._detailedloggingEnabled) { + console.log(`setWasabiConfiguration this.config.wasabi_aws_secret_access_key value: ${this.config.wasabi_aws_secret_access_key}`); + } } if (trimAndGetValue(settings.wasabi_aws_service_url)) { - this.config.wasabi_aws_service_url = addHttpsPrefix( - trimAndGetValue(settings.wasabi_aws_service_url) - ); - - if (this._detailedloggingEnabled) - console.log( - 'setWasabiConfiguration this.config.wasabi_aws_service_url value: ', - this.config.wasabi_aws_service_url - ); + this.config.wasabi_aws_service_url = addHttpsPrefix(trimAndGetValue(settings.wasabi_aws_service_url)); + + if (this._detailedloggingEnabled) { + console.log('setWasabiConfiguration this.config.wasabi_aws_service_url value: ', this.config.wasabi_aws_service_url); + } } if (trimAndGetValue(settings.wasabi_aws_default_region)) { this.config.wasabi_aws_default_region = trimAndGetValue(settings.wasabi_aws_default_region); - if (this._detailedloggingEnabled) - console.log( - 'setWasabiConfiguration this.config.wasabi_aws_default_region value: ', - this.config.wasabi_aws_default_region - ); + if (this._detailedloggingEnabled) { + console.log('setWasabiConfiguration this.config.wasabi_aws_default_region value: ', this.config.wasabi_aws_default_region); + } } if (trimAndGetValue(settings.wasabi_aws_bucket)) { this.config.wasabi_aws_bucket = trimAndGetValue(settings.wasabi_aws_bucket); - if (this._detailedloggingEnabled) - console.log( - 'setWasabiConfiguration this.config.wasabi_aws_bucket value: ', - this.config.wasabi_aws_bucket - ); + if (this._detailedloggingEnabled) { + console.log('setWasabiConfiguration this.config.wasabi_aws_bucket value: ', this.config.wasabi_aws_bucket); + } } + // Assuming trimAndGetValue() function trims and retrieves the value from settings const forcePathStyle = trimAndGetValue(settings.wasabi_aws_force_path_style); + this.config.wasabi_aws_force_path_style = forcePathStyle === 'true' || forcePathStyle === '1'; - if (forcePathStyle) { - this.config.wasabi_aws_force_path_style = forcePathStyle === 'true' || forcePathStyle === '1'; - } else { - this.config.wasabi_aws_force_path_style = false; + if (this._detailedloggingEnabled) { + console.log('setWasabiConfiguration this.config.wasabi_aws_force_path_style value: ', this.config.wasabi_aws_force_path_style); } - - if (this._detailedloggingEnabled) - console.log( - 'setWasabiConfiguration this.config.wasabi_aws_force_path_style value: ', - this.config.wasabi_aws_force_path_style - ); } } } catch (error) { @@ -313,10 +296,7 @@ export class WasabiS3Provider extends Provider { if (filename) { fileName = typeof filename === 'string' ? filename : filename(file, extension); } else { - fileName = `${prefix}-${moment().unix()}-${parseInt( - '' + Math.random() * 1000, - 10 - )}.${extension}`; + fileName = `${prefix}-${moment().unix()}-${parseInt('' + Math.random() * 1000, 10)}.${extension}`; } // Replace double backslashes with single forward slashes diff --git a/packages/core/src/tenant/tenant-setting/commands/handlers/tenant-setting.get.handler.ts b/packages/core/src/tenant/tenant-setting/commands/handlers/tenant-setting.get.handler.ts index b31a2700160..03de5acaf66 100644 --- a/packages/core/src/tenant/tenant-setting/commands/handlers/tenant-setting.get.handler.ts +++ b/packages/core/src/tenant/tenant-setting/commands/handlers/tenant-setting.get.handler.ts @@ -4,7 +4,12 @@ import { RequestContext } from './../../../../core/context'; import { TenantSettingGetCommand } from '../tenant-setting.get.command'; import { TenantSettingService } from './../../tenant-setting.service'; import { WrapSecrets } from './../../../../core/decorators'; -import { AwsS3ProviderConfigDTO, CloudinaryProviderConfigDTO, WasabiS3ProviderConfigDTO } from './../../dto'; +import { + AwsS3ProviderConfigDTO, + CloudinaryProviderConfigDTO, + DigitalOceanS3ProviderConfigDTO, + WasabiS3ProviderConfigDTO +} from './../../dto'; @CommandHandler(TenantSettingGetCommand) export class TenantSettingGetHandler @@ -13,7 +18,7 @@ export class TenantSettingGetHandler constructor( @Inject(forwardRef(() => TenantSettingService)) private readonly _tenantSettingService: TenantSettingService - ) {} + ) { } public async execute() { const tenantId = RequestContext.currentTenantId(); @@ -27,6 +32,7 @@ export class TenantSettingGetHandler WrapSecrets(settings, new WasabiS3ProviderConfigDTO()), WrapSecrets(settings, new AwsS3ProviderConfigDTO()), WrapSecrets(settings, new CloudinaryProviderConfigDTO()), + WrapSecrets(settings, new DigitalOceanS3ProviderConfigDTO()), ); } } diff --git a/packages/core/src/tenant/tenant-setting/dto/create-tenant-setting.dto.ts b/packages/core/src/tenant/tenant-setting/dto/create-tenant-setting.dto.ts index 12f5558f1d8..f2970c6b7bd 100644 --- a/packages/core/src/tenant/tenant-setting/dto/create-tenant-setting.dto.ts +++ b/packages/core/src/tenant/tenant-setting/dto/create-tenant-setting.dto.ts @@ -5,15 +5,15 @@ import { FileStorageProviderEnum } from "@gauzy/contracts"; import { AwsS3ProviderConfigDTO } from "./aws-s3-provider-config.dto"; import { WasabiS3ProviderConfigDTO } from "./wasabi-s3-provider-config.dto"; import { CloudinaryProviderConfigDTO } from "./cloudinary-provider-config.dto"; +import { DigitalOceanS3ProviderConfigDTO } from './digitalocean-s3.provider-config.dto'; /** * Tenant Setting Save Request DTO validation */ export class CreateTenantSettingDTO extends IntersectionType( - WasabiS3ProviderConfigDTO, + IntersectionType(WasabiS3ProviderConfigDTO, DigitalOceanS3ProviderConfigDTO), IntersectionType(AwsS3ProviderConfigDTO, CloudinaryProviderConfigDTO) ) { - /** * FileStorage Provider Configuration */ diff --git a/packages/core/src/tenant/tenant-setting/dto/index.ts b/packages/core/src/tenant/tenant-setting/dto/index.ts index d9fce99e2f5..af25e8c0467 100644 --- a/packages/core/src/tenant/tenant-setting/dto/index.ts +++ b/packages/core/src/tenant/tenant-setting/dto/index.ts @@ -2,3 +2,4 @@ export * from './create-tenant-setting.dto'; export * from './aws-s3-provider-config.dto'; export * from './wasabi-s3-provider-config.dto'; export * from './cloudinary-provider-config.dto'; +export * from './digitalocean-s3.provider-config.dto'; diff --git a/packages/core/src/tenant/tenant-setting/tenant-setting.controller.ts b/packages/core/src/tenant/tenant-setting/tenant-setting.controller.ts index 054a0e13c54..3b8e333e502 100644 --- a/packages/core/src/tenant/tenant-setting/tenant-setting.controller.ts +++ b/packages/core/src/tenant/tenant-setting/tenant-setting.controller.ts @@ -1,4 +1,3 @@ -import { ITenantSetting, PermissionsEnum } from '@gauzy/contracts'; import { Body, Controller, @@ -12,6 +11,7 @@ import { } from '@nestjs/common'; import { ApiOperation, ApiResponse, ApiTags } from '@nestjs/swagger'; import { CommandBus } from '@nestjs/cqrs'; +import { ITenantSetting, PermissionsEnum } from '@gauzy/contracts'; import { CrudController } from '../../core/crud'; import { Permissions } from './../../shared/decorators'; import { PermissionGuard, TenantPermissionGuard } from './../../shared/guards'; From 4f4ebb0b1abb18398ce1d195a8bc59a77b58032b Mon Sep 17 00:00:00 2001 From: RAHUL RATHORE <41804588+rahul-rocket@users.noreply.github.com> Date: Thu, 28 Mar 2024 00:47:51 +0530 Subject: [PATCH 10/10] feat: [table] migration for storage provider columns --- .../core/src/database/migration-executor.ts | 2 +- .../1711564805530-AlterStorageProvider.ts | 147 ++++++++++++++++++ 2 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 packages/core/src/database/migrations/1711564805530-AlterStorageProvider.ts diff --git a/packages/core/src/database/migration-executor.ts b/packages/core/src/database/migration-executor.ts index c6bd883b02e..87761882af5 100644 --- a/packages/core/src/database/migration-executor.ts +++ b/packages/core/src/database/migration-executor.ts @@ -266,8 +266,8 @@ function queryParams(parameters: any[] | undefined): string { function getTemplate(connection: DataSource, name: string, timestamp: number, upSqls: string[], downSqls: string[]): string { return ` import { MigrationInterface, QueryRunner } from "typeorm"; -import { DatabaseTypeEnum } from "@gauzy/config"; import { yellow } from "chalk"; +import { DatabaseTypeEnum } from "@gauzy/config"; export class ${camelCase(name, true)}${timestamp} implements MigrationInterface { diff --git a/packages/core/src/database/migrations/1711564805530-AlterStorageProvider.ts b/packages/core/src/database/migrations/1711564805530-AlterStorageProvider.ts new file mode 100644 index 00000000000..8d3f0a7c111 --- /dev/null +++ b/packages/core/src/database/migrations/1711564805530-AlterStorageProvider.ts @@ -0,0 +1,147 @@ + +import { MigrationInterface, QueryRunner } from "typeorm"; +import { yellow } from "chalk"; +import { DatabaseTypeEnum } from "@gauzy/config"; + +export class AlterStorageProvider1711564805530 implements MigrationInterface { + + name = 'AlterStorageProvider1711564805530'; + + /** + * Up Migration + * + * @param queryRunner + */ + public async up(queryRunner: QueryRunner): Promise { + console.log(yellow(this.name + ' start running!')); + + switch (queryRunner.connection.options.type) { + case DatabaseTypeEnum.sqlite: + case DatabaseTypeEnum.betterSqlite3: + await this.sqliteUpQueryRunner(queryRunner); + break; + case DatabaseTypeEnum.postgres: + await this.postgresUpQueryRunner(queryRunner); + break; + case DatabaseTypeEnum.mysql: + await this.mysqlUpQueryRunner(queryRunner); + break; + default: + throw Error(`Unsupported database: ${queryRunner.connection.options.type}`); + } + } + + /** + * Down Migration + * + * @param queryRunner + */ + public async down(queryRunner: QueryRunner): Promise { + switch (queryRunner.connection.options.type) { + case DatabaseTypeEnum.sqlite: + case DatabaseTypeEnum.betterSqlite3: + await this.sqliteDownQueryRunner(queryRunner); + break; + case DatabaseTypeEnum.postgres: + await this.postgresDownQueryRunner(queryRunner); + break; + case DatabaseTypeEnum.mysql: + await this.mysqlDownQueryRunner(queryRunner); + break; + default: + throw Error(`Unsupported database: ${queryRunner.connection.options.type}`); + } + } + + /** + * PostgresDB Up Migration + * + * @param queryRunner + */ + public async postgresUpQueryRunner(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TYPE "public"."image_asset_storageprovider_enum" RENAME TO "image_asset_storageprovider_enum_old"`); + await queryRunner.query(`CREATE TYPE "public"."image_asset_storageprovider_enum" AS ENUM('LOCAL', 'S3', 'WASABI', 'CLOUDINARY', 'DIGITALOCEAN')`); + await queryRunner.query(`ALTER TABLE "image_asset" ALTER COLUMN "storageProvider" TYPE "public"."image_asset_storageprovider_enum" USING "storageProvider"::"text"::"public"."image_asset_storageprovider_enum"`); + await queryRunner.query(`DROP TYPE "public"."image_asset_storageprovider_enum_old"`); + await queryRunner.query(`ALTER TYPE "public"."screenshot_storageprovider_enum" RENAME TO "screenshot_storageprovider_enum_old"`); + await queryRunner.query(`CREATE TYPE "public"."screenshot_storageprovider_enum" AS ENUM('LOCAL', 'S3', 'WASABI', 'CLOUDINARY', 'DIGITALOCEAN')`); + await queryRunner.query(`ALTER TABLE "screenshot" ALTER COLUMN "storageProvider" TYPE "public"."screenshot_storageprovider_enum" USING "storageProvider"::"text"::"public"."screenshot_storageprovider_enum"`); + await queryRunner.query(`DROP TYPE "public"."screenshot_storageprovider_enum_old"`); + } + + /** + * PostgresDB Down Migration + * + * @param queryRunner + */ + public async postgresDownQueryRunner(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE TYPE "public"."screenshot_storageprovider_enum_old" AS ENUM('LOCAL', 'S3', 'WASABI', 'CLOUDINARY')`); + await queryRunner.query(`ALTER TABLE "screenshot" ALTER COLUMN "storageProvider" TYPE "public"."screenshot_storageprovider_enum_old" USING "storageProvider"::"text"::"public"."screenshot_storageprovider_enum_old"`); + await queryRunner.query(`DROP TYPE "public"."screenshot_storageprovider_enum"`); + await queryRunner.query(`ALTER TYPE "public"."screenshot_storageprovider_enum_old" RENAME TO "screenshot_storageprovider_enum"`); + await queryRunner.query(`CREATE TYPE "public"."image_asset_storageprovider_enum_old" AS ENUM('LOCAL', 'S3', 'WASABI', 'CLOUDINARY')`); + await queryRunner.query(`ALTER TABLE "image_asset" ALTER COLUMN "storageProvider" TYPE "public"."image_asset_storageprovider_enum_old" USING "storageProvider"::"text"::"public"."image_asset_storageprovider_enum_old"`); + await queryRunner.query(`DROP TYPE "public"."image_asset_storageprovider_enum"`); + await queryRunner.query(`ALTER TYPE "public"."image_asset_storageprovider_enum_old" RENAME TO "image_asset_storageprovider_enum"`); + } + + /** + * SqliteDB and BetterSQlite3DB Up Migration + * + * @param queryRunner + */ + public async sqliteUpQueryRunner(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "IDX_af1a212cb378bb0eed51c1b2bc"`); + await queryRunner.query(`DROP INDEX "IDX_9d44ce9eb8689e578b941a6a54"`); + await queryRunner.query(`DROP INDEX "IDX_d3675304df9971cccf96d9a7c3"`); + await queryRunner.query(`DROP INDEX "IDX_01856a9a730b7e79d70aa661cb"`); + await queryRunner.query(`CREATE TABLE "temporary_image_asset" ("id" varchar PRIMARY KEY NOT NULL, "createdAt" datetime NOT NULL DEFAULT (datetime('now')), "updatedAt" datetime NOT NULL DEFAULT (datetime('now')), "tenantId" varchar, "organizationId" varchar, "name" varchar, "url" varchar NOT NULL, "width" integer NOT NULL DEFAULT (0), "height" integer NOT NULL DEFAULT (0), "isFeatured" boolean NOT NULL DEFAULT (0), "thumb" varchar, "size" numeric, "externalProviderId" varchar, "storageProvider" varchar CHECK( "storageProvider" IN ('LOCAL','S3','WASABI','CLOUDINARY','DIGITALOCEAN') ), "deletedAt" datetime, "isActive" boolean DEFAULT (1), "isArchived" boolean DEFAULT (0), CONSTRAINT "FK_d3675304df9971cccf96d9a7c34" FOREIGN KEY ("organizationId") REFERENCES "organization" ("id") ON DELETE CASCADE ON UPDATE CASCADE, CONSTRAINT "FK_01856a9a730b7e79d70aa661cb0" FOREIGN KEY ("tenantId") REFERENCES "tenant" ("id") ON DELETE CASCADE ON UPDATE NO ACTION)`); + await queryRunner.query(`INSERT INTO "temporary_image_asset"("id", "createdAt", "updatedAt", "tenantId", "organizationId", "name", "url", "width", "height", "isFeatured", "thumb", "size", "externalProviderId", "storageProvider", "deletedAt", "isActive", "isArchived") SELECT "id", "createdAt", "updatedAt", "tenantId", "organizationId", "name", "url", "width", "height", "isFeatured", "thumb", "size", "externalProviderId", "storageProvider", "deletedAt", "isActive", "isArchived" FROM "image_asset"`); + await queryRunner.query(`DROP TABLE "image_asset"`); + await queryRunner.query(`ALTER TABLE "temporary_image_asset" RENAME TO "image_asset"`); + await queryRunner.query(`CREATE INDEX "IDX_af1a212cb378bb0eed51c1b2bc" ON "image_asset" ("isArchived") `); + await queryRunner.query(`CREATE INDEX "IDX_9d44ce9eb8689e578b941a6a54" ON "image_asset" ("isActive") `); + await queryRunner.query(`CREATE INDEX "IDX_d3675304df9971cccf96d9a7c3" ON "image_asset" ("organizationId") `); + await queryRunner.query(`CREATE INDEX "IDX_01856a9a730b7e79d70aa661cb" ON "image_asset" ("tenantId") `); + } + + /** + * SqliteDB and BetterSQlite3DB Down Migration + * + * @param queryRunner + */ + public async sqliteDownQueryRunner(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "IDX_01856a9a730b7e79d70aa661cb"`); + await queryRunner.query(`DROP INDEX "IDX_d3675304df9971cccf96d9a7c3"`); + await queryRunner.query(`DROP INDEX "IDX_9d44ce9eb8689e578b941a6a54"`); + await queryRunner.query(`DROP INDEX "IDX_af1a212cb378bb0eed51c1b2bc"`); + await queryRunner.query(`ALTER TABLE "image_asset" RENAME TO "temporary_image_asset"`); + await queryRunner.query(`CREATE TABLE "image_asset" ("id" varchar PRIMARY KEY NOT NULL, "createdAt" datetime NOT NULL DEFAULT (datetime('now')), "updatedAt" datetime NOT NULL DEFAULT (datetime('now')), "tenantId" varchar, "organizationId" varchar, "name" varchar, "url" varchar NOT NULL, "width" integer NOT NULL DEFAULT (0), "height" integer NOT NULL DEFAULT (0), "isFeatured" boolean NOT NULL DEFAULT (0), "thumb" varchar, "size" numeric, "externalProviderId" varchar, "storageProvider" varchar CHECK( "storageProvider" IN ('LOCAL','S3','WASABI','CLOUDINARY') ), "deletedAt" datetime, "isActive" boolean DEFAULT (1), "isArchived" boolean DEFAULT (0), CONSTRAINT "FK_d3675304df9971cccf96d9a7c34" FOREIGN KEY ("organizationId") REFERENCES "organization" ("id") ON DELETE CASCADE ON UPDATE CASCADE, CONSTRAINT "FK_01856a9a730b7e79d70aa661cb0" FOREIGN KEY ("tenantId") REFERENCES "tenant" ("id") ON DELETE CASCADE ON UPDATE NO ACTION)`); + await queryRunner.query(`INSERT INTO "image_asset"("id", "createdAt", "updatedAt", "tenantId", "organizationId", "name", "url", "width", "height", "isFeatured", "thumb", "size", "externalProviderId", "storageProvider", "deletedAt", "isActive", "isArchived") SELECT "id", "createdAt", "updatedAt", "tenantId", "organizationId", "name", "url", "width", "height", "isFeatured", "thumb", "size", "externalProviderId", "storageProvider", "deletedAt", "isActive", "isArchived" FROM "temporary_image_asset"`); + await queryRunner.query(`DROP TABLE "temporary_image_asset"`); + await queryRunner.query(`CREATE INDEX "IDX_01856a9a730b7e79d70aa661cb" ON "image_asset" ("tenantId") `); + await queryRunner.query(`CREATE INDEX "IDX_d3675304df9971cccf96d9a7c3" ON "image_asset" ("organizationId") `); + await queryRunner.query(`CREATE INDEX "IDX_9d44ce9eb8689e578b941a6a54" ON "image_asset" ("isActive") `); + await queryRunner.query(`CREATE INDEX "IDX_af1a212cb378bb0eed51c1b2bc" ON "image_asset" ("isArchived") `); + } + + /** + * MySQL Up Migration + * + * @param queryRunner + */ + public async mysqlUpQueryRunner(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`image_asset\` CHANGE \`storageProvider\` \`storageProvider\` enum ('LOCAL', 'S3', 'WASABI', 'CLOUDINARY', 'DIGITALOCEAN') NULL`); + await queryRunner.query(`ALTER TABLE \`screenshot\` CHANGE \`storageProvider\` \`storageProvider\` enum ('LOCAL', 'S3', 'WASABI', 'CLOUDINARY', 'DIGITALOCEAN') NULL`); + } + + /** + * MySQL Down Migration + * + * @param queryRunner + */ + public async mysqlDownQueryRunner(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`screenshot\` CHANGE \`storageProvider\` \`storageProvider\` enum ('LOCAL', 'S3', 'WASABI', 'CLOUDINARY') NULL`); + await queryRunner.query(`ALTER TABLE \`image_asset\` CHANGE \`storageProvider\` \`storageProvider\` enum ('LOCAL', 'S3', 'WASABI', 'CLOUDINARY') NULL`); + } +}