fix: upload to local provider (#1581)

* fix: upload to local provider * fix: remove log --------- Co-authored-by: Lior Zamir <liorz@ballerine.com>
ballerine-io · Nov 7, 2023 · 622baf0 · 622baf0
1 parent 3e526ed
commit 622baf0
Show file tree

Hide file tree

Showing 2 changed files with 52 additions and 20 deletions.
diff --git a/services/workflows-service/src/providers/file/file-provider/local-file.service.ts b/services/workflows-service/src/providers/file/file-provider/local-file.service.ts
@@ -38,16 +38,45 @@ export class LocalFileService implements IFileProvider {
     localFilePath: TLocalFilePath,
     remoteFileConfig: TRemoteFileConfig,
   ): Promise<TRemoteFileConfig> {
-    const toLocalFilePath = remoteFileConfig as TLocalFilePath;
-    this.client
-      .createReadStream(this.__removeFilePrefix(localFilePath))
-      .pipe(this.client.createWriteStream(this.__removeFilePrefix(toLocalFilePath)));
+    return new Promise((resolve, reject) => {
+      const distFilePath = this.__removeFilePrefix(remoteFileConfig as TRemoteUri);
 
-    return Promise.resolve(toLocalFilePath);
+      const destDirectory = path.dirname(distFilePath);
+      if (!fs.existsSync(destDirectory)) {
+        fs.mkdirSync(destDirectory, { recursive: true });
+      }
+
+      // Create a readable stream from the source file
+      const readStream = this.client.createReadStream(this.__removeFilePrefix(localFilePath));
+
+      // Create a writable stream to the destination file
+      const writeStream = this.client.createWriteStream(this.__removeFilePrefix(distFilePath));
+
+      readStream.pipe(writeStream);
+
+      writeStream.on('finish', () => {
+        resolve(distFilePath);
+      });
+
+      writeStream.on('error', err => {
+        reject(err);
+      });
+    });
   }
 
-  generateRemotePath({ fileName }: { fileName: string }): string {
-    return path.join(os.tmpdir(), this.__removeFilePrefix(fileName));
+  generateRemotePath({
+    fileName,
+    customerName,
+    directory,
+  }: {
+    fileName: string;
+    customerName: string;
+    directory?: string;
+  }): string {
+    const desiredPath = [customerName, directory, this.__removeFilePrefix(fileName)]
+      .filter(pathPart => !!pathPart)
+      .join('/');
+    return path.join(os.homedir(), '.ballerine', desiredPath);
   }
 
   private __removeFilePrefix(fileName: string): string {

diff --git a/services/workflows-service/src/providers/file/file.service.ts b/services/workflows-service/src/providers/file/file.service.ts
@@ -16,7 +16,6 @@ import { LocalFileService } from '@/providers/file/file-provider/local-file.serv
 import { fromBuffer, fromFile } from 'file-type';
 import { streamToBuffer } from '@/common/stream-to-buffer/stream-to-buffer';
 import { Readable } from 'stream';
-import { TDocumentWithoutPageType } from '@/common/types';
 import * as fs from 'fs/promises';
 
 @Injectable()
@@ -219,14 +218,18 @@ export class FileService {
     targetRemoteFileConfig: TRemoteFileConfig;
     remoteFileNameInDirectory: string;
   } {
+    const properties = {
+      customerName,
+      fileName,
+      directory: entityId,
+    };
+
     if (this.__fetchBucketName(process.env, false)) {
       const s3ClientConfig = AwsS3FileConfig.fetchClientConfig(process.env);
       const awsFileService = new AwsS3FileService(s3ClientConfig);
-      const remoteFileNameInDocument = awsFileService.generateRemotePath({
-        customerName,
-        fileName,
-        directory: entityId,
-      });
+
+      const remoteFileNameInDocument = awsFileService.generateRemotePath(properties);
+
       const awsConfigForClient = this.__fetchAwsConfigForFileNameInBucket(remoteFileNameInDocument);
 
       return {
@@ -237,7 +240,7 @@ export class FileService {
     }
 
     const localFileService = new LocalFileService();
-    const toFileStoragePath = localFileService.generateRemotePath({ fileName });
+    const toFileStoragePath = localFileService.generateRemotePath(properties);
 
     return {
       targetServiceProvider: localFileService,
@@ -261,23 +264,23 @@ export class FileService {
     const { sourceServiceProvider, sourceRemoteFileConfig } =
       this.__fetchSourceServiceProviders(fileDetails);
 
-    const remoteFileNamePrefix = fileDetails.id! || getDocumentId(fileDetails, false);
+    const remoteFileNamePrefix = fileDetails.id || getDocumentId(fileDetails, false);
 
-    let localFilePath;
+    let tmpLocalFilePath;
     if (shouldDownloadFromSource) {
       const tmpFile = tmp.fileSync();
-      localFilePath = await sourceServiceProvider.download(sourceRemoteFileConfig, tmpFile.name);
+      tmpLocalFilePath = await sourceServiceProvider.download(sourceRemoteFileConfig, tmpFile.name);
     }
 
-    const file = await fromFile(localFilePath ?? fileDetails.uri);
+    const file = await fromFile(tmpLocalFilePath ?? fileDetails.uri);
 
     const remoteFileName = `${remoteFileNamePrefix}_${randomUUID()}${
       file?.ext ? `.${file?.ext}` : ''
     }`;
 
-    if (localFilePath) {
+    if (tmpLocalFilePath) {
       try {
-        await fs.unlink(localFilePath);
+        await fs.unlink(tmpLocalFilePath);
       } catch (err) {
         // TODO: should we log non succesful deletetion ?
       }