Browser download

src/helpers/DownloadUploadUtil.ts

@@ -0,0 +1,116 @@
+import { spawn } from "child_process";
+import * as path from "path";
+
+import { ConnectorError } from "../utils/errors";
+import * as Helper from "./Helper"; 
+import connectionPool from "../connectors/ConnectionPool";
+import { callableFunction, SSH } from "../utils/types";
+import FolderUtil from "./FolderUtil";
+
+export default class DownloadUploadUtil {
+  private async exec(command: string, options: any): Promise<{ stdout: string, stderr: string }> {
+    return new Promise((resolve, reject) => {
+      console.log("executing command", command);
+      const child = spawn(command, { shell: true, ...options });
+
+      let stdout = '';
+      let stderr = '';
+
+      child.stdout.on('data', (data) => {
+        stdout += data;
+      });
+
+      child.stderr.on('data', (data) => {
+        stderr += data;
+      });
+
+      child.on('close', (code) => {
+        if (code === 0) {
+          resolve({ stdout, stderr });
+        } else {
+          reject(new Error(`Command failed with exit code ${code}: ${stderr}`));
+        }
+      });
+
+      child.on('error', (err) => {
+        reject(new Error(`Exec failed with ${err}`));
+      });
+      console.log("executed command", command);
+    });
+  }
+
+  private async ssh(hpc: string) : Promise<SSH> {
+    const connection = connectionPool[hpc].ssh;
+    if (!connection) {
+      throw new ConnectorError(`No connection found for HPC: ${hpc}`);
+    }
+    console.log("ssh connection found", connection);
+    try {
+      await Helper.runCommandWithBackoff((async (ssh: SSH) => {
+        if (!ssh.connection.isConnected()) {
+          await ssh.connection.connect(ssh.config);
+        }
+        await ssh.connection.execCommand("echo");
+      }) as callableFunction, [connection], null);
+    } catch (e) {
+      console.log("error connecting to HPC", hpc, e);
+      throw new ConnectorError(`Failed to connect to HPC: ${hpc} with error ${e}`);
+    }
+    return connection;
+  }
+
+  private async zip(
+    from: string,
+    to: string,
+    hpc: string
+  ): Promise<string | null> {
+    console.log("zipping file from", from, "to", to);
+    const ssh = await this.ssh(hpc);
+    const command = `zip -q -r ${to} ${path.basename(from)}`;
+    const { stdout, stderr } = await ssh.connection.execCommand(command, { cwd: path.dirname(from) });
+    if (stderr) {
+      throw new Error(`Failed to zip file: ${stderr}`);
+    }
+    console.log("zipped file from", from, "to", to);
+    return stdout;
+  }
+
+  private async rm(
+    path: string
+  ): Promise<string | null> {
+    console.log("removing path", path);
+    const out = await this.exec(`rm -rf ${path};`, {});
+    console.log("removed path", path);
+    return out.stdout;
+  }
+
+  async download(from: string, to: string, hpc: string): Promise<void> {
+    console.log("downloading file from", from, "to", to);
+    if (to === undefined)
+      throw new ConnectorError("please init input file first");
+
+    // create from/to zip paths from raw files and zip the from file
+    const fromZipFilePath = from.endsWith(".zip") ? from : `${from}.zip`;
+    const toZipFilePath = `${to}.zip`;
+    await this.zip(from, fromZipFilePath, hpc);
+
+    console.log("start download");
+
+    try {      
+      // try to get the from file via ssh/scp and remove the compressed folder afterwards
+      // wraps command with backoff -> takes lambda function and array of inputs to execute command
+      await Helper.runCommandWithBackoff.call(this, (async (to1: string, zipPath: string) => {
+        const ssh = await this.ssh(hpc);
+        await ssh.connection.getFile(to1, zipPath);
+      }) as callableFunction, [to, fromZipFilePath], "Trying to download file again");
+      await this.rm(fromZipFilePath);
+
+      // decompress the transferred file into the toZipFilePath directory
+      await FolderUtil.putFileFromZip(to, toZipFilePath);
+    } catch (e) {
+      const error = `unable to get file from ${from} to ${to}: ` + Helper.assertError(e).toString();
+      throw new ConnectorError(error);
+    }
+    console.log("downloaded file from", from, "to", to);
+  }
+}

src/server/FolderRoutes.ts

@@ -13,10 +13,15 @@ import dataSource from "../utils/DB";
 import type {
   updateFolderBody,
   initGlobusDownloadBody,
-  GlobusFolder
+  GlobusFolder,
+  initBrowserDownloadBody
 } from "../utils/types";
 
 import { authMiddleWare, requestErrors, validator, schemas, prepareDataForDB, globusTaskList } from "./ServerUtil";
+import * as fs from "fs";
+import e = require("express");
+import DownloadUploadUtil from "../helpers/DownloadUploadUtil";
+import { Job } from "../models/Job";
 
 const folderRouter = express.Router();
 
@@ -344,4 +349,126 @@ folderRouter.get(
   }
 );
 
-export default folderRouter;
+/**
+   * @openapi
+   * /folder/:folderId/download/browser:
+   *  post:
+   *      description: Get sends a request to initiate a download of the specified folder (Authentication REQUIRED)
+   *      responses:
+   *          200:
+   *              description: Download of the specific folder is successful and file returned
+   *          402:
+   *              description: Returns "invalid input" and a list of errors with the format of the req body or "invalid token" if a valid jupyter token authentication is not provided
+   *          403:
+   *              description: Returns error when the folder ID cannot be found, when the hpc config for globus cannot be found, when the globus download fails, or when a download is already running for the folder
+   */
+folderRouter.get(
+  "/:folderId/download/browser", 
+  authMiddleWare, 
+  async function (req, res) {
+    const errors = requestErrors(
+      validator.validate(req.body, schemas.initBrowserDownload)
+    );
+
+    if (errors.length > 0) {
+      res.status(402).json({ error: "invalid input", messages: errors });
+      return;
+    }
+
+    const body = req.body as initBrowserDownloadBody;
+
+    if (!res.locals.username) {
+      res.status(402).json({ error: "invalid token" });
+      return;
+    }
+
+    // get jobId from body
+    const jobId = body.jobId;
+
+    // get folder; if not found, error out
+    const folderId = req.params.folderId;
+    const folder = await (dataSource
+      .getRepository(Folder)
+      .findOneByOrFail({
+        id: folderId
+      })
+    );
+
+    if (!folder) {
+      res.status(403).json({ error: `cannot find folder with id ${folderId}` });
+      return;
+    }
+
+    var downloadPath = path.join(__dirname, 'uploads');
+    if (!fs.existsSync(downloadPath)) {
+      fs.mkdirSync(downloadPath);
+    }
+
+    const jobs = await dataSource.getRepository(Job).find({
+      where: { userId: res.locals.username as string },
+      relations: [
+        "remoteDataFolder",
+        "remoteResultFolder",
+        "remoteExecutableFolder",
+      ],
+    });
+
+    var curr_job = null;
+
+    for (const job of jobs) {
+      if (job.id == jobId) {
+        curr_job = job;
+        break;
+      }
+    }
+
+    if (curr_job == null) {
+      res.status(403).json({ error: `cannot find job with id ${jobId}` });
+      return;
+    }
+
+    var hpcPath = null;
+    var hpc = null;
+
+    try {
+      if (curr_job.remoteExecutableFolder!.id == folderId) {
+        hpcPath = curr_job.remoteExecutableFolder!.hpcPath;
+        hpc = curr_job.remoteExecutableFolder!.hpc;
+      } else {
+        hpcPath = curr_job.remoteResultFolder!.hpcPath;
+        hpc = curr_job.remoteResultFolder!.hpc;
+      }
+    } catch (err) {
+      res.status(403).json({ error: `failed to get hpc path or hpc with error: ${Helper.assertError(err).toString()}` });
+      return;
+    }
+
+    if (hpcPath == null) {
+      res.status(403).json({ error: `cannot find hpc path with folderId ${folderId}` });
+      return;
+    }
+
+    if (hpc == null) {
+      res.status(403).json({ error: `cannot find hpc with folderId ${folderId}` });
+      return;
+    }
+
+    try {
+      // res.download(path.join(__dirname, 'FolderRoutes.js'));
+      downloadPath = path.join(downloadPath, folderId + '.zip');
+      const download_util = new DownloadUploadUtil();
+      await download_util.download(hpcPath, downloadPath, hpc);
+      const file = downloadPath;
+      res.download(file);
+    } catch (err) {
+      res
+        .status(403)
+        .json({ 
+          error: `failed to download with error: ${Helper.assertError(err).toString()}`
+        });
+      return;
+    }
+  }
+);
+
+export default folderRouter;

src/server/ServerUtil.ts

@@ -85,6 +85,14 @@ export const schemas = {
     },
     required: ["jupyterhubApiToken", "toEndpoint", "toPath"],
   },
+  initBrowserDownload: {
+    type: "object",
+    properties: {
+      jobId: { type: "string" },
+      jupyterhubApiToken: { type: "string" },
+    },
+    required: ["jupyterhubApiToken", "jobId"],
+  },
   refreshCache: {
     type: "object",
     properties: {
@@ -160,4 +168,4 @@ export const authMiddleWare = async (
       { error: "Malformed input. No jupyterhub api token passed with request." }
     );
   }
-};
+};

src/utils/types.ts

@@ -365,6 +365,11 @@ export interface initGlobusDownloadBody {
   fromPath?: string 
 }
 
+export interface initBrowserDownloadBody { 
+  jupyterhubApiToken: string,
+  jobId: string,
+}
+
 export interface createJobBody { 
   jupyterhubApiToken: string, 
   maintainer?: string, 
@@ -384,4 +389,4 @@ export interface updateJobBody {
   remoteExecutableFolder?: object,
 }
 
-export type callableFunction = (..._args: unknown[]) => unknown;
+export type callableFunction = (..._args: unknown[]) => unknown;