diff --git a/src/helpers/DownloadUploadUtil.ts b/src/helpers/DownloadUploadUtil.ts new file mode 100644 index 00000000..f79f5e25 --- /dev/null +++ b/src/helpers/DownloadUploadUtil.ts @@ -0,0 +1,116 @@ +import { spawn } from "child_process"; +import * as path from "path"; + +import { ConnectorError } from "../utils/errors"; +import * as Helper from "./Helper"; +import connectionPool from "../connectors/ConnectionPool"; +import { callableFunction, SSH } from "../utils/types"; +import FolderUtil from "./FolderUtil"; + +export default class DownloadUploadUtil { + private async exec(command: string, options: any): Promise<{ stdout: string, stderr: string }> { + return new Promise((resolve, reject) => { + console.log("executing command", command); + const child = spawn(command, { shell: true, ...options }); + + let stdout = ''; + let stderr = ''; + + child.stdout.on('data', (data) => { + stdout += data; + }); + + child.stderr.on('data', (data) => { + stderr += data; + }); + + child.on('close', (code) => { + if (code === 0) { + resolve({ stdout, stderr }); + } else { + reject(new Error(`Command failed with exit code ${code}: ${stderr}`)); + } + }); + + child.on('error', (err) => { + reject(new Error(`Exec failed with ${err}`)); + }); + console.log("executed command", command); + }); + } + + private async ssh(hpc: string) : Promise { + const connection = connectionPool[hpc].ssh; + if (!connection) { + throw new ConnectorError(`No connection found for HPC: ${hpc}`); + } + console.log("ssh connection found", connection); + try { + await Helper.runCommandWithBackoff((async (ssh: SSH) => { + if (!ssh.connection.isConnected()) { + await ssh.connection.connect(ssh.config); + } + await ssh.connection.execCommand("echo"); + }) as callableFunction, [connection], null); + } catch (e) { + console.log("error connecting to HPC", hpc, e); + throw new ConnectorError(`Failed to connect to HPC: ${hpc} with error ${e}`); + } + return connection; + } + + private async zip( + from: string, + to: string, + hpc: string + ): Promise { + console.log("zipping file from", from, "to", to); + const ssh = await this.ssh(hpc); + const command = `zip -q -r ${to} ${path.basename(from)}`; + const { stdout, stderr } = await ssh.connection.execCommand(command, { cwd: path.dirname(from) }); + if (stderr) { + throw new Error(`Failed to zip file: ${stderr}`); + } + console.log("zipped file from", from, "to", to); + return stdout; + } + + private async rm( + path: string + ): Promise { + console.log("removing path", path); + const out = await this.exec(`rm -rf ${path};`, {}); + console.log("removed path", path); + return out.stdout; + } + + async download(from: string, to: string, hpc: string): Promise { + console.log("downloading file from", from, "to", to); + if (to === undefined) + throw new ConnectorError("please init input file first"); + + // create from/to zip paths from raw files and zip the from file + const fromZipFilePath = from.endsWith(".zip") ? from : `${from}.zip`; + const toZipFilePath = `${to}.zip`; + await this.zip(from, fromZipFilePath, hpc); + + console.log("start download"); + + try { + // try to get the from file via ssh/scp and remove the compressed folder afterwards + // wraps command with backoff -> takes lambda function and array of inputs to execute command + await Helper.runCommandWithBackoff.call(this, (async (to1: string, zipPath: string) => { + const ssh = await this.ssh(hpc); + await ssh.connection.getFile(to1, zipPath); + }) as callableFunction, [to, fromZipFilePath], "Trying to download file again"); + await this.rm(fromZipFilePath); + + // decompress the transferred file into the toZipFilePath directory + await FolderUtil.putFileFromZip(to, toZipFilePath); + } catch (e) { + const error = `unable to get file from ${from} to ${to}: ` + Helper.assertError(e).toString(); + throw new ConnectorError(error); + } + console.log("downloaded file from", from, "to", to); + } +} diff --git a/src/server/FolderRoutes.ts b/src/server/FolderRoutes.ts index f1ee7c5a..5eed560d 100644 --- a/src/server/FolderRoutes.ts +++ b/src/server/FolderRoutes.ts @@ -13,10 +13,15 @@ import dataSource from "../utils/DB"; import type { updateFolderBody, initGlobusDownloadBody, - GlobusFolder + GlobusFolder, + initBrowserDownloadBody } from "../utils/types"; import { authMiddleWare, requestErrors, validator, schemas, prepareDataForDB, globusTaskList } from "./ServerUtil"; +import * as fs from "fs"; +import e = require("express"); +import DownloadUploadUtil from "../helpers/DownloadUploadUtil"; +import { Job } from "../models/Job"; const folderRouter = express.Router(); @@ -344,4 +349,126 @@ folderRouter.get( } ); -export default folderRouter; \ No newline at end of file +/** + * @openapi + * /folder/:folderId/download/browser: + * post: + * description: Get sends a request to initiate a download of the specified folder (Authentication REQUIRED) + * responses: + * 200: + * description: Download of the specific folder is successful and file returned + * 402: + * description: Returns "invalid input" and a list of errors with the format of the req body or "invalid token" if a valid jupyter token authentication is not provided + * 403: + * description: Returns error when the folder ID cannot be found, when the hpc config for globus cannot be found, when the globus download fails, or when a download is already running for the folder + */ +folderRouter.get( + "/:folderId/download/browser", + authMiddleWare, + async function (req, res) { + const errors = requestErrors( + validator.validate(req.body, schemas.initBrowserDownload) + ); + + if (errors.length > 0) { + res.status(402).json({ error: "invalid input", messages: errors }); + return; + } + + const body = req.body as initBrowserDownloadBody; + + if (!res.locals.username) { + res.status(402).json({ error: "invalid token" }); + return; + } + + // get jobId from body + const jobId = body.jobId; + + // get folder; if not found, error out + const folderId = req.params.folderId; + const folder = await (dataSource + .getRepository(Folder) + .findOneByOrFail({ + id: folderId + }) + ); + + if (!folder) { + res.status(403).json({ error: `cannot find folder with id ${folderId}` }); + return; + } + + var downloadPath = path.join(__dirname, 'uploads'); + if (!fs.existsSync(downloadPath)) { + fs.mkdirSync(downloadPath); + } + + const jobs = await dataSource.getRepository(Job).find({ + where: { userId: res.locals.username as string }, + relations: [ + "remoteDataFolder", + "remoteResultFolder", + "remoteExecutableFolder", + ], + }); + + var curr_job = null; + + for (const job of jobs) { + if (job.id == jobId) { + curr_job = job; + break; + } + } + + if (curr_job == null) { + res.status(403).json({ error: `cannot find job with id ${jobId}` }); + return; + } + + var hpcPath = null; + var hpc = null; + + try { + if (curr_job.remoteExecutableFolder!.id == folderId) { + hpcPath = curr_job.remoteExecutableFolder!.hpcPath; + hpc = curr_job.remoteExecutableFolder!.hpc; + } else { + hpcPath = curr_job.remoteResultFolder!.hpcPath; + hpc = curr_job.remoteResultFolder!.hpc; + } + } catch (err) { + res.status(403).json({ error: `failed to get hpc path or hpc with error: ${Helper.assertError(err).toString()}` }); + return; + } + + if (hpcPath == null) { + res.status(403).json({ error: `cannot find hpc path with folderId ${folderId}` }); + return; + } + + if (hpc == null) { + res.status(403).json({ error: `cannot find hpc with folderId ${folderId}` }); + return; + } + + try { + // res.download(path.join(__dirname, 'FolderRoutes.js')); + downloadPath = path.join(downloadPath, folderId + '.zip'); + const download_util = new DownloadUploadUtil(); + await download_util.download(hpcPath, downloadPath, hpc); + const file = downloadPath; + res.download(file); + } catch (err) { + res + .status(403) + .json({ + error: `failed to download with error: ${Helper.assertError(err).toString()}` + }); + return; + } + } +); + +export default folderRouter; diff --git a/src/server/ServerUtil.ts b/src/server/ServerUtil.ts index c5492ecd..245865df 100644 --- a/src/server/ServerUtil.ts +++ b/src/server/ServerUtil.ts @@ -85,6 +85,14 @@ export const schemas = { }, required: ["jupyterhubApiToken", "toEndpoint", "toPath"], }, + initBrowserDownload: { + type: "object", + properties: { + jobId: { type: "string" }, + jupyterhubApiToken: { type: "string" }, + }, + required: ["jupyterhubApiToken", "jobId"], + }, refreshCache: { type: "object", properties: { @@ -160,4 +168,4 @@ export const authMiddleWare = async ( { error: "Malformed input. No jupyterhub api token passed with request." } ); } -}; \ No newline at end of file +}; diff --git a/src/utils/types.ts b/src/utils/types.ts index c4aa11b0..3c6b519b 100644 --- a/src/utils/types.ts +++ b/src/utils/types.ts @@ -365,6 +365,11 @@ export interface initGlobusDownloadBody { fromPath?: string } +export interface initBrowserDownloadBody { + jupyterhubApiToken: string, + jobId: string, +} + export interface createJobBody { jupyterhubApiToken: string, maintainer?: string, @@ -384,4 +389,4 @@ export interface updateJobBody { remoteExecutableFolder?: object, } -export type callableFunction = (..._args: unknown[]) => unknown; \ No newline at end of file +export type callableFunction = (..._args: unknown[]) => unknown;