From 62c103916dc0918eb17d0b2dedd187fa060f3934 Mon Sep 17 00:00:00 2001 From: John Speaks <113151039+JTSIV1@users.noreply.github.com> Date: Mon, 11 Nov 2024 11:02:14 -0600 Subject: [PATCH 1/7] add schema for download req --- src/server/ServerUtil.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/server/ServerUtil.ts b/src/server/ServerUtil.ts index c5492ecd..2724e5c8 100644 --- a/src/server/ServerUtil.ts +++ b/src/server/ServerUtil.ts @@ -85,6 +85,14 @@ export const schemas = { }, required: ["jupyterhubApiToken", "toEndpoint", "toPath"], }, + initBrowserDownload: { + type: "object", + properties: { + jobId: { type: "string" }, + jupyterhubApiToken: { type: "string" }, + }, + required: ["jupyterhubApiToken"], + }, refreshCache: { type: "object", properties: { @@ -160,4 +168,4 @@ export const authMiddleWare = async ( { error: "Malformed input. No jupyterhub api token passed with request." } ); } -}; \ No newline at end of file +}; From 7b00e20821066e347be50fbce0f4e83be25ef191 Mon Sep 17 00:00:00 2001 From: John Speaks <113151039+JTSIV1@users.noreply.github.com> Date: Mon, 11 Nov 2024 11:05:10 -0600 Subject: [PATCH 2/7] initial work on download request in api --- src/server/FolderRoutes.ts | 133 ++++++++++++++++++++++++++++++++++++- 1 file changed, 131 insertions(+), 2 deletions(-) diff --git a/src/server/FolderRoutes.ts b/src/server/FolderRoutes.ts index f1ee7c5a..f6be77d2 100644 --- a/src/server/FolderRoutes.ts +++ b/src/server/FolderRoutes.ts @@ -13,10 +13,19 @@ import dataSource from "../utils/DB"; import type { updateFolderBody, initGlobusDownloadBody, - GlobusFolder + GlobusFolder, + initBrowserDownloadBody } from "../utils/types"; import { authMiddleWare, requestErrors, validator, schemas, prepareDataForDB, globusTaskList } from "./ServerUtil"; +import multer from "multer"; +import Busboy from 'busboy'; +import type { BusboyConfig } from 'busboy'; +import * as fs from "fs"; +import * as os from "os"; +import e = require("express"); +import FolderUtil from "../helpers/FolderUtil"; +import DownloadUploadUtil from "../helpers/DownloadUploadUtil"; const folderRouter = express.Router(); @@ -344,4 +353,124 @@ folderRouter.get( } ); -export default folderRouter; \ No newline at end of file +// // const storage = multer.memoryStorage(); +// // const upload = multer({ storage: storage }).single("file"); +// const busboy = require('busboy'); + +// folderRouter.post( +// "/:folderId/uploadFile", +// async function (req, res) { +// const bb = busboy({ headers: req.headers }); +// const uploadPath = path.join(__dirname, 'uploads'); +// console.log(`Upload path: ${uploadPath}`); + +// if (!fs.existsSync(uploadPath)) { +// fs.mkdirSync(uploadPath); +// } + +// bb.on('file', (fieldname: string, file: NodeJS.ReadableStream, filename: string, encoding: string, mimetype: string) => { +// console.log(`Received file: ${filename}`); +// const saveTo = path.join(uploadPath, filename); +// file.pipe(fs.createWriteStream(saveTo)); +// }); + +// bb.on('finish', () => { +// res.writeHead(200, { 'Connection': 'close' }); +// res.end("File upload complete"); +// }); + +// bb.on('error', (err: any) => { +// console.error('Busboy error:', err); +// res.status(500).send({ error: 'File upload failed' }); +// }); + +// req.pipe(bb); + +// req.on('aborted', () => { +// bb.destroy(); +// }); + +// req.on('error', (err) => { +// console.error('Request stream error:', err); +// res.status(500).send({ error: 'Request stream failed' }); +// }); +// } +// ); + +/** + * @openapi + * /folder/:folderId/download/globus-init: + * post: + * description: Posts a request to initiate a globus download of the specified folder (Authentication REQUIRED) + * responses: + * 200: + * description: Globus download of the specific folder is successful + * 402: + * description: Returns "invalid input" and a list of errors with the format of the req body or "invalid token" if a valid jupyter token authentication is not provided + * 403: + * description: Returns error when the folder ID cannot be found, when the hpc config for globus cannot be found, when the globus download fails, or when a download is already running for the folder + */ +folderRouter.get( + "/:folderId/download/browser", + authMiddleWare, + async function (req, res) { + const errors = requestErrors( + validator.validate(req.body, schemas.initBrowserDownload) + ); + + if (errors.length > 0) { + res.status(402).json({ error: "invalid input", messages: errors }); + return; + } + + const body = req.body as initBrowserDownloadBody; + + if (!res.locals.username) { + res.status(402).json({ error: "invalid token" }); + return; + } + + // get jobId from body + const jobId = body.jobId; + + // get folder; if not found, error out + const folderId = req.params.folderId; + const folder = await (dataSource + .getRepository(Folder) + .findOneByOrFail({ + id: folderId + }) + ); + + if (!folder) { + res.status(403).json({ error: `cannot find folder with id ${folderId}` }); + return; + } + + const downloadPath = path.join(__dirname, 'uploads'); + if (!fs.existsSync(downloadPath)) { + fs.mkdirSync(downloadPath); + } + try { + // const connector = new BaseConnector(hpcConfigMap[folder.hpc]); + // const connector = new BaseConnector(folder.hpc); + // await connector.ssh(); // Ensure the connector is connected + // await connector.download(folder.globusPath, downloadPath); + // const file = path.join(downloadPath, folder.globusPath); + const download_util = new DownloadUploadUtil(); + await download_util.download(folder.globusPath, downloadPath, jobId!); + const file = downloadPath; + res.download(file); + res.status(200).json({ success: true }); + } catch (err) { + res + .status(403) + .json({ + error: `failed to download with error: ${Helper.assertError(err).toString()}` + }); + return; + } + } +); + +export default folderRouter; From fc5deb2e6bef8713fd1c1db0c87ae75497b507ba Mon Sep 17 00:00:00 2001 From: John Speaks <113151039+JTSIV1@users.noreply.github.com> Date: Mon, 11 Nov 2024 11:07:00 -0600 Subject: [PATCH 3/7] Create DownloadUploadUtil.ts --- src/helpers/DownloadUploadUtil.ts | 94 +++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 src/helpers/DownloadUploadUtil.ts diff --git a/src/helpers/DownloadUploadUtil.ts b/src/helpers/DownloadUploadUtil.ts new file mode 100644 index 00000000..c5dd3e87 --- /dev/null +++ b/src/helpers/DownloadUploadUtil.ts @@ -0,0 +1,94 @@ +import { spawn } from "child_process"; +import * as path from "path"; + +import { ConnectorError } from "../utils/errors"; +import * as Helper from "./Helper"; +import connectionPool from "../connectors/ConnectionPool"; +import { callableFunction, SSH } from "../utils/types"; +import FolderUtil from "./FolderUtil"; + +export default class DownloadUploadUtil { + private async exec(command: string, options: any): Promise<{ stdout: string, stderr: string }> { + return new Promise((resolve, reject) => { + const child = spawn(command, { shell: true, ...options }); + + let stdout = ''; + let stderr = ''; + + child.stdout.on('data', (data) => { + stdout += data; + }); + + child.stderr.on('data', (data) => { + stderr += data; + }); + + child.on('close', (code) => { + if (code === 0) { + resolve({ stdout, stderr }); + } else { + reject(new Error(`Command failed with exit code ${code}: ${stderr}`)); + } + }); + + child.on('error', (err) => { + reject(err); + }); + }); + } + + private ssh(jobId: string) : SSH { + return connectionPool[jobId!].ssh; + } + + private async zip( + from: string, + to: string + ): Promise { + const out = await this.exec( + `zip -q -r ${to} . ${path.basename(from)}`, // quiet, recursive, to to at the current directory from the from directory path + Object.assign( + { + cwd: from, // set cwd to spawn child in the from directory + }, + {} + ) + ); + + return out.stdout; + } + + private async rm( + path: string + ): Promise { + + const out = await this.exec(`rm -rf ${path};`, {}); + return out.stdout; + } + + async download(from: string, to: string, jobId: string) { + if (to === undefined) + throw new ConnectorError("please init input file first"); + + // create from/to zip paths from raw files and zip the from file + const fromZipFilePath = from.endsWith(".zip") ? from : `${from}.zip`; + const toZipFilePath = `${to}.zip`; + await this.zip(from, fromZipFilePath); + + try { + // try to get the from file via ssh/scp and remove the compressed folder afterwards + // wraps command with backoff -> takes lambda function and array of inputs to execute command + await Helper.runCommandWithBackoff.call(this, (async (to1: string, zipPath: string) => { + await this.ssh(jobId).connection.getFile(to1, zipPath); + }) as callableFunction, [to, fromZipFilePath], "Trying to download file again"); + await this.rm(fromZipFilePath); + + // decompress the transferred file into the toZipFilePath directory + await FolderUtil.putFileFromZip(to, toZipFilePath); + } catch (e) { + const error = `unable to get file from ${from} to ${to}: ` + Helper.assertError(e).toString(); + throw new ConnectorError(error); + } + } +} + From 0ff07ab0208e251852c786a88e26eddd8a1bf2a0 Mon Sep 17 00:00:00 2001 From: John Speaks <113151039+JTSIV1@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:02:54 -0600 Subject: [PATCH 4/7] New browser download route --- src/server/FolderRoutes.ts | 118 ++++++++++++++++++------------------- 1 file changed, 58 insertions(+), 60 deletions(-) diff --git a/src/server/FolderRoutes.ts b/src/server/FolderRoutes.ts index f6be77d2..5eed560d 100644 --- a/src/server/FolderRoutes.ts +++ b/src/server/FolderRoutes.ts @@ -18,14 +18,10 @@ import type { } from "../utils/types"; import { authMiddleWare, requestErrors, validator, schemas, prepareDataForDB, globusTaskList } from "./ServerUtil"; -import multer from "multer"; -import Busboy from 'busboy'; -import type { BusboyConfig } from 'busboy'; import * as fs from "fs"; -import * as os from "os"; import e = require("express"); -import FolderUtil from "../helpers/FolderUtil"; import DownloadUploadUtil from "../helpers/DownloadUploadUtil"; +import { Job } from "../models/Job"; const folderRouter = express.Router(); @@ -353,58 +349,14 @@ folderRouter.get( } ); -// // const storage = multer.memoryStorage(); -// // const upload = multer({ storage: storage }).single("file"); -// const busboy = require('busboy'); - -// folderRouter.post( -// "/:folderId/uploadFile", -// async function (req, res) { -// const bb = busboy({ headers: req.headers }); -// const uploadPath = path.join(__dirname, 'uploads'); -// console.log(`Upload path: ${uploadPath}`); - -// if (!fs.existsSync(uploadPath)) { -// fs.mkdirSync(uploadPath); -// } - -// bb.on('file', (fieldname: string, file: NodeJS.ReadableStream, filename: string, encoding: string, mimetype: string) => { -// console.log(`Received file: ${filename}`); -// const saveTo = path.join(uploadPath, filename); -// file.pipe(fs.createWriteStream(saveTo)); -// }); - -// bb.on('finish', () => { -// res.writeHead(200, { 'Connection': 'close' }); -// res.end("File upload complete"); -// }); - -// bb.on('error', (err: any) => { -// console.error('Busboy error:', err); -// res.status(500).send({ error: 'File upload failed' }); -// }); - -// req.pipe(bb); - -// req.on('aborted', () => { -// bb.destroy(); -// }); - -// req.on('error', (err) => { -// console.error('Request stream error:', err); -// res.status(500).send({ error: 'Request stream failed' }); -// }); -// } -// ); - /** * @openapi - * /folder/:folderId/download/globus-init: + * /folder/:folderId/download/browser: * post: - * description: Posts a request to initiate a globus download of the specified folder (Authentication REQUIRED) + * description: Get sends a request to initiate a download of the specified folder (Authentication REQUIRED) * responses: * 200: - * description: Globus download of the specific folder is successful + * description: Download of the specific folder is successful and file returned * 402: * description: Returns "invalid input" and a list of errors with the format of the req body or "invalid token" if a valid jupyter token authentication is not provided * 403: @@ -447,21 +399,67 @@ folderRouter.get( return; } - const downloadPath = path.join(__dirname, 'uploads'); + var downloadPath = path.join(__dirname, 'uploads'); if (!fs.existsSync(downloadPath)) { fs.mkdirSync(downloadPath); } + + const jobs = await dataSource.getRepository(Job).find({ + where: { userId: res.locals.username as string }, + relations: [ + "remoteDataFolder", + "remoteResultFolder", + "remoteExecutableFolder", + ], + }); + + var curr_job = null; + + for (const job of jobs) { + if (job.id == jobId) { + curr_job = job; + break; + } + } + + if (curr_job == null) { + res.status(403).json({ error: `cannot find job with id ${jobId}` }); + return; + } + + var hpcPath = null; + var hpc = null; + + try { + if (curr_job.remoteExecutableFolder!.id == folderId) { + hpcPath = curr_job.remoteExecutableFolder!.hpcPath; + hpc = curr_job.remoteExecutableFolder!.hpc; + } else { + hpcPath = curr_job.remoteResultFolder!.hpcPath; + hpc = curr_job.remoteResultFolder!.hpc; + } + } catch (err) { + res.status(403).json({ error: `failed to get hpc path or hpc with error: ${Helper.assertError(err).toString()}` }); + return; + } + + if (hpcPath == null) { + res.status(403).json({ error: `cannot find hpc path with folderId ${folderId}` }); + return; + } + + if (hpc == null) { + res.status(403).json({ error: `cannot find hpc with folderId ${folderId}` }); + return; + } + try { - // const connector = new BaseConnector(hpcConfigMap[folder.hpc]); - // const connector = new BaseConnector(folder.hpc); - // await connector.ssh(); // Ensure the connector is connected - // await connector.download(folder.globusPath, downloadPath); - // const file = path.join(downloadPath, folder.globusPath); + // res.download(path.join(__dirname, 'FolderRoutes.js')); + downloadPath = path.join(downloadPath, folderId + '.zip'); const download_util = new DownloadUploadUtil(); - await download_util.download(folder.globusPath, downloadPath, jobId!); + await download_util.download(hpcPath, downloadPath, hpc); const file = downloadPath; res.download(file); - res.status(200).json({ success: true }); } catch (err) { res .status(403) From de4423de62a3f18832e1355cb73f10075347b833 Mon Sep 17 00:00:00 2001 From: John Speaks <113151039+JTSIV1@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:03:48 -0600 Subject: [PATCH 5/7] add type for download req --- src/utils/types.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/utils/types.ts b/src/utils/types.ts index c4aa11b0..3c6b519b 100644 --- a/src/utils/types.ts +++ b/src/utils/types.ts @@ -365,6 +365,11 @@ export interface initGlobusDownloadBody { fromPath?: string } +export interface initBrowserDownloadBody { + jupyterhubApiToken: string, + jobId: string, +} + export interface createJobBody { jupyterhubApiToken: string, maintainer?: string, @@ -384,4 +389,4 @@ export interface updateJobBody { remoteExecutableFolder?: object, } -export type callableFunction = (..._args: unknown[]) => unknown; \ No newline at end of file +export type callableFunction = (..._args: unknown[]) => unknown; From 9a2bd87b58e8d151d35563e61cc4e08a2c1f73f0 Mon Sep 17 00:00:00 2001 From: John Speaks <113151039+JTSIV1@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:04:44 -0600 Subject: [PATCH 6/7] working download helper --- src/helpers/DownloadUploadUtil.ts | 62 +++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/src/helpers/DownloadUploadUtil.ts b/src/helpers/DownloadUploadUtil.ts index c5dd3e87..f79f5e25 100644 --- a/src/helpers/DownloadUploadUtil.ts +++ b/src/helpers/DownloadUploadUtil.ts @@ -10,6 +10,7 @@ import FolderUtil from "./FolderUtil"; export default class DownloadUploadUtil { private async exec(command: string, options: any): Promise<{ stdout: string, stderr: string }> { return new Promise((resolve, reject) => { + console.log("executing command", command); const child = spawn(command, { shell: true, ...options }); let stdout = ''; @@ -32,54 +33,75 @@ export default class DownloadUploadUtil { }); child.on('error', (err) => { - reject(err); + reject(new Error(`Exec failed with ${err}`)); }); + console.log("executed command", command); }); } - private ssh(jobId: string) : SSH { - return connectionPool[jobId!].ssh; + private async ssh(hpc: string) : Promise { + const connection = connectionPool[hpc].ssh; + if (!connection) { + throw new ConnectorError(`No connection found for HPC: ${hpc}`); + } + console.log("ssh connection found", connection); + try { + await Helper.runCommandWithBackoff((async (ssh: SSH) => { + if (!ssh.connection.isConnected()) { + await ssh.connection.connect(ssh.config); + } + await ssh.connection.execCommand("echo"); + }) as callableFunction, [connection], null); + } catch (e) { + console.log("error connecting to HPC", hpc, e); + throw new ConnectorError(`Failed to connect to HPC: ${hpc} with error ${e}`); + } + return connection; } private async zip( from: string, - to: string + to: string, + hpc: string ): Promise { - const out = await this.exec( - `zip -q -r ${to} . ${path.basename(from)}`, // quiet, recursive, to to at the current directory from the from directory path - Object.assign( - { - cwd: from, // set cwd to spawn child in the from directory - }, - {} - ) - ); - - return out.stdout; + console.log("zipping file from", from, "to", to); + const ssh = await this.ssh(hpc); + const command = `zip -q -r ${to} ${path.basename(from)}`; + const { stdout, stderr } = await ssh.connection.execCommand(command, { cwd: path.dirname(from) }); + if (stderr) { + throw new Error(`Failed to zip file: ${stderr}`); + } + console.log("zipped file from", from, "to", to); + return stdout; } private async rm( path: string ): Promise { - + console.log("removing path", path); const out = await this.exec(`rm -rf ${path};`, {}); + console.log("removed path", path); return out.stdout; } - async download(from: string, to: string, jobId: string) { + async download(from: string, to: string, hpc: string): Promise { + console.log("downloading file from", from, "to", to); if (to === undefined) throw new ConnectorError("please init input file first"); // create from/to zip paths from raw files and zip the from file const fromZipFilePath = from.endsWith(".zip") ? from : `${from}.zip`; const toZipFilePath = `${to}.zip`; - await this.zip(from, fromZipFilePath); + await this.zip(from, fromZipFilePath, hpc); + + console.log("start download"); try { // try to get the from file via ssh/scp and remove the compressed folder afterwards // wraps command with backoff -> takes lambda function and array of inputs to execute command await Helper.runCommandWithBackoff.call(this, (async (to1: string, zipPath: string) => { - await this.ssh(jobId).connection.getFile(to1, zipPath); + const ssh = await this.ssh(hpc); + await ssh.connection.getFile(to1, zipPath); }) as callableFunction, [to, fromZipFilePath], "Trying to download file again"); await this.rm(fromZipFilePath); @@ -89,6 +111,6 @@ export default class DownloadUploadUtil { const error = `unable to get file from ${from} to ${to}: ` + Helper.assertError(e).toString(); throw new ConnectorError(error); } + console.log("downloaded file from", from, "to", to); } } - From c09f399e96d536af5abd4474e2da84bf636d1031 Mon Sep 17 00:00:00 2001 From: John Speaks <113151039+JTSIV1@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:06:19 -0600 Subject: [PATCH 7/7] require jobid --- src/server/ServerUtil.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/ServerUtil.ts b/src/server/ServerUtil.ts index 2724e5c8..245865df 100644 --- a/src/server/ServerUtil.ts +++ b/src/server/ServerUtil.ts @@ -91,7 +91,7 @@ export const schemas = { jobId: { type: "string" }, jupyterhubApiToken: { type: "string" }, }, - required: ["jupyterhubApiToken"], + required: ["jupyterhubApiToken", "jobId"], }, refreshCache: { type: "object",