From 5476808687eeca1a273ebcfe180796565d64513b Mon Sep 17 00:00:00 2001 From: AlexanderGeere Date: Tue, 12 Nov 2024 10:12:50 +0200 Subject: [PATCH 01/14] Remove provider and create a signer --- mod/provider/s3.js | 102 --------------------------------------------- mod/sign/_sign.js | 4 +- mod/sign/s3.js | 97 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 103 deletions(-) delete mode 100644 mod/provider/s3.js create mode 100644 mod/sign/s3.js diff --git a/mod/provider/s3.js b/mod/provider/s3.js deleted file mode 100644 index 1c728ebbd6..0000000000 --- a/mod/provider/s3.js +++ /dev/null @@ -1,102 +0,0 @@ -/** -@module /provider/s3 -*/ - -const { - S3Client, - PutObjectCommand, - GetObjectCommand, - DeleteObjectCommand, - ListObjectsCommand -} = require('@aws-sdk/client-s3'); - -const { getSignedUrl } = require('@aws-sdk/s3-request-presigner'); - -module.exports = async (req, res) => { - - const credentials = Object.fromEntries(new URLSearchParams(process.env.AWS_S3_CLIENT)) - - const s3Client = new S3Client({ - credentials, - region: req.params.region - }) - - - const commands = { - get, - put, - trash, - list - } - - if (!Object.hasOwn(commands, req.params.command)) { - return res.status(400).send(`S3 command validation failed.`) - } - - return commands[req.params.command](s3Client, req) -} - -async function trash(s3Client, req) { - - const command = new DeleteObjectCommand({ - Key: req.params.key, - Bucket: req.params.bucket - }) - - return s3Client.send(command); -} - -async function get(s3Client, req) { - - try { - - const command = new GetObjectCommand({ - Key: req.params.key, - Bucket: req.params.bucket - }) - - const signedURL = await getSignedUrl(s3Client, command, { - expiresIn: 3600, - }); - - return JSON.stringify(signedURL); - - } catch (err) { - console.error(err) - } -} - -async function put(s3Client, req) { - - try { - - const command = new PutObjectCommand({ - Key: req.params.key, - Bucket: req.params.bucket, - Region: req.params.region - }); - - const signedURL = await getSignedUrl(s3Client, command, { - expiresIn: 3600, - }); - - return JSON.stringify(signedURL); - - } catch (err) { - console.error(err) - } -} - -async function list(s3Client, req) { - - try { - - const command = new ListObjectsCommand({ Bucket: req.params.bucket }) - const response = await s3Client.send(command); - - return response - - } catch (err) { - console.error(err) - } -} \ No newline at end of file diff --git a/mod/sign/_sign.js b/mod/sign/_sign.js index c6115e62e8..7f0a467ba9 100644 --- a/mod/sign/_sign.js +++ b/mod/sign/_sign.js @@ -9,9 +9,11 @@ The sign module provides access to different request signer methods. */ const cloudinary = require('./cloudinary') +const s3 = require('./s3') const signerModules = { - cloudinary + cloudinary, + s3 } /** diff --git a/mod/sign/s3.js b/mod/sign/s3.js new file mode 100644 index 0000000000..31c180ce26 --- /dev/null +++ b/mod/sign/s3.js @@ -0,0 +1,97 @@ +/** +@module /sign/s3 + +Signs requests to S3. Provides functions for get, list, delete and put to S3. +*/ + +const { + S3Client, + PutObjectCommand, + GetObjectCommand, + DeleteObjectCommand, + ListObjectsCommand +} = require('@aws-sdk/client-s3') + +const { getSignedUrl } = require('@aws-sdk/s3-request-presigner'); + +/** +@function s3 +@async + +@description +The s3 signer method signs requests for the s3 service. + +Provides methods for list, get, trash and put + +@param {Object} req HTTP request. +@param {Object} res HTTP response. +@param {Object} req.params Request parameter. +@param {string} params.region +@param {string} params.bucket +@param {string} params.key +@param {string} params.command + +@returns {Promise} The signed url associated to the request params. +**/ +module.exports = async function s3(req, res){ + + //Read credentials from an env key + const credentials = Object.fromEntries(new URLSearchParams(process.env.AWS_S3_CLIENT)) + + const s3Client = new S3Client({ + credentials, + region: req.params.region + }) + + req.params.s3Client = s3Client + + //Assign the corresponding function to the requested command + const commands = { + get: () => objectAction(req.params,GetObjectCommand), + trash: () => objectAction(req.params,DeleteObjectCommand), + put: () => objectAction(req.params,PutObjectCommand), + list: () => objectAction(req.params,ListObjectsCommand) + } + + if (!Object.hasOwn(commands, req.params.command)) { + return res.status(400).send(`S3 command validation failed.`) + } + + return commands[req.params.command]() +} + +async function objectAction(reqParams, objectCommand) { + + //The parameters required per action for S3 + //S3 Parameters are capitalised + const actionParams = { + get: {'key':'Key','bucket': 'Bucket'}, + list: {'bucket': 'Bucket'}, + put: {'key': 'Key','bucket': 'Bucket','region': 'Region'}, + trash: {'key': 'Key','bucket': 'Bucket'} + } + + try { + + //Transfrom our keys into aws key names + const commandParams = Object.keys(reqParams) + .filter(key => Object.keys(actionParams[reqParams.command]).includes(key)) + .reduce((acc, key) => ({ + ...acc, + ...{ [actionParams[reqParams.command][key]]: reqParams[key]} + }), + {} + ) + + const command = new objectCommand(commandParams) + + const signedURL = await getSignedUrl(reqParams.s3Client, command, { + expiresIn: 3600, + }); + + return JSON.stringify(signedURL); + + } catch (err) { + console.error(err) + } +} From 6b9157d5ef83ac98a63c4c1e6d295d70b80bbbe4 Mon Sep 17 00:00:00 2001 From: AlexanderGeere Date: Tue, 12 Nov 2024 10:21:57 +0200 Subject: [PATCH 02/14] remove s3 references in provider --- mod/provider/_provider.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mod/provider/_provider.js b/mod/provider/_provider.js index 542889c732..09b0ffb495 100644 --- a/mod/provider/_provider.js +++ b/mod/provider/_provider.js @@ -6,14 +6,11 @@ const file = require('./file') const cloudfront = require('./cloudfront') -const s3 = require('./s3') - module.exports = async (req, res) => { const provider = { cloudfront, - file, - s3 + file } if (!Object.hasOwn(provider, req.params.provider)) { From 5113539d1df0289bca7bde951ddc0a51e9c8457a Mon Sep 17 00:00:00 2001 From: AlexanderGeere Date: Tue, 12 Nov 2024 16:15:12 +0200 Subject: [PATCH 03/14] function documentation --- mod/sign/s3.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/mod/sign/s3.js b/mod/sign/s3.js index 31c180ce26..63045e1e2a 100644 --- a/mod/sign/s3.js +++ b/mod/sign/s3.js @@ -60,6 +60,22 @@ module.exports = async function s3(req, res){ return commands[req.params.command]() } +/** +@function objectAction +@async + +@description +Generates the signed url for the command and parameters specified from the request + +@param {Function} objectCommand The S3 function to be carried out. +@param {Object} reqParams Request parameters. +@property {string} reqParams.region +@property {string} reqParams.bucket +@property {string} reqParams.key +@property {string} reqParams.command + +@returns {String} The signed url associated to the request params. +**/ async function objectAction(reqParams, objectCommand) { //The parameters required per action for S3 From 8461242bb7fb63d8efd04a60f45ee12a1fae4e2b Mon Sep 17 00:00:00 2001 From: AlexanderGeere Date: Wed, 13 Nov 2024 12:28:12 +0200 Subject: [PATCH 04/14] redirect and exporting null --- mod/provider/_provider.js | 32 +++++++++++++++++++-- mod/provider/s3.js | 44 +++++++++++++++++++++++++++++ mod/sign/_sign.js | 4 +-- mod/sign/s3.js | 59 +++++++++++++++++++++++++++++++-------- 4 files changed, 123 insertions(+), 16 deletions(-) create mode 100644 mod/provider/s3.js diff --git a/mod/provider/_provider.js b/mod/provider/_provider.js index 09b0ffb495..0b37dc264c 100644 --- a/mod/provider/_provider.js +++ b/mod/provider/_provider.js @@ -1,23 +1,49 @@ /** @module /provider + +@description +Functions for handling 3rd party service provider requests */ const file = require('./file') const cloudfront = require('./cloudfront') -module.exports = async (req, res) => { +const s3 = require('./s3') + +/** +@function provider +@async + +@description +The provider method looks up a provider module method matching the provider request parameter and passes the req/res objects as argument to the matched method. + +The response from the method is returned with the HTTP response. + +@param {Object} req HTTP request. +@param {Object} res HTTP response. +@param {Object} req.params Request parameter. +@param {string} params.signer Provider module to handle the request. + +@returns {Promise} The promise resolves into the response from the provider modules method. +*/ +module.exports = async function provider(req, res){ const provider = { cloudfront, - file + file, + s3: {deprecated: true, fn: s3} + } + + if (provider[req.params.provider] && provider[req.params.provider].deprecated){ + return await provider[req.params.provider].fn(req, res) } if (!Object.hasOwn(provider, req.params.provider)) { return res.send(`Failed to validate 'provider' param.`) } - const response = await provider[req.params.provider](req) + const response = await provider[req.params.provider](req, res) req.params.content_type && res.setHeader('content-type', req.params.content_type) diff --git a/mod/provider/s3.js b/mod/provider/s3.js new file mode 100644 index 0000000000..49e4d6a9f6 --- /dev/null +++ b/mod/provider/s3.js @@ -0,0 +1,44 @@ +/** +@module /provider/s3 +@deprecated + +@description +Provides a way to call get, put, trash and list. Directly interfaces with s3 + +This module has been deprecated and replaced with {@link module:/sign/s3~s3} +*/ + +/** +@function s3 +@async + +@description +The s3 provider method will redirect requests to the signer. + +Provides methods for list, get, trash and put + +@param {Object} req HTTP request. +@param {Object} res HTTP response. +@param {Object} req.params Request parameter. + +@returns {Request} A redirect to the s3 signer. +**/ +module.exports = async function s3(req, res) { + + const commands = { + get, + put, + trash, + list + } + + if (!Object.hasOwn(commands, req.params.command)) { + return res.status(400).send(`S3 command validation failed.`) + } + + const paramString = Object.keys(req.params).filter(param => ['command','bucket','key','region'].includes(param)).map(param => `${param}=${req.params[param]}`).join('&') + + res.setHeader('location', `${process.env.DIR}/api/sign/s3?${paramString}`) + + return res.status(301).send() +} \ No newline at end of file diff --git a/mod/sign/_sign.js b/mod/sign/_sign.js index 7f0a467ba9..bded49f8cc 100644 --- a/mod/sign/_sign.js +++ b/mod/sign/_sign.js @@ -34,8 +34,8 @@ The response from the method is returned with the HTTP response. */ module.exports = async function signer(req, res) { - if (!Object.hasOwn(signerModules, req.params.signer)) { - return res.send(`Failed to validate 'provider' param.`) + if (!Object.hasOwn(signerModules, req.params.signer) || !signerModules[req.params.signer]) { + return res.send(`Failed to validate 'sign' param.`) } const response = await signerModules[req.params.signer](req, res) diff --git a/mod/sign/s3.js b/mod/sign/s3.js index 63045e1e2a..d869d7e4a1 100644 --- a/mod/sign/s3.js +++ b/mod/sign/s3.js @@ -1,18 +1,55 @@ /** @module /sign/s3 - +@requires aws-sdk/client-s3 +@requires aws-sdk/s3-request-presigner Signs requests to S3. Provides functions for get, list, delete and put to S3. */ -const { - S3Client, - PutObjectCommand, - GetObjectCommand, - DeleteObjectCommand, - ListObjectsCommand -} = require('@aws-sdk/client-s3') +//The S3 packages are optional. +//Need a temporary assignment to determine if they are available. +let + S3Client, + PutObjectCommand, + GetObjectCommand, + DeleteObjectCommand, + ListObjectsCommand, + getSignedUrl; + +//Check for credentials +if(!process.env?.AWS_S3_CLIENT){ + console.log('S3 Sign: Missing credentials from env: AWS_S3_CLIENT') + module.exports = null +} +else{ + + //Attempt import if credentials are found + try{ + + //Assign constructors and functions from the sdks. + const clientSDK = require('@aws-sdk/client-s3'); + + S3Client = clientSDK.S3Client + PutObjectCommand = clientSDK.PutObjectCommand + GetObjectCommand = clientSDK.GetObjectCommand + DeleteObjectCommand = clientSDK.DeleteObjectCommand + ListObjectsCommand = clientSDK.ListObjectsCommand + + getSignedUrl = require('@aws-sdk/s3-request-presigner').getSignedUrl; + + //Export the function . + module.exports = s3 + } + catch(err){ + + //The module has not been installed. + if(err.code === 'MODULE_NOT_FOUND'){ + console.log('AWS-SDK is not available') + module.exports = null + } + else throw err + } +} -const { getSignedUrl } = require('@aws-sdk/s3-request-presigner'); /** @function s3 @@ -33,8 +70,8 @@ Provides methods for list, get, trash and put @returns {Promise} The signed url associated to the request params. **/ -module.exports = async function s3(req, res){ - +async function s3(req, res){ + //Read credentials from an env key const credentials = Object.fromEntries(new URLSearchParams(process.env.AWS_S3_CLIENT)) From 3859e3fd4219c60ea773440d473ae56de77d5d83 Mon Sep 17 00:00:00 2001 From: AlexanderGeere Date: Wed, 13 Nov 2024 12:30:48 +0200 Subject: [PATCH 05/14] sonarcloud fix --- mod/provider/_provider.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod/provider/_provider.js b/mod/provider/_provider.js index 0b37dc264c..ec92465a7f 100644 --- a/mod/provider/_provider.js +++ b/mod/provider/_provider.js @@ -35,7 +35,7 @@ module.exports = async function provider(req, res){ s3: {deprecated: true, fn: s3} } - if (provider[req.params.provider] && provider[req.params.provider].deprecated){ + if (provider[req.params.provider]?.deprecated){ return await provider[req.params.provider].fn(req, res) } From 20bd27be228225dc57439ac128e3e66ca0df07fd Mon Sep 17 00:00:00 2001 From: Robert Hurst Date: Thu, 14 Nov 2024 12:22:41 +0200 Subject: [PATCH 06/14] Optional dependencies and formatting --- mod/sign/s3.js | 95 +++++++++++++++++++++++++------------------------- package.json | 6 ++-- 2 files changed, 51 insertions(+), 50 deletions(-) diff --git a/mod/sign/s3.js b/mod/sign/s3.js index d869d7e4a1..3529f15de6 100644 --- a/mod/sign/s3.js +++ b/mod/sign/s3.js @@ -7,7 +7,7 @@ Signs requests to S3. Provides functions for get, list, delete and put to S3. //The S3 packages are optional. //Need a temporary assignment to determine if they are available. -let +let S3Client, PutObjectCommand, GetObjectCommand, @@ -16,33 +16,32 @@ let getSignedUrl; //Check for credentials -if(!process.env?.AWS_S3_CLIENT){ +if (!process.env?.AWS_S3_CLIENT) { console.log('S3 Sign: Missing credentials from env: AWS_S3_CLIENT') module.exports = null -} -else{ +} +else { //Attempt import if credentials are found - try{ + try { //Assign constructors and functions from the sdks. const clientSDK = require('@aws-sdk/client-s3'); - + getSignedUrl = require('@aws-sdk/s3-request-presigner').getSignedUrl; + S3Client = clientSDK.S3Client PutObjectCommand = clientSDK.PutObjectCommand GetObjectCommand = clientSDK.GetObjectCommand DeleteObjectCommand = clientSDK.DeleteObjectCommand ListObjectsCommand = clientSDK.ListObjectsCommand - - getSignedUrl = require('@aws-sdk/s3-request-presigner').getSignedUrl; - + //Export the function . module.exports = s3 } - catch(err){ - + catch (err) { + //The module has not been installed. - if(err.code === 'MODULE_NOT_FOUND'){ + if (err.code === 'MODULE_NOT_FOUND') { console.log('AWS-SDK is not available') module.exports = null } @@ -70,7 +69,7 @@ Provides methods for list, get, trash and put @returns {Promise} The signed url associated to the request params. **/ -async function s3(req, res){ +async function s3(req, res) { //Read credentials from an env key const credentials = Object.fromEntries(new URLSearchParams(process.env.AWS_S3_CLIENT)) @@ -84,10 +83,10 @@ async function s3(req, res){ //Assign the corresponding function to the requested command const commands = { - get: () => objectAction(req.params,GetObjectCommand), - trash: () => objectAction(req.params,DeleteObjectCommand), - put: () => objectAction(req.params,PutObjectCommand), - list: () => objectAction(req.params,ListObjectsCommand) + get: () => objectAction(req.params, GetObjectCommand), + trash: () => objectAction(req.params, DeleteObjectCommand), + put: () => objectAction(req.params, PutObjectCommand), + list: () => objectAction(req.params, ListObjectsCommand) } if (!Object.hasOwn(commands, req.params.command)) { @@ -115,36 +114,36 @@ Generates the signed url for the command and parameters specified from the reque **/ async function objectAction(reqParams, objectCommand) { - //The parameters required per action for S3 - //S3 Parameters are capitalised - const actionParams = { - get: {'key':'Key','bucket': 'Bucket'}, - list: {'bucket': 'Bucket'}, - put: {'key': 'Key','bucket': 'Bucket','region': 'Region'}, - trash: {'key': 'Key','bucket': 'Bucket'} - } + //The parameters required per action for S3 + //S3 Parameters are capitalised + const actionParams = { + get: { 'key': 'Key', 'bucket': 'Bucket' }, + list: { 'bucket': 'Bucket' }, + put: { 'key': 'Key', 'bucket': 'Bucket', 'region': 'Region' }, + trash: { 'key': 'Key', 'bucket': 'Bucket' } + } - try { - - //Transfrom our keys into aws key names - const commandParams = Object.keys(reqParams) - .filter(key => Object.keys(actionParams[reqParams.command]).includes(key)) - .reduce((acc, key) => ({ - ...acc, - ...{ [actionParams[reqParams.command][key]]: reqParams[key]} - }), - {} - ) - - const command = new objectCommand(commandParams) - - const signedURL = await getSignedUrl(reqParams.s3Client, command, { - expiresIn: 3600, - }); - - return JSON.stringify(signedURL); - - } catch (err) { - console.error(err) - } + try { + + //Transfrom our keys into aws key names + const commandParams = Object.keys(reqParams) + .filter(key => Object.keys(actionParams[reqParams.command]).includes(key)) + .reduce((acc, key) => ({ + ...acc, + ...{ [actionParams[reqParams.command][key]]: reqParams[key] } + }), + {} + ) + + const command = new objectCommand(commandParams) + + const signedURL = await getSignedUrl(reqParams.s3Client, command, { + expiresIn: 3600, + }); + + return JSON.stringify(signedURL); + + } catch (err) { + console.error(err) + } } diff --git a/package.json b/package.json index dddebad17c..845422e8ba 100644 --- a/package.json +++ b/package.json @@ -20,10 +20,12 @@ "test-docs": "jsdoc --configure jsdoc_test.json --verbose" }, "license": "MIT", - "dependencies": { + "optionalDependencies": { "@aws-sdk/client-s3": "^3.621.0", + "@aws-sdk/s3-request-presigner": "^3.621.0" + }, + "dependencies": { "@aws-sdk/cloudfront-signer": "^3.621.0", - "@aws-sdk/s3-request-presigner": "^3.621.0", "jsonwebtoken": "^9.0.2", "nodemailer": "^6.9.7", "pg": "^8.7.3", From 90bc15e2b23d727d8e110b00af4b4f74022414cc Mon Sep 17 00:00:00 2001 From: dbauszus-glx Date: Thu, 14 Nov 2024 12:08:13 +0000 Subject: [PATCH 07/14] add sign docs --- mod/provider/s3.js | 2 +- mod/sign/_sign.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mod/provider/s3.js b/mod/provider/s3.js index 49e4d6a9f6..1941c80192 100644 --- a/mod/provider/s3.js +++ b/mod/provider/s3.js @@ -41,4 +41,4 @@ module.exports = async function s3(req, res) { res.setHeader('location', `${process.env.DIR}/api/sign/s3?${paramString}`) return res.status(301).send() -} \ No newline at end of file +} diff --git a/mod/sign/_sign.js b/mod/sign/_sign.js index bded49f8cc..59d68d638d 100644 --- a/mod/sign/_sign.js +++ b/mod/sign/_sign.js @@ -1,9 +1,10 @@ /** ## /sign -The sign module provides access to different request signer methods. +The sign API provides access to different request signer modules. Signer modules which are unavailable will export as null and won't be available from the signerModules object methods. @requires /sign/cloudinary +@requires /sign/s3 @module /sign */ From 75f48a2d31a58f72a76612b98cfffdf35dd1eb6c Mon Sep 17 00:00:00 2001 From: dbauszus-glx Date: Thu, 14 Nov 2024 12:28:37 +0000 Subject: [PATCH 08/14] s3 module docs --- mod/provider/s3.js | 18 +++++++++--------- mod/sign/s3.js | 26 ++++++++++++++++---------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/mod/provider/s3.js b/mod/provider/s3.js index 1941c80192..affb0f046d 100644 --- a/mod/provider/s3.js +++ b/mod/provider/s3.js @@ -25,16 +25,16 @@ Provides methods for list, get, trash and put **/ module.exports = async function s3(req, res) { - const commands = { - get, - put, - trash, - list - } + // const commands = { + // get, + // put, + // trash, + // list + // } - if (!Object.hasOwn(commands, req.params.command)) { - return res.status(400).send(`S3 command validation failed.`) - } + // if (!Object.hasOwn(commands, req.params.command)) { + // return res.status(400).send(`S3 command validation failed.`) + // } const paramString = Object.keys(req.params).filter(param => ['command','bucket','key','region'].includes(param)).map(param => `${param}=${req.params[param]}`).join('&') diff --git a/mod/sign/s3.js b/mod/sign/s3.js index 3529f15de6..7c52a17fba 100644 --- a/mod/sign/s3.js +++ b/mod/sign/s3.js @@ -1,13 +1,19 @@ /** -@module /sign/s3 +### /sign/s3 +Signs requests to S3. Provides functions for get, list, delete and put to S3. + +The module requires AWS_S3_CLIENT credentials in the process.env and will export as null if the credentials are not provided. The credentials consist of two parts: an access key ID and a secret access key eg: `AWS_S3_CLIENT="accessKeyId=AKIAIOSFODNN7EXAMPLE&secretAccessKey=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"`. [Both the access key ID and secret access key together are required to authenticate your requests]{@link https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html}. + +The aws-sdk/client-s3 and aws-sdk/s3-request-presigner are optional dependencies. The require will fail and the module will export as null if these optional dependencies are not installed. + @requires aws-sdk/client-s3 @requires aws-sdk/s3-request-presigner -Signs requests to S3. Provides functions for get, list, delete and put to S3. + +@module /sign/s3 */ -//The S3 packages are optional. -//Need a temporary assignment to determine if they are available. let + credentials, S3Client, PutObjectCommand, GetObjectCommand, @@ -15,12 +21,15 @@ let ListObjectsCommand, getSignedUrl; -//Check for credentials +// Check for credentials if (!process.env?.AWS_S3_CLIENT) { console.log('S3 Sign: Missing credentials from env: AWS_S3_CLIENT') module.exports = null -} -else { + +} else { + + //Read credentials from an env key + credentials = Object.fromEntries(new URLSearchParams(process.env.AWS_S3_CLIENT)) //Attempt import if credentials are found try { @@ -71,9 +80,6 @@ Provides methods for list, get, trash and put **/ async function s3(req, res) { - //Read credentials from an env key - const credentials = Object.fromEntries(new URLSearchParams(process.env.AWS_S3_CLIENT)) - const s3Client = new S3Client({ credentials, region: req.params.region From 2a8523161ca95e38cef83d25c99dc22f05e9a32d Mon Sep 17 00:00:00 2001 From: dbauszus-glx Date: Thu, 14 Nov 2024 12:40:31 +0000 Subject: [PATCH 09/14] s3_provider --- mod/provider/_provider.js | 8 ++---- mod/provider/s3.js | 51 ++++++++++++++++----------------------- 2 files changed, 23 insertions(+), 36 deletions(-) diff --git a/mod/provider/_provider.js b/mod/provider/_provider.js index ec92465a7f..32cbdc0d7e 100644 --- a/mod/provider/_provider.js +++ b/mod/provider/_provider.js @@ -32,11 +32,7 @@ module.exports = async function provider(req, res){ const provider = { cloudfront, file, - s3: {deprecated: true, fn: s3} - } - - if (provider[req.params.provider]?.deprecated){ - return await provider[req.params.provider].fn(req, res) + s3, } if (!Object.hasOwn(provider, req.params.provider)) { @@ -48,4 +44,4 @@ module.exports = async function provider(req, res){ req.params.content_type && res.setHeader('content-type', req.params.content_type) res.send(response) -} \ No newline at end of file +} diff --git a/mod/provider/s3.js b/mod/provider/s3.js index affb0f046d..cd284395ed 100644 --- a/mod/provider/s3.js +++ b/mod/provider/s3.js @@ -1,44 +1,35 @@ /** -@module /provider/s3 -@deprecated +### /provider/s3 -@description -Provides a way to call get, put, trash and list. Directly interfaces with s3 +The S3 provider module requires the [S3 signer]{@link module:/sign/s3} and will return a signed request URL. + +@requires /sign/s3 -This module has been deprecated and replaced with {@link module:/sign/s3~s3} +@module /provider/s3 */ +const s3_signer = require('../sign/s3') + +if (!s3_signer) { + + module.exports = null +} else { + + module.exports = s3_provider +} + /** -@function s3 -@async +@function s3_provider @description -The s3 provider method will redirect requests to the signer. - -Provides methods for list, get, trash and put +The s3_provider method returns the s3_signer method. @param {Object} req HTTP request. @param {Object} res HTTP response. -@param {Object} req.params Request parameter. -@returns {Request} A redirect to the s3 signer. +@returns {Function} The s3_signer module method. **/ -module.exports = async function s3(req, res) { - - // const commands = { - // get, - // put, - // trash, - // list - // } - - // if (!Object.hasOwn(commands, req.params.command)) { - // return res.status(400).send(`S3 command validation failed.`) - // } - - const paramString = Object.keys(req.params).filter(param => ['command','bucket','key','region'].includes(param)).map(param => `${param}=${req.params[param]}`).join('&') - - res.setHeader('location', `${process.env.DIR}/api/sign/s3?${paramString}`) - - return res.status(301).send() +async function s3_provider(req, res) { + + return s3_signer(req, res) } From 7071f1b0c6fa9ec563c9c415a5ed5570134cd541 Mon Sep 17 00:00:00 2001 From: AlexanderGeere Date: Thu, 14 Nov 2024 16:00:45 +0200 Subject: [PATCH 10/14] Provide a way connect to a public bucket --- mod/provider/_provider.js | 2 +- mod/sign/s3.js | 76 ++++++++++++++++++++++++++++----------- 2 files changed, 56 insertions(+), 22 deletions(-) diff --git a/mod/provider/_provider.js b/mod/provider/_provider.js index 32cbdc0d7e..b729d7760f 100644 --- a/mod/provider/_provider.js +++ b/mod/provider/_provider.js @@ -32,7 +32,7 @@ module.exports = async function provider(req, res){ const provider = { cloudfront, file, - s3, + s3 } if (!Object.hasOwn(provider, req.params.provider)) { diff --git a/mod/sign/s3.js b/mod/sign/s3.js index 7c52a17fba..0d9edd7221 100644 --- a/mod/sign/s3.js +++ b/mod/sign/s3.js @@ -13,7 +13,6 @@ The aws-sdk/client-s3 and aws-sdk/s3-request-presigner are optional dependencies */ let - credentials, S3Client, PutObjectCommand, GetObjectCommand, @@ -21,15 +20,16 @@ let ListObjectsCommand, getSignedUrl; -// Check for credentials -if (!process.env?.AWS_S3_CLIENT) { - console.log('S3 Sign: Missing credentials from env: AWS_S3_CLIENT') - module.exports = null +if(!process.env.AWS_S3_CLIENT){ -} else { + //Assume the bucket is public if no credentials are supplied + console.log('Sign S3: AWS_S3_CLIENT was not found in the env') - //Read credentials from an env key - credentials = Object.fromEntries(new URLSearchParams(process.env.AWS_S3_CLIENT)) + s3 = public_s3 + module.exports = s3 + +} +else{ //Attempt import if credentials are found try { @@ -37,27 +37,66 @@ if (!process.env?.AWS_S3_CLIENT) { //Assign constructors and functions from the sdks. const clientSDK = require('@aws-sdk/client-s3'); getSignedUrl = require('@aws-sdk/s3-request-presigner').getSignedUrl; - + S3Client = clientSDK.S3Client PutObjectCommand = clientSDK.PutObjectCommand GetObjectCommand = clientSDK.GetObjectCommand DeleteObjectCommand = clientSDK.DeleteObjectCommand ListObjectsCommand = clientSDK.ListObjectsCommand - + //Export the function . module.exports = s3 } catch (err) { - + //The module has not been installed. if (err.code === 'MODULE_NOT_FOUND') { console.log('AWS-SDK is not available') module.exports = null } else throw err - } + } } +//Assign the corresponding function to the requested command +const commands = { + get: (req) => objectAction(req.params, GetObjectCommand), + trash: (req) => objectAction(req.params, DeleteObjectCommand), + put: (req) => objectAction(req.params, PutObjectCommand), + list: (req) => objectAction(req.params, ListObjectsCommand) +} + +/** +@function public_s3 +@async + +@description +The public s3 method returns a url for the bucket. + +Provides methods for list, get, trash and put. + +@param {Object} req HTTP request. +@param {Object} res HTTP response. +@param {Object} req.params Request parameter. +@param {string} params.region +@param {string} params.bucket +@param {string} params.key +@param {string} params.command + +@returns {Promise} The signed url associated to the request params. +**/ +async function public_s3(req, res){ + + if (!Object.hasOwn(commands, req.params.command)) { + return res.status(400).send(`S3 command validation failed.`) + } + + //Public bucket URL + let signedUrl = `https://${req.params.bucket}.s3.${req.params.region}.amazonaws.com` + signedUrl = req.params.command !== 'list' ? signedUrl+`/${req.params.key}` : signedUrl + + return (async () => {return JSON.stringify(signedUrl)})() +} /** @function s3 @@ -80,6 +119,9 @@ Provides methods for list, get, trash and put **/ async function s3(req, res) { + //Read credentials from an env key + const credentials = Object.fromEntries(new URLSearchParams(process.env.AWS_S3_CLIENT)) + const s3Client = new S3Client({ credentials, region: req.params.region @@ -87,19 +129,11 @@ async function s3(req, res) { req.params.s3Client = s3Client - //Assign the corresponding function to the requested command - const commands = { - get: () => objectAction(req.params, GetObjectCommand), - trash: () => objectAction(req.params, DeleteObjectCommand), - put: () => objectAction(req.params, PutObjectCommand), - list: () => objectAction(req.params, ListObjectsCommand) - } - if (!Object.hasOwn(commands, req.params.command)) { return res.status(400).send(`S3 command validation failed.`) } - return commands[req.params.command]() + return commands[req.params.command](req) } /** From 71c86675e5bbe0d33c7a6be57b218eead9312048 Mon Sep 17 00:00:00 2001 From: AlexanderGeere Date: Thu, 14 Nov 2024 17:07:41 +0200 Subject: [PATCH 11/14] public requests are not signed --- mod/sign/s3.js | 35 +---------------------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/mod/sign/s3.js b/mod/sign/s3.js index 0d9edd7221..494c69a85b 100644 --- a/mod/sign/s3.js +++ b/mod/sign/s3.js @@ -25,8 +25,7 @@ if(!process.env.AWS_S3_CLIENT){ //Assume the bucket is public if no credentials are supplied console.log('Sign S3: AWS_S3_CLIENT was not found in the env') - s3 = public_s3 - module.exports = s3 + module.exports = null } else{ @@ -66,38 +65,6 @@ const commands = { list: (req) => objectAction(req.params, ListObjectsCommand) } -/** -@function public_s3 -@async - -@description -The public s3 method returns a url for the bucket. - -Provides methods for list, get, trash and put. - -@param {Object} req HTTP request. -@param {Object} res HTTP response. -@param {Object} req.params Request parameter. -@param {string} params.region -@param {string} params.bucket -@param {string} params.key -@param {string} params.command - -@returns {Promise} The signed url associated to the request params. -**/ -async function public_s3(req, res){ - - if (!Object.hasOwn(commands, req.params.command)) { - return res.status(400).send(`S3 command validation failed.`) - } - - //Public bucket URL - let signedUrl = `https://${req.params.bucket}.s3.${req.params.region}.amazonaws.com` - signedUrl = req.params.command !== 'list' ? signedUrl+`/${req.params.key}` : signedUrl - - return (async () => {return JSON.stringify(signedUrl)})() -} - /** @function s3 @async From 7e73a1589bbb1477666dfeaeed835b689f36a9da Mon Sep 17 00:00:00 2001 From: dbauszus-glx Date: Thu, 14 Nov 2024 16:02:10 +0000 Subject: [PATCH 12/14] package bump; ListObjectsV2Command --- mod/sign/s3.js | 18 ++++++++---------- package.json | 4 ++-- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/mod/sign/s3.js b/mod/sign/s3.js index 494c69a85b..d1d3c52548 100644 --- a/mod/sign/s3.js +++ b/mod/sign/s3.js @@ -13,6 +13,7 @@ The aws-sdk/client-s3 and aws-sdk/s3-request-presigner are optional dependencies */ let + clientSDK, S3Client, PutObjectCommand, GetObjectCommand, @@ -26,22 +27,21 @@ if(!process.env.AWS_S3_CLIENT){ console.log('Sign S3: AWS_S3_CLIENT was not found in the env') module.exports = null - -} -else{ +} else{ //Attempt import if credentials are found try { //Assign constructors and functions from the sdks. - const clientSDK = require('@aws-sdk/client-s3'); + clientSDK = require('@aws-sdk/client-s3'); + getSignedUrl = require('@aws-sdk/s3-request-presigner').getSignedUrl; S3Client = clientSDK.S3Client PutObjectCommand = clientSDK.PutObjectCommand GetObjectCommand = clientSDK.GetObjectCommand DeleteObjectCommand = clientSDK.DeleteObjectCommand - ListObjectsCommand = clientSDK.ListObjectsCommand + ListObjectsCommand = clientSDK.ListObjectsV2Command //Export the function . module.exports = s3 @@ -89,13 +89,11 @@ async function s3(req, res) { //Read credentials from an env key const credentials = Object.fromEntries(new URLSearchParams(process.env.AWS_S3_CLIENT)) - const s3Client = new S3Client({ + req.params.S3Client = new clientSDK.S3Client({ credentials, region: req.params.region }) - req.params.s3Client = s3Client - if (!Object.hasOwn(commands, req.params.command)) { return res.status(400).send(`S3 command validation failed.`) } @@ -144,11 +142,11 @@ async function objectAction(reqParams, objectCommand) { const command = new objectCommand(commandParams) - const signedURL = await getSignedUrl(reqParams.s3Client, command, { + const signedURL = await getSignedUrl(reqParams.S3Client, command, { expiresIn: 3600, }); - return JSON.stringify(signedURL); + return signedURL; } catch (err) { console.error(err) diff --git a/package.json b/package.json index 845422e8ba..977679e91f 100644 --- a/package.json +++ b/package.json @@ -21,8 +21,8 @@ }, "license": "MIT", "optionalDependencies": { - "@aws-sdk/client-s3": "^3.621.0", - "@aws-sdk/s3-request-presigner": "^3.621.0" + "@aws-sdk/client-s3": "^3.691.0", + "@aws-sdk/s3-request-presigner": "^3.691.0" }, "dependencies": { "@aws-sdk/cloudfront-signer": "^3.621.0", From 9cce4f17d1e651693a0ff978a04a02bb25db2baf Mon Sep 17 00:00:00 2001 From: dbauszus-glx Date: Thu, 14 Nov 2024 16:50:07 +0000 Subject: [PATCH 13/14] Update commands --- mod/sign/s3.js | 168 ++++++++++++++++++++----------------------------- 1 file changed, 68 insertions(+), 100 deletions(-) diff --git a/mod/sign/s3.js b/mod/sign/s3.js index d1d3c52548..0f6234b598 100644 --- a/mod/sign/s3.js +++ b/mod/sign/s3.js @@ -4,6 +4,45 @@ Signs requests to S3. Provides functions for get, list, delete and put to S3. The module requires AWS_S3_CLIENT credentials in the process.env and will export as null if the credentials are not provided. The credentials consist of two parts: an access key ID and a secret access key eg: `AWS_S3_CLIENT="accessKeyId=AKIAIOSFODNN7EXAMPLE&secretAccessKey=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"`. [Both the access key ID and secret access key together are required to authenticate your requests]{@link https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html}. +Sample requests for common S3 SDK commands. Please refer to the S3 SDK documentation for detailed information in regards to the Command methods. + +```js +let url; // URL to be signed. + +// List S3 Bucket +url = `${host}/api/sign/s3?` + mapp.utils.paramString({ + command: 'ListObjectsV2Command', + Bucket, + Region}) + +// Get object from S3 Bucket +url = `${host}/api/sign/s3?` + mapp.utils.paramString({ + command: 'GetObjectCommand', + Key, //file name + Bucket, + Region}) + +// Get object from S3 Bucket +url = `${host}/api/sign/s3?` + mapp.utils.paramString({ + command: 'PutObjectCommand', + Key, //file name + Bucket, + Region}) + +// Get object from S3 Bucket +url = `${host}/api/sign/s3?` + mapp.utils.paramString({ + command: 'DeleteObjectCommand', + Key, //file name + Bucket, + Region}) + +// Sign URL +const signedURL = await mapp.utils.xhr({ + url, + responseType: 'text' +}) +``` + The aws-sdk/client-s3 and aws-sdk/s3-request-presigner are optional dependencies. The require will fail and the module will export as null if these optional dependencies are not installed. @requires aws-sdk/client-s3 @@ -12,14 +51,9 @@ The aws-sdk/client-s3 and aws-sdk/s3-request-presigner are optional dependencies @module /sign/s3 */ -let - clientSDK, - S3Client, - PutObjectCommand, - GetObjectCommand, - DeleteObjectCommand, - ListObjectsCommand, - getSignedUrl; +let clientSDK; +let getSignedUrl; +let credentials; if(!process.env.AWS_S3_CLIENT){ @@ -27,128 +61,62 @@ if(!process.env.AWS_S3_CLIENT){ console.log('Sign S3: AWS_S3_CLIENT was not found in the env') module.exports = null -} else{ +} else { //Attempt import if credentials are found try { - //Assign constructors and functions from the sdks. - clientSDK = require('@aws-sdk/client-s3'); + // Create credentials object from AWS_S3_CLIENT + credentials = Object.fromEntries(new URLSearchParams(process.env.AWS_S3_CLIENT)) + // Require will err if installed without optional dependencies. + clientSDK = require('@aws-sdk/client-s3'); getSignedUrl = require('@aws-sdk/s3-request-presigner').getSignedUrl; - - S3Client = clientSDK.S3Client - PutObjectCommand = clientSDK.PutObjectCommand - GetObjectCommand = clientSDK.GetObjectCommand - DeleteObjectCommand = clientSDK.DeleteObjectCommand - ListObjectsCommand = clientSDK.ListObjectsV2Command - - //Export the function . - module.exports = s3 + + module.exports = s3_signer } catch (err) { - - //The module has not been installed. - if (err.code === 'MODULE_NOT_FOUND') { - console.log('AWS-SDK is not available') - module.exports = null - } - else throw err - } -} -//Assign the corresponding function to the requested command -const commands = { - get: (req) => objectAction(req.params, GetObjectCommand), - trash: (req) => objectAction(req.params, DeleteObjectCommand), - put: (req) => objectAction(req.params, PutObjectCommand), - list: (req) => objectAction(req.params, ListObjectsCommand) + module.exports = null + } } /** -@function s3 +@function s3_signer @async @description -The s3 signer method signs requests for the s3 service. +The S3 signer method checks whether the command string parameter exists in the S3 clientSDK. -Provides methods for list, get, trash and put +The provided request params will be spread into the Command object created from the S3 clientSDK. @param {Object} req HTTP request. @param {Object} res HTTP response. -@param {Object} req.params Request parameter. -@param {string} params.region -@param {string} params.bucket -@param {string} params.key -@param {string} params.command +@property {Object} req.params Request parameter. +@property {String} params.command S3Client SDK command. @returns {Promise} The signed url associated to the request params. **/ -async function s3(req, res) { - - //Read credentials from an env key - const credentials = Object.fromEntries(new URLSearchParams(process.env.AWS_S3_CLIENT)) +async function s3_signer(req, res) { - req.params.S3Client = new clientSDK.S3Client({ + const S3Client = new clientSDK.S3Client({ credentials, - region: req.params.region + region: req.params.Region }) - if (!Object.hasOwn(commands, req.params.command)) { - return res.status(400).send(`S3 command validation failed.`) + if (!Object.hasOwn(clientSDK, req.params.command)) { + return res.status(400).send(`S3 clientSDK command validation failed.`) } - return commands[req.params.command](req) -} + // Spread req.params into the clientSDK Command. + const Command = new clientSDK[req.params.command]({...req.params}) -/** -@function objectAction -@async - -@description -Generates the signed url for the command and parameters specified from the request - -@param {Function} objectCommand The S3 function to be carried out. -@param {Object} reqParams Request parameters. -@property {string} reqParams.region -@property {string} reqParams.bucket -@property {string} reqParams.key -@property {string} reqParams.command - -@returns {String} The signed url associated to the request params. -**/ -async function objectAction(reqParams, objectCommand) { - - //The parameters required per action for S3 - //S3 Parameters are capitalised - const actionParams = { - get: { 'key': 'Key', 'bucket': 'Bucket' }, - list: { 'bucket': 'Bucket' }, - put: { 'key': 'Key', 'bucket': 'Bucket', 'region': 'Region' }, - trash: { 'key': 'Key', 'bucket': 'Bucket' } - } - - try { - - //Transfrom our keys into aws key names - const commandParams = Object.keys(reqParams) - .filter(key => Object.keys(actionParams[reqParams.command]).includes(key)) - .reduce((acc, key) => ({ - ...acc, - ...{ [actionParams[reqParams.command][key]]: reqParams[key] } - }), - {} - ) - - const command = new objectCommand(commandParams) - - const signedURL = await getSignedUrl(reqParams.S3Client, command, { + const signedURL = await getSignedUrl( + S3Client, + Command, + { expiresIn: 3600, }); - return signedURL; - - } catch (err) { - console.error(err) - } + return signedURL; } From 82641d643bec2d58a499dd1dbbbe224328952278 Mon Sep 17 00:00:00 2001 From: Robert Hurst Date: Fri, 15 Nov 2024 16:44:35 +0200 Subject: [PATCH 14/14] Update s3 sign docs --- mod/sign/s3.js | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/mod/sign/s3.js b/mod/sign/s3.js index 0f6234b598..8b31d52eb6 100644 --- a/mod/sign/s3.js +++ b/mod/sign/s3.js @@ -2,6 +2,9 @@ ### /sign/s3 Signs requests to S3. Provides functions for get, list, delete and put to S3. +> For public buckets you do not need to use the s3 sign in order to get or list from the bucket. +> See bellow for examples of how public interactions + The module requires AWS_S3_CLIENT credentials in the process.env and will export as null if the credentials are not provided. The credentials consist of two parts: an access key ID and a secret access key eg: `AWS_S3_CLIENT="accessKeyId=AKIAIOSFODNN7EXAMPLE&secretAccessKey=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"`. [Both the access key ID and secret access key together are required to authenticate your requests]{@link https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html}. Sample requests for common S3 SDK commands. Please refer to the S3 SDK documentation for detailed information in regards to the Command methods. @@ -41,7 +44,15 @@ const signedURL = await mapp.utils.xhr({ url, responseType: 'text' }) + +// Public Bucket Operations (No credentials needed) +// List bucket contents +url = `https://${Bucket}.s3.${Region}.amazonaws.com?list-type=2` + +// Get object +url = `https://${Bucket}.s3.${Region}.amazonaws.com/${Key}` ``` +Note: Write operations (PUT, DELETE) are not available for public buckets. The aws-sdk/client-s3 and aws-sdk/s3-request-presigner are optional dependencies. The require will fail and the module will export as null if these optional dependencies are not installed. @@ -55,7 +66,7 @@ let clientSDK; let getSignedUrl; let credentials; -if(!process.env.AWS_S3_CLIENT){ +if (!process.env.AWS_S3_CLIENT) { //Assume the bucket is public if no credentials are supplied console.log('Sign S3: AWS_S3_CLIENT was not found in the env') @@ -72,13 +83,13 @@ if(!process.env.AWS_S3_CLIENT){ // Require will err if installed without optional dependencies. clientSDK = require('@aws-sdk/client-s3'); getSignedUrl = require('@aws-sdk/s3-request-presigner').getSignedUrl; - + module.exports = s3_signer } catch (err) { module.exports = null - } + } } /** @@ -109,7 +120,7 @@ async function s3_signer(req, res) { } // Spread req.params into the clientSDK Command. - const Command = new clientSDK[req.params.command]({...req.params}) + const Command = new clientSDK[req.params.command]({ ...req.params }) const signedURL = await getSignedUrl( S3Client,