diff --git a/docs/openapi3.yaml b/docs/openapi3.yaml index 2f901ff0d..1dbc5b936 100644 --- a/docs/openapi3.yaml +++ b/docs/openapi3.yaml @@ -2012,6 +2012,9 @@ components: description: Javascript as a string. processor_type: type: string + enum: + - raw_javascript + - file_download description: | The type of the processor resource. Can be one of the following: * `raw_javascript`: Raw javascript string that will be stored and persisted as it is. diff --git a/package.json b/package.json index c8aaa3e27..9d4822844 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "copy-dir": "^0.3.0", "cron": "^1.7.1", "dockerode": "^2.5.8", + "esprima": "^4.0.1", "express": "^4.17.1", "express-ajv-swagger-validation": "^0.9.0", "express-easy-zip": "^1.1.4", diff --git a/src/app.js b/src/app.js index 02efb6164..17db4fe54 100644 --- a/src/app.js +++ b/src/app.js @@ -8,6 +8,7 @@ let reportsRouter = require('./reports/routes/reportsRoute.js'); let configRouter = require('./configManager/routes/configRoute.js'); let dslRouter = require('./tests/routes/dslRoute.js'); let testsRouter = require('./tests/routes/testsRoute.js'); +let processorssRouter = require('./processors/routes/processorsRoute.js'); let swaggerValidator = require('express-ajv-swagger-validation'); let audit = require('express-requests-logger'); @@ -56,6 +57,7 @@ module.exports = () => { app.use('/v1/dsl', dslRouter); app.use('/v1/tests', reportsRouter); app.use('/v1/tests', testsRouter); + app.use('/v1/processors', processorssRouter); app.use('/', function (req, res, next) { res.redirect('/ui'); diff --git a/src/common/consts.js b/src/common/consts.js index 0657b1e26..5665a135e 100644 --- a/src/common/consts.js +++ b/src/common/consts.js @@ -1,6 +1,9 @@ module.exports = { TEST_TYPE_BASIC: 'basic', TEST_TYPE_DSL: 'dsl', + PROCESSOR_TYPE_FILE_DOWNLOAD: 'file_download', + PROCESSOR_TYPE_RAW_JAVASCRIPT: 'raw_javascript', + ERROR_MESSAGES: { NOT_FOUND: 'Not found', DSL_DEF_ALREADY_EXIST: 'Definition already exists' diff --git a/src/database/cassandra-handler/cassandra.js b/src/database/cassandra-handler/cassandra.js index 0a88b38df..a0b4c2059 100644 --- a/src/database/cassandra-handler/cassandra.js +++ b/src/database/cassandra-handler/cassandra.js @@ -4,6 +4,7 @@ const schedulerCassandraConnector = require('../../jobs/models/database/cassandr const reportsCassandraConnector = require('../../reports/models/database/cassandra/cassandraConnector'); const testsCassandraConnector = require('../../tests/models/database/cassandra/cassandraConnector'); const configCassandraConnector = require('../../configManager/models/database/cassandra/cassandraConnector'); +const processorsCassandraConnector = require('../../processors/models/database/cassandra/cassandraConnector'); const databaseConfig = require('../../config/databaseConfig'); const cassandraMigration = require('./cassandraMigration'); const logger = require('../../common/logger'); @@ -17,6 +18,7 @@ module.exports.init = async () => { await schedulerCassandraConnector.init(cassandraClient); await testsCassandraConnector.init(cassandraClient); await configCassandraConnector.init(cassandraClient); + await processorsCassandraConnector.init(cassandraClient); logger.info('cassandra client initialized'); }; diff --git a/src/database/cassandra-handler/init-scripts/14__create_processors_table.cql b/src/database/cassandra-handler/init-scripts/14__create_processors_table.cql new file mode 100644 index 000000000..1808df32e --- /dev/null +++ b/src/database/cassandra-handler/init-scripts/14__create_processors_table.cql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS processors +( + processor_id uuid, + name text, + description text, + type text, + file_url text, + javascript text, + created_at timestamp, + updated_at timestamp, + PRIMARY KEY (processor_id) +); diff --git a/src/database/sequlize-handler/sequlize.js b/src/database/sequlize-handler/sequlize.js index acda53091..556da6235 100644 --- a/src/database/sequlize-handler/sequlize.js +++ b/src/database/sequlize-handler/sequlize.js @@ -6,6 +6,7 @@ const schedulerSequlizeConnector = require('../../jobs/models/database/sequelize const reportsSequlizeConnector = require('../../reports/models/database/sequelize/sequelizeConnector'); const testsSequlizeConnector = require('../../tests/models/database/sequelize/sequelizeConnector'); const configSequlizeConnector = require('../../configManager/models/database/sequelize/sequelizeConnector'); +const processorsSequlizeConnector = require('../../processors/models/database/sequelize/sequelizeConnector'); const logger = require('../../../src/common/logger'); const databaseConfig = require('../../config/databaseConfig'); const Sequelize = require('sequelize'); @@ -17,6 +18,7 @@ module.exports.init = async () => { await reportsSequlizeConnector.init(sequlizeClient); await testsSequlizeConnector.init(sequlizeClient); await configSequlizeConnector.init(sequlizeClient); + await processorsSequlizeConnector.init(sequlizeClient); await runSequlizeMigrations(); }; diff --git a/src/processors/controllers/processorController.js b/src/processors/controllers/processorController.js new file mode 100644 index 000000000..78a598241 --- /dev/null +++ b/src/processors/controllers/processorController.js @@ -0,0 +1,12 @@ +'use strict'; +let processorManager = require('../models/processorsManager'); + +module.exports.createProcessor = function (req, res, next) { + return processorManager.createProcessor(req.body) + .then(function (result) { + return res.status(201).json(result); + }) + .catch(function (err) { + return next(err); + }); +}; \ No newline at end of file diff --git a/src/processors/models/database/cassandra/cassandraConnector.js b/src/processors/models/database/cassandra/cassandraConnector.js new file mode 100644 index 000000000..87b645bfd --- /dev/null +++ b/src/processors/models/database/cassandra/cassandraConnector.js @@ -0,0 +1,38 @@ +let logger = require('../../../../common/logger'); +let databaseConfig = require('../../../../config/databaseConfig'); +let client; + +const INSERT_PROCESSOR = 'INSERT INTO processors(processor_id, name, description, type, file_url, javascript, created_at, updated_at) values(?,?,?,?,?,?,?,?)'; + +module.exports = { + init, + insertProcessor +}; + +let queryOptions = { + consistency: databaseConfig.cassandraConsistency, + prepare: true +}; + +async function init(cassandraClient) { + client = cassandraClient; +} + +function insertProcessor(processorId, processorInfo) { + let params = [processorId, processorInfo.name, processorInfo.description, processorInfo.type, processorInfo.file_url, processorInfo.javascript, Date.now(), Date.now()]; + return executeQuery(INSERT_PROCESSOR, params, queryOptions); +} + +function executeQuery(query, params, queryOptions) { + return client.execute(query, params, { prepare: true }, queryOptions).then((result) => { + logger.trace('Query result', { + query: query, + params: params, + rows_returned: result.rowLength + }); + return Promise.resolve(result.rows ? result.rows : []); + }).catch((exception) => { + logger.error(`Cassandra query failed \n ${JSON.stringify({ query, params, queryOptions })}`, exception); + return Promise.reject(new Error('Error occurred in communication with cassandra')); + }); +} \ No newline at end of file diff --git a/src/processors/models/database/databaseConnector.js b/src/processors/models/database/databaseConnector.js new file mode 100644 index 000000000..254f56ef3 --- /dev/null +++ b/src/processors/models/database/databaseConnector.js @@ -0,0 +1,24 @@ +'use strict'; + +let databaseConfig = require('../../../config/databaseConfig'); +let cassandraConnector = require('./cassandra/cassandraConnector'); +let sequelizeConnector = require('./sequelize/sequelizeConnector'); +let databaseConnector = databaseConfig.type.toLowerCase() === 'cassandra' ? cassandraConnector : sequelizeConnector; + +module.exports = { + init, + closeConnection, + insertProcessor +}; + +async function insertProcessor(jobId, jobInfo) { + return databaseConnector.insertProcessor(jobId, jobInfo); +} + +async function init() { + return databaseConnector.init(); +} + +function closeConnection() { + return databaseConnector.closeConnection(); +} \ No newline at end of file diff --git a/src/processors/models/database/sequelize/sequelizeConnector.js b/src/processors/models/database/sequelize/sequelizeConnector.js new file mode 100644 index 000000000..3afca9b97 --- /dev/null +++ b/src/processors/models/database/sequelize/sequelizeConnector.js @@ -0,0 +1,60 @@ +'use strict'; + +const Sequelize = require('sequelize'); +let client; + +module.exports = { + init, + insertProcessor +}; + +async function init(sequelizeClient) { + client = sequelizeClient; + await initSchemas(); +} + +async function insertProcessor(processorId, processorInfo) { + const processor = client.model('processor'); + let params = { + processor_id: processorId, + name: processorInfo.name, + description: processorInfo.description, + type: processorInfo.type, + file_url: processorInfo.file_url, + javascript: processorInfo.javascript, + created_at: Date.now(), + updated_at: Date.now() + }; + return processor.create(params); +} + +async function initSchemas() { + const processorsFiles = client.define('processor', { + processor_id: { + type: Sequelize.DataTypes.UUID, + primaryKey: true + }, + name: { + type: Sequelize.DataTypes.TEXT('medium') + }, + description: { + type: Sequelize.DataTypes.TEXT('long') + }, + type: { + type: Sequelize.DataTypes.TEXT('medium') + }, + file_url: { + type: Sequelize.DataTypes.TEXT('long') + }, + javascript: { + type: Sequelize.DataTypes.TEXT('long') + }, + created_at: { + type: Sequelize.DataTypes.DATE + }, + updated_at: { + type: Sequelize.DataTypes.DATE + } + }); + await processorsFiles.sync(); +} diff --git a/src/processors/models/processorsManager.js b/src/processors/models/processorsManager.js new file mode 100644 index 000000000..30a92ae6f --- /dev/null +++ b/src/processors/models/processorsManager.js @@ -0,0 +1,24 @@ +'use strict'; + +const uuid = require('uuid'); + +const logger = require('../../common/logger'), + databaseConnector = require('./database/databaseConnector'), + common = require('../../common/consts.js'), + fileManager = require('../../tests/models/fileManager.js'); + +module.exports.createProcessor = async function (processor) { + let processorId = uuid.v4(); + try { + if (processor.type === common.PROCESSOR_TYPE_FILE_DOWNLOAD) { + processor.javascript = await fileManager.downloadFile(processor.file_url); + } + fileManager.validateJavascriptContent(processor.javascript); + await databaseConnector.insertProcessor(processorId, processor); + logger.info('Processor saved successfully to database'); + return processor; + } catch (error) { + logger.error(error, 'Error occurred trying to create new processor'); + return Promise.reject(error); + } +}; \ No newline at end of file diff --git a/src/processors/routes/processorsRoute.js b/src/processors/routes/processorsRoute.js new file mode 100644 index 000000000..111cec906 --- /dev/null +++ b/src/processors/routes/processorsRoute.js @@ -0,0 +1,11 @@ +'use strict'; + +let swaggerValidator = require('express-ajv-swagger-validation'); +let express = require('express'); +let router = express.Router(); + +let processors = require('../controllers/processorController'); + +router.post('/', swaggerValidator.validate, processors.createProcessor); + +module.exports = router; \ No newline at end of file diff --git a/src/tests/models/fileManager.js b/src/tests/models/fileManager.js index 97af0dd88..9b4d9befc 100644 --- a/src/tests/models/fileManager.js +++ b/src/tests/models/fileManager.js @@ -1,12 +1,16 @@ 'use strict'; -const database = require('./database'), - uuid = require('uuid'), +const uuid = require('uuid'), request = require('request-promise-native'), + esprima = require('esprima'); + +const database = require('./database'), { ERROR_MESSAGES } = require('../../common/consts'); module.exports = { createFileFromUrl, - getFile + downloadFile, + getFile, + validateJavascriptContent }; async function createFileFromUrl(testRawData) { if (testRawData['processor_file_url']) { @@ -15,16 +19,16 @@ async function createFileFromUrl(testRawData) { } return undefined; } + async function downloadFile(fileUrl) { const options = { url: fileUrl }; try { const response = await request.get(options); - const base64Value = Buffer.from(response).toString('base64'); - return base64Value; + return response; } catch (err) { - const errMsg = 'Error to read file, throw exception: ' + err; + const errMsg = 'Error to download file: ' + err; const error = new Error(errMsg); error.statusCode = 422; throw error; @@ -45,6 +49,19 @@ async function getFile(fileId) { async function saveFile(fileUrl) { const id = uuid(); const fileToSave = await downloadFile(fileUrl); - await database.saveFile(id, fileToSave); + const fileBase64Value = Buffer.from(fileToSave).toString('base64'); + await database.saveFile(id, fileBase64Value); return id; } + +function validateJavascriptContent (javascriptFileContent) { + let error, errorMessage; + try { + esprima.parseScript(javascriptFileContent); + } catch (err) { + errorMessage = err.description; + error = new Error('javascript syntax validation failed with error: ' + errorMessage); + error.statusCode = 422; + throw error; + } +} \ No newline at end of file diff --git a/tests/integration-tests/processors/helpers/requestCreator.js b/tests/integration-tests/processors/helpers/requestCreator.js new file mode 100644 index 000000000..ae9de1011 --- /dev/null +++ b/tests/integration-tests/processors/helpers/requestCreator.js @@ -0,0 +1,25 @@ + +const request = require('supertest'), + expressApp = require('../../../../src/app'); +let app; +module.exports = { + init, + createProcessor +}; +async function init() { + try { + app = await expressApp(); + } catch (err){ + console.log(err); + process.exit(1); + } +} + +function createProcessor(body, headers) { + return request(app).post('/v1/processors') + .send(body) + .set(headers) + .expect(function(res){ + return res; + }); +} \ No newline at end of file diff --git a/tests/integration-tests/processors/processors-test.js b/tests/integration-tests/processors/processors-test.js new file mode 100644 index 000000000..c90c9c001 --- /dev/null +++ b/tests/integration-tests/processors/processors-test.js @@ -0,0 +1,134 @@ +const should = require('should'), + nock = require('nock'); + +let validHeaders = { 'Content-Type': 'application/json' }; +const requestSender = require('./helpers/requestCreator'); +describe('Processors api', function() { + this.timeout(5000000); + before(async function () { + await requestSender.init(); + }); + + describe('Good requests', function() { + it('Create processor with type file_download', async () => { + nock('https://authentication.predator.dev').get('/?dl=1').reply(200, + `{ + const uuid = require('uuid/v4'); + module.exports = { + createAuthToken + }; + + function createAuthToken(userContext, events, done) { + userContext.vars.token = uuid(); + return done(); + } + }` + ); + + const requestBody = { + name: 'authentication', + description: 'Creates authorization token and saves it in the context', + type: 'file_download', + file_url: 'https://authentication.predator.dev/?dl=1' + }; + let createProcessorResponse = await requestSender.createProcessor(requestBody, validHeaders); + createProcessorResponse.statusCode.should.eql(201); + }); + + it('Create processor with type raw_javascript', async () => { + const requestBody = { + name: 'authentication', + description: 'Creates authorization token and saves it in the context', + type: 'raw_javascript', + javascript: + `{ + const uuid = require('uuid/v4'); + module.exports = { + createAuthToken + }; + + function createAuthToken(userContext, events, done) { + userContext.vars.token = uuid(); + return done(); + } + }` + }; + let createProcessorResponse = await requestSender.createProcessor(requestBody, validHeaders); + createProcessorResponse.statusCode.should.eql(201); + }); + }); + + describe('Bad requests', function () { + it('Create processor with unknown type', async () => { + const requestBody = { + name: 'bad-processor', + description: 'Processor with unknown type', + type: 'unknown' + }; + let createProcessorResponse = await requestSender.createProcessor(requestBody, validHeaders); + createProcessorResponse.statusCode.should.eql(400); + }); + + it('Create processor with type file_download and no url', async () => { + const requestBody = { + name: 'download-me', + description: 'Processor with no file url', + type: 'file_download' + }; + let createProcessorResponse = await requestSender.createProcessor(requestBody, validHeaders); + createProcessorResponse.statusCode.should.eql(400); + }); + + it('Create processor with type raw_javascript and no js', async () => { + const requestBody = { + name: 'javascript-me', + description: 'Processor with no js', + type: 'raw_javascript', + file_url: 'bad' + }; + let createProcessorResponse = await requestSender.createProcessor(requestBody, validHeaders); + createProcessorResponse.statusCode.should.eql(400); + }); + }); + + describe('Sad requests', function () { + it('Create processor with type file_download and invalid file_url', async () => { + nock('https://authentication.predator.dev').get('/?dl=1').replyWithError('error downloading file'); + + const requestBody = { + name: 'authentication', + description: 'Creates authorization token and saves it in the context', + type: 'file_download', + file_url: 'https://authentication.predator.dev/?dl=1' + }; + let createProcessorResponse = await requestSender.createProcessor(requestBody, validHeaders); + createProcessorResponse.statusCode.should.eql(422); + }); + + it('Create processor with type file_download and invalid js syntax', async () => { + nock('https://authentication.predator.dev').get('/?dl=1').reply(200, + `{ + const uuid = require('uuid/v4'); + module.exports = { + createAuthToken + }; + + function createAuthToken(userContext, events, done) { + userContext.vars.token = uuid(); + return done(); + } + + this is not valid javascript + }` + ); + const requestBody = { + name: 'authentication', + description: 'Creates authorization token and saves it in the context', + type: 'file_download', + file_url: 'https://authentication.predator.dev/?dl=1' + }; + let createProcessorResponse = await requestSender.createProcessor(requestBody, validHeaders); + createProcessorResponse.statusCode.should.eql(422); + }); + }); +}); \ No newline at end of file diff --git a/tests/integration-tests/tests/tests-test.js b/tests/integration-tests/tests/tests-test.js index f5eac52d6..589780e55 100644 --- a/tests/integration-tests/tests/tests-test.js +++ b/tests/integration-tests/tests/tests-test.js @@ -39,7 +39,7 @@ describe('the tests api', function() { let requestBody = Object.assign({ processor_file_url: 'https://www.notRealUrl.com' }, simpleTest.test); const res = await requestSender.createTest(requestBody, validHeaders); res.statusCode.should.eql(422); - res.body.message.should.eql('Error to read file, throw exception: RequestError: Error: getaddrinfo ENOTFOUND www.notrealurl.com www.notrealurl.com:443'); + res.body.message.should.eql('Error to download file: RequestError: Error: getaddrinfo ENOTFOUND www.notrealurl.com www.notrealurl.com:443'); }); let badBodyScenarios = ['Body_with_illegal_artillery', 'Body_with_no_artillery_schema', 'Body_with_no_test_type', 'Body_with_no_description', 'Body_with_no_name', 'Body_with_no_scenarios', 'Body_with_no_step_action', 'Body_with_no_steps']; diff --git a/tests/unit-tests/tests/models/fileManager-test.js b/tests/unit-tests/tests/models/fileManager-test.js new file mode 100644 index 000000000..d4ce87598 --- /dev/null +++ b/tests/unit-tests/tests/models/fileManager-test.js @@ -0,0 +1,36 @@ +'use strict'; +const should = require('should'); + +const fileManager = require('../../../../src/tests/models/fileManager'); + +describe('Javascript validation', function () { + it('Should pass javascript validation', function () { + fileManager.validateJavascriptContent(` + { + let i = 10; + i++; + console.log(i); + } + `) + }); + + it('Should fail javascript validation with error thrown', function () { + let error; + try { + fileManager.validateJavascriptContent(` + { + return 10; + + function xyz() { + console.log('xyz') + } + } + `) + } catch(e) { + error = e; + } + + should(error.statusCode).eql(422); + should(error.message).containDeep('javascript syntax validation failed with error: Illegal return statement'); + }); +});