feat(custom-js): implement POST /processors

.circleci/config.yml

@@ -6,7 +6,7 @@ jobs:
     steps:
     - checkout
     - setup_remote_docker:
-        docker_layer_caching: true
+        docker_layer_caching: false
     - restore_cache:
         key: dependency-cache-{{ checksum "package.json" }}
     - run: npm install
@@ -22,7 +22,7 @@ jobs:
     steps:
       - checkout
       - setup_remote_docker:
-          docker_layer_caching: true
+          docker_layer_caching: false
       - restore_cache:
           key: dependency-cache-{{ checksum "package.json" }}
       - run: npm audit
@@ -49,7 +49,7 @@ jobs:
     steps:
     - checkout
     - setup_remote_docker:
-        docker_layer_caching: true
+        docker_layer_caching: false
     - restore_cache:
         key: dependency-cache-{{ checksum "package.json" }}
     - run:
@@ -88,15 +88,15 @@ jobs:
     steps:
     - checkout
     - setup_remote_docker:
-        docker_layer_caching: true
+        docker_layer_caching: false
     - run: .circleci/dockerBuild.sh
   end-to-end-tests:
     docker:
     - image: circleci/node:8
     steps:
     - checkout
     - setup_remote_docker:
-        docker_layer_caching: true
+        docker_layer_caching: false
     - restore_cache:
         key: dependency-cache-{{ checksum "package.json" }}
     - run: .circleci/dockerRun.sh
@@ -107,7 +107,7 @@ jobs:
     steps:
     - checkout
     - setup_remote_docker:
-        docker_layer_caching: true
+        docker_layer_caching: false
     - restore_cache:
         key: dependency-cache-{{ checksum "package.json" }}
     - add_ssh_keys:
@@ -120,7 +120,7 @@ jobs:
     steps:
     - checkout
     - setup_remote_docker:
-        docker_layer_caching: true
+        docker_layer_caching: false
     - restore_cache:
         key: dependency-cache-{{ checksum "package.json" }}
     - add_ssh_keys:
@@ -133,7 +133,7 @@ jobs:
     steps:
     - checkout
     - setup_remote_docker:
-        docker_layer_caching: true
+        docker_layer_caching: false
     - restore_cache:
         key: dependency-cache-{{ checksum "package.json" }}
     - add_ssh_keys:

docs/openapi3.yaml

@@ -2012,6 +2012,9 @@ components:
           description: Javascript as a string.
     processor_type:
       type: string
+      enum:
+        - raw_javascript
+        - file_download
       description: |
         The type of the processor resource. Can be one of the following:
         * `raw_javascript`: Raw javascript string that will be stored and persisted as it is.

package.json

@@ -44,6 +44,7 @@
     "copy-dir": "^0.3.0",
     "cron": "^1.7.1",
     "dockerode": "^2.5.8",
+    "esprima": "^4.0.1",
     "express": "^4.17.1",
     "express-ajv-swagger-validation": "^0.9.0",
     "express-easy-zip": "^1.1.4",

src/app.js

@@ -8,6 +8,7 @@ let reportsRouter = require('./reports/routes/reportsRoute.js');
 let configRouter = require('./configManager/routes/configRoute.js');
 let dslRouter = require('./tests/routes/dslRoute.js');
 let testsRouter = require('./tests/routes/testsRoute.js');
+let processorssRouter = require('./processors/routes/processorsRoute.js');
 
 let swaggerValidator = require('express-ajv-swagger-validation');
 let audit = require('express-requests-logger');
@@ -56,6 +57,7 @@ module.exports = () => {
             app.use('/v1/dsl', dslRouter);
             app.use('/v1/tests', reportsRouter);
             app.use('/v1/tests', testsRouter);
+            app.use('/v1/processors', processorssRouter);
 
             app.use('/', function (req, res, next) {
                 res.redirect('/ui');

src/common/consts.js

@@ -1,6 +1,9 @@
 module.exports = {
     TEST_TYPE_BASIC: 'basic',
     TEST_TYPE_DSL: 'dsl',
+    PROCESSOR_TYPE_FILE_DOWNLOAD: 'file_download',
+    PROCESSOR_TYPE_RAW_JAVASCRIPT: 'raw_javascript',
+
     ERROR_MESSAGES: {
         NOT_FOUND: 'Not found',
         DSL_DEF_ALREADY_EXIST: 'Definition already exists'

src/database/cassandra-handler/cassandra.js

@@ -4,6 +4,7 @@ const schedulerCassandraConnector = require('../../jobs/models/database/cassandr
 const reportsCassandraConnector = require('../../reports/models/database/cassandra/cassandraConnector');
 const testsCassandraConnector = require('../../tests/models/database/cassandra/cassandraConnector');
 const configCassandraConnector = require('../../configManager/models/database/cassandra/cassandraConnector');
+const processorsCassandraConnector = require('../../processors/models/database/cassandra/cassandraConnector');
 const databaseConfig = require('../../config/databaseConfig');
 const cassandraMigration = require('./cassandraMigration');
 const logger = require('../../common/logger');
@@ -17,6 +18,7 @@ module.exports.init = async () => {
     await schedulerCassandraConnector.init(cassandraClient);
     await testsCassandraConnector.init(cassandraClient);
     await configCassandraConnector.init(cassandraClient);
+    await processorsCassandraConnector.init(cassandraClient);
     logger.info('cassandra client initialized');
 };
 

src/database/cassandra-handler/init-scripts/14__create_processors_table.cql

@@ -0,0 +1,12 @@
+CREATE TABLE IF NOT EXISTS processors
+(
+    processor_id       uuid,
+    name               text,
+    description        text,
+    type               text,
+    file_url           text,
+    javascript         text,
+    created_at timestamp,
+    updated_at timestamp,
+    PRIMARY KEY (processor_id)
+);

src/database/sequlize-handler/sequlize.js

@@ -6,6 +6,7 @@ const schedulerSequlizeConnector = require('../../jobs/models/database/sequelize
 const reportsSequlizeConnector = require('../../reports/models/database/sequelize/sequelizeConnector');
 const testsSequlizeConnector = require('../../tests/models/database/sequelize/sequelizeConnector');
 const configSequlizeConnector = require('../../configManager/models/database/sequelize/sequelizeConnector');
+const processorsSequlizeConnector = require('../../processors/models/database/sequelize/sequelizeConnector');
 const logger = require('../../../src/common/logger');
 const databaseConfig = require('../../config/databaseConfig');
 const Sequelize = require('sequelize');
@@ -17,6 +18,7 @@ module.exports.init = async () => {
     await reportsSequlizeConnector.init(sequlizeClient);
     await testsSequlizeConnector.init(sequlizeClient);
     await configSequlizeConnector.init(sequlizeClient);
+    await processorsSequlizeConnector.init(sequlizeClient);
     await runSequlizeMigrations();
 };
 

src/processors/controllers/processorController.js

@@ -0,0 +1,12 @@
+'use strict';
+let processorManager = require('../models/processorsManager');
+
+module.exports.createProcessor = function (req, res, next) {
+    return processorManager.createProcessor(req.body)
+        .then(function (result) {
+            return res.status(201).json(result);
+        })
+        .catch(function (err) {
+            return next(err);
+        });
+};

src/processors/models/database/cassandra/cassandraConnector.js

@@ -0,0 +1,38 @@
+let logger = require('../../../../common/logger');
+let databaseConfig = require('../../../../config/databaseConfig');
+let client;
+
+const INSERT_PROCESSOR = 'INSERT INTO processors(processor_id, name, description, type, file_url, javascript, created_at, updated_at) values(?,?,?,?,?,?,?,?)';
+
+module.exports = {
+    init,
+    insertProcessor
+};
+
+let queryOptions = {
+    consistency: databaseConfig.cassandraConsistency,
+    prepare: true
+};
+
+async function init(cassandraClient) {
+    client = cassandraClient;
+}
+
+function insertProcessor(processorId, processorInfo) {
+    let params = [processorId, processorInfo.name, processorInfo.description, processorInfo.type, processorInfo.file_url, processorInfo.javascript, Date.now(), Date.now()];
+    return executeQuery(INSERT_PROCESSOR, params, queryOptions);
+}
+
+function executeQuery(query, params, queryOptions) {
+    return client.execute(query, params, { prepare: true }, queryOptions).then((result) => {
+        logger.trace('Query result', {
+            query: query,
+            params: params,
+            rows_returned: result.rowLength
+        });
+        return Promise.resolve(result.rows ? result.rows : []);
+    }).catch((exception) => {
+        logger.error(`Cassandra query failed \n ${JSON.stringify({ query, params, queryOptions })}`, exception);
+        return Promise.reject(new Error('Error occurred in communication with cassandra'));
+    });
+}

src/processors/models/database/databaseConnector.js

@@ -0,0 +1,24 @@
+'use strict';
+
+let databaseConfig = require('../../../config/databaseConfig');
+let cassandraConnector = require('./cassandra/cassandraConnector');
+let sequelizeConnector = require('./sequelize/sequelizeConnector');
+let databaseConnector = databaseConfig.type.toLowerCase() === 'cassandra' ? cassandraConnector : sequelizeConnector;
+
+module.exports = {
+    init,
+    closeConnection,
+    insertProcessor
+};
+
+async function insertProcessor(jobId, jobInfo) {
+    return databaseConnector.insertProcessor(jobId, jobInfo);
+}
+
+async function init() {
+    return databaseConnector.init();
+}
+
+function closeConnection() {
+    return databaseConnector.closeConnection();
+}

src/processors/models/database/sequelize/sequelizeConnector.js

@@ -0,0 +1,60 @@
+'use strict';
+
+const Sequelize = require('sequelize');
+let client;
+
+module.exports = {
+    init,
+    insertProcessor
+};
+
+async function init(sequelizeClient) {
+    client = sequelizeClient;
+    await initSchemas();
+}
+
+async function insertProcessor(processorId, processorInfo) {
+    const processor = client.model('processor');
+    let params = {
+        processor_id: processorId,
+        name: processorInfo.name,
+        description: processorInfo.description,
+        type: processorInfo.type,
+        file_url: processorInfo.file_url,
+        javascript: processorInfo.javascript,
+        created_at: Date.now(),
+        updated_at: Date.now()
+    };
+    return processor.create(params);
+}
+
+async function initSchemas() {
+    const processorsFiles = client.define('processor', {
+        processor_id: {
+            type: Sequelize.DataTypes.UUID,
+            primaryKey: true
+        },
+        name: {
+            type: Sequelize.DataTypes.TEXT('medium')
+        },
+        description: {
+            type: Sequelize.DataTypes.TEXT('long')
+        },
+        type: {
+            type: Sequelize.DataTypes.TEXT('medium')
+        },
+        file_url: {
+            type: Sequelize.DataTypes.TEXT('long')
+        },
+        javascript: {
+            type: Sequelize.DataTypes.TEXT('long')
+        },
+        created_at: {
+            type: Sequelize.DataTypes.DATE
+        },
+        updated_at: {
+            type: Sequelize.DataTypes.DATE
+        }
+    });
+    await processorsFiles.sync();
+}

src/processors/models/processorsManager.js

@@ -0,0 +1,24 @@
+'use strict';
+
+const uuid = require('uuid');
+
+const logger = require('../../common/logger'),
+    databaseConnector = require('./database/databaseConnector'),
+    common = require('../../common/consts.js'),
+    fileManager = require('../../tests/models/fileManager.js');
+
+module.exports.createProcessor = async function (processor) {
+    let processorId = uuid.v4();
+    try {
+        if (processor.type === common.PROCESSOR_TYPE_FILE_DOWNLOAD) {
+            processor.javascript = await fileManager.downloadFile(processor.file_url);
+        }
+        fileManager.validateJavascriptContent(processor.javascript);
+        await databaseConnector.insertProcessor(processorId, processor);
+        logger.info('Processor saved successfully to database');
+        return processor;
+    } catch (error) {
+        logger.error(error, 'Error occurred trying to create new processor');
+        return Promise.reject(error);
+    }
+};

src/processors/routes/processorsRoute.js

@@ -0,0 +1,11 @@
+'use strict';
+
+let swaggerValidator = require('express-ajv-swagger-validation');
+let express = require('express');
+let router = express.Router();
+
+let processors = require('../controllers/processorController');
+
+router.post('/', swaggerValidator.validate, processors.createProcessor);
+
+module.exports = router;