Skip to content

Commit

Permalink
[OGUI-1370] Migrate to newer NPM module for mysql connection (#2581)
Browse files Browse the repository at this point in the history 
* migrate away from `mysql` to `mariadb` npm module for infologger to match DB used on server
* adds middleware for converting SQL errors to JS native errors
* updates and adds tests accordingly
* update server to use new querying service
  • Loading branch information
@graduta
graduta authored Sep 8, 2024
1 parent de3f0ea commit fa85d9f
Show file tree
Hide file tree
Showing 12 changed files with 522 additions and 318 deletions.
10 changes: 4 additions & 6 deletions InfoLogger/lib/api.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@
* or submit itself to any jurisdiction.
*/

const { InfoLoggerReceiver, MySQL } = require('@aliceo2/web-ui');
const { InfoLoggerReceiver } = require('@aliceo2/web-ui');

const { StatusController } = require('./controller/StatusController.js');
const { QueryController } = require('./controller/QueryController.js');
const { LiveService } = require('./services/LiveService.js');
const { QueryService } = require('./services/QueryService.js');

const ProfileService = require('./ProfileService.js');
const JsonFileConnector = require('./JSONFileConnector.js');

Expand All @@ -26,21 +28,17 @@ const projPackage = require('./../package.json');
const config = require('./configProvider.js');

let liveService = null;
let sqlService = null;
let queryService = null;

module.exports.attachTo = async (http, ws) => {
const { QueryController } = await import('./controller/QueryController.mjs');

if (config.infoLoggerServer) {
const infoLoggerReceiver = new InfoLoggerReceiver();
liveService = new LiveService(ws, config.infoLoggerServer, infoLoggerReceiver);
liveService.initialize();
}

if (config.mysql) {
sqlService = new MySQL(config.mysql);
queryService = new QueryService(sqlService, config.mysql);
queryService = new QueryService(config.mysql);
}
const queryController = new QueryController(queryService);

Expand Down
15 changes: 11 additions & 4 deletions ...Logger/lib/controller/QueryController.mjs → InfoLogger/lib/controller/QueryController.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,12 @@
* or submit itself to any jurisdiction.
*/

import { LogManager, updateAndSendExpressResponseFromNativeError } from '@aliceo2/web-ui';
const { LogManager, updateAndSendExpressResponseFromNativeError } = require('@aliceo2/web-ui');

/**
* Gateway for all calls that are to query InfoLogger database
*/
export class QueryController {
class QueryController {
/**
* Setup QueryController to be used in the API router
* @param {SQLDataSource} queryService - service to be used to query information on the logs
Expand All @@ -39,9 +39,14 @@ export class QueryController {
async getLogs(req, res) {
try {
const { body: { criterias, options } } = req;
if (!criterias || Object.keys(criterias).length === 0) {
res.status(400).json({ error: 'Invalid query parameters provided' });
return;
}
const logs = await this._queryService.queryFromFilters(criterias, options);
res.status(200).json(logs);
} catch (error) {
this._logger.errorMessage(error.toString());
updateAndSendExpressResponseFromNativeError(res, error);
}
}
Expand All @@ -62,9 +67,11 @@ export class QueryController {
const stats = await this._queryService.queryGroupCountLogsBySeverity(runNumber);
res.status(200).json(stats);
} catch (error) {
this._logger.errorMessage(error.toString(), { level: 99, facility: 'ilg/query-ctrl', run: runNumber });
res.status(502).json({ error: `Unable to serve query on stats for runNumber: ${runNumber}` });
this._logger.errorMessage(error.toString());
updateAndSendExpressResponseFromNativeError(res, error);
}
}
}
}

exports.QueryController = QueryController;
22 changes: 7 additions & 15 deletions InfoLogger/lib/controller/StatusController.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ class StatusController {
const result = {
'infoLogger-gui': this._getProjectInfo(),
infoLoggerServer: this._getLiveSourceStatus(ilgServerConfig ?? {}),
mysql: await this._getDataSourceStatus(dataSourceConfig ?? {}),
mysql: this._getDataSourceStatus(dataSourceConfig ?? {}),
};

res.status(200).json(result);
Expand Down Expand Up @@ -122,30 +122,22 @@ class StatusController {
}

/**
* Build object with information and status about data source
* Build object with information and latest known status about data source
* @param {object} config used for retrieving data form data source
* @param {string} config.host - host of the data source
* @param {number} config.port - port of the data source
* @param {string} config.database - database name
* @returns {object} - information on statue of the data source
*/
async _getDataSourceStatus({ host, port, database }) {
const dataSourceStatus = {
_getDataSourceStatus({ host, port, database }) {
return {
host,
port,
database,
status: this?._querySource?.isAvailable
? { ok: true }
: { ok: false, message: 'Data source is not available' },
};
if (this._querySource) {
try {
await this._querySource.isConnectionUpAndRunning();
dataSourceStatus.status = { ok: true };
} catch (error) {
dataSourceStatus.status = { ok: false, message: error.message || error };
}
} else {
dataSourceStatus.status = { ok: false, message: 'There was no data source set up' };
}
return dataSourceStatus;
}
}

Expand Down
236 changes: 123 additions & 113 deletions InfoLogger/lib/services/QueryService.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,28 +12,134 @@
* or submit itself to any jurisdiction.
*/

const logger = require('@aliceo2/web-ui').LogManager
.getLogger(`${process.env.npm_config_log_label ?? 'ilg'}/sql`);
const mariadb = require('mariadb');
const { LogManager } = require('@aliceo2/web-ui');
const { fromSqlToNativeError } = require('../utils/fromSqlToNativeError');

class QueryService {
/**
* Query service that is to be used to map the InfoLogger parameters to SQL query and retrieve data
* @param {MySql} connection - mysql connection
* @param {object} configMySql - mysql config
*/
constructor(connection, configMySql) {
this.configMySql = configMySql;
this.connection = connection;
constructor(configMySql = {}) {
configMySql._user = configMySql?.user ?? 'gui';
configMySql._password = configMySql?.password ?? '';
configMySql._host = configMySql?.host ?? 'localhost';
configMySql._port = configMySql?.port ?? 3306;
configMySql._database = configMySql?.database ?? 'info_logger';
configMySql._connectionLimit = configMySql?.connectionLimit ?? 25;
this._timeout = configMySql?.timeout ?? 10000;

this._pool = mariadb.createPool(configMySql);
this._isAvailable = false;
this._logger = LogManager.getLogger(`${process.env.npm_config_log_label ?? 'ilg'}/query-service`);
}

/**
* Method to test connection of mysql connector once initialized
* @param {number} timeout - timeout for the connection test
* @returns {Promise} - a promise that resolves if connection is successful
*/
async checkConnection(timeout = this._timeout) {
try {
await this._pool.query({
sql: 'SELECT 1',
timeout,
});
this._isAvailable = true;
} catch (error) {
this._isAvailable = false;
fromSqlToNativeError(error);
}
}

/**
* Ask DB for a part of rows and the total count
* - total: how many rows available (limited to 1M)
* - more: true if has more than 1M rows
* - limit: options.limit or 100k
* - rows: the first `limit` rows
* - count: how many rows inside `rows`
* - time: how much did it take, in ms
* @param {object} filters - criteria like MongoDB
* @param {object} options - specific options for the query
* @param {number} options.limit - how many rows to get
* @returns {Promise.<object>} - {total, more, limit, rows, count, time}
*/
async queryFromFilters(filters, options) {
const { limit = 100000 } = options;
const { criteria, values } = this._filtersToSqlConditions(filters);
const criteriaString = this._getCriteriaAsString(criteria);

const requestRows = `SELECT * FROM \`messages\` ${criteriaString} ORDER BY \`TIMESTAMP\` LIMIT ?;`;
const startTime = Date.now(); // ms

let rows = [];
try {
rows = await this._pool.query(
{
sql: requestRows,
timeout: this._timeout,
},
[...values, limit],
);
} catch (error) {
fromSqlToNativeError(error);
}

const totalTime = Date.now() - startTime; // ms
return {
rows,
count: rows.length,
limit: limit,
time: totalTime, // ms
queryAsString: this._getSQLQueryAsString(criteriaString, limit),
};
}

/**
* Method to check if mysql driver connection is up
* @returns {Promise} - resolves/rejects
* Given a runNumber, query logs for it and return a count of the logs grouped by severity
* @param {number|string} runNumber - number of the run for which the query should be performed
* @returns {Promise.<object>} - object containing the count of logs grouped by severity
*/
async isConnectionUpAndRunning() {
await this.connection.query('select timestamp from messages LIMIT 1000;');
const url = `${this.configMySql.host}:${this.configMySql.port}/${this.configMySql.database}`;
logger.infoMessage(`Connected to infoLogger database ${url}`);
async queryGroupCountLogsBySeverity(runNumber) {
const groupByStatement =
'SELECT severity, COUNT(*) FROM messages WHERE run=? and severity '
+ `in ('D', 'I', 'W', 'E', 'F') GROUP BY severity;`;
let data = [];
try {
data = await this._pool.query({
sql: groupByStatement,
timeout: this._timeout,
}, [runNumber]);
} catch (error) {
fromSqlToNativeError(error);
}
const result = { D: 0, I: 0, W: 0, E: 0, F: 0 };

data.forEach((group) => {
result[group['severity']] = group['COUNT(*)'];
});
return result;
}

/**
* Method to fill criteria and return it as string
* @param {Array} criteria Array of criteria set by the user
* @returns {string} - criteria as string in SQL format
*/
_getCriteriaAsString(criteria) {
return criteria && criteria.length ? `WHERE ${criteria.join(' AND ')}` : '';
}

/**
* Get the SQL Query used as a string
* @param {string} criteriaVerbose - criteria as string in SQL format
* @param {number} limit - limit of number of messages
* @returns {string} - SQL Query as string
*/
_getSQLQueryAsString(criteriaVerbose, limit) {
return `SELECT * FROM \`messages\` ${criteriaVerbose} ORDER BY \`TIMESTAMP\` LIMIT ${limit}`;
}

/**
Expand Down Expand Up @@ -150,7 +256,7 @@ class QueryService {
criteria.push(`\`${field}\` IN (?)`);
break;
default:
logger.warn(`unknown operator ${operator}`);
this._logger.warn(`unknown operator ${operator}`);
break;
}
}
Expand All @@ -159,107 +265,11 @@ class QueryService {
}

/**
* Ask DB for a part of rows and the total count
* - total: how many rows available (limited to 1M)
* - more: true if has more than 1M rows
* - limit: options.limit or 100k
* - rows: the first `limit` rows
* - count: how many rows inside `rows`
* - time: how much did it take, in ms
* @param {object} filters - criteria like MongoDB
* @param {object} options - limit, etc.
* @returns {Promise.<object>} - {total, more, limit, rows, count, time}
*/
async queryFromFilters(filters, options) {
if (!filters) {
throw new Error('filters parameter is mandatory');
}
options = { limit: 100000, ...options };

const startTime = Date.now(); // ms
const { criteria, values } = this._filtersToSqlConditions(filters);
const criteriaString = this._getCriteriaAsString(criteria);

const rows = await this._queryMessagesOnOptions(criteriaString, options, values)
.catch((error) => {
logger.error(error);
throw error;
});

const totalTime = Date.now() - startTime; // ms
logger.debug(`Query done in ${totalTime}ms`);
return {
rows,
count: rows.length,
limit: options.limit,
time: totalTime, // ms
queryAsString: this._getSQLQueryAsString(criteriaString, options.limit),
};
}

/**
* Given a runNumber, query logs for it and return a count of the logs grouped by severity
* @param {number|string} runNumber - number of the run for which the query should be performed
* @returns {Promise.<object>} - object containing the count of logs grouped by severity
* Getter for the availability of the service
* @returns {boolean} - true if service is available, false otherwise
*/
async queryGroupCountLogsBySeverity(runNumber) {
const groupByStatement =
'SELECT severity, COUNT(*) FROM messages WHERE run=? and severity '
+ `in ('D', 'I', 'W', 'E', 'F') GROUP BY severity;`;
return this.connection.query(groupByStatement, [runNumber]).then((data) => {
const result = {
D: 0,
I: 0,
W: 0,
E: 0,
F: 0,
};

/**
* data is of structure:
* [
* RowDataPacket { severity: 'E', 'COUNT(*)': 102 }
* ]
*/
data.forEach((group) => {
result[group['severity']] = group['COUNT(*)'];
});
return result;
});
}

/**
* Method to fill criteria and return it as string
* @param {Array} criteria Array of criteria set by the user
* @returns {string} - criteria as string in SQL format
*/
_getCriteriaAsString(criteria) {
return criteria && criteria.length ? `WHERE ${criteria.join(' AND ')}` : '';
}

/**
* Get the SQL Query used as a string
* @param {string} criteriaVerbose - criteria as string in SQL format
* @param {number} limit - limit of number of messages
* @returns {string} - SQL Query as string
*/
_getSQLQueryAsString(criteriaVerbose, limit) {
return `SELECT * FROM \`messages\` ${criteriaVerbose} ORDER BY \`TIMESTAMP\` LIMIT ${limit}`;
}

/**
* Method to retrieve the messages based on passed Options
* @param {string} criteriaString as a string
* @param {object} options containing limit on messages
* @param {Array} values of filter parameters
* @returns {Promise} rows
*/
_queryMessagesOnOptions(criteriaString, options, values) {
// The rows asked with a limit
const requestRows = `SELECT * FROM \`messages\` ${criteriaString} ORDER BY \`TIMESTAMP\` LIMIT ${options.limit}`;

return this.connection.query(requestRows, values)
.then((data) => data);
get isAvailable() {
return this._isAvailable;
}
};

Expand Down
Loading

0 comments on commit fa85d9f

Please sign in to comment.