Skip to content

Commit

Permalink
feat(reporter): implement aggregate mathematics
Browse files Browse the repository at this point in the history
  • Loading branch information
@NivLipetz
NivLipetz committed Mar 4, 2019
1 parent 0d7b132 commit 69b58ea
Show file tree
Hide file tree
Showing 7 changed files with 214 additions and 80 deletions.
55 changes: 55 additions & 0 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
"graceful-shutdown-express": "^2.0.1",
"jsck": "^0.3.2",
"lodash": "^4.17.10",
"mathjs": "^5.6.0",
"mysql2": "^1.6.4",
"node-cmd": "^3.0.0",
"nodemailer": "^5.1.1",
Expand Down
109 changes: 85 additions & 24 deletions src/reports/models/artilleryReportGenerator.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,18 @@
let fs = require('fs');
let path = require('path');
let _ = require('lodash');
let math = require('mathjs');

let logger = require('../../common/logger');
let databaseConnector = require('./databaseConnector');
let reportsManager = require('./reportsManager');

const STATS_INTERVAL = 15;

module.exports.createArtilleryReport = async (testId, reportId) => {
let stats;
try {
stats = await databaseConnector.getStats(testId, reportId);
} catch (error) {
logger.error(error, 'Failed to get stats from database');
error.statusCode = 500;
return Promise.reject(error);
}
let stats, report;
report = await reportsManager.getReport(testId, reportId);
stats = await databaseConnector.getStats(testId, reportId);

if (stats.length === 0) {
let errorMessage = `Can not generate artillery report as testId: ${testId} and reportId: ${reportId} is not found`;
Expand All @@ -25,39 +24,80 @@ module.exports.createArtilleryReport = async (testId, reportId) => {
return Promise.reject(error);
}

let reportInput = { intermediate: [] };
let reportInput = { intermediate: [], final_report: [] };
reportInput.duration = math.min(report.duration, Math.floor(report.duration_seconds));
reportInput.start_time = report.start_time;
reportInput.parallelism = report.parallelism;
reportInput.status = mapReportStatus(report.status);

stats.forEach(stat => {
let data = JSON.parse(stat.data);
data.bucket = Math.floor((new Date(data.timestamp).getTime() - new Date(report.start_time).getTime()) / 1000 / STATS_INTERVAL) * STATS_INTERVAL;
switch (stat.phase_status) {
case 'intermediate':
reportInput.intermediate.push(JSON.parse(stat.data));
reportInput.intermediate.push(data);
break;
case 'aggregate':
reportInput.aggregate = JSON.parse(stat.data);
case 'final_report':
reportInput.final_report.push(data);
break;
default:
logger.warn(stat, 'Received unknown stat from database while creating report');
break;
}
});

if (!reportInput.aggregate) {
if (report.parallelism > 1) {
const bucketIntemerdiates = _.groupBy(reportInput.intermediate, 'bucket');
reportInput.intermediate = _.keysIn(bucketIntemerdiates).map((bucket) => {
return createAggregateManually(bucketIntemerdiates[bucket]);
});
}

if (_.isEmpty(reportInput.final_report)) {
reportInput.aggregate = createAggregateManually(reportInput.intermediate);
} else {
reportInput.aggregate = createAggregateManually(reportInput.final_report);
}

let reportOutput = generateReportFromTemplate(reportInput);
return reportOutput;
};

function createAggregateManually(intermediateStats) {
let result = { requestsCompleted: 0, scenariosCreated: 0, scenariosAvoided: 0, scenariosCompleted: 0, pendingRequests: 0, scenarioCounts: {}, errors: {}, codes: {}, latency: { median: 0, max: 0, min: 9999999 } };
_.each(intermediateStats, function(stats) {
result.latency.median += stats.latency.median;
if (stats.latency.max >= result.latency.max) {
result.latency.max = stats.latency.max;
}
if (stats.latency.min <= result.latency.min) {
result.latency.min = stats.latency.min;
}
function createAggregateManually(listOfStats) {
let medians = [], maxs = [], mins = [], scenario95 = [], scenario99 = [], request95 = [], request99 = [];
let result = {
bucket: 0,
requestsCompleted: 0,
scenariosCreated: 0,
scenariosAvoided: 0,
scenariosCompleted: 0,
pendingRequests: 0,
scenarioCounts: {},
errors: {},
concurrency: 0,
codes: {},
latency: {
median: 0,
max: 0,
min: 0
},
rps: {
mean: 0,
count: 0
},
scenarioDuration: { }
};
_.each(listOfStats, function(stats) {
result.bucket = stats.bucket;
result.concurrency += stats.concurrency;

medians.push(stats.latency.median);
maxs.push(stats.latency.max);
mins.push(stats.latency.min);
request95.push(stats.latency.p95 * stats.requestsCompleted);
request99.push(stats.latency.p99 * stats.requestsCompleted);
scenario95.push(stats.scenarioDuration.p95 * stats.scenariosCompleted);
scenario99.push(stats.scenarioDuration.p99 * stats.scenariosCompleted);

result.scenariosCreated += stats.scenariosCreated;
result.scenariosAvoided += stats.scenariosAvoided;
Expand All @@ -83,12 +123,22 @@ function createAggregateManually(intermediateStats) {
result.errors[error] = count;
}
});

result.rps.count += stats.rps.count;
result.requestsCompleted += stats.requestsCompleted;

result.pendingRequests += stats.pendingRequests;
});

result.latency.median = result.latency.median / intermediateStats.length;
result.rps.mean = result.rps.count / STATS_INTERVAL;
result.latency.median = math.median(medians);
result.latency.min = math.min(mins);
result.latency.max = math.max(maxs);
result.latency.p95 = math.sum(request95) / result.requestsCompleted;
result.latency.p99 = math.sum(request99) / result.requestsCompleted;

result.scenarioDuration.p95 = math.sum(scenario95) / result.scenariosCompleted;
result.scenarioDuration.p99 = math.sum(scenario99) / result.scenariosCompleted;

return result;
}
Expand All @@ -101,4 +151,15 @@ function generateReportFromTemplate(reportInput) {
let compiledTemplate = _.template(template);
let html = compiledTemplate({ report: JSON.stringify(reportInput, null, 2) });
return html;
}

function mapReportStatus(status) {
const mapper = {
'in_progress': 'In progress',
'partially_finished': 'Partially finished',
'finished': 'Finished',
'aborted': 'Aborted',
'failed': 'Failed'
};
return mapper[status];
}
48 changes: 20 additions & 28 deletions src/reports/models/reportsManager.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,7 @@ const databaseConnector = require('./databaseConnector');
const jobConnector = require('../../jobs/models/jobManager');
const serviceConfig = require('../../config/serviceConfig');
const statsConsumer = require('./statsConsumer');

const SUBSCRIBER_DONE_STAGE = 'done';
const SUBSCRIBER_ABORTED_STAGE = 'aborted';
const SUBSCRIBER_FAILED_STAGE = 'error';

const REPORT_STARTED_STATUS = 'started';
const REPORT_IN_PROGRESS_STATUS = 'in_progress';
const REPORT_FINISHED_STATUS = 'finished';
const REPORT_ABORTED_STATUS = 'aborted';
const REPORT_FAILED_STATUS = 'failed';
const REPORT_PARTIALLY_FINISHED_STATUS = 'partially_finished';
const constants = require('../utils/constants');

module.exports.getReport = async (testId, reportId) => {
let reportSummary = await databaseConnector.getReport(testId, reportId);
Expand Down Expand Up @@ -51,7 +41,7 @@ module.exports.postReport = async (testId, reportBody) => {
arrival_rate: job.arrival_rate,
duration: job.duration,
ramp_to: job.ramp_to,
parallelism: job.parallelism,
parallelism: job.parallelism || 1,
max_virtual_users: job.max_virtual_users,
environment: job.environment
};
Expand Down Expand Up @@ -115,25 +105,27 @@ function generateGraphanaUrl(report) {
}

module.exports.updateReport = async (report, status, stats, statsTime) => {
const delayedTimeInMs = Math.max(report.duration * 0.01, 30000);

const subscribersStages = getListOfSubscribersStages(report);
const uniqueSubscribersStages = _.uniq(subscribersStages);

if (status === REPORT_FINISHED_STATUS) {
if (status === constants.REPORT_FINISHED_STATUS) {
if (allSubscribersFinished(uniqueSubscribersStages)) {
await databaseConnector.updateReport(report.test_id, report.report_id, REPORT_FINISHED_STATUS, report.phase, stats.data, statsTime);
await databaseConnector.updateReport(report.test_id, report.report_id, constants.REPORT_FINISHED_STATUS, report.phase, stats.data, statsTime);
} else {
setTimeout(async () => {
await delayedUpdateOfReportStatus(report, stats, statsTime);
}, 30000);
}, delayedTimeInMs);
}
} else if (status === REPORT_ABORTED_STATUS && allSubscribersAborted(uniqueSubscribersStages)) {
await databaseConnector.updateReport(report.test_id, report.report_id, REPORT_ABORTED_STATUS, report.phase, stats.data, statsTime);
} else if (status === REPORT_FAILED_STATUS && allSubscribersFailed(uniqueSubscribersStages)) {
await databaseConnector.updateReport(report.test_id, report.report_id, REPORT_FAILED_STATUS, report.phase, stats.data, statsTime);
} else if (status === REPORT_IN_PROGRESS_STATUS) {
await databaseConnector.updateReport(report.test_id, report.report_id, REPORT_IN_PROGRESS_STATUS, report.phase, stats.data, undefined);
} else if (status === constants.REPORT_ABORTED_STATUS && allSubscribersAborted(uniqueSubscribersStages)) {
await databaseConnector.updateReport(report.test_id, report.report_id, constants.REPORT_ABORTED_STATUS, report.phase, stats.data, statsTime);
} else if (status === constants.REPORT_FAILED_STATUS && allSubscribersFailed(uniqueSubscribersStages)) {
await databaseConnector.updateReport(report.test_id, report.report_id, constants.REPORT_FAILED_STATUS, report.phase, stats.data, statsTime);
} else if (status === constants.REPORT_IN_PROGRESS_STATUS) {
await databaseConnector.updateReport(report.test_id, report.report_id, constants.REPORT_IN_PROGRESS_STATUS, report.phase, stats.data, undefined);
} else {
await databaseConnector.updateReport(report.test_id, report.report_id, REPORT_STARTED_STATUS, report.phase, undefined, undefined);
await databaseConnector.updateReport(report.test_id, report.report_id, constants.REPORT_STARTED_STATUS, report.phase, undefined, undefined);
}
};

Expand All @@ -146,21 +138,21 @@ async function delayedUpdateOfReportStatus (report, stats, statsTime) {
const mostUpToDateReportVersion = await module.exports.getReport(report.test_id, report.report_id);
const subscribersStages = getListOfSubscribersStages(mostUpToDateReportVersion);
const uniqueSubscribersStages = _.uniq(subscribersStages);
if (mostUpToDateReportVersion.status !== REPORT_FINISHED_STATUS && uniqueSubscribersStages.length === 1 && uniqueSubscribersStages[0] === SUBSCRIBER_DONE_STAGE) {
await databaseConnector.updateReport(report.test_id, report.report_id, REPORT_FINISHED_STATUS, report.phase, stats.data, statsTime);
if (mostUpToDateReportVersion.status !== constants.REPORT_FINISHED_STATUS && uniqueSubscribersStages.length === 1 && uniqueSubscribersStages[0] === constants.SUBSCRIBER_DONE_STAGE) {
await databaseConnector.updateReport(report.test_id, report.report_id, constants.REPORT_FINISHED_STATUS, report.phase, stats.data, statsTime);
} else {
await databaseConnector.updateReport(report.test_id, report.report_id, REPORT_PARTIALLY_FINISHED_STATUS, report.phase, stats.data, statsTime);
await databaseConnector.updateReport(report.test_id, report.report_id, constants.REPORT_PARTIALLY_FINISHED_STATUS, report.phase, stats.data, statsTime);
}
}

function allSubscribersFinished (subscribers) {
return subscribers.length === 1 && subscribers[0] === SUBSCRIBER_DONE_STAGE;
return subscribers.length === 1 && subscribers[0] === constants.SUBSCRIBER_DONE_STAGE;
}

function allSubscribersAborted (subscribers) {
return subscribers.length === 1 && subscribers[0] === SUBSCRIBER_ABORTED_STAGE;
return subscribers.length === 1 && subscribers[0] === constants.SUBSCRIBER_ABORTED_STAGE;
}

function allSubscribersFailed (subscribers) {
return subscribers.length === 1 && subscribers[0] === SUBSCRIBER_FAILED_STAGE;
return subscribers.length === 1 && subscribers[0] === constants.SUBSCRIBER_FAILED_STAGE;
}
Loading

0 comments on commit 69b58ea

Please sign in to comment.