From c2527d1ddc3e72c73b08f0e944db6e8871d168ca Mon Sep 17 00:00:00 2001 From: Vince Zarola Date: Tue, 21 May 2019 10:23:23 +0100 Subject: [PATCH 1/4] Seperate Stats Parser classes into their own files I am planning on adding unit tests for these parsers. Seperating them out should make this easier, it keeps the files smaller and more readable, plus it keeps things more organised. --- server/controllers/stats.js | 235 +------------------------ server/helpers/app-stats-parser.js | 120 +++++++++++++ server/helpers/general-stats-parser.js | 65 +++++++ server/helpers/label-parser.js | 34 ++++ server/helpers/stats-parser.js | 19 ++ 5 files changed, 241 insertions(+), 232 deletions(-) create mode 100644 server/helpers/app-stats-parser.js create mode 100644 server/helpers/general-stats-parser.js create mode 100644 server/helpers/label-parser.js create mode 100644 server/helpers/stats-parser.js diff --git a/server/controllers/stats.js b/server/controllers/stats.js index 60cb2ec..1b1b815 100644 --- a/server/controllers/stats.js +++ b/server/controllers/stats.js @@ -1,7 +1,9 @@ var express = require('express'); var router = express.Router(); + +var AppStatsParser = require('../helpers/app-stats-parser.js'); +var GeneralStatsParser = require('../helpers/general-stats-parser.js'); var helpers = require('../helpers/app-helpers.js'); -var logger = require('config-logger'); router.get('/', function(req, res) { var successcb = function(data){ @@ -23,235 +25,4 @@ router.get('/', function(req, res) { }); -/** - * The Stats Parser class provides base functionality that individual stats - * parsers can reuse. It's designed for individual parsers to inherit from. - */ -class StatsParser { - constructor() { - // Captures the key in a Javascript Object Literal (i.e. everything before the equals sign) - var labelNameRE = '([^=]+)'; - - // Captures the value in a Javascript Object Literal (i.e. anything between the double quotes) - var labelValueRE = '"([^"]*)"'; - - // Matches the key/value pair in a Javascript Object Literal - this._labelRE = labelNameRE + '=' + labelValueRE; - - this._spacesRE = '\\s+'; - this._valueRE = '(\\d+)'; - } -} - -/** - * This class is used to parse Javascript literal object data e.g. - * {appId="01FDACB01",fnID="01FDACC02",image="fnproject/hello"} - */ -class LabelParser extends StatsParser { - constructor() { - super(); - - this._regex = RegExp(this._labelRE + ',?', 'gm'); - - this._WHOLE_MATCH = 0; - this._LABEL_KEY = 1; - this._LABEL_VALUE = 2; - } - - parse(data) { - // Remove the start and end curly braces as they're not part of the data - data = data.replace(/^{|}$/g, ''); - - var jsonData = {}; - - var labelData; - while((labelData = this._regex.exec(data)) !== null) { - var labelKey = labelData[this._LABEL_KEY]; - var labelValue = labelData[this._LABEL_VALUE]; - - jsonData[labelKey] = labelValue; - } - - return jsonData; - } -} - -/** - * This parser is used to parse the fn_queue, fn_running and fn_complete data - * from the Fn server's metrics API. This data isn't app specific, but is for - * the whole system. - */ -class GeneralStatsParser extends StatsParser { - constructor() { - super(); - - this._metricNames = { - 'fn_queued': 'Queue', - 'fn_running': 'Running', - 'fn_completed': 'Complete', - }; - - var metricNameRE = '(' + Object.keys(this._metricNames).join('|') + ')'; - var metricsRE = '^' + metricNameRE + this._spacesRE + this._valueRE; - - this._regex = RegExp(metricsRE, 'gm'); - - //regexMatch[0] = the whole match - //regexMatch[1] = metric name (e.g. fn_completed) - //regexMatch[2] = metric value (integer) - this._WHOLE_MATCH = 0; - this._METRIC_NAME = 1; - this._METRIC_VALUE = 2; - } - - /* - * Parse the Fn server stats data and return an object containing the - * results. - * - * Example data structure for object being returned: - * Complete: 3 - * Queue: 0 - * Running: 1 - * - * @param {String} data the data to parse. - * - * @return {Object} an object representing the parsed data as per the - * documentation above. - */ - parse(data) { - var jsonData = {}; - - var metricData; - while((metricData = this._regex.exec(data)) !== null) { - logger.debug( - "Processing General Stat: " + metricData[this._WHOLE_MATCH] - ); - - var metricsName = metricData[this._METRIC_NAME]; - var metricsHumanName = this._metricNames[metricsName]; - var metricsValue = parseInt(metricData[this._METRIC_VALUE]); - - jsonData[metricsHumanName] = metricsValue; - } - - return jsonData; - } -} - -/** - * This class is used to parse app specific stats from the Fn server's metrics - * API. - */ -class AppStatsParser extends StatsParser { - constructor() { - super(); - - this._metricNames = { - 'fn_container_start_total': 'Starting', - 'fn_container_busy_total': 'Busy', - 'fn_container_idle_total': 'Idling', - 'fn_container_paused_total': 'Paused', - 'fn_container_wait_total': 'Waiting', - }; - - var metricNameRE = '(' + Object.keys(this._metricNames).join('|') + ')'; - - // Match fn container info e.g. {app_id="01D8JQSKDENG8G00GZJ000000B"} - var fnDataRE = '{' + '[^}]+' + '}'; - - // unfortunately we cannot use ((?:, labelRE)+) as it won't capture the - // middle key-value pair - var metricsRE = '^' + metricNameRE + '(' + fnDataRE + ')' + this._spacesRE + this._valueRE; - - this._regex = RegExp(metricsRE, 'gm'); - - //regexMatch[0] = the whole match - //regexMatch[1] = metric name (e.g. fn_container_busy_total) - //regexMatch[2] = fn data in javascript object notation (e.g. {app_id="01D7"}) - //regexMatch[3] = metric value (integer) - this._WHOLE_MATCH = 0; - this._METRIC_NAME = 1; - this._FN_DATA = 2; - this._METRIC_VALUE = 3; - } - - /* - * Parse the stats data and return an object containing the app specific - * stats. - * - * Example data structure for object being returned: - * 01D8JQSKDENG8G00GZJ000000B - * Functions - * 01D8JQSQ2VNG8G00GZJ000000C - * Busy: 1 - * Idling: 0 - * Paused: 0 - * Starting: 0 - * Waiting: 0 - * 01D8JQSQ2VNG8G00GZJ000000D - * Busy: 0 - * Idling: 0 - * Paused: 0 - * Starting: 0 - * Waiting: 1 - * - * @param {String} data the data to parse. - * - * @return {Object} an object representing the parsed data as per the - * documentation above. - */ - parse(data) { - var jsonData = {}; - - var labelParser = new LabelParser(); - var metricData; - while((metricData = this._regex.exec(data)) !== null) { - logger.debug("Processing App Stat: " + metricData[0]); - - var metricsName = metricData[this._METRIC_NAME]; - var metricsHumanName = this._metricNames[metricsName]; - var metricsValue = parseInt(metricData[this._METRIC_VALUE]); - - var rawFnData = metricData[this._FN_DATA]; - var fnData = labelParser.parse(rawFnData); - - jsonData = this._addData(jsonData, fnData.app_id, fnData.fn_id, - metricsHumanName, metricsValue - ); - } - - return jsonData; - } - - /** - * Adds App Data to the object that we're going to return. - * - * @param {Object} data the object to append the data to. - * @param {String} appId the ID of the Fn App which this data belongs to. - * @param {String} fnId the ID of the Fn function which this data belongs to. - * @param {String} metricsHumanName the human readable name of the metric being recorded. - * @param {Int} metricsValue the value of the metric that was parsed. - * - * @return {Object} the data object with the app data added. - */ - _addData(data, appId, fnId, metricsHumanName, metricsValue) { - if(data[appId] === undefined) { - data[appId] = {'Functions': {}}; - } - - if(data[appId].Functions[fnId] === undefined) { - data[appId].Functions[fnId] = {}; - } - - // Aggregate data for all fn images - if(metricsHumanName in data[appId].Functions[fnId]) { - data[appId].Functions[fnId][metricsHumanName] += metricsValue; - } else { - data[appId].Functions[fnId][metricsHumanName] = metricsValue; - } - - return data; - } -} - module.exports = router; diff --git a/server/helpers/app-stats-parser.js b/server/helpers/app-stats-parser.js new file mode 100644 index 0000000..aa5ff24 --- /dev/null +++ b/server/helpers/app-stats-parser.js @@ -0,0 +1,120 @@ +const logger = require('config-logger'); + +const LabelParser = require('./label-parser.js'); +const StatsParser = require('./stats-parser.js'); + +/** + * This class is used to parse app specific stats from the Fn server's metrics + * API. + */ +module.exports = class AppStatsParser extends StatsParser { + constructor() { + super(); + + this._metricNames = { + 'fn_container_start_total': 'Starting', + 'fn_container_busy_total': 'Busy', + 'fn_container_idle_total': 'Idling', + 'fn_container_paused_total': 'Paused', + 'fn_container_wait_total': 'Waiting', + }; + + var metricNameRE = '(' + Object.keys(this._metricNames).join('|') + ')'; + + // Match fn container info e.g. {app_id="01D8JQSKDENG8G00GZJ000000B"} + var fnDataRE = '{' + '[^}]+' + '}'; + + // unfortunately we cannot use ((?:, labelRE)+) as it won't capture the + // middle key-value pair + var metricsRE = '^' + metricNameRE + '(' + fnDataRE + ')' + this._spacesRE + this._valueRE; + + this._regex = RegExp(metricsRE, 'gm'); + + //regexMatch[0] = the whole match + //regexMatch[1] = metric name (e.g. fn_container_busy_total) + //regexMatch[2] = fn data in javascript object notation (e.g. {app_id="01D7"}) + //regexMatch[3] = metric value (integer) + this._WHOLE_MATCH = 0; + this._METRIC_NAME = 1; + this._FN_DATA = 2; + this._METRIC_VALUE = 3; + } + + /* + * Parse the stats data and return an object containing the app specific + * stats. + * + * Example data structure for object being returned: + * 01D8JQSKDENG8G00GZJ000000B + * Functions + * 01D8JQSQ2VNG8G00GZJ000000C + * Busy: 1 + * Idling: 0 + * Paused: 0 + * Starting: 0 + * Waiting: 0 + * 01D8JQSQ2VNG8G00GZJ000000D + * Busy: 0 + * Idling: 0 + * Paused: 0 + * Starting: 0 + * Waiting: 1 + * + * @param {String} data the data to parse. + * + * @return {Object} an object representing the parsed data as per the + * documentation above. + */ + parse(data) { + var jsonData = {}; + + var labelParser = new LabelParser(); + var metricData; + while((metricData = this._regex.exec(data)) !== null) { + logger.debug("Processing App Stat: " + metricData[0]); + + var metricsName = metricData[this._METRIC_NAME]; + var metricsHumanName = this._metricNames[metricsName]; + var metricsValue = parseInt(metricData[this._METRIC_VALUE]); + + var rawFnData = metricData[this._FN_DATA]; + var fnData = labelParser.parse(rawFnData); + + jsonData = this._addData(jsonData, fnData.app_id, fnData.fn_id, + metricsHumanName, metricsValue + ); + } + + return jsonData; + } + + /** + * Adds App Data to the object that we're going to return. + * + * @param {Object} data the object to append the data to. + * @param {String} appId the ID of the Fn App which this data belongs to. + * @param {String} fnId the ID of the Fn function which this data belongs to. + * @param {String} metricsHumanName the human readable name of the metric being recorded. + * @param {Int} metricsValue the value of the metric that was parsed. + * + * @return {Object} the data object with the app data added. + */ + _addData(data, appId, fnId, metricsHumanName, metricsValue) { + if(data[appId] === undefined) { + data[appId] = {'Functions': {}}; + } + + if(data[appId].Functions[fnId] === undefined) { + data[appId].Functions[fnId] = {}; + } + + // Aggregate data for all fn images + if(metricsHumanName in data[appId].Functions[fnId]) { + data[appId].Functions[fnId][metricsHumanName] += metricsValue; + } else { + data[appId].Functions[fnId][metricsHumanName] = metricsValue; + } + + return data; + } +}; diff --git a/server/helpers/general-stats-parser.js b/server/helpers/general-stats-parser.js new file mode 100644 index 0000000..f793047 --- /dev/null +++ b/server/helpers/general-stats-parser.js @@ -0,0 +1,65 @@ +const logger = require('config-logger'); + +const StatsParser = require('./stats-parser.js'); + +/** + * This parser is used to parse the fn_queue, fn_running and fn_complete data + * from the Fn server's metrics API. This data isn't app specific, but is for + * the whole system. + */ +module.exports = class GeneralStatsParser extends StatsParser { + constructor() { + super(); + + this._metricNames = { + 'fn_queued': 'Queue', + 'fn_running': 'Running', + 'fn_completed': 'Complete', + }; + + var metricNameRE = '(' + Object.keys(this._metricNames).join('|') + ')'; + var metricsRE = '^' + metricNameRE + this._spacesRE + this._valueRE; + + this._regex = RegExp(metricsRE, 'gm'); + + //regexMatch[0] = the whole match + //regexMatch[1] = metric name (e.g. fn_completed) + //regexMatch[2] = metric value (integer) + this._WHOLE_MATCH = 0; + this._METRIC_NAME = 1; + this._METRIC_VALUE = 2; + } + + /* + * Parse the Fn server stats data and return an object containing the + * results. + * + * Example data structure for object being returned: + * Complete: 3 + * Queue: 0 + * Running: 1 + * + * @param {String} data the data to parse. + * + * @return {Object} an object representing the parsed data as per the + * documentation above. + */ + parse(data) { + var jsonData = {}; + + var metricData; + while((metricData = this._regex.exec(data)) !== null) { + logger.debug( + "Processing General Stat: " + metricData[this._WHOLE_MATCH] + ); + + var metricsName = metricData[this._METRIC_NAME]; + var metricsHumanName = this._metricNames[metricsName]; + var metricsValue = parseInt(metricData[this._METRIC_VALUE]); + + jsonData[metricsHumanName] = metricsValue; + } + + return jsonData; + } +}; diff --git a/server/helpers/label-parser.js b/server/helpers/label-parser.js new file mode 100644 index 0000000..a48aead --- /dev/null +++ b/server/helpers/label-parser.js @@ -0,0 +1,34 @@ +const StatsParser = require('./stats-parser.js'); + +/** + * This class is used to parse Javascript literal object data e.g. + * {appId="01FDACB01",fnID="01FDACC02",image="fnproject/hello"} + */ +module.exports = class LabelParser extends StatsParser { + constructor() { + super(); + + this._regex = RegExp(this._labelRE + ',?', 'gm'); + + this._WHOLE_MATCH = 0; + this._LABEL_KEY = 1; + this._LABEL_VALUE = 2; + } + + parse(data) { + // Remove the start and end curly braces as they're not part of the data + data = data.replace(/^{|}$/g, ''); + + var jsonData = {}; + + var labelData; + while((labelData = this._regex.exec(data)) !== null) { + var labelKey = labelData[this._LABEL_KEY]; + var labelValue = labelData[this._LABEL_VALUE]; + + jsonData[labelKey] = labelValue; + } + + return jsonData; + } +}; diff --git a/server/helpers/stats-parser.js b/server/helpers/stats-parser.js new file mode 100644 index 0000000..1a616d8 --- /dev/null +++ b/server/helpers/stats-parser.js @@ -0,0 +1,19 @@ +/** + * The Stats Parser class provides base functionality that individual stats + * parsers can reuse. It's designed for individual parsers to inherit from. + */ +module.exports = class StatsParser { + constructor() { + // Captures the key in a Javascript Object Literal (i.e. everything before the equals sign) + var labelNameRE = '([^=]+)'; + + // Captures the value in a Javascript Object Literal (i.e. anything between the double quotes) + var labelValueRE = '"([^"]*)"'; + + // Matches the key/value pair in a Javascript Object Literal + this._labelRE = labelNameRE + '=' + labelValueRE; + + this._spacesRE = '\\s+'; + this._valueRE = '(\\d+)'; + } +}; From 28c3f5c5cd5160ed89b16769c5c81f5868a44c17 Mon Sep 17 00:00:00 2001 From: Vince Zarola Date: Thu, 13 Jun 2019 21:32:15 +0100 Subject: [PATCH 2/4] Add Unit Tests for Stats Parser Classes Make a start on adding Unit Tests to the repo so that we can automatically test code e.g. on pull requests. --- package.json | 1 + test/lib/shared-stats-parser-libs.js | 21 +++ test/lib/shared-stats-parser-tests.js | 52 +++++++ test/lib/unit-test-data.js | 29 ++++ test/test_app_stats_parser.js | 195 ++++++++++++++++++++++++++ test/test_general_stats_parser.js | 46 ++++++ 6 files changed, 344 insertions(+) create mode 100644 test/lib/shared-stats-parser-libs.js create mode 100644 test/lib/shared-stats-parser-tests.js create mode 100644 test/lib/unit-test-data.js create mode 100644 test/test_app_stats_parser.js create mode 100644 test/test_general_stats_parser.js diff --git a/package.json b/package.json index 7b316c7..efa74b9 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "private": true, "scripts": { "start": "node server", + "test": "./node_modules/.bin/mocha test/test_*.js", "test-integration": "./node_modules/.bin/mocha test_integration/test_*.js" }, "dependencies": { diff --git a/test/lib/shared-stats-parser-libs.js b/test/lib/shared-stats-parser-libs.js new file mode 100644 index 0000000..0132304 --- /dev/null +++ b/test/lib/shared-stats-parser-libs.js @@ -0,0 +1,21 @@ +const assert = require('assert'); + +/** + * Given a parser and an array of UnitTestData objects, parse each test's input + * and check if the parsed results match the expected output. + * + * @param {StatsParser} parser + * @param {Array} + */ +exports.run_tests = function(parser, tests) { + for (let i = 0; i < tests.length; i++) { + let test = tests[i]; + + it(test.description(), function() { + let actualResult = parser.parse(test.inputData); + assert.deepStrictEqual( + actualResult, this.test.expectedResult, this.test.failureMessage() + ); + }.bind({test: test})); + } +}; diff --git a/test/lib/shared-stats-parser-tests.js b/test/lib/shared-stats-parser-tests.js new file mode 100644 index 0000000..7eb5496 --- /dev/null +++ b/test/lib/shared-stats-parser-tests.js @@ -0,0 +1,52 @@ +/** + * This module contains stats parser tests that are applicable to multiple + * Stats parsers + */ + +const UnitTestData = require('./unit-test-data.js'); + +/** + * Test the parsers ignore Prometheus data that they aren't looking for + * + * @return {UnitTestData} + */ +exports.irrelevantStatsData = function() { + let test = 'ignore irrelevant data'; + + let inputData = ` +# HELP fn_api_latency Latency distribution of API requests +# TYPE fn_api_latency histogram +fn_api_latency_bucket{blame="service",method="GET",path="/metrics",status="200",le="1.0"} 6.0 +go_memstats_mspan_inuse_bytes 72200.0 +go_threads 13.0 +fn_calls 2.0 +fn_errors 1.0 +fn_util_mem_used 0.0 +fn_container_wait_duration_seconds_bucket{app_id="01D8JQSKDENG8G00GZJ000000B",fn_id="01D8RVJ0QANG8G00GZJ000000J",image_name="fndemouser/sleeplonger:0.0.2",le="120000.0"} 3.0 +`; + + let expectedResult = {}; + + return new UnitTestData(test, inputData, expectedResult); +}; + +/** + * Test the parsers ignore invalid Prometheus data + * + * @return {UnitTestData} + */ +exports.invalidStatsData = function() { + let test = 'handle invalid data'; + + // Test things such as negative/non-numeric fn stats and also stats that + // contain app data which shouldn't + let inputData = ` +fn_queued {app_id="01D8JQSKDENG8G00GZJ000000B",fn_id="01D8JQSQ2VNG8G00GZJ000000C",image_name="fndemouser/sleepy:0.0.10"} 9.0 +fn_running invalid +fn_complete -54 +`; + + let expectedResult = {}; + + return new UnitTestData(test, inputData, expectedResult); +}; diff --git a/test/lib/unit-test-data.js b/test/lib/unit-test-data.js new file mode 100644 index 0000000..32d6c69 --- /dev/null +++ b/test/lib/unit-test-data.js @@ -0,0 +1,29 @@ +/** + * Helper class to hold unit test data + */ +module.exports = class UnitTestData { + constructor(test, inputData, expectedResult) { + /** @public {String} a description of the test */ + this.test = test; + + /** @public {String} the data to be processed */ + this.inputData = inputData; + + /** @public {Object} the expected result after processing the input data */ + this.expectedResult = expectedResult; + } + + /** + * Return a description for Mocha to use for this test + */ + description() { + return 'can ' + this.test; + } + + /** + * Return a failure message for Mocha to use for this test + */ + failureMessage() { + return 'parser did not ' + this.test; + } +}; diff --git a/test/test_app_stats_parser.js b/test/test_app_stats_parser.js new file mode 100644 index 0000000..8f9970d --- /dev/null +++ b/test/test_app_stats_parser.js @@ -0,0 +1,195 @@ +/** + * Test the AppStatsParser class + */ + +const AppStatsParser = require('../server/helpers/app-stats-parser.js'); +const sharedStatsParserLibs = require('./lib/shared-stats-parser-libs.js'); +const sharedStatsParserTests = require('./lib/shared-stats-parser-tests.js'); +const UnitTestData = require('./lib/unit-test-data.js'); + +describe('Test App Stats Parser', function() { + let tests = [ + expectedStatsData(), + missingLabelData(), + additionalLabelData(), + reorderedLabelData(), + sharedStatsParserTests.irrelevantStatsData(), + sharedStatsParserTests.invalidStatsData(), + unexpectedData(), + ]; + + sharedStatsParserLibs.run_tests(new AppStatsParser(), tests); +}); + +/** + * Test the parser correctly handles data it is expected to parse + */ +function expectedStatsData() { + let test = 'parse expected data'; + + let inputData = ` +# HELP fn_container_start_total containers in state container_start_total +# TYPE fn_container_start_total untyped +fn_container_start_total{app_id="01D7MD7GTBNG8G00GZJ0000001",fn_id="01D7MD7M48NG8G00GZJ0000002",image_name="fndemouser/testapp:0.0.2"} 3.0 +fn_container_start_total{app_id="01D8JQSKDENG8G00GZJ000000B",fn_id="01D8JQSQ2VNG8G00GZJ000000C",image_name="fndemouser/sleepy:0.0.10"} 0.0 +fn_container_start_total{app_id="01D8JQSKDENG8G00GZJ000000B",fn_id="01D8RVJ0QANG8G00GZJ000000J",image_name="fndemouser/sleeplonger:0.0.2"} 9.0 +# HELP fn_container_busy_total containers in state container_busy_total +# TYPE fn_container_busy_total untyped +fn_container_busy_total{app_id="01D7MD7GTBNG8G00GZJ0000001",fn_id="01D7MD7M48NG8G00GZJ0000002",image_name="fndemouser/testapp:0.0.2"} 0.0 +fn_container_busy_total{app_id="01D8JQSKDENG8G00GZJ000000B",fn_id="01D8JQSQ2VNG8G00GZJ000000C",image_name="fndemouser/sleepy:0.0.10"} 1.0 +fn_container_busy_total{app_id="01D8JQSKDENG8G00GZJ000000B",fn_id="01D8RVJ0QANG8G00GZJ000000J",image_name="fndemouser/sleeplonger:0.0.2"} 54.0 +# HELP fn_container_idle_total containers in state container_idle_total +# TYPE fn_container_idle_total untyped +fn_container_idle_total{app_id="01D7MD7GTBNG8G00GZJ0000001",fn_id="01D7MD7M48NG8G00GZJ0000002",image_name="fndemouser/testapp:0.0.2"} 2.0 +fn_container_idle_total{app_id="01D8JQSKDENG8G00GZJ000000B",fn_id="01D8JQSQ2VNG8G00GZJ000000C",image_name="fndemouser/sleepy:0.0.10"} 1.0 +fn_container_idle_total{app_id="01D8JQSKDENG8G00GZJ000000B",fn_id="01D8RVJ0QANG8G00GZJ000000J",image_name="fndemouser/sleeplonger:0.0.2"} 0.0 +# HELP fn_container_paused_total containers in state container_paused_total +# TYPE fn_container_paused_total untyped +fn_container_paused_total{app_id="01D7MD7GTBNG8G00GZJ0000001",fn_id="01D7MD7M48NG8G00GZJ0000002",image_name="fndemouser/testapp:0.0.2"} 0.0 +fn_container_paused_total{app_id="01D8JQSKDENG8G00GZJ000000B",fn_id="01D8RVJ0QANG8G00GZJ000000J",image_name="fndemouser/sleeplonger:0.0.2"} 3.0 +# HELP fn_container_wait_total containers in state container_wait_total +# TYPE fn_container_wait_total untyped +fn_container_wait_total{app_id="01D7MD7GTBNG8G00GZJ0000001",fn_id="01D7MD7M48NG8G00GZJ0000002",image_name="fndemouser/testapp:0.0.2"} 107.0 +fn_container_wait_total{app_id="01D8JQSKDENG8G00GZJ000000B",fn_id="01D8JQSQ2VNG8G00GZJ000000C",image_name="fndemouser/sleepy:0.0.10"} 5.0 +fn_container_wait_total{app_id="01D8JQSKDENG8G00GZJ000000B",fn_id="01D8RVJ0QANG8G00GZJ000000J",image_name="fndemouser/sleeplonger:0.0.2"} 0.0 +`; + + let expectedResult = { + '01D7MD7GTBNG8G00GZJ0000001': { + 'Functions': { + '01D7MD7M48NG8G00GZJ0000002': { + 'Busy': 0, + 'Idling': 2, + 'Paused': 0, + 'Starting': 3, + 'Waiting': 107, + }, + }, + }, + '01D8JQSKDENG8G00GZJ000000B': { + 'Functions': { + '01D8JQSQ2VNG8G00GZJ000000C': { + 'Busy': 1, + 'Idling': 1, + 'Starting': 0, + 'Waiting': 5, + }, + '01D8RVJ0QANG8G00GZJ000000J': { + 'Busy': 54, + 'Idling': 0, + 'Paused': 3, + 'Starting': 9, + 'Waiting': 0, + }, + }, + }, + }; + + return new UnitTestData(test, inputData, expectedResult); +} + +/** + * Test how the parser copes with missing label data + */ +function missingLabelData() { + let test = 'handle missing label data'; + + let inputData =` +fn_container_busy_total{app_id="01D7MD7GTBNG8G00GZJ0000001",fn_id="01D7MD7M48NG8G00GZJ0000002"} 1.0 +fn_container_wait_total{app_id="01D7MD7GTBNG8G00GZJ0000001",image_name="fndemouser/testapp:0.0.2"} 2.0 +fn_container_paused_total{fn_id="01D7MD7M48NG8G00GZJ0000002",image_name="fndemouser/testapp:0.0.2"} 3.0 +fn_container_start_total 4.0 +`; + + let expectedResult = { + '01D7MD7GTBNG8G00GZJ0000001': { + 'Functions': { + '01D7MD7M48NG8G00GZJ0000002': { + 'Busy': 1, + }, + undefined: { + 'Waiting': 2, + } + }, + }, + undefined: { + 'Functions': { + '01D7MD7M48NG8G00GZJ0000002': { + 'Paused': 3, + }, + }, + }, + }; + + return new UnitTestData(test, inputData, expectedResult); +} + +/** + * Test how the parser copes with label data it isn't expecting to see + */ +function additionalLabelData() { + let test = 'handle additional label data'; + + let inputData = ` +fn_container_start_total{app_id="01D7MD7GTBNG8G00GZJ0000001",fn_id="01D7MD7M48NG8G00GZJ0000002",image_name="fndemouser/testapp:0.0.2",extra_data="additional_data"} 1.0 +`; + + let expectedResult = { + '01D7MD7GTBNG8G00GZJ0000001': { + 'Functions': { + '01D7MD7M48NG8G00GZJ0000002': { + 'Starting': 1, + }, + }, + }, + }; + + return new UnitTestData(test, inputData, expectedResult); +} + +/** + * Test the parser is able to parse the label data, no matter what order it's + * in + */ +function reorderedLabelData() { + let test = 'handle re-ordered label data'; + + let inputData = ` +fn_container_start_total{fn_id="01D7MD7M48NG8G00GZJ0000002",app_id="01D7MD7GTBNG8G00GZJ0000001",image_name="fndemouser/testapp:0.0.2"} 3.0 +`; + + let expectedResult = { + '01D7MD7GTBNG8G00GZJ0000001': { + 'Functions': { + '01D7MD7M48NG8G00GZJ0000002': { + 'Starting': 3, + }, + }, + }, + }; + + return new UnitTestData(test, inputData, expectedResult); +} + +/** + * Test the parser can handle data that it isn't expecting + */ +function unexpectedData() { + let test = 'handle unexpected data'; + + let inputData = ` +fn_container_wait_total{app_id="",fn_id="",image_name=""} 1.1 +`; + + let expectedResult = { + '': { + 'Functions': { + '': { + 'Waiting': 1, + }, + }, + }, + }; + + return new UnitTestData(test, inputData, expectedResult); +} diff --git a/test/test_general_stats_parser.js b/test/test_general_stats_parser.js new file mode 100644 index 0000000..f7a121d --- /dev/null +++ b/test/test_general_stats_parser.js @@ -0,0 +1,46 @@ +/** + * Test the GeneralStatsParser class + */ + +const GeneralStatsParser = require('../server/helpers/general-stats-parser.js'); +const sharedStatsParserLibs = require('./lib/shared-stats-parser-libs.js'); +const sharedStatsParserTests = require('./lib/shared-stats-parser-tests.js'); +const UnitTestData = require('./lib/unit-test-data.js'); + +describe('Test General Stats Parser', function() { + let tests = [ + expectedStatsData(), + sharedStatsParserTests.irrelevantStatsData(), + sharedStatsParserTests.invalidStatsData(), + ]; + + sharedStatsParserLibs.run_tests(new GeneralStatsParser(), tests); +}); + +/** + * Test the parser correctly handles data it is expected to parse + */ +function expectedStatsData() { + let test = 'parse expected data'; + + // Test things like Prometheus comments and multi-digit values + let inputData = ` +# HELP fn_queued calls currently queued against agent +# TYPE fn_queued untyped +fn_queued 9.0 +# HELP fn_running calls currently running in agent +# TYPE fn_running untyped +fn_running 987.0 +# HELP fn_completed calls completed in agent +# TYPE fn_completed untyped +fn_completed 9876.0 +`; + + let expectedResult = { + 'Queue': 9, + 'Running': 987, + 'Complete': 9876, + }; + + return new UnitTestData(test, inputData, expectedResult); +} From 398f3466208b5708ed0c60d7e19b3dc9149488b7 Mon Sep 17 00:00:00 2001 From: Vince Zarola Date: Wed, 26 Jun 2019 22:37:52 +0100 Subject: [PATCH 3/4] Run Unit Tests as part of CircleCI We have recently made a start on adding Unit Tests. Run these Unit Tests as part of CircleCI so that any PRs are automatically checked for issues. --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 47b9779..4319d9e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,6 +11,12 @@ jobs: npm install --loglevel error --only dev ./node_modules/.bin/eslint . + - run: + name: "Run the unit tests" + command: | + npm install --loglevel silent --no-save + npm test + - run: name: "Update Docker" command: | From ca76e291241e42a4828ed841b2433f3eba9d26ec Mon Sep 17 00:00:00 2001 From: Vince Zarola Date: Wed, 26 Jun 2019 22:42:17 +0100 Subject: [PATCH 4/4] Add liniting as an npm run-script This means devs can now lint their code by running "npm run lint". This is alot easier to remember and makes it easier to change the linter should we choose to use another one in the future --- .circleci/config.yml | 2 +- package.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4319d9e..6abe4e3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,7 +9,7 @@ jobs: name: "Lint the code" command: | npm install --loglevel error --only dev - ./node_modules/.bin/eslint . + npm run lint - run: name: "Run the unit tests" diff --git a/package.json b/package.json index efa74b9..b053f34 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "private": true, "scripts": { "start": "node server", + "lint": "./node_modules/.bin/eslint .", "test": "./node_modules/.bin/mocha test/test_*.js", "test-integration": "./node_modules/.bin/mocha test_integration/test_*.js" },