From ae4083b33e77303076f6a4b3e780d4e01a115afd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Tue, 20 Aug 2024 05:17:29 +0000 Subject: [PATCH] chore: Major update - refactored entire code base from CJS > ESM Implements #859 --- .eslintrc.yml | 14 - eslint.config.js | 37 + package-lock.json | 103 +- package.json | 4 + src/butler-sos.js | 92 +- src/docker-healthcheck.js | 2 +- src/globals.js | 1682 ++++++++++++++----------- src/lib/appnamesextract.js | 25 +- src/lib/config-file-schema.js | 6 +- src/lib/config-file-verify.js | 52 +- src/lib/healthmetrics.js | 47 +- src/lib/heartbeat.js | 10 +- src/lib/log-event-categorise.js | 18 +- src/lib/logdb.js | 12 +- src/lib/post-to-influxdb.js | 22 +- src/lib/post-to-mqtt.js | 20 +- src/lib/post-to-new-relic.js | 24 +- src/lib/prom-client.js | 18 +- src/lib/proxysessionmetrics.js | 38 +- src/lib/serverheaders.js | 8 +- src/lib/servertags.js | 7 +- src/lib/service_uptime.js | 24 +- src/lib/telemetry.js | 11 +- src/lib/udp_handlers_log_events.js | 23 +- src/lib/udp_handlers_user_activity.js | 24 +- 25 files changed, 1253 insertions(+), 1070 deletions(-) delete mode 100644 .eslintrc.yml create mode 100644 eslint.config.js diff --git a/.eslintrc.yml b/.eslintrc.yml deleted file mode 100644 index 4b247b58..00000000 --- a/.eslintrc.yml +++ /dev/null @@ -1,14 +0,0 @@ -env: - es6: true - es2021: true - node: true -extends: - - airbnb-base - - prettier -parserOptions: - ecmaVersion: 12 - sourceType: module -rules: - prettier/prettier: error -plugins: - - prettier diff --git a/eslint.config.js b/eslint.config.js new file mode 100644 index 00000000..44fc837b --- /dev/null +++ b/eslint.config.js @@ -0,0 +1,37 @@ +import prettier from 'eslint-plugin-prettier'; +import globals from 'globals'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import js from '@eslint/js'; +import { FlatCompat } from '@eslint/eslintrc'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const compat = new FlatCompat({ + baseDirectory: __dirname, + recommendedConfig: js.configs.recommended, + allConfig: js.configs.all, +}); + +// export default [...compat.extends("airbnb-base", "prettier"), { +export default [ + ...compat.extends('prettier'), + { + plugins: { + prettier, + }, + + languageOptions: { + globals: { + ...globals.node, + }, + + ecmaVersion: 12, + sourceType: 'module', + }, + + rules: { + 'prettier/prettier': 'error', + }, + }, +]; diff --git a/package-lock.json b/package-lock.json index 1262358e..82d34f95 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,10 +36,13 @@ "yaml-validator": "^5.0.1" }, "devDependencies": { + "@eslint/js": "^9.9.0", "esbuild": "^0.23.1", "eslint": "^9.9.0", "eslint-config-prettier": "^9.1.0", "eslint-formatter-table": "^7.32.1", + "eslint-plugin-prettier": "^5.2.1", + "globals": "^15.9.0", "prettier": "^3.3.3", "snyk": "^1.1292.4" } @@ -564,6 +567,19 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@eslint/eslintrc/node_modules/globals": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", + "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/@eslint/js": { "version": "9.9.0", "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.9.0.tgz", @@ -726,6 +742,19 @@ "node": ">=8.0.0" } }, + "node_modules/@pkgr/core": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.1.1.tgz", + "integrity": "sha512-cq8o4cWH0ibXh9VGi5P20Tu9XF/0fFXl9EUinr9QfTM7a7p0oTA4iJRCQWppXR1Pg8dSM0UCItCkPwsk9qWWYA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/unts" + } + }, "node_modules/@sentry-internal/tracing": { "version": "7.64.0", "resolved": "https://registry.npmjs.org/@sentry-internal/tracing/-/tracing-7.64.0.tgz", @@ -1506,6 +1535,37 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/eslint-plugin-prettier": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-5.2.1.tgz", + "integrity": "sha512-gH3iR3g4JfF+yYPaJYkN7jEl9QbweL/YfkoRlNnuIEHEz1vHVlCmWOS+eGGiRuzHQXdJFCOTxRgvju9b8VUmrw==", + "dev": true, + "license": "MIT", + "dependencies": { + "prettier-linter-helpers": "^1.0.0", + "synckit": "^0.9.1" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint-plugin-prettier" + }, + "peerDependencies": { + "@types/eslint": ">=8.0.0", + "eslint": ">=8.0.0", + "eslint-config-prettier": "*", + "prettier": ">=3.0.0" + }, + "peerDependenciesMeta": { + "@types/eslint": { + "optional": true + }, + "eslint-config-prettier": { + "optional": true + } + } + }, "node_modules/eslint-scope": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.0.2.tgz", @@ -1658,6 +1718,13 @@ "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, + "node_modules/fast-diff": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.3.0.tgz", + "integrity": "sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", @@ -2053,9 +2120,9 @@ } }, "node_modules/globals": { - "version": "14.0.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", - "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", + "version": "15.9.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-15.9.0.tgz", + "integrity": "sha512-SmSKyLLKFbSr6rptvP8izbyxJL4ILwqO9Jg23UA0sDlGlu58V59D1//I3vlc0KJphVdUR7vMjHIplYnzBxorQA==", "dev": true, "license": "MIT", "engines": { @@ -2890,6 +2957,19 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, + "node_modules/prettier-linter-helpers": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz", + "integrity": "sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-diff": "^1.1.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/process": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", @@ -3332,6 +3412,23 @@ "node": ">=8" } }, + "node_modules/synckit": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.9.1.tgz", + "integrity": "sha512-7gr8p9TQP6RAHusBOSLs46F4564ZrjV8xFmw5zCmgmhGUcw2hxsShhJ6CEiHQMgPDwAQ1fWHPM0ypc4RMAig4A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@pkgr/core": "^0.1.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/unts" + } + }, "node_modules/systeminformation": { "version": "5.23.4", "resolved": "https://registry.npmjs.org/systeminformation/-/systeminformation-5.23.4.tgz", diff --git a/package.json b/package.json index 5cb03f1e..7a170459 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "url": "https://github.com/ptarmiganlabs/butler-sos/issues" }, "homepage": "https://github.com/ptarmiganlabs/butler-sos#readme", + "type": "module", "dependencies": { "@breejs/later": "^4.2.0", "@influxdata/influxdb-client": "^1.35.0", @@ -56,10 +57,13 @@ "yaml-validator": "^5.0.1" }, "devDependencies": { + "@eslint/js": "^9.9.0", "esbuild": "^0.23.1", "eslint": "^9.9.0", "eslint-config-prettier": "^9.1.0", "eslint-formatter-table": "^7.32.1", + "eslint-plugin-prettier": "^5.2.1", + "globals": "^15.9.0", "prettier": "^3.3.3", "snyk": "^1.1292.4" }, diff --git a/src/butler-sos.js b/src/butler-sos.js index e7113696..e1a26383 100755 --- a/src/butler-sos.js +++ b/src/butler-sos.js @@ -1,30 +1,29 @@ // Add dependencies -const path = require('path'); -const FastifyHealthcheck = require('fastify-healthcheck'); -const Fastify = require('fastify'); +import path from 'path'; +import FastifyHealthcheck from 'fastify-healthcheck'; +import Fastify from 'fastify'; const promServer = Fastify({ logger: false }); const promFastifyMetricsServer = Fastify({ logger: false }); const dockerHealthCheckServer = Fastify({ logger: false }); -const metricsPlugin = require('fastify-metrics'); +import metricsPlugin from 'fastify-metrics'; promServer.server.keepAliveTimeout = 0; promFastifyMetricsServer.register(metricsPlugin, { endpoint: '/metrics' }); // Load code from sub modules -const globals = require('./globals'); -const healthMetrics = require('./lib/healthmetrics'); -const logDb = require('./lib/logdb'); -const proxySessionMetrics = require('./lib/proxysessionmetrics'); -const appNamesExtract = require('./lib/appnamesextract'); -const heartbeat = require('./lib/heartbeat'); -const serviceUptime = require('./lib/service_uptime'); -const udpUserActivity = require('./lib/udp_handlers_user_activity'); -const udpLogEvents = require('./lib/udp_handlers_log_events'); -const telemetry = require('./lib/telemetry'); -const promClient = require('./lib/prom-client'); -const { verifyConfigFile } = require('./lib/config-file-verify'); +import { setupHealthMetricsTimer } from './lib/healthmetrics.js'; +import { setupLogDbTimer } from './lib/logdb.js'; +import { setupUserSessionsTimer } from './lib/proxysessionmetrics.js'; +import { setupAppNamesExtractTimer } from './lib/appnamesextract.js'; +import { setupHeartbeatTimer } from './lib/heartbeat.js'; +import { serviceUptimeStart } from './lib/service_uptime.js'; +import { udpInitUserActivityServer } from './lib/udp_handlers_user_activity.js'; +import { udpInitLogEventServer } from './lib/udp_handlers_log_events.js'; +import { setupAnonUsageReportTimer } from './lib/telemetry.js'; +import { setupPromClient } from './lib/prom-client.js'; +import { verifyConfigFile } from './lib/config-file-verify.js'; // Suppress experimental warnings // https://stackoverflow.com/questions/55778283/how-to-disable-warnings-when-node-is-launched-via-a-global-shell-script @@ -56,9 +55,14 @@ async function sleep(ms) { } async function mainScript() { + // Load globals dynamically/async to ensure singleton pattern works + const settingsObj = (await import('./globals.js')).default; + const globals = await settingsObj.init(); + globals.logger.verbose(`START: Globals init done: ${globals.initialised}`); + // Verify that the config file has the correct format // Only do this if the command line option no-config-file-verify is NOT set - let configFileVerify = false + let configFileVerify = false; if (globals.options.skipConfigVerification) { globals.logger.warn('MAIN: Skipping config file verification'); } else { @@ -71,24 +75,32 @@ async function mainScript() { process.exit(1); } + // Ensure that initialisation of globals is complete + // Sleep 5 seconds otherwise to llow globals to be initialised - // Sleep 5 seconds to allow inits to complete - globals.logger.info('MAIN: Waiting 5 seconds for initializations to complete...'); - globals.logger.info('5...'); - await sleep(1000); - globals.logger.info('4...'); - await sleep(1000); - globals.logger.info('3...'); - await sleep(1000); - globals.logger.info('2...'); - await sleep(1000); - globals.logger.info('1...'); - await sleep(1000); + function sleepLocal(ms) { + // eslint-disable-next-line no-promise-executor-return + return new Promise((resolve) => setTimeout(resolve, ms)); + } - await globals.initInfluxDB(); + if (!globals.initialised) { + globals.logger.info('START: Sleeping 5 seconds to allow globals to be initialised.'); + globals.logger.info('5...'); + await sleepLocal(1000); + globals.logger.info('4...'); + await sleepLocal(1000); + globals.logger.info('3...'); + await sleepLocal(1000); + globals.logger.info('2...'); + await sleepLocal(1000); + globals.logger.info('1...'); + await sleepLocal(1000); + } else { + globals.logger.info('START: Globals initialised - all good.'); + } if (globals.config.get('Butler-SOS.uptimeMonitor.enable') === true) { - serviceUptime.serviceUptimeStart(); + serviceUptimeStart(); } // Load certificates to use when connecting to healthcheck API @@ -101,7 +113,7 @@ async function mainScript() { // Set up heartbeats, if enabled in the config file if (globals.config.get('Butler-SOS.heartbeat.enable') === true) { - heartbeat.setupHeartbeatTimer(globals.config, globals.logger); + setupHeartbeatTimer(globals.config, globals.logger); } try { @@ -140,7 +152,7 @@ async function mainScript() { // Set up anon usage reports, if enabled if (globals.config.get('Butler-SOS.anonTelemetry') === true) { - telemetry.setupAnonUsageReportTimer(); + setupAnonUsageReportTimer(); globals.logger.verbose('MAIN: Anonymous telemetry reporting has been set up.'); globals.logger.verbose( 'MAIN: ❤️ Thank you for supporting Butler SOS by allowing telemetry! ❤️' @@ -152,7 +164,7 @@ async function mainScript() { // Set up UDP handler for user activity/events if (globals.config.get('Butler-SOS.userEvents.enable')) { - udpUserActivity.udpInitUserActivityServer(); + udpInitUserActivityServer(); globals.logger.debug( `MAIN: Server for user activity/events UDP server: ${globals.udpServerUserActivity.host}` @@ -171,7 +183,7 @@ async function mainScript() { globals.config.get('Butler-SOS.logEvents.source.scheduler.enable') || globals.config.get('Butler-SOS.logEvents.source.proxy.enable') ) { - udpLogEvents.udpInitLogEventServer(); + udpInitLogEventServer(); globals.logger.debug( `MAIN: Server for user activity/events UDP server: ${globals.udpServerLogEvents.host}` @@ -224,7 +236,7 @@ async function mainScript() { globals.logger.info( `MAIN: Starting Prometheus Butler SOS endpoint on ${promHost}:${promPort}.` ); - promClient.setupPromClient(promServer, promPort, promHost); + setupPromClient(promServer, promPort, promHost); } catch (err) { globals.logger.error( `MAIN: Error while starting Prometheus Butler SOS endpoint on ${promHost}:${promPort}.` @@ -250,20 +262,20 @@ async function mainScript() { // Set up extraction of data from log db if (globals.config.get('Butler-SOS.logdb.enable') === true) { - logDb.setupLogDbTimer(); + setupLogDbTimer(); } // Set up extraction of sessions data if (globals.config.get('Butler-SOS.userSessions.enableSessionExtract') === true) { - proxySessionMetrics.setupUserSessionsTimer(); + setupUserSessionsTimer(); } // Set up extraction on main metrics data (i.e. the Sense healthcheck API) - healthMetrics.setupHealthMetricsTimer(); + setupHealthMetricsTimer(); // Set up extraction of app IDs and names if (globals.config.get('Butler-SOS.appNames.enableAppNameExtract') === true) { - appNamesExtract.setupAppNamesExtractTimer(); + setupAppNamesExtractTimer(); } } diff --git a/src/docker-healthcheck.js b/src/docker-healthcheck.js index 31376a0a..e604a416 100755 --- a/src/docker-healthcheck.js +++ b/src/docker-healthcheck.js @@ -1,7 +1,7 @@ /* eslint-disable no-console */ // Set up REST endpoint for Docker healthchecks -const httpHealth = require('http'); +import httpHealth from 'http'; const optionsHealth = { host: 'localhost', diff --git a/src/globals.js b/src/globals.js index f67ce2a4..257a43c0 100755 --- a/src/globals.js +++ b/src/globals.js @@ -1,843 +1,987 @@ -const path = require('path'); -const dgram = require('dgram'); -const os = require('os'); -const crypto = require('crypto'); - -const mqtt = require('mqtt'); -const fs = require('fs-extra'); -const winston = require('winston'); -require('winston-daily-rotate-file'); -const si = require('systeminformation'); -const { Command, Option } = require('commander'); -const Influx = require('influx'); -const { InfluxDB, HttpError, DEFAULT_WriteOptions } = require('@influxdata/influxdb-client'); -const { OrgsAPI, BucketsAPI } = require('@influxdata/influxdb-client-apis'); - -const { getServerTags } = require('./lib/servertags'); - -const InfluxDB2 = InfluxDB; - -const { Pool } = require('pg'); - -function checkFileExistsSync(filepath) { - let flag = true; - try { - fs.accessSync(filepath, fs.constants.F_OK); - } catch (e) { - flag = false; - } - return flag; -} +import upath from 'path'; +import dgram from 'dgram'; +import os from 'os'; +import crypto from 'crypto'; +import mqtt from 'mqtt'; +import fs from 'fs-extra'; +import winston from 'winston'; +import 'winston-daily-rotate-file'; +import si from 'systeminformation'; +import pg from 'pg'; +import { readFileSync } from 'fs'; +import Influx from 'influx'; +import { Command, Option } from 'commander'; +import { InfluxDB, HttpError, DEFAULT_WriteOptions } from '@influxdata/influxdb-client'; +import { OrgsAPI, BucketsAPI } from '@influxdata/influxdb-client-apis'; + +import { getServerTags } from './lib/servertags.js'; +import { fileURLToPath } from 'url'; + +let instance = null; + +class Settings { + constructor() { + if (!instance) { + instance = this; + } -// Get app version from package.json file -const appVersion = require('../package.json').version; - -// Command line parameters -const program = new Command(); -program - .version(appVersion) - .name('butler-sos') - .description( - 'Butler SenseOps Stats ("Butler-SOS") is a microservice publishing operational Qlik Sense metrics to InfluxDB, Prometheus and New Relic.\nUser events and log events can be forwarded from Sense to Butler SOS and then acted upon there. Events can be stored in InfluxDB and sent to New Relic.\nAdd Grafana for great looking dashboards and you get real-time monitoring of what happens inside a Qlik Sense environment.' - ) - .option('-c, --configfile ', 'path to config file') - .addOption( - new Option('-l, --loglevel ', 'log level').choices([ - 'error', - 'warn', - 'info', - 'verbose', - 'debug', - 'silly', - ]) - ) - .option( - '--new-relic-account-name ', - 'New Relic account name. Used within Butler SOS to differentiate between different target New Relic accounts' - ) - .option('--new-relic-api-key ', 'insert API key to use with New Relic') - .option('--new-relic-account-id ', 'New Relic account ID') - - .option('--skip-config-verification', 'Disable config file verification', false); - -// Parse command line params -program.parse(process.argv); -const options = program.opts(); - -// Is there a config file specified on the command line? -let configFileOption; -let configFileExpanded; -let configFilePath; -let configFileBasename; -let configFileExtension = '.yaml'; -if (options.configfile && options.configfile.length > 0) { - configFileOption = options.configfile; - configFileExpanded = path.resolve(options.configfile); - configFilePath = path.dirname(configFileExpanded); - configFileExtension = path.extname(configFileExpanded); - configFileBasename = path.basename(configFileExpanded, configFileExtension); - - if (configFileExtension.toLowerCase() !== '.yaml') { - // eslint-disable-next-line no-console - console.log('Error: Config file extension must be yaml'); - process.exit(1); - } + // Flag to keep track of initialisation status of globals object + this.initialised = false; - if (checkFileExistsSync(options.configfile)) { - process.env.NODE_CONFIG_DIR = configFilePath; - process.env.NODE_ENV = configFileBasename; - } else { - // eslint-disable-next-line no-console - console.log('Error: Specified config file does not exist'); - process.exit(1); - } -} else { - // Set default values of environment variables controlling config file location and name - if (process.env.NODE_CONFIG_DIR === undefined) { - process.env.NODE_CONFIG_DIR = path.join(process.cwd(), 'config'); + return instance; } - if (process.env.NODE_ENV === undefined) { - process.env.NODE_ENV = 'production'; - } -} + async init() { + // Get app version from package.json file + const filenamePackage = `./package.json`; + let a; + let b; + let c; + // Are we running as a packaged app? + if (process.pkg) { + // Get path to JS file + a = process.pkg.defaultEntrypoint; + + // Strip off the filename + b = upath.dirname(a); + + // Add path to package.json file + c = upath.join(b, filenamePackage); + + // Set base path of the executable + this.appBasePath = upath.join(b); + } else { + // Get path to JS file + a = fileURLToPath(import.meta.url); + + // Strip off the filename + b = upath.dirname(a); + + // Add path to package.json file + c = upath.join(b, '..', filenamePackage); + + // Set base path of the executable + this.appBasePath = upath.join(b, '..'); + } -// Set global variable conttaining the name and full path of the config file -const configFile = path.join( - process.env.NODE_CONFIG_DIR, - `${process.env.NODE_ENV}${configFileExtension}` -); - -// Are we running as standalone app or not? -const isPkg = typeof process.pkg !== 'undefined'; -if (isPkg && configFileOption === undefined) { - // Show help if running as standalone app and mandatory options (e.g. config file) are not specified - program.help({ error: true }); -} + const { version } = JSON.parse(readFileSync(c)); + this.appVersion = version; + + // Make copy of influxdb client + const InfluxDB2 = InfluxDB; + + // Command line parameters + const program = new Command(); + program + .version(this.appVersion) + .name('butler-sos') + .description( + 'Butler SenseOps Stats ("Butler-SOS") is a tool publishing operational Qlik Sense metrics to InfluxDB, Prometheus, New Relic and other destinations.\nUser events and log events can be forwarded from Sense to Butler SOS and then acted upon there. Events can be stored in InfluxDB and sent to New Relic.\nAdd Grafana for great looking dashboards and you get real-time monitoring of what happens inside a Qlik Sense environment.\n\nMore info at https://butler-sos.ptarmiganlabs.com' + ) + .option('-c, --configfile ', 'path to config file') + .addOption( + new Option('-l, --loglevel ', 'log level').choices([ + 'error', + 'warn', + 'info', + 'verbose', + 'debug', + 'silly', + ]) + ) + .option( + '--new-relic-account-name ', + 'New Relic account name. Used within Butler SOS to differentiate between different target New Relic accounts' + ) + .option('--new-relic-api-key ', 'insert API key to use with New Relic') + .option('--new-relic-account-id ', 'New Relic account ID') + + .option('--skip-config-verification', 'Disable config file verification', false); + + // Parse command line params + program.parse(process.argv); + this.options = program.opts(); + + // Utility functions + this.checkFileExistsSync = Settings.checkFileExistsSync; + this.sleep = Settings.sleep; + + // Is there a config file specified on the command line? + let configFileOption; + this.configFile = null; + let configFilePath; + let configFileBasename; + let configFileExtension; + if (this.options.configfile && this.options.configfile.length > 0) { + configFileOption = this.options.configfile; + + // Full path to config file + this.configFile = upath.resolve(this.options.configfile); + configFilePath = upath.dirname(this.configFile); + configFileExtension = upath.extname(this.configFile); + configFileBasename = upath.basename(this.configFile, configFileExtension); + + if (configFileExtension.toLowerCase() !== '.yaml') { + // eslint-disable-next-line no-console + console.log('Error: Config file extension must be yaml'); + process.exit(1); + } -// eslint-disable-next-line import/order -const config = require('config'); + if (this.checkFileExistsSync(this.options.configfile)) { + process.env.NODE_CONFIG_DIR = configFilePath; + process.env.NODE_ENV = configFileBasename; + } else { + // eslint-disable-next-line no-console + console.log('Error: Specified config file does not exist'); + process.exit(1); + } + } else { + // Get value of env variable NODE_ENV + const env = process.env.NODE_ENV; + + // Get path to config file + const filename = fileURLToPath(import.meta.url); + const dirname = upath.dirname(filename); + this.configFile = upath.resolve(dirname, `./config/${env}.yaml`); + } -// Is there a log level file specified on the command line? -if (options.loglevel && options.loglevel.length > 0) { - config['Butler-SOS'].logLevel = options.loglevel; -} + this.config = (await import('config')).default; -// Set up array for storing app ids and names -const appNames = []; - -// Set up logger with timestamps and colors, and optional logging to disk file -const logTransports = []; - -logTransports.push( - new winston.transports.Console({ - name: 'console', - level: config.get('Butler-SOS.logLevel'), - format: winston.format.combine( - winston.format.errors({ stack: true }), - winston.format.timestamp(), - winston.format.colorize(), - winston.format.simple(), - winston.format.printf((info) => `${info.timestamp} ${info.level}: ${info.message}`) - ), - }) -); - -if (config.get('Butler-SOS.fileLogging')) { - logTransports.push( - new winston.transports.DailyRotateFile({ - dirname: path.join(process.cwd(), config.get('Butler-SOS.logDirectory')), - filename: 'butler-sos.%DATE%.log', - level: config.get('Butler-SOS.logLevel'), - datePattern: 'YYYY-MM-DD', - maxFiles: '30d', - }) - ); -} + this.execPath = this.isPkg ? upath.dirname(process.execPath) : process.cwd(); -const logger = winston.createLogger({ - transports: logTransports, - format: winston.format.combine( - winston.format.timestamp(), - winston.format.printf((info) => `${info.timestamp} ${info.level}: ${info.message}`) - ), -}); - -// Show contents of environment variables controlling config file location and name -logger.debug(`NODE_CONFIG_DIR: ${process.env.NODE_CONFIG_DIR}`); -logger.debug(`NODE_ENV: ${process.env.NODE_ENV}`); - -// Output config file name and path to log -logger.info(`Using config file: ${configFile}`); - -// Function to get current logging level -const getLoggingLevel = () => logTransports.find((transport) => transport.name === 'console').level; - -// Are there New Relic account name(s), API key(s) and account ID(s) specified on the command line? -// There must be the same number of each specified! -// If so, replace any info from the config file with data from command line options -if ( - options?.newRelicAccountName?.length > 0 && - options?.newRelicApiKey?.length > 0 && - options?.newRelicAccountId?.length > 0 && - options?.newRelicAccountName?.length === options?.newRelicApiKey?.length && - options?.newRelicApiKey?.length === options?.newRelicAccountId?.length -) { - config['Butler-SOS'].thirdPartyToolsCredentials.newRelic = []; - - for (let index = 0; index < options.newRelicApiKey.length; index += 1) { - const accountName = options.newRelicAccountName[index]; - const accountId = options.newRelicAccountId[index]; - const insertApiKey = options.newRelicApiKey[index]; - - config['Butler-SOS'].thirdPartyToolsCredentials.newRelic.push({ - accountName, - accountId, - insertApiKey, - }); - } -} else if ( - options?.newRelicAccountName?.length > 0 || - options?.newRelicApiKey?.length > 0 || - options?.newRelicAccountId?.length > 0 -) { - logger.error( - 'Incorrect command line parameters: Number of New Relic account names/IDs/API keys must match.' - ); - process.exit(1); -} + // Are we running as standalone app or not? + this.isPkg = typeof process.pkg !== 'undefined'; + if (this.isPkg && configFileOption === undefined) { + // Show help if running as standalone app and mandatory options (e.g. config file) are not specified + program.help({ error: true }); + } -// ------------------------------------ -// User activity UDP server -const udpServerUserActivity = {}; + // Is there a log level file specified on the command line? + if (this.options.loglevel && this.options.loglevel.length > 0) { + this.config['Butler-SOS'].logLevel = this.options.loglevel; + } -try { - udpServerUserActivity.host = config.get('Butler-SOS.userEvents.udpServerConfig.serverHost'); + // Set up array for storing app ids and names + this.appNames = []; + + // Set up logger with timestamps and colors, and optional logging to disk file + this.logTransports = []; + + this.logTransports.push( + new winston.transports.Console({ + name: 'console', + level: this.config.get('Butler-SOS.logLevel'), + format: winston.format.combine( + winston.format.errors({ stack: true }), + winston.format.timestamp(), + winston.format.colorize(), + winston.format.simple(), + winston.format.printf( + (info) => `${info.timestamp} ${info.level}: ${info.message}` + ) + ), + }) + ); - // Prepare to listen on port X for incoming UDP connections regarding user activity events - udpServerUserActivity.socket = dgram.createSocket({ - type: 'udp4', - reuseAddr: true, - }); + if ( + this.config['Butler-SOS'].logLevel === 'verbose' || + this.config['Butler-SOS'].logLevel === 'debug' || + this.config['Butler-SOS'].logLevel === 'silly' + ) { + // We don't have a logging object yet, so use plain console.log + + // Are we in a packaged app? + if (this.isPkg) { + // eslint-disable-next-line no-console + console.log(`Running in packaged app. Executable path: ${this.execPath}`); + } else { + // eslint-disable-next-line no-console + console.log( + `Running in non-packaged environment. Executable path: ${this.execPath}` + ); + } - udpServerUserActivity.portUserActivity = config.get( - 'Butler-SOS.userEvents.udpServerConfig.portUserActivityEvents' - ); -} catch (err) { - logger.error(`CONFIG: Setting up UDP user activity listener: ${err}`); -} + // eslint-disable-next-line no-console + console.log( + `Log file directory: ${upath.join(this.execPath, this.config.get('Butler-SOS.logDirectory'))}` + ); -// ------------------------------------ -// Log events UDP server -const udpServerLogEvents = {}; + // eslint-disable-next-line no-console + console.log(`upath.dirname(process.execPath): ${upath.dirname(process.execPath)}`); -try { - udpServerLogEvents.host = config.get('Butler-SOS.logEvents.udpServerConfig.serverHost'); + // eslint-disable-next-line no-console + console.log(`process.cwd(): ${process.cwd()}`); + } - // Prepare to listen on port X for incoming UDP connections regarding user activity events - udpServerLogEvents.socket = dgram.createSocket({ - type: 'udp4', - reuseAddr: true, - }); + if (this.config.get('Butler-SOS.fileLogging')) { + this.logTransports.push( + new winston.transports.DailyRotateFile({ + dirname: upath.join(this.execPath, this.config.get('Butler-SOS.logDirectory')), + filename: 'butler-sos.%DATE%.log', + level: this.config.get('Butler-SOS.logLevel'), + datePattern: 'YYYY-MM-DD', + maxFiles: '30d', + }) + ); + } - udpServerLogEvents.port = config.get('Butler-SOS.logEvents.udpServerConfig.portLogEvents'); -} catch (err) { - logger.error(`CONFIG: Setting up UDP log events listener: ${err}`); -} + this.logger = winston.createLogger({ + transports: this.logTransports, + format: winston.format.combine( + winston.format.timestamp(), + winston.format.printf((info) => `${info.timestamp} ${info.level}: ${info.message}`) + ), + }); -// ------------------------------------ -// Get info on what servers to monitor -const serverList = config.get('Butler-SOS.serversToMonitor.servers'); - -// Only set up connection pool for accessing Qlik Sense log db if that feature is enabled -let pgPool; -if (config.get('Butler-SOS.logdb.enable') === true) { - // Set up connection pool for accessing Qlik Sense log db - pgPool = new Pool({ - host: config.get('Butler-SOS.logdb.host'), - database: 'QLogs', - user: config.get('Butler-SOS.logdb.qlogsReaderUser'), - password: config.get('Butler-SOS.logdb.qlogsReaderPwd'), - port: config.get('Butler-SOS.logdb.port'), - }); - - // the pool will emit an error on behalf of any idle clients - // it contains if a backend error or network partition happens - // eslint-disable-next-line no-unused-vars - pgPool.on('error', (err, client) => { - logger.error(`CONFIG: Unexpected error on idle client: ${err}`); - // process.exit(-1); - }); -} + // Show contents of environment variables controlling config file location and name + this.logger.debug(`NODE_CONFIG_DIR: ${process.env.NODE_CONFIG_DIR}`); + this.logger.debug(`NODE_ENV: ${process.env.NODE_ENV}`); -// Get list of standard and user configurable tags -// ..begin with standard tags -const tagValues = ['host', 'server_name', 'server_description']; - -// ..check if there are any extra tags for this Butler SOS instance that should be sent to InfluxDB -if ( - config.has('Butler-SOS.serversToMonitor.serverTagsDefinition') && - config.get('Butler-SOS.serversToMonitor.serverTagsDefinition') !== null -) { - // Loop over all tags defined for the current server, adding them to the data structure that will later be passed to Influxdb - config.get('Butler-SOS.serversToMonitor.serverTagsDefinition').forEach((entry) => { - logger.debug(`CONFIG: Setting up new Influx database: Found server tag : ${entry}`); - - tagValues.push(entry); - }); -} + // Output config file name and path to log + this.logger.info(`Using config file: ${this.configFile}`); + + // Function to get current logging level + this.getLoggingLevel = () => + this.logTransports.find((transport) => transport.name === 'console').level; + + // Are there New Relic account name(s), API key(s) and account ID(s) specified on the command line? + // There must be the same number of each specified! + // If so, replace any info from the config file with data from command line options + if ( + this.options?.newRelicAccountName?.length > 0 && + this.options?.newRelicApiKey?.length > 0 && + this.options?.newRelicAccountId?.length > 0 && + this.options?.newRelicAccountName?.length === this.options?.newRelicApiKey?.length && + this.options?.newRelicApiKey?.length === this.options?.newRelicAccountId?.length + ) { + this.config['Butler-SOS'].thirdPartyToolsCredentials.newRelic = []; + + for (let index = 0; index < this.options.newRelicApiKey.length; index += 1) { + const accountName = this.options.newRelicAccountName[index]; + const accountId = this.options.newRelicAccountId[index]; + const insertApiKey = this.options.newRelicApiKey[index]; + + this.config['Butler-SOS'].thirdPartyToolsCredentials.newRelic.push({ + accountName, + accountId, + insertApiKey, + }); + } + } else if ( + this.options?.newRelicAccountName?.length > 0 || + this.options?.newRelicApiKey?.length > 0 || + this.options?.newRelicAccountId?.length > 0 + ) { + this.logger.error( + 'Incorrect command line parameters: Number of New Relic account names/IDs/API keys must match.' + ); + process.exit(1); + } -// Add tags for log events -const tagValuesLogEvent = tagValues.slice(); -tagValuesLogEvent.push('level'); -tagValuesLogEvent.push('source'); -tagValuesLogEvent.push('log_row'); -tagValuesLogEvent.push('subsystem'); -tagValuesLogEvent.push('user_full'); -tagValuesLogEvent.push('user_directory'); -tagValuesLogEvent.push('user_id'); -tagValuesLogEvent.push('task_id'); -tagValuesLogEvent.push('task_name'); -tagValuesLogEvent.push('app_id'); -tagValuesLogEvent.push('app_name'); -tagValuesLogEvent.push('result_code'); -tagValuesLogEvent.push('windows_user'); -tagValuesLogEvent.push('engine_exe_version'); - -// Check if there are any extra log event tags in the config file -if (config.has('Butler-SOS.logEvents.tags') && config.get('Butler-SOS.logEvents.tags') !== null) { - config.get('Butler-SOS.logEvents.tags').forEach((entry) => { - logger.debug( - `CONFIG: Setting up new Influx database: Found log event tag in config file: ${JSON.stringify( - entry + // Verbose: Show what New Relic account names/API keys/account IDs have been defined (on command line or in config file) + this.logger.verbose( + `New Relic account names/API keys/account IDs (via command line or config file): ${JSON.stringify( + this.config['Butler-SOS'].thirdPartyToolsCredentials.newRelic, + null, + 2 )}` ); - tagValuesLogEvent.push(entry.tag); - }); -} + // Get certificate file paths for QRS connection + const filename = fileURLToPath(import.meta.url); + const dirname = upath.dirname(filename); + this.certPath = upath.resolve(dirname, this.config.get('Butler-SOS.cert.clientCert')); + this.keyPath = upath.resolve(dirname, this.config.get('Butler-SOS.cert.clientCertKey')); + this.caPath = upath.resolve(dirname, this.config.get('Butler-SOS.cert.clientCertCA')); -// Add tags for log events categories, if enabled and configured -if ( - config.has('Butler-SOS.logEvents.categorise.enable') && - config.get('Butler-SOS.logEvents.categorise.enable') === true && - config.has('Butler-SOS.logEvents.categorise.rules') -) { - // Add tags from Butler-SOS.logEvents.categorise.rules[].category[], where each object has properties 'name' and 'value' - config.get('Butler-SOS.logEvents.categorise.rules').forEach((rule) => { - rule.category.forEach((category) => { - tagValuesLogEvent.push(category.name); - }); - }); - - // Add default rule categories, if enabled - if ( - config.has('Butler-SOS.logEvents.categorise.ruleDefault.enable') && - config.get('Butler-SOS.logEvents.categorise.ruleDefault.enable') === true && - config.has('Butler-SOS.logEvents.categorise.ruleDefault.category') - ) { - config.get('Butler-SOS.logEvents.categorise.ruleDefault.category').forEach((category) => { - tagValuesLogEvent.push(category.name); - }); - } -} + // ------------------------------------ + // User activity UDP server + this.udpServerUserActivity = {}; -// Create InfluxDB tags for data coming from log db -const tagValuesLogEventLogDb = tagValues.slice(); -tagValuesLogEventLogDb.push('source_process'); -tagValuesLogEventLogDb.push('log_level'); - -// Create tags for user sessions -const tagValuesUserProxySessions = tagValues.slice(); -tagValuesUserProxySessions.push('user_session_virtual_proxy'); -tagValuesUserProxySessions.push('user_session_host'); - -// Show Influxdb config -if (config.get('Butler-SOS.influxdbConfig.enable') === true) { - logger.info(`CONFIG: Influxdb enabled: true`); - logger.info(`CONFIG: Influxdb host IP: ${config.get('Butler-SOS.influxdbConfig.host')}`); - logger.info(`CONFIG: Influxdb host port: ${config.get('Butler-SOS.influxdbConfig.port')}`); - logger.info(`CONFIG: Influxdb version: ${config.get('Butler-SOS.influxdbConfig.version')}`); - - // Version specific configs - if (config.get('Butler-SOS.influxdbConfig.version') === 1) { - logger.info( - `CONFIG: Influxdb db name: ${config.get('Butler-SOS.influxdbConfig.v1Config.dbName')}` - ); - logger.info( - `CONFIG: Influxdb retention policy: ${config.get('Butler-SOS.influxdbConfig.v1Config.retentionPolicy.name')}` - ); - } else if (config.get('Butler-SOS.influxdbConfig.version') === 2) { - logger.info( - `CONFIG: Influxdb organisation: ${config.get('Butler-SOS.influxdbConfig.v2Config.org')}` - ); - logger.info( - `CONFIG: Influxdb bucket name: ${config.get('Butler-SOS.influxdbConfig.v2Config.bucket')}` - ); - logger.info( - `CONFIG: Influxdb retention policy duration: ${config.get('Butler-SOS.influxdbConfig.v2Config.retentionDuration')}` - ); - } else { - logger.error( - `CONFIG: Influxdb version ${config.get('Butler-SOS.influxdbConfig.version')} is not supported!` - ); - } -} else { - logger.info(`CONFIG: Influxdb enabled: false`); -} + try { + this.udpServerUserActivity.host = this.config.get( + 'Butler-SOS.userEvents.udpServerConfig.serverHost' + ); + + // Prepare to listen on port X for incoming UDP connections regarding user activity events + this.udpServerUserActivity.socket = dgram.createSocket({ + type: 'udp4', + reuseAddr: true, + }); -// Set up Influxdb client -let influx; -const influxWriteApi = []; -if (config.get('Butler-SOS.influxdbConfig.enable') === true) { - if (config.get('Butler-SOS.influxdbConfig.version') === 1) { - // Set up Influxdb v1 client - influx = new Influx.InfluxDB({ - host: config.get('Butler-SOS.influxdbConfig.host'), - port: config.get('Butler-SOS.influxdbConfig.port'), - database: config.get('Butler-SOS.influxdbConfig.v1Config.dbName'), - username: `${ - config.get('Butler-SOS.influxdbConfig.v1Config.auth.enable') - ? config.get('Butler-SOS.influxdbConfig.v1Config.auth.username') - : '' - }`, - password: `${ - config.get('Butler-SOS.influxdbConfig.v1Config.auth.enable') - ? config.get('Butler-SOS.influxdbConfig.v1Config.auth.password') - : '' - }`, - schema: [ - { - measurement: 'sense_server', - fields: { - version: Influx.FieldType.STRING, - started: Influx.FieldType.STRING, - uptime: Influx.FieldType.STRING, - }, - tags: tagValues, - }, - { - measurement: 'mem', - fields: { - comitted: Influx.FieldType.INTEGER, - allocated: Influx.FieldType.INTEGER, - free: Influx.FieldType.INTEGER, - }, - tags: tagValues, - }, - { - measurement: 'apps', - fields: { - active_docs_count: Influx.FieldType.INTEGER, - loaded_docs_count: Influx.FieldType.INTEGER, - in_memory_docs_count: Influx.FieldType.INTEGER, - active_docs: Influx.FieldType.STRING, - active_docs_names: Influx.FieldType.STRING, - active_session_docs_names: Influx.FieldType.STRING, - loaded_docs: Influx.FieldType.STRING, - loaded_docs_names: Influx.FieldType.STRING, - loaded_session_docs_names: Influx.FieldType.STRING, - in_memory_docs: Influx.FieldType.STRING, - in_memory_docs_names: Influx.FieldType.STRING, - in_memory_session_docs_names: Influx.FieldType.STRING, - calls: Influx.FieldType.INTEGER, - selections: Influx.FieldType.INTEGER, - }, - tags: tagValues, - }, - { - measurement: 'cpu', - fields: { - total: Influx.FieldType.INTEGER, - }, - tags: tagValues, - }, - { - measurement: 'session', - fields: { - active: Influx.FieldType.INTEGER, - total: Influx.FieldType.INTEGER, - }, - tags: tagValues, - }, - { - measurement: 'users', - fields: { - active: Influx.FieldType.INTEGER, - total: Influx.FieldType.INTEGER, - }, - tags: tagValues, - }, - { - measurement: 'cache', - fields: { - hits: Influx.FieldType.INTEGER, - lookups: Influx.FieldType.INTEGER, - added: Influx.FieldType.INTEGER, - replaced: Influx.FieldType.INTEGER, - bytes_added: Influx.FieldType.INTEGER, - }, - tags: tagValues, - }, - { - measurement: 'log_event_logdb', - fields: { - message: Influx.FieldType.STRING, - }, - tags: tagValuesLogEventLogDb, - }, - { - measurement: 'log_event', - fields: { - message: Influx.FieldType.STRING, - exception_message: Influx.FieldType.STRING, - app_name: Influx.FieldType.STRING, - app_id: Influx.FieldType.STRING, - execution_id: Influx.FieldType.STRING, - command: Influx.FieldType.STRING, - result_code: Influx.FieldType.STRING, - origin: Influx.FieldType.STRING, - context: Influx.FieldType.STRING, - session_id: Influx.FieldType.STRING, - raw_event: Influx.FieldType.STRING, - }, - tags: tagValuesLogEvent, - }, - { - measurement: 'butlersos_memory_usage', - fields: { - heap_used: Influx.FieldType.FLOAT, - heap_total: Influx.FieldType.FLOAT, - external: Influx.FieldType.FLOAT, - process_memory: Influx.FieldType.FLOAT, - }, - tags: ['butler_sos_instance', 'version'], - }, - { - measurement: 'user_session_summary', - fields: { - session_count: Influx.FieldType.INTEGER, - session_user_id_list: Influx.FieldType.STRING, - }, - tags: tagValuesUserProxySessions, - }, - { - measurement: 'user_session_list', - fields: { - session_user_id_list: Influx.FieldType.STRING, - }, - tags: tagValuesUserProxySessions, - }, - // { - // measurement: 'user_events', - // fields: { - // userFull: Influx.FieldType.STRING, - // userId: Influx.FieldType.STRING - // }, - // tags: ['host', 'event_action', 'userFull', 'userDirectory', 'userId', 'origin'] - // }, - ], - }); - } else if (config.get('Butler-SOS.influxdbConfig.version') === 2) { - // Set up Influxdb v2 client - const url = `http://${config.get('Butler-SOS.influxdbConfig.host')}:${config.get( - 'Butler-SOS.influxdbConfig.port' - )}`; - const token = config.get('Butler-SOS.influxdbConfig.v2Config.token'); + this.udpServerUserActivity.portUserActivity = this.config.get( + 'Butler-SOS.userEvents.udpServerConfig.portUserActivityEvents' + ); + } catch (err) { + this.logger.error(`CONFIG: Setting up UDP user activity listener: ${err}`); + } + + // ------------------------------------ + // Log events UDP server + this.udpServerLogEvents = {}; try { - influx = new InfluxDB2({ url, token }); + this.udpServerLogEvents.host = this.config.get( + 'Butler-SOS.logEvents.udpServerConfig.serverHost' + ); + + // Prepare to listen on port X for incoming UDP connections regarding user activity events + this.udpServerLogEvents.socket = dgram.createSocket({ + type: 'udp4', + reuseAddr: true, + }); + + this.udpServerLogEvents.port = this.config.get( + 'Butler-SOS.logEvents.udpServerConfig.portLogEvents' + ); } catch (err) { - logger.error(`INFLUXDB2 INIT: Error creating InfluxDB 2 client: ${err}`); - logger.error(`INFLUXDB2 INIT: Exiting.`); + this.logger.error(`CONFIG: Setting up UDP log events listener: ${err}`); } - } else { - logger.error( - `CONFIG: Influxdb version ${config.get('Butler-SOS.influxdbConfig.version')} is not supported!` - ); - } -} -async function initInfluxDB() { - let enableInfluxdb = false; + // ------------------------------------ + // Get info on what servers to monitor + this.serverList = this.config.get('Butler-SOS.serversToMonitor.servers'); + + // Only set up connection pool for accessing Qlik Sense log db if that feature is enabled + this.pgPool; + if (this.config.get('Butler-SOS.logdb.enable') === true) { + const { Pool } = pg; + + // Set up connection pool for accessing Qlik Sense log db + this.pgPool = new Pool({ + host: this.config.get('Butler-SOS.logdb.host'), + database: 'QLogs', + user: this.config.get('Butler-SOS.logdb.qlogsReaderUser'), + password: this.config.get('Butler-SOS.logdb.qlogsReaderPwd'), + port: this.config.get('Butler-SOS.logdb.port'), + }); + + // the pool will emit an error on behalf of any idle clients + // it contains if a backend error or network partition happens + // eslint-disable-next-line no-unused-vars + this.pgPool.on('error', (err, client) => { + this.logger.error(`CONFIG: Unexpected error on idle client: ${err}`); + // process.exit(-1); + }); + } - // Handle InfluxDB v1 - if (config.get('Butler-SOS.influxdbConfig.version') === 1) { - const dbName = config.get('Butler-SOS.influxdbConfig.v1Config.dbName'); + // Get list of standard and user configurable tags + // ..begin with standard tags + const tagValues = ['host', 'server_name', 'server_description']; + // ..check if there are any extra tags for this Butler SOS instance that should be sent to InfluxDB if ( - influx && - config.get('Butler-SOS.influxdbConfig.enable') === true && - dbName?.length > 0 + this.config.has('Butler-SOS.serversToMonitor.serverTagsDefinition') && + this.config.get('Butler-SOS.serversToMonitor.serverTagsDefinition') !== null ) { - enableInfluxdb = true; + // Loop over all tags defined for the current server, adding them to the data structure that will later be passed to Influxdb + this.config.get('Butler-SOS.serversToMonitor.serverTagsDefinition').forEach((entry) => { + this.logger.debug( + `CONFIG: Setting up new Influx database: Found server tag : ${entry}` + ); + + tagValues.push(entry); + }); } - if (enableInfluxdb) { - try { - const names = await influx.getDatabaseNames(); - if (!names.includes(dbName)) { - try { - const res = await influx.createDatabase(dbName); - logger.info(`CONFIG: Created new InfluxDB v1 database: ${dbName}`); + // Add tags for log events + const tagValuesLogEvent = tagValues.slice(); + tagValuesLogEvent.push('level'); + tagValuesLogEvent.push('source'); + tagValuesLogEvent.push('log_row'); + tagValuesLogEvent.push('subsystem'); + tagValuesLogEvent.push('user_full'); + tagValuesLogEvent.push('user_directory'); + tagValuesLogEvent.push('user_id'); + tagValuesLogEvent.push('task_id'); + tagValuesLogEvent.push('task_name'); + tagValuesLogEvent.push('app_id'); + tagValuesLogEvent.push('app_name'); + tagValuesLogEvent.push('result_code'); + tagValuesLogEvent.push('windows_user'); + tagValuesLogEvent.push('engine_exe_version'); + + // Check if there are any extra log event tags in the config file + if ( + this.config.has('Butler-SOS.logEvents.tags') && + this.config.get('Butler-SOS.logEvents.tags') !== null + ) { + this.config.get('Butler-SOS.logEvents.tags').forEach((entry) => { + this.logger.debug( + `CONFIG: Setting up new Influx database: Found log event tag in config file: ${JSON.stringify( + entry + )}` + ); + + tagValuesLogEvent.push(entry.tag); + }); + } - const newPolicy = config.get( - 'Butler-SOS.influxdbConfig.v1Config.retentionPolicy' - ); + // Add tags for log events categories, if enabled and configured + if ( + this.config.has('Butler-SOS.logEvents.categorise.enable') && + this.config.get('Butler-SOS.logEvents.categorise.enable') === true && + this.config.has('Butler-SOS.logEvents.categorise.rules') + ) { + // Add tags from Butler-SOS.logEvents.categorise.rules[].category[], where each object has properties 'name' and 'value' + this.config.get('Butler-SOS.logEvents.categorise.rules').forEach((rule) => { + rule.category.forEach((category) => { + tagValuesLogEvent.push(category.name); + }); + }); + + // Add default rule categories, if enabled + if ( + this.config.has('Butler-SOS.logEvents.categorise.ruleDefault.enable') && + this.config.get('Butler-SOS.logEvents.categorise.ruleDefault.enable') === true && + this.config.has('Butler-SOS.logEvents.categorise.ruleDefault.category') + ) { + this.config + .get('Butler-SOS.logEvents.categorise.ruleDefault.category') + .forEach((category) => { + tagValuesLogEvent.push(category.name); + }); + } + } + + // Create InfluxDB tags for data coming from log db + const tagValuesLogEventLogDb = tagValues.slice(); + tagValuesLogEventLogDb.push('source_process'); + tagValuesLogEventLogDb.push('log_level'); + + // Create tags for user sessions + const tagValuesUserProxySessions = tagValues.slice(); + tagValuesUserProxySessions.push('user_session_virtual_proxy'); + tagValuesUserProxySessions.push('user_session_host'); + + // Show Influxdb config + if (this.config.get('Butler-SOS.influxdbConfig.enable') === true) { + this.logger.info(`CONFIG: Influxdb enabled: true`); + this.logger.info( + `CONFIG: Influxdb host IP: ${this.config.get('Butler-SOS.influxdbConfig.host')}` + ); + this.logger.info( + `CONFIG: Influxdb host port: ${this.config.get('Butler-SOS.influxdbConfig.port')}` + ); + this.logger.info( + `CONFIG: Influxdb version: ${this.config.get('Butler-SOS.influxdbConfig.version')}` + ); + + // Version specific configs + if (this.config.get('Butler-SOS.influxdbConfig.version') === 1) { + this.logger.info( + `CONFIG: Influxdb db name: ${this.config.get('Butler-SOS.influxdbConfig.v1Config.dbName')}` + ); + this.logger.info( + `CONFIG: Influxdb retention policy: ${this.config.get('Butler-SOS.influxdbConfig.v1Config.retentionPolicy.name')}` + ); + } else if (this.config.get('Butler-SOS.influxdbConfig.version') === 2) { + this.logger.info( + `CONFIG: Influxdb organisation: ${this.config.get('Butler-SOS.influxdbConfig.v2Config.org')}` + ); + this.logger.info( + `CONFIG: Influxdb bucket name: ${this.config.get('Butler-SOS.influxdbConfig.v2Config.bucket')}` + ); + this.logger.info( + `CONFIG: Influxdb retention policy duration: ${this.config.get('Butler-SOS.influxdbConfig.v2Config.retentionDuration')}` + ); + } else { + this.logger.error( + `CONFIG: Influxdb version ${this.config.get('Butler-SOS.influxdbConfig.version')} is not supported!` + ); + } + } else { + this.logger.info(`CONFIG: Influxdb enabled: false`); + } + + this.influxWriteApi = []; + if (this.config.get('Butler-SOS.influxdbConfig.enable') === true) { + if (this.config.get('Butler-SOS.influxdbConfig.version') === 1) { + // Set up Influxdb v1 client + this.influx = new Influx.InfluxDB({ + host: this.config.get('Butler-SOS.influxdbConfig.host'), + port: this.config.get('Butler-SOS.influxdbConfig.port'), + database: this.config.get('Butler-SOS.influxdbConfig.v1Config.dbName'), + username: `${ + this.config.get('Butler-SOS.influxdbConfig.v1Config.auth.enable') + ? this.config.get('Butler-SOS.influxdbConfig.v1Config.auth.username') + : '' + }`, + password: `${ + this.config.get('Butler-SOS.influxdbConfig.v1Config.auth.enable') + ? this.config.get('Butler-SOS.influxdbConfig.v1Config.auth.password') + : '' + }`, + schema: [ + { + measurement: 'sense_server', + fields: { + version: Influx.FieldType.STRING, + started: Influx.FieldType.STRING, + uptime: Influx.FieldType.STRING, + }, + tags: tagValues, + }, + { + measurement: 'mem', + fields: { + comitted: Influx.FieldType.INTEGER, + allocated: Influx.FieldType.INTEGER, + free: Influx.FieldType.INTEGER, + }, + tags: tagValues, + }, + { + measurement: 'apps', + fields: { + active_docs_count: Influx.FieldType.INTEGER, + loaded_docs_count: Influx.FieldType.INTEGER, + in_memory_docs_count: Influx.FieldType.INTEGER, + active_docs: Influx.FieldType.STRING, + active_docs_names: Influx.FieldType.STRING, + active_session_docs_names: Influx.FieldType.STRING, + loaded_docs: Influx.FieldType.STRING, + loaded_docs_names: Influx.FieldType.STRING, + loaded_session_docs_names: Influx.FieldType.STRING, + in_memory_docs: Influx.FieldType.STRING, + in_memory_docs_names: Influx.FieldType.STRING, + in_memory_session_docs_names: Influx.FieldType.STRING, + calls: Influx.FieldType.INTEGER, + selections: Influx.FieldType.INTEGER, + }, + tags: tagValues, + }, + { + measurement: 'cpu', + fields: { + total: Influx.FieldType.INTEGER, + }, + tags: tagValues, + }, + { + measurement: 'session', + fields: { + active: Influx.FieldType.INTEGER, + total: Influx.FieldType.INTEGER, + }, + tags: tagValues, + }, + { + measurement: 'users', + fields: { + active: Influx.FieldType.INTEGER, + total: Influx.FieldType.INTEGER, + }, + tags: tagValues, + }, + { + measurement: 'cache', + fields: { + hits: Influx.FieldType.INTEGER, + lookups: Influx.FieldType.INTEGER, + added: Influx.FieldType.INTEGER, + replaced: Influx.FieldType.INTEGER, + bytes_added: Influx.FieldType.INTEGER, + }, + tags: tagValues, + }, + { + measurement: 'log_event_logdb', + fields: { + message: Influx.FieldType.STRING, + }, + tags: tagValuesLogEventLogDb, + }, + { + measurement: 'log_event', + fields: { + message: Influx.FieldType.STRING, + exception_message: Influx.FieldType.STRING, + app_name: Influx.FieldType.STRING, + app_id: Influx.FieldType.STRING, + execution_id: Influx.FieldType.STRING, + command: Influx.FieldType.STRING, + result_code: Influx.FieldType.STRING, + origin: Influx.FieldType.STRING, + context: Influx.FieldType.STRING, + session_id: Influx.FieldType.STRING, + raw_event: Influx.FieldType.STRING, + }, + tags: tagValuesLogEvent, + }, + { + measurement: 'butlersos_memory_usage', + fields: { + heap_used: Influx.FieldType.FLOAT, + heap_total: Influx.FieldType.FLOAT, + external: Influx.FieldType.FLOAT, + process_memory: Influx.FieldType.FLOAT, + }, + tags: ['butler_sos_instance', 'version'], + }, + { + measurement: 'user_session_summary', + fields: { + session_count: Influx.FieldType.INTEGER, + session_user_id_list: Influx.FieldType.STRING, + }, + tags: tagValuesUserProxySessions, + }, + { + measurement: 'user_session_list', + fields: { + session_user_id_list: Influx.FieldType.STRING, + }, + tags: tagValuesUserProxySessions, + }, + // { + // measurement: 'user_events', + // fields: { + // userFull: Influx.FieldType.STRING, + // userId: Influx.FieldType.STRING + // }, + // tags: ['host', 'event_action', 'userFull', 'userDirectory', 'userId', 'origin'] + // }, + ], + }); + } else if (this.config.get('Butler-SOS.influxdbConfig.version') === 2) { + // Set up Influxdb v2 client + const url = `http://${this.config.get('Butler-SOS.influxdbConfig.host')}:${this.config.get( + 'Butler-SOS.influxdbConfig.port' + )}`; + const token = this.config.get('Butler-SOS.influxdbConfig.v2Config.token'); + + try { + this.influx = new InfluxDB2({ url, token }); + } catch (err) { + this.logger.error(`INFLUXDB2 INIT: Error creating InfluxDB 2 client: ${err}`); + this.logger.error(`INFLUXDB2 INIT: Exiting.`); + } + } else { + this.logger.error( + `CONFIG: Influxdb version ${this.config.get('Butler-SOS.influxdbConfig.version')} is not supported!` + ); + } + } + + // Now initialise InfluxDB + await this.initInfluxDB(); + + // ------------------------------------ + // Create MQTT client object and connect to MQTT broker + // Only do this if MQTT is enabled + // ------------------------------------ + if (this.config.get('Butler-SOS.mqttConfig.enable') === true) { + this.mqttClient = mqtt.connect({ + port: this.config.get('Butler-SOS.mqttConfig.brokerPort'), + host: this.config.get('Butler-SOS.mqttConfig.brokerHost'), + }); + } + + /* + Following might be needed for conecting to older Mosquitto versions + var mqttClient = mqtt.connect('mqtt://', { + protocolId: 'MQIsdp', + protocolVersion: 3 + }); + */ + + // Anon telemetry reporting + this.hostInfo = await this.initHostInfo(); - // Create new default retention policy + // Indicate that we have finished initialising + this.initialised = true; + + this.logger.verbose('GLOBALS: Init done'); + + // eslint-disable-next-line no-constructor-return + return instance; + } + + // Static function to check if a file exists + static checkFileExistsSync(filepath) { + let flag = true; + try { + fs.accessSync(filepath, fs.constants.F_OK); + } catch (e) { + flag = false; + } + return flag; + } + + // Static sleep function + static sleep(ms) { + // eslint-disable-next-line no-promise-executor-return + return new Promise((resolve) => setTimeout(resolve, ms)); + } + + // Static function to check if Butler is running in a Docker container + static isRunningInDocker() { + try { + fs.accessSync('/.dockerenv'); + return true; + } catch (_) { + return false; + } + } + + async initInfluxDB() { + let enableInfluxdb = false; + + // Handle InfluxDB v1 + if (this.config.get('Butler-SOS.influxdbConfig.version') === 1) { + const dbName = this.config.get('Butler-SOS.influxdbConfig.v1Config.dbName'); + + if ( + influx && + this.config.get('Butler-SOS.influxdbConfig.enable') === true && + dbName?.length > 0 + ) { + enableInfluxdb = true; + } + + if (enableInfluxdb) { + try { + const names = await this.influx.getDatabaseNames(); + if (!names.includes(dbName)) { + // Create new database try { - const res2 = await influx.createRetentionPolicy(newPolicy.name, { - database: dbName, - duration: newPolicy.duration, - replication: 1, - isDefault: true, - }); + const res = await this.influx.createDatabase(dbName); + this.logger.info(`CONFIG: Created new InfluxDB v1 database: ${dbName}`); - logger.info( - `CONFIG: Created new InfluxDB v1 retention policy: ${newPolicy.name}` + const newPolicy = this.config.get( + 'Butler-SOS.influxdbConfig.v1Config.retentionPolicy' ); + + // Create new default retention policy + try { + const res2 = await this.influx.createRetentionPolicy( + newPolicy.name, + { + database: dbName, + duration: newPolicy.duration, + replication: 1, + isDefault: true, + } + ); + + this.logger.info( + `CONFIG: Created new InfluxDB v1 retention policy: ${newPolicy.name}` + ); + } catch (err) { + this.logger.error( + `CONFIG: Error creating new InfluxDB v1 retention policy "${newPolicy.name}"! ${err.stack}` + ); + } } catch (err) { - logger.error( - `CONFIG: Error creating new InfluxDB v1 retention policy "${newPolicy.name}"! ${err.stack}` + this.logger.error( + `CONFIG: Error creating new InfluxDB v1 database "${dbName}"! ${err.stack}` ); } - } catch (err) { - logger.error( - `CONFIG: Error creating new InfluxDB v1 database "${dbName}"! ${err.stack}` - ); + } else { + this.logger.info(`CONFIG: Found InfluxDB v1 database: ${dbName}`); } - } else { - logger.info(`CONFIG: Found InfluxDB v1 database: ${dbName}`); + } catch (err) { + this.logger.error( + `CONFIG: Error getting list of InfluxDB v1 databases. ${err.stack}` + ); } - } catch (err) { - logger.error(`CONFIG: Error getting list of InfluxDB v1 databases. ${err.stack}`); } - } - } else if (config.get('Butler-SOS.influxdbConfig.version') === 2) { - // Get config - const org = config.get('Butler-SOS.influxdbConfig.v2Config.org'); - const bucketName = config.get('Butler-SOS.influxdbConfig.v2Config.bucket'); - const description = config.get('Butler-SOS.influxdbConfig.v2Config.description'); - const token = config.get('Butler-SOS.influxdbConfig.v2Config.token'); - const retentionDuration = config.get( - 'Butler-SOS.influxdbConfig.v2Config.retentionDuration' - ); - - if ( - influx && - config.get('Butler-SOS.influxdbConfig.enable') === true && - org?.length > 0 && - bucketName?.length > 0 && - token?.length > 0 && - retentionDuration?.length > 0 - ) { - enableInfluxdb = true; - } + } else if (this.config.get('Butler-SOS.influxdbConfig.version') === 2) { + // Get config + const org = this.config.get('Butler-SOS.influxdbConfig.v2Config.org'); + const bucketName = this.config.get('Butler-SOS.influxdbConfig.v2Config.bucket'); + const description = this.config.get('Butler-SOS.influxdbConfig.v2Config.description'); + const token = this.config.get('Butler-SOS.influxdbConfig.v2Config.token'); + const retentionDuration = this.config.get( + 'Butler-SOS.influxdbConfig.v2Config.retentionDuration' + ); + + if ( + this.influx && + this.config.get('Butler-SOS.influxdbConfig.enable') === true && + org?.length > 0 && + bucketName?.length > 0 && + token?.length > 0 && + retentionDuration?.length > 0 + ) { + enableInfluxdb = true; + } - if (enableInfluxdb) { - let orgID; + if (enableInfluxdb) { + let orgID; - try { - // Get organisation by name - const orgsAPI = new OrgsAPI(influx); - const organizations = await orgsAPI.getOrgs({ org }); - if (!organizations || !organizations.orgs || !organizations.orgs.length) { - logger.error(`INFLUXDB2: No organization named "${org}" found!`); + try { + // Get organisation by name + const orgsAPI = new OrgsAPI(this.influx); + const organizations = await orgsAPI.getOrgs({ org }); + if (!organizations || !organizations.orgs || !organizations.orgs.length) { + this.logger.error(`INFLUXDB2: No organization named "${org}" found!`); + } + orgID = organizations.orgs[0].id; + this.logger.info( + `INFLUXDB2: Using organization "${org}" identified by "${orgID}"` + ); + } catch (err) { + this.logger.error(`INFLUXDB2: Error getting organisation: ${err}`); } - orgID = organizations.orgs[0].id; - logger.info(`INFLUXDB2: Using organization "${org}" identified by "${orgID}"`); - } catch (err) { - logger.error(`INFLUXDB2: Error getting organisation: ${err}`); - } - try { - // Get buckets by name - const bucketsAPI = new BucketsAPI(influx); try { - const buckets = await bucketsAPI.getBuckets({ orgID, name: bucketName }); - if (buckets && buckets.buckets && buckets.buckets.length > 0) { - const bucketID = buckets.buckets[0].id; - logger.info( - `INFLUXDB2: Bucket named "${bucketName}" already exists, bucket ID="${bucketID}"` - ); - } - } catch (e) { - if (e instanceof HttpError && e.statusCode === 404) { - // Bucket not found. Let's create it - logger.info( - `INFLUXDB2: Bucket named "${bucketName}" not found, creating it...` - ); + // Get buckets by name + const bucketsAPI = new BucketsAPI(this.influx); + try { + const buckets = await bucketsAPI.getBuckets({ orgID, name: bucketName }); + if (buckets && buckets.buckets && buckets.buckets.length > 0) { + const bucketID = buckets.buckets[0].id; + this.logger.info( + `INFLUXDB2: Bucket named "${bucketName}" already exists, bucket ID="${bucketID}"` + ); + } + } catch (e) { + if (e instanceof HttpError && e.statusCode === 404) { + // Bucket not found. Let's create it + this.logger.info( + `INFLUXDB2: Bucket named "${bucketName}" not found, creating it...` + ); - // creates a bucket, entity properties are specified in the "body" property - const newBucket = await bucketsAPI.postBuckets({ - body: { orgID, name: bucketName, description, rp: retentionDuration }, - }); + // creates a bucket, entity properties are specified in the "body" property + const newBucket = await bucketsAPI.postBuckets({ + body: { + orgID, + name: bucketName, + description, + rp: retentionDuration, + }, + }); - logger.verbose( - `INFLUXDB2: New bucket: ${JSON.stringify( - newBucket, - (key, value) => (key === 'links' ? undefined : value), - 2 - )}` - ); - } else { - throw e; + this.logger.verbose( + `INFLUXDB2: New bucket: ${JSON.stringify( + newBucket, + (key, value) => (key === 'links' ? undefined : value), + 2 + )}` + ); + } else { + throw e; + } } + } catch (err) { + this.logger.error(`INFLUXDB2: Error getting bucket: ${err}`); } - } catch (err) { - logger.error(`INFLUXDB2: Error getting bucket: ${err}`); - } - // Get write API + // Get write API - // Create array of per-server writeAPI objects - // Each object has two properties: host and writeAPI, where host can be used as key later on - serverList.forEach((server) => { - // Get per-server tags - const tags = getServerTags(logger, server); + // Create array of per-server writeAPI objects + // Each object has two properties: host and writeAPI, where host can be used as key later on + this.serverList.forEach((server) => { + // Get per-server tags + const tags = getServerTags(this.logger, server); - // advanced write options - const writeOptions = { - /* the maximum points/lines to send in a single batch to InfluxDB server */ - // batchSize: flushBatchSize + 1, // don't let automatically flush data + // advanced write options + const writeOptions = { + /* the maximum points/lines to send in a single batch to InfluxDB server */ + // batchSize: flushBatchSize + 1, // don't let automatically flush data - /* default tags to add to every point */ - defaultTags: tags, + /* default tags to add to every point */ + defaultTags: tags, - /* maximum time in millis to keep points in an unflushed batch, 0 means don't periodically flush */ - flushInterval: 5000, + /* maximum time in millis to keep points in an unflushed batch, 0 means don't periodically flush */ + flushInterval: 5000, - /* maximum size of the retry buffer - it contains items that could not be sent for the first time */ - // maxBufferLines: 30_000, + /* maximum size of the retry buffer - it contains items that could not be sent for the first time */ + // maxBufferLines: 30_000, - /* the count of internally-scheduled retries upon write failure, the delays between write attempts follow an exponential backoff strategy if there is no Retry-After HTTP header */ - maxRetries: 2, // do not retry writes + /* the count of internally-scheduled retries upon write failure, the delays between write attempts follow an exponential backoff strategy if there is no Retry-After HTTP header */ + maxRetries: 2, // do not retry writes - // ... there are more write options that can be customized, see - // https://influxdata.github.io/influxdb-client-js/influxdb-client.writeoptions.html and - // https://influxdata.github.io/influxdb-client-js/influxdb-client.writeretryoptions.html - }; + // ... there are more write options that can be customized, see + // https://influxdata.github.io/influxdb-client-js/influxdb-client.writeoptions.html and + // https://influxdata.github.io/influxdb-client-js/influxdb-client.writeretryoptions.html + }; - try { - const serverWriteApi = influx.getWriteApi(org, bucketName, 'ns', writeOptions); + try { + const serverWriteApi = this.influx.getWriteApi( + org, + bucketName, + 'ns', + writeOptions + ); - // Save to global variable, using hostNamre as key - influxWriteApi.push({ - serverName: server.serverName, - writeAPI: serverWriteApi, - }); - } catch (err) { - logger.error(`INFLUXDB2: Error getting write API: ${err}`); - } - }); + // Save to global variable, using hostNamre as key + this.influxWriteApi.push({ + serverName: server.serverName, + writeAPI: serverWriteApi, + }); + } catch (err) { + this.logger.error(`INFLUXDB2: Error getting write API: ${err}`); + } + }); + } } } -} -// ------------------------------------ -// Create MQTT client object and connect to MQTT broker -// Only do this if MQTT is enabled -// ------------------------------------ -let mqttClient; - -if (config.get('Butler-SOS.mqttConfig.enable') === true) { - mqttClient = mqtt.connect({ - port: config.get('Butler-SOS.mqttConfig.brokerPort'), - host: config.get('Butler-SOS.mqttConfig.brokerHost'), - }); -} - -/* - Following might be needed for conecting to older Mosquitto versions - var mqttClient = mqtt.connect('mqtt://', { - protocolId: 'MQIsdp', - protocolVersion: 3 - }); -*/ - -// Anon telemetry reporting -let hostInfo; - -async function initHostInfo() { - try { - const siCPU = await si.cpu(); - const siSystem = await si.system(); - const siMem = await si.mem(); - const siOS = await si.osInfo(); - const siDocker = await si.dockerInfo(); - const siNetwork = await si.networkInterfaces(); - const siNetworkDefault = await si.networkInterfaceDefault(); - - const defaultNetworkInterface = siNetworkDefault; - - const networkInterface = siNetwork.filter((item) => item.iface === defaultNetworkInterface); - - const idSrc = - networkInterface[0].mac + - networkInterface[0].ip4 + - config.get('Butler-SOS.logdb.host') + - siSystem.uuid; - const salt = networkInterface[0].mac; - const hash = crypto.createHmac('sha256', salt); - hash.update(idSrc); - - // Get first 50 characters of hash - const id = hash.digest('hex'); - - hostInfo = { - id, - node: { - nodeVersion: process.version, - versions: process.versions, - }, - os: { - platform: os.platform(), - release: os.release(), - version: os.version(), - arch: os.arch(), - cpuCores: os.cpus().length, - type: os.type(), - totalmem: os.totalmem(), - }, - si: { - cpu: siCPU, - system: siSystem, - memory: { - total: siMem.total, + async initHostInfo() { + try { + const siCPU = await si.cpu(); + const siSystem = await si.system(); + const siMem = await si.mem(); + const siOS = await si.osInfo(); + const siDocker = await si.dockerInfo(); + const siNetwork = await si.networkInterfaces(); + const siNetworkDefault = await si.networkInterfaceDefault(); + + const defaultNetworkInterface = siNetworkDefault; + + const networkInterface = siNetwork.filter( + (item) => item.iface === defaultNetworkInterface + ); + + const idSrc = + networkInterface[0].mac + + networkInterface[0].ip4 + + this.config.get('Butler-SOS.logdb.host') + + siSystem.uuid; + const salt = networkInterface[0].mac; + const hash = crypto.createHmac('sha256', salt); + hash.update(idSrc); + + // Get first 50 characters of hash + const id = hash.digest('hex'); + + const hostInfo = { + id, + node: { + nodeVersion: process.version, + versions: process.versions, }, - os: siOS, - network: siNetwork, - networkDefault: siNetworkDefault, - docker: siDocker, - }, - }; - - return hostInfo; - } catch (err) { - logger.error(`CONFIG: Getting host info: ${err}`); - return null; + os: { + platform: os.platform(), + release: os.release(), + version: os.version(), + arch: os.arch(), + cpuCores: os.cpus().length, + type: os.type(), + totalmem: os.totalmem(), + }, + si: { + cpu: siCPU, + system: siSystem, + memory: { + total: siMem.total, + }, + os: siOS, + network: siNetwork, + networkDefault: siNetworkDefault, + docker: siDocker, + }, + }; + + return hostInfo; + } catch (err) { + this.logger.error(`CONFIG: Getting host info: ${err}`); + return null; + } } } -module.exports = { - config, - mqttClient, - logger, - getLoggingLevel, - influx, - influxWriteApi, - pgPool, - appVersion, - serverList, - initInfluxDB, - appNames, - udpServerUserActivity, - udpServerLogEvents, - initHostInfo, - hostInfo, - isPkg, - checkFileExistsSync, - configFile, - options, -}; +export default new Settings(); diff --git a/src/lib/appnamesextract.js b/src/lib/appnamesextract.js index 9aaa47cf..43f70dcb 100755 --- a/src/lib/appnamesextract.js +++ b/src/lib/appnamesextract.js @@ -1,14 +1,9 @@ // Get app names from the Qlik Repository Service (QRS) API +import path from 'path'; +import qrsInteract from 'qrs-interact'; +import clonedeep from 'lodash.clonedeep'; -const path = require('path'); - -const qrsInteract = require('qrs-interact'); -const clonedeep = require('lodash.clonedeep'); -const globals = require('../globals'); - -const certPath = path.resolve(process.cwd(), globals.config.get('Butler-SOS.cert.clientCert')); -const keyPath = path.resolve(process.cwd(), globals.config.get('Butler-SOS.cert.clientCertKey')); -// caPath = path.resolve(process.cwd(), globals.config.get('Butler-SOS.cert.clientCertCA')); +import globals from '../globals.js'; function getAppNames() { globals.logger.verbose(`APP NAMES: Start getting app names from repository db`); @@ -18,8 +13,8 @@ function getAppNames() { hostname: globals.config.get('Butler-SOS.appNames.hostIP'), portNumber: 4242, certificates: { - certFile: certPath, - keyFile: keyPath, + certFile: globals.certPath, + keyFile: globals.keyPath, }, }; @@ -27,7 +22,6 @@ function getAppNames() { 'X-Qlik-User': 'UserDirectory=Internal; UserId=sa_repository', }; - // eslint-disable-next-line new-cap const qrsInteractInstance = new qrsInteract(configQRS); const appList = []; @@ -61,7 +55,7 @@ function getAppNames() { } } -function setupAppNamesExtractTimer() { +export function setupAppNamesExtractTimer() { // Configure timer for getting app names data setInterval(() => { globals.logger.verbose('APP NAMES: Event started: Get app names'); @@ -69,8 +63,3 @@ function setupAppNamesExtractTimer() { getAppNames(); }, globals.config.get('Butler-SOS.appNames.extractInterval')); } - -module.exports = { - setupAppNamesExtractTimer, - getAppNames, -}; diff --git a/src/lib/config-file-schema.js b/src/lib/config-file-schema.js index 466f022b..778bc0a9 100755 --- a/src/lib/config-file-schema.js +++ b/src/lib/config-file-schema.js @@ -1,4 +1,4 @@ -const confifgFileSchema = { +export const confifgFileSchema = { 'Butler-SOS': { logLevel: 'string', fileLogging: 'boolean', @@ -409,7 +409,3 @@ const confifgFileSchema = { }, }, }; - -module.exports = { - confifgFileSchema, -}; diff --git a/src/lib/config-file-verify.js b/src/lib/config-file-verify.js index ea47e85b..04267d25 100755 --- a/src/lib/config-file-verify.js +++ b/src/lib/config-file-verify.js @@ -1,9 +1,9 @@ -const { config, configFile, logger } = require('../globals'); -const { confifgFileSchema } = require('./config-file-schema'); +import globals from '../globals.js'; +import { confifgFileSchema } from './config-file-schema.js'; // Function to verify that the config file has the correct format // Use yaml-validator to validate the config file -async function verifyConfigFile() { +export async function verifyConfigFile() { // try { // Dynamically load yaml-validator @@ -12,7 +12,7 @@ async function verifyConfigFile() { // Options for yaml-validator const verifyOptions = { onWarning(error, filepath) { - logger.warn(`${filepath} has error: ${error}`); + globals.logger.warn(`${filepath} has error: ${error}`); }, log: false, structure: confifgFileSchema, @@ -23,21 +23,21 @@ async function verifyConfigFile() { const validator = new YamlValidator(verifyOptions); // File names to validate in array - const files = [configFile]; + const files = [globals.configFile]; // Verify the config file validator.validate(files); // Exit app if there are errors in the config file's structure if (validator.logs.length > 0) { - logger.verbose(`VERIFY CONFIG FILE: Logs length: ${validator.logs.length}`); - logger.verbose(validator.logs); + globals.logger.verbose(`VERIFY CONFIG FILE: Logs length: ${validator.logs.length}`); + globals.logger.verbose(validator.logs); - logger.error(`VERIFY CONFIG FILE: Errors found in config file. Exiting.`); - logger.error( + globals.logger.error(`VERIFY CONFIG FILE: Errors found in config file. Exiting.`); + globals.logger.error( `Tip: Start Butler SOS with --no-config-file-verify option to skip this check and start with provided config file. ` ); - logger.error(`${validator.logs}`); + globals.logger.error(`${validator.logs}`); process.exit(1); } @@ -46,10 +46,10 @@ async function verifyConfigFile() { // If InfluxDB is enabled, check if the version is valid // Valid values: 1 and 2 - if (config.get('Butler-SOS.influxdbConfig.enable') === true) { - const influxdbVersion = config.get('Butler-SOS.influxdbConfig.version'); + if (globals.config.get('Butler-SOS.influxdbConfig.enable') === true) { + const influxdbVersion = globals.config.get('Butler-SOS.influxdbConfig.version'); if (influxdbVersion !== 1 && influxdbVersion !== 2) { - logger.error( + globals.logger.error( `VERIFY CONFIG FILE: Butler-SOS.influxdbConfig.enable (=InfluxDB version) ${influxdbVersion} is invalid. Exiting.` ); process.exit(1); @@ -64,23 +64,23 @@ async function verifyConfigFile() { // If either of the conditions above is false, an error should be logged and Butler SOS should not start. try { // Loop over all defined server tags - const serverTagsDefinition = config.get( + const serverTagsDefinition = globals.config.get( 'Butler-SOS.serversToMonitor.serverTagsDefinition' ); // eslint-disable-next-line no-restricted-syntax for (const tag of serverTagsDefinition) { // Check that all servers have this tag - const servers = config.get('Butler-SOS.serversToMonitor.servers'); + const servers = globals.config.get('Butler-SOS.serversToMonitor.servers'); // eslint-disable-next-line no-restricted-syntax for (const server of servers) { // Check if server.serverTags.tag is defined if (server?.serverTags === null || !server?.serverTags[tag]) { - logger.error( + globals.logger.error( `VERIFY CONFIG FILE: Server tag "${tag}" is not defined for server "${server.serverName}". Exiting.` ); process.exit(1); } else { - logger.verbose( + globals.logger.verbose( `VERIFY CONFIG FILE: Server tag "${tag}" is defined for server "${server.serverName}".` ); } @@ -88,38 +88,36 @@ async function verifyConfigFile() { } // Now ensure that the tags defined for each server are valid and that there are no extra tags there - const servers = config.get('Butler-SOS.serversToMonitor.servers'); + const servers = globals.config.get('Butler-SOS.serversToMonitor.servers'); // eslint-disable-next-line no-restricted-syntax for (const server of servers) { // eslint-disable-next-line no-restricted-syntax for (const tag in server.serverTags) { if (!serverTagsDefinition.includes(tag)) { - logger.error( + globals.logger.error( `VERIFY CONFIG FILE: Server tag "${tag}" for server "${server.serverName}" is not defined in Butler-SOS.serversToMonitor.serverTagsDefinition. Exiting.` ); process.exit(1); } else { - logger.verbose( + globals.logger.verbose( `VERIFY CONFIG FILE: Server tag "${tag}" is defined in Butler-SOS.serversToMonitor.serverTagsDefinition.` ); } } } } catch (err) { - logger.error(`VERIFY CONFIG FILE: Server tags verification failed. ${err}`); + globals.logger.error(`VERIFY CONFIG FILE: Server tags verification failed. ${err}`); process.exit(1); } - logger.info(`VERIFY CONFIG FILE: Your config file at ${configFile} is valid, good work!`); + globals.logger.info( + `VERIFY CONFIG FILE: Your config file at ${globals.configFile} is valid, good work!` + ); return true; } catch (err) { - logger.error(`VERIFY CONFIG FILE: ${err}`); + globals.logger.error(`VERIFY CONFIG FILE: ${err}`); return false; } } - -module.exports = { - verifyConfigFile, -}; diff --git a/src/lib/healthmetrics.js b/src/lib/healthmetrics.js index a90e0042..64d65d70 100755 --- a/src/lib/healthmetrics.js +++ b/src/lib/healthmetrics.js @@ -2,17 +2,18 @@ * Get metrics from the Sense health check API */ -const path = require('path'); -const https = require('https'); -const fs = require('fs'); -const axios = require('axios'); -const globals = require('../globals'); -const postToInfluxdb = require('./post-to-influxdb'); -const postToNewRelic = require('./post-to-new-relic'); -const postToMQTT = require('./post-to-mqtt'); -const { getServerTags } = require('./servertags'); -const serverHeaders = require('./serverheaders'); -const prometheus = require('./prom-client'); +import path from 'path'; +import https from 'https'; +import fs from 'fs'; +import axios from 'axios'; + +import globals from '../globals.js'; +import { postHealthMetricsToInfluxdb } from './post-to-influxdb.js'; +import { postHealthMetricsToNewRelic } from './post-to-new-relic.js'; +import { postHealthToMQTT } from './post-to-mqtt.js'; +import { getServerHeaders } from './serverheaders.js'; +import { getServerTags } from './servertags.js'; +import { saveHealthMetricsToPrometheus } from './prom-client.js'; function getCertificates(options) { const certificate = {}; @@ -24,7 +25,7 @@ function getCertificates(options) { return certificate; } -function getHealthStatsFromSense(serverName, host, tags, headers) { +export function getHealthStatsFromSense(serverName, host, tags, headers) { globals.logger.debug(`HEALTH: URL=https://${host}/engine/healthcheck/`); const options = {}; @@ -98,30 +99,25 @@ function getHealthStatsFromSense(serverName, host, tags, headers) { // Post to MQTT if (globals.config.get('Butler-SOS.mqttConfig.enable') === true) { globals.logger.debug('HEALTH: Calling HEALTH metrics MQTT posting method'); - postToMQTT.postHealthToMQTT(host, tags.host, response.data); + postHealthToMQTT(host, tags.host, response.data); } // Post to Influxdb if (globals.config.get('Butler-SOS.influxdbConfig.enable') === true) { globals.logger.debug('HEALTH: Calling HEALTH metrics Influxdb posting method'); - postToInfluxdb.postHealthMetricsToInfluxdb( - serverName, - host, - response.data, - tags - ); + postHealthMetricsToInfluxdb(serverName, host, response.data, tags); } // Post to New Relic if (globals.config.get('Butler-SOS.newRelic.enable') === true) { globals.logger.debug('HEALTH: Calling HEALTH metrics New Relic posting method'); - postToNewRelic.postHealthMetricsToNewRelic(host, response.data, tags); + postHealthMetricsToNewRelic(host, response.data, tags); } // Save latest available data for Prometheus if (globals.config.get('Butler-SOS.prometheus.enable') === true) { globals.logger.debug('HEALTH: Calling HEALTH metrics Prometheus method'); - prometheus.saveHealthMetrics(host, response.data, tags); + saveHealthMetricsToPrometheus(host, response.data, tags); } } }) @@ -130,7 +126,7 @@ function getHealthStatsFromSense(serverName, host, tags, headers) { }); } -function setupHealthMetricsTimer() { +export function setupHealthMetricsTimer() { // Configure timer for getting healthcheck data setInterval(() => { globals.logger.verbose('HEALTH: Event started: Statistics collection'); @@ -152,14 +148,9 @@ function setupHealthMetricsTimer() { // }); // Get per-server headers - const headers = serverHeaders.getServerHeaders(server); + const headers = getServerHeaders(server); getHealthStatsFromSense(server.serverName, server.host, tags, headers); }); }, globals.config.get('Butler-SOS.serversToMonitor.pollingInterval')); } - -module.exports = { - setupHealthMetricsTimer, - getHealthStatsFromSense, -}; diff --git a/src/lib/heartbeat.js b/src/lib/heartbeat.js index 4aa29c08..c1c849a2 100644 --- a/src/lib/heartbeat.js +++ b/src/lib/heartbeat.js @@ -1,6 +1,6 @@ /* eslint-disable no-unused-vars */ -const later = require('@breejs/later'); -const axios = require('axios'); +import later from '@breejs/later'; +import axios from 'axios'; const callRemoteURL = function callRemoteURL(remoteURL, logger) { axios @@ -15,7 +15,7 @@ const callRemoteURL = function callRemoteURL(remoteURL, logger) { }); }; -function setupHeartbeatTimer(config, logger) { +export function setupHeartbeatTimer(config, logger) { try { logger.debug( `HEARTBEAT: Setting up heartbeat to remote: ${config.get( @@ -34,7 +34,3 @@ function setupHeartbeatTimer(config, logger) { logger.error(`HEARTBEAT: Error ${err}`); } } - -module.exports = { - setupHeartbeatTimer, -}; diff --git a/src/lib/log-event-categorise.js b/src/lib/log-event-categorise.js index 35807de5..0d9e076f 100644 --- a/src/lib/log-event-categorise.js +++ b/src/lib/log-event-categorise.js @@ -1,4 +1,4 @@ -const { config, logger } = require('../globals'); +import globals from '../globals.js'; // Function to categorise log events // @@ -13,7 +13,7 @@ const { config, logger } = require('../globals'); // - name: The name of the category // - value: The value of the category // - actionTaken: The action taken for the log event. Possible values are 'categorised', 'dropped' -function categoriseLogEvent(logLevel, logMessage) { +export function categoriseLogEvent(logLevel, logMessage) { const logEventCategory = []; try { @@ -21,7 +21,7 @@ function categoriseLogEvent(logLevel, logMessage) { // Loop over all rules in the config file // eslint-disable-next-line no-restricted-syntax - for (const rule of config.get('Butler-SOS.logEvents.categorise.rules')) { + for (const rule of globals.config.get('Butler-SOS.logEvents.categorise.rules')) { // Check if the log event matches any of the rule's log levels (which are found in the array 'logLevel' property) // Make the check case insensitive if (rule.logLevel.map((x) => x.toLowerCase()).includes(logLevel.toLowerCase())) { @@ -73,7 +73,7 @@ function categoriseLogEvent(logLevel, logMessage) { // Warn if the filter type is not recognised if (!['sw', 'ew', 'so'].includes(filter.type)) { - logger.warn( + globals.logger.warn( `LOG EVENT CATEGORISATION: Filter type '${filter.type}' is not recognised` ); } @@ -97,22 +97,18 @@ function categoriseLogEvent(logLevel, logMessage) { // If no rule matched, then use default rule (if enabled in the config file) if ( match === false && - config.get('Butler-SOS.logEvents.categorise.ruleDefault.enable') === true + globals.config.get('Butler-SOS.logEvents.categorise.ruleDefault.enable') === true ) { // Deep copy the categories from the default rule to the log event uniqueCategories.push( - ...config.get('Butler-SOS.logEvents.categorise.ruleDefault.category') + ...globals.config.get('Butler-SOS.logEvents.categorise.ruleDefault.category') ); } // Return the log event category and the action taken return { category: uniqueCategories, actionTaken: 'categorised' }; } catch (err) { - logger.error(`LOG EVENT CATEGORISATION: Error processing log event: ${err}`); + globals.logger.error(`LOG EVENT CATEGORISATION: Error processing log event: ${err}`); return null; } } - -module.exports = { - categoriseLogEvent, -}; diff --git a/src/lib/logdb.js b/src/lib/logdb.js index dc38d7ad..95826464 100755 --- a/src/lib/logdb.js +++ b/src/lib/logdb.js @@ -1,9 +1,9 @@ /* eslint-disable no-unused-vars */ -const globals = require('../globals'); -const postToMQTT = require('./post-to-mqtt'); +import globals from '../globals.js'; +import { postLogDbToMQTT } from './post-to-mqtt.js'; -function setupLogDbTimer() { +export function setupLogDbTimer() { // Get query period from config file. const queryPeriod = globals.config.get('Butler-SOS.logdb.queryPeriod'); @@ -155,7 +155,7 @@ function setupLogDbTimer() { // Post to MQTT (if enabled) if (globals.config.get('Butler-SOS.mqttConfig.enable') === true) { globals.logger.silly('LOGDB: Posting log db data to MQTT...'); - postToMQTT.postLogDbToMQTT( + postLogDbToMQTT( row.process_host, row.process_name, row.entry_level, @@ -181,7 +181,3 @@ function setupLogDbTimer() { }); }, globals.config.get('Butler-SOS.logdb.pollingInterval')); } - -module.exports = { - setupLogDbTimer, -}; diff --git a/src/lib/post-to-influxdb.js b/src/lib/post-to-influxdb.js index 1cc9780e..c0cf49a8 100755 --- a/src/lib/post-to-influxdb.js +++ b/src/lib/post-to-influxdb.js @@ -1,9 +1,9 @@ /* eslint-disable prefer-destructuring */ /* eslint-disable no-unused-vars */ -const { Point } = require('@influxdata/influxdb-client'); +import { Point } from '@influxdata/influxdb-client'; -const globals = require('../globals'); +import globals from '../globals.js'; const sessionAppPrefix = 'SessionApp'; @@ -42,7 +42,7 @@ function getFormattedTime(serverStarted) { return `${days} days, ${hours}h ${minutes.substr(-2)}m ${seconds.substr(-2)}s`; } -async function postHealthMetricsToInfluxdb(serverName, host, body, serverTags) { +export async function postHealthMetricsToInfluxdb(serverName, host, body, serverTags) { // Calculate server uptime const formattedTime = getFormattedTime(body.started); @@ -485,7 +485,7 @@ async function postHealthMetricsToInfluxdb(serverName, host, body, serverTags) { } } -async function postProxySessionsToInfluxdb(userSessions) { +export async function postProxySessionsToInfluxdb(userSessions) { globals.logger.debug(`PROXY SESSIONS: User sessions: ${JSON.stringify(userSessions)}`); globals.logger.silly( @@ -562,7 +562,7 @@ async function postProxySessionsToInfluxdb(userSessions) { } } -async function postButlerSOSMemoryUsageToInfluxdb(memory) { +export async function postButlerSOSMemoryUsageToInfluxdb(memory) { globals.logger.debug(`MEMORY USAGE: Memory usage ${JSON.stringify(memory, null, 2)})`); // Get Butler version @@ -678,7 +678,7 @@ async function postButlerSOSMemoryUsageToInfluxdb(memory) { } } -async function postUserEventToInfluxdb(msg) { +export async function postUserEventToInfluxdb(msg) { globals.logger.debug(`USER EVENT INFLUXDB: ${msg})`); // Only write to influuxdb if the global influx object has been initialized @@ -873,7 +873,7 @@ async function postUserEventToInfluxdb(msg) { } } -async function postLogEventToInfluxdb(msg) { +export async function postLogEventToInfluxdb(msg) { globals.logger.debug(`LOG EVENT INFLUXDB: ${msg})`); try { @@ -1258,11 +1258,3 @@ async function postLogEventToInfluxdb(msg) { globals.logger.error(`LOG EVENT INFLUXDB 2: Error saving log event to InfluxDB! ${err}`); } } - -module.exports = { - postHealthMetricsToInfluxdb, - postProxySessionsToInfluxdb, - postButlerSOSMemoryUsageToInfluxdb, - postUserEventToInfluxdb, - postLogEventToInfluxdb, -}; diff --git a/src/lib/post-to-mqtt.js b/src/lib/post-to-mqtt.js index 7560bc5f..fa7bfe09 100755 --- a/src/lib/post-to-mqtt.js +++ b/src/lib/post-to-mqtt.js @@ -1,9 +1,9 @@ /* eslint-disable prefer-destructuring */ /* eslint-disable no-unused-vars */ -const globals = require('../globals'); +import globals from '../globals.js'; -function postLogDbToMQTT(processHost, processName, entryLevel, message, _timestamp) { +export function postLogDbToMQTT(processHost, processName, entryLevel, message, _timestamp) { // Get base MQTT topic const baseTopic = globals.config.get('Butler-SOS.mqttConfig.baseTopic'); @@ -11,7 +11,7 @@ function postLogDbToMQTT(processHost, processName, entryLevel, message, _timesta globals.mqttClient.publish(`${baseTopic + processHost}/${processName}/${entryLevel}`, message); } -function postHealthToMQTT(_host, serverName, body) { +export function postHealthToMQTT(_host, serverName, body) { // Get base MQTT topic const baseTopic = globals.config.get('Butler-SOS.mqttConfig.baseTopic'); @@ -93,7 +93,7 @@ function postHealthToMQTT(_host, serverName, body) { globals.mqttClient.publish(`${baseTopic + serverName}/saturated`, body.saturated.toString()); } -function postUserSessionsToMQTT(host, virtualProxy, body) { +export function postUserSessionsToMQTT(host, virtualProxy, body) { // Get base MQTT topic const baseTopic = globals.config.get('Butler-SOS.mqttConfig.baseTopic'); @@ -101,7 +101,7 @@ function postUserSessionsToMQTT(host, virtualProxy, body) { globals.mqttClient.publish(`${baseTopic + host}/usersession${virtualProxy}`, body); } -function postUserEventToMQTT(msg) { +export function postUserEventToMQTT(msg) { try { // Create payload const payload = { @@ -203,7 +203,7 @@ function postUserEventToMQTT(msg) { } } -function postLogEventToMQTT(msg) { +export function postLogEventToMQTT(msg) { try { // Get MQTT root topic let baseTopic = globals.config.get('Butler-SOS.logEvents.sendToMQTT.baseTopic'); @@ -252,11 +252,3 @@ function postLogEventToMQTT(msg) { globals.logger.error(`LOG EVENT MQTT: Failed posting message to MQTT ${err}.`); } } - -module.exports = { - postLogDbToMQTT, - postHealthToMQTT, - postUserSessionsToMQTT, - postUserEventToMQTT, - postLogEventToMQTT, -}; diff --git a/src/lib/post-to-new-relic.js b/src/lib/post-to-new-relic.js index 168a9b55..335711ac 100755 --- a/src/lib/post-to-new-relic.js +++ b/src/lib/post-to-new-relic.js @@ -1,10 +1,10 @@ /* eslint-disable prefer-destructuring */ /* eslint-disable no-unused-vars */ -const crypto = require('crypto'); -const axios = require('axios'); +import crypto from 'crypto'; +import axios from 'axios'; -const globals = require('../globals'); +import globals from '../globals.js'; // const sessionAppPrefix = 'SessionApp'; @@ -54,7 +54,7 @@ function getFormattedTime(serverStarted) { * @param {*} body * @param {*} tags */ -async function postHealthMetricsToNewRelic(_host, body, tags) { +export async function postHealthMetricsToNewRelic(_host, body, tags) { // Calculate server uptime const formattedTime = getFormattedTime(body.started); @@ -347,7 +347,7 @@ async function postHealthMetricsToNewRelic(_host, body, tags) { * * @param {*} userSessions */ -async function postProxySessionsToNewRelic(userSessions) { +export async function postProxySessionsToNewRelic(userSessions) { globals.logger.debug( `PROXY SESSIONS NEW RELIC: User sessions: ${JSON.stringify(userSessions)}` ); @@ -494,7 +494,7 @@ async function postProxySessionsToNewRelic(userSessions) { } } -async function postButlerSOSUptimeToNewRelic(fields) { +export async function postButlerSOSUptimeToNewRelic(fields) { globals.logger.debug( `MEMORY USAGE NEW RELIC: Memory usage ${JSON.stringify(fields, null, 2)})` ); @@ -651,7 +651,7 @@ async function postButlerSOSUptimeToNewRelic(fields) { } } -async function postUserEventToNewRelic(msg) { +export async function postUserEventToNewRelic(msg) { globals.logger.debug(`USER EVENT NEW RELIC 1: ${JSON.stringify(msg, null, 2)})`); try { @@ -912,7 +912,7 @@ function sendNRLogEventYesNo(sourceService, sourceLogLevel) { * * @param {*} msg */ -async function postLogEventToNewRelic(msg) { +export async function postLogEventToNewRelic(msg) { globals.logger.debug(`LOG EVENT NEW RELIC: ${msg})`); try { @@ -1084,11 +1084,3 @@ async function postLogEventToNewRelic(msg) { globals.logger.error(`LOG EVENT NEW RELIC: Error saving event to New Relic! ${err}`); } } - -module.exports = { - postHealthMetricsToNewRelic, - postProxySessionsToNewRelic, - postButlerSOSUptimeToNewRelic, - postUserEventToNewRelic, - postLogEventToNewRelic, -}; diff --git a/src/lib/prom-client.js b/src/lib/prom-client.js index 7c715910..adb33c6b 100755 --- a/src/lib/prom-client.js +++ b/src/lib/prom-client.js @@ -1,9 +1,9 @@ -const client = require('prom-client'); +import client from 'prom-client'; // import { collectDefaultMetrics, register } from 'prom-client'; // Load global variables and functions -const globals = require('../globals'); -const { getServerTags } = require('./servertags'); +import globals from '../globals.js'; +import { getServerTags } from './servertags.js'; let promLabels = null; @@ -38,7 +38,7 @@ let promMetricUserSessionsTotal = null; // client.collectDefaultMetrics(); -async function setupPromClient(promServer, promPort, promHost) { +export async function setupPromClient(promServer, promPort, promHost) { try { // Create array with all defined server tags that should be used as Prometheus labels globals.serverList.forEach((server) => { @@ -210,7 +210,7 @@ async function setupPromClient(promServer, promPort, promHost) { } } -function saveHealthMetrics(host, data, labels) { +export function saveHealthMetricsToPrometheus(host, data, labels) { try { globals.logger.silly(`PROM: Health metrics (host): ${host}`); globals.logger.silly(`PROM: Health metrics (data): ${JSON.stringify(data)}`); @@ -257,7 +257,7 @@ function saveHealthMetrics(host, data, labels) { } } -function saveUserSessionMetrics(userSessionsData) { +export function saveUserSessionMetricsToPrometheus(userSessionsData) { try { globals.logger.silly(`PROM: Session metrics (host): ${userSessionsData.host}`); globals.logger.silly( @@ -279,9 +279,3 @@ function saveUserSessionMetrics(userSessionsData) { globals.logger.error(`PROM: Error saving health data for Prometheus! ${err.stack}`); } } - -module.exports = { - setupPromClient, - saveHealthMetrics, - saveUserSessionMetrics, -}; diff --git a/src/lib/proxysessionmetrics.js b/src/lib/proxysessionmetrics.js index 8dc0b4b8..57ecb1aa 100755 --- a/src/lib/proxysessionmetrics.js +++ b/src/lib/proxysessionmetrics.js @@ -2,18 +2,18 @@ * Get metrics from Sense repository service */ -const https = require('https'); -const fs = require('fs'); -const path = require('path'); -const axios = require('axios'); -const { Point } = require('@influxdata/influxdb-client'); - -const globals = require('../globals'); -const postToInfluxdb = require('./post-to-influxdb'); -const postToNewRelic = require('./post-to-new-relic'); -const postToMQTT = require('./post-to-mqtt'); -const { getServerTags } = require('./servertags'); -const prometheus = require('./prom-client'); +import https from 'https'; +import fs from 'fs'; +import path from 'path'; +import axios from 'axios'; +import { Point } from '@influxdata/influxdb-client'; + +import globals from '../globals.js'; +import { postProxySessionsToInfluxdb } from './post-to-influxdb.js'; +import { postProxySessionsToNewRelic } from './post-to-new-relic.js'; +import { postUserSessionsToMQTT } from './post-to-mqtt.js'; +import { getServerTags } from './servertags.js'; +import { saveUserSessionMetricsToPrometheus } from './prom-client.js'; function getCertificates(options) { const certificate = {}; @@ -280,7 +280,7 @@ function getProxySessionStatsFromSense(serverName, host, virtualProxy, influxTag 'PROXY SESSIONS: Calling user sessions MQTT posting method' ); - postToMQTT.postUserSessionsToMQTT( + postUserSessionsToMQTT( host.split(':')[0], // response.request._headers.xvirtualproxy, virtualProxy, @@ -302,7 +302,7 @@ function getProxySessionStatsFromSense(serverName, host, virtualProxy, influxTag 'PROXY SESSIONS: Calling user sessions Influxdb posting method' ); - postToInfluxdb.postProxySessionsToInfluxdb(userProxySessionsData); + postProxySessionsToInfluxdb(userProxySessionsData); } // Post to New Relic @@ -316,13 +316,13 @@ function getProxySessionStatsFromSense(serverName, host, virtualProxy, influxTag 'PROXY SESSIONS: Calling user sessions New Relic posting method' ); - postToNewRelic.postProxySessionsToNewRelic(userProxySessionsData); + postProxySessionsToNewRelic(userProxySessionsData); } // Save latest available data for Prometheus if (globals.config.get('Butler-SOS.prometheus.enable') === true) { globals.logger.debug('HEALTH: Calling SESSIONS metrics Prometheus method'); - prometheus.saveUserSessionMetrics(userProxySessionsData); + saveUserSessionMetricsToPrometheus(userProxySessionsData); } } }) @@ -332,7 +332,7 @@ function getProxySessionStatsFromSense(serverName, host, virtualProxy, influxTag } // Get info on what sessions currently exist -function setupUserSessionsTimer() { +export function setupUserSessionsTimer() { globals.logger.debug( `PROXY SESSIONS: Monitor user sessions for these servers/virtual proxies: ${JSON.stringify( globals.serverList, @@ -366,7 +366,3 @@ function setupUserSessionsTimer() { }); }, globals.config.get('Butler-SOS.userSessions.pollingInterval')); } - -module.exports = { - setupUserSessionsTimer, -}; diff --git a/src/lib/serverheaders.js b/src/lib/serverheaders.js index 3526b588..2c449137 100755 --- a/src/lib/serverheaders.js +++ b/src/lib/serverheaders.js @@ -1,6 +1,6 @@ -const globals = require('../globals'); +import globals from '../globals.js'; -function getServerHeaders(server) { +export function getServerHeaders(server) { try { const headers = {}; @@ -25,7 +25,3 @@ function getServerHeaders(server) { return []; } } - -module.exports = { - getServerHeaders, -}; diff --git a/src/lib/servertags.js b/src/lib/servertags.js index 7d929378..88adbdfc 100755 --- a/src/lib/servertags.js +++ b/src/lib/servertags.js @@ -1,4 +1,5 @@ -function getServerTags(logger, server) { +// Get tag values from the server object +export function getServerTags(logger, server) { try { let tags = { host: server.host.split(':')[0], @@ -34,7 +35,3 @@ function getServerTags(logger, server) { return []; } } - -module.exports = { - getServerTags, -}; diff --git a/src/lib/service_uptime.js b/src/lib/service_uptime.js index f0a5f105..8e5d4b35 100644 --- a/src/lib/service_uptime.js +++ b/src/lib/service_uptime.js @@ -1,18 +1,18 @@ /* eslint-disable no-bitwise */ -const later = require('@breejs/later'); -const luxon = require('luxon'); +import later from '@breejs/later'; +import { Duration } from 'luxon'; -const globals = require('../globals'); -const postToInfluxdb = require('./post-to-influxdb'); -const postToNewRelic = require('./post-to-new-relic'); +import globals from '../globals.js'; +import { postButlerSOSMemoryUsageToInfluxdb } from './post-to-influxdb.js'; +import { postButlerSOSUptimeToNewRelic } from './post-to-new-relic.js'; const fullUnits = ['years', 'months', 'days', 'hours', 'minutes', 'seconds']; -luxon.Duration.prototype.toFull = function convToFull() { +Duration.prototype.toFull = function convToFull() { // return this.shiftTo.apply(this, fullUnits); return this.shiftTo(...fullUnits); // Suggested bt GitHub Copilot }; -function serviceUptimeStart() { +export function serviceUptimeStart() { const uptimeLogLevel = globals.config.get('Butler-SOS.uptimeMonitor.logLevel'); const uptimeInterval = globals.config.get('Butler-SOS.uptimeMonitor.frequency'); @@ -51,7 +51,7 @@ function serviceUptimeStart() { startIterations += 1; const uptimeMilliSec = Date.now() - startTime; - const d = luxon.Duration.fromMillis(uptimeMilliSec).toFull().toObject(); + const d = Duration.fromMillis(uptimeMilliSec).toFull().toObject(); // Round to whole seconds d.seconds = Math.round(d.seconds); const uptimeString = `${d.months} months, ${d.days} days, ${d.hours} hours, ${d.minutes} minutes, ${d.seconds} seconds`; @@ -92,7 +92,7 @@ function serviceUptimeStart() { true && enableInfluxDB === true ) { - postToInfluxdb.postButlerSOSMemoryUsageToInfluxdb({ + postButlerSOSMemoryUsageToInfluxdb({ instanceTag: butlerSosMemoryInfluxTag, heapUsedMByte, heapTotalMByte, @@ -103,7 +103,7 @@ function serviceUptimeStart() { // Send to New Relic if (globals.config.get('Butler-SOS.uptimeMonitor.storeNewRelic.enable') === true) { - postToNewRelic.postButlerSOSUptimeToNewRelic({ + postButlerSOSUptimeToNewRelic({ intervalMillisec, heapUsed, heapTotal, @@ -116,7 +116,3 @@ function serviceUptimeStart() { } }, later.parse.text(uptimeInterval)); } - -module.exports = { - serviceUptimeStart, -}; diff --git a/src/lib/telemetry.js b/src/lib/telemetry.js index 0187c6ea..c60f6f0b 100644 --- a/src/lib/telemetry.js +++ b/src/lib/telemetry.js @@ -1,7 +1,6 @@ -const { PostHog } = require('posthog-node'); +import { PostHog } from 'posthog-node'; -const globals = require('../globals'); -const { log } = require('winston'); +import globals from '../globals.js'; // Define variable to hold the PostHog client let posthogClient; @@ -251,7 +250,7 @@ const callRemoteURL = async function reportTelemetry() { } }; -function setupAnonUsageReportTimer(logger, hostInfo) { +export function setupAnonUsageReportTimer(logger, hostInfo) { try { // Setup PostHog client posthogClient = new PostHog('phc_5cmKiX9OubQjsSfOZuaolWaxo2z7WXqd295eB0uOtTb', { @@ -276,7 +275,3 @@ function setupAnonUsageReportTimer(logger, hostInfo) { logger.error(`TELEMETRY: ${err}`); } } - -module.exports = { - setupAnonUsageReportTimer, -}; diff --git a/src/lib/udp_handlers_log_events.js b/src/lib/udp_handlers_log_events.js index a68b9bed..b5b61f30 100644 --- a/src/lib/udp_handlers_log_events.js +++ b/src/lib/udp_handlers_log_events.js @@ -1,17 +1,16 @@ -/* eslint-disable prefer-destructuring */ /* eslint-disable no-unused-vars */ // Load global variables and functions -const globals = require('../globals'); -const postToInfluxdb = require('./post-to-influxdb'); -const postToNewRelic = require('./post-to-new-relic'); -const postToMQTT = require('./post-to-mqtt'); -const { categoriseLogEvent } = require('./log-event-categorise'); +import globals from '../globals.js'; +import { postLogEventToInfluxdb } from './post-to-influxdb.js'; +import { postLogEventToNewRelic } from './post-to-new-relic.js'; +import { postLogEventToMQTT } from './post-to-mqtt.js'; +import { categoriseLogEvent } from './log-event-categorise.js'; // -------------------------------------------------------- // Set up UDP server for acting on Sense log events // -------------------------------------------------------- -function udpInitLogEventServer() { +export function udpInitLogEventServer() { // Handler for UDP server startup event globals.udpServerLogEvents.socket.on('listening', (_message, _remote) => { const address = globals.udpServerLogEvents.socket.address(); @@ -294,7 +293,7 @@ function udpInitLogEventServer() { globals.config.get('Butler-SOS.logEvents.sendToMQTT.enable') ) { globals.logger.debug('LOG EVENT: Calling log event MQTT posting method'); - postToMQTT.postLogEventToMQTT(msgObj); + postLogEventToMQTT(msgObj); } // Post to Influxdb (if enabled) @@ -303,7 +302,7 @@ function udpInitLogEventServer() { globals.config.get('Butler-SOS.logEvents.sendToInfluxdb.enable') ) { globals.logger.debug('LOG EVENT: Calling log event Influxdb posting method'); - postToInfluxdb.postLogEventToInfluxdb(msgObj); + postLogEventToInfluxdb(msgObj); } // Post to New Relic (if enabled) @@ -312,7 +311,7 @@ function udpInitLogEventServer() { globals.config.get('Butler-SOS.logEvents.sendToNewRelic.enable') ) { globals.logger.debug('LOG EVENT: Calling log event New Relic posting method'); - postToNewRelic.postLogEventToNewRelic(msgObj); + postLogEventToNewRelic(msgObj); } } } catch (err) { @@ -320,7 +319,3 @@ function udpInitLogEventServer() { } }); } - -module.exports = { - udpInitLogEventServer, -}; diff --git a/src/lib/udp_handlers_user_activity.js b/src/lib/udp_handlers_user_activity.js index c95ecd04..2456ba31 100644 --- a/src/lib/udp_handlers_user_activity.js +++ b/src/lib/udp_handlers_user_activity.js @@ -1,17 +1,17 @@ /* eslint-disable no-unused-vars */ -const { validate } = require('uuid'); -const parser = require('ua-parser-js'); +import { validate } from 'uuid'; +import parser from 'ua-parser-js'; // Load global variables and functions -const globals = require('../globals'); -const postToInfluxdb = require('./post-to-influxdb'); -const postToNewRelic = require('./post-to-new-relic'); -const postToMQTT = require('./post-to-mqtt'); +import globals from '../globals.js'; +import { postUserEventToInfluxdb } from './post-to-influxdb.js'; +import { postUserEventToNewRelic } from './post-to-new-relic.js'; +import { postUserEventToMQTT } from './post-to-mqtt.js'; // -------------------------------------------------------- // Set up UDP server for acting on Sense user activity events // -------------------------------------------------------- -function udpInitUserActivityServer() { +export function udpInitUserActivityServer() { // Handler for UDP server startup event globals.udpServerUserActivity.socket.on('listening', (_message, _remote) => { const address = globals.udpServerUserActivity.socket.address(); @@ -161,7 +161,7 @@ function udpInitUserActivityServer() { globals.config.get('Butler-SOS.userEvents.sendToMQTT.enable') ) { globals.logger.debug('USER EVENT: Calling user sessions MQTT posting method'); - postToMQTT.postUserEventToMQTT(msgObj); + postUserEventToMQTT(msgObj); } // Post to Influxdb @@ -170,7 +170,7 @@ function udpInitUserActivityServer() { globals.config.get('Butler-SOS.userEvents.sendToInfluxdb.enable') ) { globals.logger.debug('USER EVENT: Calling user sessions Influxdb posting method'); - postToInfluxdb.postUserEventToInfluxdb(msgObj); + postUserEventToInfluxdb(msgObj); } // Post to New Relic @@ -179,14 +179,10 @@ function udpInitUserActivityServer() { globals.config.get('Butler-SOS.userEvents.sendToNewRelic.enable') ) { globals.logger.debug('USER EVENT: Calling user event New Relic posting method'); - postToNewRelic.postUserEventToNewRelic(msgObj); + postUserEventToNewRelic(msgObj); } } catch (err) { globals.logger.error(`USER EVENT: Error processing user activity event: ${err}`); } }); } - -module.exports = { - udpInitUserActivityServer, -};