init

.gitignore

@@ -0,0 +1,2 @@
+node_modules
+package-lock.json

README.md

@@ -0,0 +1,45 @@
+<div align="center">
+    <img width="460" height="300" src="./media/bullmq-monitor.png">
+</div>
+
+Service that acts as a central component to monitor BullMQ:
+
+- Exposes a BullMQ dashboard which is per default behind a login page
+- Acts as a [Prometheus-Exporter](https://prometheus.io/docs/instrumenting/exporters/) to collect metrics about queues in BullMQ
+
+## Implementation
+
+The following section will provide a brief overview of the libraries and practices used in the implementation of this service.
+
+
+### BullMQ Dashboard
+
+Implemented by using [@bull-board](https://github.com/felixmosh/bull-board) and secured using [passport](https://www.passportjs.org/).
+
+
+### Prometheus Exporter
+
+Strongly influenced by [bull_exporter](https://github.com/UpHabit/bull_exporter). Which uses the old bull library.
+
+Implemented by using the [bullmq](https://docs.bullmq.io/) library (specifically the [QueueEvents](https://docs.bullmq.io/guide/events) class) and [prom-client](https://github.com/siimon/prom-client).
+
+For each queue a class extending the QueueEvents class is created. This class listens for the following events: `completed`. Whenever an eventListener is triggered, a [histogram](https://prometheus.io/docs/concepts/metric_types/#histogram) is updated with
+
+1. the duration between the start of the processing and the end of the job
+2. the duration between the creation of the job and the end of its processing.
+
+Furthermore, a cron job is executed every n seconds which collects the current status of the queues (`completed`, `active`, `delayed`, `failed`, `waiting` jobs) and writes them to a [gauge](https://prometheus.io/docs/concepts/metric_types/#gauge).
+
+Thus, the following metrics are collected:
+
+| Metric                    | type      | description |
+|---------------------------|-----------|-------------|
+| bullmq_processed_duration | histogram | Processing time for completed jobs                      |
+| bullmq_completed_duration | histogram | Completion time for jobs                                |
+| bullmq_completed          | gauge     | Total number of completed jobs                          |
+| bullmq_active             | gauge     | Total number of active jobs (currently being processed) |
+| bullmq_delayed            | gauge     | Total number of jobs that will run in the future        |
+| bullmq_failed             | gauge     | Total number of failed jobs                             |
+| bullmq_waiting            | gauge     | Total number of jobs waiting to be processed            |
+
+Each metric also has the attribute `queue` which indicated which queue the metric is associated with.

configs/config-local.json

@@ -0,0 +1,22 @@
+{
+  "redis": {
+    "host": "localhost:49153/",
+    "username": "default",
+    "password": "redispw",
+    "ssl": false
+  },
+  "cookieSecret": "myCookieSecret123!",
+  "cookieMaxAge": "1h",
+  "users": [
+    {
+      "username": "admin",
+      "password": "password",
+      "role": "admin"
+    },
+    {
+      "username": "user",
+      "password": "password",
+      "role": "user"
+    }
+  ]
+}

docker/Dockerfile

@@ -0,0 +1,59 @@
+ARG VERSION
+ARG COMMIT
+ARG NPM_REGISTRY
+ARG BASE_REPO
+ARG NODE_VERSION
+ARG ALPINE_VERSION
+
+# STAGE 1: Build app
+FROM $BASE_REPO/node:$NODE_VERSION-alpine$ALPINE_VERSION as builder
+
+ENV VERSION=${VERSION}
+ENV COMMIT=${COMMIT}
+
+ARG NPM_REGISTRY
+ARG UID=510
+ARG GID=510
+
+RUN apk update && \
+    apk upgrade
+
+RUN mkdir -p /usr/src/app/node_modules && \
+    addgroup -g ${GID} app && \
+    adduser -h /usr/src/app -G app -u ${UID} -D app && \
+    chown -R ${UID}:${GID} /usr/src/app
+
+WORKDIR /usr/src/app
+COPY --chown=${UID}:${GID} package*.json .npmrc ./ 
+USER app
+COPY --chown=${UID}:${GID} . .
+RUN npm install --registry=${NPM_REGISTRY}
+RUN npm run build
+
+# STAGE 2: Run app
+FROM $BASE_REPO/node:$NODE_VERSION-alpine$ALPINE_VERSION
+
+ENV VERSION=${VERSION}
+ENV COMMIT=${COMMIT}
+
+ARG NPM_REGISTRY
+ARG UID=510
+ARG GID=510
+
+RUN apk update && \
+    apk upgrade
+
+RUN mkdir -p /usr/src/app/node_modules && \
+    addgroup -g ${GID} app && \
+    adduser -h /usr/src/app -G app -u ${UID} -D app && \
+    chown -R ${UID}:${GID} /usr/src/app
+
+WORKDIR /usr/src/app
+COPY  --chown=${UID}:${GID} package*.json .npmrc ./
+USER app
+RUN npm ci --only=production --registry=${NPM_REGISTRY}
+COPY --chown=${UID}:${GID} --from=builder /usr/src/app/dist ./dist
+
+# EXPOSE 8080
+CMD [ "node", "dist/server.js" ]
+

package.json

@@ -0,0 +1,58 @@
+{
+  "name": "bullmq-prometheus",
+  "version": "1.0.0",
+  "description": "Service that can be used to monitor BullMQ by providing Prometheus metrics and a Bullmq dashboard secured behind a login wall.",
+  "main": "src/server.ts",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 0",
+    "lint": "eslint . --ext ts",
+    "prettier": "prettier -w src/**/*.ts",
+    "build": "tsc",
+    "run": "node dist/server.js",
+    "ts-node": "ts-node src/server.ts",
+    "nodemon": "nodemon",
+    "prepare": "node -e \"if (process.env.NODE_ENV !== 'production'){process.exit(1)} \" || husky install"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@bull-board/api": "^4.2.2",
+    "@bull-board/express": "^4.2.2",
+    "bullmq": "^1.87.1",
+    "connect-ensure-login": "^0.1.1",
+    "ejs": "^3.1.8",
+    "express": "^4.18.1",
+    "express-session": "^1.17.3",
+    "ioredis": "^5.2.2",
+    "parse-duration": "^1.0.2",
+    "passport": "^0.6.0",
+    "passport-local": "^1.0.0",
+    "pino": "^8.5.0",
+    "pino-http": "^8.2.0",
+    "prom-client": "^14.0.1"
+  },
+  "devDependencies": {
+    "@types/connect-ensure-login": "^0.1.7",
+    "@types/ejs": "^3.1.1",
+    "@types/express": "^4.17.13",
+    "@types/express-session": "^1.17.5",
+    "@types/passport": "^1.0.9",
+    "@types/passport-local": "^1.0.34",
+    "@typescript-eslint/eslint-plugin": "^5.33.0",
+    "@typescript-eslint/parser": "^5.33.0",
+    "eslint": "^8.21.0",
+    "husky": "^8.0.1",
+    "nodemon": "^2.0.19",
+    "prettier": "^2.7.1",
+    "ts-node": "^10.9.1",
+    "typescript": "^4.7.4"
+  },
+  "nodemonConfig": {
+    "watch": [
+      "src"
+    ],
+    "ext": "ts",
+    "exec": "npx ts-node src/server.ts"
+  }
+}

src/app.ts

@@ -0,0 +1,71 @@
+import express from 'express';
+import Redis from 'ioredis';
+import config from './config';
+import { ConfigureRoutes as ConfigureDashboardRoutes, User } from './controllers/dashboard';
+import { ConfigureRoutes as ConfigureMetricsRoute } from './controllers/metrics';
+import logger from './logger';
+import { PrometheusMetricsCollector } from './monitor/promMetricsCollector';
+import { formatConnectionString, handleFutureShutdown } from './utils';
+
+const log = logger.child({ pkg: "app" })
+export const app = express();
+app.disable('x-powered-by');
+
+const pino = require('pino-http')()
+app.use(pino)
+
+app.get('/health', async (req, res) => {
+  res.status(200).send('OK');
+});
+
+const username = config.redis.username
+const password = config.redis.password
+const host = config.redis.host
+
+if (username === undefined || password === undefined || host === undefined) {
+  process.exit(125);
+}
+
+const enableSsl = config.redis.ssl
+const prefix = process.env.NODE_ENV?.toLowerCase() || 'local';
+const cookieSecret = config.cookieSecret
+const cookieMaxAge = config.cookieMaxAge
+const defaultUsers: Array<User> = [
+  { username: 'admin', password: 'secret', role: 'admin' },
+  { username: 'user', password: 'secret', role: 'user' },
+];
+
+const users = config.users || defaultUsers
+
+const redisConnString = formatConnectionString(host, username, password, enableSsl);
+
+export const metricsCollector = new PrometheusMetricsCollector('monitor', {
+  bullmqOpts: {
+    prefix: prefix,
+  },
+  client: new Redis(redisConnString, { maxRetriesPerRequest: null }),
+  queues: [],
+});
+
+handleFutureShutdown(metricsCollector);
+
+const dashboardRouter = express.Router();
+app.use('/bullmq', dashboardRouter);
+
+metricsCollector
+  .discoverAllQueues()
+  .then((queues) => {
+    log.info(`Discovered ${queues.length} queues`);
+    ConfigureDashboardRoutes(dashboardRouter, {
+      basePath: '/bullmq',
+      queues: metricsCollector.monitoredQueues.map((q) => q.queue),
+      cookieSecret: cookieSecret,
+      cookieMaxAge: cookieMaxAge,
+      users: users,
+    });
+    ConfigureMetricsRoute(app, metricsCollector);
+  })
+  .catch((err) => {
+    console.error(err);
+    process.exit(125);
+  });

src/config.ts

@@ -0,0 +1,19 @@
+import { readFileSync } from 'fs'
+
+export interface Config {
+  redis: {
+    host: string,
+    username: string,
+    password: string,
+    ssl: boolean
+  },
+  cookieSecret: string,
+  cookieMaxAge: string,
+  users?: Array<any>
+}
+
+const prefix = process.env.NODE_ENV?.toLowerCase() || 'local';
+const jsonRaw = readFileSync(`./configs/config-${prefix}.json`)
+const config = JSON.parse(jsonRaw.toString())
+
+export default config as Config;