From 9f204d9c18e29c0c91ee733464491d514a7d8ba0 Mon Sep 17 00:00:00 2001 From: cornholio <0@mcornholio.ru> Date: Wed, 19 Jun 2024 11:20:02 +0200 Subject: [PATCH] Prometheus metrics --- .../otv-backend/templates/servicemonitor.yaml | 14 ++ charts/otv-backend/templates/statefulset.yaml | 2 +- charts/otv-backend/values.yaml | 6 + package.json | 1 + packages/common/src/ApiHandler/ApiHandler.ts | 8 +- .../src/ApiHandler/__mocks__/ApiHandler.ts | 4 +- packages/common/src/__mocks__/metrics.ts | 9 + packages/common/src/config.ts | 1 + packages/common/src/db/index.ts | 3 + packages/common/src/index.ts | 2 + packages/common/src/metrics.ts | 166 ++++++++++++++++++ packages/common/src/nominator/nominator.ts | 2 + .../common/src/scorekeeper/jobs/JobsClass.ts | 1 + .../src/scorekeeper/jobs/cron/SetupCronJob.ts | 4 + .../jobs/specificJobs/BlockDataJob.ts | 2 + .../test/ApiHandler/ApiHandler.int.test.ts | 2 +- .../test/chaindata/chaindata.int.test.ts | 4 +- .../test/nominator/nominator.unit.test.ts | 2 +- .../scorekeeper/NumNominations.unit.test.ts | 4 +- packages/common/test/testUtils/chaindata.ts | 4 +- packages/common/test/testUtils/dbUtils.ts | 4 +- packages/common/test/testUtils/scorekeeper.ts | 4 +- packages/core/src/index.ts | 14 +- packages/gateway/src/routes/setupRoutes.ts | 16 +- packages/telemetry/src/Telemetry/Telemetry.ts | 13 +- packages/telemetry/test/utils.ts | 14 +- yarn.lock | 34 ++++ 27 files changed, 319 insertions(+), 21 deletions(-) create mode 100644 charts/otv-backend/templates/servicemonitor.yaml create mode 100644 packages/common/src/__mocks__/metrics.ts create mode 100644 packages/common/src/metrics.ts diff --git a/charts/otv-backend/templates/servicemonitor.yaml b/charts/otv-backend/templates/servicemonitor.yaml new file mode 100644 index 000000000..b3a88a58c --- /dev/null +++ b/charts/otv-backend/templates/servicemonitor.yaml @@ -0,0 +1,14 @@ +{{ if and .Values.serviceMonitor.enabled ( ne .Values.environment "ci" ) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ .Release.Name }} + labels: +{{ toYaml .Values.serviceMonitor.labels | indent 4 }} +spec: + selector: + matchLabels: + app: {{ .Release.Name }} + endpoints: + - port: backend +{{ end }} diff --git a/charts/otv-backend/templates/statefulset.yaml b/charts/otv-backend/templates/statefulset.yaml index 57a06577e..2c94c4965 100644 --- a/charts/otv-backend/templates/statefulset.yaml +++ b/charts/otv-backend/templates/statefulset.yaml @@ -19,7 +19,7 @@ spec: labels: app: {{ .Release.Name }} annotations: - checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} spec: containers: diff --git a/charts/otv-backend/values.yaml b/charts/otv-backend/values.yaml index bcda562b3..edc45e643 100644 --- a/charts/otv-backend/values.yaml +++ b/charts/otv-backend/values.yaml @@ -32,6 +32,12 @@ storageSize: 20Gi # memory: 400Mi resources: {} +serviceMonitor: + enabled: true + labels: + group: w3f + release: prometheus-operator + secret: | { "matrix": { diff --git a/package.json b/package.json index d8afa8b59..2dc5bb67e 100644 --- a/package.json +++ b/package.json @@ -103,6 +103,7 @@ "mongodb-memory-server": "^9.1.7", "mongoose": "^8.2.1", "prettier": "^3.2.4", + "prom-client": "^15.1.2", "semver": "^7.6.0", "swagger-jsdoc": "^6.2.8", "swagger2": "^4.0.3", diff --git a/packages/common/src/ApiHandler/ApiHandler.ts b/packages/common/src/ApiHandler/ApiHandler.ts index ac1142f22..4ed95f0de 100644 --- a/packages/common/src/ApiHandler/ApiHandler.ts +++ b/packages/common/src/ApiHandler/ApiHandler.ts @@ -3,6 +3,7 @@ import EventEmitter from "eventemitter3"; import logger from "../logger"; import { API_PROVIDER_TIMEOUT, POLKADOT_API_TIMEOUT } from "../constants"; +import { setChainRpcConnectivity } from "../metrics"; export const apiLabel = { label: "ApiHandler" }; @@ -20,9 +21,12 @@ class ApiHandler extends EventEmitter { public upSince = -1; public isConnected = false; + public chain: string; - constructor(endpoints: string[]) { + // chain is a name for type of chain: relay, people. Used for Prometheus metrics. + constructor(chain: string, endpoints: string[]) { super(); + this.chain = chain; this.endpoints = endpoints.sort(() => Math.random() - 0.5); } @@ -47,6 +51,7 @@ class ApiHandler extends EventEmitter { } this.isConnected = false; + setChainRpcConnectivity(this.chain, false); this.connectionAttempt = new Promise(async (resolve) => { try { await this.wsProvider.connect(); @@ -63,6 +68,7 @@ class ApiHandler extends EventEmitter { this.connectionAttempt = null; this.upSince = Date.now(); this.isConnected = true; + setChainRpcConnectivity(this.chain, true); logger.info(`Connected to ${this.currentEndpoint()}`, apiLabel); resolve(); } catch (err) { diff --git a/packages/common/src/ApiHandler/__mocks__/ApiHandler.ts b/packages/common/src/ApiHandler/__mocks__/ApiHandler.ts index a411b7636..107d09ae0 100644 --- a/packages/common/src/ApiHandler/__mocks__/ApiHandler.ts +++ b/packages/common/src/ApiHandler/__mocks__/ApiHandler.ts @@ -19,9 +19,11 @@ const createMockApiPromise = (): any => ({ class ApiHandlerMock extends EventEmitter { private endpoints: string[]; private api: ApiPromise = createMockApiPromise as unknown as ApiPromise; + public chain: string; - constructor(endpoints: string[]) { + constructor(chain: string, endpoints: string[]) { super(); + this.chain = chain; // Initialize with mock data or behavior as needed this.endpoints = endpoints.sort(() => Math.random() - 0.5); this.api = createMockApiPromise(); diff --git a/packages/common/src/__mocks__/metrics.ts b/packages/common/src/__mocks__/metrics.ts new file mode 100644 index 000000000..4a27088d6 --- /dev/null +++ b/packages/common/src/__mocks__/metrics.ts @@ -0,0 +1,9 @@ +import { vi } from "vitest"; + +export const setupMetrics = vi.fn(); +export const registerBlockScan = vi.fn(); +export const registerNomination = vi.fn(); +export const setDbConnectivity = vi.fn(); +export const setTelemetryConnectivity = vi.fn(); +export const setChainRpcConnectivity = vi.fn(); +export const renderMetrics = vi.fn(() => Promise.resolve("")); diff --git a/packages/common/src/config.ts b/packages/common/src/config.ts index b34c2e813..7aa8a6f2d 100644 --- a/packages/common/src/config.ts +++ b/packages/common/src/config.ts @@ -92,6 +92,7 @@ export type ConfigSchema = { kusamaBootstrapEndpoint: string; polkadotBootstrapEndpoint: string; candidatesUrl: string; + prometheusPrefix?: string; }; matrix: { accessToken: string; diff --git a/packages/common/src/db/index.ts b/packages/common/src/db/index.ts index 743b65ca2..81f9a3e18 100644 --- a/packages/common/src/db/index.ts +++ b/packages/common/src/db/index.ts @@ -3,6 +3,7 @@ import mongoose from "mongoose"; import logger from "../logger"; import * as queries from "./queries"; +import { setDbConnectivity } from "../metrics"; // [name, client, version, null, networkId] export type NodeDetails = [ @@ -37,6 +38,8 @@ export class Db { resolve(true); }); + mongoose.connection.on("open", () => setDbConnectivity(true)); + mongoose.connection.on("disconnected", () => setDbConnectivity(false)); mongoose.connection.on("error", (err) => { logger.error(`MongoDB connection issue: ${err}`, dbLabel); reject(err); diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts index fb7678ebb..18e1d3df9 100644 --- a/packages/common/src/index.ts +++ b/packages/common/src/index.ts @@ -14,6 +14,7 @@ import * as Models from "./db/models"; import ScoreKeeper from "./scorekeeper/scorekeeper"; import * as Jobs from "./scorekeeper/jobs/specificJobs"; import MatrixBot from "./matrix"; +import * as metrics from "./metrics"; export { ApiHandler, @@ -31,4 +32,5 @@ export { ScoreKeeper, Jobs, MatrixBot, + metrics, }; diff --git a/packages/common/src/metrics.ts b/packages/common/src/metrics.ts new file mode 100644 index 000000000..119a2185e --- /dev/null +++ b/packages/common/src/metrics.ts @@ -0,0 +1,166 @@ +import promClient from "prom-client"; + +import { ConfigSchema } from "./config"; +import { jobStatusEmitter } from "./Events"; +import { JobStatus } from "./scorekeeper/jobs/JobsClass"; +import logger from "./logger"; + +export const metricsLabel = { label: "Metrics" }; + +export type Metrics = { + counters: { + nominations: promClient.Counter; + blocksScanned: promClient.Counter; + jobRuns: promClient.Counter; + }; + gauges: { + dbConnectivity: promClient.Gauge; + telemetryConnectivity: promClient.Gauge; + chainRpcConnectivity: promClient.Gauge; + jobProgress: promClient.Gauge; + latestBlock: promClient.Gauge; + }; + histograms: { + jobExecutionTime: promClient.Histogram; + }; +}; + +let metrics: Metrics | null = null; + +export function setupMetrics(config: ConfigSchema): void { + const prefix = config.global.prometheusPrefix ?? "otv_backend_"; + metrics = { + counters: { + blocksScanned: new promClient.Counter({ + name: `${prefix}blocks_scanned_total`, + help: "amount of blocks scanned", + }), + jobRuns: new promClient.Counter({ + name: `${prefix}jobs_runs_total`, + help: "job events by name and status", + labelNames: ["name", "status"], + }), + nominations: new promClient.Counter({ + name: `${prefix}nominations_total`, + help: "nominations count by account", + labelNames: ["account"], + }), + }, + gauges: { + dbConnectivity: new promClient.Gauge({ + name: `${prefix}db_connectivity`, + help: "database connection status; 0 or 1", + }), + telemetryConnectivity: new promClient.Gauge({ + name: `${prefix}telemetry_connectivity`, + help: "telemetry connection status; 0 or 1", + }), + chainRpcConnectivity: new promClient.Gauge({ + name: `${prefix}chain_rpc_connectivity`, + help: "chain RPC connection status by chain (relay, people); 0 or 1", + labelNames: ["chain"], + }), + jobProgress: new promClient.Gauge({ + name: `${prefix}job_progress`, + help: "job progress by name, in percents from 0 to 100", + labelNames: ["name"], + }), + latestBlock: new promClient.Gauge({ + name: `${prefix}latest_block`, + help: "latest scanned block number", + }), + }, + histograms: { + jobExecutionTime: new promClient.Histogram({ + name: `${prefix}job_execution_time_seconds`, + help: "job timings; only successful jobs are counted", + labelNames: ["name"], + buckets: growingBuckets(0, 10, 30), + }), + }, + }; + + promClient.collectDefaultMetrics({ prefix }); + followJobs(metrics); +} + +function getMetrics(): Metrics { + if (metrics === null) { + throw new Error("getMetrics() was called before setupMetrics()"); + } + + return metrics; +} + +export function registerBlockScan(blockNumber: number): void { + const metrics = getMetrics(); + metrics.counters.blocksScanned.inc(); + metrics.gauges.latestBlock.set(blockNumber); +} + +export function registerNomination(account: string): void { + getMetrics().counters.nominations.inc({ account }); +} + +export function setDbConnectivity(isConnected: boolean): void { + getMetrics().gauges.dbConnectivity.set(isConnected ? 1 : 0); +} + +export function setTelemetryConnectivity(isConnected: boolean): void { + getMetrics().gauges.telemetryConnectivity.set(isConnected ? 1 : 0); +} + +export function setChainRpcConnectivity( + chain: string, + isConnected: boolean, +): void { + getMetrics().gauges.chainRpcConnectivity.set({ chain }, isConnected ? 1 : 0); +} + +function followJobs(metrics: Metrics): void { + const updateJob = (data: JobStatus) => { + metrics.counters.jobRuns.inc({ name: data.name, status: data.status }); + }; + jobStatusEmitter.on("jobStarted", updateJob); + jobStatusEmitter.on("jobRunning", updateJob); + jobStatusEmitter.on("jobErrored", updateJob); + jobStatusEmitter.on("jobFinished", (data: JobStatus) => { + updateJob(data); + if (!data.executedAt) { + logger.error( + `${data.name}: jobFinished event was emitted, but jobStatus.executedAt is empty`, + metricsLabel, + ); + return; + } + const duration = Math.ceil((Date.now() - data.executedAt) / 1000); + metrics.histograms.jobExecutionTime.observe({ name: data.name }, duration); + }); + + jobStatusEmitter.on("jobProgress", (data: JobStatus) => { + metrics.gauges.jobProgress.set({ name: data.name }, data.progress); + }); +} + +// prom-client has two types of bucket generation: linear, which would create +// too much buckets for our case, and exponential, which would either create too +// little of buckets, or buckets with floating points. +// This creates buckets like [10, 30, 60, 100, 150, 210, 280, 360, 450, ...] +function growingBuckets( + start: number, + factor: number, + count: number, +): number[] { + const buckets = []; + + let currentBucket = start; + for (let i = 0; i < count; i++) { + currentBucket += factor * i; + buckets.push(currentBucket); + } + return buckets; +} + +export async function renderMetrics(): Promise { + return promClient.register.metrics(); +} diff --git a/packages/common/src/nominator/nominator.ts b/packages/common/src/nominator/nominator.ts index 2407154c2..9340a0281 100644 --- a/packages/common/src/nominator/nominator.ts +++ b/packages/common/src/nominator/nominator.ts @@ -8,6 +8,7 @@ import EventEmitter from "eventemitter3"; import { sendProxyDelayTx, sendProxyTx } from "./NominatorTx"; import { getNominatorChainInfo } from "./NominatorChainInfo"; import { NominatorState, NominatorStatus } from "../types"; +import { registerNomination } from "../metrics"; export const nominatorLabel = { label: "Nominator" }; @@ -354,6 +355,7 @@ export default class Nominator extends EventEmitter { tx = api.tx.staking.nominate(targets); await this.sendStakingTx(tx, targets); } + registerNomination(await this.stash()); await queries.setLastNominatedEraIndex(currentEra); return true; } catch (e) { diff --git a/packages/common/src/scorekeeper/jobs/JobsClass.ts b/packages/common/src/scorekeeper/jobs/JobsClass.ts index b3b944fa0..a1f17885b 100644 --- a/packages/common/src/scorekeeper/jobs/JobsClass.ts +++ b/packages/common/src/scorekeeper/jobs/JobsClass.ts @@ -30,6 +30,7 @@ export type JobConfig = { export interface JobStatus { name: string; updated: number; + executedAt?: number; enabled?: boolean; runCount?: number; status: string; diff --git a/packages/common/src/scorekeeper/jobs/cron/SetupCronJob.ts b/packages/common/src/scorekeeper/jobs/cron/SetupCronJob.ts index 3ca4d564d..c9b8d07eb 100644 --- a/packages/common/src/scorekeeper/jobs/cron/SetupCronJob.ts +++ b/packages/common/src/scorekeeper/jobs/cron/SetupCronJob.ts @@ -39,12 +39,14 @@ export const setupCronJob = async ( } isRunning = true; + const executedAt = Date.now(); logger.info(`Executing ${name}.`, loggerLabel); const runningStatus: JobStatus = { status: "running", name: name, runCount: jobRunCount, updated: Date.now(), + executedAt, }; jobStatusEmitter.emit("jobRunning", runningStatus); @@ -58,6 +60,7 @@ export const setupCronJob = async ( runCount: jobRunCount, updated: Date.now(), error: JSON.stringify(e), + executedAt, }; jobStatusEmitter.emit("jobErrored", errorStatus); @@ -69,6 +72,7 @@ export const setupCronJob = async ( name: name, runCount: jobRunCount, updated: Date.now(), + executedAt, }; jobStatusEmitter.emit("jobFinished", finishedStatus); } diff --git a/packages/common/src/scorekeeper/jobs/specificJobs/BlockDataJob.ts b/packages/common/src/scorekeeper/jobs/specificJobs/BlockDataJob.ts index 3d8536eab..e02498022 100644 --- a/packages/common/src/scorekeeper/jobs/specificJobs/BlockDataJob.ts +++ b/packages/common/src/scorekeeper/jobs/specificJobs/BlockDataJob.ts @@ -8,6 +8,7 @@ import { Block, EventRecord, Phase } from "@polkadot/types/interfaces"; import type { FrameSystemEventRecord } from "@polkadot/types/lookup"; import { Exposure } from "../../../chaindata/queries/ValidatorPref"; import { JobNames } from "../JobConfigs"; +import { registerBlockScan } from "../../../metrics"; export const blockdataLabel = { label: "Block" }; @@ -200,6 +201,7 @@ export const processBlock = async ( await queries.setBlockIndex(blockNumber, blockIndex?.latest); } const end = Date.now(); + registerBlockScan(blockNumber); logger.info( `Done processing block #${blockNumber} (${(end - start) / 1000}s)`, blockdataLabel, diff --git a/packages/common/test/ApiHandler/ApiHandler.int.test.ts b/packages/common/test/ApiHandler/ApiHandler.int.test.ts index e3af62b7e..0f1961549 100644 --- a/packages/common/test/ApiHandler/ApiHandler.int.test.ts +++ b/packages/common/test/ApiHandler/ApiHandler.int.test.ts @@ -7,7 +7,7 @@ describe("ApiHandler Integration Tests", () => { let handler: ApiHandler; beforeAll(async () => { - handler = new ApiHandler(KusamaEndpoints); + handler = new ApiHandler("relay", KusamaEndpoints); }, TIMEOUT_DURATION); it( diff --git a/packages/common/test/chaindata/chaindata.int.test.ts b/packages/common/test/chaindata/chaindata.int.test.ts index 6c3ac12e1..d36c61cff 100644 --- a/packages/common/test/chaindata/chaindata.int.test.ts +++ b/packages/common/test/chaindata/chaindata.int.test.ts @@ -11,8 +11,8 @@ describe("ChainData Integration Tests", () => { beforeAll(async () => { const apiHandlers: ApiHandlers = { - relay: new ApiHandler(KusamaEndpoints), - people: new ApiHandler(KusamaPeopleEndpoints), + relay: new ApiHandler("relay", KusamaEndpoints), + people: new ApiHandler("people", KusamaPeopleEndpoints), }; chainData = new ChainData(apiHandlers); }, TIMEOUT_DURATION); diff --git a/packages/common/test/nominator/nominator.unit.test.ts b/packages/common/test/nominator/nominator.unit.test.ts index 4101c5591..1c6c68ebb 100644 --- a/packages/common/test/nominator/nominator.unit.test.ts +++ b/packages/common/test/nominator/nominator.unit.test.ts @@ -14,7 +14,7 @@ describe("Nominator Mock Class Unit Tests", () => { const signerAddress = "DvDsrjvaJpXNW7XLvtFtEB3D9nnBKMqzvrijFffwpe7CCc6"; beforeAll(async () => { - handler = new ApiHandler(["Constants.KusamaEndpoints"]); + handler = new ApiHandler("relay", ["Constants.KusamaEndpoints"]); await handler.getApi(); nominatorConfig = { diff --git a/packages/common/test/scorekeeper/NumNominations.unit.test.ts b/packages/common/test/scorekeeper/NumNominations.unit.test.ts index 9acca94e1..d931e0122 100644 --- a/packages/common/test/scorekeeper/NumNominations.unit.test.ts +++ b/packages/common/test/scorekeeper/NumNominations.unit.test.ts @@ -72,8 +72,8 @@ describe("autoNumNominations", () => { }, }); - const relayApiHandler = new ApiHandler(KusamaEndpoints); - const peopleApiHandler = new ApiHandler(KusamaPeopleEndpoints); + const relayApiHandler = new ApiHandler("relay", KusamaEndpoints); + const peopleApiHandler = new ApiHandler("people", KusamaPeopleEndpoints); chaindata = new ChainData({ relay: relayApiHandler, diff --git a/packages/common/test/testUtils/chaindata.ts b/packages/common/test/testUtils/chaindata.ts index 7f045c1fc..703bb64da 100644 --- a/packages/common/test/testUtils/chaindata.ts +++ b/packages/common/test/testUtils/chaindata.ts @@ -3,8 +3,8 @@ import ApiHandler from "../../src/ApiHandler/ApiHandler"; import { KusamaEndpoints, KusamaPeopleEndpoints } from "../../src/constants"; export async function getKusamaChainData(): Promise { - const relayApiHandler = new ApiHandler(KusamaEndpoints); - const peopleApiHandler = new ApiHandler(KusamaPeopleEndpoints); + const relayApiHandler = new ApiHandler("relay", KusamaEndpoints); + const peopleApiHandler = new ApiHandler("people", KusamaPeopleEndpoints); await Promise.all([relayApiHandler.getApi(), peopleApiHandler.getApi()]); return new ChainData({ diff --git a/packages/common/test/testUtils/dbUtils.ts b/packages/common/test/testUtils/dbUtils.ts index e23ab82ef..b3a3fb26f 100644 --- a/packages/common/test/testUtils/dbUtils.ts +++ b/packages/common/test/testUtils/dbUtils.ts @@ -2,7 +2,7 @@ import { MongoMemoryServer } from "mongodb-memory-server"; import { Db } from "../../src"; import mongoose from "mongoose"; -import { afterAll, afterEach, beforeAll, beforeEach } from "vitest"; +import { afterAll, afterEach, beforeAll, beforeEach, vi } from "vitest"; import logger from "../../src/logger"; import { deleteAllDb } from "./deleteAll"; @@ -13,6 +13,8 @@ interface ObjectWithId { type ObjectOrArray = T | T[]; +vi.mock("../../src/metrics"); + export const omitId = >( obj: ObjectOrArray, ): ObjectOrArray> => { diff --git a/packages/common/test/testUtils/scorekeeper.ts b/packages/common/test/testUtils/scorekeeper.ts index 0fa3d1b01..c0eb4e886 100644 --- a/packages/common/test/testUtils/scorekeeper.ts +++ b/packages/common/test/testUtils/scorekeeper.ts @@ -5,8 +5,8 @@ import ApiHandler from "../../src/ApiHandler/ApiHandler"; import { KusamaEndpoints, KusamaPeopleEndpoints } from "../../src/constants"; export const getAndStartScorekeeper = async () => { - const relayApiHandler = new ApiHandler(KusamaEndpoints); - const peopleApiHandler = new ApiHandler(KusamaPeopleEndpoints); + const relayApiHandler = new ApiHandler("relay", KusamaEndpoints); + const peopleApiHandler = new ApiHandler("people", KusamaPeopleEndpoints); await Promise.all([relayApiHandler.getApi(), peopleApiHandler.getApi()]); const chaindata = new ChainData({ diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index d307bb558..6221e6592 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -16,6 +16,7 @@ import { import { Server } from "@1kv/gateway"; import { TelemetryClient } from "@1kv/telemetry"; import { ConfigSchema } from "@1kv/common/build/config"; +import { registerBlockScan, setupMetrics } from "@1kv/common/build/metrics"; const isCI = process.env.CI; @@ -42,10 +43,10 @@ export const createAPIHandlers = async ( ? config.global.apiEndpoints : Constants.LocalEndpoints; - const relayHandler = new ApiHandler(endpoints); + const relayHandler = new ApiHandler("relay", endpoints); const peopleHandler = config.global.apiPeopleEndpoints - ? new ApiHandler(config.global.apiPeopleEndpoints) + ? new ApiHandler("people", config.global.apiPeopleEndpoints) : relayHandler; return { relay: relayHandler, people: peopleHandler }; @@ -209,9 +210,17 @@ export const initScorekeeper = async ( } }; +// If this gauge won't set, latest block metric would be zero +async function setInitialLatestBlock() { + const index = await queries.getBlockIndex(); + registerBlockScan(index.latest); +} + const start = async (cmd: { config: string }) => { try { const config = await Config.loadConfigDir(cmd.config); + setupMetrics(config); + const winstonLabel = { label: "start" }; logger.info(`Starting the backend services. ${version}`, winstonLabel); @@ -221,6 +230,7 @@ const start = async (cmd: { config: string }) => { // Create the Database. await createDB(config); + await setInitialLatestBlock(); // Set the chain metadata await setChainMetadata(config); diff --git a/packages/gateway/src/routes/setupRoutes.ts b/packages/gateway/src/routes/setupRoutes.ts index e29c63a92..aa6fae669 100644 --- a/packages/gateway/src/routes/setupRoutes.ts +++ b/packages/gateway/src/routes/setupRoutes.ts @@ -1,10 +1,10 @@ import Router from "@koa/router"; -import { ApiHandler, Config, logger, ScoreKeeper } from "@1kv/common"; -import { response } from "../controllers"; +import { ApiHandler, Config, logger, ScoreKeeper, metrics } from "@1kv/common"; import path from "path"; import mount from "koa-mount"; import yamljs from "yamljs"; import { koaSwagger } from "koa2-swagger-ui"; +import { response } from "../controllers"; import Koa from "koa"; @@ -56,6 +56,16 @@ export const setupHealthCheckRoute = ( } }; +export const setupMetricsRoute = async ( + routerInstance: Router, +): Promise => { + routerInstance.get("/metrics", async (ctx) => { + ctx.status = 200; + ctx.body = await metrics.renderMetrics(); + ctx.type = "text/plain"; + }); +}; + export const setupScorekeeperRoutes = ( router: Router, app: Koa, @@ -180,6 +190,7 @@ export const setupRoutes = async ( // Set up the health check route on the healthRouter setupHealthCheckRoute(healthRouter, handler); + setupMetricsRoute(healthRouter); setupScorekeeperRoutes(healthRouter, app, scorekeeper); app.use(healthRouter.routes()); @@ -187,6 +198,7 @@ export const setupRoutes = async ( logger.info(`Setting up all routes`, { label: "Gateway" }); setupCache(app, cache); setupHealthCheckRoute(router, handler); + setupMetricsRoute(router); setupScorekeeperRoutes(router, app, scorekeeper); setupDocs(app, config); diff --git a/packages/telemetry/src/Telemetry/Telemetry.ts b/packages/telemetry/src/Telemetry/Telemetry.ts index 85eb3bb84..f233575fe 100644 --- a/packages/telemetry/src/Telemetry/Telemetry.ts +++ b/packages/telemetry/src/Telemetry/Telemetry.ts @@ -1,6 +1,6 @@ import WebSocket from "ws"; -import { Config, Constants, logger, queries, Util } from "@1kv/common"; +import { Config, Constants, logger, queries, Util, metrics } from "@1kv/common"; import { registerTelemetryWs } from "./TelemetryWS"; export default class TelemetryClient { @@ -21,7 +21,16 @@ export default class TelemetryClient { private _memNodes = {}; - public isConnected = false; + private _isConnected = false; + get isConnected(): boolean { + return this._isConnected; + } + + set isConnected(isConnected: boolean) { + this._isConnected = isConnected; + metrics.setTelemetryConnectivity(isConnected); + } + constructor(config: Config.ConfigSchema) { this.config = config; this._host = diff --git a/packages/telemetry/test/utils.ts b/packages/telemetry/test/utils.ts index 7071e8b72..ede8228de 100644 --- a/packages/telemetry/test/utils.ts +++ b/packages/telemetry/test/utils.ts @@ -1,11 +1,22 @@ import { MongoMemoryServer } from "mongodb-memory-server"; -import { Db, logger } from "@1kv/common"; +import { Config, Db, logger, metrics } from "@1kv/common"; import { afterAll, afterEach, beforeAll, beforeEach } from "vitest"; import mongoose from "mongoose"; +import path from "path"; +import fs from "fs"; let mongoServer: MongoMemoryServer | null = null; let mongoUri: string | null = null; +function getConfig(): Config.ConfigSchema { + const jsonPath = path.resolve( + __dirname, + "../../../packages/core/config/kusama.current.sample.json", + ); + const jsonData = fs.readFileSync(jsonPath, "utf-8"); + return JSON.parse(jsonData); +} + export const createTestServer = async () => { const isCI = process.env.CI === "true"; @@ -18,6 +29,7 @@ export const createTestServer = async () => { mongoUri = mongoServer.getUri(); } + await metrics.setupMetrics(getConfig()); logger.info("Connecting to MongoDB at URI:", mongoUri); await Db.create(mongoUri); logger.info("Connected to MongoDB"); diff --git a/yarn.lock b/yarn.lock index b14e1fe89..34973e4a7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -55,6 +55,7 @@ __metadata: nodemon: ^3.1.0 open-cli: ^8.0.0 prettier: ^3.2.4 + prom-client: ^15.1.2 semver: ^7.6.0 swagger-jsdoc: ^6.2.8 swagger2: ^4.0.3 @@ -1321,6 +1322,13 @@ __metadata: languageName: node linkType: hard +"@opentelemetry/api@npm:^1.4.0": + version: 1.9.0 + resolution: "@opentelemetry/api@npm:1.9.0" + checksum: 9e88e59d53ced668f3daaecfd721071c5b85a67dd386f1c6f051d1be54375d850016c881f656ffbe9a03bedae85f7e89c2f2b635313f9c9b195ad033cdc31020 + languageName: node + linkType: hard + "@pkgjs/parseargs@npm:^0.11.0": version: 0.11.0 resolution: "@pkgjs/parseargs@npm:0.11.0" @@ -3394,6 +3402,13 @@ __metadata: languageName: node linkType: hard +"bintrees@npm:1.0.2": + version: 1.0.2 + resolution: "bintrees@npm:1.0.2" + checksum: 56a52b7d3634e30002b1eda740d2517a22fa8e9e2eb088e919f37c030a0ed86e364ab59e472fc770fc8751308054bb1c892979d150e11d9e11ac33bcc1b5d16e + languageName: node + linkType: hard + "bn.js@npm:^5.2.1": version: 5.2.1 resolution: "bn.js@npm:5.2.1" @@ -7670,6 +7685,16 @@ __metadata: languageName: node linkType: hard +"prom-client@npm:^15.1.2": + version: 15.1.2 + resolution: "prom-client@npm:15.1.2" + dependencies: + "@opentelemetry/api": ^1.4.0 + tdigest: ^0.1.1 + checksum: b9b2f439588a462c0aec840e8aa857bb0a77284174d6587ca042eb13ea6ac36ba13277f45ae6ed3696b3007a1020c5ee2c5ee46b23be033a7bb45207a5365c21 + languageName: node + linkType: hard + "promise-retry@npm:^2.0.1": version: 2.0.1 resolution: "promise-retry@npm:2.0.1" @@ -8812,6 +8837,15 @@ __metadata: languageName: node linkType: hard +"tdigest@npm:^0.1.1": + version: 0.1.2 + resolution: "tdigest@npm:0.1.2" + dependencies: + bintrees: 1.0.2 + checksum: 44de8246752b6f8c2924685f969fd3d94c36949f22b0907e99bef2b2220726dd8467f4730ea96b06040b9aa2587c0866049640039d1b956952dfa962bc2075a3 + languageName: node + linkType: hard + "temp-dir@npm:^3.0.0": version: 3.0.0 resolution: "temp-dir@npm:3.0.0"