diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3830feed..f0d73338 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -23,6 +23,8 @@ There are 3 folders in `src`, corresponding to the 3 processes that the VHS runs * `/network/validator/:publicKey`: Returns information about a specific validator. * `/network/validator/:publicKey/manifests`: Returns the manifests of a specific validator. * `/network/validator/:publicKey/reports`: Returns more detailed information about the reliability of a specific validator. + * `/network/amendments/info`: Returns general information about known amendments. + * `/network/amendments/info/:param`: Returns general information about a specific amendment by name or ID. ## SQL Table Schemas @@ -106,6 +108,30 @@ This table keeps track of the manifests of the validators. | `revoked` |Whether the manifest has been revoked. | | `seq` |The sequence number of this manifest. | +### `amendments_info` + +This table keeps track of the general information of all known amendments. + +| Key | Definition | +|----------------------|------------------------------------------------------------| +| `id` |The amendment id. | +| `name` |The name of the amendment. | +| `rippled_version` |The rippled version when the amendment is first enabled | +| `deprecated` |Whether the amendment has been deprecated/retired | + +### `ballot` + +This table keeps track of the most current voting data for the validators. + +| Key | Definition | +|----------------------|-------------------------------------------------------------------| +| `signing_key` |The signing key of the validator. | +| `ledger_index` |The most recent ledger index where voting data was retrieved. | +| `amendments` |The amendments this validator wants to be added to the protocol. | +| `base_fee` |The unscaled transaction cost this validator wants to set. | +| `reserve_base` |The minimum reserve requirement this validator wants to set. | +| `reserve_inc` |The increment in the reserve requirement this validator wants to set.| + ### `ballot` diff --git a/package-lock.json b/package-lock.json index 6eefcc62..98fe654b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@types/bunyan": "^1.8.7", "axios": "^0.21.1", "bunyan": "^1.8.15", + "create-hash": "^1.2.0", "dotenv": "^16.3.1", "express": "4.18.2", "knex": "2.5.1", @@ -28,6 +29,7 @@ }, "devDependencies": { "@types/axios": "^0.14.0", + "@types/create-hash": "^1.2.2", "@types/express": "4.17.20", "@types/jest": "^26.0.19", "@types/nconf": "^0.10.0", @@ -1418,6 +1420,15 @@ "@types/node": "*" } }, + "node_modules/@types/create-hash": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@types/create-hash/-/create-hash-1.2.2.tgz", + "integrity": "sha512-Fg8/kfMJObbETFU/Tn+Y0jieYewryLrbKwLCEIwPyklZZVY2qB+64KFjhplGSw+cseZosfFXctXO+PyIYD8iZQ==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/express": { "version": "4.17.20", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.20.tgz", @@ -12004,6 +12015,15 @@ "@types/node": "*" } }, + "@types/create-hash": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@types/create-hash/-/create-hash-1.2.2.tgz", + "integrity": "sha512-Fg8/kfMJObbETFU/Tn+Y0jieYewryLrbKwLCEIwPyklZZVY2qB+64KFjhplGSw+cseZosfFXctXO+PyIYD8iZQ==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/express": { "version": "4.17.20", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.20.tgz", diff --git a/package.json b/package.json index 892e56d9..8a1dd05d 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "homepage": "https://github.com/ripple/validator-history-service#readme", "devDependencies": { "@types/axios": "^0.14.0", + "@types/create-hash": "^1.2.2", "@types/express": "4.17.20", "@types/jest": "^26.0.19", "@types/nconf": "^0.10.0", @@ -61,6 +62,7 @@ "@types/bunyan": "^1.8.7", "axios": "^0.21.1", "bunyan": "^1.8.15", + "create-hash": "^1.2.0", "dotenv": "^16.3.1", "express": "4.18.2", "knex": "2.5.1", diff --git a/src/api/routes/v1/amendments.ts b/src/api/routes/v1/amendments.ts new file mode 100644 index 00000000..6335792e --- /dev/null +++ b/src/api/routes/v1/amendments.ts @@ -0,0 +1,123 @@ +import { Request, Response } from 'express' + +import { query } from '../../../shared/database' +import { AmendmentsInfo } from '../../../shared/types' +import { isEarlierVersion } from '../../../shared/utils' +import logger from '../../../shared/utils/logger' + +interface AmendmentsInfoResponse { + result: 'success' | 'error' + count: number + amendments: AmendmentsInfo[] +} + +interface SingleAmendmentInfoResponse { + result: 'success' | 'error' + amendment: AmendmentsInfo +} + +interface Cache { + amendments: AmendmentsInfo[] + time: number +} + +const log = logger({ name: 'api-amendments' }) + +const cache: Cache = { + amendments: [], + time: Date.now(), +} + +/** + * Updates amendments in cache. + * + * @returns Void. + */ +async function cacheAmendmentsInfo(): Promise { + try { + cache.amendments = await query('amendments_info').select('*') + cache.amendments.sort((prev: AmendmentsInfo, next: AmendmentsInfo) => { + if (isEarlierVersion(prev.rippled_version, next.rippled_version)) { + return 1 + } + return -1 + }) + cache.time = Date.now() + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- TODO: clean up + } catch (err: any) { + log.error(err.toString()) + } +} + +void cacheAmendmentsInfo() + +/** + * Handles Amendments Info request. + * + * @param _u - Unused express request. + * @param res - Express response. + */ +export async function handleAmendmentsInfo( + _u: Request, + res: Response, +): Promise { + try { + if (Date.now() - cache.time > 60 * 1000) { + await cacheAmendmentsInfo() + } + const amendments: AmendmentsInfo[] = cache.amendments + const response: AmendmentsInfoResponse = { + result: 'success', + count: amendments.length, + amendments, + } + res.send(response) + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- TODO: clean up + } catch (err: any) { + res.send({ result: 'error', message: 'internal error' }) + log.error(err.toString()) + } +} + +/** + * Handles Amendment Info request. + * + * @param req - Unused express request. + * @param res - Express response. + */ +export async function handleAmendmentInfo( + req: Request, + res: Response, +): Promise { + try { + const { param } = req.params + if (Date.now() - cache.time > 60 * 1000) { + await cacheAmendmentsInfo() + } + const amendments: AmendmentsInfo[] = cache.amendments.filter( + (amend) => amend.name === param || amend.id === param, + ) + if (amendments.length === 0) { + res.send({ result: 'error', message: "incorrect amendment's id/name" }) + return + } + if (amendments.length > 1) { + res.send({ + result: 'error', + message: + "there's a duplicate amendment's id/name on the server, please try again later", + }) + log.error("there's a duplicate amendment's id/name on the server", param) + return + } + const response: SingleAmendmentInfoResponse = { + result: 'success', + amendment: amendments[0], + } + res.send(response) + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- TODO: clean up + } catch (err: any) { + res.send({ result: 'error', message: 'internal error' }) + log.error(err.toString()) + } +} diff --git a/src/api/routes/v1/index.ts b/src/api/routes/v1/index.ts index 8f2b4773..4259c67d 100644 --- a/src/api/routes/v1/index.ts +++ b/src/api/routes/v1/index.ts @@ -1,5 +1,7 @@ +/* eslint-disable import/max-dependencies -- Disabled since this module requires multiple dependencies. */ import { Router as createRouter } from 'express' +import { handleAmendmentsInfo, handleAmendmentInfo } from './amendments' import handleDailyScores from './daily-report' import getNetworkOrAdd from './get-network' import handleHealth from './health' @@ -12,6 +14,9 @@ const api = createRouter() api.use('/health', handleHealth) api.use('/network/validator_reports', handleDailyScores) +api.use('/network/amendment/info/:param', handleAmendmentInfo) +api.use('/network/amendments/info', handleAmendmentsInfo) + api.use('/network/get_network/:entryUrl', getNetworkOrAdd) api.use('/network/topology/nodes/:network', handleNodes) diff --git a/src/api/routes/v1/info.ts b/src/api/routes/v1/info.ts index 49ef5a0c..998a818e 100644 --- a/src/api/routes/v1/info.ts +++ b/src/api/routes/v1/info.ts @@ -56,6 +56,16 @@ const info = { route: '/v1/network/validator_reports', example: 'https://data.xrpl.org/v1/network/validator_reports', }, + { + action: 'Get Amendments General Information', + route: '/v1/network/amendments/info', + example: 'https://data.xrpl.org/v1/network/amendments/info', + }, + { + action: 'Get Amendment General Information by Name or ID', + route: '/v1/network/amendment/info/{amendment}', + example: 'https://data.xrpl.org/v1/network/amendment/info/{amendment}', + }, ], } diff --git a/src/shared/database/amendments.ts b/src/shared/database/amendments.ts new file mode 100644 index 00000000..a3a4ec0a --- /dev/null +++ b/src/shared/database/amendments.ts @@ -0,0 +1,141 @@ +import axios from 'axios' +import createHash from 'create-hash' + +import { AmendmentsInfo } from '../types' +import logger from '../utils/logger' + +import { query } from './utils' + +const log = logger({ name: 'amendments' }) + +const amendmentIDs = new Map() +const rippledVersions = new Map() + +const ACTIVE_AMENDMENT_REGEX = + /^\s*REGISTER_F[A-Z]+\s*\((?\S+),\s*.*$/u +const RETIRED_AMENDMENT_REGEX = + /^ .*retireFeature\("(?\S+)"\)[,;].*$/u + +const AMENDMENT_VERSION_REGEX = + /\| \[(?[a-zA-Z0-9_]+)\][^\n]+\| (?v[0-9]*\.[0-9]*\.[0-9]*|TBD) *\|/u + +// TODO: Clean this up when this PR is merged: +// https://github.com/XRPLF/rippled/pull/4781 +/** + * Fetch a list of amendments names from rippled file. + * + * @returns The list of amendment names. + */ +async function fetchAmendmentNames(): Promise | null> { + try { + const response = await axios.get( + 'https://raw.githubusercontent.com/ripple/rippled/develop/src/ripple/protocol/impl/Feature.cpp', + ) + const text = response.data + const amendmentNames: Map = new Map() + text.split('\n').forEach((line: string) => { + const name = ACTIVE_AMENDMENT_REGEX.exec(line) + if (name) { + amendmentNames.set(name[1], name[0].includes('VoteBehavior::Obsolete')) + } else { + const name2 = RETIRED_AMENDMENT_REGEX.exec(line) + if (name2) { + amendmentNames.set(name2[1], true) + } + } + }) + return amendmentNames + } catch (err) { + log.error('Error getting amendment names', err) + return null + } +} + +/** + * Extracts Amendment ID from Amendment name inside a buffer. + * + * @param buffer -- The buffer containing the amendment name. + * + * @returns The amendment ID string. + */ +function sha512Half(buffer: Buffer): string { + return createHash('sha512') + .update(buffer) + .digest('hex') + .toUpperCase() + .slice(0, 64) +} + +/** + * Maps the id of Amendments to its corresponding names. + * + * @returns Void. + */ +async function nameOfAmendmentID(): Promise { + // The Amendment ID is the hash of the Amendment name + const amendmentNames = await fetchAmendmentNames() + if (amendmentNames !== null) { + amendmentNames.forEach((deprecated, name) => { + amendmentIDs.set(sha512Half(Buffer.from(name, 'ascii')), { + name, + deprecated, + }) + }) + } +} + +/** + * Fetches the versions when amendments are first enabled. + * + * @returns Void. + */ +async function fetchMinRippledVersions(): Promise { + try { + const response = await axios.get( + 'https://raw.githubusercontent.com/XRPLF/xrpl-dev-portal/master/content/resources/known-amendments.md', + ) + const text = response.data + + text.split('\n').forEach((line: string) => { + const found = AMENDMENT_VERSION_REGEX.exec(line) + if (found) { + rippledVersions.set( + found[1], + found[2].startsWith('v') ? found[2].slice(1) : found[2], + ) + } + }) + } catch (err) { + log.error('Error getting amendment rippled versions', err) + } +} + +/** + * Saves a validator to the database. + * + * @param amendment - The amendment to be saved. + * @returns Void. + */ +async function saveAmendmentInfo(amendment: AmendmentsInfo): Promise { + await query('amendments_info') + .insert(amendment) + .onConflict('id') + .merge() + .catch((err) => log.error('Error Saving AmendmentInfo', err)) +} + +export default async function fetchAmendmentInfo(): Promise { + log.info('Fetch amendments info from data sources...') + await nameOfAmendmentID() + await fetchMinRippledVersions() + amendmentIDs.forEach(async (value, id) => { + const amendment: AmendmentsInfo = { + id, + name: value.name, + rippled_version: rippledVersions.get(value.name), + deprecated: value.deprecated, + } + await saveAmendmentInfo(amendment) + }) + log.info('Finish fetching amendments info from data sources...') +} diff --git a/src/shared/database/setup.ts b/src/shared/database/setup.ts index f5cf404b..5bb82c33 100644 --- a/src/shared/database/setup.ts +++ b/src/shared/database/setup.ts @@ -1,5 +1,6 @@ import logger from '../utils/logger' +import fetchAmendmentInfo from './amendments' import networks from './networks' import { db, query } from './utils' @@ -19,7 +20,9 @@ export default async function setupTables(): Promise { await setupHourlyAgreementTable() await setupDailyAgreementTable() await setupNetworksTable() + await setupAmendmentsInfoTable() await setupBallotTable() + await fetchAmendmentInfo() } async function setupCrawlsTable(): Promise { @@ -204,6 +207,19 @@ async function setupNetworksTable(): Promise { } } +async function setupAmendmentsInfoTable(): Promise { + const hasAmendmentsInfo = await db().schema.hasTable('amendments_info') + if (!hasAmendmentsInfo) { + await db().schema.createTable('amendments_info', (table) => { + table.string('id') + table.string('name') + table.string('rippled_version') + table.boolean('deprecated') + table.primary(['id']) + }) + } +} + async function setupBallotTable(): Promise { const hasBallot = await db().schema.hasTable('ballot') if (!hasBallot) { diff --git a/src/shared/types/index.ts b/src/shared/types/index.ts index 1b4aa232..604bd823 100644 --- a/src/shared/types/index.ts +++ b/src/shared/types/index.ts @@ -194,6 +194,13 @@ interface DailyAgreement { agreement: AgreementScore } +interface AmendmentsInfo { + id: string + name: string + rippled_version?: string + deprecated: boolean +} + export { Node, Crawl, @@ -217,4 +224,5 @@ export { StreamLedger, Chain, ValidatorKeys, + AmendmentsInfo, } diff --git a/src/shared/utils/index.ts b/src/shared/utils/index.ts index d1d4c12f..c0ef8f60 100644 --- a/src/shared/utils/index.ts +++ b/src/shared/utils/index.ts @@ -155,3 +155,77 @@ export function overlaps( } return false } +/** + * Determines whether the source rippled version is not later than the target rippled version. + * Example usage: isNotLaterRippledVersion('1.10.0', '1.11.0') returns true. + * IsNotLaterRippledVersion('1.10.0', '1.10.0-b1') returns false. + * + * @param source -- The source rippled version. + * @param target -- The target rippled version. + * @returns True if source is earlier than target, false otherwise. + */ +// eslint-disable-next-line max-statements, max-lines-per-function, complexity -- Disabled for this util function. +export function isEarlierVersion( + source: string | null | undefined, + target: string | null | undefined, +): boolean { + if (source === target) { + return false + } + if (source === 'TBD' || !source) { + return false + } + if (target === 'TBD' || !target) { + return true + } + const sourceDecomp = source.split('.') + const targetDecomp = target.split('.') + const sourceMajor = parseInt(sourceDecomp[0], 10) + const sourceMinor = parseInt(sourceDecomp[1], 10) + const targetMajor = parseInt(targetDecomp[0], 10) + const targetMinor = parseInt(targetDecomp[1], 10) + // Compare major version + if (sourceMajor !== targetMajor) { + return sourceMajor < targetMajor + } + // Compare minor version + if (sourceMinor !== targetMinor) { + return sourceMinor < targetMinor + } + const sourcePatch = sourceDecomp[2].split('-') + const targetPatch = targetDecomp[2].split('-') + + const sourcePatchVersion = parseInt(sourcePatch[0], 10) + const targetPatchVersion = parseInt(targetPatch[0], 10) + + // Compare patch version + if (sourcePatchVersion !== targetPatchVersion) { + return sourcePatchVersion < targetPatchVersion + } + + // Compare release version + if (sourcePatch.length !== targetPatch.length) { + return sourcePatch.length > targetPatch.length + } + + if (sourcePatch.length === 2) { + // Compare different release types + if (!sourcePatch[1][0].startsWith(targetPatch[1][0])) { + return sourcePatch[1] < targetPatch[1] + } + // Compare beta version + if (sourcePatch[1].startsWith('b')) { + return ( + parseInt(sourcePatch[1].slice(1), 10) < + parseInt(targetPatch[1].slice(1), 10) + ) + } + // Compare rc version + return ( + parseInt(sourcePatch[1].slice(2), 10) < + parseInt(targetPatch[1].slice(2), 10) + ) + } + + return false +}