From 713b209e94a95a749defeda9127c97d12dcaaa50 Mon Sep 17 00:00:00 2001 From: canonbrother Date: Wed, 22 Nov 2023 23:40:49 +0800 Subject: [PATCH 01/11] add dump db --- .../database.spec/specifications.ts | 5 +++++ apps/whale-api/src/module.database/database.ts | 2 ++ .../provider.level/level.database.ts | 16 ++++++++++++++++ .../provider.memory/memory.database.spec.ts | 4 ++++ 4 files changed, 27 insertions(+) diff --git a/apps/whale-api/src/module.database/database.spec/specifications.ts b/apps/whale-api/src/module.database/database.spec/specifications.ts index d646380fcb..4c1dc898e5 100644 --- a/apps/whale-api/src/module.database/database.spec/specifications.ts +++ b/apps/whale-api/src/module.database/database.spec/specifications.ts @@ -33,6 +33,11 @@ export async function shouldGetById (database: Database): Promise { } } +export async function shouldDump (database: Database): Promise { + const data = await database.dump() + expect(data.length).toStrictEqual(40) +} + export async function shouldGetByPartitionKey (database: Database): Promise { const index = PartitionMapping.index for (const data of PARTITIONS) { diff --git a/apps/whale-api/src/module.database/database.ts b/apps/whale-api/src/module.database/database.ts index 57ef00384c..412c4e8ecc 100644 --- a/apps/whale-api/src/module.database/database.ts +++ b/apps/whale-api/src/module.database/database.ts @@ -38,6 +38,8 @@ export abstract class Database { mapping: ModelMapping, id: string ): Promise + + abstract dump (): Promise>> } export enum SortOrder { diff --git a/apps/whale-api/src/module.database/provider.level/level.database.ts b/apps/whale-api/src/module.database/provider.level/level.database.ts index 6e1da834d0..b9b4588ed2 100644 --- a/apps/whale-api/src/module.database/provider.level/level.database.ts +++ b/apps/whale-api/src/module.database/provider.level/level.database.ts @@ -24,6 +24,22 @@ export abstract class LevelUpDatabase extends Database { await this.root.close() } + async dump (): Promise>> { + const items: any[] = [] + return await new Promise((resolve, reject) => { + this.root.createReadStream({ keyAsBuffer: false, valueAsBuffer: false }) + .on('data', function (data) { + items.push(data) + }).on('error', function (err) { + reject(err) + }).on('close', function () { + reject(new Error('stream closed')) + }).on('end', function () { + resolve(items) + }) + }) + } + /** * Sub index space for model indexes. */ diff --git a/apps/whale-api/src/module.database/provider.memory/memory.database.spec.ts b/apps/whale-api/src/module.database/provider.memory/memory.database.spec.ts index 975e3f83bc..ac9cf473a6 100644 --- a/apps/whale-api/src/module.database/provider.memory/memory.database.spec.ts +++ b/apps/whale-api/src/module.database/provider.memory/memory.database.spec.ts @@ -39,6 +39,10 @@ it('should delete and be deleted', async () => { await spec.shouldDelete(database) }) +it('should dump', async () => { + await spec.shouldDump(database) +}) + it('should query by partition pagination', async () => { await spec.shouldQueryPartitionPagination(database) }) From 5310d222c4ec53990c5952998bbe866a39b04be8 Mon Sep 17 00:00:00 2001 From: canonbrother Date: Thu, 23 Nov 2023 10:55:32 +0800 Subject: [PATCH 02/11] add dumpdb api --- apps/whale-api/src/app.configuration.ts | 6 ++++-- apps/whale-api/src/module.api/_module.ts | 2 ++ .../src/module.api/debug.controller.ts | 20 +++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 apps/whale-api/src/module.api/debug.controller.ts diff --git a/apps/whale-api/src/app.configuration.ts b/apps/whale-api/src/app.configuration.ts index 92ef71d0f6..2832430650 100644 --- a/apps/whale-api/src/app.configuration.ts +++ b/apps/whale-api/src/app.configuration.ts @@ -23,7 +23,8 @@ export function AppConfiguration (): any { level: { location: process.env.WHALE_DATABASE_LEVEL_LOCATION } - } + }, + debug: process.env.WHALE_DEBUG } } @@ -34,6 +35,7 @@ export function ENV_VALIDATION_SCHEMA (): any { WHALE_NETWORK: Joi.string().valid('mainnet', 'testnet', 'regtest', 'devnet', 'changi').default('regtest'), WHALE_DEFID_URL: Joi.string().optional(), WHALE_DATABASE_PROVIDER: Joi.string().optional(), - WHALE_DATABASE_LEVEL_LOCATION: Joi.string().optional() + WHALE_DATABASE_LEVEL_LOCATION: Joi.string().optional(), + WHALE_DEBUG: Joi.bool().optional() }) } diff --git a/apps/whale-api/src/module.api/_module.ts b/apps/whale-api/src/module.api/_module.ts index 3c3589292b..ce68eb4a49 100644 --- a/apps/whale-api/src/module.api/_module.ts +++ b/apps/whale-api/src/module.api/_module.ts @@ -3,6 +3,7 @@ import { CacheModule, Global, Module } from '@nestjs/common' import { RpcController } from './rpc.controller' import { ActuatorController } from './actuator.controller' import { TransactionController } from './transaction.controller' +import { DebugController } from './debug.controller' import { ApiValidationPipe } from './pipes/api.validation.pipe' import { AddressController } from './address.controller' import { PoolPairController } from './poolpair.controller' @@ -62,6 +63,7 @@ import { GovernanceService } from './governance.service' LoanController, LegacyController, ConsortiumController, + DebugController, GovernanceController ], providers: [ diff --git a/apps/whale-api/src/module.api/debug.controller.ts b/apps/whale-api/src/module.api/debug.controller.ts new file mode 100644 index 0000000000..88983c863d --- /dev/null +++ b/apps/whale-api/src/module.api/debug.controller.ts @@ -0,0 +1,20 @@ +import { Controller, ForbiddenException, Get, Inject } from '@nestjs/common' +import { NetworkName } from '@defichain/jellyfish-network' +import { Database } from '../module.database/database' + +@Controller('/debug') +export class DebugController { + constructor ( + protected readonly database: Database, + @Inject('NETWORK') protected readonly network: NetworkName + ) { + } + + @Get('/dumpdb') + async dumpDb (): Promise>> { + if (process.env.WHALE_DEBUG === undefined) { + throw new ForbiddenException('Debug mode is not enabled') + } + return await this.database.dump() + } +} From 334b97073bcc24c2976245f193ad42933d3a0b1e Mon Sep 17 00:00:00 2001 From: canonbrother Date: Thu, 23 Nov 2023 15:42:21 +0800 Subject: [PATCH 03/11] fastcsv + dumpdb to csv --- apps/package.json | 5 +- .../database.spec/specifications.ts | 2 +- .../provider.level/level.database.ts | 13 +- package-lock.json | 257 ++++++++++++++++++ 4 files changed, 270 insertions(+), 7 deletions(-) diff --git a/apps/package.json b/apps/package.json index 5b6b43534e..b9b30010e5 100644 --- a/apps/package.json +++ b/apps/package.json @@ -43,16 +43,17 @@ "class-transformer": "0.5.1", "class-validator": "0.14.0", "cross-fetch": "3.1.5", + "fast-csv": "^4.3.6", "graphology": "0.25.1", "graphology-components": "1.5.4", "graphology-simple-path": "0.1.2", "joi": "17.8.4", - "lodash": "4.17.21", "level": "7.0.0", "level-js": "6.0.0", "level-packager": "6.0.0", "leveldown": "6.0.0", "lexicographic-integer-encoding": "1.0.1", + "lodash": "4.17.21", "memdown": "6.1.1", "pg": "8.9.0", "reflect-metadata": "0.1.13", @@ -67,9 +68,9 @@ "@types/cache-manager": "4.0.2", "@types/cron": "2.0.1", "@types/express": "4.17.17", - "@types/lodash": "4.14.194", "@types/level": "^6.0.0", "@types/levelup": "4.3.1", + "@types/lodash": "4.14.194", "@types/lossless-json": "1.0.1", "@types/memdown": "3.0.1", "@types/node": "16.11.58", diff --git a/apps/whale-api/src/module.database/database.spec/specifications.ts b/apps/whale-api/src/module.database/database.spec/specifications.ts index 4c1dc898e5..470fc77b1d 100644 --- a/apps/whale-api/src/module.database/database.spec/specifications.ts +++ b/apps/whale-api/src/module.database/database.spec/specifications.ts @@ -35,7 +35,7 @@ export async function shouldGetById (database: Database): Promise { export async function shouldDump (database: Database): Promise { const data = await database.dump() - expect(data.length).toStrictEqual(40) + expect((data as any).rowFormatter.rowCount).toStrictEqual(40) } export async function shouldGetByPartitionKey (database: Database): Promise { diff --git a/apps/whale-api/src/module.database/provider.level/level.database.ts b/apps/whale-api/src/module.database/provider.level/level.database.ts index b9b4588ed2..8f864ecb53 100644 --- a/apps/whale-api/src/module.database/provider.level/level.database.ts +++ b/apps/whale-api/src/module.database/provider.level/level.database.ts @@ -1,3 +1,5 @@ +import fs from 'fs' +import { format } from '@fast-csv/format' import sub from 'subleveldown' import level from 'level' import { LevelUp } from 'levelup' @@ -24,18 +26,21 @@ export abstract class LevelUpDatabase extends Database { await this.root.close() } - async dump (): Promise>> { - const items: any[] = [] + async dump (): Promise { + const writer = fs.createWriteStream('out.csv') + const csv = format() + csv.pipe(writer) + return await new Promise((resolve, reject) => { this.root.createReadStream({ keyAsBuffer: false, valueAsBuffer: false }) .on('data', function (data) { - items.push(data) + csv.write(data) }).on('error', function (err) { reject(err) }).on('close', function () { reject(new Error('stream closed')) }).on('end', function () { - resolve(items) + resolve(csv.end()) }) }) } diff --git a/package-lock.json b/package-lock.json index 2654cf33b3..8d1b966e86 100644 --- a/package-lock.json +++ b/package-lock.json @@ -68,6 +68,7 @@ "class-transformer": "0.5.1", "class-validator": "0.14.0", "cross-fetch": "3.1.5", + "fast-csv": "^4.3.6", "graphology": "0.25.1", "graphology-components": "1.5.4", "graphology-simple-path": "0.1.2", @@ -4052,6 +4053,43 @@ "@ethersproject/strings": "^5.7.0" } }, + "node_modules/@fast-csv/format": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz", + "integrity": "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A==", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.isboolean": "^3.0.3", + "lodash.isequal": "^4.5.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0" + } + }, + "node_modules/@fast-csv/format/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==" + }, + "node_modules/@fast-csv/parse": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-4.3.6.tgz", + "integrity": "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA==", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + } + }, + "node_modules/@fast-csv/parse/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==" + }, "node_modules/@fastify/ajv-compiler": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/@fastify/ajv-compiler/-/ajv-compiler-3.4.0.tgz", @@ -14707,6 +14745,18 @@ "resolved": "https://registry.npmjs.org/fast-content-type-parse/-/fast-content-type-parse-1.0.0.tgz", "integrity": "sha512-Xbc4XcysUXcsP5aHUU7Nq3OwvHq97C+WnbkeIefpeYLX+ryzFJlU6OStFJhs6Ol0LkUGpcK+wL0JwfM+FCU5IA==" }, + "node_modules/fast-csv": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-4.3.6.tgz", + "integrity": "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw==", + "dependencies": { + "@fast-csv/format": "4.3.5", + "@fast-csv/parse": "4.3.6" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/fast-decode-uri-component": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/fast-decode-uri-component/-/fast-decode-uri-component-1.0.1.tgz", @@ -19436,6 +19486,11 @@ "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", "integrity": "sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw=" }, + "node_modules/lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==" + }, "node_modules/lodash.flatten": { "version": "4.4.0", "license": "MIT" @@ -19445,17 +19500,47 @@ "resolved": "https://registry.npmjs.org/lodash.flow/-/lodash.flow-3.5.0.tgz", "integrity": "sha512-ff3BX/tSioo+XojX4MOsOMhJw0nZoUEF011LX8g8d3gvjVbxd89cCio4BCXronjxcTUIJUoqKEUA+n4CqvvRPw==" }, + "node_modules/lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==" + }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==" + }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==" + }, + "node_modules/lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==" + }, "node_modules/lodash.ismatch": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.ismatch/-/lodash.ismatch-4.4.0.tgz", "integrity": "sha512-fPMfXjGQEV9Xsq/8MTSgUf255gawYRbjwMyDbcvDhXgV7enSZA0hynz6vMPnpAb5iONEzBHBPsT+0zes5Z301g==", "dev": true }, + "node_modules/lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==" + }, "node_modules/lodash.isplainobject": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=" }, + "node_modules/lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==" + }, "node_modules/lodash.memoize": { "version": "4.1.2", "license": "MIT" @@ -30834,6 +30919,7 @@ "class-transformer": "0.5.1", "class-validator": "0.14.0", "cross-fetch": "3.1.5", + "fast-csv": "^4.3.6", "graphology": "0.25.1", "graphology-components": "1.5.4", "graphology-simple-path": "0.1.2", @@ -32207,6 +32293,47 @@ "@ethersproject/strings": "^5.7.0" } }, + "@fast-csv/format": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz", + "integrity": "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A==", + "requires": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.isboolean": "^3.0.3", + "lodash.isequal": "^4.5.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0" + }, + "dependencies": { + "@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==" + } + } + }, + "@fast-csv/parse": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-4.3.6.tgz", + "integrity": "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA==", + "requires": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + }, + "dependencies": { + "@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==" + } + } + }, "@fastify/ajv-compiler": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/@fastify/ajv-compiler/-/ajv-compiler-3.4.0.tgz", @@ -40496,6 +40623,7 @@ "class-transformer": "0.5.1", "class-validator": "0.14.0", "cross-fetch": "3.1.5", + "fast-csv": "^4.3.6", "graphology": "0.25.1", "graphology-components": "1.5.4", "graphology-simple-path": "0.1.2", @@ -41869,6 +41997,47 @@ "@ethersproject/strings": "^5.7.0" } }, + "@fast-csv/format": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz", + "integrity": "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A==", + "requires": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.isboolean": "^3.0.3", + "lodash.isequal": "^4.5.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0" + }, + "dependencies": { + "@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==" + } + } + }, + "@fast-csv/parse": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-4.3.6.tgz", + "integrity": "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA==", + "requires": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + }, + "dependencies": { + "@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==" + } + } + }, "@fastify/ajv-compiler": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/@fastify/ajv-compiler/-/ajv-compiler-3.4.0.tgz", @@ -49653,6 +49822,15 @@ "resolved": "https://registry.npmjs.org/fast-content-type-parse/-/fast-content-type-parse-1.0.0.tgz", "integrity": "sha512-Xbc4XcysUXcsP5aHUU7Nq3OwvHq97C+WnbkeIefpeYLX+ryzFJlU6OStFJhs6Ol0LkUGpcK+wL0JwfM+FCU5IA==" }, + "fast-csv": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-4.3.6.tgz", + "integrity": "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw==", + "requires": { + "@fast-csv/format": "4.3.5", + "@fast-csv/parse": "4.3.6" + } + }, "fast-decode-uri-component": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/fast-decode-uri-component/-/fast-decode-uri-component-1.0.1.tgz", @@ -52929,6 +53107,11 @@ "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", "integrity": "sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw=" }, + "lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==" + }, "lodash.flatten": { "version": "4.4.0" }, @@ -52937,17 +53120,47 @@ "resolved": "https://registry.npmjs.org/lodash.flow/-/lodash.flow-3.5.0.tgz", "integrity": "sha512-ff3BX/tSioo+XojX4MOsOMhJw0nZoUEF011LX8g8d3gvjVbxd89cCio4BCXronjxcTUIJUoqKEUA+n4CqvvRPw==" }, + "lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==" + }, + "lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==" + }, + "lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==" + }, + "lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==" + }, "lodash.ismatch": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.ismatch/-/lodash.ismatch-4.4.0.tgz", "integrity": "sha512-fPMfXjGQEV9Xsq/8MTSgUf255gawYRbjwMyDbcvDhXgV7enSZA0hynz6vMPnpAb5iONEzBHBPsT+0zes5Z301g==", "dev": true }, + "lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==" + }, "lodash.isplainobject": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=" }, + "lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==" + }, "lodash.memoize": { "version": "4.1.2" }, @@ -60483,6 +60696,15 @@ "resolved": "https://registry.npmjs.org/fast-content-type-parse/-/fast-content-type-parse-1.0.0.tgz", "integrity": "sha512-Xbc4XcysUXcsP5aHUU7Nq3OwvHq97C+WnbkeIefpeYLX+ryzFJlU6OStFJhs6Ol0LkUGpcK+wL0JwfM+FCU5IA==" }, + "fast-csv": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-4.3.6.tgz", + "integrity": "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw==", + "requires": { + "@fast-csv/format": "4.3.5", + "@fast-csv/parse": "4.3.6" + } + }, "fast-decode-uri-component": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/fast-decode-uri-component/-/fast-decode-uri-component-1.0.1.tgz", @@ -63759,6 +63981,11 @@ "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", "integrity": "sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw=" }, + "lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==" + }, "lodash.flatten": { "version": "4.4.0" }, @@ -63767,17 +63994,47 @@ "resolved": "https://registry.npmjs.org/lodash.flow/-/lodash.flow-3.5.0.tgz", "integrity": "sha512-ff3BX/tSioo+XojX4MOsOMhJw0nZoUEF011LX8g8d3gvjVbxd89cCio4BCXronjxcTUIJUoqKEUA+n4CqvvRPw==" }, + "lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==" + }, + "lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==" + }, + "lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==" + }, + "lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==" + }, "lodash.ismatch": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.ismatch/-/lodash.ismatch-4.4.0.tgz", "integrity": "sha512-fPMfXjGQEV9Xsq/8MTSgUf255gawYRbjwMyDbcvDhXgV7enSZA0hynz6vMPnpAb5iONEzBHBPsT+0zes5Z301g==", "dev": true }, + "lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==" + }, "lodash.isplainobject": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=" }, + "lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==" + }, "lodash.memoize": { "version": "4.1.2" }, From 9004ab27a7e05ab1d991b5da97a15fc2752716b4 Mon Sep 17 00:00:00 2001 From: canonbrother Date: Thu, 23 Nov 2023 16:33:04 +0800 Subject: [PATCH 04/11] dumpdb api return void --- apps/whale-api/src/module.api/debug.controller.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/whale-api/src/module.api/debug.controller.ts b/apps/whale-api/src/module.api/debug.controller.ts index 88983c863d..242ceabcbc 100644 --- a/apps/whale-api/src/module.api/debug.controller.ts +++ b/apps/whale-api/src/module.api/debug.controller.ts @@ -11,10 +11,10 @@ export class DebugController { } @Get('/dumpdb') - async dumpDb (): Promise>> { + async dumpDb (): Promise { if (process.env.WHALE_DEBUG === undefined) { throw new ForbiddenException('Debug mode is not enabled') } - return await this.database.dump() + await this.database.dump() } } From c5a07c9f68830d510c345c9747c9e14b2896108f Mon Sep 17 00:00:00 2001 From: canonbrother Date: Thu, 23 Nov 2023 22:13:10 +0800 Subject: [PATCH 05/11] dumpdb with drain check --- .../provider.level/level.database.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/apps/whale-api/src/module.database/provider.level/level.database.ts b/apps/whale-api/src/module.database/provider.level/level.database.ts index 8f864ecb53..5bbfab5f8a 100644 --- a/apps/whale-api/src/module.database/provider.level/level.database.ts +++ b/apps/whale-api/src/module.database/provider.level/level.database.ts @@ -32,15 +32,24 @@ export abstract class LevelUpDatabase extends Database { csv.pipe(writer) return await new Promise((resolve, reject) => { - this.root.createReadStream({ keyAsBuffer: false, valueAsBuffer: false }) + console.time('dumpdb done in: ') + this.root.createReadStream() // query stream .on('data', function (data) { - csv.write(data) + void (async () => { + const ok = csv.write(data) + if (!ok) { + await new Promise((resolve) => { + csv.once('drain', resolve) + }) + } + })() }).on('error', function (err) { reject(err) }).on('close', function () { reject(new Error('stream closed')) }).on('end', function () { resolve(csv.end()) + console.timeEnd('dumpdb done in: ') }) }) } From 36c9414bbf00b2ff6ac853c7f34c0516c1a7b9a4 Mon Sep 17 00:00:00 2001 From: canonbrother Date: Fri, 24 Nov 2023 15:12:56 +0800 Subject: [PATCH 06/11] split file on dump --- .../provider.level/level.database.ts | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/apps/whale-api/src/module.database/provider.level/level.database.ts b/apps/whale-api/src/module.database/provider.level/level.database.ts index 5bbfab5f8a..74e704cc90 100644 --- a/apps/whale-api/src/module.database/provider.level/level.database.ts +++ b/apps/whale-api/src/module.database/provider.level/level.database.ts @@ -1,4 +1,5 @@ import fs from 'fs' +import path from 'path' import { format } from '@fast-csv/format' import sub from 'subleveldown' import level from 'level' @@ -27,29 +28,35 @@ export abstract class LevelUpDatabase extends Database { } async dump (): Promise { - const writer = fs.createWriteStream('out.csv') - const csv = format() - csv.pipe(writer) - + let id = 0 + const maxSize = 500_000 + const dir = path.join(process.cwd(), 'dump') + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir) + } + let items: string[] = [] return await new Promise((resolve, reject) => { - console.time('dumpdb done in: ') + console.time('dump done in: ') this.root.createReadStream() // query stream .on('data', function (data) { - void (async () => { - const ok = csv.write(data) - if (!ok) { - await new Promise((resolve) => { - csv.once('drain', resolve) - }) - } - })() + items.push(data) + if (items.length >= maxSize) { + const writer = fs.createWriteStream(`${dir}/dump-${id}.csv`) + const csv = format() + csv.pipe(writer) + items.forEach(i => csv.write(i)) + csv.end() + + items = [] + id += 1 + } }).on('error', function (err) { reject(err) }).on('close', function () { reject(new Error('stream closed')) }).on('end', function () { - resolve(csv.end()) - console.timeEnd('dumpdb done in: ') + resolve(true) + console.timeEnd('dump done in: ') }) }) } From f645d4fa3f49c574d92c636c4d6a7dedb4f85b9c Mon Sep 17 00:00:00 2001 From: canonbrother Date: Fri, 24 Nov 2023 15:15:26 +0800 Subject: [PATCH 07/11] git ignore dump --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index a28feb453f..330531166e 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,10 @@ dist # debug log lerna-debug.log +# debug dump +dump + + coverage # vscode From 85527905e95fbe4c52e5e38bc943e2f7dada6e55 Mon Sep 17 00:00:00 2001 From: canonbrother Date: Fri, 24 Nov 2023 15:23:25 +0800 Subject: [PATCH 08/11] update test --- .../src/module.database/database.spec/specifications.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/whale-api/src/module.database/database.spec/specifications.ts b/apps/whale-api/src/module.database/database.spec/specifications.ts index 470fc77b1d..776d484516 100644 --- a/apps/whale-api/src/module.database/database.spec/specifications.ts +++ b/apps/whale-api/src/module.database/database.spec/specifications.ts @@ -34,8 +34,8 @@ export async function shouldGetById (database: Database): Promise { } export async function shouldDump (database: Database): Promise { - const data = await database.dump() - expect((data as any).rowFormatter.rowCount).toStrictEqual(40) + const dump = await database.dump() + expect(dump).toBeTruthy() } export async function shouldGetByPartitionKey (database: Database): Promise { From 8e7a861083d160542eae0066cecf2971b1a69915 Mon Sep 17 00:00:00 2001 From: canonbrother Date: Fri, 24 Nov 2023 15:24:51 +0800 Subject: [PATCH 09/11] rm br --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 330531166e..b0fdf16783 100644 --- a/.gitignore +++ b/.gitignore @@ -36,7 +36,6 @@ lerna-debug.log # debug dump dump - coverage # vscode From bb0e6f9f0c23eb8e7d19759766461c78ad377600 Mon Sep 17 00:00:00 2001 From: canonbrother Date: Fri, 24 Nov 2023 15:27:08 +0800 Subject: [PATCH 10/11] refine type --- apps/whale-api/src/module.database/database.ts | 2 +- .../src/module.database/provider.level/level.database.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/whale-api/src/module.database/database.ts b/apps/whale-api/src/module.database/database.ts index 412c4e8ecc..345a595b0c 100644 --- a/apps/whale-api/src/module.database/database.ts +++ b/apps/whale-api/src/module.database/database.ts @@ -39,7 +39,7 @@ export abstract class Database { id: string ): Promise - abstract dump (): Promise>> + abstract dump (): Promise } export enum SortOrder { diff --git a/apps/whale-api/src/module.database/provider.level/level.database.ts b/apps/whale-api/src/module.database/provider.level/level.database.ts index 74e704cc90..5885c0f2bd 100644 --- a/apps/whale-api/src/module.database/provider.level/level.database.ts +++ b/apps/whale-api/src/module.database/provider.level/level.database.ts @@ -27,7 +27,7 @@ export abstract class LevelUpDatabase extends Database { await this.root.close() } - async dump (): Promise { + async dump (): Promise { let id = 0 const maxSize = 500_000 const dir = path.join(process.cwd(), 'dump') From 7bf7598908835e026293829b80cc9ccd7eb6898e Mon Sep 17 00:00:00 2001 From: canonbrother Date: Fri, 24 Nov 2023 15:44:42 +0800 Subject: [PATCH 11/11] rm network di --- apps/whale-api/src/module.api/debug.controller.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/whale-api/src/module.api/debug.controller.ts b/apps/whale-api/src/module.api/debug.controller.ts index 242ceabcbc..924624ce04 100644 --- a/apps/whale-api/src/module.api/debug.controller.ts +++ b/apps/whale-api/src/module.api/debug.controller.ts @@ -1,12 +1,10 @@ -import { Controller, ForbiddenException, Get, Inject } from '@nestjs/common' -import { NetworkName } from '@defichain/jellyfish-network' +import { Controller, ForbiddenException, Get } from '@nestjs/common' import { Database } from '../module.database/database' @Controller('/debug') export class DebugController { constructor ( - protected readonly database: Database, - @Inject('NETWORK') protected readonly network: NetworkName + protected readonly database: Database ) { }