From 7c444cbb70a109eb6d70f9c4e1e8650ab137ecf8 Mon Sep 17 00:00:00 2001 From: Gabriel Rocheleau Date: Wed, 26 Apr 2023 16:58:20 -0400 Subject: [PATCH 01/27] blockchain: remove level dependency from blockchain --- packages/blockchain/package.json | 1 - packages/blockchain/test/util.ts | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/blockchain/package.json b/packages/blockchain/package.json index a92383cca3..6717f5c704 100644 --- a/packages/blockchain/package.json +++ b/packages/blockchain/package.json @@ -48,7 +48,6 @@ "abstract-level": "^1.0.3", "debug": "^4.3.3", "ethereum-cryptography": "^1.1.2", - "level": "^8.0.0", "lru-cache": "^5.1.1", "memory-level": "^1.0.0" }, diff --git a/packages/blockchain/test/util.ts b/packages/blockchain/test/util.ts index 10d175c068..448255be46 100644 --- a/packages/blockchain/test/util.ts +++ b/packages/blockchain/test/util.ts @@ -8,7 +8,7 @@ import { MemoryLevel } from 'memory-level' import { Blockchain } from '../src' -import type { Level } from 'level' +import type { AbstractLevel } from 'abstract-level' export const generateBlocks = (numberOfBlocks: number, existingBlocks?: Block[]): Block[] => { const blocks = existingBlocks ? existingBlocks : [] @@ -115,7 +115,9 @@ export const isConsecutive = (blocks: Block[]) => { }) } -export const createTestDB = async (): Promise<[Level, Block]> => { +export const createTestDB = async (): Promise< + [AbstractLevel, Block] +> => { const common = new Common({ chain: Chain.Mainnet, hardfork: Hardfork.Chainstart }) const genesis = Block.fromBlockData({ header: { number: 0 } }, { common }) const db = new MemoryLevel() From ec3a913075e59ea6ecf081a666e6ae946342f0b7 Mon Sep 17 00:00:00 2001 From: Gabriel Rocheleau Date: Thu, 27 Apr 2023 10:43:58 -0400 Subject: [PATCH 02/27] util: refactor DB types from trie to util --- packages/client/lib/execution/level.ts | 2 +- .../lib/sync/fetcher/bytecodefetcher.ts | 3 +- packages/trie/src/db/checkpoint.ts | 4 +- packages/trie/src/db/map.ts | 3 +- packages/trie/src/trie/trie.ts | 13 +++-- packages/trie/src/types.ts | 48 +------------------ packages/trie/test/db/checkpoint.spec.ts | 4 +- packages/trie/test/db/db.spec.ts | 4 +- packages/trie/test/proof/range.spec.ts | 3 +- packages/trie/test/stream.spec.ts | 3 +- packages/trie/test/trie/checkpoint.spec.ts | 4 +- packages/util/src/db.ts | 46 ++++++++++++++++++ packages/util/src/index.ts | 5 ++ 13 files changed, 71 insertions(+), 71 deletions(-) create mode 100644 packages/util/src/db.ts diff --git a/packages/client/lib/execution/level.ts b/packages/client/lib/execution/level.ts index 78f88a5ad6..c574ed6d65 100644 --- a/packages/client/lib/execution/level.ts +++ b/packages/client/lib/execution/level.ts @@ -1,6 +1,6 @@ import { MemoryLevel } from 'memory-level' -import type { BatchDBOp, DB } from '@ethereumjs/trie' +import type { BatchDBOp, DB } from '@ethereumjs/util' import type { AbstractLevel } from 'abstract-level' export const ENCODING_OPTS = { keyEncoding: 'view', valueEncoding: 'view' } diff --git a/packages/client/lib/sync/fetcher/bytecodefetcher.ts b/packages/client/lib/sync/fetcher/bytecodefetcher.ts index cf7dc907ed..c79bfb75d6 100644 --- a/packages/client/lib/sync/fetcher/bytecodefetcher.ts +++ b/packages/client/lib/sync/fetcher/bytecodefetcher.ts @@ -1,6 +1,6 @@ import { CODEHASH_PREFIX } from '@ethereumjs/statemanager' import { Trie } from '@ethereumjs/trie' -import { bytesToHex, concatBytes, equalsBytes } from '@ethereumjs/util' +import { BatchDBOp, bytesToHex, concatBytes, equalsBytes } from '@ethereumjs/util' import { debug as createDebugLogger } from 'debug' import { keccak256 } from 'ethereum-cryptography/keccak' @@ -9,7 +9,6 @@ import { Fetcher } from './fetcher' import type { Peer } from '../../net/peer' import type { FetcherOptions } from './fetcher' import type { Job } from './types' -import type { BatchDBOp } from '@ethereumjs/trie' import type { Debugger } from 'debug' type ByteCodeDataResponse = Uint8Array[] & { completed?: boolean } diff --git a/packages/trie/src/db/checkpoint.ts b/packages/trie/src/db/checkpoint.ts index 6d033a5aae..9991d93cf2 100644 --- a/packages/trie/src/db/checkpoint.ts +++ b/packages/trie/src/db/checkpoint.ts @@ -1,6 +1,6 @@ -import { bytesToHex, hexStringToBytes } from '@ethereumjs/util' +import { BatchDBOp, DB, bytesToHex, hexStringToBytes } from '@ethereumjs/util' -import type { BatchDBOp, Checkpoint, CheckpointDBOpts, DB } from '../types' +import type { Checkpoint, CheckpointDBOpts } from '../types' import type LRUCache from 'lru-cache' const LRU = require('lru-cache') diff --git a/packages/trie/src/db/map.ts b/packages/trie/src/db/map.ts index 1a45b2879b..0f2dd2a23a 100644 --- a/packages/trie/src/db/map.ts +++ b/packages/trie/src/db/map.ts @@ -1,7 +1,6 @@ +import { DB, BatchDBOp } from '@ethereumjs/util' import { bytesToHex } from 'ethereum-cryptography/utils' -import type { BatchDBOp, DB } from '../types' - export class MapDB implements DB { _database: Map diff --git a/packages/trie/src/trie/trie.ts b/packages/trie/src/trie/trie.ts index 6c01279958..717a7d85f7 100644 --- a/packages/trie/src/trie/trie.ts +++ b/packages/trie/src/trie/trie.ts @@ -1,4 +1,12 @@ -import { RLP_EMPTY_STRING, bytesToHex, bytesToUtf8, equalsBytes } from '@ethereumjs/util' +import { + BatchDBOp, + DB, + PutBatch, + RLP_EMPTY_STRING, + bytesToHex, + bytesToUtf8, + equalsBytes, +} from '@ethereumjs/util' import { keccak256 } from 'ethereum-cryptography/keccak' import { CheckpointDB, MapDB } from '../db' @@ -12,13 +20,10 @@ import { WalkController } from '../util/walkController' import { BranchNode, ExtensionNode, LeafNode, decodeNode, decodeRawNode, isRawNode } from './node' import type { - BatchDBOp, - DB, EmbeddedNode, FoundNodeFunction, Nibbles, Proof, - PutBatch, TrieNode, TrieOpts, TrieOptsWithDefaults, diff --git a/packages/trie/src/types.ts b/packages/trie/src/types.ts index d267e847e6..c5897eacd6 100644 --- a/packages/trie/src/types.ts +++ b/packages/trie/src/types.ts @@ -2,6 +2,7 @@ import { utf8ToBytes } from 'ethereum-cryptography/utils' import type { BranchNode, ExtensionNode, LeafNode } from './trie' import type { WalkController } from './util/walkController' +import { DB } from '@ethereumjs/util' export type TrieNode = BranchNode | ExtensionNode | LeafNode @@ -91,53 +92,6 @@ export interface CheckpointDBOpts { cacheSize?: number } -export type BatchDBOp = PutBatch | DelBatch - -export interface PutBatch { - type: 'put' - key: Uint8Array - value: Uint8Array -} - -export interface DelBatch { - type: 'del' - key: Uint8Array -} - -export interface DB { - /** - * Retrieves a raw value from leveldb. - * @param key - * @returns A Promise that resolves to `Uint8Array` if a value is found or `null` if no value is found. - */ - get(key: Uint8Array): Promise - - /** - * Writes a value directly to leveldb. - * @param key The key as a `Uint8Array` - * @param value The value to be stored - */ - put(key: Uint8Array, val: Uint8Array): Promise - - /** - * Removes a raw value in the underlying leveldb. - * @param keys - */ - del(key: Uint8Array): Promise - - /** - * Performs a batch operation on db. - * @param opStack A stack of levelup operations - */ - batch(opStack: BatchDBOp[]): Promise - - /** - * Returns a copy of the DB instance, with a reference - * to the **same** underlying leveldb instance. - */ - copy(): DB -} - export type Checkpoint = { // We cannot use a Uint8Array => Uint8Array map directly. If you create two Uint8Arrays with the same internal value, // then when setting a value on the Map, it actually creates two indices. diff --git a/packages/trie/test/db/checkpoint.spec.ts b/packages/trie/test/db/checkpoint.spec.ts index 02d4129939..486a939921 100644 --- a/packages/trie/test/db/checkpoint.spec.ts +++ b/packages/trie/test/db/checkpoint.spec.ts @@ -1,10 +1,8 @@ -import { hexStringToBytes, utf8ToBytes } from '@ethereumjs/util' +import { BatchDBOp, hexStringToBytes, utf8ToBytes } from '@ethereumjs/util' import * as tape from 'tape' import { CheckpointDB, MapDB } from '../../src' -import type { BatchDBOp } from '../../src' - tape('DB tests', (t) => { const k = utf8ToBytes('k1') const v = utf8ToBytes('v1') diff --git a/packages/trie/test/db/db.spec.ts b/packages/trie/test/db/db.spec.ts index 0f3f1969e5..a4a2d2fdea 100644 --- a/packages/trie/test/db/db.spec.ts +++ b/packages/trie/test/db/db.spec.ts @@ -1,10 +1,8 @@ -import { equalsBytes, utf8ToBytes } from '@ethereumjs/util' +import { BatchDBOp, equalsBytes, utf8ToBytes } from '@ethereumjs/util' import * as tape from 'tape' import { MapDB } from '../../src' -import type { BatchDBOp } from '../../src' - tape('DB tests', (t) => { const db = new MapDB() diff --git a/packages/trie/test/proof/range.spec.ts b/packages/trie/test/proof/range.spec.ts index f1c2118f63..d11cf1b46c 100644 --- a/packages/trie/test/proof/range.spec.ts +++ b/packages/trie/test/proof/range.spec.ts @@ -1,4 +1,5 @@ import { + DB, compareBytes, concatBytes, hexStringToBytes, @@ -10,8 +11,6 @@ import * as tape from 'tape' import { MapDB, Trie } from '../../src' -import type { DB } from '../../src' - // reference: https://github.com/ethereum/go-ethereum/blob/20356e57b119b4e70ce47665a71964434e15200d/trie/proof_test.go const TRIE_SIZE = 512 diff --git a/packages/trie/test/stream.spec.ts b/packages/trie/test/stream.spec.ts index c08606befa..04b2ca528e 100644 --- a/packages/trie/test/stream.spec.ts +++ b/packages/trie/test/stream.spec.ts @@ -2,8 +2,7 @@ import { utf8ToBytes } from 'ethereum-cryptography/utils' import * as tape from 'tape' import { Trie } from '../src' - -import type { BatchDBOp } from '../src' +import { BatchDBOp } from '@ethereumjs/util' tape('kv stream test', function (tester) { const it = tester.test diff --git a/packages/trie/test/trie/checkpoint.spec.ts b/packages/trie/test/trie/checkpoint.spec.ts index ebd737adfe..6b6bc1b7f4 100644 --- a/packages/trie/test/trie/checkpoint.spec.ts +++ b/packages/trie/test/trie/checkpoint.spec.ts @@ -1,12 +1,10 @@ -import { bytesToHex, bytesToUtf8, equalsBytes, utf8ToBytes } from '@ethereumjs/util' +import { BatchDBOp, bytesToHex, bytesToUtf8, equalsBytes, utf8ToBytes } from '@ethereumjs/util' import { createHash } from 'crypto' import { keccak256 } from 'ethereum-cryptography/keccak' import * as tape from 'tape' import { MapDB, ROOT_DB_KEY, Trie } from '../../src' -import type { BatchDBOp } from '../../src' - tape('testing checkpoints', function (tester) { const it = tester.test diff --git a/packages/util/src/db.ts b/packages/util/src/db.ts new file mode 100644 index 0000000000..710ae42dc1 --- /dev/null +++ b/packages/util/src/db.ts @@ -0,0 +1,46 @@ +export type BatchDBOp = PutBatch | DelBatch + +export interface PutBatch { + type: 'put' + key: Uint8Array + value: Uint8Array +} + +export interface DelBatch { + type: 'del' + key: Uint8Array +} + +export interface DB { + /** + * Retrieves a raw value from db. + * @param key + * @returns A Promise that resolves to `Uint8Array` if a value is found or `null` if no value is found. + */ + get(key: Uint8Array): Promise + + /** + * Writes a value directly to db. + * @param key The key as a `Uint8Array` + * @param value The value to be stored + */ + put(key: Uint8Array, val: Uint8Array): Promise + + /** + * Removes a raw value in the underlying db. + * @param keys + */ + del(key: Uint8Array): Promise + + /** + * Performs a batch operation on db. + * @param opStack A stack of levelup operations + */ + batch(opStack: BatchDBOp[]): Promise + + /** + * Returns a copy of the DB instance, with a reference + * to the **same** underlying db instance. + */ + copy(): DB +} diff --git a/packages/util/src/index.ts b/packages/util/src/index.ts index 9a05428d06..84a4c7f070 100644 --- a/packages/util/src/index.ts +++ b/packages/util/src/index.ts @@ -18,6 +18,11 @@ export * from './account' */ export * from './address' +/** + * DB type + */ +export * from './db' + /** * Withdrawal type */ From 510b0c6239f6210d0745e838c0f21457bd4d933e Mon Sep 17 00:00:00 2001 From: Gabriel Rocheleau Date: Thu, 27 Apr 2023 16:53:39 -0400 Subject: [PATCH 03/27] util: make db interface generic --- packages/blockchain/src/blockchain.ts | 9 +++---- packages/util/src/db.ts | 35 +++++++++++++++++---------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/packages/blockchain/src/blockchain.ts b/packages/blockchain/src/blockchain.ts index 9b7280ad56..135c3c4f9e 100644 --- a/packages/blockchain/src/blockchain.ts +++ b/packages/blockchain/src/blockchain.ts @@ -1,8 +1,7 @@ import { Block, BlockHeader } from '@ethereumjs/block' import { Chain, Common, ConsensusAlgorithm, ConsensusType, Hardfork } from '@ethereumjs/common' -import { KECCAK256_RLP, Lock, concatBytesNoTypeCheck } from '@ethereumjs/util' +import { DB, KECCAK256_RLP, Lock, concatBytesNoTypeCheck } from '@ethereumjs/util' import { bytesToHex, equalsBytes, hexToBytes } from 'ethereum-cryptography/utils' -import { MemoryLevel } from 'memory-level' import { CasperConsensus, CliqueConsensus, EthashConsensus } from './consensus' import { DBOp, DBSaveLookups, DBSetBlockOrHeader, DBSetHashToNumber, DBSetTD } from './db/helpers' @@ -17,14 +16,14 @@ import type { BlockchainInterface, BlockchainOptions, OnBlock } from './types' import type { BlockData } from '@ethereumjs/block' import type { CliqueConfig } from '@ethereumjs/common' import type { BigIntLike } from '@ethereumjs/util' -import type { AbstractLevel } from 'abstract-level' +import { MapDB } from './db/map' /** * This class stores and interacts with blocks. */ export class Blockchain implements BlockchainInterface { consensus: Consensus - db: AbstractLevel + db: DB dbManager: DBManager private _genesisBlock?: Block /** The genesis block of this blockchain */ @@ -115,7 +114,7 @@ export class Blockchain implements BlockchainInterface { this._validateBlocks = opts.validateBlocks ?? true this._customGenesisState = opts.genesisState - this.db = opts.db ? opts.db : new MemoryLevel() + this.db = opts.db ? opts.db : new MapDB() this.dbManager = new DBManager(this.db, this._common) if (opts.consensus) { diff --git a/packages/util/src/db.ts b/packages/util/src/db.ts index 710ae42dc1..93a4d26a91 100644 --- a/packages/util/src/db.ts +++ b/packages/util/src/db.ts @@ -1,46 +1,55 @@ -export type BatchDBOp = PutBatch | DelBatch +export type BatchDBOp< + TKey extends Uint8Array | string = Uint8Array, + TValue extends Uint8Array | string = Uint8Array +> = PutBatch | DelBatch -export interface PutBatch { +export interface PutBatch< + TKey extends Uint8Array | string = Uint8Array, + TValue extends Uint8Array | string = Uint8Array +> { type: 'put' - key: Uint8Array - value: Uint8Array + key: TKey + value: TValue } -export interface DelBatch { +export interface DelBatch { type: 'del' - key: Uint8Array + key: TKey } -export interface DB { +export interface DB< + TKey extends Uint8Array | string = Uint8Array, + TValue extends Uint8Array | string = Uint8Array +> { /** * Retrieves a raw value from db. * @param key * @returns A Promise that resolves to `Uint8Array` if a value is found or `null` if no value is found. */ - get(key: Uint8Array): Promise + get(key: TKey): Promise /** * Writes a value directly to db. - * @param key The key as a `Uint8Array` + * @param key The key as a `TValue` * @param value The value to be stored */ - put(key: Uint8Array, val: Uint8Array): Promise + put(key: TKey, val: TValue): Promise /** * Removes a raw value in the underlying db. * @param keys */ - del(key: Uint8Array): Promise + del(key: TKey): Promise /** * Performs a batch operation on db. * @param opStack A stack of levelup operations */ - batch(opStack: BatchDBOp[]): Promise + batch(opStack: BatchDBOp[]): Promise /** * Returns a copy of the DB instance, with a reference * to the **same** underlying db instance. */ - copy(): DB + copy(): DB } From 89f4741d7ffbcb3c84c1040b4f8066c0b028add1 Mon Sep 17 00:00:00 2001 From: Gabriel Rocheleau Date: Thu, 27 Apr 2023 16:55:35 -0400 Subject: [PATCH 04/27] blockchain: remove abstract-level usage in favor of DB interface and MapDB --- packages/blockchain/package.json | 1 - packages/blockchain/src/consensus/clique.ts | 28 +++++++------- packages/blockchain/src/db/manager.ts | 22 ++++++----- packages/blockchain/src/db/map.ts | 42 +++++++++++++++++++++ packages/blockchain/src/types.ts | 13 ++----- packages/blockchain/test/index.spec.ts | 3 +- packages/blockchain/test/util.ts | 6 +-- packages/client/bin/cli.ts | 3 +- packages/client/lib/blockchain/chain.ts | 6 +-- packages/trie/src/db/map.ts | 6 +-- 10 files changed, 84 insertions(+), 46 deletions(-) create mode 100644 packages/blockchain/src/db/map.ts diff --git a/packages/blockchain/package.json b/packages/blockchain/package.json index 6717f5c704..515bf15ac7 100644 --- a/packages/blockchain/package.json +++ b/packages/blockchain/package.json @@ -45,7 +45,6 @@ "@ethereumjs/trie": "^5.0.4", "@ethereumjs/tx": "^4.1.1", "@ethereumjs/util": "^8.0.5", - "abstract-level": "^1.0.3", "debug": "^4.3.3", "ethereum-cryptography": "^1.1.2", "lru-cache": "^5.1.1", diff --git a/packages/blockchain/src/consensus/clique.ts b/packages/blockchain/src/consensus/clique.ts index b89aa64fc4..ce03035d87 100644 --- a/packages/blockchain/src/consensus/clique.ts +++ b/packages/blockchain/src/consensus/clique.ts @@ -2,7 +2,7 @@ import { ConsensusAlgorithm } from '@ethereumjs/common' import { RLP } from '@ethereumjs/rlp' import { Address, TypeOutput, bigIntToBytes, bytesToBigInt, toType } from '@ethereumjs/util' import { debug as createDebugLogger } from 'debug' -import { equalsBytes, hexToBytes } from 'ethereum-cryptography/utils' +import { equalsBytes, hexToBytes, utf8ToBytes } from 'ethereum-cryptography/utils' import type { Blockchain } from '..' import type { Consensus, ConsensusOptions } from './interface' @@ -253,7 +253,7 @@ export class CliqueConsensus implements Consensus { bigIntToBytes(state[0]), state[1].map((a) => a.toBytes()), ]) - await this.blockchain!.db.put(CLIQUE_SIGNERS_KEY, RLP.encode(formatted), DB_OPTS) + await this.blockchain!.db.put(CLIQUE_SIGNERS_KEY, RLP.encode(formatted) /* , DB_OPTS */) // Output active signers for debugging purposes if (signerState !== undefined) { let i = 0 @@ -414,7 +414,7 @@ export class CliqueConsensus implements Consensus { bigIntToBytes(v[0]), [v[1][0].toBytes(), v[1][1].toBytes(), v[1][2]], ]) - await this.blockchain!.db.put(CLIQUE_VOTES_KEY, RLP.encode(formatted), DB_OPTS) + await this.blockchain!.db.put(CLIQUE_VOTES_KEY, RLP.encode(formatted) /* , DB_OPTS */) } /** @@ -523,7 +523,10 @@ export class CliqueConsensus implements Consensus { bigIntToBytes(b[0]), b[1].toBytes(), ]) - await this.blockchain!.db.put(CLIQUE_BLOCK_SIGNERS_SNAPSHOT_KEY, RLP.encode(formatted), DB_OPTS) + await this.blockchain!.db.put( + CLIQUE_BLOCK_SIGNERS_SNAPSHOT_KEY, + RLP.encode(formatted) /* , DB_OPTS */ + ) } /** @@ -532,9 +535,9 @@ export class CliqueConsensus implements Consensus { */ private async getCliqueLatestSignerStates(): Promise { try { - const signerStates = await this.blockchain!.db.get( - CLIQUE_SIGNERS_KEY, - DB_OPTS + const signerStates = await this.blockchain!.db.get( + CLIQUE_SIGNERS_KEY + /* DB_OPTS */ ) const states = RLP.decode(signerStates) as [Uint8Array, Uint8Array[]] return states.map((state) => { @@ -556,9 +559,9 @@ export class CliqueConsensus implements Consensus { */ private async getCliqueLatestVotes(): Promise { try { - const signerVotes = await this.blockchain!.db.get( - CLIQUE_VOTES_KEY, - DB_OPTS + const signerVotes = await this.blockchain!.db.get( + CLIQUE_VOTES_KEY + /* DB_OPTS */ ) const votes = RLP.decode(signerVotes) as [Uint8Array, [Uint8Array, Uint8Array, Uint8Array]] return votes.map((vote) => { @@ -582,9 +585,8 @@ export class CliqueConsensus implements Consensus { */ private async getCliqueLatestBlockSigners(): Promise { try { - const blockSigners = await this.blockchain!.db.get( - CLIQUE_BLOCK_SIGNERS_SNAPSHOT_KEY, - DB_OPTS + const blockSigners = await this.blockchain!.db.get( + CLIQUE_BLOCK_SIGNERS_SNAPSHOT_KEY /* DB_OPTS */ ) const signers = RLP.decode(blockSigners) as [Uint8Array, Uint8Array][] return signers.map((s) => { diff --git a/packages/blockchain/src/db/manager.ts b/packages/blockchain/src/db/manager.ts index 02aec83939..21fe238d2e 100644 --- a/packages/blockchain/src/db/manager.ts +++ b/packages/blockchain/src/db/manager.ts @@ -1,6 +1,13 @@ import { Block, BlockHeader, valuesArrayToHeaderData } from '@ethereumjs/block' import { RLP } from '@ethereumjs/rlp' -import { KECCAK256_RLP, KECCAK256_RLP_ARRAY, bytesToBigInt, equalsBytes } from '@ethereumjs/util' +import { + DB, + KECCAK256_RLP, + KECCAK256_RLP_ARRAY, + bytesToBigInt, + equalsBytes, + toBytes, +} from '@ethereumjs/util' import { hexToBytes } from 'ethereum-cryptography/utils' import { Cache } from './cache' @@ -9,7 +16,6 @@ import { DBOp, DBTarget } from './operation' import type { DBOpData, DatabaseKey } from './operation' import type { BlockBodyBytes, BlockBytes, BlockOptions } from '@ethereumjs/block' import type { Common } from '@ethereumjs/common' -import type { AbstractLevel } from 'abstract-level' class NotFoundError extends Error { public code: string = 'LEVEL_NOT_FOUND' @@ -43,12 +49,9 @@ export type CacheMap = { [key: string]: Cache } export class DBManager { private _cache: CacheMap private _common: Common - private _db: AbstractLevel + private _db: DB - constructor( - db: AbstractLevel, - common: Common - ) { + constructor(db: DB, common: Common) { this._db = db this._common = common this._cache = { @@ -219,9 +222,10 @@ export class DBManager { } let value = this._cache[cacheString].get(dbKey) if (!value) { - value = await this._db.get(dbKey, dbOpts) + value = ((await this._db.get(dbKey)) as Uint8Array | null) ?? undefined if (value !== undefined) { + // TODO: Check if this comment is still valid // Always cast values to Uint8Array since db sometimes returns values as `Buffer` this._cache[cacheString].set(dbKey, Uint8Array.from(value)) } @@ -230,7 +234,7 @@ export class DBManager { return value } - return this._db.get(dbKey, dbOpts) + return this._db.get(dbKey /* , dbOpts */) } /** diff --git a/packages/blockchain/src/db/map.ts b/packages/blockchain/src/db/map.ts new file mode 100644 index 0000000000..4fcd5500c9 --- /dev/null +++ b/packages/blockchain/src/db/map.ts @@ -0,0 +1,42 @@ +import { DB, BatchDBOp } from '@ethereumjs/util' +import { bytesToHex } from 'ethereum-cryptography/utils' + +export class MapDB + implements DB +{ + _database: Map + + constructor(database?: Map) { + this._database = database ?? new Map() + } + + async get(key: TKey): Promise { + const result = this._database.get(key) + + return result ?? null + } + + async put(key: TKey, val: TValue): Promise { + this._database.set(key, val) + } + + async del(key: TKey): Promise { + this._database.delete(key) + } + + async batch(opStack: BatchDBOp[]): Promise { + for (const op of opStack) { + if (op.type === 'del') { + await this.del(op.key) + } + + if (op.type === 'put') { + await this.put(op.key, op.value) + } + } + } + + copy(): DB { + return new MapDB(this._database) + } +} diff --git a/packages/blockchain/src/types.ts b/packages/blockchain/src/types.ts index a1396722af..a5a8958cc4 100644 --- a/packages/blockchain/src/types.ts +++ b/packages/blockchain/src/types.ts @@ -1,8 +1,8 @@ +import { DB } from '@ethereumjs/util' import type { Consensus } from './consensus' import type { GenesisState } from './genesisStates' import type { Block, BlockHeader } from '@ethereumjs/block' import type { Common } from '@ethereumjs/common' -import type { AbstractLevel } from 'abstract-level' export type OnBlock = (block: Block, reorg: boolean) => Promise | void @@ -105,14 +105,9 @@ export interface BlockchainOptions { /** * Database to store blocks and metadata. - * Should be an `abstract-leveldown` compliant store - * wrapped with `encoding-down`. - * For example: - * `levelup(encode(leveldown('./db1')))` - * or use the `level` convenience package: - * `new MemoryLevel('./db1')` - */ - db?: AbstractLevel + * Can be any database implementation that adheres to the `DB` interface + */ + db?: DB /** * This flags indicates if a block should be validated along the consensus algorithm diff --git a/packages/blockchain/test/index.spec.ts b/packages/blockchain/test/index.spec.ts index 7238639bc9..3b9b3d5a63 100644 --- a/packages/blockchain/test/index.spec.ts +++ b/packages/blockchain/test/index.spec.ts @@ -11,6 +11,7 @@ import * as testDataPreLondon from './testdata/testdata_pre-london.json' import { createTestDB, generateBlockchain, generateBlocks, isConsecutive } from './util' import type { BlockOptions } from '@ethereumjs/block' +import { MapDB } from '@ethereumjs/trie' tape('blockchain test', (t) => { t.test('should not crash on getting head of a blockchain without a genesis', async (st) => { @@ -609,7 +610,7 @@ tape('blockchain test', (t) => { }) t.test('should save headers', async (st) => { - const db = new MemoryLevel() + const db = new MapDB() const gasLimit = 8000000 const common = new Common({ chain: Chain.Mainnet, hardfork: Hardfork.Istanbul }) diff --git a/packages/blockchain/test/util.ts b/packages/blockchain/test/util.ts index 448255be46..d23f59f373 100644 --- a/packages/blockchain/test/util.ts +++ b/packages/blockchain/test/util.ts @@ -1,15 +1,13 @@ import { Block, BlockHeader } from '@ethereumjs/block' import { Chain, Common, Hardfork } from '@ethereumjs/common' import { RLP } from '@ethereumjs/rlp' -import { toBytes } from '@ethereumjs/util' +import { DB, toBytes } from '@ethereumjs/util' import { keccak256 } from 'ethereum-cryptography/keccak' import { bytesToHex, equalsBytes, hexToBytes, utf8ToBytes } from 'ethereum-cryptography/utils' import { MemoryLevel } from 'memory-level' import { Blockchain } from '../src' -import type { AbstractLevel } from 'abstract-level' - export const generateBlocks = (numberOfBlocks: number, existingBlocks?: Block[]): Block[] => { const blocks = existingBlocks ? existingBlocks : [] @@ -116,7 +114,7 @@ export const isConsecutive = (blocks: Block[]) => { } export const createTestDB = async (): Promise< - [AbstractLevel, Block] + [DB, Block] > => { const common = new Common({ chain: Chain.Mainnet, hardfork: Hardfork.Chainstart }) const genesis = Block.fromBlockData({ header: { number: 0 } }, { common }) diff --git a/packages/client/bin/cli.ts b/packages/client/bin/cli.ts index e1e24e43dd..5f5eb39125 100755 --- a/packages/client/bin/cli.ts +++ b/packages/client/bin/cli.ts @@ -6,6 +6,7 @@ import { Chain, Common, ConsensusAlgorithm, Hardfork } from '@ethereumjs/common' import { RLP } from '@ethereumjs/rlp' import { Address, + DB, bytesToHex, bytesToPrefixedHexString, hexStringToBytes, @@ -424,7 +425,7 @@ async function startClient(config: Config, customGenesisState?: GenesisState) { if (customGenesisState !== undefined) { const validateConsensus = config.chainCommon.consensusAlgorithm() === ConsensusAlgorithm.Clique blockchain = await Blockchain.create({ - db: dbs.chainDB, + db: dbs.chainDB as any, genesisState: customGenesisState, common: config.chainCommon, hardforkByHeadBlockNumber: true, diff --git a/packages/client/lib/blockchain/chain.ts b/packages/client/lib/blockchain/chain.ts index b26610fee8..2d3b6becd4 100644 --- a/packages/client/lib/blockchain/chain.ts +++ b/packages/client/lib/blockchain/chain.ts @@ -155,7 +155,7 @@ export class Chain { this.config = options.config this.blockchain = options.blockchain! - this.chainDB = this.blockchain.db + this.chainDB = this.blockchain.db as any this.opened = false } @@ -213,7 +213,7 @@ export class Chain { */ async open(): Promise { if (this.opened) return false - await this.blockchain.db.open() + await (this.blockchain.db as any)?.open() await (this.blockchain as any)._init() this.opened = true await this.update(false) @@ -231,7 +231,7 @@ export class Chain { async close(): Promise { if (!this.opened) return false this.reset() - await this.blockchain.db.close() + await (this.blockchain.db as any)?.close() this.opened = false } diff --git a/packages/trie/src/db/map.ts b/packages/trie/src/db/map.ts index 0f2dd2a23a..22bb653b51 100644 --- a/packages/trie/src/db/map.ts +++ b/packages/trie/src/db/map.ts @@ -11,11 +11,7 @@ export class MapDB implements DB { async get(key: Uint8Array): Promise { const result = this._database.get(bytesToHex(key)) - if (result !== undefined) { - return result - } - - return null + return result ?? null } async put(key: Uint8Array, val: Uint8Array): Promise { From 6c9d0b190f878689c938338026a82aeff95be501 Mon Sep 17 00:00:00 2001 From: acolytec3 Date: Wed, 3 May 2023 22:07:59 -0400 Subject: [PATCH 05/27] WIP removal of level DB stuff (#2673) Co-authored-by: Gabriel Rocheleau --- package-lock.json | 4 - packages/blockchain/src/blockchain.ts | 149 ++++++++---------- packages/blockchain/src/consensus/clique.ts | 90 ++++------- packages/blockchain/src/consensus/ethash.ts | 3 +- packages/blockchain/src/db/manager.ts | 84 ++++------ packages/blockchain/src/db/map.ts | 20 ++- packages/blockchain/src/types.ts | 2 +- packages/blockchain/test/index.spec.ts | 14 +- packages/blockchain/test/util.ts | 24 +-- packages/client/bin/cli.ts | 4 +- packages/client/browser/index.ts | 3 +- packages/client/lib/blockchain/chain.ts | 7 +- packages/client/lib/execution/level.ts | 13 +- packages/client/lib/execution/vmexecution.ts | 3 +- packages/client/lib/miner/miner.ts | 3 +- .../client/lib/sync/fetcher/accountfetcher.ts | 3 +- .../lib/sync/fetcher/bytecodefetcher.ts | 3 +- .../client/lib/sync/fetcher/storagefetcher.ts | 5 +- .../test/net/protocol/snapprotocol.spec.ts | 6 +- .../getTransactionByBlockHashAndIndex.spec.ts | 2 +- packages/client/test/sim/simutils.ts | 3 +- packages/ethash/src/index.ts | 60 ++----- packages/ethash/src/level.ts | 69 ++++++++ packages/ethash/test/block.spec.ts | 3 +- packages/ethash/test/miner.spec.ts | 3 +- packages/trie/recipes/level-legacy.ts | 10 +- packages/trie/recipes/level.ts | 6 +- packages/trie/recipes/lmdb.ts | 8 +- packages/trie/src/db/checkpoint.ts | 22 +-- packages/trie/src/db/map.ts | 13 +- packages/trie/src/trie/trie.ts | 11 +- packages/trie/src/types.ts | 4 +- packages/trie/test/db/checkpoint.spec.ts | 8 +- packages/trie/test/db/db.spec.ts | 4 +- packages/trie/test/proof/range.spec.ts | 3 +- packages/trie/test/stream.spec.ts | 3 +- packages/trie/test/trie/checkpoint.spec.ts | 4 +- packages/trie/test/trie/trie.spec.ts | 16 +- packages/util/src/db.ts | 10 +- packages/vm/test/api/runBlock.spec.ts | 5 +- packages/vm/test/api/utils.ts | 3 +- packages/vm/test/level.ts | 78 +++++++++ .../tester/runners/BlockchainTestsRunner.ts | 9 +- 43 files changed, 445 insertions(+), 352 deletions(-) create mode 100644 packages/ethash/src/level.ts create mode 100644 packages/vm/test/level.ts diff --git a/package-lock.json b/package-lock.json index 4c8b86f565..b1caecd730 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17845,10 +17845,8 @@ "@ethereumjs/trie": "^5.0.4", "@ethereumjs/tx": "^4.1.1", "@ethereumjs/util": "^8.0.5", - "abstract-level": "^1.0.3", "debug": "^4.3.3", "ethereum-cryptography": "^1.1.2", - "level": "^8.0.0", "lru-cache": "^5.1.1", "memory-level": "^1.0.0" }, @@ -19575,10 +19573,8 @@ "@types/async": "^2.4.1", "@types/level-errors": "^3.0.0", "@types/lru-cache": "^5.1.0", - "abstract-level": "^1.0.3", "debug": "^4.3.3", "ethereum-cryptography": "^1.1.2", - "level": "^8.0.0", "lru-cache": "^5.1.1", "memory-level": "^1.0.0" } diff --git a/packages/blockchain/src/blockchain.ts b/packages/blockchain/src/blockchain.ts index 135c3c4f9e..c8fe29de24 100644 --- a/packages/blockchain/src/blockchain.ts +++ b/packages/blockchain/src/blockchain.ts @@ -1,11 +1,17 @@ import { Block, BlockHeader } from '@ethereumjs/block' import { Chain, Common, ConsensusAlgorithm, ConsensusType, Hardfork } from '@ethereumjs/common' -import { DB, KECCAK256_RLP, Lock, concatBytesNoTypeCheck } from '@ethereumjs/util' +import { + KECCAK256_RLP, + Lock, + bytesToPrefixedHexString, + concatBytesNoTypeCheck, +} from '@ethereumjs/util' import { bytesToHex, equalsBytes, hexToBytes } from 'ethereum-cryptography/utils' import { CasperConsensus, CliqueConsensus, EthashConsensus } from './consensus' import { DBOp, DBSaveLookups, DBSetBlockOrHeader, DBSetHashToNumber, DBSetTD } from './db/helpers' import { DBManager } from './db/manager' +import { MapDB } from './db/map' import { DBTarget } from './db/operation' import { genesisStateRoot } from './genesisStates' import {} from './utils' @@ -15,8 +21,7 @@ import type { GenesisState } from './genesisStates' import type { BlockchainInterface, BlockchainOptions, OnBlock } from './types' import type { BlockData } from '@ethereumjs/block' import type { CliqueConfig } from '@ethereumjs/common' -import type { BigIntLike } from '@ethereumjs/util' -import { MapDB } from './db/map' +import type { BigIntLike, DB } from '@ethereumjs/util' /** * This class stores and interacts with blocks. @@ -114,7 +119,8 @@ export class Blockchain implements BlockchainInterface { this._validateBlocks = opts.validateBlocks ?? true this._customGenesisState = opts.genesisState - this.db = opts.db ? opts.db : new MapDB() + this.db = opts.db !== undefined ? opts.db : new MapDB() + this.dbManager = new DBManager(this.db, this._common) if (opts.consensus) { @@ -190,17 +196,13 @@ export class Blockchain implements BlockchainInterface { await this.consensus.setup({ blockchain: this }) if (this._isInitialized) return - let dbGenesisBlock - try { - const genesisHash = await this.dbManager.numberToHash(BigInt(0)) - dbGenesisBlock = await this.dbManager.getBlock(genesisHash) - } catch (error: any) { - if (error.code !== 'LEVEL_NOT_FOUND') { - throw error - } - } - if (!genesisBlock) { + let genesisHash = await this.dbManager.numberToHash(BigInt(0)) + + const dbGenesisBlock = + genesisHash !== undefined ? await this.dbManager.getBlock(genesisHash) : undefined + + if (genesisBlock === undefined) { let stateRoot if (this._common.chainId() === BigInt(1) && this._customGenesisState === undefined) { // For mainnet use the known genesis stateRoot to quicken setup @@ -213,13 +215,13 @@ export class Blockchain implements BlockchainInterface { // If the DB has a genesis block, then verify that the genesis block in the // DB is indeed the Genesis block generated or assigned. - if (dbGenesisBlock && !equalsBytes(genesisBlock.hash(), dbGenesisBlock.hash())) { + if (dbGenesisBlock !== undefined && !equalsBytes(genesisBlock.hash(), dbGenesisBlock.hash())) { throw new Error( 'The genesis block in the DB has a different hash than the provided genesis block.' ) } - const genesisHash = genesisBlock.hash() + genesisHash = genesisBlock.hash() if (!dbGenesisBlock) { // If there is no genesis block put the genesis block in the DB. @@ -238,37 +240,16 @@ export class Blockchain implements BlockchainInterface { this._genesisBlock = genesisBlock // load verified iterator heads - try { - const heads = await this.dbManager.getHeads() - this._heads = heads - } catch (error: any) { - if (error.code !== 'LEVEL_NOT_FOUND') { - throw error - } - this._heads = {} - } + const heads = await this.dbManager.getHeads() + this._heads = heads !== undefined ? heads : {} // load headerchain head - try { - const hash = await this.dbManager.getHeadHeader() - this._headHeaderHash = hash - } catch (error: any) { - if (error.code !== 'LEVEL_NOT_FOUND') { - throw error - } - this._headHeaderHash = genesisHash - } + let hash = await this.dbManager.getHeadHeader() + this._headHeaderHash = hash !== undefined ? hash : genesisHash // load blockchain head - try { - const hash = await this.dbManager.getHeadBlock() - this._headBlockHash = hash - } catch (error: any) { - if (error.code !== 'LEVEL_NOT_FOUND') { - throw error - } - this._headBlockHash = genesisHash - } + hash = await this.dbManager.getHeadBlock() + this._headBlockHash = hash !== undefined ? hash : genesisHash if (this._hardforkByHeadBlockNumber) { const latestHeader = await this._getHeader(this._headHeaderHash) @@ -422,6 +403,9 @@ export class Blockchain implements BlockchainInterface { async resetCanonicalHead(canonicalHead: bigint) { await this.runWithLock(async () => { const hash = await this.dbManager.numberToHash(canonicalHead) + if (hash === undefined) { + throw new Error(`no block for ${canonicalHead} found in DB`) + } const header = await this._getHeader(hash, canonicalHead) const td = await this.getParentTD(header) @@ -737,18 +721,16 @@ export class Blockchain implements BlockchainInterface { // in the `VM` if we encounter a `BLOCKHASH` opcode: then a bigint is used we // need to then read the block from the canonical chain Q: is this safe? We // know it is OK if we call it from the iterator... (runBlock) - try { - return await this.dbManager.getBlock(blockId) - } catch (error: any) { - if (error.code === 'LEVEL_NOT_FOUND') { - if (typeof blockId === 'object') { - error.message = `Block with hash ${bytesToHex(blockId)} not found in DB (NotFound)` - } else { - error.message = `Block number ${blockId} not found in DB (NotFound)` - } + const block = await this.dbManager.getBlock(blockId) + + if (block === undefined) { + if (typeof blockId === 'object') { + throw new Error(`Block with hash ${bytesToHex(blockId)} not found in DB`) + } else { + throw new Error(`Block number ${blockId} not found in DB`) } - throw error } + return block } /** @@ -757,6 +739,9 @@ export class Blockchain implements BlockchainInterface { public async getTotalDifficulty(hash: Uint8Array, number?: bigint): Promise { if (number === undefined) { number = await this.dbManager.hashToNumber(hash) + if (number === undefined) { + throw new Error(`Block with hash ${bytesToPrefixedHexString(hash)} not found in DB`) + } } return this.dbManager.getTotalDifficulty(hash, number) } @@ -793,12 +778,12 @@ export class Blockchain implements BlockchainInterface { let block try { block = await this.getBlock(blockId) - } catch (error: any) { - if (error.code !== 'LEVEL_NOT_FOUND') { - throw error - } - return + } catch (err: any) { + if (err.message.includes('not found in DB') === true) { + return + } else throw err } + i++ const nextBlockNumber = block.header.number + BigInt(reverse ? -1 : 1) if (i !== 0 && skip && i % (skip + 1) !== 0) { @@ -834,11 +819,12 @@ export class Blockchain implements BlockchainInterface { let number try { number = await this.dbManager.hashToNumber(hashes[mid]) - } catch (error: any) { - if (error.code !== 'LEVEL_NOT_FOUND') { - throw error - } + } catch (err: any) { + if (err.message.includes('not found in DB') === true) { + number = undefined + } else throw err } + if (number !== undefined) { min = mid + 1 } else { @@ -944,9 +930,9 @@ export class Blockchain implements BlockchainInterface { try { const childHeader = await this.getCanonicalHeader(blockNumber + BigInt(1)) await this._delChild(childHeader.hash(), childHeader.number, headHash, ops) - } catch (error: any) { - if (error.code !== 'LEVEL_NOT_FOUND') { - throw error + } catch (err: any) { + if (err.message.includes('not found in canonical chain') !== true) { + throw err } } } @@ -976,7 +962,8 @@ export class Blockchain implements BlockchainInterface { } let headBlockNumber = await this.dbManager.hashToNumber(headHash) - let nextBlockNumber = headBlockNumber + BigInt(1) + // `headBlockNumber` should always exist since it defaults to the genesis block + let nextBlockNumber = headBlockNumber! + BigInt(1) let blocksRanCounter = 0 let lastBlock: Block | undefined @@ -990,7 +977,7 @@ export class Blockchain implements BlockchainInterface { // If reorg has happened, the _heads must have been updated so lets reload the counters headHash = this._heads[name] ?? this.genesisBlock.hash() headBlockNumber = await this.dbManager.hashToNumber(headHash) - nextBlockNumber = headBlockNumber + BigInt(1) + nextBlockNumber = headBlockNumber! + BigInt(1) nextBlock = await this.getBlock(nextBlockNumber) } this._heads[name] = nextBlock.hash() @@ -1008,7 +995,7 @@ export class Blockchain implements BlockchainInterface { nextBlockNumber++ blocksRanCounter++ } catch (error: any) { - if (error.code === 'LEVEL_NOT_FOUND') { + if ((error.message as string).includes('not found in DB')) { break } else { throw error @@ -1181,13 +1168,9 @@ export class Blockchain implements BlockchainInterface { staleHeadBlock = true } - try { - header = await this._getHeader(header.parentHash, --currentNumber) - } catch (error: any) { + header = await this._getHeader(header.parentHash, --currentNumber) + if (header === undefined) { staleHeads = [] - if (error.code !== 'LEVEL_NOT_FOUND') { - throw error - } break } } @@ -1241,6 +1224,8 @@ export class Blockchain implements BlockchainInterface { private async _getHeader(hash: Uint8Array, number?: bigint) { if (number === undefined) { number = await this.dbManager.hashToNumber(hash) + if (number === undefined) + throw new Error(`no header for ${bytesToPrefixedHexString(hash)} found in DB`) } return this.dbManager.getHeader(hash, number) } @@ -1284,25 +1269,21 @@ export class Blockchain implements BlockchainInterface { */ async getCanonicalHeader(number: bigint) { const hash = await this.dbManager.numberToHash(number) + if (hash === undefined) { + throw new Error(`header with number ${number} not found in canonical chain`) + } return this._getHeader(hash, number) } /** * This method either returns a Uint8Array if there exists one in the DB or if it - * does not exist (DB throws a `NotFoundError`) then return false If DB throws + * does not exist then return false If DB throws * any other error, this function throws. * @param number */ async safeNumberToHash(number: bigint): Promise { - try { - const hash = await this.dbManager.numberToHash(number) - return hash - } catch (error: any) { - if (error.code !== 'LEVEL_NOT_FOUND') { - throw error - } - return false - } + const hash = await this.dbManager.numberToHash(number) + return hash !== undefined ? hash : false } /** diff --git a/packages/blockchain/src/consensus/clique.ts b/packages/blockchain/src/consensus/clique.ts index ce03035d87..17b2ac8881 100644 --- a/packages/blockchain/src/consensus/clique.ts +++ b/packages/blockchain/src/consensus/clique.ts @@ -2,7 +2,7 @@ import { ConsensusAlgorithm } from '@ethereumjs/common' import { RLP } from '@ethereumjs/rlp' import { Address, TypeOutput, bigIntToBytes, bytesToBigInt, toType } from '@ethereumjs/util' import { debug as createDebugLogger } from 'debug' -import { equalsBytes, hexToBytes, utf8ToBytes } from 'ethereum-cryptography/utils' +import { equalsBytes, hexToBytes } from 'ethereum-cryptography/utils' import type { Blockchain } from '..' import type { Consensus, ConsensusOptions } from './interface' @@ -25,11 +25,6 @@ export const CLIQUE_DIFF_INTURN = BigInt(2) // Block difficulty for out-of-turn signatures export const CLIQUE_DIFF_NOTURN = BigInt(1) -const DB_OPTS = { - keyEncoding: 'view', - valueEncoding: 'view', -} - // Clique Signer State type CliqueSignerState = [blockNumber: bigint, signers: Address[]] type CliqueLatestSignerStates = CliqueSignerState[] @@ -523,10 +518,7 @@ export class CliqueConsensus implements Consensus { bigIntToBytes(b[0]), b[1].toBytes(), ]) - await this.blockchain!.db.put( - CLIQUE_BLOCK_SIGNERS_SNAPSHOT_KEY, - RLP.encode(formatted) /* , DB_OPTS */ - ) + await this.blockchain!.db.put(CLIQUE_BLOCK_SIGNERS_SNAPSHOT_KEY, RLP.encode(formatted)) } /** @@ -534,23 +526,14 @@ export class CliqueConsensus implements Consensus { * @hidden */ private async getCliqueLatestSignerStates(): Promise { - try { - const signerStates = await this.blockchain!.db.get( - CLIQUE_SIGNERS_KEY - /* DB_OPTS */ - ) - const states = RLP.decode(signerStates) as [Uint8Array, Uint8Array[]] - return states.map((state) => { - const blockNum = bytesToBigInt(state[0] as Uint8Array) - const addrs = (state[1]).map((bytes: Uint8Array) => new Address(bytes)) - return [blockNum, addrs] - }) as CliqueLatestSignerStates - } catch (error: any) { - if (error.code === 'LEVEL_NOT_FOUND') { - return [] - } - throw error - } + const signerStates = await this.blockchain!.db.get(CLIQUE_SIGNERS_KEY) + if (signerStates === undefined) return [] + const states = RLP.decode(signerStates) as [Uint8Array, Uint8Array[]] + return states.map((state) => { + const blockNum = bytesToBigInt(state[0] as Uint8Array) + const addrs = (state[1]).map((bytes: Uint8Array) => new Address(bytes)) + return [blockNum, addrs] + }) as CliqueLatestSignerStates } /** @@ -558,25 +541,16 @@ export class CliqueConsensus implements Consensus { * @hidden */ private async getCliqueLatestVotes(): Promise { - try { - const signerVotes = await this.blockchain!.db.get( - CLIQUE_VOTES_KEY - /* DB_OPTS */ - ) - const votes = RLP.decode(signerVotes) as [Uint8Array, [Uint8Array, Uint8Array, Uint8Array]] - return votes.map((vote) => { - const blockNum = bytesToBigInt(vote[0] as Uint8Array) - const signer = new Address((vote[1] as any)[0]) - const beneficiary = new Address((vote[1] as any)[1]) - const nonce = (vote[1] as any)[2] - return [blockNum, [signer, beneficiary, nonce]] - }) as CliqueLatestVotes - } catch (error: any) { - if (error.code === 'LEVEL_NOT_FOUND') { - return [] - } - throw error - } + const signerVotes = await this.blockchain!.db.get(CLIQUE_VOTES_KEY) + if (signerVotes === undefined) return [] + const votes = RLP.decode(signerVotes) as [Uint8Array, [Uint8Array, Uint8Array, Uint8Array]] + return votes.map((vote) => { + const blockNum = bytesToBigInt(vote[0] as Uint8Array) + const signer = new Address((vote[1] as any)[0]) + const beneficiary = new Address((vote[1] as any)[1]) + const nonce = (vote[1] as any)[2] + return [blockNum, [signer, beneficiary, nonce]] + }) as CliqueLatestVotes } /** @@ -584,22 +558,14 @@ export class CliqueConsensus implements Consensus { * @hidden */ private async getCliqueLatestBlockSigners(): Promise { - try { - const blockSigners = await this.blockchain!.db.get( - CLIQUE_BLOCK_SIGNERS_SNAPSHOT_KEY /* DB_OPTS */ - ) - const signers = RLP.decode(blockSigners) as [Uint8Array, Uint8Array][] - return signers.map((s) => { - const blockNum = bytesToBigInt(s[0] as Uint8Array) - const signer = new Address(s[1] as any) - return [blockNum, signer] - }) as CliqueLatestBlockSigners - } catch (error: any) { - if (error.code === 'LEVEL_NOT_FOUND') { - return [] - } - throw error - } + const blockSigners = await this.blockchain!.db.get(CLIQUE_BLOCK_SIGNERS_SNAPSHOT_KEY) + if (blockSigners === undefined) return [] + const signers = RLP.decode(blockSigners) as [Uint8Array, Uint8Array][] + return signers.map((s) => { + const blockNum = bytesToBigInt(s[0] as Uint8Array) + const signer = new Address(s[1] as any) + return [blockNum, signer] + }) as CliqueLatestBlockSigners } /** diff --git a/packages/blockchain/src/consensus/ethash.ts b/packages/blockchain/src/consensus/ethash.ts index 7a13245c13..76e5d6a7fa 100644 --- a/packages/blockchain/src/consensus/ethash.ts +++ b/packages/blockchain/src/consensus/ethash.ts @@ -4,7 +4,6 @@ import { Ethash } from '@ethereumjs/ethash' import type { Blockchain } from '..' import type { Consensus, ConsensusOptions } from './interface' import type { Block, BlockHeader } from '@ethereumjs/block' -import type { EthashCacheDB } from '@ethereumjs/ethash' /** * This class encapsulates Ethash-related consensus functionality when used with the Blockchain class. @@ -45,7 +44,7 @@ export class EthashConsensus implements Consensus { public async genesisInit(): Promise {} public async setup({ blockchain }: ConsensusOptions): Promise { this.blockchain = blockchain - this._ethash = new Ethash(this.blockchain.db as unknown as EthashCacheDB) + this._ethash = new Ethash(this.blockchain.db as any) } public async newBlock(): Promise {} } diff --git a/packages/blockchain/src/db/manager.ts b/packages/blockchain/src/db/manager.ts index 21fe238d2e..9da7d93cc6 100644 --- a/packages/blockchain/src/db/manager.ts +++ b/packages/blockchain/src/db/manager.ts @@ -1,12 +1,11 @@ import { Block, BlockHeader, valuesArrayToHeaderData } from '@ethereumjs/block' import { RLP } from '@ethereumjs/rlp' import { - DB, KECCAK256_RLP, KECCAK256_RLP_ARRAY, bytesToBigInt, + bytesToPrefixedHexString, equalsBytes, - toBytes, } from '@ethereumjs/util' import { hexToBytes } from 'ethereum-cryptography/utils' @@ -16,19 +15,7 @@ import { DBOp, DBTarget } from './operation' import type { DBOpData, DatabaseKey } from './operation' import type { BlockBodyBytes, BlockBytes, BlockOptions } from '@ethereumjs/block' import type { Common } from '@ethereumjs/common' - -class NotFoundError extends Error { - public code: string = 'LEVEL_NOT_FOUND' - - constructor(blockNumber: bigint) { - super(`Key ${blockNumber.toString()} was not found`) - - // `Error.captureStackTrace` is not defined in some browser contexts - if (typeof Error.captureStackTrace !== 'undefined') { - Error.captureStackTrace(this, this.constructor) - } - } -} +import type { DB } from '@ethereumjs/util' /** * @hidden @@ -68,6 +55,7 @@ export class DBManager { */ async getHeads(): Promise<{ [key: string]: Uint8Array }> { const heads = await this.get(DBTarget.Heads) + if (heads === undefined) return heads for (const key of Object.keys(heads)) { // Heads are stored in DB as hex strings since Level converts Uint8Arrays // to nested JSON objects when they are included in a value being stored @@ -80,14 +68,14 @@ export class DBManager { /** * Fetches header of the head block. */ - async getHeadHeader(): Promise { + async getHeadHeader(): Promise { return this.get(DBTarget.HeadHeader) } /** * Fetches head block. */ - async getHeadBlock(): Promise { + async getHeadBlock(): Promise { return this.get(DBTarget.HeadBlock) } @@ -95,13 +83,14 @@ export class DBManager { * Fetches a block (header and body) given a block id, * which can be either its hash or its number. */ - async getBlock(blockId: Uint8Array | bigint | number): Promise { + async getBlock(blockId: Uint8Array | bigint | number): Promise { if (typeof blockId === 'number' && Number.isInteger(blockId)) { blockId = BigInt(blockId) } let number let hash + if (blockId === undefined) return undefined if (blockId instanceof Uint8Array) { hash = blockId number = await this.hashToNumber(blockId) @@ -112,30 +101,27 @@ export class DBManager { throw new Error('Unknown blockId type') } + if (hash === undefined || number === undefined) return undefined const header = await this.getHeader(hash, number) - let body: BlockBodyBytes - try { - body = await this.getBody(hash, number) - } catch (error: any) { - if (error.code !== 'LEVEL_NOT_FOUND') { - throw error - } - + const body = await this.getBody(hash, number) + if (body[0].length === 0 && body[1].length === 0) { // Do extra validations on the header since we are assuming empty transactions and uncles - if ( - !equalsBytes(header.transactionsTrie, KECCAK256_RLP) || - !equalsBytes(header.uncleHash, KECCAK256_RLP_ARRAY) - ) { - throw error + if (!equalsBytes(header.transactionsTrie, KECCAK256_RLP)) { + throw new Error('transactionsTrie root should be equal to hash of null') + } + if (!equalsBytes(header.uncleHash, KECCAK256_RLP_ARRAY)) { + throw new Error('uncle hash should be equal to hash of empty array') } - body = [[], []] // If this block had empty withdrawals push an empty array in body if (header.withdrawalsRoot !== undefined) { // Do extra validations for withdrawal before assuming empty withdrawals - if (!equalsBytes(header.withdrawalsRoot, KECCAK256_RLP)) { - throw error + if ( + !equalsBytes(header.withdrawalsRoot, KECCAK256_RLP) && + (body.length !== 3 || body[2]?.length === 0) + ) { + throw new Error('withdrawals root shoot be equal to hash of null when no withdrawals') } - body.push([]) + if (body.length !== 3) body.push([]) } } @@ -154,6 +140,9 @@ export class DBManager { */ async getBody(blockHash: Uint8Array, blockNumber: bigint): Promise { const body = await this.get(DBTarget.Body, { blockHash, blockNumber }) + if (body === undefined) { + return [[], []] + } return RLP.decode(body) as BlockBodyBytes } @@ -188,20 +177,20 @@ export class DBManager { /** * Performs a block hash to block number lookup. */ - async hashToNumber(blockHash: Uint8Array): Promise { + async hashToNumber(blockHash: Uint8Array): Promise { const value = await this.get(DBTarget.HashToNumber, { blockHash }) - return bytesToBigInt(value) + if (value === undefined) { + throw new Error(`value for ${bytesToPrefixedHexString(blockHash)} not found in DB`) + } + return value !== undefined ? bytesToBigInt(value) : undefined } /** * Performs a block number to block hash lookup. */ - async numberToHash(blockNumber: bigint): Promise { - if (blockNumber < BigInt(0)) { - throw new NotFoundError(blockNumber) - } - - return this.get(DBTarget.NumberToHash, { blockNumber }) + async numberToHash(blockNumber: bigint): Promise { + const value = await this.get(DBTarget.NumberToHash, { blockNumber }) + return value } /** @@ -214,16 +203,14 @@ export class DBManager { const cacheString = dbGetOperation.cacheString const dbKey = dbGetOperation.baseDBOp.key - const dbOpts = dbGetOperation.baseDBOp if (cacheString !== undefined) { if (this._cache[cacheString] === undefined) { throw new Error(`Invalid cache: ${cacheString}`) } let value = this._cache[cacheString].get(dbKey) - if (!value) { - value = ((await this._db.get(dbKey)) as Uint8Array | null) ?? undefined - + if (value === undefined) { + value = (await this._db.get(dbKey)) as Uint8Array | undefined if (value !== undefined) { // TODO: Check if this comment is still valid // Always cast values to Uint8Array since db sometimes returns values as `Buffer` @@ -233,8 +220,7 @@ export class DBManager { return value } - - return this._db.get(dbKey /* , dbOpts */) + return this._db.get(dbKey) } /** diff --git a/packages/blockchain/src/db/map.ts b/packages/blockchain/src/db/map.ts index 4fcd5500c9..19d06a8379 100644 --- a/packages/blockchain/src/db/map.ts +++ b/packages/blockchain/src/db/map.ts @@ -1,5 +1,4 @@ -import { DB, BatchDBOp } from '@ethereumjs/util' -import { bytesToHex } from 'ethereum-cryptography/utils' +import type { BatchDBOp, DB } from '@ethereumjs/util' export class MapDB implements DB @@ -10,18 +9,19 @@ export class MapDB() } - async get(key: TKey): Promise { - const result = this._database.get(key) - - return result ?? null + async get(key: TKey): Promise { + const dbKey = key.toString() + return this._database.get(dbKey as TKey) } async put(key: TKey, val: TValue): Promise { - this._database.set(key, val) + const dbKey = key.toString() + this._database.set(dbKey as TKey, val) } async del(key: TKey): Promise { - this._database.delete(key) + const dbKey = key.toString() + this._database.delete(dbKey as TKey) } async batch(opStack: BatchDBOp[]): Promise { @@ -39,4 +39,8 @@ export class MapDB { return new MapDB(this._database) } + + open() { + return Promise.resolve() + } } diff --git a/packages/blockchain/src/types.ts b/packages/blockchain/src/types.ts index a5a8958cc4..7b44fb0303 100644 --- a/packages/blockchain/src/types.ts +++ b/packages/blockchain/src/types.ts @@ -1,8 +1,8 @@ -import { DB } from '@ethereumjs/util' import type { Consensus } from './consensus' import type { GenesisState } from './genesisStates' import type { Block, BlockHeader } from '@ethereumjs/block' import type { Common } from '@ethereumjs/common' +import type { DB } from '@ethereumjs/util' export type OnBlock = (block: Block, reorg: boolean) => Promise | void diff --git a/packages/blockchain/test/index.spec.ts b/packages/blockchain/test/index.spec.ts index 3b9b3d5a63..fffe1c4ea5 100644 --- a/packages/blockchain/test/index.spec.ts +++ b/packages/blockchain/test/index.spec.ts @@ -1,17 +1,16 @@ import { Block, BlockHeader } from '@ethereumjs/block' import { Chain, Common, Hardfork } from '@ethereumjs/common' import { bytesToHex, equalsBytes, hexToBytes, utf8ToBytes } from 'ethereum-cryptography/utils' -import { MemoryLevel } from 'memory-level' import * as tape from 'tape' import { Blockchain } from '../src' +import { MapDB } from '../src/db/map' import * as blocksData from './testdata/blocks_mainnet.json' import * as testDataPreLondon from './testdata/testdata_pre-london.json' import { createTestDB, generateBlockchain, generateBlocks, isConsecutive } from './util' import type { BlockOptions } from '@ethereumjs/block' -import { MapDB } from '@ethereumjs/trie' tape('blockchain test', (t) => { t.test('should not crash on getting head of a blockchain without a genesis', async (st) => { @@ -214,14 +213,20 @@ tape('blockchain test', (t) => { await blockchain.getBlock(5) st.fail('should throw an exception') } catch (e: any) { - st.ok(e.message.includes('NotFound'), `should throw for non-existing block-by-number request`) + st.ok( + e.message.includes('not found in DB'), + `should throw for non-existing block-by-number request` + ) } try { await blockchain.getBlock(hexToBytes('1234')) st.fail('should throw an exception') } catch (e: any) { - st.ok(e.message.includes('NotFound'), `should throw for non-existing block-by-hash request`) + st.ok( + e.message.includes('not found in DB'), + `should throw for non-existing block-by-hash request` + ) } st.end() @@ -528,6 +533,7 @@ tape('blockchain test', (t) => { await blockchain.getBlock(BigInt(1)) const block2HeaderValuesArray = blocks[2].header.raw() + block2HeaderValuesArray[1] = new Uint8Array(32) const block2Header = BlockHeader.fromValuesArray(block2HeaderValuesArray, { common: blocks[2]._common, diff --git a/packages/blockchain/test/util.ts b/packages/blockchain/test/util.ts index d23f59f373..247754e436 100644 --- a/packages/blockchain/test/util.ts +++ b/packages/blockchain/test/util.ts @@ -1,12 +1,14 @@ import { Block, BlockHeader } from '@ethereumjs/block' import { Chain, Common, Hardfork } from '@ethereumjs/common' import { RLP } from '@ethereumjs/rlp' -import { DB, toBytes } from '@ethereumjs/util' +import { toBytes } from '@ethereumjs/util' import { keccak256 } from 'ethereum-cryptography/keccak' import { bytesToHex, equalsBytes, hexToBytes, utf8ToBytes } from 'ethereum-cryptography/utils' -import { MemoryLevel } from 'memory-level' import { Blockchain } from '../src' +import { MapDB } from '../src/db/map' + +import type { DB } from '@ethereumjs/util' export const generateBlocks = (numberOfBlocks: number, existingBlocks?: Block[]): Block[] => { const blocks = existingBlocks ? existingBlocks : [] @@ -118,34 +120,27 @@ export const createTestDB = async (): Promise< > => { const common = new Common({ chain: Chain.Mainnet, hardfork: Hardfork.Chainstart }) const genesis = Block.fromBlockData({ header: { number: 0 } }, { common }) - const db = new MemoryLevel() + const db = new MapDB() + await db.batch([ { type: 'put', key: hexToBytes('6800000000000000006e'), - keyEncoding: 'view', - valueEncoding: 'view', value: genesis.hash(), }, { type: 'put', key: hexToBytes('48d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3'), - keyEncoding: 'view', - valueEncoding: 'view', value: hexToBytes('00'), }, { type: 'put', key: 'LastHeader', - keyEncoding: 'view', - valueEncoding: 'view', value: genesis.hash(), }, { type: 'put', key: 'LastBlock', - keyEncoding: 'view', - valueEncoding: 'view', value: genesis.hash(), }, { @@ -153,8 +148,6 @@ export const createTestDB = async (): Promise< key: hexToBytes( '680000000000000000d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3' ), - keyEncoding: 'view', - valueEncoding: 'view', value: genesis.header.serialize(), }, { @@ -162,8 +155,6 @@ export const createTestDB = async (): Promise< key: hexToBytes( '680000000000000000d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa374' ), - keyEncoding: 'view', - valueEncoding: 'view', value: RLP.encode(toBytes(17179869184)), }, { @@ -171,14 +162,11 @@ export const createTestDB = async (): Promise< key: hexToBytes( '620000000000000000d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3' ), - keyEncoding: 'view', - valueEncoding: 'view', value: RLP.encode(genesis.raw().slice(1)), }, { type: 'put', key: 'heads', - valueEncoding: 'json', value: { head0: bytesToHex(Uint8Array.from([171, 205])) }, }, ]) diff --git a/packages/client/bin/cli.ts b/packages/client/bin/cli.ts index 5f5eb39125..c8c6299915 100755 --- a/packages/client/bin/cli.ts +++ b/packages/client/bin/cli.ts @@ -6,7 +6,6 @@ import { Chain, Common, ConsensusAlgorithm, Hardfork } from '@ethereumjs/common' import { RLP } from '@ethereumjs/rlp' import { Address, - DB, bytesToHex, bytesToPrefixedHexString, hexStringToBytes, @@ -25,6 +24,7 @@ import * as readline from 'readline' import { EthereumClient } from '../lib/client' import { Config, DataDirectory, SyncMode } from '../lib/config' +import { LevelDB } from '../lib/execution/level' import { getLogger } from '../lib/logging' import { Event } from '../lib/types' import { parseMultiaddrs } from '../lib/util' @@ -425,7 +425,7 @@ async function startClient(config: Config, customGenesisState?: GenesisState) { if (customGenesisState !== undefined) { const validateConsensus = config.chainCommon.consensusAlgorithm() === ConsensusAlgorithm.Clique blockchain = await Blockchain.create({ - db: dbs.chainDB as any, + db: new LevelDB(dbs.chainDB), genesisState: customGenesisState, common: config.chainCommon, hardforkByHeadBlockNumber: true, diff --git a/packages/client/browser/index.ts b/packages/client/browser/index.ts index 71744f2191..9527bf71c2 100644 --- a/packages/client/browser/index.ts +++ b/packages/client/browser/index.ts @@ -5,6 +5,7 @@ import { Level } from 'level' import { EthereumClient } from '../lib/client' import { Config } from '../lib/config' +import { LevelDB } from '../lib/execution/level' import { parseMultiaddrs } from '../lib/util' import { getLogger } from './logging' @@ -75,7 +76,7 @@ export async function createClient(args: any) { ) const blockchain = await Blockchain.create({ - db: chainDB, + db: new LevelDB(chainDB), common: config.chainCommon, hardforkByHeadBlockNumber: true, validateBlocks: true, diff --git a/packages/client/lib/blockchain/chain.ts b/packages/client/lib/blockchain/chain.ts index 2d3b6becd4..6260441e9a 100644 --- a/packages/client/lib/blockchain/chain.ts +++ b/packages/client/lib/blockchain/chain.ts @@ -3,6 +3,7 @@ import { Blockchain } from '@ethereumjs/blockchain' import { ConsensusAlgorithm, Hardfork } from '@ethereumjs/common' import { equalsBytes } from 'ethereum-cryptography/utils' +import { LevelDB } from '../execution/level' import { Event } from '../types' import type { Config } from '../config' @@ -133,7 +134,7 @@ export class Chain { options.blockchain = options.blockchain ?? new (Blockchain as any)({ - db: options.chainDB, + db: new LevelDB(options.chainDB), common: options.config.chainCommon, hardforkByHeadBlockNumber: true, validateBlocks: true, @@ -213,7 +214,7 @@ export class Chain { */ async open(): Promise { if (this.opened) return false - await (this.blockchain.db as any)?.open() + await this.blockchain.db.open() await (this.blockchain as any)._init() this.opened = true await this.update(false) @@ -231,7 +232,7 @@ export class Chain { async close(): Promise { if (!this.opened) return false this.reset() - await (this.blockchain.db as any)?.close() + await (this.blockchain.db as any)?.close?.() this.opened = false } diff --git a/packages/client/lib/execution/level.ts b/packages/client/lib/execution/level.ts index c574ed6d65..4c2b6d318c 100644 --- a/packages/client/lib/execution/level.ts +++ b/packages/client/lib/execution/level.ts @@ -26,10 +26,12 @@ export class LevelDB implements DB { /** * @inheritDoc */ - async get(key: Uint8Array): Promise { - let value = null + // @ts-expect-error + async get(key: Uint8Array): Promise { + let value try { value = await this._leveldb.get(key, ENCODING_OPTS) + if (value === null) return undefined } catch (error: any) { // https://github.com/Level/abstract-level/blob/915ad1317694d0ce8c580b5ab85d81e1e78a3137/abstract-level.js#L309 // This should be `true` if the error came from LevelDB @@ -38,7 +40,7 @@ export class LevelDB implements DB { throw error } } - return value as Uint8Array | null + return value } /** @@ -66,6 +68,11 @@ export class LevelDB implements DB { * @inheritDoc */ copy(): DB { + //@ts-expect-error return new LevelDB(this._leveldb) } + + open() { + return this._leveldb.open() + } } diff --git a/packages/client/lib/execution/vmexecution.ts b/packages/client/lib/execution/vmexecution.ts index 297c13b766..54d356a97c 100644 --- a/packages/client/lib/execution/vmexecution.ts +++ b/packages/client/lib/execution/vmexecution.ts @@ -21,6 +21,7 @@ import { ReceiptsManager } from './receipt' import type { ExecutionOptions } from './execution' import type { Block } from '@ethereumjs/block' +import type { DB } from '@ethereumjs/util' import type { RunBlockOpts, TxReceipt } from '@ethereumjs/vm' export class VMExecution extends Execution { @@ -49,7 +50,7 @@ export class VMExecution extends Execution { if (this.config.vm === undefined) { const trie = new Trie({ - db: new LevelDB(this.stateDB), + db: new LevelDB(this.stateDB) as DB, useKeyHashing: true, cacheSize: this.config.trieCache, }) diff --git a/packages/client/lib/miner/miner.ts b/packages/client/lib/miner/miner.ts index ba673c2e05..b59064c5d6 100644 --- a/packages/client/lib/miner/miner.ts +++ b/packages/client/lib/miner/miner.ts @@ -5,6 +5,7 @@ import { bytesToPrefixedHexString } from '@ethereumjs/util' import { bytesToHex, equalsBytes } from 'ethereum-cryptography/utils' import { MemoryLevel } from 'memory-level' +import { LevelDB } from '../execution/level' import { Event } from '../types' import type { Config } from '../config' @@ -65,7 +66,7 @@ export class Miner { ((this.config.chainCommon.consensusConfig() as CliqueConfig).period ?? this.DEFAULT_PERIOD) * 1000 // defined in ms for setTimeout use if (this.config.chainCommon.consensusType() === ConsensusType.ProofOfWork) { - this.ethash = new Ethash(new MemoryLevel()) + this.ethash = new Ethash(new LevelDB(new MemoryLevel()) as any) } } diff --git a/packages/client/lib/sync/fetcher/accountfetcher.ts b/packages/client/lib/sync/fetcher/accountfetcher.ts index 4aea59c10b..3ba858beab 100644 --- a/packages/client/lib/sync/fetcher/accountfetcher.ts +++ b/packages/client/lib/sync/fetcher/accountfetcher.ts @@ -25,6 +25,7 @@ import type { EventBusType } from '../../types' import type { FetcherOptions } from './fetcher' import type { StorageRequest } from './storagefetcher' import type { Job } from './types' +import type { DB } from '@ethereumjs/util' import type { Debugger } from 'debug' type AccountDataResponse = AccountData[] & { completed?: boolean } @@ -202,7 +203,7 @@ export class AccountFetcher extends Fetcher } } - const trie = new Trie({ db: new LevelDB() }) + const trie = new Trie({ db: new LevelDB() as DB }) const keys = accounts.map((acc: any) => acc.hash) const values = accounts.map((acc: any) => accountBodyToRLP(acc.body)) // convert the request to the right values diff --git a/packages/client/lib/sync/fetcher/bytecodefetcher.ts b/packages/client/lib/sync/fetcher/bytecodefetcher.ts index c79bfb75d6..99507407ce 100644 --- a/packages/client/lib/sync/fetcher/bytecodefetcher.ts +++ b/packages/client/lib/sync/fetcher/bytecodefetcher.ts @@ -1,6 +1,6 @@ import { CODEHASH_PREFIX } from '@ethereumjs/statemanager' import { Trie } from '@ethereumjs/trie' -import { BatchDBOp, bytesToHex, concatBytes, equalsBytes } from '@ethereumjs/util' +import { bytesToHex, concatBytes, equalsBytes } from '@ethereumjs/util' import { debug as createDebugLogger } from 'debug' import { keccak256 } from 'ethereum-cryptography/keccak' @@ -9,6 +9,7 @@ import { Fetcher } from './fetcher' import type { Peer } from '../../net/peer' import type { FetcherOptions } from './fetcher' import type { Job } from './types' +import type { BatchDBOp } from '@ethereumjs/util' import type { Debugger } from 'debug' type ByteCodeDataResponse = Uint8Array[] & { completed?: boolean } diff --git a/packages/client/lib/sync/fetcher/storagefetcher.ts b/packages/client/lib/sync/fetcher/storagefetcher.ts index 672c125144..ffbc2f1639 100644 --- a/packages/client/lib/sync/fetcher/storagefetcher.ts +++ b/packages/client/lib/sync/fetcher/storagefetcher.ts @@ -18,6 +18,7 @@ import type { Peer } from '../../net/peer' import type { StorageData } from '../../net/protocol/snapprotocol' import type { FetcherOptions } from './fetcher' import type { Job } from './types' +import type { DB } from '@ethereumjs/util' import type { Debugger } from 'debug' const TOTAL_RANGE_END = BigInt(2) ** BigInt(256) - BigInt(1) @@ -124,7 +125,7 @@ export class StorageFetcher extends Fetcher slot.hash) const values = slots.map((slot: any) => slot.body) return await trie.verifyRangeProof( @@ -155,7 +156,7 @@ export class StorageFetcher extends Fetcher { return { diff --git a/packages/client/test/net/protocol/snapprotocol.spec.ts b/packages/client/test/net/protocol/snapprotocol.spec.ts index 9dca7985b0..76b5a34003 100644 --- a/packages/client/test/net/protocol/snapprotocol.spec.ts +++ b/packages/client/test/net/protocol/snapprotocol.spec.ts @@ -18,6 +18,8 @@ import { Chain } from '../../../lib/blockchain' import { Config } from '../../../lib/config' import { LevelDB } from '../../../lib/execution/level' import { SnapProtocol } from '../../../lib/net/protocol' + +import type { DB } from '@ethereumjs/util' ;(BigInt.prototype as any).toJSON = function () { return this.toString() } @@ -200,7 +202,7 @@ tape('[SnapProtocol]', (t) => { resData ) - const trie = new Trie({ db: new LevelDB() }) + const trie = new Trie({ db: new LevelDB() as DB }) try { const keys = accounts.map((acc: any) => acc.hash) const values = accounts.map((acc: any) => accountBodyToRLP(acc.body)) @@ -340,7 +342,7 @@ tape('[SnapProtocol]', (t) => { // lastAccount const lastAccountSlots = slots[0] const lastAccountStorageRoot = (lastAccount.body as any)[2] - const trie = new Trie({ db: new LevelDB() }) + const trie = new Trie({ db: new LevelDB() as DB }) try { const keys = lastAccountSlots.map((acc: any) => acc.hash) const values = lastAccountSlots.map((acc: any) => acc.body) diff --git a/packages/client/test/rpc/eth/getTransactionByBlockHashAndIndex.spec.ts b/packages/client/test/rpc/eth/getTransactionByBlockHashAndIndex.spec.ts index 1df8700f6b..b15158871f 100644 --- a/packages/client/test/rpc/eth/getTransactionByBlockHashAndIndex.spec.ts +++ b/packages/client/test/rpc/eth/getTransactionByBlockHashAndIndex.spec.ts @@ -62,7 +62,7 @@ tape(`${method}: call with unknown block hash`, async (t) => { const mockTxIndex = '0x1' const req = params(method, [mockBlockHash, mockTxIndex]) - const expectRes = checkError(t, INVALID_PARAMS, 'NotFound') + const expectRes = checkError(t, INVALID_PARAMS, 'not found in DB') await baseRequest(t, server, req, 200, expectRes) }) diff --git a/packages/client/test/sim/simutils.ts b/packages/client/test/sim/simutils.ts index e410e14f21..5d432a9ea3 100644 --- a/packages/client/test/sim/simutils.ts +++ b/packages/client/test/sim/simutils.ts @@ -20,6 +20,7 @@ import * as net from 'node:net' import { EthereumClient } from '../../lib/client' import { Config } from '../../lib/config' +import { LevelDB } from '../../lib/execution/level' import type { Common } from '@ethereumjs/common' import type { TxOptions } from '@ethereumjs/tx' @@ -432,7 +433,7 @@ export async function createInlineClient(config: any, common: any, customGenesis ) const blockchain = await Blockchain.create({ - db: chainDB, + db: new LevelDB(chainDB), genesisState: customGenesisState, common: config.chainCommon, hardforkByHeadBlockNumber: true, diff --git a/packages/ethash/src/index.ts b/packages/ethash/src/index.ts index 217f374579..a26855657f 100644 --- a/packages/ethash/src/index.ts +++ b/packages/ethash/src/index.ts @@ -25,7 +25,7 @@ import { } from './util' import type { BlockData, HeaderData } from '@ethereumjs/block' -import type { AbstractLevel } from 'abstract-level' +import type { DB } from '@ethereumjs/util' function xor(a: Uint8Array, b: Uint8Array) { const len = Math.max(a.length, b.length) @@ -154,27 +154,16 @@ export class Miner { } } -export type EthashCacheDB = AbstractLevel< - string | Uint8Array, - string | Uint8Array, - { - cache: string[] - fullSize: number - cacheSize: number - seed: string - } -> - export class Ethash { dbOpts: Object - cacheDB?: EthashCacheDB + cacheDB?: DB cache: Uint8Array[] epoc?: number fullSize?: number cacheSize?: number seed?: Uint8Array - constructor(cacheDB?: EthashCacheDB) { + constructor(cacheDB?: DB) { this.dbOpts = { valueEncoding: 'json', } @@ -302,21 +291,15 @@ export class Ethash { if (epoc === 0) { return [zeros(32), 0] } - let data - try { - const dbData = await this.cacheDB!.get(epoc, this.dbOpts) - data = { - cache: dbData.cache.map((el) => hexToBytes(el)), + + const dbData = await this.cacheDB!.get(epoc) + if (dbData !== undefined) { + const data = { + cache: dbData.cache.map((el: string) => hexToBytes(el)), fullSize: dbData.fullSize, cacheSize: dbData.cacheSize, seed: hexToBytes(dbData.seed), } - } catch (error: any) { - if (error.code !== 'LEVEL_NOT_FOUND') { - throw error - } - } - if (data) { return [data.seed, epoc] } else { return findLastSeed(epoc - 1) @@ -324,20 +307,15 @@ export class Ethash { } let data - try { - const dbData = await this.cacheDB!.get(epoc, this.dbOpts) + const dbData = await this.cacheDB!.get(epoc) + if (dbData !== undefined) { data = { - cache: dbData.cache.map((el) => hexToBytes(el)), + cache: dbData.cache.map((el: string) => hexToBytes(el)), fullSize: dbData.fullSize, cacheSize: dbData.cacheSize, seed: hexToBytes(dbData.seed), } - } catch (error: any) { - if (error.code !== 'LEVEL_NOT_FOUND') { - throw error - } } - if (!data) { this.cacheSize = await getCacheSize(epoc) this.fullSize = await getFullSize(epoc) @@ -346,16 +324,12 @@ export class Ethash { this.seed = getSeed(seed, foundEpoc, epoc) const cache = this.mkcache(this.cacheSize!, this.seed!) // store the generated cache - await this.cacheDB!.put( - epoc, - { - cacheSize: this.cacheSize, - fullSize: this.fullSize, - seed: bytesToHex(this.seed), - cache: cache.map((el) => bytesToHex(el)), - }, - this.dbOpts - ) + await this.cacheDB!.put(epoc as any, { + cacheSize: this.cacheSize, + fullSize: this.fullSize, + seed: bytesToHex(this.seed), + cache: cache.map((el) => bytesToHex(el)), + }) } else { this.cache = data.cache.map((a: Uint8Array) => { return Uint8Array.from(a) diff --git a/packages/ethash/src/level.ts b/packages/ethash/src/level.ts new file mode 100644 index 0000000000..1647ace310 --- /dev/null +++ b/packages/ethash/src/level.ts @@ -0,0 +1,69 @@ +import { MemoryLevel } from 'memory-level' + +import type { BatchDBOp, DB } from '@ethereumjs/util' +import type { AbstractLevel } from 'abstract-level' + +const ENCODING_OPTS = { valueEncoding: 'json' } + +export class LevelDB implements DB { + readonly _leveldb: AbstractLevel< + string | Uint8Array, + string | Uint8Array, + { + cache: string[] + fullSize: number + cacheSize: number + seed: string + } + > + + constructor( + leveldb?: AbstractLevel< + string | Uint8Array, + string | Uint8Array, + { + cache: string[] + fullSize: number + cacheSize: number + seed: string + } + > + ) { + this._leveldb = leveldb ?? new MemoryLevel(ENCODING_OPTS) + } + + async get(key: any): Promise { + let value + try { + value = await this._leveldb.get(key, ENCODING_OPTS) + } catch (error: any) { + // https://github.com/Level/abstract-level/blob/915ad1317694d0ce8c580b5ab85d81e1e78a3137/abstract-level.js#L309 + // This should be `true` if the error came from LevelDB + // so we can check for `NOT true` to identify any non-404 errors + if (error.notFound !== true) { + throw error + } + } + return value + } + + async put(key: Uint8Array, val: any): Promise { + await this._leveldb.put(key, val, ENCODING_OPTS) + } + + async del(key: any): Promise { + await this._leveldb.del(key) + } + + async batch(opStack: BatchDBOp[]): Promise { + await this._leveldb.batch(opStack, ENCODING_OPTS) + } + + copy(): DB { + return new LevelDB(this._leveldb) + } + + open() { + return this._leveldb.open() + } +} diff --git a/packages/ethash/test/block.spec.ts b/packages/ethash/test/block.spec.ts index 11aa520e7a..0793837432 100644 --- a/packages/ethash/test/block.spec.ts +++ b/packages/ethash/test/block.spec.ts @@ -7,10 +7,11 @@ import { MemoryLevel } from 'memory-level' import * as tape from 'tape' import { Ethash } from '../src' +import { LevelDB } from '../src/level' import type { BlockBytes } from '@ethereumjs/block' -const cacheDB = new MemoryLevel() +const cacheDB = new LevelDB(new MemoryLevel()) const { validBlockRlp, invalidBlockRlp } = require('./ethash_block_rlp_tests.json') diff --git a/packages/ethash/test/miner.spec.ts b/packages/ethash/test/miner.spec.ts index 2d01dd51c3..3b9765c4fa 100644 --- a/packages/ethash/test/miner.spec.ts +++ b/packages/ethash/test/miner.spec.ts @@ -4,9 +4,10 @@ import { MemoryLevel } from 'memory-level' import * as tape from 'tape' import { Ethash } from '../src' +import { LevelDB } from '../src/level' import type { BlockHeader } from '@ethereumjs/block' -const cacheDb = new MemoryLevel() +const cacheDb = new LevelDB(new MemoryLevel()) const common = new Common({ chain: Chain.Ropsten, hardfork: Hardfork.Petersburg }) tape('Check if miner works as expected', async function (t) { diff --git a/packages/trie/recipes/level-legacy.ts b/packages/trie/recipes/level-legacy.ts index ff37097c8b..dc880953c5 100644 --- a/packages/trie/recipes/level-legacy.ts +++ b/packages/trie/recipes/level-legacy.ts @@ -1,6 +1,6 @@ import level from 'level-mem' -import type { BatchDBOp, DB } from '@ethereumjs/trie' +import type { BatchDBOp, DB } from '@ethereumjs/util' import type { LevelUp } from 'levelup' const ENCODING_OPTS = { keyEncoding: 'view', valueEncoding: 'view' } @@ -12,8 +12,8 @@ export class LevelDB implements DB { this._leveldb = leveldb ?? level() } - async get(key: Uint8Array): Promise { - let value: Uint8Array | null = null + async get(key: Uint8Array): Promise { + let value try { value = await this._leveldb.get(key, ENCODING_OPTS) } catch (error: any) { @@ -42,4 +42,8 @@ export class LevelDB implements DB { copy(): DB { return new LevelDB(this._leveldb) } + + open() { + return Promise.resolve() + } } diff --git a/packages/trie/recipes/level.ts b/packages/trie/recipes/level.ts index 6d6814d70d..00ba87ef08 100644 --- a/packages/trie/recipes/level.ts +++ b/packages/trie/recipes/level.ts @@ -1,6 +1,6 @@ import { MemoryLevel } from 'memory-level' -import type { BatchDBOp, DB } from '@ethereumjs/trie' +import type { BatchDBOp, DB } from '@ethereumjs/util' import type { AbstractLevel } from 'abstract-level' const ENCODING_OPTS = { keyEncoding: 'view', valueEncoding: 'view' } @@ -22,8 +22,8 @@ export class LevelDB implements DB { this._leveldb = leveldb ?? new MemoryLevel(ENCODING_OPTS) } - async get(key: Uint8Array): Promise { - let value: Uint8Array | null = null + async get(key: Uint8Array): Promise { + let value try { value = await this._leveldb.get(key, ENCODING_OPTS) } catch (error: any) { diff --git a/packages/trie/recipes/lmdb.ts b/packages/trie/recipes/lmdb.ts index 07d2ba4290..d4f3e04f8f 100644 --- a/packages/trie/recipes/lmdb.ts +++ b/packages/trie/recipes/lmdb.ts @@ -1,6 +1,6 @@ import { Database, open } from 'lmdb' -import type { BatchDBOp, DB } from '@ethereumjs/trie' +import type { BatchDBOp, DB } from '@ethereumjs/util' export class LMDB implements DB { readonly _path: string @@ -15,7 +15,7 @@ export class LMDB implements DB { }) } - async get(key: Uint8Array): Promise { + async get(key: Uint8Array): Promise { return this._database.get(key) } @@ -42,4 +42,8 @@ export class LMDB implements DB { copy(): DB { return new LMDB(this._path) } + + open() { + return Promise.resolve() + } } diff --git a/packages/trie/src/db/checkpoint.ts b/packages/trie/src/db/checkpoint.ts index 9991d93cf2..f4ab7a71bf 100644 --- a/packages/trie/src/db/checkpoint.ts +++ b/packages/trie/src/db/checkpoint.ts @@ -1,6 +1,7 @@ -import { BatchDBOp, DB, bytesToHex, hexStringToBytes } from '@ethereumjs/util' +import { bytesToHex, hexStringToBytes } from '@ethereumjs/util' import type { Checkpoint, CheckpointDBOpts } from '../types' +import type { BatchDBOp, DB } from '@ethereumjs/util' import type LRUCache from 'lru-cache' const LRU = require('lru-cache') @@ -14,7 +15,7 @@ export class CheckpointDB implements DB { public db: DB public readonly cacheSize: number - protected _cache?: LRUCache + protected _cache?: LRUCache _stats = { cache: { @@ -85,7 +86,7 @@ export class CheckpointDB implements DB { // This was the final checkpoint, we should now commit and flush everything to disk const batchOp: BatchDBOp[] = [] for (const [key, value] of keyValueMap.entries()) { - if (value === null) { + if (value === undefined) { batchOp.push({ type: 'del', key: hexStringToBytes(key), @@ -119,7 +120,7 @@ export class CheckpointDB implements DB { /** * @inheritDoc */ - async get(key: Uint8Array): Promise { + async get(key: Uint8Array): Promise { const keyHex = bytesToHex(key) if (this._cache !== undefined) { const value = this._cache.get(keyHex) @@ -132,9 +133,8 @@ export class CheckpointDB implements DB { // Lookup the value in our diff cache. We return the latest checkpointed value (which should be the value on disk) for (let index = this.checkpoints.length - 1; index >= 0; index--) { - const value = this.checkpoints[index].keyValueMap.get(bytesToHex(key)) - if (value !== undefined) { - return value + if (this.checkpoints[index].keyValueMap.has(bytesToHex(key))) { + return this.checkpoints[index].keyValueMap.get(bytesToHex(key)) } } // Nothing has been found in diff cache, look up from disk @@ -177,12 +177,12 @@ export class CheckpointDB implements DB { async del(key: Uint8Array): Promise { const keyHex = bytesToHex(key) if (this._cache !== undefined) { - this._cache.set(keyHex, null) + this._cache.set(keyHex, undefined) this._stats.cache.writes += 1 } if (this.hasCheckpoints()) { // delete the value in the current diff cache - this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(bytesToHex(key), null) + this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(bytesToHex(key), undefined) } else { // delete the value on disk await this.db.del(key) @@ -231,4 +231,8 @@ export class CheckpointDB implements DB { copy(): CheckpointDB { return new CheckpointDB({ db: this.db, cacheSize: this.cacheSize }) } + + open() { + return Promise.resolve() + } } diff --git a/packages/trie/src/db/map.ts b/packages/trie/src/db/map.ts index 22bb653b51..75ea2cf3a9 100644 --- a/packages/trie/src/db/map.ts +++ b/packages/trie/src/db/map.ts @@ -1,6 +1,7 @@ -import { DB, BatchDBOp } from '@ethereumjs/util' import { bytesToHex } from 'ethereum-cryptography/utils' +import type { BatchDBOp, DB } from '@ethereumjs/util' + export class MapDB implements DB { _database: Map @@ -8,10 +9,8 @@ export class MapDB implements DB { this._database = database ?? new Map() } - async get(key: Uint8Array): Promise { - const result = this._database.get(bytesToHex(key)) - - return result ?? null + async get(key: Uint8Array): Promise { + return this._database.get(bytesToHex(key)) } async put(key: Uint8Array, val: Uint8Array): Promise { @@ -37,4 +36,8 @@ export class MapDB implements DB { copy(): DB { return new MapDB(this._database) } + + open() { + return Promise.resolve(undefined) + } } diff --git a/packages/trie/src/trie/trie.ts b/packages/trie/src/trie/trie.ts index 717a7d85f7..ac4d1b3638 100644 --- a/packages/trie/src/trie/trie.ts +++ b/packages/trie/src/trie/trie.ts @@ -1,12 +1,4 @@ -import { - BatchDBOp, - DB, - PutBatch, - RLP_EMPTY_STRING, - bytesToHex, - bytesToUtf8, - equalsBytes, -} from '@ethereumjs/util' +import { RLP_EMPTY_STRING, bytesToHex, bytesToUtf8, equalsBytes } from '@ethereumjs/util' import { keccak256 } from 'ethereum-cryptography/keccak' import { CheckpointDB, MapDB } from '../db' @@ -28,6 +20,7 @@ import type { TrieOpts, TrieOptsWithDefaults, } from '../types' +import type { BatchDBOp, DB, PutBatch } from '@ethereumjs/util' interface Path { node: TrieNode | null diff --git a/packages/trie/src/types.ts b/packages/trie/src/types.ts index c5897eacd6..891fb01ed1 100644 --- a/packages/trie/src/types.ts +++ b/packages/trie/src/types.ts @@ -2,7 +2,7 @@ import { utf8ToBytes } from 'ethereum-cryptography/utils' import type { BranchNode, ExtensionNode, LeafNode } from './trie' import type { WalkController } from './util/walkController' -import { DB } from '@ethereumjs/util' +import type { DB } from '@ethereumjs/util' export type TrieNode = BranchNode | ExtensionNode | LeafNode @@ -95,7 +95,7 @@ export interface CheckpointDBOpts { export type Checkpoint = { // We cannot use a Uint8Array => Uint8Array map directly. If you create two Uint8Arrays with the same internal value, // then when setting a value on the Map, it actually creates two indices. - keyValueMap: Map + keyValueMap: Map root: Uint8Array } diff --git a/packages/trie/test/db/checkpoint.spec.ts b/packages/trie/test/db/checkpoint.spec.ts index 486a939921..33a1a0161d 100644 --- a/packages/trie/test/db/checkpoint.spec.ts +++ b/packages/trie/test/db/checkpoint.spec.ts @@ -1,8 +1,10 @@ -import { BatchDBOp, hexStringToBytes, utf8ToBytes } from '@ethereumjs/util' +import { hexStringToBytes, utf8ToBytes } from '@ethereumjs/util' import * as tape from 'tape' import { CheckpointDB, MapDB } from '../../src' +import type { BatchDBOp } from '@ethereumjs/util' + tape('DB tests', (t) => { const k = utf8ToBytes('k1') const v = utf8ToBytes('v1') @@ -15,7 +17,7 @@ tape('DB tests', (t) => { await db.put(k, v) st.deepEqual(await db.get(k), v, 'before revert: v1') await db.revert() - st.deepEqual(await db.get(k), null, 'after revert: null') + st.deepEqual(await db.get(k), undefined, 'after revert: null') st.end() }) @@ -52,7 +54,7 @@ tape('DB tests', (t) => { st.deepEqual(await db.get(k), v, 'before CP: v1') db.checkpoint(hexStringToBytes('01')) await db.del(k) - st.deepEqual(await db.get(k), null, 'before revert: null') + st.deepEqual(await db.get(k), undefined, 'before revert: undefined') await db.revert() st.deepEqual(await db.get(k), v, 'after revert: v1') st.end() diff --git a/packages/trie/test/db/db.spec.ts b/packages/trie/test/db/db.spec.ts index a4a2d2fdea..fcf83b5a50 100644 --- a/packages/trie/test/db/db.spec.ts +++ b/packages/trie/test/db/db.spec.ts @@ -1,8 +1,10 @@ -import { BatchDBOp, equalsBytes, utf8ToBytes } from '@ethereumjs/util' +import { equalsBytes, utf8ToBytes } from '@ethereumjs/util' import * as tape from 'tape' import { MapDB } from '../../src' +import type { BatchDBOp } from '@ethereumjs/util' + tape('DB tests', (t) => { const db = new MapDB() diff --git a/packages/trie/test/proof/range.spec.ts b/packages/trie/test/proof/range.spec.ts index d11cf1b46c..8fd053b141 100644 --- a/packages/trie/test/proof/range.spec.ts +++ b/packages/trie/test/proof/range.spec.ts @@ -1,5 +1,4 @@ import { - DB, compareBytes, concatBytes, hexStringToBytes, @@ -11,6 +10,8 @@ import * as tape from 'tape' import { MapDB, Trie } from '../../src' +import type { DB } from '@ethereumjs/util' + // reference: https://github.com/ethereum/go-ethereum/blob/20356e57b119b4e70ce47665a71964434e15200d/trie/proof_test.go const TRIE_SIZE = 512 diff --git a/packages/trie/test/stream.spec.ts b/packages/trie/test/stream.spec.ts index 04b2ca528e..e7e49b8723 100644 --- a/packages/trie/test/stream.spec.ts +++ b/packages/trie/test/stream.spec.ts @@ -2,7 +2,8 @@ import { utf8ToBytes } from 'ethereum-cryptography/utils' import * as tape from 'tape' import { Trie } from '../src' -import { BatchDBOp } from '@ethereumjs/util' + +import type { BatchDBOp } from '@ethereumjs/util' tape('kv stream test', function (tester) { const it = tester.test diff --git a/packages/trie/test/trie/checkpoint.spec.ts b/packages/trie/test/trie/checkpoint.spec.ts index 6b6bc1b7f4..63568c535e 100644 --- a/packages/trie/test/trie/checkpoint.spec.ts +++ b/packages/trie/test/trie/checkpoint.spec.ts @@ -1,10 +1,12 @@ -import { BatchDBOp, bytesToHex, bytesToUtf8, equalsBytes, utf8ToBytes } from '@ethereumjs/util' +import { bytesToHex, bytesToUtf8, equalsBytes, utf8ToBytes } from '@ethereumjs/util' import { createHash } from 'crypto' import { keccak256 } from 'ethereum-cryptography/keccak' import * as tape from 'tape' import { MapDB, ROOT_DB_KEY, Trie } from '../../src' +import type { BatchDBOp } from '@ethereumjs/util' + tape('testing checkpoints', function (tester) { const it = tester.test diff --git a/packages/trie/test/trie/trie.spec.ts b/packages/trie/test/trie/trie.spec.ts index 8ffcf94607..ea846a732c 100644 --- a/packages/trie/test/trie/trie.spec.ts +++ b/packages/trie/test/trie/trie.spec.ts @@ -98,7 +98,7 @@ for (const { constructor, defaults, title } of [ }) // @ts-expect-error - st.equal(await trie._db.get(ROOT_DB_KEY), null) + st.equal(await trie._db.get(ROOT_DB_KEY), undefined) await trie.put(utf8ToBytes('foo'), utf8ToBytes('bar')) @@ -137,12 +137,12 @@ for (const { constructor, defaults, title } of [ }) // @ts-expect-error - st.equal(await trie._db.get(ROOT_DB_KEY), null) + st.equal(await trie._db.get(ROOT_DB_KEY), undefined) await trie.put(utf8ToBytes('do_not_persist_with_db'), utf8ToBytes('bar')) // @ts-expect-error - st.equal(await trie._db.get(ROOT_DB_KEY), null) + st.equal(await trie._db.get(ROOT_DB_KEY), undefined) st.end() } @@ -152,12 +152,12 @@ for (const { constructor, defaults, title } of [ const trie = await constructor.create({ ...defaults, useRootPersistence: true }) // @ts-expect-error - st.equal(await trie._db.get(ROOT_DB_KEY), null) + st.equal(await trie._db.get(ROOT_DB_KEY), undefined) await trie.put(utf8ToBytes('do_not_persist_without_db'), utf8ToBytes('bar')) // @ts-expect-error - st.notEqual(await trie._db.get(ROOT_DB_KEY), null) + st.notEqual(await trie._db.get(ROOT_DB_KEY), undefined) st.end() }) @@ -167,7 +167,7 @@ for (const { constructor, defaults, title } of [ const trie = await constructor.create({ ...defaults, db, useRootPersistence: true }) // @ts-expect-error - st.equal(await trie._db.get(ROOT_DB_KEY), null) + st.equal(await trie._db.get(ROOT_DB_KEY), undefined) await trie.put(utf8ToBytes('foo'), utf8ToBytes('bar')) // @ts-expect-error st.equal(bytesToHex(await trie._db.get(ROOT_DB_KEY)), EXPECTED_ROOTS) @@ -184,7 +184,7 @@ for (const { constructor, defaults, title } of [ useRootPersistence: true, }) // @ts-expect-error - st.equal(await empty._db.get(ROOT_DB_KEY), null) + st.equal(await empty._db.get(ROOT_DB_KEY), undefined) st.end() }) @@ -195,7 +195,7 @@ for (const { constructor, defaults, title } of [ try { await trie.put(BASE_DB_KEY, utf8ToBytes('bar')) st.fail("Attempting to set '__root__' should fail but it did not.") - } catch ({ message }) { + } catch ({ message }: any) { st.equal(message, "Attempted to set '__root__' key but it is not allowed.") } diff --git a/packages/util/src/db.ts b/packages/util/src/db.ts index 93a4d26a91..081a62f050 100644 --- a/packages/util/src/db.ts +++ b/packages/util/src/db.ts @@ -24,9 +24,9 @@ export interface DB< /** * Retrieves a raw value from db. * @param key - * @returns A Promise that resolves to `Uint8Array` if a value is found or `null` if no value is found. + * @returns A Promise that resolves to `Uint8Array` if a value is found or `undefined` if no value is found. */ - get(key: TKey): Promise + get(key: TKey): Promise /** * Writes a value directly to db. @@ -52,4 +52,10 @@ export interface DB< * to the **same** underlying db instance. */ copy(): DB + + /** + * Opens the database -- if applicable + */ + open(): Promise + // TODO - decide if we actually need open/close - it's not required for maps and Level automatically opens the DB when you instantiate it } diff --git a/packages/vm/test/api/runBlock.spec.ts b/packages/vm/test/api/runBlock.spec.ts index 85709e3cc2..5e8725de5c 100644 --- a/packages/vm/test/api/runBlock.spec.ts +++ b/packages/vm/test/api/runBlock.spec.ts @@ -237,7 +237,10 @@ tape('runBlock() -> API parameter usage/data errors', async (t) => { .runBlock({ block }) .then(() => t.fail('should have returned error')) .catch((e) => { - t.ok(e.code.includes('LEVEL_NOT_FOUND'), 'block failed validation due to no parent header') + t.ok( + e.message.includes('not found in DB'), + 'block failed validation due to no parent header' + ) }) }) diff --git a/packages/vm/test/api/utils.ts b/packages/vm/test/api/utils.ts index 8e7c6c6afc..6807315076 100644 --- a/packages/vm/test/api/utils.ts +++ b/packages/vm/test/api/utils.ts @@ -6,6 +6,7 @@ import { hexToBytes } from 'ethereum-cryptography/utils' import { MemoryLevel } from 'memory-level' import { VM } from '../../src/vm' +import { LevelDB } from '../level' import type { VMOpts } from '../../src/types' import type { Block } from '@ethereumjs/block' @@ -24,7 +25,7 @@ export async function setBalance(vm: VM, address: Address, balance = BigInt(1000 } export async function setupVM(opts: VMOpts & { genesisBlock?: Block } = {}) { - const db: any = new MemoryLevel() + const db: any = new LevelDB(new MemoryLevel()) const { common, genesisBlock } = opts if (opts.blockchain === undefined) { opts.blockchain = await Blockchain.create({ diff --git a/packages/vm/test/level.ts b/packages/vm/test/level.ts new file mode 100644 index 0000000000..4c2b6d318c --- /dev/null +++ b/packages/vm/test/level.ts @@ -0,0 +1,78 @@ +import { MemoryLevel } from 'memory-level' + +import type { BatchDBOp, DB } from '@ethereumjs/util' +import type { AbstractLevel } from 'abstract-level' + +export const ENCODING_OPTS = { keyEncoding: 'view', valueEncoding: 'view' } + +/** + * LevelDB is a thin wrapper around the underlying levelup db, + * which validates inputs and sets encoding type. + */ +export class LevelDB implements DB { + _leveldb: AbstractLevel + + /** + * Initialize a DB instance. If `leveldb` is not provided, DB + * defaults to an [in-memory store](https://github.com/Level/memdown). + * @param leveldb - An abstract-leveldown compliant store + */ + constructor( + leveldb?: AbstractLevel + ) { + this._leveldb = leveldb ?? new MemoryLevel(ENCODING_OPTS) + } + + /** + * @inheritDoc + */ + // @ts-expect-error + async get(key: Uint8Array): Promise { + let value + try { + value = await this._leveldb.get(key, ENCODING_OPTS) + if (value === null) return undefined + } catch (error: any) { + // https://github.com/Level/abstract-level/blob/915ad1317694d0ce8c580b5ab85d81e1e78a3137/abstract-level.js#L309 + // This should be `true` if the error came from LevelDB + // so we can check for `NOT true` to identify any non-404 errors + if (error.notFound !== true) { + throw error + } + } + return value + } + + /** + * @inheritDoc + */ + async put(key: Uint8Array, val: Uint8Array): Promise { + await this._leveldb.put(key, val, ENCODING_OPTS) + } + + /** + * @inheritDoc + */ + async del(key: Uint8Array): Promise { + await this._leveldb.del(key, ENCODING_OPTS) + } + + /** + * @inheritDoc + */ + async batch(opStack: BatchDBOp[]): Promise { + await this._leveldb.batch(opStack, ENCODING_OPTS) + } + + /** + * @inheritDoc + */ + copy(): DB { + //@ts-expect-error + return new LevelDB(this._leveldb) + } + + open() { + return this._leveldb.open() + } +} diff --git a/packages/vm/test/tester/runners/BlockchainTestsRunner.ts b/packages/vm/test/tester/runners/BlockchainTestsRunner.ts index 98cc69a2fd..8d4698bcd6 100644 --- a/packages/vm/test/tester/runners/BlockchainTestsRunner.ts +++ b/packages/vm/test/tester/runners/BlockchainTestsRunner.ts @@ -1,6 +1,7 @@ import { Block } from '@ethereumjs/block' import { Blockchain } from '@ethereumjs/blockchain' import { ConsensusAlgorithm } from '@ethereumjs/common' +import { LevelDB } from '@ethereumjs/ethash/dist/level' import { RLP } from '@ethereumjs/rlp' import { DefaultStateManager } from '@ethereumjs/statemanager' import { Trie } from '@ethereumjs/trie' @@ -34,7 +35,7 @@ export async function runBlockchainTest(options: any, testData: any, t: tape.Tes // fix for BlockchainTests/GeneralStateTests/stRandom/* testData.lastblockhash = stripHexPrefix(testData.lastblockhash) - let cacheDB = new Level('./.cachedb') + let cacheDB = new LevelDB(new Level('./.cachedb')) let state = new Trie({ useKeyHashing: true }) let stateManager = new DefaultStateManager({ trie: state, @@ -65,7 +66,7 @@ export async function runBlockchainTest(options: any, testData: any, t: tape.Tes } let blockchain = await Blockchain.create({ - db: new MemoryLevel(), + db: new LevelDB(new MemoryLevel()), common, validateBlocks: true, validateConsensus: validatePow, @@ -214,7 +215,7 @@ export async function runBlockchainTest(options: any, testData: any, t: tape.Tes throw e } - await cacheDB.close() + await cacheDB._leveldb.close() if (expectException !== false) { t.fail(`expected exception but test did not throw an exception: ${expectException}`) @@ -235,7 +236,7 @@ export async function runBlockchainTest(options: any, testData: any, t: tape.Tes const end = Date.now() const timeSpent = `${(end - begin) / 1000} secs` t.comment(`Time: ${timeSpent}`) - await cacheDB.close() + await cacheDB._leveldb.close() // @ts-ignore Explicitly delete objects for memory optimization (early GC) common = blockchain = state = stateManager = vm = cacheDB = null // eslint-disable-line From 98dd481b5a513b9c870dff28ecf6c9e441942b2e Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Thu, 4 May 2023 11:55:51 -0400 Subject: [PATCH 06/27] Add correct value encoding for keys in level --- packages/client/lib/execution/level.ts | 39 +++++++++++++++---- packages/client/test/blockchain/chain.spec.ts | 2 +- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/packages/client/lib/execution/level.ts b/packages/client/lib/execution/level.ts index 4c2b6d318c..b99e16e011 100644 --- a/packages/client/lib/execution/level.ts +++ b/packages/client/lib/execution/level.ts @@ -1,9 +1,10 @@ +import { RLP } from '@ethereumjs/rlp' import { MemoryLevel } from 'memory-level' import type { BatchDBOp, DB } from '@ethereumjs/util' import type { AbstractLevel } from 'abstract-level' -export const ENCODING_OPTS = { keyEncoding: 'view', valueEncoding: 'view' } +//export const ENCODING_OPTS = { keyEncoding: 'view', valueEncoding: 'view' } /** * LevelDB is a thin wrapper around the underlying levelup db, @@ -20,17 +21,31 @@ export class LevelDB implements DB { constructor( leveldb?: AbstractLevel ) { - this._leveldb = leveldb ?? new MemoryLevel(ENCODING_OPTS) + this._leveldb = leveldb ?? new MemoryLevel() } /** * @inheritDoc */ // @ts-expect-error - async get(key: Uint8Array): Promise { + async get(key: Uint8Array | string): Promise { let value + let encoding = undefined + // Set value encoding based on key type or specific key names so values are interpreted correctly by Level + if ( + key instanceof Uint8Array || + key === 'CliqueSigners' || + key === 'CliqueVotes' || + key === 'CliqueBlockSignersSnapshot' + ) + encoding = 'view' + if (key === 'heads' || typeof key === 'number') { + encoding = 'json' + } try { - value = await this._leveldb.get(key, ENCODING_OPTS) + value = await this._leveldb.get(key, { + valueEncoding: encoding, + }) if (value === null) return undefined } catch (error: any) { // https://github.com/Level/abstract-level/blob/915ad1317694d0ce8c580b5ab85d81e1e78a3137/abstract-level.js#L309 @@ -40,28 +55,36 @@ export class LevelDB implements DB { throw error } } + // eslint-disable-next-line + if (value instanceof Buffer) value = Uint8Array.from(value) return value } /** * @inheritDoc */ - async put(key: Uint8Array, val: Uint8Array): Promise { - await this._leveldb.put(key, val, ENCODING_OPTS) + async put(key: Uint8Array | string, val: Uint8Array | string | any): Promise { + let valEncoding: string + if (typeof val === 'string') { + valEncoding = 'string' + } else if (val instanceof Uint8Array) { + valEncoding = 'view' + } else valEncoding = 'json' + await this._leveldb.put(key, val, { valueEncoding: valEncoding }) } /** * @inheritDoc */ async del(key: Uint8Array): Promise { - await this._leveldb.del(key, ENCODING_OPTS) + await this._leveldb.del(key) } /** * @inheritDoc */ async batch(opStack: BatchDBOp[]): Promise { - await this._leveldb.batch(opStack, ENCODING_OPTS) + await this._leveldb.batch(opStack) } /** diff --git a/packages/client/test/blockchain/chain.spec.ts b/packages/client/test/blockchain/chain.spec.ts index 287c20b5c7..86afdf8e4d 100644 --- a/packages/client/test/blockchain/chain.spec.ts +++ b/packages/client/test/blockchain/chain.spec.ts @@ -23,7 +23,7 @@ tape('[Chain]', (t) => { await db.put(testKey, testValue) const value = await db.get(testKey) - t.equal(testValue, value, 'read value matches written value') + t.equal(value, testValue, 'read value matches written value') t.end() }) From 9c2e40d91622616875ee4e4935c79306e6a4e56b Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Thu, 4 May 2023 14:17:05 -0400 Subject: [PATCH 07/27] Apply correct db interface typing --- packages/blockchain/src/db/manager.ts | 11 ++++++----- packages/client/lib/execution/level.ts | 1 - packages/ethash/src/index.ts | 18 +++++++++--------- packages/util/src/db.ts | 17 ++++++++++------- 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/packages/blockchain/src/db/manager.ts b/packages/blockchain/src/db/manager.ts index 9da7d93cc6..7220dbd819 100644 --- a/packages/blockchain/src/db/manager.ts +++ b/packages/blockchain/src/db/manager.ts @@ -15,7 +15,7 @@ import { DBOp, DBTarget } from './operation' import type { DBOpData, DatabaseKey } from './operation' import type { BlockBodyBytes, BlockBytes, BlockOptions } from '@ethereumjs/block' import type { Common } from '@ethereumjs/common' -import type { DB } from '@ethereumjs/util' +import type { DB, DBObject } from '@ethereumjs/util' /** * @hidden @@ -36,7 +36,7 @@ export type CacheMap = { [key: string]: Cache } export class DBManager { private _cache: CacheMap private _common: Common - private _db: DB + private _db: DB constructor(db: DB, common: Common) { this._db = db @@ -54,15 +54,16 @@ export class DBManager { * Fetches iterator heads from the db. */ async getHeads(): Promise<{ [key: string]: Uint8Array }> { - const heads = await this.get(DBTarget.Heads) + const heads = (await this.get(DBTarget.Heads)) as DBObject if (heads === undefined) return heads + const decodedHeads: { [key: string]: Uint8Array } = {} for (const key of Object.keys(heads)) { // Heads are stored in DB as hex strings since Level converts Uint8Arrays // to nested JSON objects when they are included in a value being stored // in the DB - heads[key] = hexToBytes(heads[key]) + decodedHeads[key] = hexToBytes(heads[key] as string) } - return heads + return decodedHeads } /** diff --git a/packages/client/lib/execution/level.ts b/packages/client/lib/execution/level.ts index b99e16e011..6b0033bb3b 100644 --- a/packages/client/lib/execution/level.ts +++ b/packages/client/lib/execution/level.ts @@ -1,4 +1,3 @@ -import { RLP } from '@ethereumjs/rlp' import { MemoryLevel } from 'memory-level' import type { BatchDBOp, DB } from '@ethereumjs/util' diff --git a/packages/ethash/src/index.ts b/packages/ethash/src/index.ts index a26855657f..7df5122a4c 100644 --- a/packages/ethash/src/index.ts +++ b/packages/ethash/src/index.ts @@ -25,7 +25,7 @@ import { } from './util' import type { BlockData, HeaderData } from '@ethereumjs/block' -import type { DB } from '@ethereumjs/util' +import type { DB, DBObject } from '@ethereumjs/util' function xor(a: Uint8Array, b: Uint8Array) { const len = Math.max(a.length, b.length) @@ -156,14 +156,14 @@ export class Miner { export class Ethash { dbOpts: Object - cacheDB?: DB + cacheDB?: DB cache: Uint8Array[] epoc?: number fullSize?: number cacheSize?: number seed?: Uint8Array - constructor(cacheDB?: DB) { + constructor(cacheDB?: DB) { this.dbOpts = { valueEncoding: 'json', } @@ -295,10 +295,10 @@ export class Ethash { const dbData = await this.cacheDB!.get(epoc) if (dbData !== undefined) { const data = { - cache: dbData.cache.map((el: string) => hexToBytes(el)), + cache: (dbData.cache as string[]).map((el: string) => hexToBytes(el)), fullSize: dbData.fullSize, cacheSize: dbData.cacheSize, - seed: hexToBytes(dbData.seed), + seed: hexToBytes(dbData.seed as string), } return [data.seed, epoc] } else { @@ -310,10 +310,10 @@ export class Ethash { const dbData = await this.cacheDB!.get(epoc) if (dbData !== undefined) { data = { - cache: dbData.cache.map((el: string) => hexToBytes(el)), + cache: (dbData.cache as string[]).map((el: string) => hexToBytes(el)), fullSize: dbData.fullSize, cacheSize: dbData.cacheSize, - seed: hexToBytes(dbData.seed), + seed: hexToBytes(dbData.seed as string), } } if (!data) { @@ -334,8 +334,8 @@ export class Ethash { this.cache = data.cache.map((a: Uint8Array) => { return Uint8Array.from(a) }) - this.cacheSize = data.cacheSize - this.fullSize = data.fullSize + this.cacheSize = data.cacheSize as number + this.fullSize = data.fullSize as number this.seed = Uint8Array.from(data.seed) } } diff --git a/packages/util/src/db.ts b/packages/util/src/db.ts index 081a62f050..19c6ada280 100644 --- a/packages/util/src/db.ts +++ b/packages/util/src/db.ts @@ -1,25 +1,28 @@ +export type DBObject = { + [key: string]: string | string[] | number +} export type BatchDBOp< - TKey extends Uint8Array | string = Uint8Array, - TValue extends Uint8Array | string = Uint8Array + TKey extends Uint8Array | string | number = Uint8Array, + TValue extends Uint8Array | string | DBObject = Uint8Array > = PutBatch | DelBatch export interface PutBatch< - TKey extends Uint8Array | string = Uint8Array, - TValue extends Uint8Array | string = Uint8Array + TKey extends Uint8Array | string | number = Uint8Array, + TValue extends Uint8Array | string | DBObject = Uint8Array > { type: 'put' key: TKey value: TValue } -export interface DelBatch { +export interface DelBatch { type: 'del' key: TKey } export interface DB< - TKey extends Uint8Array | string = Uint8Array, - TValue extends Uint8Array | string = Uint8Array + TKey extends Uint8Array | string | number = Uint8Array, + TValue extends Uint8Array | string | DBObject = Uint8Array > { /** * Retrieves a raw value from db. From 29579c6a43dc337e0351c45ef1e726f3cfb51310 Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Thu, 4 May 2023 15:30:35 -0400 Subject: [PATCH 08/27] Fix level encoding for strings --- packages/client/lib/blockchain/chain.ts | 5 +++-- packages/client/lib/execution/level.ts | 10 +++++----- packages/client/test/blockchain/chain.spec.ts | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/client/lib/blockchain/chain.ts b/packages/client/lib/blockchain/chain.ts index 6260441e9a..1360a44403 100644 --- a/packages/client/lib/blockchain/chain.ts +++ b/packages/client/lib/blockchain/chain.ts @@ -7,6 +7,7 @@ import { LevelDB } from '../execution/level' import { Event } from '../types' import type { Config } from '../config' +import type { DB } from '@ethereumjs/util' import type { AbstractLevel } from 'abstract-level' /** @@ -99,7 +100,7 @@ export interface ChainHeaders { */ export class Chain { public config: Config - public chainDB: AbstractLevel + public chainDB: DB public blockchain: Blockchain public opened: boolean @@ -156,7 +157,7 @@ export class Chain { this.config = options.config this.blockchain = options.blockchain! - this.chainDB = this.blockchain.db as any + this.chainDB = this.blockchain.db this.opened = false } diff --git a/packages/client/lib/execution/level.ts b/packages/client/lib/execution/level.ts index 6b0033bb3b..c817a3b2ba 100644 --- a/packages/client/lib/execution/level.ts +++ b/packages/client/lib/execution/level.ts @@ -1,6 +1,6 @@ import { MemoryLevel } from 'memory-level' -import type { BatchDBOp, DB } from '@ethereumjs/util' +import type { BatchDBOp, DB, DBObject } from '@ethereumjs/util' import type { AbstractLevel } from 'abstract-level' //export const ENCODING_OPTS = { keyEncoding: 'view', valueEncoding: 'view' } @@ -27,7 +27,7 @@ export class LevelDB implements DB { * @inheritDoc */ // @ts-expect-error - async get(key: Uint8Array | string): Promise { + async get(key: Uint8Array | string): Promise { let value let encoding = undefined // Set value encoding based on key type or specific key names so values are interpreted correctly by Level @@ -62,10 +62,10 @@ export class LevelDB implements DB { /** * @inheritDoc */ - async put(key: Uint8Array | string, val: Uint8Array | string | any): Promise { - let valEncoding: string + async put(key: Uint8Array | string, val: Uint8Array | string | DBObject): Promise { + let valEncoding: string = '' if (typeof val === 'string') { - valEncoding = 'string' + valEncoding = 'utf8' } else if (val instanceof Uint8Array) { valEncoding = 'view' } else valEncoding = 'json' diff --git a/packages/client/test/blockchain/chain.spec.ts b/packages/client/test/blockchain/chain.spec.ts index 86afdf8e4d..fb1f6315e2 100644 --- a/packages/client/test/blockchain/chain.spec.ts +++ b/packages/client/test/blockchain/chain.spec.ts @@ -20,8 +20,8 @@ tape('[Chain]', (t) => { const db = chain.chainDB const testKey = 'name' const testValue = 'test' - await db.put(testKey, testValue) + const value = await db.get(testKey) t.equal(value, testValue, 'read value matches written value') t.end() From 9810b927ce18f7cd43dd853ab7e7cb1c83efe067 Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Thu, 4 May 2023 16:05:17 -0400 Subject: [PATCH 09/27] More typing fixes --- packages/blockchain/src/blockchain.ts | 4 ++-- packages/blockchain/src/consensus/clique.ts | 9 ++++++--- packages/blockchain/src/db/manager.ts | 2 +- packages/blockchain/src/types.ts | 4 ++-- packages/client/bin/cli.ts | 2 +- packages/client/lib/blockchain/chain.ts | 4 ++-- 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/packages/blockchain/src/blockchain.ts b/packages/blockchain/src/blockchain.ts index c8fe29de24..e9d66a6af8 100644 --- a/packages/blockchain/src/blockchain.ts +++ b/packages/blockchain/src/blockchain.ts @@ -21,14 +21,14 @@ import type { GenesisState } from './genesisStates' import type { BlockchainInterface, BlockchainOptions, OnBlock } from './types' import type { BlockData } from '@ethereumjs/block' import type { CliqueConfig } from '@ethereumjs/common' -import type { BigIntLike, DB } from '@ethereumjs/util' +import type { BigIntLike, DB, DBObject } from '@ethereumjs/util' /** * This class stores and interacts with blocks. */ export class Blockchain implements BlockchainInterface { consensus: Consensus - db: DB + db: DB dbManager: DBManager private _genesisBlock?: Block /** The genesis block of this blockchain */ diff --git a/packages/blockchain/src/consensus/clique.ts b/packages/blockchain/src/consensus/clique.ts index 17b2ac8881..3825de74b0 100644 --- a/packages/blockchain/src/consensus/clique.ts +++ b/packages/blockchain/src/consensus/clique.ts @@ -528,7 +528,7 @@ export class CliqueConsensus implements Consensus { private async getCliqueLatestSignerStates(): Promise { const signerStates = await this.blockchain!.db.get(CLIQUE_SIGNERS_KEY) if (signerStates === undefined) return [] - const states = RLP.decode(signerStates) as [Uint8Array, Uint8Array[]] + const states = RLP.decode(signerStates as Uint8Array) as [Uint8Array, Uint8Array[]] return states.map((state) => { const blockNum = bytesToBigInt(state[0] as Uint8Array) const addrs = (state[1]).map((bytes: Uint8Array) => new Address(bytes)) @@ -543,7 +543,10 @@ export class CliqueConsensus implements Consensus { private async getCliqueLatestVotes(): Promise { const signerVotes = await this.blockchain!.db.get(CLIQUE_VOTES_KEY) if (signerVotes === undefined) return [] - const votes = RLP.decode(signerVotes) as [Uint8Array, [Uint8Array, Uint8Array, Uint8Array]] + const votes = RLP.decode(signerVotes as Uint8Array) as [ + Uint8Array, + [Uint8Array, Uint8Array, Uint8Array] + ] return votes.map((vote) => { const blockNum = bytesToBigInt(vote[0] as Uint8Array) const signer = new Address((vote[1] as any)[0]) @@ -560,7 +563,7 @@ export class CliqueConsensus implements Consensus { private async getCliqueLatestBlockSigners(): Promise { const blockSigners = await this.blockchain!.db.get(CLIQUE_BLOCK_SIGNERS_SNAPSHOT_KEY) if (blockSigners === undefined) return [] - const signers = RLP.decode(blockSigners) as [Uint8Array, Uint8Array][] + const signers = RLP.decode(blockSigners as Uint8Array) as [Uint8Array, Uint8Array][] return signers.map((s) => { const blockNum = bytesToBigInt(s[0] as Uint8Array) const signer = new Address(s[1] as any) diff --git a/packages/blockchain/src/db/manager.ts b/packages/blockchain/src/db/manager.ts index 7220dbd819..5005ab8ca4 100644 --- a/packages/blockchain/src/db/manager.ts +++ b/packages/blockchain/src/db/manager.ts @@ -38,7 +38,7 @@ export class DBManager { private _common: Common private _db: DB - constructor(db: DB, common: Common) { + constructor(db: DB, common: Common) { this._db = db this._common = common this._cache = { diff --git a/packages/blockchain/src/types.ts b/packages/blockchain/src/types.ts index 7b44fb0303..c35f5c78d8 100644 --- a/packages/blockchain/src/types.ts +++ b/packages/blockchain/src/types.ts @@ -2,7 +2,7 @@ import type { Consensus } from './consensus' import type { GenesisState } from './genesisStates' import type { Block, BlockHeader } from '@ethereumjs/block' import type { Common } from '@ethereumjs/common' -import type { DB } from '@ethereumjs/util' +import type { DB, DBObject } from '@ethereumjs/util' export type OnBlock = (block: Block, reorg: boolean) => Promise | void @@ -107,7 +107,7 @@ export interface BlockchainOptions { * Database to store blocks and metadata. * Can be any database implementation that adheres to the `DB` interface */ - db?: DB + db?: DB /** * This flags indicates if a block should be validated along the consensus algorithm diff --git a/packages/client/bin/cli.ts b/packages/client/bin/cli.ts index c8c6299915..a2141cd2a0 100755 --- a/packages/client/bin/cli.ts +++ b/packages/client/bin/cli.ts @@ -425,7 +425,7 @@ async function startClient(config: Config, customGenesisState?: GenesisState) { if (customGenesisState !== undefined) { const validateConsensus = config.chainCommon.consensusAlgorithm() === ConsensusAlgorithm.Clique blockchain = await Blockchain.create({ - db: new LevelDB(dbs.chainDB), + db: new LevelDB(dbs.chainDB) as any, genesisState: customGenesisState, common: config.chainCommon, hardforkByHeadBlockNumber: true, diff --git a/packages/client/lib/blockchain/chain.ts b/packages/client/lib/blockchain/chain.ts index 1360a44403..f447147432 100644 --- a/packages/client/lib/blockchain/chain.ts +++ b/packages/client/lib/blockchain/chain.ts @@ -7,7 +7,7 @@ import { LevelDB } from '../execution/level' import { Event } from '../types' import type { Config } from '../config' -import type { DB } from '@ethereumjs/util' +import type { DB, DBObject } from '@ethereumjs/util' import type { AbstractLevel } from 'abstract-level' /** @@ -100,7 +100,7 @@ export interface ChainHeaders { */ export class Chain { public config: Config - public chainDB: DB + public chainDB: DB public blockchain: Blockchain public opened: boolean From c7ea77af7597ec2cca7618214344ef63fb842887 Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Mon, 8 May 2023 15:18:51 -0400 Subject: [PATCH 10/27] Fix batch value encoding --- packages/client/lib/execution/level.ts | 32 ++++++++++++++++++-------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/packages/client/lib/execution/level.ts b/packages/client/lib/execution/level.ts index c817a3b2ba..59c071496d 100644 --- a/packages/client/lib/execution/level.ts +++ b/packages/client/lib/execution/level.ts @@ -3,7 +3,15 @@ import { MemoryLevel } from 'memory-level' import type { BatchDBOp, DB, DBObject } from '@ethereumjs/util' import type { AbstractLevel } from 'abstract-level' -//export const ENCODING_OPTS = { keyEncoding: 'view', valueEncoding: 'view' } +// Helper to infer the `valueEncoding` option for `putting` a value in a levelDB +const inferValueEncoding = (value: Uint8Array | string | DBObject) => { + if (typeof value === 'string') { + return 'utf8' + } else if (value instanceof Uint8Array) { + return 'view' + } + return 'json' +} /** * LevelDB is a thin wrapper around the underlying levelup db, @@ -63,19 +71,14 @@ export class LevelDB implements DB { * @inheritDoc */ async put(key: Uint8Array | string, val: Uint8Array | string | DBObject): Promise { - let valEncoding: string = '' - if (typeof val === 'string') { - valEncoding = 'utf8' - } else if (val instanceof Uint8Array) { - valEncoding = 'view' - } else valEncoding = 'json' - await this._leveldb.put(key, val, { valueEncoding: valEncoding }) + const encoding = inferValueEncoding(val) + await this._leveldb.put(key, val, { valueEncoding: encoding }) } /** * @inheritDoc */ - async del(key: Uint8Array): Promise { + async del(key: Uint8Array | string): Promise { await this._leveldb.del(key) } @@ -83,7 +86,16 @@ export class LevelDB implements DB { * @inheritDoc */ async batch(opStack: BatchDBOp[]): Promise { - await this._leveldb.batch(opStack) + const levelOps = [] + for (const op of opStack) { + if (op.type === 'put') { + const encoding = inferValueEncoding(op.value) + levelOps.push({ ...op, valueEncoding: encoding }) + } else { + levelOps.push(op) + } + } + await this._leveldb.batch(levelOps) } /** From ccd8be33748118f3afb532a9e94242c593571d12 Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Mon, 8 May 2023 15:22:13 -0400 Subject: [PATCH 11/27] fix deprecated property usage --- packages/trie/src/db/checkpoint.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/trie/src/db/checkpoint.ts b/packages/trie/src/db/checkpoint.ts index c9382f0613..a7186130b1 100644 --- a/packages/trie/src/db/checkpoint.ts +++ b/packages/trie/src/db/checkpoint.ts @@ -210,7 +210,7 @@ export class CheckpointDB implements DB { } stats(reset = true) { - const stats = { ...this._stats, size: this._cache?.length ?? 0 } + const stats = { ...this._stats, size: this._cache?.size ?? 0 } if (reset) { this._stats = { cache: { From abcd64dc0dc15fdae7fe8944b92173ba4428d62e Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Tue, 9 May 2023 10:27:22 -0400 Subject: [PATCH 12/27] Replace level with mapdb in VM --- packages/vm/package.json | 2 - packages/vm/test/tester/mapDB.ts | 46 +++++++++++++++++++ .../tester/runners/BlockchainTestsRunner.ts | 11 ++--- 3 files changed, 50 insertions(+), 9 deletions(-) create mode 100644 packages/vm/test/tester/mapDB.ts diff --git a/packages/vm/package.json b/packages/vm/package.json index c03a914755..1c1eb18e8d 100644 --- a/packages/vm/package.json +++ b/packages/vm/package.json @@ -78,8 +78,6 @@ "@types/node-dir": "^0.0.34", "benchmark": "^2.1.4", "c-kzg": "^2.0.4", - "level": "^8.0.0", - "memory-level": "^1.0.0", "minimist": "^1.2.5", "node-dir": "^0.1.17", "nyc": "^15.1.0", diff --git a/packages/vm/test/tester/mapDB.ts b/packages/vm/test/tester/mapDB.ts new file mode 100644 index 0000000000..19d06a8379 --- /dev/null +++ b/packages/vm/test/tester/mapDB.ts @@ -0,0 +1,46 @@ +import type { BatchDBOp, DB } from '@ethereumjs/util' + +export class MapDB + implements DB +{ + _database: Map + + constructor(database?: Map) { + this._database = database ?? new Map() + } + + async get(key: TKey): Promise { + const dbKey = key.toString() + return this._database.get(dbKey as TKey) + } + + async put(key: TKey, val: TValue): Promise { + const dbKey = key.toString() + this._database.set(dbKey as TKey, val) + } + + async del(key: TKey): Promise { + const dbKey = key.toString() + this._database.delete(dbKey as TKey) + } + + async batch(opStack: BatchDBOp[]): Promise { + for (const op of opStack) { + if (op.type === 'del') { + await this.del(op.key) + } + + if (op.type === 'put') { + await this.put(op.key, op.value) + } + } + } + + copy(): DB { + return new MapDB(this._database) + } + + open() { + return Promise.resolve() + } +} diff --git a/packages/vm/test/tester/runners/BlockchainTestsRunner.ts b/packages/vm/test/tester/runners/BlockchainTestsRunner.ts index 8d4698bcd6..c0e26cf354 100644 --- a/packages/vm/test/tester/runners/BlockchainTestsRunner.ts +++ b/packages/vm/test/tester/runners/BlockchainTestsRunner.ts @@ -1,17 +1,15 @@ import { Block } from '@ethereumjs/block' import { Blockchain } from '@ethereumjs/blockchain' import { ConsensusAlgorithm } from '@ethereumjs/common' -import { LevelDB } from '@ethereumjs/ethash/dist/level' import { RLP } from '@ethereumjs/rlp' import { DefaultStateManager } from '@ethereumjs/statemanager' import { Trie } from '@ethereumjs/trie' import { TransactionFactory } from '@ethereumjs/tx' import { bytesToBigInt, isHexPrefixed, stripHexPrefix, toBytes } from '@ethereumjs/util' import { bytesToHex, hexToBytes } from 'ethereum-cryptography/utils' -import { Level } from 'level' -import { MemoryLevel } from 'memory-level' import { setupPreConditions, verifyPostConditions } from '../../util' +import { MapDB } from '../mapDB' import type { EthashConsensus } from '@ethereumjs/blockchain' import type { Common } from '@ethereumjs/common' @@ -35,7 +33,7 @@ export async function runBlockchainTest(options: any, testData: any, t: tape.Tes // fix for BlockchainTests/GeneralStateTests/stRandom/* testData.lastblockhash = stripHexPrefix(testData.lastblockhash) - let cacheDB = new LevelDB(new Level('./.cachedb')) + let cacheDB = new MapDB() let state = new Trie({ useKeyHashing: true }) let stateManager = new DefaultStateManager({ trie: state, @@ -66,7 +64,6 @@ export async function runBlockchainTest(options: any, testData: any, t: tape.Tes } let blockchain = await Blockchain.create({ - db: new LevelDB(new MemoryLevel()), common, validateBlocks: true, validateConsensus: validatePow, @@ -215,7 +212,7 @@ export async function runBlockchainTest(options: any, testData: any, t: tape.Tes throw e } - await cacheDB._leveldb.close() + // await cacheDB._leveldb.close() if (expectException !== false) { t.fail(`expected exception but test did not throw an exception: ${expectException}`) @@ -236,7 +233,7 @@ export async function runBlockchainTest(options: any, testData: any, t: tape.Tes const end = Date.now() const timeSpent = `${(end - begin) / 1000} secs` t.comment(`Time: ${timeSpent}`) - await cacheDB._leveldb.close() + // await cacheDB._leveldb.close() // @ts-ignore Explicitly delete objects for memory optimization (early GC) common = blockchain = state = stateManager = vm = cacheDB = null // eslint-disable-line From d35960e6e6d3c30f77121d55cd99f997155b506e Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Tue, 9 May 2023 10:57:55 -0400 Subject: [PATCH 13/27] move mapDB to util --- package-lock.json | 4 -- packages/blockchain/src/blockchain.ts | 2 +- packages/blockchain/src/db/map.ts | 46 ------------------- packages/blockchain/test/index.spec.ts | 2 +- packages/blockchain/test/util.ts | 3 +- packages/client/bin/cli.ts | 2 +- packages/trie/benchmarks/index.ts | 2 +- packages/trie/src/db/index.ts | 1 - packages/trie/src/db/map.ts | 43 ----------------- packages/trie/src/trie/trie.ts | 6 +-- packages/trie/test/db/checkpoint.spec.ts | 4 +- packages/trie/test/db/db.spec.ts | 6 +-- packages/trie/test/proof/range.spec.ts | 3 +- packages/trie/test/trie/checkpoint.spec.ts | 4 +- packages/trie/test/trie/secure.spec.ts | 5 +- packages/trie/test/trie/trie.spec.ts | 6 +-- packages/util/src/index.ts | 1 + .../{vm/test/tester => util/src}/mapDB.ts | 10 ++-- .../tester/runners/BlockchainTestsRunner.ts | 3 +- 19 files changed, 30 insertions(+), 123 deletions(-) delete mode 100644 packages/blockchain/src/db/map.ts delete mode 100644 packages/trie/src/db/map.ts rename packages/{vm/test/tester => util/src}/mapDB.ts (73%) diff --git a/package-lock.json b/package-lock.json index 283a114c1b..c529d3141d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18318,8 +18318,6 @@ "@types/node-dir": "^0.0.34", "benchmark": "^2.1.4", "c-kzg": "^2.0.4", - "level": "^8.0.0", - "memory-level": "^1.0.0", "minimist": "^1.2.5", "node-dir": "^0.1.17", "nyc": "^15.1.0", @@ -19898,9 +19896,7 @@ "c-kzg": "^2.0.4", "debug": "^4.3.3", "ethereum-cryptography": "^1.1.2", - "level": "^8.0.0", "mcl-wasm": "^0.7.1", - "memory-level": "^1.0.0", "minimist": "^1.2.5", "node-dir": "^0.1.17", "nyc": "^15.1.0", diff --git a/packages/blockchain/src/blockchain.ts b/packages/blockchain/src/blockchain.ts index 341689a0d9..2dd7a90bca 100644 --- a/packages/blockchain/src/blockchain.ts +++ b/packages/blockchain/src/blockchain.ts @@ -3,6 +3,7 @@ import { Chain, Common, ConsensusAlgorithm, ConsensusType, Hardfork } from '@eth import { KECCAK256_RLP, Lock, + MapDB, bytesToPrefixedHexString, concatBytesNoTypeCheck, } from '@ethereumjs/util' @@ -11,7 +12,6 @@ import { bytesToHex, equalsBytes, hexToBytes } from 'ethereum-cryptography/utils import { CasperConsensus, CliqueConsensus, EthashConsensus } from './consensus' import { DBOp, DBSaveLookups, DBSetBlockOrHeader, DBSetHashToNumber, DBSetTD } from './db/helpers' import { DBManager } from './db/manager' -import { MapDB } from './db/map' import { DBTarget } from './db/operation' import { genesisStateRoot } from './genesisStates' import {} from './utils' diff --git a/packages/blockchain/src/db/map.ts b/packages/blockchain/src/db/map.ts deleted file mode 100644 index 19d06a8379..0000000000 --- a/packages/blockchain/src/db/map.ts +++ /dev/null @@ -1,46 +0,0 @@ -import type { BatchDBOp, DB } from '@ethereumjs/util' - -export class MapDB - implements DB -{ - _database: Map - - constructor(database?: Map) { - this._database = database ?? new Map() - } - - async get(key: TKey): Promise { - const dbKey = key.toString() - return this._database.get(dbKey as TKey) - } - - async put(key: TKey, val: TValue): Promise { - const dbKey = key.toString() - this._database.set(dbKey as TKey, val) - } - - async del(key: TKey): Promise { - const dbKey = key.toString() - this._database.delete(dbKey as TKey) - } - - async batch(opStack: BatchDBOp[]): Promise { - for (const op of opStack) { - if (op.type === 'del') { - await this.del(op.key) - } - - if (op.type === 'put') { - await this.put(op.key, op.value) - } - } - } - - copy(): DB { - return new MapDB(this._database) - } - - open() { - return Promise.resolve() - } -} diff --git a/packages/blockchain/test/index.spec.ts b/packages/blockchain/test/index.spec.ts index fffe1c4ea5..7e42f73752 100644 --- a/packages/blockchain/test/index.spec.ts +++ b/packages/blockchain/test/index.spec.ts @@ -1,10 +1,10 @@ import { Block, BlockHeader } from '@ethereumjs/block' import { Chain, Common, Hardfork } from '@ethereumjs/common' +import { MapDB } from '@ethereumjs/util' import { bytesToHex, equalsBytes, hexToBytes, utf8ToBytes } from 'ethereum-cryptography/utils' import * as tape from 'tape' import { Blockchain } from '../src' -import { MapDB } from '../src/db/map' import * as blocksData from './testdata/blocks_mainnet.json' import * as testDataPreLondon from './testdata/testdata_pre-london.json' diff --git a/packages/blockchain/test/util.ts b/packages/blockchain/test/util.ts index 247754e436..0f303811d0 100644 --- a/packages/blockchain/test/util.ts +++ b/packages/blockchain/test/util.ts @@ -1,12 +1,11 @@ import { Block, BlockHeader } from '@ethereumjs/block' import { Chain, Common, Hardfork } from '@ethereumjs/common' import { RLP } from '@ethereumjs/rlp' -import { toBytes } from '@ethereumjs/util' +import { MapDB, toBytes } from '@ethereumjs/util' import { keccak256 } from 'ethereum-cryptography/keccak' import { bytesToHex, equalsBytes, hexToBytes, utf8ToBytes } from 'ethereum-cryptography/utils' import { Blockchain } from '../src' -import { MapDB } from '../src/db/map' import type { DB } from '@ethereumjs/util' diff --git a/packages/client/bin/cli.ts b/packages/client/bin/cli.ts index 4785920aeb..47123e3456 100755 --- a/packages/client/bin/cli.ts +++ b/packages/client/bin/cli.ts @@ -433,7 +433,7 @@ async function startClient(config: Config, customGenesisState?: GenesisState) { if (customGenesisState !== undefined) { const validateConsensus = config.chainCommon.consensusAlgorithm() === ConsensusAlgorithm.Clique blockchain = await Blockchain.create({ - db: new LevelDB(dbs.chainDB) as any, + db: new LevelDB(dbs.chainDB), genesisState: customGenesisState, common: config.chainCommon, hardforkByHeadBlockNumber: true, diff --git a/packages/trie/benchmarks/index.ts b/packages/trie/benchmarks/index.ts index 7c94b11361..a32436d8f6 100644 --- a/packages/trie/benchmarks/index.ts +++ b/packages/trie/benchmarks/index.ts @@ -1,6 +1,6 @@ import { createSuite } from './suite' import { LevelDB } from './engines/level' -import { MapDB } from '../dist' +import { MapDB } from '@ethereumjs/util' createSuite(new MapDB()) createSuite(new LevelDB()) diff --git a/packages/trie/src/db/index.ts b/packages/trie/src/db/index.ts index 0cf4fafadb..2199ccf124 100644 --- a/packages/trie/src/db/index.ts +++ b/packages/trie/src/db/index.ts @@ -1,2 +1 @@ export * from './checkpoint' -export * from './map' diff --git a/packages/trie/src/db/map.ts b/packages/trie/src/db/map.ts deleted file mode 100644 index 75ea2cf3a9..0000000000 --- a/packages/trie/src/db/map.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { bytesToHex } from 'ethereum-cryptography/utils' - -import type { BatchDBOp, DB } from '@ethereumjs/util' - -export class MapDB implements DB { - _database: Map - - constructor(database?: Map) { - this._database = database ?? new Map() - } - - async get(key: Uint8Array): Promise { - return this._database.get(bytesToHex(key)) - } - - async put(key: Uint8Array, val: Uint8Array): Promise { - this._database.set(bytesToHex(key), val) - } - - async del(key: Uint8Array): Promise { - this._database.delete(bytesToHex(key)) - } - - async batch(opStack: BatchDBOp[]): Promise { - for (const op of opStack) { - if (op.type === 'del') { - await this.del(op.key) - } - - if (op.type === 'put') { - await this.put(op.key, op.value) - } - } - } - - copy(): DB { - return new MapDB(this._database) - } - - open() { - return Promise.resolve(undefined) - } -} diff --git a/packages/trie/src/trie/trie.ts b/packages/trie/src/trie/trie.ts index a5dfa5f9d0..a619d2c507 100644 --- a/packages/trie/src/trie/trie.ts +++ b/packages/trie/src/trie/trie.ts @@ -1,7 +1,7 @@ -import { RLP_EMPTY_STRING, bytesToHex, bytesToUtf8, equalsBytes } from '@ethereumjs/util' +import { MapDB, RLP_EMPTY_STRING, bytesToHex, bytesToUtf8, equalsBytes } from '@ethereumjs/util' import { keccak256 } from 'ethereum-cryptography/keccak' -import { CheckpointDB, MapDB } from '../db' +import { CheckpointDB } from '../db' import { verifyRangeProof } from '../proof/range' import { ROOT_DB_KEY } from '../types' import { Lock } from '../util/lock' @@ -60,7 +60,7 @@ export class Trie { this._opts = { ...this._opts, ...opts } } - this.database(opts?.db ?? new MapDB()) + this.database(opts?.db ?? new MapDB()) this.EMPTY_TRIE_ROOT = this.hash(RLP_EMPTY_STRING) this._hashLen = this.EMPTY_TRIE_ROOT.length diff --git a/packages/trie/test/db/checkpoint.spec.ts b/packages/trie/test/db/checkpoint.spec.ts index 33a1a0161d..6610760ff7 100644 --- a/packages/trie/test/db/checkpoint.spec.ts +++ b/packages/trie/test/db/checkpoint.spec.ts @@ -1,7 +1,7 @@ -import { hexStringToBytes, utf8ToBytes } from '@ethereumjs/util' +import { MapDB, hexStringToBytes, utf8ToBytes } from '@ethereumjs/util' import * as tape from 'tape' -import { CheckpointDB, MapDB } from '../../src' +import { CheckpointDB } from '../../src' import type { BatchDBOp } from '@ethereumjs/util' diff --git a/packages/trie/test/db/db.spec.ts b/packages/trie/test/db/db.spec.ts index fcf83b5a50..6e9880fac8 100644 --- a/packages/trie/test/db/db.spec.ts +++ b/packages/trie/test/db/db.spec.ts @@ -1,12 +1,10 @@ -import { equalsBytes, utf8ToBytes } from '@ethereumjs/util' +import { MapDB, equalsBytes, utf8ToBytes } from '@ethereumjs/util' import * as tape from 'tape' -import { MapDB } from '../../src' - import type { BatchDBOp } from '@ethereumjs/util' tape('DB tests', (t) => { - const db = new MapDB() + const db = new MapDB() const k = utf8ToBytes('k1') const v = utf8ToBytes('v1') diff --git a/packages/trie/test/proof/range.spec.ts b/packages/trie/test/proof/range.spec.ts index 8fd053b141..247dfe0bd4 100644 --- a/packages/trie/test/proof/range.spec.ts +++ b/packages/trie/test/proof/range.spec.ts @@ -1,4 +1,5 @@ import { + MapDB, compareBytes, concatBytes, hexStringToBytes, @@ -8,7 +9,7 @@ import { import * as crypto from 'crypto' import * as tape from 'tape' -import { MapDB, Trie } from '../../src' +import { Trie } from '../../src' import type { DB } from '@ethereumjs/util' diff --git a/packages/trie/test/trie/checkpoint.spec.ts b/packages/trie/test/trie/checkpoint.spec.ts index 50616de239..8f35fb8d74 100644 --- a/packages/trie/test/trie/checkpoint.spec.ts +++ b/packages/trie/test/trie/checkpoint.spec.ts @@ -1,9 +1,9 @@ -import { bytesToHex, bytesToUtf8, equalsBytes, utf8ToBytes } from '@ethereumjs/util' +import { MapDB, bytesToHex, bytesToUtf8, equalsBytes, utf8ToBytes } from '@ethereumjs/util' import { createHash } from 'crypto' import { keccak256 } from 'ethereum-cryptography/keccak' import * as tape from 'tape' -import { MapDB, ROOT_DB_KEY, Trie } from '../../src' +import { ROOT_DB_KEY, Trie } from '../../src' import type { BatchDBOp } from '@ethereumjs/util' diff --git a/packages/trie/test/trie/secure.spec.ts b/packages/trie/test/trie/secure.spec.ts index 30645e3fe2..590eefb312 100644 --- a/packages/trie/test/trie/secure.spec.ts +++ b/packages/trie/test/trie/secure.spec.ts @@ -1,4 +1,5 @@ import { + MapDB, bytesToPrefixedHexString, bytesToUtf8, equalsBytes, @@ -8,7 +9,7 @@ import { import { createHash } from 'crypto' import * as tape from 'tape' -import { MapDB, ROOT_DB_KEY, Trie } from '../../src' +import { ROOT_DB_KEY, Trie } from '../../src' tape('SecureTrie', function (t) { const trie = new Trie({ useKeyHashing: true, db: new MapDB() }) @@ -89,7 +90,7 @@ tape('SecureTrie', function (t) { await trie.put(ROOT_DB_KEY, utf8ToBytes('bar')) st.fail("Attempting to set '__root__' should fail but it did not.") - } catch ({ message }) { + } catch ({ message }: any) { st.equal(message, "Attempted to set '__root__' key but it is not allowed.") } }) diff --git a/packages/trie/test/trie/trie.spec.ts b/packages/trie/test/trie/trie.spec.ts index ea846a732c..5f3f4eb21a 100644 --- a/packages/trie/test/trie/trie.spec.ts +++ b/packages/trie/test/trie/trie.spec.ts @@ -1,8 +1,8 @@ -import { KECCAK256_RLP, bytesToHex, equalsBytes, utf8ToBytes } from '@ethereumjs/util' +import { KECCAK256_RLP, MapDB, bytesToHex, equalsBytes, utf8ToBytes } from '@ethereumjs/util' import { keccak256 } from 'ethereum-cryptography/keccak' import * as tape from 'tape' -import { ROOT_DB_KEY as BASE_DB_KEY, MapDB, Trie } from '../../src' +import { ROOT_DB_KEY as BASE_DB_KEY, Trie } from '../../src' for (const { constructor, defaults, title } of [ { @@ -163,7 +163,7 @@ for (const { constructor, defaults, title } of [ }) t.test('persist and restore the root', async function (st) { - const db = new MapDB() + const db = new MapDB() const trie = await constructor.create({ ...defaults, db, useRootPersistence: true }) // @ts-expect-error diff --git a/packages/util/src/index.ts b/packages/util/src/index.ts index 5e3162d874..7ba80184ca 100644 --- a/packages/util/src/index.ts +++ b/packages/util/src/index.ts @@ -72,4 +72,5 @@ export { } from './internal' export * from './kzg' export * from './lock' +export * from './mapDB' export * from './provider' diff --git a/packages/vm/test/tester/mapDB.ts b/packages/util/src/mapDB.ts similarity index 73% rename from packages/vm/test/tester/mapDB.ts rename to packages/util/src/mapDB.ts index 19d06a8379..6085c0acd9 100644 --- a/packages/vm/test/tester/mapDB.ts +++ b/packages/util/src/mapDB.ts @@ -1,4 +1,6 @@ -import type { BatchDBOp, DB } from '@ethereumjs/util' +import { bytesToHex } from 'ethereum-cryptography/utils' + +import type { BatchDBOp, DB } from './db' export class MapDB implements DB @@ -10,17 +12,17 @@ export class MapDB { - const dbKey = key.toString() + const dbKey = key instanceof Uint8Array ? bytesToHex(key) : key.toString() return this._database.get(dbKey as TKey) } async put(key: TKey, val: TValue): Promise { - const dbKey = key.toString() + const dbKey = key instanceof Uint8Array ? bytesToHex(key) : key.toString() this._database.set(dbKey as TKey, val) } async del(key: TKey): Promise { - const dbKey = key.toString() + const dbKey = key instanceof Uint8Array ? bytesToHex(key) : key.toString() this._database.delete(dbKey as TKey) } diff --git a/packages/vm/test/tester/runners/BlockchainTestsRunner.ts b/packages/vm/test/tester/runners/BlockchainTestsRunner.ts index c0e26cf354..6958ddb9a0 100644 --- a/packages/vm/test/tester/runners/BlockchainTestsRunner.ts +++ b/packages/vm/test/tester/runners/BlockchainTestsRunner.ts @@ -5,11 +5,10 @@ import { RLP } from '@ethereumjs/rlp' import { DefaultStateManager } from '@ethereumjs/statemanager' import { Trie } from '@ethereumjs/trie' import { TransactionFactory } from '@ethereumjs/tx' -import { bytesToBigInt, isHexPrefixed, stripHexPrefix, toBytes } from '@ethereumjs/util' +import { MapDB, bytesToBigInt, isHexPrefixed, stripHexPrefix, toBytes } from '@ethereumjs/util' import { bytesToHex, hexToBytes } from 'ethereum-cryptography/utils' import { setupPreConditions, verifyPostConditions } from '../../util' -import { MapDB } from '../mapDB' import type { EthashConsensus } from '@ethereumjs/blockchain' import type { Common } from '@ethereumjs/common' From 86cea58a787c9f901390d39ec0a5a7dfb15a88d0 Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Tue, 9 May 2023 11:14:46 -0400 Subject: [PATCH 14/27] Cleanup --- packages/blockchain/src/consensus/clique.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/blockchain/src/consensus/clique.ts b/packages/blockchain/src/consensus/clique.ts index 3825de74b0..83e6a7d212 100644 --- a/packages/blockchain/src/consensus/clique.ts +++ b/packages/blockchain/src/consensus/clique.ts @@ -248,7 +248,7 @@ export class CliqueConsensus implements Consensus { bigIntToBytes(state[0]), state[1].map((a) => a.toBytes()), ]) - await this.blockchain!.db.put(CLIQUE_SIGNERS_KEY, RLP.encode(formatted) /* , DB_OPTS */) + await this.blockchain!.db.put(CLIQUE_SIGNERS_KEY, RLP.encode(formatted)) // Output active signers for debugging purposes if (signerState !== undefined) { let i = 0 @@ -409,7 +409,7 @@ export class CliqueConsensus implements Consensus { bigIntToBytes(v[0]), [v[1][0].toBytes(), v[1][1].toBytes(), v[1][2]], ]) - await this.blockchain!.db.put(CLIQUE_VOTES_KEY, RLP.encode(formatted) /* , DB_OPTS */) + await this.blockchain!.db.put(CLIQUE_VOTES_KEY, RLP.encode(formatted)) } /** From 2b1e081bfa92ee897875a7f12ff13f29c5e1617e Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Wed, 10 May 2023 12:41:20 -0400 Subject: [PATCH 15/27] Add key/value encodings back to interface --- packages/client/lib/execution/level.ts | 81 +++++++++++-------- .../client/lib/sync/fetcher/accountfetcher.ts | 2 +- packages/trie/src/db/checkpoint.ts | 16 +++- packages/trie/src/trie/trie.ts | 27 ++++++- packages/trie/src/types.ts | 2 +- packages/util/src/db.ts | 24 +++++- 6 files changed, 106 insertions(+), 46 deletions(-) diff --git a/packages/client/lib/execution/level.ts b/packages/client/lib/execution/level.ts index 59c071496d..1bab7de2fe 100644 --- a/packages/client/lib/execution/level.ts +++ b/packages/client/lib/execution/level.ts @@ -1,23 +1,45 @@ +import { KeyEncoding, ValueEncoding } from '@ethereumjs/util' import { MemoryLevel } from 'memory-level' -import type { BatchDBOp, DB, DBObject } from '@ethereumjs/util' +import type { BatchDBOp, DB, DBObject, EncodingOpts } from '@ethereumjs/util' import type { AbstractLevel } from 'abstract-level' // Helper to infer the `valueEncoding` option for `putting` a value in a levelDB -const inferValueEncoding = (value: Uint8Array | string | DBObject) => { - if (typeof value === 'string') { - return 'utf8' - } else if (value instanceof Uint8Array) { - return 'view' +const getEncodings = (opts: EncodingOpts = {}) => { + const encodings = { keyEncoding: '', valueEncoding: '' } + switch (opts.valueEncoding) { + case ValueEncoding.String: + encodings.valueEncoding = 'utf8' + break + case ValueEncoding.Bytes: + encodings.valueEncoding = 'view' + break + case ValueEncoding.JSON: + encodings.valueEncoding = 'json' + break + default: + encodings.valueEncoding = 'view' } - return 'json' + switch (opts.keyEncoding) { + case KeyEncoding.Bytes: + encodings.keyEncoding = 'view' + break + case KeyEncoding.Number: + case KeyEncoding.String: + encodings.keyEncoding = 'utf8' + break + default: + encodings.keyEncoding = 'utf8' + } + + return encodings } /** * LevelDB is a thin wrapper around the underlying levelup db, * which validates inputs and sets encoding type. */ -export class LevelDB implements DB { +export class LevelDB implements DB { _leveldb: AbstractLevel /** @@ -34,25 +56,15 @@ export class LevelDB implements DB { /** * @inheritDoc */ - // @ts-expect-error - async get(key: Uint8Array | string): Promise { + async get( + key: Uint8Array | string, + opts?: EncodingOpts + ): Promise { let value - let encoding = undefined - // Set value encoding based on key type or specific key names so values are interpreted correctly by Level - if ( - key instanceof Uint8Array || - key === 'CliqueSigners' || - key === 'CliqueVotes' || - key === 'CliqueBlockSignersSnapshot' - ) - encoding = 'view' - if (key === 'heads' || typeof key === 'number') { - encoding = 'json' - } + const encodings = getEncodings(opts) + try { - value = await this._leveldb.get(key, { - valueEncoding: encoding, - }) + value = await this._leveldb.get(key, encodings) if (value === null) return undefined } catch (error: any) { // https://github.com/Level/abstract-level/blob/915ad1317694d0ce8c580b5ab85d81e1e78a3137/abstract-level.js#L309 @@ -70,9 +82,13 @@ export class LevelDB implements DB { /** * @inheritDoc */ - async put(key: Uint8Array | string, val: Uint8Array | string | DBObject): Promise { - const encoding = inferValueEncoding(val) - await this._leveldb.put(key, val, { valueEncoding: encoding }) + async put( + key: Uint8Array | string, + val: Uint8Array | string | DBObject, + opts?: {} + ): Promise { + const encodings = getEncodings(opts) + await this._leveldb.put(key, val, encodings) } /** @@ -88,13 +104,10 @@ export class LevelDB implements DB { async batch(opStack: BatchDBOp[]): Promise { const levelOps = [] for (const op of opStack) { - if (op.type === 'put') { - const encoding = inferValueEncoding(op.value) - levelOps.push({ ...op, valueEncoding: encoding }) - } else { - levelOps.push(op) - } + const encodings = getEncodings(op.opts) + levelOps.push({ ...op, ...encodings }) } + await this._leveldb.batch(levelOps) } diff --git a/packages/client/lib/sync/fetcher/accountfetcher.ts b/packages/client/lib/sync/fetcher/accountfetcher.ts index 3ba858beab..79387b3da7 100644 --- a/packages/client/lib/sync/fetcher/accountfetcher.ts +++ b/packages/client/lib/sync/fetcher/accountfetcher.ts @@ -203,7 +203,7 @@ export class AccountFetcher extends Fetcher } } - const trie = new Trie({ db: new LevelDB() as DB }) + const trie = new Trie({ db: new LevelDB() as DB }) const keys = accounts.map((acc: any) => acc.hash) const values = accounts.map((acc: any) => accountBodyToRLP(acc.body)) // convert the request to the right values diff --git a/packages/trie/src/db/checkpoint.ts b/packages/trie/src/db/checkpoint.ts index a7186130b1..6d6b231ed5 100644 --- a/packages/trie/src/db/checkpoint.ts +++ b/packages/trie/src/db/checkpoint.ts @@ -1,4 +1,4 @@ -import { bytesToHex, hexStringToBytes } from '@ethereumjs/util' +import { KeyEncoding, ValueEncoding, bytesToHex, hexStringToBytes } from '@ethereumjs/util' import type { Checkpoint, CheckpointDBOpts } from '../types' import type { BatchDBOp, DB } from '@ethereumjs/util' @@ -138,7 +138,10 @@ export class CheckpointDB implements DB { } } // Nothing has been found in diff cache, look up from disk - const value = await this.db.get(key) + const value = await this.db.get(key, { + keyEncoding: KeyEncoding.Bytes, + valueEncoding: ValueEncoding.Bytes, + }) this._stats.db.reads += 1 if (value !== null) { this._stats.db.hits += 1 @@ -162,7 +165,10 @@ export class CheckpointDB implements DB { // put value in diff cache this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(keyHex, value) } else { - await this.db.put(key, value) + await this.db.put(key, value, { + keyEncoding: KeyEncoding.Bytes, + valueEncoding: ValueEncoding.Bytes, + }) this._stats.db.writes += 1 if (this._cache !== undefined) { @@ -182,7 +188,9 @@ export class CheckpointDB implements DB { this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(keyHex, undefined) } else { // delete the value on disk - await this.db.del(key) + await this.db.del(key, { + keyEncoding: KeyEncoding.Bytes, + }) this._stats.db.writes += 1 if (this._cache !== undefined) { diff --git a/packages/trie/src/trie/trie.ts b/packages/trie/src/trie/trie.ts index a619d2c507..26e4961261 100644 --- a/packages/trie/src/trie/trie.ts +++ b/packages/trie/src/trie/trie.ts @@ -1,4 +1,12 @@ -import { MapDB, RLP_EMPTY_STRING, bytesToHex, bytesToUtf8, equalsBytes } from '@ethereumjs/util' +import { + KeyEncoding, + MapDB, + RLP_EMPTY_STRING, + ValueEncoding, + bytesToHex, + bytesToUtf8, + equalsBytes, +} from '@ethereumjs/util' import { keccak256 } from 'ethereum-cryptography/keccak' import { CheckpointDB } from '../db' @@ -80,9 +88,16 @@ export class Trie { if (opts?.db !== undefined && opts?.useRootPersistence === true) { if (opts?.root === undefined) { - opts.root = (await opts?.db.get(key)) ?? undefined + opts.root = + (await opts?.db.get(key, { + keyEncoding: KeyEncoding.Bytes, + valueEncoding: ValueEncoding.Bytes, + })) ?? undefined } else { - await opts?.db.put(key, opts.root) + await opts?.db.put(key, opts.root, { + keyEncoding: KeyEncoding.Bytes, + valueEncoding: ValueEncoding.Bytes, + }) } } @@ -191,6 +206,9 @@ export class Trie { return { type: 'del', key: e, + opts: { + keyEncoding: KeyEncoding.Bytes, + }, } }) } @@ -227,6 +245,9 @@ export class Trie { return { type: 'del', key: e, + opts: { + keyEncoding: KeyEncoding.Bytes, + }, } }) } diff --git a/packages/trie/src/types.ts b/packages/trie/src/types.ts index 891fb01ed1..ebb6984193 100644 --- a/packages/trie/src/types.ts +++ b/packages/trie/src/types.ts @@ -27,7 +27,7 @@ export interface TrieOpts { /** * A database instance. */ - db?: DB + db?: DB /** * A `Uint8Array` for the root of a previously stored trie diff --git a/packages/util/src/db.ts b/packages/util/src/db.ts index 19c6ada280..fd79f07775 100644 --- a/packages/util/src/db.ts +++ b/packages/util/src/db.ts @@ -6,6 +6,22 @@ export type BatchDBOp< TValue extends Uint8Array | string | DBObject = Uint8Array > = PutBatch | DelBatch +export enum KeyEncoding { + String = 'string', + Bytes = 'view', + Number = 'number', +} + +export enum ValueEncoding { + String = 'string', + Bytes = 'view', + JSON = 'json', +} + +export type EncodingOpts = { + keyEncoding?: KeyEncoding + valueEncoding?: ValueEncoding +} export interface PutBatch< TKey extends Uint8Array | string | number = Uint8Array, TValue extends Uint8Array | string | DBObject = Uint8Array @@ -13,11 +29,13 @@ export interface PutBatch< type: 'put' key: TKey value: TValue + opts?: EncodingOpts } export interface DelBatch { type: 'del' key: TKey + opts?: EncodingOpts } export interface DB< @@ -29,20 +47,20 @@ export interface DB< * @param key * @returns A Promise that resolves to `Uint8Array` if a value is found or `undefined` if no value is found. */ - get(key: TKey): Promise + get(key: TKey, opts?: EncodingOpts): Promise /** * Writes a value directly to db. * @param key The key as a `TValue` * @param value The value to be stored */ - put(key: TKey, val: TValue): Promise + put(key: TKey, val: TValue, opts?: EncodingOpts): Promise /** * Removes a raw value in the underlying db. * @param keys */ - del(key: TKey): Promise + del(key: TKey, opts?: EncodingOpts): Promise /** * Performs a batch operation on db. From 238626dcaf0c6df51ae6be244960a57d2969ea74 Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Wed, 10 May 2023 12:52:34 -0400 Subject: [PATCH 16/27] Remove level dependency from ethash --- packages/ethash/package.json | 4 +--- packages/ethash/src/index.ts | 31 ++++++++++++++++++++++-------- packages/ethash/test/miner.spec.ts | 9 +++++---- packages/util/src/mapDB.ts | 8 +++++--- 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/packages/ethash/package.json b/packages/ethash/package.json index 2756032064..1736ef714d 100644 --- a/packages/ethash/package.json +++ b/packages/ethash/package.json @@ -40,13 +40,11 @@ "@ethereumjs/block": "^4.2.1", "@ethereumjs/rlp": "^4.0.1", "@ethereumjs/util": "^8.0.5", - "abstract-level": "^1.0.3", "bigint-crypto-utils": "^3.0.23", "ethereum-cryptography": "^1.1.2" }, "devDependencies": { - "@ethereumjs/common": "^3.1.1", - "memory-level": "^1.0.0" + "@ethereumjs/common": "^3.1.1" }, "engines": { "node": ">=16" diff --git a/packages/ethash/src/index.ts b/packages/ethash/src/index.ts index 7df5122a4c..a53d981aea 100644 --- a/packages/ethash/src/index.ts +++ b/packages/ethash/src/index.ts @@ -1,7 +1,9 @@ import { Block, BlockHeader } from '@ethereumjs/block' import { RLP } from '@ethereumjs/rlp' import { + KeyEncoding, TWO_POW256, + ValueEncoding, bigIntToBytes, bytesToBigInt, bytesToHex, @@ -292,7 +294,10 @@ export class Ethash { return [zeros(32), 0] } - const dbData = await this.cacheDB!.get(epoc) + const dbData = await this.cacheDB!.get(epoc, { + keyEncoding: KeyEncoding.Number, + valueEncoding: ValueEncoding.JSON, + }) if (dbData !== undefined) { const data = { cache: (dbData.cache as string[]).map((el: string) => hexToBytes(el)), @@ -307,7 +312,10 @@ export class Ethash { } let data - const dbData = await this.cacheDB!.get(epoc) + const dbData = await this.cacheDB!.get(epoc, { + keyEncoding: KeyEncoding.Number, + valueEncoding: ValueEncoding.JSON, + }) if (dbData !== undefined) { data = { cache: (dbData.cache as string[]).map((el: string) => hexToBytes(el)), @@ -324,12 +332,19 @@ export class Ethash { this.seed = getSeed(seed, foundEpoc, epoc) const cache = this.mkcache(this.cacheSize!, this.seed!) // store the generated cache - await this.cacheDB!.put(epoc as any, { - cacheSize: this.cacheSize, - fullSize: this.fullSize, - seed: bytesToHex(this.seed), - cache: cache.map((el) => bytesToHex(el)), - }) + await this.cacheDB!.put( + epoc, + { + cacheSize: this.cacheSize, + fullSize: this.fullSize, + seed: bytesToHex(this.seed), + cache: cache.map((el) => bytesToHex(el)), + }, + { + keyEncoding: KeyEncoding.Number, + valueEncoding: ValueEncoding.JSON, + } + ) } else { this.cache = data.cache.map((a: Uint8Array) => { return Uint8Array.from(a) diff --git a/packages/ethash/test/miner.spec.ts b/packages/ethash/test/miner.spec.ts index 3b9765c4fa..c85128d872 100644 --- a/packages/ethash/test/miner.spec.ts +++ b/packages/ethash/test/miner.spec.ts @@ -1,17 +1,18 @@ import { Block } from '@ethereumjs/block' import { Chain, Common, Hardfork } from '@ethereumjs/common' -import { MemoryLevel } from 'memory-level' +import { MapDB } from '@ethereumjs/util' import * as tape from 'tape' import { Ethash } from '../src' -import { LevelDB } from '../src/level' import type { BlockHeader } from '@ethereumjs/block' -const cacheDb = new LevelDB(new MemoryLevel()) +import type { DBObject } from '@ethereumjs/util' + +const cacheDb = new MapDB() const common = new Common({ chain: Chain.Ropsten, hardfork: Hardfork.Petersburg }) tape('Check if miner works as expected', async function (t) { - const e = new Ethash(cacheDb as any) + const e = new Ethash(cacheDb) const block = Block.fromBlockData( { diff --git a/packages/util/src/mapDB.ts b/packages/util/src/mapDB.ts index 6085c0acd9..efb17da7f8 100644 --- a/packages/util/src/mapDB.ts +++ b/packages/util/src/mapDB.ts @@ -1,9 +1,11 @@ import { bytesToHex } from 'ethereum-cryptography/utils' -import type { BatchDBOp, DB } from './db' +import type { BatchDBOp, DB, DBObject } from './db' -export class MapDB - implements DB +export class MapDB< + TKey extends Uint8Array | string | number, + TValue extends Uint8Array | string | DBObject +> implements DB { _database: Map From e1faad9413234370eaf0eb06d4ecb2994563f976 Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Wed, 10 May 2023 14:38:55 -0400 Subject: [PATCH 17/27] Add db ops to blockchain --- packages/blockchain/src/db/manager.ts | 40 ++++++++++++++----- packages/blockchain/src/db/operation.ts | 20 ++++++---- packages/client/test/blockchain/chain.spec.ts | 15 +++++-- 3 files changed, 55 insertions(+), 20 deletions(-) diff --git a/packages/blockchain/src/db/manager.ts b/packages/blockchain/src/db/manager.ts index 5005ab8ca4..08cf567d4f 100644 --- a/packages/blockchain/src/db/manager.ts +++ b/packages/blockchain/src/db/manager.ts @@ -12,10 +12,10 @@ import { hexToBytes } from 'ethereum-cryptography/utils' import { Cache } from './cache' import { DBOp, DBTarget } from './operation' -import type { DBOpData, DatabaseKey } from './operation' +import type { DatabaseKey } from './operation' import type { BlockBodyBytes, BlockBytes, BlockOptions } from '@ethereumjs/block' import type { Common } from '@ethereumjs/common' -import type { DB, DBObject } from '@ethereumjs/util' +import type { BatchDBOp, DB, DBObject, DelBatch, PutBatch } from '@ethereumjs/util' /** * @hidden @@ -211,26 +211,48 @@ export class DBManager { } let value = this._cache[cacheString].get(dbKey) if (value === undefined) { - value = (await this._db.get(dbKey)) as Uint8Array | undefined + value = (await this._db.get(dbKey, { + keyEncoding: dbGetOperation.baseDBOp.keyEncoding, + valueEncoding: dbGetOperation.baseDBOp.valueEncoding, + })) as Uint8Array | undefined if (value !== undefined) { - // TODO: Check if this comment is still valid - // Always cast values to Uint8Array since db sometimes returns values as `Buffer` - this._cache[cacheString].set(dbKey, Uint8Array.from(value)) + this._cache[cacheString].set(dbKey, value) } } return value } - return this._db.get(dbKey) + return this._db.get(dbKey, { + keyEncoding: dbGetOperation.baseDBOp.keyEncoding, + valueEncoding: dbGetOperation.baseDBOp.valueEncoding, + }) } /** * Performs a batch operation on db. */ async batch(ops: DBOp[]) { - const convertedOps: DBOpData[] = ops.map((op) => op.baseDBOp) + const convertedOps: BatchDBOp[] = ops.map((op) => { + const type = + op.baseDBOp.type !== undefined + ? op.baseDBOp.type + : op.baseDBOp.value !== undefined + ? 'put' + : 'del' + const convertedOp = { + key: op.baseDBOp.key, + value: op.baseDBOp.value, + type, + opts: { + keyEncoding: op.baseDBOp.keyEncoding, + valueEncoding: op.baseDBOp.valueEncoding, + }, + } + if (type === 'put') return convertedOp as PutBatch + else return convertedOp as DelBatch + }) // update the current cache for each operation ops.map((op) => op.updateCache(this._cache)) - return this._db.batch(convertedOps as any) + return this._db.batch(convertedOps) } } diff --git a/packages/blockchain/src/db/operation.ts b/packages/blockchain/src/db/operation.ts index 84730ba853..fcce734c97 100644 --- a/packages/blockchain/src/db/operation.ts +++ b/packages/blockchain/src/db/operation.ts @@ -1,3 +1,5 @@ +import { KeyEncoding, ValueEncoding } from '@ethereumjs/util' + import { HEADS_KEY, HEAD_BLOCK_KEY, @@ -30,10 +32,10 @@ export enum DBTarget { * @hidden */ export interface DBOpData { - type?: string + type?: 'put' | 'del' key: Uint8Array | string - keyEncoding: string - valueEncoding?: string + keyEncoding: KeyEncoding + valueEncoding?: ValueEncoding value?: Uint8Array | object } @@ -56,22 +58,24 @@ export class DBOp { this.baseDBOp = { key: '', - keyEncoding: 'view', - valueEncoding: 'view', + keyEncoding: KeyEncoding.Bytes, + valueEncoding: ValueEncoding.Bytes, } switch (operationTarget) { case DBTarget.Heads: { this.baseDBOp.key = HEADS_KEY - this.baseDBOp.valueEncoding = 'json' + this.baseDBOp.valueEncoding = ValueEncoding.JSON break } case DBTarget.HeadHeader: { this.baseDBOp.key = HEAD_HEADER_KEY + this.baseDBOp.keyEncoding = KeyEncoding.String break } case DBTarget.HeadBlock: { this.baseDBOp.key = HEAD_BLOCK_KEY + this.baseDBOp.keyEncoding = KeyEncoding.String break } case DBTarget.HashToNumber: { @@ -117,9 +121,9 @@ export class DBOp { dbOperation.baseDBOp.type = 'put' if (operationTarget === DBTarget.Heads) { - dbOperation.baseDBOp.valueEncoding = 'json' + dbOperation.baseDBOp.valueEncoding = ValueEncoding.JSON } else { - dbOperation.baseDBOp.valueEncoding = 'view' + dbOperation.baseDBOp.valueEncoding = ValueEncoding.Bytes } return dbOperation diff --git a/packages/client/test/blockchain/chain.spec.ts b/packages/client/test/blockchain/chain.spec.ts index fb1f6315e2..b0baa22e3c 100644 --- a/packages/client/test/blockchain/chain.spec.ts +++ b/packages/client/test/blockchain/chain.spec.ts @@ -2,13 +2,16 @@ // needed for karma-typescript bundling import { Block } from '@ethereumjs/block' import { Blockchain } from '@ethereumjs/blockchain' +import { KeyEncoding, ValueEncoding } from '@ethereumjs/util' import { bytesToHex, equalsBytes } from 'ethereum-cryptography/utils' import * as tape from 'tape' import * as util from 'util' // eslint-disable-line @typescript-eslint/no-unused-vars +import { string } from 'yargs' import { Chain } from '../../lib/blockchain' import { Config } from '../../lib/config' +import type { LevelDB } from '../../lib/execution/level' import type { BlockData, HeaderData } from '@ethereumjs/block' const config = new Config({ accountCache: 10000, storageCache: 1000 }) @@ -17,12 +20,18 @@ tape('[Chain]', (t) => { t.test('should test blockchain DB is initialized', async (t) => { const chain = await Chain.create({ config }) - const db = chain.chainDB + const db = chain.chainDB as LevelDB const testKey = 'name' const testValue = 'test' - await db.put(testKey, testValue) + await db.put(testKey, testValue, { + keyEncoding: KeyEncoding.String, + valueEncoding: ValueEncoding.String, + }) - const value = await db.get(testKey) + const value = await db.get(testKey, { + keyEncoding: KeyEncoding.String, + valueEncoding: ValueEncoding.String, + }) t.equal(value, testValue, 'read value matches written value') t.end() }) From c8d95df054624e27be161ba5db4fd13d916b85d5 Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Wed, 10 May 2023 14:45:10 -0400 Subject: [PATCH 18/27] remove unused imports --- packages/client/test/blockchain/chain.spec.ts | 2 - packages/ethash/src/level.ts | 69 ------------------- 2 files changed, 71 deletions(-) delete mode 100644 packages/ethash/src/level.ts diff --git a/packages/client/test/blockchain/chain.spec.ts b/packages/client/test/blockchain/chain.spec.ts index b0baa22e3c..e95904b4b1 100644 --- a/packages/client/test/blockchain/chain.spec.ts +++ b/packages/client/test/blockchain/chain.spec.ts @@ -5,8 +5,6 @@ import { Blockchain } from '@ethereumjs/blockchain' import { KeyEncoding, ValueEncoding } from '@ethereumjs/util' import { bytesToHex, equalsBytes } from 'ethereum-cryptography/utils' import * as tape from 'tape' -import * as util from 'util' // eslint-disable-line @typescript-eslint/no-unused-vars -import { string } from 'yargs' import { Chain } from '../../lib/blockchain' import { Config } from '../../lib/config' diff --git a/packages/ethash/src/level.ts b/packages/ethash/src/level.ts deleted file mode 100644 index 1647ace310..0000000000 --- a/packages/ethash/src/level.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { MemoryLevel } from 'memory-level' - -import type { BatchDBOp, DB } from '@ethereumjs/util' -import type { AbstractLevel } from 'abstract-level' - -const ENCODING_OPTS = { valueEncoding: 'json' } - -export class LevelDB implements DB { - readonly _leveldb: AbstractLevel< - string | Uint8Array, - string | Uint8Array, - { - cache: string[] - fullSize: number - cacheSize: number - seed: string - } - > - - constructor( - leveldb?: AbstractLevel< - string | Uint8Array, - string | Uint8Array, - { - cache: string[] - fullSize: number - cacheSize: number - seed: string - } - > - ) { - this._leveldb = leveldb ?? new MemoryLevel(ENCODING_OPTS) - } - - async get(key: any): Promise { - let value - try { - value = await this._leveldb.get(key, ENCODING_OPTS) - } catch (error: any) { - // https://github.com/Level/abstract-level/blob/915ad1317694d0ce8c580b5ab85d81e1e78a3137/abstract-level.js#L309 - // This should be `true` if the error came from LevelDB - // so we can check for `NOT true` to identify any non-404 errors - if (error.notFound !== true) { - throw error - } - } - return value - } - - async put(key: Uint8Array, val: any): Promise { - await this._leveldb.put(key, val, ENCODING_OPTS) - } - - async del(key: any): Promise { - await this._leveldb.del(key) - } - - async batch(opStack: BatchDBOp[]): Promise { - await this._leveldb.batch(opStack, ENCODING_OPTS) - } - - copy(): DB { - return new LevelDB(this._leveldb) - } - - open() { - return this._leveldb.open() - } -} From aca0e1d4e42165b2bcf4bbcf3beb49d05db2330f Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Wed, 10 May 2023 14:50:46 -0400 Subject: [PATCH 19/27] remove leveldb --- packages/ethash/test/block.spec.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/ethash/test/block.spec.ts b/packages/ethash/test/block.spec.ts index 0793837432..12aa5b567c 100644 --- a/packages/ethash/test/block.spec.ts +++ b/packages/ethash/test/block.spec.ts @@ -1,17 +1,15 @@ import { Block } from '@ethereumjs/block' import { Chain, Common, Hardfork } from '@ethereumjs/common' import { RLP } from '@ethereumjs/rlp' -import { toBytes } from '@ethereumjs/util' +import { MapDB, toBytes } from '@ethereumjs/util' import { hexToBytes } from 'ethereum-cryptography/utils' -import { MemoryLevel } from 'memory-level' import * as tape from 'tape' import { Ethash } from '../src' -import { LevelDB } from '../src/level' import type { BlockBytes } from '@ethereumjs/block' -const cacheDB = new LevelDB(new MemoryLevel()) +const cacheDB = new MapDB() const { validBlockRlp, invalidBlockRlp } = require('./ethash_block_rlp_tests.json') From ef57f9bfed81396d0e6c44eb009ca8581e749639 Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Wed, 10 May 2023 21:30:02 -0400 Subject: [PATCH 20/27] update trie to use strings --- package-lock.json | 8 ++---- packages/trie/src/db/checkpoint.ts | 43 +++++++++++++++++++----------- packages/trie/src/trie/trie.ts | 22 ++++++++------- packages/trie/src/types.ts | 4 +-- 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/package-lock.json b/package-lock.json index c529d3141d..eef7a07f5b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18108,13 +18108,11 @@ "@ethereumjs/block": "^4.2.1", "@ethereumjs/rlp": "^4.0.1", "@ethereumjs/util": "^8.0.5", - "abstract-level": "^1.0.3", "bigint-crypto-utils": "^3.0.23", "ethereum-cryptography": "^1.1.2" }, "devDependencies": { - "@ethereumjs/common": "^3.1.1", - "memory-level": "^1.0.0" + "@ethereumjs/common": "^3.1.1" }, "engines": { "node": ">=16" @@ -19758,10 +19756,8 @@ "@ethereumjs/common": "^3.1.1", "@ethereumjs/rlp": "^4.0.1", "@ethereumjs/util": "^8.0.5", - "abstract-level": "^1.0.3", "bigint-crypto-utils": "^3.0.23", - "ethereum-cryptography": "^1.1.2", - "memory-level": "^1.0.0" + "ethereum-cryptography": "^1.1.2" } }, "@ethereumjs/evm": { diff --git a/packages/trie/src/db/checkpoint.ts b/packages/trie/src/db/checkpoint.ts index 6d6b231ed5..59157d55d7 100644 --- a/packages/trie/src/db/checkpoint.ts +++ b/packages/trie/src/db/checkpoint.ts @@ -1,7 +1,8 @@ import { KeyEncoding, ValueEncoding, bytesToHex, hexStringToBytes } from '@ethereumjs/util' +import { hexToBytes } from 'ethereum-cryptography/utils' import type { Checkpoint, CheckpointDBOpts } from '../types' -import type { BatchDBOp, DB } from '@ethereumjs/util' +import type { BatchDBOp, DB, DelBatch, PutBatch } from '@ethereumjs/util' import type LRUCache from 'lru-cache' const LRU = require('lru-cache') @@ -12,7 +13,7 @@ const LRU = require('lru-cache') */ export class CheckpointDB implements DB { public checkpoints: Checkpoint[] - public db: DB + public db: DB public readonly cacheSize: number protected _cache?: LRUCache @@ -133,24 +134,25 @@ export class CheckpointDB implements DB { // Lookup the value in our diff cache. We return the latest checkpointed value (which should be the value on disk) for (let index = this.checkpoints.length - 1; index >= 0; index--) { - if (this.checkpoints[index].keyValueMap.has(bytesToHex(key))) { - return this.checkpoints[index].keyValueMap.get(bytesToHex(key)) + if (this.checkpoints[index].keyValueMap.has(keyHex)) { + return this.checkpoints[index].keyValueMap.get(keyHex) } } // Nothing has been found in diff cache, look up from disk - const value = await this.db.get(key, { - keyEncoding: KeyEncoding.Bytes, - valueEncoding: ValueEncoding.Bytes, + const valueHex = await this.db.get(keyHex, { + keyEncoding: KeyEncoding.String, + valueEncoding: ValueEncoding.String, }) this._stats.db.reads += 1 - if (value !== null) { + if (valueHex !== undefined) { this._stats.db.hits += 1 } + const value = valueHex !== undefined ? hexToBytes(valueHex) : undefined this._cache?.set(keyHex, value) if (this.hasCheckpoints()) { // Since we are a checkpoint, put this value in diff cache, // so future `get` calls will not look the key up again from disk. - this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(bytesToHex(key), value) + this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(keyHex, value) } return value @@ -161,13 +163,14 @@ export class CheckpointDB implements DB { */ async put(key: Uint8Array, value: Uint8Array): Promise { const keyHex = bytesToHex(key) + const valueHex = bytesToHex(value) if (this.hasCheckpoints()) { // put value in diff cache this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(keyHex, value) } else { - await this.db.put(key, value, { - keyEncoding: KeyEncoding.Bytes, - valueEncoding: ValueEncoding.Bytes, + await this.db.put(keyHex, valueHex, { + keyEncoding: KeyEncoding.String, + valueEncoding: ValueEncoding.String, }) this._stats.db.writes += 1 @@ -188,8 +191,8 @@ export class CheckpointDB implements DB { this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(keyHex, undefined) } else { // delete the value on disk - await this.db.del(key, { - keyEncoding: KeyEncoding.Bytes, + await this.db.del(keyHex, { + keyEncoding: KeyEncoding.String, }) this._stats.db.writes += 1 @@ -213,7 +216,17 @@ export class CheckpointDB implements DB { } } } else { - await this.db.batch(opStack) + const convertedOps = opStack.map((op) => { + const convertedOp = { + key: bytesToHex(op.key), + value: op.type === 'put' ? bytesToHex(op.value) : undefined, + type: op.type, + opts: op.opts, + } + if (op.type === 'put') return convertedOp as PutBatch + else return convertedOp as DelBatch + }) + await this.db.batch(convertedOps) } } diff --git a/packages/trie/src/trie/trie.ts b/packages/trie/src/trie/trie.ts index 26e4961261..61e7a97a98 100644 --- a/packages/trie/src/trie/trie.ts +++ b/packages/trie/src/trie/trie.ts @@ -8,6 +8,7 @@ import { equalsBytes, } from '@ethereumjs/util' import { keccak256 } from 'ethereum-cryptography/keccak' +import { hexToBytes } from 'ethereum-cryptography/utils' import { CheckpointDB } from '../db' import { verifyRangeProof } from '../proof/range' @@ -68,7 +69,7 @@ export class Trie { this._opts = { ...this._opts, ...opts } } - this.database(opts?.db ?? new MapDB()) + this.database(opts?.db ?? new MapDB()) this.EMPTY_TRIE_ROOT = this.hash(RLP_EMPTY_STRING) this._hashLen = this.EMPTY_TRIE_ROOT.length @@ -88,15 +89,15 @@ export class Trie { if (opts?.db !== undefined && opts?.useRootPersistence === true) { if (opts?.root === undefined) { - opts.root = - (await opts?.db.get(key, { - keyEncoding: KeyEncoding.Bytes, - valueEncoding: ValueEncoding.Bytes, - })) ?? undefined + const rootHex = await opts?.db.get(bytesToHex(key), { + keyEncoding: KeyEncoding.String, + valueEncoding: ValueEncoding.String, + }) + opts.root = rootHex !== undefined ? hexToBytes(rootHex) : undefined } else { - await opts?.db.put(key, opts.root, { - keyEncoding: KeyEncoding.Bytes, - valueEncoding: ValueEncoding.Bytes, + await opts?.db.put(bytesToHex(key), bytesToHex(opts.root), { + keyEncoding: KeyEncoding.String, + valueEncoding: ValueEncoding.String, }) } } @@ -104,7 +105,7 @@ export class Trie { return new Trie(opts) } - database(db?: DB) { + database(db?: DB) { if (db !== undefined) { if (db instanceof CheckpointDB) { throw new Error('Cannot pass in an instance of CheckpointDB') @@ -392,6 +393,7 @@ export class Trie { const toSave: BatchDBOp[] = [] const lastNode = stack.pop() if (!lastNode) { + console.trace('no last node!') throw new Error('Stack underflow') } diff --git a/packages/trie/src/types.ts b/packages/trie/src/types.ts index ebb6984193..ccb8148c5f 100644 --- a/packages/trie/src/types.ts +++ b/packages/trie/src/types.ts @@ -27,7 +27,7 @@ export interface TrieOpts { /** * A database instance. */ - db?: DB + db?: DB /** * A `Uint8Array` for the root of a previously stored trie @@ -84,7 +84,7 @@ export interface CheckpointDBOpts { /** * A database instance. */ - db: DB + db: DB /** * Cache size (default: 0) From b5fa536646a368066553d4d6e4da63515f21a50f Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Wed, 10 May 2023 21:41:19 -0400 Subject: [PATCH 21/27] Slight fixes --- packages/client/lib/execution/level.ts | 2 +- packages/client/lib/execution/vmexecution.ts | 2 +- packages/client/lib/sync/fetcher/accountfetcher.ts | 2 +- packages/client/lib/sync/fetcher/storagefetcher.ts | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/client/lib/execution/level.ts b/packages/client/lib/execution/level.ts index 1bab7de2fe..31780aaef4 100644 --- a/packages/client/lib/execution/level.ts +++ b/packages/client/lib/execution/level.ts @@ -37,7 +37,7 @@ const getEncodings = (opts: EncodingOpts = {}) => { /** * LevelDB is a thin wrapper around the underlying levelup db, - * which validates inputs and sets encoding type. + * corresponding to the {@link DB} */ export class LevelDB implements DB { _leveldb: AbstractLevel diff --git a/packages/client/lib/execution/vmexecution.ts b/packages/client/lib/execution/vmexecution.ts index 8f64893b93..8e370e7300 100644 --- a/packages/client/lib/execution/vmexecution.ts +++ b/packages/client/lib/execution/vmexecution.ts @@ -51,7 +51,7 @@ export class VMExecution extends Execution { if (this.config.vm === undefined) { const trie = new Trie({ - db: new LevelDB(this.stateDB) as DB, + db: new LevelDB(this.stateDB) as any, useKeyHashing: true, cacheSize: this.config.trieCache, }) diff --git a/packages/client/lib/sync/fetcher/accountfetcher.ts b/packages/client/lib/sync/fetcher/accountfetcher.ts index 79387b3da7..8439b5d407 100644 --- a/packages/client/lib/sync/fetcher/accountfetcher.ts +++ b/packages/client/lib/sync/fetcher/accountfetcher.ts @@ -203,7 +203,7 @@ export class AccountFetcher extends Fetcher } } - const trie = new Trie({ db: new LevelDB() as DB }) + const trie = new Trie({ db: new LevelDB() as any }) const keys = accounts.map((acc: any) => acc.hash) const values = accounts.map((acc: any) => accountBodyToRLP(acc.body)) // convert the request to the right values diff --git a/packages/client/lib/sync/fetcher/storagefetcher.ts b/packages/client/lib/sync/fetcher/storagefetcher.ts index ffbc2f1639..a3b231eda5 100644 --- a/packages/client/lib/sync/fetcher/storagefetcher.ts +++ b/packages/client/lib/sync/fetcher/storagefetcher.ts @@ -125,7 +125,7 @@ export class StorageFetcher extends Fetcher slot.hash) const values = slots.map((slot: any) => slot.body) return await trie.verifyRangeProof( @@ -156,7 +156,7 @@ export class StorageFetcher extends Fetcher { return { From 7f1d7e0bba93ce5ceea0b81b09a35dceb4d00f10 Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Wed, 10 May 2023 21:45:25 -0400 Subject: [PATCH 22/27] lint --- packages/client/lib/execution/vmexecution.ts | 1 - packages/client/lib/sync/fetcher/accountfetcher.ts | 1 - packages/client/lib/sync/fetcher/storagefetcher.ts | 1 - packages/trie/src/trie/trie.ts | 1 - 4 files changed, 4 deletions(-) diff --git a/packages/client/lib/execution/vmexecution.ts b/packages/client/lib/execution/vmexecution.ts index 8e370e7300..669f534f03 100644 --- a/packages/client/lib/execution/vmexecution.ts +++ b/packages/client/lib/execution/vmexecution.ts @@ -21,7 +21,6 @@ import { ReceiptsManager } from './receipt' import type { ExecutionOptions } from './execution' import type { Block } from '@ethereumjs/block' -import type { DB } from '@ethereumjs/util' import type { RunBlockOpts, TxReceipt } from '@ethereumjs/vm' export class VMExecution extends Execution { diff --git a/packages/client/lib/sync/fetcher/accountfetcher.ts b/packages/client/lib/sync/fetcher/accountfetcher.ts index 8439b5d407..e6459ded93 100644 --- a/packages/client/lib/sync/fetcher/accountfetcher.ts +++ b/packages/client/lib/sync/fetcher/accountfetcher.ts @@ -25,7 +25,6 @@ import type { EventBusType } from '../../types' import type { FetcherOptions } from './fetcher' import type { StorageRequest } from './storagefetcher' import type { Job } from './types' -import type { DB } from '@ethereumjs/util' import type { Debugger } from 'debug' type AccountDataResponse = AccountData[] & { completed?: boolean } diff --git a/packages/client/lib/sync/fetcher/storagefetcher.ts b/packages/client/lib/sync/fetcher/storagefetcher.ts index a3b231eda5..93abb2ccc0 100644 --- a/packages/client/lib/sync/fetcher/storagefetcher.ts +++ b/packages/client/lib/sync/fetcher/storagefetcher.ts @@ -18,7 +18,6 @@ import type { Peer } from '../../net/peer' import type { StorageData } from '../../net/protocol/snapprotocol' import type { FetcherOptions } from './fetcher' import type { Job } from './types' -import type { DB } from '@ethereumjs/util' import type { Debugger } from 'debug' const TOTAL_RANGE_END = BigInt(2) ** BigInt(256) - BigInt(1) diff --git a/packages/trie/src/trie/trie.ts b/packages/trie/src/trie/trie.ts index 61e7a97a98..c1bfcb3f7d 100644 --- a/packages/trie/src/trie/trie.ts +++ b/packages/trie/src/trie/trie.ts @@ -393,7 +393,6 @@ export class Trie { const toSave: BatchDBOp[] = [] const lastNode = stack.pop() if (!lastNode) { - console.trace('no last node!') throw new Error('Stack underflow') } From 7d807cf466899bca136b46ff14e3bdc74b9ae0ba Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Thu, 11 May 2023 08:17:17 -0400 Subject: [PATCH 23/27] fix trie tests --- packages/trie/test/proof/range.spec.ts | 2 +- packages/trie/test/trie/checkpoint.spec.ts | 2 +- packages/trie/test/trie/trie.spec.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/trie/test/proof/range.spec.ts b/packages/trie/test/proof/range.spec.ts index 247dfe0bd4..01cdaf722b 100644 --- a/packages/trie/test/proof/range.spec.ts +++ b/packages/trie/test/proof/range.spec.ts @@ -22,7 +22,7 @@ const TRIE_SIZE = 512 * @param addKey - whether to add 100 ordered keys * @returns Trie object and sorted entries */ -async function randomTrie(db: DB, addKey: boolean = true) { +async function randomTrie(db: DB, addKey: boolean = true) { const entries: [Uint8Array, Uint8Array][] = [] const trie = new Trie({ db }) diff --git a/packages/trie/test/trie/checkpoint.spec.ts b/packages/trie/test/trie/checkpoint.spec.ts index 8f35fb8d74..551059b2a5 100644 --- a/packages/trie/test/trie/checkpoint.spec.ts +++ b/packages/trie/test/trie/checkpoint.spec.ts @@ -277,7 +277,7 @@ tape('testing checkpoints', function (tester) { // I.e. the trie is pruned. t.deepEqual( // @ts-expect-error - [...CommittedState._db.db._database.values()].map((value) => bytesToHex(value)), + [...CommittedState._db.db._database.values()].map((value) => value), [ 'd7eba6ee0f011acb031b79554d57001c42fbfabb150eb9fdd3b6d434f7b791eb', 'e3a1202418cf7414b1e6c2c8d92b4673eecdb4aac88f7f58623e3be903aefb2fd4655c32', diff --git a/packages/trie/test/trie/trie.spec.ts b/packages/trie/test/trie/trie.spec.ts index 5f3f4eb21a..6657511611 100644 --- a/packages/trie/test/trie/trie.spec.ts +++ b/packages/trie/test/trie/trie.spec.ts @@ -163,7 +163,7 @@ for (const { constructor, defaults, title } of [ }) t.test('persist and restore the root', async function (st) { - const db = new MapDB() + const db = new MapDB() const trie = await constructor.create({ ...defaults, db, useRootPersistence: true }) // @ts-expect-error From cd0ab876e4471a3a91ab608809ee2933dc313817 Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Thu, 11 May 2023 08:18:40 -0400 Subject: [PATCH 24/27] cast trie db as any --- packages/client/test/net/protocol/snapprotocol.spec.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/client/test/net/protocol/snapprotocol.spec.ts b/packages/client/test/net/protocol/snapprotocol.spec.ts index 76b5a34003..84aa0e719c 100644 --- a/packages/client/test/net/protocol/snapprotocol.spec.ts +++ b/packages/client/test/net/protocol/snapprotocol.spec.ts @@ -18,8 +18,6 @@ import { Chain } from '../../../lib/blockchain' import { Config } from '../../../lib/config' import { LevelDB } from '../../../lib/execution/level' import { SnapProtocol } from '../../../lib/net/protocol' - -import type { DB } from '@ethereumjs/util' ;(BigInt.prototype as any).toJSON = function () { return this.toString() } @@ -202,7 +200,7 @@ tape('[SnapProtocol]', (t) => { resData ) - const trie = new Trie({ db: new LevelDB() as DB }) + const trie = new Trie({ db: new LevelDB() as any }) try { const keys = accounts.map((acc: any) => acc.hash) const values = accounts.map((acc: any) => accountBodyToRLP(acc.body)) @@ -342,7 +340,7 @@ tape('[SnapProtocol]', (t) => { // lastAccount const lastAccountSlots = slots[0] const lastAccountStorageRoot = (lastAccount.body as any)[2] - const trie = new Trie({ db: new LevelDB() as DB }) + const trie = new Trie({ db: new LevelDB() as any }) try { const keys = lastAccountSlots.map((acc: any) => acc.hash) const values = lastAccountSlots.map((acc: any) => acc.body) From b1346f20dc175b9ae8b95060d0006a30b423855f Mon Sep 17 00:00:00 2001 From: Gabriel Rocheleau Date: Thu, 11 May 2023 19:30:31 -0400 Subject: [PATCH 25/27] client: improve levelDB types and remove unnecessary typecasting --- packages/client/lib/config.ts | 2 +- packages/client/lib/execution/level.ts | 31 +++++++++---------- packages/client/lib/execution/vmexecution.ts | 2 +- .../client/lib/sync/fetcher/accountfetcher.ts | 2 +- .../client/lib/sync/fetcher/storagefetcher.ts | 4 +-- packages/client/test/sim/snapsync.spec.ts | 2 +- 6 files changed, 20 insertions(+), 23 deletions(-) diff --git a/packages/client/lib/config.ts b/packages/client/lib/config.ts index c184d492fb..75b72df938 100644 --- a/packages/client/lib/config.ts +++ b/packages/client/lib/config.ts @@ -569,7 +569,7 @@ export class Config { * Returns the config level db. */ static getConfigDB(networkDir: string) { - return new Level(`${networkDir}/config` as any) + return new Level(`${networkDir}/config`) } /** diff --git a/packages/client/lib/execution/level.ts b/packages/client/lib/execution/level.ts index 1bab7de2fe..528bf8a868 100644 --- a/packages/client/lib/execution/level.ts +++ b/packages/client/lib/execution/level.ts @@ -39,7 +39,11 @@ const getEncodings = (opts: EncodingOpts = {}) => { * LevelDB is a thin wrapper around the underlying levelup db, * which validates inputs and sets encoding type. */ -export class LevelDB implements DB { +export class LevelDB< + TKey extends Uint8Array | string = Uint8Array | string, + TValue extends Uint8Array | string | DBObject = Uint8Array | string | DBObject +> implements DB +{ _leveldb: AbstractLevel /** @@ -56,10 +60,7 @@ export class LevelDB implements DB { + async get(key: TKey, opts?: EncodingOpts): Promise { let value const encodings = getEncodings(opts) @@ -76,17 +77,13 @@ export class LevelDB implements DB { + async put(key: TKey, val: TValue, opts?: {}): Promise { const encodings = getEncodings(opts) await this._leveldb.put(key, val, encodings) } @@ -94,29 +91,29 @@ export class LevelDB implements DB { + async del(key: TKey): Promise { await this._leveldb.del(key) } /** * @inheritDoc */ - async batch(opStack: BatchDBOp[]): Promise { + async batch(opStack: BatchDBOp[]): Promise { const levelOps = [] for (const op of opStack) { const encodings = getEncodings(op.opts) levelOps.push({ ...op, ...encodings }) } - await this._leveldb.batch(levelOps) + // TODO: Investigate why as any is necessary + await this._leveldb.batch(levelOps as any) } /** * @inheritDoc */ - copy(): DB { - //@ts-expect-error - return new LevelDB(this._leveldb) + copy(): DB { + return new LevelDB(this._leveldb) } open() { diff --git a/packages/client/lib/execution/vmexecution.ts b/packages/client/lib/execution/vmexecution.ts index 8f64893b93..5c7db5b714 100644 --- a/packages/client/lib/execution/vmexecution.ts +++ b/packages/client/lib/execution/vmexecution.ts @@ -51,7 +51,7 @@ export class VMExecution extends Execution { if (this.config.vm === undefined) { const trie = new Trie({ - db: new LevelDB(this.stateDB) as DB, + db: new LevelDB(this.stateDB), useKeyHashing: true, cacheSize: this.config.trieCache, }) diff --git a/packages/client/lib/sync/fetcher/accountfetcher.ts b/packages/client/lib/sync/fetcher/accountfetcher.ts index 79387b3da7..405840ef35 100644 --- a/packages/client/lib/sync/fetcher/accountfetcher.ts +++ b/packages/client/lib/sync/fetcher/accountfetcher.ts @@ -203,7 +203,7 @@ export class AccountFetcher extends Fetcher } } - const trie = new Trie({ db: new LevelDB() as DB }) + const trie = new Trie({ db: new LevelDB() }) const keys = accounts.map((acc: any) => acc.hash) const values = accounts.map((acc: any) => accountBodyToRLP(acc.body)) // convert the request to the right values diff --git a/packages/client/lib/sync/fetcher/storagefetcher.ts b/packages/client/lib/sync/fetcher/storagefetcher.ts index ffbc2f1639..488decb0f6 100644 --- a/packages/client/lib/sync/fetcher/storagefetcher.ts +++ b/packages/client/lib/sync/fetcher/storagefetcher.ts @@ -125,7 +125,7 @@ export class StorageFetcher extends Fetcher slot.hash) const values = slots.map((slot: any) => slot.body) return await trie.verifyRangeProof( @@ -156,7 +156,7 @@ export class StorageFetcher extends Fetcher { return { diff --git a/packages/client/test/sim/snapsync.spec.ts b/packages/client/test/sim/snapsync.spec.ts index 5be6083f8e..22b012e418 100644 --- a/packages/client/test/sim/snapsync.spec.ts +++ b/packages/client/test/sim/snapsync.spec.ts @@ -139,7 +139,7 @@ tape('simple mainnet test run', async (t) => { try { if (ejsClient !== null && snapCompleted !== undefined) { // call sync if not has been called yet - void ejsClient.services[0].synchronizer.sync() + void ejsClient.services[0].synchronizer?.sync() // wait on the sync promise to complete if it has been called independently const snapSyncTimeout = new Promise((_resolve, reject) => setTimeout(reject, 40000)) try { From 16a5ab9847e7f385c35f3b38fd20791b8ef32478 Mon Sep 17 00:00:00 2001 From: Gabriel Rocheleau Date: Thu, 11 May 2023 19:34:09 -0400 Subject: [PATCH 26/27] client: remove additional unnecessary typecasting --- packages/client/test/net/protocol/snapprotocol.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/client/test/net/protocol/snapprotocol.spec.ts b/packages/client/test/net/protocol/snapprotocol.spec.ts index 84aa0e719c..9dca7985b0 100644 --- a/packages/client/test/net/protocol/snapprotocol.spec.ts +++ b/packages/client/test/net/protocol/snapprotocol.spec.ts @@ -200,7 +200,7 @@ tape('[SnapProtocol]', (t) => { resData ) - const trie = new Trie({ db: new LevelDB() as any }) + const trie = new Trie({ db: new LevelDB() }) try { const keys = accounts.map((acc: any) => acc.hash) const values = accounts.map((acc: any) => accountBodyToRLP(acc.body)) @@ -340,7 +340,7 @@ tape('[SnapProtocol]', (t) => { // lastAccount const lastAccountSlots = slots[0] const lastAccountStorageRoot = (lastAccount.body as any)[2] - const trie = new Trie({ db: new LevelDB() as any }) + const trie = new Trie({ db: new LevelDB() }) try { const keys = lastAccountSlots.map((acc: any) => acc.hash) const values = lastAccountSlots.map((acc: any) => acc.body) From 9d3e04bd0e6afa61ec75e6cdaa04449de7277465 Mon Sep 17 00:00:00 2001 From: Gabriel Rocheleau Date: Thu, 11 May 2023 19:42:32 -0400 Subject: [PATCH 27/27] client: type issue --- packages/client/lib/execution/vmexecution.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/lib/execution/vmexecution.ts b/packages/client/lib/execution/vmexecution.ts index 0d5a5fe031..83b82a48f5 100644 --- a/packages/client/lib/execution/vmexecution.ts +++ b/packages/client/lib/execution/vmexecution.ts @@ -50,7 +50,7 @@ export class VMExecution extends Execution { if (this.config.vm === undefined) { const trie = new Trie({ - db: new LevelDB(this.stateDB), + db: new LevelDB(this.stateDB), useKeyHashing: true, cacheSize: this.config.trieCache, })