diff --git a/package-lock.json b/package-lock.json index f8a2659734..191f1c226a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16938,8 +16938,7 @@ "@types/readable-stream": "^2.3.13", "debug": "^4.3.4", "ethereum-cryptography": "^2.2.1", - "lru-cache": "10.1.0", - "readable-stream": "^3.6.0" + "lru-cache": "10.1.0" }, "devDependencies": { "@ethereumjs/genesis": "^0.2.2", diff --git a/packages/statemanager/src/stateManager.ts b/packages/statemanager/src/stateManager.ts index 84a0da6e8a..e4261d9163 100644 --- a/packages/statemanager/src/stateManager.ts +++ b/packages/statemanager/src/stateManager.ts @@ -9,7 +9,6 @@ import { KECCAK256_RLP, KECCAK256_RLP_S, bigIntToHex, - bytesToBigInt, bytesToHex, bytesToUnprefixedHex, concatBytes, @@ -231,7 +230,6 @@ export class DefaultStateManager implements EVMStateManagerInterface { type: opts.accountCacheOpts?.type ?? CacheType.ORDERED_MAP, size: opts.accountCacheOpts?.size ?? 100000, } - if (!this._accountCacheSettings.deactivate) { this._accountCache = new AccountCache({ size: this._accountCacheSettings.size, @@ -245,7 +243,6 @@ export class DefaultStateManager implements EVMStateManagerInterface { type: opts.storageCacheOpts?.type ?? CacheType.ORDERED_MAP, size: opts.storageCacheOpts?.size ?? 20000, } - if (!this._storageCacheSettings.deactivate) { this._storageCache = new StorageCache({ size: this._storageCacheSettings.size, @@ -259,7 +256,6 @@ export class DefaultStateManager implements EVMStateManagerInterface { type: opts.codeCacheOpts?.type ?? CacheType.ORDERED_MAP, size: opts.codeCacheOpts?.size ?? 20000, } - if (!this._codeCacheSettings.deactivate) { this._codeCache = new CodeCache({ size: this._codeCacheSettings.size, @@ -991,19 +987,8 @@ export class DefaultStateManager implements EVMStateManagerInterface { } const trie = this._getStorageTrie(address, account) - return new Promise((resolve, reject) => { - const storage: StorageDump = {} - const stream = trie.createReadStream() - - stream.on('data', (val: any) => { - storage[bytesToHex(val.key)] = bytesToHex(val.value) - }) - stream.on('end', () => { - resolve(storage) - }) - stream.on('error', (e) => { - reject(e) - }) + return trie.getValueMap().then((value) => { + return value.values }) } @@ -1026,44 +1011,24 @@ export class DefaultStateManager implements EVMStateManagerInterface { if (!account) { throw new Error(`Account does not exist.`) } - const trie = this._getStorageTrie(address, account) - return new Promise((resolve, reject) => { - let inRange = false - let i = 0 - - /** Object conforming to {@link StorageRange.storage}. */ - const storageMap: StorageRange['storage'] = {} - const stream = trie.createReadStream() - - stream.on('data', (val: any) => { - if (!inRange) { - // Check if the key is already in the correct range. - if (bytesToBigInt(val.key) >= startKey) { - inRange = true - } else { - return - } - } + const trie = this._getStorageTrie(address, account) - if (i < limit) { - storageMap[bytesToHex(val.key)] = { key: null, value: bytesToHex(val.value) } - i++ - } else if (i === limit) { - resolve({ - storage: storageMap, - nextKey: bytesToHex(val.key), - }) + return trie.getValueMap(startKey, limit).then((value) => { + const values = value.values + const dump = Object.create(null) + for (const key of Object.keys(values)) { + const val = values[key] + dump[key] = { + key: null, + value: val, } - }) + } - stream.on('end', () => { - resolve({ - storage: storageMap, - nextKey: null, - }) - }) - stream.on('error', (e) => reject(e)) + return { + storage: dump, + nextKey: value.nextKey, + } }) } diff --git a/packages/trie/package.json b/packages/trie/package.json index 17251fd4de..3eed919db0 100644 --- a/packages/trie/package.json +++ b/packages/trie/package.json @@ -59,8 +59,7 @@ "@types/readable-stream": "^2.3.13", "debug": "^4.3.4", "lru-cache": "10.1.0", - "ethereum-cryptography": "^2.2.1", - "readable-stream": "^3.6.0" + "ethereum-cryptography": "^2.2.1" }, "devDependencies": { "@ethereumjs/genesis": "^0.2.2", diff --git a/packages/trie/src/trie.ts b/packages/trie/src/trie.ts index 4cb59f7afe..703c099519 100644 --- a/packages/trie/src/trie.ts +++ b/packages/trie/src/trie.ts @@ -3,11 +3,13 @@ import { RLP } from '@ethereumjs/rlp' import { + BIGINT_0, KeyEncoding, Lock, MapDB, RLP_EMPTY_STRING, ValueEncoding, + bytesToBigInt, bytesToHex, bytesToUnprefixedHex, bytesToUtf8, @@ -29,8 +31,8 @@ import { import { verifyRangeProof } from './proof/range.js' import { ROOT_DB_KEY } from './types.js' import { _walkTrie } from './util/asyncWalk.js' +import { nibbleTypeToPackedBytes } from './util/encoding.js' import { bytesToNibbles, matchingNibbleLength } from './util/nibbles.js' -import { TrieReadStream as ReadStream } from './util/readStream.js' import { WalkController } from './util/walkController.js' import type { @@ -1081,14 +1083,6 @@ export class Trie { return true } - /** - * The `data` event is given an `Object` that has two properties; the `key` and the `value`. Both should be Uint8Arrays. - * @return Returns a [stream](https://nodejs.org/dist/latest-v12.x/docs/api/stream.html#stream_class_stream_readable) of the contents of the `trie` - */ - createReadStream(): ReadStream { - return new ReadStream(this) - } - /** * Returns a copy of the underlying trie. * @@ -1227,4 +1221,45 @@ export class Trie { this.debug(`Deleting ${this._db.checkpoints.length} checkpoints.`, ['FLUSH_CHECKPOINTS']) this._db.checkpoints = [] } + + /** + * Returns a list of values stored in the trie + * @param startKey first unhashed key in the range to be returned (defaults to 0) + * @param limit - the number of keys to be returned (undefined means all keys) + * @returns an object with two properties (a map of all key/value pairs in the trie - or in the specified range) and then a `nextKey` reference if a range is specified + */ + async getValueMap( + startKey = BIGINT_0, + limit?: number + ): Promise<{ values: { [key: string]: string }; nextKey: null | string }> { + // If limit is undefined, all keys are inRange + let inRange = limit !== undefined ? false : true + let i = 0 + const values: { [key: string]: string } = {} + let nextKey: string | null = null + await this.walkAllValueNodes(async (node: TrieNode, currentKey: number[]) => { + if (node instanceof LeafNode) { + const keyBytes = nibbleTypeToPackedBytes(currentKey.concat(node.key())) + if (!inRange) { + // Check if the key is already in the correct range. + if (bytesToBigInt(keyBytes) >= startKey) { + inRange = true + } else { + return + } + } + + if (limit === undefined || i < limit) { + values[bytesToHex(keyBytes)] = bytesToHex(node._value) + i++ + } else if (i === limit) { + nextKey = bytesToHex(keyBytes) + } + } + }) + return { + values, + nextKey, + } + } } diff --git a/packages/trie/src/util/index.ts b/packages/trie/src/util/index.ts index 1297a1dbca..d86ae67d7d 100644 --- a/packages/trie/src/util/index.ts +++ b/packages/trie/src/util/index.ts @@ -1,5 +1,4 @@ export * from './encoding.js' export * from './genesisState.js' -export * from './readStream.js' export * from './tasks.js' export * from './walkController.js' diff --git a/packages/trie/src/util/readStream.ts b/packages/trie/src/util/readStream.ts deleted file mode 100644 index dfc002d29c..0000000000 --- a/packages/trie/src/util/readStream.ts +++ /dev/null @@ -1,66 +0,0 @@ -// eslint-disable-next-line implicit-dependencies/no-implicit -import { Readable } from 'readable-stream' - -import { BranchNode, LeafNode } from '../node/index.js' - -import { nibblestoBytes } from './nibbles.js' - -import type { Trie } from '../trie.js' -import type { FoundNodeFunction } from '../types.js' - -const _findValueNodes = async (trie: Trie, onFound: FoundNodeFunction): Promise => { - const outerOnFound: FoundNodeFunction = async (nodeRef, node, key, walkController) => { - let fullKey = key - if (node instanceof LeafNode) { - fullKey = key.concat(node.key()) - // found leaf node! - onFound(nodeRef, node, fullKey, walkController) - } else if (node instanceof BranchNode && node.value()) { - // found branch with value - onFound(nodeRef, node, fullKey, walkController) - } else { - // keep looking for value nodes - if (node !== null) { - walkController.allChildren(node, key) - } - } - } - await trie.walkTrie(trie.root(), outerOnFound) -} - -export class TrieReadStream extends Readable { - private trie: Trie - private _started: boolean - - constructor(trie: Trie) { - super({ objectMode: true }) - - this.trie = trie - this._started = false - } - - async _read() { - if (this._started) { - return - } - this._started = true - try { - await _findValueNodes(this.trie, async (_, node, key, walkController) => { - if (node !== null) { - this.push({ - key: nibblestoBytes(key), - value: node.value(), - }) - walkController.allChildren(node, key) - } - }) - } catch (error: any) { - if (error.message === 'Missing node in DB') { - // pass - } else { - throw error - } - } - this.push(null) - } -} diff --git a/packages/trie/test/stream.spec.ts b/packages/trie/test/stream.spec.ts deleted file mode 100644 index 5c4d05ff43..0000000000 --- a/packages/trie/test/stream.spec.ts +++ /dev/null @@ -1,164 +0,0 @@ -import { utf8ToBytes } from '@ethereumjs/util' -import { assert, describe, it } from 'vitest' - -import { Trie } from '../src/index.js' - -import type { BatchDBOp } from '@ethereumjs/util' - -describe('kv stream test', () => { - const trie = new Trie() - const ops = [ - { - type: 'del', - key: utf8ToBytes('father'), - }, - { - type: 'put', - key: utf8ToBytes('name'), - value: utf8ToBytes('Yuri Irsenovich Kim'), - }, - { - type: 'put', - key: utf8ToBytes('dob'), - value: utf8ToBytes('16 February 1941'), - }, - { - type: 'put', - key: utf8ToBytes('spouse'), - value: utf8ToBytes('Kim Young-sook'), - }, - { - type: 'put', - key: utf8ToBytes('occupation'), - value: utf8ToBytes('Clown'), - }, - { - type: 'put', - key: utf8ToBytes('nameads'), - value: utf8ToBytes('Yuri Irsenovich Kim'), - }, - { - type: 'put', - key: utf8ToBytes('namfde'), - value: utf8ToBytes('Yuri Irsenovich Kim'), - }, - { - type: 'put', - key: utf8ToBytes('namsse'), - value: utf8ToBytes('Yuri Irsenovich Kim'), - }, - { - type: 'put', - key: utf8ToBytes('dofab'), - value: utf8ToBytes('16 February 1941'), - }, - { - type: 'put', - key: utf8ToBytes('spoudse'), - value: utf8ToBytes('Kim Young-sook'), - }, - { - type: 'put', - key: utf8ToBytes('occupdsation'), - value: utf8ToBytes('Clown'), - }, - { - type: 'put', - key: utf8ToBytes('dozzzb'), - value: utf8ToBytes('16 February 1941'), - }, - { - type: 'put', - key: utf8ToBytes('spouszze'), - value: utf8ToBytes('Kim Young-sook'), - }, - { - type: 'put', - key: utf8ToBytes('occupatdfion'), - value: utf8ToBytes('Clown'), - }, - { - type: 'put', - key: utf8ToBytes('dssob'), - value: utf8ToBytes('16 February 1941'), - }, - { - type: 'put', - key: utf8ToBytes('spossuse'), - value: utf8ToBytes('Kim Young-sook'), - }, - { - type: 'put', - key: utf8ToBytes('occupssation'), - value: utf8ToBytes('Clown'), - }, - ] as BatchDBOp[] - - const valObj1 = {} as any - const valObj2 = {} as any - for (const op of ops) { - if (op.type === 'put') { - valObj1[op.key.toString()] = op.value.toString() - valObj2[op.key.toString()] = op.value.toString() - } - } - - it('should populate trie', async () => { - await trie.batch(ops) - }) - - it('should fetch all of the nodes', () => { - const stream = trie.createReadStream() - stream.on('data', (d: any) => { - const key = d.key.toString() - const value = d.value.toString() - assert.equal(valObj1[key], value) - delete valObj1[key] - }) - stream.on('end', () => { - const keys = Object.keys(valObj1) - assert.equal(keys.length, 0) - }) - }) -}) - -describe('db stream test', () => { - const trie = new Trie() - const ops = [ - { - type: 'put', - key: utf8ToBytes('color'), - value: utf8ToBytes('purple'), - }, - { - type: 'put', - key: utf8ToBytes('food'), - value: utf8ToBytes('sushi'), - }, - { - type: 'put', - key: utf8ToBytes('fight'), - value: utf8ToBytes('fire'), - }, - { - type: 'put', - key: utf8ToBytes('colo'), - value: utf8ToBytes('trolo'), - }, - { - type: 'put', - key: utf8ToBytes('color'), - value: utf8ToBytes('blue'), - }, - { - type: 'put', - key: utf8ToBytes('color'), - value: utf8ToBytes('pink'), - }, - ] as BatchDBOp[] - - it('should populate trie', async () => { - trie.checkpoint() - await trie.batch(ops) - }) -}) diff --git a/packages/trie/test/trie/trie.spec.ts b/packages/trie/test/trie/trie.spec.ts index 9f06d99b08..a3f35ca252 100644 --- a/packages/trie/test/trie/trie.spec.ts +++ b/packages/trie/test/trie/trie.spec.ts @@ -1,6 +1,7 @@ import { KECCAK256_RLP, MapDB, + bigIntToBytes, bytesToHex, concatBytes, equalsBytes, @@ -286,3 +287,19 @@ describe('keyHashingFunction', async () => { assert.equal(bytesToHex(trieWithCommonCopy.root()), '0x80', 'used hash function from common') }) }) + +describe('getValueMap', () => { + it('should return a map of all hashed keys and values', async () => { + const trie = await createTrie({}) + const entries: [Uint8Array, string][] = [ + [bigIntToBytes(1n), '0x' + '0a'.repeat(32)], + [bigIntToBytes(2n), '0x' + '0b'.repeat(32)], + [bigIntToBytes(3n), '0x' + '0c'.repeat(32)], + ] + for (const entry of entries) { + await trie.put(entry[0], utf8ToBytes(entry[1])) + } + const dump = await trie.getValueMap() + assert.equal(Object.entries(dump.values).length, 3) + }) +})