From fc8cc7dc12eb1e9d16607e83cae6e9b8960133dd Mon Sep 17 00:00:00 2001 From: Amir Date: Thu, 18 Jul 2024 11:25:45 -0700 Subject: [PATCH 01/16] Refactor dumpStorage to use walkAllValueNodes --- packages/statemanager/src/stateManager.ts | 30 +++++++++++++++-------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/statemanager/src/stateManager.ts b/packages/statemanager/src/stateManager.ts index 6ba58ebfd7..5e45e5bcd8 100644 --- a/packages/statemanager/src/stateManager.ts +++ b/packages/statemanager/src/stateManager.ts @@ -41,6 +41,10 @@ import type { } from '@ethereumjs/common' import type { DB, PrefixedHexString } from '@ethereumjs/util' import type { Debugger } from 'debug' +import { TrieNode } from '@ethereumjs/trie' +import { LeafNode } from '@ethereumjs/trie' +import { Nibbles } from '@ethereumjs/trie' +import { nibblesToBytes } from '@ethereumjs/trie' export type StorageProof = { key: PrefixedHexString @@ -984,6 +988,15 @@ export class DefaultStateManager implements EVMStateManagerInterface { * Both are represented as hex strings without the `0x` prefix. */ async dumpStorage(address: Address): Promise { + function nibblestoBytes(arr: Nibbles): Uint8Array { + const buf = new Uint8Array(arr.length / 2) + for (let i = 0; i < buf.length; i++) { + let q = i * 2 + buf[i] = (arr[q] << 4) + arr[++q] + } + return buf + } + await this.flush() const account = await this.getAccount(address) if (!account) { @@ -993,17 +1006,14 @@ export class DefaultStateManager implements EVMStateManagerInterface { return new Promise((resolve, reject) => { const storage: StorageDump = {} - const stream = trie.createReadStream() - stream.on('data', (val: any) => { - storage[bytesToHex(val.key)] = bytesToHex(val.value) - }) - stream.on('end', () => { - resolve(storage) - }) - stream.on('error', (e) => { - reject(e) - }) + return trie + .walkAllValueNodes(async (node: TrieNode, key: number[]) => { + if (node instanceof LeafNode) { + storage[bytesToHex(nibblestoBytes(node._nibbles))] = bytesToHex(node._value) + } + }) + .then((_) => resolve(storage)) }) } From 879e3fbb59e313629eb1712e677918d8f5e7a914 Mon Sep 17 00:00:00 2001 From: Amir Date: Thu, 18 Jul 2024 11:46:30 -0700 Subject: [PATCH 02/16] Refactor dumpStorageRange to use walkAllValueNodes --- packages/statemanager/src/stateManager.ts | 66 ++++++++++++++--------- 1 file changed, 40 insertions(+), 26 deletions(-) diff --git a/packages/statemanager/src/stateManager.ts b/packages/statemanager/src/stateManager.ts index 5e45e5bcd8..9af42b4fcc 100644 --- a/packages/statemanager/src/stateManager.ts +++ b/packages/statemanager/src/stateManager.ts @@ -1027,6 +1027,15 @@ export class DefaultStateManager implements EVMStateManagerInterface { The object will also contain `nextKey`, the next (hashed) storage key after the range included in `storage`. */ async dumpStorageRange(address: Address, startKey: bigint, limit: number): Promise { + function nibblestoBytes(arr: Nibbles): Uint8Array { + const buf = new Uint8Array(arr.length / 2) + for (let i = 0; i < buf.length; i++) { + let q = i * 2 + buf[i] = (arr[q] << 4) + arr[++q] + } + return buf + } + if (!Number.isSafeInteger(limit) || limit < 0) { throw new Error(`Limit is not a proper uint.`) } @@ -1036,6 +1045,7 @@ export class DefaultStateManager implements EVMStateManagerInterface { if (!account) { throw new Error(`Account does not exist.`) } + const trie = this._getStorageTrie(address, account) return new Promise((resolve, reject) => { @@ -1044,36 +1054,40 @@ export class DefaultStateManager implements EVMStateManagerInterface { /** Object conforming to {@link StorageRange.storage}. */ const storageMap: StorageRange['storage'] = {} - const stream = trie.createReadStream() - - stream.on('data', (val: any) => { - if (!inRange) { - // Check if the key is already in the correct range. - if (bytesToBigInt(val.key) >= startKey) { - inRange = true - } else { - return - } - } - if (i < limit) { - storageMap[bytesToHex(val.key)] = { key: null, value: bytesToHex(val.value) } - i++ - } else if (i === limit) { + return trie + .walkAllValueNodes(async (node: TrieNode, key: number[]) => { + if (node instanceof LeafNode) { + // storage[bytesToHex(nibblestoBytes(node._nibbles))] = bytesToHex(node._value) + + const keyBytes = nibblestoBytes(node._nibbles) + + if (!inRange) { + // Check if the key is already in the correct range. + if (bytesToBigInt(keyBytes) >= startKey) { + inRange = true + } else { + return + } + } + + if (i < limit) { + storageMap[bytesToHex(keyBytes)] = { key: null, value: bytesToHex(node._value) } + i++ + } else if (i === limit) { + resolve({ + storage: storageMap, + nextKey: bytesToHex(keyBytes), + }) + } + } + }) + .then((_) => resolve({ storage: storageMap, - nextKey: bytesToHex(val.key), + nextKey: null, }) - } - }) - - stream.on('end', () => { - resolve({ - storage: storageMap, - nextKey: null, - }) - }) - stream.on('error', (e) => reject(e)) + ) }) } From 29a3468468029fd76d608edf868b323b3ca6476e Mon Sep 17 00:00:00 2001 From: Amir Date: Thu, 18 Jul 2024 11:56:03 -0700 Subject: [PATCH 03/16] Fix linting issues --- packages/statemanager/src/stateManager.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/packages/statemanager/src/stateManager.ts b/packages/statemanager/src/stateManager.ts index 9af42b4fcc..41f5c73898 100644 --- a/packages/statemanager/src/stateManager.ts +++ b/packages/statemanager/src/stateManager.ts @@ -1,6 +1,6 @@ import { Chain, Common } from '@ethereumjs/common' import { RLP } from '@ethereumjs/rlp' -import { Trie, createTrieFromProof, verifyTrieProof } from '@ethereumjs/trie' +import { LeafNode, Trie, createTrieFromProof, verifyTrieProof } from '@ethereumjs/trie' import { Account, Address, @@ -39,12 +39,9 @@ import type { StorageDump, StorageRange, } from '@ethereumjs/common' +import type { Nibbles, TrieNode } from '@ethereumjs/trie' import type { DB, PrefixedHexString } from '@ethereumjs/util' import type { Debugger } from 'debug' -import { TrieNode } from '@ethereumjs/trie' -import { LeafNode } from '@ethereumjs/trie' -import { Nibbles } from '@ethereumjs/trie' -import { nibblesToBytes } from '@ethereumjs/trie' export type StorageProof = { key: PrefixedHexString @@ -1004,11 +1001,11 @@ export class DefaultStateManager implements EVMStateManagerInterface { } const trie = this._getStorageTrie(address, account) - return new Promise((resolve, reject) => { + return new Promise((resolve, _) => { const storage: StorageDump = {} return trie - .walkAllValueNodes(async (node: TrieNode, key: number[]) => { + .walkAllValueNodes(async (node: TrieNode, _) => { if (node instanceof LeafNode) { storage[bytesToHex(nibblestoBytes(node._nibbles))] = bytesToHex(node._value) } @@ -1048,7 +1045,7 @@ export class DefaultStateManager implements EVMStateManagerInterface { const trie = this._getStorageTrie(address, account) - return new Promise((resolve, reject) => { + return new Promise((resolve, _) => { let inRange = false let i = 0 @@ -1056,7 +1053,7 @@ export class DefaultStateManager implements EVMStateManagerInterface { const storageMap: StorageRange['storage'] = {} return trie - .walkAllValueNodes(async (node: TrieNode, key: number[]) => { + .walkAllValueNodes(async (node: TrieNode, _) => { if (node instanceof LeafNode) { // storage[bytesToHex(nibblestoBytes(node._nibbles))] = bytesToHex(node._value) From b0f6f5ccd0c1cbaf09ee778f06a1f001eaba817b Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Thu, 18 Jul 2024 19:43:56 -0400 Subject: [PATCH 04/16] append nibbles to current key --- packages/statemanager/src/stateManager.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/statemanager/src/stateManager.ts b/packages/statemanager/src/stateManager.ts index 41f5c73898..44b599392d 100644 --- a/packages/statemanager/src/stateManager.ts +++ b/packages/statemanager/src/stateManager.ts @@ -1053,12 +1053,11 @@ export class DefaultStateManager implements EVMStateManagerInterface { const storageMap: StorageRange['storage'] = {} return trie - .walkAllValueNodes(async (node: TrieNode, _) => { + .walkAllValueNodes(async (node: TrieNode, currentKey) => { if (node instanceof LeafNode) { // storage[bytesToHex(nibblestoBytes(node._nibbles))] = bytesToHex(node._value) - const keyBytes = nibblestoBytes(node._nibbles) - + const keyBytes = nibblestoBytes(currentKey.concat(node.key())) if (!inRange) { // Check if the key is already in the correct range. if (bytesToBigInt(keyBytes) >= startKey) { From 862c3f751127b14d4b07003cb259a00a2d9a9f11 Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Thu, 18 Jul 2024 19:55:12 -0400 Subject: [PATCH 05/16] remove readable-stream --- package-lock.json | 3 +- packages/evm/vite.config.bundler.ts | 2 +- packages/trie/package.json | 3 +- packages/trie/src/trie.ts | 9 ---- packages/trie/src/util/index.ts | 1 - packages/trie/src/util/readStream.ts | 66 ---------------------------- 6 files changed, 3 insertions(+), 81 deletions(-) delete mode 100644 packages/trie/src/util/readStream.ts diff --git a/package-lock.json b/package-lock.json index dc5d3c1479..a306dfb925 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16938,8 +16938,7 @@ "@types/readable-stream": "^2.3.13", "debug": "^4.3.4", "ethereum-cryptography": "^2.2.1", - "lru-cache": "10.1.0", - "readable-stream": "^3.6.0" + "lru-cache": "10.1.0" }, "devDependencies": { "@ethereumjs/genesis": "^0.2.2", diff --git a/packages/evm/vite.config.bundler.ts b/packages/evm/vite.config.bundler.ts index 198b7cc30a..c135e233f4 100644 --- a/packages/evm/vite.config.bundler.ts +++ b/packages/evm/vite.config.bundler.ts @@ -11,7 +11,7 @@ export default defineConfig({ treeshake: 'safest', }, lib: { - entry: 'src/index.ts', + entry: '../trie/examples/trieWalking.ts', name: '@ethereumjs/evm', fileName: (format) => `ethereumjs-evm-bundle.${format}.js`, // only build for es diff --git a/packages/trie/package.json b/packages/trie/package.json index 17251fd4de..3eed919db0 100644 --- a/packages/trie/package.json +++ b/packages/trie/package.json @@ -59,8 +59,7 @@ "@types/readable-stream": "^2.3.13", "debug": "^4.3.4", "lru-cache": "10.1.0", - "ethereum-cryptography": "^2.2.1", - "readable-stream": "^3.6.0" + "ethereum-cryptography": "^2.2.1" }, "devDependencies": { "@ethereumjs/genesis": "^0.2.2", diff --git a/packages/trie/src/trie.ts b/packages/trie/src/trie.ts index 4cb59f7afe..62c62bb881 100644 --- a/packages/trie/src/trie.ts +++ b/packages/trie/src/trie.ts @@ -30,7 +30,6 @@ import { verifyRangeProof } from './proof/range.js' import { ROOT_DB_KEY } from './types.js' import { _walkTrie } from './util/asyncWalk.js' import { bytesToNibbles, matchingNibbleLength } from './util/nibbles.js' -import { TrieReadStream as ReadStream } from './util/readStream.js' import { WalkController } from './util/walkController.js' import type { @@ -1081,14 +1080,6 @@ export class Trie { return true } - /** - * The `data` event is given an `Object` that has two properties; the `key` and the `value`. Both should be Uint8Arrays. - * @return Returns a [stream](https://nodejs.org/dist/latest-v12.x/docs/api/stream.html#stream_class_stream_readable) of the contents of the `trie` - */ - createReadStream(): ReadStream { - return new ReadStream(this) - } - /** * Returns a copy of the underlying trie. * diff --git a/packages/trie/src/util/index.ts b/packages/trie/src/util/index.ts index 1297a1dbca..d86ae67d7d 100644 --- a/packages/trie/src/util/index.ts +++ b/packages/trie/src/util/index.ts @@ -1,5 +1,4 @@ export * from './encoding.js' export * from './genesisState.js' -export * from './readStream.js' export * from './tasks.js' export * from './walkController.js' diff --git a/packages/trie/src/util/readStream.ts b/packages/trie/src/util/readStream.ts deleted file mode 100644 index dfc002d29c..0000000000 --- a/packages/trie/src/util/readStream.ts +++ /dev/null @@ -1,66 +0,0 @@ -// eslint-disable-next-line implicit-dependencies/no-implicit -import { Readable } from 'readable-stream' - -import { BranchNode, LeafNode } from '../node/index.js' - -import { nibblestoBytes } from './nibbles.js' - -import type { Trie } from '../trie.js' -import type { FoundNodeFunction } from '../types.js' - -const _findValueNodes = async (trie: Trie, onFound: FoundNodeFunction): Promise => { - const outerOnFound: FoundNodeFunction = async (nodeRef, node, key, walkController) => { - let fullKey = key - if (node instanceof LeafNode) { - fullKey = key.concat(node.key()) - // found leaf node! - onFound(nodeRef, node, fullKey, walkController) - } else if (node instanceof BranchNode && node.value()) { - // found branch with value - onFound(nodeRef, node, fullKey, walkController) - } else { - // keep looking for value nodes - if (node !== null) { - walkController.allChildren(node, key) - } - } - } - await trie.walkTrie(trie.root(), outerOnFound) -} - -export class TrieReadStream extends Readable { - private trie: Trie - private _started: boolean - - constructor(trie: Trie) { - super({ objectMode: true }) - - this.trie = trie - this._started = false - } - - async _read() { - if (this._started) { - return - } - this._started = true - try { - await _findValueNodes(this.trie, async (_, node, key, walkController) => { - if (node !== null) { - this.push({ - key: nibblestoBytes(key), - value: node.value(), - }) - walkController.allChildren(node, key) - } - }) - } catch (error: any) { - if (error.message === 'Missing node in DB') { - // pass - } else { - throw error - } - } - this.push(null) - } -} From 682ddd00b8bbf5ceec5e5fd5216131ce692603c9 Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Thu, 18 Jul 2024 20:06:24 -0400 Subject: [PATCH 06/16] Revert bundler config change --- packages/evm/vite.config.bundler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/evm/vite.config.bundler.ts b/packages/evm/vite.config.bundler.ts index c135e233f4..198b7cc30a 100644 --- a/packages/evm/vite.config.bundler.ts +++ b/packages/evm/vite.config.bundler.ts @@ -11,7 +11,7 @@ export default defineConfig({ treeshake: 'safest', }, lib: { - entry: '../trie/examples/trieWalking.ts', + entry: 'src/index.ts', name: '@ethereumjs/evm', fileName: (format) => `ethereumjs-evm-bundle.${format}.js`, // only build for es From c7a5e09dac9cbfee543446eef3ed84a5dd9c421a Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Thu, 18 Jul 2024 20:08:03 -0400 Subject: [PATCH 07/16] type parameter --- packages/statemanager/src/stateManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/statemanager/src/stateManager.ts b/packages/statemanager/src/stateManager.ts index 44b599392d..0aaee94f54 100644 --- a/packages/statemanager/src/stateManager.ts +++ b/packages/statemanager/src/stateManager.ts @@ -1053,7 +1053,7 @@ export class DefaultStateManager implements EVMStateManagerInterface { const storageMap: StorageRange['storage'] = {} return trie - .walkAllValueNodes(async (node: TrieNode, currentKey) => { + .walkAllValueNodes(async (node: TrieNode, currentKey: Number[]) => { if (node instanceof LeafNode) { // storage[bytesToHex(nibblestoBytes(node._nibbles))] = bytesToHex(node._value) From fad55923db4db5f5a79cea8618eeacafaad69426 Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Thu, 18 Jul 2024 20:41:53 -0400 Subject: [PATCH 08/16] Remove duplicate code --- packages/statemanager/src/stateManager.ts | 32 +++++++---------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/packages/statemanager/src/stateManager.ts b/packages/statemanager/src/stateManager.ts index 0aaee94f54..2efd29def7 100644 --- a/packages/statemanager/src/stateManager.ts +++ b/packages/statemanager/src/stateManager.ts @@ -1,6 +1,12 @@ import { Chain, Common } from '@ethereumjs/common' import { RLP } from '@ethereumjs/rlp' -import { LeafNode, Trie, createTrieFromProof, verifyTrieProof } from '@ethereumjs/trie' +import { + LeafNode, + Trie, + createTrieFromProof, + nibbleTypeToPackedBytes, + verifyTrieProof, +} from '@ethereumjs/trie' import { Account, Address, @@ -985,15 +991,6 @@ export class DefaultStateManager implements EVMStateManagerInterface { * Both are represented as hex strings without the `0x` prefix. */ async dumpStorage(address: Address): Promise { - function nibblestoBytes(arr: Nibbles): Uint8Array { - const buf = new Uint8Array(arr.length / 2) - for (let i = 0; i < buf.length; i++) { - let q = i * 2 - buf[i] = (arr[q] << 4) + arr[++q] - } - return buf - } - await this.flush() const account = await this.getAccount(address) if (!account) { @@ -1007,7 +1004,7 @@ export class DefaultStateManager implements EVMStateManagerInterface { return trie .walkAllValueNodes(async (node: TrieNode, _) => { if (node instanceof LeafNode) { - storage[bytesToHex(nibblestoBytes(node._nibbles))] = bytesToHex(node._value) + storage[bytesToHex(nibbleTypeToPackedBytes(node._nibbles))] = bytesToHex(node._value) } }) .then((_) => resolve(storage)) @@ -1024,15 +1021,6 @@ export class DefaultStateManager implements EVMStateManagerInterface { The object will also contain `nextKey`, the next (hashed) storage key after the range included in `storage`. */ async dumpStorageRange(address: Address, startKey: bigint, limit: number): Promise { - function nibblestoBytes(arr: Nibbles): Uint8Array { - const buf = new Uint8Array(arr.length / 2) - for (let i = 0; i < buf.length; i++) { - let q = i * 2 - buf[i] = (arr[q] << 4) + arr[++q] - } - return buf - } - if (!Number.isSafeInteger(limit) || limit < 0) { throw new Error(`Limit is not a proper uint.`) } @@ -1053,11 +1041,11 @@ export class DefaultStateManager implements EVMStateManagerInterface { const storageMap: StorageRange['storage'] = {} return trie - .walkAllValueNodes(async (node: TrieNode, currentKey: Number[]) => { + .walkAllValueNodes(async (node: TrieNode, currentKey: number[]) => { if (node instanceof LeafNode) { // storage[bytesToHex(nibblestoBytes(node._nibbles))] = bytesToHex(node._value) - const keyBytes = nibblestoBytes(currentKey.concat(node.key())) + const keyBytes = nibbleTypeToPackedBytes(currentKey.concat(node.key())) if (!inRange) { // Check if the key is already in the correct range. if (bytesToBigInt(keyBytes) >= startKey) { From 09289d8c3bf1813540329614aac8c0c3cbe447c5 Mon Sep 17 00:00:00 2001 From: Holger Drewes Date: Fri, 19 Jul 2024 09:48:06 +0200 Subject: [PATCH 09/16] Micro change to re-trigger CI --- packages/statemanager/src/stateManager.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/statemanager/src/stateManager.ts b/packages/statemanager/src/stateManager.ts index 2efd29def7..282bc55296 100644 --- a/packages/statemanager/src/stateManager.ts +++ b/packages/statemanager/src/stateManager.ts @@ -236,7 +236,6 @@ export class DefaultStateManager implements EVMStateManagerInterface { type: opts.accountCacheOpts?.type ?? CacheType.ORDERED_MAP, size: opts.accountCacheOpts?.size ?? 100000, } - if (!this._accountCacheSettings.deactivate) { this._accountCache = new AccountCache({ size: this._accountCacheSettings.size, @@ -250,7 +249,6 @@ export class DefaultStateManager implements EVMStateManagerInterface { type: opts.storageCacheOpts?.type ?? CacheType.ORDERED_MAP, size: opts.storageCacheOpts?.size ?? 20000, } - if (!this._storageCacheSettings.deactivate) { this._storageCache = new StorageCache({ size: this._storageCacheSettings.size, @@ -264,7 +262,6 @@ export class DefaultStateManager implements EVMStateManagerInterface { type: opts.codeCacheOpts?.type ?? CacheType.ORDERED_MAP, size: opts.codeCacheOpts?.size ?? 20000, } - if (!this._codeCacheSettings.deactivate) { this._codeCache = new CodeCache({ size: this._codeCacheSettings.size, From b4ba2c0172f7e4a8fb8462f6a103b1622bcdaa37 Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Fri, 19 Jul 2024 07:58:46 -0400 Subject: [PATCH 10/16] lint and obsolete test cleanup --- packages/statemanager/src/stateManager.ts | 2 +- packages/trie/test/stream.spec.ts | 164 ---------------------- 2 files changed, 1 insertion(+), 165 deletions(-) delete mode 100644 packages/trie/test/stream.spec.ts diff --git a/packages/statemanager/src/stateManager.ts b/packages/statemanager/src/stateManager.ts index 282bc55296..2627ee602a 100644 --- a/packages/statemanager/src/stateManager.ts +++ b/packages/statemanager/src/stateManager.ts @@ -45,7 +45,7 @@ import type { StorageDump, StorageRange, } from '@ethereumjs/common' -import type { Nibbles, TrieNode } from '@ethereumjs/trie' +import type { TrieNode } from '@ethereumjs/trie' import type { DB, PrefixedHexString } from '@ethereumjs/util' import type { Debugger } from 'debug' diff --git a/packages/trie/test/stream.spec.ts b/packages/trie/test/stream.spec.ts deleted file mode 100644 index 5c4d05ff43..0000000000 --- a/packages/trie/test/stream.spec.ts +++ /dev/null @@ -1,164 +0,0 @@ -import { utf8ToBytes } from '@ethereumjs/util' -import { assert, describe, it } from 'vitest' - -import { Trie } from '../src/index.js' - -import type { BatchDBOp } from '@ethereumjs/util' - -describe('kv stream test', () => { - const trie = new Trie() - const ops = [ - { - type: 'del', - key: utf8ToBytes('father'), - }, - { - type: 'put', - key: utf8ToBytes('name'), - value: utf8ToBytes('Yuri Irsenovich Kim'), - }, - { - type: 'put', - key: utf8ToBytes('dob'), - value: utf8ToBytes('16 February 1941'), - }, - { - type: 'put', - key: utf8ToBytes('spouse'), - value: utf8ToBytes('Kim Young-sook'), - }, - { - type: 'put', - key: utf8ToBytes('occupation'), - value: utf8ToBytes('Clown'), - }, - { - type: 'put', - key: utf8ToBytes('nameads'), - value: utf8ToBytes('Yuri Irsenovich Kim'), - }, - { - type: 'put', - key: utf8ToBytes('namfde'), - value: utf8ToBytes('Yuri Irsenovich Kim'), - }, - { - type: 'put', - key: utf8ToBytes('namsse'), - value: utf8ToBytes('Yuri Irsenovich Kim'), - }, - { - type: 'put', - key: utf8ToBytes('dofab'), - value: utf8ToBytes('16 February 1941'), - }, - { - type: 'put', - key: utf8ToBytes('spoudse'), - value: utf8ToBytes('Kim Young-sook'), - }, - { - type: 'put', - key: utf8ToBytes('occupdsation'), - value: utf8ToBytes('Clown'), - }, - { - type: 'put', - key: utf8ToBytes('dozzzb'), - value: utf8ToBytes('16 February 1941'), - }, - { - type: 'put', - key: utf8ToBytes('spouszze'), - value: utf8ToBytes('Kim Young-sook'), - }, - { - type: 'put', - key: utf8ToBytes('occupatdfion'), - value: utf8ToBytes('Clown'), - }, - { - type: 'put', - key: utf8ToBytes('dssob'), - value: utf8ToBytes('16 February 1941'), - }, - { - type: 'put', - key: utf8ToBytes('spossuse'), - value: utf8ToBytes('Kim Young-sook'), - }, - { - type: 'put', - key: utf8ToBytes('occupssation'), - value: utf8ToBytes('Clown'), - }, - ] as BatchDBOp[] - - const valObj1 = {} as any - const valObj2 = {} as any - for (const op of ops) { - if (op.type === 'put') { - valObj1[op.key.toString()] = op.value.toString() - valObj2[op.key.toString()] = op.value.toString() - } - } - - it('should populate trie', async () => { - await trie.batch(ops) - }) - - it('should fetch all of the nodes', () => { - const stream = trie.createReadStream() - stream.on('data', (d: any) => { - const key = d.key.toString() - const value = d.value.toString() - assert.equal(valObj1[key], value) - delete valObj1[key] - }) - stream.on('end', () => { - const keys = Object.keys(valObj1) - assert.equal(keys.length, 0) - }) - }) -}) - -describe('db stream test', () => { - const trie = new Trie() - const ops = [ - { - type: 'put', - key: utf8ToBytes('color'), - value: utf8ToBytes('purple'), - }, - { - type: 'put', - key: utf8ToBytes('food'), - value: utf8ToBytes('sushi'), - }, - { - type: 'put', - key: utf8ToBytes('fight'), - value: utf8ToBytes('fire'), - }, - { - type: 'put', - key: utf8ToBytes('colo'), - value: utf8ToBytes('trolo'), - }, - { - type: 'put', - key: utf8ToBytes('color'), - value: utf8ToBytes('blue'), - }, - { - type: 'put', - key: utf8ToBytes('color'), - value: utf8ToBytes('pink'), - }, - ] as BatchDBOp[] - - it('should populate trie', async () => { - trie.checkpoint() - await trie.batch(ops) - }) -}) From aead8fdf1d148afdf50d93fd85c03f19b4da108e Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Tue, 23 Jul 2024 11:37:23 -0400 Subject: [PATCH 11/16] add getValueMap to api [no ci] --- packages/trie/src/trie.ts | 44 ++++++++++++++++++++++++++++ packages/trie/test/trie/trie.spec.ts | 18 ++++++++++++ 2 files changed, 62 insertions(+) diff --git a/packages/trie/src/trie.ts b/packages/trie/src/trie.ts index 62c62bb881..703c099519 100644 --- a/packages/trie/src/trie.ts +++ b/packages/trie/src/trie.ts @@ -3,11 +3,13 @@ import { RLP } from '@ethereumjs/rlp' import { + BIGINT_0, KeyEncoding, Lock, MapDB, RLP_EMPTY_STRING, ValueEncoding, + bytesToBigInt, bytesToHex, bytesToUnprefixedHex, bytesToUtf8, @@ -29,6 +31,7 @@ import { import { verifyRangeProof } from './proof/range.js' import { ROOT_DB_KEY } from './types.js' import { _walkTrie } from './util/asyncWalk.js' +import { nibbleTypeToPackedBytes } from './util/encoding.js' import { bytesToNibbles, matchingNibbleLength } from './util/nibbles.js' import { WalkController } from './util/walkController.js' @@ -1218,4 +1221,45 @@ export class Trie { this.debug(`Deleting ${this._db.checkpoints.length} checkpoints.`, ['FLUSH_CHECKPOINTS']) this._db.checkpoints = [] } + + /** + * Returns a list of values stored in the trie + * @param startKey first unhashed key in the range to be returned (defaults to 0) + * @param limit - the number of keys to be returned (undefined means all keys) + * @returns an object with two properties (a map of all key/value pairs in the trie - or in the specified range) and then a `nextKey` reference if a range is specified + */ + async getValueMap( + startKey = BIGINT_0, + limit?: number + ): Promise<{ values: { [key: string]: string }; nextKey: null | string }> { + // If limit is undefined, all keys are inRange + let inRange = limit !== undefined ? false : true + let i = 0 + const values: { [key: string]: string } = {} + let nextKey: string | null = null + await this.walkAllValueNodes(async (node: TrieNode, currentKey: number[]) => { + if (node instanceof LeafNode) { + const keyBytes = nibbleTypeToPackedBytes(currentKey.concat(node.key())) + if (!inRange) { + // Check if the key is already in the correct range. + if (bytesToBigInt(keyBytes) >= startKey) { + inRange = true + } else { + return + } + } + + if (limit === undefined || i < limit) { + values[bytesToHex(keyBytes)] = bytesToHex(node._value) + i++ + } else if (i === limit) { + nextKey = bytesToHex(keyBytes) + } + } + }) + return { + values, + nextKey, + } + } } diff --git a/packages/trie/test/trie/trie.spec.ts b/packages/trie/test/trie/trie.spec.ts index 9f06d99b08..b3b35efccd 100644 --- a/packages/trie/test/trie/trie.spec.ts +++ b/packages/trie/test/trie/trie.spec.ts @@ -1,6 +1,7 @@ import { KECCAK256_RLP, MapDB, + bigIntToBytes, bytesToHex, concatBytes, equalsBytes, @@ -286,3 +287,20 @@ describe('keyHashingFunction', async () => { assert.equal(bytesToHex(trieWithCommonCopy.root()), '0x80', 'used hash function from common') }) }) + +describe('getValueMap', () => { + it.only('should return a map of all hashed keys and values', async () => { + const trie = await createTrie({}) + const entries: [Uint8Array, string][] = [ + [bigIntToBytes(1n), 'bingo'], + [bigIntToBytes(2n), 'dingo'], + [bigIntToBytes(3n), 'mingo'], + ] + for (const entry of entries) { + await trie.put(entry[0], utf8ToBytes(entry[1])) + } + + const dump = await trie.getValueMap() + assert.equal(Object.entries(dump.values).length, 3) + }) +}) From 987b64615026f6c7a57ccd35a524d95d3dabfd15 Mon Sep 17 00:00:00 2001 From: Amir Date: Tue, 23 Jul 2024 12:11:09 -0700 Subject: [PATCH 12/16] Fix test --- packages/trie/test/trie/trie.spec.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/trie/test/trie/trie.spec.ts b/packages/trie/test/trie/trie.spec.ts index b3b35efccd..510ac8b0b0 100644 --- a/packages/trie/test/trie/trie.spec.ts +++ b/packages/trie/test/trie/trie.spec.ts @@ -5,6 +5,7 @@ import { bytesToHex, concatBytes, equalsBytes, + hexToBytes, randomBytes, unprefixedHexToBytes, utf8ToBytes, @@ -292,14 +293,13 @@ describe('getValueMap', () => { it.only('should return a map of all hashed keys and values', async () => { const trie = await createTrie({}) const entries: [Uint8Array, string][] = [ - [bigIntToBytes(1n), 'bingo'], - [bigIntToBytes(2n), 'dingo'], - [bigIntToBytes(3n), 'mingo'], + [bigIntToBytes(1n), '0x' + '0a'.repeat(32)], + [bigIntToBytes(2n), '0x' + '0b'.repeat(32)], + [bigIntToBytes(3n), '0x' + '0c'.repeat(32)], ] for (const entry of entries) { await trie.put(entry[0], utf8ToBytes(entry[1])) } - const dump = await trie.getValueMap() assert.equal(Object.entries(dump.values).length, 3) }) From 6a2c60078ac0a2aeefea704917f0ebf0b9419ec2 Mon Sep 17 00:00:00 2001 From: Amir Date: Tue, 23 Jul 2024 12:15:32 -0700 Subject: [PATCH 13/16] Fix lint issue --- packages/trie/test/trie/trie.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/trie/test/trie/trie.spec.ts b/packages/trie/test/trie/trie.spec.ts index 510ac8b0b0..ad613ce9fe 100644 --- a/packages/trie/test/trie/trie.spec.ts +++ b/packages/trie/test/trie/trie.spec.ts @@ -5,7 +5,6 @@ import { bytesToHex, concatBytes, equalsBytes, - hexToBytes, randomBytes, unprefixedHexToBytes, utf8ToBytes, From 92dc4bd727f10974a370d21f30ce9bacb6b205a8 Mon Sep 17 00:00:00 2001 From: Amir Date: Tue, 23 Jul 2024 14:49:46 -0700 Subject: [PATCH 14/16] Replace walkAllNodes usage with getValueMap in statemanager --- packages/statemanager/src/stateManager.ts | 66 ++++++----------------- 1 file changed, 17 insertions(+), 49 deletions(-) diff --git a/packages/statemanager/src/stateManager.ts b/packages/statemanager/src/stateManager.ts index c9d754e53f..93e411ca59 100644 --- a/packages/statemanager/src/stateManager.ts +++ b/packages/statemanager/src/stateManager.ts @@ -995,16 +995,8 @@ export class DefaultStateManager implements EVMStateManagerInterface { } const trie = this._getStorageTrie(address, account) - return new Promise((resolve, _) => { - const storage: StorageDump = {} - - return trie - .walkAllValueNodes(async (node: TrieNode, _) => { - if (node instanceof LeafNode) { - storage[bytesToHex(nibbleTypeToPackedBytes(node._nibbles))] = bytesToHex(node._value) - } - }) - .then((_) => resolve(storage)) + return trie.getValueMap().then((value) => { + return value.values }) } @@ -1030,45 +1022,21 @@ export class DefaultStateManager implements EVMStateManagerInterface { const trie = this._getStorageTrie(address, account) - return new Promise((resolve, _) => { - let inRange = false - let i = 0 - - /** Object conforming to {@link StorageRange.storage}. */ - const storageMap: StorageRange['storage'] = {} - - return trie - .walkAllValueNodes(async (node: TrieNode, currentKey: number[]) => { - if (node instanceof LeafNode) { - // storage[bytesToHex(nibblestoBytes(node._nibbles))] = bytesToHex(node._value) - - const keyBytes = nibbleTypeToPackedBytes(currentKey.concat(node.key())) - if (!inRange) { - // Check if the key is already in the correct range. - if (bytesToBigInt(keyBytes) >= startKey) { - inRange = true - } else { - return - } - } - - if (i < limit) { - storageMap[bytesToHex(keyBytes)] = { key: null, value: bytesToHex(node._value) } - i++ - } else if (i === limit) { - resolve({ - storage: storageMap, - nextKey: bytesToHex(keyBytes), - }) - } - } - }) - .then((_) => - resolve({ - storage: storageMap, - nextKey: null, - }) - ) + return trie.getValueMap(startKey, limit).then((value) => { + const values = value.values + const dump = Object.create(null) + for (const key of Object.keys(values)) { + const val = values[key] + dump[key] = { + key: null, + value: val, + } + } + + return { + storage: dump, + nextKey: value.nextKey, + } }) } From 68ebd675a93fe895656acc3904f69d50821c8763 Mon Sep 17 00:00:00 2001 From: Amir Date: Tue, 23 Jul 2024 14:55:06 -0700 Subject: [PATCH 15/16] Update test --- packages/trie/test/trie/trie.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/trie/test/trie/trie.spec.ts b/packages/trie/test/trie/trie.spec.ts index ad613ce9fe..a3f35ca252 100644 --- a/packages/trie/test/trie/trie.spec.ts +++ b/packages/trie/test/trie/trie.spec.ts @@ -289,7 +289,7 @@ describe('keyHashingFunction', async () => { }) describe('getValueMap', () => { - it.only('should return a map of all hashed keys and values', async () => { + it('should return a map of all hashed keys and values', async () => { const trie = await createTrie({}) const entries: [Uint8Array, string][] = [ [bigIntToBytes(1n), '0x' + '0a'.repeat(32)], From a12aabb81a422a36928672e3642fb02d6a8098e5 Mon Sep 17 00:00:00 2001 From: Amir Date: Tue, 23 Jul 2024 15:05:13 -0700 Subject: [PATCH 16/16] Fix lint issues --- packages/statemanager/src/stateManager.ts | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/packages/statemanager/src/stateManager.ts b/packages/statemanager/src/stateManager.ts index 93e411ca59..e4261d9163 100644 --- a/packages/statemanager/src/stateManager.ts +++ b/packages/statemanager/src/stateManager.ts @@ -1,12 +1,6 @@ import { Chain, Common } from '@ethereumjs/common' import { RLP } from '@ethereumjs/rlp' -import { - LeafNode, - Trie, - createTrieFromProof, - nibbleTypeToPackedBytes, - verifyTrieProof, -} from '@ethereumjs/trie' +import { Trie, createTrieFromProof, verifyTrieProof } from '@ethereumjs/trie' import { Account, Address, @@ -15,7 +9,6 @@ import { KECCAK256_RLP, KECCAK256_RLP_S, bigIntToHex, - bytesToBigInt, bytesToHex, bytesToUnprefixedHex, concatBytes, @@ -47,7 +40,6 @@ import type { StorageDump, StorageRange, } from '@ethereumjs/common' -import type { TrieNode } from '@ethereumjs/trie' import type { DB, PrefixedHexString } from '@ethereumjs/util' import type { Debugger } from 'debug'