From f7d16f644e6c34e85987dd1befa62aef3c6b9a0a Mon Sep 17 00:00:00 2001 From: garyschulte Date: Fri, 7 Jul 2023 15:27:56 -0700 Subject: [PATCH 01/13] create a bonsai based reference test worldstate Signed-off-by: garyschulte --- .../bonsai/worldview/BonsaiWorldState.java | 56 +++---- ethereum/referencetests/build.gradle | 1 + .../BonsaiReferenceTestWorldState.java | 138 ++++++++++++++++++ .../DefaultReferenceTestWorldState.java | 50 +++++++ .../ReferenceTestWorldState.java | 28 +--- 5 files changed, 215 insertions(+), 58 deletions(-) create mode 100644 ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java create mode 100644 ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/DefaultReferenceTestWorldState.java diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java index 865471eca99..4de046313ad 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java @@ -27,6 +27,7 @@ import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount; import org.hyperledger.besu.ethereum.bonsai.BonsaiValue; import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; @@ -67,7 +68,8 @@ public class BonsaiWorldState private BonsaiWorldStateKeyValueStorage worldStateStorage; - private final BonsaiWorldStateProvider archive; + private final CachedMerkleTrieLoader cachedMerkleTrieLoader; + private final TrieLogManager trieLogManager; private final BonsaiWorldStateUpdateAccumulator accumulator; private Hash worldStateRootHash; @@ -78,38 +80,29 @@ public class BonsaiWorldState public BonsaiWorldState( final BonsaiWorldStateProvider archive, final BonsaiWorldStateKeyValueStorage worldStateStorage) { - this.archive = archive; - this.worldStateStorage = worldStateStorage; - worldStateRootHash = - Hash.wrap( - Bytes32.wrap(worldStateStorage.getWorldStateRootHash().orElse(Hash.EMPTY_TRIE_HASH))); - worldStateBlockHash = - Hash.wrap(Bytes32.wrap(worldStateStorage.getWorldStateBlockHash().orElse(Hash.ZERO))); - accumulator = - new BonsaiWorldStateUpdateAccumulator( - this, - (addr, value) -> - archive - .getCachedMerkleTrieLoader() - .preLoadAccount(getWorldStateStorage(), worldStateRootHash, addr), - (addr, value) -> - archive - .getCachedMerkleTrieLoader() - .preLoadStorageSlot(getWorldStateStorage(), addr, value)); + this(worldStateStorage, archive.getCachedMerkleTrieLoader(), archive.getTrieLogManager()); } - public BonsaiWorldState( - final BonsaiWorldStateProvider archive, + protected BonsaiWorldState( final BonsaiWorldStateKeyValueStorage worldStateStorage, - final BonsaiWorldStateUpdateAccumulator updater) { - this.archive = archive; + final CachedMerkleTrieLoader cachedMerkleTrieLoader, + final TrieLogManager trieLogManager) { this.worldStateStorage = worldStateStorage; this.worldStateRootHash = Hash.wrap( Bytes32.wrap(worldStateStorage.getWorldStateRootHash().orElse(Hash.EMPTY_TRIE_HASH))); this.worldStateBlockHash = Hash.wrap(Bytes32.wrap(worldStateStorage.getWorldStateBlockHash().orElse(Hash.ZERO))); - this.accumulator = updater; + accumulator = + new BonsaiWorldStateUpdateAccumulator( + this, + (addr, value) -> + cachedMerkleTrieLoader.preLoadAccount( + getWorldStateStorage(), worldStateRootHash, addr), + (addr, value) -> + cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value)); + this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; + this.trieLogManager = trieLogManager; } /** @@ -130,10 +123,6 @@ public Hash getWorldStateRootHash() { return worldStateRootHash; } - public BonsaiWorldStateProvider getArchive() { - return archive; - } - @Override public boolean isPersisted() { return isPersisted(worldStateStorage); @@ -189,9 +178,7 @@ private Hash calculateRootHash( final StoredMerklePatriciaTrie accountTrie = createTrie( (location, hash) -> - archive - .getCachedMerkleTrieLoader() - .getAccountStateTrieNode(worldStateStorage, location, hash), + cachedMerkleTrieLoader.getAccountStateTrieNode(worldStateStorage, location, hash), worldStateRootHash); // for manicured tries and composting, collect branches here (not implemented) @@ -277,10 +264,8 @@ private void updateAccountStorageState( final StoredMerklePatriciaTrie storageTrie = createTrie( (location, key) -> - archive - .getCachedMerkleTrieLoader() - .getAccountStorageTrieNode( - worldStateStorage, updatedAddressHash, location, key), + cachedMerkleTrieLoader.getAccountStorageTrieNode( + worldStateStorage, updatedAddressHash, location, key), storageRoot); // for manicured tries and composting, collect branches here (not implemented) @@ -405,7 +390,6 @@ public void persist(final BlockHeader blockHeader) { } saveTrieLog = () -> { - final TrieLogManager trieLogManager = archive.getTrieLogManager(); trieLogManager.saveTrieLog(localCopy, newWorldStateRootHash, blockHeader, this); // not save a frozen state in the cache if (!isFrozen) { diff --git a/ethereum/referencetests/build.gradle b/ethereum/referencetests/build.gradle index a0b7db03d48..5692c415f02 100644 --- a/ethereum/referencetests/build.gradle +++ b/ethereum/referencetests/build.gradle @@ -147,6 +147,7 @@ dependencies { implementation project(':crypto:algorithms') implementation project(':datatypes') implementation project(':ethereum:core') + implementation project(':metrics:core') implementation project(path: ':ethereum:core', configuration: 'testSupportArtifacts') implementation project(':ethereum:rlp') implementation project(':evm') diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java new file mode 100644 index 00000000000..e76e7d9749b --- /dev/null +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -0,0 +1,138 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.referencetests; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogAddedEvent; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; +import org.hyperledger.besu.evm.worldstate.WorldUpdater; +import org.hyperledger.besu.metrics.ObservableMetricsSystem; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.services.trielogs.TrieLog; +import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent; +import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory; +import org.hyperledger.besu.util.Subscribers; + +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +import com.fasterxml.jackson.annotation.JsonCreator; + +public class BonsaiReferenceTestWorldState extends BonsaiWorldState + implements ReferenceTestWorldState { + + protected BonsaiReferenceTestWorldState( + final BonsaiWorldStateKeyValueStorage worldStateStorage, + final CachedMerkleTrieLoader cachedMerkleTrieLoader, + final TrieLogManager trieLogManager) { + super(worldStateStorage, cachedMerkleTrieLoader, trieLogManager); + } + + @JsonCreator + public static BonsaiReferenceTestWorldState create( + final Map accounts) { + final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem(); + final CachedMerkleTrieLoader cachedMerkleTrieLoader = new CachedMerkleTrieLoader(metricsSystem); + final TrieLogManager trieLogManager = new NoOpTrieLogManager(); + final BonsaiWorldStateKeyValueStorage worldStateStorage = + new BonsaiWorldStateKeyValueStorage(new InMemoryKeyValueStorageProvider(), metricsSystem); + final BonsaiReferenceTestWorldState worldState = + new BonsaiReferenceTestWorldState( + worldStateStorage, cachedMerkleTrieLoader, trieLogManager); + + final WorldUpdater updater = worldState.updater(); + for (final Map.Entry entry : accounts.entrySet()) { + ReferenceTestWorldState.insertAccount( + updater, Address.fromHexString(entry.getKey()), entry.getValue()); + } + updater.commit(); + return worldState; + } + + static class NoOpTrieLogManager implements TrieLogManager { + private final Subscribers trieLogObservers = Subscribers.create(); + private final TrieLogFactory trieLogFactory = new TrieLogFactoryImpl(); + + @Override + public void saveTrieLog( + final BonsaiWorldStateUpdateAccumulator localUpdater, + final Hash forWorldStateRootHash, + final BlockHeader forBlockHeader, + final BonsaiWorldState forWorldState) { + // notify trie log added observers, synchronously + TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); + trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); + } + + @Override + public void addCachedLayer( + final BlockHeader blockHeader, + final Hash worldStateRootHash, + final BonsaiWorldState forWorldState) {} + + @Override + public boolean containWorldStateStorage(final Hash blockHash) { + return false; + } + + @Override + public Optional getWorldState(final Hash blockHash) { + return Optional.empty(); + } + + @Override + public Optional getNearestWorldState(final BlockHeader blockHeader) { + return Optional.empty(); + } + + @Override + public Optional getHeadWorldState( + final Function> hashBlockHeaderFunction) { + return Optional.empty(); + } + + @Override + public long getMaxLayersToLoad() { + return 0; + } + + @Override + public void reset() {} + + @Override + public Optional getTrieLogLayer(final Hash blockHash) { + return Optional.empty(); + } + + @Override + public synchronized long subscribe(final TrieLogEvent.TrieLogObserver sub) { + return trieLogObservers.subscribe(sub); + } + + @Override + public synchronized void unsubscribe(final long id) { + trieLogObservers.unsubscribe(id); + } + } +} diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/DefaultReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/DefaultReferenceTestWorldState.java new file mode 100644 index 00000000000..7f3829511fa --- /dev/null +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/DefaultReferenceTestWorldState.java @@ -0,0 +1,50 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.referencetests; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; +import org.hyperledger.besu.ethereum.worldstate.DefaultMutableWorldState; +import org.hyperledger.besu.evm.worldstate.WorldUpdater; +import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; + +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonCreator; + +class DefaultReferenceTestWorldState extends DefaultMutableWorldState + implements ReferenceTestWorldState { + + DefaultReferenceTestWorldState() { + super( + new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()), + new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage())); + } + + @JsonCreator + static ReferenceTestWorldState create(final Map accounts) { + final ReferenceTestWorldState worldState = new DefaultReferenceTestWorldState(); + final WorldUpdater updater = worldState.updater(); + + for (final Map.Entry entry : accounts.entrySet()) { + ReferenceTestWorldState.insertAccount( + updater, Address.fromHexString(entry.getKey()), entry.getValue()); + } + + updater.commit(); + return worldState; + } +} diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java index 659847c1211..7008eb25313 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java @@ -17,12 +17,9 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Wei; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; -import org.hyperledger.besu.ethereum.worldstate.DefaultMutableWorldState; +import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.worldstate.WorldUpdater; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.util.HashMap; import java.util.Map; @@ -35,9 +32,9 @@ /** Represent a worldState for testing. */ @JsonIgnoreProperties(ignoreUnknown = true) -public class ReferenceTestWorldState extends DefaultMutableWorldState { +public interface ReferenceTestWorldState extends MutableWorldState { - public static class AccountMock { + class AccountMock { private final long nonce; private final Wei balance; private final Bytes code; @@ -91,21 +88,8 @@ static void insertAccount( } @JsonCreator - public static ReferenceTestWorldState create(final Map accounts) { - final ReferenceTestWorldState worldState = new ReferenceTestWorldState(); - final WorldUpdater updater = worldState.updater(); - - for (final Map.Entry entry : accounts.entrySet()) { - insertAccount(updater, Address.fromHexString(entry.getKey()), entry.getValue()); - } - - updater.commit(); - return worldState; - } - - public ReferenceTestWorldState() { - super( - new WorldStateKeyValueStorage(new InMemoryKeyValueStorage()), - new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage())); + static ReferenceTestWorldState create(final Map accounts) { + // delegate to a Bonsai reference test world state: + return BonsaiReferenceTestWorldState.create(accounts); } } From 4b6c7a633493a0369de7b455e155ffe90f969e18 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Wed, 26 Jul 2023 13:25:16 -0700 Subject: [PATCH 02/13] fix some tests and initialization Signed-off-by: garyschulte --- .../besu/ethereum/bonsai/BonsaiWorldStateProvider.java | 2 +- .../besu/ethereum/mainnet/MainnetBlockProcessorTest.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java index 986e736745a..6388d62a584 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateProvider.java @@ -102,8 +102,8 @@ public BonsaiWorldStateProvider( pluginContext); this.blockchain = blockchain; this.worldStateStorage = worldStateStorage; - this.persistedState = new BonsaiWorldState(this, worldStateStorage); this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; + this.persistedState = new BonsaiWorldState(this, worldStateStorage); blockchain .getBlockHeader(persistedState.getWorldStateBlockHash()) .ifPresent( diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetBlockProcessorTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetBlockProcessorTest.java index 6bd6978acb1..3648b3f4bd8 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetBlockProcessorTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetBlockProcessorTest.java @@ -94,6 +94,7 @@ public void accountCreatedWhenBlockRewardIsZeroAndNotSkipped() { final BlockHeader emptyBlockHeader = new BlockHeaderTestFixture() .transactionsRoot(Hash.EMPTY_LIST_HASH) + .stateRoot(Hash.fromHexString("0xa6b5d50f7b3c39b969c2fe8fed091939c674fef49b4826309cb6994361e39b71")) .ommersHash(Hash.EMPTY_LIST_HASH) .buildHeader(); blockProcessor.processBlock(blockchain, worldState, emptyBlockHeader, emptyList(), emptyList()); From 4afe52fdfa924ffb4c9620d400921fed249443b3 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Fri, 4 Aug 2023 14:23:38 -0700 Subject: [PATCH 03/13] interim commit, using Function as interface rather than PreImageStorage bonsai worldstate tests fixes and hacks -> getOrCreate in BonsaiWorldStateUpdateAccumulator - do not throw if we discover an empty account in a non-null BonsaiValue -> add curentStateRoot to t8n -> preImageHasher function in BonsaiWorldState / Accumulator .. hacky setting/copying of preImageHasher on construction/copying just in Layered. if we keep this route, we need to do it for snapshots also -> storageEntriesFrom and streamAccounts implemented in BonsaiWorldStateKeyValueStorage Signed-off-by: garyschulte --- .../besu/ethereum/bonsai/BonsaiAccount.java | 5 +- .../bonsai/storage/BonsaiPreImageProxy.java | 2 + .../BonsaiWorldStateKeyValueStorage.java | 38 ++++++++++++++- .../storage/BonsaiWorldStateLayerStorage.java | 1 + .../bonsai/worldview/BonsaiWorldState.java | 33 ++++++------- .../BonsaiWorldStateUpdateAccumulator.java | 36 ++++++++++---- .../mainnet/MainnetBlockProcessorTest.java | 4 +- .../backwardsync/BackwardSyncContextTest.java | 7 ++- .../besu/evmtool/StateTestSubCommand.java | 6 +-- .../hyperledger/besu/evmtool/T8nExecutor.java | 6 +-- .../besu/evmtool/T8nSubCommand.java | 3 +- .../t8n/berlin-calculate-difficulty.json | 3 +- .../besu/evmtool/t8n/berlin-example-yul.json | 3 +- .../t8n/berlin-negative-cli-reward.json | 3 +- .../evmtool/t8n/berlin-no-cli-reward.json | 3 +- .../besu/evmtool/t8n/berlin-no-tx.json | 3 +- .../besu/evmtool/t8n/berlin-simple.json | 3 +- .../t8n/cancun-6780-selfdestruct-sweep.json | 3 +- .../t8n/cancun-6780-selfdestruct-to-self.json | 3 +- .../cancun-6780-selfdestruct-transient.json | 3 +- .../besu/evmtool/t8n/cancun-blobs-per-tx.json | 3 +- .../evmtool/t8n/istanbul-cumulative-gas.json | 1 + .../evmtool/t8n/london-env-no-basefee.json | 3 +- .../besu/evmtool/t8n/london-hex.json | 3 +- .../besu/evmtool/t8n/shanghai-blockhash.json | 3 +- .../besu/evmtool/t8n/shanghai-init-code.json | 3 +- .../besu/evmtool/t8n/shanghai-invalidTx.json | 3 +- .../t8n/shanghai-withdrawals-no-nonce.json | 3 +- .../t8n/shanghai-withdrawals-overflow.json | 3 +- .../evmtool/t8n/shanghai-withdrawals.json | 3 +- .../BonsaiReferenceTestWorldState.java | 48 ++++++++++++++++++- .../DefaultReferenceTestWorldState.java | 14 +++++- .../GeneralStateTestCaseEipSpec.java | 7 ++- .../referencetests/ReferenceTestEnv.java | 3 +- .../ReferenceTestWorldState.java | 2 + .../vm/GeneralStateReferenceTestTools.java | 5 +- 36 files changed, 206 insertions(+), 69 deletions(-) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiPreImageProxy.java diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java index a5488cb9a2e..523dd4424b7 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java @@ -35,6 +35,9 @@ import java.util.Map; import java.util.NavigableMap; import java.util.Objects; +import java.util.Optional; +import java.util.TreeMap; +import java.util.stream.Collectors; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; @@ -219,7 +222,7 @@ public UInt256 getOriginalStorageValue(final UInt256 key) { @Override public NavigableMap storageEntriesFrom( final Bytes32 startKeyHash, final int limit) { - throw new RuntimeException("Bonsai Tries does not currently support enumerating storage"); + return context.getWorldStateStorage().storageEntriesFrom(this.addressHash, startKeyHash, limit); } public Bytes serializeAccount() { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiPreImageProxy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiPreImageProxy.java new file mode 100644 index 00000000000..c681ba241e8 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiPreImageProxy.java @@ -0,0 +1,2 @@ +package org.hyperledger.besu.ethereum.bonsai.storage;public interface BonsaiPreImageProxy { +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java index 4ceb63ef219..d98f86a6413 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java @@ -19,6 +19,7 @@ import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; +import org.apache.tuweni.units.bigints.UInt256; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.bonsai.storage.flat.FlatDbReaderStrategy; @@ -31,6 +32,7 @@ import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; +import org.hyperledger.besu.evm.account.AccountStorageEntry; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; @@ -41,10 +43,14 @@ import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.Optional; +import java.util.TreeMap; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; +import java.util.stream.Collectors; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; @@ -53,7 +59,7 @@ @SuppressWarnings("unused") public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoCloseable { - + Bytes32 BYTES32_MAX_VALUE = Bytes32.fromHexString("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateKeyValueStorage.class); // 0x776f726c64526f6f74 @@ -79,6 +85,9 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC protected final Subscribers subscribers = Subscribers.create(); + // no-op default hash preImage mapper: + protected Function> hashPreImageMapper = __ -> Optional.empty(); + public BonsaiWorldStateKeyValueStorage( final StorageProvider provider, final ObservableMetricsSystem metricsSystem) { this.composedWorldStateStorage = @@ -250,6 +259,33 @@ public Map streamFlatStorages( composedWorldStateStorage, accountHash, startKeyHash, endKeyHash, max); } + /** + * Provide a function to map hash values to an Optional wrapping their corresponding preImage + * or empty if the preImage is not available. + * + * @param hashPreImageMapper preImage mapper function + */ + public void setPreImageMapper(final Function> hashPreImageMapper) { + this.hashPreImageMapper = hashPreImageMapper; + } + + public NavigableMap storageEntriesFrom(final Hash addressHash, + final Bytes32 startKeyHash, final int limit) { + return streamFlatStorages(addressHash, startKeyHash, BYTES32_MAX_VALUE, limit) + .entrySet() + .stream() + .collect(Collectors.toMap( + e -> e.getKey(), + e -> AccountStorageEntry.create( + UInt256.fromBytes(e.getValue()), + Hash.wrap(e.getKey()), + hashPreImageMapper.apply(Hash.wrap(e.getKey())) + .map(UInt256::fromBytes)), + (a,b) -> a, + TreeMap::new + )); + } + @Override public Optional getNodeData(final Bytes location, final Bytes32 hash) { return Optional.empty(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java index aa96354789a..1e46312287c 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java @@ -39,6 +39,7 @@ public BonsaiWorldStateLayerStorage( final BonsaiWorldStateKeyValueStorage parent, final ObservableMetricsSystem metricsSystem) { super(parent, composedWorldStateStorage, trieLogStorage, metricsSystem); + setPreImageMapper(parent.hashPreImageMapper); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java index 4de046313ad..878b82b799a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java @@ -16,11 +16,9 @@ package org.hyperledger.besu.ethereum.bonsai.worldview; -import static org.hyperledger.besu.ethereum.bonsai.BonsaiAccount.fromRLP; -import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; -import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; -import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; - +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; @@ -47,19 +45,20 @@ import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import java.util.Map; import java.util.Optional; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Stream; -import javax.annotation.Nonnull; -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.units.bigints.UInt256; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.hyperledger.besu.ethereum.bonsai.BonsaiAccount.fromRLP; +import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; +import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; public class BonsaiWorldState implements MutableWorldState, BonsaiWorldView, BonsaiStorageSubscriber { @@ -68,8 +67,8 @@ public class BonsaiWorldState private BonsaiWorldStateKeyValueStorage worldStateStorage; - private final CachedMerkleTrieLoader cachedMerkleTrieLoader; - private final TrieLogManager trieLogManager; + protected final CachedMerkleTrieLoader cachedMerkleTrieLoader; + protected final TrieLogManager trieLogManager; private final BonsaiWorldStateUpdateAccumulator accumulator; private Hash worldStateRootHash; @@ -80,13 +79,14 @@ public class BonsaiWorldState public BonsaiWorldState( final BonsaiWorldStateProvider archive, final BonsaiWorldStateKeyValueStorage worldStateStorage) { - this(worldStateStorage, archive.getCachedMerkleTrieLoader(), archive.getTrieLogManager()); + this(worldStateStorage, archive.getCachedMerkleTrieLoader(), archive.getTrieLogManager(), Hash::hash); } protected BonsaiWorldState( final BonsaiWorldStateKeyValueStorage worldStateStorage, final CachedMerkleTrieLoader cachedMerkleTrieLoader, - final TrieLogManager trieLogManager) { + final TrieLogManager trieLogManager, + final Function preImageHasher) { this.worldStateStorage = worldStateStorage; this.worldStateRootHash = Hash.wrap( @@ -100,7 +100,8 @@ protected BonsaiWorldState( cachedMerkleTrieLoader.preLoadAccount( getWorldStateStorage(), worldStateRootHash, addr), (addr, value) -> - cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value)); + cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value), + preImageHasher); this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; this.trieLogManager = trieLogManager; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java index b1cd64378b9..407ff996ba1 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java @@ -66,6 +66,8 @@ public class BonsaiWorldStateUpdateAccumulator private final Map> codeToUpdate = new ConcurrentHashMap<>(); private final Set
storageToClear = Collections.synchronizedSet(new HashSet<>()); + private final Function preImageHasher; + // storage sub mapped by _hashed_ key. This is because in self_destruct calls we need to // enumerate the old storage and delete it. Those are trie stored by hashed key by spec and the // alternative was to keep a giant pre-image cache of the entire trie. @@ -78,11 +80,21 @@ public BonsaiWorldStateUpdateAccumulator( final BonsaiWorldView world, final Consumer> accountPreloader, final Consumer storagePreloader) { - super(world); - this.accountsToUpdate = new AccountConsumingMap<>(new ConcurrentHashMap<>(), accountPreloader); - this.accountPreloader = accountPreloader; - this.storagePreloader = storagePreloader; - this.isAccumulatorStateChanged = false; + // create accumulator without a preImage-aware Hash function: + this(world, accountPreloader, storagePreloader, Hash::hash); + } + + protected BonsaiWorldStateUpdateAccumulator( + final BonsaiWorldView world, + final Consumer> accountPreloader, + final Consumer storagePreloader, + final Function preImageHasher) { + super(world); + this.accountsToUpdate = new AccountConsumingMap<>(new ConcurrentHashMap<>(), accountPreloader); + this.accountPreloader = accountPreloader; + this.storagePreloader = storagePreloader; + this.isAccumulatorStateChanged = false; + this.preImageHasher = preImageHasher; } public BonsaiWorldStateUpdateAccumulator copy() { @@ -127,14 +139,18 @@ public MutableAccount createAccount(final Address address, final long nonce, fin bonsaiValue = new BonsaiValue<>(null, null); accountsToUpdate.put(address, bonsaiValue); } else if (bonsaiValue.getUpdated() != null) { - throw new IllegalStateException("Cannot create an account when one already exists"); + if (bonsaiValue.getUpdated().isEmpty()) { + return new WrappedEvmAccount(track(new UpdateTrackingAccount<>(bonsaiValue.getUpdated()))); + } else { + throw new IllegalStateException("Cannot create an account when one already exists"); + } } final BonsaiAccount newAccount = new BonsaiAccount( this, address, - address.addressHash(), + preImageHasher.apply(address), nonce, balance, Hash.EMPTY_TRIE_HASH, @@ -355,7 +371,7 @@ public void commit() { entries.forEach( storageUpdate -> { final UInt256 keyUInt = storageUpdate.getKey(); - final Hash slotHash = Hash.hash(keyUInt); + final Hash slotHash = preImageHasher.apply(keyUInt); final StorageSlotKey slotKey = new StorageSlotKey(slotHash, Optional.of(keyUInt)); final UInt256 value = storageUpdate.getValue(); @@ -399,7 +415,7 @@ public Optional getCode(final Address address, final Hash codeHash) { @Override public UInt256 getStorageValue(final Address address, final UInt256 slotKey) { - StorageSlotKey storageSlotKey = new StorageSlotKey(Hash.hash(slotKey), Optional.of(slotKey)); + StorageSlotKey storageSlotKey = new StorageSlotKey(preImageHasher.apply(slotKey), Optional.of(slotKey)); return getStorageValueByStorageSlotKey(address, storageSlotKey).orElse(UInt256.ZERO); } @@ -443,7 +459,7 @@ public Optional getStorageValueByStorageSlotKey( public UInt256 getPriorStorageValue(final Address address, final UInt256 storageKey) { // TODO maybe log the read into the trie layer? StorageSlotKey storageSlotKey = - new StorageSlotKey(Hash.hash(storageKey), Optional.of(storageKey)); + new StorageSlotKey(preImageHasher.apply(storageKey), Optional.of(storageKey)); final Map> localAccountStorage = storageToUpdate.get(address); if (localAccountStorage != null) { diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetBlockProcessorTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetBlockProcessorTest.java index 3648b3f4bd8..1b106686bfc 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetBlockProcessorTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetBlockProcessorTest.java @@ -94,7 +94,9 @@ public void accountCreatedWhenBlockRewardIsZeroAndNotSkipped() { final BlockHeader emptyBlockHeader = new BlockHeaderTestFixture() .transactionsRoot(Hash.EMPTY_LIST_HASH) - .stateRoot(Hash.fromHexString("0xa6b5d50f7b3c39b969c2fe8fed091939c674fef49b4826309cb6994361e39b71")) + .stateRoot( + Hash.fromHexString( + "0xa6b5d50f7b3c39b969c2fe8fed091939c674fef49b4826309cb6994361e39b71")) .ommersHash(Hash.EMPTY_LIST_HASH) .buildHeader(); blockProcessor.processBlock(blockchain, worldState, emptyBlockHeader, emptyList(), emptyList()); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java index 3ed28e66f4d..b5909c4d429 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java @@ -47,7 +47,7 @@ import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; -import org.hyperledger.besu.ethereum.referencetests.ReferenceTestWorldState; +import org.hyperledger.besu.ethereum.referencetests.DefaultReferenceTestWorldState; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; @@ -149,7 +149,10 @@ public void setup() { return new BlockProcessingResult( Optional.of( new BlockProcessingOutputs( - new ReferenceTestWorldState(), blockDataGenerator.receipts(block)))); + // use forest-based worldstate since it does not require + // blockheader stateroot to match actual worldstate root + DefaultReferenceTestWorldState.create(Collections.emptyMap()), + blockDataGenerator.receipts(block)))); }); backwardChain = inMemoryBackwardChain(); diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java index 95af770c589..152259abe21 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java @@ -37,12 +37,10 @@ import org.hyperledger.besu.ethereum.referencetests.ReferenceTestBlockchain; import org.hyperledger.besu.ethereum.referencetests.ReferenceTestProtocolSchedules; import org.hyperledger.besu.ethereum.rlp.RLP; -import org.hyperledger.besu.ethereum.worldstate.DefaultMutableWorldState; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.log.Log; import org.hyperledger.besu.evm.tracing.OperationTracer; import org.hyperledger.besu.evm.tracing.StandardJsonTracer; -import org.hyperledger.besu.evm.worldstate.WorldState; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.evmtool.exception.UnsupportedForkException; import org.hyperledger.besu.util.LogConfigurator; @@ -200,9 +198,7 @@ private void traceTestSpecs(final String test, final List transactions, final List rejections, final TracerManager tracerManager) { @@ -238,7 +238,7 @@ static T8nResult runTest( ReferenceTestProtocolSchedules.create( new StubGenesisConfigOptions().chainId(BigInteger.valueOf(chainId))); - final MutableWorldState worldState = new DefaultMutableWorldState(initialWorldState); + final MutableWorldState worldState = initialWorldState.copy(); final ProtocolSchedule protocolSchedule = referenceTestProtocolSchedules.getByName(fork); if (protocolSchedule == null) { diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nSubCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nSubCommand.java index 1571dc3551e..ea2984c5c90 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nSubCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nSubCommand.java @@ -21,7 +21,6 @@ import org.hyperledger.besu.crypto.SignatureAlgorithmFactory; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.referencetests.ReferenceTestEnv; import org.hyperledger.besu.ethereum.referencetests.ReferenceTestWorldState; @@ -175,7 +174,7 @@ public void run() { final ObjectMapper objectMapper = JsonUtils.createObjectMapper(); final ObjectReader t8nReader = objectMapper.reader(); - MutableWorldState initialWorldState; + ReferenceTestWorldState initialWorldState; ReferenceTestEnv referenceTestEnv; List transactions = new ArrayList<>(); List rejections = new ArrayList<>(); diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-calculate-difficulty.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-calculate-difficulty.json index 1a3c7a178c6..005cb1498ad 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-calculate-difficulty.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-calculate-difficulty.json @@ -48,6 +48,7 @@ "currentGasLimit": "100000000000000000", "currentNumber": "1", "currentTimestamp": "1000", + "currentStateRoot": "0xd921e74bfe864514fa508003336b8f66eb98d748c5163749827029a1ed7db265", "parentDifficulty": "131072", "parentGasUsed": "50000000000000000", "parentGasLimit": "100000000000000000", @@ -101,4 +102,4 @@ "gasUsed": "0xa8ad" } } -} \ No newline at end of file +} diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-example-yul.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-example-yul.json index 1a3c7a178c6..005cb1498ad 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-example-yul.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-example-yul.json @@ -48,6 +48,7 @@ "currentGasLimit": "100000000000000000", "currentNumber": "1", "currentTimestamp": "1000", + "currentStateRoot": "0xd921e74bfe864514fa508003336b8f66eb98d748c5163749827029a1ed7db265", "parentDifficulty": "131072", "parentGasUsed": "50000000000000000", "parentGasLimit": "100000000000000000", @@ -101,4 +102,4 @@ "gasUsed": "0xa8ad" } } -} \ No newline at end of file +} diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-negative-cli-reward.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-negative-cli-reward.json index 4ef273955c4..f41eb4989c4 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-negative-cli-reward.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-negative-cli-reward.json @@ -46,6 +46,7 @@ "env": { "currentCoinbase": "0x0000000000000000000000000000000000000000", "currentDifficulty": "0x0", + "currentStateRoot": "0xde762e485411037c02119ff1115b422945b37efe91f4135dd442c3346629d0ef", "currentGasLimit": "0x0", "currentNumber": "0", "currentTimestamp": "0" @@ -93,4 +94,4 @@ "gasUsed": "0xa8ad" } } -} \ No newline at end of file +} diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-no-cli-reward.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-no-cli-reward.json index d06c75ccfd2..4aba3a31a1d 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-no-cli-reward.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-no-cli-reward.json @@ -45,6 +45,7 @@ "env": { "currentCoinbase": "0x0000000000000000000000000000000000000000", "currentDifficulty": "0x0", + "currentStateRoot": "0x9580f6eec8ad54773eff8e370d12dfea7b3216e929657c48a509af64c4190be6", "currentGasLimit": "0x0", "currentNumber": "0", "currentTimestamp": "0" @@ -92,4 +93,4 @@ "gasUsed": "0xa8ad" } } -} \ No newline at end of file +} diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-no-tx.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-no-tx.json index 5abcf7bb347..8c655b71ec9 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-no-tx.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-no-tx.json @@ -30,6 +30,7 @@ "env": { "currentCoinbase": "0x0000000000000000000000000000000000000000", "currentDifficulty": "0x0", + "currentStateRoot": "0x369ba8b2e5b32d675d07933d6fb851d97d3ca66c60a829f7356163d92ae0439a", "currentGasLimit": "0x0", "currentNumber": "0", "currentTimestamp": "0" @@ -57,4 +58,4 @@ "gasUsed": "0x0" } } -} \ No newline at end of file +} diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-simple.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-simple.json index e4555f220b3..3fef5b5a296 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-simple.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/berlin-simple.json @@ -46,6 +46,7 @@ "env": { "currentCoinbase": "0x0000000000000000000000000000000000000000", "currentDifficulty": "0x0", + "currentStateRoot": "0xde762e485411037c02119ff1115b422945b37efe91f4135dd442c3346629d0ef", "currentGasLimit": "0x0", "currentNumber": "0", "currentTimestamp": "0" @@ -93,4 +94,4 @@ "gasUsed": "0xa8ad" } } -} \ No newline at end of file +} diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/cancun-6780-selfdestruct-sweep.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/cancun-6780-selfdestruct-sweep.json index bc7d113ec79..7e1ffaf68d1 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/cancun-6780-selfdestruct-sweep.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/cancun-6780-selfdestruct-sweep.json @@ -49,6 +49,7 @@ "currentTimestamp": "1000", "currentRandom": "0", "currentDifficulty": "0", + "currentStateRoot": "0x3a0e532de836d767cae901aba671040fedc07557d277f7203066f640ed95f78d", "parentDifficulty": "0", "parentBaseFee": "7", "parentGasUsed": "0", @@ -113,4 +114,4 @@ "currentExcessBlobGas": "0x0" } } -} \ No newline at end of file +} diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/cancun-6780-selfdestruct-to-self.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/cancun-6780-selfdestruct-to-self.json index 7d8907d9bfb..d922c54baa7 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/cancun-6780-selfdestruct-to-self.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/cancun-6780-selfdestruct-to-self.json @@ -51,6 +51,7 @@ "currentTimestamp": "1000", "currentRandom": "0", "currentDifficulty": "0", + "currentStateRoot" : "0xddd3a541e86e2dd0293959736de63e1fad74ae95149f34740b1173378e82527a", "parentDifficulty": "0", "parentBaseFee": "7", "parentGasUsed": "0", @@ -112,4 +113,4 @@ "currentExcessBlobGas": "0x0" } } -} \ No newline at end of file +} diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/cancun-6780-selfdestruct-transient.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/cancun-6780-selfdestruct-transient.json index 1860990b0f7..997b29f76bd 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/cancun-6780-selfdestruct-transient.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/cancun-6780-selfdestruct-transient.json @@ -43,6 +43,7 @@ "currentTimestamp": "1000", "currentRandom": "0", "currentDifficulty": "0", + "currentStateRoot": "0xe8a9461dcfdbaa48bbddca4f4baa439e45f1489827e5deeb2797363323e34769", "parentDifficulty": "0", "parentBaseFee": "7", "parentGasUsed": "0", @@ -97,4 +98,4 @@ "currentExcessBlobGas": "0x0" } } -} \ No newline at end of file +} diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/cancun-blobs-per-tx.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/cancun-blobs-per-tx.json index b2ed324b3d6..0912b52b81a 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/cancun-blobs-per-tx.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/cancun-blobs-per-tx.json @@ -53,6 +53,7 @@ "currentTimestamp": "24", "currentRandom": "0", "currentDifficulty": "0", + "currentStateRoot": "0xd29f5a8dd1a63e0a299009f546bdf447fb61f1604d95e737bd8eb3c089d78060", "parentDifficulty": "0", "parentBaseFee": "7", "parentGasUsed": "21000", @@ -114,4 +115,4 @@ "currentExcessBlobGas": "0x0" } } -} \ No newline at end of file +} diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/istanbul-cumulative-gas.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/istanbul-cumulative-gas.json index d3dd7021df7..087eae7745b 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/istanbul-cumulative-gas.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/istanbul-cumulative-gas.json @@ -363,6 +363,7 @@ "currentGasLimit": "100000000000000000", "currentNumber": "1", "currentTimestamp": "1000", + "currentStateRoot": "0xf4f4aed0d1813d2880d8bb1cb5697303c20002ac6bfc206635645f43805ccbcb", "parentDifficulty": "131072", "parentGasUsed": "50000000000000000", "parentGasLimit": "100000000000000000", diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/london-env-no-basefee.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/london-env-no-basefee.json index 052ebb5260b..bb2d2da5dbc 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/london-env-no-basefee.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/london-env-no-basefee.json @@ -48,6 +48,7 @@ "currentGasLimit": "100000000000000000", "currentNumber": "1", "currentTimestamp": "1000", + "currentStateRoot": "0x5c0f211cd1bea44ac01bd3fd7e0ab742d316adabd139bcf60a7aac6c458d596d", "parentDifficulty": "131072", "parentBaseFee": "7", "parentGasUsed": "50000000000000000", @@ -103,4 +104,4 @@ "currentBaseFee": "0x7" } } -} \ No newline at end of file +} diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/london-hex.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/london-hex.json index 502ac628de6..1ccce1739d6 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/london-hex.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/london-hex.json @@ -48,6 +48,7 @@ "currentNumber": "0x1", "currentTimestamp": "0x0", "currentGasLimit": "0xff112233445566", + "currentStateRoot": "0x5775e461ac316f8c02466d23e4492d24bbe546336f6fc632dd1311d6a5b67b22", "previousHash": "0xb271e9e5796d0ff5a2fd519ba666393e42d4f38680854761121d84a7a96ff017", "parentTimestamp": "0x00", "parentDifficulty": "0x00", @@ -102,4 +103,4 @@ "currentBaseFee": "0xa" } } -} \ No newline at end of file +} diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-blockhash.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-blockhash.json index e1c331206d2..9b2908976fc 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-blockhash.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-blockhash.json @@ -48,6 +48,7 @@ "currentTimestamp": "24", "currentRandom": "0", "currentDifficulty": "0", + "currentStateRoot": "0x89412c0f1bb31b983d0317e6e2801a4e604e1ef9987a132ab63c52f2d5a3994b", "parentDifficulty": "0", "parentGasUsed": "0", "parentGasLimit": "100000000000000000", @@ -106,4 +107,4 @@ "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" } } -} \ No newline at end of file +} diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-init-code.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-init-code.json index d536a6a7260..6c502e80cc8 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-init-code.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-init-code.json @@ -42,6 +42,7 @@ "currentTimestamp": "12", "currentRandom": "0", "currentDifficulty": "0", + "currentStateRoot": "0x70c42824108fafccadbfce71e6e22660c4fad89be18be324cd15ef351969a8c8", "parentDifficulty": "0", "parentBaseFee": "7", "parentGasUsed": "0", @@ -81,4 +82,4 @@ "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" } } -} \ No newline at end of file +} diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-invalidTx.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-invalidTx.json index 7ec8a1d7763..79087bf545a 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-invalidTx.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-invalidTx.json @@ -42,6 +42,7 @@ "currentNumber": "1", "currentTimestamp": "1000", "currentDifficulty": "0x20000", + "currentStateRoot": "0xec92f4c572101075d17eb5aaf15c33df92b6d5519cbed635fc53353b99e8e6da", "currentBaseFee": "0xa", "blockHashes": { "0": "0x5e20a0453cecd065ea59c37ac63e079ee08998b6045136a8ce6635c7912ec0b6" @@ -84,4 +85,4 @@ "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" } } -} \ No newline at end of file +} diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-withdrawals-no-nonce.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-withdrawals-no-nonce.json index 86cefab468f..4e9f1febebe 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-withdrawals-no-nonce.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-withdrawals-no-nonce.json @@ -40,6 +40,7 @@ "currentTimestamp": "24", "currentRandom": "0", "currentDifficulty": "0", + "currentStateRoot": "0x41855ebd4b425591ae60f57de6ec700c8a1128c7761b63371b09671b5f1abb3f", "parentDifficulty": "0", "parentGasUsed": "0", "parentGasLimit": "100000000000000000", @@ -98,4 +99,4 @@ "withdrawalsRoot": "0xa485a3bd07e29cb8234b5f093d5216eb8b965fb2693c66fea254f6cacef97a6f" } } -} \ No newline at end of file +} diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-withdrawals-overflow.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-withdrawals-overflow.json index 8f8b29b7525..e334d94cbff 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-withdrawals-overflow.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-withdrawals-overflow.json @@ -34,6 +34,7 @@ "currentTimestamp": "12", "currentRandom": "0", "currentDifficulty": "0", + "currentStateRoot": "0x2a23ea102cd739aa3ad83a9be1555d621e50b3a3f780a7369ec2a6c03f22e18d", "parentDifficulty": "0", "parentBaseFee": "7", "parentGasUsed": "0", @@ -78,4 +79,4 @@ "withdrawalsRoot": "0x7bae13217931ff0d7a6c5823c996492f28cd92da5a7788a9a68856bb2206741a" } } -} \ No newline at end of file +} diff --git a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-withdrawals.json b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-withdrawals.json index 5a4efc2eea0..946a97ceac6 100644 --- a/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-withdrawals.json +++ b/ethereum/evmtool/src/test/resources/org/hyperledger/besu/evmtool/t8n/shanghai-withdrawals.json @@ -35,6 +35,7 @@ "currentTimestamp": "12", "currentRandom": "0", "currentDifficulty": "0", + "currentStateRoot": "0x7075d070d6ec70a91216fc2dba6ceae6b8d30815166f6b7b7249776767fb9855", "parentDifficulty": "0", "parentBaseFee": "7", "parentGasUsed": "0", @@ -81,4 +82,4 @@ "withdrawalsRoot": "0xa485a3bd07e29cb8234b5f093d5216eb8b965fb2693c66fea254f6cacef97a6f" } } -} \ No newline at end of file +} diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java index e76e7d9749b..9fcacb77880 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -14,10 +14,14 @@ */ package org.hyperledger.besu.ethereum.referencetests; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount; import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateLayerStorage; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogAddedEvent; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogFactoryImpl; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogManager; @@ -25,6 +29,7 @@ import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; +import org.hyperledger.besu.evm.account.AccountStorageEntry; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -33,20 +38,44 @@ import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory; import org.hyperledger.besu.util.Subscribers; +import java.util.Comparator; +import java.util.HashMap; import java.util.Map; +import java.util.NavigableMap; import java.util.Optional; +import java.util.TreeMap; import java.util.function.Function; +import java.util.stream.Stream; import com.fasterxml.jackson.annotation.JsonCreator; public class BonsaiReferenceTestWorldState extends BonsaiWorldState implements ReferenceTestWorldState { + static final Map preImageCache = new HashMap<>(); + protected BonsaiReferenceTestWorldState( final BonsaiWorldStateKeyValueStorage worldStateStorage, final CachedMerkleTrieLoader cachedMerkleTrieLoader, final TrieLogManager trieLogManager) { - super(worldStateStorage, cachedMerkleTrieLoader, trieLogManager); + super(worldStateStorage, cachedMerkleTrieLoader, trieLogManager, BonsaiReferenceTestWorldState::preImageHasher); + worldStateStorage.setPreImageMapper(hash -> Optional.ofNullable(preImageCache.get(hash))); + } + + private static Hash preImageHasher(final Bytes key) { + Hash hash = Hash.hash(key); + preImageCache.putIfAbsent(hash, key); + return hash; + } + + @Override + public ReferenceTestWorldState copy() { + var layerCopy = new BonsaiWorldStateLayerStorage(worldStateStorage); + layerCopy.setPreImageMapper(hash -> Optional.ofNullable(preImageCache.get(hash))); + return new BonsaiReferenceTestWorldState( + layerCopy, + cachedMerkleTrieLoader, + trieLogManager); } @JsonCreator @@ -70,6 +99,23 @@ public static BonsaiReferenceTestWorldState create( return worldState; } + @Override + public Stream streamAccounts(final Bytes32 startKeyHash, final int limit){ + return worldStateStorage + .streamFlatAccounts( + //TODO: use a constant for the end range + startKeyHash, Bytes32.fromHexStringLenient("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), Long.MAX_VALUE) + .entrySet() + // map to addresses: + .stream() + .map(entry -> { + Address address = Address.wrap(preImageCache.get(entry.getKey())); + return new StreamableAccount(Optional.of(address), + BonsaiAccount.fromRLP(this, address, entry.getValue(), false)); + }) + .sorted(Comparator.comparing(account -> account.getAddress().orElse(Address.ZERO))); + } + static class NoOpTrieLogManager implements TrieLogManager { private final Subscribers trieLogObservers = Subscribers.create(); private final TrieLogFactory trieLogFactory = new TrieLogFactoryImpl(); diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/DefaultReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/DefaultReferenceTestWorldState.java index 7f3829511fa..ec0b058ba99 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/DefaultReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/DefaultReferenceTestWorldState.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DefaultMutableWorldState; +import org.hyperledger.besu.evm.worldstate.WorldState; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; @@ -25,7 +26,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; -class DefaultReferenceTestWorldState extends DefaultMutableWorldState +public class DefaultReferenceTestWorldState extends DefaultMutableWorldState implements ReferenceTestWorldState { DefaultReferenceTestWorldState() { @@ -34,8 +35,17 @@ class DefaultReferenceTestWorldState extends DefaultMutableWorldState new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage())); } + public DefaultReferenceTestWorldState(final WorldState worldState) { + super(worldState); + } + + @Override + public ReferenceTestWorldState copy() { + return new DefaultReferenceTestWorldState(this); + } + @JsonCreator - static ReferenceTestWorldState create(final Map accounts) { + public static ReferenceTestWorldState create(final Map accounts) { final ReferenceTestWorldState worldState = new DefaultReferenceTestWorldState(); final WorldUpdater updater = worldState.updater(); diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/GeneralStateTestCaseEipSpec.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/GeneralStateTestCaseEipSpec.java index b396eff98f8..def8c14f17e 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/GeneralStateTestCaseEipSpec.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/GeneralStateTestCaseEipSpec.java @@ -18,7 +18,6 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.Transaction; -import org.hyperledger.besu.evm.worldstate.WorldState; import java.util.function.Supplier; @@ -37,7 +36,7 @@ public class GeneralStateTestCaseEipSpec { // is why this is a supplier: calling get() actually does the signing. private final Supplier transactionSupplier; - private final WorldState initialWorldState; + private final ReferenceTestWorldState initialWorldState; private final Hash expectedRootHash; @@ -54,7 +53,7 @@ public class GeneralStateTestCaseEipSpec { GeneralStateTestCaseEipSpec( final String fork, final Supplier transactionSupplier, - final WorldState initialWorldState, + final ReferenceTestWorldState initialWorldState, final Hash expectedRootHash, final Hash expectedLogsHash, final BlockHeader blockHeader, @@ -78,7 +77,7 @@ public String getFork() { return fork; } - public WorldState getInitialWorldState() { + public ReferenceTestWorldState getInitialWorldState() { return initialWorldState; } diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestEnv.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestEnv.java index 79356331fb6..5f7af60719a 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestEnv.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestEnv.java @@ -104,6 +104,7 @@ public ReferenceTestEnv( @JsonProperty("currentBaseFee") final String baseFee, @JsonProperty("currentTimestamp") final String timestamp, @JsonProperty("currentRandom") final String random, + @JsonProperty("currentStateRoot") final String stateRoot, @JsonProperty("previousHash") final String previousHash, @JsonProperty("parentDifficulty") final String parentDifficulty, @JsonProperty("parentBaseFee") final String parentBaseFee, @@ -127,7 +128,7 @@ public ReferenceTestEnv( generateTestBlockHash(previousHash, number), Hash.EMPTY_LIST_HASH, // ommersHash Address.fromHexString(coinbase), - Hash.EMPTY, // stateRoot + Optional.ofNullable(stateRoot).map(Hash::fromHexString).orElse(Hash.EMPTY), // stateRoot Hash.EMPTY, // transactionsRoot Hash.EMPTY, // receiptsRoot new LogsBloomFilter(), diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java index 7008eb25313..825d7a2cd4b 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java @@ -87,6 +87,8 @@ static void insertAccount( } } + ReferenceTestWorldState copy(); + @JsonCreator static ReferenceTestWorldState create(final Map accounts) { // delegate to a Bonsai reference test world state: diff --git a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java index e01e08c0414..471adda77da 100644 --- a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java +++ b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java @@ -31,6 +31,7 @@ import org.hyperledger.besu.ethereum.referencetests.GeneralStateTestCaseSpec; import org.hyperledger.besu.ethereum.referencetests.ReferenceTestBlockchain; import org.hyperledger.besu.ethereum.referencetests.ReferenceTestProtocolSchedules; +import org.hyperledger.besu.ethereum.referencetests.ReferenceTestWorldState; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.worldstate.DefaultMutableWorldState; import org.hyperledger.besu.evm.account.Account; @@ -123,7 +124,7 @@ public static Collection generateTestParametersForConfig(final String[ public static void executeTest(final GeneralStateTestCaseEipSpec spec) { final BlockHeader blockHeader = spec.getBlockHeader(); - final WorldState initialWorldState = spec.getInitialWorldState(); + final ReferenceTestWorldState initialWorldState = spec.getInitialWorldState(); final Transaction transaction = spec.getTransaction(); // Sometimes the tests ask us assemble an invalid transaction. If we have @@ -136,7 +137,7 @@ public static void executeTest(final GeneralStateTestCaseEipSpec spec) { return; } - final MutableWorldState worldState = new DefaultMutableWorldState(initialWorldState); + final MutableWorldState worldState = initialWorldState.copy(); // Several of the GeneralStateTests check if the transaction could potentially // consume more gas than is left for the block it's attempted to be included in. // This check is performed within the `BlockImporter` rather than inside the From d9efe768932fcc9778e83f61eac75a456319dff8 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Fri, 4 Aug 2023 16:58:42 -0700 Subject: [PATCH 04/13] move hashing and preImage functions into a PreImageProxy class Signed-off-by: garyschulte --- .../besu/ethereum/bonsai/BonsaiAccount.java | 3 - .../bonsai/storage/BonsaiPreImageProxy.java | 86 ++++++++++++++++++- ...nsaiSnapshotWorldStateKeyValueStorage.java | 9 +- .../BonsaiWorldStateKeyValueStorage.java | 86 ++++++++++++------- .../storage/BonsaiWorldStateLayerStorage.java | 12 +-- .../bonsai/worldview/BonsaiWorldState.java | 35 +++++--- .../BonsaiWorldStateUpdateAccumulator.java | 46 +++++----- .../bonsai/trielog/TrieLogManagerTests.java | 8 +- .../backwardsync/ForwardSyncStepTest.java | 5 +- ethereum/referencetests/build.gradle | 1 + .../BonsaiReferenceTestWorldState.java | 52 +++-------- 11 files changed, 219 insertions(+), 124 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java index 523dd4424b7..7362d345ce4 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java @@ -35,9 +35,6 @@ import java.util.Map; import java.util.NavigableMap; import java.util.Objects; -import java.util.Optional; -import java.util.TreeMap; -import java.util.stream.Collectors; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiPreImageProxy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiPreImageProxy.java index c681ba241e8..a05d50f7601 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiPreImageProxy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiPreImageProxy.java @@ -1,2 +1,86 @@ -package org.hyperledger.besu.ethereum.bonsai.storage;public interface BonsaiPreImageProxy { +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.bonsai.storage; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; + +import java.util.Optional; + +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; + +/** Acts as both a Hasher and PreImageStorage for Bonsai storage format. */ +public interface BonsaiPreImageProxy extends WorldStatePreimageStorage { + /** + * If this value is not already present, save in preImage store and return the hash value. + * + * @param value value to hash + * @return Hash of value + */ + Hash hashAndSavePreImage(Bytes value); + + /** PreImageProxy which does not store or cache preImages and only implements hashing. */ + class NoOpPreImageProxy implements BonsaiPreImageProxy { + + @Override + public Hash hashAndSavePreImage(final Bytes value) { + return Hash.hash(value); + } + + @Override + public Optional getStorageTrieKeyPreimage(final Bytes32 trieKey) { + return Optional.empty(); + } + + @Override + public Optional
getAccountTrieKeyPreimage(final Bytes32 trieKey) { + return Optional.empty(); + } + + @Override + public Updater updater() { + throw new UnsupportedOperationException("NoOpPreImageProxy does not implement an updater"); + } + } + + /** + * A caching PreImageProxy suitable for ReferenceTestWorldState which saves hashes in an unbounded + * BiMap. + */ + class BonsaiReferenceTestPreImageProxy extends NoOpPreImageProxy { + BiMap preImageCache = HashBiMap.create(); + + @Override + public Hash hashAndSavePreImage(final Bytes value) { + return preImageCache.inverse().computeIfAbsent(value, Hash::hash); + } + + @Override + public Optional getStorageTrieKeyPreimage(final Bytes32 trieKey) { + return Optional.ofNullable(preImageCache.get(trieKey)).map(UInt256::fromBytes); + } + + @Override + public Optional
getAccountTrieKeyPreimage(final Bytes32 trieKey) { + return Optional.ofNullable(preImageCache.get(trieKey)).map(Address::wrap); + } + } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java index 44c79c46b8d..2ebaeb38230 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java @@ -44,13 +44,15 @@ public BonsaiSnapshotWorldStateKeyValueStorage( final BonsaiWorldStateKeyValueStorage parentWorldStateStorage, final SnappedKeyValueStorage segmentedWorldStateStorage, final KeyValueStorage trieLogStorage, - final ObservableMetricsSystem metricsSystem) { + final ObservableMetricsSystem metricsSystem, + final BonsaiPreImageProxy preImageProxy) { super( parentWorldStateStorage.flatDbMode, parentWorldStateStorage.flatDbReaderStrategy, segmentedWorldStateStorage, trieLogStorage, - metricsSystem); + metricsSystem, + preImageProxy); this.parentWorldStateStorage = parentWorldStateStorage; this.subscribeParentId = parentWorldStateStorage.subscribe(this); } @@ -62,7 +64,8 @@ public BonsaiSnapshotWorldStateKeyValueStorage( worldStateStorage, ((SnappableKeyValueStorage) worldStateStorage.composedWorldStateStorage).takeSnapshot(), worldStateStorage.trieLogStorage, - metricsSystem); + metricsSystem, + worldStateStorage.preImageProxy); } private boolean isClosedGet() { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java index d98f86a6413..a251d7d7b56 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java @@ -19,12 +19,14 @@ import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; -import org.apache.tuweni.units.bigints.UInt256; +import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount; import org.hyperledger.besu.ethereum.bonsai.storage.flat.FlatDbReaderStrategy; import org.hyperledger.besu.ethereum.bonsai.storage.flat.FullFlatDbReaderStrategy; import org.hyperledger.besu.ethereum.bonsai.storage.flat.PartialFlatDbReaderStrategy; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldView; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.trie.MerkleTrie; @@ -33,6 +35,7 @@ import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.evm.account.AccountStorageEntry; +import org.hyperledger.besu.evm.worldstate.WorldState; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; @@ -41,25 +44,28 @@ import org.hyperledger.besu.util.Subscribers; import java.nio.charset.StandardCharsets; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.NavigableMap; import java.util.Optional; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @SuppressWarnings("unused") public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoCloseable { - Bytes32 BYTES32_MAX_VALUE = Bytes32.fromHexString("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); + Bytes32 BYTES32_MAX_VALUE = + Bytes32.fromHexString("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateKeyValueStorage.class); // 0x776f726c64526f6f74 @@ -85,11 +91,17 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC protected final Subscribers subscribers = Subscribers.create(); - // no-op default hash preImage mapper: - protected Function> hashPreImageMapper = __ -> Optional.empty(); + final BonsaiPreImageProxy preImageProxy; public BonsaiWorldStateKeyValueStorage( final StorageProvider provider, final ObservableMetricsSystem metricsSystem) { + this(provider, metricsSystem, new BonsaiPreImageProxy.NoOpPreImageProxy()); + } + + public BonsaiWorldStateKeyValueStorage( + final StorageProvider provider, + final ObservableMetricsSystem metricsSystem, + final BonsaiPreImageProxy preImageProxy) { this.composedWorldStateStorage = provider.getStorageBySegmentIdentifiers( List.of( @@ -97,6 +109,7 @@ public BonsaiWorldStateKeyValueStorage( this.trieLogStorage = provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); this.metricsSystem = metricsSystem; + this.preImageProxy = preImageProxy; loadFlatDbStrategy(); } @@ -105,12 +118,14 @@ public BonsaiWorldStateKeyValueStorage( final FlatDbReaderStrategy flatDbReaderStrategy, final SegmentedKeyValueStorage composedWorldStateStorage, final KeyValueStorage trieLogStorage, - final ObservableMetricsSystem metricsSystem) { + final ObservableMetricsSystem metricsSystem, + final BonsaiPreImageProxy preImageProxy) { this.flatDbMode = flatDbMode; this.flatDbReaderStrategy = flatDbReaderStrategy; this.composedWorldStateStorage = composedWorldStateStorage; this.trieLogStorage = trieLogStorage; this.metricsSystem = metricsSystem; + this.preImageProxy = preImageProxy; } public void loadFlatDbStrategy() { @@ -259,31 +274,42 @@ public Map streamFlatStorages( composedWorldStateStorage, accountHash, startKeyHash, endKeyHash, max); } - /** - * Provide a function to map hash values to an Optional wrapping their corresponding preImage - * or empty if the preImage is not available. - * - * @param hashPreImageMapper preImage mapper function - */ - public void setPreImageMapper(final Function> hashPreImageMapper) { - this.hashPreImageMapper = hashPreImageMapper; - } - - public NavigableMap storageEntriesFrom(final Hash addressHash, - final Bytes32 startKeyHash, final int limit) { - return streamFlatStorages(addressHash, startKeyHash, BYTES32_MAX_VALUE, limit) - .entrySet() - .stream() - .collect(Collectors.toMap( + public NavigableMap storageEntriesFrom( + final Hash addressHash, final Bytes32 startKeyHash, final int limit) { + return streamFlatStorages(addressHash, startKeyHash, BYTES32_MAX_VALUE, limit) + .entrySet() + // map back to slot keys using preImage provider: + .stream() + .collect( + Collectors.toMap( e -> e.getKey(), - e -> AccountStorageEntry.create( - UInt256.fromBytes(e.getValue()), - Hash.wrap(e.getKey()), - hashPreImageMapper.apply(Hash.wrap(e.getKey())) - .map(UInt256::fromBytes)), - (a,b) -> a, - TreeMap::new - )); + e -> + AccountStorageEntry.create( + UInt256.fromBytes(e.getValue()), + Hash.wrap(e.getKey()), + preImageProxy.getStorageTrieKeyPreimage(e.getKey())), + (a, b) -> a, + TreeMap::new)); + } + + public Stream streamAccounts( + final BonsaiWorldView context, final Bytes32 startKeyHash, final int limit) { + return streamFlatAccounts(startKeyHash, BYTES32_MAX_VALUE, Long.MAX_VALUE) + .entrySet() + // map back to addresses using preImage provider: + .stream() + .map( + entry -> + preImageProxy + .getAccountTrieKeyPreimage(entry.getKey()) + .map( + address -> + new WorldState.StreamableAccount( + Optional.of(address), + BonsaiAccount.fromRLP(context, address, entry.getValue(), false)))) + .filter(Optional::isPresent) + .map(Optional::get) + .sorted(Comparator.comparing(account -> account.getAddress().orElse(Address.ZERO))); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java index 1e46312287c..95cef8f56b8 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java @@ -30,16 +30,17 @@ public BonsaiWorldStateLayerStorage(final BonsaiWorldStateKeyValueStorage parent new LayeredKeyValueStorage(parent.composedWorldStateStorage), parent.trieLogStorage, parent, - parent.metricsSystem); + parent.metricsSystem, + parent.preImageProxy); } public BonsaiWorldStateLayerStorage( final SnappedKeyValueStorage composedWorldStateStorage, final KeyValueStorage trieLogStorage, final BonsaiWorldStateKeyValueStorage parent, - final ObservableMetricsSystem metricsSystem) { - super(parent, composedWorldStateStorage, trieLogStorage, metricsSystem); - setPreImageMapper(parent.hashPreImageMapper); + final ObservableMetricsSystem metricsSystem, + final BonsaiPreImageProxy preImageProxy) { + super(parent, composedWorldStateStorage, trieLogStorage, metricsSystem, preImageProxy); } @Override @@ -53,6 +54,7 @@ public BonsaiWorldStateLayerStorage clone() { ((LayeredKeyValueStorage) composedWorldStateStorage).clone(), trieLogStorage, parentWorldStateStorage, - metricsSystem); + metricsSystem, + preImageProxy); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java index 878b82b799a..855366e9f68 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java @@ -16,9 +16,11 @@ package org.hyperledger.besu.ethereum.bonsai.worldview; -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.units.bigints.UInt256; +import static org.hyperledger.besu.ethereum.bonsai.BonsaiAccount.fromRLP; +import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; +import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; + import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; @@ -26,6 +28,7 @@ import org.hyperledger.besu.ethereum.bonsai.BonsaiValue; import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiPreImageProxy; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; @@ -45,20 +48,19 @@ import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import javax.annotation.Nonnull; import java.util.Map; import java.util.Optional; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Stream; +import javax.annotation.Nonnull; -import static org.hyperledger.besu.ethereum.bonsai.BonsaiAccount.fromRLP; -import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; -import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; -import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class BonsaiWorldState implements MutableWorldState, BonsaiWorldView, BonsaiStorageSubscriber { @@ -73,20 +75,24 @@ public class BonsaiWorldState private Hash worldStateRootHash; Hash worldStateBlockHash; - + public final BonsaiPreImageProxy preImageProxy; private boolean isFrozen; public BonsaiWorldState( final BonsaiWorldStateProvider archive, final BonsaiWorldStateKeyValueStorage worldStateStorage) { - this(worldStateStorage, archive.getCachedMerkleTrieLoader(), archive.getTrieLogManager(), Hash::hash); + this( + worldStateStorage, + archive.getCachedMerkleTrieLoader(), + archive.getTrieLogManager(), + new BonsaiPreImageProxy.NoOpPreImageProxy()); } protected BonsaiWorldState( final BonsaiWorldStateKeyValueStorage worldStateStorage, final CachedMerkleTrieLoader cachedMerkleTrieLoader, final TrieLogManager trieLogManager, - final Function preImageHasher) { + final BonsaiPreImageProxy preImageProxy) { this.worldStateStorage = worldStateStorage; this.worldStateRootHash = Hash.wrap( @@ -101,9 +107,10 @@ protected BonsaiWorldState( getWorldStateStorage(), worldStateRootHash, addr), (addr, value) -> cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value), - preImageHasher); + preImageProxy); this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; this.trieLogManager = trieLogManager; + this.preImageProxy = preImageProxy; } /** diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java index 407ff996ba1..24c9020d927 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java @@ -23,6 +23,7 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount; import org.hyperledger.besu.ethereum.bonsai.BonsaiValue; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiPreImageProxy; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; @@ -66,7 +67,7 @@ public class BonsaiWorldStateUpdateAccumulator private final Map> codeToUpdate = new ConcurrentHashMap<>(); private final Set
storageToClear = Collections.synchronizedSet(new HashSet<>()); - private final Function preImageHasher; + private final BonsaiPreImageProxy preImageProxy; // storage sub mapped by _hashed_ key. This is because in self_destruct calls we need to // enumerate the old storage and delete it. Those are trie stored by hashed key by spec and the @@ -77,30 +78,22 @@ public class BonsaiWorldStateUpdateAccumulator private boolean isAccumulatorStateChanged; public BonsaiWorldStateUpdateAccumulator( - final BonsaiWorldView world, - final Consumer> accountPreloader, - final Consumer storagePreloader) { - // create accumulator without a preImage-aware Hash function: - this(world, accountPreloader, storagePreloader, Hash::hash); - } - - protected BonsaiWorldStateUpdateAccumulator( final BonsaiWorldView world, final Consumer> accountPreloader, final Consumer storagePreloader, - final Function preImageHasher) { - super(world); - this.accountsToUpdate = new AccountConsumingMap<>(new ConcurrentHashMap<>(), accountPreloader); - this.accountPreloader = accountPreloader; - this.storagePreloader = storagePreloader; - this.isAccumulatorStateChanged = false; - this.preImageHasher = preImageHasher; + final BonsaiPreImageProxy preImageProxy) { + super(world); + this.accountsToUpdate = new AccountConsumingMap<>(new ConcurrentHashMap<>(), accountPreloader); + this.accountPreloader = accountPreloader; + this.storagePreloader = storagePreloader; + this.isAccumulatorStateChanged = false; + this.preImageProxy = preImageProxy; } public BonsaiWorldStateUpdateAccumulator copy() { final BonsaiWorldStateUpdateAccumulator copy = new BonsaiWorldStateUpdateAccumulator( - wrappedWorldView(), accountPreloader, storagePreloader); + wrappedWorldView(), accountPreloader, storagePreloader, preImageProxy); copy.cloneFromUpdater(this); return copy; } @@ -139,18 +132,18 @@ public MutableAccount createAccount(final Address address, final long nonce, fin bonsaiValue = new BonsaiValue<>(null, null); accountsToUpdate.put(address, bonsaiValue); } else if (bonsaiValue.getUpdated() != null) { - if (bonsaiValue.getUpdated().isEmpty()) { - return new WrappedEvmAccount(track(new UpdateTrackingAccount<>(bonsaiValue.getUpdated()))); - } else { - throw new IllegalStateException("Cannot create an account when one already exists"); - } + if (bonsaiValue.getUpdated().isEmpty()) { + return new WrappedEvmAccount(track(new UpdateTrackingAccount<>(bonsaiValue.getUpdated()))); + } else { + throw new IllegalStateException("Cannot create an account when one already exists"); + } } final BonsaiAccount newAccount = new BonsaiAccount( this, address, - preImageHasher.apply(address), + preImageProxy.hashAndSavePreImage(address), nonce, balance, Hash.EMPTY_TRIE_HASH, @@ -371,7 +364,7 @@ public void commit() { entries.forEach( storageUpdate -> { final UInt256 keyUInt = storageUpdate.getKey(); - final Hash slotHash = preImageHasher.apply(keyUInt); + final Hash slotHash = preImageProxy.hashAndSavePreImage(keyUInt); final StorageSlotKey slotKey = new StorageSlotKey(slotHash, Optional.of(keyUInt)); final UInt256 value = storageUpdate.getValue(); @@ -415,7 +408,8 @@ public Optional getCode(final Address address, final Hash codeHash) { @Override public UInt256 getStorageValue(final Address address, final UInt256 slotKey) { - StorageSlotKey storageSlotKey = new StorageSlotKey(preImageHasher.apply(slotKey), Optional.of(slotKey)); + StorageSlotKey storageSlotKey = + new StorageSlotKey(preImageProxy.hashAndSavePreImage(slotKey), Optional.of(slotKey)); return getStorageValueByStorageSlotKey(address, storageSlotKey).orElse(UInt256.ZERO); } @@ -459,7 +453,7 @@ public Optional getStorageValueByStorageSlotKey( public UInt256 getPriorStorageValue(final Address address, final UInt256 storageKey) { // TODO maybe log the read into the trie layer? StorageSlotKey storageSlotKey = - new StorageSlotKey(preImageHasher.apply(storageKey), Optional.of(storageKey)); + new StorageSlotKey(preImageProxy.hashAndSavePreImage(storageKey), Optional.of(storageKey)); final Map> localAccountStorage = storageToUpdate.get(address); if (localAccountStorage != null) { diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java index 35ce1b77c06..d54f3aca00c 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java @@ -21,6 +21,7 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.bonsai.cache.CachedWorldStorageManager; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiPreImageProxy; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; @@ -51,7 +52,12 @@ public class TrieLogManagerTests { @Mock BonsaiWorldStateProvider archive; @Mock Blockchain blockchain; BonsaiWorldStateUpdateAccumulator bonsaiUpdater = - spy(new BonsaiWorldStateUpdateAccumulator(worldState, (__, ___) -> {}, (__, ___) -> {})); + spy( + new BonsaiWorldStateUpdateAccumulator( + worldState, + (__, ___) -> {}, + (__, ___) -> {}, + new BonsaiPreImageProxy.NoOpPreImageProxy())); TrieLogManager trieLogManager; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java index 77308ebcc3b..e38a9647b79 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/ForwardSyncStepTest.java @@ -38,7 +38,7 @@ import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.referencetests.ReferenceTestWorldState; +import org.hyperledger.besu.ethereum.referencetests.DefaultReferenceTestWorldState; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; import java.nio.charset.StandardCharsets; @@ -138,7 +138,8 @@ public void setup() { return new BlockProcessingResult( Optional.of( new BlockProcessingOutputs( - new ReferenceTestWorldState(), blockDataGenerator.receipts(block)))); + DefaultReferenceTestWorldState.create(Collections.emptyMap()), + blockDataGenerator.receipts(block)))); }); } diff --git a/ethereum/referencetests/build.gradle b/ethereum/referencetests/build.gradle index 5692c415f02..db30e8b940a 100644 --- a/ethereum/referencetests/build.gradle +++ b/ethereum/referencetests/build.gradle @@ -148,6 +148,7 @@ dependencies { implementation project(':datatypes') implementation project(':ethereum:core') implementation project(':metrics:core') + implementation project(':util') implementation project(path: ':ethereum:core', configuration: 'testSupportArtifacts') implementation project(':ethereum:rlp') implementation project(':evm') diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java index 9fcacb77880..e5abca00ef7 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -14,12 +14,10 @@ */ package org.hyperledger.besu.ethereum.referencetests; -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.bytes.Bytes32; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount; import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiPreImageProxy; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateLayerStorage; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogAddedEvent; @@ -29,7 +27,6 @@ import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; -import org.hyperledger.besu.evm.account.AccountStorageEntry; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -38,44 +35,30 @@ import org.hyperledger.besu.plugin.services.trielogs.TrieLogFactory; import org.hyperledger.besu.util.Subscribers; -import java.util.Comparator; -import java.util.HashMap; import java.util.Map; -import java.util.NavigableMap; import java.util.Optional; -import java.util.TreeMap; import java.util.function.Function; import java.util.stream.Stream; import com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.tuweni.bytes.Bytes32; public class BonsaiReferenceTestWorldState extends BonsaiWorldState implements ReferenceTestWorldState { - static final Map preImageCache = new HashMap<>(); - protected BonsaiReferenceTestWorldState( final BonsaiWorldStateKeyValueStorage worldStateStorage, final CachedMerkleTrieLoader cachedMerkleTrieLoader, - final TrieLogManager trieLogManager) { - super(worldStateStorage, cachedMerkleTrieLoader, trieLogManager, BonsaiReferenceTestWorldState::preImageHasher); - worldStateStorage.setPreImageMapper(hash -> Optional.ofNullable(preImageCache.get(hash))); - } - - private static Hash preImageHasher(final Bytes key) { - Hash hash = Hash.hash(key); - preImageCache.putIfAbsent(hash, key); - return hash; + final TrieLogManager trieLogManager, + final BonsaiPreImageProxy preImageProxy) { + super(worldStateStorage, cachedMerkleTrieLoader, trieLogManager, preImageProxy); } @Override public ReferenceTestWorldState copy() { var layerCopy = new BonsaiWorldStateLayerStorage(worldStateStorage); - layerCopy.setPreImageMapper(hash -> Optional.ofNullable(preImageCache.get(hash))); return new BonsaiReferenceTestWorldState( - layerCopy, - cachedMerkleTrieLoader, - trieLogManager); + layerCopy, cachedMerkleTrieLoader, trieLogManager, preImageProxy); } @JsonCreator @@ -84,11 +67,14 @@ public static BonsaiReferenceTestWorldState create( final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem(); final CachedMerkleTrieLoader cachedMerkleTrieLoader = new CachedMerkleTrieLoader(metricsSystem); final TrieLogManager trieLogManager = new NoOpTrieLogManager(); + final BonsaiPreImageProxy preImageProxy = + new BonsaiPreImageProxy.BonsaiReferenceTestPreImageProxy(); final BonsaiWorldStateKeyValueStorage worldStateStorage = - new BonsaiWorldStateKeyValueStorage(new InMemoryKeyValueStorageProvider(), metricsSystem); + new BonsaiWorldStateKeyValueStorage( + new InMemoryKeyValueStorageProvider(), metricsSystem, preImageProxy); final BonsaiReferenceTestWorldState worldState = new BonsaiReferenceTestWorldState( - worldStateStorage, cachedMerkleTrieLoader, trieLogManager); + worldStateStorage, cachedMerkleTrieLoader, trieLogManager, preImageProxy); final WorldUpdater updater = worldState.updater(); for (final Map.Entry entry : accounts.entrySet()) { @@ -100,20 +86,8 @@ public static BonsaiReferenceTestWorldState create( } @Override - public Stream streamAccounts(final Bytes32 startKeyHash, final int limit){ - return worldStateStorage - .streamFlatAccounts( - //TODO: use a constant for the end range - startKeyHash, Bytes32.fromHexStringLenient("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), Long.MAX_VALUE) - .entrySet() - // map to addresses: - .stream() - .map(entry -> { - Address address = Address.wrap(preImageCache.get(entry.getKey())); - return new StreamableAccount(Optional.of(address), - BonsaiAccount.fromRLP(this, address, entry.getValue(), false)); - }) - .sorted(Comparator.comparing(account -> account.getAddress().orElse(Address.ZERO))); + public Stream streamAccounts(final Bytes32 startKeyHash, final int limit) { + return worldStateStorage.streamAccounts(this, startKeyHash, limit); } static class NoOpTrieLogManager implements TrieLogManager { From e689332208c4132340499e6295af90d0546218f5 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Thu, 24 Aug 2023 16:39:32 -0700 Subject: [PATCH 05/13] a few fixes for preImageProxy usage, account and storage streaming Signed-off-by: garyschulte --- .../bonsai/storage/BonsaiWorldStateKeyValueStorage.java | 3 ++- .../ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java | 2 +- .../besu/ethereum/bonsai/worldview/BonsaiWorldState.java | 7 ++++--- .../worldview/BonsaiWorldStateUpdateAccumulator.java | 3 ++- .../org/hyperledger/besu/evmtool/StateTestSubCommand.java | 1 + .../ethereum/referencetests/ReferenceTestWorldState.java | 1 + .../besu/services/kvstore/LayeredKeyValueStorage.java | 4 ++-- .../services/kvstore/SegmentedKeyValueStorageAdapter.java | 5 +++-- 8 files changed, 16 insertions(+), 10 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java index a251d7d7b56..0f4691ea136 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java @@ -58,6 +58,7 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.rlp.RLP; import org.apache.tuweni.units.bigints.UInt256; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -285,7 +286,7 @@ public NavigableMap storageEntriesFrom( e -> e.getKey(), e -> AccountStorageEntry.create( - UInt256.fromBytes(e.getValue()), + UInt256.fromBytes(RLP.decodeValue(e.getValue())), Hash.wrap(e.getKey()), preImageProxy.getStorageTrieKeyPreimage(e.getKey())), (a, b) -> a, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java index 1a35dbb93fe..55dd13622b0 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java @@ -158,7 +158,7 @@ public Map streamStorageFlatDatabase( .streamFromKey( ACCOUNT_STORAGE_STORAGE, Bytes.concatenate(accountHash, startKeyHash).toArrayUnsafe()) - .takeWhile(pair -> Bytes.wrap(pair.getKey()).slice(0, Hash.SIZE).equals(accountHash)) + .filter(pair -> Bytes.wrap(pair.getKey()).slice(0, Hash.SIZE).equals(accountHash)) .limit(max) .map( pair -> diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java index 855366e9f68..cf60c96d2f8 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java @@ -67,7 +67,7 @@ public class BonsaiWorldState private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldState.class); - private BonsaiWorldStateKeyValueStorage worldStateStorage; + protected BonsaiWorldStateKeyValueStorage worldStateStorage; protected final CachedMerkleTrieLoader cachedMerkleTrieLoader; protected final TrieLogManager trieLogManager; @@ -218,7 +218,7 @@ private void updateTheAccounts( final BonsaiAccount updatedAccount = bonsaiValue.getUpdated(); try { if (updatedAccount == null) { - final Hash addressHash = Hash.hash(accountKey); + final Hash addressHash = preImageProxy.hashAndSavePreImage(accountKey); accountTrie.remove(addressHash); maybeStateUpdater.ifPresent( bonsaiUpdater -> bonsaiUpdater.removeAccountInfoState(addressHash)); @@ -227,7 +227,8 @@ private void updateTheAccounts( final Bytes accountValue = updatedAccount.serializeAccount(); maybeStateUpdater.ifPresent( bonsaiUpdater -> - bonsaiUpdater.putAccountInfoState(Hash.hash(accountKey), accountValue)); + bonsaiUpdater.putAccountInfoState( + preImageProxy.hashAndSavePreImage(accountKey), accountValue)); accountTrie.put(addressHash, accountValue); } } catch (MerkleTrieException e) { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java index 24c9020d927..9eff3797591 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java @@ -338,7 +338,8 @@ public void commit() { pendingCode.setUpdated(updatedAccount.getCode()); } - // This is especially to avoid unnecessary computation for withdrawals + // This is especially to avoid unnecessary computation for withdrawals and + // self-destruct beneficiaries if (updatedAccount.getUpdatedStorage().isEmpty()) { return; } diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java index 152259abe21..82f41c6558d 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java @@ -262,6 +262,7 @@ private void traceTestSpecs(final String test, final List accounts) { // delegate to a Bonsai reference test world state: return BonsaiReferenceTestWorldState.create(accounts); + // return DefaultReferenceTestWorldState.create(accounts); } } diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java index 9a028becdf8..06256e3c177 100644 --- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java @@ -121,9 +121,9 @@ public Stream> stream(final SegmentIdentifier segmentId) { @Override public Stream> streamFromKey( - final SegmentIdentifier segmentId, final byte[] startKey) { + final SegmentIdentifier segmentId, final byte[] startKeyHash) { return stream(segmentId) - .filter(e -> Bytes.wrap(startKey).compareTo(Bytes.wrap(e.getKey())) <= 0); + .filter(e -> Bytes.wrap(startKeyHash).compareTo(Bytes.wrap(e.getKey())) <= 0); } @Override diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorageAdapter.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorageAdapter.java index 81a492624f2..fd4dffce8f1 100644 --- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorageAdapter.java +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorageAdapter.java @@ -88,8 +88,9 @@ public Stream> stream() { } @Override - public Stream> streamFromKey(final byte[] startKey) throws StorageException { - return storage.streamFromKey(segmentIdentifier, startKey); + public Stream> streamFromKey(final byte[] startKeyHash) + throws StorageException { + return storage.streamFromKey(segmentIdentifier, startKeyHash); } @Override From 6659509603137589003473f030138cfe190f96ae Mon Sep 17 00:00:00 2001 From: garyschulte Date: Fri, 25 Aug 2023 14:03:35 -0700 Subject: [PATCH 06/13] reference tests fix and a little cleanup Signed-off-by: garyschulte --- .../hyperledger/besu/evmtool/StateTestSubCommand.java | 2 +- .../besu/evmtool/state-test/blockhash.json | 4 ++-- .../referencetests/GeneralStateTestCaseSpec.java | 11 ++++++++++- .../ethereum/vm/GeneralStateReferenceTestTools.java | 3 +++ 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java index 82f41c6558d..2bc5f815309 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java @@ -262,7 +262,7 @@ private void traceTestSpecs(final String test, final List> finalStateSpecs; + private static final BlockHeaderFunctions MAINNET_FUNCTIONS = new MainnetBlockHeaderFunctions(); @JsonCreator public GeneralStateTestCaseSpec( @@ -49,7 +53,7 @@ public GeneralStateTestCaseSpec( } private Map> generate( - final BlockHeader blockHeader, + final BlockHeader rawBlockHeader, final ReferenceTestWorldState initialWorldState, final Map> postSections, final StateTestVersionedTransaction versionedTransaction) { @@ -62,6 +66,11 @@ private Map> generate( final List post = entry.getValue(); final List specs = new ArrayList<>(post.size()); for (final PostSection p : post) { + final BlockHeader blockHeader = + BlockHeaderBuilder.fromHeader(rawBlockHeader) + .stateRoot(p.rootHash) + .blockHeaderFunctions(MAINNET_FUNCTIONS) + .buildBlockHeader(); final Supplier txSupplier = () -> versionedTransaction.get(p.indexes); specs.add( new GeneralStateTestCaseEipSpec( diff --git a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java index 471adda77da..83388fe2526 100644 --- a/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java +++ b/ethereum/referencetests/src/reference-test/java/org/hyperledger/besu/ethereum/vm/GeneralStateReferenceTestTools.java @@ -21,8 +21,10 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderBuilder; +import org.hyperledger.besu.ethereum.core.BlockHeaderFunctions; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.mainnet.TransactionValidationParams; @@ -179,6 +181,7 @@ public static void executeTest(final GeneralStateTestCaseEipSpec spec) { worldStateUpdater.deleteAccount(coinbase.getAddress()); } worldStateUpdater.commit(); + worldState.persist(blockHeader); // Check the world state root hash. final Hash expectedRootHash = spec.getExpectedRootHash(); From ee7c22ca8b6f591e378b5cec34f2289b1711490e Mon Sep 17 00:00:00 2001 From: Karim TAAM Date: Tue, 29 Aug 2023 16:54:54 +0200 Subject: [PATCH 07/13] fix tests collisions Signed-off-by: Karim TAAM --- .../BonsaiWorldStateUpdateAccumulator.java | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java index 9eff3797591..2ea0957a37e 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java @@ -297,6 +297,19 @@ public void commit() { final BonsaiAccount updatedAccount; final BonsaiValue updatedAccountValue = accountsToUpdate.get(updatedAddress); + + final Map> pendingStorageUpdates = + storageToUpdate.computeIfAbsent( + updatedAddress, + k -> + new StorageConsumingMap<>( + updatedAddress, new ConcurrentHashMap<>(), storagePreloader)); + + if (tracked.getStorageWasCleared()) { + storageToClear.add(updatedAddress); + pendingStorageUpdates.clear(); + } + if (tracked.getWrappedAccount() == null) { updatedAccount = new BonsaiAccount(this, tracked); tracked.setWrappedAccount(updatedAccount); @@ -316,6 +329,17 @@ public void commit() { } if (tracked.getStorageWasCleared()) { updatedAccount.clearStorage(); + wrappedWorldView() + .getAllAccountStorage(updatedAddress, updatedAccount.getStorageRoot()) + .forEach( + (keyHash, entryValue) -> { + final StorageSlotKey storageSlotKey = + new StorageSlotKey(Hash.wrap(keyHash), Optional.empty()); + final UInt256 value = UInt256.fromBytes(RLP.decodeOne(entryValue)); + pendingStorageUpdates.put( + storageSlotKey, new BonsaiValue<>(value, null, true)); + }); + updatedAccount.setStorageRoot(Hash.EMPTY_TRIE_HASH); } tracked.getUpdatedStorage().forEach(updatedAccount::setStorageValue); } @@ -344,19 +368,6 @@ public void commit() { return; } - final StorageConsumingMap> - pendingStorageUpdates = - storageToUpdate.computeIfAbsent( - updatedAddress, - __ -> - new StorageConsumingMap<>( - updatedAddress, new ConcurrentHashMap<>(), storagePreloader)); - - if (tracked.getStorageWasCleared()) { - storageToClear.add(updatedAddress); - pendingStorageUpdates.clear(); - } - final TreeSet> entries = new TreeSet<>(Map.Entry.comparingByKey()); entries.addAll(updatedAccount.getUpdatedStorage().entrySet()); From 30d21d279f26bd901f0d45132cd6c58e65d6ccb5 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Tue, 29 Aug 2023 09:41:38 -0700 Subject: [PATCH 08/13] pass limit into streamFlatAccounts, make ref test preimageproxy synchronized to avoid concurrency issues Signed-off-by: garyschulte --- .../bonsai/storage/BonsaiPreImageProxy.java | 2 +- .../storage/BonsaiWorldStateKeyValueStorage.java | 2 +- .../bonsai/storage/flat/FlatDbReaderStrategy.java | 15 +++++++-------- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiPreImageProxy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiPreImageProxy.java index a05d50f7601..42a86ce36b2 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiPreImageProxy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiPreImageProxy.java @@ -69,7 +69,7 @@ class BonsaiReferenceTestPreImageProxy extends NoOpPreImageProxy { BiMap preImageCache = HashBiMap.create(); @Override - public Hash hashAndSavePreImage(final Bytes value) { + public synchronized Hash hashAndSavePreImage(final Bytes value) { return preImageCache.inverse().computeIfAbsent(value, Hash::hash); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java index 0f4691ea136..1289a05f4a5 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java @@ -295,7 +295,7 @@ public NavigableMap storageEntriesFrom( public Stream streamAccounts( final BonsaiWorldView context, final Bytes32 startKeyHash, final int limit) { - return streamFlatAccounts(startKeyHash, BYTES32_MAX_VALUE, Long.MAX_VALUE) + return streamFlatAccounts(startKeyHash, BYTES32_MAX_VALUE, limit) .entrySet() // map back to addresses using preImage provider: .stream() diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java index 55dd13622b0..704877f832b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java @@ -15,10 +15,10 @@ */ package org.hyperledger.besu.ethereum.bonsai.storage.flat; -import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; -import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; -import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; - +import kotlin.Pair; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.rlp.RLP; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.trie.NodeLoader; @@ -34,10 +34,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import kotlin.Pair; -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.rlp.RLP; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; /** * This class represents a FlatDbReaderStrategy, which is responsible for reading data from flat From 9e6f18bd918ba2e6f191d60054c1a12bc4756c5d Mon Sep 17 00:00:00 2001 From: garyschulte Date: Tue, 29 Aug 2023 09:53:58 -0700 Subject: [PATCH 09/13] spotless vs intellij imports Signed-off-by: garyschulte --- .../bonsai/storage/flat/FlatDbReaderStrategy.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java index 704877f832b..55dd13622b0 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java @@ -15,10 +15,10 @@ */ package org.hyperledger.besu.ethereum.bonsai.storage.flat; -import kotlin.Pair; -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.rlp.RLP; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; + import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.trie.NodeLoader; @@ -34,9 +34,10 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; -import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; -import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; +import kotlin.Pair; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.rlp.RLP; /** * This class represents a FlatDbReaderStrategy, which is responsible for reading data from flat From 5c18c90079d5de834b1163573b3c62932a3fdfe9 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Tue, 29 Aug 2023 10:48:34 -0700 Subject: [PATCH 10/13] remove commented out Forest worldstate Signed-off-by: garyschulte --- .../besu/ethereum/referencetests/ReferenceTestWorldState.java | 1 - 1 file changed, 1 deletion(-) diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java index 95b7b2e4a47..825d7a2cd4b 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestWorldState.java @@ -93,6 +93,5 @@ static void insertAccount( static ReferenceTestWorldState create(final Map accounts) { // delegate to a Bonsai reference test world state: return BonsaiReferenceTestWorldState.create(accounts); - // return DefaultReferenceTestWorldState.create(accounts); } } From 3ae839852914beff9e2e0fc6deabb72c29e4b222 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Thu, 31 Aug 2023 18:40:20 -0700 Subject: [PATCH 11/13] implement feedback from Karim, isolate BonsaiPreImageProxy, add endKey version of streamFromKey Signed-off-by: garyschulte --- ...nsaiSnapshotWorldStateKeyValueStorage.java | 9 +- .../BonsaiWorldStateKeyValueStorage.java | 61 +------------ .../storage/BonsaiWorldStateLayerStorage.java | 11 +-- .../storage/flat/FlatDbReaderStrategy.java | 4 +- .../bonsai/worldview/BonsaiWorldState.java | 41 +++++---- .../BonsaiWorldStateUpdateAccumulator.java | 22 ++--- .../bonsai/trielog/TrieLogManagerTests.java | 8 +- ethereum/referencetests/build.gradle | 1 + .../BonsaiReferenceTestUpdateAccumulator.java | 44 ++++++++++ .../BonsaiReferenceTestWorldState.java | 40 +++++++-- .../BonsaiReferenceTestWorldStateStorage.java | 85 +++++++++++++++++++ plugin-api/build.gradle | 2 +- .../services/storage/KeyValueStorage.java | 11 +++ .../storage/SegmentedKeyValueStorage.java | 13 +++ .../RocksDBColumnarKeyValueSnapshot.java | 9 +- .../RocksDBColumnarKeyValueStorage.java | 11 +++ .../segmented/RocksDBSnapshotTransaction.java | 24 ++++++ .../kvstore/LayeredKeyValueStorage.java | 14 ++- .../LimitedInMemoryKeyValueStorage.java | 7 ++ .../SegmentedInMemoryKeyValueStorage.java | 13 ++- .../SegmentedKeyValueStorageAdapter.java | 5 ++ 21 files changed, 311 insertions(+), 124 deletions(-) create mode 100644 ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java create mode 100644 ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java index 2ebaeb38230..44c79c46b8d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java @@ -44,15 +44,13 @@ public BonsaiSnapshotWorldStateKeyValueStorage( final BonsaiWorldStateKeyValueStorage parentWorldStateStorage, final SnappedKeyValueStorage segmentedWorldStateStorage, final KeyValueStorage trieLogStorage, - final ObservableMetricsSystem metricsSystem, - final BonsaiPreImageProxy preImageProxy) { + final ObservableMetricsSystem metricsSystem) { super( parentWorldStateStorage.flatDbMode, parentWorldStateStorage.flatDbReaderStrategy, segmentedWorldStateStorage, trieLogStorage, - metricsSystem, - preImageProxy); + metricsSystem); this.parentWorldStateStorage = parentWorldStateStorage; this.subscribeParentId = parentWorldStateStorage.subscribe(this); } @@ -64,8 +62,7 @@ public BonsaiSnapshotWorldStateKeyValueStorage( worldStateStorage, ((SnappableKeyValueStorage) worldStateStorage.composedWorldStateStorage).takeSnapshot(), worldStateStorage.trieLogStorage, - metricsSystem, - worldStateStorage.preImageProxy); + metricsSystem); } private boolean isClosedGet() { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java index 1289a05f4a5..1201fffb3b8 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java @@ -19,14 +19,11 @@ import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; -import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount; import org.hyperledger.besu.ethereum.bonsai.storage.flat.FlatDbReaderStrategy; import org.hyperledger.besu.ethereum.bonsai.storage.flat.FullFlatDbReaderStrategy; import org.hyperledger.besu.ethereum.bonsai.storage.flat.PartialFlatDbReaderStrategy; -import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldView; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.trie.MerkleTrie; @@ -35,7 +32,6 @@ import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage; import org.hyperledger.besu.evm.account.AccountStorageEntry; -import org.hyperledger.besu.evm.worldstate.WorldState; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; @@ -44,29 +40,21 @@ import org.hyperledger.besu.util.Subscribers; import java.nio.charset.StandardCharsets; -import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.NavigableMap; import java.util.Optional; -import java.util.TreeMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Predicate; import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.rlp.RLP; -import org.apache.tuweni.units.bigints.UInt256; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @SuppressWarnings("unused") public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoCloseable { - Bytes32 BYTES32_MAX_VALUE = - Bytes32.fromHexString("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateKeyValueStorage.class); // 0x776f726c64526f6f74 @@ -92,17 +80,8 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC protected final Subscribers subscribers = Subscribers.create(); - final BonsaiPreImageProxy preImageProxy; - public BonsaiWorldStateKeyValueStorage( final StorageProvider provider, final ObservableMetricsSystem metricsSystem) { - this(provider, metricsSystem, new BonsaiPreImageProxy.NoOpPreImageProxy()); - } - - public BonsaiWorldStateKeyValueStorage( - final StorageProvider provider, - final ObservableMetricsSystem metricsSystem, - final BonsaiPreImageProxy preImageProxy) { this.composedWorldStateStorage = provider.getStorageBySegmentIdentifiers( List.of( @@ -110,7 +89,6 @@ public BonsaiWorldStateKeyValueStorage( this.trieLogStorage = provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); this.metricsSystem = metricsSystem; - this.preImageProxy = preImageProxy; loadFlatDbStrategy(); } @@ -119,14 +97,12 @@ public BonsaiWorldStateKeyValueStorage( final FlatDbReaderStrategy flatDbReaderStrategy, final SegmentedKeyValueStorage composedWorldStateStorage, final KeyValueStorage trieLogStorage, - final ObservableMetricsSystem metricsSystem, - final BonsaiPreImageProxy preImageProxy) { + final ObservableMetricsSystem metricsSystem) { this.flatDbMode = flatDbMode; this.flatDbReaderStrategy = flatDbReaderStrategy; this.composedWorldStateStorage = composedWorldStateStorage; this.trieLogStorage = trieLogStorage; this.metricsSystem = metricsSystem; - this.preImageProxy = preImageProxy; } public void loadFlatDbStrategy() { @@ -277,40 +253,7 @@ public Map streamFlatStorages( public NavigableMap storageEntriesFrom( final Hash addressHash, final Bytes32 startKeyHash, final int limit) { - return streamFlatStorages(addressHash, startKeyHash, BYTES32_MAX_VALUE, limit) - .entrySet() - // map back to slot keys using preImage provider: - .stream() - .collect( - Collectors.toMap( - e -> e.getKey(), - e -> - AccountStorageEntry.create( - UInt256.fromBytes(RLP.decodeValue(e.getValue())), - Hash.wrap(e.getKey()), - preImageProxy.getStorageTrieKeyPreimage(e.getKey())), - (a, b) -> a, - TreeMap::new)); - } - - public Stream streamAccounts( - final BonsaiWorldView context, final Bytes32 startKeyHash, final int limit) { - return streamFlatAccounts(startKeyHash, BYTES32_MAX_VALUE, limit) - .entrySet() - // map back to addresses using preImage provider: - .stream() - .map( - entry -> - preImageProxy - .getAccountTrieKeyPreimage(entry.getKey()) - .map( - address -> - new WorldState.StreamableAccount( - Optional.of(address), - BonsaiAccount.fromRLP(context, address, entry.getValue(), false)))) - .filter(Optional::isPresent) - .map(Optional::get) - .sorted(Comparator.comparing(account -> account.getAddress().orElse(Address.ZERO))); + throw new RuntimeException("Bonsai Tries does not currently support enumerating storage"); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java index 95cef8f56b8..aa96354789a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateLayerStorage.java @@ -30,17 +30,15 @@ public BonsaiWorldStateLayerStorage(final BonsaiWorldStateKeyValueStorage parent new LayeredKeyValueStorage(parent.composedWorldStateStorage), parent.trieLogStorage, parent, - parent.metricsSystem, - parent.preImageProxy); + parent.metricsSystem); } public BonsaiWorldStateLayerStorage( final SnappedKeyValueStorage composedWorldStateStorage, final KeyValueStorage trieLogStorage, final BonsaiWorldStateKeyValueStorage parent, - final ObservableMetricsSystem metricsSystem, - final BonsaiPreImageProxy preImageProxy) { - super(parent, composedWorldStateStorage, trieLogStorage, metricsSystem, preImageProxy); + final ObservableMetricsSystem metricsSystem) { + super(parent, composedWorldStateStorage, trieLogStorage, metricsSystem); } @Override @@ -54,7 +52,6 @@ public BonsaiWorldStateLayerStorage clone() { ((LayeredKeyValueStorage) composedWorldStateStorage).clone(), trieLogStorage, parentWorldStateStorage, - metricsSystem, - preImageProxy); + metricsSystem); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java index 55dd13622b0..edc1b1792fb 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java @@ -157,8 +157,8 @@ public Map streamStorageFlatDatabase( storage .streamFromKey( ACCOUNT_STORAGE_STORAGE, - Bytes.concatenate(accountHash, startKeyHash).toArrayUnsafe()) - .filter(pair -> Bytes.wrap(pair.getKey()).slice(0, Hash.SIZE).equals(accountHash)) + Bytes.concatenate(accountHash, startKeyHash).toArrayUnsafe(), + Bytes.concatenate(accountHash, endKeyHash).toArrayUnsafe()) .limit(max) .map( pair -> diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java index cf60c96d2f8..f28e24b0970 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java @@ -28,7 +28,6 @@ import org.hyperledger.besu.ethereum.bonsai.BonsaiValue; import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiPreImageProxy; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; @@ -71,46 +70,48 @@ public class BonsaiWorldState protected final CachedMerkleTrieLoader cachedMerkleTrieLoader; protected final TrieLogManager trieLogManager; - private final BonsaiWorldStateUpdateAccumulator accumulator; + private BonsaiWorldStateUpdateAccumulator accumulator; - private Hash worldStateRootHash; + protected Hash worldStateRootHash; Hash worldStateBlockHash; - public final BonsaiPreImageProxy preImageProxy; private boolean isFrozen; public BonsaiWorldState( final BonsaiWorldStateProvider archive, final BonsaiWorldStateKeyValueStorage worldStateStorage) { - this( - worldStateStorage, - archive.getCachedMerkleTrieLoader(), - archive.getTrieLogManager(), - new BonsaiPreImageProxy.NoOpPreImageProxy()); + this(worldStateStorage, archive.getCachedMerkleTrieLoader(), archive.getTrieLogManager()); } protected BonsaiWorldState( final BonsaiWorldStateKeyValueStorage worldStateStorage, final CachedMerkleTrieLoader cachedMerkleTrieLoader, - final TrieLogManager trieLogManager, - final BonsaiPreImageProxy preImageProxy) { + final TrieLogManager trieLogManager) { this.worldStateStorage = worldStateStorage; this.worldStateRootHash = Hash.wrap( Bytes32.wrap(worldStateStorage.getWorldStateRootHash().orElse(Hash.EMPTY_TRIE_HASH))); this.worldStateBlockHash = Hash.wrap(Bytes32.wrap(worldStateStorage.getWorldStateBlockHash().orElse(Hash.ZERO))); - accumulator = + this.accumulator = new BonsaiWorldStateUpdateAccumulator( this, (addr, value) -> cachedMerkleTrieLoader.preLoadAccount( getWorldStateStorage(), worldStateRootHash, addr), (addr, value) -> - cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value), - preImageProxy); + cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value)); this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; this.trieLogManager = trieLogManager; - this.preImageProxy = preImageProxy; + } + + /** + * Having a protected method to override the accumulator solves the chicken-egg problem of needing + * a worldstate reference (this) when construction the Accumulator. + * + * @param accumulator accumulator to use. + */ + protected void setAccumulator(final BonsaiWorldStateUpdateAccumulator accumulator) { + this.accumulator = accumulator; } /** @@ -218,7 +219,7 @@ private void updateTheAccounts( final BonsaiAccount updatedAccount = bonsaiValue.getUpdated(); try { if (updatedAccount == null) { - final Hash addressHash = preImageProxy.hashAndSavePreImage(accountKey); + final Hash addressHash = hashAndSavePreImage(accountKey); accountTrie.remove(addressHash); maybeStateUpdater.ifPresent( bonsaiUpdater -> bonsaiUpdater.removeAccountInfoState(addressHash)); @@ -227,8 +228,7 @@ private void updateTheAccounts( final Bytes accountValue = updatedAccount.serializeAccount(); maybeStateUpdater.ifPresent( bonsaiUpdater -> - bonsaiUpdater.putAccountInfoState( - preImageProxy.hashAndSavePreImage(accountKey), accountValue)); + bonsaiUpdater.putAccountInfoState(hashAndSavePreImage(accountKey), accountValue)); accountTrie.put(addressHash, accountValue); } } catch (MerkleTrieException e) { @@ -619,4 +619,9 @@ private void closeFrozenStorage() { // no op } } + + protected Hash hashAndSavePreImage(final Bytes value) { + // by default do not save has preImages + return Hash.hash(value); + } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java index 2ea0957a37e..f0348f54325 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java @@ -23,7 +23,6 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount; import org.hyperledger.besu.ethereum.bonsai.BonsaiValue; -import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiPreImageProxy; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; @@ -67,8 +66,6 @@ public class BonsaiWorldStateUpdateAccumulator private final Map> codeToUpdate = new ConcurrentHashMap<>(); private final Set
storageToClear = Collections.synchronizedSet(new HashSet<>()); - private final BonsaiPreImageProxy preImageProxy; - // storage sub mapped by _hashed_ key. This is because in self_destruct calls we need to // enumerate the old storage and delete it. Those are trie stored by hashed key by spec and the // alternative was to keep a giant pre-image cache of the entire trie. @@ -80,20 +77,18 @@ public class BonsaiWorldStateUpdateAccumulator public BonsaiWorldStateUpdateAccumulator( final BonsaiWorldView world, final Consumer> accountPreloader, - final Consumer storagePreloader, - final BonsaiPreImageProxy preImageProxy) { + final Consumer storagePreloader) { super(world); this.accountsToUpdate = new AccountConsumingMap<>(new ConcurrentHashMap<>(), accountPreloader); this.accountPreloader = accountPreloader; this.storagePreloader = storagePreloader; this.isAccumulatorStateChanged = false; - this.preImageProxy = preImageProxy; } public BonsaiWorldStateUpdateAccumulator copy() { final BonsaiWorldStateUpdateAccumulator copy = new BonsaiWorldStateUpdateAccumulator( - wrappedWorldView(), accountPreloader, storagePreloader, preImageProxy); + wrappedWorldView(), accountPreloader, storagePreloader); copy.cloneFromUpdater(this); return copy; } @@ -143,7 +138,7 @@ public MutableAccount createAccount(final Address address, final long nonce, fin new BonsaiAccount( this, address, - preImageProxy.hashAndSavePreImage(address), + hashAndSavePreImage(address), nonce, balance, Hash.EMPTY_TRIE_HASH, @@ -376,7 +371,7 @@ public void commit() { entries.forEach( storageUpdate -> { final UInt256 keyUInt = storageUpdate.getKey(); - final Hash slotHash = preImageProxy.hashAndSavePreImage(keyUInt); + final Hash slotHash = hashAndSavePreImage(keyUInt); final StorageSlotKey slotKey = new StorageSlotKey(slotHash, Optional.of(keyUInt)); final UInt256 value = storageUpdate.getValue(); @@ -421,7 +416,7 @@ public Optional getCode(final Address address, final Hash codeHash) { @Override public UInt256 getStorageValue(final Address address, final UInt256 slotKey) { StorageSlotKey storageSlotKey = - new StorageSlotKey(preImageProxy.hashAndSavePreImage(slotKey), Optional.of(slotKey)); + new StorageSlotKey(hashAndSavePreImage(slotKey), Optional.of(slotKey)); return getStorageValueByStorageSlotKey(address, storageSlotKey).orElse(UInt256.ZERO); } @@ -465,7 +460,7 @@ public Optional getStorageValueByStorageSlotKey( public UInt256 getPriorStorageValue(final Address address, final UInt256 storageKey) { // TODO maybe log the read into the trie layer? StorageSlotKey storageSlotKey = - new StorageSlotKey(preImageProxy.hashAndSavePreImage(storageKey), Optional.of(storageKey)); + new StorageSlotKey(hashAndSavePreImage(storageKey), Optional.of(storageKey)); final Map> localAccountStorage = storageToUpdate.get(address); if (localAccountStorage != null) { @@ -838,4 +833,9 @@ protected Map delegate() { public interface Consumer { void process(final Address address, T value); } + + protected Hash hashAndSavePreImage(final Bytes bytes) { + // by default do not save hash preImages + return Hash.hash(bytes); + } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java index d54f3aca00c..35ce1b77c06 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/trielog/TrieLogManagerTests.java @@ -21,7 +21,6 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.bonsai.cache.CachedWorldStorageManager; -import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiPreImageProxy; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; @@ -52,12 +51,7 @@ public class TrieLogManagerTests { @Mock BonsaiWorldStateProvider archive; @Mock Blockchain blockchain; BonsaiWorldStateUpdateAccumulator bonsaiUpdater = - spy( - new BonsaiWorldStateUpdateAccumulator( - worldState, - (__, ___) -> {}, - (__, ___) -> {}, - new BonsaiPreImageProxy.NoOpPreImageProxy())); + spy(new BonsaiWorldStateUpdateAccumulator(worldState, (__, ___) -> {}, (__, ___) -> {})); TrieLogManager trieLogManager; diff --git a/ethereum/referencetests/build.gradle b/ethereum/referencetests/build.gradle index db30e8b940a..cf1a6e1f8a8 100644 --- a/ethereum/referencetests/build.gradle +++ b/ethereum/referencetests/build.gradle @@ -154,6 +154,7 @@ dependencies { implementation project(':evm') implementation project(':services:kvstore') + implementation 'io.tmio:tuweni-rlp' implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.google.guava:guava' diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java new file mode 100644 index 00000000000..c8349b7f790 --- /dev/null +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java @@ -0,0 +1,44 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.referencetests; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.bonsai.BonsaiValue; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiPreImageProxy; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldView; + +import org.apache.tuweni.bytes.Bytes; + +public class BonsaiReferenceTestUpdateAccumulator extends BonsaiWorldStateUpdateAccumulator { + private final BonsaiPreImageProxy preImageProxy; + + public BonsaiReferenceTestUpdateAccumulator( + final BonsaiWorldView world, + final Consumer> accountPreloader, + final Consumer storagePreloader, + final BonsaiPreImageProxy preImageProxy) { + super(world, accountPreloader, storagePreloader); + this.preImageProxy = preImageProxy; + } + + @Override + protected Hash hashAndSavePreImage(final Bytes bytes) { + // by default do not save hash preImages + return preImageProxy.hashAndSavePreImage(bytes); + } +} diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java index e5abca00ef7..e40b3774a97 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -19,7 +19,6 @@ import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiPreImageProxy; import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateLayerStorage; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogAddedEvent; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogFactoryImpl; import org.hyperledger.besu.ethereum.bonsai.trielog.TrieLogManager; @@ -41,22 +40,37 @@ import java.util.stream.Stream; import com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; public class BonsaiReferenceTestWorldState extends BonsaiWorldState implements ReferenceTestWorldState { + private final BonsaiReferenceTestWorldStateStorage refTestStorage; + private final BonsaiPreImageProxy preImageProxy; + protected BonsaiReferenceTestWorldState( - final BonsaiWorldStateKeyValueStorage worldStateStorage, + final BonsaiReferenceTestWorldStateStorage worldStateStorage, final CachedMerkleTrieLoader cachedMerkleTrieLoader, final TrieLogManager trieLogManager, final BonsaiPreImageProxy preImageProxy) { - super(worldStateStorage, cachedMerkleTrieLoader, trieLogManager, preImageProxy); + super(worldStateStorage, cachedMerkleTrieLoader, trieLogManager); + this.refTestStorage = worldStateStorage; + this.preImageProxy = preImageProxy; + setAccumulator( + new BonsaiReferenceTestUpdateAccumulator( + this, + (addr, value) -> + cachedMerkleTrieLoader.preLoadAccount( + getWorldStateStorage(), worldStateRootHash, addr), + (addr, value) -> + cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value), + preImageProxy)); } @Override public ReferenceTestWorldState copy() { - var layerCopy = new BonsaiWorldStateLayerStorage(worldStateStorage); + var layerCopy = new BonsaiReferenceTestWorldStateStorage(worldStateStorage, preImageProxy); return new BonsaiReferenceTestWorldState( layerCopy, cachedMerkleTrieLoader, trieLogManager, preImageProxy); } @@ -69,9 +83,13 @@ public static BonsaiReferenceTestWorldState create( final TrieLogManager trieLogManager = new NoOpTrieLogManager(); final BonsaiPreImageProxy preImageProxy = new BonsaiPreImageProxy.BonsaiReferenceTestPreImageProxy(); - final BonsaiWorldStateKeyValueStorage worldStateStorage = - new BonsaiWorldStateKeyValueStorage( - new InMemoryKeyValueStorageProvider(), metricsSystem, preImageProxy); + + final BonsaiReferenceTestWorldStateStorage worldStateStorage = + new BonsaiReferenceTestWorldStateStorage( + new BonsaiWorldStateKeyValueStorage( + new InMemoryKeyValueStorageProvider(), metricsSystem), + preImageProxy); + final BonsaiReferenceTestWorldState worldState = new BonsaiReferenceTestWorldState( worldStateStorage, cachedMerkleTrieLoader, trieLogManager, preImageProxy); @@ -87,7 +105,7 @@ public static BonsaiReferenceTestWorldState create( @Override public Stream streamAccounts(final Bytes32 startKeyHash, final int limit) { - return worldStateStorage.streamAccounts(this, startKeyHash, limit); + return this.refTestStorage.streamAccounts(this, startKeyHash, limit); } static class NoOpTrieLogManager implements TrieLogManager { @@ -155,4 +173,10 @@ public synchronized void unsubscribe(final long id) { trieLogObservers.unsubscribe(id); } } + + @Override + protected Hash hashAndSavePreImage(final Bytes value) { + // by default do not save has preImages + return preImageProxy.hashAndSavePreImage(value); + } } diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java new file mode 100644 index 00000000000..b998bf7d25b --- /dev/null +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java @@ -0,0 +1,85 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.referencetests; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiPreImageProxy; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.bonsai.worldview.BonsaiWorldView; +import org.hyperledger.besu.evm.account.AccountStorageEntry; +import org.hyperledger.besu.evm.worldstate.WorldState; + +import java.util.Comparator; +import java.util.NavigableMap; +import java.util.Optional; +import java.util.TreeMap; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.rlp.RLP; +import org.apache.tuweni.units.bigints.UInt256; + +public class BonsaiReferenceTestWorldStateStorage extends BonsaiWorldStateLayerStorage { + private final BonsaiPreImageProxy preImageProxy; + + public BonsaiReferenceTestWorldStateStorage( + final BonsaiWorldStateKeyValueStorage parent, final BonsaiPreImageProxy preImageProxy) { + super(parent); + this.preImageProxy = preImageProxy; + } + + @Override + public NavigableMap storageEntriesFrom( + final Hash addressHash, final Bytes32 startKeyHash, final int limit) { + return streamFlatStorages(addressHash, startKeyHash, UInt256.MAX_VALUE, limit) + .entrySet() + // map back to slot keys using preImage provider: + .stream() + .collect( + Collectors.toMap( + e -> e.getKey(), + e -> + AccountStorageEntry.create( + UInt256.fromBytes(RLP.decodeValue(e.getValue())), + Hash.wrap(e.getKey()), + preImageProxy.getStorageTrieKeyPreimage(e.getKey())), + (a, b) -> a, + TreeMap::new)); + } + + public Stream streamAccounts( + final BonsaiWorldView context, final Bytes32 startKeyHash, final int limit) { + return streamFlatAccounts(startKeyHash, UInt256.MAX_VALUE, limit) + .entrySet() + // map back to addresses using preImage provider: + .stream() + .map( + entry -> + preImageProxy + .getAccountTrieKeyPreimage(entry.getKey()) + .map( + address -> + new WorldState.StreamableAccount( + Optional.of(address), + BonsaiAccount.fromRLP(context, address, entry.getValue(), false)))) + .filter(Optional::isPresent) + .map(Optional::get) + .sorted(Comparator.comparing(account -> account.getAddress().orElse(Address.ZERO))); + } +} diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index aedc706bab1..7aa85e8120c 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -69,7 +69,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = 'fPOd/MnNB1PwfTV7HDTXc3oYRcoeUzMJrlzDUdg/HNk=' + knownHash = 'A44Xsq60UEFaeGmScHw+MVOgqWvJanMmEAqDtiaDA8k=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/KeyValueStorage.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/KeyValueStorage.java index 6ccdb3b0a24..3d0e9e5efea 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/KeyValueStorage.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/KeyValueStorage.java @@ -82,6 +82,17 @@ public interface KeyValueStorage extends Closeable { */ Stream> streamFromKey(final byte[] startKey); + /** + * Returns a stream of key-value pairs starting from the specified key, ending at the specified + * key. This method is used to retrieve a stream of data from the storage, starting from the given + * key. If no data is available from the specified key onwards, an empty stream is returned. + * + * @param startKey The key from which the stream should start. + * @param endKey The key at which the stream should stop. + * @return A stream of key-value pairs starting from the specified key. + */ + Stream> streamFromKey(final byte[] startKey, final byte[] endKey); + /** * Returns a stream of all keys. * diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentedKeyValueStorage.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentedKeyValueStorage.java index 21e6ecca6f6..55d04d470c4 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentedKeyValueStorage.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentedKeyValueStorage.java @@ -78,6 +78,19 @@ default boolean containsKey(final SegmentIdentifier segment, final byte[] key) Stream> streamFromKey( final SegmentIdentifier segmentIdentifier, final byte[] startKey); + /** + * Returns a stream of key-value pairs starting from the specified key, ending at the specified + * key. This method is used to retrieve a stream of data from the storage, starting from the given + * key. If no data is available from the specified key onwards, an empty stream is returned. + * + * @param segmentIdentifier The segment identifier whose keys we want to stream. + * @param startKey The key from which the stream should start. + * @param endKey The key at which the stream should stop. + * @return A stream of key-value pairs starting from the specified key. + */ + Stream> streamFromKey( + final SegmentIdentifier segmentIdentifier, final byte[] startKey, final byte[] endKey); + /** * Stream keys. * diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueSnapshot.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueSnapshot.java index d51bdb310b8..6a123069001 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueSnapshot.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueSnapshot.java @@ -33,7 +33,6 @@ import java.util.stream.Stream; import org.apache.commons.lang3.tuple.Pair; -import org.apache.tuweni.bytes.Bytes; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.OptimisticTransactionDB; import org.slf4j.Logger; @@ -83,7 +82,13 @@ public Stream> stream(final SegmentIdentifier segment) { @Override public Stream> streamFromKey( final SegmentIdentifier segment, final byte[] startKey) { - return stream(segment).filter(e -> Bytes.wrap(startKey).compareTo(Bytes.wrap(e.getKey())) <= 0); + return snapTx.streamFromKey(segment, startKey); + } + + @Override + public Stream> streamFromKey( + final SegmentIdentifier segment, final byte[] startKey, final byte[] endKey) { + return snapTx.streamFromKey(segment, startKey, endKey); } @Override diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java index 3d98be37523..9462f9bcb14 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java @@ -333,6 +333,17 @@ public Stream> streamFromKey( return RocksDbIterator.create(rocksIterator).toStream(); } + @Override + public Stream> streamFromKey( + final SegmentIdentifier segmentIdentifier, final byte[] startKey, final byte[] endKey) { + final Bytes endKeyBytes = Bytes.wrap(endKey); + final RocksIterator rocksIterator = getDB().newIterator(safeColumnHandle(segmentIdentifier)); + rocksIterator.seek(startKey); + return RocksDbIterator.create(rocksIterator) + .toStream() + .takeWhile(e -> endKeyBytes.compareTo(Bytes.wrap(e.getKey())) >= 0); + } + @Override public Stream streamKeys(final SegmentIdentifier segmentIdentifier) { final RocksIterator rocksIterator = getDB().newIterator(safeColumnHandle(segmentIdentifier)); diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBSnapshotTransaction.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBSnapshotTransaction.java index c1c404628d9..5d058bbaddb 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBSnapshotTransaction.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBSnapshotTransaction.java @@ -28,6 +28,7 @@ import java.util.stream.Stream; import org.apache.commons.lang3.tuple.Pair; +import org.apache.tuweni.bytes.Bytes; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.OptimisticTransactionDB; import org.rocksdb.ReadOptions; @@ -166,6 +167,29 @@ public Stream streamKeys(final SegmentIdentifier segmentId) { return RocksDbIterator.create(rocksIterator).toStreamKeys(); } + public Stream> streamFromKey( + final SegmentIdentifier segment, final byte[] startKey) { + throwIfClosed(); + + final RocksIterator rocksIterator = + db.newIterator(columnFamilyMapper.apply(segment), readOptions); + rocksIterator.seek(startKey); + return RocksDbIterator.create(rocksIterator).toStream(); + } + + public Stream> streamFromKey( + final SegmentIdentifier segment, final byte[] startKey, final byte[] endKey) { + throwIfClosed(); + final Bytes endKeyBytes = Bytes.wrap(endKey); + + final RocksIterator rocksIterator = + db.newIterator(columnFamilyMapper.apply(segment), readOptions); + rocksIterator.seek(startKey); + return RocksDbIterator.create(rocksIterator) + .toStream() + .takeWhile(e -> endKeyBytes.compareTo(Bytes.wrap(e.getKey())) >= 0); + } + @Override public void commit() throws StorageException { // no-op diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java index 06256e3c177..cc1ed63ef42 100644 --- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java @@ -121,9 +121,19 @@ public Stream> stream(final SegmentIdentifier segmentId) { @Override public Stream> streamFromKey( - final SegmentIdentifier segmentId, final byte[] startKeyHash) { + final SegmentIdentifier segmentId, final byte[] startKey) { + final Bytes startKeyBytes = Bytes.wrap(startKey); + return stream(segmentId).filter(e -> startKeyBytes.compareTo(Bytes.wrap(e.getKey())) <= 0); + } + + @Override + public Stream> streamFromKey( + final SegmentIdentifier segmentId, final byte[] startKey, final byte[] endKey) { + final Bytes startKeyBytes = Bytes.wrap(startKey); + final Bytes endKeyBytes = Bytes.wrap(endKey); return stream(segmentId) - .filter(e -> Bytes.wrap(startKeyHash).compareTo(Bytes.wrap(e.getKey())) <= 0); + .filter(e -> startKeyBytes.compareTo(Bytes.wrap(e.getKey())) <= 0) + .takeWhile(e -> endKeyBytes.compareTo(Bytes.wrap(e.getKey())) >= 0); } @Override diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LimitedInMemoryKeyValueStorage.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LimitedInMemoryKeyValueStorage.java index 7ab04ec26e8..aa52e49284f 100644 --- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LimitedInMemoryKeyValueStorage.java +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LimitedInMemoryKeyValueStorage.java @@ -118,6 +118,13 @@ public Stream> streamFromKey(final byte[] startKey) { return stream().filter(e -> Bytes.wrap(startKey).compareTo(Bytes.wrap(e.getKey())) <= 0); } + @Override + public Stream> streamFromKey(final byte[] startKey, final byte[] endKey) { + return stream() + .filter(e -> Bytes.wrap(startKey).compareTo(Bytes.wrap(e.getKey())) <= 0) + .takeWhile(e -> Bytes.wrap(endKey).compareTo(Bytes.wrap(e.getKey())) >= 0); + } + @Override public Stream streamKeys() { final Lock lock = rwLock.readLock(); diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedInMemoryKeyValueStorage.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedInMemoryKeyValueStorage.java index adc9a4c71dc..153c86415cc 100644 --- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedInMemoryKeyValueStorage.java +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedInMemoryKeyValueStorage.java @@ -149,8 +149,19 @@ public Stream> stream(final SegmentIdentifier segmentIdenti @Override public Stream> streamFromKey( final SegmentIdentifier segmentIdentifier, final byte[] startKey) { + final Bytes startKeyBytes = Bytes.wrap(startKey); return stream(segmentIdentifier) - .filter(e -> Bytes.wrap(startKey).compareTo(Bytes.wrap(e.getKey())) <= 0); + .filter(e -> startKeyBytes.compareTo(Bytes.wrap(e.getKey())) <= 0); + } + + @Override + public Stream> streamFromKey( + final SegmentIdentifier segmentIdentifier, final byte[] startKey, final byte[] endKey) { + final Bytes startKeyHash = Bytes.wrap(startKey); + final Bytes endKeyHash = Bytes.wrap(endKey); + return stream(segmentIdentifier) + .filter(e -> startKeyHash.compareTo(Bytes.wrap(e.getKey())) <= 0) + .takeWhile(e -> endKeyHash.compareTo(Bytes.wrap(e.getKey())) >= 0); } @Override diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorageAdapter.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorageAdapter.java index fd4dffce8f1..a02e9284fa6 100644 --- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorageAdapter.java +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedKeyValueStorageAdapter.java @@ -93,6 +93,11 @@ public Stream> streamFromKey(final byte[] startKeyHash) return storage.streamFromKey(segmentIdentifier, startKeyHash); } + @Override + public Stream> streamFromKey(final byte[] startKey, final byte[] endKey) { + return storage.streamFromKey(segmentIdentifier, startKey, endKey); + } + @Override public Stream streamKeys() { throwIfClosed(); From 27d1e744fa1cdb15aa94df666b375fc1ce88cf3f Mon Sep 17 00:00:00 2001 From: garyschulte Date: Thu, 31 Aug 2023 18:53:02 -0700 Subject: [PATCH 12/13] filter inMemory key value storage rather than takeWhile to avoid sort Signed-off-by: garyschulte --- .../bonsai/storage/flat/FlatDbReaderStrategy.java | 9 ++++----- .../besu/services/kvstore/LayeredKeyValueStorage.java | 2 +- .../kvstore/SegmentedInMemoryKeyValueStorage.java | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java index edc1b1792fb..74c7b101c25 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/flat/FlatDbReaderStrategy.java @@ -135,10 +135,10 @@ public Map streamAccountFlatDatabase( final long max) { final Stream> pairStream = storage - .streamFromKey(ACCOUNT_INFO_STATE, startKeyHash.toArrayUnsafe()) + .streamFromKey( + ACCOUNT_INFO_STATE, startKeyHash.toArrayUnsafe(), endKeyHash.toArrayUnsafe()) .limit(max) - .map(pair -> new Pair<>(Bytes32.wrap(pair.getKey()), Bytes.wrap(pair.getValue()))) - .takeWhile(pair -> pair.getFirst().compareTo(endKeyHash) <= 0); + .map(pair -> new Pair<>(Bytes32.wrap(pair.getKey()), Bytes.wrap(pair.getValue()))); final TreeMap collected = pairStream.collect( @@ -164,8 +164,7 @@ public Map streamStorageFlatDatabase( pair -> new Pair<>( Bytes32.wrap(Bytes.wrap(pair.getKey()).slice(Hash.SIZE)), - RLP.encodeValue(Bytes.wrap(pair.getValue()).trimLeadingZeros()))) - .takeWhile(pair -> pair.getFirst().compareTo(endKeyHash) <= 0); + RLP.encodeValue(Bytes.wrap(pair.getValue()).trimLeadingZeros()))); final TreeMap collected = pairStream.collect( diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java index cc1ed63ef42..ffc57530682 100644 --- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/LayeredKeyValueStorage.java @@ -133,7 +133,7 @@ public Stream> streamFromKey( final Bytes endKeyBytes = Bytes.wrap(endKey); return stream(segmentId) .filter(e -> startKeyBytes.compareTo(Bytes.wrap(e.getKey())) <= 0) - .takeWhile(e -> endKeyBytes.compareTo(Bytes.wrap(e.getKey())) >= 0); + .filter(e -> endKeyBytes.compareTo(Bytes.wrap(e.getKey())) >= 0); } @Override diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedInMemoryKeyValueStorage.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedInMemoryKeyValueStorage.java index 153c86415cc..66d70bac4dc 100644 --- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedInMemoryKeyValueStorage.java +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/SegmentedInMemoryKeyValueStorage.java @@ -161,7 +161,7 @@ public Stream> streamFromKey( final Bytes endKeyHash = Bytes.wrap(endKey); return stream(segmentIdentifier) .filter(e -> startKeyHash.compareTo(Bytes.wrap(e.getKey())) <= 0) - .takeWhile(e -> endKeyHash.compareTo(Bytes.wrap(e.getKey())) >= 0); + .filter(e -> endKeyHash.compareTo(Bytes.wrap(e.getKey())) >= 0); } @Override From a4073185dc02bb6e6f0048fc9f8693ae2883adff Mon Sep 17 00:00:00 2001 From: garyschulte Date: Wed, 13 Sep 2023 17:13:11 -0700 Subject: [PATCH 13/13] rebase, javadoc and a bit of cleanup Signed-off-by: garyschulte --- .../bonsai/storage/BonsaiPreImageProxy.java | 32 ++++--------------- .../BonsaiWorldStateUpdateAccumulator.java | 2 +- plugin-api/build.gradle | 2 +- .../segmented/RocksDBSnapshotTransaction.java | 21 +++++++++++- 4 files changed, 29 insertions(+), 28 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiPreImageProxy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiPreImageProxy.java index 42a86ce36b2..248e5d895fd 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiPreImageProxy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiPreImageProxy.java @@ -37,35 +37,11 @@ public interface BonsaiPreImageProxy extends WorldStatePreimageStorage { */ Hash hashAndSavePreImage(Bytes value); - /** PreImageProxy which does not store or cache preImages and only implements hashing. */ - class NoOpPreImageProxy implements BonsaiPreImageProxy { - - @Override - public Hash hashAndSavePreImage(final Bytes value) { - return Hash.hash(value); - } - - @Override - public Optional getStorageTrieKeyPreimage(final Bytes32 trieKey) { - return Optional.empty(); - } - - @Override - public Optional
getAccountTrieKeyPreimage(final Bytes32 trieKey) { - return Optional.empty(); - } - - @Override - public Updater updater() { - throw new UnsupportedOperationException("NoOpPreImageProxy does not implement an updater"); - } - } - /** * A caching PreImageProxy suitable for ReferenceTestWorldState which saves hashes in an unbounded * BiMap. */ - class BonsaiReferenceTestPreImageProxy extends NoOpPreImageProxy { + class BonsaiReferenceTestPreImageProxy implements BonsaiPreImageProxy { BiMap preImageCache = HashBiMap.create(); @Override @@ -82,5 +58,11 @@ public Optional getStorageTrieKeyPreimage(final Bytes32 trieKey) { public Optional
getAccountTrieKeyPreimage(final Bytes32 trieKey) { return Optional.ofNullable(preImageCache.get(trieKey)).map(Address::wrap); } + + @Override + public Updater updater() { + throw new UnsupportedOperationException( + "BonsaiReferenceTestPreImageProxy does not implement an updater"); + } } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java index f0348f54325..23fd7625fa3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java @@ -128,7 +128,7 @@ public MutableAccount createAccount(final Address address, final long nonce, fin accountsToUpdate.put(address, bonsaiValue); } else if (bonsaiValue.getUpdated() != null) { if (bonsaiValue.getUpdated().isEmpty()) { - return new WrappedEvmAccount(track(new UpdateTrackingAccount<>(bonsaiValue.getUpdated()))); + return track(new UpdateTrackingAccount<>(bonsaiValue.getUpdated())); } else { throw new IllegalStateException("Cannot create an account when one already exists"); } diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index 7aa85e8120c..e544419b8ab 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -69,7 +69,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = 'A44Xsq60UEFaeGmScHw+MVOgqWvJanMmEAqDtiaDA8k=' + knownHash = 'tpSnjt4HgqSiOTJhBbYdB0r1nFX4QZbicjfloI71Wf0=' } check.dependsOn('checkAPIChanges') diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBSnapshotTransaction.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBSnapshotTransaction.java index 5d058bbaddb..0897493eb19 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBSnapshotTransaction.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBSnapshotTransaction.java @@ -166,7 +166,15 @@ public Stream streamKeys(final SegmentIdentifier segmentId) { rocksIterator.seekToFirst(); return RocksDbIterator.create(rocksIterator).toStreamKeys(); } - + /** + * Returns a stream of key-value pairs starting from the specified key. This method is used to + * retrieve a stream of data reading through the transaction, starting from the given key. If no + * data is available from the specified key onwards, an empty stream is returned. + * + * @param segment The segment identifier whose keys we want to stream. + * @param startKey The key from which the stream should start. + * @return A stream of key-value pairs starting from the specified key. + */ public Stream> streamFromKey( final SegmentIdentifier segment, final byte[] startKey) { throwIfClosed(); @@ -177,6 +185,17 @@ public Stream> streamFromKey( return RocksDbIterator.create(rocksIterator).toStream(); } + /** + * Returns a stream of key-value pairs starting from the specified key, ending at the specified + * key. This method is used to retrieve a stream of data reading through the transaction, starting + * from the given key. If no data is available from the specified key onwards, an empty stream is + * returned. + * + * @param segment The segment identifier whose keys we want to stream. + * @param startKey The key from which the stream should start. + * @param endKey The key at which the stream should stop. + * @return A stream of key-value pairs starting from the specified key. + */ public Stream> streamFromKey( final SegmentIdentifier segment, final byte[] startKey, final byte[] endKey) { throwIfClosed();