From f2c2512ef64a5c41e69e325c4d2f76837180cbbd Mon Sep 17 00:00:00 2001 From: Karim TAAM Date: Wed, 27 Mar 2024 11:17:40 +0100 Subject: [PATCH] storage format refactor for preparing verkle trie integration (#6721) Splitting Bonsai code: This commit aims to divide Bonsai into two packages - Common Classes with a Prefix: The common part will include classes prefixed with "DiffBased." These classes are designed to provide a base for both Bonsai and any future storage format that might use this diff-based approach. - Bonsai's Specifics: Bonsai will retain its unique features in its own package. This means that while it shares the diff-based infrastructure with the common part, it also has its own specific functionalities that are not shared with other storage format. - Extension to Verkle: this modification add the possibility of adding "Verkle" as a new storage format based on the diff-based architecture. Like Bonsai, Verkle would use the common diff-based classes but also have its own specific features. --------- Signed-off-by: Karim TAAM Signed-off-by: Karim Taam --- .../subcommands/storage/TrieLogHelper.java | 6 +- .../storage/TrieLogSubCommand.java | 4 +- .../besu/components/BesuComponent.java | 8 +- .../controller/BesuControllerBuilder.java | 23 +- .../storage/TrieLogHelperTest.java | 6 +- .../controller/BesuControllerBuilderTest.java | 8 +- .../internal/pojoadapter/AccountAdapter.java | 2 +- .../mainnet/AbstractBlockProcessor.java | 4 +- .../keyvalue/KeyValueStorageProvider.java | 2 +- .../common/GenesisWorldStateProvider.java | 18 +- .../{ => diffbased}/bonsai/BonsaiAccount.java | 185 ++------- .../bonsai/BonsaiWorldStateProvider.java | 145 +++++++ .../cache/BonsaiCachedMerkleTrieLoader.java} | 10 +- .../BonsaiCachedMerkleTrieLoaderModule.java} | 8 +- .../BonsaiCachedWorldStorageManager.java | 60 +++ .../NoOpBonsaiCachedWorldStorageManager.java} | 18 +- .../bonsai/storage/BonsaiPreImageProxy.java | 2 +- ...nsaiSnapshotWorldStateKeyValueStorage.java | 14 +- .../BonsaiWorldStateKeyValueStorage.java | 188 +-------- .../storage/BonsaiWorldStateLayerStorage.java | 11 +- .../storage/flat/FullFlatDbStrategy.java | 4 +- .../storage/flat/PartialFlatDbStrategy.java | 4 +- .../bonsai/trielog/TrieLogFactoryImpl.java | 19 +- .../bonsai/worldview/BonsaiWorldState.java | 361 ++++-------------- .../BonsaiWorldStateUpdateAccumulator.java | 98 +++++ .../diffbased/common/DiffBasedAccount.java | 206 ++++++++++ .../common/DiffBasedValue.java} | 25 +- .../common/DiffBasedWorldStateProvider.java} | 171 +++------ .../diffbased/common/StorageSubscriber.java | 25 ++ .../DiffBasedCachedWorldStorageManager.java} | 97 ++--- .../cache/DiffBasedCachedWorldView.java} | 22 +- ...BasedLayeredWorldStateKeyValueStorage.java | 22 ++ ...asedSnapshotWorldStateKeyValueStorage.java | 21 + .../DiffBasedWorldStateKeyValueStorage.java | 234 ++++++++++++ .../flat/AccountHashCodeStorageStrategy.java | 2 +- .../flat/CodeHashCodeStorageStrategy.java | 2 +- .../storage/flat/CodeStorageStrategy.java | 2 +- .../common}/storage/flat/FlatDbStrategy.java | 2 +- .../storage/flat/FlatDbStrategyProvider.java | 4 +- .../common}/trielog/NoOpTrieLogManager.java | 10 +- .../common}/trielog/TrieLogAddedEvent.java | 2 +- .../common}/trielog/TrieLogLayer.java | 50 +-- .../common}/trielog/TrieLogManager.java | 23 +- .../common}/trielog/TrieLogPruner.java | 12 +- .../common/worldview/DiffBasedWorldState.java | 334 ++++++++++++++++ .../common/worldview/DiffBasedWorldView.java} | 11 +- ...DiffBasedWorldStateUpdateAccumulator.java} | 339 ++++++++-------- .../preload/AccountConsumingMap.java | 49 +++ .../accumulator/preload/Consumer.java | 21 + .../preload/StorageConsumingMap.java | 53 +++ .../WorldStateStorageCoordinator.java | 2 +- .../core/InMemoryKeyValueStorageProvider.java | 12 +- .../besu/ethereum/core/TrieGenerator.java | 2 +- .../BlockImportExceptionHandlingTest.java | 6 +- .../bonsai/AbstractIsolationTests.java | 8 +- .../BonsaiCachedMerkleTrieLoaderTest.java} | 12 +- .../bonsai/BonsaiSnapshotIsolationTests.java | 2 +- .../bonsai/BonsaiWorldStateProviderTest.java | 36 +- .../bonsai/LogRollingTests.java | 12 +- .../{ => diffbased}/bonsai/RollingImport.java | 12 +- .../BonsaiWorldStateKeyValueStorageTest.java | 4 +- .../bonsai/trielog/TrieLogFactoryTests.java | 3 +- .../bonsai/trielog/TrieLogManagerTests.java | 9 +- .../bonsai/trielog/TrieLogPrunerTest.java | 7 +- .../worldview/BonsaiWorldStateTest.java | 30 +- .../flat/FlatDbStrategyProviderTest.java | 4 +- .../common}/trielog/TrieLogLayerTests.java | 2 +- ...tKeyValueStorageWorldStateStorageTest.java | 1 + .../eth/sync/DefaultSynchronizer.java | 2 +- .../eth/sync/fastsync/FastSyncDownloader.java | 2 +- .../eth/sync/snapsync/PersistDataStep.java | 2 +- .../sync/snapsync/SnapWorldDownloadState.java | 2 +- .../snapsync/SnapWorldStateDownloader.java | 2 +- .../request/AccountRangeDataRequest.java | 2 +- .../request/StorageRangeDataRequest.java | 2 +- ...ccountFlatDatabaseHealingRangeRequest.java | 2 +- ...torageFlatDatabaseHealingRangeRequest.java | 2 +- .../CheckPointSyncChainDownloaderTest.java | 2 +- .../fastsync/FastDownloaderFactoryTest.java | 2 +- .../sync/fastsync/FastSyncDownloaderTest.java | 2 +- .../FastWorldDownloadStateTest.java | 2 +- .../worldstate/LoadLocalDataStepTest.java | 2 +- .../snapsync/AccountHealingTrackingTest.java | 2 +- .../sync/snapsync/PersistDataStepTest.java | 2 +- .../snapsync/SnapWorldDownloadStateTest.java | 2 +- .../eth/sync/snapsync/TaskGenerator.java | 3 +- ...ntFlatDatabaseHealingRangeRequestTest.java | 2 +- ...geFlatDatabaseHealingRangeRequestTest.java | 3 +- .../StorageTrieNodeHealingRequestTest.java | 2 +- .../BonsaiReferenceTestUpdateAccumulator.java | 18 +- .../BonsaiReferenceTestWorldState.java | 58 +-- .../BonsaiReferenceTestWorldStateStorage.java | 12 +- gradle.properties | 2 +- 93 files changed, 1968 insertions(+), 1277 deletions(-) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/BonsaiAccount.java (55%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/cache/CachedMerkleTrieLoader.java => diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java} (94%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/cache/CachedMerkleTrieLoaderModule.java => diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoaderModule.java} (77%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/cache/NoOpCachedWorldStorageManager.java => diffbased/bonsai/cache/NoOpBonsaiCachedWorldStorageManager.java} (68%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/BonsaiPreImageProxy.java (97%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java (93%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/BonsaiWorldStateKeyValueStorage.java (67%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/BonsaiWorldStateLayerStorage.java (79%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/flat/FullFlatDbStrategy.java (93%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/flat/PartialFlatDbStrategy.java (96%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/trielog/TrieLogFactoryImpl.java (91%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/worldview/BonsaiWorldState.java (59%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/BonsaiValue.java => diffbased/common/DiffBasedValue.java} (80%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/BonsaiWorldStateProvider.java => diffbased/common/DiffBasedWorldStateProvider.java} (62%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/StorageSubscriber.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/cache/CachedWorldStorageManager.java => diffbased/common/cache/DiffBasedCachedWorldStorageManager.java} (62%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/cache/CachedBonsaiWorldView.java => diffbased/common/cache/DiffBasedCachedWorldView.java} (71%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedLayeredWorldStateKeyValueStorage.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedWorldStateKeyValueStorage.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/storage/flat/AccountHashCodeStorageStrategy.java (96%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/storage/flat/CodeHashCodeStorageStrategy.java (96%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/storage/flat/CodeStorageStrategy.java (94%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/storage/flat/FlatDbStrategy.java (99%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/storage/flat/FlatDbStrategyProvider.java (95%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/trielog/NoOpTrieLogManager.java (79%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/trielog/TrieLogAddedEvent.java (92%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/trielog/TrieLogLayer.java (79%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/trielog/TrieLogManager.java (87%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/trielog/TrieLogPruner.java (94%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/worldview/BonsaiWorldView.java => diffbased/common/worldview/DiffBasedWorldView.java} (85%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java => diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java} (71%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/AccountConsumingMap.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/Consumer.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/StorageConsumingMap.java rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/AbstractIsolationTests.java (97%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{bonsai/CachedMerkleTrieLoaderTest.java => diffbased/bonsai/BonsaiCachedMerkleTrieLoaderTest.java} (94%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/BonsaiSnapshotIsolationTests.java (99%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/BonsaiWorldStateProviderTest.java (89%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/LogRollingTests.java (96%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/RollingImport.java (90%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java (99%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/trielog/TrieLogFactoryTests.java (94%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/trielog/TrieLogManagerTests.java (84%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/trielog/TrieLogPrunerTest.java (96%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/worldview/BonsaiWorldStateTest.java (81%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/storage/flat/FlatDbStrategyProviderTest.java (97%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/trielog/TrieLogLayerTests.java (98%) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 8cebe77618c..d94c19d8bdb 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -27,9 +27,9 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.rlp.RLP; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import java.io.File; diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index a47271e247b..0ee2e4532c6 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -25,8 +25,8 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; diff --git a/besu/src/main/java/org/hyperledger/besu/components/BesuComponent.java b/besu/src/main/java/org/hyperledger/besu/components/BesuComponent.java index f0093243a6c..65ba2d931d2 100644 --- a/besu/src/main/java/org/hyperledger/besu/components/BesuComponent.java +++ b/besu/src/main/java/org/hyperledger/besu/components/BesuComponent.java @@ -19,8 +19,8 @@ import org.hyperledger.besu.cli.BesuCommand; import org.hyperledger.besu.ethereum.eth.transactions.BlobCache; import org.hyperledger.besu.ethereum.eth.transactions.BlobCacheModule; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoaderModule; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoaderModule; import org.hyperledger.besu.metrics.MetricsSystemModule; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.services.BesuPluginContextImpl; @@ -37,7 +37,7 @@ modules = { BesuCommandModule.class, MetricsSystemModule.class, - CachedMerkleTrieLoaderModule.class, + BonsaiCachedMerkleTrieLoaderModule.class, BesuPluginContextModule.class, BlobCacheModule.class }) @@ -55,7 +55,7 @@ public interface BesuComponent { * * @return CachedMerkleTrieLoader */ - CachedMerkleTrieLoader getCachedMerkleTrieLoader(); + BonsaiCachedMerkleTrieLoader getCachedMerkleTrieLoader(); /** * a metrics system that is observable by a Prometheus or OTEL metrics collection subsystem diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index cdc1a317088..8990ba9e447 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -81,11 +81,11 @@ import org.hyperledger.besu.ethereum.p2p.config.SubProtocolConfiguration; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive; import org.hyperledger.besu.ethereum.trie.forest.pruner.MarkSweepPruner; import org.hyperledger.besu.ethereum.trie.forest.pruner.Pruner; @@ -576,13 +576,14 @@ public BesuController build() { dataDirectory.toString(), numberOfBlocksToCache); - final CachedMerkleTrieLoader cachedMerkleTrieLoader = + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader = besuComponent .map(BesuComponent::getCachedMerkleTrieLoader) - .orElseGet(() -> new CachedMerkleTrieLoader(metricsSystem)); + .orElseGet(() -> new BonsaiCachedMerkleTrieLoader(metricsSystem)); final WorldStateArchive worldStateArchive = - createWorldStateArchive(worldStateStorageCoordinator, blockchain, cachedMerkleTrieLoader); + createWorldStateArchive( + worldStateStorageCoordinator, blockchain, bonsaiCachedMerkleTrieLoader); if (blockchain.getChainHeadBlockNumber() < 1) { genesisState.writeStateTo(worldStateArchive.getMutable()); @@ -1048,7 +1049,7 @@ private Optional createSnapProtocolManager( WorldStateArchive createWorldStateArchive( final WorldStateStorageCoordinator worldStateStorageCoordinator, final Blockchain blockchain, - final CachedMerkleTrieLoader cachedMerkleTrieLoader) { + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader) { return switch (dataStorageConfiguration.getDataStorageFormat()) { case BONSAI -> { final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage = @@ -1057,7 +1058,7 @@ yield new BonsaiWorldStateProvider( worldStateKeyValueStorage, blockchain, Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()), - cachedMerkleTrieLoader, + bonsaiCachedMerkleTrieLoader, besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null), evmConfiguration); } @@ -1067,6 +1068,8 @@ yield new BonsaiWorldStateProvider( yield new ForestWorldStateArchive( worldStateStorageCoordinator, preimageStorage, evmConfiguration); } + default -> throw new IllegalStateException( + "Unexpected value: " + dataStorageConfiguration.getDataStorageFormat()); }; } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index c2789c68085..d4a6dd2dc71 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -31,9 +31,9 @@ import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; diff --git a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java index a06e1907d2c..5eb14201cd8 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java @@ -43,9 +43,9 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; @@ -189,7 +189,7 @@ public void shouldDisablePruningIfBonsaiIsEnabled() { .createWorldStateArchive( any(WorldStateStorageCoordinator.class), any(Blockchain.class), - any(CachedMerkleTrieLoader.class)); + any(BonsaiCachedMerkleTrieLoader.class)); doReturn(mockWorldState).when(worldStateArchive).getMutable(); when(storageProvider.createWorldStateStorageCoordinator(dataStorageConfiguration)) .thenReturn(new WorldStateStorageCoordinator(bonsaiWorldStateStorage)); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/AccountAdapter.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/AccountAdapter.java index 13caadab6ab..faf42aafc8e 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/AccountAdapter.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/AccountAdapter.java @@ -17,7 +17,7 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.account.AccountState; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java index 1a70ce49381..c6cf6898eb1 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java @@ -31,8 +31,8 @@ import org.hyperledger.besu.ethereum.privacy.storage.PrivateMetadataUpdater; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.vm.BlockHashLookup; import org.hyperledger.besu.ethereum.vm.CachingBlockHashLookup; import org.hyperledger.besu.evm.gascalculator.CancunGasCalculator; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java index 004790ee170..944f668cd68 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java @@ -19,7 +19,7 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java index 121fec3f4dc..7f89ff7185c 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java @@ -19,11 +19,11 @@ import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.NoOpCachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.NoOpTrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.NoOpBonsaiCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.NoOpTrieLogManager; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; @@ -59,8 +59,8 @@ public static MutableWorldState createGenesisWorldState( * @return a mutable world state for the Genesis block */ private static MutableWorldState createGenesisBonsaiWorldState() { - final CachedMerkleTrieLoader cachedMerkleTrieLoader = - new CachedMerkleTrieLoader(new NoOpMetricsSystem()); + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader = + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()); final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage = new BonsaiWorldStateKeyValueStorage( new KeyValueStorageProvider( @@ -71,8 +71,8 @@ private static MutableWorldState createGenesisBonsaiWorldState() { DataStorageConfiguration.DEFAULT_BONSAI_CONFIG); return new BonsaiWorldState( bonsaiWorldStateKeyValueStorage, - cachedMerkleTrieLoader, - new NoOpCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage), + bonsaiCachedMerkleTrieLoader, + new NoOpBonsaiCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage), new NoOpTrieLogManager(), EvmConfiguration.DEFAULT); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiAccount.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccount.java similarity index 55% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiAccount.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccount.java index d93f550cafa..6eea664e048 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiAccount.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccount.java @@ -14,48 +14,34 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import org.hyperledger.besu.datatypes.AccountValue; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; -import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.rlp.RLPException; import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.rlp.RLPOutput; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; import org.hyperledger.besu.evm.ModificationNotAllowedException; import org.hyperledger.besu.evm.account.AccountStorageEntry; -import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.worldstate.UpdateTrackingAccount; -import java.util.HashMap; -import java.util.Map; import java.util.NavigableMap; import java.util.Objects; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.units.bigints.UInt256; -public class BonsaiAccount implements MutableAccount, AccountValue { - private final BonsaiWorldView context; - private boolean immutable; - - private final Address address; - private final Hash addressHash; - private Hash codeHash; - private long nonce; - private Wei balance; +public class BonsaiAccount extends DiffBasedAccount { private Hash storageRoot; - private Bytes code; - - private final Map updatedStorage = new HashMap<>(); public BonsaiAccount( - final BonsaiWorldView context, + final DiffBasedWorldView context, final Address address, final Hash addressHash, final long nonce, @@ -63,31 +49,24 @@ public BonsaiAccount( final Hash storageRoot, final Hash codeHash, final boolean mutable) { - this.context = context; - this.address = address; - this.addressHash = addressHash; - this.nonce = nonce; - this.balance = balance; + super(context, address, addressHash, nonce, balance, codeHash, mutable); this.storageRoot = storageRoot; - this.codeHash = codeHash; - - this.immutable = !mutable; } public BonsaiAccount( - final BonsaiWorldView context, + final DiffBasedWorldView context, final Address address, final AccountValue stateTrieAccount, final boolean mutable) { - this( + super( context, address, address.addressHash(), stateTrieAccount.getNonce(), stateTrieAccount.getBalance(), - stateTrieAccount.getStorageRoot(), stateTrieAccount.getCodeHash(), mutable); + this.storageRoot = stateTrieAccount.getStorageRoot(); } public BonsaiAccount(final BonsaiAccount toCopy) { @@ -95,37 +74,35 @@ public BonsaiAccount(final BonsaiAccount toCopy) { } public BonsaiAccount( - final BonsaiAccount toCopy, final BonsaiWorldView context, final boolean mutable) { - this.context = context; - this.address = toCopy.address; - this.addressHash = toCopy.addressHash; - this.nonce = toCopy.nonce; - this.balance = toCopy.balance; + final BonsaiAccount toCopy, final DiffBasedWorldView context, final boolean mutable) { + super( + context, + toCopy.address, + toCopy.addressHash, + toCopy.nonce, + toCopy.balance, + toCopy.codeHash, + mutable); this.storageRoot = toCopy.storageRoot; - this.codeHash = toCopy.codeHash; - this.code = toCopy.code; updatedStorage.putAll(toCopy.updatedStorage); - - this.immutable = !mutable; } public BonsaiAccount( - final BonsaiWorldView context, final UpdateTrackingAccount tracked) { - this.context = context; - this.address = tracked.getAddress(); - this.addressHash = tracked.getAddressHash(); - this.nonce = tracked.getNonce(); - this.balance = tracked.getBalance(); + final DiffBasedWorldView context, final UpdateTrackingAccount tracked) { + super( + context, + tracked.getAddress(), + tracked.getAddressHash(), + tracked.getNonce(), + tracked.getBalance(), + tracked.getCodeHash(), + true); this.storageRoot = Hash.EMPTY_TRIE_HASH; - this.codeHash = tracked.getCodeHash(); - this.code = tracked.getCode(); updatedStorage.putAll(tracked.getUpdatedStorage()); - - this.immutable = false; } public static BonsaiAccount fromRLP( - final BonsaiWorldView context, + final DiffBasedWorldView context, final Address address, final Bytes encoded, final boolean mutable) @@ -144,88 +121,11 @@ public static BonsaiAccount fromRLP( context, address, address.addressHash(), nonce, balance, storageRoot, codeHash, mutable); } - @Override - public Address getAddress() { - return address; - } - - @Override - public Hash getAddressHash() { - return addressHash; - } - - @Override - public long getNonce() { - return nonce; - } - - @Override - public void setNonce(final long value) { - if (immutable) { - throw new ModificationNotAllowedException(); - } - nonce = value; - } - - @Override - public Wei getBalance() { - return balance; - } - - @Override - public void setBalance(final Wei value) { - if (immutable) { - throw new ModificationNotAllowedException(); - } - balance = value; - } - - @Override - public Bytes getCode() { - if (code == null) { - code = context.getCode(address, codeHash).orElse(Bytes.EMPTY); - } - return code; - } - - @Override - public void setCode(final Bytes code) { - if (immutable) { - throw new ModificationNotAllowedException(); - } - this.code = code; - if (code == null || code.isEmpty()) { - this.codeHash = Hash.EMPTY; - } else { - this.codeHash = Hash.hash(code); - } - } - - @Override - public Hash getCodeHash() { - return codeHash; - } - - @Override - public UInt256 getStorageValue(final UInt256 key) { - return context.getStorageValue(address, key); - } - - @Override - public UInt256 getOriginalStorageValue(final UInt256 key) { - return context.getPriorStorageValue(address, key); - } - @Override public NavigableMap storageEntriesFrom( final Bytes32 startKeyHash, final int limit) { - return context.getWorldStateStorage().storageEntriesFrom(this.addressHash, startKeyHash, limit); - } - - public Bytes serializeAccount() { - final BytesValueRLPOutput out = new BytesValueRLPOutput(); - writeTo(out); - return out.encoded(); + return ((BonsaiWorldStateKeyValueStorage) context.getWorldStateStorage()) + .storageEntriesFrom(this.addressHash, startKeyHash, limit); } @Override @@ -240,24 +140,6 @@ public void writeTo(final RLPOutput out) { out.endList(); } - @Override - public void setStorageValue(final UInt256 key, final UInt256 value) { - if (immutable) { - throw new ModificationNotAllowedException(); - } - updatedStorage.put(key, value); - } - - @Override - public void clearStorage() { - updatedStorage.clear(); - } - - @Override - public Map getUpdatedStorage() { - return updatedStorage; - } - @Override public Hash getStorageRoot() { return storageRoot; @@ -270,11 +152,6 @@ public void setStorageRoot(final Hash storageRoot) { this.storageRoot = storageRoot; } - @Override - public void becomeImmutable() { - immutable = true; - } - @Override public String toString() { return "AccountState{" diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java new file mode 100644 index 00000000000..bfd295091c3 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java @@ -0,0 +1,145 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; +import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; +import org.hyperledger.besu.evm.internal.EvmConfiguration; +import org.hyperledger.besu.plugin.BesuContext; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.tuweni.bytes.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BonsaiWorldStateProvider extends DiffBasedWorldStateProvider { + + private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateProvider.class); + private final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader; + + public BonsaiWorldStateProvider( + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, + final Blockchain blockchain, + final Optional maxLayersToLoad, + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader, + final BesuContext pluginContext, + final EvmConfiguration evmConfiguration) { + super(worldStateKeyValueStorage, blockchain, maxLayersToLoad, pluginContext); + this.bonsaiCachedMerkleTrieLoader = bonsaiCachedMerkleTrieLoader; + provideCachedWorldStorageManager( + new BonsaiCachedWorldStorageManager(this, worldStateKeyValueStorage)); + loadPersistedState(new BonsaiWorldState(this, worldStateKeyValueStorage, evmConfiguration)); + } + + @VisibleForTesting + BonsaiWorldStateProvider( + final BonsaiCachedWorldStorageManager bonsaiCachedWorldStorageManager, + final TrieLogManager trieLogManager, + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, + final Blockchain blockchain, + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader, + final EvmConfiguration evmConfiguration) { + super(worldStateKeyValueStorage, blockchain, trieLogManager); + this.bonsaiCachedMerkleTrieLoader = bonsaiCachedMerkleTrieLoader; + provideCachedWorldStorageManager(bonsaiCachedWorldStorageManager); + loadPersistedState(new BonsaiWorldState(this, worldStateKeyValueStorage, evmConfiguration)); + } + + public BonsaiCachedMerkleTrieLoader getCachedMerkleTrieLoader() { + return bonsaiCachedMerkleTrieLoader; + } + + private BonsaiWorldStateKeyValueStorage getWorldStateKeyValueStorage() { + return (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage; + } + /** + * Prepares the state healing process for a given address and location. It prepares the state + * healing, including retrieving data from storage, identifying invalid slots or nodes, removing + * account and slot from the state trie, and committing the changes. Finally, it downgrades the + * world state storage to partial flat database mode. + */ + public void prepareStateHealing(final Address address, final Bytes location) { + final Set keysToDelete = new HashSet<>(); + final BonsaiWorldStateKeyValueStorage.Updater updater = + getWorldStateKeyValueStorage().updater(); + final Hash accountHash = address.addressHash(); + final StoredMerklePatriciaTrie accountTrie = + new StoredMerklePatriciaTrie<>( + (l, h) -> { + final Optional node = + getWorldStateKeyValueStorage().getAccountStateTrieNode(l, h); + if (node.isPresent()) { + keysToDelete.add(l); + } + return node; + }, + persistedState.getWorldStateRootHash(), + Function.identity(), + Function.identity()); + try { + accountTrie + .get(accountHash) + .map(RLP::input) + .map(StateTrieAccountValue::readFrom) + .ifPresent( + account -> { + final StoredMerklePatriciaTrie storageTrie = + new StoredMerklePatriciaTrie<>( + (l, h) -> { + Optional node = + getWorldStateKeyValueStorage() + .getAccountStorageTrieNode(accountHash, l, h); + if (node.isPresent()) { + keysToDelete.add(Bytes.concatenate(accountHash, l)); + } + return node; + }, + account.getStorageRoot(), + Function.identity(), + Function.identity()); + try { + storageTrie.getPath(location); + } catch (Exception eA) { + LOG.warn("Invalid slot found for account {} at location {}", address, location); + // ignore + } + }); + } catch (Exception eA) { + LOG.warn("Invalid node for account {} at location {}", address, location); + // ignore + } + keysToDelete.forEach(updater::removeAccountStateTrieNode); + updater.commit(); + + getWorldStateKeyValueStorage().downgradeToPartialFlatDbMode(); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoader.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java similarity index 94% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoader.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java index cbd1fc820d8..9f3251660dc 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoader.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java @@ -13,14 +13,15 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.cache; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.metrics.ObservableMetricsSystem; @@ -37,8 +38,7 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -public class CachedMerkleTrieLoader - implements BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber { +public class BonsaiCachedMerkleTrieLoader implements StorageSubscriber { private static final int ACCOUNT_CACHE_SIZE = 100_000; private static final int STORAGE_CACHE_SIZE = 200_000; @@ -47,7 +47,7 @@ public class CachedMerkleTrieLoader private final Cache storageNodes = CacheBuilder.newBuilder().recordStats().maximumSize(STORAGE_CACHE_SIZE).build(); - public CachedMerkleTrieLoader(final ObservableMetricsSystem metricsSystem) { + public BonsaiCachedMerkleTrieLoader(final ObservableMetricsSystem metricsSystem) { CacheMetricsCollector cacheMetrics = new CacheMetricsCollector(); cacheMetrics.addCache("accountsNodes", accountNodes); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoaderModule.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoaderModule.java similarity index 77% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoaderModule.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoaderModule.java index 63c8051f253..04a8efc4514 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoaderModule.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoaderModule.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.cache; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache; import org.hyperledger.besu.metrics.ObservableMetricsSystem; @@ -21,11 +21,11 @@ import dagger.Provides; @Module -public class CachedMerkleTrieLoaderModule { +public class BonsaiCachedMerkleTrieLoaderModule { @Provides - CachedMerkleTrieLoader provideCachedMerkleTrieLoaderModule( + BonsaiCachedMerkleTrieLoader provideCachedMerkleTrieLoaderModule( final ObservableMetricsSystem metricsSystem) { - return new CachedMerkleTrieLoader(metricsSystem); + return new BonsaiCachedMerkleTrieLoader(metricsSystem); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java new file mode 100644 index 00000000000..a7bee477c12 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java @@ -0,0 +1,60 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache; + +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.evm.internal.EvmConfiguration; + +public class BonsaiCachedWorldStorageManager extends DiffBasedCachedWorldStorageManager { + + public BonsaiCachedWorldStorageManager( + final BonsaiWorldStateProvider archive, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage) { + super(archive, worldStateKeyValueStorage); + } + + @Override + public DiffBasedWorldState createWorldState( + final DiffBasedWorldStateProvider archive, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final EvmConfiguration evmConfiguration) { + return new BonsaiWorldState( + (BonsaiWorldStateProvider) archive, + (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage, + evmConfiguration); + } + + @Override + public DiffBasedWorldStateKeyValueStorage createLayeredKeyValueStorage( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage) { + return new BonsaiWorldStateLayerStorage( + (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage); + } + + @Override + public DiffBasedWorldStateKeyValueStorage createSnapshotKeyValueStorage( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage) { + return new BonsaiSnapshotWorldStateKeyValueStorage( + (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/NoOpBonsaiCachedWorldStorageManager.java similarity index 68% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/NoOpBonsaiCachedWorldStorageManager.java index d86128b13f1..0e4ee9cbffd 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/NoOpBonsaiCachedWorldStorageManager.java @@ -12,19 +12,19 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.cache; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; import java.util.Optional; import java.util.function.Function; -public class NoOpCachedWorldStorageManager extends CachedWorldStorageManager { +public class NoOpBonsaiCachedWorldStorageManager extends BonsaiCachedWorldStorageManager { - public NoOpCachedWorldStorageManager( + public NoOpBonsaiCachedWorldStorageManager( final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage) { super(null, bonsaiWorldStateKeyValueStorage); } @@ -33,7 +33,7 @@ public NoOpCachedWorldStorageManager( public synchronized void addCachedLayer( final BlockHeader blockHeader, final Hash worldStateRootHash, - final BonsaiWorldState forWorldState) { + final DiffBasedWorldState forWorldState) { // no cache } @@ -43,17 +43,17 @@ public boolean containWorldStateStorage(final Hash blockHash) { } @Override - public Optional getWorldState(final Hash blockHash) { + public Optional getWorldState(final Hash blockHash) { return Optional.empty(); } @Override - public Optional getNearestWorldState(final BlockHeader blockHeader) { + public Optional getNearestWorldState(final BlockHeader blockHeader) { return Optional.empty(); } @Override - public Optional getHeadWorldState( + public Optional getHeadWorldState( final Function> hashBlockHeaderFunction) { return Optional.empty(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiPreImageProxy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiPreImageProxy.java similarity index 97% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiPreImageProxy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiPreImageProxy.java index 16cda3104f0..4bae322a855 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiPreImageProxy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiPreImageProxy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java similarity index 93% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java index e20d844738e..c2972e04598 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java @@ -13,11 +13,12 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedSnapshotWorldStateKeyValueStorage; import org.hyperledger.besu.plugin.services.exception.StorageException; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SnappableKeyValueStorage; @@ -32,7 +33,7 @@ import org.slf4j.LoggerFactory; public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKeyValueStorage - implements BonsaiStorageSubscriber { + implements DiffBasedSnapshotWorldStateKeyValueStorage, StorageSubscriber { protected final BonsaiWorldStateKeyValueStorage parentWorldStateStorage; private static final Logger LOG = @@ -53,9 +54,9 @@ public BonsaiSnapshotWorldStateKeyValueStorage( final BonsaiWorldStateKeyValueStorage worldStateStorageKeyValueStorage) { this( worldStateStorageKeyValueStorage, - ((SnappableKeyValueStorage) worldStateStorageKeyValueStorage.composedWorldStateStorage) + ((SnappableKeyValueStorage) worldStateStorageKeyValueStorage.getComposedWorldStateStorage()) .takeSnapshot(), - worldStateStorageKeyValueStorage.trieLogStorage); + worldStateStorageKeyValueStorage.getTrieLogStorage()); } private boolean isClosedGet() { @@ -207,7 +208,7 @@ public void onClearTrieLog() { protected synchronized void doClose() throws Exception { if (!isClosedGet()) { // alert any subscribers we are closing: - subscribers.forEach(BonsaiStorageSubscriber::onCloseStorage); + subscribers.forEach(StorageSubscriber::onCloseStorage); // close all of the SnappedKeyValueStorages: composedWorldStateStorage.close(); @@ -220,6 +221,7 @@ protected synchronized void doClose() throws Exception { } } + @Override public BonsaiWorldStateKeyValueStorage getParentWorldStateStorage() { return parentWorldStateStorage; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorage.java similarity index 67% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorage.java index 5f477afebaf..a58316fc62c 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorage.java @@ -12,7 +12,7 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; @@ -24,8 +24,9 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.trie.MerkleTrie; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.flat.FlatDbStrategy; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.flat.FlatDbStrategyProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategyProvider; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; @@ -37,52 +38,28 @@ import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; -import org.hyperledger.besu.util.Subscribers; -import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.Map; import java.util.NavigableMap; import java.util.Optional; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; -import java.util.stream.Stream; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class BonsaiWorldStateKeyValueStorage implements WorldStateKeyValueStorage, AutoCloseable { - private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateKeyValueStorage.class); - - // 0x776f726c64526f6f74 - public static final byte[] WORLD_ROOT_HASH_KEY = "worldRoot".getBytes(StandardCharsets.UTF_8); - // 0x776f726c64426c6f636b48617368 - public static final byte[] WORLD_BLOCK_HASH_KEY = - "worldBlockHash".getBytes(StandardCharsets.UTF_8); +public class BonsaiWorldStateKeyValueStorage extends DiffBasedWorldStateKeyValueStorage + implements WorldStateKeyValueStorage { protected final FlatDbStrategyProvider flatDbStrategyProvider; - protected final SegmentedKeyValueStorage composedWorldStateStorage; - protected final KeyValueStorage trieLogStorage; - - private final AtomicBoolean shouldClose = new AtomicBoolean(false); - - protected final AtomicBoolean isClosed = new AtomicBoolean(false); - - protected final Subscribers subscribers = Subscribers.create(); - public BonsaiWorldStateKeyValueStorage( final StorageProvider provider, final MetricsSystem metricsSystem, final DataStorageConfiguration dataStorageConfiguration) { - this.composedWorldStateStorage = + super( provider.getStorageBySegmentIdentifiers( List.of( - ACCOUNT_INFO_STATE, CODE_STORAGE, ACCOUNT_STORAGE_STORAGE, TRIE_BRANCH_STORAGE)); - this.trieLogStorage = - provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); + ACCOUNT_INFO_STATE, CODE_STORAGE, ACCOUNT_STORAGE_STORAGE, TRIE_BRANCH_STORAGE)), + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE)); this.flatDbStrategyProvider = new FlatDbStrategyProvider(metricsSystem, dataStorageConfiguration); flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage); @@ -92,9 +69,8 @@ public BonsaiWorldStateKeyValueStorage( final FlatDbStrategyProvider flatDbStrategyProvider, final SegmentedKeyValueStorage composedWorldStateStorage, final KeyValueStorage trieLogStorage) { + super(composedWorldStateStorage, trieLogStorage); this.flatDbStrategyProvider = flatDbStrategyProvider; - this.composedWorldStateStorage = composedWorldStateStorage; - this.trieLogStorage = trieLogStorage; } @Override @@ -102,6 +78,7 @@ public DataStorageFormat getDataStorageFormat() { return DataStorageFormat.BONSAI; } + @Override public FlatDbMode getFlatDbMode() { return flatDbStrategyProvider.getFlatDbMode(); } @@ -155,31 +132,6 @@ public Optional getTrieNodeUnsafe(final Bytes key) { .map(Bytes::wrap); } - public Optional getTrieLog(final Hash blockHash) { - return trieLogStorage.get(blockHash.toArrayUnsafe()); - } - - public Stream streamTrieLogKeys(final long limit) { - return trieLogStorage.streamKeys().limit(limit); - } - - public Optional getStateTrieNode(final Bytes location) { - return composedWorldStateStorage - .get(TRIE_BRANCH_STORAGE, location.toArrayUnsafe()) - .map(Bytes::wrap); - } - - public Optional getWorldStateRootHash() { - return composedWorldStateStorage.get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY).map(Bytes::wrap); - } - - public Optional getWorldStateBlockHash() { - return composedWorldStateStorage - .get(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY) - .map(Bytes32::wrap) - .map(Hash::wrap); - } - public Optional getStorageValueByStorageSlotKey( final Hash accountHash, final StorageSlotKey storageSlotKey) { return getStorageValueByStorageSlotKey( @@ -209,34 +161,11 @@ public Optional getStorageValueByStorageSlotKey( composedWorldStateStorage); } - public Map streamFlatAccounts( - final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { - return flatDbStrategyProvider - .getFlatDbStrategy(composedWorldStateStorage) - .streamAccountFlatDatabase(composedWorldStateStorage, startKeyHash, endKeyHash, max); - } - - public Map streamFlatStorages( - final Hash accountHash, final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { - return flatDbStrategyProvider - .getFlatDbStrategy(composedWorldStateStorage) - .streamStorageFlatDatabase( - composedWorldStateStorage, accountHash, startKeyHash, endKeyHash, max); - } - public NavigableMap storageEntriesFrom( final Hash addressHash, final Bytes32 startKeyHash, final int limit) { throw new RuntimeException("Bonsai Tries does not currently support enumerating storage"); } - public boolean isWorldStateAvailable(final Bytes32 rootHash, final Hash blockHash) { - return composedWorldStateStorage - .get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY) - .map(Bytes32::wrap) - .map(hash -> hash.equals(rootHash) || trieLogStorage.containsKey(blockHash.toArrayUnsafe())) - .orElse(false); - } - public void upgradeToFullFlatDbMode() { flatDbStrategyProvider.upgradeToFullFlatDbMode(composedWorldStateStorage); } @@ -247,26 +176,14 @@ public void downgradeToPartialFlatDbMode() { @Override public void clear() { - subscribers.forEach(BonsaiStorageSubscriber::onClearStorage); - flatDbStrategyProvider - .getFlatDbStrategy(composedWorldStateStorage) - .clearAll(composedWorldStateStorage); - composedWorldStateStorage.clear(TRIE_BRANCH_STORAGE); - trieLogStorage.clear(); + super.clear(); flatDbStrategyProvider.loadFlatDbStrategy( composedWorldStateStorage); // force reload of flat db reader strategy } - public void clearTrieLog() { - subscribers.forEach(BonsaiStorageSubscriber::onClearTrieLog); - trieLogStorage.clear(); - } - - public void clearFlatDatabase() { - subscribers.forEach(BonsaiStorageSubscriber::onClearFlatDatabaseStorage); - flatDbStrategyProvider - .getFlatDbStrategy(composedWorldStateStorage) - .resetOnResync(composedWorldStateStorage); + @Override + public FlatDbStrategy getFlatDbStrategy() { + return flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage); } @Override @@ -277,20 +194,7 @@ public Updater updater() { flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage)); } - public boolean pruneTrieLog(final Hash blockHash) { - try { - return trieLogStorage.tryDelete(blockHash.toArrayUnsafe()); - } catch (Exception e) { - LOG.error("Error pruning trie log for block hash {}", blockHash, e); - return false; - } - } - - public FlatDbStrategy getFlatDbStrategy() { - return flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage); - } - - public static class Updater implements WorldStateKeyValueStorage.Updater { + public static class Updater implements DiffBasedWorldStateKeyValueStorage.Updater { private final SegmentedKeyValueStorageTransaction composedWorldStateTransaction; private final KeyValueStorageTransaction trieLogStorageTransaction; @@ -340,6 +244,7 @@ public Updater putAccountInfoState(final Hash accountHash, final Bytes accountVa return this; } + @Override public Updater saveWorldState(final Bytes blockHash, final Bytes32 nodeHash, final Bytes node) { composedWorldStateTransaction.put( TRIE_BRANCH_STORAGE, Bytes.EMPTY.toArrayUnsafe(), node.toArrayUnsafe()); @@ -392,10 +297,12 @@ public synchronized void removeStorageValueBySlotHash( composedWorldStateTransaction, accountHash, slotHash); } + @Override public SegmentedKeyValueStorageTransaction getWorldStateTransaction() { return composedWorldStateTransaction; } + @Override public KeyValueStorageTransaction getTrieLogStorageTransaction() { return trieLogStorageTransaction; } @@ -407,65 +314,10 @@ public void commit() { composedWorldStateTransaction.commit(); } + @Override public void rollback() { composedWorldStateTransaction.rollback(); trieLogStorageTransaction.rollback(); } } - - @Override - public synchronized void close() throws Exception { - // when the storage clears, close - shouldClose.set(true); - tryClose(); - } - - public synchronized long subscribe(final BonsaiStorageSubscriber sub) { - if (isClosed.get()) { - throw new RuntimeException("Storage is marked to close or has already closed"); - } - return subscribers.subscribe(sub); - } - - public synchronized void unSubscribe(final long id) { - subscribers.unsubscribe(id); - try { - tryClose(); - } catch (Exception e) { - LOG.atWarn() - .setMessage("exception while trying to close : {}") - .addArgument(e::getMessage) - .log(); - } - } - - protected synchronized void tryClose() throws Exception { - if (shouldClose.get() && subscribers.getSubscriberCount() < 1) { - doClose(); - } - } - - protected synchronized void doClose() throws Exception { - if (!isClosed.get()) { - // alert any subscribers we are closing: - subscribers.forEach(BonsaiStorageSubscriber::onCloseStorage); - - // close all of the KeyValueStorages: - composedWorldStateStorage.close(); - trieLogStorage.close(); - - // set storage closed - isClosed.set(true); - } - } - - public interface BonsaiStorageSubscriber { - default void onClearStorage() {} - - default void onClearFlatDatabaseStorage() {} - - default void onClearTrieLog() {} - - default void onCloseStorage() {} - } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateLayerStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateLayerStorage.java similarity index 79% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateLayerStorage.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateLayerStorage.java index 92d4f54fee3..045376f6135 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateLayerStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateLayerStorage.java @@ -13,21 +13,22 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedLayeredWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SnappedKeyValueStorage; import org.hyperledger.besu.services.kvstore.LayeredKeyValueStorage; public class BonsaiWorldStateLayerStorage extends BonsaiSnapshotWorldStateKeyValueStorage - implements BonsaiStorageSubscriber { + implements DiffBasedLayeredWorldStateKeyValueStorage, StorageSubscriber { public BonsaiWorldStateLayerStorage(final BonsaiWorldStateKeyValueStorage parent) { this( - new LayeredKeyValueStorage(parent.composedWorldStateStorage), - parent.trieLogStorage, + new LayeredKeyValueStorage(parent.getComposedWorldStateStorage()), + parent.getTrieLogStorage(), parent); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FullFlatDbStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/FullFlatDbStrategy.java similarity index 93% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FullFlatDbStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/FullFlatDbStrategy.java index 83f7f030c73..f8cbf2c3d2b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FullFlatDbStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/FullFlatDbStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; @@ -21,6 +21,8 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.trie.NodeLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.CodeStorageStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/PartialFlatDbStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/PartialFlatDbStrategy.java similarity index 96% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/PartialFlatDbStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/PartialFlatDbStrategy.java index cfe18dbe2ad..9e693d63e46 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/PartialFlatDbStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/PartialFlatDbStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; @@ -21,6 +21,8 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.trie.NodeLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.CodeStorageStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory; import org.hyperledger.besu.metrics.BesuMetricCategory; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryImpl.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryImpl.java similarity index 91% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryImpl.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryImpl.java index 2915a671820..6ebda666dbb 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryImpl.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryImpl.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog; import org.hyperledger.besu.datatypes.AccountValue; import org.hyperledger.besu.datatypes.Address; @@ -23,7 +23,8 @@ import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.rlp.RLPOutput; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.plugin.data.BlockHeader; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; @@ -150,7 +151,7 @@ public static TrieLogLayer readFrom(final RLPInput input) { final TrieLogLayer newLayer = new TrieLogLayer(); input.enterList(); - newLayer.blockHash = Hash.wrap(input.readBytes32()); + newLayer.setBlockHash(Hash.wrap(input.readBytes32())); while (!input.isEndOfCurrentList()) { input.enterList(); @@ -164,7 +165,9 @@ public static TrieLogLayer readFrom(final RLPInput input) { final StateTrieAccountValue newValue = nullOrValue(input, StateTrieAccountValue::readFrom); final boolean isCleared = getOptionalIsCleared(input); input.leaveList(); - newLayer.accounts.put(address, new BonsaiValue<>(oldValue, newValue, isCleared)); + newLayer + .getAccountChanges() + .put(address, new DiffBasedValue<>(oldValue, newValue, isCleared)); } if (input.nextIsNull()) { @@ -175,13 +178,13 @@ public static TrieLogLayer readFrom(final RLPInput input) { final Bytes newCode = nullOrValue(input, RLPInput::readBytes); final boolean isCleared = getOptionalIsCleared(input); input.leaveList(); - newLayer.code.put(address, new BonsaiValue<>(oldCode, newCode, isCleared)); + newLayer.getCodeChanges().put(address, new DiffBasedValue<>(oldCode, newCode, isCleared)); } if (input.nextIsNull()) { input.skipNext(); } else { - final Map> storageChanges = new TreeMap<>(); + final Map> storageChanges = new TreeMap<>(); input.enterList(); while (!input.isEndOfCurrentList()) { input.enterList(); @@ -190,11 +193,11 @@ public static TrieLogLayer readFrom(final RLPInput input) { final UInt256 oldValue = nullOrValue(input, RLPInput::readUInt256Scalar); final UInt256 newValue = nullOrValue(input, RLPInput::readUInt256Scalar); final boolean isCleared = getOptionalIsCleared(input); - storageChanges.put(storageSlotKey, new BonsaiValue<>(oldValue, newValue, isCleared)); + storageChanges.put(storageSlotKey, new DiffBasedValue<>(oldValue, newValue, isCleared)); input.leaveList(); } input.leaveList(); - newLayer.storage.put(address, storageChanges); + newLayer.getStorageChanges().put(address, storageChanges); } // TODO add trie nodes diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java similarity index 59% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java index 9e5472f5706..3b3af540d7b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java @@ -14,39 +14,33 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai.worldview; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; -import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; -import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; -import static org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldView.encodeTrieValue; +import static org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView.encodeTrieValue; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; import org.hyperledger.besu.ethereum.trie.NodeLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateLayerStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator.StorageConsumingMap; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.StorageConsumingMap; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.internal.EvmConfiguration; -import org.hyperledger.besu.evm.worldstate.WorldUpdater; -import org.hyperledger.besu.plugin.services.exception.StorageException; -import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; @@ -63,24 +57,10 @@ import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.rlp.RLP; import org.apache.tuweni.units.bigints.UInt256; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class BonsaiWorldState - implements MutableWorldState, BonsaiWorldView, BonsaiStorageSubscriber { +public class BonsaiWorldState extends DiffBasedWorldState { - private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldState.class); - - protected BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage; - - protected final CachedMerkleTrieLoader cachedMerkleTrieLoader; - protected final CachedWorldStorageManager cachedWorldStorageManager; - protected final TrieLogManager trieLogManager; - private BonsaiWorldStateUpdateAccumulator accumulator; - - protected Hash worldStateRootHash; - Hash worldStateBlockHash; - private boolean isFrozen; + protected final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader; public BonsaiWorldState( final BonsaiWorldStateProvider archive, @@ -96,90 +76,39 @@ public BonsaiWorldState( public BonsaiWorldState( final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, - final CachedMerkleTrieLoader cachedMerkleTrieLoader, - final CachedWorldStorageManager cachedWorldStorageManager, + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader, + final DiffBasedCachedWorldStorageManager cachedWorldStorageManager, final TrieLogManager trieLogManager, final EvmConfiguration evmConfiguration) { - this.worldStateKeyValueStorage = worldStateKeyValueStorage; - this.worldStateRootHash = - Hash.wrap( - Bytes32.wrap( - worldStateKeyValueStorage.getWorldStateRootHash().orElse(getEmptyTrieHash()))); - this.worldStateBlockHash = - Hash.wrap( - Bytes32.wrap(worldStateKeyValueStorage.getWorldStateBlockHash().orElse(Hash.ZERO))); - this.accumulator = + super(worldStateKeyValueStorage, cachedWorldStorageManager, trieLogManager); + this.bonsaiCachedMerkleTrieLoader = bonsaiCachedMerkleTrieLoader; + this.setAccumulator( new BonsaiWorldStateUpdateAccumulator( this, (addr, value) -> - cachedMerkleTrieLoader.preLoadAccount( - getWorldStateStorage(), worldStateRootHash, addr), + bonsaiCachedMerkleTrieLoader.preLoadAccount( + worldStateKeyValueStorage, worldStateRootHash, addr), (addr, value) -> - cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value), - evmConfiguration); - this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; - this.cachedWorldStorageManager = cachedWorldStorageManager; - this.trieLogManager = trieLogManager; - } - - /** - * Override the accumulator solves the chicken-egg problem of needing a worldstate reference - * (this) when construction the Accumulator. - * - * @param accumulator accumulator to use. - */ - public void setAccumulator(final BonsaiWorldStateUpdateAccumulator accumulator) { - this.accumulator = accumulator; - } - - /** - * Returns the world state block hash of this world state - * - * @return the world state block hash. - */ - public Hash getWorldStateBlockHash() { - return worldStateBlockHash; - } - - /** - * Returns the world state root hash of this world state - * - * @return the world state root hash. - */ - public Hash getWorldStateRootHash() { - return worldStateRootHash; + bonsaiCachedMerkleTrieLoader.preLoadStorageSlot( + getWorldStateStorage(), addr, value), + evmConfiguration)); } @Override - public boolean isPersisted() { - return isPersisted(worldStateKeyValueStorage); - } - - private boolean isPersisted(final WorldStateKeyValueStorage worldStateKeyValueStorage) { - return !(worldStateKeyValueStorage instanceof BonsaiSnapshotWorldStateKeyValueStorage); - } - - @Override - public Optional getCode(@Nonnull final Address address, final Hash codeHash) { - return worldStateKeyValueStorage.getCode(codeHash, address.addressHash()); - } - - /** - * Reset the worldState to this block header - * - * @param blockHeader block to use - */ - public void resetWorldStateTo(final BlockHeader blockHeader) { - worldStateBlockHash = blockHeader.getBlockHash(); - worldStateRootHash = blockHeader.getStateRoot(); + public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { + return (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage; } @Override - public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { - return worldStateKeyValueStorage; + protected Hash calculateRootHash( + final Optional maybeStateUpdater, + final DiffBasedWorldStateUpdateAccumulator worldStateUpdater) { + return internalCalculateRootHash( + maybeStateUpdater.map(BonsaiWorldStateKeyValueStorage.Updater.class::cast), + (BonsaiWorldStateUpdateAccumulator) worldStateUpdater); } - private Hash calculateRootHash( + private Hash internalCalculateRootHash( final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater) { @@ -187,7 +116,7 @@ private Hash calculateRootHash( // This must be done before updating the accounts so // that we can get the storage state hash - Stream>>> + Stream>>> storageStream = worldStateUpdater.getStorageToUpdate().entrySet().stream(); if (maybeStateUpdater.isEmpty()) { storageStream = @@ -205,8 +134,8 @@ private Hash calculateRootHash( final StoredMerklePatriciaTrie accountTrie = createTrie( (location, hash) -> - cachedMerkleTrieLoader.getAccountStateTrieNode( - worldStateKeyValueStorage, location, hash), + bonsaiCachedMerkleTrieLoader.getAccountStateTrieNode( + getWorldStateStorage(), location, hash), worldStateRootHash); // for manicured tries and composting, collect branches here (not implemented) @@ -231,10 +160,10 @@ private void updateTheAccounts( final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater, final StoredMerklePatriciaTrie accountTrie) { - for (final Map.Entry> accountUpdate : + for (final Map.Entry> accountUpdate : worldStateUpdater.getAccountsToUpdate().entrySet()) { final Bytes accountKey = accountUpdate.getKey(); - final BonsaiValue bonsaiValue = accountUpdate.getValue(); + final DiffBasedValue bonsaiValue = accountUpdate.getValue(); final BonsaiAccount updatedAccount = bonsaiValue.getUpdated(); try { if (updatedAccount == null) { @@ -264,7 +193,7 @@ protected void updateCode( final BonsaiWorldStateUpdateAccumulator worldStateUpdater) { maybeStateUpdater.ifPresent( bonsaiUpdater -> { - for (final Map.Entry> codeUpdate : + for (final Map.Entry> codeUpdate : worldStateUpdater.getCodeToUpdate().entrySet()) { final Bytes updatedCode = codeUpdate.getValue().getUpdated(); final Hash accountHash = codeUpdate.getKey().addressHash(); @@ -294,12 +223,12 @@ private boolean codeIsEmpty(final Bytes value) { private void updateAccountStorageState( final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater, - final Map.Entry>> + final Map.Entry>> storageAccountUpdate) { final Address updatedAddress = storageAccountUpdate.getKey(); final Hash updatedAddressHash = updatedAddress.addressHash(); if (worldStateUpdater.getAccountsToUpdate().containsKey(updatedAddress)) { - final BonsaiValue accountValue = + final DiffBasedValue accountValue = worldStateUpdater.getAccountsToUpdate().get(updatedAddress); final BonsaiAccount accountOriginal = accountValue.getPrior(); final Hash storageRoot = @@ -310,12 +239,12 @@ private void updateAccountStorageState( final StoredMerklePatriciaTrie storageTrie = createTrie( (location, key) -> - cachedMerkleTrieLoader.getAccountStorageTrieNode( - worldStateKeyValueStorage, updatedAddressHash, location, key), + bonsaiCachedMerkleTrieLoader.getAccountStorageTrieNode( + getWorldStateStorage(), updatedAddressHash, location, key), storageRoot); // for manicured tries and composting, collect branches here (not implemented) - for (final Map.Entry> storageUpdate : + for (final Map.Entry> storageUpdate : storageAccountUpdate.getValue().entrySet()) { final Hash slotHash = storageUpdate.getKey().getSlotHash(); final UInt256 updatedStorage = storageUpdate.getValue().getUpdated(); @@ -360,11 +289,10 @@ private void updateAccountStorageState( private void clearStorage( final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater) { - for (final Address address : worldStateUpdater.getStorageToClear()) { // because we are clearing persisted values we need the account root as persisted final BonsaiAccount oldAccount = - worldStateKeyValueStorage + getWorldStateStorage() .getAccount(address.addressHash()) .map(bytes -> BonsaiAccount.fromRLP(BonsaiWorldState.this, address, bytes, true)) .orElse(null); @@ -379,7 +307,7 @@ private void clearStorage( (location, key) -> getStorageTrieNode(addressHash, location, key), oldAccount.getStorageRoot()); try { - final StorageConsumingMap> storageToDelete = + final StorageConsumingMap> storageToDelete = worldStateUpdater.getStorageToUpdate().get(address); Map entriesToDelete = storageTrie.entriesFrom(Bytes32.ZERO, 256); while (!entriesToDelete.isEmpty()) { @@ -394,7 +322,7 @@ private void clearStorage( address.addressHash(), storageSlotKey.getSlotHash())); storageToDelete .computeIfAbsent( - storageSlotKey, key -> new BonsaiValue<>(slotValue, null, true)) + storageSlotKey, key -> new DiffBasedValue<>(slotValue, null, true)) .setPrior(slotValue); }); entriesToDelete.keySet().forEach(storageTrie::remove); @@ -412,138 +340,6 @@ private void clearStorage( } } - @Override - public void persist(final BlockHeader blockHeader) { - final Optional maybeBlockHeader = Optional.ofNullable(blockHeader); - LOG.atDebug() - .setMessage("Persist world state for block {}") - .addArgument(maybeBlockHeader) - .log(); - - final BonsaiWorldStateUpdateAccumulator localCopy = accumulator.copy(); - - boolean success = false; - - final BonsaiWorldStateKeyValueStorage.Updater stateUpdater = - worldStateKeyValueStorage.updater(); - Runnable saveTrieLog = () -> {}; - - try { - final Hash newWorldStateRootHash = - calculateRootHash(isFrozen ? Optional.empty() : Optional.of(stateUpdater), accumulator); - // if we are persisted with a block header, and the prior state is the parent - // then persist the TrieLog for that transition. - // If specified but not a direct descendant simply store the new block hash. - if (blockHeader != null) { - verifyWorldStateRoot(newWorldStateRootHash, blockHeader); - saveTrieLog = - () -> { - trieLogManager.saveTrieLog(localCopy, newWorldStateRootHash, blockHeader, this); - // not save a frozen state in the cache - if (!isFrozen) { - cachedWorldStorageManager.addCachedLayer(blockHeader, newWorldStateRootHash, this); - } - }; - - stateUpdater - .getWorldStateTransaction() - .put(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY, blockHeader.getHash().toArrayUnsafe()); - worldStateBlockHash = blockHeader.getHash(); - } else { - stateUpdater.getWorldStateTransaction().remove(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY); - worldStateBlockHash = null; - } - - stateUpdater - .getWorldStateTransaction() - .put(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY, newWorldStateRootHash.toArrayUnsafe()); - worldStateRootHash = newWorldStateRootHash; - success = true; - } finally { - if (success) { - stateUpdater.commit(); - accumulator.reset(); - saveTrieLog.run(); - } else { - stateUpdater.rollback(); - accumulator.reset(); - } - } - } - - protected void verifyWorldStateRoot(final Hash calculatedStateRoot, final BlockHeader header) { - if (!calculatedStateRoot.equals(header.getStateRoot())) { - throw new RuntimeException( - "World State Root does not match expected value, header " - + header.getStateRoot().toHexString() - + " calculated " - + calculatedStateRoot.toHexString()); - } - } - - @Override - public WorldUpdater updater() { - return accumulator; - } - - @Override - public Hash rootHash() { - if (isFrozen && accumulator.isAccumulatorStateChanged()) { - worldStateRootHash = calculateRootHash(Optional.empty(), accumulator.copy()); - accumulator.resetAccumulatorStateChanged(); - } - return Hash.wrap(worldStateRootHash); - } - - static final KeyValueStorageTransaction noOpTx = - new KeyValueStorageTransaction() { - - @Override - public void put(final byte[] key, final byte[] value) { - // no-op - } - - @Override - public void remove(final byte[] key) { - // no-op - } - - @Override - public void commit() throws StorageException { - // no-op - } - - @Override - public void rollback() { - // no-op - } - }; - - static final SegmentedKeyValueStorageTransaction noOpSegmentedTx = - new SegmentedKeyValueStorageTransaction() { - - @Override - public void put( - final SegmentIdentifier segmentIdentifier, final byte[] key, final byte[] value) { - // no-op - } - - @Override - public void remove(final SegmentIdentifier segmentIdentifier, final byte[] key) { - // no-op - } - - @Override - public void commit() throws StorageException { - // no-op - } - - @Override - public void rollback() { - // no-op - } - }; - @Override public Hash frontierRootHash() { return calculateRootHash( @@ -553,25 +349,28 @@ public Hash frontierRootHash() { accumulator.copy()); } - public Hash blockHash() { - return worldStateBlockHash; - } - @Override - public Stream streamAccounts(final Bytes32 startKeyHash, final int limit) { - throw new RuntimeException("Bonsai Tries do not provide account streaming."); + public MutableWorldState freeze() { + this.isFrozen = true; + this.worldStateKeyValueStorage = new BonsaiWorldStateLayerStorage(getWorldStateStorage()); + return this; } @Override public Account get(final Address address) { - return worldStateKeyValueStorage + return getWorldStateStorage() .getAccount(address.addressHash()) .map(bytes -> BonsaiAccount.fromRLP(accumulator, address, bytes, true)) .orElse(null); } + @Override + public Optional getCode(@Nonnull final Address address, final Hash codeHash) { + return getWorldStateStorage().getCode(codeHash, address.addressHash()); + } + protected Optional getAccountStateTrieNode(final Bytes location, final Bytes32 nodeHash) { - return worldStateKeyValueStorage.getAccountStateTrieNode(location, nodeHash); + return getWorldStateStorage().getAccountStateTrieNode(location, nodeHash); } private void writeTrieNode( @@ -584,7 +383,7 @@ private void writeTrieNode( protected Optional getStorageTrieNode( final Hash accountHash, final Bytes location, final Bytes32 nodeHash) { - return worldStateKeyValueStorage.getAccountStorageTrieNode(accountHash, location, nodeHash); + return getWorldStateStorage().getAccountStorageTrieNode(accountHash, location, nodeHash); } private void writeStorageTrieNode( @@ -605,7 +404,7 @@ public UInt256 getStorageValue(final Address address, final UInt256 storageKey) @Override public Optional getStorageValueByStorageSlotKey( final Address address, final StorageSlotKey storageSlotKey) { - return worldStateKeyValueStorage + return getWorldStateStorage() .getStorageValueByStorageSlotKey(address.addressHash(), storageSlotKey) .map(UInt256::fromBytes); } @@ -614,7 +413,7 @@ public Optional getStorageValueByStorageSlotKey( final Supplier> storageRootSupplier, final Address address, final StorageSlotKey storageSlotKey) { - return worldStateKeyValueStorage + return getWorldStateStorage() .getStorageValueByStorageSlotKey(storageRootSupplier, address.addressHash(), storageSlotKey) .map(UInt256::fromBytes); } @@ -632,50 +431,18 @@ public Map getAllAccountStorage(final Address address, final Has return storageTrie.entriesFrom(Bytes32.ZERO, Integer.MAX_VALUE); } - @Override - public MutableWorldState freeze() { - this.isFrozen = true; - this.worldStateKeyValueStorage = new BonsaiWorldStateLayerStorage(worldStateKeyValueStorage); - return this; - } - private StoredMerklePatriciaTrie createTrie( final NodeLoader nodeLoader, final Bytes32 rootHash) { return new StoredMerklePatriciaTrie<>( nodeLoader, rootHash, Function.identity(), Function.identity()); } - @Override - public void close() { - try { - if (!isPersisted()) { - this.worldStateKeyValueStorage.close(); - if (isFrozen) { - closeFrozenStorage(); - } - } - } catch (Exception e) { - // no op - } - } - - private void closeFrozenStorage() { - try { - final BonsaiWorldStateLayerStorage worldStateLayerStorage = - (BonsaiWorldStateLayerStorage) worldStateKeyValueStorage; - if (!isPersisted(worldStateLayerStorage.getParentWorldStateStorage())) { - worldStateLayerStorage.getParentWorldStateStorage().close(); - } - } catch (Exception e) { - // no op - } - } - protected Hash hashAndSavePreImage(final Bytes value) { // by default do not save has preImages return Hash.hash(value); } + @Override protected Hash getEmptyTrieHash() { return Hash.EMPTY_TRIE_HASH; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java new file mode 100644 index 00000000000..ff2e696e163 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java @@ -0,0 +1,98 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview; + +import org.hyperledger.besu.datatypes.AccountValue; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.Consumer; +import org.hyperledger.besu.evm.internal.EvmConfiguration; +import org.hyperledger.besu.evm.worldstate.UpdateTrackingAccount; + +public class BonsaiWorldStateUpdateAccumulator + extends DiffBasedWorldStateUpdateAccumulator { + public BonsaiWorldStateUpdateAccumulator( + final DiffBasedWorldView world, + final Consumer> accountPreloader, + final Consumer storagePreloader, + final EvmConfiguration evmConfiguration) { + super(world, accountPreloader, storagePreloader, evmConfiguration); + } + + @Override + public DiffBasedWorldStateUpdateAccumulator copy() { + final BonsaiWorldStateUpdateAccumulator copy = + new BonsaiWorldStateUpdateAccumulator( + wrappedWorldView(), + getAccountPreloader(), + getStoragePreloader(), + getEvmConfiguration()); + copy.cloneFromUpdater(this); + return copy; + } + + @Override + protected BonsaiAccount copyAccount(final BonsaiAccount account) { + return new BonsaiAccount(account); + } + + @Override + protected BonsaiAccount copyAccount( + final BonsaiAccount toCopy, final DiffBasedWorldView context, final boolean mutable) { + return new BonsaiAccount(toCopy, context, mutable); + } + + @Override + protected BonsaiAccount createAccount( + final DiffBasedWorldView context, + final Address address, + final AccountValue stateTrieAccount, + final boolean mutable) { + return new BonsaiAccount(context, address, stateTrieAccount, mutable); + } + + @Override + protected BonsaiAccount createAccount( + final DiffBasedWorldView context, + final Address address, + final Hash addressHash, + final long nonce, + final Wei balance, + final Hash storageRoot, + final Hash codeHash, + final boolean mutable) { + return new BonsaiAccount( + context, address, addressHash, nonce, balance, storageRoot, codeHash, mutable); + } + + @Override + protected BonsaiAccount createAccount( + final DiffBasedWorldView context, final UpdateTrackingAccount tracked) { + return new BonsaiAccount(context, tracked); + } + + @Override + protected void assertCloseEnoughForDiffing( + final BonsaiAccount source, final AccountValue account, final String context) { + BonsaiAccount.assertCloseEnoughForDiffing(source, account, context); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java new file mode 100644 index 00000000000..044deb45a01 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java @@ -0,0 +1,206 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.ethereum.trie.diffbased.common; + +import org.hyperledger.besu.datatypes.AccountValue; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; +import org.hyperledger.besu.evm.ModificationNotAllowedException; +import org.hyperledger.besu.evm.account.MutableAccount; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; + +public abstract class DiffBasedAccount implements MutableAccount, AccountValue { + protected final DiffBasedWorldView context; + protected boolean immutable; + protected final Address address; + protected final Hash addressHash; + protected Hash codeHash; + protected long nonce; + protected Wei balance; + protected Bytes code; + + protected final Map updatedStorage = new HashMap<>(); + + public DiffBasedAccount( + final DiffBasedWorldView context, + final Address address, + final Hash addressHash, + final long nonce, + final Wei balance, + final Hash codeHash, + final boolean mutable) { + this.context = context; + this.address = address; + this.addressHash = addressHash; + this.nonce = nonce; + this.balance = balance; + this.codeHash = codeHash; + + this.immutable = !mutable; + } + + public DiffBasedAccount( + final DiffBasedWorldView context, + final Address address, + final AccountValue stateTrieAccount, + final boolean mutable) { + this( + context, + address, + address.addressHash(), + stateTrieAccount.getNonce(), + stateTrieAccount.getBalance(), + stateTrieAccount.getCodeHash(), + mutable); + } + + public DiffBasedAccount( + final DiffBasedAccount toCopy, final DiffBasedWorldView context, final boolean mutable) { + this.context = context; + this.address = toCopy.address; + this.addressHash = toCopy.addressHash; + this.nonce = toCopy.nonce; + this.balance = toCopy.balance; + this.codeHash = toCopy.codeHash; + this.code = toCopy.code; + updatedStorage.putAll(toCopy.updatedStorage); + + this.immutable = !mutable; + } + + @Override + public Address getAddress() { + return address; + } + + @Override + public Hash getAddressHash() { + return addressHash; + } + + @Override + public long getNonce() { + return nonce; + } + + @Override + public void setNonce(final long value) { + if (immutable) { + throw new ModificationNotAllowedException(); + } + nonce = value; + } + + @Override + public Wei getBalance() { + return balance; + } + + @Override + public void setBalance(final Wei value) { + if (immutable) { + throw new ModificationNotAllowedException(); + } + balance = value; + } + + @Override + public Bytes getCode() { + if (code == null) { + code = context.getCode(address, codeHash).orElse(Bytes.EMPTY); + } + return code; + } + + @Override + public void setCode(final Bytes code) { + if (immutable) { + throw new ModificationNotAllowedException(); + } + this.code = code; + if (code == null || code.isEmpty()) { + this.codeHash = Hash.EMPTY; + } else { + this.codeHash = Hash.hash(code); + } + } + + @Override + public Hash getCodeHash() { + return codeHash; + } + + @Override + public UInt256 getStorageValue(final UInt256 key) { + return context.getStorageValue(address, key); + } + + @Override + public UInt256 getOriginalStorageValue(final UInt256 key) { + return context.getPriorStorageValue(address, key); + } + + public Bytes serializeAccount() { + final BytesValueRLPOutput out = new BytesValueRLPOutput(); + writeTo(out); + return out.encoded(); + } + + @Override + public void setStorageValue(final UInt256 key, final UInt256 value) { + if (immutable) { + throw new ModificationNotAllowedException(); + } + updatedStorage.put(key, value); + } + + @Override + public void clearStorage() { + updatedStorage.clear(); + } + + @Override + public Map getUpdatedStorage() { + return updatedStorage; + } + + @Override + public void becomeImmutable() { + immutable = true; + } + + @Override + public String toString() { + return "AccountState{" + + "address=" + + address + + ", nonce=" + + nonce + + ", balance=" + + balance + + ", codeHash=" + + codeHash + + '}'; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedValue.java similarity index 80% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedValue.java index c99da3fe24b..bd9c3e3f514 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedValue.java @@ -1,5 +1,5 @@ /* - * Copyright ConsenSys AG. + * Copyright Hyperledger Besu Contributors. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -11,38 +11,37 @@ * specific language governing permissions and limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.common; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; -public class BonsaiValue implements TrieLog.LogTuple { +public class DiffBasedValue implements TrieLog.LogTuple { private T prior; private T updated; private boolean lastStepCleared; private boolean clearedAtLeastOnce; - public BonsaiValue(final T prior, final T updated) { + public DiffBasedValue(final T prior, final T updated) { this.prior = prior; this.updated = updated; this.lastStepCleared = false; this.clearedAtLeastOnce = false; } - public BonsaiValue(final T prior, final T updated, final boolean lastStepCleared) { + public DiffBasedValue(final T prior, final T updated, final boolean lastStepCleared) { this.prior = prior; this.updated = updated; this.lastStepCleared = lastStepCleared; this.clearedAtLeastOnce = lastStepCleared; } - public BonsaiValue( + public DiffBasedValue( final T prior, final T updated, final boolean lastStepCleared, @@ -63,12 +62,12 @@ public T getUpdated() { return updated; } - public BonsaiValue setPrior(final T prior) { + public DiffBasedValue setPrior(final T prior) { this.prior = prior; return this; } - public BonsaiValue setUpdated(final T updated) { + public DiffBasedValue setUpdated(final T updated) { this.lastStepCleared = updated == null; if (lastStepCleared) { this.clearedAtLeastOnce = true; @@ -94,7 +93,7 @@ public boolean isClearedAtLeastOnce() { @Override public String toString() { - return "BonsaiValue{" + return "DiffBasedValue{" + "prior=" + prior + ", updated=" @@ -112,7 +111,7 @@ public boolean equals(final Object o) { if (o == null || getClass() != o.getClass()) { return false; } - BonsaiValue that = (BonsaiValue) o; + DiffBasedValue that = (DiffBasedValue) o; return new EqualsBuilder() .append(lastStepCleared, that.lastStepCleared) .append(prior, that.prior) @@ -129,7 +128,7 @@ public int hashCode() { .toHashCode(); } - public BonsaiValue copy() { - return new BonsaiValue(prior, updated, lastStepCleared, clearedAtLeastOnce); + public DiffBasedValue copy() { + return new DiffBasedValue(prior, updated, lastStepCleared, clearedAtLeastOnce); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedWorldStateProvider.java similarity index 62% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedWorldStateProvider.java index 0230a2258bc..18bf333828b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedWorldStateProvider.java @@ -14,9 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; - -import static org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager.RETAINED_LAYERS; +package org.hyperledger.besu.ethereum.trie.diffbased.common; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; @@ -25,91 +23,75 @@ import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.proof.WorldStateProof; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; -import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; -import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; -import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldState; import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.function.Function; -import com.google.common.annotations.VisibleForTesting; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.units.bigints.UInt256; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class BonsaiWorldStateProvider implements WorldStateArchive { +public abstract class DiffBasedWorldStateProvider implements WorldStateArchive { + + private static final Logger LOG = LoggerFactory.getLogger(DiffBasedWorldStateProvider.class); - private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateProvider.class); + protected final Blockchain blockchain; - private final Blockchain blockchain; + protected final TrieLogManager trieLogManager; + protected DiffBasedCachedWorldStorageManager cachedWorldStorageManager; + protected DiffBasedWorldState persistedState; - private final CachedWorldStorageManager cachedWorldStorageManager; - private final TrieLogManager trieLogManager; - private final BonsaiWorldState persistedState; - private final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage; - private final CachedMerkleTrieLoader cachedMerkleTrieLoader; + protected final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage; - public BonsaiWorldStateProvider( - final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, + public DiffBasedWorldStateProvider( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, final Blockchain blockchain, final Optional maxLayersToLoad, - final CachedMerkleTrieLoader cachedMerkleTrieLoader, - final BesuContext pluginContext, - final EvmConfiguration evmConfiguration) { + final BesuContext pluginContext) { this.worldStateKeyValueStorage = worldStateKeyValueStorage; - this.cachedWorldStorageManager = new CachedWorldStorageManager(this, worldStateKeyValueStorage); - // TODO: de-dup constructors this.trieLogManager = new TrieLogManager( blockchain, worldStateKeyValueStorage, - maxLayersToLoad.orElse(RETAINED_LAYERS), + maxLayersToLoad.orElse(DiffBasedCachedWorldStorageManager.RETAINED_LAYERS), pluginContext); this.blockchain = blockchain; - this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; - this.persistedState = new BonsaiWorldState(this, worldStateKeyValueStorage, evmConfiguration); - blockchain - .getBlockHeader(persistedState.getWorldStateBlockHash()) - .ifPresent( - blockHeader -> - this.cachedWorldStorageManager.addCachedLayer( - blockHeader, persistedState.getWorldStateRootHash(), persistedState)); } - @VisibleForTesting - BonsaiWorldStateProvider( - final CachedWorldStorageManager cachedWorldStorageManager, - final TrieLogManager trieLogManager, - final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, + public DiffBasedWorldStateProvider( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, final Blockchain blockchain, - final CachedMerkleTrieLoader cachedMerkleTrieLoader, - final EvmConfiguration evmConfiguration) { - this.cachedWorldStorageManager = cachedWorldStorageManager; + final TrieLogManager trieLogManager) { + + this.worldStateKeyValueStorage = worldStateKeyValueStorage; + // TODO: de-dup constructors this.trieLogManager = trieLogManager; this.blockchain = blockchain; - this.worldStateKeyValueStorage = worldStateKeyValueStorage; - this.persistedState = new BonsaiWorldState(this, worldStateKeyValueStorage, evmConfiguration); - this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; + } + + protected void provideCachedWorldStorageManager( + final DiffBasedCachedWorldStorageManager cachedWorldStorageManager) { + this.cachedWorldStorageManager = cachedWorldStorageManager; + } + + protected void loadPersistedState(final DiffBasedWorldState persistedState) { + this.persistedState = persistedState; blockchain .getBlockHeader(persistedState.getWorldStateBlockHash()) .ifPresent( @@ -158,9 +140,7 @@ public Optional getMutable( .getWorldState(blockHeader.getHash()) .or(() -> cachedWorldStorageManager.getNearestWorldState(blockHeader)) .or(() -> cachedWorldStorageManager.getHeadWorldState(blockchain::getBlockHeader)) - .flatMap( - bonsaiWorldState -> - rollMutableStateToBlockHash(bonsaiWorldState, blockHeader.getHash())) + .flatMap(worldState -> rollMutableStateToBlockHash(worldState, blockHeader.getHash())) .map(MutableWorldState::freeze); } } @@ -172,7 +152,7 @@ public synchronized Optional getMutable( } Optional rollMutableStateToBlockHash( - final BonsaiWorldState mutableState, final Hash blockHash) { + final DiffBasedWorldState mutableState, final Hash blockHash) { if (blockHash.equals(mutableState.blockHash())) { return Optional.of(mutableState); } else { @@ -221,19 +201,19 @@ Optional rollMutableStateToBlockHash( } // attempt the state rolling - final BonsaiWorldStateUpdateAccumulator bonsaiUpdater = - (BonsaiWorldStateUpdateAccumulator) mutableState.updater(); + final DiffBasedWorldStateUpdateAccumulator diffBasedUpdater = + (DiffBasedWorldStateUpdateAccumulator) mutableState.updater(); try { for (final TrieLog rollBack : rollBacks) { LOG.debug("Attempting Rollback of {}", rollBack.getBlockHash()); - bonsaiUpdater.rollBack(rollBack); + diffBasedUpdater.rollBack(rollBack); } for (int i = rollForwards.size() - 1; i >= 0; i--) { final var forward = rollForwards.get(i); LOG.debug("Attempting Rollforward of {}", rollForwards.get(i).getBlockHash()); - bonsaiUpdater.rollForward(forward); + diffBasedUpdater.rollForward(forward); } - bonsaiUpdater.commit(); + diffBasedUpdater.commit(); mutableState.persist(blockchain.getBlockHeader(blockHash).get()); @@ -247,7 +227,7 @@ Optional rollMutableStateToBlockHash( throw re; } catch (final Exception e) { // if we fail we must clean up the updater - bonsaiUpdater.reset(); + diffBasedUpdater.reset(); LOG.debug( "State rolling failed on " + mutableState.getWorldStateStorage().getClass().getSimpleName() @@ -269,80 +249,16 @@ Optional rollMutableStateToBlockHash( } } - public CachedMerkleTrieLoader getCachedMerkleTrieLoader() { - return cachedMerkleTrieLoader; - } - @Override public MutableWorldState getMutable() { return persistedState; } - /** - * Prepares the state healing process for a given address and location. It prepares the state - * healing, including retrieving data from storage, identifying invalid slots or nodes, removing - * account and slot from the state trie, and committing the changes. Finally, it downgrades the - * world state storage to partial flat database mode. - */ - public void prepareStateHealing(final Address address, final Bytes location) { - final Set keysToDelete = new HashSet<>(); - final BonsaiWorldStateKeyValueStorage.Updater updater = worldStateKeyValueStorage.updater(); - final Hash accountHash = address.addressHash(); - final StoredMerklePatriciaTrie accountTrie = - new StoredMerklePatriciaTrie<>( - (l, h) -> { - final Optional node = worldStateKeyValueStorage.getAccountStateTrieNode(l, h); - if (node.isPresent()) { - keysToDelete.add(l); - } - return node; - }, - persistedState.getWorldStateRootHash(), - Function.identity(), - Function.identity()); - try { - accountTrie - .get(accountHash) - .map(RLP::input) - .map(StateTrieAccountValue::readFrom) - .ifPresent( - account -> { - final StoredMerklePatriciaTrie storageTrie = - new StoredMerklePatriciaTrie<>( - (l, h) -> { - Optional node = - worldStateKeyValueStorage.getAccountStorageTrieNode( - accountHash, l, h); - if (node.isPresent()) { - keysToDelete.add(Bytes.concatenate(accountHash, l)); - } - return node; - }, - account.getStorageRoot(), - Function.identity(), - Function.identity()); - try { - storageTrie.getPath(location); - } catch (Exception eA) { - LOG.warn("Invalid slot found for account {} at location {}", address, location); - // ignore - } - }); - } catch (Exception eA) { - LOG.warn("Invalid node for account {} at location {}", address, location); - // ignore - } - keysToDelete.forEach(bytes -> updater.removeAccountStateTrieNode(bytes)); - updater.commit(); - - worldStateKeyValueStorage.downgradeToPartialFlatDbMode(); - } - public TrieLogManager getTrieLogManager() { return trieLogManager; } - public CachedWorldStorageManager getCachedWorldStorageManager() { + public DiffBasedCachedWorldStorageManager getCachedWorldStorageManager() { return cachedWorldStorageManager; } @@ -360,7 +276,8 @@ public Optional getAccountProof( final Address accountAddress, final List accountStorageKeys, final Function, ? extends Optional> mapper) { - try (BonsaiWorldState ws = (BonsaiWorldState) getMutable(blockHeader, false).orElse(null)) { + try (DiffBasedWorldState ws = + (DiffBasedWorldState) getMutable(blockHeader, false).orElse(null)) { if (ws != null) { final WorldStateProofProvider worldStateProofProvider = new WorldStateProofProvider( diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/StorageSubscriber.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/StorageSubscriber.java new file mode 100644 index 00000000000..fe16cf381e9 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/StorageSubscriber.java @@ -0,0 +1,25 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common; + +public interface StorageSubscriber { + default void onClearStorage() {} + + default void onClearFlatDatabaseStorage() {} + + default void onClearTrieLog() {} + + default void onCloseStorage() {} +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldStorageManager.java similarity index 62% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedWorldStorageManager.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldStorageManager.java index 3944b72a732..287213aa57a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedWorldStorageManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldStorageManager.java @@ -12,15 +12,15 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.cache; +package org.hyperledger.besu.ethereum.trie.diffbased.common.cache; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateLayerStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedLayeredWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; import org.hyperledger.besu.evm.internal.EvmConfiguration; import java.util.ArrayList; @@ -35,20 +35,20 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class CachedWorldStorageManager - implements BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber { +public abstract class DiffBasedCachedWorldStorageManager implements StorageSubscriber { public static final long RETAINED_LAYERS = 512; // at least 256 + typical rollbacks - private static final Logger LOG = LoggerFactory.getLogger(CachedWorldStorageManager.class); - private final BonsaiWorldStateProvider archive; + private static final Logger LOG = + LoggerFactory.getLogger(DiffBasedCachedWorldStorageManager.class); + private final DiffBasedWorldStateProvider archive; private final EvmConfiguration evmConfiguration; - private final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; - private final Map cachedWorldStatesByHash; + private final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage; + private final Map cachedWorldStatesByHash; - private CachedWorldStorageManager( - final BonsaiWorldStateProvider archive, - final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, - final Map cachedWorldStatesByHash, + private DiffBasedCachedWorldStorageManager( + final DiffBasedWorldStateProvider archive, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final Map cachedWorldStatesByHash, final EvmConfiguration evmConfiguration) { worldStateKeyValueStorage.subscribe(this); this.rootWorldStateStorage = worldStateKeyValueStorage; @@ -57,32 +57,32 @@ private CachedWorldStorageManager( this.evmConfiguration = evmConfiguration; } - public CachedWorldStorageManager( - final BonsaiWorldStateProvider archive, - final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage) { + public DiffBasedCachedWorldStorageManager( + final DiffBasedWorldStateProvider archive, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage) { this(archive, worldStateKeyValueStorage, new ConcurrentHashMap<>(), EvmConfiguration.DEFAULT); } public synchronized void addCachedLayer( final BlockHeader blockHeader, final Hash worldStateRootHash, - final BonsaiWorldState forWorldState) { - final Optional cachedBonsaiWorldView = + final DiffBasedWorldState forWorldState) { + final Optional cachedDiffBasedWorldView = Optional.ofNullable(this.cachedWorldStatesByHash.get(blockHeader.getBlockHash())); - if (cachedBonsaiWorldView.isPresent()) { + if (cachedDiffBasedWorldView.isPresent()) { // only replace if it is a layered storage if (forWorldState.isPersisted() - && cachedBonsaiWorldView.get().getWorldStateStorage() - instanceof BonsaiWorldStateLayerStorage) { + && cachedDiffBasedWorldView.get().getWorldStateStorage() + instanceof DiffBasedLayeredWorldStateKeyValueStorage) { LOG.atDebug() .setMessage("updating layered world state for block {}, state root hash {}") .addArgument(blockHeader::toLogString) .addArgument(worldStateRootHash::toShortHexString) .log(); - cachedBonsaiWorldView + cachedDiffBasedWorldView .get() .updateWorldStateStorage( - new BonsaiSnapshotWorldStateKeyValueStorage(forWorldState.getWorldStateStorage())); + createSnapshotKeyValueStorage(forWorldState.getWorldStateStorage())); } } else { LOG.atDebug() @@ -93,16 +93,16 @@ public synchronized void addCachedLayer( if (forWorldState.isPersisted()) { cachedWorldStatesByHash.put( blockHeader.getHash(), - new CachedBonsaiWorldView( - blockHeader, - new BonsaiSnapshotWorldStateKeyValueStorage(forWorldState.getWorldStateStorage()))); + new DiffBasedCachedWorldView( + blockHeader, createSnapshotKeyValueStorage(forWorldState.getWorldStateStorage()))); } else { // otherwise, add the layer to the cache cachedWorldStatesByHash.put( blockHeader.getHash(), - new CachedBonsaiWorldView( + new DiffBasedCachedWorldView( blockHeader, - ((BonsaiWorldStateLayerStorage) forWorldState.getWorldStateStorage()).clone())); + ((DiffBasedLayeredWorldStateKeyValueStorage) forWorldState.getWorldStateStorage()) + .clone())); } } scrubCachedLayers(blockHeader.getNumber()); @@ -122,15 +122,15 @@ private synchronized void scrubCachedLayers(final long newMaxHeight) { } } - public Optional getWorldState(final Hash blockHash) { + public Optional getWorldState(final Hash blockHash) { if (cachedWorldStatesByHash.containsKey(blockHash)) { // return a new worldstate using worldstate storage and an isolated copy of the updater return Optional.ofNullable(cachedWorldStatesByHash.get(blockHash)) .map( cached -> - new BonsaiWorldState( + createWorldState( archive, - new BonsaiWorldStateLayerStorage(cached.getWorldStateStorage()), + createLayeredKeyValueStorage(cached.getWorldStateStorage()), evmConfiguration)); } LOG.atDebug() @@ -141,7 +141,7 @@ public Optional getWorldState(final Hash blockHash) { return Optional.empty(); } - public Optional getNearestWorldState(final BlockHeader blockHeader) { + public Optional getNearestWorldState(final BlockHeader blockHeader) { LOG.atDebug() .setMessage("getting nearest worldstate for {}") .addArgument(blockHeader.toLogString()) @@ -149,7 +149,7 @@ public Optional getNearestWorldState(final BlockHeader blockHe return Optional.ofNullable( cachedWorldStatesByHash.get(blockHeader.getParentHash())) // search parent block - .map(CachedBonsaiWorldView::getWorldStateStorage) + .map(DiffBasedCachedWorldView::getWorldStateStorage) .or( () -> { // or else search the nearest state in the cache @@ -158,22 +158,22 @@ public Optional getNearestWorldState(final BlockHeader blockHe .addArgument(blockHeader.toLogString()) .log(); - final List cachedBonsaiWorldViews = + final List cachedDiffBasedWorldViews = new ArrayList<>(cachedWorldStatesByHash.values()); - return cachedBonsaiWorldViews.stream() + return cachedDiffBasedWorldViews.stream() .sorted( Comparator.comparingLong( view -> Math.abs(blockHeader.getNumber() - view.getBlockNumber()))) - .map(CachedBonsaiWorldView::getWorldStateStorage) + .map(DiffBasedCachedWorldView::getWorldStateStorage) .findFirst(); }) .map( storage -> - new BonsaiWorldState( // wrap the state in a layered worldstate - archive, new BonsaiWorldStateLayerStorage(storage), evmConfiguration)); + createWorldState( // wrap the state in a layered worldstate + archive, createLayeredKeyValueStorage(storage), evmConfiguration)); } - public Optional getHeadWorldState( + public Optional getHeadWorldState( final Function> hashBlockHeaderFunction) { LOG.atDebug().setMessage("getting head worldstate").log(); @@ -187,7 +187,7 @@ public Optional getHeadWorldState( addCachedLayer( blockHeader, blockHeader.getStateRoot(), - new BonsaiWorldState(archive, rootWorldStateStorage, evmConfiguration)); + createWorldState(archive, rootWorldStateStorage, evmConfiguration)); return getWorldState(blockHeader.getHash()); }); } @@ -219,4 +219,15 @@ public void onClearTrieLog() { public void onCloseStorage() { this.cachedWorldStatesByHash.clear(); } + + public abstract DiffBasedWorldState createWorldState( + final DiffBasedWorldStateProvider archive, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final EvmConfiguration evmConfiguration); + + public abstract DiffBasedWorldStateKeyValueStorage createLayeredKeyValueStorage( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage); + + public abstract DiffBasedWorldStateKeyValueStorage createSnapshotKeyValueStorage( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedBonsaiWorldView.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldView.java similarity index 71% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedBonsaiWorldView.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldView.java index 2b8fd69d114..951941f0ce1 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedBonsaiWorldView.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldView.java @@ -13,30 +13,30 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.cache; +package org.hyperledger.besu.ethereum.trie.diffbased.common.cache; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class CachedBonsaiWorldView - implements BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber { - private BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage; +public class DiffBasedCachedWorldView implements StorageSubscriber { + private DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage; private final BlockHeader blockHeader; private long worldViewSubscriberId; - private static final Logger LOG = LoggerFactory.getLogger(CachedBonsaiWorldView.class); + private static final Logger LOG = LoggerFactory.getLogger(DiffBasedCachedWorldView.class); - public CachedBonsaiWorldView( - final BlockHeader blockHeader, final BonsaiWorldStateKeyValueStorage worldView) { + public DiffBasedCachedWorldView( + final BlockHeader blockHeader, final DiffBasedWorldStateKeyValueStorage worldView) { this.blockHeader = blockHeader; this.worldStateKeyValueStorage = worldView; this.worldViewSubscriberId = worldStateKeyValueStorage.subscribe(this); } - public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { + public DiffBasedWorldStateKeyValueStorage getWorldStateStorage() { return worldStateKeyValueStorage; } @@ -58,10 +58,10 @@ public synchronized void close() { } public synchronized void updateWorldStateStorage( - final BonsaiWorldStateKeyValueStorage newWorldStateStorage) { + final DiffBasedWorldStateKeyValueStorage newWorldStateStorage) { long newSubscriberId = newWorldStateStorage.subscribe(this); this.worldStateKeyValueStorage.unSubscribe(this.worldViewSubscriberId); - BonsaiWorldStateKeyValueStorage oldWorldStateStorage = this.worldStateKeyValueStorage; + final DiffBasedWorldStateKeyValueStorage oldWorldStateStorage = this.worldStateKeyValueStorage; this.worldStateKeyValueStorage = newWorldStateStorage; this.worldViewSubscriberId = newSubscriberId; try { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedLayeredWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedLayeredWorldStateKeyValueStorage.java new file mode 100644 index 00000000000..a561072dc05 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedLayeredWorldStateKeyValueStorage.java @@ -0,0 +1,22 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage; + +public interface DiffBasedLayeredWorldStateKeyValueStorage + extends DiffBasedSnapshotWorldStateKeyValueStorage { + + DiffBasedWorldStateKeyValueStorage clone(); +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java new file mode 100644 index 00000000000..9c1a691cf31 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java @@ -0,0 +1,21 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage; + +public interface DiffBasedSnapshotWorldStateKeyValueStorage { + + DiffBasedWorldStateKeyValueStorage getParentWorldStateStorage(); +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedWorldStateKeyValueStorage.java new file mode 100644 index 00000000000..2424828ee60 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedWorldStateKeyValueStorage.java @@ -0,0 +1,234 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage; + +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.storage.StorageProvider; +import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy; +import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; +import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; +import org.hyperledger.besu.util.Subscribers; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Stream; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class DiffBasedWorldStateKeyValueStorage + implements WorldStateKeyValueStorage, AutoCloseable { + private static final Logger LOG = + LoggerFactory.getLogger(DiffBasedWorldStateKeyValueStorage.class); + + // 0x776f726c64526f6f74 + public static final byte[] WORLD_ROOT_HASH_KEY = "worldRoot".getBytes(StandardCharsets.UTF_8); + // 0x776f726c64426c6f636b48617368 + public static final byte[] WORLD_BLOCK_HASH_KEY = + "worldBlockHash".getBytes(StandardCharsets.UTF_8); + + private final AtomicBoolean shouldClose = new AtomicBoolean(false); + + protected final AtomicBoolean isClosed = new AtomicBoolean(false); + + protected final Subscribers subscribers = Subscribers.create(); + protected final SegmentedKeyValueStorage composedWorldStateStorage; + protected final KeyValueStorage trieLogStorage; + + public DiffBasedWorldStateKeyValueStorage(final StorageProvider provider) { + this.composedWorldStateStorage = + provider.getStorageBySegmentIdentifiers( + List.of( + ACCOUNT_INFO_STATE, CODE_STORAGE, ACCOUNT_STORAGE_STORAGE, TRIE_BRANCH_STORAGE)); + this.trieLogStorage = + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); + } + + public DiffBasedWorldStateKeyValueStorage( + final SegmentedKeyValueStorage composedWorldStateStorage, + final KeyValueStorage trieLogStorage) { + this.composedWorldStateStorage = composedWorldStateStorage; + this.trieLogStorage = trieLogStorage; + } + + public abstract FlatDbMode getFlatDbMode(); + + public abstract FlatDbStrategy getFlatDbStrategy(); + + @Override + public abstract DataStorageFormat getDataStorageFormat(); + + public SegmentedKeyValueStorage getComposedWorldStateStorage() { + return composedWorldStateStorage; + } + + public KeyValueStorage getTrieLogStorage() { + return trieLogStorage; + } + + public Optional getTrieLog(final Hash blockHash) { + return trieLogStorage.get(blockHash.toArrayUnsafe()); + } + + public Stream streamTrieLogKeys(final long limit) { + return trieLogStorage.streamKeys().limit(limit); + } + + public Optional getStateTrieNode(final Bytes location) { + return composedWorldStateStorage + .get(TRIE_BRANCH_STORAGE, location.toArrayUnsafe()) + .map(Bytes::wrap); + } + + public Optional getWorldStateRootHash() { + return composedWorldStateStorage.get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY).map(Bytes::wrap); + } + + public Optional getWorldStateBlockHash() { + return composedWorldStateStorage + .get(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY) + .map(Bytes32::wrap) + .map(Hash::wrap); + } + + public Map streamFlatAccounts( + final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { + return getFlatDbStrategy() + .streamAccountFlatDatabase(composedWorldStateStorage, startKeyHash, endKeyHash, max); + } + + public Map streamFlatStorages( + final Hash accountHash, final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { + return getFlatDbStrategy() + .streamStorageFlatDatabase( + composedWorldStateStorage, accountHash, startKeyHash, endKeyHash, max); + } + + public boolean isWorldStateAvailable(final Bytes32 rootHash, final Hash blockHash) { + return composedWorldStateStorage + .get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY) + .map(Bytes32::wrap) + .map(hash -> hash.equals(rootHash) || trieLogStorage.containsKey(blockHash.toArrayUnsafe())) + .orElse(false); + } + + @Override + public void clear() { + subscribers.forEach(StorageSubscriber::onClearStorage); + getFlatDbStrategy().clearAll(composedWorldStateStorage); + composedWorldStateStorage.clear(TRIE_BRANCH_STORAGE); + trieLogStorage.clear(); + } + + public void clearTrieLog() { + subscribers.forEach(StorageSubscriber::onClearTrieLog); + trieLogStorage.clear(); + } + + public void clearFlatDatabase() { + subscribers.forEach(StorageSubscriber::onClearFlatDatabaseStorage); + getFlatDbStrategy().resetOnResync(composedWorldStateStorage); + } + + @Override + public abstract Updater updater(); + + public boolean pruneTrieLog(final Hash blockHash) { + try { + return trieLogStorage.tryDelete(blockHash.toArrayUnsafe()); + } catch (Exception e) { + LOG.error("Error pruning trie log for block hash {}", blockHash, e); + return false; + } + } + + @Override + public synchronized void close() throws Exception { + // when the storage clears, close + shouldClose.set(true); + tryClose(); + } + + public synchronized long subscribe(final StorageSubscriber sub) { + if (isClosed.get()) { + throw new RuntimeException("Storage is marked to close or has already closed"); + } + return subscribers.subscribe(sub); + } + + public synchronized void unSubscribe(final long id) { + subscribers.unsubscribe(id); + try { + tryClose(); + } catch (Exception e) { + LOG.atWarn() + .setMessage("exception while trying to close : {}") + .addArgument(e::getMessage) + .log(); + } + } + + protected synchronized void tryClose() throws Exception { + if (shouldClose.get() && subscribers.getSubscriberCount() < 1) { + doClose(); + } + } + + protected synchronized void doClose() throws Exception { + if (!isClosed.get()) { + // alert any subscribers we are closing: + subscribers.forEach(StorageSubscriber::onCloseStorage); + + // close all of the KeyValueStorages: + composedWorldStateStorage.close(); + trieLogStorage.close(); + + // set storage closed + isClosed.set(true); + } + } + + public interface Updater extends WorldStateKeyValueStorage.Updater { + + DiffBasedWorldStateKeyValueStorage.Updater saveWorldState( + final Bytes blockHash, final Bytes32 nodeHash, final Bytes node); + + SegmentedKeyValueStorageTransaction getWorldStateTransaction(); + + KeyValueStorageTransaction getTrieLogStorageTransaction(); + + @Override + void commit(); + + void rollback(); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/AccountHashCodeStorageStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/AccountHashCodeStorageStrategy.java similarity index 96% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/AccountHashCodeStorageStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/AccountHashCodeStorageStrategy.java index 7d77ff1a6fc..446ed3262a1 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/AccountHashCodeStorageStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/AccountHashCodeStorageStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeHashCodeStorageStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/CodeHashCodeStorageStrategy.java similarity index 96% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeHashCodeStorageStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/CodeHashCodeStorageStrategy.java index 6f4d8bb8147..4263e423b86 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeHashCodeStorageStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/CodeHashCodeStorageStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeStorageStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/CodeStorageStrategy.java similarity index 94% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeStorageStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/CodeStorageStrategy.java index 8767be24788..91da4eff066 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeStorageStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/CodeStorageStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategy.java similarity index 99% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategy.java index d85f4932516..37c2877b167 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java similarity index 95% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java index e251f28c175..7614c1499b8 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java @@ -13,11 +13,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.FullFlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.PartialFlatDbStrategy; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.plugin.services.MetricsSystem; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java similarity index 79% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java index 4c9a520bd2f..d74a6804d92 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java @@ -12,12 +12,12 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import java.util.Optional; @@ -30,10 +30,10 @@ public NoOpTrieLogManager() { @Override public synchronized void saveTrieLog( - final BonsaiWorldStateUpdateAccumulator localUpdater, + final DiffBasedWorldStateUpdateAccumulator localUpdater, final Hash forWorldStateRootHash, final BlockHeader forBlockHeader, - final BonsaiWorldState forWorldState) { + final DiffBasedWorldState forWorldState) { // notify trie log added observers, synchronously TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogAddedEvent.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogAddedEvent.java similarity index 92% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogAddedEvent.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogAddedEvent.java index 3e1387997b4..28edc5cc608 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogAddedEvent.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogAddedEvent.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayer.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayer.java similarity index 79% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayer.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayer.java index ff1e29c41b8..ff6ef226ef7 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayer.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayer.java @@ -14,7 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import static com.google.common.base.Preconditions.checkState; @@ -22,7 +22,7 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import java.util.HashMap; @@ -49,21 +49,21 @@ public class TrieLogLayer implements TrieLog { protected Hash blockHash; protected Optional blockNumber = Optional.empty(); - Map> getAccounts() { + Map> getAccounts() { return accounts; } - Map> getCode() { + Map> getCode() { return code; } - Map>> getStorage() { + Map>> getStorage() { return storage; } - protected final Map> accounts; - protected final Map> code; - protected final Map>> storage; + protected final Map> accounts; + protected final Map> code; + protected final Map>> storage; protected boolean frozen = false; public TrieLogLayer() { @@ -104,14 +104,14 @@ public TrieLogLayer setBlockNumber(final long blockNumber) { public TrieLogLayer addAccountChange( final Address address, final AccountValue oldValue, final AccountValue newValue) { checkState(!frozen, "Layer is Frozen"); - accounts.put(address, new BonsaiValue<>(oldValue, newValue)); + accounts.put(address, new DiffBasedValue<>(oldValue, newValue)); return this; } public TrieLogLayer addCodeChange( final Address address, final Bytes oldValue, final Bytes newValue, final Hash blockHash) { checkState(!frozen, "Layer is Frozen"); - code.put(address, new BonsaiValue<>(oldValue, newValue, newValue == null)); + code.put(address, new DiffBasedValue<>(oldValue, newValue, newValue == null)); return this; } @@ -123,22 +123,22 @@ public TrieLogLayer addStorageChange( checkState(!frozen, "Layer is Frozen"); storage .computeIfAbsent(address, a -> new TreeMap<>()) - .put(slot, new BonsaiValue<>(oldValue, newValue)); + .put(slot, new DiffBasedValue<>(oldValue, newValue)); return this; } @Override - public Map> getAccountChanges() { + public Map> getAccountChanges() { return accounts; } @Override - public Map> getCodeChanges() { + public Map> getCodeChanges() { return code; } @Override - public Map>> getStorageChanges() { + public Map>> getStorageChanges() { return storage; } @@ -147,18 +147,18 @@ public boolean hasStorageChanges(final Address address) { } @Override - public Map> getStorageChanges(final Address address) { + public Map> getStorageChanges(final Address address) { return storage.getOrDefault(address, Map.of()); } @Override public Optional getPriorCode(final Address address) { - return Optional.ofNullable(code.get(address)).map(BonsaiValue::getPrior); + return Optional.ofNullable(code.get(address)).map(DiffBasedValue::getPrior); } @Override public Optional getCode(final Address address) { - return Optional.ofNullable(code.get(address)).map(BonsaiValue::getUpdated); + return Optional.ofNullable(code.get(address)).map(DiffBasedValue::getUpdated); } @Override @@ -166,7 +166,7 @@ public Optional getPriorStorageByStorageSlotKey( final Address address, final StorageSlotKey storageSlotKey) { return Optional.ofNullable(storage.get(address)) .map(i -> i.get(storageSlotKey)) - .map(BonsaiValue::getPrior); + .map(DiffBasedValue::getPrior); } @Override @@ -174,24 +174,24 @@ public Optional getStorageByStorageSlotKey( final Address address, final StorageSlotKey storageSlotKey) { return Optional.ofNullable(storage.get(address)) .map(i -> i.get(storageSlotKey)) - .map(BonsaiValue::getUpdated); + .map(DiffBasedValue::getUpdated); } @Override public Optional getPriorAccount(final Address address) { - return Optional.ofNullable(accounts.get(address)).map(BonsaiValue::getPrior); + return Optional.ofNullable(accounts.get(address)).map(DiffBasedValue::getPrior); } @Override public Optional getAccount(final Address address) { - return Optional.ofNullable(accounts.get(address)).map(BonsaiValue::getUpdated); + return Optional.ofNullable(accounts.get(address)).map(DiffBasedValue::getUpdated); } public String dump() { final StringBuilder sb = new StringBuilder(); sb.append("TrieLog{" + "blockHash=").append(blockHash).append(frozen).append('}'); sb.append("accounts\n"); - for (final Map.Entry> account : accounts.entrySet()) { + for (final Map.Entry> account : accounts.entrySet()) { sb.append(" : ").append(account.getKey()).append("\n"); if (Objects.equals(account.getValue().getPrior(), account.getValue().getUpdated())) { sb.append(" = ").append(account.getValue().getUpdated()).append("\n"); @@ -201,7 +201,7 @@ public String dump() { } } sb.append("code").append("\n"); - for (final Map.Entry> code : code.entrySet()) { + for (final Map.Entry> code : code.entrySet()) { sb.append(" : ").append(code.getKey()).append("\n"); if (Objects.equals(code.getValue().getPrior(), code.getValue().getUpdated())) { sb.append(" = ").append(code.getValue().getPrior()).append("\n"); @@ -211,10 +211,10 @@ public String dump() { } } sb.append("Storage").append("\n"); - for (final Map.Entry>> storage : + for (final Map.Entry>> storage : storage.entrySet()) { sb.append(" : ").append(storage.getKey()).append("\n"); - for (final Map.Entry> slot : + for (final Map.Entry> slot : storage.getValue().entrySet()) { final UInt256 originalValue = slot.getValue().getPrior(); final UInt256 updatedValue = slot.getValue().getUpdated(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManager.java similarity index 87% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManager.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManager.java index 19ac6285f5c..5233cda6f66 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManager.java @@ -13,14 +13,15 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.services.TrieLogService; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; @@ -41,7 +42,7 @@ public class TrieLogManager { private static final Logger LOG = LoggerFactory.getLogger(TrieLogManager.class); public static final long LOG_RANGE_LIMIT = 1000; // restrict trielog range queries to 1k logs protected final Blockchain blockchain; - protected final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; + protected final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage; protected final long maxLayersToLoad; protected final Subscribers trieLogObservers = Subscribers.create(); @@ -50,7 +51,7 @@ public class TrieLogManager { public TrieLogManager( final Blockchain blockchain, - final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, final long maxLayersToLoad, final BesuContext pluginContext) { this.blockchain = blockchain; @@ -60,15 +61,15 @@ public TrieLogManager( } public synchronized void saveTrieLog( - final BonsaiWorldStateUpdateAccumulator localUpdater, + final DiffBasedWorldStateUpdateAccumulator localUpdater, final Hash forWorldStateRootHash, final BlockHeader forBlockHeader, - final BonsaiWorldState forWorldState) { + final DiffBasedWorldState forWorldState) { // do not overwrite a trielog layer that already exists in the database. // if it's only in memory we need to save it // for example, in case of reorg we don't replace a trielog layer if (rootWorldStateStorage.getTrieLog(forBlockHeader.getHash()).isEmpty()) { - final BonsaiWorldStateKeyValueStorage.Updater stateUpdater = + final DiffBasedWorldStateKeyValueStorage.Updater stateUpdater = forWorldState.getWorldStateStorage().updater(); boolean success = false; try { @@ -90,7 +91,7 @@ public synchronized void saveTrieLog( } private TrieLog prepareTrieLog( - final BlockHeader blockHeader, final BonsaiWorldStateUpdateAccumulator localUpdater) { + final BlockHeader blockHeader, final DiffBasedWorldStateUpdateAccumulator localUpdater) { LOG.atDebug() .setMessage("Adding layered world state for {}") .addArgument(blockHeader::toLogString) @@ -104,7 +105,7 @@ private void persistTrieLog( final BlockHeader blockHeader, final Hash worldStateRootHash, final TrieLog trieLog, - final BonsaiWorldStateKeyValueStorage.Updater stateUpdater) { + final DiffBasedWorldStateKeyValueStorage.Updater stateUpdater) { LOG.atDebug() .setMessage("Persisting trie log for block hash {} and world state root {}") .addArgument(blockHeader::toLogString) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java similarity index 94% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java index b72796f2225..60a728b3fb6 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java @@ -13,13 +13,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent; import java.util.Comparator; @@ -41,7 +41,7 @@ public class TrieLogPruner implements TrieLogEvent.TrieLogObserver { private final int pruningLimit; private final int loadingLimit; - private final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; + private final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage; private final Blockchain blockchain; private final Consumer executeAsync; private final long numBlocksToRetain; @@ -51,7 +51,7 @@ public class TrieLogPruner implements TrieLogEvent.TrieLogObserver { TreeMultimap.create(Comparator.reverseOrder(), Comparator.naturalOrder()); public TrieLogPruner( - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage, final Blockchain blockchain, final Consumer executeAsync, final long numBlocksToRetain, @@ -100,7 +100,7 @@ private int preloadQueue() { } } - void addToPruneQueue(final long blockNumber, final Hash blockHash) { + public void addToPruneQueue(final long blockNumber, final Hash blockHash) { LOG.atTrace() .setMessage("adding trie log to queue for later pruning blockNumber {}; blockHash {}") .addArgument(blockNumber) @@ -109,7 +109,7 @@ void addToPruneQueue(final long blockNumber, final Hash blockHash) { trieLogBlocksAndForksByDescendingBlockNumber.put(blockNumber, blockHash); } - int pruneFromQueue() { + public int pruneFromQueue() { final long retainAboveThisBlock = blockchain.getChainHeadBlockNumber() - numBlocksToRetain; final Optional finalized = blockchain.getFinalized(); if (requireFinalizedBlock && finalized.isEmpty()) { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java new file mode 100644 index 00000000000..7a909cbf8e1 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java @@ -0,0 +1,334 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview; + +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; +import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; +import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedLayeredWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedSnapshotWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; +import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.evm.worldstate.WorldUpdater; +import org.hyperledger.besu.plugin.services.exception.StorageException; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; +import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; + +import java.util.Optional; +import java.util.stream.Stream; +import javax.annotation.Nonnull; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class DiffBasedWorldState + implements MutableWorldState, DiffBasedWorldView, StorageSubscriber { + + private static final Logger LOG = LoggerFactory.getLogger(DiffBasedWorldState.class); + + protected DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage; + protected final DiffBasedCachedWorldStorageManager cachedWorldStorageManager; + protected final TrieLogManager trieLogManager; + protected DiffBasedWorldStateUpdateAccumulator accumulator; + + protected Hash worldStateRootHash; + protected Hash worldStateBlockHash; + protected boolean isFrozen; + + protected DiffBasedWorldState( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final DiffBasedCachedWorldStorageManager cachedWorldStorageManager, + final TrieLogManager trieLogManager) { + this.worldStateKeyValueStorage = worldStateKeyValueStorage; + this.worldStateRootHash = + Hash.wrap( + Bytes32.wrap( + worldStateKeyValueStorage.getWorldStateRootHash().orElse(getEmptyTrieHash()))); + this.worldStateBlockHash = + Hash.wrap( + Bytes32.wrap(worldStateKeyValueStorage.getWorldStateBlockHash().orElse(Hash.ZERO))); + this.cachedWorldStorageManager = cachedWorldStorageManager; + this.trieLogManager = trieLogManager; + } + + /** + * Having a protected method to override the accumulator solves the chicken-egg problem of needing + * a worldstate reference (this) when construction the Accumulator. + * + * @param accumulator accumulator to use. + */ + public void setAccumulator(final DiffBasedWorldStateUpdateAccumulator accumulator) { + this.accumulator = accumulator; + } + + /** + * Returns the world state block hash of this world state + * + * @return the world state block hash. + */ + public Hash getWorldStateBlockHash() { + return worldStateBlockHash; + } + + /** + * Returns the world state root hash of this world state + * + * @return the world state root hash. + */ + public Hash getWorldStateRootHash() { + return worldStateRootHash; + } + + @Override + public boolean isPersisted() { + return isPersisted(worldStateKeyValueStorage); + } + + private boolean isPersisted(final WorldStateKeyValueStorage worldStateKeyValueStorage) { + return !(worldStateKeyValueStorage instanceof DiffBasedSnapshotWorldStateKeyValueStorage); + } + + /** + * Reset the worldState to this block header + * + * @param blockHeader block to use + */ + public void resetWorldStateTo(final BlockHeader blockHeader) { + worldStateBlockHash = blockHeader.getBlockHash(); + worldStateRootHash = blockHeader.getStateRoot(); + } + + @Override + public DiffBasedWorldStateKeyValueStorage getWorldStateStorage() { + return worldStateKeyValueStorage; + } + + public DiffBasedWorldStateUpdateAccumulator getAccumulator() { + return accumulator; + } + + @Override + public void persist(final BlockHeader blockHeader) { + final Optional maybeBlockHeader = Optional.ofNullable(blockHeader); + LOG.atDebug() + .setMessage("Persist world state for block {}") + .addArgument(maybeBlockHeader) + .log(); + + final DiffBasedWorldStateUpdateAccumulator localCopy = accumulator.copy(); + + boolean success = false; + + final DiffBasedWorldStateKeyValueStorage.Updater stateUpdater = + worldStateKeyValueStorage.updater(); + Runnable saveTrieLog = () -> {}; + + try { + final Hash newWorldStateRootHash = + calculateRootHash(isFrozen ? Optional.empty() : Optional.of(stateUpdater), accumulator); + // if we are persisted with a block header, and the prior state is the parent + // then persist the TrieLog for that transition. + // If specified but not a direct descendant simply store the new block hash. + if (blockHeader != null) { + verifyWorldStateRoot(newWorldStateRootHash, blockHeader); + saveTrieLog = + () -> { + trieLogManager.saveTrieLog(localCopy, newWorldStateRootHash, blockHeader, this); + // not save a frozen state in the cache + if (!isFrozen) { + cachedWorldStorageManager.addCachedLayer(blockHeader, newWorldStateRootHash, this); + } + }; + + stateUpdater + .getWorldStateTransaction() + .put(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY, blockHeader.getHash().toArrayUnsafe()); + worldStateBlockHash = blockHeader.getHash(); + } else { + stateUpdater.getWorldStateTransaction().remove(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY); + worldStateBlockHash = null; + } + + stateUpdater + .getWorldStateTransaction() + .put(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY, newWorldStateRootHash.toArrayUnsafe()); + worldStateRootHash = newWorldStateRootHash; + success = true; + } finally { + if (success) { + stateUpdater.commit(); + accumulator.reset(); + saveTrieLog.run(); + } else { + stateUpdater.rollback(); + accumulator.reset(); + } + } + } + + protected void verifyWorldStateRoot(final Hash calculatedStateRoot, final BlockHeader header) { + if (!calculatedStateRoot.equals(header.getStateRoot())) { + throw new RuntimeException( + "World State Root does not match expected value, header " + + header.getStateRoot().toHexString() + + " calculated " + + calculatedStateRoot.toHexString()); + } + } + + @Override + public WorldUpdater updater() { + return accumulator; + } + + @Override + public Hash rootHash() { + if (isFrozen && accumulator.isAccumulatorStateChanged()) { + worldStateRootHash = calculateRootHash(Optional.empty(), accumulator.copy()); + accumulator.resetAccumulatorStateChanged(); + } + return Hash.wrap(worldStateRootHash); + } + + protected static final KeyValueStorageTransaction noOpTx = + new KeyValueStorageTransaction() { + + @Override + public void put(final byte[] key, final byte[] value) { + // no-op + } + + @Override + public void remove(final byte[] key) { + // no-op + } + + @Override + public void commit() throws StorageException { + // no-op + } + + @Override + public void rollback() { + // no-op + } + }; + + protected static final SegmentedKeyValueStorageTransaction noOpSegmentedTx = + new SegmentedKeyValueStorageTransaction() { + + @Override + public void put( + final SegmentIdentifier segmentIdentifier, final byte[] key, final byte[] value) { + // no-op + } + + @Override + public void remove(final SegmentIdentifier segmentIdentifier, final byte[] key) { + // no-op + } + + @Override + public void commit() throws StorageException { + // no-op + } + + @Override + public void rollback() { + // no-op + } + }; + + public Hash blockHash() { + return worldStateBlockHash; + } + + @Override + public Stream streamAccounts(final Bytes32 startKeyHash, final int limit) { + throw new RuntimeException("storage format do not provide account streaming."); + } + + @Override + public UInt256 getPriorStorageValue(final Address address, final UInt256 storageKey) { + return getStorageValue(address, storageKey); + } + + @Override + public void close() { + try { + if (!isPersisted()) { + this.worldStateKeyValueStorage.close(); + if (isFrozen) { + closeFrozenStorage(); + } + } + } catch (Exception e) { + // no op + } + } + + private void closeFrozenStorage() { + try { + final DiffBasedLayeredWorldStateKeyValueStorage worldStateLayerStorage = + (DiffBasedLayeredWorldStateKeyValueStorage) worldStateKeyValueStorage; + if (!isPersisted(worldStateLayerStorage.getParentWorldStateStorage())) { + worldStateLayerStorage.getParentWorldStateStorage().close(); + } + } catch (Exception e) { + // no op + } + } + + @Override + public abstract Hash frontierRootHash(); + + @Override + public abstract MutableWorldState freeze(); + + @Override + public abstract Account get(final Address address); + + @Override + public abstract UInt256 getStorageValue(final Address address, final UInt256 storageKey); + + @Override + public abstract Optional getStorageValueByStorageSlotKey( + final Address address, final StorageSlotKey storageSlotKey); + + @Override + public abstract Optional getCode(@Nonnull final Address address, final Hash codeHash); + + protected abstract Hash calculateRootHash( + final Optional maybeStateUpdater, + final DiffBasedWorldStateUpdateAccumulator worldStateUpdater); + + protected abstract Hash getEmptyTrieHash(); +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldView.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldView.java similarity index 85% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldView.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldView.java index cd16e7dec8a..626584aa2c3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldView.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldView.java @@ -1,5 +1,5 @@ /* - * Copyright ConsenSys AG. + * Copyright Hyperledger Besu Contributors. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -11,16 +11,15 @@ * specific language governing permissions and limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - * */ -package org.hyperledger.besu.ethereum.trie.bonsai.worldview; +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.evm.worldstate.WorldView; @@ -31,7 +30,7 @@ import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; -public interface BonsaiWorldView extends WorldView { +public interface DiffBasedWorldView extends WorldView { Optional getCode(Address address, final Hash codeHash); @@ -59,7 +58,7 @@ static Bytes encodeTrieValue(final Bytes bytes) { boolean isPersisted(); - BonsaiWorldStateKeyValueStorage getWorldStateStorage(); + DiffBasedWorldStateKeyValueStorage getWorldStateStorage(); WorldUpdater updater(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java similarity index 71% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java index 44c18176bff..66169ee85a3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java @@ -1,5 +1,5 @@ /* - * Copyright ConsenSys AG. + * Copyright Hyperledger Besu Contributors. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -11,10 +11,9 @@ * specific language governing permissions and limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - * */ -package org.hyperledger.besu.ethereum.trie.bonsai.worldview; +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator; import org.hyperledger.besu.datatypes.AccountValue; import org.hyperledger.besu.datatypes.Address; @@ -23,9 +22,14 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.AccountConsumingMap; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.Consumer; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.StorageConsumingMap; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -44,42 +48,40 @@ import java.util.Set; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.function.Function; -import javax.annotation.Nonnull; -import com.google.common.collect.ForwardingMap; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class BonsaiWorldStateUpdateAccumulator - extends AbstractWorldUpdater - implements BonsaiWorldView, TrieLogAccumulator { +@SuppressWarnings("unchecked") +public abstract class DiffBasedWorldStateUpdateAccumulator + extends AbstractWorldUpdater + implements DiffBasedWorldView, TrieLogAccumulator { private static final Logger LOG = - LoggerFactory.getLogger(BonsaiWorldStateUpdateAccumulator.class); - protected final Consumer> accountPreloader; + LoggerFactory.getLogger(DiffBasedWorldStateUpdateAccumulator.class); + protected final Consumer> accountPreloader; protected final Consumer storagePreloader; - private final AccountConsumingMap> accountsToUpdate; - private final Map> codeToUpdate = new ConcurrentHashMap<>(); + private final AccountConsumingMap> accountsToUpdate; + private final Map> codeToUpdate = new ConcurrentHashMap<>(); private final Set
storageToClear = Collections.synchronizedSet(new HashSet<>()); protected final EvmConfiguration evmConfiguration; // storage sub mapped by _hashed_ key. This is because in self_destruct calls we need to // enumerate the old storage and delete it. Those are trie stored by hashed key by spec and the // alternative was to keep a giant pre-image cache of the entire trie. - private final Map>> + private final Map>> storageToUpdate = new ConcurrentHashMap<>(); private final Map storageKeyHashLookup = new ConcurrentHashMap<>(); protected boolean isAccumulatorStateChanged; - public BonsaiWorldStateUpdateAccumulator( - final BonsaiWorldView world, - final Consumer> accountPreloader, + public DiffBasedWorldStateUpdateAccumulator( + final DiffBasedWorldView world, + final Consumer> accountPreloader, final Consumer storagePreloader, final EvmConfiguration evmConfiguration) { super(world, evmConfiguration); @@ -90,15 +92,7 @@ public BonsaiWorldStateUpdateAccumulator( this.evmConfiguration = evmConfiguration; } - public BonsaiWorldStateUpdateAccumulator copy() { - final BonsaiWorldStateUpdateAccumulator copy = - new BonsaiWorldStateUpdateAccumulator( - wrappedWorldView(), accountPreloader, storagePreloader, evmConfiguration); - copy.cloneFromUpdater(this); - return copy; - } - - void cloneFromUpdater(final BonsaiWorldStateUpdateAccumulator source) { + protected void cloneFromUpdater(final DiffBasedWorldStateUpdateAccumulator source) { accountsToUpdate.putAll(source.getAccountsToUpdate()); codeToUpdate.putAll(source.codeToUpdate); storageToClear.addAll(source.storageToClear); @@ -108,14 +102,25 @@ void cloneFromUpdater(final BonsaiWorldStateUpdateAccumulator source) { this.isAccumulatorStateChanged = true; } + protected Consumer> getAccountPreloader() { + return accountPreloader; + } + + protected Consumer getStoragePreloader() { + return storagePreloader; + } + + protected EvmConfiguration getEvmConfiguration() { + return evmConfiguration; + } + @Override public Account get(final Address address) { return super.get(address); } @Override - protected UpdateTrackingAccount track( - final UpdateTrackingAccount account) { + protected UpdateTrackingAccount track(final UpdateTrackingAccount account) { return super.track(account); } @@ -126,21 +131,21 @@ public MutableAccount getAccount(final Address address) { @Override public MutableAccount createAccount(final Address address, final long nonce, final Wei balance) { - BonsaiValue bonsaiValue = accountsToUpdate.get(address); - - if (bonsaiValue == null) { - bonsaiValue = new BonsaiValue<>(null, null); - accountsToUpdate.put(address, bonsaiValue); - } else if (bonsaiValue.getUpdated() != null) { - if (bonsaiValue.getUpdated().isEmpty()) { - return track(new UpdateTrackingAccount<>(bonsaiValue.getUpdated())); + DiffBasedValue diffBasedValue = accountsToUpdate.get(address); + + if (diffBasedValue == null) { + diffBasedValue = new DiffBasedValue<>(null, null); + accountsToUpdate.put(address, diffBasedValue); + } else if (diffBasedValue.getUpdated() != null) { + if (diffBasedValue.getUpdated().isEmpty()) { + return track(new UpdateTrackingAccount<>(diffBasedValue.getUpdated())); } else { throw new IllegalStateException("Cannot create an account when one already exists"); } } - final BonsaiAccount newAccount = - new BonsaiAccount( + final ACCOUNT newAccount = + createAccount( this, address, hashAndSaveAccountPreImage(address), @@ -149,17 +154,17 @@ public MutableAccount createAccount(final Address address, final long nonce, fin Hash.EMPTY_TRIE_HASH, Hash.EMPTY, true); - bonsaiValue.setUpdated(newAccount); + diffBasedValue.setUpdated(newAccount); return track(new UpdateTrackingAccount<>(newAccount)); } @Override - public Map> getAccountsToUpdate() { + public Map> getAccountsToUpdate() { return accountsToUpdate; } @Override - public Map> getCodeToUpdate() { + public Map> getCodeToUpdate() { return codeToUpdate; } @@ -168,40 +173,41 @@ public Set
getStorageToClear() { } @Override - public Map>> + public Map>> getStorageToUpdate() { return storageToUpdate; } @Override - protected BonsaiAccount getForMutation(final Address address) { - return loadAccount(address, BonsaiValue::getUpdated); + protected ACCOUNT getForMutation(final Address address) { + return loadAccount(address, DiffBasedValue::getUpdated); } - protected BonsaiAccount loadAccount( - final Address address, - final Function, BonsaiAccount> bonsaiAccountFunction) { + protected ACCOUNT loadAccount( + final Address address, final Function, ACCOUNT> accountFunction) { try { - final BonsaiValue bonsaiValue = accountsToUpdate.get(address); - if (bonsaiValue == null) { + final DiffBasedValue diffBasedValue = accountsToUpdate.get(address); + if (diffBasedValue == null) { final Account account; - if (wrappedWorldView() - instanceof BonsaiWorldStateUpdateAccumulator bonsaiWorldStateUpdateAccumulator) { - account = bonsaiWorldStateUpdateAccumulator.loadAccount(address, bonsaiAccountFunction); + if (wrappedWorldView() instanceof DiffBasedWorldStateUpdateAccumulator) { + final DiffBasedWorldStateUpdateAccumulator worldStateUpdateAccumulator = + (DiffBasedWorldStateUpdateAccumulator) wrappedWorldView(); + account = worldStateUpdateAccumulator.loadAccount(address, accountFunction); } else { account = wrappedWorldView().get(address); } - if (account instanceof BonsaiAccount bonsaiAccount) { - BonsaiAccount mutableAccount = new BonsaiAccount(bonsaiAccount, this, true); - accountsToUpdate.put(address, new BonsaiValue<>(bonsaiAccount, mutableAccount)); + if (account instanceof DiffBasedAccount diffBasedAccount) { + ACCOUNT mutableAccount = copyAccount((ACCOUNT) diffBasedAccount, this, true); + accountsToUpdate.put( + address, new DiffBasedValue<>((ACCOUNT) diffBasedAccount, mutableAccount)); return mutableAccount; } else { // add the empty read in accountsToUpdate - accountsToUpdate.put(address, new BonsaiValue<>(null, null)); + accountsToUpdate.put(address, new DiffBasedValue<>(null, null)); return null; } } else { - return bonsaiAccountFunction.apply(bonsaiValue); + return accountFunction.apply(diffBasedValue); } } catch (MerkleTrieException e) { // need to throw to trigger the heal @@ -229,12 +235,12 @@ public void revert() { public void commit() { this.isAccumulatorStateChanged = true; for (final Address deletedAddress : getDeletedAccounts()) { - final BonsaiValue accountValue = + final DiffBasedValue accountValue = accountsToUpdate.computeIfAbsent( deletedAddress, - __ -> loadAccountFromParent(deletedAddress, new BonsaiValue<>(null, null, true))); + __ -> loadAccountFromParent(deletedAddress, new DiffBasedValue<>(null, null, true))); storageToClear.add(deletedAddress); - final BonsaiValue codeValue = codeToUpdate.get(deletedAddress); + final DiffBasedValue codeValue = codeToUpdate.get(deletedAddress); if (codeValue != null) { codeValue.setUpdated(null).setCleared(); } else { @@ -242,26 +248,27 @@ public void commit() { .getCode( deletedAddress, Optional.ofNullable(accountValue) - .map(BonsaiValue::getPrior) - .map(BonsaiAccount::getCodeHash) + .map(DiffBasedValue::getPrior) + .map(DiffBasedAccount::getCodeHash) .orElse(Hash.EMPTY)) .ifPresent( deletedCode -> - codeToUpdate.put(deletedAddress, new BonsaiValue<>(deletedCode, null, true))); + codeToUpdate.put( + deletedAddress, new DiffBasedValue<>(deletedCode, null, true))); } // mark all updated storage as to be cleared - final Map> deletedStorageUpdates = + final Map> deletedStorageUpdates = storageToUpdate.computeIfAbsent( deletedAddress, k -> new StorageConsumingMap<>( deletedAddress, new ConcurrentHashMap<>(), storagePreloader)); - final Iterator>> iter = + final Iterator>> iter = deletedStorageUpdates.entrySet().iterator(); while (iter.hasNext()) { - final Map.Entry> updateEntry = iter.next(); - final BonsaiValue updatedSlot = updateEntry.getValue(); + final Map.Entry> updateEntry = iter.next(); + final DiffBasedValue updatedSlot = updateEntry.getValue(); if (updatedSlot.getPrior() == null || updatedSlot.getPrior().isZero()) { iter.remove(); } else { @@ -269,7 +276,7 @@ public void commit() { } } - final BonsaiAccount originalValue = accountValue.getPrior(); + final ACCOUNT originalValue = accountValue.getPrior(); if (originalValue != null) { // Enumerate and delete addresses not updated wrappedWorldView() @@ -280,7 +287,8 @@ public void commit() { new StorageSlotKey(Hash.wrap(keyHash), Optional.empty()); if (!deletedStorageUpdates.containsKey(storageSlotKey)) { final UInt256 value = UInt256.fromBytes(RLP.decodeOne(entryValue)); - deletedStorageUpdates.put(storageSlotKey, new BonsaiValue<>(value, null, true)); + deletedStorageUpdates.put( + storageSlotKey, new DiffBasedValue<>(value, null, true)); } }); } @@ -294,11 +302,11 @@ public void commit() { .forEach( tracked -> { final Address updatedAddress = tracked.getAddress(); - final BonsaiAccount updatedAccount; - final BonsaiValue updatedAccountValue = + final ACCOUNT updatedAccount; + final DiffBasedValue updatedAccountValue = accountsToUpdate.get(updatedAddress); - final Map> pendingStorageUpdates = + final Map> pendingStorageUpdates = storageToUpdate.computeIfAbsent( updatedAddress, k -> @@ -311,12 +319,12 @@ public void commit() { } if (tracked.getWrappedAccount() == null) { - updatedAccount = new BonsaiAccount(this, tracked); + updatedAccount = createAccount(this, tracked); tracked.setWrappedAccount(updatedAccount); if (updatedAccountValue == null) { - accountsToUpdate.put(updatedAddress, new BonsaiValue<>(null, updatedAccount)); + accountsToUpdate.put(updatedAddress, new DiffBasedValue<>(null, updatedAccount)); codeToUpdate.put( - updatedAddress, new BonsaiValue<>(null, updatedAccount.getCode())); + updatedAddress, new DiffBasedValue<>(null, updatedAccount.getCode())); } else { updatedAccountValue.setUpdated(updatedAccount); } @@ -334,17 +342,17 @@ public void commit() { } if (tracked.codeWasUpdated()) { - final BonsaiValue pendingCode = + final DiffBasedValue pendingCode = codeToUpdate.computeIfAbsent( updatedAddress, addr -> - new BonsaiValue<>( + new DiffBasedValue<>( wrappedWorldView() .getCode( addr, Optional.ofNullable(updatedAccountValue) - .map(BonsaiValue::getPrior) - .map(BonsaiAccount::getCodeHash) + .map(DiffBasedValue::getPrior) + .map(DiffBasedAccount::getCodeHash) .orElse(Hash.EMPTY)) .orElse(null), null)); @@ -368,12 +376,11 @@ public void commit() { final StorageSlotKey slotKey = new StorageSlotKey(hashAndSaveSlotPreImage(keyUInt), Optional.of(keyUInt)); final UInt256 value = storageUpdate.getValue(); - final BonsaiValue pendingValue = pendingStorageUpdates.get(slotKey); - + final DiffBasedValue pendingValue = pendingStorageUpdates.get(slotKey); if (pendingValue == null) { pendingStorageUpdates.put( slotKey, - new BonsaiValue<>( + new DiffBasedValue<>( updatedAccount.getOriginalStorageValue(keyUInt), value)); } else { pendingValue.setUpdated(value); @@ -394,7 +401,7 @@ public void commit() { @Override public Optional getCode(final Address address, final Hash codeHash) { - final BonsaiValue localCode = codeToUpdate.get(address); + final DiffBasedValue localCode = codeToUpdate.get(address); if (localCode == null) { final Optional code = wrappedWorldView().getCode(address, codeHash); if (code.isEmpty() && !codeHash.equals(Hash.EMPTY)) { @@ -417,31 +424,26 @@ public UInt256 getStorageValue(final Address address, final UInt256 slotKey) { @Override public Optional getStorageValueByStorageSlotKey( final Address address, final StorageSlotKey storageSlotKey) { - final Map> localAccountStorage = + final Map> localAccountStorage = storageToUpdate.get(address); if (localAccountStorage != null) { - final BonsaiValue value = localAccountStorage.get(storageSlotKey); + final DiffBasedValue value = localAccountStorage.get(storageSlotKey); if (value != null) { return Optional.ofNullable(value.getUpdated()); } } try { final Optional valueUInt = - (wrappedWorldView() instanceof BonsaiWorldState bonsaiWorldState) - ? bonsaiWorldState.getStorageValueByStorageSlotKey( - () -> - Optional.ofNullable(loadAccount(address, BonsaiValue::getPrior)) - .map(BonsaiAccount::getStorageRoot), - address, - storageSlotKey) + (wrappedWorldView() instanceof DiffBasedWorldState worldState) + ? worldState.getStorageValueByStorageSlotKey(address, storageSlotKey) : wrappedWorldView().getStorageValueByStorageSlotKey(address, storageSlotKey); storageToUpdate .computeIfAbsent( address, key -> new StorageConsumingMap<>(address, new ConcurrentHashMap<>(), storagePreloader)) - .put(storageSlotKey, new BonsaiValue<>(valueUInt.orElse(null), valueUInt.orElse(null))); - + .put( + storageSlotKey, new DiffBasedValue<>(valueUInt.orElse(null), valueUInt.orElse(null))); return valueUInt; } catch (MerkleTrieException e) { // need to throw to trigger the heal @@ -455,10 +457,10 @@ public UInt256 getPriorStorageValue(final Address address, final UInt256 storage // TODO maybe log the read into the trie layer? StorageSlotKey storageSlotKey = new StorageSlotKey(hashAndSaveSlotPreImage(storageKey), Optional.of(storageKey)); - final Map> localAccountStorage = + final Map> localAccountStorage = storageToUpdate.get(address); if (localAccountStorage != null) { - final BonsaiValue value = localAccountStorage.get(storageSlotKey); + final DiffBasedValue value = localAccountStorage.get(storageSlotKey); if (value != null) { if (value.isLastStepCleared()) { return UInt256.ZERO; @@ -482,11 +484,11 @@ public UInt256 getPriorStorageValue(final Address address, final UInt256 storage @Override public Map getAllAccountStorage(final Address address, final Hash rootHash) { final Map results = wrappedWorldView().getAllAccountStorage(address, rootHash); - final StorageConsumingMap> bonsaiValueStorage = + final StorageConsumingMap> diffBasedValueStorage = storageToUpdate.get(address); - if (bonsaiValueStorage != null) { + if (diffBasedValueStorage != null) { // hash the key to match the implied storage interface of hashed slotKey - bonsaiValueStorage.forEach( + diffBasedValueStorage.forEach( (key, value) -> results.put(key.getSlotHash(), value.getUpdated())); } return results; @@ -498,7 +500,7 @@ public boolean isPersisted() { } @Override - public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { + public DiffBasedWorldStateKeyValueStorage getWorldStateStorage() { return wrappedWorldView().getWorldStateStorage(); } @@ -550,7 +552,7 @@ private void rollAccountChange( // non-change, a cached read. return; } - BonsaiValue accountValue = accountsToUpdate.get(address); + DiffBasedValue accountValue = accountsToUpdate.get(address); if (accountValue == null) { accountValue = loadAccountFromParent(address, accountValue); } @@ -558,7 +560,7 @@ private void rollAccountChange( if (expectedValue == null && replacementValue != null) { accountsToUpdate.put( address, - new BonsaiValue<>(null, new BonsaiAccount(this, address, replacementValue, true))); + new DiffBasedValue<>(null, createAccount(this, address, replacementValue, true))); } else { throw new IllegalStateException( String.format( @@ -572,7 +574,7 @@ private void rollAccountChange( "Expected to create account, but the account exists. Address=%s", address)); } } else { - BonsaiAccount.assertCloseEnoughForDiffing( + assertCloseEnoughForDiffing( accountValue.getUpdated(), expectedValue, "Address=" + address + " Prior Value in Rolling Change"); @@ -586,19 +588,18 @@ private void rollAccountChange( accountValue.setUpdated(null); } } else { - accountValue.setUpdated( - new BonsaiAccount(wrappedWorldView(), address, replacementValue, true)); + accountValue.setUpdated(createAccount(wrappedWorldView(), address, replacementValue, true)); } } } - private BonsaiValue loadAccountFromParent( - final Address address, final BonsaiValue defaultValue) { + private DiffBasedValue loadAccountFromParent( + final Address address, final DiffBasedValue defaultValue) { try { final Account parentAccount = wrappedWorldView().get(address); - if (parentAccount instanceof BonsaiAccount account) { - final BonsaiValue loadedAccountValue = - new BonsaiValue<>(new BonsaiAccount(account), account); + if (parentAccount instanceof DiffBasedAccount account) { + final DiffBasedValue loadedAccountValue = + new DiffBasedValue<>(copyAccount((ACCOUNT) account), ((ACCOUNT) account)); accountsToUpdate.put(address, loadedAccountValue); return loadedAccountValue; } else { @@ -617,7 +618,7 @@ private void rollCodeChange( // non-change, a cached read. return; } - BonsaiValue codeValue = codeToUpdate.get(address); + DiffBasedValue codeValue = codeToUpdate.get(address); if (codeValue == null) { final Bytes storedCode = wrappedWorldView() @@ -625,14 +626,14 @@ private void rollCodeChange( address, Optional.ofNullable(expectedCode).map(Hash::hash).orElse(Hash.EMPTY)) .orElse(Bytes.EMPTY); if (!storedCode.isEmpty()) { - codeValue = new BonsaiValue<>(storedCode, storedCode); + codeValue = new DiffBasedValue<>(storedCode, storedCode); codeToUpdate.put(address, codeValue); } } if (codeValue == null) { if ((expectedCode == null || expectedCode.isEmpty()) && replacementCode != null) { - codeToUpdate.put(address, new BonsaiValue<>(null, replacementCode)); + codeToUpdate.put(address, new DiffBasedValue<>(null, replacementCode)); } else { throw new IllegalStateException( String.format( @@ -660,10 +661,10 @@ private void rollCodeChange( } } - private Map> maybeCreateStorageMap( - final Map> storageMap, final Address address) { + private Map> maybeCreateStorageMap( + final Map> storageMap, final Address address) { if (storageMap == null) { - final StorageConsumingMap> newMap = + final StorageConsumingMap> newMap = new StorageConsumingMap<>(address, new ConcurrentHashMap<>(), storagePreloader); storageToUpdate.put(address, newMap); return newMap; @@ -685,13 +686,13 @@ private void rollStorageChange( // corner case on deletes, non-change return; } - final Map> storageMap = storageToUpdate.get(address); - BonsaiValue slotValue = storageMap == null ? null : storageMap.get(storageSlotKey); + final Map> storageMap = storageToUpdate.get(address); + DiffBasedValue slotValue = storageMap == null ? null : storageMap.get(storageSlotKey); if (slotValue == null) { final Optional storageValue = wrappedWorldView().getStorageValueByStorageSlotKey(address, storageSlotKey); if (storageValue.isPresent()) { - slotValue = new BonsaiValue<>(storageValue.get(), storageValue.get()); + slotValue = new DiffBasedValue<>(storageValue.get(), storageValue.get()); storageToUpdate .computeIfAbsent( address, @@ -703,7 +704,7 @@ private void rollStorageChange( if (slotValue == null) { if ((expectedValue == null || expectedValue.isZero()) && replacementValue != null) { maybeCreateStorageMap(storageMap, address) - .put(storageSlotKey, new BonsaiValue<>(null, replacementValue)); + .put(storageSlotKey, new DiffBasedValue<>(null, replacementValue)); } else { throw new IllegalStateException( String.format( @@ -730,7 +731,7 @@ private void rollStorageChange( existingSlotValue == null ? "null" : existingSlotValue.toShortHexString())); } if (replacementValue == null && slotValue.getPrior() == null) { - final Map> thisStorageUpdate = + final Map> thisStorageUpdate = maybeCreateStorageMap(storageMap, address); thisStorageUpdate.remove(storageSlotKey); if (thisStorageUpdate.isEmpty()) { @@ -769,67 +770,6 @@ public void reset() { storageKeyHashLookup.clear(); } - public static class AccountConsumingMap extends ForwardingMap { - - private final ConcurrentMap accounts; - private final Consumer consumer; - - public AccountConsumingMap( - final ConcurrentMap accounts, final Consumer consumer) { - this.accounts = accounts; - this.consumer = consumer; - } - - @Override - public T put(@Nonnull final Address address, @Nonnull final T value) { - consumer.process(address, value); - return accounts.put(address, value); - } - - public Consumer getConsumer() { - return consumer; - } - - @Override - protected Map delegate() { - return accounts; - } - } - - public static class StorageConsumingMap extends ForwardingMap { - - private final Address address; - - private final ConcurrentMap storages; - private final Consumer consumer; - - public StorageConsumingMap( - final Address address, final ConcurrentMap storages, final Consumer consumer) { - this.address = address; - this.storages = storages; - this.consumer = consumer; - } - - @Override - public T put(@Nonnull final K slotKey, @Nonnull final T value) { - consumer.process(address, slotKey); - return storages.put(slotKey, value); - } - - public Consumer getConsumer() { - return consumer; - } - - @Override - protected Map delegate() { - return storages; - } - } - - public interface Consumer { - void process(final Address address, T value); - } - protected Hash hashAndSaveAccountPreImage(final Address address) { // no need to save account preimage by default return Hash.hash(address); @@ -843,4 +783,33 @@ protected Hash hashAndSaveSlotPreImage(final UInt256 slotKey) { } return hash; } + + public abstract DiffBasedWorldStateUpdateAccumulator copy(); + + protected abstract ACCOUNT copyAccount(final ACCOUNT account); + + protected abstract ACCOUNT copyAccount( + final ACCOUNT toCopy, final DiffBasedWorldView context, final boolean mutable); + + protected abstract ACCOUNT createAccount( + final DiffBasedWorldView context, + final Address address, + final AccountValue stateTrieAccount, + final boolean mutable); + + protected abstract ACCOUNT createAccount( + final DiffBasedWorldView context, + final Address address, + final Hash addressHash, + final long nonce, + final Wei balance, + final Hash storageRoot, + final Hash codeHash, + final boolean mutable); + + protected abstract ACCOUNT createAccount( + final DiffBasedWorldView context, final UpdateTrackingAccount tracked); + + protected abstract void assertCloseEnoughForDiffing( + final ACCOUNT source, final AccountValue account, final String context); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/AccountConsumingMap.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/AccountConsumingMap.java new file mode 100644 index 00000000000..63c5b9233f2 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/AccountConsumingMap.java @@ -0,0 +1,49 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload; + +import org.hyperledger.besu.datatypes.Address; + +import java.util.Map; +import java.util.concurrent.ConcurrentMap; +import javax.annotation.Nonnull; + +import com.google.common.collect.ForwardingMap; + +public class AccountConsumingMap extends ForwardingMap { + + private final ConcurrentMap accounts; + private final Consumer consumer; + + public AccountConsumingMap(final ConcurrentMap accounts, final Consumer consumer) { + this.accounts = accounts; + this.consumer = consumer; + } + + @Override + public T put(@Nonnull final Address address, @Nonnull final T value) { + consumer.process(address, value); + return accounts.put(address, value); + } + + public Consumer getConsumer() { + return consumer; + } + + @Override + protected Map delegate() { + return accounts; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/Consumer.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/Consumer.java new file mode 100644 index 00000000000..577b0d8c551 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/Consumer.java @@ -0,0 +1,21 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload; + +import org.hyperledger.besu.datatypes.Address; + +public interface Consumer { + void process(final Address address, T value); +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/StorageConsumingMap.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/StorageConsumingMap.java new file mode 100644 index 00000000000..13b42d7b4a4 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/StorageConsumingMap.java @@ -0,0 +1,53 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload; + +import org.hyperledger.besu.datatypes.Address; + +import java.util.Map; +import java.util.concurrent.ConcurrentMap; +import javax.annotation.Nonnull; + +import com.google.common.collect.ForwardingMap; + +public class StorageConsumingMap extends ForwardingMap { + + private final Address address; + + private final ConcurrentMap storages; + private final Consumer consumer; + + public StorageConsumingMap( + final Address address, final ConcurrentMap storages, final Consumer consumer) { + this.address = address; + this.storages = storages; + this.consumer = consumer; + } + + @Override + public T put(@Nonnull final K slotKey, @Nonnull final T value) { + consumer.process(address, slotKey); + return storages.put(slotKey, value); + } + + public Consumer getConsumer() { + return consumer; + } + + @Override + protected Map delegate() { + return storages; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java index f689410a9de..c35fe423aee 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java @@ -15,7 +15,7 @@ package org.hyperledger.besu.ethereum.worldstate; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java index 398472ef3ae..9bac9254940 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java @@ -25,9 +25,9 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; @@ -95,15 +95,15 @@ public static BonsaiWorldStateProvider createBonsaiInMemoryWorldStateArchive( final Blockchain blockchain, final EvmConfiguration evmConfiguration) { final InMemoryKeyValueStorageProvider inMemoryKeyValueStorageProvider = new InMemoryKeyValueStorageProvider(); - final CachedMerkleTrieLoader cachedMerkleTrieLoader = - new CachedMerkleTrieLoader(new NoOpMetricsSystem()); + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader = + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()); return new BonsaiWorldStateProvider( (BonsaiWorldStateKeyValueStorage) inMemoryKeyValueStorageProvider.createWorldStateStorage( DataStorageConfiguration.DEFAULT_BONSAI_CONFIG), blockchain, Optional.empty(), - cachedMerkleTrieLoader, + bonsaiCachedMerkleTrieLoader, null, evmConfiguration); } diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java index c549c66eb93..a04592d3093 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java @@ -20,7 +20,7 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrie; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java index f8fe794586f..d451a6fb7b2 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java @@ -41,9 +41,9 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java similarity index 97% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java index ab9e87d5f61..b2fbc713869 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider.createInMemoryBlockchain; import static org.mockito.ArgumentMatchers.any; @@ -66,8 +66,8 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -159,7 +159,7 @@ public void createStorage() { (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage, blockchain, Optional.of(16L), - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), null, EvmConfiguration.DEFAULT); var ws = archive.getMutable(); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiCachedMerkleTrieLoaderTest.java similarity index 94% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiCachedMerkleTrieLoaderTest.java index d96c6990cb2..d4e42cd516b 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiCachedMerkleTrieLoaderTest.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.assertj.core.api.Assertions.assertThat; @@ -26,8 +26,8 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; @@ -45,9 +45,9 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; -class CachedMerkleTrieLoaderTest { +class BonsaiCachedMerkleTrieLoaderTest { - private CachedMerkleTrieLoader merkleTrieLoader; + private BonsaiCachedMerkleTrieLoader merkleTrieLoader; private final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); private final BonsaiWorldStateKeyValueStorage inMemoryWorldState = Mockito.spy( @@ -69,7 +69,7 @@ public void setup() { TrieGenerator.generateTrie( worldStateStorageCoordinator, accounts.stream().map(Address::addressHash).collect(Collectors.toList())); - merkleTrieLoader = new CachedMerkleTrieLoader(new NoOpMetricsSystem()); + merkleTrieLoader = new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()); } @Test diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiSnapshotIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiSnapshotIsolationTests.java similarity index 99% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiSnapshotIsolationTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiSnapshotIsolationTests.java index ce983e10a9a..507047604bc 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiSnapshotIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiSnapshotIsolationTests.java @@ -14,7 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProviderTest.java similarity index 89% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProviderTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProviderTest.java index 10d3381c7e0..879183ecc38 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProviderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProviderTest.java @@ -13,13 +13,13 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.BLOCKCHAIN; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; -import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; -import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; +import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; +import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.eq; @@ -36,13 +36,13 @@ import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -77,7 +77,7 @@ class BonsaiWorldStateProviderTest { @Mock SegmentedKeyValueStorageTransaction segmentedKeyValueStorageTransaction; BonsaiWorldStateProvider bonsaiWorldStateArchive; - @Mock CachedWorldStorageManager cachedWorldStorageManager; + @Mock BonsaiCachedWorldStorageManager cachedWorldStorageManager; @Mock TrieLogManager trieLogManager; @BeforeEach @@ -111,7 +111,7 @@ void testGetMutableReturnPersistedStateWhenNeeded() { new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_BONSAI_CONFIG), blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT); assertThat(bonsaiWorldStateArchive.getMutable(chainHead, true)) @@ -128,7 +128,7 @@ void testGetMutableReturnEmptyWhenLoadMoreThanLimitLayersBack() { DataStorageConfiguration.DEFAULT_BONSAI_CONFIG), blockchain, Optional.of(512L), - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), null, EvmConfiguration.DEFAULT); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); @@ -150,7 +150,7 @@ void testGetMutableWhenLoadLessThanLimitLayersBack() { new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_BONSAI_CONFIG), blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); final BlockHeader chainHead = blockBuilder.number(511).buildHeader(); @@ -185,7 +185,7 @@ void testGetMutableWithStorageInconsistencyRollbackTheState() { trieLogManager, worldStateKeyValueStorage, blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); @@ -214,7 +214,7 @@ void testGetMutableWithStorageConsistencyNotRollbackTheState() { trieLogManager, worldStateKeyValueStorage, blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); @@ -254,7 +254,7 @@ void testGetMutableWithStorageConsistencyToRollbackAndRollForwardTheState() { trieLogManager, worldStateKeyValueStorage, blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); // initial persisted state hash key @@ -297,7 +297,7 @@ void testGetMutableWithRollbackNotOverrideTrieLogLayer() { new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_BONSAI_CONFIG), blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); // initial persisted state hash key diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/LogRollingTests.java similarity index 96% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/LogRollingTests.java index ee63fab968f..331c6b1afdb 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/LogRollingTests.java @@ -14,7 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; @@ -29,11 +29,11 @@ import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.internal.EvmConfiguration; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/RollingImport.java similarity index 90% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/RollingImport.java index a43fce8f402..134d094a4dd 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/RollingImport.java @@ -14,7 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static com.google.common.base.Preconditions.checkArgument; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; @@ -25,11 +25,11 @@ import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java similarity index 99% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java index 911ff10005e..c4769248e9c 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java @@ -12,11 +12,11 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage; import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; -import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; +import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryTests.java similarity index 94% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryTests.java index 64f96d8b734..59c208792e4 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryTests.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; @@ -24,6 +24,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogManagerTests.java similarity index 84% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogManagerTests.java index e5c716e413a..a6ff8d3a7c8 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogManagerTests.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.spy; @@ -22,9 +22,10 @@ import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; import org.hyperledger.besu.evm.internal.EvmConfiguration; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogPrunerTest.java similarity index 96% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogPrunerTest.java index ac84d35be40..19e63f6f940 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogPrunerTest.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; @@ -26,7 +26,10 @@ import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogAddedEvent; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; import java.util.Optional; import java.util.function.Consumer; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateTest.java similarity index 81% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateTest.java index f875f098bdc..8abae1f2b6b 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateTest.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.worldview; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -23,8 +23,8 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; import org.hyperledger.besu.evm.internal.EvmConfiguration; import java.util.HashMap; @@ -69,8 +69,8 @@ void setup() { @MethodSource("priorAndUpdatedEmptyAndNullBytes") void codeUpdateDoesNothingWhenMarkedAsDeletedButAlreadyDeleted( final Bytes prior, final Bytes updated) { - final Map> codeToUpdate = - Map.of(Address.ZERO, new BonsaiValue<>(prior, updated)); + final Map> codeToUpdate = + Map.of(Address.ZERO, new DiffBasedValue<>(prior, updated)); when(bonsaiWorldStateUpdateAccumulator.getCodeToUpdate()).thenReturn(codeToUpdate); worldState.updateCode(Optional.of(bonsaiUpdater), bonsaiWorldStateUpdateAccumulator); @@ -79,8 +79,8 @@ void codeUpdateDoesNothingWhenMarkedAsDeletedButAlreadyDeleted( @Test void codeUpdateDoesNothingWhenAddingSameAsExistingValue() { - final Map> codeToUpdate = - Map.of(Address.ZERO, new BonsaiValue<>(CODE, CODE)); + final Map> codeToUpdate = + Map.of(Address.ZERO, new DiffBasedValue<>(CODE, CODE)); when(bonsaiWorldStateUpdateAccumulator.getCodeToUpdate()).thenReturn(codeToUpdate); worldState.updateCode(Optional.of(bonsaiUpdater), bonsaiWorldStateUpdateAccumulator); @@ -90,8 +90,8 @@ void codeUpdateDoesNothingWhenAddingSameAsExistingValue() { @ParameterizedTest @MethodSource("emptyAndNullBytes") void removesCodeWhenMarkedAsDeleted(final Bytes updated) { - final Map> codeToUpdate = - Map.of(Address.ZERO, new BonsaiValue<>(CODE, updated)); + final Map> codeToUpdate = + Map.of(Address.ZERO, new DiffBasedValue<>(CODE, updated)); when(bonsaiWorldStateUpdateAccumulator.getCodeToUpdate()).thenReturn(codeToUpdate); worldState.updateCode(Optional.of(bonsaiUpdater), bonsaiWorldStateUpdateAccumulator); @@ -101,8 +101,8 @@ void removesCodeWhenMarkedAsDeleted(final Bytes updated) { @ParameterizedTest @MethodSource("codeValueAndEmptyAndNullBytes") void addsCodeForNewCodeValue(final Bytes prior) { - final Map> codeToUpdate = - Map.of(ACCOUNT, new BonsaiValue<>(prior, CODE)); + final Map> codeToUpdate = + Map.of(ACCOUNT, new DiffBasedValue<>(prior, CODE)); when(bonsaiWorldStateUpdateAccumulator.getCodeToUpdate()).thenReturn(codeToUpdate); worldState.updateCode(Optional.of(bonsaiUpdater), bonsaiWorldStateUpdateAccumulator); @@ -112,10 +112,10 @@ void addsCodeForNewCodeValue(final Bytes prior) { @Test void updateCodeForMultipleValues() { - final Map> codeToUpdate = new HashMap<>(); - codeToUpdate.put(Address.fromHexString("0x1"), new BonsaiValue<>(null, CODE)); - codeToUpdate.put(Address.fromHexString("0x2"), new BonsaiValue<>(CODE, null)); - codeToUpdate.put(Address.fromHexString("0x3"), new BonsaiValue<>(Bytes.of(9), CODE)); + final Map> codeToUpdate = new HashMap<>(); + codeToUpdate.put(Address.fromHexString("0x1"), new DiffBasedValue<>(null, CODE)); + codeToUpdate.put(Address.fromHexString("0x2"), new DiffBasedValue<>(CODE, null)); + codeToUpdate.put(Address.fromHexString("0x3"), new DiffBasedValue<>(Bytes.of(9), CODE)); when(bonsaiWorldStateUpdateAccumulator.getCodeToUpdate()).thenReturn(codeToUpdate); worldState.updateCode(Optional.of(bonsaiUpdater), bonsaiWorldStateUpdateAccumulator); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProviderTest.java similarity index 97% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProviderTest.java index 2340a6e8b05..5199bc19db4 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProviderTest.java @@ -13,13 +13,15 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat; import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.FullFlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.PartialFlatDbStrategy; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayerTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayerTests.java similarity index 98% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayerTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayerTests.java index d7d49bace04..d967ca1a1b4 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayerTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayerTests.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import org.hyperledger.besu.datatypes.AccountValue; import org.hyperledger.besu.datatypes.Address; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestKeyValueStorageWorldStateStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestKeyValueStorageWorldStateStorageTest.java index 08dfc86d5c0..d60f92448cc 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestKeyValueStorageWorldStateStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestKeyValueStorageWorldStateStorageTest.java @@ -66,6 +66,7 @@ public void getCode_saveAndGetSpecialValues() { storage.updater().putCode(MerkleTrie.EMPTY_TRIE_NODE).putCode(Bytes.EMPTY).commit(); assertThat(storage.getCode(Hash.EMPTY_TRIE_HASH)).contains(MerkleTrie.EMPTY_TRIE_NODE); + assertThat(storage.getCode(Hash.EMPTY)).contains(Bytes.EMPTY); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java index 34c906bd90d..1a82d56355f 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java @@ -34,7 +34,7 @@ import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.trie.forest.pruner.Pruner; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.BesuMetricCategory; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java index 87a021136e2..7b64f885c29 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java @@ -21,7 +21,7 @@ import org.hyperledger.besu.ethereum.eth.sync.TrailingPeerRequirements; import org.hyperledger.besu.ethereum.eth.sync.worldstate.StalledDownloadException; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.services.tasks.TaskCollection; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStep.java index 264b29017d7..e9fb43ffeaf 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStep.java @@ -20,7 +20,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.TrieNodeHealingRequest; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.exception.StorageException; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java index 707634664bb..3d93b852297 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java @@ -30,7 +30,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.StorageFlatDatabaseHealingRangeRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldDownloadState; import org.hyperledger.besu.ethereum.trie.RangeManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java index 76a07a4d8ea..471c84a6321 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java @@ -28,7 +28,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; import org.hyperledger.besu.ethereum.trie.RangeManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java index 94ba92b5356..bb98534fd0e 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.trie.NodeUpdater; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java index 48c25fadcd5..81b4897a863 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.trie.CompactEncoding; import org.hyperledger.besu.ethereum.trie.NodeUpdater; import org.hyperledger.besu.ethereum.trie.RangeManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java index 5c114d647a1..f2514b34055 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java index 82a9e3f5510..ed81fe65d50 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java @@ -27,7 +27,7 @@ import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java index 822ac5095d4..e5c8ce55583 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java @@ -36,7 +36,7 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.checkpoint.ImmutableCheckpoint; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java index 765eac4c5c7..e100c036989 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate.FastDownloaderFactory; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java index 0c625c61344..ae252a39f47 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java @@ -34,7 +34,7 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate.NodeDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.StalledDownloadException; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java index 3555a9b7179..44847ffb93e 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java @@ -26,7 +26,7 @@ import org.hyperledger.besu.ethereum.eth.manager.task.EthTask; import org.hyperledger.besu.ethereum.eth.sync.worldstate.StalledDownloadException; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloadProcess; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java index e4a827b7f00..6707ed97054 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java @@ -22,7 +22,7 @@ import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java index 296952cf029..03aa292fabf 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java index 9a53a4418b4..0de00f5ab2d 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java @@ -25,7 +25,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.BytecodeRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.StorageRangeDataRequest; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java index 18ab5f02fa7..35c5de5daca 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java @@ -39,7 +39,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloadProcess; import org.hyperledger.besu.ethereum.trie.RangeManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java index 9b5d9355168..81b9d385763 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java @@ -27,7 +27,7 @@ import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; @@ -51,6 +51,7 @@ public static List> createAccountRequest(final boolean wit new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_BONSAI_CONFIG); + final WorldStateStorageCoordinator worldStateStorageCoordinator = new WorldStateStorageCoordinator(worldStateKeyValueStorage); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java index 6c3ba6f0daa..dba37b5afa3 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java @@ -29,7 +29,7 @@ import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java index eb6eedfb4c4..da5354f71bf 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; @@ -89,7 +89,6 @@ public void setup() { DataStorageConfiguration.DEFAULT_BONSAI_CONFIG); worldStateStorageCoordinator = new WorldStateStorageCoordinator(worldStateKeyValueStorage); proofProvider = new WorldStateProofProvider(worldStateStorageCoordinator); - trie = TrieGenerator.generateTrie( worldStateStorageCoordinator, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java index 1f87a2d783e..5cec07c152f 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java @@ -21,7 +21,7 @@ import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java index c64bbec90d8..ee4730ffda3 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java @@ -17,11 +17,13 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiPreImageProxy; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiPreImageProxy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.Consumer; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.StorageConsumingMap; import org.hyperledger.besu.evm.internal.EvmConfiguration; import java.util.concurrent.ConcurrentHashMap; @@ -32,8 +34,8 @@ public class BonsaiReferenceTestUpdateAccumulator extends BonsaiWorldStateUpdate private final BonsaiPreImageProxy preImageProxy; public BonsaiReferenceTestUpdateAccumulator( - final BonsaiWorldView world, - final Consumer> accountPreloader, + final DiffBasedWorldView world, + final Consumer> accountPreloader, final Consumer storagePreloader, final BonsaiPreImageProxy preImageProxy, final EvmConfiguration evmConfiguration) { @@ -65,7 +67,7 @@ public BonsaiReferenceTestUpdateAccumulator createDetachedAccumulator() { getStorageToUpdate() .forEach( (k, v) -> { - StorageConsumingMap> newMap = + StorageConsumingMap> newMap = new StorageConsumingMap<>(k, new ConcurrentHashMap<>(), v.getConsumer()); v.forEach((key, value) -> newMap.put(key, value.copy())); copy.getStorageToUpdate().put(k, newMap); diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java index 1c536e96a38..f16452f9360 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -18,16 +18,18 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.NoOpCachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiPreImageProxy; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateLayerStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogAddedEvent; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.NoOpBonsaiCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiPreImageProxy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogAddedEvent; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldUpdater; @@ -54,14 +56,14 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState protected BonsaiReferenceTestWorldState( final BonsaiReferenceTestWorldStateStorage worldStateKeyValueStorage, - final CachedMerkleTrieLoader cachedMerkleTrieLoader, - final CachedWorldStorageManager cachedWorldStorageManager, + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader, + final DiffBasedCachedWorldStorageManager cachedWorldStorageManager, final TrieLogManager trieLogManager, final BonsaiPreImageProxy preImageProxy, final EvmConfiguration evmConfiguration) { super( worldStateKeyValueStorage, - cachedMerkleTrieLoader, + bonsaiCachedMerkleTrieLoader, cachedWorldStorageManager, trieLogManager, evmConfiguration); @@ -72,21 +74,21 @@ protected BonsaiReferenceTestWorldState( new BonsaiReferenceTestUpdateAccumulator( this, (addr, value) -> - cachedMerkleTrieLoader.preLoadAccount( + bonsaiCachedMerkleTrieLoader.preLoadAccount( getWorldStateStorage(), worldStateRootHash, addr), (addr, value) -> - cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value), + bonsaiCachedMerkleTrieLoader.preLoadStorageSlot( + getWorldStateStorage(), addr, value), preImageProxy, evmConfiguration)); } @Override public ReferenceTestWorldState copy() { - var layerCopy = - new BonsaiReferenceTestWorldStateStorage(worldStateKeyValueStorage, preImageProxy); + var layerCopy = new BonsaiReferenceTestWorldStateStorage(getWorldStateStorage(), preImageProxy); return new BonsaiReferenceTestWorldState( layerCopy, - cachedMerkleTrieLoader, + bonsaiCachedMerkleTrieLoader, cachedWorldStorageManager, trieLogManager, preImageProxy, @@ -187,8 +189,9 @@ private void generateTrieLogFromState( private BonsaiWorldState createBonsaiWorldState(final boolean isFrozen) { BonsaiWorldState bonsaiWorldState = new BonsaiWorldState( - new BonsaiWorldStateLayerStorage(worldStateKeyValueStorage), - cachedMerkleTrieLoader, + new BonsaiWorldStateLayerStorage( + (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage), + bonsaiCachedMerkleTrieLoader, cachedWorldStorageManager, trieLogManager, evmConfiguration); @@ -209,8 +212,11 @@ public static BonsaiReferenceTestWorldState create( final Map accounts, final EvmConfiguration evmConfiguration) { final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem(); - final CachedMerkleTrieLoader cachedMerkleTrieLoader = new CachedMerkleTrieLoader(metricsSystem); + + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader = + new BonsaiCachedMerkleTrieLoader(metricsSystem); final TrieLogManager trieLogManager = new ReferenceTestsInMemoryTrieLogManager(); + final BonsaiPreImageProxy preImageProxy = new BonsaiPreImageProxy.BonsaiReferenceTestPreImageProxy(); @@ -223,13 +229,13 @@ public static BonsaiReferenceTestWorldState create( final BonsaiReferenceTestWorldStateStorage worldStateKeyValueStorage = new BonsaiReferenceTestWorldStateStorage(bonsaiWorldStateKeyValueStorage, preImageProxy); - final NoOpCachedWorldStorageManager noOpCachedWorldStorageManager = - new NoOpCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage); + final NoOpBonsaiCachedWorldStorageManager noOpCachedWorldStorageManager = + new NoOpBonsaiCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage); final BonsaiReferenceTestWorldState worldState = new BonsaiReferenceTestWorldState( worldStateKeyValueStorage, - cachedMerkleTrieLoader, + bonsaiCachedMerkleTrieLoader, noOpCachedWorldStorageManager, trieLogManager, preImageProxy, @@ -260,10 +266,10 @@ public ReferenceTestsInMemoryTrieLogManager() { @Override public synchronized void saveTrieLog( - final BonsaiWorldStateUpdateAccumulator localUpdater, + final DiffBasedWorldStateUpdateAccumulator localUpdater, final Hash forWorldStateRootHash, final BlockHeader forBlockHeader, - final BonsaiWorldState forWorldState) { + final DiffBasedWorldState forWorldState) { // notify trie log added observers, synchronously TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); trieLogCache.put(forBlockHeader.getHash(), trieLogFactory.serialize(trieLog)); diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java index 68057523a08..0993d4425be 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java @@ -16,11 +16,11 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiPreImageProxy; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateLayerStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiPreImageProxy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; import org.hyperledger.besu.evm.account.AccountStorageEntry; import org.hyperledger.besu.evm.worldstate.WorldState; @@ -64,7 +64,7 @@ public NavigableMap storageEntriesFrom( } public Stream streamAccounts( - final BonsaiWorldView context, final Bytes32 startKeyHash, final int limit) { + final DiffBasedWorldView context, final Bytes32 startKeyHash, final int limit) { return streamFlatAccounts(startKeyHash, UInt256.MAX_VALUE, limit) .entrySet() // map back to addresses using preImage provider: diff --git a/gradle.properties b/gradle.properties index 265507cea43..61b69b61768 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,4 +13,4 @@ org.gradle.jvmargs=-Xmx4g \ --add-opens jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED \ --add-opens jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED # Could be moved to sonar properties after https://sonarsource.atlassian.net/browse/SONARGRADL-134 -systemProp.sonar.gradle.skipCompile=true +systemProp.sonar.gradle.skipCompile=true \ No newline at end of file