Skip to content

Commit

Permalink
storage format refactor for preparing verkle trie integration (hyperl…
Browse files Browse the repository at this point in the history
…edger#6721)

Splitting Bonsai code: This commit aims to divide Bonsai into two packages

- Common Classes with a Prefix: The common part will include classes prefixed with "DiffBased." These classes are designed to provide a base for both Bonsai and any future storage format that might use this diff-based approach.

- Bonsai's Specifics: Bonsai will retain its unique features in its own package. This means that while it shares the diff-based infrastructure with the common part, it also has its own specific functionalities that are not shared with other storage format.

- Extension to Verkle: this modification add the possibility of adding "Verkle" as a new storage format based on the diff-based architecture. Like Bonsai, Verkle would use the common diff-based classes but also have its own specific features.

---------

Signed-off-by: Karim TAAM <karim.t2am@gmail.com>
Signed-off-by: Karim Taam <karim.t2am@gmail.com>
  • Loading branch information
matkt authored Mar 27, 2024
1 parent 7e46889 commit f2c2512
Show file tree
Hide file tree
Showing 93 changed files with 1,968 additions and 1,277 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@
import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput;
import org.hyperledger.besu.ethereum.rlp.RLP;
import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl;
import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl;
import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;

import java.io.File;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
import org.hyperledger.besu.ethereum.storage.StorageProvider;
import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
import org.hyperledger.besu.cli.BesuCommand;
import org.hyperledger.besu.ethereum.eth.transactions.BlobCache;
import org.hyperledger.besu.ethereum.eth.transactions.BlobCacheModule;
import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader;
import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoaderModule;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoaderModule;
import org.hyperledger.besu.metrics.MetricsSystemModule;
import org.hyperledger.besu.metrics.ObservableMetricsSystem;
import org.hyperledger.besu.services.BesuPluginContextImpl;
Expand All @@ -37,7 +37,7 @@
modules = {
BesuCommandModule.class,
MetricsSystemModule.class,
CachedMerkleTrieLoaderModule.class,
BonsaiCachedMerkleTrieLoaderModule.class,
BesuPluginContextModule.class,
BlobCacheModule.class
})
Expand All @@ -55,7 +55,7 @@ public interface BesuComponent {
*
* @return CachedMerkleTrieLoader
*/
CachedMerkleTrieLoader getCachedMerkleTrieLoader();
BonsaiCachedMerkleTrieLoader getCachedMerkleTrieLoader();

/**
* a metrics system that is observable by a Prometheus or OTEL metrics collection subsystem
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,11 +81,11 @@
import org.hyperledger.besu.ethereum.p2p.config.SubProtocolConfiguration;
import org.hyperledger.besu.ethereum.storage.StorageProvider;
import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier;
import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider;
import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader;
import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager;
import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager;
import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner;
import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive;
import org.hyperledger.besu.ethereum.trie.forest.pruner.MarkSweepPruner;
import org.hyperledger.besu.ethereum.trie.forest.pruner.Pruner;
Expand Down Expand Up @@ -576,13 +576,14 @@ public BesuController build() {
dataDirectory.toString(),
numberOfBlocksToCache);

final CachedMerkleTrieLoader cachedMerkleTrieLoader =
final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader =
besuComponent
.map(BesuComponent::getCachedMerkleTrieLoader)
.orElseGet(() -> new CachedMerkleTrieLoader(metricsSystem));
.orElseGet(() -> new BonsaiCachedMerkleTrieLoader(metricsSystem));

final WorldStateArchive worldStateArchive =
createWorldStateArchive(worldStateStorageCoordinator, blockchain, cachedMerkleTrieLoader);
createWorldStateArchive(
worldStateStorageCoordinator, blockchain, bonsaiCachedMerkleTrieLoader);

if (blockchain.getChainHeadBlockNumber() < 1) {
genesisState.writeStateTo(worldStateArchive.getMutable());
Expand Down Expand Up @@ -1048,7 +1049,7 @@ private Optional<SnapProtocolManager> createSnapProtocolManager(
WorldStateArchive createWorldStateArchive(
final WorldStateStorageCoordinator worldStateStorageCoordinator,
final Blockchain blockchain,
final CachedMerkleTrieLoader cachedMerkleTrieLoader) {
final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader) {
return switch (dataStorageConfiguration.getDataStorageFormat()) {
case BONSAI -> {
final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage =
Expand All @@ -1057,7 +1058,7 @@ yield new BonsaiWorldStateProvider(
worldStateKeyValueStorage,
blockchain,
Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()),
cachedMerkleTrieLoader,
bonsaiCachedMerkleTrieLoader,
besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null),
evmConfiguration);
}
Expand All @@ -1067,6 +1068,8 @@ yield new BonsaiWorldStateProvider(
yield new ForestWorldStateArchive(
worldStateStorageCoordinator, preimageStorage, evmConfiguration);
}
default -> throw new IllegalStateException(
"Unexpected value: " + dataStorageConfiguration.getDataStorageFormat());
};
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@
import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider;
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput;
import org.hyperledger.besu.ethereum.storage.StorageProvider;
import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl;
import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl;
import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,9 @@
import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier;
import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage;
import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader;
import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState;
import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration;
import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
Expand Down Expand Up @@ -189,7 +189,7 @@ public void shouldDisablePruningIfBonsaiIsEnabled() {
.createWorldStateArchive(
any(WorldStateStorageCoordinator.class),
any(Blockchain.class),
any(CachedMerkleTrieLoader.class));
any(BonsaiCachedMerkleTrieLoader.class));
doReturn(mockWorldState).when(worldStateArchive).getMutable();
when(storageProvider.createWorldStateStorageCoordinator(dataStorageConfiguration))
.thenReturn(new WorldStateStorageCoordinator(bonsaiWorldStateStorage));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Wei;
import org.hyperledger.besu.ethereum.api.query.BlockchainQueries;
import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount;
import org.hyperledger.besu.evm.account.Account;
import org.hyperledger.besu.evm.account.AccountState;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@
import org.hyperledger.besu.ethereum.privacy.storage.PrivateMetadataUpdater;
import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult;
import org.hyperledger.besu.ethereum.trie.MerkleTrieException;
import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState;
import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator;
import org.hyperledger.besu.ethereum.vm.BlockHashLookup;
import org.hyperledger.besu.ethereum.vm.CachingBlockHashLookup;
import org.hyperledger.besu.evm.gascalculator.CancunGasCalculator;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions;
import org.hyperledger.besu.ethereum.storage.StorageProvider;
import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@
import org.hyperledger.besu.ethereum.core.MutableWorldState;
import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider;
import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader;
import org.hyperledger.besu.ethereum.trie.bonsai.cache.NoOpCachedWorldStorageManager;
import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.bonsai.trielog.NoOpTrieLogManager;
import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.NoOpBonsaiCachedWorldStorageManager;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState;
import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.NoOpTrieLogManager;
import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
Expand Down Expand Up @@ -59,8 +59,8 @@ public static MutableWorldState createGenesisWorldState(
* @return a mutable world state for the Genesis block
*/
private static MutableWorldState createGenesisBonsaiWorldState() {
final CachedMerkleTrieLoader cachedMerkleTrieLoader =
new CachedMerkleTrieLoader(new NoOpMetricsSystem());
final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader =
new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem());
final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage =
new BonsaiWorldStateKeyValueStorage(
new KeyValueStorageProvider(
Expand All @@ -71,8 +71,8 @@ private static MutableWorldState createGenesisBonsaiWorldState() {
DataStorageConfiguration.DEFAULT_BONSAI_CONFIG);
return new BonsaiWorldState(
bonsaiWorldStateKeyValueStorage,
cachedMerkleTrieLoader,
new NoOpCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage),
bonsaiCachedMerkleTrieLoader,
new NoOpBonsaiCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage),
new NoOpTrieLogManager(),
EvmConfiguration.DEFAULT);
}
Expand Down
Loading

0 comments on commit f2c2512

Please sign in to comment.