From bb647d14432f39177899d02d4ead5a51645dccff Mon Sep 17 00:00:00 2001 From: ahamlat Date: Tue, 26 Mar 2024 17:25:15 +0100 Subject: [PATCH 01/24] Modify the message when the selection of transactions is interrupted because it reached the maximum configured duration (#6814) Signed-off-by: Ameziane H --- .../blockcreation/txselection/BlockTransactionSelector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java index b37cff7e43a..c12061f4cbe 100644 --- a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java +++ b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java @@ -192,7 +192,7 @@ private void timeLimitedSelection() { isTimeout.set(true); } LOG.warn( - "Interrupting transaction selection since it is taking more than the max configured time of " + "Interrupting the selection of transactions for block inclusion as it exceeds the maximum configured duration of " + blockTxsSelectionMaxTime + "ms", e); From 56e184441592c4980261f5adb1136c2983c7e426 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Tue, 26 Mar 2024 13:09:37 -0400 Subject: [PATCH 02/24] Block on skipped matrix (#6818) * also fail on cancelled and skipped * DCO correction --------- Signed-off-by: Justin Florentine Signed-off-by: Sally MacFarlane Signed-off-by: stefan.pingel@consensys.net Signed-off-by: Simon Dudley Signed-off-by: Usman Saleem Signed-off-by: Fabio Di Fabio Signed-off-by: Lucas Saldanha Signed-off-by: Jason Frame Signed-off-by: Karim Taam Co-authored-by: Sally MacFarlane Co-authored-by: Stefan Pingel <16143240+pinges@users.noreply.github.com> Co-authored-by: Simon Dudley Co-authored-by: Usman Saleem Co-authored-by: Fabio Di Fabio Co-authored-by: Lucas Saldanha Co-authored-by: Jason Frame Co-authored-by: Karim TAAM --- .github/workflows/acceptance-tests.yml | 13 +++++++++++-- .github/workflows/pre-review.yml | 13 +++++++++++-- .github/workflows/reference-tests.yml | 14 +++++++++++--- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 0d241424944..74ba75c2f91 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -77,6 +77,15 @@ jobs: permissions: checks: write statuses: write + if: always() steps: - - name: consolidation - run: echo "consolidating statuses" + # Fail if any `needs` job was not a success. + # Along with `if: always()`, this allows this job to act as a single required status check for the entire workflow. + - name: Fail on workflow error + run: exit 1 + if: >- + ${{ + contains(needs.*.result, 'failure') + || contains(needs.*.result, 'cancelled') + || contains(needs.*.result, 'skipped') + }} \ No newline at end of file diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index e7dc683dea4..1fa7d3d31d8 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -135,6 +135,15 @@ jobs: permissions: checks: write statuses: write + if: always() steps: - - name: consolidation - run: echo "consolidating statuses" + # Fail if any `needs` job was not a success. + # Along with `if: always()`, this allows this job to act as a single required status check for the entire workflow. + - name: Fail on workflow error + run: exit 1 + if: >- + ${{ + contains(needs.*.result, 'failure') + || contains(needs.*.result, 'cancelled') + || contains(needs.*.result, 'skipped') + }} diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index c83022526e4..68310944af4 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -69,7 +69,15 @@ jobs: permissions: checks: write statuses: write + if: always() steps: - - name: consolidation - run: echo "consolidating statuses" - + # Fail if any `needs` job was not a success. + # Along with `if: always()`, this allows this job to act as a single required status check for the entire workflow. + - name: Fail on workflow error + run: exit 1 + if: >- + ${{ + contains(needs.*.result, 'failure') + || contains(needs.*.result, 'cancelled') + || contains(needs.*.result, 'skipped') + }} From e954537fcc5236a42887c95be8ccfd0a661481e6 Mon Sep 17 00:00:00 2001 From: Usman Saleem Date: Wed, 27 Mar 2024 06:17:40 +1000 Subject: [PATCH 03/24] build - Refactor Besu custom error prone dependency (#6692) Move Besu custom error-prone checks into its own repository and use it as an external dependency. This allows to move to a newer version of Google errorprone checks as well while cleaning up build.gradle file. Key changes resulted due to this change: * String toLowerCase and toUpperCase to use Locale.ROOT as argument * Use interface such as List,Map or NavigatableMap instead of concrete class where appropriate. * Simplify StringBuilder to plain String * Suppress warnings where appropriate. ----- Signed-off-by: Usman Saleem --- .../tests/acceptance/dsl/node/BesuNode.java | 5 +- .../DeploySmartContractTransaction.java | 3 +- .../org/hyperledger/besu/RunnerBuilder.java | 7 +- .../org/hyperledger/besu/cli/BesuCommand.java | 5 +- .../besu/cli/config/NetworkName.java | 3 +- .../besu/cli/config/ProfileName.java | 4 +- .../converter/MetricCategoryConverter.java | 3 +- .../options/stable/DataStorageOptions.java | 3 +- .../options/stable/LoggingLevelOption.java | 5 +- .../operator/GenerateBlockchainConfig.java | 13 +- .../TransitionBesuControllerBuilder.java | 1 + .../besu/services/RpcEndpointServiceImpl.java | 9 +- build.gradle | 38 +-- .../org/hyperledger/besu/config/BftFork.java | 3 +- .../besu/config/JsonBftConfigOptions.java | 3 +- .../org/hyperledger/besu/config/JsonUtil.java | 3 +- .../common/bft/inttest/NetworkLayout.java | 9 +- .../crypto/SignatureAlgorithmFactory.java | 6 +- .../besu/crypto/SignatureAlgorithmType.java | 8 +- .../besu/crypto/SECP256R1Test.java | 2 +- .../org/hyperledger/besu/datatypes/Wei.java | 3 +- .../hyperledger/besu/enclave/TlsHelpers.java | 3 +- errorprone-checks/README.md | 9 - errorprone-checks/build.gradle | 70 ----- .../errorpronechecks/BannedMethod.java | 60 ---- .../DoNotCreateSecureRandomDirectly.java | 59 ---- .../DoNotInvokeMessageDigestDirectly.java | 44 --- .../DoNotReturnNullOptionals.java | 69 ----- ...ntalCliOptionMustBeCorrectlyDisplayed.java | 76 ----- .../MethodInputParametersMustBeFinal.java | 131 --------- .../PreferCommonAnnotations.java | 75 ----- .../PrivateStaticFinalLoggers.java | 70 ----- .../errorpronechecks/BannedMethodTest.java | 39 --- .../DoNotCreateSecureRandomDirectlyTest.java | 40 --- .../DoNotInvokeMessageDigestDirectlyTest.java | 40 --- .../DoNotReturnNullOptionalsTest.java | 40 --- ...CliOptionMustBeCorrectlyDisplayedTest.java | 45 --- .../MethodInputParametersMustBeFinalTest.java | 54 ---- .../PreferCommonAnnotationsTest.java | 40 --- .../PrivateStaticFinalLoggersTest.java | 40 --- .../BannedMethodNegativeCases.java | 29 -- .../BannedMethodPositiveCases.java | 50 ---- ...eateSecureRandomDirectlyNegativeCases.java | 34 --- ...eateSecureRandomDirectlyPositiveCases.java | 41 --- ...okeMessageDigestDirectlyNegativeCases.java | 26 -- ...okeMessageDigestDirectlyPositiveCases.java | 26 -- ...DoNotReturnNullOptionalsNegativeCases.java | 36 --- ...DoNotReturnNullOptionalsPositiveCases.java | 35 --- ...MustBeCorrectlyDisplayedNegativeCases.java | 50 ---- ...MustBeCorrectlyDisplayedPositiveCases.java | 41 --- ...tersMustBeFinalInterfaceNegativeCases.java | 40 --- ...tersMustBeFinalInterfacePositiveCases.java | 24 -- ...putParametersMustBeFinalNegativeCases.java | 63 ---- ...putParametersMustBeFinalPositiveCases.java | 41 --- .../PreferCommonAnnotationsNegativeCases.java | 32 --- .../PreferCommonAnnotationsPositiveCases.java | 27 -- ...rivateStaticFinalLoggersNegativeCases.java | 25 -- ...rivateStaticFinalLoggersPositiveCases.java | 25 -- .../api/graphql/GraphQLHttpService.java | 3 +- .../api/graphql/internal/Scalars.java | 4 +- .../api/jsonrpc/JsonRpcHttpService.java | 3 +- .../internal/parameters/BlockParameter.java | 3 +- .../parameters/BlockParameterOrBlockHash.java | 3 +- .../parameters/PendingTransactionsParams.java | 3 +- .../jsonrpc/websocket/WebSocketService.java | 4 +- .../api/jsonrpc/JsonRpcHttpServiceTest.java | 3 +- ethereum/core/build.gradle | 1 + .../mainnet/ProtocolScheduleBuilder.java | 5 +- .../privacy/PrivateTransactionReceipt.java | 14 +- .../besu/ethereum/mainnet/EthHashTest.java | 50 ++-- .../messages/snap/AccountRangeMessage.java | 3 +- .../messages/snap/StorageRangeMessage.java | 9 +- .../besu/ethereum/eth/sync/SyncMode.java | 5 +- .../sync/snapsync/SnapWorldDownloadState.java | 7 +- .../snapsync/SnapWorldStateDownloader.java | 4 +- .../ethereum/eth/sync/snapsync/StackTrie.java | 7 +- .../SnapSyncStatePersistenceManager.java | 3 +- .../request/AccountRangeDataRequest.java | 6 +- .../snapsync/request/SnapDataRequest.java | 6 +- .../request/StorageRangeDataRequest.java | 6 +- ...ccountFlatDatabaseHealingRangeRequest.java | 7 +- .../heal/AccountTrieNodeHealingRequest.java | 7 +- ...torageFlatDatabaseHealingRangeRequest.java | 5 +- .../AbstractPrioritizedTransactions.java | 3 +- .../AbstractSequentialTransactionsLayer.java | 3 +- .../layered/AbstractTransactionsLayer.java | 4 +- .../layered/ReadyTransactions.java | 6 +- .../layered/SparseTransactions.java | 3 +- .../layered/TransactionsLayer.java | 3 +- .../snap/StorageRangeMessageTest.java | 3 +- .../evmtool/benchmarks/BenchmarkExecutor.java | 3 +- .../p2p/discovery/PeerDiscoveryStatus.java | 4 +- .../besu/ethereum/p2p/peers/EnodeURLImpl.java | 3 +- .../ethereum/p2p/rlpx/wire/Capability.java | 2 +- .../besu/ethereum/p2p/rlpx/wire/PeerInfo.java | 2 +- .../PeerDiscoveryTimestampsTest.java | 31 +- .../MockConnectionInitializer.java | 3 +- .../rlpx/connections/netty/DeFramerTest.java | 4 +- .../ReferenceTestBlockchain.java | 1 + .../besu/ethereum/trie/RangeManager.java | 5 +- .../ethereum/trie/patricia/BranchNode.java | 2 +- gradle/verification-metadata.xml | 270 ++++++++---------- gradle/versions.gradle | 7 +- .../prometheus/MetricsHttpService.java | 6 +- .../besu/metrics/rocksdb/RocksDBStats.java | 5 +- .../besu/nat/docker/HostBasedIpDetector.java | 1 + settings.gradle | 1 - .../hyperledger/besu/util/Subscribers.java | 4 +- .../besu/util/log/LogUtilTest.java | 1 + 109 files changed, 325 insertions(+), 2011 deletions(-) delete mode 100644 errorprone-checks/README.md delete mode 100644 errorprone-checks/build.gradle delete mode 100644 errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/BannedMethod.java delete mode 100644 errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectly.java delete mode 100644 errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectly.java delete mode 100644 errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotReturnNullOptionals.java delete mode 100644 errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayed.java delete mode 100644 errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinal.java delete mode 100644 errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/PreferCommonAnnotations.java delete mode 100644 errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggers.java delete mode 100644 errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/BannedMethodTest.java delete mode 100644 errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyTest.java delete mode 100644 errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyTest.java delete mode 100644 errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsTest.java delete mode 100644 errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedTest.java delete mode 100644 errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalTest.java delete mode 100644 errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/PreferCommonAnnotationsTest.java delete mode 100644 errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersTest.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/BannedMethodNegativeCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/BannedMethodPositiveCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyNegativeCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyPositiveCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyNegativeCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyPositiveCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsNegativeCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsPositiveCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedNegativeCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedPositiveCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalInterfaceNegativeCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalInterfacePositiveCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalNegativeCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalPositiveCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PreferCommonAnnotationsNegativeCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PreferCommonAnnotationsPositiveCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersNegativeCases.java delete mode 100644 errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersPositiveCases.java diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java index c822ce899ce..80fbf7f727d 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java @@ -62,6 +62,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Properties; @@ -431,7 +432,9 @@ public NodeRequests nodeRequests() { getGenesisConfig() .map( gc -> - gc.toLowerCase().contains("ibft") ? ConsensusType.IBFT2 : ConsensusType.QBFT) + gc.toLowerCase(Locale.ROOT).contains("ibft") + ? ConsensusType.IBFT2 + : ConsensusType.QBFT) .orElse(ConsensusType.IBFT2); nodeRequests = diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/DeploySmartContractTransaction.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/DeploySmartContractTransaction.java index 14bc9a550cb..97e5c9d3878 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/DeploySmartContractTransaction.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/DeploySmartContractTransaction.java @@ -20,6 +20,7 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.stream.Collectors; import org.web3j.crypto.Credentials; @@ -83,7 +84,7 @@ && parameterTypesAreEqual(i.getParameterTypes(), parameterObjects)) @SuppressWarnings("rawtypes") private boolean parameterTypesAreEqual( - final Class[] expectedTypes, final ArrayList actualObjects) { + final Class[] expectedTypes, final List actualObjects) { if (expectedTypes.length != actualObjects.size()) { return false; } diff --git a/besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java b/besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java index 8d0a693e27f..6379e2c1a64 100644 --- a/besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java @@ -133,6 +133,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -800,7 +801,7 @@ public Runner build() { metricsSystem, supportedCapabilities, jsonRpcConfiguration.getRpcApis().stream() - .filter(apiGroup -> !apiGroup.toLowerCase().startsWith("engine")) + .filter(apiGroup -> !apiGroup.toLowerCase(Locale.ROOT).startsWith("engine")) .collect(Collectors.toList()), filterManager, accountLocalConfigPermissioningController, @@ -938,7 +939,7 @@ public Runner build() { metricsSystem, supportedCapabilities, webSocketConfiguration.getRpcApis().stream() - .filter(apiGroup -> !apiGroup.toLowerCase().startsWith("engine")) + .filter(apiGroup -> !apiGroup.toLowerCase(Locale.ROOT).startsWith("engine")) .collect(Collectors.toList()), filterManager, accountLocalConfigPermissioningController, @@ -1021,7 +1022,7 @@ public Runner build() { metricsSystem, supportedCapabilities, jsonRpcIpcConfiguration.getEnabledApis().stream() - .filter(apiGroup -> !apiGroup.toLowerCase().startsWith("engine")) + .filter(apiGroup -> !apiGroup.toLowerCase(Locale.ROOT).startsWith("engine")) .collect(Collectors.toList()), filterManager, accountLocalConfigPermissioningController, diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index c993dda3d04..901c2cf2629 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -2623,10 +2623,7 @@ private void instantiateSignatureAlgorithmFactory() { SignatureAlgorithmFactory.setInstance(SignatureAlgorithmType.create(ecCurve.get())); } catch (final IllegalArgumentException e) { throw new CommandLine.InitializationException( - new StringBuilder() - .append("Invalid genesis file configuration for ecCurve. ") - .append(e.getMessage()) - .toString()); + "Invalid genesis file configuration for ecCurve. " + e.getMessage()); } } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/config/NetworkName.java b/besu/src/main/java/org/hyperledger/besu/cli/config/NetworkName.java index 5656abc2979..39a80df6952 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/config/NetworkName.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/config/NetworkName.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.cli.config; import java.math.BigInteger; +import java.util.Locale; import java.util.Optional; import org.apache.commons.lang3.StringUtils; @@ -91,7 +92,7 @@ public boolean canSnapSync() { * @return the string */ public String normalize() { - return StringUtils.capitalize(name().toLowerCase()); + return StringUtils.capitalize(name().toLowerCase(Locale.ROOT)); } /** diff --git a/besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java b/besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java index 70b3d83ed86..111261d5c80 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java @@ -14,6 +14,8 @@ */ package org.hyperledger.besu.cli.config; +import java.util.Locale; + import org.apache.commons.lang3.StringUtils; /** Enum for profile names. Each profile corresponds to a configuration file. */ @@ -51,6 +53,6 @@ public String getConfigFile() { @Override public String toString() { - return StringUtils.capitalize(name().replaceAll("_", " ").toLowerCase()); + return StringUtils.capitalize(name().replaceAll("_", " ").toLowerCase(Locale.ROOT)); } } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/converter/MetricCategoryConverter.java b/besu/src/main/java/org/hyperledger/besu/cli/converter/MetricCategoryConverter.java index 684c3ce30c0..91ca381b50d 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/converter/MetricCategoryConverter.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/converter/MetricCategoryConverter.java @@ -18,6 +18,7 @@ import java.util.EnumSet; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import com.google.common.annotations.VisibleForTesting; @@ -54,7 +55,7 @@ public & MetricCategory> void addCategories(final Class ca * @param metricCategory the metric category */ public void addRegistryCategory(final MetricCategory metricCategory) { - metricCategories.put(metricCategory.getName().toUpperCase(), metricCategory); + metricCategories.put(metricCategory.getName().toUpperCase(Locale.ROOT), metricCategory); } /** diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java index c02e96c8325..cdbee1d4164 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java @@ -30,6 +30,7 @@ import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.List; +import java.util.Locale; import org.apache.commons.lang3.StringUtils; import picocli.CommandLine; @@ -193,6 +194,6 @@ public List getCLIOptions() { * @return the normalized string */ public String normalizeDataStorageFormat() { - return StringUtils.capitalize(dataStorageFormat.toString().toLowerCase()); + return StringUtils.capitalize(dataStorageFormat.toString().toLowerCase(Locale.ROOT)); } } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/LoggingLevelOption.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/LoggingLevelOption.java index ad119ea61a0..ce5198084cf 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/LoggingLevelOption.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/LoggingLevelOption.java @@ -14,6 +14,7 @@ */ package org.hyperledger.besu.cli.options.stable; +import java.util.Locale; import java.util.Set; import picocli.CommandLine; @@ -52,8 +53,8 @@ public void setLogLevel(final String logLevel) { if ("FATAL".equalsIgnoreCase(logLevel)) { System.out.println("FATAL level is deprecated"); this.logLevel = "ERROR"; - } else if (ACCEPTED_VALUES.contains(logLevel.toUpperCase())) { - this.logLevel = logLevel.toUpperCase(); + } else if (ACCEPTED_VALUES.contains(logLevel.toUpperCase(Locale.ROOT))) { + this.logLevel = logLevel.toUpperCase(Locale.ROOT); } else { throw new CommandLine.ParameterException( spec.commandLine(), "Unknown logging value: " + logLevel); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/GenerateBlockchainConfig.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/GenerateBlockchainConfig.java index 3dc02f0f2bf..f53f3a80574 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/GenerateBlockchainConfig.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/GenerateBlockchainConfig.java @@ -178,11 +178,9 @@ private void importPublicKey(final JsonNode publicKeyJson) { if (!SIGNATURE_ALGORITHM.get().isValidPublicKey(publicKey)) { throw new IllegalArgumentException( - new StringBuilder() - .append(publicKeyText) - .append(" is not a valid public key for elliptic curve ") - .append(SIGNATURE_ALGORITHM.get().getCurveName()) - .toString()); + publicKeyText + + " is not a valid public key for elliptic curve " + + SIGNATURE_ALGORITHM.get().getCurveName()); } writeKeypair(publicKey, null); @@ -297,10 +295,7 @@ private void processEcCurve() { SignatureAlgorithmFactory.setInstance(SignatureAlgorithmType.create(ecCurve.get())); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( - new StringBuilder() - .append("Invalid parameter for ecCurve in genesis config: ") - .append(e.getMessage()) - .toString()); + "Invalid parameter for ecCurve in genesis config: " + e.getMessage()); } } diff --git a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java index 426c15aed03..090a9a9e3ad 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java @@ -249,6 +249,7 @@ protected Synchronizer createSynchronizer( return sync; } + @SuppressWarnings("UnusedVariable") private void initTransitionWatcher( final ProtocolContext protocolContext, final TransitionCoordinator composedCoordinator) { diff --git a/besu/src/main/java/org/hyperledger/besu/services/RpcEndpointServiceImpl.java b/besu/src/main/java/org/hyperledger/besu/services/RpcEndpointServiceImpl.java index 94bb5560839..34e5b78a6af 100644 --- a/besu/src/main/java/org/hyperledger/besu/services/RpcEndpointServiceImpl.java +++ b/besu/src/main/java/org/hyperledger/besu/services/RpcEndpointServiceImpl.java @@ -24,6 +24,7 @@ import java.util.Collection; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; @@ -58,7 +59,10 @@ public void registerRPCEndpoint( namespaces.stream() .anyMatch( namespace -> - entry.getKey().toUpperCase().startsWith(namespace.toUpperCase()))) + entry + .getKey() + .toUpperCase(Locale.ROOT) + .startsWith(namespace.toUpperCase(Locale.ROOT)))) .map(entry -> new PluginJsonRpcMethod(entry.getKey(), entry.getValue())) .collect(Collectors.toMap(PluginJsonRpcMethod::getName, e -> e)); } @@ -71,6 +75,7 @@ public void registerRPCEndpoint( */ public boolean hasNamespace(final String namespace) { return rpcMethods.keySet().stream() - .anyMatch(key -> key.toUpperCase().startsWith(namespace.toUpperCase())); + .anyMatch( + key -> key.toUpperCase(Locale.ROOT).startsWith(namespace.toUpperCase(Locale.ROOT))); } } diff --git a/build.gradle b/build.gradle index 00a1be20988..7646e37302d 100644 --- a/build.gradle +++ b/build.gradle @@ -29,7 +29,7 @@ plugins { id 'com.jfrog.artifactory' version '5.1.11' id 'io.spring.dependency-management' version '1.1.4' id 'me.champeau.jmh' version '0.7.2' apply false - id 'net.ltgt.errorprone' version '3.0.1' + id 'net.ltgt.errorprone' version '3.1.0' id 'maven-publish' id 'org.sonarqube' version '4.4.1.3373' } @@ -132,12 +132,12 @@ allprojects { } task sourcesJar(type: Jar, dependsOn: classes) { - classifier = 'sources' + archiveClassifier = 'sources' from sourceSets.main.allSource } task javadocJar(type: Jar, dependsOn: javadoc) { - classifier = 'javadoc' + archiveClassifier = 'javadoc' from javadoc.outputDirectory } @@ -160,6 +160,7 @@ allprojects { url 'https://splunk.jfrog.io/splunk/ext-releases-local' content { includeGroupByRegex('com\\.splunk\\..*') } } + mavenCentral() // ethereum execution spec tests fixtures. Exclusively for ethereum submodule to run ref tests @@ -181,6 +182,8 @@ allprojects { dependencies { components.all(BouncyCastleCapability) errorprone 'com.google.errorprone:error_prone_core' + // https://github.com/hyperledger/besu-errorprone-checks/ + errorprone "org.hyperledger.besu:besu-errorprone-checks" } configurations.all { @@ -216,7 +219,7 @@ allprojects { format 'sol', { target '**/*.sol' } } - tasks.withType(JavaCompile) { + tasks.withType(JavaCompile).configureEach { options.compilerArgs += [ '-Xlint:unchecked', '-Xlint:cast', @@ -229,8 +232,8 @@ allprojects { ] options.errorprone { - excludedPaths = '.*/(generated/*.*|.*ReferenceTest_.*|build/.*/annotation-output/.*)' - + excludedPaths = '.*/generated/*.*' + disableWarningsInGeneratedCode = true // Our equals need to be symmetric, this checker doesn't respect that. check('EqualsGetClass', CheckSeverity.OFF) // We like to use futures with no return values. @@ -292,7 +295,7 @@ allprojects { * */ test { - jvmArgs = [ + jvmArgs += [ '-Xmx4g', '-XX:-UseGCOverheadLimit', // Mockito and jackson-databind do some strange reflection during tests. @@ -397,7 +400,7 @@ subprojects { task testSupportJar(type: Jar) { archiveBaseName = "${project.name}-support-test" - classifier = 'test-support' + archiveClassifier = 'test-support' from sourceSets.testSupport.output } } @@ -997,7 +1000,7 @@ task checkSpdxHeader(type: CheckSpdxHeader) { jacocoTestReport { reports { - xml.enabled true + xml.required = true } } @@ -1008,25 +1011,12 @@ task jacocoRootReport(type: org.gradle.testing.jacoco.tasks.JacocoReport) { executionData.from fileTree(dir: '.', includes: ['**/jacoco/*.exec']) reports { xml.required = true - xml.enabled = true csv.required = true html.destination file("build/reports/jacocoHtml") } onlyIf = { true } } -configurations { annotationProcessor } - -// Prevent errorprone-checks being dependent upon errorprone-checks! -// However, ensure all subprojects comply with the custom rules. -configure(subprojects.findAll { it.name != 'errorprone-checks' }) { - dependencies { annotationProcessor project(":errorprone-checks") } - - tasks.withType(JavaCompile) { - options.annotationProcessorPath = configurations.annotationProcessor - } -} - // http://label-schema.org/rc1/ // using the RFC3339 format "2016-04-12T23:20:50.52Z" def buildTime() { @@ -1107,9 +1097,11 @@ tasks.register("verifyDistributions") { } dependencies { + errorprone 'com.google.errorprone:error_prone_core' + // https://github.com/hyperledger/besu-errorprone-checks/ + errorprone 'org.hyperledger.besu:besu-errorprone-checks' implementation project(':besu') implementation project(':ethereum:evmtool') - errorprone 'com.google.errorprone:error_prone_core' } @CompileStatic diff --git a/config/src/main/java/org/hyperledger/besu/config/BftFork.java b/config/src/main/java/org/hyperledger/besu/config/BftFork.java index 7e9c408bac6..de188744b3a 100644 --- a/config/src/main/java/org/hyperledger/besu/config/BftFork.java +++ b/config/src/main/java/org/hyperledger/besu/config/BftFork.java @@ -18,6 +18,7 @@ import java.math.BigInteger; import java.util.List; +import java.util.Locale; import java.util.Optional; import java.util.OptionalInt; @@ -89,7 +90,7 @@ public Optional getBlockRewardWei() { return Optional.empty(); } final String weiStr = configFileContent.get(); - if (weiStr.toLowerCase().startsWith("0x")) { + if (weiStr.toLowerCase(Locale.ROOT).startsWith("0x")) { return Optional.of(new BigInteger(1, Bytes.fromHexStringLenient(weiStr).toArrayUnsafe())); } return Optional.of(new BigInteger(weiStr)); diff --git a/config/src/main/java/org/hyperledger/besu/config/JsonBftConfigOptions.java b/config/src/main/java/org/hyperledger/besu/config/JsonBftConfigOptions.java index 4dfa49fa3a0..a3791a6e584 100644 --- a/config/src/main/java/org/hyperledger/besu/config/JsonBftConfigOptions.java +++ b/config/src/main/java/org/hyperledger/besu/config/JsonBftConfigOptions.java @@ -17,6 +17,7 @@ import org.hyperledger.besu.datatypes.Address; import java.math.BigInteger; +import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -117,7 +118,7 @@ public BigInteger getBlockRewardWei() { return BigInteger.ZERO; } final String weiStr = configFileContent.get(); - if (weiStr.toLowerCase().startsWith("0x")) { + if (weiStr.toLowerCase(Locale.ROOT).startsWith("0x")) { return new BigInteger(1, Bytes.fromHexStringLenient(weiStr).toArrayUnsafe()); } return new BigInteger(weiStr); diff --git a/config/src/main/java/org/hyperledger/besu/config/JsonUtil.java b/config/src/main/java/org/hyperledger/besu/config/JsonUtil.java index 45c933d09d8..ef19bc7b835 100644 --- a/config/src/main/java/org/hyperledger/besu/config/JsonUtil.java +++ b/config/src/main/java/org/hyperledger/besu/config/JsonUtil.java @@ -446,7 +446,8 @@ private static boolean validateType(final JsonNode node, final JsonNodeType expe final String errorMessage = String.format( "Expected %s value but got %s", - expectedType.toString().toLowerCase(), node.getNodeType().toString().toLowerCase()); + expectedType.toString().toLowerCase(Locale.ROOT), + node.getNodeType().toString().toLowerCase(Locale.ROOT)); throw new IllegalArgumentException(errorMessage); } return true; diff --git a/consensus/common/src/test-support/java/org/hyperledger/besu/consensus/common/bft/inttest/NetworkLayout.java b/consensus/common/src/test-support/java/org/hyperledger/besu/consensus/common/bft/inttest/NetworkLayout.java index 4fbeefcc86c..bb84888c2e3 100644 --- a/consensus/common/src/test-support/java/org/hyperledger/besu/consensus/common/bft/inttest/NetworkLayout.java +++ b/consensus/common/src/test-support/java/org/hyperledger/besu/consensus/common/bft/inttest/NetworkLayout.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.NavigableMap; import java.util.Set; import java.util.TreeMap; @@ -29,11 +30,11 @@ public class NetworkLayout { private final NodeParams localNode; - private final TreeMap addressKeyMap; + private final NavigableMap addressKeyMap; private final List remotePeers; public NetworkLayout( - final NodeParams localNode, final TreeMap addressKeyMap) { + final NodeParams localNode, final NavigableMap addressKeyMap) { this.localNode = localNode; this.addressKeyMap = addressKeyMap; this.remotePeers = new ArrayList<>(addressKeyMap.values()); @@ -42,14 +43,14 @@ public NetworkLayout( public static NetworkLayout createNetworkLayout( final int validatorCount, final int firstLocalNodeBlockNum) { - final TreeMap addressKeyMap = createValidators(validatorCount); + final NavigableMap addressKeyMap = createValidators(validatorCount); final NodeParams localNode = Iterables.get(addressKeyMap.values(), firstLocalNodeBlockNum); return new NetworkLayout(localNode, addressKeyMap); } - private static TreeMap createValidators(final int validatorCount) { + private static NavigableMap createValidators(final int validatorCount) { // Map is required to be sorted by address final TreeMap addressKeyMap = new TreeMap<>(); diff --git a/crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/SignatureAlgorithmFactory.java b/crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/SignatureAlgorithmFactory.java index 0aaf6d70f1b..823561b52d0 100644 --- a/crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/SignatureAlgorithmFactory.java +++ b/crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/SignatureAlgorithmFactory.java @@ -49,10 +49,8 @@ public static void setInstance(final SignatureAlgorithmType signatureAlgorithmTy if (!SignatureAlgorithmType.isDefault(instance)) { LOG.info( - new StringBuilder("The signature algorithm uses the elliptic curve ") - .append(instance.getCurveName()) - .append(". The usage of alternative elliptic curves is still experimental.") - .toString()); + "The signature algorithm uses the elliptic curve {}. The usage of alternative elliptic curves is still experimental.", + instance.getCurveName()); } } diff --git a/crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/SignatureAlgorithmType.java b/crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/SignatureAlgorithmType.java index 26e877ab931..8f2521038a9 100644 --- a/crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/SignatureAlgorithmType.java +++ b/crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/SignatureAlgorithmType.java @@ -94,11 +94,9 @@ public static boolean isDefault(final SignatureAlgorithm signatureAlgorithm) { } private static String invalidTypeErrorMessage(final String invalidEcCurve) { - return new StringBuilder() - .append(invalidEcCurve) - .append(" is not in the list of valid elliptic curves ") - .append(getEcCurvesListAsString()) - .toString(); + return invalidEcCurve + + " is not in the list of valid elliptic curves " + + getEcCurvesListAsString(); } private static String getEcCurvesListAsString() { diff --git a/crypto/algorithms/src/test/java/org/hyperledger/besu/crypto/SECP256R1Test.java b/crypto/algorithms/src/test/java/org/hyperledger/besu/crypto/SECP256R1Test.java index 26e03e5bd3c..7112d362947 100644 --- a/crypto/algorithms/src/test/java/org/hyperledger/besu/crypto/SECP256R1Test.java +++ b/crypto/algorithms/src/test/java/org/hyperledger/besu/crypto/SECP256R1Test.java @@ -160,7 +160,7 @@ public void signatureGenerationVerificationAndPubKeyRecovery() { final BigInteger recoveredPubKeyBigInt = secp256R1.recoverFromSignature( signature.getRecId(), signature.getR(), signature.getS(), dataHash); - assertThat(recoveredPubKeyBigInt).isEqualTo(recoveredPubKeyBigInt); + assertThat(recoveredPubKeyBigInt).isEqualTo(publicKeyBigInt); }); } diff --git a/datatypes/src/main/java/org/hyperledger/besu/datatypes/Wei.java b/datatypes/src/main/java/org/hyperledger/besu/datatypes/Wei.java index 2e2a8ac4f09..ee8d5ef6f74 100644 --- a/datatypes/src/main/java/org/hyperledger/besu/datatypes/Wei.java +++ b/datatypes/src/main/java/org/hyperledger/besu/datatypes/Wei.java @@ -16,6 +16,7 @@ import java.math.BigInteger; import java.util.Arrays; +import java.util.Locale; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.units.bigints.BaseUInt256Value; @@ -224,7 +225,7 @@ static Unit getPreferred(final int numOfDigits) { @Override public String toString() { - return name().toLowerCase(); + return name().toLowerCase(Locale.ROOT); } } } diff --git a/enclave/src/integration-test/java/org/hyperledger/besu/enclave/TlsHelpers.java b/enclave/src/integration-test/java/org/hyperledger/besu/enclave/TlsHelpers.java index 7eec56153e4..89ecddba272 100644 --- a/enclave/src/integration-test/java/org/hyperledger/besu/enclave/TlsHelpers.java +++ b/enclave/src/integration-test/java/org/hyperledger/besu/enclave/TlsHelpers.java @@ -32,6 +32,7 @@ import java.security.cert.X509Certificate; import java.util.Enumeration; import java.util.List; +import java.util.Locale; import java.util.Optional; import java.util.StringJoiner; @@ -93,6 +94,6 @@ private static String generateFingerprint(final X509Certificate cert) joiner.add(String.format("%02X", b)); } - return joiner.toString().toLowerCase(); + return joiner.toString().toLowerCase(Locale.ROOT); } } diff --git a/errorprone-checks/README.md b/errorprone-checks/README.md deleted file mode 100644 index 34f336cbc95..00000000000 --- a/errorprone-checks/README.md +++ /dev/null @@ -1,9 +0,0 @@ -The creation of custom errorprone checkers was largely derived from: -* https://github.com/tbroyer/gradle-errorprone-plugin -* https://errorprone.info/docs/installation -* https://github.com/google/error-prone/wiki/Writing-a-check - -To allow for debugging from within intellij, the following must be added to the VM args -in the run/debug configuration (this assumes your gradle cache is at the default location under -your home): --Xbootclasspath/p:${HOME}/.gradle/caches/./modules-2/files-2.1/com.google.errorprone/javac/9+181-r4173-1/bdf4c0aa7d540ee1f7bf14d47447aea4bbf450c5/javac-9+181-r4173-1.jar diff --git a/errorprone-checks/build.gradle b/errorprone-checks/build.gradle deleted file mode 100644 index fdb773b754c..00000000000 --- a/errorprone-checks/build.gradle +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -// we use this config to get the path of the JDK 9 javac jar, to -// stick it in the bootclasspath when running tests -configurations.maybeCreate("epJavac") - - -apply plugin: 'java' -apply plugin: 'net.ltgt.errorprone' - -sourceCompatibility = 17 -targetCompatibility = 17 - -dependencies { - api 'org.slf4j:slf4j-api' - - annotationProcessor 'com.google.auto.service:auto-service' - - implementation 'com.google.auto.service:auto-service' - implementation 'com.google.errorprone:error_prone_annotation' - implementation 'com.google.errorprone:error_prone_core' - implementation 'info.picocli:picocli' - - testImplementation 'com.google.errorprone:error_prone_test_helpers' - testImplementation 'org.assertj:assertj-core' - testImplementation 'org.junit.jupiter:junit-jupiter' - // imported to get org.jetbrains.annotations.NotNull - testImplementation 'org.jetbrains.kotlin:kotlin-stdlib' - - epJavac 'com.google.errorprone:error_prone_check_api' -} - -test { testLogging { showStandardStreams = true } } - - -tasks.withType(JavaCompile) { - options.compilerArgs += [ - '--add-exports', - 'jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED', - '--add-exports', - 'jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED', - '--add-exports', - 'jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED', - '--add-exports', - 'jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED', - '--add-exports', - 'jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED', - '--add-exports', - 'jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED', - '--add-exports', - 'jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED', - '--add-exports', - 'jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED' - ] -} - -javadoc { enabled = false } diff --git a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/BannedMethod.java b/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/BannedMethod.java deleted file mode 100644 index 8f71df921e4..00000000000 --- a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/BannedMethod.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import static com.google.errorprone.BugPattern.SeverityLevel.WARNING; -import static com.google.errorprone.bugpatterns.BugChecker.MethodInvocationTreeMatcher; -import static com.google.errorprone.matchers.Description.NO_MATCH; -import static com.google.errorprone.matchers.method.MethodMatchers.staticMethod; - -import java.util.Map; - -import com.google.auto.service.AutoService; -import com.google.common.collect.ImmutableMap; -import com.google.errorprone.BugPattern; -import com.google.errorprone.VisitorState; -import com.google.errorprone.bugpatterns.BugChecker; -import com.google.errorprone.matchers.Description; -import com.google.errorprone.matchers.Matcher; -import com.sun.source.tree.ExpressionTree; -import com.sun.source.tree.MethodInvocationTree; - -@AutoService(BugChecker.class) -@BugPattern( - summary = "Some methods should not be used, make sure that doesn't happen.", - severity = WARNING, - linkType = BugPattern.LinkType.NONE) -public class BannedMethod extends BugChecker implements MethodInvocationTreeMatcher { - - private static final ImmutableMap, String> BANNED_METHOD_LIST = - ImmutableMap.of( - staticMethod().onClass("com.google.common.base.Objects").withAnyName(), - "Do not use com.google.common.base.Objects methods, use java.util.Objects methods instead.", - staticMethod().onClass("org.junit.Assert"), - "Do not use junit assertions. Use assertj assertions instead.", - staticMethod().onClass("org.apache.logging.log4j.LogManager"), - "Do not use org.apache.logging.log4j.LogManager, use org.slf4j.LoggerFactory methods instead."); - - @Override - public Description matchMethodInvocation( - final MethodInvocationTree tree, final VisitorState state) { - for (final Map.Entry, String> entry : BANNED_METHOD_LIST.entrySet()) { - if (entry.getKey().matches(tree, state)) { - return buildDescription(tree).setMessage(entry.getValue()).build(); - } - } - return NO_MATCH; - } -} diff --git a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectly.java b/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectly.java deleted file mode 100644 index 95c7d18039c..00000000000 --- a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectly.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import static com.google.errorprone.BugPattern.SeverityLevel.WARNING; - -import com.google.auto.service.AutoService; -import com.google.errorprone.BugPattern; -import com.google.errorprone.VisitorState; -import com.google.errorprone.bugpatterns.BugChecker; -import com.google.errorprone.bugpatterns.BugChecker.MethodInvocationTreeMatcher; -import com.google.errorprone.bugpatterns.BugChecker.NewClassTreeMatcher; -import com.google.errorprone.matchers.Description; -import com.google.errorprone.util.ASTHelpers; -import com.sun.source.tree.MethodInvocationTree; -import com.sun.source.tree.NewClassTree; -import com.sun.tools.javac.code.Symbol; - -@AutoService(BugChecker.class) -@BugPattern( - summary = "Do not create SecureRandom directly.", - severity = WARNING, - linkType = BugPattern.LinkType.NONE) -public class DoNotCreateSecureRandomDirectly extends BugChecker - implements MethodInvocationTreeMatcher, NewClassTreeMatcher { - - @SuppressWarnings("TreeToString") - @Override - public Description matchMethodInvocation( - final MethodInvocationTree tree, final VisitorState state) { - if (tree.getMethodSelect().toString().equals("SecureRandom.getInstance")) { - return describeMatch(tree); - } - - return Description.NO_MATCH; - } - - @Override - public Description matchNewClass(final NewClassTree tree, final VisitorState state) { - final Symbol sym = ASTHelpers.getSymbol(tree.getIdentifier()); - if (sym != null && sym.toString().equals("java.security.SecureRandom")) { - return describeMatch(tree); - } - - return Description.NO_MATCH; - } -} diff --git a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectly.java b/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectly.java deleted file mode 100644 index 7ce0c81f05c..00000000000 --- a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectly.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import static com.google.errorprone.BugPattern.SeverityLevel.WARNING; - -import com.google.auto.service.AutoService; -import com.google.errorprone.BugPattern; -import com.google.errorprone.VisitorState; -import com.google.errorprone.bugpatterns.BugChecker; -import com.google.errorprone.bugpatterns.BugChecker.MethodInvocationTreeMatcher; -import com.google.errorprone.matchers.Description; -import com.sun.source.tree.MethodInvocationTree; - -@AutoService(BugChecker.class) -@BugPattern( - summary = "Do not invoke MessageDigest.getInstance directly.", - severity = WARNING, - linkType = BugPattern.LinkType.NONE) -public class DoNotInvokeMessageDigestDirectly extends BugChecker - implements MethodInvocationTreeMatcher { - - @SuppressWarnings("TreeToString") - @Override - public Description matchMethodInvocation( - final MethodInvocationTree tree, final VisitorState state) { - if (tree.getMethodSelect().toString().equals("MessageDigest.getInstance")) { - return describeMatch(tree); - } - return Description.NO_MATCH; - } -} diff --git a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotReturnNullOptionals.java b/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotReturnNullOptionals.java deleted file mode 100644 index 7310058e5a7..00000000000 --- a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/DoNotReturnNullOptionals.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import static com.google.errorprone.BugPattern.SeverityLevel.SUGGESTION; -import static com.google.errorprone.matchers.Matchers.contains; -import static com.sun.source.tree.Tree.Kind.NULL_LITERAL; - -import com.google.auto.service.AutoService; -import com.google.errorprone.BugPattern; -import com.google.errorprone.VisitorState; -import com.google.errorprone.bugpatterns.BugChecker; -import com.google.errorprone.bugpatterns.BugChecker.MethodTreeMatcher; -import com.google.errorprone.matchers.Description; -import com.google.errorprone.matchers.Matcher; -import com.sun.source.tree.MethodTree; -import com.sun.source.tree.ReturnTree; -import com.sun.source.tree.Tree; - -/* - * This is reworked from an example found at: - * https://github.com/google/error-prone/wiki/Writing-a-check - */ - -@AutoService(BugChecker.class) // the service descriptor -@BugPattern( - summary = "Do not return null optionals.", - severity = SUGGESTION, - linkType = BugPattern.LinkType.NONE) -public class DoNotReturnNullOptionals extends BugChecker implements MethodTreeMatcher { - - private static class ReturnNullMatcher implements Matcher { - - @Override - public boolean matches(final Tree tree, final VisitorState state) { - if ((tree instanceof ReturnTree) && (((ReturnTree) tree).getExpression() != null)) { - return ((ReturnTree) tree).getExpression().getKind() == NULL_LITERAL; - } - return false; - } - } - - private static final Matcher RETURN_NULL = new ReturnNullMatcher(); - private static final Matcher CONTAINS_RETURN_NULL = contains(RETURN_NULL); - - @SuppressWarnings("TreeToString") - @Override - public Description matchMethod(final MethodTree tree, final VisitorState state) { - if ((tree.getReturnType() == null) - || !tree.getReturnType().toString().startsWith("Optional<") - || (tree.getBody() == null) - || (!CONTAINS_RETURN_NULL.matches(tree.getBody(), state))) { - return Description.NO_MATCH; - } - return describeMatch(tree); - } -} diff --git a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayed.java b/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayed.java deleted file mode 100644 index 3cda884792c..00000000000 --- a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayed.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import static com.google.errorprone.BugPattern.SeverityLevel.WARNING; - -import java.util.Map; -import java.util.Optional; -import javax.lang.model.element.AnnotationMirror; -import javax.lang.model.element.AnnotationValue; -import javax.lang.model.element.ExecutableElement; - -import com.google.auto.service.AutoService; -import com.google.errorprone.BugPattern; -import com.google.errorprone.VisitorState; -import com.google.errorprone.bugpatterns.BugChecker; -import com.google.errorprone.bugpatterns.BugChecker.AnnotationTreeMatcher; -import com.google.errorprone.matchers.Description; -import com.google.errorprone.util.ASTHelpers; -import com.sun.source.tree.AnnotationTree; -import com.sun.tools.javac.tree.JCTree; - -@AutoService(BugChecker.class) -@BugPattern( - summary = "Experimental options must be hidden and not present in the BesuCommand class.", - severity = WARNING, - linkType = BugPattern.LinkType.NONE) -public class ExperimentalCliOptionMustBeCorrectlyDisplayed extends BugChecker - implements AnnotationTreeMatcher { - - @Override - public Description matchAnnotation(AnnotationTree tree, VisitorState state) { - final AnnotationMirror annotationMirror = ASTHelpers.getAnnotationMirror(tree); - if (annotationMirror.getAnnotationType().toString().equals("picocli.CommandLine.Option")) { - final Optional names = - getAnnotationValue(annotationMirror, "names"); - if (names.isPresent() && names.get().getValue().toString().contains("--X")) { - final JCTree.JCCompilationUnit compilation = - (JCTree.JCCompilationUnit) state.getPath().getCompilationUnit(); - if (compilation.getSourceFile().getName().endsWith("BesuCommand.java")) { - return describeMatch(tree); - } - final Optional isHidden = - getAnnotationValue(annotationMirror, "hidden"); - if (isHidden.isEmpty() || !((boolean) isHidden.get().getValue())) { - return describeMatch(tree); - } - } - } - return Description.NO_MATCH; - } - - private Optional getAnnotationValue( - final AnnotationMirror annotationMirror, final String name) { - final Map elementValues = - annotationMirror.getElementValues(); - final Optional retValue = - elementValues.keySet().stream() - .filter(k -> k.getSimpleName().toString().equals(name)) - .map(elementValues::get) - .findAny(); - return retValue; - } -} diff --git a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinal.java b/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinal.java deleted file mode 100644 index abd6f3b58d0..00000000000 --- a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinal.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import static com.google.errorprone.BugPattern.SeverityLevel.WARNING; - -import javax.lang.model.element.Modifier; - -import com.google.auto.service.AutoService; -import com.google.errorprone.BugPattern; -import com.google.errorprone.VisitorState; -import com.google.errorprone.bugpatterns.BugChecker; -import com.google.errorprone.bugpatterns.BugChecker.ClassTreeMatcher; -import com.google.errorprone.bugpatterns.BugChecker.MethodTreeMatcher; -import com.google.errorprone.matchers.Description; -import com.google.errorprone.util.ASTHelpers; -import com.sun.source.tree.ClassTree; -import com.sun.source.tree.MethodTree; -import com.sun.source.tree.ModifiersTree; -import com.sun.source.tree.VariableTree; - -@AutoService(BugChecker.class) -@BugPattern( - summary = "Method input parameters must be final.", - severity = WARNING, - linkType = BugPattern.LinkType.NONE) -public class MethodInputParametersMustBeFinal extends BugChecker - implements MethodTreeMatcher, ClassTreeMatcher { - - private boolean isAbstraction = false; - private boolean isGenerated = false; - - @Override - public Description matchClass(final ClassTree tree, final VisitorState state) { - isAbstraction = - isInterface(tree.getModifiers()) - || isAnonymousClassInAbstraction(tree) - || isEnumInAbstraction(tree); - isGenerated = ASTHelpers.hasDirectAnnotationWithSimpleName(tree, "Generated"); - return Description.NO_MATCH; - } - - @Override - public Description matchMethod(final MethodTree tree, final VisitorState state) { - if (isGenerated) { - return Description.NO_MATCH; - } - - final ModifiersTree mods = tree.getModifiers(); - - if (isAbstraction) { - if (isConcreteMethod(mods)) { - return matchParameters(tree); - } - } else if (isNotAbstract(mods)) { - return matchParameters(tree); - } - - return Description.NO_MATCH; - } - - private Description matchParameters(final MethodTree tree) { - for (final VariableTree inputParameter : tree.getParameters()) { - if (isMissingFinalModifier(inputParameter)) { - return describeMatch(tree); - } - } - - return Description.NO_MATCH; - } - - private boolean isMissingFinalModifier(final VariableTree inputParameter) { - return !inputParameter.getModifiers().getFlags().contains(Modifier.FINAL); - } - - private boolean isNotAbstract(final ModifiersTree mods) { - return !mods.getFlags().contains(Modifier.ABSTRACT); - } - - @SuppressWarnings("TreeToString") - private boolean isInterface(final ModifiersTree mods) { - return mods.toString().contains("interface"); - } - - private boolean isConcreteMethod(final ModifiersTree mods) { - return mods.getFlags().contains(Modifier.DEFAULT) || mods.getFlags().contains(Modifier.STATIC); - } - - private boolean isAnonymousClassInAbstraction(final ClassTree tree) { - return isAbstraction && isAnonymousClass(tree); - } - - private boolean isAnonymousClass(final ClassTree tree) { - return tree.getSimpleName().contentEquals(""); - } - - private boolean isEnumInAbstraction(final ClassTree tree) { - return isAbstraction && isEnum(tree); - } - - @SuppressWarnings("TreeToString") - private boolean isEnum(final ClassTree tree) { - return tree.toString().contains("enum"); - } - - @Override - public boolean equals(Object o) { - // isAbstract and isGenerated are transient calculations, not relevant to equality checks - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - return super.equals(o); - } - - @Override - public int hashCode() { - // isAbstract and isGenerated are transient calculations, not relevant to equality checks - return super.hashCode(); - } -} diff --git a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/PreferCommonAnnotations.java b/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/PreferCommonAnnotations.java deleted file mode 100644 index acc46c0f775..00000000000 --- a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/PreferCommonAnnotations.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * (c) Copyright 2023 Palantir Technologies Inc. All rights reserved. - * Copyright Hyperledger Besu contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -/* Derived from https://github.com/palantir/gradle-baseline/blob/6fe385a80291473e7fc1441f176454bec4184d6b/baseline-error-prone/src/main/java/com/palantir/baseline/errorprone/PreferCommonAnnotations.java */ - -package org.hyperledger.errorpronechecks; - -import java.util.Map; -import java.util.Objects; - -import com.google.auto.service.AutoService; -import com.google.errorprone.BugPattern; -import com.google.errorprone.BugPattern.SeverityLevel; -import com.google.errorprone.VisitorState; -import com.google.errorprone.bugpatterns.BugChecker; -import com.google.errorprone.bugpatterns.BugChecker.ImportTreeMatcher; -import com.google.errorprone.fixes.SuggestedFix; -import com.google.errorprone.matchers.Description; -import com.google.errorprone.util.ASTHelpers; -import com.sun.source.tree.ImportTree; -import com.sun.tools.javac.code.Type; - -/** - * Checker that recommends using the common version of an annotation. - * - *

Examples: - Guava's version of {@code @VisibleForTesting} over other copies. - */ -@AutoService(BugChecker.class) -@BugPattern( - summary = "Prefer the common version of annotations over other copies.", - severity = SeverityLevel.WARNING) -public final class PreferCommonAnnotations extends BugChecker implements ImportTreeMatcher { - - /** ClassName -> preferred import. */ - private static final Map PREFERRED_IMPORTS = - Map.of("org.jetbrains.annotations.NotNull", "javax.annotation.Nonnull"); - - @Override - public Description matchImport(ImportTree tree, VisitorState state) { - Type importType = ASTHelpers.getType(tree.getQualifiedIdentifier()); - if (importType == null) { - return Description.NO_MATCH; - } - String importName = importType.toString(); - for (Map.Entry entry : PREFERRED_IMPORTS.entrySet()) { - String affectedClassName = entry.getKey(); - String preferredType = entry.getValue(); - if (importName.endsWith(affectedClassName) && !Objects.equals(importName, preferredType)) { - SuggestedFix fix = - SuggestedFix.builder().removeImport(importName).addImport(preferredType).build(); - return this.buildDescription(tree) - .setMessage("Do not use " + importName + " use " + preferredType + " instead.") - .addFix(fix) - .build(); - } - } - return Description.NO_MATCH; - } -} diff --git a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggers.java b/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggers.java deleted file mode 100644 index 445f3226d80..00000000000 --- a/errorprone-checks/src/main/java/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggers.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import static com.google.errorprone.BugPattern.SeverityLevel.WARNING; -import static com.google.errorprone.fixes.SuggestedFixes.addModifiers; -import static com.google.errorprone.matchers.Description.NO_MATCH; -import static com.google.errorprone.util.ASTHelpers.getType; -import static com.google.errorprone.util.ASTHelpers.isSubtype; - -import java.util.List; -import java.util.Optional; -import javax.lang.model.element.ElementKind; -import javax.lang.model.element.Modifier; - -import com.google.auto.service.AutoService; -import com.google.errorprone.BugPattern; -import com.google.errorprone.VisitorState; -import com.google.errorprone.bugpatterns.BugChecker; -import com.google.errorprone.bugpatterns.BugChecker.VariableTreeMatcher; -import com.google.errorprone.fixes.SuggestedFix; -import com.google.errorprone.matchers.Description; -import com.google.errorprone.suppliers.Supplier; -import com.google.errorprone.suppliers.Suppliers; -import com.google.errorprone.util.ASTHelpers; -import com.sun.source.tree.VariableTree; -import com.sun.tools.javac.code.Symbol; -import com.sun.tools.javac.code.Type; - -@AutoService(BugChecker.class) -@BugPattern( - summary = "Logger classes should be private, static, and final.", - severity = WARNING, - linkType = BugPattern.LinkType.NONE) -public class PrivateStaticFinalLoggers extends BugChecker implements VariableTreeMatcher { - - static final Supplier ORG_SLF4J_LOGGER = Suppliers.typeFromString("org.slf4j.Logger"); - - @Override - public Description matchVariable(final VariableTree tree, final VisitorState state) { - final Symbol.VarSymbol sym = ASTHelpers.getSymbol(tree); - if (sym == null || sym.getKind() != ElementKind.FIELD) { - return NO_MATCH; - } - if (sym.getModifiers() - .containsAll(List.of(Modifier.PRIVATE, Modifier.STATIC, Modifier.FINAL))) { - return NO_MATCH; - } - if (!isSubtype(getType(tree), ORG_SLF4J_LOGGER.get(state), state)) { - return NO_MATCH; - } - Optional fixes = - addModifiers(tree, state, Modifier.PRIVATE, Modifier.STATIC, Modifier.FINAL); - return buildDescription(tree) - .addFix(fixes.isPresent() ? fixes.get() : SuggestedFix.emptyFix()) - .build(); - } -} diff --git a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/BannedMethodTest.java b/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/BannedMethodTest.java deleted file mode 100644 index 48f382946d8..00000000000 --- a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/BannedMethodTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import com.google.errorprone.CompilationTestHelper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class BannedMethodTest { - - private CompilationTestHelper compilationHelper; - - @BeforeEach - public void setup() { - compilationHelper = CompilationTestHelper.newInstance(BannedMethod.class, getClass()); - } - - @Test - public void bannedMethodsPositiveCases() { - compilationHelper.addSourceFile("BannedMethodPositiveCases.java").doTest(); - } - - @Test - public void bannedMethodsNegativeCases() { - compilationHelper.addSourceFile("BannedMethodNegativeCases.java").doTest(); - } -} diff --git a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyTest.java b/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyTest.java deleted file mode 100644 index 525f5a8689a..00000000000 --- a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import com.google.errorprone.CompilationTestHelper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class DoNotCreateSecureRandomDirectlyTest { - - private CompilationTestHelper compilationHelper; - - @BeforeEach - public void setup() { - compilationHelper = - CompilationTestHelper.newInstance(DoNotCreateSecureRandomDirectly.class, getClass()); - } - - @Test - public void doNotCreateSecureRandomDirectlyPositiveCases() { - compilationHelper.addSourceFile("DoNotCreateSecureRandomDirectlyPositiveCases.java").doTest(); - } - - @Test - public void doNotCreateSecureRandomDirectlyNegativeCases() { - compilationHelper.addSourceFile("DoNotCreateSecureRandomDirectlyNegativeCases.java").doTest(); - } -} diff --git a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyTest.java b/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyTest.java deleted file mode 100644 index 986aedb63c2..00000000000 --- a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import com.google.errorprone.CompilationTestHelper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class DoNotInvokeMessageDigestDirectlyTest { - - private CompilationTestHelper compilationHelper; - - @BeforeEach - public void setup() { - compilationHelper = - CompilationTestHelper.newInstance(DoNotInvokeMessageDigestDirectly.class, getClass()); - } - - @Test - public void doNotInvokeMessageDigestDirectlyPositiveCases() { - compilationHelper.addSourceFile("DoNotInvokeMessageDigestDirectlyPositiveCases.java").doTest(); - } - - @Test - public void doNotInvokeMessageDigestDirectlyNegativeCases() { - compilationHelper.addSourceFile("DoNotInvokeMessageDigestDirectlyNegativeCases.java").doTest(); - } -} diff --git a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsTest.java b/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsTest.java deleted file mode 100644 index 0055c9fc1ed..00000000000 --- a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import com.google.errorprone.CompilationTestHelper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class DoNotReturnNullOptionalsTest { - - private CompilationTestHelper compilationHelper; - - @BeforeEach - public void setup() { - compilationHelper = - CompilationTestHelper.newInstance(DoNotReturnNullOptionals.class, getClass()); - } - - @Test - public void doNotReturnNullPositiveCases() { - compilationHelper.addSourceFile("DoNotReturnNullOptionalsPositiveCases.java").doTest(); - } - - @Test - public void doNotReturnNullNegativeCases() { - compilationHelper.addSourceFile("DoNotReturnNullOptionalsNegativeCases.java").doTest(); - } -} diff --git a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedTest.java b/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedTest.java deleted file mode 100644 index 159c3905eab..00000000000 --- a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import com.google.errorprone.CompilationTestHelper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class ExperimentalCliOptionMustBeCorrectlyDisplayedTest { - - private CompilationTestHelper compilationHelper; - - @BeforeEach - public void setup() { - compilationHelper = - CompilationTestHelper.newInstance( - ExperimentalCliOptionMustBeCorrectlyDisplayed.class, getClass()); - } - - @Test - public void experimentalCliOptionMustBeHiddenPositiveCases() { - compilationHelper - .addSourceFile("ExperimentalCliOptionMustBeCorrectlyDisplayedPositiveCases.java") - .doTest(); - } - - @Test - public void experimentalCliOptionMustBeHiddenNegativeCases() { - compilationHelper - .addSourceFile("ExperimentalCliOptionMustBeCorrectlyDisplayedNegativeCases.java") - .doTest(); - } -} diff --git a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalTest.java b/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalTest.java deleted file mode 100644 index f94fa2d606c..00000000000 --- a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalTest.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import com.google.errorprone.CompilationTestHelper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class MethodInputParametersMustBeFinalTest { - - private CompilationTestHelper compilationHelper; - - @BeforeEach - public void setup() { - compilationHelper = - CompilationTestHelper.newInstance(MethodInputParametersMustBeFinal.class, getClass()); - } - - @Test - public void methodInputParametersMustBeFinalPositiveCases() { - compilationHelper.addSourceFile("MethodInputParametersMustBeFinalPositiveCases.java").doTest(); - } - - @Test - public void methodInputParametersMustBeFinalInterfacePositiveCases() { - compilationHelper - .addSourceFile("MethodInputParametersMustBeFinalInterfacePositiveCases.java") - .doTest(); - } - - @Test - public void methodInputParametersMustBeFinalNegativeCases() { - compilationHelper.addSourceFile("MethodInputParametersMustBeFinalNegativeCases.java").doTest(); - } - - @Test - public void methodInputParametersMustBeFinalInterfaceNegativeCases() { - compilationHelper - .addSourceFile("MethodInputParametersMustBeFinalInterfaceNegativeCases.java") - .doTest(); - } -} diff --git a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/PreferCommonAnnotationsTest.java b/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/PreferCommonAnnotationsTest.java deleted file mode 100644 index b8a95f7d87b..00000000000 --- a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/PreferCommonAnnotationsTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Hyperledger Besu contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import com.google.errorprone.CompilationTestHelper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class PreferCommonAnnotationsTest { - - private CompilationTestHelper compilationHelper; - - @BeforeEach - public void setup() { - compilationHelper = - CompilationTestHelper.newInstance(PreferCommonAnnotations.class, getClass()); - } - - @Test - void preferCommonAnnotationsPositiveCases() { - compilationHelper.addSourceFile("PreferCommonAnnotationsPositiveCases.java").doTest(); - } - - @Test - void preferCommonAnnotationsNegativeCases() { - compilationHelper.addSourceFile("PreferCommonAnnotationsNegativeCases.java").doTest(); - } -} diff --git a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersTest.java b/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersTest.java deleted file mode 100644 index f15dabca1d6..00000000000 --- a/errorprone-checks/src/test/java/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import com.google.errorprone.CompilationTestHelper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class PrivateStaticFinalLoggersTest { - - private CompilationTestHelper compilationHelper; - - @BeforeEach - public void setup() { - compilationHelper = - CompilationTestHelper.newInstance(PrivateStaticFinalLoggers.class, getClass()); - } - - @Test - public void privateStaticFinalLoggersPositiveCases() { - compilationHelper.addSourceFile("PrivateStaticFinalLoggersPositiveCases.java").doTest(); - } - - @Test - public void privateStaticFinalLoggersNegativeCases() { - compilationHelper.addSourceFile("PrivateStaticFinalLoggersNegativeCases.java").doTest(); - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/BannedMethodNegativeCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/BannedMethodNegativeCases.java deleted file mode 100644 index 5b105322e54..00000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/BannedMethodNegativeCases.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.hyperledger.errorpronechecks; - -import java.util.Objects; - -public class BannedMethodNegativeCases { - - public void callsObjectsEquals() throws Exception { - Objects.equals("1", "1"); - } - - public void callsObjectsHashCode() throws Exception { - Objects.hash("1", "1"); - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/BannedMethodPositiveCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/BannedMethodPositiveCases.java deleted file mode 100644 index 0b2208a42e3..00000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/BannedMethodPositiveCases.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.hyperledger.errorpronechecks; - -import com.google.common.base.Objects; - -public class BannedMethodPositiveCases { - - public void callsObjectsEquals() throws Exception { - // BUG: Diagnostic contains: Do not use com.google.common.base.Objects methods, use - // java.util.Objects methods instead. - Objects.equal("1", "1"); - } - - public void callsObjectsHashCode() throws Exception { - // BUG: Diagnostic contains: Do not use com.google.common.base.Objects methods, use - // java.util.Objects methods instead. - Objects.hashCode("1", "1"); - } - - public void usesJUnitAssertions() throws Exception { - // BUG: Diagnostic contains: Do not use junit assertions. Use assertj assertions instead. - org.junit.Assert.assertEquals(1, 1); - // BUG: Diagnostic contains: Do not use junit assertions. Use assertj assertions instead. - org.junit.Assert.assertNotEquals(1, 2); - // BUG: Diagnostic contains: Do not use junit assertions. Use assertj assertions instead. - org.junit.Assert.assertTrue(true); - // BUG: Diagnostic contains: Do not use junit assertions. Use assertj assertions instead. - org.junit.Assert.assertFalse(false); - // BUG: Diagnostic contains: Do not use junit assertions. Use assertj assertions instead. - org.junit.Assert.assertNull(null); - // BUG: Diagnostic contains: Do not use junit assertions. Use assertj assertions instead. - org.junit.Assert.assertNotNull("foo"); - // BUG: Diagnostic contains: Do not use junit assertions. Use assertj assertions instead. - org.junit.Assert.assertArrayEquals(new int[] {1}, new int[] {1}); - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyNegativeCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyNegativeCases.java deleted file mode 100644 index 704eeffa105..00000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyNegativeCases.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.hyperledger.errorpronechecks; - -import java.security.Provider; -import java.security.SecureRandom; - -public class DoNotCreateSecureRandomDirectlyNegativeCases { - - public void callsNonJRESecureRandomGetInstance() throws Exception { - TestSecureRandom.getInstance(""); - TestSecureRandom.getInstance("", ""); - TestSecureRandom.getInstance("", new Provider("", 0, "") {}); - } - - public void invokesNonJRESecureRandomConstructor() throws Exception { - new TestSecureRandom(); - } - - private class TestSecureRandom extends SecureRandom {} -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyPositiveCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyPositiveCases.java deleted file mode 100644 index 8fb932cbfb0..00000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotCreateSecureRandomDirectlyPositiveCases.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.hyperledger.errorpronechecks; - -import java.security.Provider; -import java.security.SecureRandom; - -public class DoNotCreateSecureRandomDirectlyPositiveCases { - - public void callsSecureRandomGetInstance() throws Exception { - // BUG: Diagnostic contains: Do not create SecureRandom directly. - SecureRandom.getInstance(""); - - // BUG: Diagnostic contains: Do not create SecureRandom directly. - SecureRandom.getInstance("", ""); - - // BUG: Diagnostic contains: Do not create SecureRandom directly. - SecureRandom.getInstance("", new Provider("", 0, "") {}); - } - - public void invokesSecureRandomConstructor() throws Exception { - // BUG: Diagnostic contains: Do not create SecureRandom directly. - new SecureRandom(); - - // BUG: Diagnostic contains: Do not create SecureRandom directly. - new SecureRandom(new byte[] {}); - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyNegativeCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyNegativeCases.java deleted file mode 100644 index 3f84b2fb1a4..00000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyNegativeCases.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.hyperledger.errorpronechecks; - -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; - -public class DoNotInvokeMessageDigestDirectlyNegativeCases { - - public void callsMessageDigestGetInstance() throws NoSuchAlgorithmException { - MessageDigest dig = null; - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyPositiveCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyPositiveCases.java deleted file mode 100644 index 5aac65e16d6..00000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotInvokeMessageDigestDirectlyPositiveCases.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; - -public class DoNotInvokeMessageDigestDirectlyPositiveCases { - - public void callsMessageDigestGetInstance() throws NoSuchAlgorithmException { - // BUG: Diagnostic contains: Do not invoke MessageDigest.getInstance directly. - MessageDigest dig = MessageDigest.getInstance("SHA-256"); - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsNegativeCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsNegativeCases.java deleted file mode 100644 index aa9ac813905..00000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsNegativeCases.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import java.util.Optional; -import javax.annotation.Nullable; - -public class DoNotReturnNullOptionalsNegativeCases { - - public interface allInterfacesAreValid { - public Optional ExpectToBeOverridden(); - } - - public DoNotReturnNullOptionalsNegativeCases() {} - - public Optional doesNotReturnNull() { - return Optional.of(3L); - } - - @Nullable - public Optional returnsNullButAnnotatedWithNullable() { - return Optional.empty(); - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsPositiveCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsPositiveCases.java deleted file mode 100644 index cd814d9cf99..00000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/DoNotReturnNullOptionalsPositiveCases.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.hyperledger.errorpronechecks; - -import java.util.Optional; - -public class DoNotReturnNullOptionalsPositiveCases { - - // BUG: Diagnostic contains: Do not return null optionals. - public Optional returnsNull() { - return null; - } - - // BUG: Diagnostic contains: Do not return null optionals. - public Optional sometimesReturnsNull(boolean random) { - if (random) { - - return null; - } - return Optional.of(2L); - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedNegativeCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedNegativeCases.java deleted file mode 100644 index 5362bba7935..00000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedNegativeCases.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.hyperledger.errorpronechecks; - -import picocli.CommandLine; - -public class ExperimentalCliOptionMustBeCorrectlyDisplayedNegativeCases { - - @CommandLine.Option( - hidden = true, - names = {"--Xexperimental"}) - private String experimental = ""; - - @CommandLine.Option( - hidden = false, - names = {"--notExperimental"}) - private String notExperimental = ""; - - @CommandLine.Option(names = {"--notExperimental2"}) - private String notExperimental2 = ""; - - private class AnotherClass { - @CommandLine.Option(names = {"--notExperimentalInAnotherClass"}) - private String notExperimentalInAnotherClass = ""; - - @CommandLine.Option( - hidden = true, - names = {"--XexperimentalInAnotherClass"}) - private String experimentalInAnotherClass = ""; - } - - private class BesuCommand { - - @CommandLine.Option(names = {"--notExperimentalInBesuCommandClass"}) - private String notExperimentalInBesuCommandClass = ""; - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedPositiveCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedPositiveCases.java deleted file mode 100644 index c1721b4ebf8..00000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/ExperimentalCliOptionMustBeCorrectlyDisplayedPositiveCases.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.hyperledger.errorpronechecks; - -import picocli.CommandLine; - -public class ExperimentalCliOptionMustBeCorrectlyDisplayedPositiveCases { - - // BUG: Diagnostic contains: Experimental options must be hidden and not present in the - // BesuCommand class. - @CommandLine.Option( - hidden = false, - names = {"--Xexperimental"}) - private String experimental = ""; - - // BUG: Diagnostic contains: Experimental options must be hidden and not present in the - // BesuCommand class. - @CommandLine.Option(names = {"--Xexperimental2"}) - private String experimental2 = ""; - - private class BesuCommand { - - // BUG: Diagnostic contains: Experimental options must be hidden and not present in the - // BesuCommand class. - @CommandLine.Option(names = {"--XexperimentalInBesuCommandClass"}) - private String experimentalInBesuCommandClass = ""; - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalInterfaceNegativeCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalInterfaceNegativeCases.java deleted file mode 100644 index 237e3c17372..00000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalInterfaceNegativeCases.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import java.util.Observable; -import java.util.Observer; - -public interface MethodInputParametersMustBeFinalInterfaceNegativeCases { - - void parameterCannotBeFinal(int value); - - default void concreteMethod(final long value) {} - - static void anotherConcreteMethod(final double value) {} - - static Observer annonymousClass() { - return new Observer() { - @Override - public void update(final Observable o, final Object arg) {} - }; - } - - void methodAfterAnnonymousClass(int value); - - enum Status {} - - void methodAfterEnum(int value); -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalInterfacePositiveCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalInterfacePositiveCases.java deleted file mode 100644 index 759c1859d32..00000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalInterfacePositiveCases.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -public interface MethodInputParametersMustBeFinalInterfacePositiveCases { - - // BUG: Diagnostic contains: Method input parameters must be final. - default void concreteMethod(int value) {} - - // BUG: Diagnostic contains: Method input parameters must be final. - static void concreteStaticMethodsAreIncluded(int value) {} -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalNegativeCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalNegativeCases.java deleted file mode 100644 index a86056cf9f6..00000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalNegativeCases.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -import javax.annotation.processing.Generated; - -public class MethodInputParametersMustBeFinalNegativeCases { - - public void noInputParameters() {} - - public void onlyPrimativeInputParameters(final long value) {} - - public void onlyObjectInputParameters(final Object value) {} - - public void mixedInputParameters(final Object value, final int anotherValue) {} - - public interface allInterfacesAreValid { - void parameterCannotBeFinal(int value); - } -} - -@Generated( - value = "test", - comments = "Every method is buggy, but ignored because the class has been tagged generated") -class MethodInputParametersMustBeFinalPositiveCasesBugGenerated1 { - - public void primativeInputMethod(int value) {} - - public void objectInputMethod(Object value) {} - - public void mixedInputMethod(Object value, int anotherValue) {} - - @Generated( - value = "test", - comments = "Every method is buggy, but ignored because the class has been tagged generated") - public abstract class abstractClassDefinition { - public void concreteMethodsAreIncluded(int value) {} - } - - public void varArgsInputMethod(String... value) {} -} - -@Generated( - value = "test", - comments = "Every method is buggy, but ignored because the class has been tagged generated") -class MethodInputParametersMustBeFinalPositiveCasesBugGenerated2 { - - public void mixedInputMethodFirstFinal(final Object value, int anotherValue) {} - - public void mixedInputMethodSecondFinal(Object value, final int anotherValue) {} -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalPositiveCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalPositiveCases.java deleted file mode 100644 index f18fad65183..00000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/MethodInputParametersMustBeFinalPositiveCases.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.errorpronechecks; - -public class MethodInputParametersMustBeFinalPositiveCases { - - // BUG: Diagnostic contains: Method input parameters must be final. - public void primativeInputMethod(int value) {} - - // BUG: Diagnostic contains: Method input parameters must be final. - public void objectInputMethod(Object value) {} - - // BUG: Diagnostic contains: Method input parameters must be final. - public void mixedInputMethod(Object value, int anotherValue) {} - - // BUG: Diagnostic contains: Method input parameters must be final. - public void mixedInputMethodFirstFinal(final Object value, int anotherValue) {} - - // BUG: Diagnostic contains: Method input parameters must be final. - public void mixedInputMethodSecondFinal(Object value, final int anotherValue) {} - - // BUG: Diagnostic contains: Method input parameters must be final. - public void varArgsInputMethod(String... value) {} - - public abstract class abstractClassDefinition { - // BUG: Diagnostic contains: Method input parameters must be final. - public void concreteMethodsAreIncluded(int value) {} - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PreferCommonAnnotationsNegativeCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PreferCommonAnnotationsNegativeCases.java deleted file mode 100644 index ec7cc634fa5..00000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PreferCommonAnnotationsNegativeCases.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Hyperledger Besu contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.hyperledger.errorpronechecks; - -import javax.annotation.Nonnull; - -public class PreferCommonAnnotationsNegativeCases { - - @Nonnull - public String getFoo() { - return "Foo"; - } - - // Fully Qualified Name is the "escape hatch" - @org.jetbrains.annotations.NotNull - public String getBar() { - return "Bar"; - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PreferCommonAnnotationsPositiveCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PreferCommonAnnotationsPositiveCases.java deleted file mode 100644 index 55760df18c6..00000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PreferCommonAnnotationsPositiveCases.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Hyperledger Besu contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.hyperledger.errorpronechecks; - -// BUG: Diagnostic contains: Do not use org.jetbrains.annotations.NotNull use -import org.jetbrains.annotations.NotNull; - -public class PreferCommonAnnotationsPositiveCases { - - @NotNull - public String getFoo() { - return "Foo"; - } -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersNegativeCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersNegativeCases.java deleted file mode 100644 index 99d8986ac74..00000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersNegativeCases.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.hyperledger.errorpronechecks; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class PrivateStaticFinalLoggersNegativeCases { - - private static final Logger LOG = - LoggerFactory.getLogger(PrivateStaticFinalLoggersNegativeCases.class); -} diff --git a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersPositiveCases.java b/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersPositiveCases.java deleted file mode 100644 index cbe6c772d7f..00000000000 --- a/errorprone-checks/src/test/resources/org/hyperledger/errorpronechecks/PrivateStaticFinalLoggersPositiveCases.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.hyperledger.errorpronechecks; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class PrivateStaticFinalLoggersPositiveCases { - - // BUG: Diagnostic contains: Logger classes should be private, static, and final. - private final Logger LOG = LoggerFactory.getLogger(PrivateStaticFinalLoggersPositiveCases.class); -} diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/GraphQLHttpService.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/GraphQLHttpService.java index 03cf1cb7a31..a9a12d61799 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/GraphQLHttpService.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/GraphQLHttpService.java @@ -235,8 +235,7 @@ private Optional getAndValidateHostHeader(final RoutingContext event) { private boolean hostIsInAllowlist(final String hostHeader) { if (config.getHostsAllowlist().stream() - .anyMatch( - allowlistEntry -> allowlistEntry.toLowerCase().equals(hostHeader.toLowerCase()))) { + .anyMatch(allowlistEntry -> allowlistEntry.equalsIgnoreCase(hostHeader))) { return true; } else { LOG.trace("Host not in allowlist: '{}'", hostHeader); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/Scalars.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/Scalars.java index 5b69b0708e7..5762215bc57 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/Scalars.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/Scalars.java @@ -330,7 +330,7 @@ public Number parseValue( if (input instanceof Number number) { return number; } else if (input instanceof String string) { - final String value = string.toLowerCase(); + final String value = string.toLowerCase(Locale.ROOT); if (value.startsWith("0x")) { return Bytes.fromHexStringLenient(value).toLong(); } else { @@ -352,7 +352,7 @@ public Number parseLiteral( if (input instanceof IntValue intValue) { return intValue.getValue().longValue(); } else if (input instanceof StringValue stringValue) { - final String value = stringValue.getValue().toLowerCase(); + final String value = stringValue.getValue().toLowerCase(Locale.ROOT); if (value.startsWith("0x")) { return Bytes.fromHexStringLenient(value).toLong(); } else { diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpService.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpService.java index c2a247974b0..e86bbfdb8ae 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpService.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpService.java @@ -524,8 +524,7 @@ private Optional getAndValidateHostHeader(final RoutingContext event) { private boolean hostIsInAllowlist(final String hostHeader) { if (config.getHostsAllowlist().stream() - .anyMatch( - allowlistEntry -> allowlistEntry.toLowerCase().equals(hostHeader.toLowerCase()))) { + .anyMatch(allowlistEntry -> allowlistEntry.equalsIgnoreCase(hostHeader))) { return true; } else { LOG.trace("Host not in allowlist: '{}'", hostHeader); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/BlockParameter.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/BlockParameter.java index ceed60cbc95..c90222ff3d7 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/BlockParameter.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/BlockParameter.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; +import java.util.Locale; import java.util.Objects; import java.util.Optional; @@ -38,7 +39,7 @@ public class BlockParameter { @JsonCreator public BlockParameter(final String value) { - final String normalizedValue = value.toLowerCase(); + final String normalizedValue = value.toLowerCase(Locale.ROOT); switch (normalizedValue) { case "earliest": diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/BlockParameterOrBlockHash.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/BlockParameterOrBlockHash.java index 172a34d56f6..6cc537e6922 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/BlockParameterOrBlockHash.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/BlockParameterOrBlockHash.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; +import java.util.Locale; import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; @@ -44,7 +45,7 @@ public class BlockParameterOrBlockHash { @JsonCreator public BlockParameterOrBlockHash(final Object value) throws JsonProcessingException { if (value instanceof String) { - final String normalizedValue = String.valueOf(value).toLowerCase(); + final String normalizedValue = String.valueOf(value).toLowerCase(Locale.ROOT); if (Objects.equals(normalizedValue, "earliest")) { type = BlockParameterType.EARLIEST; diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/PendingTransactionsParams.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/PendingTransactionsParams.java index 546fcec97ca..1bfe0e09aea 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/PendingTransactionsParams.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/PendingTransactionsParams.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -82,7 +83,7 @@ private Optional getFilter(final String key, final Map m } else if (!map.isEmpty()) { final Map.Entry foundEntry = map.entrySet().stream().findFirst().get(); final Predicate predicate = - Predicate.fromValue(foundEntry.getKey().toUpperCase()) + Predicate.fromValue(foundEntry.getKey().toUpperCase(Locale.ROOT)) .orElseThrow( () -> new InvalidJsonRpcParameters( diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/WebSocketService.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/WebSocketService.java index 80bb7d4fb1c..c040464dddc 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/WebSocketService.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/WebSocketService.java @@ -331,9 +331,7 @@ private boolean checkHostInAllowlist(final Optional hostHeader) { .map( header -> configuration.getHostsAllowlist().stream() - .anyMatch( - allowlistEntry -> - allowlistEntry.toLowerCase().equals(header.toLowerCase()))) + .anyMatch(allowListEntry -> allowListEntry.equalsIgnoreCase(header))) .orElse(false); } } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTest.java index 901f20ff754..7c3157d2231 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTest.java @@ -43,6 +43,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; @@ -1742,7 +1743,7 @@ private void assertBlockResultMatchesBlock(final JsonObject result, final Block assertThat(Long.decode(result.getString("timestamp"))).isEqualTo(header.getTimestamp()); assertThat(Long.decode(result.getString("number"))).isEqualTo(header.getNumber()); // Nonce is a data field and should represent 8 bytes exactly - final String nonceResult = result.getString("nonce").toLowerCase(); + final String nonceResult = result.getString("nonce").toLowerCase(Locale.ROOT); assertThat(nonceResult.length() == 18 && nonceResult.startsWith("0x")).isTrue(); assertThat(Long.parseUnsignedLong(nonceResult.substring(2), 16)).isEqualTo(header.getNonce()); assertThat(Hash.fromHexString(result.getString("hash"))).isEqualTo(header.getHash()); diff --git a/ethereum/core/build.gradle b/ethereum/core/build.gradle index dd1c0a8bd1b..fa1b462344b 100644 --- a/ethereum/core/build.gradle +++ b/ethereum/core/build.gradle @@ -32,6 +32,7 @@ dependencies { api 'org.web3j:core' annotationProcessor 'org.openjdk.jmh:jmh-generator-annprocess' + annotationProcessor 'org.hyperledger.besu:besu-errorprone-checks' implementation project(':config') implementation project(':crypto:algorithms') diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ProtocolScheduleBuilder.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ProtocolScheduleBuilder.java index 58db065155b..12d0c95c91b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ProtocolScheduleBuilder.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ProtocolScheduleBuilder.java @@ -22,6 +22,7 @@ import org.hyperledger.besu.evm.internal.EvmConfiguration; import java.math.BigInteger; +import java.util.NavigableMap; import java.util.Optional; import java.util.OptionalLong; import java.util.TreeMap; @@ -126,7 +127,7 @@ private void initSchedule( validateForkOrdering(); - final TreeMap builders = buildMilestoneMap(specFactory); + final NavigableMap builders = buildMilestoneMap(specFactory); // At this stage, all milestones are flagged with correct modifier, but ProtocolSpecs must be // inserted _AT_ the modifier block entry. @@ -290,7 +291,7 @@ private long validateForkOrder( return referenceForkBlock; } - private TreeMap buildMilestoneMap( + private NavigableMap buildMilestoneMap( final MainnetProtocolSpecFactory specFactory) { return createMilestones(specFactory) .flatMap(Optional::stream) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionReceipt.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionReceipt.java index 34ae107af92..443c5077c1f 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionReceipt.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionReceipt.java @@ -168,16 +168,14 @@ public boolean equals(final Object obj) { if (obj == this) { return true; } - if (!(obj instanceof PrivateTransactionReceipt)) { + if (!(obj instanceof PrivateTransactionReceipt other)) { return false; } - final PrivateTransactionReceipt other = (PrivateTransactionReceipt) obj; - return logs.equals(other.getLogs()) - && status == other.status - && output.equals(other.output) - && revertReason.isPresent() - ? revertReason.get().equals(other.revertReason.get()) - : true; + return !logs.equals(other.getLogs()) + || status != other.status + || !output.equals(other.output) + || revertReason.isEmpty() + || revertReason.get().equals(other.revertReason.get()); } @Override diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/EthHashTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/EthHashTest.java index 458bc741320..10a3b15b458 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/EthHashTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/EthHashTest.java @@ -82,39 +82,21 @@ public void prepareCache() { } assertThat(Hex.toHexString(buffer.array())) .isEqualTo( - new StringBuilder() - .append( - "7ce2991c951f7bf4c4c1bb119887ee07871eb5339d7b97b8588e85c742de90e5bafd5bbe6ce93a134fb6be9ad3e30db99d9528a2ea7846833f52e9ca119b6b54") - .append( - "8979480c46e19972bd0738779c932c1b43e665a2fd3122fc3ddb2691f353ceb0ed3e38b8f51fd55b6940290743563c9f8fa8822e611924657501a12aafab8a8d") - .append( - "88fb5fbae3a99d14792406672e783a06940a42799b1c38bc28715db6d37cb11f9f6b24e386dc52dd8c286bd8c36fa813dffe4448a9f56ebcbeea866b42f68d22") - .append( - "6c32aae4d695a23cab28fd74af53b0c2efcc180ceaaccc0b2e280103d097a03c1d1b0f0f26ce5f32a90238f9bc49f645db001ef9cd3d13d44743f841fad11a37") - .append( - "fa290c62c16042f703578921f30b9951465aae2af4a5dad43a7341d7b4a62750954965a47a1c3af638dc3495c4d62a9bab843168c9fc0114e79cffd1b2827b01") - .append( - "75d30ba054658f214e946cf24c43b40d3383fbb0493408e5c5392434ca21bbcf43200dfb876c713d201813934fa485f48767c5915745cf0986b1dc0f33e57748") - .append( - "bf483ee2aff4248dfe461ec0504a13628401020fc22638584a8f2f5206a13b2f233898c78359b21c8226024d0a7a93df5eb6c282bdbf005a4aab497e096f2847") - .append( - "76c71cee57932a8fb89f6d6b8743b60a4ea374899a94a2e0f218d5c55818cefb1790c8529a76dba31ebb0f4592d709b49587d2317970d39c086f18dd244291d9") - .append( - "eedb16705e53e3350591bd4ff4566a3595ac0f0ce24b5e112a3d033bc51b6fea0a92296dea7f5e20bf6ee6bc347d868fda193c395b9bb147e55e5a9f67cfe741") - .append( - "7eea7d699b155bd13804204df7ea91fa9249e4474dddf35188f77019c67d201e4c10d7079c5ad492a71afff9a23ca7e900ba7d1bdeaf3270514d8eb35eab8a0a") - .append( - "718bb7273aeb37768fa589ed8ab01fbf4027f4ebdbbae128d21e485f061c20183a9bc2e31edbda0727442e9d58eb0fe198440fe199e02e77c0f7b99973f1f74c") - .append( - "c9089a51ab96c94a84d66e6aa48b2d0a4543adb5a789039a2aa7b335ca85c91026c7d3c894da53ae364188c3fd92f78e01d080399884a47385aa792e38150cda") - .append( - "a8620b2ebeca41fbc773bb837b5e724d6eb2de570d99858df0d7d97067fb8103b21757873b735097b35d3bea8fd1c359a9e8a63c1540c76c9784cf8d975e995c") - .append( - "778401b94a2e66e6993ad67ad3ecdc2acb17779f1ea8606827ec92b11c728f8c3b6d3f04a3e6ed05ff81dd76d5dc5695a50377bc135aaf1671cf68b750315493") - .append( - "6c64510164d53312bf3c41740c7a237b05faf4a191bd8a95dafa068dbcf370255c725900ce5c934f36feadcfe55b687c440574c1f06f39d207a8553d39156a24") - .append( - "845f64fd8324bb85312979dead74f764c9677aab89801ad4f927f1c00f12e28f22422bb44200d1969d9ab377dd6b099dc6dbc3222e9321b2c1e84f8e2f07731c") - .toString()); + "7ce2991c951f7bf4c4c1bb119887ee07871eb5339d7b97b8588e85c742de90e5bafd5bbe6ce93a134fb6be9ad3e30db99d9528a2ea7846833f52e9ca119b6b54" + + "8979480c46e19972bd0738779c932c1b43e665a2fd3122fc3ddb2691f353ceb0ed3e38b8f51fd55b6940290743563c9f8fa8822e611924657501a12aafab8a8d" + + "88fb5fbae3a99d14792406672e783a06940a42799b1c38bc28715db6d37cb11f9f6b24e386dc52dd8c286bd8c36fa813dffe4448a9f56ebcbeea866b42f68d22" + + "6c32aae4d695a23cab28fd74af53b0c2efcc180ceaaccc0b2e280103d097a03c1d1b0f0f26ce5f32a90238f9bc49f645db001ef9cd3d13d44743f841fad11a37" + + "fa290c62c16042f703578921f30b9951465aae2af4a5dad43a7341d7b4a62750954965a47a1c3af638dc3495c4d62a9bab843168c9fc0114e79cffd1b2827b01" + + "75d30ba054658f214e946cf24c43b40d3383fbb0493408e5c5392434ca21bbcf43200dfb876c713d201813934fa485f48767c5915745cf0986b1dc0f33e57748" + + "bf483ee2aff4248dfe461ec0504a13628401020fc22638584a8f2f5206a13b2f233898c78359b21c8226024d0a7a93df5eb6c282bdbf005a4aab497e096f2847" + + "76c71cee57932a8fb89f6d6b8743b60a4ea374899a94a2e0f218d5c55818cefb1790c8529a76dba31ebb0f4592d709b49587d2317970d39c086f18dd244291d9" + + "eedb16705e53e3350591bd4ff4566a3595ac0f0ce24b5e112a3d033bc51b6fea0a92296dea7f5e20bf6ee6bc347d868fda193c395b9bb147e55e5a9f67cfe741" + + "7eea7d699b155bd13804204df7ea91fa9249e4474dddf35188f77019c67d201e4c10d7079c5ad492a71afff9a23ca7e900ba7d1bdeaf3270514d8eb35eab8a0a" + + "718bb7273aeb37768fa589ed8ab01fbf4027f4ebdbbae128d21e485f061c20183a9bc2e31edbda0727442e9d58eb0fe198440fe199e02e77c0f7b99973f1f74c" + + "c9089a51ab96c94a84d66e6aa48b2d0a4543adb5a789039a2aa7b335ca85c91026c7d3c894da53ae364188c3fd92f78e01d080399884a47385aa792e38150cda" + + "a8620b2ebeca41fbc773bb837b5e724d6eb2de570d99858df0d7d97067fb8103b21757873b735097b35d3bea8fd1c359a9e8a63c1540c76c9784cf8d975e995c" + + "778401b94a2e66e6993ad67ad3ecdc2acb17779f1ea8606827ec92b11c728f8c3b6d3f04a3e6ed05ff81dd76d5dc5695a50377bc135aaf1671cf68b750315493" + + "6c64510164d53312bf3c41740c7a237b05faf4a191bd8a95dafa068dbcf370255c725900ce5c934f36feadcfe55b687c440574c1f06f39d207a8553d39156a24" + + "845f64fd8324bb85312979dead74f764c9677aab89801ad4f927f1c00f12e28f22422bb44200d1969d9ab377dd6b099dc6dbc3222e9321b2c1e84f8e2f07731c"); } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/AccountRangeMessage.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/AccountRangeMessage.java index 417bbd27fcd..c9b93734a5b 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/AccountRangeMessage.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/AccountRangeMessage.java @@ -24,6 +24,7 @@ import java.math.BigInteger; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.Optional; import java.util.TreeMap; @@ -133,7 +134,7 @@ private Bytes toFullAccount(final RLPInput rlpInput) { @Value.Immutable public interface AccountRangeData { - TreeMap accounts(); + NavigableMap accounts(); ArrayDeque proofs(); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessage.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessage.java index ab4a353fea2..d89718f0fd5 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessage.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessage.java @@ -22,6 +22,7 @@ import java.math.BigInteger; import java.util.List; +import java.util.NavigableMap; import java.util.Optional; import java.util.TreeMap; @@ -49,13 +50,13 @@ public static StorageRangeMessage readFrom(final MessageData message) { } public static StorageRangeMessage create( - final ArrayDeque> slots, final List proof) { + final ArrayDeque> slots, final List proof) { return create(Optional.empty(), slots, proof); } public static StorageRangeMessage create( final Optional requestId, - final ArrayDeque> slots, + final ArrayDeque> slots, final List proof) { final BytesValueRLPOutput tmp = new BytesValueRLPOutput(); tmp.startList(); @@ -88,7 +89,7 @@ public int getCode() { } public SlotRangeData slotsData(final boolean withRequestId) { - final ArrayDeque> slots = new ArrayDeque<>(); + final ArrayDeque> slots = new ArrayDeque<>(); final ArrayDeque proofs = new ArrayDeque<>(); final RLPInput input = new BytesValueRLPInput(data, false); input.enterList(); @@ -120,7 +121,7 @@ public SlotRangeData slotsData(final boolean withRequestId) { @Value.Immutable public interface SlotRangeData { - ArrayDeque> slots(); + ArrayDeque> slots(); ArrayDeque proofs(); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SyncMode.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SyncMode.java index c2c1fc10788..b5d0c2570bb 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SyncMode.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/SyncMode.java @@ -15,6 +15,7 @@ package org.hyperledger.besu.ethereum.eth.sync; import java.util.EnumSet; +import java.util.Locale; import org.apache.commons.lang3.StringUtils; @@ -34,10 +35,10 @@ public enum SyncMode { public String normalize() { if (this.toString().startsWith("X_")) { // removes X_ at the beginning - return StringUtils.capitalize(this.toString().substring(2).toLowerCase()); + return StringUtils.capitalize(this.toString().substring(2).toLowerCase(Locale.ROOT)); } - return StringUtils.capitalize(this.toString().toLowerCase()); + return StringUtils.capitalize(this.toString().toLowerCase(Locale.ROOT)); } public static boolean isFullSync(final SyncMode syncMode) { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java index 000badc25d7..707634664bb 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java @@ -47,6 +47,7 @@ import java.util.List; import java.util.Map; import java.util.OptionalLong; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.function.Predicate; @@ -77,7 +78,7 @@ public class SnapWorldDownloadState extends WorldDownloadState protected final InMemoryTasksPriorityQueues pendingStorageFlatDatabaseHealingRequests = new InMemoryTasksPriorityQueues<>(); - private HashSet accountsHealingList = new HashSet<>(); + private Set accountsHealingList = new HashSet<>(); private DynamicPivotBlockSelector pivotBlockSelector; private final SnapSyncStatePersistenceManager snapContext; @@ -304,7 +305,7 @@ public synchronized void enqueueRequest(final SnapDataRequest request) { } } - public synchronized void setAccountsHealingList(final HashSet addAccountToHealingList) { + public synchronized void setAccountsHealingList(final Set addAccountToHealingList) { this.accountsHealingList = addAccountToHealingList; } @@ -322,7 +323,7 @@ public synchronized void addAccountToHealingList(final Bytes account) { } } - public HashSet getAccountsHealingList() { + public Set getAccountsHealingList() { return accountsHealingList; } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java index 61219882741..76a07a4d8ea 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java @@ -36,10 +36,10 @@ import org.hyperledger.besu.services.tasks.InMemoryTasksPriorityQueues; import java.time.Clock; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; @@ -154,7 +154,7 @@ public CompletableFuture run( final List currentAccountRange = snapContext.getCurrentAccountRange(); - final HashSet inconsistentAccounts = snapContext.getAccountsHealingList(); + final Set inconsistentAccounts = snapContext.getAccountsHealingList(); if (!currentAccountRange.isEmpty()) { // continue to download worldstate ranges newDownloadState.setAccountsHealingList(inconsistentAccounts); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrie.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrie.java index f9e8ea2eb7d..1cf593665a0 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrie.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrie.java @@ -28,6 +28,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.Optional; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicInteger; @@ -72,7 +73,9 @@ public StackTrie( } public void addElement( - final Bytes32 taskIdentifier, final List proofs, final TreeMap keys) { + final Bytes32 taskIdentifier, + final List proofs, + final NavigableMap keys) { this.elementsCount.addAndGet(keys.size()); this.elements.put( taskIdentifier, ImmutableTaskElement.builder().proofs(proofs).keys(keys).build()); @@ -183,7 +186,7 @@ public List proofs() { } @Value.Default - public TreeMap keys() { + public NavigableMap keys() { return new TreeMap<>(); } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/context/SnapSyncStatePersistenceManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/context/SnapSyncStatePersistenceManager.java index 75ee91352ad..6a54dbcdfe0 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/context/SnapSyncStatePersistenceManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/context/SnapSyncStatePersistenceManager.java @@ -28,6 +28,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -127,7 +128,7 @@ public List getCurrentAccountRange() { .collect(Collectors.toList()); } - public HashSet getAccountsHealingList() { + public Set getAccountsHealingList() { return healContext .streamValuesFromKeysThat(notEqualsTo(SNAP_ACCOUNT_HEALING_LIST_INDEX)) .collect(Collectors.toCollection(HashSet::new)); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java index 76897e18505..94ba92b5356 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java @@ -40,8 +40,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.Optional; -import java.util.TreeMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; @@ -155,7 +155,7 @@ protected int doPersist( public void addResponse( final WorldStateProofProvider worldStateProofProvider, - final TreeMap accounts, + final NavigableMap accounts, final ArrayDeque proofs) { if (!accounts.isEmpty() || !proofs.isEmpty()) { if (!worldStateProofProvider.isValidRangeProof( @@ -226,7 +226,7 @@ public Bytes32 getEndKeyHash() { } @VisibleForTesting - public TreeMap getAccounts() { + public NavigableMap getAccounts() { return stackTrie.getElement(startKeyHash).keys(); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/SnapDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/SnapDataRequest.java index f3fc1f933d7..d9442c94c9b 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/SnapDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/SnapDataRequest.java @@ -31,8 +31,8 @@ import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.services.tasks.TasksPriorityProvider; -import java.util.HashSet; import java.util.Optional; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; @@ -93,7 +93,7 @@ public static StorageRangeDataRequest createStorageRangeDataRequest( } public static AccountTrieNodeHealingRequest createAccountTrieNodeDataRequest( - final Hash hash, final Bytes location, final HashSet inconsistentAccounts) { + final Hash hash, final Bytes location, final Set inconsistentAccounts) { return new AccountTrieNodeHealingRequest(hash, hash, location, inconsistentAccounts); } @@ -101,7 +101,7 @@ public static AccountTrieNodeHealingRequest createAccountTrieNodeDataRequest( final Hash hash, final Hash rootHash, final Bytes location, - final HashSet inconsistentAccounts) { + final Set inconsistentAccounts) { return new AccountTrieNodeHealingRequest(hash, rootHash, location, inconsistentAccounts); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java index 646161a42e2..48c25fadcd5 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java @@ -38,8 +38,8 @@ import java.util.ArrayList; import java.util.List; +import java.util.NavigableMap; import java.util.Optional; -import java.util.TreeMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; @@ -129,7 +129,7 @@ protected int doPersist( public void addResponse( final SnapWorldDownloadState downloadState, final WorldStateProofProvider worldStateProofProvider, - final TreeMap slots, + final NavigableMap slots, final ArrayDeque proofs) { if (!slots.isEmpty() || !proofs.isEmpty()) { if (!worldStateProofProvider.isValidRangeProof( @@ -202,7 +202,7 @@ public Bytes32 getStorageRoot() { return storageRoot; } - public TreeMap getSlots() { + public NavigableMap getSlots() { return stackTrie.getElement(startKeyHash).keys(); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java index 76d6f903f12..5c114d647a1 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java @@ -41,6 +41,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.NavigableMap; import java.util.TreeMap; import java.util.function.Function; import java.util.stream.Stream; @@ -58,9 +59,9 @@ public class AccountFlatDatabaseHealingRangeRequest extends SnapDataRequest { private final Bytes32 startKeyHash; private final Bytes32 endKeyHash; - private TreeMap existingAccounts; + private NavigableMap existingAccounts; - private TreeMap flatDbAccounts; + private NavigableMap flatDbAccounts; private boolean isProofValid; public AccountFlatDatabaseHealingRangeRequest( @@ -132,7 +133,7 @@ public boolean isResponseReceived() { public void addLocalData( final WorldStateProofProvider worldStateProofProvider, - final TreeMap accounts, + final NavigableMap accounts, final ArrayDeque proofs) { if (!accounts.isEmpty() && !proofs.isEmpty()) { // very proof in order to check if the local flat database is valid or not diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountTrieNodeHealingRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountTrieNodeHealingRequest.java index e2f3353f20e..30350f1dbc8 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountTrieNodeHealingRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountTrieNodeHealingRequest.java @@ -34,6 +34,7 @@ import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.function.Function; import java.util.stream.Stream; @@ -43,13 +44,13 @@ /** Represents a healing request for an account trie node. */ public class AccountTrieNodeHealingRequest extends TrieNodeHealingRequest { - private final HashSet inconsistentAccounts; + private final Set inconsistentAccounts; public AccountTrieNodeHealingRequest( final Hash hash, final Hash originalRootHash, final Bytes location, - final HashSet inconsistentAccounts) { + final Set inconsistentAccounts) { super(hash, originalRootHash, location); this.inconsistentAccounts = inconsistentAccounts; } @@ -89,7 +90,7 @@ protected SnapDataRequest createChildNodeDataRequest(final Hash childHash, final childHash, getRootHash(), location, getSubLocation(location)); } - private HashSet getSubLocation(final Bytes location) { + private Set getSubLocation(final Bytes location) { final HashSet foundAccountsToHeal = new HashSet<>(); for (Bytes account : inconsistentAccounts) { if (account.commonPrefixLength(location) == location.size()) { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java index 26610572533..82a9e3f5510 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java @@ -36,6 +36,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.TreeMap; import java.util.function.Function; import java.util.stream.Stream; @@ -56,7 +57,7 @@ public class StorageFlatDatabaseHealingRangeRequest extends SnapDataRequest { private final Bytes32 storageRoot; private final Bytes32 startKeyHash; private final Bytes32 endKeyHash; - private TreeMap slots; + private NavigableMap slots; private boolean isProofValid; public StorageFlatDatabaseHealingRangeRequest( @@ -120,7 +121,7 @@ public boolean isResponseReceived() { public void addLocalData( final WorldStateProofProvider worldStateProofProvider, - final TreeMap slots, + final NavigableMap slots, final ArrayDeque proofs) { if (!slots.isEmpty() && !proofs.isEmpty()) { // very proof in order to check if the local flat database is valid or not diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactions.java index b796eefaca4..8929e221062 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactions.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractPrioritizedTransactions.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.Map; import java.util.NavigableMap; -import java.util.TreeMap; import java.util.TreeSet; import java.util.function.BiFunction; import java.util.function.Predicate; @@ -140,7 +139,7 @@ protected long cacheFreeSpace() { @Override protected void internalConsistencyCheck( - final Map> prevLayerTxsBySender) { + final Map> prevLayerTxsBySender) { super.internalConsistencyCheck(prevLayerTxsBySender); final var controlOrderByFee = new TreeSet<>(this::compareByFee); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractSequentialTransactionsLayer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractSequentialTransactionsLayer.java index 9596f6fefb7..245f8916413 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractSequentialTransactionsLayer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractSequentialTransactionsLayer.java @@ -27,7 +27,6 @@ import java.util.NavigableMap; import java.util.OptionalLong; import java.util.Set; -import java.util.TreeMap; import java.util.function.BiFunction; public abstract class AbstractSequentialTransactionsLayer extends AbstractTransactionsLayer { @@ -142,7 +141,7 @@ protected boolean hasExpectedNonce( @Override protected void internalConsistencyCheck( - final Map> prevLayerTxsBySender) { + final Map> prevLayerTxsBySender) { txsBySender.values().stream() .filter(senderTxs -> senderTxs.size() > 1) .map(NavigableMap::entrySet) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractTransactionsLayer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractTransactionsLayer.java index a323b6833e9..e32eb0e2291 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractTransactionsLayer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/AbstractTransactionsLayer.java @@ -569,7 +569,7 @@ public String logSender(final Address sender) { protected abstract String internalLogStats(); boolean consistencyCheck( - final Map> prevLayerTxsBySender) { + final Map> prevLayerTxsBySender) { final BinaryOperator noMergeExpected = (a, b) -> { throw new IllegalArgumentException(); @@ -606,7 +606,7 @@ boolean consistencyCheck( } protected abstract void internalConsistencyCheck( - final Map> prevLayerTxsBySender); + final Map> prevLayerTxsBySender); public BlobCache getBlobCache() { return blobCache; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReadyTransactions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReadyTransactions.java index 2820fe64b67..a3dc195d1bf 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReadyTransactions.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/ReadyTransactions.java @@ -33,7 +33,6 @@ import java.util.NavigableMap; import java.util.NavigableSet; import java.util.Optional; -import java.util.TreeMap; import java.util.TreeSet; import java.util.function.BiFunction; import java.util.function.Predicate; @@ -220,7 +219,7 @@ public String internalLogStats() { @Override protected void internalConsistencyCheck( - final Map> prevLayerTxsBySender) { + final Map> prevLayerTxsBySender) { super.internalConsistencyCheck(prevLayerTxsBySender); final var minNonceBySender = @@ -241,7 +240,6 @@ protected void internalConsistencyCheck( : "orderByMaxFee does not match pendingTransactions"; } - assert itCurrent.hasNext() == false - : "orderByMaxFee has more elements than pendingTransactions"; + assert !itCurrent.hasNext() : "orderByMaxFee has more elements than pendingTransactions"; } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/SparseTransactions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/SparseTransactions.java index 41e2f6d9c33..e036f2fd215 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/SparseTransactions.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/SparseTransactions.java @@ -39,7 +39,6 @@ import java.util.Objects; import java.util.OptionalLong; import java.util.Set; -import java.util.TreeMap; import java.util.TreeSet; import java.util.function.BiFunction; import java.util.function.Predicate; @@ -404,7 +403,7 @@ private void updateGap(final Address sender, final int currGap, final int newGap @Override protected void internalConsistencyCheck( - final Map> prevLayerTxsBySender) { + final Map> prevLayerTxsBySender) { txsBySender.values().stream() .filter(senderTxs -> senderTxs.size() > 1) .map(NavigableMap::entrySet) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/TransactionsLayer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/TransactionsLayer.java index 85227766b40..688eb6721e8 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/TransactionsLayer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/TransactionsLayer.java @@ -25,6 +25,7 @@ import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.OptionalLong; @@ -106,7 +107,7 @@ enum RemovalReason { private final String label; RemovalReason() { - this.label = name().toLowerCase(); + this.label = name().toLowerCase(Locale.ROOT); } public String label() { diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessageTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessageTest.java index c9179329b02..da14098d05b 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessageTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/StorageRangeMessageTest.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.NavigableMap; import java.util.TreeMap; import kotlin.collections.ArrayDeque; @@ -33,7 +34,7 @@ public final class StorageRangeMessageTest { @Test public void roundTripTest() { - final ArrayDeque> keys = new ArrayDeque<>(); + final ArrayDeque> keys = new ArrayDeque<>(); final TreeMap storage = new TreeMap<>(); storage.put(Hash.wrap(Bytes32.leftPad(Bytes.of(1))), Bytes32.random()); keys.add(storage); diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/benchmarks/BenchmarkExecutor.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/benchmarks/BenchmarkExecutor.java index 730dd833f95..ccc9152f757 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/benchmarks/BenchmarkExecutor.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/benchmarks/BenchmarkExecutor.java @@ -36,6 +36,7 @@ import org.hyperledger.besu.evm.precompile.PrecompiledContract; import java.io.PrintStream; +import java.util.Locale; import java.util.concurrent.TimeUnit; import com.google.common.base.Stopwatch; @@ -127,7 +128,7 @@ protected double runPrecompileBenchmark(final Bytes arg, final PrecompiledContra * @return a gas calculator */ public static GasCalculator gasCalculatorForFork(final String fork) { - return switch (EvmSpecVersion.valueOf(fork.toUpperCase())) { + return switch (EvmSpecVersion.valueOf(fork.toUpperCase(Locale.ROOT))) { case HOMESTEAD -> new HomesteadGasCalculator(); case FRONTIER -> new FrontierGasCalculator(); case BYZANTIUM -> new ByzantiumGasCalculator(); diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryStatus.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryStatus.java index dc704915658..5311a2212a5 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryStatus.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryStatus.java @@ -14,6 +14,8 @@ */ package org.hyperledger.besu.ethereum.p2p.discovery; +import java.util.Locale; + /** The status of a {@link DiscoveryPeer}, in relation to the peer discovery state machine. */ public enum PeerDiscoveryStatus { @@ -40,6 +42,6 @@ public enum PeerDiscoveryStatus { @Override public String toString() { - return name().toLowerCase(); + return name().toLowerCase(Locale.ROOT); } } diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/peers/EnodeURLImpl.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/peers/EnodeURLImpl.java index 55ca94a1ac3..e1d158f3fce 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/peers/EnodeURLImpl.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/peers/EnodeURLImpl.java @@ -23,6 +23,7 @@ import java.net.InetAddress; import java.net.URI; import java.net.UnknownHostException; +import java.util.Locale; import java.util.Objects; import java.util.Optional; import java.util.OptionalInt; @@ -157,7 +158,7 @@ public static boolean sameListeningEndpoint(final EnodeURL enodeA, final EnodeUR public static Bytes parseNodeId(final String nodeId) { int expectedSize = EnodeURLImpl.NODE_ID_SIZE * 2; - if (nodeId.toLowerCase().startsWith("0x")) { + if (nodeId.toLowerCase(Locale.ROOT).startsWith("0x")) { expectedSize += 2; } checkArgument( diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/wire/Capability.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/wire/Capability.java index 5a3a57df46d..f530337ee46 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/wire/Capability.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/wire/Capability.java @@ -84,7 +84,7 @@ public int hashCode() { } @Override - /** Returned string is sanitized since it contains user input */ + // Returned string is sanitized since it contains user input public String toString() { return Encode.forJava(name) + "/" + version; } diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/wire/PeerInfo.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/wire/PeerInfo.java index bd98270e020..492280e85d0 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/wire/PeerInfo.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/rlpx/wire/PeerInfo.java @@ -118,7 +118,7 @@ public void writeTo(final RLPOutput out) { } @Override - /** Returned string is sanitized since it contains user input */ + // Returned string is sanitized since it contains user input public String toString() { final StringBuilder sb = new StringBuilder("PeerInfo{"); sb.append("version=").append(version); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTimestampsTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTimestampsTest.java index 90ee8b422cd..d75847050d4 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTimestampsTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTimestampsTest.java @@ -21,7 +21,6 @@ import org.hyperledger.besu.ethereum.p2p.discovery.internal.Packet; import java.util.Collections; -import java.util.concurrent.atomic.AtomicLong; import org.junit.jupiter.api.Test; @@ -41,8 +40,8 @@ public void lastSeenAndFirstDiscoveredTimestampsUpdatedOnMessage() { final Packet pong = helper.createPongPacket(agent, Hash.hash(agentPing.getHash())); helper.sendMessageBetweenAgents(testAgent, agent, pong); - final AtomicLong lastSeen = new AtomicLong(); - final AtomicLong firstDiscovered = new AtomicLong(); + long lastSeen; + long firstDiscovered; assertThat(agent.streamDiscoveredPeers()).hasSize(1); @@ -50,16 +49,16 @@ public void lastSeenAndFirstDiscoveredTimestampsUpdatedOnMessage() { assertThat(p.getLastSeen()).isGreaterThan(0); assertThat(p.getFirstDiscovered()).isGreaterThan(0); - lastSeen.set(p.getLastSeen()); - firstDiscovered.set(p.getFirstDiscovered()); + lastSeen = p.getLastSeen(); + firstDiscovered = p.getFirstDiscovered(); helper.sendMessageBetweenAgents(testAgent, agent, testAgentPing); assertThat(agent.streamDiscoveredPeers()).hasSize(1); p = agent.streamDiscoveredPeers().iterator().next(); - assertThat(p.getLastSeen()).isGreaterThan(lastSeen.get()); - assertThat(p.getFirstDiscovered()).isEqualTo(firstDiscovered.get()); + assertThat(p.getLastSeen()).isGreaterThan(lastSeen); + assertThat(p.getFirstDiscovered()).isEqualTo(firstDiscovered); } @Test @@ -74,9 +73,9 @@ public void lastContactedTimestampUpdatedOnOutboundMessage() { assertThat(agent.streamDiscoveredPeers()).hasSize(1); - final AtomicLong lastContacted = new AtomicLong(); - final AtomicLong lastSeen = new AtomicLong(); - final AtomicLong firstDiscovered = new AtomicLong(); + final long lastContacted; + final long lastSeen; + final long firstDiscovered; DiscoveryPeer peer = agent.streamDiscoveredPeers().iterator().next(); final long lc = peer.getLastContacted(); @@ -87,9 +86,9 @@ public void lastContactedTimestampUpdatedOnOutboundMessage() { assertThat(ls).isGreaterThan(0); assertThat(fd).isGreaterThan(0); - lastContacted.set(lc); - lastSeen.set(ls); - firstDiscovered.set(fd); + lastContacted = lc; + lastSeen = ls; + firstDiscovered = fd; // Send another packet and ensure that timestamps are updated accordingly. // Sleep beforehand to make sure timestamps will be different. @@ -102,8 +101,8 @@ public void lastContactedTimestampUpdatedOnOutboundMessage() { peer = agent.streamDiscoveredPeers().iterator().next(); - assertThat(peer.getLastContacted()).isGreaterThan(lastContacted.get()); - assertThat(peer.getLastSeen()).isGreaterThan(lastSeen.get()); - assertThat(peer.getFirstDiscovered()).isEqualTo(firstDiscovered.get()); + assertThat(peer.getLastContacted()).isGreaterThan(lastContacted); + assertThat(peer.getLastSeen()).isGreaterThan(lastSeen); + assertThat(peer.getFirstDiscovered()).isEqualTo(firstDiscovered); } } diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/MockConnectionInitializer.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/MockConnectionInitializer.java index 814e9a797f6..64b7335cd45 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/MockConnectionInitializer.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/MockConnectionInitializer.java @@ -19,6 +19,7 @@ import org.hyperledger.besu.ethereum.p2p.rlpx.wire.messages.DisconnectMessage; import org.hyperledger.besu.util.Subscribers; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.util.HashMap; import java.util.Map; @@ -58,7 +59,7 @@ public void simulateIncomingConnection(final PeerConnection incomingConnection) @Override public CompletableFuture start() { final InetSocketAddress socketAddress = - new InetSocketAddress("127.0.0.1", NEXT_PORT.incrementAndGet()); + new InetSocketAddress(InetAddress.getLoopbackAddress(), NEXT_PORT.incrementAndGet()); return CompletableFuture.completedFuture(socketAddress); } diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java index 952ebf7c2d5..72a939f7fa3 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java @@ -55,6 +55,7 @@ import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.data.EnodeURL; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Arrays; @@ -92,7 +93,8 @@ public class DeFramerTest { private final PeerConnection peerConnection = mock(PeerConnection.class); private final CompletableFuture connectFuture = new CompletableFuture<>(); private final int remotePort = 12345; - private final InetSocketAddress remoteAddress = new InetSocketAddress("127.0.0.1", remotePort); + private final InetSocketAddress remoteAddress = + new InetSocketAddress(InetAddress.getLoopbackAddress(), remotePort); private final int p2pVersion = 5; private final String clientId = "abc"; diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestBlockchain.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestBlockchain.java index cfe577bf7af..f73e271396d 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestBlockchain.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestBlockchain.java @@ -188,6 +188,7 @@ public static class NonDeterministicOperationException extends RuntimeException } @Override + @SuppressWarnings("unused") public Comparator getBlockChoiceRule() { return (a, b) -> { throw new NonDeterministicOperationException( diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/RangeManager.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/RangeManager.java index dbc7c10932a..f0699831d94 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/RangeManager.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/RangeManager.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.Optional; import java.util.TreeMap; import java.util.function.Function; @@ -43,7 +44,7 @@ public class RangeManager { private RangeManager() {} public static int getRangeCount( - final Bytes32 min, final Bytes32 max, final TreeMap items) { + final Bytes32 min, final Bytes32 max, final NavigableMap items) { if (min.equals(MIN_RANGE) && max.equals(MAX_RANGE)) { return MAX_RANGE .toUnsignedBigInteger() @@ -120,7 +121,7 @@ public static Map generateRanges( public static Optional findNewBeginElementInRange( final Bytes32 worldstateRootHash, final List proofs, - final TreeMap receivedKeys, + final NavigableMap receivedKeys, final Bytes32 endKeyHash) { if (receivedKeys.isEmpty() || receivedKeys.lastKey().compareTo(endKeyHash) >= 0) { return Optional.empty(); diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/patricia/BranchNode.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/patricia/BranchNode.java index ac5eec524b6..b5d84513619 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/patricia/BranchNode.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/patricia/BranchNode.java @@ -55,7 +55,7 @@ public class BranchNode implements Node { public BranchNode( final Bytes location, - final ArrayList> children, + final List> children, final Optional value, final NodeFactory nodeFactory, final Function valueSerializer) { diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 9d36f782098..f012b074345 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -9,6 +9,14 @@ + + + + + + + + @@ -558,6 +566,11 @@ + + + + + @@ -603,6 +616,14 @@ + + + + + + + + @@ -622,9 +643,6 @@ - - - @@ -637,14 +655,6 @@ - - - - - - - - @@ -666,11 +676,6 @@ - - - - - @@ -739,12 +744,12 @@ - - - + + + - - + + @@ -776,20 +781,28 @@ - - - + + + + + + + + + + + - - + + - - - + + + - - + + @@ -812,20 +825,17 @@ - - - - - - + + + - - - + + + - - + + @@ -844,14 +854,6 @@ - - - - - - - - @@ -876,11 +878,6 @@ - - - - - @@ -993,6 +990,19 @@ + + + + + + + + + + + + + @@ -1001,22 +1011,9 @@ - - - - - - - - - - - - - - - - + + + @@ -1034,12 +1031,12 @@ - - - + + + - - + + @@ -1071,9 +1068,9 @@ - - - + + + @@ -1099,27 +1096,6 @@ - - - - - - - - - - - - - - - - - - - - - @@ -1443,12 +1419,25 @@ - - - + + + + + + + + + + + - - + + + + + + + @@ -2916,17 +2905,17 @@ - - - + + + - - - + + + - - + + @@ -3527,14 +3516,6 @@ - - - - - - - - @@ -3843,14 +3824,6 @@ - - - - - - - - @@ -3859,11 +3832,6 @@ - - - - - @@ -4668,14 +4636,6 @@ - - - - - - - - @@ -4689,6 +4649,14 @@ + + + + + + + + @@ -5474,14 +5442,6 @@ - - - - - - - - @@ -5548,12 +5508,12 @@ - - - + + + - - + + diff --git a/gradle/versions.gradle b/gradle/versions.gradle index dbe70c1d608..629d1598258 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -39,12 +39,7 @@ dependencyManagement { entry'dagger' } - dependencySet(group: 'com.google.errorprone', version: '2.18.0') { - entry 'error_prone_annotation' - entry 'error_prone_check_api' - entry 'error_prone_core' - entry 'error_prone_test_helpers' - } + dependency 'org.hyperledger.besu:besu-errorprone-checks:1.0.0' dependency 'com.google.guava:guava:33.0.0-jre' diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/MetricsHttpService.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/MetricsHttpService.java index 2c2023c190a..c547b3d4cd8 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/MetricsHttpService.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/MetricsHttpService.java @@ -26,6 +26,7 @@ import java.net.InetSocketAddress; import java.net.SocketException; import java.nio.charset.StandardCharsets; +import java.util.Locale; import java.util.Optional; import java.util.Set; import java.util.TreeSet; @@ -175,7 +176,10 @@ private Optional getAndValidateHostHeader(final RoutingContext event) { private boolean hostIsInAllowlist(final String hostHeader) { if (config.getHostsAllowlist().stream() .anyMatch( - allowlistEntry -> allowlistEntry.toLowerCase().equals(hostHeader.toLowerCase()))) { + allowlistEntry -> + allowlistEntry + .toLowerCase(Locale.ROOT) + .equals(hostHeader.toLowerCase(Locale.ROOT)))) { return true; } else { LOG.trace("Host not in allowlist: '{}'", hostHeader); diff --git a/metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java b/metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java index 01e7aca7a36..c8a0cb89ba9 100644 --- a/metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java +++ b/metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Locale; import io.prometheus.client.Collector; import org.rocksdb.HistogramData; @@ -175,7 +176,7 @@ public static void registerRocksDBMetrics( return; } for (final TickerType ticker : TICKERS) { - final String promCounterName = ticker.name().toLowerCase(); + final String promCounterName = ticker.name().toLowerCase(Locale.ROOT); metricsSystem.createLongGauge( category, promCounterName, @@ -192,7 +193,7 @@ private static Collector histogramToCollector( final Statistics stats, final HistogramType histogram) { return new Collector() { final String metricName = - KVSTORE_ROCKSDB_STATS.getName() + "_" + histogram.name().toLowerCase(); + KVSTORE_ROCKSDB_STATS.getName() + "_" + histogram.name().toLowerCase(Locale.ROOT); @Override public List collect() { diff --git a/nat/src/main/java/org/hyperledger/besu/nat/docker/HostBasedIpDetector.java b/nat/src/main/java/org/hyperledger/besu/nat/docker/HostBasedIpDetector.java index 34922624c7b..a7cd1c656aa 100644 --- a/nat/src/main/java/org/hyperledger/besu/nat/docker/HostBasedIpDetector.java +++ b/nat/src/main/java/org/hyperledger/besu/nat/docker/HostBasedIpDetector.java @@ -27,6 +27,7 @@ public class HostBasedIpDetector implements IpDetector { private static final String HOSTNAME = "HOST_IP"; @Override + @SuppressWarnings("AddressSelection") public Optional detectAdvertisedIp() { try { return Optional.of(InetAddress.getByName(HOSTNAME).getHostAddress()); diff --git a/settings.gradle b/settings.gradle index 15a0f8617d6..3ca01a26067 100644 --- a/settings.gradle +++ b/settings.gradle @@ -37,7 +37,6 @@ include 'crypto:services' include 'datatypes' include 'enclave' include 'evm' -include 'errorprone-checks' include 'ethereum:api' include 'ethereum:blockcreation' include 'ethereum:core' diff --git a/util/src/main/java/org/hyperledger/besu/util/Subscribers.java b/util/src/main/java/org/hyperledger/besu/util/Subscribers.java index 6704ed95773..7ca1e712744 100644 --- a/util/src/main/java/org/hyperledger/besu/util/Subscribers.java +++ b/util/src/main/java/org/hyperledger/besu/util/Subscribers.java @@ -165,9 +165,7 @@ public boolean unsubscribe(final long subscriberId) { } @Override - public void forEach(final Consumer action) { - return; - } + public void forEach(final Consumer action) {} @Override public int getSubscriberCount() { diff --git a/util/src/test/java/org/hyperledger/besu/util/log/LogUtilTest.java b/util/src/test/java/org/hyperledger/besu/util/log/LogUtilTest.java index 10fef966280..25ed58c33c9 100644 --- a/util/src/test/java/org/hyperledger/besu/util/log/LogUtilTest.java +++ b/util/src/test/java/org/hyperledger/besu/util/log/LogUtilTest.java @@ -114,6 +114,7 @@ private List assertFoundInTrace( return lines; } + @SuppressWarnings("InfiniteRecursion") private void recurseTimesAndThrow(final int times) { if (times < 1) { throw new RuntimeException("FakeStackOverflowError"); From 7df1732b89c61b5f35ea353ea6dbb39c278a7cd4 Mon Sep 17 00:00:00 2001 From: Danno Ferrin Date: Tue, 26 Mar 2024 15:23:01 -0700 Subject: [PATCH 04/24] Expose `v` field in JSON-RPC in some transaction types (#6819) * Expose `v` field in JSON-RPC in some transaction types The execution API marks the `v` field as optional for EIP-2930 and EIP-1559 transactions, preferring the `yParity` feld. However, some tooling still depends on the presence of the `v` field. For those two transaction types both `v` and `yParity` will be returned now. Signed-off-by: Danno Ferrin --- CHANGELOG.md | 1 + .../internal/results/TransactionCompleteResult.java | 2 +- .../internal/results/TransactionPendingResult.java | 2 +- .../pending/PendingTransactionDetailResult.java | 2 +- .../ethereum/api/graphql/eth_getTransaction_type2.json | 6 ++++-- .../eth/eth_getBlockByNumber_complete_shanghai.json | 2 +- .../org/hyperledger/besu/ethereum/core/Transaction.java | 8 ++++++-- 7 files changed, 15 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bacbc88d070..f5b24a6f89a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ - Fix txpool dump/restore race condition [#6665](https://github.com/hyperledger/besu/pull/6665) - Make block transaction selection max time aware of PoA transitions [#6676](https://github.com/hyperledger/besu/pull/6676) - Don't enable the BFT mining coordinator when running sub commands such as `blocks export` [#6675](https://github.com/hyperledger/besu/pull/6675) +- In JSON-RPC return optional `v` fields for type 1 and type 2 transactions [#6762](https://github.com/hyperledger/besu/pull/6762) ### Download Links diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/TransactionCompleteResult.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/TransactionCompleteResult.java index 78c1069801a..003ee07ce36 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/TransactionCompleteResult.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/TransactionCompleteResult.java @@ -126,7 +126,7 @@ public TransactionCompleteResult(final TransactionWithMetadata tx) { this.v = (transactionType == TransactionType.ACCESS_LIST || transactionType == TransactionType.EIP1559) - ? this.yParity + ? Quantity.create(transaction.getV()) : null; } this.value = Quantity.create(transaction.getValue()); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/TransactionPendingResult.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/TransactionPendingResult.java index 694284d2260..62302808af3 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/TransactionPendingResult.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/TransactionPendingResult.java @@ -116,7 +116,7 @@ public TransactionPendingResult(final Transaction transaction) { this.v = (transactionType == TransactionType.ACCESS_LIST || transactionType == TransactionType.EIP1559) - ? this.yParity + ? Quantity.create(transaction.getV()) : null; } this.value = Quantity.create(transaction.getValue()); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/pending/PendingTransactionDetailResult.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/pending/PendingTransactionDetailResult.java index 6634ded9132..acbcb85ff11 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/pending/PendingTransactionDetailResult.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/pending/PendingTransactionDetailResult.java @@ -73,7 +73,7 @@ public PendingTransactionDetailResult(final Transaction tx) { this.v = (transactionType == TransactionType.ACCESS_LIST || transactionType == TransactionType.EIP1559) - ? this.yParity + ? Quantity.create(tx.getV()) : null; } this.value = Quantity.create(tx.getValue()); diff --git a/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/graphql/eth_getTransaction_type2.json b/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/graphql/eth_getTransaction_type2.json index d88f5f51554..c2c00330ff0 100644 --- a/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/graphql/eth_getTransaction_type2.json +++ b/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/graphql/eth_getTransaction_type2.json @@ -1,5 +1,5 @@ { - "request": "{transaction (hash : \"0x3ecd2ca6cf26c864d0ea5f038a58d4cd4a46a3e242fe92f446f392fdc232dd98\") { accessList { address storageKeys } maxFeePerGas maxPriorityFeePerGas nonce type status } } ", + "request": "{transaction (hash : \"0x3ecd2ca6cf26c864d0ea5f038a58d4cd4a46a3e242fe92f446f392fdc232dd98\") { accessList { address storageKeys } maxFeePerGas maxPriorityFeePerGas nonce type status yParity v} } ", "response": { "data": { "transaction": { @@ -15,7 +15,9 @@ "maxPriorityFeePerGas": "0x3b9aca00", "nonce": "0x20", "type": "0x2", - "status": "0x1" + "status": "0x1", + "yParity": "0x0", + "v": "0x25" } } }, diff --git a/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_getBlockByNumber_complete_shanghai.json b/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_getBlockByNumber_complete_shanghai.json index fb77fccd598..481dda711a9 100644 --- a/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_getBlockByNumber_complete_shanghai.json +++ b/ethereum/api/src/test/resources/org/hyperledger/besu/ethereum/api/jsonrpc/eth/eth_getBlockByNumber_complete_shanghai.json @@ -58,7 +58,7 @@ "type": "0x2", "value": "0x0", "yParity": "0x0", - "v" : "0x0", + "v" : "0x25", "r": "0x8abbfbd4c5f2a13a8d5ed394ac50bac7d678f83a23f645818492f76e8ee17ab3", "s": "0x7bd38c6929235f775d68b45bd7dea7981264f9a265b6bea97b070e15be88389c" } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java index 6aec8e6646b..2d703752825 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java @@ -495,10 +495,14 @@ public BigInteger getS() { @Override public BigInteger getV() { - if (transactionType != null && transactionType != TransactionType.FRONTIER) { - // EIP-2718 typed transaction, use yParity: + if (transactionType != null + && transactionType != TransactionType.FRONTIER + && transactionType != TransactionType.ACCESS_LIST + && transactionType != TransactionType.EIP1559) { + // Newer transaction type lacks V, so return null return null; } else { + // Mandatory for legacy, optional for EIP-2930 and EIP-1559 TXes, prohibited for all others. final BigInteger recId = BigInteger.valueOf(signature.getRecId()); return chainId .map(bigInteger -> recId.add(REPLAY_PROTECTED_V_BASE).add(TWO.multiply(bigInteger))) From 63a53aa76f888ab83c9427de68097bb64be9ad4a Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Wed, 27 Mar 2024 16:20:51 +1000 Subject: [PATCH 05/24] Add holesky DNS server (#6824) * add dns address for holesky Signed-off-by: Gabriel Fukushima * remove typo Signed-off-by: Gabriel Fukushima --------- Signed-off-by: Gabriel Fukushima --- config/src/main/resources/holesky.json | 1 + 1 file changed, 1 insertion(+) diff --git a/config/src/main/resources/holesky.json b/config/src/main/resources/holesky.json index 63f2b53a0dc..af552a37252 100644 --- a/config/src/main/resources/holesky.json +++ b/config/src/main/resources/holesky.json @@ -17,6 +17,7 @@ "cancunTime": 1707305664, "ethash": {}, "discovery": { + "dns": "AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE@all.holesky.ethdisco.net", "bootnodes": [ "enode://ac906289e4b7f12df423d654c5a962b6ebe5b3a74cc9e06292a85221f9a64a6f1cfdd6b714ed6dacef51578f92b34c60ee91e9ede9c7f8fadc4d347326d95e2b@146.190.13.128:30303", "enode://a3435a0155a3e837c02f5e7f5662a2f1fbc25b48e4dc232016e1c51b544cb5b4510ef633ea3278c0e970fa8ad8141e2d4d0f9f95456c537ff05fdf9b31c15072@178.128.136.233:30303" From 7e46889817b718d21b385c0568b19163ca9c1372 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Wed, 27 Mar 2024 10:16:51 +0100 Subject: [PATCH 06/24] Support running acceptance tests on Windows (#6820) Signed-off-by: Fabio Di Fabio --- .../acceptance/dsl/AcceptanceTestBase.java | 9 +++------ .../tests/acceptance/dsl/node/BesuNode.java | 15 +++++++++++++++ .../dsl/node/ProcessBesuNodeRunner.java | 18 ++++++++++++------ 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/AcceptanceTestBase.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/AcceptanceTestBase.java index cac4deb9d9b..0e1f6d65d5f 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/AcceptanceTestBase.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/AcceptanceTestBase.java @@ -56,15 +56,12 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import org.junit.After; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.extension.ExtendWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Superclass for acceptance tests. For now (transition to junit5 is ongoing) this class supports - * junit4 format. - */ +/** Superclass for acceptance tests. */ @ExtendWith(AcceptanceTestBaseTestWatcher.class) public class AcceptanceTestBase { @@ -131,7 +128,7 @@ protected AcceptanceTestBase() { exitedSuccessfully = new ExitedWithCode(0); } - @After + @AfterEach public void tearDownAcceptanceTestBase() { reportMemory(); cluster.close(); diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java index 80fbf7f727d..b2f9bc0abf7 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java @@ -789,6 +789,21 @@ public void stop() { nodeRequests.shutdown(); nodeRequests = null; } + + deleteRuntimeFiles(); + } + + private void deleteRuntimeFiles() { + try { + Files.deleteIfExists(homeDirectory.resolve("besu.networks")); + } catch (IOException e) { + LOG.error("Failed to clean up besu.networks file in {}", homeDirectory, e); + } + try { + Files.deleteIfExists(homeDirectory.resolve("besu.ports")); + } catch (IOException e) { + LOG.error("Failed to clean up besu.ports file in {}", homeDirectory, e); + } } @Override diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java index 2b80ebea568..adf872e19cc 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java @@ -52,6 +52,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import org.apache.commons.lang3.SystemUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; @@ -77,8 +78,15 @@ public void startNode(final BesuNode node) { final Path dataDir = node.homeDirectory(); + final var workingDir = + new File(System.getProperty("user.dir")).getParentFile().getParentFile().toPath(); + final List params = new ArrayList<>(); - params.add("build/install/besu/bin/besu"); + if (SystemUtils.IS_OS_WINDOWS) { + params.add(workingDir.resolve("build\\install\\besu\\bin\\besu.bat").toString()); + } else { + params.add("build/install/besu/bin/besu"); + } params.add("--data-path"); params.add(dataDir.toAbsolutePath().toString()); @@ -422,15 +430,13 @@ public void startNode(final BesuNode node) { LOG.info("Creating besu process with params {}", params); final ProcessBuilder processBuilder = new ProcessBuilder(params) - .directory(new File(System.getProperty("user.dir")).getParentFile().getParentFile()) + .directory(workingDir.toFile()) .redirectErrorStream(true) .redirectInput(Redirect.INHERIT); if (!node.getPlugins().isEmpty()) { processBuilder .environment() - .put( - "BESU_OPTS", - "-Dbesu.plugins.dir=" + dataDir.resolve("plugins").toAbsolutePath().toString()); + .put("BESU_OPTS", "-Dbesu.plugins.dir=" + dataDir.resolve("plugins").toAbsolutePath()); } // Use non-blocking randomness for acceptance tests processBuilder @@ -572,7 +578,7 @@ private void killBesuProcess(final String name) { LOG.info("Killing {} process, pid {}", name, process.pid()); - process.destroy(); + process.descendants().forEach(ProcessHandle::destroy); try { process.waitFor(30, TimeUnit.SECONDS); } catch (final InterruptedException e) { From f2c2512ef64a5c41e69e325c4d2f76837180cbbd Mon Sep 17 00:00:00 2001 From: Karim TAAM Date: Wed, 27 Mar 2024 11:17:40 +0100 Subject: [PATCH 07/24] storage format refactor for preparing verkle trie integration (#6721) Splitting Bonsai code: This commit aims to divide Bonsai into two packages - Common Classes with a Prefix: The common part will include classes prefixed with "DiffBased." These classes are designed to provide a base for both Bonsai and any future storage format that might use this diff-based approach. - Bonsai's Specifics: Bonsai will retain its unique features in its own package. This means that while it shares the diff-based infrastructure with the common part, it also has its own specific functionalities that are not shared with other storage format. - Extension to Verkle: this modification add the possibility of adding "Verkle" as a new storage format based on the diff-based architecture. Like Bonsai, Verkle would use the common diff-based classes but also have its own specific features. --------- Signed-off-by: Karim TAAM Signed-off-by: Karim Taam --- .../subcommands/storage/TrieLogHelper.java | 6 +- .../storage/TrieLogSubCommand.java | 4 +- .../besu/components/BesuComponent.java | 8 +- .../controller/BesuControllerBuilder.java | 23 +- .../storage/TrieLogHelperTest.java | 6 +- .../controller/BesuControllerBuilderTest.java | 8 +- .../internal/pojoadapter/AccountAdapter.java | 2 +- .../mainnet/AbstractBlockProcessor.java | 4 +- .../keyvalue/KeyValueStorageProvider.java | 2 +- .../common/GenesisWorldStateProvider.java | 18 +- .../{ => diffbased}/bonsai/BonsaiAccount.java | 185 ++------- .../bonsai/BonsaiWorldStateProvider.java | 145 +++++++ .../cache/BonsaiCachedMerkleTrieLoader.java} | 10 +- .../BonsaiCachedMerkleTrieLoaderModule.java} | 8 +- .../BonsaiCachedWorldStorageManager.java | 60 +++ .../NoOpBonsaiCachedWorldStorageManager.java} | 18 +- .../bonsai/storage/BonsaiPreImageProxy.java | 2 +- ...nsaiSnapshotWorldStateKeyValueStorage.java | 14 +- .../BonsaiWorldStateKeyValueStorage.java | 188 +-------- .../storage/BonsaiWorldStateLayerStorage.java | 11 +- .../storage/flat/FullFlatDbStrategy.java | 4 +- .../storage/flat/PartialFlatDbStrategy.java | 4 +- .../bonsai/trielog/TrieLogFactoryImpl.java | 19 +- .../bonsai/worldview/BonsaiWorldState.java | 361 ++++-------------- .../BonsaiWorldStateUpdateAccumulator.java | 98 +++++ .../diffbased/common/DiffBasedAccount.java | 206 ++++++++++ .../common/DiffBasedValue.java} | 25 +- .../common/DiffBasedWorldStateProvider.java} | 171 +++------ .../diffbased/common/StorageSubscriber.java | 25 ++ .../DiffBasedCachedWorldStorageManager.java} | 97 ++--- .../cache/DiffBasedCachedWorldView.java} | 22 +- ...BasedLayeredWorldStateKeyValueStorage.java | 22 ++ ...asedSnapshotWorldStateKeyValueStorage.java | 21 + .../DiffBasedWorldStateKeyValueStorage.java | 234 ++++++++++++ .../flat/AccountHashCodeStorageStrategy.java | 2 +- .../flat/CodeHashCodeStorageStrategy.java | 2 +- .../storage/flat/CodeStorageStrategy.java | 2 +- .../common}/storage/flat/FlatDbStrategy.java | 2 +- .../storage/flat/FlatDbStrategyProvider.java | 4 +- .../common}/trielog/NoOpTrieLogManager.java | 10 +- .../common}/trielog/TrieLogAddedEvent.java | 2 +- .../common}/trielog/TrieLogLayer.java | 50 +-- .../common}/trielog/TrieLogManager.java | 23 +- .../common}/trielog/TrieLogPruner.java | 12 +- .../common/worldview/DiffBasedWorldState.java | 334 ++++++++++++++++ .../common/worldview/DiffBasedWorldView.java} | 11 +- ...DiffBasedWorldStateUpdateAccumulator.java} | 339 ++++++++-------- .../preload/AccountConsumingMap.java | 49 +++ .../accumulator/preload/Consumer.java | 21 + .../preload/StorageConsumingMap.java | 53 +++ .../WorldStateStorageCoordinator.java | 2 +- .../core/InMemoryKeyValueStorageProvider.java | 12 +- .../besu/ethereum/core/TrieGenerator.java | 2 +- .../BlockImportExceptionHandlingTest.java | 6 +- .../bonsai/AbstractIsolationTests.java | 8 +- .../BonsaiCachedMerkleTrieLoaderTest.java} | 12 +- .../bonsai/BonsaiSnapshotIsolationTests.java | 2 +- .../bonsai/BonsaiWorldStateProviderTest.java | 36 +- .../bonsai/LogRollingTests.java | 12 +- .../{ => diffbased}/bonsai/RollingImport.java | 12 +- .../BonsaiWorldStateKeyValueStorageTest.java | 4 +- .../bonsai/trielog/TrieLogFactoryTests.java | 3 +- .../bonsai/trielog/TrieLogManagerTests.java | 9 +- .../bonsai/trielog/TrieLogPrunerTest.java | 7 +- .../worldview/BonsaiWorldStateTest.java | 30 +- .../flat/FlatDbStrategyProviderTest.java | 4 +- .../common}/trielog/TrieLogLayerTests.java | 2 +- ...tKeyValueStorageWorldStateStorageTest.java | 1 + .../eth/sync/DefaultSynchronizer.java | 2 +- .../eth/sync/fastsync/FastSyncDownloader.java | 2 +- .../eth/sync/snapsync/PersistDataStep.java | 2 +- .../sync/snapsync/SnapWorldDownloadState.java | 2 +- .../snapsync/SnapWorldStateDownloader.java | 2 +- .../request/AccountRangeDataRequest.java | 2 +- .../request/StorageRangeDataRequest.java | 2 +- ...ccountFlatDatabaseHealingRangeRequest.java | 2 +- ...torageFlatDatabaseHealingRangeRequest.java | 2 +- .../CheckPointSyncChainDownloaderTest.java | 2 +- .../fastsync/FastDownloaderFactoryTest.java | 2 +- .../sync/fastsync/FastSyncDownloaderTest.java | 2 +- .../FastWorldDownloadStateTest.java | 2 +- .../worldstate/LoadLocalDataStepTest.java | 2 +- .../snapsync/AccountHealingTrackingTest.java | 2 +- .../sync/snapsync/PersistDataStepTest.java | 2 +- .../snapsync/SnapWorldDownloadStateTest.java | 2 +- .../eth/sync/snapsync/TaskGenerator.java | 3 +- ...ntFlatDatabaseHealingRangeRequestTest.java | 2 +- ...geFlatDatabaseHealingRangeRequestTest.java | 3 +- .../StorageTrieNodeHealingRequestTest.java | 2 +- .../BonsaiReferenceTestUpdateAccumulator.java | 18 +- .../BonsaiReferenceTestWorldState.java | 58 +-- .../BonsaiReferenceTestWorldStateStorage.java | 12 +- gradle.properties | 2 +- 93 files changed, 1968 insertions(+), 1277 deletions(-) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/BonsaiAccount.java (55%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/cache/CachedMerkleTrieLoader.java => diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java} (94%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/cache/CachedMerkleTrieLoaderModule.java => diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoaderModule.java} (77%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/cache/NoOpCachedWorldStorageManager.java => diffbased/bonsai/cache/NoOpBonsaiCachedWorldStorageManager.java} (68%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/BonsaiPreImageProxy.java (97%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java (93%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/BonsaiWorldStateKeyValueStorage.java (67%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/BonsaiWorldStateLayerStorage.java (79%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/flat/FullFlatDbStrategy.java (93%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/flat/PartialFlatDbStrategy.java (96%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/trielog/TrieLogFactoryImpl.java (91%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/worldview/BonsaiWorldState.java (59%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/BonsaiValue.java => diffbased/common/DiffBasedValue.java} (80%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/BonsaiWorldStateProvider.java => diffbased/common/DiffBasedWorldStateProvider.java} (62%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/StorageSubscriber.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/cache/CachedWorldStorageManager.java => diffbased/common/cache/DiffBasedCachedWorldStorageManager.java} (62%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/cache/CachedBonsaiWorldView.java => diffbased/common/cache/DiffBasedCachedWorldView.java} (71%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedLayeredWorldStateKeyValueStorage.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedWorldStateKeyValueStorage.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/storage/flat/AccountHashCodeStorageStrategy.java (96%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/storage/flat/CodeHashCodeStorageStrategy.java (96%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/storage/flat/CodeStorageStrategy.java (94%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/storage/flat/FlatDbStrategy.java (99%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/storage/flat/FlatDbStrategyProvider.java (95%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/trielog/NoOpTrieLogManager.java (79%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/trielog/TrieLogAddedEvent.java (92%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/trielog/TrieLogLayer.java (79%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/trielog/TrieLogManager.java (87%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/trielog/TrieLogPruner.java (94%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/worldview/BonsaiWorldView.java => diffbased/common/worldview/DiffBasedWorldView.java} (85%) rename ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/{bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java => diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java} (71%) create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/AccountConsumingMap.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/Consumer.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/StorageConsumingMap.java rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/AbstractIsolationTests.java (97%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{bonsai/CachedMerkleTrieLoaderTest.java => diffbased/bonsai/BonsaiCachedMerkleTrieLoaderTest.java} (94%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/BonsaiSnapshotIsolationTests.java (99%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/BonsaiWorldStateProviderTest.java (89%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/LogRollingTests.java (96%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/RollingImport.java (90%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java (99%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/trielog/TrieLogFactoryTests.java (94%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/trielog/TrieLogManagerTests.java (84%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/trielog/TrieLogPrunerTest.java (96%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{ => diffbased}/bonsai/worldview/BonsaiWorldStateTest.java (81%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/storage/flat/FlatDbStrategyProviderTest.java (97%) rename ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/{bonsai => diffbased/common}/trielog/TrieLogLayerTests.java (98%) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java index 8cebe77618c..d94c19d8bdb 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java @@ -27,9 +27,9 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.rlp.RLP; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import java.io.File; diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java index a47271e247b..0ee2e4532c6 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogSubCommand.java @@ -25,8 +25,8 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; diff --git a/besu/src/main/java/org/hyperledger/besu/components/BesuComponent.java b/besu/src/main/java/org/hyperledger/besu/components/BesuComponent.java index f0093243a6c..65ba2d931d2 100644 --- a/besu/src/main/java/org/hyperledger/besu/components/BesuComponent.java +++ b/besu/src/main/java/org/hyperledger/besu/components/BesuComponent.java @@ -19,8 +19,8 @@ import org.hyperledger.besu.cli.BesuCommand; import org.hyperledger.besu.ethereum.eth.transactions.BlobCache; import org.hyperledger.besu.ethereum.eth.transactions.BlobCacheModule; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoaderModule; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoaderModule; import org.hyperledger.besu.metrics.MetricsSystemModule; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.services.BesuPluginContextImpl; @@ -37,7 +37,7 @@ modules = { BesuCommandModule.class, MetricsSystemModule.class, - CachedMerkleTrieLoaderModule.class, + BonsaiCachedMerkleTrieLoaderModule.class, BesuPluginContextModule.class, BlobCacheModule.class }) @@ -55,7 +55,7 @@ public interface BesuComponent { * * @return CachedMerkleTrieLoader */ - CachedMerkleTrieLoader getCachedMerkleTrieLoader(); + BonsaiCachedMerkleTrieLoader getCachedMerkleTrieLoader(); /** * a metrics system that is observable by a Prometheus or OTEL metrics collection subsystem diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index cdc1a317088..8990ba9e447 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -81,11 +81,11 @@ import org.hyperledger.besu.ethereum.p2p.config.SubProtocolConfiguration; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogPruner; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive; import org.hyperledger.besu.ethereum.trie.forest.pruner.MarkSweepPruner; import org.hyperledger.besu.ethereum.trie.forest.pruner.Pruner; @@ -576,13 +576,14 @@ public BesuController build() { dataDirectory.toString(), numberOfBlocksToCache); - final CachedMerkleTrieLoader cachedMerkleTrieLoader = + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader = besuComponent .map(BesuComponent::getCachedMerkleTrieLoader) - .orElseGet(() -> new CachedMerkleTrieLoader(metricsSystem)); + .orElseGet(() -> new BonsaiCachedMerkleTrieLoader(metricsSystem)); final WorldStateArchive worldStateArchive = - createWorldStateArchive(worldStateStorageCoordinator, blockchain, cachedMerkleTrieLoader); + createWorldStateArchive( + worldStateStorageCoordinator, blockchain, bonsaiCachedMerkleTrieLoader); if (blockchain.getChainHeadBlockNumber() < 1) { genesisState.writeStateTo(worldStateArchive.getMutable()); @@ -1048,7 +1049,7 @@ private Optional createSnapProtocolManager( WorldStateArchive createWorldStateArchive( final WorldStateStorageCoordinator worldStateStorageCoordinator, final Blockchain blockchain, - final CachedMerkleTrieLoader cachedMerkleTrieLoader) { + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader) { return switch (dataStorageConfiguration.getDataStorageFormat()) { case BONSAI -> { final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage = @@ -1057,7 +1058,7 @@ yield new BonsaiWorldStateProvider( worldStateKeyValueStorage, blockchain, Optional.of(dataStorageConfiguration.getBonsaiMaxLayersToLoad()), - cachedMerkleTrieLoader, + bonsaiCachedMerkleTrieLoader, besuComponent.map(BesuComponent::getBesuPluginContext).orElse(null), evmConfiguration); } @@ -1067,6 +1068,8 @@ yield new BonsaiWorldStateProvider( yield new ForestWorldStateArchive( worldStateStorageCoordinator, preimageStorage, evmConfiguration); } + default -> throw new IllegalStateException( + "Unexpected value: " + dataStorageConfiguration.getDataStorageFormat()); }; } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java index c2789c68085..d4a6dd2dc71 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java @@ -31,9 +31,9 @@ import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; diff --git a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java index a06e1907d2c..5eb14201cd8 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java @@ -43,9 +43,9 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; @@ -189,7 +189,7 @@ public void shouldDisablePruningIfBonsaiIsEnabled() { .createWorldStateArchive( any(WorldStateStorageCoordinator.class), any(Blockchain.class), - any(CachedMerkleTrieLoader.class)); + any(BonsaiCachedMerkleTrieLoader.class)); doReturn(mockWorldState).when(worldStateArchive).getMutable(); when(storageProvider.createWorldStateStorageCoordinator(dataStorageConfiguration)) .thenReturn(new WorldStateStorageCoordinator(bonsaiWorldStateStorage)); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/AccountAdapter.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/AccountAdapter.java index 13caadab6ab..faf42aafc8e 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/AccountAdapter.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/AccountAdapter.java @@ -17,7 +17,7 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.account.AccountState; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java index 1a70ce49381..c6cf6898eb1 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java @@ -31,8 +31,8 @@ import org.hyperledger.besu.ethereum.privacy.storage.PrivateMetadataUpdater; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.vm.BlockHashLookup; import org.hyperledger.besu.ethereum.vm.CachingBlockHashLookup; import org.hyperledger.besu.evm.gascalculator.CancunGasCalculator; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java index 004790ee170..944f668cd68 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java @@ -19,7 +19,7 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ScheduleBasedBlockHeaderFunctions; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java index 121fec3f4dc..7f89ff7185c 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/common/GenesisWorldStateProvider.java @@ -19,11 +19,11 @@ import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.NoOpCachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.NoOpTrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.NoOpBonsaiCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.NoOpTrieLogManager; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; @@ -59,8 +59,8 @@ public static MutableWorldState createGenesisWorldState( * @return a mutable world state for the Genesis block */ private static MutableWorldState createGenesisBonsaiWorldState() { - final CachedMerkleTrieLoader cachedMerkleTrieLoader = - new CachedMerkleTrieLoader(new NoOpMetricsSystem()); + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader = + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()); final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage = new BonsaiWorldStateKeyValueStorage( new KeyValueStorageProvider( @@ -71,8 +71,8 @@ private static MutableWorldState createGenesisBonsaiWorldState() { DataStorageConfiguration.DEFAULT_BONSAI_CONFIG); return new BonsaiWorldState( bonsaiWorldStateKeyValueStorage, - cachedMerkleTrieLoader, - new NoOpCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage), + bonsaiCachedMerkleTrieLoader, + new NoOpBonsaiCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage), new NoOpTrieLogManager(), EvmConfiguration.DEFAULT); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiAccount.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccount.java similarity index 55% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiAccount.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccount.java index d93f550cafa..6eea664e048 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiAccount.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccount.java @@ -14,48 +14,34 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import org.hyperledger.besu.datatypes.AccountValue; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.Wei; -import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.rlp.RLPException; import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.rlp.RLPOutput; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; import org.hyperledger.besu.evm.ModificationNotAllowedException; import org.hyperledger.besu.evm.account.AccountStorageEntry; -import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.worldstate.UpdateTrackingAccount; -import java.util.HashMap; -import java.util.Map; import java.util.NavigableMap; import java.util.Objects; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.units.bigints.UInt256; -public class BonsaiAccount implements MutableAccount, AccountValue { - private final BonsaiWorldView context; - private boolean immutable; - - private final Address address; - private final Hash addressHash; - private Hash codeHash; - private long nonce; - private Wei balance; +public class BonsaiAccount extends DiffBasedAccount { private Hash storageRoot; - private Bytes code; - - private final Map updatedStorage = new HashMap<>(); public BonsaiAccount( - final BonsaiWorldView context, + final DiffBasedWorldView context, final Address address, final Hash addressHash, final long nonce, @@ -63,31 +49,24 @@ public BonsaiAccount( final Hash storageRoot, final Hash codeHash, final boolean mutable) { - this.context = context; - this.address = address; - this.addressHash = addressHash; - this.nonce = nonce; - this.balance = balance; + super(context, address, addressHash, nonce, balance, codeHash, mutable); this.storageRoot = storageRoot; - this.codeHash = codeHash; - - this.immutable = !mutable; } public BonsaiAccount( - final BonsaiWorldView context, + final DiffBasedWorldView context, final Address address, final AccountValue stateTrieAccount, final boolean mutable) { - this( + super( context, address, address.addressHash(), stateTrieAccount.getNonce(), stateTrieAccount.getBalance(), - stateTrieAccount.getStorageRoot(), stateTrieAccount.getCodeHash(), mutable); + this.storageRoot = stateTrieAccount.getStorageRoot(); } public BonsaiAccount(final BonsaiAccount toCopy) { @@ -95,37 +74,35 @@ public BonsaiAccount(final BonsaiAccount toCopy) { } public BonsaiAccount( - final BonsaiAccount toCopy, final BonsaiWorldView context, final boolean mutable) { - this.context = context; - this.address = toCopy.address; - this.addressHash = toCopy.addressHash; - this.nonce = toCopy.nonce; - this.balance = toCopy.balance; + final BonsaiAccount toCopy, final DiffBasedWorldView context, final boolean mutable) { + super( + context, + toCopy.address, + toCopy.addressHash, + toCopy.nonce, + toCopy.balance, + toCopy.codeHash, + mutable); this.storageRoot = toCopy.storageRoot; - this.codeHash = toCopy.codeHash; - this.code = toCopy.code; updatedStorage.putAll(toCopy.updatedStorage); - - this.immutable = !mutable; } public BonsaiAccount( - final BonsaiWorldView context, final UpdateTrackingAccount tracked) { - this.context = context; - this.address = tracked.getAddress(); - this.addressHash = tracked.getAddressHash(); - this.nonce = tracked.getNonce(); - this.balance = tracked.getBalance(); + final DiffBasedWorldView context, final UpdateTrackingAccount tracked) { + super( + context, + tracked.getAddress(), + tracked.getAddressHash(), + tracked.getNonce(), + tracked.getBalance(), + tracked.getCodeHash(), + true); this.storageRoot = Hash.EMPTY_TRIE_HASH; - this.codeHash = tracked.getCodeHash(); - this.code = tracked.getCode(); updatedStorage.putAll(tracked.getUpdatedStorage()); - - this.immutable = false; } public static BonsaiAccount fromRLP( - final BonsaiWorldView context, + final DiffBasedWorldView context, final Address address, final Bytes encoded, final boolean mutable) @@ -144,88 +121,11 @@ public static BonsaiAccount fromRLP( context, address, address.addressHash(), nonce, balance, storageRoot, codeHash, mutable); } - @Override - public Address getAddress() { - return address; - } - - @Override - public Hash getAddressHash() { - return addressHash; - } - - @Override - public long getNonce() { - return nonce; - } - - @Override - public void setNonce(final long value) { - if (immutable) { - throw new ModificationNotAllowedException(); - } - nonce = value; - } - - @Override - public Wei getBalance() { - return balance; - } - - @Override - public void setBalance(final Wei value) { - if (immutable) { - throw new ModificationNotAllowedException(); - } - balance = value; - } - - @Override - public Bytes getCode() { - if (code == null) { - code = context.getCode(address, codeHash).orElse(Bytes.EMPTY); - } - return code; - } - - @Override - public void setCode(final Bytes code) { - if (immutable) { - throw new ModificationNotAllowedException(); - } - this.code = code; - if (code == null || code.isEmpty()) { - this.codeHash = Hash.EMPTY; - } else { - this.codeHash = Hash.hash(code); - } - } - - @Override - public Hash getCodeHash() { - return codeHash; - } - - @Override - public UInt256 getStorageValue(final UInt256 key) { - return context.getStorageValue(address, key); - } - - @Override - public UInt256 getOriginalStorageValue(final UInt256 key) { - return context.getPriorStorageValue(address, key); - } - @Override public NavigableMap storageEntriesFrom( final Bytes32 startKeyHash, final int limit) { - return context.getWorldStateStorage().storageEntriesFrom(this.addressHash, startKeyHash, limit); - } - - public Bytes serializeAccount() { - final BytesValueRLPOutput out = new BytesValueRLPOutput(); - writeTo(out); - return out.encoded(); + return ((BonsaiWorldStateKeyValueStorage) context.getWorldStateStorage()) + .storageEntriesFrom(this.addressHash, startKeyHash, limit); } @Override @@ -240,24 +140,6 @@ public void writeTo(final RLPOutput out) { out.endList(); } - @Override - public void setStorageValue(final UInt256 key, final UInt256 value) { - if (immutable) { - throw new ModificationNotAllowedException(); - } - updatedStorage.put(key, value); - } - - @Override - public void clearStorage() { - updatedStorage.clear(); - } - - @Override - public Map getUpdatedStorage() { - return updatedStorage; - } - @Override public Hash getStorageRoot() { return storageRoot; @@ -270,11 +152,6 @@ public void setStorageRoot(final Hash storageRoot) { this.storageRoot = storageRoot; } - @Override - public void becomeImmutable() { - immutable = true; - } - @Override public String toString() { return "AccountState{" diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java new file mode 100644 index 00000000000..bfd295091c3 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProvider.java @@ -0,0 +1,145 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.chain.Blockchain; +import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; +import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; +import org.hyperledger.besu.evm.internal.EvmConfiguration; +import org.hyperledger.besu.plugin.BesuContext; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.tuweni.bytes.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BonsaiWorldStateProvider extends DiffBasedWorldStateProvider { + + private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateProvider.class); + private final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader; + + public BonsaiWorldStateProvider( + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, + final Blockchain blockchain, + final Optional maxLayersToLoad, + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader, + final BesuContext pluginContext, + final EvmConfiguration evmConfiguration) { + super(worldStateKeyValueStorage, blockchain, maxLayersToLoad, pluginContext); + this.bonsaiCachedMerkleTrieLoader = bonsaiCachedMerkleTrieLoader; + provideCachedWorldStorageManager( + new BonsaiCachedWorldStorageManager(this, worldStateKeyValueStorage)); + loadPersistedState(new BonsaiWorldState(this, worldStateKeyValueStorage, evmConfiguration)); + } + + @VisibleForTesting + BonsaiWorldStateProvider( + final BonsaiCachedWorldStorageManager bonsaiCachedWorldStorageManager, + final TrieLogManager trieLogManager, + final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, + final Blockchain blockchain, + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader, + final EvmConfiguration evmConfiguration) { + super(worldStateKeyValueStorage, blockchain, trieLogManager); + this.bonsaiCachedMerkleTrieLoader = bonsaiCachedMerkleTrieLoader; + provideCachedWorldStorageManager(bonsaiCachedWorldStorageManager); + loadPersistedState(new BonsaiWorldState(this, worldStateKeyValueStorage, evmConfiguration)); + } + + public BonsaiCachedMerkleTrieLoader getCachedMerkleTrieLoader() { + return bonsaiCachedMerkleTrieLoader; + } + + private BonsaiWorldStateKeyValueStorage getWorldStateKeyValueStorage() { + return (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage; + } + /** + * Prepares the state healing process for a given address and location. It prepares the state + * healing, including retrieving data from storage, identifying invalid slots or nodes, removing + * account and slot from the state trie, and committing the changes. Finally, it downgrades the + * world state storage to partial flat database mode. + */ + public void prepareStateHealing(final Address address, final Bytes location) { + final Set keysToDelete = new HashSet<>(); + final BonsaiWorldStateKeyValueStorage.Updater updater = + getWorldStateKeyValueStorage().updater(); + final Hash accountHash = address.addressHash(); + final StoredMerklePatriciaTrie accountTrie = + new StoredMerklePatriciaTrie<>( + (l, h) -> { + final Optional node = + getWorldStateKeyValueStorage().getAccountStateTrieNode(l, h); + if (node.isPresent()) { + keysToDelete.add(l); + } + return node; + }, + persistedState.getWorldStateRootHash(), + Function.identity(), + Function.identity()); + try { + accountTrie + .get(accountHash) + .map(RLP::input) + .map(StateTrieAccountValue::readFrom) + .ifPresent( + account -> { + final StoredMerklePatriciaTrie storageTrie = + new StoredMerklePatriciaTrie<>( + (l, h) -> { + Optional node = + getWorldStateKeyValueStorage() + .getAccountStorageTrieNode(accountHash, l, h); + if (node.isPresent()) { + keysToDelete.add(Bytes.concatenate(accountHash, l)); + } + return node; + }, + account.getStorageRoot(), + Function.identity(), + Function.identity()); + try { + storageTrie.getPath(location); + } catch (Exception eA) { + LOG.warn("Invalid slot found for account {} at location {}", address, location); + // ignore + } + }); + } catch (Exception eA) { + LOG.warn("Invalid node for account {} at location {}", address, location); + // ignore + } + keysToDelete.forEach(updater::removeAccountStateTrieNode); + updater.commit(); + + getWorldStateKeyValueStorage().downgradeToPartialFlatDbMode(); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoader.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java similarity index 94% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoader.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java index cbd1fc820d8..9f3251660dc 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoader.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoader.java @@ -13,14 +13,15 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.cache; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.metrics.ObservableMetricsSystem; @@ -37,8 +38,7 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -public class CachedMerkleTrieLoader - implements BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber { +public class BonsaiCachedMerkleTrieLoader implements StorageSubscriber { private static final int ACCOUNT_CACHE_SIZE = 100_000; private static final int STORAGE_CACHE_SIZE = 200_000; @@ -47,7 +47,7 @@ public class CachedMerkleTrieLoader private final Cache storageNodes = CacheBuilder.newBuilder().recordStats().maximumSize(STORAGE_CACHE_SIZE).build(); - public CachedMerkleTrieLoader(final ObservableMetricsSystem metricsSystem) { + public BonsaiCachedMerkleTrieLoader(final ObservableMetricsSystem metricsSystem) { CacheMetricsCollector cacheMetrics = new CacheMetricsCollector(); cacheMetrics.addCache("accountsNodes", accountNodes); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoaderModule.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoaderModule.java similarity index 77% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoaderModule.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoaderModule.java index 63c8051f253..04a8efc4514 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedMerkleTrieLoaderModule.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedMerkleTrieLoaderModule.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.cache; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache; import org.hyperledger.besu.metrics.ObservableMetricsSystem; @@ -21,11 +21,11 @@ import dagger.Provides; @Module -public class CachedMerkleTrieLoaderModule { +public class BonsaiCachedMerkleTrieLoaderModule { @Provides - CachedMerkleTrieLoader provideCachedMerkleTrieLoaderModule( + BonsaiCachedMerkleTrieLoader provideCachedMerkleTrieLoaderModule( final ObservableMetricsSystem metricsSystem) { - return new CachedMerkleTrieLoader(metricsSystem); + return new BonsaiCachedMerkleTrieLoader(metricsSystem); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java new file mode 100644 index 00000000000..a7bee477c12 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/BonsaiCachedWorldStorageManager.java @@ -0,0 +1,60 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache; + +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.evm.internal.EvmConfiguration; + +public class BonsaiCachedWorldStorageManager extends DiffBasedCachedWorldStorageManager { + + public BonsaiCachedWorldStorageManager( + final BonsaiWorldStateProvider archive, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage) { + super(archive, worldStateKeyValueStorage); + } + + @Override + public DiffBasedWorldState createWorldState( + final DiffBasedWorldStateProvider archive, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final EvmConfiguration evmConfiguration) { + return new BonsaiWorldState( + (BonsaiWorldStateProvider) archive, + (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage, + evmConfiguration); + } + + @Override + public DiffBasedWorldStateKeyValueStorage createLayeredKeyValueStorage( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage) { + return new BonsaiWorldStateLayerStorage( + (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage); + } + + @Override + public DiffBasedWorldStateKeyValueStorage createSnapshotKeyValueStorage( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage) { + return new BonsaiSnapshotWorldStateKeyValueStorage( + (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/NoOpBonsaiCachedWorldStorageManager.java similarity index 68% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/NoOpBonsaiCachedWorldStorageManager.java index d86128b13f1..0e4ee9cbffd 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/NoOpCachedWorldStorageManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/cache/NoOpBonsaiCachedWorldStorageManager.java @@ -12,19 +12,19 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.cache; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; import java.util.Optional; import java.util.function.Function; -public class NoOpCachedWorldStorageManager extends CachedWorldStorageManager { +public class NoOpBonsaiCachedWorldStorageManager extends BonsaiCachedWorldStorageManager { - public NoOpCachedWorldStorageManager( + public NoOpBonsaiCachedWorldStorageManager( final BonsaiWorldStateKeyValueStorage bonsaiWorldStateKeyValueStorage) { super(null, bonsaiWorldStateKeyValueStorage); } @@ -33,7 +33,7 @@ public NoOpCachedWorldStorageManager( public synchronized void addCachedLayer( final BlockHeader blockHeader, final Hash worldStateRootHash, - final BonsaiWorldState forWorldState) { + final DiffBasedWorldState forWorldState) { // no cache } @@ -43,17 +43,17 @@ public boolean containWorldStateStorage(final Hash blockHash) { } @Override - public Optional getWorldState(final Hash blockHash) { + public Optional getWorldState(final Hash blockHash) { return Optional.empty(); } @Override - public Optional getNearestWorldState(final BlockHeader blockHeader) { + public Optional getNearestWorldState(final BlockHeader blockHeader) { return Optional.empty(); } @Override - public Optional getHeadWorldState( + public Optional getHeadWorldState( final Function> hashBlockHeaderFunction) { return Optional.empty(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiPreImageProxy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiPreImageProxy.java similarity index 97% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiPreImageProxy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiPreImageProxy.java index 16cda3104f0..4bae322a855 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiPreImageProxy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiPreImageProxy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java similarity index 93% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java index e20d844738e..c2972e04598 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java @@ -13,11 +13,12 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedSnapshotWorldStateKeyValueStorage; import org.hyperledger.besu.plugin.services.exception.StorageException; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SnappableKeyValueStorage; @@ -32,7 +33,7 @@ import org.slf4j.LoggerFactory; public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKeyValueStorage - implements BonsaiStorageSubscriber { + implements DiffBasedSnapshotWorldStateKeyValueStorage, StorageSubscriber { protected final BonsaiWorldStateKeyValueStorage parentWorldStateStorage; private static final Logger LOG = @@ -53,9 +54,9 @@ public BonsaiSnapshotWorldStateKeyValueStorage( final BonsaiWorldStateKeyValueStorage worldStateStorageKeyValueStorage) { this( worldStateStorageKeyValueStorage, - ((SnappableKeyValueStorage) worldStateStorageKeyValueStorage.composedWorldStateStorage) + ((SnappableKeyValueStorage) worldStateStorageKeyValueStorage.getComposedWorldStateStorage()) .takeSnapshot(), - worldStateStorageKeyValueStorage.trieLogStorage); + worldStateStorageKeyValueStorage.getTrieLogStorage()); } private boolean isClosedGet() { @@ -207,7 +208,7 @@ public void onClearTrieLog() { protected synchronized void doClose() throws Exception { if (!isClosedGet()) { // alert any subscribers we are closing: - subscribers.forEach(BonsaiStorageSubscriber::onCloseStorage); + subscribers.forEach(StorageSubscriber::onCloseStorage); // close all of the SnappedKeyValueStorages: composedWorldStateStorage.close(); @@ -220,6 +221,7 @@ protected synchronized void doClose() throws Exception { } } + @Override public BonsaiWorldStateKeyValueStorage getParentWorldStateStorage() { return parentWorldStateStorage; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorage.java similarity index 67% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorage.java index 5f477afebaf..a58316fc62c 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorage.java @@ -12,7 +12,7 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; @@ -24,8 +24,9 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.trie.MerkleTrie; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.flat.FlatDbStrategy; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.flat.FlatDbStrategyProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategyProvider; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; @@ -37,52 +38,28 @@ import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; -import org.hyperledger.besu.util.Subscribers; -import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.Map; import java.util.NavigableMap; import java.util.Optional; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; -import java.util.stream.Stream; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class BonsaiWorldStateKeyValueStorage implements WorldStateKeyValueStorage, AutoCloseable { - private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateKeyValueStorage.class); - - // 0x776f726c64526f6f74 - public static final byte[] WORLD_ROOT_HASH_KEY = "worldRoot".getBytes(StandardCharsets.UTF_8); - // 0x776f726c64426c6f636b48617368 - public static final byte[] WORLD_BLOCK_HASH_KEY = - "worldBlockHash".getBytes(StandardCharsets.UTF_8); +public class BonsaiWorldStateKeyValueStorage extends DiffBasedWorldStateKeyValueStorage + implements WorldStateKeyValueStorage { protected final FlatDbStrategyProvider flatDbStrategyProvider; - protected final SegmentedKeyValueStorage composedWorldStateStorage; - protected final KeyValueStorage trieLogStorage; - - private final AtomicBoolean shouldClose = new AtomicBoolean(false); - - protected final AtomicBoolean isClosed = new AtomicBoolean(false); - - protected final Subscribers subscribers = Subscribers.create(); - public BonsaiWorldStateKeyValueStorage( final StorageProvider provider, final MetricsSystem metricsSystem, final DataStorageConfiguration dataStorageConfiguration) { - this.composedWorldStateStorage = + super( provider.getStorageBySegmentIdentifiers( List.of( - ACCOUNT_INFO_STATE, CODE_STORAGE, ACCOUNT_STORAGE_STORAGE, TRIE_BRANCH_STORAGE)); - this.trieLogStorage = - provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); + ACCOUNT_INFO_STATE, CODE_STORAGE, ACCOUNT_STORAGE_STORAGE, TRIE_BRANCH_STORAGE)), + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE)); this.flatDbStrategyProvider = new FlatDbStrategyProvider(metricsSystem, dataStorageConfiguration); flatDbStrategyProvider.loadFlatDbStrategy(composedWorldStateStorage); @@ -92,9 +69,8 @@ public BonsaiWorldStateKeyValueStorage( final FlatDbStrategyProvider flatDbStrategyProvider, final SegmentedKeyValueStorage composedWorldStateStorage, final KeyValueStorage trieLogStorage) { + super(composedWorldStateStorage, trieLogStorage); this.flatDbStrategyProvider = flatDbStrategyProvider; - this.composedWorldStateStorage = composedWorldStateStorage; - this.trieLogStorage = trieLogStorage; } @Override @@ -102,6 +78,7 @@ public DataStorageFormat getDataStorageFormat() { return DataStorageFormat.BONSAI; } + @Override public FlatDbMode getFlatDbMode() { return flatDbStrategyProvider.getFlatDbMode(); } @@ -155,31 +132,6 @@ public Optional getTrieNodeUnsafe(final Bytes key) { .map(Bytes::wrap); } - public Optional getTrieLog(final Hash blockHash) { - return trieLogStorage.get(blockHash.toArrayUnsafe()); - } - - public Stream streamTrieLogKeys(final long limit) { - return trieLogStorage.streamKeys().limit(limit); - } - - public Optional getStateTrieNode(final Bytes location) { - return composedWorldStateStorage - .get(TRIE_BRANCH_STORAGE, location.toArrayUnsafe()) - .map(Bytes::wrap); - } - - public Optional getWorldStateRootHash() { - return composedWorldStateStorage.get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY).map(Bytes::wrap); - } - - public Optional getWorldStateBlockHash() { - return composedWorldStateStorage - .get(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY) - .map(Bytes32::wrap) - .map(Hash::wrap); - } - public Optional getStorageValueByStorageSlotKey( final Hash accountHash, final StorageSlotKey storageSlotKey) { return getStorageValueByStorageSlotKey( @@ -209,34 +161,11 @@ public Optional getStorageValueByStorageSlotKey( composedWorldStateStorage); } - public Map streamFlatAccounts( - final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { - return flatDbStrategyProvider - .getFlatDbStrategy(composedWorldStateStorage) - .streamAccountFlatDatabase(composedWorldStateStorage, startKeyHash, endKeyHash, max); - } - - public Map streamFlatStorages( - final Hash accountHash, final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { - return flatDbStrategyProvider - .getFlatDbStrategy(composedWorldStateStorage) - .streamStorageFlatDatabase( - composedWorldStateStorage, accountHash, startKeyHash, endKeyHash, max); - } - public NavigableMap storageEntriesFrom( final Hash addressHash, final Bytes32 startKeyHash, final int limit) { throw new RuntimeException("Bonsai Tries does not currently support enumerating storage"); } - public boolean isWorldStateAvailable(final Bytes32 rootHash, final Hash blockHash) { - return composedWorldStateStorage - .get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY) - .map(Bytes32::wrap) - .map(hash -> hash.equals(rootHash) || trieLogStorage.containsKey(blockHash.toArrayUnsafe())) - .orElse(false); - } - public void upgradeToFullFlatDbMode() { flatDbStrategyProvider.upgradeToFullFlatDbMode(composedWorldStateStorage); } @@ -247,26 +176,14 @@ public void downgradeToPartialFlatDbMode() { @Override public void clear() { - subscribers.forEach(BonsaiStorageSubscriber::onClearStorage); - flatDbStrategyProvider - .getFlatDbStrategy(composedWorldStateStorage) - .clearAll(composedWorldStateStorage); - composedWorldStateStorage.clear(TRIE_BRANCH_STORAGE); - trieLogStorage.clear(); + super.clear(); flatDbStrategyProvider.loadFlatDbStrategy( composedWorldStateStorage); // force reload of flat db reader strategy } - public void clearTrieLog() { - subscribers.forEach(BonsaiStorageSubscriber::onClearTrieLog); - trieLogStorage.clear(); - } - - public void clearFlatDatabase() { - subscribers.forEach(BonsaiStorageSubscriber::onClearFlatDatabaseStorage); - flatDbStrategyProvider - .getFlatDbStrategy(composedWorldStateStorage) - .resetOnResync(composedWorldStateStorage); + @Override + public FlatDbStrategy getFlatDbStrategy() { + return flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage); } @Override @@ -277,20 +194,7 @@ public Updater updater() { flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage)); } - public boolean pruneTrieLog(final Hash blockHash) { - try { - return trieLogStorage.tryDelete(blockHash.toArrayUnsafe()); - } catch (Exception e) { - LOG.error("Error pruning trie log for block hash {}", blockHash, e); - return false; - } - } - - public FlatDbStrategy getFlatDbStrategy() { - return flatDbStrategyProvider.getFlatDbStrategy(composedWorldStateStorage); - } - - public static class Updater implements WorldStateKeyValueStorage.Updater { + public static class Updater implements DiffBasedWorldStateKeyValueStorage.Updater { private final SegmentedKeyValueStorageTransaction composedWorldStateTransaction; private final KeyValueStorageTransaction trieLogStorageTransaction; @@ -340,6 +244,7 @@ public Updater putAccountInfoState(final Hash accountHash, final Bytes accountVa return this; } + @Override public Updater saveWorldState(final Bytes blockHash, final Bytes32 nodeHash, final Bytes node) { composedWorldStateTransaction.put( TRIE_BRANCH_STORAGE, Bytes.EMPTY.toArrayUnsafe(), node.toArrayUnsafe()); @@ -392,10 +297,12 @@ public synchronized void removeStorageValueBySlotHash( composedWorldStateTransaction, accountHash, slotHash); } + @Override public SegmentedKeyValueStorageTransaction getWorldStateTransaction() { return composedWorldStateTransaction; } + @Override public KeyValueStorageTransaction getTrieLogStorageTransaction() { return trieLogStorageTransaction; } @@ -407,65 +314,10 @@ public void commit() { composedWorldStateTransaction.commit(); } + @Override public void rollback() { composedWorldStateTransaction.rollback(); trieLogStorageTransaction.rollback(); } } - - @Override - public synchronized void close() throws Exception { - // when the storage clears, close - shouldClose.set(true); - tryClose(); - } - - public synchronized long subscribe(final BonsaiStorageSubscriber sub) { - if (isClosed.get()) { - throw new RuntimeException("Storage is marked to close or has already closed"); - } - return subscribers.subscribe(sub); - } - - public synchronized void unSubscribe(final long id) { - subscribers.unsubscribe(id); - try { - tryClose(); - } catch (Exception e) { - LOG.atWarn() - .setMessage("exception while trying to close : {}") - .addArgument(e::getMessage) - .log(); - } - } - - protected synchronized void tryClose() throws Exception { - if (shouldClose.get() && subscribers.getSubscriberCount() < 1) { - doClose(); - } - } - - protected synchronized void doClose() throws Exception { - if (!isClosed.get()) { - // alert any subscribers we are closing: - subscribers.forEach(BonsaiStorageSubscriber::onCloseStorage); - - // close all of the KeyValueStorages: - composedWorldStateStorage.close(); - trieLogStorage.close(); - - // set storage closed - isClosed.set(true); - } - } - - public interface BonsaiStorageSubscriber { - default void onClearStorage() {} - - default void onClearFlatDatabaseStorage() {} - - default void onClearTrieLog() {} - - default void onCloseStorage() {} - } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateLayerStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateLayerStorage.java similarity index 79% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateLayerStorage.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateLayerStorage.java index 92d4f54fee3..045376f6135 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateLayerStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateLayerStorage.java @@ -13,21 +13,22 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedLayeredWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.SnappedKeyValueStorage; import org.hyperledger.besu.services.kvstore.LayeredKeyValueStorage; public class BonsaiWorldStateLayerStorage extends BonsaiSnapshotWorldStateKeyValueStorage - implements BonsaiStorageSubscriber { + implements DiffBasedLayeredWorldStateKeyValueStorage, StorageSubscriber { public BonsaiWorldStateLayerStorage(final BonsaiWorldStateKeyValueStorage parent) { this( - new LayeredKeyValueStorage(parent.composedWorldStateStorage), - parent.trieLogStorage, + new LayeredKeyValueStorage(parent.getComposedWorldStateStorage()), + parent.getTrieLogStorage(), parent); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FullFlatDbStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/FullFlatDbStrategy.java similarity index 93% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FullFlatDbStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/FullFlatDbStrategy.java index 83f7f030c73..f8cbf2c3d2b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FullFlatDbStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/FullFlatDbStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; @@ -21,6 +21,8 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.trie.NodeLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.CodeStorageStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.Counter; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/PartialFlatDbStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/PartialFlatDbStrategy.java similarity index 96% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/PartialFlatDbStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/PartialFlatDbStrategy.java index cfe18dbe2ad..9e693d63e46 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/PartialFlatDbStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/PartialFlatDbStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; @@ -21,6 +21,8 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.trie.NodeLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.CodeStorageStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory; import org.hyperledger.besu.metrics.BesuMetricCategory; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryImpl.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryImpl.java similarity index 91% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryImpl.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryImpl.java index 2915a671820..6ebda666dbb 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryImpl.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryImpl.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog; import org.hyperledger.besu.datatypes.AccountValue; import org.hyperledger.besu.datatypes.Address; @@ -23,7 +23,8 @@ import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.rlp.RLPOutput; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.plugin.data.BlockHeader; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; @@ -150,7 +151,7 @@ public static TrieLogLayer readFrom(final RLPInput input) { final TrieLogLayer newLayer = new TrieLogLayer(); input.enterList(); - newLayer.blockHash = Hash.wrap(input.readBytes32()); + newLayer.setBlockHash(Hash.wrap(input.readBytes32())); while (!input.isEndOfCurrentList()) { input.enterList(); @@ -164,7 +165,9 @@ public static TrieLogLayer readFrom(final RLPInput input) { final StateTrieAccountValue newValue = nullOrValue(input, StateTrieAccountValue::readFrom); final boolean isCleared = getOptionalIsCleared(input); input.leaveList(); - newLayer.accounts.put(address, new BonsaiValue<>(oldValue, newValue, isCleared)); + newLayer + .getAccountChanges() + .put(address, new DiffBasedValue<>(oldValue, newValue, isCleared)); } if (input.nextIsNull()) { @@ -175,13 +178,13 @@ public static TrieLogLayer readFrom(final RLPInput input) { final Bytes newCode = nullOrValue(input, RLPInput::readBytes); final boolean isCleared = getOptionalIsCleared(input); input.leaveList(); - newLayer.code.put(address, new BonsaiValue<>(oldCode, newCode, isCleared)); + newLayer.getCodeChanges().put(address, new DiffBasedValue<>(oldCode, newCode, isCleared)); } if (input.nextIsNull()) { input.skipNext(); } else { - final Map> storageChanges = new TreeMap<>(); + final Map> storageChanges = new TreeMap<>(); input.enterList(); while (!input.isEndOfCurrentList()) { input.enterList(); @@ -190,11 +193,11 @@ public static TrieLogLayer readFrom(final RLPInput input) { final UInt256 oldValue = nullOrValue(input, RLPInput::readUInt256Scalar); final UInt256 newValue = nullOrValue(input, RLPInput::readUInt256Scalar); final boolean isCleared = getOptionalIsCleared(input); - storageChanges.put(storageSlotKey, new BonsaiValue<>(oldValue, newValue, isCleared)); + storageChanges.put(storageSlotKey, new DiffBasedValue<>(oldValue, newValue, isCleared)); input.leaveList(); } input.leaveList(); - newLayer.storage.put(address, storageChanges); + newLayer.getStorageChanges().put(address, storageChanges); } // TODO add trie nodes diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java similarity index 59% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java index 9e5472f5706..3b3af540d7b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldState.java @@ -14,39 +14,33 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai.worldview; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; -import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; -import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; -import static org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldView.encodeTrieValue; +import static org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView.encodeTrieValue; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; import org.hyperledger.besu.ethereum.trie.NodeLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateLayerStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator.StorageConsumingMap; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.StorageConsumingMap; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.internal.EvmConfiguration; -import org.hyperledger.besu.evm.worldstate.WorldUpdater; -import org.hyperledger.besu.plugin.services.exception.StorageException; -import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; @@ -63,24 +57,10 @@ import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.rlp.RLP; import org.apache.tuweni.units.bigints.UInt256; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class BonsaiWorldState - implements MutableWorldState, BonsaiWorldView, BonsaiStorageSubscriber { +public class BonsaiWorldState extends DiffBasedWorldState { - private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldState.class); - - protected BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage; - - protected final CachedMerkleTrieLoader cachedMerkleTrieLoader; - protected final CachedWorldStorageManager cachedWorldStorageManager; - protected final TrieLogManager trieLogManager; - private BonsaiWorldStateUpdateAccumulator accumulator; - - protected Hash worldStateRootHash; - Hash worldStateBlockHash; - private boolean isFrozen; + protected final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader; public BonsaiWorldState( final BonsaiWorldStateProvider archive, @@ -96,90 +76,39 @@ public BonsaiWorldState( public BonsaiWorldState( final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, - final CachedMerkleTrieLoader cachedMerkleTrieLoader, - final CachedWorldStorageManager cachedWorldStorageManager, + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader, + final DiffBasedCachedWorldStorageManager cachedWorldStorageManager, final TrieLogManager trieLogManager, final EvmConfiguration evmConfiguration) { - this.worldStateKeyValueStorage = worldStateKeyValueStorage; - this.worldStateRootHash = - Hash.wrap( - Bytes32.wrap( - worldStateKeyValueStorage.getWorldStateRootHash().orElse(getEmptyTrieHash()))); - this.worldStateBlockHash = - Hash.wrap( - Bytes32.wrap(worldStateKeyValueStorage.getWorldStateBlockHash().orElse(Hash.ZERO))); - this.accumulator = + super(worldStateKeyValueStorage, cachedWorldStorageManager, trieLogManager); + this.bonsaiCachedMerkleTrieLoader = bonsaiCachedMerkleTrieLoader; + this.setAccumulator( new BonsaiWorldStateUpdateAccumulator( this, (addr, value) -> - cachedMerkleTrieLoader.preLoadAccount( - getWorldStateStorage(), worldStateRootHash, addr), + bonsaiCachedMerkleTrieLoader.preLoadAccount( + worldStateKeyValueStorage, worldStateRootHash, addr), (addr, value) -> - cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value), - evmConfiguration); - this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; - this.cachedWorldStorageManager = cachedWorldStorageManager; - this.trieLogManager = trieLogManager; - } - - /** - * Override the accumulator solves the chicken-egg problem of needing a worldstate reference - * (this) when construction the Accumulator. - * - * @param accumulator accumulator to use. - */ - public void setAccumulator(final BonsaiWorldStateUpdateAccumulator accumulator) { - this.accumulator = accumulator; - } - - /** - * Returns the world state block hash of this world state - * - * @return the world state block hash. - */ - public Hash getWorldStateBlockHash() { - return worldStateBlockHash; - } - - /** - * Returns the world state root hash of this world state - * - * @return the world state root hash. - */ - public Hash getWorldStateRootHash() { - return worldStateRootHash; + bonsaiCachedMerkleTrieLoader.preLoadStorageSlot( + getWorldStateStorage(), addr, value), + evmConfiguration)); } @Override - public boolean isPersisted() { - return isPersisted(worldStateKeyValueStorage); - } - - private boolean isPersisted(final WorldStateKeyValueStorage worldStateKeyValueStorage) { - return !(worldStateKeyValueStorage instanceof BonsaiSnapshotWorldStateKeyValueStorage); - } - - @Override - public Optional getCode(@Nonnull final Address address, final Hash codeHash) { - return worldStateKeyValueStorage.getCode(codeHash, address.addressHash()); - } - - /** - * Reset the worldState to this block header - * - * @param blockHeader block to use - */ - public void resetWorldStateTo(final BlockHeader blockHeader) { - worldStateBlockHash = blockHeader.getBlockHash(); - worldStateRootHash = blockHeader.getStateRoot(); + public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { + return (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage; } @Override - public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { - return worldStateKeyValueStorage; + protected Hash calculateRootHash( + final Optional maybeStateUpdater, + final DiffBasedWorldStateUpdateAccumulator worldStateUpdater) { + return internalCalculateRootHash( + maybeStateUpdater.map(BonsaiWorldStateKeyValueStorage.Updater.class::cast), + (BonsaiWorldStateUpdateAccumulator) worldStateUpdater); } - private Hash calculateRootHash( + private Hash internalCalculateRootHash( final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater) { @@ -187,7 +116,7 @@ private Hash calculateRootHash( // This must be done before updating the accounts so // that we can get the storage state hash - Stream>>> + Stream>>> storageStream = worldStateUpdater.getStorageToUpdate().entrySet().stream(); if (maybeStateUpdater.isEmpty()) { storageStream = @@ -205,8 +134,8 @@ private Hash calculateRootHash( final StoredMerklePatriciaTrie accountTrie = createTrie( (location, hash) -> - cachedMerkleTrieLoader.getAccountStateTrieNode( - worldStateKeyValueStorage, location, hash), + bonsaiCachedMerkleTrieLoader.getAccountStateTrieNode( + getWorldStateStorage(), location, hash), worldStateRootHash); // for manicured tries and composting, collect branches here (not implemented) @@ -231,10 +160,10 @@ private void updateTheAccounts( final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater, final StoredMerklePatriciaTrie accountTrie) { - for (final Map.Entry> accountUpdate : + for (final Map.Entry> accountUpdate : worldStateUpdater.getAccountsToUpdate().entrySet()) { final Bytes accountKey = accountUpdate.getKey(); - final BonsaiValue bonsaiValue = accountUpdate.getValue(); + final DiffBasedValue bonsaiValue = accountUpdate.getValue(); final BonsaiAccount updatedAccount = bonsaiValue.getUpdated(); try { if (updatedAccount == null) { @@ -264,7 +193,7 @@ protected void updateCode( final BonsaiWorldStateUpdateAccumulator worldStateUpdater) { maybeStateUpdater.ifPresent( bonsaiUpdater -> { - for (final Map.Entry> codeUpdate : + for (final Map.Entry> codeUpdate : worldStateUpdater.getCodeToUpdate().entrySet()) { final Bytes updatedCode = codeUpdate.getValue().getUpdated(); final Hash accountHash = codeUpdate.getKey().addressHash(); @@ -294,12 +223,12 @@ private boolean codeIsEmpty(final Bytes value) { private void updateAccountStorageState( final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater, - final Map.Entry>> + final Map.Entry>> storageAccountUpdate) { final Address updatedAddress = storageAccountUpdate.getKey(); final Hash updatedAddressHash = updatedAddress.addressHash(); if (worldStateUpdater.getAccountsToUpdate().containsKey(updatedAddress)) { - final BonsaiValue accountValue = + final DiffBasedValue accountValue = worldStateUpdater.getAccountsToUpdate().get(updatedAddress); final BonsaiAccount accountOriginal = accountValue.getPrior(); final Hash storageRoot = @@ -310,12 +239,12 @@ private void updateAccountStorageState( final StoredMerklePatriciaTrie storageTrie = createTrie( (location, key) -> - cachedMerkleTrieLoader.getAccountStorageTrieNode( - worldStateKeyValueStorage, updatedAddressHash, location, key), + bonsaiCachedMerkleTrieLoader.getAccountStorageTrieNode( + getWorldStateStorage(), updatedAddressHash, location, key), storageRoot); // for manicured tries and composting, collect branches here (not implemented) - for (final Map.Entry> storageUpdate : + for (final Map.Entry> storageUpdate : storageAccountUpdate.getValue().entrySet()) { final Hash slotHash = storageUpdate.getKey().getSlotHash(); final UInt256 updatedStorage = storageUpdate.getValue().getUpdated(); @@ -360,11 +289,10 @@ private void updateAccountStorageState( private void clearStorage( final Optional maybeStateUpdater, final BonsaiWorldStateUpdateAccumulator worldStateUpdater) { - for (final Address address : worldStateUpdater.getStorageToClear()) { // because we are clearing persisted values we need the account root as persisted final BonsaiAccount oldAccount = - worldStateKeyValueStorage + getWorldStateStorage() .getAccount(address.addressHash()) .map(bytes -> BonsaiAccount.fromRLP(BonsaiWorldState.this, address, bytes, true)) .orElse(null); @@ -379,7 +307,7 @@ private void clearStorage( (location, key) -> getStorageTrieNode(addressHash, location, key), oldAccount.getStorageRoot()); try { - final StorageConsumingMap> storageToDelete = + final StorageConsumingMap> storageToDelete = worldStateUpdater.getStorageToUpdate().get(address); Map entriesToDelete = storageTrie.entriesFrom(Bytes32.ZERO, 256); while (!entriesToDelete.isEmpty()) { @@ -394,7 +322,7 @@ private void clearStorage( address.addressHash(), storageSlotKey.getSlotHash())); storageToDelete .computeIfAbsent( - storageSlotKey, key -> new BonsaiValue<>(slotValue, null, true)) + storageSlotKey, key -> new DiffBasedValue<>(slotValue, null, true)) .setPrior(slotValue); }); entriesToDelete.keySet().forEach(storageTrie::remove); @@ -412,138 +340,6 @@ private void clearStorage( } } - @Override - public void persist(final BlockHeader blockHeader) { - final Optional maybeBlockHeader = Optional.ofNullable(blockHeader); - LOG.atDebug() - .setMessage("Persist world state for block {}") - .addArgument(maybeBlockHeader) - .log(); - - final BonsaiWorldStateUpdateAccumulator localCopy = accumulator.copy(); - - boolean success = false; - - final BonsaiWorldStateKeyValueStorage.Updater stateUpdater = - worldStateKeyValueStorage.updater(); - Runnable saveTrieLog = () -> {}; - - try { - final Hash newWorldStateRootHash = - calculateRootHash(isFrozen ? Optional.empty() : Optional.of(stateUpdater), accumulator); - // if we are persisted with a block header, and the prior state is the parent - // then persist the TrieLog for that transition. - // If specified but not a direct descendant simply store the new block hash. - if (blockHeader != null) { - verifyWorldStateRoot(newWorldStateRootHash, blockHeader); - saveTrieLog = - () -> { - trieLogManager.saveTrieLog(localCopy, newWorldStateRootHash, blockHeader, this); - // not save a frozen state in the cache - if (!isFrozen) { - cachedWorldStorageManager.addCachedLayer(blockHeader, newWorldStateRootHash, this); - } - }; - - stateUpdater - .getWorldStateTransaction() - .put(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY, blockHeader.getHash().toArrayUnsafe()); - worldStateBlockHash = blockHeader.getHash(); - } else { - stateUpdater.getWorldStateTransaction().remove(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY); - worldStateBlockHash = null; - } - - stateUpdater - .getWorldStateTransaction() - .put(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY, newWorldStateRootHash.toArrayUnsafe()); - worldStateRootHash = newWorldStateRootHash; - success = true; - } finally { - if (success) { - stateUpdater.commit(); - accumulator.reset(); - saveTrieLog.run(); - } else { - stateUpdater.rollback(); - accumulator.reset(); - } - } - } - - protected void verifyWorldStateRoot(final Hash calculatedStateRoot, final BlockHeader header) { - if (!calculatedStateRoot.equals(header.getStateRoot())) { - throw new RuntimeException( - "World State Root does not match expected value, header " - + header.getStateRoot().toHexString() - + " calculated " - + calculatedStateRoot.toHexString()); - } - } - - @Override - public WorldUpdater updater() { - return accumulator; - } - - @Override - public Hash rootHash() { - if (isFrozen && accumulator.isAccumulatorStateChanged()) { - worldStateRootHash = calculateRootHash(Optional.empty(), accumulator.copy()); - accumulator.resetAccumulatorStateChanged(); - } - return Hash.wrap(worldStateRootHash); - } - - static final KeyValueStorageTransaction noOpTx = - new KeyValueStorageTransaction() { - - @Override - public void put(final byte[] key, final byte[] value) { - // no-op - } - - @Override - public void remove(final byte[] key) { - // no-op - } - - @Override - public void commit() throws StorageException { - // no-op - } - - @Override - public void rollback() { - // no-op - } - }; - - static final SegmentedKeyValueStorageTransaction noOpSegmentedTx = - new SegmentedKeyValueStorageTransaction() { - - @Override - public void put( - final SegmentIdentifier segmentIdentifier, final byte[] key, final byte[] value) { - // no-op - } - - @Override - public void remove(final SegmentIdentifier segmentIdentifier, final byte[] key) { - // no-op - } - - @Override - public void commit() throws StorageException { - // no-op - } - - @Override - public void rollback() { - // no-op - } - }; - @Override public Hash frontierRootHash() { return calculateRootHash( @@ -553,25 +349,28 @@ public Hash frontierRootHash() { accumulator.copy()); } - public Hash blockHash() { - return worldStateBlockHash; - } - @Override - public Stream streamAccounts(final Bytes32 startKeyHash, final int limit) { - throw new RuntimeException("Bonsai Tries do not provide account streaming."); + public MutableWorldState freeze() { + this.isFrozen = true; + this.worldStateKeyValueStorage = new BonsaiWorldStateLayerStorage(getWorldStateStorage()); + return this; } @Override public Account get(final Address address) { - return worldStateKeyValueStorage + return getWorldStateStorage() .getAccount(address.addressHash()) .map(bytes -> BonsaiAccount.fromRLP(accumulator, address, bytes, true)) .orElse(null); } + @Override + public Optional getCode(@Nonnull final Address address, final Hash codeHash) { + return getWorldStateStorage().getCode(codeHash, address.addressHash()); + } + protected Optional getAccountStateTrieNode(final Bytes location, final Bytes32 nodeHash) { - return worldStateKeyValueStorage.getAccountStateTrieNode(location, nodeHash); + return getWorldStateStorage().getAccountStateTrieNode(location, nodeHash); } private void writeTrieNode( @@ -584,7 +383,7 @@ private void writeTrieNode( protected Optional getStorageTrieNode( final Hash accountHash, final Bytes location, final Bytes32 nodeHash) { - return worldStateKeyValueStorage.getAccountStorageTrieNode(accountHash, location, nodeHash); + return getWorldStateStorage().getAccountStorageTrieNode(accountHash, location, nodeHash); } private void writeStorageTrieNode( @@ -605,7 +404,7 @@ public UInt256 getStorageValue(final Address address, final UInt256 storageKey) @Override public Optional getStorageValueByStorageSlotKey( final Address address, final StorageSlotKey storageSlotKey) { - return worldStateKeyValueStorage + return getWorldStateStorage() .getStorageValueByStorageSlotKey(address.addressHash(), storageSlotKey) .map(UInt256::fromBytes); } @@ -614,7 +413,7 @@ public Optional getStorageValueByStorageSlotKey( final Supplier> storageRootSupplier, final Address address, final StorageSlotKey storageSlotKey) { - return worldStateKeyValueStorage + return getWorldStateStorage() .getStorageValueByStorageSlotKey(storageRootSupplier, address.addressHash(), storageSlotKey) .map(UInt256::fromBytes); } @@ -632,50 +431,18 @@ public Map getAllAccountStorage(final Address address, final Has return storageTrie.entriesFrom(Bytes32.ZERO, Integer.MAX_VALUE); } - @Override - public MutableWorldState freeze() { - this.isFrozen = true; - this.worldStateKeyValueStorage = new BonsaiWorldStateLayerStorage(worldStateKeyValueStorage); - return this; - } - private StoredMerklePatriciaTrie createTrie( final NodeLoader nodeLoader, final Bytes32 rootHash) { return new StoredMerklePatriciaTrie<>( nodeLoader, rootHash, Function.identity(), Function.identity()); } - @Override - public void close() { - try { - if (!isPersisted()) { - this.worldStateKeyValueStorage.close(); - if (isFrozen) { - closeFrozenStorage(); - } - } - } catch (Exception e) { - // no op - } - } - - private void closeFrozenStorage() { - try { - final BonsaiWorldStateLayerStorage worldStateLayerStorage = - (BonsaiWorldStateLayerStorage) worldStateKeyValueStorage; - if (!isPersisted(worldStateLayerStorage.getParentWorldStateStorage())) { - worldStateLayerStorage.getParentWorldStateStorage().close(); - } - } catch (Exception e) { - // no op - } - } - protected Hash hashAndSavePreImage(final Bytes value) { // by default do not save has preImages return Hash.hash(value); } + @Override protected Hash getEmptyTrieHash() { return Hash.EMPTY_TRIE_HASH; } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java new file mode 100644 index 00000000000..ff2e696e163 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java @@ -0,0 +1,98 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview; + +import org.hyperledger.besu.datatypes.AccountValue; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.Consumer; +import org.hyperledger.besu.evm.internal.EvmConfiguration; +import org.hyperledger.besu.evm.worldstate.UpdateTrackingAccount; + +public class BonsaiWorldStateUpdateAccumulator + extends DiffBasedWorldStateUpdateAccumulator { + public BonsaiWorldStateUpdateAccumulator( + final DiffBasedWorldView world, + final Consumer> accountPreloader, + final Consumer storagePreloader, + final EvmConfiguration evmConfiguration) { + super(world, accountPreloader, storagePreloader, evmConfiguration); + } + + @Override + public DiffBasedWorldStateUpdateAccumulator copy() { + final BonsaiWorldStateUpdateAccumulator copy = + new BonsaiWorldStateUpdateAccumulator( + wrappedWorldView(), + getAccountPreloader(), + getStoragePreloader(), + getEvmConfiguration()); + copy.cloneFromUpdater(this); + return copy; + } + + @Override + protected BonsaiAccount copyAccount(final BonsaiAccount account) { + return new BonsaiAccount(account); + } + + @Override + protected BonsaiAccount copyAccount( + final BonsaiAccount toCopy, final DiffBasedWorldView context, final boolean mutable) { + return new BonsaiAccount(toCopy, context, mutable); + } + + @Override + protected BonsaiAccount createAccount( + final DiffBasedWorldView context, + final Address address, + final AccountValue stateTrieAccount, + final boolean mutable) { + return new BonsaiAccount(context, address, stateTrieAccount, mutable); + } + + @Override + protected BonsaiAccount createAccount( + final DiffBasedWorldView context, + final Address address, + final Hash addressHash, + final long nonce, + final Wei balance, + final Hash storageRoot, + final Hash codeHash, + final boolean mutable) { + return new BonsaiAccount( + context, address, addressHash, nonce, balance, storageRoot, codeHash, mutable); + } + + @Override + protected BonsaiAccount createAccount( + final DiffBasedWorldView context, final UpdateTrackingAccount tracked) { + return new BonsaiAccount(context, tracked); + } + + @Override + protected void assertCloseEnoughForDiffing( + final BonsaiAccount source, final AccountValue account, final String context) { + BonsaiAccount.assertCloseEnoughForDiffing(source, account, context); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java new file mode 100644 index 00000000000..044deb45a01 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java @@ -0,0 +1,206 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.ethereum.trie.diffbased.common; + +import org.hyperledger.besu.datatypes.AccountValue; +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; +import org.hyperledger.besu.evm.ModificationNotAllowedException; +import org.hyperledger.besu.evm.account.MutableAccount; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; + +public abstract class DiffBasedAccount implements MutableAccount, AccountValue { + protected final DiffBasedWorldView context; + protected boolean immutable; + protected final Address address; + protected final Hash addressHash; + protected Hash codeHash; + protected long nonce; + protected Wei balance; + protected Bytes code; + + protected final Map updatedStorage = new HashMap<>(); + + public DiffBasedAccount( + final DiffBasedWorldView context, + final Address address, + final Hash addressHash, + final long nonce, + final Wei balance, + final Hash codeHash, + final boolean mutable) { + this.context = context; + this.address = address; + this.addressHash = addressHash; + this.nonce = nonce; + this.balance = balance; + this.codeHash = codeHash; + + this.immutable = !mutable; + } + + public DiffBasedAccount( + final DiffBasedWorldView context, + final Address address, + final AccountValue stateTrieAccount, + final boolean mutable) { + this( + context, + address, + address.addressHash(), + stateTrieAccount.getNonce(), + stateTrieAccount.getBalance(), + stateTrieAccount.getCodeHash(), + mutable); + } + + public DiffBasedAccount( + final DiffBasedAccount toCopy, final DiffBasedWorldView context, final boolean mutable) { + this.context = context; + this.address = toCopy.address; + this.addressHash = toCopy.addressHash; + this.nonce = toCopy.nonce; + this.balance = toCopy.balance; + this.codeHash = toCopy.codeHash; + this.code = toCopy.code; + updatedStorage.putAll(toCopy.updatedStorage); + + this.immutable = !mutable; + } + + @Override + public Address getAddress() { + return address; + } + + @Override + public Hash getAddressHash() { + return addressHash; + } + + @Override + public long getNonce() { + return nonce; + } + + @Override + public void setNonce(final long value) { + if (immutable) { + throw new ModificationNotAllowedException(); + } + nonce = value; + } + + @Override + public Wei getBalance() { + return balance; + } + + @Override + public void setBalance(final Wei value) { + if (immutable) { + throw new ModificationNotAllowedException(); + } + balance = value; + } + + @Override + public Bytes getCode() { + if (code == null) { + code = context.getCode(address, codeHash).orElse(Bytes.EMPTY); + } + return code; + } + + @Override + public void setCode(final Bytes code) { + if (immutable) { + throw new ModificationNotAllowedException(); + } + this.code = code; + if (code == null || code.isEmpty()) { + this.codeHash = Hash.EMPTY; + } else { + this.codeHash = Hash.hash(code); + } + } + + @Override + public Hash getCodeHash() { + return codeHash; + } + + @Override + public UInt256 getStorageValue(final UInt256 key) { + return context.getStorageValue(address, key); + } + + @Override + public UInt256 getOriginalStorageValue(final UInt256 key) { + return context.getPriorStorageValue(address, key); + } + + public Bytes serializeAccount() { + final BytesValueRLPOutput out = new BytesValueRLPOutput(); + writeTo(out); + return out.encoded(); + } + + @Override + public void setStorageValue(final UInt256 key, final UInt256 value) { + if (immutable) { + throw new ModificationNotAllowedException(); + } + updatedStorage.put(key, value); + } + + @Override + public void clearStorage() { + updatedStorage.clear(); + } + + @Override + public Map getUpdatedStorage() { + return updatedStorage; + } + + @Override + public void becomeImmutable() { + immutable = true; + } + + @Override + public String toString() { + return "AccountState{" + + "address=" + + address + + ", nonce=" + + nonce + + ", balance=" + + balance + + ", codeHash=" + + codeHash + + '}'; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedValue.java similarity index 80% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedValue.java index c99da3fe24b..bd9c3e3f514 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiValue.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedValue.java @@ -1,5 +1,5 @@ /* - * Copyright ConsenSys AG. + * Copyright Hyperledger Besu Contributors. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -11,38 +11,37 @@ * specific language governing permissions and limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.common; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; -public class BonsaiValue implements TrieLog.LogTuple { +public class DiffBasedValue implements TrieLog.LogTuple { private T prior; private T updated; private boolean lastStepCleared; private boolean clearedAtLeastOnce; - public BonsaiValue(final T prior, final T updated) { + public DiffBasedValue(final T prior, final T updated) { this.prior = prior; this.updated = updated; this.lastStepCleared = false; this.clearedAtLeastOnce = false; } - public BonsaiValue(final T prior, final T updated, final boolean lastStepCleared) { + public DiffBasedValue(final T prior, final T updated, final boolean lastStepCleared) { this.prior = prior; this.updated = updated; this.lastStepCleared = lastStepCleared; this.clearedAtLeastOnce = lastStepCleared; } - public BonsaiValue( + public DiffBasedValue( final T prior, final T updated, final boolean lastStepCleared, @@ -63,12 +62,12 @@ public T getUpdated() { return updated; } - public BonsaiValue setPrior(final T prior) { + public DiffBasedValue setPrior(final T prior) { this.prior = prior; return this; } - public BonsaiValue setUpdated(final T updated) { + public DiffBasedValue setUpdated(final T updated) { this.lastStepCleared = updated == null; if (lastStepCleared) { this.clearedAtLeastOnce = true; @@ -94,7 +93,7 @@ public boolean isClearedAtLeastOnce() { @Override public String toString() { - return "BonsaiValue{" + return "DiffBasedValue{" + "prior=" + prior + ", updated=" @@ -112,7 +111,7 @@ public boolean equals(final Object o) { if (o == null || getClass() != o.getClass()) { return false; } - BonsaiValue that = (BonsaiValue) o; + DiffBasedValue that = (DiffBasedValue) o; return new EqualsBuilder() .append(lastStepCleared, that.lastStepCleared) .append(prior, that.prior) @@ -129,7 +128,7 @@ public int hashCode() { .toHashCode(); } - public BonsaiValue copy() { - return new BonsaiValue(prior, updated, lastStepCleared, clearedAtLeastOnce); + public DiffBasedValue copy() { + return new DiffBasedValue(prior, updated, lastStepCleared, clearedAtLeastOnce); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedWorldStateProvider.java similarity index 62% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedWorldStateProvider.java index 0230a2258bc..18bf333828b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedWorldStateProvider.java @@ -14,9 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; - -import static org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager.RETAINED_LAYERS; +package org.hyperledger.besu.ethereum.trie.diffbased.common; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; @@ -25,91 +23,75 @@ import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.proof.WorldStateProof; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; -import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; -import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; -import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldState; import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.function.Function; -import com.google.common.annotations.VisibleForTesting; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.units.bigints.UInt256; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class BonsaiWorldStateProvider implements WorldStateArchive { +public abstract class DiffBasedWorldStateProvider implements WorldStateArchive { + + private static final Logger LOG = LoggerFactory.getLogger(DiffBasedWorldStateProvider.class); - private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateProvider.class); + protected final Blockchain blockchain; - private final Blockchain blockchain; + protected final TrieLogManager trieLogManager; + protected DiffBasedCachedWorldStorageManager cachedWorldStorageManager; + protected DiffBasedWorldState persistedState; - private final CachedWorldStorageManager cachedWorldStorageManager; - private final TrieLogManager trieLogManager; - private final BonsaiWorldState persistedState; - private final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage; - private final CachedMerkleTrieLoader cachedMerkleTrieLoader; + protected final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage; - public BonsaiWorldStateProvider( - final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, + public DiffBasedWorldStateProvider( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, final Blockchain blockchain, final Optional maxLayersToLoad, - final CachedMerkleTrieLoader cachedMerkleTrieLoader, - final BesuContext pluginContext, - final EvmConfiguration evmConfiguration) { + final BesuContext pluginContext) { this.worldStateKeyValueStorage = worldStateKeyValueStorage; - this.cachedWorldStorageManager = new CachedWorldStorageManager(this, worldStateKeyValueStorage); - // TODO: de-dup constructors this.trieLogManager = new TrieLogManager( blockchain, worldStateKeyValueStorage, - maxLayersToLoad.orElse(RETAINED_LAYERS), + maxLayersToLoad.orElse(DiffBasedCachedWorldStorageManager.RETAINED_LAYERS), pluginContext); this.blockchain = blockchain; - this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; - this.persistedState = new BonsaiWorldState(this, worldStateKeyValueStorage, evmConfiguration); - blockchain - .getBlockHeader(persistedState.getWorldStateBlockHash()) - .ifPresent( - blockHeader -> - this.cachedWorldStorageManager.addCachedLayer( - blockHeader, persistedState.getWorldStateRootHash(), persistedState)); } - @VisibleForTesting - BonsaiWorldStateProvider( - final CachedWorldStorageManager cachedWorldStorageManager, - final TrieLogManager trieLogManager, - final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, + public DiffBasedWorldStateProvider( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, final Blockchain blockchain, - final CachedMerkleTrieLoader cachedMerkleTrieLoader, - final EvmConfiguration evmConfiguration) { - this.cachedWorldStorageManager = cachedWorldStorageManager; + final TrieLogManager trieLogManager) { + + this.worldStateKeyValueStorage = worldStateKeyValueStorage; + // TODO: de-dup constructors this.trieLogManager = trieLogManager; this.blockchain = blockchain; - this.worldStateKeyValueStorage = worldStateKeyValueStorage; - this.persistedState = new BonsaiWorldState(this, worldStateKeyValueStorage, evmConfiguration); - this.cachedMerkleTrieLoader = cachedMerkleTrieLoader; + } + + protected void provideCachedWorldStorageManager( + final DiffBasedCachedWorldStorageManager cachedWorldStorageManager) { + this.cachedWorldStorageManager = cachedWorldStorageManager; + } + + protected void loadPersistedState(final DiffBasedWorldState persistedState) { + this.persistedState = persistedState; blockchain .getBlockHeader(persistedState.getWorldStateBlockHash()) .ifPresent( @@ -158,9 +140,7 @@ public Optional getMutable( .getWorldState(blockHeader.getHash()) .or(() -> cachedWorldStorageManager.getNearestWorldState(blockHeader)) .or(() -> cachedWorldStorageManager.getHeadWorldState(blockchain::getBlockHeader)) - .flatMap( - bonsaiWorldState -> - rollMutableStateToBlockHash(bonsaiWorldState, blockHeader.getHash())) + .flatMap(worldState -> rollMutableStateToBlockHash(worldState, blockHeader.getHash())) .map(MutableWorldState::freeze); } } @@ -172,7 +152,7 @@ public synchronized Optional getMutable( } Optional rollMutableStateToBlockHash( - final BonsaiWorldState mutableState, final Hash blockHash) { + final DiffBasedWorldState mutableState, final Hash blockHash) { if (blockHash.equals(mutableState.blockHash())) { return Optional.of(mutableState); } else { @@ -221,19 +201,19 @@ Optional rollMutableStateToBlockHash( } // attempt the state rolling - final BonsaiWorldStateUpdateAccumulator bonsaiUpdater = - (BonsaiWorldStateUpdateAccumulator) mutableState.updater(); + final DiffBasedWorldStateUpdateAccumulator diffBasedUpdater = + (DiffBasedWorldStateUpdateAccumulator) mutableState.updater(); try { for (final TrieLog rollBack : rollBacks) { LOG.debug("Attempting Rollback of {}", rollBack.getBlockHash()); - bonsaiUpdater.rollBack(rollBack); + diffBasedUpdater.rollBack(rollBack); } for (int i = rollForwards.size() - 1; i >= 0; i--) { final var forward = rollForwards.get(i); LOG.debug("Attempting Rollforward of {}", rollForwards.get(i).getBlockHash()); - bonsaiUpdater.rollForward(forward); + diffBasedUpdater.rollForward(forward); } - bonsaiUpdater.commit(); + diffBasedUpdater.commit(); mutableState.persist(blockchain.getBlockHeader(blockHash).get()); @@ -247,7 +227,7 @@ Optional rollMutableStateToBlockHash( throw re; } catch (final Exception e) { // if we fail we must clean up the updater - bonsaiUpdater.reset(); + diffBasedUpdater.reset(); LOG.debug( "State rolling failed on " + mutableState.getWorldStateStorage().getClass().getSimpleName() @@ -269,80 +249,16 @@ Optional rollMutableStateToBlockHash( } } - public CachedMerkleTrieLoader getCachedMerkleTrieLoader() { - return cachedMerkleTrieLoader; - } - @Override public MutableWorldState getMutable() { return persistedState; } - /** - * Prepares the state healing process for a given address and location. It prepares the state - * healing, including retrieving data from storage, identifying invalid slots or nodes, removing - * account and slot from the state trie, and committing the changes. Finally, it downgrades the - * world state storage to partial flat database mode. - */ - public void prepareStateHealing(final Address address, final Bytes location) { - final Set keysToDelete = new HashSet<>(); - final BonsaiWorldStateKeyValueStorage.Updater updater = worldStateKeyValueStorage.updater(); - final Hash accountHash = address.addressHash(); - final StoredMerklePatriciaTrie accountTrie = - new StoredMerklePatriciaTrie<>( - (l, h) -> { - final Optional node = worldStateKeyValueStorage.getAccountStateTrieNode(l, h); - if (node.isPresent()) { - keysToDelete.add(l); - } - return node; - }, - persistedState.getWorldStateRootHash(), - Function.identity(), - Function.identity()); - try { - accountTrie - .get(accountHash) - .map(RLP::input) - .map(StateTrieAccountValue::readFrom) - .ifPresent( - account -> { - final StoredMerklePatriciaTrie storageTrie = - new StoredMerklePatriciaTrie<>( - (l, h) -> { - Optional node = - worldStateKeyValueStorage.getAccountStorageTrieNode( - accountHash, l, h); - if (node.isPresent()) { - keysToDelete.add(Bytes.concatenate(accountHash, l)); - } - return node; - }, - account.getStorageRoot(), - Function.identity(), - Function.identity()); - try { - storageTrie.getPath(location); - } catch (Exception eA) { - LOG.warn("Invalid slot found for account {} at location {}", address, location); - // ignore - } - }); - } catch (Exception eA) { - LOG.warn("Invalid node for account {} at location {}", address, location); - // ignore - } - keysToDelete.forEach(bytes -> updater.removeAccountStateTrieNode(bytes)); - updater.commit(); - - worldStateKeyValueStorage.downgradeToPartialFlatDbMode(); - } - public TrieLogManager getTrieLogManager() { return trieLogManager; } - public CachedWorldStorageManager getCachedWorldStorageManager() { + public DiffBasedCachedWorldStorageManager getCachedWorldStorageManager() { return cachedWorldStorageManager; } @@ -360,7 +276,8 @@ public Optional getAccountProof( final Address accountAddress, final List accountStorageKeys, final Function, ? extends Optional> mapper) { - try (BonsaiWorldState ws = (BonsaiWorldState) getMutable(blockHeader, false).orElse(null)) { + try (DiffBasedWorldState ws = + (DiffBasedWorldState) getMutable(blockHeader, false).orElse(null)) { if (ws != null) { final WorldStateProofProvider worldStateProofProvider = new WorldStateProofProvider( diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/StorageSubscriber.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/StorageSubscriber.java new file mode 100644 index 00000000000..fe16cf381e9 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/StorageSubscriber.java @@ -0,0 +1,25 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common; + +public interface StorageSubscriber { + default void onClearStorage() {} + + default void onClearFlatDatabaseStorage() {} + + default void onClearTrieLog() {} + + default void onCloseStorage() {} +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedWorldStorageManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldStorageManager.java similarity index 62% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedWorldStorageManager.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldStorageManager.java index 3944b72a732..287213aa57a 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedWorldStorageManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldStorageManager.java @@ -12,15 +12,15 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.cache; +package org.hyperledger.besu.ethereum.trie.diffbased.common.cache; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateLayerStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedLayeredWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; import org.hyperledger.besu.evm.internal.EvmConfiguration; import java.util.ArrayList; @@ -35,20 +35,20 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class CachedWorldStorageManager - implements BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber { +public abstract class DiffBasedCachedWorldStorageManager implements StorageSubscriber { public static final long RETAINED_LAYERS = 512; // at least 256 + typical rollbacks - private static final Logger LOG = LoggerFactory.getLogger(CachedWorldStorageManager.class); - private final BonsaiWorldStateProvider archive; + private static final Logger LOG = + LoggerFactory.getLogger(DiffBasedCachedWorldStorageManager.class); + private final DiffBasedWorldStateProvider archive; private final EvmConfiguration evmConfiguration; - private final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; - private final Map cachedWorldStatesByHash; + private final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage; + private final Map cachedWorldStatesByHash; - private CachedWorldStorageManager( - final BonsaiWorldStateProvider archive, - final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, - final Map cachedWorldStatesByHash, + private DiffBasedCachedWorldStorageManager( + final DiffBasedWorldStateProvider archive, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final Map cachedWorldStatesByHash, final EvmConfiguration evmConfiguration) { worldStateKeyValueStorage.subscribe(this); this.rootWorldStateStorage = worldStateKeyValueStorage; @@ -57,32 +57,32 @@ private CachedWorldStorageManager( this.evmConfiguration = evmConfiguration; } - public CachedWorldStorageManager( - final BonsaiWorldStateProvider archive, - final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage) { + public DiffBasedCachedWorldStorageManager( + final DiffBasedWorldStateProvider archive, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage) { this(archive, worldStateKeyValueStorage, new ConcurrentHashMap<>(), EvmConfiguration.DEFAULT); } public synchronized void addCachedLayer( final BlockHeader blockHeader, final Hash worldStateRootHash, - final BonsaiWorldState forWorldState) { - final Optional cachedBonsaiWorldView = + final DiffBasedWorldState forWorldState) { + final Optional cachedDiffBasedWorldView = Optional.ofNullable(this.cachedWorldStatesByHash.get(blockHeader.getBlockHash())); - if (cachedBonsaiWorldView.isPresent()) { + if (cachedDiffBasedWorldView.isPresent()) { // only replace if it is a layered storage if (forWorldState.isPersisted() - && cachedBonsaiWorldView.get().getWorldStateStorage() - instanceof BonsaiWorldStateLayerStorage) { + && cachedDiffBasedWorldView.get().getWorldStateStorage() + instanceof DiffBasedLayeredWorldStateKeyValueStorage) { LOG.atDebug() .setMessage("updating layered world state for block {}, state root hash {}") .addArgument(blockHeader::toLogString) .addArgument(worldStateRootHash::toShortHexString) .log(); - cachedBonsaiWorldView + cachedDiffBasedWorldView .get() .updateWorldStateStorage( - new BonsaiSnapshotWorldStateKeyValueStorage(forWorldState.getWorldStateStorage())); + createSnapshotKeyValueStorage(forWorldState.getWorldStateStorage())); } } else { LOG.atDebug() @@ -93,16 +93,16 @@ public synchronized void addCachedLayer( if (forWorldState.isPersisted()) { cachedWorldStatesByHash.put( blockHeader.getHash(), - new CachedBonsaiWorldView( - blockHeader, - new BonsaiSnapshotWorldStateKeyValueStorage(forWorldState.getWorldStateStorage()))); + new DiffBasedCachedWorldView( + blockHeader, createSnapshotKeyValueStorage(forWorldState.getWorldStateStorage()))); } else { // otherwise, add the layer to the cache cachedWorldStatesByHash.put( blockHeader.getHash(), - new CachedBonsaiWorldView( + new DiffBasedCachedWorldView( blockHeader, - ((BonsaiWorldStateLayerStorage) forWorldState.getWorldStateStorage()).clone())); + ((DiffBasedLayeredWorldStateKeyValueStorage) forWorldState.getWorldStateStorage()) + .clone())); } } scrubCachedLayers(blockHeader.getNumber()); @@ -122,15 +122,15 @@ private synchronized void scrubCachedLayers(final long newMaxHeight) { } } - public Optional getWorldState(final Hash blockHash) { + public Optional getWorldState(final Hash blockHash) { if (cachedWorldStatesByHash.containsKey(blockHash)) { // return a new worldstate using worldstate storage and an isolated copy of the updater return Optional.ofNullable(cachedWorldStatesByHash.get(blockHash)) .map( cached -> - new BonsaiWorldState( + createWorldState( archive, - new BonsaiWorldStateLayerStorage(cached.getWorldStateStorage()), + createLayeredKeyValueStorage(cached.getWorldStateStorage()), evmConfiguration)); } LOG.atDebug() @@ -141,7 +141,7 @@ public Optional getWorldState(final Hash blockHash) { return Optional.empty(); } - public Optional getNearestWorldState(final BlockHeader blockHeader) { + public Optional getNearestWorldState(final BlockHeader blockHeader) { LOG.atDebug() .setMessage("getting nearest worldstate for {}") .addArgument(blockHeader.toLogString()) @@ -149,7 +149,7 @@ public Optional getNearestWorldState(final BlockHeader blockHe return Optional.ofNullable( cachedWorldStatesByHash.get(blockHeader.getParentHash())) // search parent block - .map(CachedBonsaiWorldView::getWorldStateStorage) + .map(DiffBasedCachedWorldView::getWorldStateStorage) .or( () -> { // or else search the nearest state in the cache @@ -158,22 +158,22 @@ public Optional getNearestWorldState(final BlockHeader blockHe .addArgument(blockHeader.toLogString()) .log(); - final List cachedBonsaiWorldViews = + final List cachedDiffBasedWorldViews = new ArrayList<>(cachedWorldStatesByHash.values()); - return cachedBonsaiWorldViews.stream() + return cachedDiffBasedWorldViews.stream() .sorted( Comparator.comparingLong( view -> Math.abs(blockHeader.getNumber() - view.getBlockNumber()))) - .map(CachedBonsaiWorldView::getWorldStateStorage) + .map(DiffBasedCachedWorldView::getWorldStateStorage) .findFirst(); }) .map( storage -> - new BonsaiWorldState( // wrap the state in a layered worldstate - archive, new BonsaiWorldStateLayerStorage(storage), evmConfiguration)); + createWorldState( // wrap the state in a layered worldstate + archive, createLayeredKeyValueStorage(storage), evmConfiguration)); } - public Optional getHeadWorldState( + public Optional getHeadWorldState( final Function> hashBlockHeaderFunction) { LOG.atDebug().setMessage("getting head worldstate").log(); @@ -187,7 +187,7 @@ public Optional getHeadWorldState( addCachedLayer( blockHeader, blockHeader.getStateRoot(), - new BonsaiWorldState(archive, rootWorldStateStorage, evmConfiguration)); + createWorldState(archive, rootWorldStateStorage, evmConfiguration)); return getWorldState(blockHeader.getHash()); }); } @@ -219,4 +219,15 @@ public void onClearTrieLog() { public void onCloseStorage() { this.cachedWorldStatesByHash.clear(); } + + public abstract DiffBasedWorldState createWorldState( + final DiffBasedWorldStateProvider archive, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final EvmConfiguration evmConfiguration); + + public abstract DiffBasedWorldStateKeyValueStorage createLayeredKeyValueStorage( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage); + + public abstract DiffBasedWorldStateKeyValueStorage createSnapshotKeyValueStorage( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedBonsaiWorldView.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldView.java similarity index 71% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedBonsaiWorldView.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldView.java index 2b8fd69d114..951941f0ce1 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/cache/CachedBonsaiWorldView.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/cache/DiffBasedCachedWorldView.java @@ -13,30 +13,30 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.cache; +package org.hyperledger.besu.ethereum.trie.diffbased.common.cache; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class CachedBonsaiWorldView - implements BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber { - private BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage; +public class DiffBasedCachedWorldView implements StorageSubscriber { + private DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage; private final BlockHeader blockHeader; private long worldViewSubscriberId; - private static final Logger LOG = LoggerFactory.getLogger(CachedBonsaiWorldView.class); + private static final Logger LOG = LoggerFactory.getLogger(DiffBasedCachedWorldView.class); - public CachedBonsaiWorldView( - final BlockHeader blockHeader, final BonsaiWorldStateKeyValueStorage worldView) { + public DiffBasedCachedWorldView( + final BlockHeader blockHeader, final DiffBasedWorldStateKeyValueStorage worldView) { this.blockHeader = blockHeader; this.worldStateKeyValueStorage = worldView; this.worldViewSubscriberId = worldStateKeyValueStorage.subscribe(this); } - public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { + public DiffBasedWorldStateKeyValueStorage getWorldStateStorage() { return worldStateKeyValueStorage; } @@ -58,10 +58,10 @@ public synchronized void close() { } public synchronized void updateWorldStateStorage( - final BonsaiWorldStateKeyValueStorage newWorldStateStorage) { + final DiffBasedWorldStateKeyValueStorage newWorldStateStorage) { long newSubscriberId = newWorldStateStorage.subscribe(this); this.worldStateKeyValueStorage.unSubscribe(this.worldViewSubscriberId); - BonsaiWorldStateKeyValueStorage oldWorldStateStorage = this.worldStateKeyValueStorage; + final DiffBasedWorldStateKeyValueStorage oldWorldStateStorage = this.worldStateKeyValueStorage; this.worldStateKeyValueStorage = newWorldStateStorage; this.worldViewSubscriberId = newSubscriberId; try { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedLayeredWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedLayeredWorldStateKeyValueStorage.java new file mode 100644 index 00000000000..a561072dc05 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedLayeredWorldStateKeyValueStorage.java @@ -0,0 +1,22 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage; + +public interface DiffBasedLayeredWorldStateKeyValueStorage + extends DiffBasedSnapshotWorldStateKeyValueStorage { + + DiffBasedWorldStateKeyValueStorage clone(); +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java new file mode 100644 index 00000000000..9c1a691cf31 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedSnapshotWorldStateKeyValueStorage.java @@ -0,0 +1,21 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage; + +public interface DiffBasedSnapshotWorldStateKeyValueStorage { + + DiffBasedWorldStateKeyValueStorage getParentWorldStateStorage(); +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedWorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedWorldStateKeyValueStorage.java new file mode 100644 index 00000000000..2424828ee60 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedWorldStateKeyValueStorage.java @@ -0,0 +1,234 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage; + +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.storage.StorageProvider; +import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat.FlatDbStrategy; +import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; +import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; +import org.hyperledger.besu.util.Subscribers; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Stream; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class DiffBasedWorldStateKeyValueStorage + implements WorldStateKeyValueStorage, AutoCloseable { + private static final Logger LOG = + LoggerFactory.getLogger(DiffBasedWorldStateKeyValueStorage.class); + + // 0x776f726c64526f6f74 + public static final byte[] WORLD_ROOT_HASH_KEY = "worldRoot".getBytes(StandardCharsets.UTF_8); + // 0x776f726c64426c6f636b48617368 + public static final byte[] WORLD_BLOCK_HASH_KEY = + "worldBlockHash".getBytes(StandardCharsets.UTF_8); + + private final AtomicBoolean shouldClose = new AtomicBoolean(false); + + protected final AtomicBoolean isClosed = new AtomicBoolean(false); + + protected final Subscribers subscribers = Subscribers.create(); + protected final SegmentedKeyValueStorage composedWorldStateStorage; + protected final KeyValueStorage trieLogStorage; + + public DiffBasedWorldStateKeyValueStorage(final StorageProvider provider) { + this.composedWorldStateStorage = + provider.getStorageBySegmentIdentifiers( + List.of( + ACCOUNT_INFO_STATE, CODE_STORAGE, ACCOUNT_STORAGE_STORAGE, TRIE_BRANCH_STORAGE)); + this.trieLogStorage = + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); + } + + public DiffBasedWorldStateKeyValueStorage( + final SegmentedKeyValueStorage composedWorldStateStorage, + final KeyValueStorage trieLogStorage) { + this.composedWorldStateStorage = composedWorldStateStorage; + this.trieLogStorage = trieLogStorage; + } + + public abstract FlatDbMode getFlatDbMode(); + + public abstract FlatDbStrategy getFlatDbStrategy(); + + @Override + public abstract DataStorageFormat getDataStorageFormat(); + + public SegmentedKeyValueStorage getComposedWorldStateStorage() { + return composedWorldStateStorage; + } + + public KeyValueStorage getTrieLogStorage() { + return trieLogStorage; + } + + public Optional getTrieLog(final Hash blockHash) { + return trieLogStorage.get(blockHash.toArrayUnsafe()); + } + + public Stream streamTrieLogKeys(final long limit) { + return trieLogStorage.streamKeys().limit(limit); + } + + public Optional getStateTrieNode(final Bytes location) { + return composedWorldStateStorage + .get(TRIE_BRANCH_STORAGE, location.toArrayUnsafe()) + .map(Bytes::wrap); + } + + public Optional getWorldStateRootHash() { + return composedWorldStateStorage.get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY).map(Bytes::wrap); + } + + public Optional getWorldStateBlockHash() { + return composedWorldStateStorage + .get(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY) + .map(Bytes32::wrap) + .map(Hash::wrap); + } + + public Map streamFlatAccounts( + final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { + return getFlatDbStrategy() + .streamAccountFlatDatabase(composedWorldStateStorage, startKeyHash, endKeyHash, max); + } + + public Map streamFlatStorages( + final Hash accountHash, final Bytes startKeyHash, final Bytes32 endKeyHash, final long max) { + return getFlatDbStrategy() + .streamStorageFlatDatabase( + composedWorldStateStorage, accountHash, startKeyHash, endKeyHash, max); + } + + public boolean isWorldStateAvailable(final Bytes32 rootHash, final Hash blockHash) { + return composedWorldStateStorage + .get(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY) + .map(Bytes32::wrap) + .map(hash -> hash.equals(rootHash) || trieLogStorage.containsKey(blockHash.toArrayUnsafe())) + .orElse(false); + } + + @Override + public void clear() { + subscribers.forEach(StorageSubscriber::onClearStorage); + getFlatDbStrategy().clearAll(composedWorldStateStorage); + composedWorldStateStorage.clear(TRIE_BRANCH_STORAGE); + trieLogStorage.clear(); + } + + public void clearTrieLog() { + subscribers.forEach(StorageSubscriber::onClearTrieLog); + trieLogStorage.clear(); + } + + public void clearFlatDatabase() { + subscribers.forEach(StorageSubscriber::onClearFlatDatabaseStorage); + getFlatDbStrategy().resetOnResync(composedWorldStateStorage); + } + + @Override + public abstract Updater updater(); + + public boolean pruneTrieLog(final Hash blockHash) { + try { + return trieLogStorage.tryDelete(blockHash.toArrayUnsafe()); + } catch (Exception e) { + LOG.error("Error pruning trie log for block hash {}", blockHash, e); + return false; + } + } + + @Override + public synchronized void close() throws Exception { + // when the storage clears, close + shouldClose.set(true); + tryClose(); + } + + public synchronized long subscribe(final StorageSubscriber sub) { + if (isClosed.get()) { + throw new RuntimeException("Storage is marked to close or has already closed"); + } + return subscribers.subscribe(sub); + } + + public synchronized void unSubscribe(final long id) { + subscribers.unsubscribe(id); + try { + tryClose(); + } catch (Exception e) { + LOG.atWarn() + .setMessage("exception while trying to close : {}") + .addArgument(e::getMessage) + .log(); + } + } + + protected synchronized void tryClose() throws Exception { + if (shouldClose.get() && subscribers.getSubscriberCount() < 1) { + doClose(); + } + } + + protected synchronized void doClose() throws Exception { + if (!isClosed.get()) { + // alert any subscribers we are closing: + subscribers.forEach(StorageSubscriber::onCloseStorage); + + // close all of the KeyValueStorages: + composedWorldStateStorage.close(); + trieLogStorage.close(); + + // set storage closed + isClosed.set(true); + } + } + + public interface Updater extends WorldStateKeyValueStorage.Updater { + + DiffBasedWorldStateKeyValueStorage.Updater saveWorldState( + final Bytes blockHash, final Bytes32 nodeHash, final Bytes node); + + SegmentedKeyValueStorageTransaction getWorldStateTransaction(); + + KeyValueStorageTransaction getTrieLogStorageTransaction(); + + @Override + void commit(); + + void rollback(); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/AccountHashCodeStorageStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/AccountHashCodeStorageStrategy.java similarity index 96% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/AccountHashCodeStorageStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/AccountHashCodeStorageStrategy.java index 7d77ff1a6fc..446ed3262a1 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/AccountHashCodeStorageStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/AccountHashCodeStorageStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeHashCodeStorageStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/CodeHashCodeStorageStrategy.java similarity index 96% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeHashCodeStorageStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/CodeHashCodeStorageStrategy.java index 6f4d8bb8147..4263e423b86 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeHashCodeStorageStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/CodeHashCodeStorageStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeStorageStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/CodeStorageStrategy.java similarity index 94% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeStorageStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/CodeStorageStrategy.java index 8767be24788..91da4eff066 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/CodeStorageStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/CodeStorageStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategy.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategy.java similarity index 99% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategy.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategy.java index d85f4932516..37c2877b167 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategy.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategy.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java similarity index 95% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java index e251f28c175..7614c1499b8 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProvider.java @@ -13,11 +13,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.FullFlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.PartialFlatDbStrategy; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.plugin.services.MetricsSystem; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java similarity index 79% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java index 4c9a520bd2f..d74a6804d92 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/NoOpTrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/NoOpTrieLogManager.java @@ -12,12 +12,12 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import java.util.Optional; @@ -30,10 +30,10 @@ public NoOpTrieLogManager() { @Override public synchronized void saveTrieLog( - final BonsaiWorldStateUpdateAccumulator localUpdater, + final DiffBasedWorldStateUpdateAccumulator localUpdater, final Hash forWorldStateRootHash, final BlockHeader forBlockHeader, - final BonsaiWorldState forWorldState) { + final DiffBasedWorldState forWorldState) { // notify trie log added observers, synchronously TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); trieLogObservers.forEach(o -> o.onTrieLogAdded(new TrieLogAddedEvent(trieLog))); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogAddedEvent.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogAddedEvent.java similarity index 92% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogAddedEvent.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogAddedEvent.java index 3e1387997b4..28edc5cc608 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogAddedEvent.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogAddedEvent.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayer.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayer.java similarity index 79% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayer.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayer.java index ff1e29c41b8..ff6ef226ef7 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayer.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayer.java @@ -14,7 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import static com.google.common.base.Preconditions.checkState; @@ -22,7 +22,7 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; import java.util.HashMap; @@ -49,21 +49,21 @@ public class TrieLogLayer implements TrieLog { protected Hash blockHash; protected Optional blockNumber = Optional.empty(); - Map> getAccounts() { + Map> getAccounts() { return accounts; } - Map> getCode() { + Map> getCode() { return code; } - Map>> getStorage() { + Map>> getStorage() { return storage; } - protected final Map> accounts; - protected final Map> code; - protected final Map>> storage; + protected final Map> accounts; + protected final Map> code; + protected final Map>> storage; protected boolean frozen = false; public TrieLogLayer() { @@ -104,14 +104,14 @@ public TrieLogLayer setBlockNumber(final long blockNumber) { public TrieLogLayer addAccountChange( final Address address, final AccountValue oldValue, final AccountValue newValue) { checkState(!frozen, "Layer is Frozen"); - accounts.put(address, new BonsaiValue<>(oldValue, newValue)); + accounts.put(address, new DiffBasedValue<>(oldValue, newValue)); return this; } public TrieLogLayer addCodeChange( final Address address, final Bytes oldValue, final Bytes newValue, final Hash blockHash) { checkState(!frozen, "Layer is Frozen"); - code.put(address, new BonsaiValue<>(oldValue, newValue, newValue == null)); + code.put(address, new DiffBasedValue<>(oldValue, newValue, newValue == null)); return this; } @@ -123,22 +123,22 @@ public TrieLogLayer addStorageChange( checkState(!frozen, "Layer is Frozen"); storage .computeIfAbsent(address, a -> new TreeMap<>()) - .put(slot, new BonsaiValue<>(oldValue, newValue)); + .put(slot, new DiffBasedValue<>(oldValue, newValue)); return this; } @Override - public Map> getAccountChanges() { + public Map> getAccountChanges() { return accounts; } @Override - public Map> getCodeChanges() { + public Map> getCodeChanges() { return code; } @Override - public Map>> getStorageChanges() { + public Map>> getStorageChanges() { return storage; } @@ -147,18 +147,18 @@ public boolean hasStorageChanges(final Address address) { } @Override - public Map> getStorageChanges(final Address address) { + public Map> getStorageChanges(final Address address) { return storage.getOrDefault(address, Map.of()); } @Override public Optional getPriorCode(final Address address) { - return Optional.ofNullable(code.get(address)).map(BonsaiValue::getPrior); + return Optional.ofNullable(code.get(address)).map(DiffBasedValue::getPrior); } @Override public Optional getCode(final Address address) { - return Optional.ofNullable(code.get(address)).map(BonsaiValue::getUpdated); + return Optional.ofNullable(code.get(address)).map(DiffBasedValue::getUpdated); } @Override @@ -166,7 +166,7 @@ public Optional getPriorStorageByStorageSlotKey( final Address address, final StorageSlotKey storageSlotKey) { return Optional.ofNullable(storage.get(address)) .map(i -> i.get(storageSlotKey)) - .map(BonsaiValue::getPrior); + .map(DiffBasedValue::getPrior); } @Override @@ -174,24 +174,24 @@ public Optional getStorageByStorageSlotKey( final Address address, final StorageSlotKey storageSlotKey) { return Optional.ofNullable(storage.get(address)) .map(i -> i.get(storageSlotKey)) - .map(BonsaiValue::getUpdated); + .map(DiffBasedValue::getUpdated); } @Override public Optional getPriorAccount(final Address address) { - return Optional.ofNullable(accounts.get(address)).map(BonsaiValue::getPrior); + return Optional.ofNullable(accounts.get(address)).map(DiffBasedValue::getPrior); } @Override public Optional getAccount(final Address address) { - return Optional.ofNullable(accounts.get(address)).map(BonsaiValue::getUpdated); + return Optional.ofNullable(accounts.get(address)).map(DiffBasedValue::getUpdated); } public String dump() { final StringBuilder sb = new StringBuilder(); sb.append("TrieLog{" + "blockHash=").append(blockHash).append(frozen).append('}'); sb.append("accounts\n"); - for (final Map.Entry> account : accounts.entrySet()) { + for (final Map.Entry> account : accounts.entrySet()) { sb.append(" : ").append(account.getKey()).append("\n"); if (Objects.equals(account.getValue().getPrior(), account.getValue().getUpdated())) { sb.append(" = ").append(account.getValue().getUpdated()).append("\n"); @@ -201,7 +201,7 @@ public String dump() { } } sb.append("code").append("\n"); - for (final Map.Entry> code : code.entrySet()) { + for (final Map.Entry> code : code.entrySet()) { sb.append(" : ").append(code.getKey()).append("\n"); if (Objects.equals(code.getValue().getPrior(), code.getValue().getUpdated())) { sb.append(" = ").append(code.getValue().getPrior()).append("\n"); @@ -211,10 +211,10 @@ public String dump() { } } sb.append("Storage").append("\n"); - for (final Map.Entry>> storage : + for (final Map.Entry>> storage : storage.entrySet()) { sb.append(" : ").append(storage.getKey()).append("\n"); - for (final Map.Entry> slot : + for (final Map.Entry> slot : storage.getValue().entrySet()) { final UInt256 originalValue = slot.getValue().getPrior(); final UInt256 updatedValue = slot.getValue().getUpdated(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManager.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManager.java similarity index 87% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManager.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManager.java index 19ac6285f5c..5233cda6f66 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManager.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogManager.java @@ -13,14 +13,15 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; import org.hyperledger.besu.plugin.BesuContext; import org.hyperledger.besu.plugin.services.TrieLogService; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; @@ -41,7 +42,7 @@ public class TrieLogManager { private static final Logger LOG = LoggerFactory.getLogger(TrieLogManager.class); public static final long LOG_RANGE_LIMIT = 1000; // restrict trielog range queries to 1k logs protected final Blockchain blockchain; - protected final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; + protected final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage; protected final long maxLayersToLoad; protected final Subscribers trieLogObservers = Subscribers.create(); @@ -50,7 +51,7 @@ public class TrieLogManager { public TrieLogManager( final Blockchain blockchain, - final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage, + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, final long maxLayersToLoad, final BesuContext pluginContext) { this.blockchain = blockchain; @@ -60,15 +61,15 @@ public TrieLogManager( } public synchronized void saveTrieLog( - final BonsaiWorldStateUpdateAccumulator localUpdater, + final DiffBasedWorldStateUpdateAccumulator localUpdater, final Hash forWorldStateRootHash, final BlockHeader forBlockHeader, - final BonsaiWorldState forWorldState) { + final DiffBasedWorldState forWorldState) { // do not overwrite a trielog layer that already exists in the database. // if it's only in memory we need to save it // for example, in case of reorg we don't replace a trielog layer if (rootWorldStateStorage.getTrieLog(forBlockHeader.getHash()).isEmpty()) { - final BonsaiWorldStateKeyValueStorage.Updater stateUpdater = + final DiffBasedWorldStateKeyValueStorage.Updater stateUpdater = forWorldState.getWorldStateStorage().updater(); boolean success = false; try { @@ -90,7 +91,7 @@ public synchronized void saveTrieLog( } private TrieLog prepareTrieLog( - final BlockHeader blockHeader, final BonsaiWorldStateUpdateAccumulator localUpdater) { + final BlockHeader blockHeader, final DiffBasedWorldStateUpdateAccumulator localUpdater) { LOG.atDebug() .setMessage("Adding layered world state for {}") .addArgument(blockHeader::toLogString) @@ -104,7 +105,7 @@ private void persistTrieLog( final BlockHeader blockHeader, final Hash worldStateRootHash, final TrieLog trieLog, - final BonsaiWorldStateKeyValueStorage.Updater stateUpdater) { + final DiffBasedWorldStateKeyValueStorage.Updater stateUpdater) { LOG.atDebug() .setMessage("Persisting trie log for block hash {} and world state root {}") .addArgument(blockHeader::toLogString) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java similarity index 94% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java index b72796f2225..60a728b3fb6 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPruner.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogPruner.java @@ -13,13 +13,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import org.hyperledger.besu.plugin.services.trielogs.TrieLogEvent; import java.util.Comparator; @@ -41,7 +41,7 @@ public class TrieLogPruner implements TrieLogEvent.TrieLogObserver { private final int pruningLimit; private final int loadingLimit; - private final BonsaiWorldStateKeyValueStorage rootWorldStateStorage; + private final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage; private final Blockchain blockchain; private final Consumer executeAsync; private final long numBlocksToRetain; @@ -51,7 +51,7 @@ public class TrieLogPruner implements TrieLogEvent.TrieLogObserver { TreeMultimap.create(Comparator.reverseOrder(), Comparator.naturalOrder()); public TrieLogPruner( - final BonsaiWorldStateKeyValueStorage rootWorldStateStorage, + final DiffBasedWorldStateKeyValueStorage rootWorldStateStorage, final Blockchain blockchain, final Consumer executeAsync, final long numBlocksToRetain, @@ -100,7 +100,7 @@ private int preloadQueue() { } } - void addToPruneQueue(final long blockNumber, final Hash blockHash) { + public void addToPruneQueue(final long blockNumber, final Hash blockHash) { LOG.atTrace() .setMessage("adding trie log to queue for later pruning blockNumber {}; blockHash {}") .addArgument(blockNumber) @@ -109,7 +109,7 @@ void addToPruneQueue(final long blockNumber, final Hash blockHash) { trieLogBlocksAndForksByDescendingBlockNumber.put(blockNumber, blockHash); } - int pruneFromQueue() { + public int pruneFromQueue() { final long retainAboveThisBlock = blockchain.getChainHeadBlockNumber() - numBlocksToRetain; final Optional finalized = blockchain.getFinalized(); if (requireFinalizedBlock && finalized.isEmpty()) { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java new file mode 100644 index 00000000000..7a909cbf8e1 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldState.java @@ -0,0 +1,334 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview; + +import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; +import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; +import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.StorageSlotKey; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.StorageSubscriber; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedLayeredWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedSnapshotWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; +import org.hyperledger.besu.evm.account.Account; +import org.hyperledger.besu.evm.worldstate.WorldUpdater; +import org.hyperledger.besu.plugin.services.exception.StorageException; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; +import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; +import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction; + +import java.util.Optional; +import java.util.stream.Stream; +import javax.annotation.Nonnull; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class DiffBasedWorldState + implements MutableWorldState, DiffBasedWorldView, StorageSubscriber { + + private static final Logger LOG = LoggerFactory.getLogger(DiffBasedWorldState.class); + + protected DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage; + protected final DiffBasedCachedWorldStorageManager cachedWorldStorageManager; + protected final TrieLogManager trieLogManager; + protected DiffBasedWorldStateUpdateAccumulator accumulator; + + protected Hash worldStateRootHash; + protected Hash worldStateBlockHash; + protected boolean isFrozen; + + protected DiffBasedWorldState( + final DiffBasedWorldStateKeyValueStorage worldStateKeyValueStorage, + final DiffBasedCachedWorldStorageManager cachedWorldStorageManager, + final TrieLogManager trieLogManager) { + this.worldStateKeyValueStorage = worldStateKeyValueStorage; + this.worldStateRootHash = + Hash.wrap( + Bytes32.wrap( + worldStateKeyValueStorage.getWorldStateRootHash().orElse(getEmptyTrieHash()))); + this.worldStateBlockHash = + Hash.wrap( + Bytes32.wrap(worldStateKeyValueStorage.getWorldStateBlockHash().orElse(Hash.ZERO))); + this.cachedWorldStorageManager = cachedWorldStorageManager; + this.trieLogManager = trieLogManager; + } + + /** + * Having a protected method to override the accumulator solves the chicken-egg problem of needing + * a worldstate reference (this) when construction the Accumulator. + * + * @param accumulator accumulator to use. + */ + public void setAccumulator(final DiffBasedWorldStateUpdateAccumulator accumulator) { + this.accumulator = accumulator; + } + + /** + * Returns the world state block hash of this world state + * + * @return the world state block hash. + */ + public Hash getWorldStateBlockHash() { + return worldStateBlockHash; + } + + /** + * Returns the world state root hash of this world state + * + * @return the world state root hash. + */ + public Hash getWorldStateRootHash() { + return worldStateRootHash; + } + + @Override + public boolean isPersisted() { + return isPersisted(worldStateKeyValueStorage); + } + + private boolean isPersisted(final WorldStateKeyValueStorage worldStateKeyValueStorage) { + return !(worldStateKeyValueStorage instanceof DiffBasedSnapshotWorldStateKeyValueStorage); + } + + /** + * Reset the worldState to this block header + * + * @param blockHeader block to use + */ + public void resetWorldStateTo(final BlockHeader blockHeader) { + worldStateBlockHash = blockHeader.getBlockHash(); + worldStateRootHash = blockHeader.getStateRoot(); + } + + @Override + public DiffBasedWorldStateKeyValueStorage getWorldStateStorage() { + return worldStateKeyValueStorage; + } + + public DiffBasedWorldStateUpdateAccumulator getAccumulator() { + return accumulator; + } + + @Override + public void persist(final BlockHeader blockHeader) { + final Optional maybeBlockHeader = Optional.ofNullable(blockHeader); + LOG.atDebug() + .setMessage("Persist world state for block {}") + .addArgument(maybeBlockHeader) + .log(); + + final DiffBasedWorldStateUpdateAccumulator localCopy = accumulator.copy(); + + boolean success = false; + + final DiffBasedWorldStateKeyValueStorage.Updater stateUpdater = + worldStateKeyValueStorage.updater(); + Runnable saveTrieLog = () -> {}; + + try { + final Hash newWorldStateRootHash = + calculateRootHash(isFrozen ? Optional.empty() : Optional.of(stateUpdater), accumulator); + // if we are persisted with a block header, and the prior state is the parent + // then persist the TrieLog for that transition. + // If specified but not a direct descendant simply store the new block hash. + if (blockHeader != null) { + verifyWorldStateRoot(newWorldStateRootHash, blockHeader); + saveTrieLog = + () -> { + trieLogManager.saveTrieLog(localCopy, newWorldStateRootHash, blockHeader, this); + // not save a frozen state in the cache + if (!isFrozen) { + cachedWorldStorageManager.addCachedLayer(blockHeader, newWorldStateRootHash, this); + } + }; + + stateUpdater + .getWorldStateTransaction() + .put(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY, blockHeader.getHash().toArrayUnsafe()); + worldStateBlockHash = blockHeader.getHash(); + } else { + stateUpdater.getWorldStateTransaction().remove(TRIE_BRANCH_STORAGE, WORLD_BLOCK_HASH_KEY); + worldStateBlockHash = null; + } + + stateUpdater + .getWorldStateTransaction() + .put(TRIE_BRANCH_STORAGE, WORLD_ROOT_HASH_KEY, newWorldStateRootHash.toArrayUnsafe()); + worldStateRootHash = newWorldStateRootHash; + success = true; + } finally { + if (success) { + stateUpdater.commit(); + accumulator.reset(); + saveTrieLog.run(); + } else { + stateUpdater.rollback(); + accumulator.reset(); + } + } + } + + protected void verifyWorldStateRoot(final Hash calculatedStateRoot, final BlockHeader header) { + if (!calculatedStateRoot.equals(header.getStateRoot())) { + throw new RuntimeException( + "World State Root does not match expected value, header " + + header.getStateRoot().toHexString() + + " calculated " + + calculatedStateRoot.toHexString()); + } + } + + @Override + public WorldUpdater updater() { + return accumulator; + } + + @Override + public Hash rootHash() { + if (isFrozen && accumulator.isAccumulatorStateChanged()) { + worldStateRootHash = calculateRootHash(Optional.empty(), accumulator.copy()); + accumulator.resetAccumulatorStateChanged(); + } + return Hash.wrap(worldStateRootHash); + } + + protected static final KeyValueStorageTransaction noOpTx = + new KeyValueStorageTransaction() { + + @Override + public void put(final byte[] key, final byte[] value) { + // no-op + } + + @Override + public void remove(final byte[] key) { + // no-op + } + + @Override + public void commit() throws StorageException { + // no-op + } + + @Override + public void rollback() { + // no-op + } + }; + + protected static final SegmentedKeyValueStorageTransaction noOpSegmentedTx = + new SegmentedKeyValueStorageTransaction() { + + @Override + public void put( + final SegmentIdentifier segmentIdentifier, final byte[] key, final byte[] value) { + // no-op + } + + @Override + public void remove(final SegmentIdentifier segmentIdentifier, final byte[] key) { + // no-op + } + + @Override + public void commit() throws StorageException { + // no-op + } + + @Override + public void rollback() { + // no-op + } + }; + + public Hash blockHash() { + return worldStateBlockHash; + } + + @Override + public Stream streamAccounts(final Bytes32 startKeyHash, final int limit) { + throw new RuntimeException("storage format do not provide account streaming."); + } + + @Override + public UInt256 getPriorStorageValue(final Address address, final UInt256 storageKey) { + return getStorageValue(address, storageKey); + } + + @Override + public void close() { + try { + if (!isPersisted()) { + this.worldStateKeyValueStorage.close(); + if (isFrozen) { + closeFrozenStorage(); + } + } + } catch (Exception e) { + // no op + } + } + + private void closeFrozenStorage() { + try { + final DiffBasedLayeredWorldStateKeyValueStorage worldStateLayerStorage = + (DiffBasedLayeredWorldStateKeyValueStorage) worldStateKeyValueStorage; + if (!isPersisted(worldStateLayerStorage.getParentWorldStateStorage())) { + worldStateLayerStorage.getParentWorldStateStorage().close(); + } + } catch (Exception e) { + // no op + } + } + + @Override + public abstract Hash frontierRootHash(); + + @Override + public abstract MutableWorldState freeze(); + + @Override + public abstract Account get(final Address address); + + @Override + public abstract UInt256 getStorageValue(final Address address, final UInt256 storageKey); + + @Override + public abstract Optional getStorageValueByStorageSlotKey( + final Address address, final StorageSlotKey storageSlotKey); + + @Override + public abstract Optional getCode(@Nonnull final Address address, final Hash codeHash); + + protected abstract Hash calculateRootHash( + final Optional maybeStateUpdater, + final DiffBasedWorldStateUpdateAccumulator worldStateUpdater); + + protected abstract Hash getEmptyTrieHash(); +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldView.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldView.java similarity index 85% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldView.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldView.java index cd16e7dec8a..626584aa2c3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldView.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/DiffBasedWorldView.java @@ -1,5 +1,5 @@ /* - * Copyright ConsenSys AG. + * Copyright Hyperledger Besu Contributors. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -11,16 +11,15 @@ * specific language governing permissions and limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - * */ -package org.hyperledger.besu.ethereum.trie.bonsai.worldview; +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; import org.hyperledger.besu.evm.worldstate.WorldUpdater; import org.hyperledger.besu.evm.worldstate.WorldView; @@ -31,7 +30,7 @@ import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; -public interface BonsaiWorldView extends WorldView { +public interface DiffBasedWorldView extends WorldView { Optional getCode(Address address, final Hash codeHash); @@ -59,7 +58,7 @@ static Bytes encodeTrieValue(final Bytes bytes) { boolean isPersisted(); - BonsaiWorldStateKeyValueStorage getWorldStateStorage(); + DiffBasedWorldStateKeyValueStorage getWorldStateStorage(); WorldUpdater updater(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java similarity index 71% rename from ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java rename to ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java index 44c18176bff..66169ee85a3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateUpdateAccumulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/DiffBasedWorldStateUpdateAccumulator.java @@ -1,5 +1,5 @@ /* - * Copyright ConsenSys AG. + * Copyright Hyperledger Besu Contributors. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -11,10 +11,9 @@ * specific language governing permissions and limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - * */ -package org.hyperledger.besu.ethereum.trie.bonsai.worldview; +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator; import org.hyperledger.besu.datatypes.AccountValue; import org.hyperledger.besu.datatypes.Address; @@ -23,9 +22,14 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrieException; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.AccountConsumingMap; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.Consumer; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.StorageConsumingMap; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -44,42 +48,40 @@ import java.util.Set; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.function.Function; -import javax.annotation.Nonnull; -import com.google.common.collect.ForwardingMap; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; import org.apache.tuweni.units.bigints.UInt256; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class BonsaiWorldStateUpdateAccumulator - extends AbstractWorldUpdater - implements BonsaiWorldView, TrieLogAccumulator { +@SuppressWarnings("unchecked") +public abstract class DiffBasedWorldStateUpdateAccumulator + extends AbstractWorldUpdater + implements DiffBasedWorldView, TrieLogAccumulator { private static final Logger LOG = - LoggerFactory.getLogger(BonsaiWorldStateUpdateAccumulator.class); - protected final Consumer> accountPreloader; + LoggerFactory.getLogger(DiffBasedWorldStateUpdateAccumulator.class); + protected final Consumer> accountPreloader; protected final Consumer storagePreloader; - private final AccountConsumingMap> accountsToUpdate; - private final Map> codeToUpdate = new ConcurrentHashMap<>(); + private final AccountConsumingMap> accountsToUpdate; + private final Map> codeToUpdate = new ConcurrentHashMap<>(); private final Set

storageToClear = Collections.synchronizedSet(new HashSet<>()); protected final EvmConfiguration evmConfiguration; // storage sub mapped by _hashed_ key. This is because in self_destruct calls we need to // enumerate the old storage and delete it. Those are trie stored by hashed key by spec and the // alternative was to keep a giant pre-image cache of the entire trie. - private final Map>> + private final Map>> storageToUpdate = new ConcurrentHashMap<>(); private final Map storageKeyHashLookup = new ConcurrentHashMap<>(); protected boolean isAccumulatorStateChanged; - public BonsaiWorldStateUpdateAccumulator( - final BonsaiWorldView world, - final Consumer> accountPreloader, + public DiffBasedWorldStateUpdateAccumulator( + final DiffBasedWorldView world, + final Consumer> accountPreloader, final Consumer storagePreloader, final EvmConfiguration evmConfiguration) { super(world, evmConfiguration); @@ -90,15 +92,7 @@ public BonsaiWorldStateUpdateAccumulator( this.evmConfiguration = evmConfiguration; } - public BonsaiWorldStateUpdateAccumulator copy() { - final BonsaiWorldStateUpdateAccumulator copy = - new BonsaiWorldStateUpdateAccumulator( - wrappedWorldView(), accountPreloader, storagePreloader, evmConfiguration); - copy.cloneFromUpdater(this); - return copy; - } - - void cloneFromUpdater(final BonsaiWorldStateUpdateAccumulator source) { + protected void cloneFromUpdater(final DiffBasedWorldStateUpdateAccumulator source) { accountsToUpdate.putAll(source.getAccountsToUpdate()); codeToUpdate.putAll(source.codeToUpdate); storageToClear.addAll(source.storageToClear); @@ -108,14 +102,25 @@ void cloneFromUpdater(final BonsaiWorldStateUpdateAccumulator source) { this.isAccumulatorStateChanged = true; } + protected Consumer> getAccountPreloader() { + return accountPreloader; + } + + protected Consumer getStoragePreloader() { + return storagePreloader; + } + + protected EvmConfiguration getEvmConfiguration() { + return evmConfiguration; + } + @Override public Account get(final Address address) { return super.get(address); } @Override - protected UpdateTrackingAccount track( - final UpdateTrackingAccount account) { + protected UpdateTrackingAccount track(final UpdateTrackingAccount account) { return super.track(account); } @@ -126,21 +131,21 @@ public MutableAccount getAccount(final Address address) { @Override public MutableAccount createAccount(final Address address, final long nonce, final Wei balance) { - BonsaiValue bonsaiValue = accountsToUpdate.get(address); - - if (bonsaiValue == null) { - bonsaiValue = new BonsaiValue<>(null, null); - accountsToUpdate.put(address, bonsaiValue); - } else if (bonsaiValue.getUpdated() != null) { - if (bonsaiValue.getUpdated().isEmpty()) { - return track(new UpdateTrackingAccount<>(bonsaiValue.getUpdated())); + DiffBasedValue diffBasedValue = accountsToUpdate.get(address); + + if (diffBasedValue == null) { + diffBasedValue = new DiffBasedValue<>(null, null); + accountsToUpdate.put(address, diffBasedValue); + } else if (diffBasedValue.getUpdated() != null) { + if (diffBasedValue.getUpdated().isEmpty()) { + return track(new UpdateTrackingAccount<>(diffBasedValue.getUpdated())); } else { throw new IllegalStateException("Cannot create an account when one already exists"); } } - final BonsaiAccount newAccount = - new BonsaiAccount( + final ACCOUNT newAccount = + createAccount( this, address, hashAndSaveAccountPreImage(address), @@ -149,17 +154,17 @@ public MutableAccount createAccount(final Address address, final long nonce, fin Hash.EMPTY_TRIE_HASH, Hash.EMPTY, true); - bonsaiValue.setUpdated(newAccount); + diffBasedValue.setUpdated(newAccount); return track(new UpdateTrackingAccount<>(newAccount)); } @Override - public Map> getAccountsToUpdate() { + public Map> getAccountsToUpdate() { return accountsToUpdate; } @Override - public Map> getCodeToUpdate() { + public Map> getCodeToUpdate() { return codeToUpdate; } @@ -168,40 +173,41 @@ public Set
getStorageToClear() { } @Override - public Map>> + public Map>> getStorageToUpdate() { return storageToUpdate; } @Override - protected BonsaiAccount getForMutation(final Address address) { - return loadAccount(address, BonsaiValue::getUpdated); + protected ACCOUNT getForMutation(final Address address) { + return loadAccount(address, DiffBasedValue::getUpdated); } - protected BonsaiAccount loadAccount( - final Address address, - final Function, BonsaiAccount> bonsaiAccountFunction) { + protected ACCOUNT loadAccount( + final Address address, final Function, ACCOUNT> accountFunction) { try { - final BonsaiValue bonsaiValue = accountsToUpdate.get(address); - if (bonsaiValue == null) { + final DiffBasedValue diffBasedValue = accountsToUpdate.get(address); + if (diffBasedValue == null) { final Account account; - if (wrappedWorldView() - instanceof BonsaiWorldStateUpdateAccumulator bonsaiWorldStateUpdateAccumulator) { - account = bonsaiWorldStateUpdateAccumulator.loadAccount(address, bonsaiAccountFunction); + if (wrappedWorldView() instanceof DiffBasedWorldStateUpdateAccumulator) { + final DiffBasedWorldStateUpdateAccumulator worldStateUpdateAccumulator = + (DiffBasedWorldStateUpdateAccumulator) wrappedWorldView(); + account = worldStateUpdateAccumulator.loadAccount(address, accountFunction); } else { account = wrappedWorldView().get(address); } - if (account instanceof BonsaiAccount bonsaiAccount) { - BonsaiAccount mutableAccount = new BonsaiAccount(bonsaiAccount, this, true); - accountsToUpdate.put(address, new BonsaiValue<>(bonsaiAccount, mutableAccount)); + if (account instanceof DiffBasedAccount diffBasedAccount) { + ACCOUNT mutableAccount = copyAccount((ACCOUNT) diffBasedAccount, this, true); + accountsToUpdate.put( + address, new DiffBasedValue<>((ACCOUNT) diffBasedAccount, mutableAccount)); return mutableAccount; } else { // add the empty read in accountsToUpdate - accountsToUpdate.put(address, new BonsaiValue<>(null, null)); + accountsToUpdate.put(address, new DiffBasedValue<>(null, null)); return null; } } else { - return bonsaiAccountFunction.apply(bonsaiValue); + return accountFunction.apply(diffBasedValue); } } catch (MerkleTrieException e) { // need to throw to trigger the heal @@ -229,12 +235,12 @@ public void revert() { public void commit() { this.isAccumulatorStateChanged = true; for (final Address deletedAddress : getDeletedAccounts()) { - final BonsaiValue accountValue = + final DiffBasedValue accountValue = accountsToUpdate.computeIfAbsent( deletedAddress, - __ -> loadAccountFromParent(deletedAddress, new BonsaiValue<>(null, null, true))); + __ -> loadAccountFromParent(deletedAddress, new DiffBasedValue<>(null, null, true))); storageToClear.add(deletedAddress); - final BonsaiValue codeValue = codeToUpdate.get(deletedAddress); + final DiffBasedValue codeValue = codeToUpdate.get(deletedAddress); if (codeValue != null) { codeValue.setUpdated(null).setCleared(); } else { @@ -242,26 +248,27 @@ public void commit() { .getCode( deletedAddress, Optional.ofNullable(accountValue) - .map(BonsaiValue::getPrior) - .map(BonsaiAccount::getCodeHash) + .map(DiffBasedValue::getPrior) + .map(DiffBasedAccount::getCodeHash) .orElse(Hash.EMPTY)) .ifPresent( deletedCode -> - codeToUpdate.put(deletedAddress, new BonsaiValue<>(deletedCode, null, true))); + codeToUpdate.put( + deletedAddress, new DiffBasedValue<>(deletedCode, null, true))); } // mark all updated storage as to be cleared - final Map> deletedStorageUpdates = + final Map> deletedStorageUpdates = storageToUpdate.computeIfAbsent( deletedAddress, k -> new StorageConsumingMap<>( deletedAddress, new ConcurrentHashMap<>(), storagePreloader)); - final Iterator>> iter = + final Iterator>> iter = deletedStorageUpdates.entrySet().iterator(); while (iter.hasNext()) { - final Map.Entry> updateEntry = iter.next(); - final BonsaiValue updatedSlot = updateEntry.getValue(); + final Map.Entry> updateEntry = iter.next(); + final DiffBasedValue updatedSlot = updateEntry.getValue(); if (updatedSlot.getPrior() == null || updatedSlot.getPrior().isZero()) { iter.remove(); } else { @@ -269,7 +276,7 @@ public void commit() { } } - final BonsaiAccount originalValue = accountValue.getPrior(); + final ACCOUNT originalValue = accountValue.getPrior(); if (originalValue != null) { // Enumerate and delete addresses not updated wrappedWorldView() @@ -280,7 +287,8 @@ public void commit() { new StorageSlotKey(Hash.wrap(keyHash), Optional.empty()); if (!deletedStorageUpdates.containsKey(storageSlotKey)) { final UInt256 value = UInt256.fromBytes(RLP.decodeOne(entryValue)); - deletedStorageUpdates.put(storageSlotKey, new BonsaiValue<>(value, null, true)); + deletedStorageUpdates.put( + storageSlotKey, new DiffBasedValue<>(value, null, true)); } }); } @@ -294,11 +302,11 @@ public void commit() { .forEach( tracked -> { final Address updatedAddress = tracked.getAddress(); - final BonsaiAccount updatedAccount; - final BonsaiValue updatedAccountValue = + final ACCOUNT updatedAccount; + final DiffBasedValue updatedAccountValue = accountsToUpdate.get(updatedAddress); - final Map> pendingStorageUpdates = + final Map> pendingStorageUpdates = storageToUpdate.computeIfAbsent( updatedAddress, k -> @@ -311,12 +319,12 @@ public void commit() { } if (tracked.getWrappedAccount() == null) { - updatedAccount = new BonsaiAccount(this, tracked); + updatedAccount = createAccount(this, tracked); tracked.setWrappedAccount(updatedAccount); if (updatedAccountValue == null) { - accountsToUpdate.put(updatedAddress, new BonsaiValue<>(null, updatedAccount)); + accountsToUpdate.put(updatedAddress, new DiffBasedValue<>(null, updatedAccount)); codeToUpdate.put( - updatedAddress, new BonsaiValue<>(null, updatedAccount.getCode())); + updatedAddress, new DiffBasedValue<>(null, updatedAccount.getCode())); } else { updatedAccountValue.setUpdated(updatedAccount); } @@ -334,17 +342,17 @@ public void commit() { } if (tracked.codeWasUpdated()) { - final BonsaiValue pendingCode = + final DiffBasedValue pendingCode = codeToUpdate.computeIfAbsent( updatedAddress, addr -> - new BonsaiValue<>( + new DiffBasedValue<>( wrappedWorldView() .getCode( addr, Optional.ofNullable(updatedAccountValue) - .map(BonsaiValue::getPrior) - .map(BonsaiAccount::getCodeHash) + .map(DiffBasedValue::getPrior) + .map(DiffBasedAccount::getCodeHash) .orElse(Hash.EMPTY)) .orElse(null), null)); @@ -368,12 +376,11 @@ public void commit() { final StorageSlotKey slotKey = new StorageSlotKey(hashAndSaveSlotPreImage(keyUInt), Optional.of(keyUInt)); final UInt256 value = storageUpdate.getValue(); - final BonsaiValue pendingValue = pendingStorageUpdates.get(slotKey); - + final DiffBasedValue pendingValue = pendingStorageUpdates.get(slotKey); if (pendingValue == null) { pendingStorageUpdates.put( slotKey, - new BonsaiValue<>( + new DiffBasedValue<>( updatedAccount.getOriginalStorageValue(keyUInt), value)); } else { pendingValue.setUpdated(value); @@ -394,7 +401,7 @@ public void commit() { @Override public Optional getCode(final Address address, final Hash codeHash) { - final BonsaiValue localCode = codeToUpdate.get(address); + final DiffBasedValue localCode = codeToUpdate.get(address); if (localCode == null) { final Optional code = wrappedWorldView().getCode(address, codeHash); if (code.isEmpty() && !codeHash.equals(Hash.EMPTY)) { @@ -417,31 +424,26 @@ public UInt256 getStorageValue(final Address address, final UInt256 slotKey) { @Override public Optional getStorageValueByStorageSlotKey( final Address address, final StorageSlotKey storageSlotKey) { - final Map> localAccountStorage = + final Map> localAccountStorage = storageToUpdate.get(address); if (localAccountStorage != null) { - final BonsaiValue value = localAccountStorage.get(storageSlotKey); + final DiffBasedValue value = localAccountStorage.get(storageSlotKey); if (value != null) { return Optional.ofNullable(value.getUpdated()); } } try { final Optional valueUInt = - (wrappedWorldView() instanceof BonsaiWorldState bonsaiWorldState) - ? bonsaiWorldState.getStorageValueByStorageSlotKey( - () -> - Optional.ofNullable(loadAccount(address, BonsaiValue::getPrior)) - .map(BonsaiAccount::getStorageRoot), - address, - storageSlotKey) + (wrappedWorldView() instanceof DiffBasedWorldState worldState) + ? worldState.getStorageValueByStorageSlotKey(address, storageSlotKey) : wrappedWorldView().getStorageValueByStorageSlotKey(address, storageSlotKey); storageToUpdate .computeIfAbsent( address, key -> new StorageConsumingMap<>(address, new ConcurrentHashMap<>(), storagePreloader)) - .put(storageSlotKey, new BonsaiValue<>(valueUInt.orElse(null), valueUInt.orElse(null))); - + .put( + storageSlotKey, new DiffBasedValue<>(valueUInt.orElse(null), valueUInt.orElse(null))); return valueUInt; } catch (MerkleTrieException e) { // need to throw to trigger the heal @@ -455,10 +457,10 @@ public UInt256 getPriorStorageValue(final Address address, final UInt256 storage // TODO maybe log the read into the trie layer? StorageSlotKey storageSlotKey = new StorageSlotKey(hashAndSaveSlotPreImage(storageKey), Optional.of(storageKey)); - final Map> localAccountStorage = + final Map> localAccountStorage = storageToUpdate.get(address); if (localAccountStorage != null) { - final BonsaiValue value = localAccountStorage.get(storageSlotKey); + final DiffBasedValue value = localAccountStorage.get(storageSlotKey); if (value != null) { if (value.isLastStepCleared()) { return UInt256.ZERO; @@ -482,11 +484,11 @@ public UInt256 getPriorStorageValue(final Address address, final UInt256 storage @Override public Map getAllAccountStorage(final Address address, final Hash rootHash) { final Map results = wrappedWorldView().getAllAccountStorage(address, rootHash); - final StorageConsumingMap> bonsaiValueStorage = + final StorageConsumingMap> diffBasedValueStorage = storageToUpdate.get(address); - if (bonsaiValueStorage != null) { + if (diffBasedValueStorage != null) { // hash the key to match the implied storage interface of hashed slotKey - bonsaiValueStorage.forEach( + diffBasedValueStorage.forEach( (key, value) -> results.put(key.getSlotHash(), value.getUpdated())); } return results; @@ -498,7 +500,7 @@ public boolean isPersisted() { } @Override - public BonsaiWorldStateKeyValueStorage getWorldStateStorage() { + public DiffBasedWorldStateKeyValueStorage getWorldStateStorage() { return wrappedWorldView().getWorldStateStorage(); } @@ -550,7 +552,7 @@ private void rollAccountChange( // non-change, a cached read. return; } - BonsaiValue accountValue = accountsToUpdate.get(address); + DiffBasedValue accountValue = accountsToUpdate.get(address); if (accountValue == null) { accountValue = loadAccountFromParent(address, accountValue); } @@ -558,7 +560,7 @@ private void rollAccountChange( if (expectedValue == null && replacementValue != null) { accountsToUpdate.put( address, - new BonsaiValue<>(null, new BonsaiAccount(this, address, replacementValue, true))); + new DiffBasedValue<>(null, createAccount(this, address, replacementValue, true))); } else { throw new IllegalStateException( String.format( @@ -572,7 +574,7 @@ private void rollAccountChange( "Expected to create account, but the account exists. Address=%s", address)); } } else { - BonsaiAccount.assertCloseEnoughForDiffing( + assertCloseEnoughForDiffing( accountValue.getUpdated(), expectedValue, "Address=" + address + " Prior Value in Rolling Change"); @@ -586,19 +588,18 @@ private void rollAccountChange( accountValue.setUpdated(null); } } else { - accountValue.setUpdated( - new BonsaiAccount(wrappedWorldView(), address, replacementValue, true)); + accountValue.setUpdated(createAccount(wrappedWorldView(), address, replacementValue, true)); } } } - private BonsaiValue loadAccountFromParent( - final Address address, final BonsaiValue defaultValue) { + private DiffBasedValue loadAccountFromParent( + final Address address, final DiffBasedValue defaultValue) { try { final Account parentAccount = wrappedWorldView().get(address); - if (parentAccount instanceof BonsaiAccount account) { - final BonsaiValue loadedAccountValue = - new BonsaiValue<>(new BonsaiAccount(account), account); + if (parentAccount instanceof DiffBasedAccount account) { + final DiffBasedValue loadedAccountValue = + new DiffBasedValue<>(copyAccount((ACCOUNT) account), ((ACCOUNT) account)); accountsToUpdate.put(address, loadedAccountValue); return loadedAccountValue; } else { @@ -617,7 +618,7 @@ private void rollCodeChange( // non-change, a cached read. return; } - BonsaiValue codeValue = codeToUpdate.get(address); + DiffBasedValue codeValue = codeToUpdate.get(address); if (codeValue == null) { final Bytes storedCode = wrappedWorldView() @@ -625,14 +626,14 @@ private void rollCodeChange( address, Optional.ofNullable(expectedCode).map(Hash::hash).orElse(Hash.EMPTY)) .orElse(Bytes.EMPTY); if (!storedCode.isEmpty()) { - codeValue = new BonsaiValue<>(storedCode, storedCode); + codeValue = new DiffBasedValue<>(storedCode, storedCode); codeToUpdate.put(address, codeValue); } } if (codeValue == null) { if ((expectedCode == null || expectedCode.isEmpty()) && replacementCode != null) { - codeToUpdate.put(address, new BonsaiValue<>(null, replacementCode)); + codeToUpdate.put(address, new DiffBasedValue<>(null, replacementCode)); } else { throw new IllegalStateException( String.format( @@ -660,10 +661,10 @@ private void rollCodeChange( } } - private Map> maybeCreateStorageMap( - final Map> storageMap, final Address address) { + private Map> maybeCreateStorageMap( + final Map> storageMap, final Address address) { if (storageMap == null) { - final StorageConsumingMap> newMap = + final StorageConsumingMap> newMap = new StorageConsumingMap<>(address, new ConcurrentHashMap<>(), storagePreloader); storageToUpdate.put(address, newMap); return newMap; @@ -685,13 +686,13 @@ private void rollStorageChange( // corner case on deletes, non-change return; } - final Map> storageMap = storageToUpdate.get(address); - BonsaiValue slotValue = storageMap == null ? null : storageMap.get(storageSlotKey); + final Map> storageMap = storageToUpdate.get(address); + DiffBasedValue slotValue = storageMap == null ? null : storageMap.get(storageSlotKey); if (slotValue == null) { final Optional storageValue = wrappedWorldView().getStorageValueByStorageSlotKey(address, storageSlotKey); if (storageValue.isPresent()) { - slotValue = new BonsaiValue<>(storageValue.get(), storageValue.get()); + slotValue = new DiffBasedValue<>(storageValue.get(), storageValue.get()); storageToUpdate .computeIfAbsent( address, @@ -703,7 +704,7 @@ private void rollStorageChange( if (slotValue == null) { if ((expectedValue == null || expectedValue.isZero()) && replacementValue != null) { maybeCreateStorageMap(storageMap, address) - .put(storageSlotKey, new BonsaiValue<>(null, replacementValue)); + .put(storageSlotKey, new DiffBasedValue<>(null, replacementValue)); } else { throw new IllegalStateException( String.format( @@ -730,7 +731,7 @@ private void rollStorageChange( existingSlotValue == null ? "null" : existingSlotValue.toShortHexString())); } if (replacementValue == null && slotValue.getPrior() == null) { - final Map> thisStorageUpdate = + final Map> thisStorageUpdate = maybeCreateStorageMap(storageMap, address); thisStorageUpdate.remove(storageSlotKey); if (thisStorageUpdate.isEmpty()) { @@ -769,67 +770,6 @@ public void reset() { storageKeyHashLookup.clear(); } - public static class AccountConsumingMap extends ForwardingMap { - - private final ConcurrentMap accounts; - private final Consumer consumer; - - public AccountConsumingMap( - final ConcurrentMap accounts, final Consumer consumer) { - this.accounts = accounts; - this.consumer = consumer; - } - - @Override - public T put(@Nonnull final Address address, @Nonnull final T value) { - consumer.process(address, value); - return accounts.put(address, value); - } - - public Consumer getConsumer() { - return consumer; - } - - @Override - protected Map delegate() { - return accounts; - } - } - - public static class StorageConsumingMap extends ForwardingMap { - - private final Address address; - - private final ConcurrentMap storages; - private final Consumer consumer; - - public StorageConsumingMap( - final Address address, final ConcurrentMap storages, final Consumer consumer) { - this.address = address; - this.storages = storages; - this.consumer = consumer; - } - - @Override - public T put(@Nonnull final K slotKey, @Nonnull final T value) { - consumer.process(address, slotKey); - return storages.put(slotKey, value); - } - - public Consumer getConsumer() { - return consumer; - } - - @Override - protected Map delegate() { - return storages; - } - } - - public interface Consumer { - void process(final Address address, T value); - } - protected Hash hashAndSaveAccountPreImage(final Address address) { // no need to save account preimage by default return Hash.hash(address); @@ -843,4 +783,33 @@ protected Hash hashAndSaveSlotPreImage(final UInt256 slotKey) { } return hash; } + + public abstract DiffBasedWorldStateUpdateAccumulator copy(); + + protected abstract ACCOUNT copyAccount(final ACCOUNT account); + + protected abstract ACCOUNT copyAccount( + final ACCOUNT toCopy, final DiffBasedWorldView context, final boolean mutable); + + protected abstract ACCOUNT createAccount( + final DiffBasedWorldView context, + final Address address, + final AccountValue stateTrieAccount, + final boolean mutable); + + protected abstract ACCOUNT createAccount( + final DiffBasedWorldView context, + final Address address, + final Hash addressHash, + final long nonce, + final Wei balance, + final Hash storageRoot, + final Hash codeHash, + final boolean mutable); + + protected abstract ACCOUNT createAccount( + final DiffBasedWorldView context, final UpdateTrackingAccount tracked); + + protected abstract void assertCloseEnoughForDiffing( + final ACCOUNT source, final AccountValue account, final String context); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/AccountConsumingMap.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/AccountConsumingMap.java new file mode 100644 index 00000000000..63c5b9233f2 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/AccountConsumingMap.java @@ -0,0 +1,49 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload; + +import org.hyperledger.besu.datatypes.Address; + +import java.util.Map; +import java.util.concurrent.ConcurrentMap; +import javax.annotation.Nonnull; + +import com.google.common.collect.ForwardingMap; + +public class AccountConsumingMap extends ForwardingMap { + + private final ConcurrentMap accounts; + private final Consumer consumer; + + public AccountConsumingMap(final ConcurrentMap accounts, final Consumer consumer) { + this.accounts = accounts; + this.consumer = consumer; + } + + @Override + public T put(@Nonnull final Address address, @Nonnull final T value) { + consumer.process(address, value); + return accounts.put(address, value); + } + + public Consumer getConsumer() { + return consumer; + } + + @Override + protected Map delegate() { + return accounts; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/Consumer.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/Consumer.java new file mode 100644 index 00000000000..577b0d8c551 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/Consumer.java @@ -0,0 +1,21 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload; + +import org.hyperledger.besu.datatypes.Address; + +public interface Consumer { + void process(final Address address, T value); +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/StorageConsumingMap.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/StorageConsumingMap.java new file mode 100644 index 00000000000..13b42d7b4a4 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/worldview/accumulator/preload/StorageConsumingMap.java @@ -0,0 +1,53 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload; + +import org.hyperledger.besu.datatypes.Address; + +import java.util.Map; +import java.util.concurrent.ConcurrentMap; +import javax.annotation.Nonnull; + +import com.google.common.collect.ForwardingMap; + +public class StorageConsumingMap extends ForwardingMap { + + private final Address address; + + private final ConcurrentMap storages; + private final Consumer consumer; + + public StorageConsumingMap( + final Address address, final ConcurrentMap storages, final Consumer consumer) { + this.address = address; + this.storages = storages; + this.consumer = consumer; + } + + @Override + public T put(@Nonnull final K slotKey, @Nonnull final T value) { + consumer.process(address, slotKey); + return storages.put(slotKey, value); + } + + public Consumer getConsumer() { + return consumer; + } + + @Override + protected Map delegate() { + return storages; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java index f689410a9de..c35fe423aee 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorageCoordinator.java @@ -15,7 +15,7 @@ package org.hyperledger.besu.ethereum.worldstate; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java index 398472ef3ae..9bac9254940 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java @@ -25,9 +25,9 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.worldview.ForestMutableWorldState; @@ -95,15 +95,15 @@ public static BonsaiWorldStateProvider createBonsaiInMemoryWorldStateArchive( final Blockchain blockchain, final EvmConfiguration evmConfiguration) { final InMemoryKeyValueStorageProvider inMemoryKeyValueStorageProvider = new InMemoryKeyValueStorageProvider(); - final CachedMerkleTrieLoader cachedMerkleTrieLoader = - new CachedMerkleTrieLoader(new NoOpMetricsSystem()); + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader = + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()); return new BonsaiWorldStateProvider( (BonsaiWorldStateKeyValueStorage) inMemoryKeyValueStorageProvider.createWorldStateStorage( DataStorageConfiguration.DEFAULT_BONSAI_CONFIG), blockchain, Optional.empty(), - cachedMerkleTrieLoader, + bonsaiCachedMerkleTrieLoader, null, evmConfiguration); } diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java index c549c66eb93..a04592d3093 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TrieGenerator.java @@ -20,7 +20,7 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrie; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java index f8fe794586f..d451a6fb7b2 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/BlockImportExceptionHandlingTest.java @@ -41,9 +41,9 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java similarity index 97% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java index ab9e87d5f61..b2fbc713869 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/AbstractIsolationTests.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider.createInMemoryBlockchain; import static org.mockito.ArgumentMatchers.any; @@ -66,8 +66,8 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -159,7 +159,7 @@ public void createStorage() { (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage, blockchain, Optional.of(16L), - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), null, EvmConfiguration.DEFAULT); var ws = archive.getMutable(); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiCachedMerkleTrieLoaderTest.java similarity index 94% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiCachedMerkleTrieLoaderTest.java index d96c6990cb2..d4e42cd516b 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/CachedMerkleTrieLoaderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiCachedMerkleTrieLoaderTest.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.assertj.core.api.Assertions.assertThat; @@ -26,8 +26,8 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; @@ -45,9 +45,9 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; -class CachedMerkleTrieLoaderTest { +class BonsaiCachedMerkleTrieLoaderTest { - private CachedMerkleTrieLoader merkleTrieLoader; + private BonsaiCachedMerkleTrieLoader merkleTrieLoader; private final StorageProvider storageProvider = new InMemoryKeyValueStorageProvider(); private final BonsaiWorldStateKeyValueStorage inMemoryWorldState = Mockito.spy( @@ -69,7 +69,7 @@ public void setup() { TrieGenerator.generateTrie( worldStateStorageCoordinator, accounts.stream().map(Address::addressHash).collect(Collectors.toList())); - merkleTrieLoader = new CachedMerkleTrieLoader(new NoOpMetricsSystem()); + merkleTrieLoader = new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()); } @Test diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiSnapshotIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiSnapshotIsolationTests.java similarity index 99% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiSnapshotIsolationTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiSnapshotIsolationTests.java index ce983e10a9a..507047604bc 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiSnapshotIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiSnapshotIsolationTests.java @@ -14,7 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProviderTest.java similarity index 89% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProviderTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProviderTest.java index 10d3381c7e0..879183ecc38 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/BonsaiWorldStateProviderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiWorldStateProviderTest.java @@ -13,13 +13,13 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.BLOCKCHAIN; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; -import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; -import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; +import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY; +import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.eq; @@ -36,13 +36,13 @@ import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; @@ -77,7 +77,7 @@ class BonsaiWorldStateProviderTest { @Mock SegmentedKeyValueStorageTransaction segmentedKeyValueStorageTransaction; BonsaiWorldStateProvider bonsaiWorldStateArchive; - @Mock CachedWorldStorageManager cachedWorldStorageManager; + @Mock BonsaiCachedWorldStorageManager cachedWorldStorageManager; @Mock TrieLogManager trieLogManager; @BeforeEach @@ -111,7 +111,7 @@ void testGetMutableReturnPersistedStateWhenNeeded() { new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_BONSAI_CONFIG), blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT); assertThat(bonsaiWorldStateArchive.getMutable(chainHead, true)) @@ -128,7 +128,7 @@ void testGetMutableReturnEmptyWhenLoadMoreThanLimitLayersBack() { DataStorageConfiguration.DEFAULT_BONSAI_CONFIG), blockchain, Optional.of(512L), - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), null, EvmConfiguration.DEFAULT); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); @@ -150,7 +150,7 @@ void testGetMutableWhenLoadLessThanLimitLayersBack() { new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_BONSAI_CONFIG), blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); final BlockHeader chainHead = blockBuilder.number(511).buildHeader(); @@ -185,7 +185,7 @@ void testGetMutableWithStorageInconsistencyRollbackTheState() { trieLogManager, worldStateKeyValueStorage, blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); @@ -214,7 +214,7 @@ void testGetMutableWithStorageConsistencyNotRollbackTheState() { trieLogManager, worldStateKeyValueStorage, blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); final BlockHeader blockHeader = blockBuilder.number(0).buildHeader(); @@ -254,7 +254,7 @@ void testGetMutableWithStorageConsistencyToRollbackAndRollForwardTheState() { trieLogManager, worldStateKeyValueStorage, blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); // initial persisted state hash key @@ -297,7 +297,7 @@ void testGetMutableWithRollbackNotOverrideTrieLogLayer() { new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_BONSAI_CONFIG), blockchain, - new CachedMerkleTrieLoader(new NoOpMetricsSystem()), + new BonsaiCachedMerkleTrieLoader(new NoOpMetricsSystem()), EvmConfiguration.DEFAULT)); // initial persisted state hash key diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/LogRollingTests.java similarity index 96% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/LogRollingTests.java index ee63fab968f..331c6b1afdb 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/LogRollingTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/LogRollingTests.java @@ -14,7 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; @@ -29,11 +29,11 @@ import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.account.MutableAccount; import org.hyperledger.besu.evm.internal.EvmConfiguration; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/RollingImport.java similarity index 90% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/RollingImport.java index a43fce8f402..134d094a4dd 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/RollingImport.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/RollingImport.java @@ -14,7 +14,7 @@ * */ -package org.hyperledger.besu.ethereum.trie.bonsai; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; import static com.google.common.base.Preconditions.checkArgument; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE; @@ -25,11 +25,11 @@ import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogFactoryImpl; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogLayer; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog.TrieLogFactoryImpl; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java similarity index 99% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java index 911ff10005e..c4769248e9c 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/BonsaiWorldStateKeyValueStorageTest.java @@ -12,11 +12,11 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage; import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE; -import static org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; +import static org.hyperledger.besu.ethereum.trie.diffbased.common.storage.DiffBasedWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryTests.java similarity index 94% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryTests.java index 64f96d8b734..59c208792e4 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogFactoryTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogFactoryTests.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; @@ -24,6 +24,7 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; import org.hyperledger.besu.ethereum.core.BlockchainSetupUtil; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.trielogs.TrieLog; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogManagerTests.java similarity index 84% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogManagerTests.java index e5c716e413a..a6ff8d3a7c8 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogManagerTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogManagerTests.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.spy; @@ -22,9 +22,10 @@ import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; import org.hyperledger.besu.evm.internal.EvmConfiguration; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogPrunerTest.java similarity index 96% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogPrunerTest.java index ac84d35be40..19e63f6f940 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogPrunerTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/trielog/TrieLogPrunerTest.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.trielog; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; @@ -26,7 +26,10 @@ import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogAddedEvent; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogLayer; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; import java.util.Optional; import java.util.function.Consumer; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateTest.java similarity index 81% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateTest.java index f875f098bdc..8abae1f2b6b 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/worldview/BonsaiWorldStateTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiWorldStateTest.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.worldview; +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -23,8 +23,8 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; import org.hyperledger.besu.evm.internal.EvmConfiguration; import java.util.HashMap; @@ -69,8 +69,8 @@ void setup() { @MethodSource("priorAndUpdatedEmptyAndNullBytes") void codeUpdateDoesNothingWhenMarkedAsDeletedButAlreadyDeleted( final Bytes prior, final Bytes updated) { - final Map> codeToUpdate = - Map.of(Address.ZERO, new BonsaiValue<>(prior, updated)); + final Map> codeToUpdate = + Map.of(Address.ZERO, new DiffBasedValue<>(prior, updated)); when(bonsaiWorldStateUpdateAccumulator.getCodeToUpdate()).thenReturn(codeToUpdate); worldState.updateCode(Optional.of(bonsaiUpdater), bonsaiWorldStateUpdateAccumulator); @@ -79,8 +79,8 @@ void codeUpdateDoesNothingWhenMarkedAsDeletedButAlreadyDeleted( @Test void codeUpdateDoesNothingWhenAddingSameAsExistingValue() { - final Map> codeToUpdate = - Map.of(Address.ZERO, new BonsaiValue<>(CODE, CODE)); + final Map> codeToUpdate = + Map.of(Address.ZERO, new DiffBasedValue<>(CODE, CODE)); when(bonsaiWorldStateUpdateAccumulator.getCodeToUpdate()).thenReturn(codeToUpdate); worldState.updateCode(Optional.of(bonsaiUpdater), bonsaiWorldStateUpdateAccumulator); @@ -90,8 +90,8 @@ void codeUpdateDoesNothingWhenAddingSameAsExistingValue() { @ParameterizedTest @MethodSource("emptyAndNullBytes") void removesCodeWhenMarkedAsDeleted(final Bytes updated) { - final Map> codeToUpdate = - Map.of(Address.ZERO, new BonsaiValue<>(CODE, updated)); + final Map> codeToUpdate = + Map.of(Address.ZERO, new DiffBasedValue<>(CODE, updated)); when(bonsaiWorldStateUpdateAccumulator.getCodeToUpdate()).thenReturn(codeToUpdate); worldState.updateCode(Optional.of(bonsaiUpdater), bonsaiWorldStateUpdateAccumulator); @@ -101,8 +101,8 @@ void removesCodeWhenMarkedAsDeleted(final Bytes updated) { @ParameterizedTest @MethodSource("codeValueAndEmptyAndNullBytes") void addsCodeForNewCodeValue(final Bytes prior) { - final Map> codeToUpdate = - Map.of(ACCOUNT, new BonsaiValue<>(prior, CODE)); + final Map> codeToUpdate = + Map.of(ACCOUNT, new DiffBasedValue<>(prior, CODE)); when(bonsaiWorldStateUpdateAccumulator.getCodeToUpdate()).thenReturn(codeToUpdate); worldState.updateCode(Optional.of(bonsaiUpdater), bonsaiWorldStateUpdateAccumulator); @@ -112,10 +112,10 @@ void addsCodeForNewCodeValue(final Bytes prior) { @Test void updateCodeForMultipleValues() { - final Map> codeToUpdate = new HashMap<>(); - codeToUpdate.put(Address.fromHexString("0x1"), new BonsaiValue<>(null, CODE)); - codeToUpdate.put(Address.fromHexString("0x2"), new BonsaiValue<>(CODE, null)); - codeToUpdate.put(Address.fromHexString("0x3"), new BonsaiValue<>(Bytes.of(9), CODE)); + final Map> codeToUpdate = new HashMap<>(); + codeToUpdate.put(Address.fromHexString("0x1"), new DiffBasedValue<>(null, CODE)); + codeToUpdate.put(Address.fromHexString("0x2"), new DiffBasedValue<>(CODE, null)); + codeToUpdate.put(Address.fromHexString("0x3"), new DiffBasedValue<>(Bytes.of(9), CODE)); when(bonsaiWorldStateUpdateAccumulator.getCodeToUpdate()).thenReturn(codeToUpdate); worldState.updateCode(Optional.of(bonsaiUpdater), bonsaiWorldStateUpdateAccumulator); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProviderTest.java similarity index 97% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProviderTest.java index 2340a6e8b05..5199bc19db4 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/storage/flat/FlatDbStrategyProviderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/flat/FlatDbStrategyProviderTest.java @@ -13,13 +13,15 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat; +package org.hyperledger.besu.ethereum.trie.diffbased.common.storage.flat; import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.FullFlatDbStrategy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.PartialFlatDbStrategy; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayerTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayerTests.java similarity index 98% rename from ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayerTests.java rename to ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayerTests.java index d7d49bace04..d967ca1a1b4 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/trielog/TrieLogLayerTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/common/trielog/TrieLogLayerTests.java @@ -13,7 +13,7 @@ * SPDX-License-Identifier: Apache-2.0 * */ -package org.hyperledger.besu.ethereum.trie.bonsai.trielog; +package org.hyperledger.besu.ethereum.trie.diffbased.common.trielog; import org.hyperledger.besu.datatypes.AccountValue; import org.hyperledger.besu.datatypes.Address; diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestKeyValueStorageWorldStateStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestKeyValueStorageWorldStateStorageTest.java index 08dfc86d5c0..d60f92448cc 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestKeyValueStorageWorldStateStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/storage/ForestKeyValueStorageWorldStateStorageTest.java @@ -66,6 +66,7 @@ public void getCode_saveAndGetSpecialValues() { storage.updater().putCode(MerkleTrie.EMPTY_TRIE_NODE).putCode(Bytes.EMPTY).commit(); assertThat(storage.getCode(Hash.EMPTY_TRIE_HASH)).contains(MerkleTrie.EMPTY_TRIE_NODE); + assertThat(storage.getCode(Hash.EMPTY)).contains(Bytes.EMPTY); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java index 34c906bd90d..1a82d56355f 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java @@ -34,7 +34,7 @@ import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiWorldStateProvider; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; import org.hyperledger.besu.ethereum.trie.forest.pruner.Pruner; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.BesuMetricCategory; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java index 87a021136e2..7b64f885c29 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloader.java @@ -21,7 +21,7 @@ import org.hyperledger.besu.ethereum.eth.sync.TrailingPeerRequirements; import org.hyperledger.besu.ethereum.eth.sync.worldstate.StalledDownloadException; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.services.tasks.TaskCollection; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStep.java index 264b29017d7..e9fb43ffeaf 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStep.java @@ -20,7 +20,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.TrieNodeHealingRequest; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.exception.StorageException; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java index 707634664bb..3d93b852297 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java @@ -30,7 +30,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.StorageFlatDatabaseHealingRangeRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldDownloadState; import org.hyperledger.besu.ethereum.trie.RangeManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java index 76a07a4d8ea..471c84a6321 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java @@ -28,7 +28,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; import org.hyperledger.besu.ethereum.trie.RangeManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java index 94ba92b5356..bb98534fd0e 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.trie.NodeUpdater; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java index 48c25fadcd5..81b4897a863 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.trie.CompactEncoding; import org.hyperledger.besu.ethereum.trie.NodeUpdater; import org.hyperledger.besu.ethereum.trie.RangeManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java index 5c114d647a1..f2514b34055 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java index 82a9e3f5510..ed81fe65d50 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java @@ -27,7 +27,7 @@ import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java index 822ac5095d4..e5c8ce55583 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointSyncChainDownloaderTest.java @@ -36,7 +36,7 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.checkpoint.ImmutableCheckpoint; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java index 765eac4c5c7..e100c036989 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastDownloaderFactoryTest.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate.FastDownloaderFactory; import org.hyperledger.besu.ethereum.eth.sync.state.SyncState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java index 0c625c61344..ae252a39f47 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncDownloaderTest.java @@ -34,7 +34,7 @@ import org.hyperledger.besu.ethereum.eth.sync.fastsync.worldstate.NodeDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.StalledDownloadException; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java index 3555a9b7179..44847ffb93e 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/FastWorldDownloadStateTest.java @@ -26,7 +26,7 @@ import org.hyperledger.besu.ethereum.eth.manager.task.EthTask; import org.hyperledger.besu.ethereum.eth.sync.worldstate.StalledDownloadException; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloadProcess; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java index e4a827b7f00..6707ed97054 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/worldstate/LoadLocalDataStepTest.java @@ -22,7 +22,7 @@ import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java index 296952cf029..03aa292fabf 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java index 9a53a4418b4..0de00f5ab2d 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/PersistDataStepTest.java @@ -25,7 +25,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.BytecodeRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.StorageRangeDataRequest; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java index 18ab5f02fa7..35c5de5daca 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java @@ -39,7 +39,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloadProcess; import org.hyperledger.besu.ethereum.trie.RangeManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java index 9b5d9355168..81b9d385763 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java @@ -27,7 +27,7 @@ import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; @@ -51,6 +51,7 @@ public static List> createAccountRequest(final boolean wit new InMemoryKeyValueStorageProvider(), new NoOpMetricsSystem(), DataStorageConfiguration.DEFAULT_BONSAI_CONFIG); + final WorldStateStorageCoordinator worldStateStorageCoordinator = new WorldStateStorageCoordinator(worldStateKeyValueStorage); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java index 6c3ba6f0daa..dba37b5afa3 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java @@ -29,7 +29,7 @@ import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java index eb6eedfb4c4..da5354f71bf 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java @@ -31,7 +31,7 @@ import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; @@ -89,7 +89,6 @@ public void setup() { DataStorageConfiguration.DEFAULT_BONSAI_CONFIG); worldStateStorageCoordinator = new WorldStateStorageCoordinator(worldStateKeyValueStorage); proofProvider = new WorldStateProofProvider(worldStateStorageCoordinator); - trie = TrieGenerator.generateTrie( worldStateStorageCoordinator, diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java index 1f87a2d783e..5cec07c152f 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageTrieNodeHealingRequestTest.java @@ -21,7 +21,7 @@ import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java index c64bbec90d8..ee4730ffda3 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestUpdateAccumulator.java @@ -17,11 +17,13 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.datatypes.StorageSlotKey; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiValue; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiPreImageProxy; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiPreImageProxy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.DiffBasedValue; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.Consumer; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.preload.StorageConsumingMap; import org.hyperledger.besu.evm.internal.EvmConfiguration; import java.util.concurrent.ConcurrentHashMap; @@ -32,8 +34,8 @@ public class BonsaiReferenceTestUpdateAccumulator extends BonsaiWorldStateUpdate private final BonsaiPreImageProxy preImageProxy; public BonsaiReferenceTestUpdateAccumulator( - final BonsaiWorldView world, - final Consumer> accountPreloader, + final DiffBasedWorldView world, + final Consumer> accountPreloader, final Consumer storagePreloader, final BonsaiPreImageProxy preImageProxy, final EvmConfiguration evmConfiguration) { @@ -65,7 +67,7 @@ public BonsaiReferenceTestUpdateAccumulator createDetachedAccumulator() { getStorageToUpdate() .forEach( (k, v) -> { - StorageConsumingMap> newMap = + StorageConsumingMap> newMap = new StorageConsumingMap<>(k, new ConcurrentHashMap<>(), v.getConsumer()); v.forEach((key, value) -> newMap.put(key, value.copy())); copy.getStorageToUpdate().put(k, newMap); diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java index 1c536e96a38..f16452f9360 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldState.java @@ -18,16 +18,18 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.CachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.cache.NoOpCachedWorldStorageManager; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiPreImageProxy; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateLayerStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogAddedEvent; -import org.hyperledger.besu.ethereum.trie.bonsai.trielog.TrieLogManager; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.NoOpBonsaiCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiPreImageProxy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldStateUpdateAccumulator; +import org.hyperledger.besu.ethereum.trie.diffbased.common.cache.DiffBasedCachedWorldStorageManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogAddedEvent; +import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldState; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.accumulator.DiffBasedWorldStateUpdateAccumulator; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.internal.EvmConfiguration; import org.hyperledger.besu.evm.worldstate.WorldUpdater; @@ -54,14 +56,14 @@ public class BonsaiReferenceTestWorldState extends BonsaiWorldState protected BonsaiReferenceTestWorldState( final BonsaiReferenceTestWorldStateStorage worldStateKeyValueStorage, - final CachedMerkleTrieLoader cachedMerkleTrieLoader, - final CachedWorldStorageManager cachedWorldStorageManager, + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader, + final DiffBasedCachedWorldStorageManager cachedWorldStorageManager, final TrieLogManager trieLogManager, final BonsaiPreImageProxy preImageProxy, final EvmConfiguration evmConfiguration) { super( worldStateKeyValueStorage, - cachedMerkleTrieLoader, + bonsaiCachedMerkleTrieLoader, cachedWorldStorageManager, trieLogManager, evmConfiguration); @@ -72,21 +74,21 @@ protected BonsaiReferenceTestWorldState( new BonsaiReferenceTestUpdateAccumulator( this, (addr, value) -> - cachedMerkleTrieLoader.preLoadAccount( + bonsaiCachedMerkleTrieLoader.preLoadAccount( getWorldStateStorage(), worldStateRootHash, addr), (addr, value) -> - cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value), + bonsaiCachedMerkleTrieLoader.preLoadStorageSlot( + getWorldStateStorage(), addr, value), preImageProxy, evmConfiguration)); } @Override public ReferenceTestWorldState copy() { - var layerCopy = - new BonsaiReferenceTestWorldStateStorage(worldStateKeyValueStorage, preImageProxy); + var layerCopy = new BonsaiReferenceTestWorldStateStorage(getWorldStateStorage(), preImageProxy); return new BonsaiReferenceTestWorldState( layerCopy, - cachedMerkleTrieLoader, + bonsaiCachedMerkleTrieLoader, cachedWorldStorageManager, trieLogManager, preImageProxy, @@ -187,8 +189,9 @@ private void generateTrieLogFromState( private BonsaiWorldState createBonsaiWorldState(final boolean isFrozen) { BonsaiWorldState bonsaiWorldState = new BonsaiWorldState( - new BonsaiWorldStateLayerStorage(worldStateKeyValueStorage), - cachedMerkleTrieLoader, + new BonsaiWorldStateLayerStorage( + (BonsaiWorldStateKeyValueStorage) worldStateKeyValueStorage), + bonsaiCachedMerkleTrieLoader, cachedWorldStorageManager, trieLogManager, evmConfiguration); @@ -209,8 +212,11 @@ public static BonsaiReferenceTestWorldState create( final Map accounts, final EvmConfiguration evmConfiguration) { final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem(); - final CachedMerkleTrieLoader cachedMerkleTrieLoader = new CachedMerkleTrieLoader(metricsSystem); + + final BonsaiCachedMerkleTrieLoader bonsaiCachedMerkleTrieLoader = + new BonsaiCachedMerkleTrieLoader(metricsSystem); final TrieLogManager trieLogManager = new ReferenceTestsInMemoryTrieLogManager(); + final BonsaiPreImageProxy preImageProxy = new BonsaiPreImageProxy.BonsaiReferenceTestPreImageProxy(); @@ -223,13 +229,13 @@ public static BonsaiReferenceTestWorldState create( final BonsaiReferenceTestWorldStateStorage worldStateKeyValueStorage = new BonsaiReferenceTestWorldStateStorage(bonsaiWorldStateKeyValueStorage, preImageProxy); - final NoOpCachedWorldStorageManager noOpCachedWorldStorageManager = - new NoOpCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage); + final NoOpBonsaiCachedWorldStorageManager noOpCachedWorldStorageManager = + new NoOpBonsaiCachedWorldStorageManager(bonsaiWorldStateKeyValueStorage); final BonsaiReferenceTestWorldState worldState = new BonsaiReferenceTestWorldState( worldStateKeyValueStorage, - cachedMerkleTrieLoader, + bonsaiCachedMerkleTrieLoader, noOpCachedWorldStorageManager, trieLogManager, preImageProxy, @@ -260,10 +266,10 @@ public ReferenceTestsInMemoryTrieLogManager() { @Override public synchronized void saveTrieLog( - final BonsaiWorldStateUpdateAccumulator localUpdater, + final DiffBasedWorldStateUpdateAccumulator localUpdater, final Hash forWorldStateRootHash, final BlockHeader forBlockHeader, - final BonsaiWorldState forWorldState) { + final DiffBasedWorldState forWorldState) { // notify trie log added observers, synchronously TrieLog trieLog = trieLogFactory.create(localUpdater, forBlockHeader); trieLogCache.put(forBlockHeader.getHash(), trieLogFactory.serialize(trieLog)); diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java index 68057523a08..0993d4425be 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BonsaiReferenceTestWorldStateStorage.java @@ -16,11 +16,11 @@ import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.trie.bonsai.BonsaiAccount; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiPreImageProxy; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateLayerStorage; -import org.hyperledger.besu.ethereum.trie.bonsai.worldview.BonsaiWorldView; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiAccount; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiPreImageProxy; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateLayerStorage; +import org.hyperledger.besu.ethereum.trie.diffbased.common.worldview.DiffBasedWorldView; import org.hyperledger.besu.evm.account.AccountStorageEntry; import org.hyperledger.besu.evm.worldstate.WorldState; @@ -64,7 +64,7 @@ public NavigableMap storageEntriesFrom( } public Stream streamAccounts( - final BonsaiWorldView context, final Bytes32 startKeyHash, final int limit) { + final DiffBasedWorldView context, final Bytes32 startKeyHash, final int limit) { return streamFlatAccounts(startKeyHash, UInt256.MAX_VALUE, limit) .entrySet() // map back to addresses using preImage provider: diff --git a/gradle.properties b/gradle.properties index 265507cea43..61b69b61768 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,4 +13,4 @@ org.gradle.jvmargs=-Xmx4g \ --add-opens jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED \ --add-opens jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED # Could be moved to sonar properties after https://sonarsource.atlassian.net/browse/SONARGRADL-134 -systemProp.sonar.gradle.skipCompile=true +systemProp.sonar.gradle.skipCompile=true \ No newline at end of file From 5bc81ae1811c2aeec29792f567335fa258f38010 Mon Sep 17 00:00:00 2001 From: Matt Whitehead Date: Wed, 27 Mar 2024 17:05:00 +0000 Subject: [PATCH 08/24] Ensure empty withdrawal lists are set in BFT blocks when the protocol spec is shanghai or higher (#6765) * Ensure empty withdrawal lists are set in BFT blocks when the protocol schedule is shanghai or higher Signed-off-by: Matthew Whitehead * Unit tests missing mock Signed-off-by: Matthew Whitehead * Unit tests missing mock Signed-off-by: Matthew Whitehead * Refactor and reduce code duplication Signed-off-by: Matthew Whitehead --------- Signed-off-by: Matthew Whitehead Signed-off-by: Matt Whitehead --- CHANGELOG.md | 1 + .../bft/BaseBftProtocolScheduleBuilder.java | 4 - .../bft/blockcreation/BftBlockCreator.java | 23 ++++ .../blockcreation/BftBlockCreatorTest.java | 121 ++++++++++++++++++ .../ibft/statemachine/IbftRoundTest.java | 4 + .../blockcreation/PkiQbftBlockCreator.java | 44 ++++++- .../QbftBlockCreatorFactory.java | 6 +- .../PkiQbftBlockCreatorTest.java | 31 ++++- .../qbft/messagewrappers/ProposalTest.java | 6 +- .../blockcreation/AbstractBlockCreator.java | 20 +++ .../ethereum/blockcreation/BlockCreator.java | 2 + 11 files changed, 250 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5b24a6f89a..508f4925048 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ - Make block transaction selection max time aware of PoA transitions [#6676](https://github.com/hyperledger/besu/pull/6676) - Don't enable the BFT mining coordinator when running sub commands such as `blocks export` [#6675](https://github.com/hyperledger/besu/pull/6675) - In JSON-RPC return optional `v` fields for type 1 and type 2 transactions [#6762](https://github.com/hyperledger/besu/pull/6762) +- Fix Shanghai/QBFT block import bug when syncing new nodes [#6765](https://github.com/hyperledger/besu/pull/6765) ### Download Links diff --git a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/BaseBftProtocolScheduleBuilder.java b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/BaseBftProtocolScheduleBuilder.java index 2578a805686..166a2d84a36 100644 --- a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/BaseBftProtocolScheduleBuilder.java +++ b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/BaseBftProtocolScheduleBuilder.java @@ -30,7 +30,6 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolScheduleBuilder; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpecAdapters; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpecBuilder; -import org.hyperledger.besu.ethereum.mainnet.WithdrawalsValidator; import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -125,9 +124,6 @@ private ProtocolSpecBuilder applyBftChanges( .skipZeroBlockRewards(true) .blockHeaderFunctions(BftBlockHeaderFunctions.forOnchainBlock(bftExtraDataCodec)) .blockReward(Wei.of(configOptions.getBlockRewardWei())) - .withdrawalsValidator( - new WithdrawalsValidator - .ProhibitedWithdrawals()) // QBFT/IBFT doesn't support withdrawals .miningBeneficiaryCalculator( header -> configOptions.getMiningBeneficiary().orElseGet(header::getCoinbase)); } diff --git a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/blockcreation/BftBlockCreator.java b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/blockcreation/BftBlockCreator.java index e6a981cec78..b84cd665f14 100644 --- a/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/blockcreation/BftBlockCreator.java +++ b/consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/blockcreation/BftBlockCreator.java @@ -19,6 +19,7 @@ import org.hyperledger.besu.consensus.common.bft.BftBlockHeaderFunctions; import org.hyperledger.besu.consensus.common.bft.BftExtraDataCodec; import org.hyperledger.besu.consensus.common.bft.BftHelpers; +import org.hyperledger.besu.consensus.common.bft.BftProtocolSchedule; import org.hyperledger.besu.datatypes.Address; import org.hyperledger.besu.ethereum.ProtocolContext; import org.hyperledger.besu.ethereum.blockcreation.AbstractBlockCreator; @@ -29,7 +30,10 @@ import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.mainnet.WithdrawalsValidator; +import java.util.Collections; import java.util.Optional; /** The Bft block creator. */ @@ -77,12 +81,31 @@ public BftBlockCreator( this.bftExtraDataCodec = bftExtraDataCodec; } + @Override + public BlockCreationResult createBlock(final long timestamp) { + ProtocolSpec protocolSpec = + ((BftProtocolSchedule) protocolSchedule) + .getByBlockNumberOrTimestamp(parentHeader.getNumber() + 1, timestamp); + + if (protocolSpec.getWithdrawalsValidator() instanceof WithdrawalsValidator.AllowedWithdrawals) { + return createEmptyWithdrawalsBlock(timestamp); + } else { + return createBlock(Optional.empty(), Optional.empty(), timestamp); + } + } + private static MiningBeneficiaryCalculator miningBeneficiaryCalculator( final Address localAddress, final ForksSchedule forksSchedule) { return blockNum -> forksSchedule.getFork(blockNum).getValue().getMiningBeneficiary().orElse(localAddress); } + @Override + public BlockCreationResult createEmptyWithdrawalsBlock(final long timestamp) { + return createBlock( + Optional.empty(), Optional.empty(), Optional.of(Collections.emptyList()), timestamp); + } + @Override protected BlockHeader createFinalBlockHeader(final SealableBlockHeader sealableBlockHeader) { final BlockHeaderBuilder builder = diff --git a/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/blockcreation/BftBlockCreatorTest.java b/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/blockcreation/BftBlockCreatorTest.java index 25c5c1d10b8..44e082af7f1 100644 --- a/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/blockcreation/BftBlockCreatorTest.java +++ b/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/blockcreation/BftBlockCreatorTest.java @@ -213,4 +213,125 @@ public BlockHeaderValidator.Builder createBlockHeaderRuleset( new BftBlockHashing(bftExtraDataEncoder) .calculateDataHashForCommittedSeal(header, extraData)); } + + @Test + public void testBlockCreationResultsInEmptyWithdrawalsListShanghaiOnwards() { + // Construct a parent block. + final BlockHeaderTestFixture blockHeaderBuilder = new BlockHeaderTestFixture(); + blockHeaderBuilder.gasLimit(5000); // required to pass validation rule checks. + final BlockHeader parentHeader = blockHeaderBuilder.buildHeader(); + final Optional optionalHeader = Optional.of(parentHeader); + + // Construct a blockchain and world state + final MutableBlockchain blockchain = mock(MutableBlockchain.class); + when(blockchain.getChainHeadHash()).thenReturn(parentHeader.getHash()); + when(blockchain.getBlockHeader(any())).thenReturn(optionalHeader); + final BlockHeader blockHeader = mock(BlockHeader.class); + when(blockHeader.getBaseFee()).thenReturn(Optional.empty()); + when(blockchain.getChainHeadHeader()).thenReturn(blockHeader); + + final List
initialValidatorList = Lists.newArrayList(); + for (int i = 0; i < 4; i++) { + initialValidatorList.add(AddressHelpers.ofValue(i)); + } + + final IbftExtraDataCodec bftExtraDataEncoder = new IbftExtraDataCodec(); + + final BaseBftProtocolScheduleBuilder bftProtocolSchedule = + new BaseBftProtocolScheduleBuilder() { + @Override + public BlockHeaderValidator.Builder createBlockHeaderRuleset( + final BftConfigOptions config, final FeeMarket feeMarket) { + return IbftBlockHeaderValidationRulesetFactory.blockHeaderValidator( + 5, Optional.empty()); + } + }; + final GenesisConfigOptions configOptions = + GenesisConfigFile.fromConfig("{\"config\": {\"shanghaiTime\":0}}").getConfigOptions(); + final ForksSchedule forksSchedule = + new ForksSchedule<>(List.of(new ForkSpec<>(0, configOptions.getBftConfigOptions()))); + final ProtocolSchedule protocolSchedule = + bftProtocolSchedule.createProtocolSchedule( + configOptions, + forksSchedule, + PrivacyParameters.DEFAULT, + false, + bftExtraDataEncoder, + EvmConfiguration.DEFAULT, + MiningParameters.MINING_DISABLED, + new BadBlockManager()); + final ProtocolContext protContext = + new ProtocolContext( + blockchain, + createInMemoryWorldStateArchive(), + setupContextWithBftExtraDataEncoder(initialValidatorList, bftExtraDataEncoder), + new BadBlockManager()); + + final TransactionPoolConfiguration poolConf = + ImmutableTransactionPoolConfiguration.builder().txPoolMaxSize(1).build(); + + final GasPricePendingTransactionsSorter pendingTransactions = + new GasPricePendingTransactionsSorter( + poolConf, + TestClock.system(ZoneId.systemDefault()), + metricsSystem, + blockchain::getChainHeadHeader); + + final EthContext ethContext = mock(EthContext.class, RETURNS_DEEP_STUBS); + when(ethContext.getEthPeers().subscribeConnect(any())).thenReturn(1L); + + final TransactionPool transactionPool = + new TransactionPool( + () -> pendingTransactions, + protocolSchedule, + protContext, + mock(TransactionBroadcaster.class), + ethContext, + new TransactionPoolMetrics(metricsSystem), + poolConf); + + transactionPool.setEnabled(); + + final MiningParameters miningParameters = + ImmutableMiningParameters.builder() + .mutableInitValues( + MutableInitValues.builder() + .extraData( + bftExtraDataEncoder.encode( + new BftExtraData( + Bytes.wrap(new byte[32]), + Collections.emptyList(), + Optional.empty(), + 0, + initialValidatorList))) + .minTransactionGasPrice(Wei.ZERO) + .coinbase(AddressHelpers.ofValue(1)) + .build()) + .build(); + + final BftBlockCreator blockCreator = + new BftBlockCreator( + miningParameters, + forksSchedule, + initialValidatorList.get(0), + parent -> + bftExtraDataEncoder.encode( + new BftExtraData( + Bytes.wrap(new byte[32]), + Collections.emptyList(), + Optional.empty(), + 0, + initialValidatorList)), + transactionPool, + protContext, + protocolSchedule, + parentHeader, + bftExtraDataEncoder, + new DeterministicEthScheduler()); + + final Block block = blockCreator.createBlock(parentHeader.getTimestamp() + 1).getBlock(); + + // Test that a BFT block contains an empty withdrawals list (not an optional empty) + assertThat(block.getBody().getWithdrawals().isPresent()).isTrue(); + } } diff --git a/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/statemachine/IbftRoundTest.java b/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/statemachine/IbftRoundTest.java index 5057b057409..e93f2f9f40c 100644 --- a/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/statemachine/IbftRoundTest.java +++ b/consensus/ibft/src/test/java/org/hyperledger/besu/consensus/ibft/statemachine/IbftRoundTest.java @@ -59,6 +59,7 @@ import org.hyperledger.besu.ethereum.core.BlockImporter; import org.hyperledger.besu.ethereum.mainnet.BlockImportResult; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.mainnet.WithdrawalsValidator; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.plugin.services.securitymodule.SecurityModuleException; import org.hyperledger.besu.util.Subscribers; @@ -273,6 +274,9 @@ public void twoValidatorNetworkSendsPrepareOnProposalReceptionThenSendsCommitOnC @Test public void localNodeProposesToNetworkOfTwoValidatorsImportsOnReceptionOfCommitFromPeer() { + lenient() + .when(protocolSpec.getWithdrawalsValidator()) + .thenReturn(new WithdrawalsValidator.AllowedWithdrawals()); final RoundState roundState = new RoundState(roundIdentifier, 2, messageValidator); final IbftRound round = new IbftRound( diff --git a/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/blockcreation/PkiQbftBlockCreator.java b/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/blockcreation/PkiQbftBlockCreator.java index 4ec6ea9f8aa..ea4c851b7f2 100644 --- a/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/blockcreation/PkiQbftBlockCreator.java +++ b/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/blockcreation/PkiQbftBlockCreator.java @@ -19,6 +19,7 @@ import org.hyperledger.besu.consensus.common.bft.BftBlockHeaderFunctions; import org.hyperledger.besu.consensus.common.bft.BftExtraData; import org.hyperledger.besu.consensus.common.bft.BftExtraDataCodec; +import org.hyperledger.besu.consensus.common.bft.BftProtocolSchedule; import org.hyperledger.besu.consensus.qbft.pki.PkiBlockCreationConfiguration; import org.hyperledger.besu.consensus.qbft.pki.PkiQbftBlockHeaderFunctions; import org.hyperledger.besu.consensus.qbft.pki.PkiQbftExtraData; @@ -29,6 +30,9 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderBuilder; import org.hyperledger.besu.ethereum.core.Transaction; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.mainnet.WithdrawalsValidator; import org.hyperledger.besu.pki.cms.CmsCreator; import java.util.Collections; @@ -48,6 +52,8 @@ public class PkiQbftBlockCreator implements BlockCreator { private final BlockCreator blockCreator; private final PkiQbftExtraDataCodec pkiQbftExtraDataCodec; private final CmsCreator cmsCreator; + private final BlockHeader parentHeader; + private final ProtocolSchedule protocolSchedule; /** * Instantiates a new Pki qbft block creator. @@ -55,17 +61,24 @@ public class PkiQbftBlockCreator implements BlockCreator { * @param blockCreator the block creator * @param pkiBlockCreationConfiguration the pki block creation configuration * @param pkiQbftExtraDataCodec the pki qbft extra data codec + * @param parentHeader the block header of the parent block + * @param protocolSchedule the protocol schedule (the type of block can vary based on the current + * protocol spec) */ public PkiQbftBlockCreator( final BlockCreator blockCreator, final PkiBlockCreationConfiguration pkiBlockCreationConfiguration, - final BftExtraDataCodec pkiQbftExtraDataCodec) { + final BftExtraDataCodec pkiQbftExtraDataCodec, + final BlockHeader parentHeader, + final ProtocolSchedule protocolSchedule) { this( blockCreator, new CmsCreator( pkiBlockCreationConfiguration.getKeyStore(), pkiBlockCreationConfiguration.getCertificateAlias()), - pkiQbftExtraDataCodec); + pkiQbftExtraDataCodec, + parentHeader, + protocolSchedule); } /** @@ -74,14 +87,21 @@ public PkiQbftBlockCreator( * @param blockCreator the block creator * @param cmsCreator the cms creator * @param bftExtraDataCodec the bft extra data codec + * @param parentHeader the block header of the parent block + * @param protocolSchedule the protocol schedule (the type of block can vary based on the current + * protocol spec) */ @VisibleForTesting public PkiQbftBlockCreator( final BlockCreator blockCreator, final CmsCreator cmsCreator, - final BftExtraDataCodec bftExtraDataCodec) { + final BftExtraDataCodec bftExtraDataCodec, + final BlockHeader parentHeader, + final ProtocolSchedule protocolSchedule) { this.blockCreator = blockCreator; this.cmsCreator = cmsCreator; + this.protocolSchedule = protocolSchedule; + this.parentHeader = parentHeader; checkArgument( bftExtraDataCodec instanceof PkiQbftExtraDataCodec, @@ -91,7 +111,16 @@ public PkiQbftBlockCreator( @Override public BlockCreationResult createBlock(final long timestamp) { - final BlockCreationResult blockCreationResult = blockCreator.createBlock(timestamp); + ProtocolSpec protocolSpec = + ((BftProtocolSchedule) protocolSchedule) + .getByBlockNumberOrTimestamp(parentHeader.getNumber() + 1, timestamp); + + final BlockCreationResult blockCreationResult; + if (protocolSpec.getWithdrawalsValidator() instanceof WithdrawalsValidator.AllowedWithdrawals) { + blockCreationResult = blockCreator.createEmptyWithdrawalsBlock(timestamp); + } else { + blockCreationResult = blockCreator.createBlock(timestamp); + } return replaceCmsInBlock(blockCreationResult); } @@ -114,6 +143,13 @@ public BlockCreationResult createBlock( timestamp); } + @Override + public BlockCreationResult createEmptyWithdrawalsBlock(final long timestamp) { + final BlockCreationResult blockCreationResult = + blockCreator.createEmptyWithdrawalsBlock(timestamp); + return replaceCmsInBlock(blockCreationResult); + } + private BlockCreationResult replaceCmsInBlock(final BlockCreationResult blockCreationResult) { final Block block = blockCreationResult.getBlock(); final BlockHeader blockHeader = block.getHeader(); diff --git a/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/blockcreation/QbftBlockCreatorFactory.java b/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/blockcreation/QbftBlockCreatorFactory.java index 5bb87363b39..9cbc64e787e 100644 --- a/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/blockcreation/QbftBlockCreatorFactory.java +++ b/consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/blockcreation/QbftBlockCreatorFactory.java @@ -77,7 +77,11 @@ public BlockCreator create(final BlockHeader parentHeader, final int round) { return blockCreator; } else { return new PkiQbftBlockCreator( - blockCreator, qbftContext.getPkiBlockCreationConfiguration().get(), bftExtraDataCodec); + blockCreator, + qbftContext.getPkiBlockCreationConfiguration().get(), + bftExtraDataCodec, + parentHeader, + protocolSchedule); } } diff --git a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/blockcreation/PkiQbftBlockCreatorTest.java b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/blockcreation/PkiQbftBlockCreatorTest.java index 669563d6f5b..1a597b790e6 100644 --- a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/blockcreation/PkiQbftBlockCreatorTest.java +++ b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/blockcreation/PkiQbftBlockCreatorTest.java @@ -18,6 +18,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.hyperledger.besu.consensus.common.bft.BftExtraDataFixture.createExtraData; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -25,6 +26,7 @@ import org.hyperledger.besu.consensus.common.bft.BftBlockHeaderFunctions; import org.hyperledger.besu.consensus.common.bft.BftExtraData; +import org.hyperledger.besu.consensus.common.bft.BftProtocolSchedule; import org.hyperledger.besu.consensus.qbft.QbftExtraDataCodec; import org.hyperledger.besu.consensus.qbft.pki.PkiQbftBlockHeaderFunctions; import org.hyperledger.besu.consensus.qbft.pki.PkiQbftExtraData; @@ -37,6 +39,8 @@ import org.hyperledger.besu.ethereum.core.BlockBody; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.BlockHeaderTestFixture; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.mainnet.WithdrawalsValidator; import org.hyperledger.besu.pki.cms.CmsCreator; import java.util.Collections; @@ -53,21 +57,38 @@ public class PkiQbftBlockCreatorTest { private CmsCreator cmsCreator; private PkiQbftBlockCreator pkiQbftBlockCreator; private BlockHeaderTestFixture blockHeaderBuilder; + private BlockHeader blockHeader; + private BftProtocolSchedule protocolSchedule; + private ProtocolSpec protocolSpec; @BeforeEach public void before() { blockCreator = mock(BlockCreator.class); cmsCreator = mock(CmsCreator.class); + blockHeader = mock(BlockHeader.class); + protocolSchedule = mock(BftProtocolSchedule.class); + protocolSpec = mock(ProtocolSpec.class); - pkiQbftBlockCreator = new PkiQbftBlockCreator(blockCreator, cmsCreator, extraDataCodec); + pkiQbftBlockCreator = + new PkiQbftBlockCreator( + blockCreator, cmsCreator, extraDataCodec, blockHeader, protocolSchedule); blockHeaderBuilder = new BlockHeaderTestFixture(); + + when(protocolSchedule.getByBlockNumberOrTimestamp(anyLong(), anyLong())) + .thenReturn(protocolSpec); } @Test public void createProposalBehaviourWithNonPkiCodecFails() { assertThatThrownBy( - () -> new PkiQbftBlockCreator(blockCreator, cmsCreator, new QbftExtraDataCodec())) + () -> + new PkiQbftBlockCreator( + blockCreator, + cmsCreator, + new QbftExtraDataCodec(), + blockHeader, + protocolSchedule)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("PkiQbftBlockCreator must use PkiQbftExtraDataCodec"); } @@ -75,6 +96,8 @@ public void createProposalBehaviourWithNonPkiCodecFails() { @Test public void cmsInProposedBlockHasValueCreatedByCmsCreator() { createBlockBeingProposed(); + when(protocolSpec.getWithdrawalsValidator()) + .thenReturn(new WithdrawalsValidator.AllowedWithdrawals()); final Bytes cms = Bytes.random(32); when(cmsCreator.create(any(Bytes.class))).thenReturn(cms); @@ -89,6 +112,8 @@ public void cmsInProposedBlockHasValueCreatedByCmsCreator() { @Test public void cmsIsCreatedWithCorrectHashingFunction() { + when(protocolSpec.getWithdrawalsValidator()) + .thenReturn(new WithdrawalsValidator.ProhibitedWithdrawals()); final Block block = createBlockBeingProposed(); final Hash expectedHashForCmsCreation = PkiQbftBlockHeaderFunctions.forCmsSignature(extraDataCodec).hash(block.getHeader()); @@ -124,6 +149,8 @@ private Block createBlockBeingProposed() { new BlockBody(Collections.emptyList(), Collections.emptyList())); when(blockCreator.createBlock(eq(1L))) .thenReturn(new BlockCreationResult(block, new TransactionSelectionResults())); + when(blockCreator.createEmptyWithdrawalsBlock(anyLong())) + .thenReturn(new BlockCreationResult(block, new TransactionSelectionResults())); return block; } diff --git a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/messagewrappers/ProposalTest.java b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/messagewrappers/ProposalTest.java index 46a079ef00a..b3c462656ac 100644 --- a/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/messagewrappers/ProposalTest.java +++ b/consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/messagewrappers/ProposalTest.java @@ -51,7 +51,11 @@ public class ProposalTest { private static final Block BLOCK = new Block( new BlockHeaderTestFixture().extraData(bftExtraDataCodec.encode(extraData)).buildHeader(), - new BlockBody(Collections.emptyList(), Collections.emptyList())); + new BlockBody( + Collections.emptyList(), + Collections.emptyList(), + Optional.of(Collections.emptyList()), + Optional.empty())); @Test public void canRoundTripProposalMessage() { diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockCreator.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockCreator.java index f7503c65791..451b337b178 100644 --- a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockCreator.java +++ b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockCreator.java @@ -159,6 +159,26 @@ public BlockCreationResult createBlock( true); } + @Override + public BlockCreationResult createEmptyWithdrawalsBlock(final long timestamp) { + throw new UnsupportedOperationException("Only used by BFT block creators"); + } + + public BlockCreationResult createBlock( + final Optional> maybeTransactions, + final Optional> maybeOmmers, + final Optional> maybeWithdrawals, + final long timestamp) { + return createBlock( + maybeTransactions, + maybeOmmers, + maybeWithdrawals, + Optional.empty(), + Optional.empty(), + timestamp, + true); + } + protected BlockCreationResult createBlock( final Optional> maybeTransactions, final Optional> maybeOmmers, diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/BlockCreator.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/BlockCreator.java index 5931d49f369..389c0c16f81 100644 --- a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/BlockCreator.java +++ b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/BlockCreator.java @@ -44,6 +44,8 @@ public TransactionSelectionResults getTransactionSelectionResults() { BlockCreationResult createBlock(final long timestamp); + BlockCreationResult createEmptyWithdrawalsBlock(final long timestamp); + BlockCreationResult createBlock( final List transactions, final List ommers, final long timestamp); From ceafa2a1e6df1069f2fccb4f4441a5c0d53ac615 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Thu, 28 Mar 2024 17:08:45 +0100 Subject: [PATCH 09/24] Fix two flacky acceptance tests (#6837) Signed-off-by: Fabio Di Fabio --- .../EthSendRawTransactionAcceptanceTest.java | 18 +++++++++++------- .../NewPendingTransactionAcceptanceTest.java | 13 ++++++++++--- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EthSendRawTransactionAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EthSendRawTransactionAcceptanceTest.java index 164393664bb..30c32546c5a 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EthSendRawTransactionAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/EthSendRawTransactionAcceptanceTest.java @@ -25,7 +25,6 @@ import java.util.function.UnaryOperator; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class EthSendRawTransactionAcceptanceTest extends AcceptanceTestBase { @@ -45,14 +44,20 @@ public void setUp() throws Exception { strictNode = besu.createArchiveNode("strictNode", configureNode((true))); miningNode = besu.createMinerNode("strictMiningNode", configureNode((true))); cluster.start(lenientNode, strictNode, miningNode); - // verify all nodes are done syncing so the tx pool will be enabled - lenientNode.verify(eth.syncingStatus(false)); - strictNode.verify(eth.syncingStatus(false)); - miningNode.verify(eth.syncingStatus(false)); + + // verify nodes are fully connected otherwise tx could not be propagated + lenientNode.verify(net.awaitPeerCount(2)); + strictNode.verify(net.awaitPeerCount(2)); + miningNode.verify(net.awaitPeerCount(2)); + + // verify that the miner started producing blocks and all other nodes are syncing from it + waitForBlockHeight(miningNode, 1); + final var minerChainHead = miningNode.execute(ethTransactions.block()); + lenientNode.verify(blockchain.minimumHeight(minerChainHead.getNumber().longValue())); + strictNode.verify(blockchain.minimumHeight(minerChainHead.getNumber().longValue())); } @Test - @Disabled("flaky with timeout") public void shouldSendSuccessfullyToLenientNodeWithoutChainId() { final TransferTransaction tx = createTransactionWithoutChainId(); final String rawTx = tx.signedTransactionData(); @@ -60,7 +65,6 @@ public void shouldSendSuccessfullyToLenientNodeWithoutChainId() { lenientNode.verify(eth.expectSuccessfulEthRawTransaction(rawTx)); - // this line is where the test is flaky // Tx should be included on-chain miningNode.verify(eth.expectSuccessfulTransactionReceipt(txHash)); } diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/pubsub/NewPendingTransactionAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/pubsub/NewPendingTransactionAcceptanceTest.java index 9c0368abb88..f230054dcfd 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/pubsub/NewPendingTransactionAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/pubsub/NewPendingTransactionAcceptanceTest.java @@ -41,12 +41,19 @@ public void setUp() throws Exception { minerNode = besu.createMinerNode("miner-node1"); archiveNode = besu.createArchiveNode("full-node1"); cluster.start(minerNode, archiveNode); + + // verify nodes are fully connected otherwise tx could not be propagated + minerNode.verify(net.awaitPeerCount(1)); + archiveNode.verify(net.awaitPeerCount(1)); + accountOne = accounts.createAccount("account-one"); minerWebSocket = new WebSocket(vertx, minerNode.getConfiguration()); archiveWebSocket = new WebSocket(vertx, archiveNode.getConfiguration()); - // verify all nodes are done syncing so the tx pool will be enabled - archiveNode.verify(eth.syncingStatus(false)); - minerNode.verify(eth.syncingStatus(false)); + + // verify that the miner started producing blocks and all other nodes are syncing from it + waitForBlockHeight(minerNode, 1); + final var minerChainHead = minerNode.execute(ethTransactions.block()); + archiveNode.verify(blockchain.minimumHeight(minerChainHead.getNumber().longValue())); } @AfterEach From 3a2eb4e71e196e6a29c7f560bb385fe7831b1af2 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Thu, 28 Mar 2024 18:09:39 +0100 Subject: [PATCH 10/24] Fix to avoid broadcasting full blob txs (#6835) * separate queue for tx hashes Signed-off-by: Gabriel Fukushima * Refinements Signed-off-by: Fabio Di Fabio * Update tests Signed-off-by: Fabio Di Fabio * Update CHANGELOG Signed-off-by: Fabio Di Fabio * Refinements Signed-off-by: Fabio Di Fabio --------- Signed-off-by: Gabriel Fukushima Signed-off-by: Fabio Di Fabio Co-authored-by: Gabriel Fukushima --- CHANGELOG.md | 1 + ...wPooledTransactionHashesMessageSender.java | 2 +- .../transactions/PeerTransactionTracker.java | 22 +++++++ .../transactions/TransactionBroadcaster.java | 37 ++++++++++-- ...ledTransactionHashesMessageSenderTest.java | 9 +-- .../TransactionBroadcasterTest.java | 58 +++++++++++++------ 6 files changed, 100 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 508f4925048..306cc8b26d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ - Don't enable the BFT mining coordinator when running sub commands such as `blocks export` [#6675](https://github.com/hyperledger/besu/pull/6675) - In JSON-RPC return optional `v` fields for type 1 and type 2 transactions [#6762](https://github.com/hyperledger/besu/pull/6762) - Fix Shanghai/QBFT block import bug when syncing new nodes [#6765](https://github.com/hyperledger/besu/pull/6765) +- Fix to avoid broadcasting full blob txs, instead of only the tx announcement, to a subset of nodes [#6835](https://github.com/hyperledger/besu/pull/6835) ### Download Links diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/NewPooledTransactionHashesMessageSender.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/NewPooledTransactionHashesMessageSender.java index 2071226b605..bbc1dd3cd4f 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/NewPooledTransactionHashesMessageSender.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/NewPooledTransactionHashesMessageSender.java @@ -45,7 +45,7 @@ public void sendTransactionHashesToPeer(final EthPeer peer) { final Capability capability = peer.getConnection().capability(EthProtocol.NAME); for (final List txBatch : Iterables.partition( - transactionTracker.claimTransactionsToSendToPeer(peer), MAX_TRANSACTIONS_HASHES)) { + transactionTracker.claimTransactionHashesToSendToPeer(peer), MAX_TRANSACTIONS_HASHES)) { try { final List txHashes = toHashList(txBatch); LOG.atTrace() diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PeerTransactionTracker.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PeerTransactionTracker.java index fc6b0f1d819..f9773ddd23c 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PeerTransactionTracker.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/PeerTransactionTracker.java @@ -32,10 +32,12 @@ public class PeerTransactionTracker implements EthPeer.DisconnectCallback { private static final int MAX_TRACKED_SEEN_TRANSACTIONS = 100_000; private final Map> seenTransactions = new ConcurrentHashMap<>(); private final Map> transactionsToSend = new ConcurrentHashMap<>(); + private final Map> transactionHashesToSend = new ConcurrentHashMap<>(); public void reset() { seenTransactions.clear(); transactionsToSend.clear(); + transactionHashesToSend.clear(); } public synchronized void markTransactionsAsSeen( @@ -55,6 +57,15 @@ public synchronized void addToPeerSendQueue(final EthPeer peer, final Transactio } } + public synchronized void addToPeerHashSendQueue( + final EthPeer peer, final Transaction transaction) { + if (!hasPeerSeenTransaction(peer, transaction)) { + transactionHashesToSend + .computeIfAbsent(peer, key -> createTransactionsSet()) + .add(transaction); + } + } + public Iterable getEthPeersWithUnsentTransactions() { return transactionsToSend.keySet(); } @@ -69,6 +80,16 @@ public synchronized Set claimTransactionsToSendToPeer(final EthPeer } } + public synchronized Set claimTransactionHashesToSendToPeer(final EthPeer peer) { + final Set transactionHashesToSend = this.transactionHashesToSend.remove(peer); + if (transactionHashesToSend != null) { + markTransactionHashesAsSeen(peer, toHashList(transactionHashesToSend)); + return transactionHashesToSend; + } else { + return emptySet(); + } + } + public boolean hasSeenTransaction(final Hash txHash) { return seenTransactions.values().stream().anyMatch(seen -> seen.contains(txHash)); } @@ -100,5 +121,6 @@ protected boolean removeEldestEntry(final Map.Entry eldest) { public void onDisconnect(final EthPeer peer) { seenTransactions.remove(peer); transactionsToSend.remove(peer); + transactionHashesToSend.remove(peer); } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionBroadcaster.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionBroadcaster.java index f64f4e7973c..69c30211a31 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionBroadcaster.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionBroadcaster.java @@ -30,8 +30,10 @@ import java.util.EnumSet; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.stream.Collectors; +import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,16 +49,33 @@ public class TransactionBroadcaster implements TransactionBatchAddedListener { private final TransactionsMessageSender transactionsMessageSender; private final NewPooledTransactionHashesMessageSender newPooledTransactionHashesMessageSender; private final EthContext ethContext; + private final Random random; public TransactionBroadcaster( final EthContext ethContext, final PeerTransactionTracker transactionTracker, final TransactionsMessageSender transactionsMessageSender, final NewPooledTransactionHashesMessageSender newPooledTransactionHashesMessageSender) { + this( + ethContext, + transactionTracker, + transactionsMessageSender, + newPooledTransactionHashesMessageSender, + null); + } + + @VisibleForTesting + protected TransactionBroadcaster( + final EthContext ethContext, + final PeerTransactionTracker transactionTracker, + final TransactionsMessageSender transactionsMessageSender, + final NewPooledTransactionHashesMessageSender newPooledTransactionHashesMessageSender, + final Long seed) { this.transactionTracker = transactionTracker; this.transactionsMessageSender = transactionsMessageSender; this.newPooledTransactionHashesMessageSender = newPooledTransactionHashesMessageSender; this.ethContext = ethContext; + this.random = seed != null ? new Random(seed) : new Random(); } public void relayTransactionPoolTo( @@ -65,7 +84,13 @@ public void relayTransactionPoolTo( if (peer.hasSupportForMessage(EthPV65.NEW_POOLED_TRANSACTION_HASHES)) { sendTransactionHashes(toTransactionList(pendingTransactions), List.of(peer)); } else { - sendFullTransactions(toTransactionList(pendingTransactions), List.of(peer)); + // we need to exclude txs that support hash only broadcasting + final var fullBroadcastTxs = + pendingTransactions.stream() + .map(PendingTransaction::getTransaction) + .filter(tx -> !ANNOUNCE_HASH_ONLY_TX_TYPES.contains(tx.getType())) + .toList(); + sendFullTransactions(fullBroadcastTxs, List.of(peer)); } } } @@ -77,7 +102,7 @@ public void onTransactionsAdded(final Collection transactions) { return; } - final int numPeersToSendFullTransactions = (int) Math.ceil(Math.sqrt(currPeerCount)); + final int numPeersToSendFullTransactions = (int) Math.round(Math.sqrt(currPeerCount)); final Map> transactionByBroadcastMode = transactions.stream() @@ -107,7 +132,7 @@ public void onTransactionsAdded(final Collection transactions) { numPeersToSendFullTransactions - sendOnlyFullTransactionPeers.size(), sendOnlyHashPeers.size()); - Collections.shuffle(sendOnlyHashPeers); + Collections.shuffle(sendOnlyHashPeers, random); // move peers from the mixed list to reach the required size for full transaction peers movePeersBetweenLists(sendOnlyHashPeers, sendMixedPeers, delta); @@ -121,7 +146,7 @@ public void onTransactionsAdded(final Collection transactions) { .addArgument(sendOnlyHashPeers::size) .addArgument(sendMixedPeers::size) .addArgument(sendOnlyFullTransactionPeers) - .addArgument(() -> sendOnlyHashPeers.toString() + sendMixedPeers.toString()) + .addArgument(() -> sendOnlyHashPeers.toString() + sendMixedPeers) .log(); sendToFullTransactionsPeers( @@ -141,7 +166,7 @@ private void sendToOnlyHashPeers( final Map> txsByHashOnlyBroadcast, final List hashOnlyPeers) { final List allTransactions = - txsByHashOnlyBroadcast.values().stream().flatMap(List::stream).collect(Collectors.toList()); + txsByHashOnlyBroadcast.values().stream().flatMap(List::stream).toList(); sendTransactionHashes(allTransactions, hashOnlyPeers); } @@ -175,7 +200,7 @@ private void sendTransactionHashes( .forEach( peer -> { transactions.forEach( - transaction -> transactionTracker.addToPeerSendQueue(peer, transaction)); + transaction -> transactionTracker.addToPeerHashSendQueue(peer, transaction)); ethContext .getScheduler() .scheduleSyncWorkerTask( diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/NewPooledTransactionHashesMessageSenderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/NewPooledTransactionHashesMessageSenderTest.java index ec6f772c796..6a72271bb1b 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/NewPooledTransactionHashesMessageSenderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/NewPooledTransactionHashesMessageSenderTest.java @@ -78,9 +78,9 @@ public void setUp() { @Test public void shouldSendPendingTransactionsToEachPeer() throws Exception { - transactionTracker.addToPeerSendQueue(peer1, transaction1); - transactionTracker.addToPeerSendQueue(peer1, transaction2); - transactionTracker.addToPeerSendQueue(peer2, transaction3); + transactionTracker.addToPeerHashSendQueue(peer1, transaction1); + transactionTracker.addToPeerHashSendQueue(peer1, transaction2); + transactionTracker.addToPeerHashSendQueue(peer2, transaction3); List.of(peer1, peer2).forEach(messageSender::sendTransactionHashesToPeer); @@ -96,7 +96,8 @@ public void shouldSendTransactionsInBatchesWithLimit() throws Exception { final Set transactions = generator.transactions(6000).stream().collect(Collectors.toSet()); - transactions.forEach(transaction -> transactionTracker.addToPeerSendQueue(peer1, transaction)); + transactions.forEach( + transaction -> transactionTracker.addToPeerHashSendQueue(peer1, transaction)); messageSender.sendTransactionHashesToPeer(peer1); final ArgumentCaptor messageDataArgumentCaptor = diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionBroadcasterTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionBroadcasterTest.java index b7d6d054f93..0615a25f5ad 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionBroadcasterTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionBroadcasterTest.java @@ -55,7 +55,7 @@ @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) public class TransactionBroadcasterTest { - + private static final Long FIXED_RANDOM_SEED = 0L; @Mock private EthContext ethContext; @Mock private EthPeers ethPeers; @Mock private EthScheduler ethScheduler; @@ -92,12 +92,14 @@ public void setUp() { when(ethContext.getEthPeers()).thenReturn(ethPeers); when(ethContext.getScheduler()).thenReturn(ethScheduler); + // we use the fixed random seed to have a predictable shuffle of peers txBroadcaster = new TransactionBroadcaster( ethContext, transactionTracker, transactionsMessageSender, - newPooledTransactionHashesMessageSender); + newPooledTransactionHashesMessageSender, + FIXED_RANDOM_SEED); } @Test @@ -132,7 +134,7 @@ public void relayTransactionHashesFromPoolWhenPeerSupportEth65() { txBroadcaster.relayTransactionPoolTo(ethPeerWithEth65, pendingTxs); - verifyTransactionAddedToPeerSendingQueue(ethPeerWithEth65, txs); + verifyTransactionAddedToPeerHashSendingQueue(ethPeerWithEth65, txs); sendTaskCapture.getValue().run(); @@ -177,14 +179,16 @@ public void onTransactionsAddedWithOnlyFewEth65PeersSendFullTransactions() { List txs = toTransactionList(setupTransactionPool(1, 1)); txBroadcaster.onTransactionsAdded(txs); - - verifyTransactionAddedToPeerSendingQueue(ethPeerWithEth65, txs); + // the shuffled hash only peer list is always: + // [ethPeerWithEth65_3, ethPeerWithEth65_2, ethPeerWithEth65] + // so ethPeerWithEth65 and ethPeerWithEth65_2 are moved to the mixed broadcast list + verifyTransactionAddedToPeerHashSendingQueue(ethPeerWithEth65, txs); verifyTransactionAddedToPeerSendingQueue(ethPeerWithEth65_2, txs); sendTaskCapture.getAllValues().forEach(Runnable::run); - verify(transactionsMessageSender, times(2)).sendTransactionsToPeer(any(EthPeer.class)); - verifyNoInteractions(newPooledTransactionHashesMessageSender); + verify(transactionsMessageSender).sendTransactionsToPeer(ethPeerWithEth65_2); + verify(newPooledTransactionHashesMessageSender).sendTransactionHashesToPeer(ethPeerWithEth65); } @Test @@ -196,10 +200,12 @@ public void onTransactionsAddedWithOnlyEth65PeersSendFullTransactionsAndTransact List txs = toTransactionList(setupTransactionPool(1, 1)); txBroadcaster.onTransactionsAdded(txs); - + // the shuffled hash only peer list is always: + // [ethPeerWithEth65_3, ethPeerWithEth65_2, ethPeerWithEth65] + // so ethPeerWithEth65 and ethPeerWithEth65_2 are moved to the mixed broadcast list verifyTransactionAddedToPeerSendingQueue(ethPeerWithEth65, txs); verifyTransactionAddedToPeerSendingQueue(ethPeerWithEth65_2, txs); - verifyTransactionAddedToPeerSendingQueue(ethPeerWithEth65_3, txs); + verifyTransactionAddedToPeerHashSendingQueue(ethPeerWithEth65_3, txs); sendTaskCapture.getAllValues().forEach(Runnable::run); @@ -218,8 +224,10 @@ public void onTransactionsAddedWithMixedPeersSendFullTransactionsAndTransactionH List txs = toTransactionList(setupTransactionPool(1, 1)); txBroadcaster.onTransactionsAdded(txs); - - verifyTransactionAddedToPeerSendingQueue(ethPeerWithEth65, txs); + // the shuffled hash only peer list is always: + // [ethPeerWithEth65, ethPeerWithEth65_2] + // so ethPeerWithEth65_2 is moved to the mixed broadcast list + verifyTransactionAddedToPeerHashSendingQueue(ethPeerWithEth65, txs); verifyTransactionAddedToPeerSendingQueue(ethPeerWithEth65_2, txs); verifyTransactionAddedToPeerSendingQueue(ethPeerNoEth65, txs); @@ -250,9 +258,11 @@ public void onTransactionsAddedWithMixedPeersSendFullTransactionsAndTransactionH List txs = toTransactionList(setupTransactionPool(BLOB, 0, 1)); txBroadcaster.onTransactionsAdded(txs); - - verifyTransactionAddedToPeerSendingQueue(ethPeerWithEth65, txs); - verifyTransactionAddedToPeerSendingQueue(ethPeerWithEth65_2, txs); + // the shuffled hash only peer list is always: + // [ethPeerWithEth65, ethPeerWithEth65_2] + // so ethPeerWithEth65_2 is moved to the mixed broadcast list + verifyTransactionAddedToPeerHashSendingQueue(ethPeerWithEth65, txs); + verifyTransactionAddedToPeerHashSendingQueue(ethPeerWithEth65_2, txs); verifyNoTransactionAddedToPeerSendingQueue(ethPeerNoEth65); sendTaskCapture.getAllValues().forEach(Runnable::run); @@ -268,7 +278,6 @@ public void onTransactionsAddedWithMixedPeersSendFullTransactionsAndTransactionH @Test public void onTransactionsAddedWithMixedPeersAndMixedBroadcastKind() { - List eth65Peers = List.of(ethPeerWithEth65, ethPeerWithEth65_2); when(ethPeers.peerCount()).thenReturn(3); @@ -285,9 +294,12 @@ public void onTransactionsAddedWithMixedPeersAndMixedBroadcastKind() { mixedTxs.addAll(hashBroadcastTxs); txBroadcaster.onTransactionsAdded(mixedTxs); - - verifyTransactionAddedToPeerSendingQueue(ethPeerWithEth65, mixedTxs); - verifyTransactionAddedToPeerSendingQueue(ethPeerWithEth65_2, mixedTxs); + // the shuffled hash only peer list is always: + // [ethPeerWithEth65, ethPeerWithEth65_2] + // so ethPeerWithEth65_2 is moved to the mixed broadcast list + verifyTransactionAddedToPeerHashSendingQueue(ethPeerWithEth65, mixedTxs); + verifyTransactionAddedToPeerHashSendingQueue(ethPeerWithEth65_2, hashBroadcastTxs); + verifyTransactionAddedToPeerSendingQueue(ethPeerWithEth65_2, fullBroadcastTxs); verifyTransactionAddedToPeerSendingQueue(ethPeerNoEth65, fullBroadcastTxs); sendTaskCapture.getAllValues().forEach(Runnable::run); @@ -348,6 +360,16 @@ private void verifyTransactionAddedToPeerSendingQueue( .containsExactlyInAnyOrderElementsOf(transactions); } + private void verifyTransactionAddedToPeerHashSendingQueue( + final EthPeer peer, final Collection transactions) { + + ArgumentCaptor trackedTransactions = ArgumentCaptor.forClass(Transaction.class); + verify(transactionTracker, times(transactions.size())) + .addToPeerHashSendQueue(eq(peer), trackedTransactions.capture()); + assertThat(trackedTransactions.getAllValues()) + .containsExactlyInAnyOrderElementsOf(transactions); + } + private void verifyNoTransactionAddedToPeerSendingQueue(final EthPeer peer) { verify(transactionTracker, times(0)).addToPeerSendQueue(eq(peer), any()); From 464cd2606aae5783e74f3c34df0ec0030f9e40f4 Mon Sep 17 00:00:00 2001 From: Matt Nelson <85905982+non-fungible-nelson@users.noreply.github.com> Date: Thu, 28 Mar 2024 15:14:25 -0400 Subject: [PATCH 11/24] logging fix for historical queries (#6830) Signed-off-by: Matt Nelson <85905982+non-fungible-nelson@users.noreply.github.com> --- .../trie/diffbased/common/DiffBasedWorldStateProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedWorldStateProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedWorldStateProvider.java index 18bf333828b..bda86f7dfac 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedWorldStateProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedWorldStateProvider.java @@ -132,7 +132,7 @@ public Optional getMutable( if (chainHeadBlockHeader.getNumber() - blockHeader.getNumber() >= trieLogManager.getMaxLayersToLoad()) { LOG.warn( - "Exceeded the limit of back layers that can be loaded ({})", + "Exceeded the limit of historical blocks that can be loaded ({}). If you need to make older historical queries, configure your `--bonsai-historical-block-limit`.", trieLogManager.getMaxLayersToLoad()); return Optional.empty(); } From a2ef6c49daa21f9e93d9f7047fe4d63bd8ad3989 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Fri, 29 Mar 2024 11:56:09 +1000 Subject: [PATCH 12/24] refactor to check for null peer (#6841) Signed-off-by: Sally MacFarlane --- .../besu/ethereum/eth/manager/EthPeers.java | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java index b379edb4a80..c2362c5dd16 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthPeers.java @@ -177,14 +177,21 @@ private List getIncompleteConnections(final Bytes id) { public boolean registerDisconnect(final PeerConnection connection) { final EthPeer peer = peer(connection); - return registerDisconnect(peer.getId(), peer, connection); + return registerDisconnect(peer, connection); } - private boolean registerDisconnect( - final Bytes id, final EthPeer peer, final PeerConnection connection) { + private boolean registerDisconnect(final EthPeer peer, final PeerConnection connection) { incompleteConnections.invalidate(connection); boolean removed = false; - if (peer != null && peer.getConnection().equals(connection)) { + if (peer == null) { + LOG.atTrace() + .setMessage("attempt to remove null peer with connection {}") + .addArgument(connection) + .log(); + return false; + } + if (peer.getConnection().equals(connection)) { + final Bytes id = peer.getId(); if (!peerHasIncompleteConnection(id)) { removed = completeConnections.remove(id, peer); disconnectCallbacks.forEach(callback -> callback.onDisconnect(peer)); @@ -297,7 +304,7 @@ private void removeDisconnectedPeers() { .forEach( ep -> { if (ep.isDisconnected()) { - registerDisconnect(ep.getId(), ep, ep.getConnection()); + registerDisconnect(ep, ep.getConnection()); } }); } From 6c1991a07584662860869b5e2d049a1c21fb070d Mon Sep 17 00:00:00 2001 From: Matt Nelson <85905982+non-fungible-nelson@users.noreply.github.com> Date: Thu, 28 Mar 2024 22:42:49 -0400 Subject: [PATCH 13/24] update broken link, issue template (#6829) Signed-off-by: Matt Nelson <85905982+non-fungible-nelson@users.noreply.github.com> Co-authored-by: Sally MacFarlane --- .github/issue_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/issue_template.md b/.github/issue_template.md index cfae3400266..54acf4c1968 100644 --- a/.github/issue_template.md +++ b/.github/issue_template.md @@ -3,7 +3,7 @@ - + From ad49e21bf0d927b2c56afc5e6f23e49680f51379 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Fri, 29 Mar 2024 13:50:36 +1000 Subject: [PATCH 14/24] Prevent startup with privacy and bonsai enabled (#6809) * prevent startup with bonsai and privacy Signed-off-by: Sally MacFarlane * tests for privacy specify forest Signed-off-by: Sally MacFarlane * changelog Signed-off-by: Sally MacFarlane --------- Signed-off-by: Sally MacFarlane --- CHANGELOG.md | 1 + .../org/hyperledger/besu/cli/BesuCommand.java | 3 ++ .../besu/cli/PrivacyOptionsTest.java | 32 +++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 306cc8b26d1..92afdaf8877 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ - Extend error handling of plugin RPC methods [#6759](https://github.com/hyperledger/besu/pull/6759) - Added engine_newPayloadV4 and engine_getPayloadV4 methods [#6783](https://github.com/hyperledger/besu/pull/6783) - Reduce storage size of receipts [#6602](https://github.com/hyperledger/besu/pull/6602) +- Prevent startup with BONSAI and privacy enabled [#6809](https://github.com/hyperledger/besu/pull/6809) ### Bug fixes - Fix txpool dump/restore race condition [#6665](https://github.com/hyperledger/besu/pull/6665) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 901c2cf2629..023133f7ae2 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -1979,6 +1979,9 @@ private PrivacyParameters privacyParameters() { throw new ParameterException( commandLine, String.format("%s %s", "Checkpoint sync", errorSuffix)); } + if (getDataStorageConfiguration().getDataStorageFormat().equals(DataStorageFormat.BONSAI)) { + throw new ParameterException(commandLine, String.format("%s %s", "Bonsai", errorSuffix)); + } if (isPruningEnabled()) { throw new ParameterException(commandLine, String.format("%s %s", "Pruning", errorSuffix)); } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/PrivacyOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/PrivacyOptionsTest.java index 84988aa2f09..3485e527be3 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/PrivacyOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/PrivacyOptionsTest.java @@ -30,6 +30,9 @@ import java.net.URI; import java.net.URL; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.Optional; import org.junit.jupiter.api.Test; @@ -192,6 +195,26 @@ public void privacyWithCheckpointSyncMustError() { assertThat(commandOutput.toString(UTF_8)).isEmpty(); } + @Test + public void privacyWithBonsaiDefaultMustError() { + // bypass overridden parseCommand method which specifies bonsai + super.parseCommand("--privacy-enabled"); + + assertThat(commandErrorOutput.toString(UTF_8)) + .contains("Bonsai cannot be enabled with privacy."); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + + @Test + public void privacyWithBonsaiExplicitMustError() { + // bypass overridden parseCommand method which specifies bonsai + super.parseCommand("--privacy-enabled", "--data-storage-format", "BONSAI"); + + assertThat(commandErrorOutput.toString(UTF_8)) + .contains("Bonsai cannot be enabled with privacy."); + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + } + @Test public void privacyWithPruningMustError() { parseCommand("--pruning-enabled", "--privacy-enabled"); @@ -483,4 +506,13 @@ public void eeaWsApisWithPrivacyDisabledLogsWarning() { assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } + + @Override + protected TestBesuCommand parseCommand(final String... args) { + // privacy requires forest to be specified + final List argsPlusForest = new ArrayList<>(Arrays.stream(args).toList()); + argsPlusForest.add("--data-storage-format"); + argsPlusForest.add("FOREST"); + return super.parseCommand(argsPlusForest.toArray(String[]::new)); + } } From 1679525ae28b3baacbd85c9c869ec92897d6cb50 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Fri, 29 Mar 2024 15:48:15 +0100 Subject: [PATCH 15/24] Dedicated log marker for invalid txs removed from the txpool (#6826) Signed-off-by: Fabio Di Fabio Signed-off-by: Justin Florentine Co-authored-by: Justin Florentine --- CHANGELOG.md | 2 ++ besu/src/main/resources/log4j2.xml | 3 +++ .../layered/LayeredPendingTransactions.java | 22 +++++++++++++++++-- .../AbstractPendingTransactionsSorter.java | 22 +++++++++++++++++++ 4 files changed, 47 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92afdaf8877..93b631a1855 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,8 +27,10 @@ - Extend error handling of plugin RPC methods [#6759](https://github.com/hyperledger/besu/pull/6759) - Added engine_newPayloadV4 and engine_getPayloadV4 methods [#6783](https://github.com/hyperledger/besu/pull/6783) - Reduce storage size of receipts [#6602](https://github.com/hyperledger/besu/pull/6602) +- Dedicated log marker for invalid txs removed from the txpool [#6826](https://github.com/hyperledger/besu/pull/6826) - Prevent startup with BONSAI and privacy enabled [#6809](https://github.com/hyperledger/besu/pull/6809) + ### Bug fixes - Fix txpool dump/restore race condition [#6665](https://github.com/hyperledger/besu/pull/6665) - Make block transaction selection max time aware of PoA transitions [#6676](https://github.com/hyperledger/besu/pull/6676) diff --git a/besu/src/main/resources/log4j2.xml b/besu/src/main/resources/log4j2.xml index 80ba817fe7c..d47cc9a40fc 100644 --- a/besu/src/main/resources/log4j2.xml +++ b/besu/src/main/resources/log4j2.xml @@ -48,6 +48,9 @@ + + + diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactions.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactions.java index d9b6b60814d..5b2eadf87f0 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactions.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/layered/LayeredPendingTransactions.java @@ -37,6 +37,7 @@ import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.account.AccountState; +import org.hyperledger.besu.plugin.data.TransactionSelectionResult; import java.util.ArrayDeque; import java.util.ArrayList; @@ -53,10 +54,13 @@ import kotlin.ranges.LongRange; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.Marker; +import org.slf4j.MarkerFactory; public class LayeredPendingTransactions implements PendingTransactions { private static final Logger LOG = LoggerFactory.getLogger(LayeredPendingTransactions.class); private static final Logger LOG_FOR_REPLAY = LoggerFactory.getLogger("LOG_FOR_REPLAY"); + private static final Marker INVALID_TX_REMOVED = MarkerFactory.getMarker("INVALID_TX_REMOVED"); private final TransactionPoolConfiguration poolConfig; private final AbstractPrioritizedTransactions prioritizedTransactions; @@ -234,7 +238,8 @@ private void logTransactionForReplayAdd( .log(); } - private void logTransactionForReplayDelete(final PendingTransaction pendingTransaction) { + private void logDiscardedTransaction( + final PendingTransaction pendingTransaction, final TransactionSelectionResult result) { // csv fields: sequence, addedAt, sender, nonce, type, hash, rlp LOG_FOR_REPLAY .atTrace() @@ -252,6 +257,19 @@ private void logTransactionForReplayDelete(final PendingTransaction pendingTrans return rlp.encoded().toHexString(); }) .log(); + LOG.atInfo() + .addMarker(INVALID_TX_REMOVED) + .addKeyValue("txhash", pendingTransaction::getHash) + .addKeyValue("txlog", pendingTransaction::toTraceLog) + .addKeyValue("reason", result) + .addKeyValue( + "txrlp", + () -> { + final BytesValueRLPOutput rlp = new BytesValueRLPOutput(); + pendingTransaction.getTransaction().writeTo(rlp); + return rlp.encoded().toHexString(); + }) + .log(); } private TransactionAddedResult nonceChecks( @@ -333,7 +351,7 @@ public synchronized void selectTransactions( if (res.discard()) { invalidTransactions.add(candidatePendingTx); - logTransactionForReplayDelete(candidatePendingTx); + logDiscardedTransaction(candidatePendingTx, res); } if (res.stop()) { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/AbstractPendingTransactionsSorter.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/AbstractPendingTransactionsSorter.java index bb53a8e4fbd..fa2efa53c84 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/AbstractPendingTransactionsSorter.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/AbstractPendingTransactionsSorter.java @@ -32,6 +32,7 @@ import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolReplacementHandler; import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; import org.hyperledger.besu.evm.account.Account; import org.hyperledger.besu.evm.account.AccountState; import org.hyperledger.besu.metrics.BesuMetricCategory; @@ -61,6 +62,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.Marker; +import org.slf4j.MarkerFactory; /** * Holds the current set of pending transactions with the ability to iterate them based on priority @@ -71,6 +74,7 @@ public abstract class AbstractPendingTransactionsSorter implements PendingTransactions { private static final Logger LOG = LoggerFactory.getLogger(AbstractPendingTransactionsSorter.class); + private static final Marker INVALID_TX_REMOVED = MarkerFactory.getMarker("INVALID_TX_REMOVED"); protected final Clock clock; protected final TransactionPoolConfiguration poolConfig; @@ -247,6 +251,7 @@ public void selectTransactions(final TransactionSelector selector) { if (result.discard()) { transactionsToRemove.add(transactionToProcess.getTransaction()); + logDiscardedTransaction(transactionToProcess, result); } if (result.stop()) { @@ -259,6 +264,23 @@ public void selectTransactions(final TransactionSelector selector) { } } + private void logDiscardedTransaction( + final PendingTransaction pendingTransaction, final TransactionSelectionResult result) { + LOG.atInfo() + .addMarker(INVALID_TX_REMOVED) + .addKeyValue("txhash", pendingTransaction::getHash) + .addKeyValue("txlog", pendingTransaction::toTraceLog) + .addKeyValue("reason", result) + .addKeyValue( + "txrlp", + () -> { + final BytesValueRLPOutput rlp = new BytesValueRLPOutput(); + pendingTransaction.getTransaction().writeTo(rlp); + return rlp.encoded().toHexString(); + }) + .log(); + } + private AccountTransactionOrder createSenderTransactionOrder(final Address address) { return new AccountTransactionOrder( transactionsBySender.get(address).streamPendingTransactions()); From d8e1e1710cf0f5a2bb60c27210bee609045f3d7d Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Fri, 29 Mar 2024 16:57:45 +0100 Subject: [PATCH 16/24] Remove deprecated Forest pruning (#6810) * Remove deprecated Forest pruning Signed-off-by: Fabio Di Fabio * Update CHANGELOG.md Co-authored-by: Sally MacFarlane Signed-off-by: Fabio Di Fabio --------- Signed-off-by: Fabio Di Fabio Co-authored-by: Sally MacFarlane --- CHANGELOG.md | 3 +- .../org/hyperledger/besu/cli/BesuCommand.java | 46 --- .../controller/BesuControllerBuilder.java | 52 --- ...onsensusScheduleBesuControllerBuilder.java | 15 - .../TransitionBesuControllerBuilder.java | 16 - .../org/hyperledger/besu/cli/launcher.json | 6 - .../besu/cli/CommandTestAbstract.java | 2 - .../besu/cli/PrivacyOptionsTest.java | 9 - .../besu/cli/PruningOptionsTest.java | 126 ------- .../controller/BesuControllerBuilderTest.java | 218 ------------ .../src/test/resources/everything_config.toml | 5 - .../forest/pruner/PrunerIntegrationTest.java | 267 --------------- .../trie/forest/pruner/MarkSweepPruner.java | 317 ------------------ .../ethereum/trie/forest/pruner/Pruner.java | 199 ----------- .../forest/pruner/PrunerConfiguration.java | 42 --- .../forest/pruner/MarkSweepPrunerTest.java | 293 ---------------- .../trie/forest/pruner/PrunerTest.java | 214 ------------ .../eth/sync/DefaultSynchronizer.java | 13 +- 18 files changed, 3 insertions(+), 1840 deletions(-) delete mode 100644 besu/src/test/java/org/hyperledger/besu/cli/PruningOptionsTest.java delete mode 100644 besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java delete mode 100644 ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerIntegrationTest.java delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/pruner/MarkSweepPruner.java delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/pruner/Pruner.java delete mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerConfiguration.java delete mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/MarkSweepPrunerTest.java delete mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 93b631a1855..1d7730f0e3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Breaking Changes - RocksDB database metadata format has changed to be more expressive, the migration of an existing metadata file to the new format is automatic at startup. Before performing a downgrade to a previous version it is mandatory to revert to the original format using the subcommand `besu --data-path=/path/to/besu/datadir storage revert-metadata v2-to-v1`. - BFT networks won't start with SNAP or CHECKPOINT sync (previously Besu would start with this config but quietly fail to sync, so it's now more obvious that it won't work) [#6625](https://github.com/hyperledger/besu/pull/6625), [#6667](https://github.com/hyperledger/besu/pull/6667) +- Forest pruning has been removed, it was deprecated since 24.1.0. In case you are still using it you must now remove any of the following options: `pruning-enabled`, `pruning-blocks-retained` and `pruning-block-confirmations`, from your configuration, and you may want to consider switching to Bonsai. ### Upcoming Breaking Changes - Receipt compaction will be enabled by default in a future version of Besu. After this change it will not be possible to downgrade to the previous Besu version. @@ -29,7 +30,7 @@ - Reduce storage size of receipts [#6602](https://github.com/hyperledger/besu/pull/6602) - Dedicated log marker for invalid txs removed from the txpool [#6826](https://github.com/hyperledger/besu/pull/6826) - Prevent startup with BONSAI and privacy enabled [#6809](https://github.com/hyperledger/besu/pull/6809) - +- Remove deprecated Forest pruning [#6810](https://github.com/hyperledger/besu/pull/6810) ### Bug fixes - Fix txpool dump/restore race condition [#6665](https://github.com/hyperledger/besu/pull/6665) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 023133f7ae2..1c63329ac25 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -140,7 +140,6 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; import org.hyperledger.besu.ethereum.transaction.TransactionSimulator; -import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.evm.precompile.AbstractAltBnPrecompiledContract; import org.hyperledger.besu.evm.precompile.BigIntegerModularExponentiationPrecompiledContract; @@ -797,12 +796,6 @@ static class MetricsOptionGroup { "How deep a chain reorganization must be in order for it to be logged (default: ${DEFAULT-VALUE})") private final Long reorgLoggingThreshold = 6L; - @Option( - names = {"--pruning-enabled"}, - description = - "Enable disk-space saving optimization that removes old state that is unlikely to be required (default: ${DEFAULT-VALUE})") - private final Boolean pruningEnabled = false; - // Permission Option Group @CommandLine.ArgGroup(validate = false, heading = "@|bold Permissions Options|@%n") PermissionsOptions permissionsOptions = new PermissionsOptions(); @@ -852,23 +845,6 @@ static class MetricsOptionGroup { private final Map genesisConfigOverrides = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - @Option( - names = {"--pruning-blocks-retained"}, - paramLabel = "", - description = - "Minimum number of recent blocks for which to keep entire world state (default: ${DEFAULT-VALUE})", - arity = "1") - private final Integer pruningBlocksRetained = PrunerConfiguration.DEFAULT_PRUNING_BLOCKS_RETAINED; - - @Option( - names = {"--pruning-block-confirmations"}, - paramLabel = "", - description = - "Minimum number of confirmations on a block before marking begins (default: ${DEFAULT-VALUE})", - arity = "1") - private final Integer pruningBlockConfirmations = - PrunerConfiguration.DEFAULT_PRUNING_BLOCK_CONFIRMATIONS; - @CommandLine.Option( names = {"--pid-path"}, paramLabel = MANDATORY_PATH_FORMAT_HELP, @@ -1683,18 +1659,6 @@ && isOptionSet(commandLine, "--sync-min-peers")) { "--node-private-key-file", "--security-module=" + DEFAULT_SECURITY_MODULE); } - - if (isPruningEnabled()) { - if (dataStorageOptions - .toDomainObject() - .getDataStorageFormat() - .equals(DataStorageFormat.BONSAI)) { - logger.warn("Forest pruning is ignored with Bonsai data storage format."); - } else { - logger.warn( - "Forest pruning is deprecated and will be removed soon. To save disk space consider switching to Bonsai data storage format."); - } - } } private void configure() throws Exception { @@ -1840,9 +1804,6 @@ public BesuControllerBuilder getControllerBuilder() { .clock(Clock.systemUTC()) .isRevertReasonEnabled(isRevertReasonEnabled) .storageProvider(storageProvider) - .isPruningEnabled(isPruningEnabled()) - .pruningConfiguration( - new PrunerConfiguration(pruningBlockConfirmations, pruningBlocksRetained)) .genesisConfigOverrides(genesisConfigOverrides) .gasLimitCalculator( getMiningParameters().getTargetGasLimit().isPresent() @@ -1982,9 +1943,6 @@ private PrivacyParameters privacyParameters() { if (getDataStorageConfiguration().getDataStorageFormat().equals(DataStorageFormat.BONSAI)) { throw new ParameterException(commandLine, String.format("%s %s", "Bonsai", errorSuffix)); } - if (isPruningEnabled()) { - throw new ParameterException(commandLine, String.format("%s %s", "Pruning", errorSuffix)); - } if (Boolean.TRUE.equals(privacyOptionGroup.isPrivacyMultiTenancyEnabled) && Boolean.FALSE.equals(jsonRpcConfiguration.isAuthenticationEnabled()) @@ -2206,10 +2164,6 @@ private OptionalInt getGenesisBlockPeriodSeconds( return OptionalInt.empty(); } - private boolean isPruningEnabled() { - return pruningEnabled; - } - // Blockchain synchronization from peers. private Runner synchronize( final BesuController controller, diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 8990ba9e447..d70590c37d5 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -87,9 +87,6 @@ import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogManager; import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner; import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive; -import org.hyperledger.besu.ethereum.trie.forest.pruner.MarkSweepPruner; -import org.hyperledger.besu.ethereum.trie.forest.pruner.Pruner; -import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; @@ -159,10 +156,6 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides GasLimitCalculator gasLimitCalculator; /** The Storage provider. */ protected StorageProvider storageProvider; - /** The Is pruning enabled. */ - protected boolean isPruningEnabled; - /** The Pruner configuration. */ - protected PrunerConfiguration prunerConfiguration; /** The Required blocks. */ protected Map requiredBlocks = Collections.emptyMap(); /** The Reorg logging threshold. */ @@ -372,28 +365,6 @@ public BesuControllerBuilder isRevertReasonEnabled(final boolean isRevertReasonE return this; } - /** - * Is pruning enabled besu controller builder. - * - * @param isPruningEnabled the is pruning enabled - * @return the besu controller builder - */ - public BesuControllerBuilder isPruningEnabled(final boolean isPruningEnabled) { - this.isPruningEnabled = isPruningEnabled; - return this; - } - - /** - * Pruning configuration besu controller builder. - * - * @param prunerConfiguration the pruner configuration - * @return the besu controller builder - */ - public BesuControllerBuilder pruningConfiguration(final PrunerConfiguration prunerConfiguration) { - this.prunerConfiguration = prunerConfiguration; - return this; - } - /** * Genesis config overrides besu controller builder. * @@ -607,25 +578,6 @@ public BesuController build() { protocolSchedule.setPublicWorldStateArchiveForPrivacyBlockProcessor( protocolContext.getWorldStateArchive()); - Optional maybePruner = Optional.empty(); - if (isPruningEnabled) { - if (dataStorageConfiguration.getDataStorageFormat().equals(DataStorageFormat.BONSAI)) { - LOG.warn( - "Cannot enable pruning with Bonsai data storage format. Disabling. Change the data storage format or disable pruning explicitly on the command line to remove this warning."); - } else { - maybePruner = - Optional.of( - new Pruner( - new MarkSweepPruner( - ((ForestWorldStateArchive) worldStateArchive).getWorldStateStorage(), - blockchain, - storageProvider.getStorageBySegmentIdentifier( - KeyValueSegmentIdentifier.PRUNING_STATE), - metricsSystem), - blockchain, - prunerConfiguration)); - } - } final int maxMessageSize = ethereumWireProtocolConfiguration.getMaxMessageSize(); final Supplier currentProtocolSpecSupplier = () -> protocolSchedule.getByBlockHeader(blockchain.getChainHeadHeader()); @@ -712,7 +664,6 @@ public BesuController build() { protocolSchedule, worldStateStorageCoordinator, protocolContext, - maybePruner, ethContext, syncState, ethProtocolManager, @@ -803,7 +754,6 @@ private TrieLogPruner createTrieLogPruner( * @param protocolSchedule the protocol schedule * @param worldStateStorageCoordinator the world state storage * @param protocolContext the protocol context - * @param maybePruner the maybe pruner * @param ethContext the eth context * @param syncState the sync state * @param ethProtocolManager the eth protocol manager @@ -814,7 +764,6 @@ protected Synchronizer createSynchronizer( final ProtocolSchedule protocolSchedule, final WorldStateStorageCoordinator worldStateStorageCoordinator, final ProtocolContext protocolContext, - final Optional maybePruner, final EthContext ethContext, final SyncState syncState, final EthProtocolManager ethProtocolManager, @@ -826,7 +775,6 @@ protected Synchronizer createSynchronizer( protocolContext, worldStateStorageCoordinator, ethProtocolManager.getBlockBroadcaster(), - maybePruner, ethContext, syncState, dataDirectory, diff --git a/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java index a41e9bb3ee0..6e02086590d 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java @@ -56,7 +56,6 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.p2p.config.SubProtocolConfiguration; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.evm.internal.EvmConfiguration; @@ -368,20 +367,6 @@ public BesuControllerBuilder isRevertReasonEnabled(final boolean isRevertReasonE return super.isRevertReasonEnabled(isRevertReasonEnabled); } - @Override - public BesuControllerBuilder isPruningEnabled(final boolean isPruningEnabled) { - besuControllerBuilderSchedule.values().forEach(b -> b.isPruningEnabled(isPruningEnabled)); - return super.isPruningEnabled(isPruningEnabled); - } - - @Override - public BesuControllerBuilder pruningConfiguration(final PrunerConfiguration prunerConfiguration) { - besuControllerBuilderSchedule - .values() - .forEach(b -> b.pruningConfiguration(prunerConfiguration)); - return super.pruningConfiguration(prunerConfiguration); - } - @Override public BesuControllerBuilder genesisConfigOverrides( final Map genesisConfigOverrides) { diff --git a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java index 090a9a9e3ad..3529797d57a 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java @@ -52,8 +52,6 @@ import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.trie.forest.pruner.Pruner; -import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; @@ -219,7 +217,6 @@ protected Synchronizer createSynchronizer( final ProtocolSchedule protocolSchedule, final WorldStateStorageCoordinator worldStateStorageCoordinator, final ProtocolContext protocolContext, - final Optional maybePruner, final EthContext ethContext, final SyncState syncState, final EthProtocolManager ethProtocolManager, @@ -231,7 +228,6 @@ protected Synchronizer createSynchronizer( protocolSchedule, worldStateStorageCoordinator, protocolContext, - maybePruner, ethContext, syncState, ethProtocolManager, @@ -387,18 +383,6 @@ public BesuControllerBuilder isRevertReasonEnabled(final boolean isRevertReasonE return propagateConfig(z -> z.isRevertReasonEnabled(isRevertReasonEnabled)); } - @Override - public BesuControllerBuilder isPruningEnabled(final boolean isPruningEnabled) { - super.isPruningEnabled(isPruningEnabled); - return propagateConfig(z -> z.isPruningEnabled(isPruningEnabled)); - } - - @Override - public BesuControllerBuilder pruningConfiguration(final PrunerConfiguration prunerConfiguration) { - super.pruningConfiguration(prunerConfiguration); - return propagateConfig(z -> z.pruningConfiguration(prunerConfiguration)); - } - @Override public BesuControllerBuilder genesisConfigOverrides( final Map genesisConfigOverrides) { diff --git a/besu/src/main/resources/org/hyperledger/besu/cli/launcher.json b/besu/src/main/resources/org/hyperledger/besu/cli/launcher.json index 6db0ef2e5bc..226ca165e3b 100644 --- a/besu/src/main/resources/org/hyperledger/besu/cli/launcher.json +++ b/besu/src/main/resources/org/hyperledger/besu/cli/launcher.json @@ -13,12 +13,6 @@ "config-key": "sync-mode", "available-options": "org.hyperledger.besu.ethereum.eth.sync.SyncMode" }, - { - "prompt-type": "CONFIRM", - "question": "Do you want to enable pruning?", - "config-key": "pruning-enabled", - "default-option": "no" - }, { "prompt-type": "INPUT", "question": "What is the data directory ?", diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java index 63f664d7469..cb77694c283 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java @@ -283,8 +283,6 @@ public void initMocks() throws Exception { when(mockControllerBuilder.clock(any())).thenReturn(mockControllerBuilder); when(mockControllerBuilder.isRevertReasonEnabled(false)).thenReturn(mockControllerBuilder); when(mockControllerBuilder.storageProvider(any())).thenReturn(mockControllerBuilder); - when(mockControllerBuilder.isPruningEnabled(anyBoolean())).thenReturn(mockControllerBuilder); - when(mockControllerBuilder.pruningConfiguration(any())).thenReturn(mockControllerBuilder); when(mockControllerBuilder.genesisConfigOverrides(any())).thenReturn(mockControllerBuilder); when(mockControllerBuilder.gasLimitCalculator(any())).thenReturn(mockControllerBuilder); when(mockControllerBuilder.requiredBlocks(any())).thenReturn(mockControllerBuilder); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/PrivacyOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/PrivacyOptionsTest.java index 3485e527be3..c214fae32d6 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/PrivacyOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/PrivacyOptionsTest.java @@ -215,15 +215,6 @@ public void privacyWithBonsaiExplicitMustError() { assertThat(commandOutput.toString(UTF_8)).isEmpty(); } - @Test - public void privacyWithPruningMustError() { - parseCommand("--pruning-enabled", "--privacy-enabled"); - - assertThat(commandErrorOutput.toString(UTF_8)) - .contains("Pruning cannot be enabled with privacy."); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - } - @Test public void privacyWithoutPrivacyPublicKeyFails() { parseCommand("--privacy-enabled", "--privacy-url", ENCLAVE_URI); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/PruningOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/PruningOptionsTest.java deleted file mode 100644 index 56d48d62581..00000000000 --- a/besu/src/test/java/org/hyperledger/besu/cli/PruningOptionsTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright Hyperledger Besu Contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.cli; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.contains; -import static org.mockito.Mockito.verify; - -import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; - -public class PruningOptionsTest extends CommandTestAbstract { - - @Disabled - public void pruningIsEnabledIfSyncModeIsFast() { - parseCommand("--sync-mode", "FAST"); - - verify(mockControllerBuilder).isPruningEnabled(true); - verify(mockControllerBuilder).build(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Disabled - public void pruningIsDisabledIfSyncModeIsFull() { - parseCommand("--sync-mode", "FULL"); - - verify(mockControllerBuilder).isPruningEnabled(false); - verify(mockControllerBuilder).build(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void pruningEnabledExplicitly() { - parseCommand("--pruning-enabled", "--sync-mode=FULL"); - - verify(mockControllerBuilder).isPruningEnabled(true); - verify(mockControllerBuilder).build(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Disabled - public void pruningDisabledExplicitly() { - parseCommand("--pruning-enabled=false", "--sync-mode=FAST"); - - verify(mockControllerBuilder).isPruningEnabled(false); - verify(mockControllerBuilder).build(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void pruningDisabledByDefault() { - parseCommand(); - - verify(mockControllerBuilder).isPruningEnabled(false); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - } - - @Test - public void pruningParametersAreCaptured() throws Exception { - parseCommand( - "--pruning-enabled", "--pruning-blocks-retained=15", "--pruning-block-confirmations=4"); - - final ArgumentCaptor pruningArg = - ArgumentCaptor.forClass(PrunerConfiguration.class); - - verify(mockControllerBuilder).pruningConfiguration(pruningArg.capture()); - verify(mockControllerBuilder).build(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - assertThat(pruningArg.getValue().getBlocksRetained()).isEqualTo(15); - assertThat(pruningArg.getValue().getBlockConfirmations()).isEqualTo(4); - } - - @Test - public void pruningLogsDeprecationWarningWithForest() { - parseCommand("--pruning-enabled", "--data-storage-format=FOREST"); - - verify(mockControllerBuilder).isPruningEnabled(true); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - verify(mockLogger) - .warn( - contains( - "Forest pruning is deprecated and will be removed soon." - + " To save disk space consider switching to Bonsai data storage format.")); - } - - @Test - public void pruningLogsIgnoredWarningWithBonsai() { - parseCommand("--pruning-enabled", "--data-storage-format=BONSAI"); - - verify(mockControllerBuilder).isPruningEnabled(true); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - verify(mockLogger).warn(contains("Forest pruning is ignored with Bonsai data storage format.")); - } -} diff --git a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java deleted file mode 100644 index 5eb14201cd8..00000000000 --- a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.controller; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import org.hyperledger.besu.config.CheckpointConfigOptions; -import org.hyperledger.besu.config.EthashConfigOptions; -import org.hyperledger.besu.config.GenesisConfigFile; -import org.hyperledger.besu.config.GenesisConfigOptions; -import org.hyperledger.besu.cryptoservices.NodeKey; -import org.hyperledger.besu.cryptoservices.NodeKeyUtils; -import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.GasLimitCalculator; -import org.hyperledger.besu.ethereum.chain.Blockchain; -import org.hyperledger.besu.ethereum.core.MiningParameters; -import org.hyperledger.besu.ethereum.core.PrivacyParameters; -import org.hyperledger.besu.ethereum.eth.EthProtocolConfiguration; -import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; -import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration; -import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; -import org.hyperledger.besu.ethereum.p2p.config.NetworkingConfiguration; -import org.hyperledger.besu.ethereum.storage.StorageProvider; -import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; -import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage; -import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.cache.BonsaiCachedMerkleTrieLoader; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; -import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration; -import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; -import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; -import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; -import org.hyperledger.besu.evm.internal.EvmConfiguration; -import org.hyperledger.besu.metrics.ObservableMetricsSystem; -import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; - -import java.math.BigInteger; -import java.nio.file.Path; -import java.time.Clock; -import java.util.OptionalLong; - -import com.google.common.collect.Range; -import org.apache.tuweni.bytes.Bytes; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.io.TempDir; -import org.mockito.Answers; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -public class BesuControllerBuilderTest { - - private BesuControllerBuilder besuControllerBuilder; - private static final NodeKey nodeKey = NodeKeyUtils.generate(); - - @Mock GenesisConfigFile genesisConfigFile; - @Mock GenesisConfigOptions genesisConfigOptions; - @Mock EthashConfigOptions ethashConfigOptions; - @Mock CheckpointConfigOptions checkpointConfigOptions; - @Mock SynchronizerConfiguration synchronizerConfiguration; - @Mock EthProtocolConfiguration ethProtocolConfiguration; - @Mock PrivacyParameters privacyParameters; - @Mock Clock clock; - @Mock StorageProvider storageProvider; - @Mock GasLimitCalculator gasLimitCalculator; - @Mock WorldStateArchive worldStateArchive; - @Mock BonsaiWorldStateKeyValueStorage bonsaiWorldStateStorage; - @Mock WorldStatePreimageStorage worldStatePreimageStorage; - private final TransactionPoolConfiguration poolConfiguration = - TransactionPoolConfiguration.DEFAULT; - private final MiningParameters miningParameters = MiningParameters.newDefault(); - - private final ObservableMetricsSystem observableMetricsSystem = new NoOpMetricsSystem(); - - BigInteger networkId = BigInteger.ONE; - - @TempDir Path tempDir; - - @BeforeEach - public void setup() { - - final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = - mock(ForestWorldStateKeyValueStorage.class); - final WorldStateStorageCoordinator worldStateStorageCoordinator = - new WorldStateStorageCoordinator(worldStateKeyValueStorage); - - when(genesisConfigFile.getParentHash()).thenReturn(Hash.ZERO.toHexString()); - when(genesisConfigFile.getDifficulty()).thenReturn(Bytes.of(0).toHexString()); - when(genesisConfigFile.getExtraData()).thenReturn(Bytes.EMPTY.toHexString()); - when(genesisConfigFile.getMixHash()).thenReturn(Hash.ZERO.toHexString()); - when(genesisConfigFile.getNonce()).thenReturn(Long.toHexString(1)); - when(genesisConfigFile.getConfigOptions(any())).thenReturn(genesisConfigOptions); - when(genesisConfigFile.getConfigOptions()).thenReturn(genesisConfigOptions); - when(genesisConfigOptions.getThanosBlockNumber()).thenReturn(OptionalLong.empty()); - when(genesisConfigOptions.getEthashConfigOptions()).thenReturn(ethashConfigOptions); - when(genesisConfigOptions.getCheckpointOptions()).thenReturn(checkpointConfigOptions); - when(ethashConfigOptions.getFixedDifficulty()).thenReturn(OptionalLong.empty()); - when(storageProvider.getStorageBySegmentIdentifier(any())) - .thenReturn(new InMemoryKeyValueStorage()); - when(storageProvider.createBlockchainStorage(any(), any(), any())) - .thenReturn( - new KeyValueStoragePrefixedKeyBlockchainStorage( - new InMemoryKeyValueStorage(), - new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions(), - false)); - when(synchronizerConfiguration.getDownloaderParallelism()).thenReturn(1); - when(synchronizerConfiguration.getTransactionsParallelism()).thenReturn(1); - when(synchronizerConfiguration.getComputationParallelism()).thenReturn(1); - - when(synchronizerConfiguration.getBlockPropagationRange()).thenReturn(Range.closed(1L, 2L)); - - lenient() - .when( - storageProvider.createWorldStateStorageCoordinator( - DataStorageConfiguration.DEFAULT_FOREST_CONFIG)) - .thenReturn(worldStateStorageCoordinator); - lenient() - .when(storageProvider.createWorldStatePreimageStorage()) - .thenReturn(worldStatePreimageStorage); - - lenient().when(worldStateKeyValueStorage.isWorldStateAvailable(any())).thenReturn(true); - lenient() - .when(worldStatePreimageStorage.updater()) - .thenReturn(mock(WorldStatePreimageStorage.Updater.class)); - lenient() - .when(worldStateKeyValueStorage.updater()) - .thenReturn(mock(ForestWorldStateKeyValueStorage.Updater.class)); - - besuControllerBuilder = spy(visitWithMockConfigs(new MainnetBesuControllerBuilder())); - } - - BesuControllerBuilder visitWithMockConfigs(final BesuControllerBuilder builder) { - return builder - .gasLimitCalculator(gasLimitCalculator) - .genesisConfigFile(genesisConfigFile) - .synchronizerConfiguration(synchronizerConfiguration) - .ethProtocolConfiguration(ethProtocolConfiguration) - .miningParameters(miningParameters) - .metricsSystem(observableMetricsSystem) - .privacyParameters(privacyParameters) - .dataDirectory(tempDir) - .clock(clock) - .transactionPoolConfiguration(poolConfiguration) - .nodeKey(nodeKey) - .storageProvider(storageProvider) - .evmConfiguration(EvmConfiguration.DEFAULT) - .networkConfiguration(NetworkingConfiguration.create()) - .networkId(networkId); - } - - @Test - public void shouldDisablePruningIfBonsaiIsEnabled() { - DataStorageConfiguration dataStorageConfiguration = - ImmutableDataStorageConfiguration.builder() - .dataStorageFormat(DataStorageFormat.BONSAI) - .bonsaiMaxLayersToLoad(DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) - .build(); - BonsaiWorldState mockWorldState = mock(BonsaiWorldState.class, Answers.RETURNS_DEEP_STUBS); - doReturn(worldStateArchive) - .when(besuControllerBuilder) - .createWorldStateArchive( - any(WorldStateStorageCoordinator.class), - any(Blockchain.class), - any(BonsaiCachedMerkleTrieLoader.class)); - doReturn(mockWorldState).when(worldStateArchive).getMutable(); - when(storageProvider.createWorldStateStorageCoordinator(dataStorageConfiguration)) - .thenReturn(new WorldStateStorageCoordinator(bonsaiWorldStateStorage)); - besuControllerBuilder.isPruningEnabled(true).dataStorageConfiguration(dataStorageConfiguration); - - besuControllerBuilder.build(); - - verify(storageProvider, never()) - .getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.PRUNING_STATE); - } - - @Test - public void shouldUsePruningIfForestIsEnabled() { - besuControllerBuilder - .isPruningEnabled(true) - .pruningConfiguration(new PrunerConfiguration(1, 2)) - .dataStorageConfiguration( - ImmutableDataStorageConfiguration.builder() - .dataStorageFormat(DataStorageFormat.FOREST) - .bonsaiMaxLayersToLoad(DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD) - .build()); - besuControllerBuilder.build(); - - verify(storageProvider).getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.PRUNING_STATE); - } -} diff --git a/besu/src/test/resources/everything_config.toml b/besu/src/test/resources/everything_config.toml index 87fc5986dd7..435ef16db52 100644 --- a/besu/src/test/resources/everything_config.toml +++ b/besu/src/test/resources/everything_config.toml @@ -151,11 +151,6 @@ Xminer-remote-sealers-limit=1000 Xminer-remote-sealers-hashrate-ttl=10 Xpos-block-creation-max-time=5 -# Pruning -pruning-enabled=true -pruning-blocks-retained=1024 -pruning-block-confirmations=10 - # Permissioning permissions-nodes-config-file-enabled=false permissions-nodes-config-file="./permissions_config.toml" diff --git a/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerIntegrationTest.java b/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerIntegrationTest.java deleted file mode 100644 index 5b723d80cdb..00000000000 --- a/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerIntegrationTest.java +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Copyright Hyperledger Besu Contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.trie.forest.pruner; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider.createInMemoryBlockchain; - -import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.chain.MutableBlockchain; -import org.hyperledger.besu.ethereum.core.Block; -import org.hyperledger.besu.ethereum.core.BlockDataGenerator; -import org.hyperledger.besu.ethereum.core.BlockDataGenerator.BlockOptions; -import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.core.MutableWorldState; -import org.hyperledger.besu.ethereum.core.TransactionReceipt; -import org.hyperledger.besu.ethereum.rlp.RLP; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.MerkleTrie; -import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive; -import org.hyperledger.besu.ethereum.trie.forest.pruner.Pruner.PruningPhase; -import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; -import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; -import org.hyperledger.besu.evm.internal.EvmConfiguration; -import org.hyperledger.besu.evm.worldstate.WorldState; -import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; -import org.hyperledger.besu.testutil.MockExecutorService; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.bytes.Bytes32; -import org.junit.jupiter.api.Test; - -public class PrunerIntegrationTest { - - private final BlockDataGenerator gen = new BlockDataGenerator(); - private final NoOpMetricsSystem metricsSystem = new NoOpMetricsSystem(); - private final Map> hashValueStore = new HashMap<>(); - private final InMemoryKeyValueStorage stateStorage = new TestInMemoryStorage(hashValueStore); - private final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = - new ForestWorldStateKeyValueStorage(stateStorage); - private final WorldStateArchive worldStateArchive = - new ForestWorldStateArchive( - new WorldStateStorageCoordinator(worldStateKeyValueStorage), - new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()), - EvmConfiguration.DEFAULT); - private final InMemoryKeyValueStorage markStorage = new InMemoryKeyValueStorage(); - private final Block genesisBlock = gen.genesisBlock(); - private final MutableBlockchain blockchain = createInMemoryBlockchain(genesisBlock); - - @Test - public void pruner_smallState_manyOpsPerTx() { - testPruner(3, 1, 1, 4, 100_000); - } - - @Test - public void pruner_largeState_fewOpsPerTx() { - testPruner(2, 5, 5, 6, 5); - } - - @Test - public void pruner_emptyBlocks() { - testPruner(5, 0, 2, 5, 10); - } - - @Test - public void pruner_markChainhead() { - testPruner(4, 2, 1, 10, 20); - } - - @Test - public void pruner_lowRelativeBlockConfirmations() { - testPruner(3, 2, 1, 4, 20); - } - - @Test - public void pruner_highRelativeBlockConfirmations() { - testPruner(3, 2, 9, 10, 20); - } - - private void testPruner( - final int numCycles, - final int accountsPerBlock, - final int blockConfirmations, - final int numBlocksToKeep, - final int opsPerTransaction) { - - final var markSweepPruner = - new MarkSweepPruner( - worldStateKeyValueStorage, blockchain, markStorage, metricsSystem, opsPerTransaction); - final var pruner = - new Pruner( - markSweepPruner, - blockchain, - new PrunerConfiguration(blockConfirmations, numBlocksToKeep), - new MockExecutorService()); - - pruner.start(); - - for (int cycle = 0; cycle < numCycles; ++cycle) { - int numBlockInCycle = - numBlocksToKeep - + 1; // +1 to get it to switch from MARKING_COMPLETE TO SWEEPING on each cycle - var fullyMarkedBlockNum = cycle * numBlockInCycle + 1; - - // This should cause a full mark and sweep cycle - assertThat(pruner.getPruningPhase()).isEqualByComparingTo(PruningPhase.IDLE); - generateBlockchainData(numBlockInCycle, accountsPerBlock); - assertThat(pruner.getPruningPhase()).isEqualByComparingTo(PruningPhase.IDLE); - - // Collect the nodes we expect to keep - final Set expectedNodes = new HashSet<>(); - for (int i = fullyMarkedBlockNum; i <= blockchain.getChainHeadBlockNumber(); i++) { - final Hash stateRoot = blockchain.getBlockHeader(i).get().getStateRoot(); - collectWorldStateNodes(stateRoot, expectedNodes); - } - - if (accountsPerBlock != 0) { - assertThat(hashValueStore.size()) - .isGreaterThanOrEqualTo(expectedNodes.size()); // Sanity check - } - - // Assert that blocks from mark point onward are still accessible - for (int i = fullyMarkedBlockNum; i <= blockchain.getChainHeadBlockNumber(); i++) { - final BlockHeader blockHeader = blockchain.getBlockHeader(i).get(); - final Hash stateRoot = blockHeader.getStateRoot(); - assertThat(worldStateArchive.get(stateRoot, blockHeader.getHash())).isPresent(); - final WorldState markedState = - worldStateArchive.get(stateRoot, blockHeader.getHash()).get(); - // Traverse accounts and make sure all are accessible - final int expectedAccounts = accountsPerBlock * i; - final long accounts = - markedState.streamAccounts(Bytes32.ZERO, expectedAccounts * 2).count(); - assertThat(accounts).isEqualTo(expectedAccounts); - // Traverse storage to ensure that all storage is accessible - markedState - .streamAccounts(Bytes32.ZERO, expectedAccounts * 2) - .forEach(a -> a.storageEntriesFrom(Bytes32.ZERO, 1000)); - } - - // All other state roots should have been removed - for (int i = 0; i < fullyMarkedBlockNum; i++) { - final BlockHeader curHeader = blockchain.getBlockHeader(i).get(); - if (!curHeader.getStateRoot().equals(Hash.EMPTY_TRIE_HASH)) { - assertThat(worldStateArchive.get(curHeader.getStateRoot(), curHeader.getHash())) - .isEmpty(); - } - } - - // Check that storage contains only the values we expect - assertThat(hashValueStore.size()).isEqualTo(expectedNodes.size()); - assertThat(hashValueStore.values().stream().map(Optional::get)) - .containsExactlyInAnyOrderElementsOf( - expectedNodes.stream().map(Bytes::toArrayUnsafe).collect(Collectors.toSet())); - } - - pruner.stop(); - } - - private void generateBlockchainData(final int numBlocks, final int numAccounts) { - Block parentBlock = blockchain.getChainHeadBlock(); - for (int i = 0; i < numBlocks; i++) { - final BlockHeader parentHeader = parentBlock.getHeader(); - final Hash parentHash = parentBlock.getHash(); - final MutableWorldState worldState = - worldStateArchive.getMutable(parentHeader.getStateRoot(), parentHash).get(); - gen.createRandomContractAccountsWithNonEmptyStorage(worldState, numAccounts); - final Hash stateRoot = worldState.rootHash(); - - final Block block = - gen.block( - BlockOptions.create() - .setStateRoot(stateRoot) - .setBlockNumber(parentHeader.getNumber() + 1L) - .setParentHash(parentHash)); - final List receipts = gen.receipts(block); - blockchain.appendBlock(block, receipts); - parentBlock = block; - } - } - - private Set collectWorldStateNodes(final Hash stateRootHash, final Set collector) { - final List storageRoots = new ArrayList<>(); - final MerkleTrie stateTrie = createStateTrie(stateRootHash); - - // Collect storage roots and code - stateTrie - .entriesFrom(Bytes32.ZERO, 1000) - .forEach( - (key, val) -> { - final StateTrieAccountValue accountValue = - StateTrieAccountValue.readFrom(RLP.input(val)); - stateStorage - .get(accountValue.getCodeHash().toArray()) - .ifPresent(v -> collector.add(Bytes.wrap(v))); - storageRoots.add(accountValue.getStorageRoot()); - }); - - // Collect state nodes - collectTrieNodes(stateTrie, collector); - // Collect storage nodes - for (Hash storageRoot : storageRoots) { - final MerkleTrie storageTrie = createStorageTrie(storageRoot); - collectTrieNodes(storageTrie, collector); - } - - return collector; - } - - private void collectTrieNodes(final MerkleTrie trie, final Set collector) { - final Bytes32 rootHash = trie.getRootHash(); - trie.visitAll( - node -> { - if (node.isReferencedByHash() || node.getHash().equals(rootHash)) { - collector.add(node.getEncodedBytes()); - } - }); - } - - private MerkleTrie createStateTrie(final Bytes32 rootHash) { - return new StoredMerklePatriciaTrie<>( - (location, hash) -> worldStateKeyValueStorage.getAccountStateTrieNode(hash), - rootHash, - Function.identity(), - Function.identity()); - } - - private MerkleTrie createStorageTrie(final Bytes32 rootHash) { - return new StoredMerklePatriciaTrie<>( - (location, hash) -> worldStateKeyValueStorage.getAccountStorageTrieNode(hash), - rootHash, - Function.identity(), - Function.identity()); - } - - // Proxy class so that we have access to the constructor that takes our own map - private static class TestInMemoryStorage extends InMemoryKeyValueStorage { - - public TestInMemoryStorage(final Map> hashValueStore) { - super(hashValueStore); - } - } -} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/pruner/MarkSweepPruner.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/pruner/MarkSweepPruner.java deleted file mode 100644 index 45b4b2ee3a0..00000000000 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/pruner/MarkSweepPruner.java +++ /dev/null @@ -1,317 +0,0 @@ -/* - * Copyright Hyperledger Besu Contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.trie.forest.pruner; - -import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.chain.MutableBlockchain; -import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.rlp.RLP; -import org.hyperledger.besu.ethereum.trie.MerkleTrie; -import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; -import org.hyperledger.besu.metrics.BesuMetricCategory; -import org.hyperledger.besu.metrics.ObservableMetricsSystem; -import org.hyperledger.besu.plugin.services.metrics.Counter; -import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; -import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; - -import java.util.Collection; -import java.util.Collections; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingDeque; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.Function; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Stopwatch; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.bytes.Bytes32; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MarkSweepPruner { - - private static final Logger LOG = LoggerFactory.getLogger(MarkSweepPruner.class); - private static final byte[] IN_USE = Bytes.of(1).toArrayUnsafe(); - - private static final int DEFAULT_OPS_PER_TRANSACTION = 10_000; - private static final int MAX_MARKING_THREAD_POOL_SIZE = 2; - - private final int operationsPerTransaction; - private final ForestWorldStateKeyValueStorage worldStateKeyValueStorage; - private final MutableBlockchain blockchain; - private final KeyValueStorage markStorage; - private final Counter markedNodesCounter; - private final Counter markOperationCounter; - private final Counter sweepOperationCounter; - private final Counter sweptNodesCounter; - private final Stopwatch markStopwatch; - private volatile long nodeAddedListenerId; - private final ReadWriteLock pendingMarksLock = new ReentrantReadWriteLock(); - private final Set pendingMarks = Collections.newSetFromMap(new ConcurrentHashMap<>()); - - public MarkSweepPruner( - final ForestWorldStateKeyValueStorage worldStateKeyValueStorage, - final MutableBlockchain blockchain, - final KeyValueStorage markStorage, - final ObservableMetricsSystem metricsSystem) { - this( - worldStateKeyValueStorage, - blockchain, - markStorage, - metricsSystem, - DEFAULT_OPS_PER_TRANSACTION); - } - - public MarkSweepPruner( - final ForestWorldStateKeyValueStorage worldStateKeyValueStorage, - final MutableBlockchain blockchain, - final KeyValueStorage markStorage, - final ObservableMetricsSystem metricsSystem, - final int operationsPerTransaction) { - this.worldStateKeyValueStorage = worldStateKeyValueStorage; - this.markStorage = markStorage; - this.blockchain = blockchain; - this.operationsPerTransaction = operationsPerTransaction; - - markedNodesCounter = - metricsSystem.createCounter( - BesuMetricCategory.PRUNER, - "marked_nodes_total", - "Total number of nodes marked as in use"); - markOperationCounter = - metricsSystem.createCounter( - BesuMetricCategory.PRUNER, - "mark_operations_total", - "Total number of mark operations performed"); - - sweptNodesCounter = - metricsSystem.createCounter( - BesuMetricCategory.PRUNER, "swept_nodes_total", "Total number of unused nodes removed"); - sweepOperationCounter = - metricsSystem.createCounter( - BesuMetricCategory.PRUNER, - "sweep_operations_total", - "Total number of sweep operations performed"); - - markStopwatch = Stopwatch.createUnstarted(); - metricsSystem.createLongGauge( - BesuMetricCategory.PRUNER, - "mark_time_duration", - "Cumulative number of seconds spent marking the state trie across all pruning cycles", - () -> markStopwatch.elapsed(TimeUnit.SECONDS)); - - LOG.debug("Using {} pruner threads", MAX_MARKING_THREAD_POOL_SIZE); - } - - public void prepare() { - // Optimization for the case where the previous cycle was interrupted (like the node was shut - // down). If the previous cycle was interrupted, there will be marks in the mark storage from - // last time, causing the first sweep to be smaller than it needs to be. - clearMarks(); - - nodeAddedListenerId = worldStateKeyValueStorage.addNodeAddedListener(this::markNodes); - } - - /** - * This is a parallel mark implementation. - * - *

The parallel task production is by sub-trie, so calling `visitAll` on a root node will - * eventually spawn up to 16 tasks (for a hexary trie). - * - *

If we marked each sub-trie in its own thread, with no common queue of tasks, our mark speed - * would be limited by the sub-trie with the maximum number of nodes. In practice for the Ethereum - * mainnet, we see a large imbalance in sub-trie size so without a common task pool the time in - * which there is only 1 thread left marking its big sub-trie would be substantial. - * - *

If we were to leave all threads to produce mark tasks before starting to mark, we would run - * out of memory quickly. - * - *

If we were to have a constant number of threads producing the mark tasks with the others - * consuming them, we would have to optimize the production/consumption balance. - * - *

To get the best of both worlds, the marking executor has a {@link - * ThreadPoolExecutor.CallerRunsPolicy} which causes the producing tasks to essentially consume - * their own mark task immediately when the task queue is full. The resulting behavior is threads - * that mark their own sub-trie until they finish that sub-trie, at which point they switch to - * marking the sub-trie tasks produced by another thread. - * - * @param rootHash The root hash of the whole state trie. Roots of storage tries will be - * discovered though traversal. - */ - public void mark(final Hash rootHash) { - markOperationCounter.inc(); - markStopwatch.start(); - final ExecutorService markingExecutorService = - new ThreadPoolExecutor( - 0, - MAX_MARKING_THREAD_POOL_SIZE, - 5L, - TimeUnit.SECONDS, - new LinkedBlockingDeque<>(16), - new ThreadFactoryBuilder() - .setDaemon(true) - .setPriority(Thread.MIN_PRIORITY) - .setNameFormat(this.getClass().getSimpleName() + "-%d") - .build(), - new ThreadPoolExecutor.CallerRunsPolicy()); - createStateTrie(rootHash) - .visitAll( - node -> { - markNode(node.getHash()); - node.getValue() - .ifPresent(value -> processAccountState(value, markingExecutorService)); - }, - markingExecutorService) - .join() /* This will block on all the marking tasks to be _produced_ but doesn't guarantee that the marking tasks have been completed. */; - markingExecutorService.shutdown(); - try { - // This ensures that the marking tasks complete. - markingExecutorService.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS); - } catch (InterruptedException e) { - LOG.info("Interrupted while marking", e); - } - markStopwatch.stop(); - LOG.debug("Completed marking used nodes for pruning"); - } - - public void sweepBefore(final long markedBlockNumber) { - sweepOperationCounter.inc(); - LOG.debug("Sweeping unused nodes"); - // Sweep state roots first, walking backwards until we get to a state root that isn't in the - // storage - long prunedNodeCount = 0; - ForestWorldStateKeyValueStorage.Updater updater = worldStateKeyValueStorage.updater(); - for (long blockNumber = markedBlockNumber - 1; blockNumber >= 0; blockNumber--) { - final BlockHeader blockHeader = blockchain.getBlockHeader(blockNumber).get(); - final Hash candidateStateRootHash = blockHeader.getStateRoot(); - if (!worldStateKeyValueStorage.isWorldStateAvailable(candidateStateRootHash)) { - break; - } - - if (!isMarked(candidateStateRootHash)) { - updater.removeAccountStateTrieNode(candidateStateRootHash); - prunedNodeCount++; - if (prunedNodeCount % operationsPerTransaction == 0) { - updater.commit(); - updater = worldStateKeyValueStorage.updater(); - } - } - } - - updater.commit(); - // Sweep non-state-root nodes - prunedNodeCount += worldStateKeyValueStorage.prune(this::isMarked); - sweptNodesCounter.inc(prunedNodeCount); - clearMarks(); - LOG.debug("Completed sweeping unused nodes"); - } - - public void cleanup() { - worldStateKeyValueStorage.removeNodeAddedListener(nodeAddedListenerId); - clearMarks(); - } - - public void clearMarks() { - markStorage.clear(); - pendingMarks.clear(); - } - - private boolean isMarked(final Bytes32 key) { - return pendingMarks.contains(key) || markStorage.containsKey(key.toArrayUnsafe()); - } - - private boolean isMarked(final byte[] key) { - return pendingMarks.contains(Bytes32.wrap(key)) || markStorage.containsKey(key); - } - - private MerkleTrie createStateTrie(final Bytes32 rootHash) { - return new StoredMerklePatriciaTrie<>( - (location, hash) -> worldStateKeyValueStorage.getAccountStateTrieNode(hash), - rootHash, - Function.identity(), - Function.identity()); - } - - private MerkleTrie createStorageTrie(final Bytes32 rootHash) { - return new StoredMerklePatriciaTrie<>( - (location, hash) -> worldStateKeyValueStorage.getAccountStorageTrieNode(hash), - rootHash, - Function.identity(), - Function.identity()); - } - - private void processAccountState(final Bytes value, final ExecutorService executorService) { - final StateTrieAccountValue accountValue = StateTrieAccountValue.readFrom(RLP.input(value)); - markNode(accountValue.getCodeHash()); - - createStorageTrie(accountValue.getStorageRoot()) - .visitAll(storageNode -> markNode(storageNode.getHash()), executorService); - } - - @VisibleForTesting - public void markNode(final Bytes32 hash) { - markThenMaybeFlush(() -> pendingMarks.add(hash), 1); - } - - private void markNodes(final Collection nodeHashes) { - markThenMaybeFlush(() -> pendingMarks.addAll(nodeHashes), nodeHashes.size()); - } - - private void markThenMaybeFlush(final Runnable nodeMarker, final int numberOfNodes) { - // We use the read lock here because pendingMarks is threadsafe and we want to allow all the - // marking threads access simultaneously. - final Lock markLock = pendingMarksLock.readLock(); - markLock.lock(); - try { - nodeMarker.run(); - } finally { - markLock.unlock(); - } - markedNodesCounter.inc(numberOfNodes); - - // However, when the size of pendingMarks grows too large, we want all the threads to stop - // adding because we're going to clear the set. - // Therefore, we need to take out a write lock. - if (pendingMarks.size() >= operationsPerTransaction) { - final Lock flushLock = pendingMarksLock.writeLock(); - flushLock.lock(); - try { - // Check once again that the condition holds. If it doesn't, that means another thread - // already flushed them. - if (pendingMarks.size() >= operationsPerTransaction) { - flushPendingMarks(); - } - } finally { - flushLock.unlock(); - } - } - } - - private void flushPendingMarks() { - final KeyValueStorageTransaction transaction = markStorage.startTransaction(); - pendingMarks.forEach(node -> transaction.put(node.toArrayUnsafe(), IN_USE)); - transaction.commit(); - pendingMarks.clear(); - } -} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/pruner/Pruner.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/pruner/Pruner.java deleted file mode 100644 index bad8b2da302..00000000000 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/pruner/Pruner.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.trie.forest.pruner; - -import static com.google.common.base.Preconditions.checkArgument; - -import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.chain.BlockAddedEvent; -import org.hyperledger.besu.ethereum.chain.Blockchain; -import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.trie.MerkleTrieException; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class Pruner { - - private static final Logger LOG = LoggerFactory.getLogger(Pruner.class); - - private final MarkSweepPruner pruningStrategy; - private final Blockchain blockchain; - private Long blockAddedObserverId; - private final long blocksRetained; - private final AtomicReference pruningPhase = - new AtomicReference<>(PruningPhase.IDLE); - private volatile long markBlockNumber = 0; - private volatile BlockHeader markedBlockHeader; - private final long blockConfirmations; - - private final AtomicReference state = new AtomicReference<>(State.IDLE); - private final ExecutorService executorService; - - @VisibleForTesting - Pruner( - final MarkSweepPruner pruningStrategy, - final Blockchain blockchain, - final PrunerConfiguration prunerConfiguration, - final ExecutorService executorService) { - this.pruningStrategy = pruningStrategy; - this.blockchain = blockchain; - this.executorService = executorService; - this.blocksRetained = prunerConfiguration.getBlocksRetained(); - this.blockConfirmations = prunerConfiguration.getBlockConfirmations(); - checkArgument( - blockConfirmations >= 0 && blockConfirmations < blocksRetained, - "blockConfirmations and blocksRetained must be non-negative. blockConfirmations must be less than blockRetained."); - } - - public Pruner( - final MarkSweepPruner pruningStrategy, - final Blockchain blockchain, - final PrunerConfiguration prunerConfiguration) { - this( - pruningStrategy, - blockchain, - prunerConfiguration, - // This is basically the out-of-the-box `Executors.newSingleThreadExecutor` except we want - // the `corePoolSize` to be 0 - new ThreadPoolExecutor( - 0, - 1, - 0L, - TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(), - new ThreadFactoryBuilder() - .setDaemon(true) - .setPriority(Thread.MIN_PRIORITY) - .setNameFormat("StatePruning-%d") - .build())); - } - - public void start() { - execute( - () -> { - if (state.compareAndSet(State.IDLE, State.RUNNING)) { - LOG.info("Starting Pruner."); - pruningStrategy.prepare(); - blockAddedObserverId = blockchain.observeBlockAdded(this::handleNewBlock); - } - }); - } - - public void stop() { - if (state.compareAndSet(State.RUNNING, State.STOPPED)) { - LOG.info("Stopping Pruner."); - pruningStrategy.cleanup(); - blockchain.removeObserver(blockAddedObserverId); - executorService.shutdownNow(); - } - } - - public void awaitStop() throws InterruptedException { - if (!executorService.awaitTermination(10, TimeUnit.SECONDS)) { - LOG.error("Failed to shutdown Pruner executor service."); - } - } - - private void handleNewBlock(final BlockAddedEvent event) { - if (!event.isNewCanonicalHead()) { - return; - } - - final long blockNumber = event.getBlock().getHeader().getNumber(); - if (pruningPhase.compareAndSet( - PruningPhase.IDLE, PruningPhase.MARK_BLOCK_CONFIRMATIONS_AWAITING)) { - markBlockNumber = blockNumber; - } else if (blockNumber >= markBlockNumber + blockConfirmations - && pruningPhase.compareAndSet( - PruningPhase.MARK_BLOCK_CONFIRMATIONS_AWAITING, PruningPhase.MARKING)) { - markedBlockHeader = blockchain.getBlockHeader(markBlockNumber).get(); - mark(markedBlockHeader); - } else if (blockNumber >= markBlockNumber + blocksRetained - && blockchain.blockIsOnCanonicalChain(markedBlockHeader.getHash()) - && pruningPhase.compareAndSet(PruningPhase.MARKING_COMPLETE, PruningPhase.SWEEPING)) { - sweep(); - } - } - - private void mark(final BlockHeader header) { - final Hash stateRoot = header.getStateRoot(); - LOG.info( - "Begin marking used nodes for pruning. Block number: {} State root: {}", - markBlockNumber, - stateRoot); - execute( - () -> { - pruningStrategy.mark(stateRoot); - pruningPhase.compareAndSet(PruningPhase.MARKING, PruningPhase.MARKING_COMPLETE); - }); - } - - private void sweep() { - LOG.info( - "Begin sweeping unused nodes for pruning. Keeping full state for blocks {} to {}", - markBlockNumber, - markBlockNumber + blocksRetained); - execute( - () -> { - pruningStrategy.sweepBefore(markBlockNumber); - pruningPhase.compareAndSet(PruningPhase.SWEEPING, PruningPhase.IDLE); - }); - } - - private void execute(final Runnable action) { - try { - executorService.execute(action); - } catch (final MerkleTrieException mte) { - LOG.error( - "An unrecoverable error occurred while pruning. The database directory must be deleted and resynced.", - mte); - System.exit(1); - } catch (final Exception e) { - LOG.error( - "An unexpected error occurred in the {} pruning phase: {}. Reattempting.", - getPruningPhase(), - e.getMessage()); - pruningStrategy.clearMarks(); - pruningPhase.set(PruningPhase.IDLE); - } - } - - PruningPhase getPruningPhase() { - return pruningPhase.get(); - } - - enum PruningPhase { - IDLE, - MARK_BLOCK_CONFIRMATIONS_AWAITING, - MARKING, - MARKING_COMPLETE, - SWEEPING; - } - - private enum State { - IDLE, - RUNNING, - STOPPED - } -} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerConfiguration.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerConfiguration.java deleted file mode 100644 index 871bdf8f300..00000000000 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerConfiguration.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.trie.forest.pruner; - -public class PrunerConfiguration { - public static final int DEFAULT_PRUNING_BLOCKS_RETAINED = 1024; - public static final int DEFAULT_PRUNING_BLOCK_CONFIRMATIONS = 10; - - private final int blocksRetainedBeforeSweeping; - private final int blockConfirmationsBeforeMarking; - - public PrunerConfiguration( - final int blockConfirmationsBeforeMarking, final int blocksRetainedBeforeSweeping) { - this.blockConfirmationsBeforeMarking = blockConfirmationsBeforeMarking; - this.blocksRetainedBeforeSweeping = blocksRetainedBeforeSweeping; - } - - public static PrunerConfiguration getDefault() { - return new PrunerConfiguration( - DEFAULT_PRUNING_BLOCK_CONFIRMATIONS, DEFAULT_PRUNING_BLOCKS_RETAINED); - } - - public int getBlocksRetained() { - return blocksRetainedBeforeSweeping; - } - - public int getBlockConfirmations() { - return blockConfirmationsBeforeMarking; - } -} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/MarkSweepPrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/MarkSweepPrunerTest.java deleted file mode 100644 index 1287800d9c3..00000000000 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/MarkSweepPrunerTest.java +++ /dev/null @@ -1,293 +0,0 @@ -/* - * Copyright Hyperledger Besu Contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.trie.forest.pruner; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider.createInMemoryBlockchain; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.inOrder; -import static org.mockito.Mockito.spy; - -import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.chain.MutableBlockchain; -import org.hyperledger.besu.ethereum.core.Block; -import org.hyperledger.besu.ethereum.core.BlockDataGenerator; -import org.hyperledger.besu.ethereum.core.BlockDataGenerator.BlockOptions; -import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.core.MutableWorldState; -import org.hyperledger.besu.ethereum.core.TransactionReceipt; -import org.hyperledger.besu.ethereum.rlp.RLP; -import org.hyperledger.besu.ethereum.storage.keyvalue.WorldStatePreimageKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.MerkleTrie; -import org.hyperledger.besu.ethereum.trie.forest.ForestWorldStateArchive; -import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; -import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; -import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; -import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; -import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; -import org.hyperledger.besu.evm.internal.EvmConfiguration; -import org.hyperledger.besu.evm.worldstate.WorldState; -import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.bytes.Bytes32; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InOrder; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -class MarkSweepPrunerTest { - - private final BlockDataGenerator gen = new BlockDataGenerator(); - private final NoOpMetricsSystem metricsSystem = new NoOpMetricsSystem(); - private final Map> hashValueStore = spy(new HashMap<>()); - private final InMemoryKeyValueStorage stateStorage = new TestInMemoryStorage(hashValueStore); - private final ForestWorldStateKeyValueStorage worldStateKeyValueStorage = - spy(new ForestWorldStateKeyValueStorage(stateStorage)); - private final WorldStateArchive worldStateArchive = - new ForestWorldStateArchive( - new WorldStateStorageCoordinator(worldStateKeyValueStorage), - new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()), - EvmConfiguration.DEFAULT); - private final InMemoryKeyValueStorage markStorage = new InMemoryKeyValueStorage(); - private final Block genesisBlock = gen.genesisBlock(); - private final MutableBlockchain blockchain = createInMemoryBlockchain(genesisBlock); - - @Test - void mark_marksAllExpectedNodes() { - final MarkSweepPruner pruner = - new MarkSweepPruner(worldStateKeyValueStorage, blockchain, markStorage, metricsSystem); - - // Generate accounts and save corresponding state root - final int numBlocks = 15; - final int numAccounts = 10; - generateBlockchainData(numBlocks, numAccounts); - - final int markBlockNumber = 10; - final BlockHeader markBlock = blockchain.getBlockHeader(markBlockNumber).get(); - // Collect the nodes we expect to keep - final Set expectedNodes = collectWorldStateNodes(markBlock.getStateRoot()); - assertThat(hashValueStore).hasSizeGreaterThan(expectedNodes.size()); // Sanity check - - // Mark and sweep - pruner.mark(markBlock.getStateRoot()); - pruner.sweepBefore(markBlock.getNumber()); - - // Assert that the block we marked is still present and all accounts are accessible - assertThat(worldStateArchive.get(markBlock.getStateRoot(), markBlock.getHash())).isPresent(); - final WorldState markedState = - worldStateArchive.get(markBlock.getStateRoot(), markBlock.getHash()).get(); - // Traverse accounts and make sure all are accessible - final int expectedAccounts = numAccounts * markBlockNumber; - final long accounts = markedState.streamAccounts(Bytes32.ZERO, expectedAccounts * 2).count(); - assertThat(accounts).isEqualTo(expectedAccounts); - // Traverse storage to ensure that all storage is accessible - markedState - .streamAccounts(Bytes32.ZERO, expectedAccounts * 2) - .forEach(a -> a.storageEntriesFrom(Bytes32.ZERO, 1000)); - - // All other state roots should have been removed - for (int i = 0; i < numBlocks; i++) { - final BlockHeader curHeader = blockchain.getBlockHeader(i + 1L).get(); - if (curHeader.getNumber() == markBlock.getNumber()) { - continue; - } - assertThat(worldStateArchive.get(curHeader.getStateRoot(), curHeader.getHash())).isEmpty(); - } - - // Check that storage contains only the values we expect - assertThat(hashValueStore).hasSameSizeAs(expectedNodes); - assertThat(hashValueStore.values().stream().map(Optional::get)) - .containsExactlyInAnyOrderElementsOf( - expectedNodes.stream().map(Bytes::toArrayUnsafe).collect(Collectors.toSet())); - } - - @Test - void sweepBefore_shouldSweepStateRootFirst() { - final MarkSweepPruner pruner = - new MarkSweepPruner(worldStateKeyValueStorage, blockchain, markStorage, metricsSystem); - - // Generate accounts and save corresponding state root - final int numBlocks = 15; - final int numAccounts = 10; - generateBlockchainData(numBlocks, numAccounts); - - final int markBlockNumber = 10; - final BlockHeader markBlock = blockchain.getBlockHeader(markBlockNumber).get(); - - // Collect state roots we expect to be swept first - final List stateRoots = new ArrayList<>(); - for (int i = markBlockNumber - 1; i >= 0; i--) { - stateRoots.add(blockchain.getBlockHeader(i).get().getStateRoot()); - } - - // Mark and sweep - pruner.mark(markBlock.getStateRoot()); - pruner.sweepBefore(markBlock.getNumber()); - - // we use individual inOrders because we only want to make sure we remove a state root before - // the full prune but without enforcing an ordering between the state root removals - stateRoots.forEach( - stateRoot -> { - final InOrder thisRootsOrdering = - inOrder(worldStateKeyValueStorage, hashValueStore, worldStateKeyValueStorage); - thisRootsOrdering.verify(worldStateKeyValueStorage).isWorldStateAvailable(stateRoot); - thisRootsOrdering.verify(hashValueStore).keySet(); - thisRootsOrdering.verify(worldStateKeyValueStorage).prune(any()); - }); - } - - @Test - void sweepBefore_shouldNotRemoveMarkedStateRoots() { - final MarkSweepPruner pruner = - new MarkSweepPruner(worldStateKeyValueStorage, blockchain, markStorage, metricsSystem); - - // Generate accounts and save corresponding state root - final int numBlocks = 15; - final int numAccounts = 10; - generateBlockchainData(numBlocks, numAccounts); - - final int markBlockNumber = 10; - final BlockHeader markBlock = blockchain.getBlockHeader(markBlockNumber).get(); - - // Collect state roots we expect to be swept first - final List stateRoots = new ArrayList<>(); - for (int i = markBlockNumber - 1; i >= 0; i--) { - stateRoots.add(blockchain.getBlockHeader(i).get().getStateRoot()); - } - - // Mark - pruner.mark(markBlock.getStateRoot()); - // Mark an extra state root - Hash markedRoot = Hash.wrap(stateRoots.remove(stateRoots.size() / 2)); - pruner.markNode(markedRoot); - // Sweep - pruner.sweepBefore(markBlock.getNumber()); - - // we use individual inOrders because we only want to make sure we remove a state root before - // the full prune but without enforcing an ordering between the state root removals - stateRoots.forEach( - stateRoot -> { - final InOrder thisRootsOrdering = - inOrder(worldStateKeyValueStorage, hashValueStore, worldStateKeyValueStorage); - thisRootsOrdering.verify(worldStateKeyValueStorage).isWorldStateAvailable(stateRoot); - thisRootsOrdering.verify(hashValueStore).keySet(); - thisRootsOrdering.verify(worldStateKeyValueStorage).prune(any()); - }); - - assertThat(stateStorage.containsKey(markedRoot.toArray())).isTrue(); - } - - private void generateBlockchainData(final int numBlocks, final int numAccounts) { - Block parentBlock = blockchain.getChainHeadBlock(); - for (int i = 0; i < numBlocks; i++) { - final BlockHeader parentHeader = parentBlock.getHeader(); - final MutableWorldState worldState = - worldStateArchive.getMutable(parentHeader.getStateRoot(), parentHeader.getHash()).get(); - gen.createRandomContractAccountsWithNonEmptyStorage(worldState, numAccounts); - final Hash stateRoot = worldState.rootHash(); - - final Block block = - gen.block( - BlockOptions.create() - .setStateRoot(stateRoot) - .setBlockNumber(parentHeader.getNumber() + 1L) - .setParentHash(parentBlock.getHash())); - final List receipts = gen.receipts(block); - blockchain.appendBlock(block, receipts); - parentBlock = block; - } - } - - private Set collectWorldStateNodes(final Hash stateRootHash) { - final Set nodeData = new HashSet<>(); - collectWorldStateNodes(stateRootHash, nodeData); - return nodeData; - } - - private Set collectWorldStateNodes(final Hash stateRootHash, final Set collector) { - final List storageRoots = new ArrayList<>(); - final MerkleTrie stateTrie = createStateTrie(stateRootHash); - - // Collect storage roots and code - stateTrie - .entriesFrom(Bytes32.ZERO, 1000) - .forEach( - (key, val) -> { - final StateTrieAccountValue accountValue = - StateTrieAccountValue.readFrom(RLP.input(val)); - stateStorage - .get(accountValue.getCodeHash().toArray()) - .ifPresent(v -> collector.add(Bytes.wrap(v))); - storageRoots.add(accountValue.getStorageRoot()); - }); - - // Collect state nodes - collectTrieNodes(stateTrie, collector); - // Collect storage nodes - for (Hash storageRoot : storageRoots) { - final MerkleTrie storageTrie = createStorageTrie(storageRoot); - collectTrieNodes(storageTrie, collector); - } - - return collector; - } - - private void collectTrieNodes(final MerkleTrie trie, final Set collector) { - final Bytes32 rootHash = trie.getRootHash(); - trie.visitAll( - (node) -> { - if (node.isReferencedByHash() || node.getHash().equals(rootHash)) { - collector.add(node.getEncodedBytes()); - } - }); - } - - private MerkleTrie createStateTrie(final Bytes32 rootHash) { - return new StoredMerklePatriciaTrie<>( - (location, hash) -> worldStateKeyValueStorage.getAccountStateTrieNode(hash), - rootHash, - Function.identity(), - Function.identity()); - } - - private MerkleTrie createStorageTrie(final Bytes32 rootHash) { - return new StoredMerklePatriciaTrie<>( - (location, hash) -> worldStateKeyValueStorage.getAccountStorageTrieNode(hash), - rootHash, - Function.identity(), - Function.identity()); - } - - // Proxy class so that we have access to the constructor that takes our own map - private static class TestInMemoryStorage extends InMemoryKeyValueStorage { - - public TestInMemoryStorage(final Map> hashValueStore) { - super(hashValueStore); - } - } -} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerTest.java deleted file mode 100644 index 308834a6a8a..00000000000 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerTest.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.trie.forest.pruner; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; - -import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.chain.Blockchain; -import org.hyperledger.besu.ethereum.chain.BlockchainStorage; -import org.hyperledger.besu.ethereum.chain.DefaultBlockchain; -import org.hyperledger.besu.ethereum.chain.MutableBlockchain; -import org.hyperledger.besu.ethereum.core.Block; -import org.hyperledger.besu.ethereum.core.BlockDataGenerator; -import org.hyperledger.besu.ethereum.core.BlockDataGenerator.BlockOptions; -import org.hyperledger.besu.ethereum.core.TransactionReceipt; -import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; -import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage; -import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage; -import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; -import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; -import org.hyperledger.besu.testutil.MockExecutorService; - -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -public class PrunerTest { - - private final NoOpMetricsSystem metricsSystem = new NoOpMetricsSystem(); - - private final BlockDataGenerator gen = new BlockDataGenerator(); - - @Mock private MarkSweepPruner markSweepPruner; - private final ExecutorService mockExecutorService = new MockExecutorService(); - - private final Block genesisBlock = gen.genesisBlock(); - - @Test - public void shouldMarkCorrectBlockAndSweep() throws ExecutionException, InterruptedException { - final BlockchainStorage blockchainStorage = - new KeyValueStoragePrefixedKeyBlockchainStorage( - new InMemoryKeyValueStorage(), - new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions(), - false); - final MutableBlockchain blockchain = - DefaultBlockchain.createMutable(genesisBlock, blockchainStorage, metricsSystem, 0); - - final Pruner pruner = - new Pruner(markSweepPruner, blockchain, new PrunerConfiguration(0, 1), mockExecutorService); - pruner.start(); - - final Block block1 = appendBlockWithParent(blockchain, genesisBlock); - appendBlockWithParent(blockchain, block1); - appendBlockWithParent(blockchain, blockchain.getChainHeadBlock()); - - verify(markSweepPruner).mark(block1.getHeader().getStateRoot()); - verify(markSweepPruner).sweepBefore(1); - pruner.stop(); - } - - @Test - public void shouldOnlySweepAfterBlockConfirmationPeriodAndRetentionPeriodEnds() { - final BlockchainStorage blockchainStorage = - new KeyValueStoragePrefixedKeyBlockchainStorage( - new InMemoryKeyValueStorage(), - new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions(), - false); - final MutableBlockchain blockchain = - DefaultBlockchain.createMutable(genesisBlock, blockchainStorage, metricsSystem, 0); - - final Pruner pruner = - new Pruner(markSweepPruner, blockchain, new PrunerConfiguration(1, 2), mockExecutorService); - pruner.start(); - - final Hash markBlockStateRootHash = - appendBlockWithParent(blockchain, genesisBlock).getHeader().getStateRoot(); - verify(markSweepPruner, never()).mark(markBlockStateRootHash); - verify(markSweepPruner, never()).sweepBefore(anyLong()); - - appendBlockWithParent(blockchain, blockchain.getChainHeadBlock()); - verify(markSweepPruner).mark(markBlockStateRootHash); - verify(markSweepPruner, never()).sweepBefore(anyLong()); - - appendBlockWithParent(blockchain, blockchain.getChainHeadBlock()); - verify(markSweepPruner).sweepBefore(1); - pruner.stop(); - } - - @Test - public void abortsPruningWhenFullyMarkedBlockNoLongerOnCanonicalChain() { - final BlockchainStorage blockchainStorage = - new KeyValueStoragePrefixedKeyBlockchainStorage( - new InMemoryKeyValueStorage(), - new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions(), - false); - final MutableBlockchain blockchain = - DefaultBlockchain.createMutable(genesisBlock, blockchainStorage, metricsSystem, 0); - - // start pruner so it can start handling block added events - final Pruner pruner = - new Pruner(markSweepPruner, blockchain, new PrunerConfiguration(0, 1), mockExecutorService); - pruner.start(); - - /* - Set up pre-marking state: - O <---- marking of this block's parent will begin when this block is added - | - | O <- this is a fork as of now (non-canonical) - O | <- this is the initially canonical block that will be marked - \/ - O <--- the common ancestor when the reorg happens - */ - final Block initiallyCanonicalBlock = appendBlockWithParent(blockchain, genesisBlock); - appendBlockWithParent(blockchain, initiallyCanonicalBlock); - final Block forkBlock = appendBlockWithParent(blockchain, genesisBlock); - /* - Cause reorg: - Set up pre-marking state: - O - | O <---- this block causes a reorg; this branch becomes canonical - | O <---- which means that state here is referring to nodes from the common ancestor block, - O | <- because this was block at which marking began - \/ - O - */ - appendBlockWithParent(blockchain, forkBlock); - verify(markSweepPruner).mark(initiallyCanonicalBlock.getHeader().getStateRoot()); - verify(markSweepPruner, never()).sweepBefore(anyLong()); - pruner.stop(); - } - - @Test - public void shouldRejectInvalidArguments() { - final Blockchain mockchain = mock(Blockchain.class); - assertThatThrownBy( - () -> - new Pruner( - markSweepPruner, - mockchain, - new PrunerConfiguration(-1, -2), - mockExecutorService)) - .isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy( - () -> - new Pruner( - markSweepPruner, - mockchain, - new PrunerConfiguration(10, 8), - mockExecutorService)) - .isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy( - () -> - new Pruner( - markSweepPruner, - mockchain, - new PrunerConfiguration(10, 10), - mockExecutorService)) - .isInstanceOf(IllegalArgumentException.class); - } - - @Test - public void shouldCleanUpPruningStrategyOnShutdown() throws InterruptedException { - final BlockchainStorage blockchainStorage = - new KeyValueStoragePrefixedKeyBlockchainStorage( - new InMemoryKeyValueStorage(), - new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions(), - false); - final MutableBlockchain blockchain = - DefaultBlockchain.createMutable(genesisBlock, blockchainStorage, metricsSystem, 0); - - final Pruner pruner = - new Pruner(markSweepPruner, blockchain, new PrunerConfiguration(0, 1), mockExecutorService); - pruner.start(); - pruner.stop(); - verify(markSweepPruner).cleanup(); - } - - private Block appendBlockWithParent(final MutableBlockchain blockchain, final Block parent) { - BlockOptions options = - new BlockOptions() - .setBlockNumber(parent.getHeader().getNumber() + 1) - .setParentHash(parent.getHash()); - final Block newBlock = gen.block(options); - final List receipts = gen.receipts(newBlock); - blockchain.appendBlock(newBlock, receipts); - return newBlock; - } -} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java index 1a82d56355f..28c1d193d19 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java @@ -35,7 +35,6 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.BonsaiWorldStateProvider; -import org.hyperledger.besu.ethereum.trie.forest.pruner.Pruner; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.data.SyncStatus; @@ -62,7 +61,6 @@ public class DefaultSynchronizer implements Synchronizer, UnverifiedForkchoiceLi private static final Logger LOG = LoggerFactory.getLogger(DefaultSynchronizer.class); - private final Optional maybePruner; private final SyncState syncState; private final AtomicBoolean running = new AtomicBoolean(false); private final Optional blockPropagationManager; @@ -79,7 +77,6 @@ public DefaultSynchronizer( final ProtocolContext protocolContext, final WorldStateStorageCoordinator worldStateStorageCoordinator, final BlockBroadcaster blockBroadcaster, - final Optional maybePruner, final EthContext ethContext, final SyncState syncState, final Path dataDirectory, @@ -88,7 +85,6 @@ public DefaultSynchronizer( final MetricsSystem metricsSystem, final SyncTerminationCondition terminationCondition, final PivotBlockSelector pivotBlockSelector) { - this.maybePruner = maybePruner; this.syncState = syncState; this.pivotBlockSelector = pivotBlockSelector; this.protocolContext = protocolContext; @@ -228,7 +224,6 @@ public void stop() { LOG.info("Stopping synchronizer"); fastSyncDownloader.ifPresent(FastSyncDownloader::stop); fullSyncDownloader.ifPresent(FullSyncDownloader::stop); - maybePruner.ifPresent(Pruner::stop); blockPropagationManager.ifPresent( manager -> { if (manager.isRunning()) { @@ -239,11 +234,7 @@ public void stop() { } @Override - public void awaitStop() throws InterruptedException { - if (maybePruner.isPresent()) { - maybePruner.get().awaitStop(); - } - } + public void awaitStop() {} private CompletableFuture handleSyncResult(final FastSyncState result) { if (!running.get()) { @@ -270,7 +261,6 @@ private CompletableFuture handleSyncResult(final FastSyncState result) { } private CompletableFuture startFullSync() { - maybePruner.ifPresent(Pruner::start); return fullSyncDownloader .map(FullSyncDownloader::start) .orElse(CompletableFuture.completedFuture(null)) @@ -376,7 +366,6 @@ private Void finalizeSync(final Void unused) { LOG.info("Stopping block propagation."); blockPropagationManager.ifPresent(BlockPropagationManager::stop); LOG.info("Stopping the pruner."); - maybePruner.ifPresent(Pruner::stop); running.set(false); return null; } From eab55f7599103c67c36ad0236fafb3447ccbc97f Mon Sep 17 00:00:00 2001 From: Karim TAAM Date: Fri, 29 Mar 2024 18:41:00 +0100 Subject: [PATCH 17/24] fix account copy issue (#6845) Signed-off-by: Karim Taam --- .../trie/diffbased/bonsai/BonsaiAccount.java | 2 + .../diffbased/common/DiffBasedAccount.java | 66 +++++++++++------ .../diffbased/bonsai/BonsaiAccountTest.java | 74 +++++++++++++++++++ 3 files changed, 121 insertions(+), 21 deletions(-) create mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccountTest.java diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccount.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccount.java index 6eea664e048..7ca6ef2b693 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccount.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccount.java @@ -82,6 +82,7 @@ public BonsaiAccount( toCopy.nonce, toCopy.balance, toCopy.codeHash, + toCopy.code, mutable); this.storageRoot = toCopy.storageRoot; updatedStorage.putAll(toCopy.updatedStorage); @@ -96,6 +97,7 @@ public BonsaiAccount( tracked.getNonce(), tracked.getBalance(), tracked.getCodeHash(), + tracked.getCode(), true); this.storageRoot = Hash.EMPTY_TRIE_HASH; updatedStorage.putAll(tracked.getUpdatedStorage()); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java index 044deb45a01..9b0a54fa468 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/DiffBasedAccount.java @@ -42,6 +42,22 @@ public abstract class DiffBasedAccount implements MutableAccount, AccountValue { protected final Map updatedStorage = new HashMap<>(); + /** + * Constructs a new DiffBasedAccount instance without the account's code. This constructor is used + * when the account's code is not required or will not be read from the database. It initializes + * the account with its context, address, address hash, nonce, balance, code hash, and mutability + * status. + * + * @param context The DiffBasedWorldView context in which this account exists. + * @param address The Ethereum address of this account. + * @param addressHash The hash of the account's address. + * @param nonce The nonce of the account, representing the number of transactions sent from this + * account. + * @param balance The balance of the account in Wei. + * @param codeHash The hash of the account's code. + * @param mutable A boolean indicating if the account is mutable. If false, the account is + * considered immutable. + */ public DiffBasedAccount( final DiffBasedWorldView context, final Address address, @@ -60,32 +76,40 @@ public DiffBasedAccount( this.immutable = !mutable; } + /** + * Constructs a new DiffBasedAccount instance with the account's code. This constructor is used + * when all account information, including its code, are available. It initializes the account + * with its context, address, address hash, nonce, balance, code hash, the actual code, and + * mutability status. + * + * @param context The DiffBasedWorldView context in which this account exists. + * @param address The Ethereum address of this account. + * @param addressHash The hash of the account's address. + * @param nonce The nonce of the account, representing the number of transactions sent from this + * account. + * @param balance The balance of the account in Wei. + * @param codeHash The hash of the account's code. + * @param code The actual bytecode of the account's smart contract. This is provided when the code + * is known and needs to be associated with the account. + * @param mutable A boolean indicating if the account is mutable. If false, the account is + * considered immutable. + */ public DiffBasedAccount( final DiffBasedWorldView context, final Address address, - final AccountValue stateTrieAccount, + final Hash addressHash, + final long nonce, + final Wei balance, + final Hash codeHash, + final Bytes code, final boolean mutable) { - this( - context, - address, - address.addressHash(), - stateTrieAccount.getNonce(), - stateTrieAccount.getBalance(), - stateTrieAccount.getCodeHash(), - mutable); - } - - public DiffBasedAccount( - final DiffBasedAccount toCopy, final DiffBasedWorldView context, final boolean mutable) { this.context = context; - this.address = toCopy.address; - this.addressHash = toCopy.addressHash; - this.nonce = toCopy.nonce; - this.balance = toCopy.balance; - this.codeHash = toCopy.codeHash; - this.code = toCopy.code; - updatedStorage.putAll(toCopy.updatedStorage); - + this.address = address; + this.addressHash = addressHash; + this.nonce = nonce; + this.balance = balance; + this.codeHash = codeHash; + this.code = code; this.immutable = !mutable; } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccountTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccountTest.java new file mode 100644 index 00000000000..60396b9789f --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/BonsaiAccountTest.java @@ -0,0 +1,74 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie.diffbased.bonsai; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.hyperledger.besu.datatypes.Address; +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.datatypes.Wei; +import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.worldview.BonsaiWorldState; +import org.hyperledger.besu.evm.worldstate.UpdateTrackingAccount; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; + +public class BonsaiAccountTest { + + @Mock BonsaiWorldState bonsaiWorldState; + + @Test + void shouldCopyTrackedBonsaiAccountCorrectly() { + final BonsaiAccount trackedAccount = + new BonsaiAccount( + bonsaiWorldState, + Address.ZERO, + Hash.hash(Address.ZERO), + 0, + Wei.ONE, + Hash.EMPTY_TRIE_HASH, + Hash.EMPTY, + true); + trackedAccount.setCode(Bytes.of(1)); + final UpdateTrackingAccount bonsaiAccountUpdateTrackingAccount = + new UpdateTrackingAccount<>(trackedAccount); + bonsaiAccountUpdateTrackingAccount.setStorageValue(UInt256.ONE, UInt256.ONE); + + final BonsaiAccount expectedAccount = new BonsaiAccount(trackedAccount, bonsaiWorldState, true); + expectedAccount.setStorageValue(UInt256.ONE, UInt256.ONE); + assertThat(new BonsaiAccount(bonsaiWorldState, bonsaiAccountUpdateTrackingAccount)) + .isEqualToComparingFieldByField(expectedAccount); + } + + @Test + void shouldCopyBonsaiAccountCorrectly() { + final BonsaiAccount account = + new BonsaiAccount( + bonsaiWorldState, + Address.ZERO, + Hash.hash(Address.ZERO), + 0, + Wei.ONE, + Hash.EMPTY_TRIE_HASH, + Hash.EMPTY, + true); + account.setCode(Bytes.of(1)); + account.setStorageValue(UInt256.ONE, UInt256.ONE); + assertThat(new BonsaiAccount(account, bonsaiWorldState, true)) + .isEqualToComparingFieldByField(account); + } +} From 9b3a2192101168f54177cd182d512fee6c91160a Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Sat, 30 Mar 2024 08:30:35 +1000 Subject: [PATCH 18/24] increase timeout (#6852) Signed-off-by: Sally MacFarlane --- .../ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java index 83355ecc291..438433c5181 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java @@ -192,7 +192,7 @@ public void shouldAwokeWhenConditionReachedAndReady() throws Exception { completionCaptor.getValue().onInitialSyncCompleted(); - voidCompletableFuture.get(200, TimeUnit.MILLISECONDS); + voidCompletableFuture.get(500, TimeUnit.MILLISECONDS); assertThat(voidCompletableFuture).isCompleted(); verify(context.getSyncState()).unsubscribeTTDReached(88L); From d926052eb21ffdef1b56d55170107a62aee5c639 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Sat, 30 Mar 2024 09:19:12 +1000 Subject: [PATCH 19/24] disable flaky test - LegacyFeeMarketBlockTransactionSelectorTest (#6851) * disable flaky test Signed-off-by: Sally MacFarlane * added message to disabled annotation Signed-off-by: Sally MacFarlane --------- Signed-off-by: Sally MacFarlane --- .../LegacyFeeMarketBlockTransactionSelectorTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/LegacyFeeMarketBlockTransactionSelectorTest.java b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/LegacyFeeMarketBlockTransactionSelectorTest.java index 39db9c05cf2..0235d8c1d46 100644 --- a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/LegacyFeeMarketBlockTransactionSelectorTest.java +++ b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/LegacyFeeMarketBlockTransactionSelectorTest.java @@ -42,6 +42,10 @@ import java.time.ZoneId; import java.util.function.Function; +import org.junit.jupiter.api.Disabled; + +@Disabled( + "disabled since it's flaky with a timeout see https://github.com/hyperledger/besu/issues/6850") public class LegacyFeeMarketBlockTransactionSelectorTest extends AbstractBlockTransactionSelectorTest { From deaea9b34da62b707509186e2955864e13212cda Mon Sep 17 00:00:00 2001 From: garyschulte Date: Fri, 29 Mar 2024 17:13:30 -0700 Subject: [PATCH 20/24] Snap client fixes (#6847) * manage empty range for storage * round rather than floor on max remote connections so that maxpeers=1 still can accept remote connections --------- Signed-off-by: garyschulte Co-authored-by: Karim Taam Co-authored-by: Sally MacFarlane --- CHANGELOG.md | 1 + .../org/hyperledger/besu/cli/BesuCommand.java | 2 +- .../eth/sync/snapsync/RequestDataStep.java | 44 ++++++++++++++----- .../ethereum/eth/sync/snapsync/StackTrie.java | 4 ++ 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d7730f0e3c..09e2439489b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ - In JSON-RPC return optional `v` fields for type 1 and type 2 transactions [#6762](https://github.com/hyperledger/besu/pull/6762) - Fix Shanghai/QBFT block import bug when syncing new nodes [#6765](https://github.com/hyperledger/besu/pull/6765) - Fix to avoid broadcasting full blob txs, instead of only the tx announcement, to a subset of nodes [#6835](https://github.com/hyperledger/besu/pull/6835) +- Snap client fixes discovered during snap server testing [#6847](https://github.com/hyperledger/besu/pull/6847) ### Download Links diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 1c63329ac25..26422b0f660 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -1554,7 +1554,7 @@ private void ensureValidPeerBoundParams() { checkState( fraction >= 0.0 && fraction <= 1.0, "Fraction of remote connections allowed must be between 0.0 and 1.0 (inclusive)."); - maxRemoteInitiatedPeers = (int) Math.floor(fraction * maxPeers); + maxRemoteInitiatedPeers = Math.round(fraction * maxPeers); } else { maxRemoteInitiatedPeers = maxPeers; } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RequestDataStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RequestDataStep.java index adbb8b736c1..f866f4c41ba 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RequestDataStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RequestDataStep.java @@ -41,6 +41,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.TreeMap; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; @@ -49,9 +50,11 @@ import kotlin.collections.ArrayDeque; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class RequestDataStep { - + private static final Logger LOG = LoggerFactory.getLogger(RequestDataStep.class); private final WorldStateStorageCoordinator worldStateStorageCoordinator; private final SnapSyncProcessState fastSyncState; private final SnapWorldDownloadState downloadState; @@ -131,15 +134,36 @@ public CompletableFuture>> requestStorage( (response, error) -> { if (response != null) { downloadState.removeOutstandingTask(getStorageRangeTask); - for (int i = 0; i < response.slots().size(); i++) { - final StorageRangeDataRequest request = - (StorageRangeDataRequest) requestTasks.get(i).getData(); - request.setRootHash(blockHeader.getStateRoot()); - request.addResponse( - downloadState, - worldStateProofProvider, - response.slots().get(i), - i < response.slots().size() - 1 ? new ArrayDeque<>() : response.proofs()); + final ArrayDeque> slots = new ArrayDeque<>(); + // Check if we have an empty range + + /* + * Checks if the response represents an "empty range". + * + * An "empty range" is defined as a response where at least one proof exists + * and either no slots are present, or the first slot is empty + */ + try { + final boolean isEmptyRange = + (response.slots().isEmpty() || response.slots().get(0).isEmpty()) + && !response.proofs().isEmpty(); + if (isEmptyRange) { // empty range detected + slots.add(new TreeMap<>()); + } else { + slots.addAll(response.slots()); + } + for (int i = 0; i < slots.size(); i++) { + final StorageRangeDataRequest request = + (StorageRangeDataRequest) requestTasks.get(i).getData(); + request.setRootHash(blockHeader.getStateRoot()); + request.addResponse( + downloadState, + worldStateProofProvider, + slots.get(i), + i < slots.size() - 1 ? new ArrayDeque<>() : response.proofs()); + } + } catch (final Exception e) { + LOG.error("Error while processing storage range response", e); } } return requestTasks; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrie.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrie.java index 1cf593665a0..7221c8da4fa 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrie.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrie.java @@ -114,6 +114,10 @@ public void commit(final FlatDatabaseUpdater flatDatabaseUpdater, final NodeUpda keys.putAll(taskElement.keys()); }); + if (keys.isEmpty()) { + return; // empty range we can ignore it + } + final Map proofsEntries = new HashMap<>(); for (Bytes proof : proofs) { proofsEntries.put(Hash.hash(proof), proof); From 34fc5eed583ec6df7903f452da3b7e0be04b6417 Mon Sep 17 00:00:00 2001 From: garyschulte Date: Fri, 29 Mar 2024 19:27:33 -0700 Subject: [PATCH 21/24] Snap server rebase (#6640) * initial snap server implementation Signed-off-by: garyschulte Co-authored-by: Sally MacFarlane --- CHANGELOG.md | 1 + .../org/hyperledger/besu/cli/BesuCommand.java | 2 + .../cli/ConfigurationOverviewBuilder.java | 16 + .../options/unstable/SynchronizerOptions.java | 23 +- .../controller/BesuControllerBuilder.java | 28 +- .../org/hyperledger/besu/datatypes/Hash.java | 3 + ethereum/core/build.gradle | 1 + .../proof/WorldStateProofProvider.java | 30 +- .../NoOpBonsaiCachedWorldStorageManager.java | 2 +- .../BonsaiWorldStateKeyValueStorage.java | 4 +- .../common/DiffBasedWorldStateProvider.java | 2 +- .../DiffBasedCachedWorldStorageManager.java | 50 +- .../DiffBasedWorldStateKeyValueStorage.java | 26 +- .../common/storage/flat/FlatDbStrategy.java | 101 ++- .../storage/flat/FlatDbStrategyProvider.java | 2 +- .../worldstate/WorldStateArchive.java | 3 - .../WorldStateStorageCoordinator.java | 5 + .../bonsai/BonsaiSnapshotIsolationTests.java | 8 +- .../eth/EthProtocolConfiguration.java | 2 +- .../eth/manager/snap/SnapProtocolManager.java | 12 +- .../ethereum/eth/manager/snap/SnapServer.java | 626 ++++++++++++++++- .../messages/snap/GetAccountRangeMessage.java | 3 + .../messages/snap/GetStorageRangeMessage.java | 2 +- .../eth/sync/DefaultSynchronizer.java | 9 + .../sync/snapsync/SnapSyncConfiguration.java | 7 + .../ethereum/eth/sync/snapsync/StackTrie.java | 65 +- .../request/AccountRangeDataRequest.java | 6 + .../request/StorageRangeDataRequest.java | 7 + ...torageFlatDatabaseHealingRangeRequest.java | 2 +- .../eth/manager/snap/SnapServerTest.java | 629 ++++++++++++++++++ .../task/SnapProtocolManagerTestUtil.java | 61 -- .../besu/ethereum/trie/CompactEncoding.java | 32 + .../kvstore/InMemoryKeyValueStorage.java | 10 +- .../kvstore/LayeredKeyValueStorage.java | 119 +++- .../SegmentedInMemoryKeyValueStorage.java | 52 +- 35 files changed, 1714 insertions(+), 237 deletions(-) create mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/snap/SnapServerTest.java delete mode 100644 ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/task/SnapProtocolManagerTestUtil.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 09e2439489b..59828b7e241 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ - Dedicated log marker for invalid txs removed from the txpool [#6826](https://github.com/hyperledger/besu/pull/6826) - Prevent startup with BONSAI and privacy enabled [#6809](https://github.com/hyperledger/besu/pull/6809) - Remove deprecated Forest pruning [#6810](https://github.com/hyperledger/besu/pull/6810) +- Experimental Snap Sync Server [#6640](https://github.com/hyperledger/besu/pull/6640) ### Bug fixes - Fix txpool dump/restore race condition [#6665](https://github.com/hyperledger/besu/pull/6665) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 26422b0f660..624699f70c5 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -2732,6 +2732,8 @@ private String generateConfigurationOverview() { getDataStorageConfiguration().getUnstable().getBonsaiTrieLogPruningWindowSize()); } + builder.setSnapServerEnabled(this.unstableSynchronizerOptions.isSnapsyncServerEnabled()); + builder.setTxPoolImplementation(buildTransactionPoolConfiguration().getTxPoolImplementation()); builder.setWorldStateUpdateMode(unstableEvmOptions.toDomainObject().worldUpdaterMode()); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java index bf03c675d8d..2b3dfd7dd51 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java @@ -55,6 +55,7 @@ public class ConfigurationOverviewBuilder { private boolean isBonsaiLimitTrieLogsEnabled = false; private long trieLogRetentionLimit = 0; private Integer trieLogsPruningWindowSize = null; + private boolean isSnapServerEnabled = false; private TransactionPoolConfiguration.Implementation txPoolImplementation; private EvmConfiguration.WorldUpdaterMode worldStateUpdateMode; private Map environment; @@ -219,6 +220,17 @@ public ConfigurationOverviewBuilder setTrieLogRetentionLimit(final long limit) { return this; } + /** + * Sets snap server enabled/disabled + * + * @param snapServerEnabled bool to indicate if snap server is enabled + * @return the builder + */ + public ConfigurationOverviewBuilder setSnapServerEnabled(final boolean snapServerEnabled) { + isSnapServerEnabled = snapServerEnabled; + return this; + } + /** * Sets trie logs pruning window size * @@ -339,6 +351,10 @@ public String build() { lines.add("Using " + worldStateUpdateMode + " worldstate update mode"); + if (isSnapServerEnabled) { + lines.add("Experimental Snap Sync server enabled"); + } + if (isBonsaiLimitTrieLogsEnabled) { final StringBuilder trieLogPruningString = new StringBuilder(); trieLogPruningString diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java index 7c9de815797..da5199f85dc 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/SynchronizerOptions.java @@ -82,6 +82,8 @@ public class SynchronizerOptions implements CLIOptions