From 6542df0361d6e113db3f5dbd4d5fb84b6ffcd178 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Thu, 21 Mar 2024 13:41:19 -0400 Subject: [PATCH 01/18] Artifactory publish (#6781) * correcting env vars * removes conditionals which result in skipped tasks being considered success * spotless fix --------- Signed-off-by: Justin Florentine --- .github/workflows/artifacts.yml | 4 ++-- .github/workflows/pre-review.yml | 1 - .../ethereum/api/graphql/scalar/Bytes32ScalarTest.java | 7 +++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index e7db5a04ae2..979745d23a9 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -48,13 +48,13 @@ jobs: compression-level: 0 - name: Artifactory Publish env: - ARTIFACTORY_USER: ${{ secrets.BESU_ARTIFACTORY }} + ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }} + ARTIFACTORY_KEY: ${{ secrets.BESU_ARTIFACTORY }} run: ./gradlew -Prelease.releaseVersion=${{ github.ref_name }} -Pversion=${{github.ref_name}} artifactoryPublish testWindows: runs-on: windows-2022 needs: artifacts timeout-minutes: 10 - if: ${{ github.actor != 'dependabot[bot]' }} steps: - name: Set up Java uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93 diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 6a0b6e79ddb..e7dc683dea4 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -35,7 +35,6 @@ jobs: - uses: gradle/wrapper-validation-action@56b90f209b02bf6d1deae490e9ef18b21a389cd4 spotless: runs-on: ubuntu-22.04 - if: ${{ github.actor != 'dependabot[bot]' }} steps: - name: Checkout Repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/scalar/Bytes32ScalarTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/scalar/Bytes32ScalarTest.java index fd7d4a61b5a..4f74fd103af 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/scalar/Bytes32ScalarTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/graphql/scalar/Bytes32ScalarTest.java @@ -44,7 +44,7 @@ public class Bytes32ScalarTest { private final StringValue strValue = StringValue.newStringValue(str).build(); private final StringValue invalidStrValue = StringValue.newStringValue("0xgh").build(); - private final VersionedHash versionedHash = new VersionedHash((byte)1, Hash.hash(value)); + private final VersionedHash versionedHash = new VersionedHash((byte) 1, Hash.hash(value)); @Test public void pareValueTest() { @@ -127,7 +127,10 @@ public void parseLiteralErrorTest2() { @Test public void parseVersionedHash() { - assertThat(scalar.getCoercing().serialize(versionedHash, GraphQLContext.newContext().build(), Locale.ENGLISH)) + assertThat( + scalar + .getCoercing() + .serialize(versionedHash, GraphQLContext.newContext().build(), Locale.ENGLISH)) .isEqualTo(versionedHash.toString()); } From bf03438fecb4f59a0d8532471be0b5928876f34b Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Thu, 21 Mar 2024 17:32:47 -0400 Subject: [PATCH 02/18] typo in env var (#6780) * get value for sonar options from vars --------- Signed-off-by: Justin Florentine --- .github/workflows/sonarcloud.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index fc926a8adcc..9f70e699178 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -37,5 +37,5 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} SONAR_ORGANIZATION: ${{ vars.SONAR_ORGANIZATION }} - SONAR_PROJECT_KEY: $ {{ vars.SONAR_PROJECT_KEY }} + SONAR_PROJECT_KEY: ${{ vars.SONAR_PROJECT_KEY }} run: ./gradlew build sonarqube --continue --info -Dorg.gradle.parallel=true -Dorg.gradle.caching=true From e3ea44e0b6f44d1761a2323fcc691b7ea8d9f1c5 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Fri, 22 Mar 2024 08:57:22 +1000 Subject: [PATCH 03/18] for fcu v2 use correct invalid params error code (#6766) * fcu v2 use invalid params error code per spec Signed-off-by: Sally MacFarlane --------- Signed-off-by: Sally MacFarlane Signed-off-by: Justin Florentine --- ...pare_payload_invalid_null_withdrawals.json | 4 ++-- .../AbstractEngineForkchoiceUpdated.java | 10 +++++++--- .../engine/EngineForkchoiceUpdatedV1.java | 2 +- .../engine/EngineForkchoiceUpdatedV2.java | 8 +------- .../engine/EngineForkchoiceUpdatedV3.java | 20 +++++++++---------- .../AbstractEngineForkchoiceUpdatedTest.java | 2 +- .../engine/EngineForkchoiceUpdatedV2Test.java | 2 +- 7 files changed, 22 insertions(+), 26 deletions(-) diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/shanghai/test-cases/05_shanghai_prepare_payload_invalid_null_withdrawals.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/shanghai/test-cases/05_shanghai_prepare_payload_invalid_null_withdrawals.json index 23ddea33af0..ac5947c79be 100644 --- a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/shanghai/test-cases/05_shanghai_prepare_payload_invalid_null_withdrawals.json +++ b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/shanghai/test-cases/05_shanghai_prepare_payload_invalid_null_withdrawals.json @@ -20,8 +20,8 @@ "jsonrpc" : "2.0", "id" : 67, "error" : { - "code" : -38003, - "message" : "Invalid payload attributes" + "code" : -32602, + "message" : "Invalid params" } }, "statusCode" : 200 diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractEngineForkchoiceUpdated.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractEngineForkchoiceUpdated.java index 442ca8448ec..0aac956d416 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractEngineForkchoiceUpdated.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractEngineForkchoiceUpdated.java @@ -172,7 +172,7 @@ public JsonRpcResponse syncResponse(final JsonRpcRequestContext requestContext) if (!getWithdrawalsValidator( protocolSchedule.get(), newHead, maybePayloadAttributes.get().getTimestamp()) .validateWithdrawals(withdrawals)) { - return new JsonRpcErrorResponse(requestId, getInvalidPayloadError()); + return new JsonRpcErrorResponse(requestId, getInvalidParametersError()); } } @@ -241,7 +241,7 @@ protected Optional isPayloadAttributeRelevantToNewHead( if (payloadAttributes.getTimestamp() <= headBlockHeader.getTimestamp()) { LOG.warn( "Payload attributes timestamp is smaller than timestamp of header in fork choice update"); - return Optional.of(new JsonRpcErrorResponse(requestId, getInvalidPayloadError())); + return Optional.of(new JsonRpcErrorResponse(requestId, getInvalidPayloadAttributesError())); } return Optional.empty(); @@ -364,10 +364,14 @@ protected boolean requireTerminalPoWBlockValidation() { return false; } - protected RpcErrorType getInvalidPayloadError() { + protected RpcErrorType getInvalidParametersError() { return RpcErrorType.INVALID_PARAMS; } + protected RpcErrorType getInvalidPayloadAttributesError() { + return RpcErrorType.INVALID_PAYLOAD_ATTRIBUTES; + } + // fcU calls are synchronous, no need to make volatile private long lastFcuInfoLog = System.currentTimeMillis(); private static final String logMessage = diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdatedV1.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdatedV1.java index 6aa5f0964b2..e38ae835027 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdatedV1.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdatedV1.java @@ -54,7 +54,7 @@ protected boolean requireTerminalPoWBlockValidation() { } @Override - protected RpcErrorType getInvalidPayloadError() { + protected RpcErrorType getInvalidParametersError() { return RpcErrorType.INVALID_PAYLOAD_ATTRIBUTES; } } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdatedV2.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdatedV2.java index 77ca45df043..98c5cc7280e 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdatedV2.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdatedV2.java @@ -62,15 +62,9 @@ protected Optional isPayloadAttributesValid( } else if (payloadAttributes.getParentBeaconBlockRoot() != null) { LOG.error( "Parent beacon block root hash present in payload attributes before cancun hardfork"); - return Optional.of( - new JsonRpcErrorResponse(requestId, RpcErrorType.INVALID_PAYLOAD_ATTRIBUTES)); + return Optional.of(new JsonRpcErrorResponse(requestId, getInvalidPayloadAttributesError())); } else { return Optional.empty(); } } - - @Override - protected RpcErrorType getInvalidPayloadError() { - return RpcErrorType.INVALID_PAYLOAD_ATTRIBUTES; - } } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdatedV3.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdatedV3.java index 51672d7280e..05b4fa29780 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdatedV3.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdatedV3.java @@ -56,16 +56,19 @@ protected ValidationResult validateParameter( final EngineForkchoiceUpdatedParameter fcuParameter, final Optional maybePayloadAttributes) { if (fcuParameter.getHeadBlockHash() == null) { - return ValidationResult.invalid(getInvalidPayloadError(), "Missing head block hash"); + return ValidationResult.invalid( + getInvalidPayloadAttributesError(), "Missing head block hash"); } else if (fcuParameter.getSafeBlockHash() == null) { - return ValidationResult.invalid(getInvalidPayloadError(), "Missing safe block hash"); + return ValidationResult.invalid( + getInvalidPayloadAttributesError(), "Missing safe block hash"); } else if (fcuParameter.getFinalizedBlockHash() == null) { - return ValidationResult.invalid(getInvalidPayloadError(), "Missing finalized block hash"); + return ValidationResult.invalid( + getInvalidPayloadAttributesError(), "Missing finalized block hash"); } if (maybePayloadAttributes.isPresent()) { if (maybePayloadAttributes.get().getParentBeaconBlockRoot() == null) { return ValidationResult.invalid( - getInvalidPayloadError(), "Missing parent beacon block root hash"); + getInvalidPayloadAttributesError(), "Missing parent beacon block root hash"); } } return ValidationResult.valid(); @@ -93,18 +96,13 @@ protected Optional isPayloadAttributesValid( if (payloadAttributes.getParentBeaconBlockRoot() == null) { LOG.error( "Parent beacon block root hash not present in payload attributes after cancun hardfork"); - return Optional.of(new JsonRpcErrorResponse(requestId, getInvalidPayloadError())); + return Optional.of(new JsonRpcErrorResponse(requestId, getInvalidPayloadAttributesError())); } else if (payloadAttributes.getTimestamp().longValue() == 0) { - return Optional.of(new JsonRpcErrorResponse(requestId, getInvalidPayloadError())); + return Optional.of(new JsonRpcErrorResponse(requestId, getInvalidPayloadAttributesError())); } else if (payloadAttributes.getTimestamp() < cancun.get().milestone()) { return Optional.of(new JsonRpcErrorResponse(requestId, RpcErrorType.UNSUPPORTED_FORK)); } else { return Optional.empty(); } } - - @Override - protected RpcErrorType getInvalidPayloadError() { - return RpcErrorType.INVALID_PAYLOAD_ATTRIBUTES; - } } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractEngineForkchoiceUpdatedTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractEngineForkchoiceUpdatedTest.java index 931d2259875..8391ede830e 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractEngineForkchoiceUpdatedTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractEngineForkchoiceUpdatedTest.java @@ -475,7 +475,7 @@ public void shouldReturnInvalidIfPayloadTimestampNotGreaterThanHead() { mockHeader.getHash(), Hash.ZERO, mockParent.getHash()), Optional.of(payloadParams)); - assertInvalidForkchoiceState(resp, expectedInvalidPayloadError()); + assertInvalidForkchoiceState(resp, RpcErrorType.INVALID_PAYLOAD_ATTRIBUTES); verify(engineCallListener, times(1)).executionEngineCalled(); } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdatedV2Test.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdatedV2Test.java index 4cb17eab4f7..b4fbccf1c72 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdatedV2Test.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdatedV2Test.java @@ -46,6 +46,6 @@ protected String getMethodName() { @Override protected RpcErrorType expectedInvalidPayloadError() { - return RpcErrorType.INVALID_PAYLOAD_ATTRIBUTES; + return RpcErrorType.INVALID_PARAMS; } } From 413e12e88a8554a3a25ffc730a9e73cb3a56e79e Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Thu, 21 Mar 2024 19:26:40 -0400 Subject: [PATCH 04/18] Cleanup dev releases (#6782) * delete develop release before creating new one --------- Signed-off-by: Justin Florentine --- .github/workflows/develop.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index 0e0cd11c78c..38c9024260f 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -36,10 +36,15 @@ jobs: echo "zipSha=$(shasum -a 256 besu*.zip)" >> $GITHUB_OUTPUT echo "tarSha=$(shasum -a 256 besu*.tar.gz)" >> $GITHUB_OUTPUT cd ../.. + - name: delete current develop release + env: + GH_TOKEN: ${{ github.token }} + run: gh release delete develop -y --cleanup-tag - name: Upload Release assets uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 with: prerelease: true + draft: false name: develop tag_name: develop fail_on_unmatched_files: true From c2e5dc29af7ffb15bcfda3bd7caf39f275b495ce Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Fri, 22 Mar 2024 11:50:52 +1000 Subject: [PATCH 05/18] extra check for syncing (#6771) Signed-off-by: Sally MacFarlane Signed-off-by: Justin Florentine --- ...SmartContractPermissioningV2AcceptanceTest.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/permissioning/NodeSmartContractPermissioningV2AcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/permissioning/NodeSmartContractPermissioningV2AcceptanceTest.java index b04bad5644d..e638614ad58 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/permissioning/NodeSmartContractPermissioningV2AcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/permissioning/NodeSmartContractPermissioningV2AcceptanceTest.java @@ -47,10 +47,7 @@ public void setUp() { permissionedNode.execute(allowNode(permissionedNode)); permissionedNode.verify(connectionIsAllowed(permissionedNode)); - allowedNode.verify(eth.syncingStatus(false)); - bootnode.verify(eth.syncingStatus(false)); - permissionedNode.verify(eth.syncingStatus(false)); - forbiddenNode.verify(eth.syncingStatus(false)); + verifyAllNodesHaveFinishedSyncing(); } @Test @@ -92,6 +89,8 @@ public void permissioningUpdatesPropagateThroughNetwork() { permissionedNode.verify(admin.addPeer(allowedNode)); permissionedNode.verify(net.awaitPeerCount(2)); + verifyAllNodesHaveFinishedSyncing(); + // permissioning changes in peer should propagate to permissioned node allowedNode.execute(allowNode(forbiddenNode)); allowedNode.verify(connectionIsAllowed(forbiddenNode)); @@ -101,6 +100,13 @@ public void permissioningUpdatesPropagateThroughNetwork() { permissionedNode.verify(net.awaitPeerCount(3)); } + private void verifyAllNodesHaveFinishedSyncing() { + allowedNode.verify(eth.syncingStatus(false)); + bootnode.verify(eth.syncingStatus(false)); + permissionedNode.verify(eth.syncingStatus(false)); + forbiddenNode.verify(eth.syncingStatus(false)); + } + @Test public void onchainPermissioningAllowlistShouldPersistAcrossRestarts() { permissionedCluster.stop(); From 5a6ab240b20be69e8ee6fe3945f295ac0dd0f370 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Fri, 22 Mar 2024 13:54:06 +1000 Subject: [PATCH 06/18] updated PR template (#6773) Signed-off-by: Sally MacFarlane Signed-off-by: Justin Florentine --- .github/pull_request_template.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 6dd62ca8b86..7e8cde29e7b 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -12,10 +12,10 @@ - [ ] Considered the changelog and included an [update if required](https://wiki.hyperledger.org/display/BESU/Changelog). - [ ] For database changes (e.g. KeyValueSegmentIdentifier) considered compatibility and performed forwards and backwards compatibility tests -### Most advanced CI tests are deferred until PR approval, but you could: +### Locally, you can run these tests to catch failures early: -- [ ] locally run all unit tests via: `./gradlew build` -- [ ] locally run all acceptance tests via: `./gradlew acceptanceTest` -- [ ] locally run all integration tests via: `./gradlew integrationTest` -- [ ] locally run all reference tests via: `./gradlew ethereum:referenceTests:referenceTests` +- [ ] unit tests: `./gradlew build` +- [ ] acceptance tests: `./gradlew acceptanceTest` +- [ ] integration tests: `./gradlew integrationTest` +- [ ] reference tests: `./gradlew ethereum:referenceTests:referenceTests` From 1f62a2804ae56efb42258d318d5bd6e7d8bc84af Mon Sep 17 00:00:00 2001 From: Stefan Pingel <16143240+pinges@users.noreply.github.com> Date: Sat, 23 Mar 2024 00:08:49 +1000 Subject: [PATCH 07/18] Fix "Backward sync stuck in a loop" #6749 (#6756) * minimal change to fix BWS Signed-off-by: stefan.pingel@consensys.net Signed-off-by: Simon Dudley Co-authored-by: Simon Dudley Signed-off-by: Justin Florentine --- .../backwardsync/BackwardSyncAlgorithm.java | 44 +++++++++++--- .../backwardsync/BackwardSyncContext.java | 2 +- .../sync/backwardsync/BackwardSyncStep.java | 3 +- .../backwardsync/BackwardSyncContextTest.java | 58 ++++++++++++++++++- gradle.properties | 1 - 5 files changed, 93 insertions(+), 15 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgorithm.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgorithm.java index 731291dc1ea..bfbd2f05dae 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgorithm.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgorithm.java @@ -22,11 +22,13 @@ import org.hyperledger.besu.ethereum.chain.MutableBlockchain; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.eth.manager.exceptions.MaxRetriesReachedException; import org.hyperledger.besu.ethereum.eth.manager.task.WaitForPeersTask; import org.hyperledger.besu.plugin.services.BesuEvents; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -61,16 +63,40 @@ public CompletableFuture executeBackwardsSync(final Void unused) { public CompletableFuture pickNextStep() { final Optional firstHash = context.getBackwardChain().getFirstHashToAppend(); if (firstHash.isPresent()) { - return executeSyncStep(firstHash.get()) - .thenAccept( - result -> { - LOG.atDebug() - .setMessage("Backward sync target block is {}") - .addArgument(result::toLogString) - .log(); - context.getBackwardChain().removeFromHashToAppend(firstHash.get()); - context.getStatus().updateTargetHeight(result.getHeader().getNumber()); + final CompletableFuture syncStep = new CompletableFuture<>(); + executeSyncStep(firstHash.get()) + .whenComplete( + (result, error) -> { + if (error != null) { + if (error instanceof CompletionException + && error.getCause() instanceof MaxRetriesReachedException) { + context.getBackwardChain().removeFromHashToAppend(firstHash.get()); + LOG.atWarn() + .setMessage( + "Unable to retrieve block {} from any peer, with {} peers available. Could be a reorged block. Waiting for the next block from the consensus client to try again.") + .addArgument(firstHash.get()) + .addArgument(context.getEthContext().getEthPeers().peerCount()) + .addArgument(context.getBackwardChain().getFirstHashToAppend()) + .log(); + LOG.atDebug() + .setMessage("Removing hash {} from hashesToAppend") + .addArgument(firstHash.get()) + .log(); + syncStep.complete(null); + } else { + syncStep.completeExceptionally(error); + } + } else { + LOG.atDebug() + .setMessage("Backward sync target block is {}") + .addArgument(result::toLogString) + .log(); + context.getBackwardChain().removeFromHashToAppend(firstHash.get()); + context.getStatus().updateTargetHeight(result.getHeader().getNumber()); + syncStep.complete(null); + } }); + return syncStep; } if (!context.isReady()) { return waitForReady(); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java index bd9c493ec40..1c1382e06b9 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContext.java @@ -408,7 +408,7 @@ private void logBlockImportProgress(final long currImportedHeight) { final float completedPercentage = 100.0f * imported / estimatedTotal; if (completedPercentage < 100.0f) { - if (currentStatus.progressLogDue()) { + if (currentStatus.progressLogDue() && targetHeight > 0) { LOG.info( String.format( "Backward sync phase 2 of 2, %.2f%% completed, imported %d blocks of at least %d (current head %d, target head %d). Peers: %d", diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStep.java index 6f9a5822de1..9903cc44666 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncStep.java @@ -61,7 +61,8 @@ protected CompletableFuture> requestHeaders(final Hash hash) { context.getProtocolContext().getBlockchain().getBlockHeader(hash); if (blockHeader.isPresent()) { LOG.debug( - "Hash {} already present in local blockchain no need to request headers to peers", hash); + "Hash {} already present in local blockchain no need to request headers from peers", + hash); return CompletableFuture.completedFuture(List.of(blockHeader.get())); } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java index 622d66587c7..945f97e75c3 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java @@ -83,7 +83,7 @@ public class BackwardSyncContextTest { public static final int REMOTE_HEIGHT = 50; public static final int UNCLE_HEIGHT = 25 - 3; - public static final int NUM_OF_RETRIES = 100; + public static final int NUM_OF_RETRIES = 1; public static final int TEST_MAX_BAD_CHAIN_EVENT_ENTRIES = 25; private BackwardSyncContext context; @@ -186,13 +186,16 @@ public void setup() { } private Block createUncle(final int i, final Hash parentHash) { + return createBlock(i, parentHash); + } + + private Block createBlock(final int i, final Hash parentHash) { final BlockDataGenerator.BlockOptions options = new BlockDataGenerator.BlockOptions() .setBlockNumber(i) .setParentHash(parentHash) .transactionTypes(TransactionType.ACCESS_LIST); - final Block block = blockDataGenerator.block(options); - return block; + return blockDataGenerator.block(options); } public static BackwardChain inMemoryBackwardChain() { @@ -438,4 +441,53 @@ public void shouldFailAfterMaxNumberOfRetries() { } } } + + @SuppressWarnings("BannedMethod") + @Test + public void whenBlockNotFoundInPeers_shouldRemoveBlockFromQueueAndProgressInNextSession() { + // This scenario can happen due to a reorg + // Expectation we progress beyond the reorg block upon receiving the next FCU + + // choose an intermediate remote block to create a reorg block from + int reorgBlockHeight = REMOTE_HEIGHT - 1; // 49 + final Hash reorgBlockParentHash = getBlockByNumber(reorgBlockHeight - 1).getHash(); + final Block reorgBlock = createBlock(reorgBlockHeight, reorgBlockParentHash); + + // represents first FCU with a block that will become reorged away + final CompletableFuture fcuBeforeReorg = context.syncBackwardsUntil(reorgBlock.getHash()); + respondUntilFutureIsDone(fcuBeforeReorg); + assertThat(localBlockchain.getChainHeadBlockNumber()).isLessThan(reorgBlockHeight); + + // represents subsequent FCU with successfully reorged version of the same block + final CompletableFuture fcuAfterReorg = + context.syncBackwardsUntil(getBlockByNumber(reorgBlockHeight).getHash()); + respondUntilFutureIsDone(fcuAfterReorg); + assertThat(localBlockchain.getChainHeadBlock()) + .isEqualTo(remoteBlockchain.getBlockByNumber(reorgBlockHeight).orElseThrow()); + } + + @SuppressWarnings("BannedMethod") + @Test + public void + whenBlockNotFoundInPeers_shouldRemoveBlockFromQueueAndProgressWithQueueInSameSession() { + // This scenario can happen due to a reorg + // Expectation we progress beyond the reorg block due to FCU we received during the same session + + // choose an intermediate remote block to create a reorg block from + int reorgBlockHeight = REMOTE_HEIGHT - 1; // 49 + final Hash reorgBlockParentHash = getBlockByNumber(reorgBlockHeight - 1).getHash(); + final Block reorgBlock = createBlock(reorgBlockHeight, reorgBlockParentHash); + + // represents first FCU with a block that will become reorged away + final CompletableFuture fcuBeforeReorg = context.syncBackwardsUntil(reorgBlock.getHash()); + // represents subsequent FCU with successfully reorged version of the same block + // received during the first FCU's BWS session + final CompletableFuture fcuAfterReorg = + context.syncBackwardsUntil(getBlockByNumber(reorgBlockHeight).getHash()); + + respondUntilFutureIsDone(fcuBeforeReorg); + respondUntilFutureIsDone(fcuAfterReorg); + assertThat(localBlockchain.getChainHeadBlock()) + .isEqualTo(remoteBlockchain.getBlockByNumber(reorgBlockHeight).orElseThrow()); + } } diff --git a/gradle.properties b/gradle.properties index 265507cea43..7887e7f1213 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,4 +13,3 @@ org.gradle.jvmargs=-Xmx4g \ --add-opens jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED \ --add-opens jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED # Could be moved to sonar properties after https://sonarsource.atlassian.net/browse/SONARGRADL-134 -systemProp.sonar.gradle.skipCompile=true From 3f1f322794c28a0da8ebde4f00ae44b476e107dc Mon Sep 17 00:00:00 2001 From: Usman Saleem Date: Sat, 23 Mar 2024 01:26:46 +1000 Subject: [PATCH 08/18] ci: Fix Publish GH workflow Artifactory secrets (#6788) Signed-off-by: Usman Saleem Co-authored-by: Justin Florentine Signed-off-by: Justin Florentine --- .github/workflows/artifacts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 979745d23a9..fbb1ed0c039 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -48,8 +48,8 @@ jobs: compression-level: 0 - name: Artifactory Publish env: - ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }} - ARTIFACTORY_KEY: ${{ secrets.BESU_ARTIFACTORY }} + ARTIFACTORY_USER: ${{ secrets.BESU_ARTIFACTORY_USER }} + ARTIFACTORY_KEY: ${{ secrets.BESU_ARTIFACTORY_TOKEN }} run: ./gradlew -Prelease.releaseVersion=${{ github.ref_name }} -Pversion=${{github.ref_name}} artifactoryPublish testWindows: runs-on: windows-2022 From 85aec93b00e54f7afec4e0e2c96c5dd0279faf06 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Fri, 22 Mar 2024 23:25:19 +0100 Subject: [PATCH 09/18] Improve PluginRpcEndpointException log (#6789) Signed-off-by: Fabio Di Fabio Signed-off-by: Justin Florentine --- .../internal/methods/PluginJsonRpcMethod.java | 4 +- .../api/jsonrpc/JsonRpcTestHelper.java | 20 +++++++-- .../api/jsonrpc/PluginJsonRpcMethodTest.java | 45 +++++++++++++++++++ plugin-api/build.gradle | 2 +- .../exception/PluginRpcEndpointException.java | 32 ++++++++----- 5 files changed, 86 insertions(+), 17 deletions(-) diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/PluginJsonRpcMethod.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/PluginJsonRpcMethod.java index 702404ae4bc..ad239b60162 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/PluginJsonRpcMethod.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/PluginJsonRpcMethod.java @@ -50,8 +50,8 @@ public JsonRpcResponse response(final JsonRpcRequestContext request) { final Object result = function.apply(() -> request.getRequest().getParams()); return new JsonRpcSuccessResponse(request.getRequest().getId(), result); } catch (final PluginRpcEndpointException ex) { - final JsonRpcError error = new JsonRpcError(ex.getRpcMethodError(), ex.getMessage()); - LOG.error("Error calling plugin JSON-RPC endpoint", ex); + final JsonRpcError error = new JsonRpcError(ex.getRpcMethodError(), ex.getData()); + LOG.debug("Error calling plugin JSON-RPC endpoint", ex); return new JsonRpcErrorResponse(request.getRequest().getId(), error); } } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcTestHelper.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcTestHelper.java index 7e5762d3336..f86476ce3e3 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcTestHelper.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcTestHelper.java @@ -37,8 +37,16 @@ protected void assertValidJsonRpcResult(final JsonObject json, final Object id) } protected void assertValidJsonRpcError( - final JsonObject json, final Object id, final int errorCode, final String errorMessage) - throws Exception { + final JsonObject json, final Object id, final int errorCode, final String errorMessage) { + assertValidJsonRpcError(json, id, errorCode, errorMessage, null); + } + + protected void assertValidJsonRpcError( + final JsonObject json, + final Object id, + final int errorCode, + final String errorMessage, + final String data) { // Check all expected fieldnames are set final Set fieldNames = json.fieldNames(); assertThat(fieldNames.size()).isEqualTo(3); @@ -53,13 +61,19 @@ protected void assertValidJsonRpcError( // Check error format final JsonObject error = json.getJsonObject("error"); final Set errorFieldNames = error.fieldNames(); - assertThat(errorFieldNames.size()).isEqualTo(2); + assertThat(errorFieldNames.size()).isEqualTo(data == null ? 2 : 3); assertThat(errorFieldNames.contains("code")).isTrue(); assertThat(errorFieldNames.contains("message")).isTrue(); + if (data != null) { + assertThat(errorFieldNames.contains("data")).isTrue(); + } // Check error field values assertThat(error.getInteger("code")).isEqualTo(errorCode); assertThat(error.getString("message")).isEqualTo(errorMessage); + if (data != null) { + assertThat(error.getString("data")).isEqualTo(data); + } } protected void assertIdMatches(final JsonObject json, final Object expectedId) { diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/PluginJsonRpcMethodTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/PluginJsonRpcMethodTest.java index 560a79a8bf8..4f7e45b3885 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/PluginJsonRpcMethodTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/PluginJsonRpcMethodTest.java @@ -24,6 +24,9 @@ import org.hyperledger.besu.plugin.services.rpc.PluginRpcRequest; import org.hyperledger.besu.plugin.services.rpc.RpcMethodError; +import java.util.Locale; +import java.util.Optional; + import io.vertx.core.json.JsonObject; import okhttp3.RequestBody; import okhttp3.Response; @@ -127,6 +130,25 @@ public void methodErrorShouldReturnErrorResponse() throws Exception { } } + @Test + public void methodErrorWithDataShouldReturnErrorResponseWithDecodedData() throws Exception { + final var wrongParamContent = + """ + {"jsonrpc":"2.0","id":1,"method":"plugin_echo","params":["data"]}"""; + try (var unused = + addRpcMethod( + "plugin_echo", + new PluginJsonRpcMethod("plugin_echo", PluginJsonRpcMethodTest::echoPluginRpcMethod))) { + final RequestBody body = RequestBody.create(wrongParamContent, JSON); + + try (final Response resp = client.newCall(buildPostRequest(body)).execute()) { + assertThat(resp.code()).isEqualTo(200); + final JsonObject json = new JsonObject(resp.body().string()); + testHelper.assertValidJsonRpcError(json, 1, -2, "Error with data: ABC", "abc"); + } + } + } + @Test public void unhandledExceptionShouldReturnInternalErrorResponse() throws Exception { final var nullParam = @@ -168,6 +190,29 @@ public String getMessage() { } }); } + + if (input.toString().equals("data")) { + throw new PluginRpcEndpointException( + new RpcMethodError() { + @Override + public int getCode() { + return -2; + } + + @Override + public String getMessage() { + return "Error with data"; + } + + @Override + public Optional decodeData(final String data) { + // just turn everything uppercase + return Optional.of(data.toUpperCase(Locale.US)); + } + }, + "abc"); + } + return input; } } diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index 7377359a165..3c4f7f6c0be 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -69,7 +69,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = '0xiYCyr3M4oSrvqYXVkLgVDzlBg2T3fmrADub5tY5a0=' + knownHash = '/FHIztl2tLW5Gzc0qnfEeuVQa6ljVfUce7YE6JLDdZU=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/exception/PluginRpcEndpointException.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/exception/PluginRpcEndpointException.java index e736a1d2cf5..b9cb8d71e8c 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/exception/PluginRpcEndpointException.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/exception/PluginRpcEndpointException.java @@ -20,6 +20,8 @@ public class PluginRpcEndpointException extends RuntimeException { /** The error */ private final RpcMethodError rpcMethodError; + /** The data associated with the exception */ + private final String data; /** * Constructs a new PluginRpcEndpointException exception with the specified error. @@ -27,20 +29,18 @@ public class PluginRpcEndpointException extends RuntimeException { * @param rpcMethodError the error. */ public PluginRpcEndpointException(final RpcMethodError rpcMethodError) { - super(); - this.rpcMethodError = rpcMethodError; + this(rpcMethodError, null); } /** * Constructs a new PluginRpcEndpointException exception with the specified error and message. * * @param rpcMethodError the error. - * @param message the detail message (which is saved for later retrieval by the {@link - * #getMessage()} method). + * @param data the data associated with the exception that could be parsed to extract more + * information to return in the error response. */ - public PluginRpcEndpointException(final RpcMethodError rpcMethodError, final String message) { - super(message); - this.rpcMethodError = rpcMethodError; + public PluginRpcEndpointException(final RpcMethodError rpcMethodError, final String data) { + this(rpcMethodError, data, null); } /** @@ -48,16 +48,17 @@ public PluginRpcEndpointException(final RpcMethodError rpcMethodError, final Str * cause. * * @param rpcMethodError the error. - * @param message the detail message (which is saved for later retrieval by the {@link - * #getMessage()} method). + * @param data the data associated with the exception that could be parsed to extract more + * information to return in the error response. * @param cause the cause (which is saved for later retrieval by the {@link #getCause()} method). * (A {@code null} value is permitted, and indicates that the cause is nonexistent or * unknown.) */ public PluginRpcEndpointException( - final RpcMethodError rpcMethodError, final String message, final Throwable cause) { - super(message, cause); + final RpcMethodError rpcMethodError, final String data, final Throwable cause) { + super(rpcMethodError.getMessage(), cause); this.rpcMethodError = rpcMethodError; + this.data = data; } /** @@ -68,4 +69,13 @@ public PluginRpcEndpointException( public RpcMethodError getRpcMethodError() { return rpcMethodError; } + + /** + * Get the data associated with the exception + * + * @return data as string, could be null. + */ + public String getData() { + return data; + } } From bd2e18e3ed9282e8e161ae5612ea7b99e7d95e1d Mon Sep 17 00:00:00 2001 From: Lucas Saldanha Date: Sun, 24 Mar 2024 11:39:46 +1300 Subject: [PATCH 10/18] Adding engine_getPayloadV4 and engine_newPayloadV4 (#6783) Moved engine_newPayload6110 and engine_getPayload6110 to V4 Signed-off-by: Lucas Saldanha Signed-off-by: Justin Florentine --- CHANGELOG.md | 3 +- ... ExecutionEnginePragueAcceptanceTest.java} | 8 ++-- .../engine/{eip6110 => prague}/genesis.json | 2 +- .../test-cases/01_cancun_prepare_payload.json | 0 .../test-cases/02_cancun_getPayloadV3.json | 0 .../test-cases/03_cancun_newPayloadV3.json | 0 .../04_cancun_forkchoiceUpdatedV3.json | 0 .../05_prague_forkchoiceUpdatedV3.json} | 0 .../test-cases/06_prague_getPayloadV4.json} | 2 +- .../07_prague_send_raw_transaction.json} | 0 ...nvalid_null_deposits_execute_payload.json} | 2 +- .../test-cases/09_prague_newPayloadV4.json} | 2 +- .../10_prague_forkchoiceUpdatedV3.json} | 0 .../test-cases/11_prague_getPayloadV4.json} | 2 +- .../besu/ethereum/api/jsonrpc/RpcMethod.java | 4 +- .../engine/EngineExchangeCapabilities.java | 1 - ...loadV6110.java => EngineGetPayloadV4.java} | 18 ++++----- ...loadV6110.java => EngineNewPayloadV4.java} | 17 ++++----- .../internal/results/BlockResultFactory.java | 4 +- ...110.java => EngineGetPayloadResultV4.java} | 4 +- .../ExecutionEngineJsonRpcMethods.java | 10 ++--- .../engine/AbstractScheduledApiTest.java | 7 +++- ...0Test.java => EngineGetPayloadV4Test.java} | 38 +++++++++---------- ...0Test.java => EngineNewPayloadV4Test.java} | 15 ++++---- .../mainnet/MainnetProtocolSpecs.java | 4 ++ 25 files changed, 75 insertions(+), 68 deletions(-) rename acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/{ExecutionEngineEip6110AcceptanceTest.java => ExecutionEnginePragueAcceptanceTest.java} (85%) rename acceptance-tests/tests/src/test/resources/jsonrpc/engine/{eip6110 => prague}/genesis.json (99%) rename acceptance-tests/tests/src/test/resources/jsonrpc/engine/{eip6110 => prague}/test-cases/01_cancun_prepare_payload.json (100%) rename acceptance-tests/tests/src/test/resources/jsonrpc/engine/{eip6110 => prague}/test-cases/02_cancun_getPayloadV3.json (100%) rename acceptance-tests/tests/src/test/resources/jsonrpc/engine/{eip6110 => prague}/test-cases/03_cancun_newPayloadV3.json (100%) rename acceptance-tests/tests/src/test/resources/jsonrpc/engine/{eip6110 => prague}/test-cases/04_cancun_forkchoiceUpdatedV3.json (100%) rename acceptance-tests/tests/src/test/resources/jsonrpc/engine/{eip6110/test-cases/05_eip6110_forkchoiceUpdatedV3.json => prague/test-cases/05_prague_forkchoiceUpdatedV3.json} (100%) rename acceptance-tests/tests/src/test/resources/jsonrpc/engine/{eip6110/test-cases/06_eip6110_getPayloadV6110.json => prague/test-cases/06_prague_getPayloadV4.json} (97%) rename acceptance-tests/tests/src/test/resources/jsonrpc/engine/{eip6110/test-cases/07_eip6110_send_raw_transaction.json => prague/test-cases/07_prague_send_raw_transaction.json} (100%) rename acceptance-tests/tests/src/test/resources/jsonrpc/engine/{eip6110/test-cases/08_eip6110_invalid_null_deposits_execute_payload.json => prague/test-cases/08_prague_invalid_null_deposits_execute_payload.json} (97%) rename acceptance-tests/tests/src/test/resources/jsonrpc/engine/{eip6110/test-cases/09_eip6110_newPayloadV6110.json => prague/test-cases/09_prague_newPayloadV4.json} (98%) rename acceptance-tests/tests/src/test/resources/jsonrpc/engine/{eip6110/test-cases/10_eip6110_forkchoiceUpdatedV3.json => prague/test-cases/10_prague_forkchoiceUpdatedV3.json} (100%) rename acceptance-tests/tests/src/test/resources/jsonrpc/engine/{eip6110/test-cases/11_eip6110_getPayloadV6110.json => prague/test-cases/11_prague_getPayloadV4.json} (97%) rename ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/{EngineGetPayloadV6110.java => EngineGetPayloadV4.java} (82%) rename ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/{EngineNewPayloadV6110.java => EngineNewPayloadV4.java} (85%) rename ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/{EngineGetPayloadResultV6110.java => EngineGetPayloadResultV4.java} (98%) rename ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/{EngineGetPayloadV6110Test.java => EngineGetPayloadV4Test.java} (85%) rename ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/{EngineNewPayloadV6110Test.java => EngineNewPayloadV4Test.java} (95%) diff --git a/CHANGELOG.md b/CHANGELOG.md index eefdbec72e6..48955bf2b0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - Transaction call object to accept both `input` and `data` field simultaneously if they are set to equal values [#6702](https://github.com/hyperledger/besu/pull/6702) - `eth_call` for blob tx allows for empty `maxFeePerBlobGas` [#6731](https://github.com/hyperledger/besu/pull/6731) - Extend error handling of plugin RPC methods [#6759](https://github.com/hyperledger/besu/pull/6759) +- Added engine_newPayloadV4 and engine_getPayloadV4 methods [#6783](https://github.com/hyperledger/besu/pull/6783) ### Bug fixes - Fix txpool dump/restore race condition [#6665](https://github.com/hyperledger/besu/pull/6665) @@ -442,7 +443,7 @@ https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.4.4/besu-23.4.4.z - Early access - layered transaction pool implementation [#5290](https://github.com/hyperledger/besu/pull/5290) - New RPC method `debug_getRawReceipts` [#5476](https://github.com/hyperledger/besu/pull/5476) - Add TrieLogFactory plugin support [#5440](https://github.com/hyperledger/besu/pull/5440) -- Ignore `min-block-occupancy-ratio` option when on PoS networks, since in some cases, it prevents to have full blocks even if enough transactions are present [#5491](https://github.com/hyperledger/besu/pull/5491) +- Ignore `min-block-occupancy-ratio` option when on PoS networks, since in some cases, it prevents to have full blocks even if enough transactions are present [#5491](https://github.com/hyperledger/besu/pull/5491) ### Bug Fixes - Fix eth_feeHistory response for the case in which blockCount is higher than highestBlock requested. [#5397](https://github.com/hyperledger/besu/pull/5397) diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/ExecutionEngineEip6110AcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/ExecutionEnginePragueAcceptanceTest.java similarity index 85% rename from acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/ExecutionEngineEip6110AcceptanceTest.java rename to acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/ExecutionEnginePragueAcceptanceTest.java index e77f011d9b4..966c02bebde 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/ExecutionEngineEip6110AcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/ExecutionEnginePragueAcceptanceTest.java @@ -22,13 +22,13 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.params.provider.Arguments; -public class ExecutionEngineEip6110AcceptanceTest extends AbstractJsonRpcTest { - private static final String GENESIS_FILE = "/jsonrpc/engine/eip6110/genesis.json"; - private static final String TEST_CASE_PATH = "/jsonrpc/engine/eip6110/test-cases/"; +public class ExecutionEnginePragueAcceptanceTest extends AbstractJsonRpcTest { + private static final String GENESIS_FILE = "/jsonrpc/engine/prague/genesis.json"; + private static final String TEST_CASE_PATH = "/jsonrpc/engine/prague/test-cases/"; private static JsonRpcTestsContext testsContext; - public ExecutionEngineEip6110AcceptanceTest() { + public ExecutionEnginePragueAcceptanceTest() { super(testsContext); } diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/genesis.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/genesis.json similarity index 99% rename from acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/genesis.json rename to acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/genesis.json index 232c577f6c8..98d388cc0f7 100644 --- a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/genesis.json +++ b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/genesis.json @@ -14,7 +14,7 @@ "londonBlock":0, "terminalTotalDifficulty":0, "cancunTime":0, - "experimentalEipsTime":20, + "pragueTime":20, "clique": { "period": 5, "epoch": 30000 diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/01_cancun_prepare_payload.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/01_cancun_prepare_payload.json similarity index 100% rename from acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/01_cancun_prepare_payload.json rename to acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/01_cancun_prepare_payload.json diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/02_cancun_getPayloadV3.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/02_cancun_getPayloadV3.json similarity index 100% rename from acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/02_cancun_getPayloadV3.json rename to acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/02_cancun_getPayloadV3.json diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/03_cancun_newPayloadV3.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/03_cancun_newPayloadV3.json similarity index 100% rename from acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/03_cancun_newPayloadV3.json rename to acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/03_cancun_newPayloadV3.json diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/04_cancun_forkchoiceUpdatedV3.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/04_cancun_forkchoiceUpdatedV3.json similarity index 100% rename from acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/04_cancun_forkchoiceUpdatedV3.json rename to acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/04_cancun_forkchoiceUpdatedV3.json diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/05_eip6110_forkchoiceUpdatedV3.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/05_prague_forkchoiceUpdatedV3.json similarity index 100% rename from acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/05_eip6110_forkchoiceUpdatedV3.json rename to acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/05_prague_forkchoiceUpdatedV3.json diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/06_eip6110_getPayloadV6110.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/06_prague_getPayloadV4.json similarity index 97% rename from acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/06_eip6110_getPayloadV6110.json rename to acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/06_prague_getPayloadV4.json index e31b2f8f8a9..74e9347df47 100644 --- a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/06_eip6110_getPayloadV6110.json +++ b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/06_prague_getPayloadV4.json @@ -1,7 +1,7 @@ { "request": { "jsonrpc": "2.0", - "method": "engine_getPayloadV6110", + "method": "engine_getPayloadV4", "params": [ "0x282643b909febddf" ], diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/07_eip6110_send_raw_transaction.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/07_prague_send_raw_transaction.json similarity index 100% rename from acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/07_eip6110_send_raw_transaction.json rename to acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/07_prague_send_raw_transaction.json diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/08_eip6110_invalid_null_deposits_execute_payload.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/08_prague_invalid_null_deposits_execute_payload.json similarity index 97% rename from acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/08_eip6110_invalid_null_deposits_execute_payload.json rename to acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/08_prague_invalid_null_deposits_execute_payload.json index ebf5de77500..0294c6b8f21 100644 --- a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/08_eip6110_invalid_null_deposits_execute_payload.json +++ b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/08_prague_invalid_null_deposits_execute_payload.json @@ -1,7 +1,7 @@ { "request": { "jsonrpc": "2.0", - "method": "engine_newPayloadV6110", + "method": "engine_newPayloadV4", "params": [ { "parentHash": "0x45811fa27a100ce9035e5e086b9669275041a4ec0ebbd920be028fd7b0aa2356", diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/09_eip6110_newPayloadV6110.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/09_prague_newPayloadV4.json similarity index 98% rename from acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/09_eip6110_newPayloadV6110.json rename to acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/09_prague_newPayloadV4.json index 760b9481295..b3b8166af22 100644 --- a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/09_eip6110_newPayloadV6110.json +++ b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/09_prague_newPayloadV4.json @@ -1,7 +1,7 @@ { "request": { "jsonrpc": "2.0", - "method": "engine_newPayloadV6110", + "method": "engine_newPayloadV4", "params": [ { "parentHash": "0x1dd4f141551d53ce393845e2873754e43396101a8ebc0fd0eeb2e6798a591315", diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/10_eip6110_forkchoiceUpdatedV3.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/10_prague_forkchoiceUpdatedV3.json similarity index 100% rename from acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/10_eip6110_forkchoiceUpdatedV3.json rename to acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/10_prague_forkchoiceUpdatedV3.json diff --git a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/11_eip6110_getPayloadV6110.json b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/11_prague_getPayloadV4.json similarity index 97% rename from acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/11_eip6110_getPayloadV6110.json rename to acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/11_prague_getPayloadV4.json index 5214ee7cd5a..7a251d599af 100644 --- a/acceptance-tests/tests/src/test/resources/jsonrpc/engine/eip6110/test-cases/11_eip6110_getPayloadV6110.json +++ b/acceptance-tests/tests/src/test/resources/jsonrpc/engine/prague/test-cases/11_prague_getPayloadV4.json @@ -1,7 +1,7 @@ { "request": { "jsonrpc": "2.0", - "method": "engine_getPayloadV6110", + "method": "engine_getPayloadV4", "params": [ "0x282643db882670cf" ], diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/RpcMethod.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/RpcMethod.java index eb18f1379c1..7b528ebd452 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/RpcMethod.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/RpcMethod.java @@ -54,11 +54,11 @@ public enum RpcMethod { ENGINE_GET_PAYLOAD_V1("engine_getPayloadV1"), ENGINE_GET_PAYLOAD_V2("engine_getPayloadV2"), ENGINE_GET_PAYLOAD_V3("engine_getPayloadV3"), - ENGINE_GET_PAYLOAD_V6110("engine_getPayloadV6110"), + ENGINE_GET_PAYLOAD_V4("engine_getPayloadV4"), ENGINE_NEW_PAYLOAD_V1("engine_newPayloadV1"), ENGINE_NEW_PAYLOAD_V2("engine_newPayloadV2"), ENGINE_NEW_PAYLOAD_V3("engine_newPayloadV3"), - ENGINE_NEW_PAYLOAD_V6110("engine_newPayloadV6110"), + ENGINE_NEW_PAYLOAD_V4("engine_newPayloadV4"), ENGINE_FORKCHOICE_UPDATED_V1("engine_forkchoiceUpdatedV1"), ENGINE_FORKCHOICE_UPDATED_V2("engine_forkchoiceUpdatedV2"), ENGINE_FORKCHOICE_UPDATED_V3("engine_forkchoiceUpdatedV3"), diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineExchangeCapabilities.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineExchangeCapabilities.java index fcc26cad641..f59681a3854 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineExchangeCapabilities.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineExchangeCapabilities.java @@ -63,7 +63,6 @@ public JsonRpcResponse syncResponse(final JsonRpcRequestContext requestContext) .filter(e -> e.getMethodName().startsWith("engine_")) .filter(e -> !e.equals(ENGINE_EXCHANGE_CAPABILITIES)) .filter(e -> !e.equals(ENGINE_PREPARE_PAYLOAD_DEBUG)) - .filter(e -> !e.getMethodName().endsWith("6110")) .map(RpcMethod::getMethodName) .collect(Collectors.toList()); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineGetPayloadV6110.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineGetPayloadV4.java similarity index 82% rename from ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineGetPayloadV6110.java rename to ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineGetPayloadV4.java index c9a9737a90a..feebfebd3d7 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineGetPayloadV6110.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineGetPayloadV4.java @@ -32,11 +32,11 @@ import io.vertx.core.Vertx; -public class EngineGetPayloadV6110 extends AbstractEngineGetPayload { +public class EngineGetPayloadV4 extends AbstractEngineGetPayload { - private final Optional eip6110; + private final Optional prague; - public EngineGetPayloadV6110( + public EngineGetPayloadV4( final Vertx vertx, final ProtocolContext protocolContext, final MergeMiningCoordinator mergeMiningCoordinator, @@ -50,12 +50,12 @@ public EngineGetPayloadV6110( mergeMiningCoordinator, blockResultFactory, engineCallListener); - this.eip6110 = schedule.hardforkFor(s -> s.fork().name().equalsIgnoreCase("ExperimentalEips")); + this.prague = schedule.hardforkFor(s -> s.fork().name().equalsIgnoreCase("Prague")); } @Override public String getName() { - return RpcMethod.ENGINE_GET_PAYLOAD_V6110.getMethodName(); + return RpcMethod.ENGINE_GET_PAYLOAD_V4.getMethodName(); } @Override @@ -66,22 +66,22 @@ protected JsonRpcResponse createResponse( return new JsonRpcSuccessResponse( request.getRequest().getId(), - blockResultFactory.payloadTransactionCompleteV6110(blockWithReceipts)); + blockResultFactory.payloadTransactionCompleteV4(blockWithReceipts)); } @Override protected ValidationResult validateForkSupported(final long blockTimestamp) { if (protocolSchedule.isPresent()) { - if (eip6110.isPresent() && blockTimestamp >= eip6110.get().milestone()) { + if (prague.isPresent() && blockTimestamp >= prague.get().milestone()) { return ValidationResult.valid(); } else { return ValidationResult.invalid( RpcErrorType.UNSUPPORTED_FORK, - "EIP-6110 configured to start at timestamp: " + eip6110.get().milestone()); + "Prague configured to start at timestamp: " + prague.get().milestone()); } } else { return ValidationResult.invalid( - RpcErrorType.UNSUPPORTED_FORK, "Configuration error, no schedule for EIP-6110 fork set"); + RpcErrorType.UNSUPPORTED_FORK, "Configuration error, no schedule for Prague fork set"); } } } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV6110.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV4.java similarity index 85% rename from ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV6110.java rename to ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV4.java index 84f203c3a73..07315535c2d 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV6110.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV4.java @@ -29,11 +29,11 @@ import io.vertx.core.Vertx; -public class EngineNewPayloadV6110 extends AbstractEngineNewPayload { +public class EngineNewPayloadV4 extends AbstractEngineNewPayload { - private final Optional eip6110; + private final Optional prague; - public EngineNewPayloadV6110( + public EngineNewPayloadV4( final Vertx vertx, final ProtocolSchedule timestampSchedule, final ProtocolContext protocolContext, @@ -42,13 +42,12 @@ public EngineNewPayloadV6110( final EngineCallListener engineCallListener) { super( vertx, timestampSchedule, protocolContext, mergeCoordinator, ethPeers, engineCallListener); - this.eip6110 = - timestampSchedule.hardforkFor(s -> s.fork().name().equalsIgnoreCase("ExperimentalEips")); + this.prague = timestampSchedule.hardforkFor(s -> s.fork().name().equalsIgnoreCase("prague")); } @Override public String getName() { - return RpcMethod.ENGINE_NEW_PAYLOAD_V6110.getMethodName(); + return RpcMethod.ENGINE_NEW_PAYLOAD_V4.getMethodName(); } @Override @@ -74,16 +73,16 @@ protected ValidationResult validateParameters( @Override protected ValidationResult validateForkSupported(final long blockTimestamp) { if (protocolSchedule.isPresent()) { - if (eip6110.isPresent() && blockTimestamp >= eip6110.get().milestone()) { + if (prague.isPresent() && blockTimestamp >= prague.get().milestone()) { return ValidationResult.valid(); } else { return ValidationResult.invalid( RpcErrorType.UNSUPPORTED_FORK, - "EIP-6110 configured to start at timestamp: " + eip6110.get().milestone()); + "Prague configured to start at timestamp: " + prague.get().milestone()); } } else { return ValidationResult.invalid( - RpcErrorType.UNSUPPORTED_FORK, "Configuration error, no schedule for EIP-6110 fork set"); + RpcErrorType.UNSUPPORTED_FORK, "Configuration error, no schedule for Prague fork set"); } } } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/BlockResultFactory.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/BlockResultFactory.java index 9596151bdfe..4bb3add6ac4 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/BlockResultFactory.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/BlockResultFactory.java @@ -154,7 +154,7 @@ public EngineGetPayloadResultV3 payloadTransactionCompleteV3( blobsBundleV1); } - public EngineGetPayloadResultV6110 payloadTransactionCompleteV6110( + public EngineGetPayloadResultV4 payloadTransactionCompleteV4( final BlockWithReceipts blockWithReceipts) { final List txs = blockWithReceipts.getBlock().getBody().getTransactions().stream() @@ -168,7 +168,7 @@ public EngineGetPayloadResultV6110 payloadTransactionCompleteV6110( final BlobsBundleV1 blobsBundleV1 = new BlobsBundleV1(blockWithReceipts.getBlock().getBody().getTransactions()); - return new EngineGetPayloadResultV6110( + return new EngineGetPayloadResultV4( blockWithReceipts.getHeader(), txs, blockWithReceipts.getBlock().getBody().getWithdrawals(), diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/EngineGetPayloadResultV6110.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/EngineGetPayloadResultV4.java similarity index 98% rename from ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/EngineGetPayloadResultV6110.java rename to ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/EngineGetPayloadResultV4.java index 4b47101fb64..721438a8746 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/EngineGetPayloadResultV6110.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/EngineGetPayloadResultV4.java @@ -30,13 +30,13 @@ import org.apache.tuweni.bytes.Bytes32; @JsonPropertyOrder({"executionPayload", "blockValue", "blobsBundle", "shouldOverrideBuilder"}) -public class EngineGetPayloadResultV6110 { +public class EngineGetPayloadResultV4 { protected final PayloadResult executionPayload; private final String blockValue; private final BlobsBundleV1 blobsBundle; private final boolean shouldOverrideBuilder; - public EngineGetPayloadResultV6110( + public EngineGetPayloadResultV4( final BlockHeader header, final List transactions, final Optional> withdrawals, diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/ExecutionEngineJsonRpcMethods.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/ExecutionEngineJsonRpcMethods.java index 9f4480b82be..0c090419cb1 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/ExecutionEngineJsonRpcMethods.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/ExecutionEngineJsonRpcMethods.java @@ -28,11 +28,11 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EngineGetPayloadV1; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EngineGetPayloadV2; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EngineGetPayloadV3; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EngineGetPayloadV6110; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EngineGetPayloadV4; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EngineNewPayloadV1; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EngineNewPayloadV2; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EngineNewPayloadV3; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EngineNewPayloadV6110; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EngineNewPayloadV4; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EnginePreparePayloadDebug; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.engine.EngineQosTimer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.BlockResultFactory; @@ -160,9 +160,9 @@ protected Map create() { protocolSchedule)); } - if (protocolSchedule.anyMatch(p -> p.spec().getName().equalsIgnoreCase("ExperimentalEips"))) { + if (protocolSchedule.anyMatch(p -> p.spec().getName().equalsIgnoreCase("prague"))) { executionEngineApisSupported.add( - new EngineGetPayloadV6110( + new EngineGetPayloadV4( consensusEngineServer, protocolContext, mergeCoordinator.get(), @@ -171,7 +171,7 @@ protected Map create() { protocolSchedule)); executionEngineApisSupported.add( - new EngineNewPayloadV6110( + new EngineNewPayloadV4( consensusEngineServer, protocolSchedule, protocolContext, diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractScheduledApiTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractScheduledApiTest.java index 6d71932ff96..9fd546dca5b 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractScheduledApiTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractScheduledApiTest.java @@ -39,8 +39,10 @@ public class AbstractScheduledApiTest { new ScheduledProtocolSpec.Hardfork("Shanghai", 20); protected final ScheduledProtocolSpec.Hardfork cancunHardfork = new ScheduledProtocolSpec.Hardfork("Cancun", 30); + protected final ScheduledProtocolSpec.Hardfork pragueHardfork = + new ScheduledProtocolSpec.Hardfork("Prague", 40); protected final ScheduledProtocolSpec.Hardfork experimentalHardfork = - new ScheduledProtocolSpec.Hardfork("ExperimentalEips", 40); + new ScheduledProtocolSpec.Hardfork("ExperimentalEips", 50); @Mock protected DefaultProtocolSchedule protocolSchedule; @@ -74,6 +76,9 @@ public void before() { lenient() .when(protocolSchedule.hardforkFor(argThat(new HardforkMatcher(cancunHardfork)))) .thenReturn(Optional.of(cancunHardfork)); + lenient() + .when(protocolSchedule.hardforkFor(argThat(new HardforkMatcher(pragueHardfork)))) + .thenReturn(Optional.of(pragueHardfork)); lenient() .when(protocolSchedule.hardforkFor(argThat(new HardforkMatcher(shanghaiHardfork)))) .thenReturn(Optional.of(shanghaiHardfork)); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineGetPayloadV6110Test.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineGetPayloadV4Test.java similarity index 85% rename from ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineGetPayloadV6110Test.java rename to ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineGetPayloadV4Test.java index 879f11e724c..3a0ee35e4d2 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineGetPayloadV6110Test.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineGetPayloadV4Test.java @@ -32,7 +32,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.EngineGetPayloadResultV6110; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.EngineGetPayloadResultV4; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.Quantity; import org.hyperledger.besu.ethereum.core.BlobTestFixture; import org.hyperledger.besu.ethereum.core.Block; @@ -59,9 +59,9 @@ @ExtendWith( MockitoExtension.class) // mocks in parent class may not be used, throwing unnecessary stubbing -public class EngineGetPayloadV6110Test extends AbstractEngineGetPayloadTest { +public class EngineGetPayloadV4Test extends AbstractEngineGetPayloadTest { - public EngineGetPayloadV6110Test() { + public EngineGetPayloadV4Test() { super(); } @@ -74,7 +74,7 @@ public void before() { .thenReturn(Optional.of(mockBlockWithReceiptsAndDeposits)); when(protocolContext.safeConsensusContext(Mockito.any())).thenReturn(Optional.of(mergeContext)); this.method = - new EngineGetPayloadV6110( + new EngineGetPayloadV4( vertx, protocolContext, mergeMiningCoordinator, @@ -86,24 +86,24 @@ public void before() { @Override @Test public void shouldReturnExpectedMethodName() { - assertThat(method.getName()).isEqualTo("engine_getPayloadV6110"); + assertThat(method.getName()).isEqualTo("engine_getPayloadV4"); } @Override @Test public void shouldReturnBlockForKnownPayloadId() { - BlockHeader eip6110Header = + BlockHeader header = new BlockHeaderTestFixture() .prevRandao(Bytes32.random()) - .timestamp(experimentalHardfork.milestone() + 1) + .timestamp(pragueHardfork.milestone() + 1) .excessBlobGas(BlobGas.of(10L)) .buildHeader(); // should return withdrawals, deposits and excessGas for a post-6110 block - PayloadIdentifier postEip6110Pid = + PayloadIdentifier payloadIdentifier = PayloadIdentifier.forPayloadParams( Hash.ZERO, - experimentalHardfork.milestone(), + pragueHardfork.milestone(), Bytes32.random(), Address.fromHexString("0x42"), Optional.empty(), @@ -124,10 +124,10 @@ public void shouldReturnBlockForKnownPayloadId() { .createTransaction(senderKeys); TransactionReceipt blobReceipt = mock(TransactionReceipt.class); when(blobReceipt.getCumulativeGasUsed()).thenReturn(100L); - BlockWithReceipts postEip6110Block = + BlockWithReceipts block = new BlockWithReceipts( new Block( - eip6110Header, + header, new BlockBody( List.of(blobTx), Collections.emptyList(), @@ -135,23 +135,23 @@ public void shouldReturnBlockForKnownPayloadId() { Optional.of(Collections.emptyList()))), List.of(blobReceipt)); - when(mergeContext.retrieveBlockById(postEip6110Pid)).thenReturn(Optional.of(postEip6110Block)); + when(mergeContext.retrieveBlockById(payloadIdentifier)).thenReturn(Optional.of(block)); - final var resp = resp(RpcMethod.ENGINE_GET_PAYLOAD_V6110.getMethodName(), postEip6110Pid); + final var resp = resp(RpcMethod.ENGINE_GET_PAYLOAD_V4.getMethodName(), payloadIdentifier); assertThat(resp).isInstanceOf(JsonRpcSuccessResponse.class); Optional.of(resp) .map(JsonRpcSuccessResponse.class::cast) .ifPresent( r -> { - assertThat(r.getResult()).isInstanceOf(EngineGetPayloadResultV6110.class); - final EngineGetPayloadResultV6110 res = (EngineGetPayloadResultV6110) r.getResult(); + assertThat(r.getResult()).isInstanceOf(EngineGetPayloadResultV4.class); + final EngineGetPayloadResultV4 res = (EngineGetPayloadResultV4) r.getResult(); assertThat(res.getExecutionPayload().getWithdrawals()).isNotNull(); assertThat(res.getExecutionPayload().getDeposits()).isNotNull(); assertThat(res.getExecutionPayload().getHash()) - .isEqualTo(eip6110Header.getHash().toString()); + .isEqualTo(header.getHash().toString()); assertThat(res.getBlockValue()).isEqualTo(Quantity.create(0)); assertThat(res.getExecutionPayload().getPrevRandao()) - .isEqualTo(eip6110Header.getPrevRandao().map(Bytes32::toString).orElse("")); + .isEqualTo(header.getPrevRandao().map(Bytes32::toString).orElse("")); // excessBlobGas: QUANTITY, 256 bits String expectedQuantityOf10 = Bytes32.leftPad(Bytes.of(10)).toQuantityHexString(); assertThat(res.getExecutionPayload().getExcessBlobGas()).isNotEmpty(); @@ -163,7 +163,7 @@ public void shouldReturnBlockForKnownPayloadId() { @Test public void shouldReturnUnsupportedFork() { - final var resp = resp(RpcMethod.ENGINE_GET_PAYLOAD_V6110.getMethodName(), mockPid); + final var resp = resp(RpcMethod.ENGINE_GET_PAYLOAD_V4.getMethodName(), mockPid); assertThat(resp).isInstanceOf(JsonRpcErrorResponse.class); assertThat(((JsonRpcErrorResponse) resp).getErrorType()) @@ -172,6 +172,6 @@ public void shouldReturnUnsupportedFork() { @Override protected String getMethodName() { - return RpcMethod.ENGINE_GET_PAYLOAD_V6110.getMethodName(); + return RpcMethod.ENGINE_GET_PAYLOAD_V4.getMethodName(); } } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV6110Test.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV4Test.java similarity index 95% rename from ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV6110Test.java rename to ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV4Test.java index 6390b8716c6..9943a10ba99 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV6110Test.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineNewPayloadV4Test.java @@ -41,7 +41,7 @@ import org.hyperledger.besu.ethereum.core.Withdrawal; import org.hyperledger.besu.ethereum.mainnet.BodyValidation; import org.hyperledger.besu.ethereum.mainnet.DepositsValidator; -import org.hyperledger.besu.evm.gascalculator.CancunGasCalculator; +import org.hyperledger.besu.evm.gascalculator.PragueGasCalculator; import java.util.Collections; import java.util.List; @@ -54,17 +54,18 @@ import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) -public class EngineNewPayloadV6110Test extends EngineNewPayloadV3Test { +public class EngineNewPayloadV4Test extends EngineNewPayloadV3Test { private static final Address depositContractAddress = Address.fromHexString("0x00000000219ab540356cbb839cbe05303d7705fa"); - public EngineNewPayloadV6110Test() {} + public EngineNewPayloadV4Test() {} @BeforeEach @Override public void before() { super.before(); maybeParentBeaconBlockRoot = Optional.of(Bytes32.ZERO); + // TODO this should be using NewPayloadV4 this.method = new EngineNewPayloadV3( vertx, @@ -73,10 +74,8 @@ public void before() { mergeCoordinator, ethPeers, engineCallListener); - lenient() - .when(protocolSchedule.hardforkFor(any())) - .thenReturn(Optional.of(super.cancunHardfork)); - lenient().when(protocolSpec.getGasCalculator()).thenReturn(new CancunGasCalculator()); + lenient().when(protocolSchedule.hardforkFor(any())).thenReturn(Optional.of(pragueHardfork)); + lenient().when(protocolSpec.getGasCalculator()).thenReturn(new PragueGasCalculator()); } @Override @@ -171,7 +170,7 @@ protected BlockHeader createBlockHeader( BlockHeader parentBlockHeader = new BlockHeaderTestFixture() .baseFeePerGas(Wei.ONE) - .timestamp(super.experimentalHardfork.milestone()) + .timestamp(pragueHardfork.milestone()) .excessBlobGas(BlobGas.ZERO) .blobGasUsed(0L) .buildHeader(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java index ca226484112..cf63c8ce175 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java @@ -733,6 +733,9 @@ static ProtocolSpecBuilder pragueDefinition( final int contractSizeLimit = configContractSizeLimit.orElse(SPURIOUS_DRAGON_CONTRACT_SIZE_LIMIT); + final Address depositContractAddress = + genesisConfigOptions.getDepositContractAddress().orElse(DEFAULT_DEPOSIT_CONTRACT_ADDRESS); + return cancunDefinition( chainId, configContractSizeLimit, @@ -760,6 +763,7 @@ static ProtocolSpecBuilder pragueDefinition( SPURIOUS_DRAGON_FORCE_DELETE_WHEN_EMPTY_ADDRESSES)) // use prague precompiled contracts .precompileContractRegistryBuilder(MainnetPrecompiledContractRegistries::prague) + .depositsValidator(new DepositsValidator.AllowedDeposits(depositContractAddress)) .name("Prague"); } From 894371e634c25c83098f144c03371251fc65b865 Mon Sep 17 00:00:00 2001 From: Simon Dudley Date: Mon, 25 Mar 2024 14:45:19 +1000 Subject: [PATCH 11/18] Refactor BackwardsSyncAlgorithm.pickNextStep for readability (#6775) * Refactor BackwardsSyncAlgorithm.pickNextStep for readability ...mostly extracting methods - Result null check - Remove redundant BannedMethod suppressions - Add sonar config back in that was removed by mistake Signed-off-by: Simon Dudley Signed-off-by: Justin Florentine --- .../backwardsync/BackwardSyncAlgorithm.java | 88 ++++++++++++------- .../backwardsync/BackwardSyncContextTest.java | 2 - gradle.properties | 1 + 3 files changed, 55 insertions(+), 36 deletions(-) diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgorithm.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgorithm.java index bfbd2f05dae..951e8586687 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgorithm.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgorithm.java @@ -63,40 +63,7 @@ public CompletableFuture executeBackwardsSync(final Void unused) { public CompletableFuture pickNextStep() { final Optional firstHash = context.getBackwardChain().getFirstHashToAppend(); if (firstHash.isPresent()) { - final CompletableFuture syncStep = new CompletableFuture<>(); - executeSyncStep(firstHash.get()) - .whenComplete( - (result, error) -> { - if (error != null) { - if (error instanceof CompletionException - && error.getCause() instanceof MaxRetriesReachedException) { - context.getBackwardChain().removeFromHashToAppend(firstHash.get()); - LOG.atWarn() - .setMessage( - "Unable to retrieve block {} from any peer, with {} peers available. Could be a reorged block. Waiting for the next block from the consensus client to try again.") - .addArgument(firstHash.get()) - .addArgument(context.getEthContext().getEthPeers().peerCount()) - .addArgument(context.getBackwardChain().getFirstHashToAppend()) - .log(); - LOG.atDebug() - .setMessage("Removing hash {} from hashesToAppend") - .addArgument(firstHash.get()) - .log(); - syncStep.complete(null); - } else { - syncStep.completeExceptionally(error); - } - } else { - LOG.atDebug() - .setMessage("Backward sync target block is {}") - .addArgument(result::toLogString) - .log(); - context.getBackwardChain().removeFromHashToAppend(firstHash.get()); - context.getStatus().updateTargetHeight(result.getHeader().getNumber()); - syncStep.complete(null); - } - }); - return syncStep; + return handleSyncStep(firstHash.get()); } if (!context.isReady()) { return waitForReady(); @@ -137,6 +104,59 @@ public CompletableFuture pickNextStep() { return executeBackwardAsync(firstAncestorHeader); } + private CompletableFuture handleSyncStep(final Hash firstHash) { + final CompletableFuture syncStep = new CompletableFuture<>(); + executeSyncStep(firstHash) + .whenComplete( + (result, error) -> { + if (error != null) { + handleSyncStepError(error, firstHash, syncStep); + } else { + handleSyncStepSuccess(result, firstHash, syncStep); + } + }); + return syncStep; + } + + private void handleSyncStepSuccess( + final Block result, final Hash firstHash, final CompletableFuture syncStep) { + if (result == null) { + LOG.atWarn().setMessage("Unexpected null result in for hash {}").addArgument(firstHash).log(); + syncStep.completeExceptionally(new BackwardSyncException("Unexpected null result", true)); + } else { + LOG.atDebug() + .setMessage("Backward sync target block is {}") + .addArgument(result::toLogString) + .log(); + context.getBackwardChain().removeFromHashToAppend(firstHash); + context.getStatus().updateTargetHeight(result.getHeader().getNumber()); + syncStep.complete(null); + } + } + + private void handleSyncStepError( + final Throwable error, final Hash firstHash, final CompletableFuture syncStep) { + if (error instanceof CompletionException + && error.getCause() instanceof MaxRetriesReachedException) { + handleEthPeerMaxRetriesException(firstHash); + syncStep.complete(null); + } else { + syncStep.completeExceptionally(error); + } + } + + private void handleEthPeerMaxRetriesException(final Hash firstHash) { + context.getBackwardChain().removeFromHashToAppend(firstHash); + LOG.atWarn() + .setMessage( + "Unable to retrieve block {} from any peer, with {} peers available. Could be a reorged block. Waiting for the next block from the consensus client to try again.") + .addArgument(firstHash) + .addArgument(context.getEthContext().getEthPeers().peerCount()) + .addArgument(context.getBackwardChain().getFirstHashToAppend()) + .log(); + LOG.atDebug().setMessage("Removing hash {} from hashesToAppend").addArgument(firstHash).log(); + } + @VisibleForTesting public CompletableFuture executeProcessKnownAncestors() { return new ProcessKnownAncestorsStep(context, context.getBackwardChain()).executeAsync(); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java index 945f97e75c3..532400c20d2 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncContextTest.java @@ -442,7 +442,6 @@ public void shouldFailAfterMaxNumberOfRetries() { } } - @SuppressWarnings("BannedMethod") @Test public void whenBlockNotFoundInPeers_shouldRemoveBlockFromQueueAndProgressInNextSession() { // This scenario can happen due to a reorg @@ -466,7 +465,6 @@ public void whenBlockNotFoundInPeers_shouldRemoveBlockFromQueueAndProgressInNext .isEqualTo(remoteBlockchain.getBlockByNumber(reorgBlockHeight).orElseThrow()); } - @SuppressWarnings("BannedMethod") @Test public void whenBlockNotFoundInPeers_shouldRemoveBlockFromQueueAndProgressWithQueueInSameSession() { diff --git a/gradle.properties b/gradle.properties index 7887e7f1213..265507cea43 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,3 +13,4 @@ org.gradle.jvmargs=-Xmx4g \ --add-opens jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED \ --add-opens jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED # Could be moved to sonar properties after https://sonarsource.atlassian.net/browse/SONARGRADL-134 +systemProp.sonar.gradle.skipCompile=true From a5d69fd16f149867afee0cacc43496909dde6a1c Mon Sep 17 00:00:00 2001 From: Jason Frame Date: Tue, 26 Mar 2024 11:00:05 +1000 Subject: [PATCH 12/18] Reduce receipt size (#6602) Signed-off-by: Jason Frame Signed-off-by: Justin Florentine --- CHANGELOG.md | 2 + .../options/stable/DataStorageOptions.java | 9 +++ .../controller/BesuControllerBuilder.java | 3 +- .../besu/services/BesuConfigurationImpl.java | 34 ++++++++ .../stable/DataStorageOptionsTest.java | 18 +++++ .../AbstractBftBesuControllerBuilderTest.java | 5 +- .../controller/BesuControllerBuilderTest.java | 5 +- .../CliqueBesuControllerBuilderTest.java | 5 +- .../MergeBesuControllerBuilderTest.java | 5 +- .../besu/services/BesuEventsImplTest.java | 3 +- .../src/test/resources/everything_config.toml | 1 + .../pojoadapter/TransactionAdapter.java | 2 +- .../internal/methods/DebugGetRawReceipts.java | 2 +- .../api/query/StateBackupService.java | 2 +- ...ewBlockHeadersSubscriptionServiceTest.java | 3 +- .../AbstractBlockTransactionSelectorTest.java | 3 +- .../ethereum/core/TransactionReceipt.java | 43 +++++++---- .../besu/ethereum/mainnet/BodyValidation.java | 3 +- .../ethereum/storage/StorageProvider.java | 4 +- ...ueStoragePrefixedKeyBlockchainStorage.java | 18 ++++- .../keyvalue/KeyValueStorageProvider.java | 7 +- .../worldstate/DataStorageConfiguration.java | 6 ++ .../core/ExecutionContextTestFixture.java | 3 +- .../core/InMemoryKeyValueStorageProvider.java | 2 +- .../ethereum/chain/ChainDataPrunerTest.java | 6 +- .../ethereum/chain/DefaultBlockchainTest.java | 3 +- .../besu/ethereum/core/LogTest.java | 31 +++++++- .../ethereum/core/TransactionReceiptTest.java | 37 ++++++++- ...oragePrefixedKeyBlockchainStorageTest.java | 8 +- .../trie/bonsai/AbstractIsolationTests.java | 16 ++++ .../trie/forest/pruner/PrunerTest.java | 12 ++- .../besu/ethereum/eth/manager/EthServer.java | 2 +- .../eth/messages/ReceiptsMessage.java | 2 +- .../ethereum/eth/manager/EthServerTest.java | 2 +- .../CheckPointBlockImportStepTest.java | 3 +- .../besu/evmtool/DataStoreModule.java | 5 +- .../ethereum/retesteth/RetestethContext.java | 2 +- .../org/hyperledger/besu/evm/log/Log.java | 66 +++++++++++++++- plugin-api/build.gradle | 2 +- .../plugin/services/BesuConfiguration.java | 10 +++ .../storage/DataStorageConfiguration.java | 40 ++++++++++ .../RocksDBKeyValuePrivacyStorageFactory.java | 29 ++++++- .../RocksDBKeyValueStorageFactory.java | 38 +++++++-- .../BaseVersionedStorageFormat.java | 28 +++++-- .../configuration/DatabaseMetadata.java | 18 ++++- .../PrivacyVersionedStorageFormat.java | 28 +++++-- ...ksDBKeyValuePrivacyStorageFactoryTest.java | 77 +++++++++++++++++-- .../RocksDBKeyValueStorageFactoryTest.java | 47 +++++++++-- 48 files changed, 601 insertions(+), 99 deletions(-) create mode 100644 plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/DataStorageConfiguration.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 48955bf2b0b..bacbc88d070 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - BFT networks won't start with SNAP or CHECKPOINT sync (previously Besu would start with this config but quietly fail to sync, so it's now more obvious that it won't work) [#6625](https://github.com/hyperledger/besu/pull/6625), [#6667](https://github.com/hyperledger/besu/pull/6667) ### Upcoming Breaking Changes +- Receipt compaction will be enabled by default in a future version of Besu. After this change it will not be possible to downgrade to the previous Besu version. ### Deprecations @@ -25,6 +26,7 @@ - `eth_call` for blob tx allows for empty `maxFeePerBlobGas` [#6731](https://github.com/hyperledger/besu/pull/6731) - Extend error handling of plugin RPC methods [#6759](https://github.com/hyperledger/besu/pull/6759) - Added engine_newPayloadV4 and engine_getPayloadV4 methods [#6783](https://github.com/hyperledger/besu/pull/6783) +- Reduce storage size of receipts [#6602](https://github.com/hyperledger/besu/pull/6602) ### Bug fixes - Fix txpool dump/restore race condition [#6665](https://github.com/hyperledger/besu/pull/6665) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java index 043f41cee6a..c02e96c8325 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/stable/DataStorageOptions.java @@ -17,6 +17,7 @@ package org.hyperledger.besu.cli.options.stable; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; +import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_RECEIPT_COMPACTION_ENABLED; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED; import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE; @@ -61,6 +62,12 @@ public class DataStorageOptions implements CLIOptions arity = "1") private Long bonsaiMaxLayersToLoad = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD; + @Option( + names = "--receipt-compaction-enabled", + description = "Enables compact storing of receipts (default: ${DEFAULT-VALUE}).", + arity = "1") + private Boolean receiptCompactionEnabled = DEFAULT_RECEIPT_COMPACTION_ENABLED; + @CommandLine.ArgGroup(validate = false) private final DataStorageOptions.Unstable unstableOptions = new Unstable(); @@ -149,6 +156,7 @@ public static DataStorageOptions fromConfig(final DataStorageConfiguration domai final DataStorageOptions dataStorageOptions = DataStorageOptions.create(); dataStorageOptions.dataStorageFormat = domainObject.getDataStorageFormat(); dataStorageOptions.bonsaiMaxLayersToLoad = domainObject.getBonsaiMaxLayersToLoad(); + dataStorageOptions.receiptCompactionEnabled = domainObject.getReceiptCompactionEnabled(); dataStorageOptions.unstableOptions.bonsaiLimitTrieLogsEnabled = domainObject.getUnstable().getBonsaiLimitTrieLogsEnabled(); dataStorageOptions.unstableOptions.bonsaiTrieLogPruningWindowSize = @@ -164,6 +172,7 @@ public DataStorageConfiguration toDomainObject() { return ImmutableDataStorageConfiguration.builder() .dataStorageFormat(dataStorageFormat) .bonsaiMaxLayersToLoad(bonsaiMaxLayersToLoad) + .receiptCompactionEnabled(receiptCompactionEnabled) .unstable( ImmutableDataStorageConfiguration.Unstable.builder() .bonsaiLimitTrieLogsEnabled(unstableOptions.bonsaiLimitTrieLogsEnabled) diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index 4229804903f..cdc1a317088 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -564,7 +564,8 @@ public BesuController build() { storageProvider.createWorldStateStorageCoordinator(dataStorageConfiguration); final BlockchainStorage blockchainStorage = - storageProvider.createBlockchainStorage(protocolSchedule, variablesStorage); + storageProvider.createBlockchainStorage( + protocolSchedule, variablesStorage, dataStorageConfiguration); final MutableBlockchain blockchain = DefaultBlockchain.createMutable( diff --git a/besu/src/main/java/org/hyperledger/besu/services/BesuConfigurationImpl.java b/besu/src/main/java/org/hyperledger/besu/services/BesuConfigurationImpl.java index e7fb8202b17..ed2e481d7da 100644 --- a/besu/src/main/java/org/hyperledger/besu/services/BesuConfigurationImpl.java +++ b/besu/src/main/java/org/hyperledger/besu/services/BesuConfigurationImpl.java @@ -67,4 +67,38 @@ public DataStorageFormat getDatabaseFormat() { public Wei getMinGasPrice() { return miningParameters.getMinTransactionGasPrice(); } + + @Override + public org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration + getDataStorageConfiguration() { + return new DataStoreConfigurationImpl(dataStorageConfiguration); + } + + /** + * A concrete implementation of DataStorageConfiguration which is used in Besu plugin framework. + */ + public static class DataStoreConfigurationImpl + implements org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration { + + private final DataStorageConfiguration dataStorageConfiguration; + + /** + * Instantiate the concrete implementation of the plugin DataStorageConfiguration. + * + * @param dataStorageConfiguration The Ethereum core module data storage configuration + */ + public DataStoreConfigurationImpl(final DataStorageConfiguration dataStorageConfiguration) { + this.dataStorageConfiguration = dataStorageConfiguration; + } + + @Override + public DataStorageFormat getDatabaseFormat() { + return dataStorageConfiguration.getDataStorageFormat(); + } + + @Override + public boolean getReceiptCompactionEnabled() { + return dataStorageConfiguration.getReceiptCompactionEnabled(); + } + } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java index bd6985df577..94eba18ad87 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/options/stable/DataStorageOptionsTest.java @@ -110,6 +110,24 @@ public void bonsaiCodeUsingCodeHashEnabledCanBeDisabled() { "false"); } + @Test + public void receiptCompactionCanBeEnabled() { + internalTestSuccess( + dataStorageConfiguration -> + assertThat(dataStorageConfiguration.getReceiptCompactionEnabled()).isEqualTo(true), + "--receipt-compaction-enabled", + "true"); + } + + @Test + public void receiptCompactionCanBeDisabled() { + internalTestSuccess( + dataStorageConfiguration -> + assertThat(dataStorageConfiguration.getReceiptCompactionEnabled()).isEqualTo(false), + "--receipt-compaction-enabled", + "false"); + } + @Override protected DataStorageConfiguration createDefaultDomainObject() { return DataStorageConfiguration.DEFAULT_CONFIG; diff --git a/besu/src/test/java/org/hyperledger/besu/controller/AbstractBftBesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/AbstractBftBesuControllerBuilderTest.java index bcfe8a2e181..2f6924112b7 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/AbstractBftBesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/AbstractBftBesuControllerBuilderTest.java @@ -108,12 +108,13 @@ public void setup() throws JsonProcessingException { lenient().when(genesisConfigFile.getConfigOptions()).thenReturn(genesisConfigOptions); lenient().when(genesisConfigOptions.getCheckpointOptions()).thenReturn(checkpointConfigOptions); lenient() - .when(storageProvider.createBlockchainStorage(any(), any())) + .when(storageProvider.createBlockchainStorage(any(), any(), any())) .thenReturn( new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions())); + new MainnetBlockHeaderFunctions(), + false)); lenient() .when( storageProvider.createWorldStateStorageCoordinator( diff --git a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java index b7429ea7b0b..a06e1907d2c 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java @@ -124,12 +124,13 @@ public void setup() { when(ethashConfigOptions.getFixedDifficulty()).thenReturn(OptionalLong.empty()); when(storageProvider.getStorageBySegmentIdentifier(any())) .thenReturn(new InMemoryKeyValueStorage()); - when(storageProvider.createBlockchainStorage(any(), any())) + when(storageProvider.createBlockchainStorage(any(), any(), any())) .thenReturn( new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions())); + new MainnetBlockHeaderFunctions(), + false)); when(synchronizerConfiguration.getDownloaderParallelism()).thenReturn(1); when(synchronizerConfiguration.getTransactionsParallelism()).thenReturn(1); when(synchronizerConfiguration.getComputationParallelism()).thenReturn(1); diff --git a/besu/src/test/java/org/hyperledger/besu/controller/CliqueBesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/CliqueBesuControllerBuilderTest.java index b96a052c61a..631149516f4 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/CliqueBesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/CliqueBesuControllerBuilderTest.java @@ -117,12 +117,13 @@ public void setup() throws JsonProcessingException { lenient().when(genesisConfigFile.getConfigOptions()).thenReturn(genesisConfigOptions); lenient().when(genesisConfigOptions.getCheckpointOptions()).thenReturn(checkpointConfigOptions); lenient() - .when(storageProvider.createBlockchainStorage(any(), any())) + .when(storageProvider.createBlockchainStorage(any(), any(), any())) .thenReturn( new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions())); + new MainnetBlockHeaderFunctions(), + false)); lenient() .when( storageProvider.createWorldStateStorageCoordinator( diff --git a/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java index f10b504010c..e9c22a82be8 100644 --- a/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java @@ -133,12 +133,13 @@ public void setup() { when(genesisConfigOptions.getTerminalBlockHash()).thenReturn(Optional.of(Hash.ZERO)); lenient().when(genesisConfigOptions.getTerminalBlockNumber()).thenReturn(OptionalLong.of(1L)); lenient() - .when(storageProvider.createBlockchainStorage(any(), any())) + .when(storageProvider.createBlockchainStorage(any(), any(), any())) .thenReturn( new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions())); + new MainnetBlockHeaderFunctions(), + false)); lenient() .when(storageProvider.getStorageBySegmentIdentifier(any())) .thenReturn(new InMemoryKeyValueStorage()); diff --git a/besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java b/besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java index 2aa5b7c010c..321ef8a3b82 100644 --- a/besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java +++ b/besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java @@ -122,7 +122,8 @@ public void setUp() { new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions()), + new MainnetBlockHeaderFunctions(), + false), new NoOpMetricsSystem(), 0); diff --git a/besu/src/test/resources/everything_config.toml b/besu/src/test/resources/everything_config.toml index 2bcbd21b496..87fc5986dd7 100644 --- a/besu/src/test/resources/everything_config.toml +++ b/besu/src/test/resources/everything_config.toml @@ -215,6 +215,7 @@ ethstats-cacert-file="./root.cert" # Data storage data-storage-format="BONSAI" bonsai-historical-block-limit=512 +receipt-compaction-enabled=true # feature flags Xsecp256k1-native-enabled=false diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/TransactionAdapter.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/TransactionAdapter.java index f3ead13dadb..b0854e761c1 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/TransactionAdapter.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/TransactionAdapter.java @@ -274,7 +274,7 @@ public Optional getRawReceipt(final DataFetchingEnvironment environment) .map( receipt -> { final BytesValueRLPOutput rlpOutput = new BytesValueRLPOutput(); - receipt.getReceipt().writeTo(rlpOutput); + receipt.getReceipt().writeToForNetwork(rlpOutput); return rlpOutput.encoded(); }); } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugGetRawReceipts.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugGetRawReceipts.java index 9e879de5170..ba71ae0b330 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugGetRawReceipts.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugGetRawReceipts.java @@ -55,7 +55,7 @@ protected Object resultByBlockHash(final JsonRpcRequestContext request, final Ha private String[] toRLP(final List receipts) { return receipts.stream() - .map(receipt -> RLP.encode(receipt::writeTo).toHexString()) + .map(receipt -> RLP.encode(receipt::writeToForNetwork).toHexString()) .toArray(String[]::new); } } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/StateBackupService.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/StateBackupService.java index 50c589c4957..95a4923d4ff 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/StateBackupService.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/StateBackupService.java @@ -311,7 +311,7 @@ private void backupChainData() throws IOException { bodyWriter.writeBytes(bodyOutput.encoded().toArrayUnsafe()); final BytesValueRLPOutput receiptsOutput = new BytesValueRLPOutput(); - receiptsOutput.writeList(receipts.get(), TransactionReceipt::writeToWithRevertReason); + receiptsOutput.writeList(receipts.get(), (r, rlpOut) -> r.writeToForStorage(rlpOut, false)); receiptsWriter.writeBytes(receiptsOutput.encoded().toArrayUnsafe()); backupStatus.storedBlock = blockNumber; diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/blockheaders/NewBlockHeadersSubscriptionServiceTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/blockheaders/NewBlockHeadersSubscriptionServiceTest.java index 4902eb31464..ecf3e5cc691 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/blockheaders/NewBlockHeadersSubscriptionServiceTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/websocket/subscription/blockheaders/NewBlockHeadersSubscriptionServiceTest.java @@ -63,7 +63,8 @@ public class NewBlockHeadersSubscriptionServiceTest { new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions()); + new MainnetBlockHeaderFunctions(), + false); private final Block genesisBlock = gen.genesisBlock(); private final MutableBlockchain blockchain = DefaultBlockchain.createMutable(genesisBlock, blockchainStorage, new NoOpMetricsSystem(), 0); diff --git a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java index a7dab2f124e..26c3eb74cca 100644 --- a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java +++ b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/AbstractBlockTransactionSelectorTest.java @@ -167,7 +167,8 @@ public void setup() { new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions()), + new MainnetBlockHeaderFunctions(), + false), new NoOpMetricsSystem(), 0); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/TransactionReceipt.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/TransactionReceipt.java index b7419b1572d..79fb1a50149 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/TransactionReceipt.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/TransactionReceipt.java @@ -30,6 +30,7 @@ import java.util.Optional; import java.util.stream.Collectors; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import org.apache.tuweni.bytes.Bytes; @@ -169,23 +170,26 @@ private TransactionReceipt( * * @param out The RLP output to write to */ - public void writeTo(final RLPOutput out) { - writeTo(out, false); + public void writeToForNetwork(final RLPOutput out) { + writeTo(out, false, false); } - public void writeToWithRevertReason(final RLPOutput out) { - writeTo(out, true); + public void writeToForStorage(final RLPOutput out, final boolean compacted) { + writeTo(out, true, compacted); } - private void writeTo(final RLPOutput rlpOutput, final boolean withRevertReason) { + @VisibleForTesting + void writeTo(final RLPOutput rlpOutput, final boolean withRevertReason, final boolean compacted) { if (transactionType.equals(TransactionType.FRONTIER)) { - writeToForReceiptTrie(rlpOutput, withRevertReason); + writeToForReceiptTrie(rlpOutput, withRevertReason, compacted); } else { - rlpOutput.writeBytes(RLP.encode(out -> writeToForReceiptTrie(out, withRevertReason))); + rlpOutput.writeBytes( + RLP.encode(out -> writeToForReceiptTrie(out, withRevertReason, compacted))); } } - public void writeToForReceiptTrie(final RLPOutput rlpOutput, final boolean withRevertReason) { + public void writeToForReceiptTrie( + final RLPOutput rlpOutput, final boolean withRevertReason, final boolean compacted) { if (!transactionType.equals(TransactionType.FRONTIER)) { rlpOutput.writeIntScalar(transactionType.getSerializedType()); } @@ -200,8 +204,10 @@ public void writeToForReceiptTrie(final RLPOutput rlpOutput, final boolean withR rlpOutput.writeLongScalar(status); } rlpOutput.writeLongScalar(cumulativeGasUsed); - rlpOutput.writeBytes(bloomFilter); - rlpOutput.writeList(logs, Log::writeTo); + if (!compacted) { + rlpOutput.writeBytes(bloomFilter); + } + rlpOutput.writeList(logs, (log, logOutput) -> log.writeTo(logOutput, compacted)); if (withRevertReason && revertReason.isPresent()) { rlpOutput.writeBytes(revertReason.get()); } @@ -240,10 +246,21 @@ public static TransactionReceipt readFrom( // correct transaction receipt encoding to use. final RLPInput firstElement = input.readAsRlp(); final long cumulativeGas = input.readLongScalar(); - // The logs below will populate the bloom filter upon construction. + + LogsBloomFilter bloomFilter = null; + + final boolean hasLogs = !input.nextIsList() && input.nextSize() == LogsBloomFilter.BYTE_SIZE; + if (hasLogs) { + // The logs below will populate the bloom filter upon construction. + bloomFilter = LogsBloomFilter.readFrom(input); + } // TODO consider validating that the logs and bloom filter match. - final LogsBloomFilter bloomFilter = LogsBloomFilter.readFrom(input); - final List logs = input.readList(Log::readFrom); + final boolean compacted = !hasLogs; + final List logs = input.readList(logInput -> Log.readFrom(logInput, compacted)); + if (compacted) { + bloomFilter = LogsBloomFilter.builder().insertLogs(logs).build(); + } + final Optional revertReason; if (input.isEndOfCurrentList()) { revertReason = Optional.empty(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/BodyValidation.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/BodyValidation.java index f806c8d4cd1..ecf206a54ad 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/BodyValidation.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/BodyValidation.java @@ -118,7 +118,8 @@ public static Hash receiptsRoot(final List receipts) { trie.put( indexKey(i), RLP.encode( - rlpOutput -> receipts.get(i).writeToForReceiptTrie(rlpOutput, false)))); + rlpOutput -> + receipts.get(i).writeToForReceiptTrie(rlpOutput, false, false)))); return Hash.wrap(trie.getRootHash()); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java index d927165b1ab..68c71156cc9 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/StorageProvider.java @@ -33,7 +33,9 @@ public interface StorageProvider extends Closeable { VariablesStorage createVariablesStorage(); BlockchainStorage createBlockchainStorage( - ProtocolSchedule protocolSchedule, VariablesStorage variablesStorage); + ProtocolSchedule protocolSchedule, + VariablesStorage variablesStorage, + DataStorageConfiguration storageConfiguration); WorldStateKeyValueStorage createWorldStateStorage( DataStorageConfiguration dataStorageConfiguration); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorage.java index 9134ca91c01..e6f3555f60c 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorage.java @@ -59,14 +59,17 @@ public class KeyValueStoragePrefixedKeyBlockchainStorage implements BlockchainSt final KeyValueStorage blockchainStorage; final VariablesStorage variablesStorage; final BlockHeaderFunctions blockHeaderFunctions; + final boolean receiptCompaction; public KeyValueStoragePrefixedKeyBlockchainStorage( final KeyValueStorage blockchainStorage, final VariablesStorage variablesStorage, - final BlockHeaderFunctions blockHeaderFunctions) { + final BlockHeaderFunctions blockHeaderFunctions, + final boolean receiptCompaction) { this.blockchainStorage = blockchainStorage; this.variablesStorage = variablesStorage; this.blockHeaderFunctions = blockHeaderFunctions; + this.receiptCompaction = receiptCompaction; migrateVariables(); } @@ -125,7 +128,8 @@ public Optional getTransactionLocation(final Hash transacti @Override public Updater updater() { - return new Updater(blockchainStorage.startTransaction(), variablesStorage.updater()); + return new Updater( + blockchainStorage.startTransaction(), variablesStorage.updater(), receiptCompaction); } private List rlpDecodeTransactionReceipts(final Bytes bytes) { @@ -253,12 +257,15 @@ public static class Updater implements BlockchainStorage.Updater { private final KeyValueStorageTransaction blockchainTransaction; private final VariablesStorage.Updater variablesUpdater; + private final boolean receiptCompaction; Updater( final KeyValueStorageTransaction blockchainTransaction, - final VariablesStorage.Updater variablesUpdater) { + final VariablesStorage.Updater variablesUpdater, + final boolean receiptCompaction) { this.blockchainTransaction = blockchainTransaction; this.variablesUpdater = variablesUpdater; + this.receiptCompaction = receiptCompaction; } @Override @@ -365,7 +372,10 @@ private void remove(final Bytes prefix, final Bytes key) { } private Bytes rlpEncode(final List receipts) { - return RLP.encode(o -> o.writeList(receipts, TransactionReceipt::writeToWithRevertReason)); + return RLP.encode( + o -> + o.writeList( + receipts, (r, rlpOutput) -> r.writeToForStorage(rlpOutput, receiptCompaction))); } private void removeVariables() { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java index 53a3f45469d..004790ee170 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageProvider.java @@ -69,11 +69,14 @@ public VariablesStorage createVariablesStorage() { @Override public BlockchainStorage createBlockchainStorage( - final ProtocolSchedule protocolSchedule, final VariablesStorage variablesStorage) { + final ProtocolSchedule protocolSchedule, + final VariablesStorage variablesStorage, + final DataStorageConfiguration dataStorageConfiguration) { return new KeyValueStoragePrefixedKeyBlockchainStorage( getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.BLOCKCHAIN), variablesStorage, - ScheduleBasedBlockHeaderFunctions.create(protocolSchedule)); + ScheduleBasedBlockHeaderFunctions.create(protocolSchedule), + dataStorageConfiguration.getReceiptCompactionEnabled()); } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java index 064f5a961c0..7ad5362aeff 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java @@ -25,6 +25,7 @@ public interface DataStorageConfiguration { long DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD = 512; + boolean DEFAULT_RECEIPT_COMPACTION_ENABLED = false; DataStorageConfiguration DEFAULT_CONFIG = ImmutableDataStorageConfiguration.builder() @@ -50,6 +51,11 @@ public interface DataStorageConfiguration { Long getBonsaiMaxLayersToLoad(); + @Value.Default + default boolean getReceiptCompactionEnabled() { + return DEFAULT_RECEIPT_COMPACTION_ENABLED; + } + @Value.Default default Unstable getUnstable() { return Unstable.DEFAULT; diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java index 40f4c8c9312..3aa483e1a9e 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/ExecutionContextTestFixture.java @@ -64,7 +64,8 @@ private ExecutionContextTestFixture( new KeyValueStoragePrefixedKeyBlockchainStorage( blockchainKeyValueStorage, new VariablesKeyValueStorage(variablesKeyValueStorage), - new MainnetBlockHeaderFunctions()), + new MainnetBlockHeaderFunctions(), + false), new NoOpMetricsSystem(), 0); this.stateArchive = createInMemoryWorldStateArchive(); diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java index 6626f4aada6..398472ef3ae 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/InMemoryKeyValueStorageProvider.java @@ -73,7 +73,7 @@ public static MutableBlockchain createInMemoryBlockchain( return DefaultBlockchain.createMutable( genesisBlock, new KeyValueStoragePrefixedKeyBlockchainStorage( - keyValueStorage, variablesStorage, blockHeaderFunctions), + keyValueStorage, variablesStorage, blockHeaderFunctions, false), new NoOpMetricsSystem(), 0); } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/ChainDataPrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/ChainDataPrunerTest.java index 6b249e74111..7082d8ae1ce 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/ChainDataPrunerTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/ChainDataPrunerTest.java @@ -41,7 +41,8 @@ public void singleChainPruning() { new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions()); + new MainnetBlockHeaderFunctions(), + false); final ChainDataPruner chainDataPruner = new ChainDataPruner( blockchainStorage, @@ -79,7 +80,8 @@ public void forkPruning() { new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions()); + new MainnetBlockHeaderFunctions(), + false); final ChainDataPruner chainDataPruner = new ChainDataPruner( blockchainStorage, diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchainTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchainTest.java index 66c9ac5593b..69fbfcdbd65 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchainTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchainTest.java @@ -1055,7 +1055,8 @@ private BlockchainStorage createStorage( return new KeyValueStoragePrefixedKeyBlockchainStorage( kvStoreChain, new VariablesKeyValueStorage(kvStorageVariables), - new MainnetBlockHeaderFunctions()); + new MainnetBlockHeaderFunctions(), + false); } private DefaultBlockchain createMutableBlockchain( diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/LogTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/LogTest.java index 18fb8f3328d..d47efa4b074 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/LogTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/LogTest.java @@ -18,16 +18,43 @@ import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.evm.log.Log; +import org.hyperledger.besu.evm.log.LogTopic; +import java.util.List; + +import org.apache.tuweni.bytes.Bytes; import org.junit.jupiter.api.Test; public class LogTest { + final BlockDataGenerator gen = new BlockDataGenerator(); @Test public void toFromRlp() { - final BlockDataGenerator gen = new BlockDataGenerator(); - final Log log = gen.log(); + final Log log = gen.log(2); final Log copy = Log.readFrom(RLP.input(RLP.encode(log::writeTo))); assertThat(copy).isEqualTo(log); } + + @Test + public void toFromRlpCompacted() { + final Log log = gen.log(2); + final Log copy = Log.readFrom(RLP.input(RLP.encode(rlpOut -> log.writeTo(rlpOut, true))), true); + assertThat(copy).isEqualTo(log); + } + + @Test + public void toFromRlpCompactedWithLeadingZeros() { + final Bytes logData = bytesWithLeadingZeros(10, 100); + final List logTopics = + List.of( + LogTopic.of(bytesWithLeadingZeros(20, 32)), LogTopic.of(bytesWithLeadingZeros(30, 32))); + final Log log = new Log(gen.address(), logData, logTopics); + final Log copy = Log.readFrom(RLP.input(RLP.encode(rlpOut -> log.writeTo(rlpOut, true))), true); + assertThat(copy).isEqualTo(log); + } + + private Bytes bytesWithLeadingZeros(final int noLeadingZeros, final int totalSize) { + return Bytes.concatenate( + Bytes.repeat((byte) 0, noLeadingZeros), gen.bytesValue(totalSize - noLeadingZeros)); + } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/TransactionReceiptTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/TransactionReceiptTest.java index 00a04354c69..06b9367ba93 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/TransactionReceiptTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/TransactionReceiptTest.java @@ -28,7 +28,7 @@ public void toFromRlp() { final BlockDataGenerator gen = new BlockDataGenerator(); final TransactionReceipt receipt = gen.receipt(); final TransactionReceipt copy = - TransactionReceipt.readFrom(RLP.input(RLP.encode(receipt::writeToWithRevertReason)), false); + TransactionReceipt.readFrom(RLP.input(RLP.encode(receipt::writeToForNetwork)), false); assertThat(copy).isEqualTo(receipt); } @@ -37,7 +37,40 @@ public void toFromRlpWithReason() { final BlockDataGenerator gen = new BlockDataGenerator(); final TransactionReceipt receipt = gen.receipt(Bytes.fromHexString("0x1122334455667788")); final TransactionReceipt copy = - TransactionReceipt.readFrom(RLP.input(RLP.encode(receipt::writeToWithRevertReason))); + TransactionReceipt.readFrom( + RLP.input(RLP.encode(rlpOut -> receipt.writeToForReceiptTrie(rlpOut, true, false)))); assertThat(copy).isEqualTo(receipt); } + + @Test + public void toFromRlpCompacted() { + final BlockDataGenerator gen = new BlockDataGenerator(); + final TransactionReceipt receipt = gen.receipt(Bytes.fromHexString("0x1122334455667788")); + final TransactionReceipt copy = + TransactionReceipt.readFrom( + RLP.input(RLP.encode(rlpOut -> receipt.writeToForReceiptTrie(rlpOut, false, true)))); + assertThat(copy).isEqualTo(receipt); + } + + @Test + public void toFromRlpCompactedWithReason() { + final BlockDataGenerator gen = new BlockDataGenerator(); + final TransactionReceipt receipt = gen.receipt(Bytes.fromHexString("0x1122334455667788")); + final TransactionReceipt copy = + TransactionReceipt.readFrom( + RLP.input(RLP.encode(rlpOut -> receipt.writeToForReceiptTrie(rlpOut, true, true)))); + assertThat(copy).isEqualTo(receipt); + } + + @Test + public void uncompactedAndCompactedDecodeToSameReceipt() { + final BlockDataGenerator gen = new BlockDataGenerator(); + final TransactionReceipt receipt = gen.receipt(Bytes.fromHexString("0x1122334455667788")); + final Bytes compactedReceipt = + RLP.encode(rlpOut -> receipt.writeToForReceiptTrie(rlpOut, false, true)); + final Bytes unCompactedReceipt = + RLP.encode(rlpOut -> receipt.writeToForReceiptTrie(rlpOut, false, false)); + assertThat(TransactionReceipt.readFrom(RLP.input(compactedReceipt))).isEqualTo(receipt); + assertThat(TransactionReceipt.readFrom(RLP.input(unCompactedReceipt))).isEqualTo(receipt); + } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorageTest.java index 1e89581a095..526c839b69e 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStoragePrefixedKeyBlockchainStorageTest.java @@ -62,7 +62,7 @@ public void migrationToVariablesStorage() { final var blockchainStorage = new KeyValueStoragePrefixedKeyBlockchainStorage( - kvBlockchain, variablesStorage, blockHeaderFunctions); + kvBlockchain, variablesStorage, blockHeaderFunctions, false); assertNoVariablesInStorage(kvBlockchain); assertVariablesPresentInVariablesStorage(kvVariables, variableValues); @@ -80,7 +80,7 @@ public void migrationToVariablesStorageWhenSomeVariablesDoNotExist() { final var blockchainStorage = new KeyValueStoragePrefixedKeyBlockchainStorage( - kvBlockchain, variablesStorage, blockHeaderFunctions); + kvBlockchain, variablesStorage, blockHeaderFunctions, false); assertNoVariablesInStorage(kvBlockchain); assertVariablesPresentInVariablesStorage(kvVariables, variableValues); @@ -96,7 +96,7 @@ public void doesNothingIfVariablesAlreadyMigrated() { final var blockchainStorage = new KeyValueStoragePrefixedKeyBlockchainStorage( - kvBlockchain, variablesStorage, blockHeaderFunctions); + kvBlockchain, variablesStorage, blockHeaderFunctions, false); assertNoVariablesInStorage(kvBlockchain); assertVariablesPresentInVariablesStorage(kvVariables, variableValues); @@ -114,6 +114,6 @@ public void failIfInconsistencyDetectedDuringVariablesMigration() { IllegalStateException.class, () -> new KeyValueStoragePrefixedKeyBlockchainStorage( - kvBlockchain, variablesStorage, blockHeaderFunctions)); + kvBlockchain, variablesStorage, blockHeaderFunctions, false)); } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java index 2d18bee3268..ab9e87d5f61 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/bonsai/AbstractIsolationTests.java @@ -214,6 +214,22 @@ public DataStorageFormat getDatabaseFormat() { public Wei getMinGasPrice() { return MiningParameters.newDefault().getMinTransactionGasPrice(); } + + @Override + public org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration + getDataStorageConfiguration() { + return new org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration() { + @Override + public DataStorageFormat getDatabaseFormat() { + return DataStorageFormat.BONSAI; + } + + @Override + public boolean getReceiptCompactionEnabled() { + return false; + } + }; + } }) .withMetricsSystem(new NoOpMetricsSystem()) .build(); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerTest.java index 804c805624e..308834a6a8a 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/trie/forest/pruner/PrunerTest.java @@ -63,7 +63,8 @@ public void shouldMarkCorrectBlockAndSweep() throws ExecutionException, Interrup new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions()); + new MainnetBlockHeaderFunctions(), + false); final MutableBlockchain blockchain = DefaultBlockchain.createMutable(genesisBlock, blockchainStorage, metricsSystem, 0); @@ -86,7 +87,8 @@ public void shouldOnlySweepAfterBlockConfirmationPeriodAndRetentionPeriodEnds() new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions()); + new MainnetBlockHeaderFunctions(), + false); final MutableBlockchain blockchain = DefaultBlockchain.createMutable(genesisBlock, blockchainStorage, metricsSystem, 0); @@ -114,7 +116,8 @@ public void abortsPruningWhenFullyMarkedBlockNoLongerOnCanonicalChain() { new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions()); + new MainnetBlockHeaderFunctions(), + false); final MutableBlockchain blockchain = DefaultBlockchain.createMutable(genesisBlock, blockchainStorage, metricsSystem, 0); @@ -186,7 +189,8 @@ public void shouldCleanUpPruningStrategyOnShutdown() throws InterruptedException new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions()); + new MainnetBlockHeaderFunctions(), + false); final MutableBlockchain blockchain = DefaultBlockchain.createMutable(genesisBlock, blockchainStorage, metricsSystem, 0); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthServer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthServer.java index 76346f4152e..d70623d7bfd 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthServer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/EthServer.java @@ -234,7 +234,7 @@ static MessageData constructGetReceiptsResponse( } final BytesValueRLPOutput encodedReceipts = new BytesValueRLPOutput(); encodedReceipts.startList(); - maybeReceipts.get().forEach(r -> r.writeTo(encodedReceipts)); + maybeReceipts.get().forEach(r -> r.writeToForNetwork(encodedReceipts)); encodedReceipts.endList(); final int encodedSize = encodedReceipts.encodedSize(); if (responseSizeEstimate + encodedSize > maxMessageSize) { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/ReceiptsMessage.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/ReceiptsMessage.java index 9d9ed2cc7f1..7ca823cd8d7 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/ReceiptsMessage.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/ReceiptsMessage.java @@ -46,7 +46,7 @@ public static ReceiptsMessage create(final List> receip receipts.forEach( (receiptSet) -> { tmp.startList(); - receiptSet.forEach(r -> r.writeTo(tmp)); + receiptSet.forEach(r -> r.writeToForNetwork(tmp)); tmp.endList(); }); tmp.endList(); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthServerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthServerTest.java index 7c75c6ddad3..7cd7bdcdaa8 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthServerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/manager/EthServerTest.java @@ -399,7 +399,7 @@ private int calculateRlpEncodedSize(final Bytes data) { private int calculateRlpEncodedSize(final List receipts) { final BytesValueRLPOutput rlp = new BytesValueRLPOutput(); rlp.startList(); - receipts.forEach(r -> r.writeTo(rlp)); + receipts.forEach(r -> r.writeToForNetwork(rlp)); rlp.endList(); return rlp.encodedSize(); } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointBlockImportStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointBlockImportStepTest.java index 13b4718794a..4813cf84dd7 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointBlockImportStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/checkpointsync/CheckPointBlockImportStepTest.java @@ -52,7 +52,8 @@ public void setup() { new KeyValueStoragePrefixedKeyBlockchainStorage( new InMemoryKeyValueStorage(), new VariablesKeyValueStorage(new InMemoryKeyValueStorage()), - new MainnetBlockHeaderFunctions()); + new MainnetBlockHeaderFunctions(), + false); blockchain = DefaultBlockchain.createMutable( generateBlock(0), blockchainStorage, mock(MetricsSystem.class), 0); diff --git a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/DataStoreModule.java b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/DataStoreModule.java index 65585b6441e..05dff094191 100644 --- a/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/DataStoreModule.java +++ b/ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/DataStoreModule.java @@ -142,6 +142,9 @@ static BlockchainStorage provideBlockchainStorage( @Named("variables") final KeyValueStorage variablesKeyValueStorage, final BlockHeaderFunctions blockHashFunction) { return new KeyValueStoragePrefixedKeyBlockchainStorage( - keyValueStorage, new VariablesKeyValueStorage(variablesKeyValueStorage), blockHashFunction); + keyValueStorage, + new VariablesKeyValueStorage(variablesKeyValueStorage), + blockHashFunction, + false); } } diff --git a/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java b/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java index e34d9f44731..cad559ae15b 100644 --- a/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java +++ b/ethereum/retesteth/src/main/java/org/hyperledger/besu/ethereum/retesteth/RetestethContext.java @@ -279,7 +279,7 @@ private static MutableBlockchain createInMemoryBlockchain( return DefaultBlockchain.createMutable( genesisBlock, new KeyValueStoragePrefixedKeyBlockchainStorage( - keyValueStorage, variablesStorage, blockHeaderFunctions), + keyValueStorage, variablesStorage, blockHeaderFunctions, false), new NoOpMetricsSystem(), 100); } diff --git a/evm/src/main/java/org/hyperledger/besu/evm/log/Log.java b/evm/src/main/java/org/hyperledger/besu/evm/log/Log.java index 02b523445e7..0c578dfd2f9 100644 --- a/evm/src/main/java/org/hyperledger/besu/evm/log/Log.java +++ b/evm/src/main/java/org/hyperledger/besu/evm/log/Log.java @@ -26,6 +26,8 @@ import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.bytes.MutableBytes; /** * A log entry is a tuple of a logger’s address (the address of the contract that added the logs), a @@ -60,13 +62,38 @@ public Log( * @param out the output in which to encode the log entry. */ public void writeTo(final RLPOutput out) { + writeTo(out, false); + } + + /** + * Writes the log entry to the provided RLP output. + * + * @param out the output in which to encode the log entry. + * @param compacted whether to compact the rlp log entry by trimming leading zeros on topics and + * data. + */ + public void writeTo(final RLPOutput out, final boolean compacted) { out.startList(); out.writeBytes(logger); - out.writeList(topics, (topic, listOut) -> listOut.writeBytes(topic)); - out.writeBytes(data); + if (compacted) { + out.writeList(topics, (topic, listOut) -> encodeTrimmedData(listOut, topic)); + encodeTrimmedData(out, data); + } else { + out.writeList(topics, (topic, listOut) -> listOut.writeBytes(topic)); + out.writeBytes(data); + } out.endList(); } + private void encodeTrimmedData(final RLPOutput rlpOutput, final Bytes data) { + rlpOutput.startList(); + final Bytes shortData = data.trimLeadingZeros(); + final int zeroLeadDataSize = data.size() - shortData.size(); + rlpOutput.writeIntScalar(zeroLeadDataSize); + rlpOutput.writeBytes(shortData); + rlpOutput.endList(); + } + /** * Reads the log entry from the provided RLP input. * @@ -74,14 +101,45 @@ public void writeTo(final RLPOutput out) { * @return the read log entry. */ public static Log readFrom(final RLPInput in) { + return readFrom(in, false); + } + + /** + * Reads the log entry from the provided RLP input. + * + * @param in the input from which to decode the log entry. + * @param compacted whether to compact the rlp log entry by trimming leading zeros on topics and + * data. + * @return the read log entry. + */ + public static Log readFrom(final RLPInput in, final boolean compacted) { in.enterList(); final Address logger = Address.wrap(in.readBytes()); - final List topics = in.readList(listIn -> LogTopic.wrap(listIn.readBytes32())); - final Bytes data = in.readBytes(); + + final List topics; + final Bytes data; + if (compacted) { + topics = in.readList(listIn -> LogTopic.wrap(Bytes32.wrap(readTrimmedData(in)))); + data = Bytes.wrap(readTrimmedData(in)); + } else { + topics = in.readList(listIn -> LogTopic.wrap(listIn.readBytes32())); + data = in.readBytes(); + } + in.leaveList(); return new Log(logger, data, topics); } + private static Bytes readTrimmedData(final RLPInput in) { + in.enterList(); + final int zeroLeadDataSize = in.readIntScalar(); + final Bytes shortData = in.readBytes(); + final MutableBytes data = MutableBytes.create(zeroLeadDataSize + shortData.size()); + data.set(zeroLeadDataSize, shortData); + in.leaveList(); + return data; + } + /** * Gets logger address. * diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index 3c4f7f6c0be..423411b5630 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -69,7 +69,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = '/FHIztl2tLW5Gzc0qnfEeuVQa6ljVfUce7YE6JLDdZU=' + knownHash = 'YH+8rbilrhatRAh8rK8/36qxwrqkybBaaNeg+AkZ0c4=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BesuConfiguration.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BesuConfiguration.java index c28995cc6ce..ca92a12430f 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BesuConfiguration.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BesuConfiguration.java @@ -16,6 +16,7 @@ import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.plugin.Unstable; +import org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.nio.file.Path; @@ -43,6 +44,7 @@ public interface BesuConfiguration extends BesuService { * @return Database format. */ @Unstable + @Deprecated DataStorageFormat getDatabaseFormat(); /** @@ -52,4 +54,12 @@ public interface BesuConfiguration extends BesuService { */ @Unstable Wei getMinGasPrice(); + + /** + * Database storage configuration. + * + * @return Database storage configuration. + */ + @Unstable + DataStorageConfiguration getDataStorageConfiguration(); } diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/DataStorageConfiguration.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/DataStorageConfiguration.java new file mode 100644 index 00000000000..80abc936982 --- /dev/null +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/DataStorageConfiguration.java @@ -0,0 +1,40 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.hyperledger.besu.plugin.services.storage; + +import org.hyperledger.besu.plugin.Unstable; + +/** Data storage configuration */ +@Unstable +public interface DataStorageConfiguration { + + /** + * Database format. This sets the list of segmentIdentifiers that should be initialized. + * + * @return Database format. + */ + @Unstable + DataStorageFormat getDatabaseFormat(); + + /** + * Whether receipt compaction is enabled. When enabled this reduces the storage needed for + * receipts. + * + * @return Whether receipt compaction is enabled + */ + @Unstable + boolean getReceiptCompactionEnabled(); +} diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java index bf5f92165f7..909a2c5e90e 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactory.java @@ -46,7 +46,9 @@ public class RocksDBKeyValuePrivacyStorageFactory implements PrivacyKeyValueStor private static final Set SUPPORTED_VERSIONS = EnumSet.of( PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES, - PrivacyVersionedStorageFormat.BONSAI_WITH_VARIABLES); + PrivacyVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION, + PrivacyVersionedStorageFormat.BONSAI_WITH_VARIABLES, + PrivacyVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION); private static final String PRIVATE_DATABASE_PATH = "private"; private final RocksDBKeyValueStorageFactory publicFactory; private DatabaseMetadata databaseMetadata; @@ -145,7 +147,8 @@ private DatabaseMetadata readDatabaseMetadata(final BesuConfiguration commonConf privacyMetadata = existingPrivacyMetadata; final int existingPrivacyVersion = maybeExistingPrivacyVersion.getAsInt(); final var runtimeVersion = - PrivacyVersionedStorageFormat.defaultForNewDB(commonConfiguration.getDatabaseFormat()); + PrivacyVersionedStorageFormat.defaultForNewDB( + commonConfiguration.getDataStorageConfiguration()); if (existingPrivacyVersion > runtimeVersion.getPrivacyVersion().getAsInt()) { final var maybeDowngradedMetadata = @@ -194,6 +197,16 @@ private Optional handleVersionDowngrade( // In case we do an automated downgrade, then we also need to update the metadata on disk to // reflect the change to the runtime version, and return it. + // Besu supports both formats of receipts so no downgrade is needed + if (runtimeVersion == PrivacyVersionedStorageFormat.BONSAI_WITH_VARIABLES + || runtimeVersion == PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES) { + LOG.warn( + "Database contains compacted receipts but receipt compaction is not enabled, new receipts will " + + "be not stored in the compacted format. If you want to remove compacted receipts from the " + + "database it is necessary to resync Besu. Besu can support both compacted and non-compacted receipts."); + return Optional.empty(); + } + // for the moment there are supported automated downgrades, so we just fail. String error = String.format( @@ -216,6 +229,18 @@ private Optional handleVersionUpgrade( // In case we do an automated upgrade, then we also need to update the metadata on disk to // reflect the change to the runtime version, and return it. + // Besu supports both formats of receipts so no upgrade is needed other than updating metadata + if (runtimeVersion == PrivacyVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION + || runtimeVersion == PrivacyVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION) { + final DatabaseMetadata metadata = new DatabaseMetadata(runtimeVersion); + try { + metadata.writeToDirectory(dataDir); + return Optional.of(metadata); + } catch (IOException e) { + throw new StorageException("Database upgrade to use receipt compaction failed", e); + } + } + // for the moment there are no planned automated upgrades, so we just fail. String error = String.format( diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java index eaf95e82a1e..bcb86ea13a3 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java @@ -14,7 +14,9 @@ */ package org.hyperledger.besu.plugin.services.storage.rocksdb; +import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION; import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES; +import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION; import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat.FOREST_WITH_VARIABLES; import org.hyperledger.besu.plugin.services.BesuConfiguration; @@ -55,7 +57,11 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory { private static final Logger LOG = LoggerFactory.getLogger(RocksDBKeyValueStorageFactory.class); private static final EnumSet SUPPORTED_VERSIONED_FORMATS = - EnumSet.of(FOREST_WITH_VARIABLES, BONSAI_WITH_VARIABLES); + EnumSet.of( + FOREST_WITH_VARIABLES, + FOREST_WITH_RECEIPT_COMPACTION, + BONSAI_WITH_VARIABLES, + BONSAI_WITH_RECEIPT_COMPACTION); private static final String NAME = "rocksdb"; private final RocksDBMetricsFactory rocksDBMetricsFactory; private DatabaseMetadata databaseMetadata; @@ -220,12 +226,13 @@ private DatabaseMetadata readDatabaseMetadata(final BesuConfiguration commonConf if (!metadata .getVersionedStorageFormat() .getFormat() - .equals(commonConfiguration.getDatabaseFormat())) { + .equals(commonConfiguration.getDataStorageConfiguration().getDatabaseFormat())) { handleFormatMismatch(commonConfiguration, dataDir, metadata); } final var runtimeVersion = - BaseVersionedStorageFormat.defaultForNewDB(commonConfiguration.getDatabaseFormat()); + BaseVersionedStorageFormat.defaultForNewDB( + commonConfiguration.getDataStorageConfiguration()); if (metadata.getVersionedStorageFormat().getVersion() > runtimeVersion.getVersion()) { final var maybeDowngradedMetadata = @@ -247,7 +254,7 @@ private DatabaseMetadata readDatabaseMetadata(final BesuConfiguration commonConf LOG.info("Existing database at {}. Metadata {}. Processing WAL...", dataDir, metadata); } else { - metadata = DatabaseMetadata.defaultForNewDb(commonConfiguration.getDatabaseFormat()); + metadata = DatabaseMetadata.defaultForNewDb(commonConfiguration); LOG.info( "No existing database at {}. Using default metadata for new db {}", dataDir, metadata); if (!dataDirExists) { @@ -275,7 +282,7 @@ private static void handleFormatMismatch( + "Please check your config.", dataDir, existingMetadata.getVersionedStorageFormat().getFormat().name(), - commonConfiguration.getDatabaseFormat()); + commonConfiguration.getDataStorageConfiguration().getDatabaseFormat()); throw new StorageException(error); } @@ -290,6 +297,15 @@ private Optional handleVersionDowngrade( // In case we do an automated downgrade, then we also need to update the metadata on disk to // reflect the change to the runtime version, and return it. + // Besu supports both formats of receipts so no downgrade is needed + if (runtimeVersion == BONSAI_WITH_VARIABLES || runtimeVersion == FOREST_WITH_VARIABLES) { + LOG.warn( + "Database contains compacted receipts but receipt compaction is not enabled, new receipts will " + + "be not stored in the compacted format. If you want to remove compacted receipts from the " + + "database it is necessary to resync Besu. Besu can support both compacted and non-compacted receipts."); + return Optional.empty(); + } + // for the moment there are supported automated downgrades, so we just fail. String error = String.format( @@ -312,6 +328,18 @@ private Optional handleVersionUpgrade( // In case we do an automated upgrade, then we also need to update the metadata on disk to // reflect the change to the runtime version, and return it. + // Besu supports both formats of receipts so no upgrade is needed other than updating metadata + if (runtimeVersion == BONSAI_WITH_RECEIPT_COMPACTION + || runtimeVersion == FOREST_WITH_RECEIPT_COMPACTION) { + final DatabaseMetadata metadata = new DatabaseMetadata(runtimeVersion); + try { + metadata.writeToDirectory(dataDir); + return Optional.of(metadata); + } catch (IOException e) { + throw new StorageException("Database upgrade to use receipt compaction failed", e); + } + } + // for the moment there are no planned automated upgrades, so we just fail. String error = String.format( diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/BaseVersionedStorageFormat.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/BaseVersionedStorageFormat.java index a18b907a6e7..474d5e99201 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/BaseVersionedStorageFormat.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/BaseVersionedStorageFormat.java @@ -14,6 +14,7 @@ */ package org.hyperledger.besu.plugin.services.storage.rocksdb.configuration; +import org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.OptionalInt; @@ -27,13 +28,23 @@ public enum BaseVersionedStorageFormat implements VersionedStorageFormat { * make BlobDB more effective */ FOREST_WITH_VARIABLES(DataStorageFormat.FOREST, 2), + /** + * Current Forest version, with receipts using compaction, in order to make Receipts use less disk + * space + */ + FOREST_WITH_RECEIPT_COMPACTION(DataStorageFormat.FOREST, 3), /** Original Bonsai version, not used since replace by BONSAI_WITH_VARIABLES */ BONSAI_ORIGINAL(DataStorageFormat.BONSAI, 1), /** * Current Bonsai version, with blockchain variables in a dedicated column family, in order to * make BlobDB more effective */ - BONSAI_WITH_VARIABLES(DataStorageFormat.BONSAI, 2); + BONSAI_WITH_VARIABLES(DataStorageFormat.BONSAI, 2), + /** + * Current Bonsai version, with receipts using compaction, in order to make Receipts use less disk + * space + */ + BONSAI_WITH_RECEIPT_COMPACTION(DataStorageFormat.BONSAI, 3); private final DataStorageFormat format; private final int version; @@ -46,13 +57,18 @@ public enum BaseVersionedStorageFormat implements VersionedStorageFormat { /** * Return the default version for new db for a specific format * - * @param format data storage format + * @param configuration data storage configuration * @return the version to use for new db */ - public static BaseVersionedStorageFormat defaultForNewDB(final DataStorageFormat format) { - return switch (format) { - case FOREST -> FOREST_WITH_VARIABLES; - case BONSAI -> BONSAI_WITH_VARIABLES; + public static BaseVersionedStorageFormat defaultForNewDB( + final DataStorageConfiguration configuration) { + return switch (configuration.getDatabaseFormat()) { + case FOREST -> configuration.getReceiptCompactionEnabled() + ? FOREST_WITH_RECEIPT_COMPACTION + : FOREST_WITH_VARIABLES; + case BONSAI -> configuration.getReceiptCompactionEnabled() + ? BONSAI_WITH_RECEIPT_COMPACTION + : BONSAI_WITH_VARIABLES; }; } diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadata.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadata.java index 4e18e35029c..c9cdd4b8bc9 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadata.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/DatabaseMetadata.java @@ -14,6 +14,7 @@ */ package org.hyperledger.besu.plugin.services.storage.rocksdb.configuration; +import org.hyperledger.besu.plugin.services.BesuConfiguration; import org.hyperledger.besu.plugin.services.exception.StorageException; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; @@ -46,18 +47,25 @@ public class DatabaseMetadata { .enable(SerializationFeature.INDENT_OUTPUT); private final VersionedStorageFormat versionedStorageFormat; - private DatabaseMetadata(final VersionedStorageFormat versionedStorageFormat) { + /** + * Instantiates a new Database metadata. + * + * @param versionedStorageFormat the version storage format + */ + public DatabaseMetadata(final VersionedStorageFormat versionedStorageFormat) { this.versionedStorageFormat = versionedStorageFormat; } /** * Return the default metadata for new db for a specific format * - * @param dataStorageFormat data storage format + * @param besuConfiguration besu configuration * @return the metadata to use for new db */ - public static DatabaseMetadata defaultForNewDb(final DataStorageFormat dataStorageFormat) { - return new DatabaseMetadata(BaseVersionedStorageFormat.defaultForNewDB(dataStorageFormat)); + public static DatabaseMetadata defaultForNewDb(final BesuConfiguration besuConfiguration) { + return new DatabaseMetadata( + BaseVersionedStorageFormat.defaultForNewDB( + besuConfiguration.getDataStorageConfiguration())); } /** @@ -222,6 +230,7 @@ public DatabaseMetadata upgradeToPrivacy() { case FOREST -> switch (versionedStorageFormat.getVersion()) { case 1 -> PrivacyVersionedStorageFormat.FOREST_ORIGINAL; case 2 -> PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES; + case 3 -> PrivacyVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION; default -> throw new StorageException( "Unsupported database with format FOREST and version " + versionedStorageFormat.getVersion()); @@ -229,6 +238,7 @@ public DatabaseMetadata upgradeToPrivacy() { case BONSAI -> switch (versionedStorageFormat.getVersion()) { case 1 -> PrivacyVersionedStorageFormat.BONSAI_ORIGINAL; case 2 -> PrivacyVersionedStorageFormat.BONSAI_WITH_VARIABLES; + case 3 -> PrivacyVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION; default -> throw new StorageException( "Unsupported database with format BONSAI and version " + versionedStorageFormat.getVersion()); diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/PrivacyVersionedStorageFormat.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/PrivacyVersionedStorageFormat.java index 9a86d6ec15d..a883516f6a6 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/PrivacyVersionedStorageFormat.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/configuration/PrivacyVersionedStorageFormat.java @@ -14,6 +14,7 @@ */ package org.hyperledger.besu.plugin.services.storage.rocksdb.configuration; +import org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import java.util.OptionalInt; @@ -27,13 +28,23 @@ public enum PrivacyVersionedStorageFormat implements VersionedStorageFormat { * make BlobDB more effective */ FOREST_WITH_VARIABLES(BaseVersionedStorageFormat.FOREST_WITH_VARIABLES, 1), + /** + * Current Forest version, with receipts using compaction, in order to make Receipts use less disk + * space + */ + FOREST_WITH_RECEIPT_COMPACTION(BaseVersionedStorageFormat.FOREST_WITH_VARIABLES, 2), /** Original Bonsai version, not used since replace by BONSAI_WITH_VARIABLES */ BONSAI_ORIGINAL(BaseVersionedStorageFormat.BONSAI_ORIGINAL, 1), /** * Current Bonsai version, with blockchain variables in a dedicated column family, in order to * make BlobDB more effective */ - BONSAI_WITH_VARIABLES(BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES, 1); + BONSAI_WITH_VARIABLES(BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES, 1), + /** + * Current Bonsai version, with receipts using compaction, in order to make Receipts use less disk + * space + */ + BONSAI_WITH_RECEIPT_COMPACTION(BaseVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION, 2); private final VersionedStorageFormat baseVersionedStorageFormat; private final OptionalInt privacyVersion; @@ -47,13 +58,18 @@ public enum PrivacyVersionedStorageFormat implements VersionedStorageFormat { /** * Return the default version for new db for a specific format * - * @param format data storage format + * @param configuration data storage configuration * @return the version to use for new db */ - public static VersionedStorageFormat defaultForNewDB(final DataStorageFormat format) { - return switch (format) { - case FOREST -> FOREST_WITH_VARIABLES; - case BONSAI -> BONSAI_WITH_VARIABLES; + public static VersionedStorageFormat defaultForNewDB( + final DataStorageConfiguration configuration) { + return switch (configuration.getDatabaseFormat()) { + case FOREST -> configuration.getReceiptCompactionEnabled() + ? FOREST_WITH_RECEIPT_COMPACTION + : FOREST_WITH_VARIABLES; + case BONSAI -> configuration.getReceiptCompactionEnabled() + ? BONSAI_WITH_RECEIPT_COMPACTION + : BONSAI_WITH_VARIABLES; }; } diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java index 003d640be8d..5dd485cb7af 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java @@ -15,13 +15,17 @@ package org.hyperledger.besu.plugin.services.storage.rocksdb; import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.plugin.services.storage.DataStorageFormat.BONSAI; import static org.hyperledger.besu.plugin.services.storage.DataStorageFormat.FOREST; import static org.hyperledger.besu.plugin.services.storage.rocksdb.segmented.RocksDBColumnarKeyValueStorageTest.TestSegment; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; import org.hyperledger.besu.metrics.ObservableMetricsSystem; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.BesuConfiguration; +import org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration; +import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.DatabaseMetadata; @@ -35,6 +39,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -42,6 +48,7 @@ public class RocksDBKeyValuePrivacyStorageFactoryTest { @Mock private RocksDBFactoryConfiguration rocksDbConfiguration; @Mock private BesuConfiguration commonConfiguration; + @Mock private DataStorageConfiguration dataStorageConfiguration; @TempDir private Path temporaryFolder; private final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem(); private final SegmentIdentifier segment = TestSegment.BAR; @@ -94,20 +101,25 @@ public void shouldCreateCorrectMetadataFileForLatestVersion() throws Exception { } } - @Test - public void shouldUpdateCorrectMetadataFileForLatestVersion() throws Exception { + @ParameterizedTest + @EnumSource(DataStorageFormat.class) + public void shouldUpdateCorrectMetadataFileForLatestVersion( + final DataStorageFormat dataStorageFormat) throws Exception { final Path tempDataDir = temporaryFolder.resolve("data"); final Path tempDatabaseDir = temporaryFolder.resolve("db"); - mockCommonConfiguration(tempDataDir, tempDatabaseDir); + mockCommonConfiguration(tempDataDir, tempDatabaseDir, dataStorageFormat); final RocksDBKeyValueStorageFactory storageFactory = new RocksDBKeyValueStorageFactory( () -> rocksDbConfiguration, segments, RocksDBMetricsFactory.PRIVATE_ROCKS_DB_METRICS); try (final var storage = storageFactory.create(segment, commonConfiguration, metricsSystem)) { - + final BaseVersionedStorageFormat expectedBaseVersion = + dataStorageFormat == BONSAI + ? BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES + : BaseVersionedStorageFormat.FOREST_WITH_VARIABLES; assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat()) - .isEqualTo(BaseVersionedStorageFormat.FOREST_WITH_VARIABLES); + .isEqualTo(expectedBaseVersion); } storageFactory.close(); @@ -116,16 +128,67 @@ public void shouldUpdateCorrectMetadataFileForLatestVersion() throws Exception { try (final var storage = privacyStorageFactory.create(segment, commonConfiguration, metricsSystem)) { + final PrivacyVersionedStorageFormat expectedPrivacyVersion = + dataStorageFormat == BONSAI + ? PrivacyVersionedStorageFormat.BONSAI_WITH_VARIABLES + : PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES; + assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat()) + .isEqualTo(expectedPrivacyVersion); + } + privacyStorageFactory.close(); + } + + @ParameterizedTest + @EnumSource(DataStorageFormat.class) + public void shouldUpdateCorrectMetadataFileForLatestVersionWithReceiptCompaction( + final DataStorageFormat dataStorageFormat) throws Exception { + final Path tempDataDir = temporaryFolder.resolve("data"); + final Path tempDatabaseDir = temporaryFolder.resolve("db"); + mockCommonConfiguration(tempDataDir, tempDatabaseDir, dataStorageFormat); + when(dataStorageConfiguration.getReceiptCompactionEnabled()).thenReturn(true); + + final RocksDBKeyValueStorageFactory storageFactory = + new RocksDBKeyValueStorageFactory( + () -> rocksDbConfiguration, segments, RocksDBMetricsFactory.PRIVATE_ROCKS_DB_METRICS); + try (final var storage = storageFactory.create(segment, commonConfiguration, metricsSystem)) { + final BaseVersionedStorageFormat expectedBaseVersion = + dataStorageFormat == BONSAI + ? BaseVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION + : BaseVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION; assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat()) - .isEqualTo(PrivacyVersionedStorageFormat.FOREST_WITH_VARIABLES); + .isEqualTo(expectedBaseVersion); + } + storageFactory.close(); + + final RocksDBKeyValuePrivacyStorageFactory privacyStorageFactory = + new RocksDBKeyValuePrivacyStorageFactory(storageFactory); + + try (final var storage = + privacyStorageFactory.create(segment, commonConfiguration, metricsSystem)) { + final PrivacyVersionedStorageFormat expectedPrivacyVersion = + dataStorageFormat == BONSAI + ? PrivacyVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION + : PrivacyVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION; + assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat()) + .isEqualTo(expectedPrivacyVersion); } privacyStorageFactory.close(); } private void mockCommonConfiguration(final Path tempDataDir, final Path tempDatabaseDir) { + mockCommonConfiguration(tempDataDir, tempDatabaseDir, FOREST); + } + + private void mockCommonConfiguration( + final Path tempDataDir, + final Path tempDatabaseDir, + final DataStorageFormat dataStorageFormat) { when(commonConfiguration.getStoragePath()).thenReturn(tempDatabaseDir); when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); - when(commonConfiguration.getDatabaseFormat()).thenReturn(FOREST); + when(dataStorageConfiguration.getDatabaseFormat()).thenReturn(dataStorageFormat); + lenient() + .when(commonConfiguration.getDataStorageConfiguration()) + .thenReturn(dataStorageConfiguration); } } diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java index e2c56f0ac50..1748f280c1a 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java @@ -27,6 +27,7 @@ import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.plugin.services.BesuConfiguration; import org.hyperledger.besu.plugin.services.exception.StorageException; +import org.hyperledger.besu.plugin.services.storage.DataStorageConfiguration; import org.hyperledger.besu.plugin.services.storage.DataStorageFormat; import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.BaseVersionedStorageFormat; @@ -43,6 +44,8 @@ import org.junit.jupiter.api.condition.OS; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -51,16 +54,19 @@ public class RocksDBKeyValueStorageFactoryTest { @Mock private RocksDBFactoryConfiguration rocksDbConfiguration; @Mock private BesuConfiguration commonConfiguration; + @Mock private DataStorageConfiguration dataStorageConfiguration; @TempDir public Path temporaryFolder; private final ObservableMetricsSystem metricsSystem = new NoOpMetricsSystem(); private final SegmentIdentifier segment = TestSegment.FOO; private final List segments = List.of(TestSegment.DEFAULT, segment); - @Test - public void shouldCreateCorrectMetadataFileForLatestVersionForNewDb() throws Exception { + @ParameterizedTest + @EnumSource(DataStorageFormat.class) + public void shouldCreateCorrectMetadataFileForLatestVersionForNewDb( + final DataStorageFormat dataStorageFormat) throws Exception { final Path tempDataDir = temporaryFolder.resolve("data"); final Path tempDatabaseDir = temporaryFolder.resolve("db"); - mockCommonConfiguration(tempDataDir, tempDatabaseDir, FOREST); + mockCommonConfiguration(tempDataDir, tempDatabaseDir, dataStorageFormat); final RocksDBKeyValueStorageFactory storageFactory = new RocksDBKeyValueStorageFactory( @@ -68,8 +74,36 @@ public void shouldCreateCorrectMetadataFileForLatestVersionForNewDb() throws Exc try (final var storage = storageFactory.create(segment, commonConfiguration, metricsSystem)) { // Side effect is creation of the Metadata version file + final BaseVersionedStorageFormat expectedVersion = + dataStorageFormat == BONSAI + ? BaseVersionedStorageFormat.BONSAI_WITH_VARIABLES + : BaseVersionedStorageFormat.FOREST_WITH_VARIABLES; assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat()) - .isEqualTo(BaseVersionedStorageFormat.FOREST_WITH_VARIABLES); + .isEqualTo(expectedVersion); + } + } + + @ParameterizedTest + @EnumSource(DataStorageFormat.class) + public void shouldCreateCorrectMetadataFileForLatestVersionForNewDbWithReceiptCompaction( + final DataStorageFormat dataStorageFormat) throws Exception { + final Path tempDataDir = temporaryFolder.resolve("data"); + final Path tempDatabaseDir = temporaryFolder.resolve("db"); + mockCommonConfiguration(tempDataDir, tempDatabaseDir, dataStorageFormat); + when(dataStorageConfiguration.getReceiptCompactionEnabled()).thenReturn(true); + + final RocksDBKeyValueStorageFactory storageFactory = + new RocksDBKeyValueStorageFactory( + () -> rocksDbConfiguration, segments, RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS); + + try (final var storage = storageFactory.create(segment, commonConfiguration, metricsSystem)) { + // Side effect is creation of the Metadata version file + final BaseVersionedStorageFormat expectedVersion = + dataStorageFormat == BONSAI + ? BaseVersionedStorageFormat.BONSAI_WITH_RECEIPT_COMPACTION + : BaseVersionedStorageFormat.FOREST_WITH_RECEIPT_COMPACTION; + assertThat(DatabaseMetadata.lookUpFrom(tempDataDir).getVersionedStorageFormat()) + .isEqualTo(expectedVersion); } } @@ -273,6 +307,9 @@ private void mockCommonConfiguration( final Path tempDataDir, final Path tempDatabaseDir, final DataStorageFormat format) { when(commonConfiguration.getStoragePath()).thenReturn(tempDatabaseDir); when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); - lenient().when(commonConfiguration.getDatabaseFormat()).thenReturn(format); + lenient().when(dataStorageConfiguration.getDatabaseFormat()).thenReturn(format); + lenient() + .when(commonConfiguration.getDataStorageConfiguration()) + .thenReturn(dataStorageConfiguration); } } From 4373214d3d1f376a6eee848de6169453a15fa301 Mon Sep 17 00:00:00 2001 From: Simon Dudley Date: Tue, 26 Mar 2024 16:31:59 +1000 Subject: [PATCH 13/18] Remove draft config from develop release action (#6807) Trying to get develop tag to be a pre-release Signed-off-by: Simon Dudley Signed-off-by: Justin Florentine --- .github/workflows/develop.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index 38c9024260f..dc85596f469 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -44,7 +44,6 @@ jobs: uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 with: prerelease: true - draft: false name: develop tag_name: develop fail_on_unmatched_files: true From 03d8a13b416aef68077a4d78a6de1bad74f7bf33 Mon Sep 17 00:00:00 2001 From: Simon Dudley Date: Tue, 26 Mar 2024 17:03:20 +1000 Subject: [PATCH 14/18] Set noisy TransactionLogBloomCacher debug log to trace (#6808) Signed-off-by: Simon Dudley Signed-off-by: Justin Florentine --- .../ethereum/api/query/cache/TransactionLogBloomCacher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/cache/TransactionLogBloomCacher.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/cache/TransactionLogBloomCacher.java index 31132a4655e..69a9edc82f5 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/cache/TransactionLogBloomCacher.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/cache/TransactionLogBloomCacher.java @@ -159,7 +159,7 @@ void cacheLogsBloomForBlockHeader( return; } final long blockNumber = blockHeader.getNumber(); - LOG.atDebug() + LOG.atTrace() .setMessage("Caching logs bloom for block {}") .addArgument(() -> "0x" + Long.toHexString(blockNumber)) .log(); From a97108cde3c3e0fadc5e8b9e623e10f2026e2e24 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 26 Mar 2024 10:04:41 +0100 Subject: [PATCH 15/18] Support running executable Gradle tasks on Windows with WSL (#6803) Signed-off-by: Fabio Di Fabio Signed-off-by: Justin Florentine --- build.gradle | 41 ++++++++++++++++++++++------------------- wslsh.bat | 2 ++ 2 files changed, 24 insertions(+), 19 deletions(-) create mode 100644 wslsh.bat diff --git a/build.gradle b/build.gradle index d9e854fd859..00a1be20988 100644 --- a/build.gradle +++ b/build.gradle @@ -89,6 +89,9 @@ def _strListCmdArg(name) { return _strListCmdArg(name, null) } +// set the shell command to use according to os +def shell = org.gradle.internal.os.OperatingSystem.current().isWindows() ? "${projectDir}\\wslsh.bat" : '/bin/bash' + licenseReport { // This is for the allowed-licenses-file in checkLicense Task // Accepts File, URL or String path to local or remote file @@ -760,7 +763,7 @@ task distDocker { println "Building for platform ${project.getProperty('docker-platform')}" } def gitDetails = getGitCommitDetails(10) - executable "sh" + executable shell workingDir dockerBuildDir args "-c", "docker build ${dockerPlatform} --build-arg BUILD_DATE=${buildTime()} --build-arg VERSION=${dockerBuildVersion} --build-arg VCS_REF=${gitDetails.hash} -t ${image} ." } @@ -768,12 +771,12 @@ task distDocker { // tag the "default" (which is the variant in the zero position) exec { - executable "sh" + executable shell args "-c", "docker tag '${dockerImageName}:${dockerBuildVersion}-${dockerVariants[0]}' '${dockerImageName}:${dockerBuildVersion}'" } // create a static tag for the benchmark target exec { - executable "sh" + executable shell args "-c", "docker tag '${dockerImageName}:${dockerBuildVersion}-${dockerVariants[0]}' '${dockerImageName}:benchmark'" } } @@ -792,8 +795,8 @@ task testDocker { exec { def image = project.hasProperty('release.releaseVersion') ? "${dockerImageName}:" + project.property('release.releaseVersion') : "${dockerImageName}:${project.version}" workingDir "${projectDir}/docker/${variant}" - executable "sh" - args "-c", "bash ../test.sh ${image}-${variant}" + executable shell + args "-c", "../test.sh ${image}-${variant}" } } } @@ -821,7 +824,7 @@ task dockerUpload { def cmd = "docker tag '${variantImage}' '${archVariantImage}' && docker push '${archVariantImage}'" additionalTags.each { tag -> cmd += " && docker tag '${variantImage}' '${dockerImageName}:${tag.trim()}-${variant}-${architecture}' && docker push '${dockerImageName}:${tag.trim()}-${variant}-${architecture}'" } println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } } @@ -831,7 +834,7 @@ task dockerUpload { def cmd = "docker tag ${image} ${archImage} && docker push '${archImage}'" additionalTags.each { tag -> cmd += " && docker tag '${image}' '${dockerImageName}:${tag.trim()}-${architecture}' && docker push '${dockerImageName}:${tag.trim()}-${architecture}'" } println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } } @@ -848,13 +851,13 @@ task dockerUploadRelease { exec { def cmd = "docker pull '${variantImage}-${architecture}' && docker tag '${variantImage}-${architecture}' '${dockerImageName}:latest-${variant}-${architecture}'" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } exec { def cmd = "docker push '${dockerImageName}:latest-${variant}-${architecture}'" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } } @@ -863,13 +866,13 @@ task dockerUploadRelease { def archImage = "${image}-${architecture}" def cmd = "docker pull '${archImage}' && docker tag ${archImage} '${dockerImageName}:latest-${architecture}'" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } exec { def cmd = "docker push '${dockerImageName}:latest-${architecture}'" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } } @@ -899,13 +902,13 @@ task manifestDocker { exec { def cmd = "docker manifest create '${variantImage}' ${targets}" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } exec { def cmd = "docker manifest push '${variantImage}'" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } } @@ -915,13 +918,13 @@ task manifestDocker { archs.forEach { arch -> targets += "'${baseTag}-${arch}' " } def cmd = "docker manifest create '${baseTag}' ${targets}" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } exec { def cmd = "docker manifest push '${baseTag}'" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } } @@ -941,13 +944,13 @@ task manifestDockerRelease { exec { def cmd = "docker manifest create '${variantImage}' ${targets} --amend" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } exec { def cmd = "docker manifest push '${variantImage}'" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } } @@ -957,13 +960,13 @@ task manifestDockerRelease { archs.forEach { arch -> targets += "'${baseTag}-${arch}' " } def cmd = "docker manifest create '${baseTag}' ${targets} --amend" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } exec { def cmd = "docker manifest push '${baseTag}'" println "Executing '${cmd}'" - executable "sh" + executable shell args "-c", cmd } } diff --git a/wslsh.bat b/wslsh.bat new file mode 100644 index 00000000000..57e1d623070 --- /dev/null +++ b/wslsh.bat @@ -0,0 +1,2 @@ +set WSLENV=architecture/u +wsl /bin/bash %* \ No newline at end of file From fccab7a058b6629bc8ae29df95ccc0d4a067cf5e Mon Sep 17 00:00:00 2001 From: Karim TAAM Date: Tue, 26 Mar 2024 14:21:52 +0100 Subject: [PATCH 16/18] Fix issues during snap sync (#6802) The purpose of this commit is to fix some bugs detected in the snap sync. This should allow for greater stability with the snap sync and the healing of the flat DB. --------- Signed-off-by: Karim Taam Signed-off-by: Justin Florentine --- .../eth/sync/snapsync/RequestDataStep.java | 1 + .../sync/snapsync/SnapSyncMetricsManager.java | 1 + .../sync/snapsync/SnapWorldDownloadState.java | 1 + .../snapsync/SnapWorldStateDownloader.java | 1 + .../ethereum/eth/sync/snapsync/StackTrie.java | 13 +- .../request/AccountRangeDataRequest.java | 6 +- .../request/StorageRangeDataRequest.java | 22 +-- ...ccountFlatDatabaseHealingRangeRequest.java | 6 +- ...torageFlatDatabaseHealingRangeRequest.java | 4 +- .../snap/GetAccountRangeMessageTest.java | 2 +- .../snap/GetStorageRangeMessageTest.java | 2 +- .../snapsync/AccountHealingTrackingTest.java | 2 +- .../eth/sync/snapsync/RangeManagerTest.java | 1 + .../snapsync/SnapWorldDownloadStateTest.java | 1 + .../eth/sync/snapsync/StackTrieTest.java | 126 ++++++++++++++++ .../eth/sync/snapsync/TaskGenerator.java | 1 + ...ntFlatDatabaseHealingRangeRequestTest.java | 2 +- ...geFlatDatabaseHealingRangeRequestTest.java | 2 +- .../trie/InnerNodeDiscoveryManager.java | 41 ++---- .../besu/ethereum/trie}/RangeManager.java | 44 +++++- .../besu/ethereum/trie/SnapCommitVisitor.java | 138 ++++++++++++++++++ .../besu/ethereum/trie/SnapPutVisitor.java | 52 ------- .../ethereum/trie/SnapCommitVisitorTest.java | 127 ++++++++++++++++ .../ethereum/trie/SnapPutVisitorTest.java | 129 ---------------- 24 files changed, 483 insertions(+), 242 deletions(-) rename ethereum/{eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync => trie/src/main/java/org/hyperledger/besu/ethereum/trie}/RangeManager.java (74%) create mode 100644 ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/SnapCommitVisitor.java delete mode 100644 ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/SnapPutVisitor.java create mode 100644 ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/SnapCommitVisitorTest.java delete mode 100644 ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/SnapPutVisitorTest.java diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RequestDataStep.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RequestDataStep.java index fdde0a5eea9..adbb8b736c1 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RequestDataStep.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RequestDataStep.java @@ -31,6 +31,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.StorageFlatDatabaseHealingRangeRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.TrieNodeHealingRequest; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.plugin.services.MetricsSystem; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncMetricsManager.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncMetricsManager.java index da7607de7fd..b8e907b2eb7 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncMetricsManager.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapSyncMetricsManager.java @@ -18,6 +18,7 @@ import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncMetricsManager.Step.HEAL_TRIE; import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.metrics.BesuMetricCategory; import org.hyperledger.besu.plugin.services.MetricsSystem; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java index cb6ed53ce69..000badc25d7 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadState.java @@ -29,6 +29,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.AccountFlatDatabaseHealingRangeRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal.StorageFlatDatabaseHealingRangeRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldDownloadState; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java index b3dae73d934..61219882741 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldStateDownloader.java @@ -27,6 +27,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.AccountRangeDataRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.metrics.BesuMetricCategory; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrie.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrie.java index 01f17eb79b4..f9e8ea2eb7d 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrie.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrie.java @@ -15,12 +15,12 @@ package org.hyperledger.besu.ethereum.eth.sync.snapsync; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.trie.CommitVisitor; import org.hyperledger.besu.ethereum.trie.InnerNodeDiscoveryManager; import org.hyperledger.besu.ethereum.trie.MerkleTrie; import org.hyperledger.besu.ethereum.trie.Node; import org.hyperledger.besu.ethereum.trie.NodeUpdater; -import org.hyperledger.besu.ethereum.trie.SnapPutVisitor; +import org.hyperledger.besu.ethereum.trie.RangeManager; +import org.hyperledger.besu.ethereum.trie.SnapCommitVisitor; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import java.util.ArrayList; @@ -122,7 +122,7 @@ public void commit(final FlatDatabaseUpdater flatDatabaseUpdater, final NodeUpda Function.identity(), Function.identity(), startKeyHash, - keys.lastKey(), + proofs.isEmpty() ? RangeManager.MAX_RANGE : keys.lastKey(), true); final MerkleTrie trie = @@ -130,14 +130,17 @@ public void commit(final FlatDatabaseUpdater flatDatabaseUpdater, final NodeUpda snapStoredNodeFactory, proofs.isEmpty() ? MerkleTrie.EMPTY_TRIE_NODE_HASH : rootHash); for (Map.Entry entry : keys.entrySet()) { - trie.put(entry.getKey(), new SnapPutVisitor<>(snapStoredNodeFactory, entry.getValue())); + trie.put(entry.getKey(), entry.getValue()); } keys.forEach(flatDatabaseUpdater::update); trie.commit( nodeUpdater, - (new CommitVisitor<>(nodeUpdater) { + (new SnapCommitVisitor<>( + nodeUpdater, + startKeyHash, + proofs.isEmpty() ? RangeManager.MAX_RANGE : keys.lastKey()) { @Override public void maybeStoreNode(final Bytes location, final Node node) { if (!node.isHealNeeded()) { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java index 7829ad75eb8..76897e18505 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/AccountRangeDataRequest.java @@ -14,12 +14,12 @@ */ package org.hyperledger.besu.ethereum.eth.sync.snapsync.request; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MAX_RANGE; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MIN_RANGE; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.findNewBeginElementInRange; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RequestType.ACCOUNT_RANGE; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncMetricsManager.Step.DOWNLOAD; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.StackTrie.FlatDatabaseUpdater.noop; +import static org.hyperledger.besu.ethereum.trie.RangeManager.MAX_RANGE; +import static org.hyperledger.besu.ethereum.trie.RangeManager.MIN_RANGE; +import static org.hyperledger.besu.ethereum.trie.RangeManager.findNewBeginElementInRange; import static org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator.applyForStrategy; import org.hyperledger.besu.datatypes.Hash; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java index 0cb70b54635..646161a42e2 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/StorageRangeDataRequest.java @@ -14,16 +14,15 @@ */ package org.hyperledger.besu.ethereum.eth.sync.snapsync.request; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MAX_RANGE; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MIN_RANGE; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.findNewBeginElementInRange; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.getRangeCount; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RequestType.STORAGE_RANGE; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.StackTrie.FlatDatabaseUpdater.noop; +import static org.hyperledger.besu.ethereum.trie.RangeManager.MAX_RANGE; +import static org.hyperledger.besu.ethereum.trie.RangeManager.MIN_RANGE; +import static org.hyperledger.besu.ethereum.trie.RangeManager.findNewBeginElementInRange; +import static org.hyperledger.besu.ethereum.trie.RangeManager.getRangeCount; import static org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator.applyForStrategy; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; @@ -31,6 +30,7 @@ import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.trie.CompactEncoding; import org.hyperledger.besu.ethereum.trie.NodeUpdater; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.FlatDbMode; import org.hyperledger.besu.ethereum.worldstate.WorldStateKeyValueStorage; @@ -62,7 +62,7 @@ public class StorageRangeDataRequest extends SnapDataRequest { private final Bytes32 startKeyHash; private final Bytes32 endKeyHash; - private StackTrie stackTrie; + private final StackTrie stackTrie; private Optional isProofValid; protected StorageRangeDataRequest( @@ -77,7 +77,7 @@ protected StorageRangeDataRequest( this.startKeyHash = startKeyHash; this.endKeyHash = endKeyHash; this.isProofValid = Optional.empty(); - addStackTrie(Optional.empty()); + this.stackTrie = new StackTrie(Hash.wrap(getStorageRoot()), startKeyHash); LOG.trace( "create get storage range data request for account {} with root hash={} from {} to {}", accountHash, @@ -183,7 +183,6 @@ public Stream getChildRequests( final StorageRangeDataRequest storageRangeDataRequest = createStorageRangeDataRequest( getRootHash(), accountHash, storageRoot, key, value); - storageRangeDataRequest.addStackTrie(Optional.of(stackTrie)); childRequests.add(storageRangeDataRequest); }); if (startKeyHash.equals(MIN_RANGE) && endKeyHash.equals(MAX_RANGE)) { @@ -225,11 +224,4 @@ public void clear() { public void setProofValid(final boolean isProofValid) { this.isProofValid = Optional.of(isProofValid); } - - public void addStackTrie(final Optional maybeStackTrie) { - stackTrie = - maybeStackTrie - .filter(StackTrie::addSegment) - .orElse(new StackTrie(Hash.wrap(getStorageRoot()), 1, 3, startKeyHash)); - } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java index 05bbeed3da5..76d6f903f12 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequest.java @@ -14,12 +14,11 @@ */ package org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MAX_RANGE; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MIN_RANGE; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncMetricsManager.Step.HEAL_FLAT; +import static org.hyperledger.besu.ethereum.trie.RangeManager.MAX_RANGE; +import static org.hyperledger.besu.ethereum.trie.RangeManager.MIN_RANGE; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.RequestType; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; @@ -29,6 +28,7 @@ import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.CompactEncoding; import org.hyperledger.besu.ethereum.trie.MerkleTrie; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java index ac554439e91..26610572533 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequest.java @@ -14,17 +14,17 @@ */ package org.hyperledger.besu.ethereum.eth.sync.snapsync.request.heal; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.getRangeCount; import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RequestType.STORAGE_RANGE; +import static org.hyperledger.besu.ethereum.trie.RangeManager.getRangeCount; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetAccountRangeMessageTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetAccountRangeMessageTest.java index c8ef62a6d51..a486608eff1 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetAccountRangeMessageTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetAccountRangeMessageTest.java @@ -15,10 +15,10 @@ package org.hyperledger.besu.ethereum.eth.messages.snap; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.AbstractSnapMessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.RawMessage; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.apache.tuweni.bytes.Bytes32; import org.assertj.core.api.Assertions; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetStorageRangeMessageTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetStorageRangeMessageTest.java index 4ff1b09b9a0..2492528a89b 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetStorageRangeMessageTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/snap/GetStorageRangeMessageTest.java @@ -15,10 +15,10 @@ package org.hyperledger.besu.ethereum.eth.messages.snap; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.AbstractSnapMessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.RawMessage; +import org.hyperledger.besu.ethereum.trie.RangeManager; import java.util.List; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java index 28001a2a377..296952cf029 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/AccountHealingTrackingTest.java @@ -14,7 +14,7 @@ */ package org.hyperledger.besu.ethereum.eth.sync.snapsync; -import static org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager.MAX_RANGE; +import static org.hyperledger.besu.ethereum.trie.RangeManager.MAX_RANGE; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManagerTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManagerTest.java index 40b967096a7..8ee3784d2cf 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManagerTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManagerTest.java @@ -20,6 +20,7 @@ import org.hyperledger.besu.ethereum.core.TrieGenerator; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java index 0dcc204426a..18ab5f02fa7 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/SnapWorldDownloadStateTest.java @@ -38,6 +38,7 @@ import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.BytecodeRequest; import org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest; import org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloadProcess; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrieTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrieTest.java index b1b3800a1be..4fcb21d9252 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrieTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/StackTrieTest.java @@ -18,12 +18,15 @@ import org.hyperledger.besu.ethereum.core.TrieGenerator; import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.forest.storage.ForestWorldStateKeyValueStorage; +import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import org.hyperledger.besu.ethereum.worldstate.WorldStateStorageCoordinator; import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; +import java.util.ArrayList; import java.util.List; import java.util.TreeMap; @@ -143,4 +146,127 @@ public void shouldSaveTheRootWhenComplete() { worldStateKeyValueStorage.getAccountStateTrieNode(accountStateTrie.getRootHash())) .isPresent(); } + + @Test + public void shouldNotSaveNodeWithChildNotInTheRange() { + final ForestWorldStateKeyValueStorage worldStateStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final WorldStateStorageCoordinator worldStateStorageCoordinator = + new WorldStateStorageCoordinator(worldStateStorage); + + final MerkleTrie trie = + new StoredMerklePatriciaTrie<>( + (location, hash) -> worldStateStorage.getAccountStateTrieNode(hash).map(Bytes::wrap), + b -> b, + b -> b); + + trie.put(Bytes32.rightPad(Bytes.of(0x10)), Bytes.of(0x01)); + trie.put(Bytes32.rightPad(Bytes.of(0x11)), Bytes.of(0x01)); + trie.put(Bytes32.rightPad(Bytes.of(0x20)), Bytes.of(0x01)); + trie.put(Bytes32.rightPad(Bytes.of(0x21)), Bytes.of(0x01)); + trie.put(Bytes32.rightPad(Bytes.of(0x01)), Bytes.of(0x02)); + trie.put(Bytes32.rightPad(Bytes.of(0x02)), Bytes.of(0x03)); + trie.put(Bytes32.rightPad(Bytes.of(0x03)), Bytes.of(0x04)); + + final ForestWorldStateKeyValueStorage.Updater updater = worldStateStorage.updater(); + trie.commit((location, hash, value) -> updater.putAccountStateTrieNode(hash, value)); + updater.commit(); + + final Bytes32 startRange = Bytes32.rightPad(Bytes.of(0x02)); + + final RangeStorageEntriesCollector collector = + RangeStorageEntriesCollector.createCollector( + startRange, RangeManager.MAX_RANGE, 15, Integer.MAX_VALUE); + final TrieIterator visitor = RangeStorageEntriesCollector.createVisitor(collector); + final TreeMap entries = + (TreeMap) + trie.entriesFrom( + root -> + RangeStorageEntriesCollector.collectEntries( + collector, visitor, root, startRange)); + + final WorldStateProofProvider worldStateProofProvider = + new WorldStateProofProvider(worldStateStorageCoordinator); + + // generate the proof + final List proofs = + worldStateProofProvider.getAccountProofRelatedNodes( + Hash.wrap(trie.getRootHash()), startRange); + proofs.addAll( + worldStateProofProvider.getAccountProofRelatedNodes( + Hash.wrap(trie.getRootHash()), entries.lastKey())); + + // try to commit with stack trie + final ForestWorldStateKeyValueStorage recreatedWorldStateStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final StackTrie stackTrie = new StackTrie(Hash.wrap(trie.getRootHash()), 0, 256, startRange); + stackTrie.addSegment(); + stackTrie.addElement(Bytes32.random(), proofs, entries); + final ForestWorldStateKeyValueStorage.Updater updaterStackTrie = + recreatedWorldStateStorage.updater(); + stackTrie.commit( + (location, hash, value) -> updaterStackTrie.putAccountStateTrieNode(hash, value)); + updaterStackTrie.commit(); + + // verify the state of the db + Assertions.assertThat(worldStateStorage.getAccountStateTrieNode(trie.getRootHash())) + .isPresent(); + Assertions.assertThat(recreatedWorldStateStorage.getAccountStateTrieNode(trie.getRootHash())) + .isNotPresent(); + } + + @Test + public void shouldSaveNodeWithAllChildsInTheRange() { + final ForestWorldStateKeyValueStorage worldStateStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + + final MerkleTrie trie = + new StoredMerklePatriciaTrie<>( + (location, hash) -> worldStateStorage.getAccountStateTrieNode(hash).map(Bytes::wrap), + b -> b, + b -> b); + + trie.put(Bytes32.rightPad(Bytes.of(0x10)), Bytes.of(0x01)); + trie.put(Bytes32.rightPad(Bytes.of(0x11)), Bytes.of(0x01)); + trie.put(Bytes32.rightPad(Bytes.of(0x20)), Bytes.of(0x01)); + trie.put(Bytes32.rightPad(Bytes.of(0x21)), Bytes.of(0x01)); + trie.put(Bytes32.rightPad(Bytes.of(0x01)), Bytes.of(0x02)); + trie.put(Bytes32.rightPad(Bytes.of(0x02)), Bytes.of(0x03)); + trie.put(Bytes32.rightPad(Bytes.of(0x03)), Bytes.of(0x04)); + + final ForestWorldStateKeyValueStorage.Updater updater = worldStateStorage.updater(); + trie.commit((location, hash, value) -> updater.putAccountStateTrieNode(hash, value)); + updater.commit(); + + final Bytes32 startRange = Bytes32.rightPad(Bytes.of(0x00)); + + final RangeStorageEntriesCollector collector = + RangeStorageEntriesCollector.createCollector( + startRange, RangeManager.MAX_RANGE, 15, Integer.MAX_VALUE); + final TrieIterator visitor = RangeStorageEntriesCollector.createVisitor(collector); + final TreeMap entries = + (TreeMap) + trie.entriesFrom( + root -> + RangeStorageEntriesCollector.collectEntries( + collector, visitor, root, startRange)); + + // try to commit with stack trie + final ForestWorldStateKeyValueStorage recreatedWorldStateStorage = + new ForestWorldStateKeyValueStorage(new InMemoryKeyValueStorage()); + final StackTrie stackTrie = new StackTrie(Hash.wrap(trie.getRootHash()), 0, 256, startRange); + stackTrie.addSegment(); + stackTrie.addElement(Bytes32.random(), new ArrayList<>(), entries); + final ForestWorldStateKeyValueStorage.Updater updaterStackTrie = + recreatedWorldStateStorage.updater(); + stackTrie.commit( + (location, hash, value) -> updaterStackTrie.putAccountStateTrieNode(hash, value)); + updaterStackTrie.commit(); + + // verify the state of the db + Assertions.assertThat(worldStateStorage.getAccountStateTrieNode(trie.getRootHash())) + .isPresent(); + Assertions.assertThat(recreatedWorldStateStorage.getAccountStateTrieNode(trie.getRootHash())) + .isPresent(); + } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java index 63895f4e6c5..9b5d9355168 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/TaskGenerator.java @@ -24,6 +24,7 @@ import org.hyperledger.besu.ethereum.proof.WorldStateProofProvider; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.trie.MerkleTrie; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java index 2799d631926..6c3ba6f0daa 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/AccountFlatDatabaseHealingRangeRequestTest.java @@ -17,7 +17,6 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.core.TrieGenerator; -import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncMetricsManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; @@ -27,6 +26,7 @@ import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.CompactEncoding; import org.hyperledger.besu.ethereum.trie.MerkleTrie; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java index 41d02651308..eb6eedfb4c4 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/request/heal/StorageFlatDatabaseHealingRangeRequestTest.java @@ -20,7 +20,6 @@ import org.hyperledger.besu.datatypes.Hash; import org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider; import org.hyperledger.besu.ethereum.core.TrieGenerator; -import org.hyperledger.besu.ethereum.eth.sync.snapsync.RangeManager; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncProcessState; import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldDownloadState; @@ -29,6 +28,7 @@ import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.trie.MerkleTrie; +import org.hyperledger.besu.ethereum.trie.RangeManager; import org.hyperledger.besu.ethereum.trie.RangeStorageEntriesCollector; import org.hyperledger.besu.ethereum.trie.TrieIterator; import org.hyperledger.besu.ethereum.trie.bonsai.storage.BonsaiWorldStateKeyValueStorage; diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/InnerNodeDiscoveryManager.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/InnerNodeDiscoveryManager.java index 5190ada7ef9..d20b47e9e59 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/InnerNodeDiscoveryManager.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/InnerNodeDiscoveryManager.java @@ -14,6 +14,9 @@ */ package org.hyperledger.besu.ethereum.trie; +import static org.hyperledger.besu.ethereum.trie.RangeManager.createPath; +import static org.hyperledger.besu.ethereum.trie.RangeManager.isInRange; + import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.trie.patricia.BranchNode; import org.hyperledger.besu.ethereum.trie.patricia.ExtensionNode; @@ -21,7 +24,6 @@ import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.function.Function; @@ -37,7 +39,7 @@ public class InnerNodeDiscoveryManager extends StoredNodeFactory { private final List innerNodes = new ArrayList<>(); - private final Bytes startKeyHash, endKeyHash; + private final Bytes startKeyPath, endKeyPath; private final boolean allowMissingElementInRange; @@ -49,8 +51,8 @@ public InnerNodeDiscoveryManager( final Bytes32 endKeyHash, final boolean allowMissingElementInRange) { super(nodeLoader, valueSerializer, valueDeserializer); - this.startKeyHash = createPath(startKeyHash); - this.endKeyHash = createPath(endKeyHash); + this.startKeyPath = createPath(startKeyHash); + this.endKeyPath = createPath(endKeyHash); this.allowMissingElementInRange = allowMissingElementInRange; } @@ -62,7 +64,7 @@ protected Node decodeExtension( final Supplier errMessage) { final ExtensionNode vNode = (ExtensionNode) super.decodeExtension(location, path, valueRlp, errMessage); - if (isInRange(Bytes.concatenate(location, Bytes.of(0)))) { + if (isInRange(Bytes.concatenate(location, Bytes.of(0)), startKeyPath, endKeyPath)) { innerNodes.add( ImmutableInnerNode.builder() .location(location) @@ -78,7 +80,7 @@ protected BranchNode decodeBranch( final BranchNode vBranchNode = super.decodeBranch(location, nodeRLPs, errMessage); final List> children = vBranchNode.getChildren(); for (int i = 0; i < children.size(); i++) { - if (isInRange(Bytes.concatenate(location, Bytes.of(i)))) { + if (isInRange(Bytes.concatenate(location, Bytes.of(i)), startKeyPath, endKeyPath)) { innerNodes.add( ImmutableInnerNode.builder() .location(location) @@ -97,7 +99,7 @@ protected LeafNode decodeLeaf( final Supplier errMessage) { final LeafNode vLeafNode = super.decodeLeaf(location, path, valueRlp, errMessage); final Bytes concatenatePath = Bytes.concatenate(location, path); - if (isInRange(concatenatePath.slice(0, concatenatePath.size() - 1))) { + if (isInRange(concatenatePath.slice(0, concatenatePath.size() - 1), startKeyPath, endKeyPath)) { innerNodes.add(ImmutableInnerNode.builder().location(location).path(path).build()); } return vLeafNode; @@ -106,10 +108,16 @@ protected LeafNode decodeLeaf( @Override public Optional> retrieve(final Bytes location, final Bytes32 hash) throws MerkleTrieException { + return super.retrieve(location, hash) + .map( + vNode -> { + vNode.markDirty(); + return vNode; + }) .or( () -> { - if (!allowMissingElementInRange && isInRange(location)) { + if (!allowMissingElementInRange && isInRange(location, startKeyPath, endKeyPath)) { return Optional.empty(); } return Optional.of(new MissingNode<>(hash, location)); @@ -120,23 +128,6 @@ public List getInnerNodes() { return List.copyOf(innerNodes); } - private boolean isInRange(final Bytes location) { - return !location.isEmpty() - && Arrays.compare(location.toArrayUnsafe(), startKeyHash.toArrayUnsafe()) >= 0 - && Arrays.compare(location.toArrayUnsafe(), endKeyHash.toArrayUnsafe()) <= 0; - } - - private Bytes createPath(final Bytes bytes) { - final MutableBytes path = MutableBytes.create(bytes.size() * 2); - int j = 0; - for (int i = 0; i < bytes.size(); i += 1, j += 2) { - final byte b = bytes.get(i); - path.set(j, (byte) ((b >>> 4) & 0x0f)); - path.set(j + 1, (byte) (b & 0x0f)); - } - return path; - } - public static Bytes32 decodePath(final Bytes bytes) { final MutableBytes32 decoded = MutableBytes32.create(); final MutableBytes path = MutableBytes.create(Bytes32.SIZE * 2); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManager.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/RangeManager.java similarity index 74% rename from ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManager.java rename to ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/RangeManager.java index 83f58c10fbf..dbc7c10932a 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/snapsync/RangeManager.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/RangeManager.java @@ -12,14 +12,13 @@ * * SPDX-License-Identifier: Apache-2.0 */ -package org.hyperledger.besu.ethereum.eth.sync.snapsync; +package org.hyperledger.besu.ethereum.trie; import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.trie.InnerNodeDiscoveryManager; -import org.hyperledger.besu.ethereum.trie.MerkleTrieException; import org.hyperledger.besu.ethereum.trie.patricia.StoredMerklePatriciaTrie; import java.math.BigInteger; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -29,6 +28,7 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.bytes.MutableBytes; /** * This class helps to generate ranges according to several parameters (the start and the end of the @@ -152,4 +152,42 @@ public static Optional findNewBeginElementInRange( private static Bytes32 format(final BigInteger data) { return Bytes32.leftPad(Bytes.of(data.toByteArray()).trimLeadingZeros()); } + + /** + * Checks if a given location is within a specified range. This method determines whether a given + * location (represented as {@link Bytes}) falls within the range defined by a start key path and + * an end key path. + * + * @param location The location to check, represented as {@link Bytes}. + * @param startKeyPath The start of the range as path, represented as {@link Bytes}. + * @param endKeyPath The end of the range as path, represented as {@link Bytes}. + * @return {@code true} if the location is within the range (inclusive); {@code false} otherwise. + */ + public static boolean isInRange( + final Bytes location, final Bytes startKeyPath, final Bytes endKeyPath) { + final MutableBytes path = MutableBytes.create(Bytes32.SIZE * 2); + path.set(0, location); + return !location.isEmpty() + && Arrays.compare(path.toArrayUnsafe(), startKeyPath.toArrayUnsafe()) >= 0 + && Arrays.compare(path.toArrayUnsafe(), endKeyPath.toArrayUnsafe()) <= 0; + } + + /** + * Transforms a list of bytes into a path. This method processes a sequence of bytes, expanding + * each byte into two separate bytes to form a new path. The resulting path will have twice the + * length of the input byte sequence. + * + * @param bytes The byte sequence to be transformed into a path. + * @return A {@link Bytes} object representing the newly formed path. + */ + public static Bytes createPath(final Bytes bytes) { + final MutableBytes path = MutableBytes.create(bytes.size() * 2); + int j = 0; + for (int i = 0; i < bytes.size(); i += 1, j += 2) { + final byte b = bytes.get(i); + path.set(j, (byte) ((b >>> 4) & 0x0f)); + path.set(j + 1, (byte) (b & 0x0f)); + } + return path; + } } diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/SnapCommitVisitor.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/SnapCommitVisitor.java new file mode 100644 index 00000000000..099674e226b --- /dev/null +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/SnapCommitVisitor.java @@ -0,0 +1,138 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie; + +import static org.hyperledger.besu.ethereum.trie.RangeManager.createPath; + +import org.hyperledger.besu.ethereum.trie.patricia.BranchNode; +import org.hyperledger.besu.ethereum.trie.patricia.ExtensionNode; + +import java.util.Arrays; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.bytes.MutableBytes; + +/** + * Implements a visitor for persisting changes to nodes during a snap synchronization process, + * focusing specifically on nodes that are marked as "dirty" but not as "heal needed". + * + *

This visitor plays a crucial role in the snap synchronization by identifying nodes that have + * been modified and require persistence ("dirty" nodes). The key functionality of this visitor is + * its selective persistence approach: it only persists changes to dirty nodes that are not marked + * as "heal needed". This strategy is designed to prevent future inconsistencies within the tree by + * ensuring that only nodes that are both modified and currently consistent with the rest of the + * structure are persisted. Nodes marked as "heal needed" are excluded from immediate persistence to + * allow for their proper healing in a controlled manner, ensuring the integrity and consistency of + * the data structure. + */ +public class SnapCommitVisitor extends CommitVisitor implements LocationNodeVisitor { + + private final Bytes startKeyPath; + private final Bytes endKeyPath; + + public SnapCommitVisitor( + final NodeUpdater nodeUpdater, final Bytes32 startKeyHash, final Bytes32 endKeyHash) { + super(nodeUpdater); + this.startKeyPath = createPath(startKeyHash); + this.endKeyPath = createPath(endKeyHash); + } + + /** + * Visits an extension node during a traversal operation. + * + *

This method is called when visiting an extension node. It checks if the node is marked as + * "dirty" (indicating changes that have not been persisted). If the node is clean, the method + * returns immediately. For dirty nodes, it recursively visits any dirty child nodes, + * concatenating the current location with the extension node's path to form the full path to the + * child. + * + *

Additionally, it checks if the child node requires healing (e.g., if it's falls outside the + * specified range defined by {@code startKeyPath} and {@code endKeyPath}). If healing is needed, + * the extension node is marked accordingly. + * + *

Finally, it attempts to persist the extension node if applicable. + * + * @param location The current location represented as {@link Bytes}. + * @param extensionNode The extension node being visited. + */ + @Override + public void visit(final Bytes location, final ExtensionNode extensionNode) { + if (!extensionNode.isDirty()) { + return; + } + + final Node child = extensionNode.getChild(); + if (child.isDirty()) { + child.accept(Bytes.concatenate(location, extensionNode.getPath()), this); + } + if (child.isHealNeeded() + || !isInRange( + Bytes.concatenate(location, extensionNode.getPath()), startKeyPath, endKeyPath)) { + extensionNode.markHealNeeded(); // not save an incomplete node + } + + maybeStoreNode(location, extensionNode); + } + + /** + * Visits a branch node during a traversal operation. + * + *

This method is invoked when visiting a branch node. It first checks if the branch node is + * marked as "dirty" (indicating changes that have not been persisted). If the node is clean, the + * method returns immediately. + * + *

For dirty branch nodes, it iterates through each child node. For each child, if the child is + * dirty, it recursively visits the child, passing along the concatenated path (current location + * plus the child's index) to the child's accept method. + * + *

Additionally, it checks if the child node requires healing (e.g., if it's falls outside the + * specified range of interest defined by {@code startKeyPath} and {@code endKeyPath}). If healing + * is needed, the branch node is marked accordingly. + * + *

Finally, it attempts to persist the branch node if applicable. + * + * @param location The current location represented as {@link Bytes}. + * @param branchNode The branch node being visited. + */ + @Override + public void visit(final Bytes location, final BranchNode branchNode) { + if (!branchNode.isDirty()) { + return; + } + + for (int i = 0; i < branchNode.maxChild(); ++i) { + Bytes index = Bytes.of(i); + final Node child = branchNode.child((byte) i); + if (child.isDirty()) { + child.accept(Bytes.concatenate(location, index), this); + } + if (child.isHealNeeded() + || !isInRange(Bytes.concatenate(location, index), startKeyPath, endKeyPath)) { + branchNode.markHealNeeded(); // not save an incomplete node + } + } + + maybeStoreNode(location, branchNode); + } + + private boolean isInRange( + final Bytes location, final Bytes startKeyPath, final Bytes endKeyPath) { + final MutableBytes path = MutableBytes.create(Bytes32.SIZE * 2); + path.set(0, location); + return Arrays.compare(path.toArrayUnsafe(), startKeyPath.toArrayUnsafe()) >= 0 + && Arrays.compare(path.toArrayUnsafe(), endKeyPath.toArrayUnsafe()) <= 0; + } +} diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/SnapPutVisitor.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/SnapPutVisitor.java deleted file mode 100644 index 0cce55dcf4a..00000000000 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/SnapPutVisitor.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright Hyperledger Besu Contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.trie; - -import org.hyperledger.besu.ethereum.trie.patricia.BranchNode; -import org.hyperledger.besu.ethereum.trie.patricia.ExtensionNode; -import org.hyperledger.besu.ethereum.trie.patricia.PutVisitor; - -import org.apache.tuweni.bytes.Bytes; - -public class SnapPutVisitor extends PutVisitor { - - public SnapPutVisitor(final NodeFactory nodeFactory, final V value) { - super(nodeFactory, value); - } - - @Override - public Node visit(final BranchNode branchNode, final Bytes path) { - final Node visit = super.visit(branchNode, path); - for (Node child : visit.getChildren()) { - if (child.isHealNeeded() || (child instanceof StoredNode && child.getValue().isEmpty())) { - visit.markHealNeeded(); // not save an incomplete node - return visit; - } - } - return visit; - } - - @Override - public Node visit(final ExtensionNode extensionNode, final Bytes path) { - final Node visit = super.visit(extensionNode, path); - for (Node child : visit.getChildren()) { - if (child.isHealNeeded() || (child instanceof StoredNode && child.getValue().isEmpty())) { - visit.markHealNeeded(); // not save an incomplete node - return visit; - } - } - return visit; - } -} diff --git a/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/SnapCommitVisitorTest.java b/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/SnapCommitVisitorTest.java new file mode 100644 index 00000000000..5b281529893 --- /dev/null +++ b/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/SnapCommitVisitorTest.java @@ -0,0 +1,127 @@ +/* + * Copyright Hyperledger Besu Contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie; + +import static org.mockito.Mockito.mock; + +import org.hyperledger.besu.datatypes.Hash; +import org.hyperledger.besu.ethereum.trie.patricia.BranchNode; +import org.hyperledger.besu.ethereum.trie.patricia.ExtensionNode; +import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory; + +import java.util.ArrayList; +import java.util.Optional; +import java.util.function.Function; + +import org.apache.tuweni.bytes.Bytes; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +@SuppressWarnings("unchecked") +public class SnapCommitVisitorTest { + + @Test + public void shouldDetectValidBranch() { + final StoredNodeFactory storedNodeFactory = mock(StoredNodeFactory.class); + final ArrayList> children = new ArrayList<>(); + for (int i = 0; i < 16; i++) { + children.add( + new StoredNode<>( + storedNodeFactory, Bytes.concatenate(Bytes.of(0x00), Bytes.of(i)), Hash.ZERO)); + } + final BranchNode validBranchNode = + new BranchNode<>( + Bytes.of(0x00), + children, + Optional.of(Bytes.of(0x00)), + storedNodeFactory, + Function.identity()); + validBranchNode.markDirty(); + final SnapCommitVisitor snapCommitVisitor = + new SnapCommitVisitor<>( + (location, hash, value) -> {}, RangeManager.MIN_RANGE, RangeManager.MAX_RANGE); + Assertions.assertThat(validBranchNode.isHealNeeded()).isFalse(); + snapCommitVisitor.visit(validBranchNode.getLocation().get(), validBranchNode); + Assertions.assertThat(validBranchNode.isHealNeeded()).isFalse(); + } + + @Test + public void shouldDetectBranchWithChildrenNotInTheRange() { + final StoredNodeFactory storedNodeFactory = mock(StoredNodeFactory.class); + final ArrayList> children = new ArrayList<>(); + for (int i = 0; i < 16; i++) { + children.add( + new StoredNode<>( + storedNodeFactory, Bytes.concatenate(Bytes.of(0x01), Bytes.of(i)), Hash.ZERO)); + } + + final BranchNode invalidBranchNode = + new BranchNode<>( + Bytes.of(0x01), + children, + Optional.of(Bytes.of(0x00)), + storedNodeFactory, + Function.identity()); + invalidBranchNode.markDirty(); + final SnapCommitVisitor snapCommitVisitor = + new SnapCommitVisitor<>( + (location, hash, value) -> {}, + RangeManager.MIN_RANGE, + Hash.fromHexString( + "0x1effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")); + Assertions.assertThat(invalidBranchNode.isHealNeeded()).isFalse(); + snapCommitVisitor.visit(invalidBranchNode.getLocation().get(), invalidBranchNode); + Assertions.assertThat(invalidBranchNode.isHealNeeded()).isTrue(); + } + + @Test + public void shouldDetectValidExtension() { + final StoredNodeFactory storedNodeFactory = mock(StoredNodeFactory.class); + final ExtensionNode validExtensionNode = + new ExtensionNode<>( + Bytes.of(0x00), + Bytes.of(0x01), + new StoredNode<>(storedNodeFactory, Bytes.of((byte) 0x00, (byte) 0x01), Hash.ZERO), + storedNodeFactory); + validExtensionNode.markDirty(); + final SnapCommitVisitor snapCommitVisitor = + new SnapCommitVisitor<>( + (location, hash, value) -> {}, RangeManager.MIN_RANGE, RangeManager.MAX_RANGE); + Assertions.assertThat(validExtensionNode.isHealNeeded()).isFalse(); + snapCommitVisitor.visit(validExtensionNode.getLocation().get(), validExtensionNode); + Assertions.assertThat(validExtensionNode.isHealNeeded()).isFalse(); + } + + @Test + public void shouldDetectExtensionWithChildNotInRange() { + final StoredNodeFactory storedNodeFactory = mock(StoredNodeFactory.class); + final ExtensionNode inValidExtensionNode = + new ExtensionNode<>( + Bytes.of(0x00), + Bytes.of(0x03), + new StoredNode<>(storedNodeFactory, Bytes.of((byte) 0x00, (byte) 0x03), Hash.ZERO), + storedNodeFactory); + inValidExtensionNode.markDirty(); + final SnapCommitVisitor snapCommitVisitor = + new SnapCommitVisitor<>( + (location, hash, value) -> {}, + RangeManager.MIN_RANGE, + Hash.fromHexString( + "0x02ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")); + Assertions.assertThat(inValidExtensionNode.isHealNeeded()).isFalse(); + snapCommitVisitor.visit(inValidExtensionNode.getLocation().get(), inValidExtensionNode); + Assertions.assertThat(inValidExtensionNode.isHealNeeded()).isTrue(); + } +} diff --git a/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/SnapPutVisitorTest.java b/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/SnapPutVisitorTest.java deleted file mode 100644 index cbedbe9a288..00000000000 --- a/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/SnapPutVisitorTest.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright Hyperledger Besu Contributors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ -package org.hyperledger.besu.ethereum.trie; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyByte; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import org.hyperledger.besu.datatypes.Hash; -import org.hyperledger.besu.ethereum.trie.patricia.BranchNode; -import org.hyperledger.besu.ethereum.trie.patricia.ExtensionNode; -import org.hyperledger.besu.ethereum.trie.patricia.LeafNode; -import org.hyperledger.besu.ethereum.trie.patricia.StoredNodeFactory; - -import java.util.ArrayList; -import java.util.Optional; -import java.util.function.Function; - -import org.apache.tuweni.bytes.Bytes; -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; - -@SuppressWarnings("unchecked") -public class SnapPutVisitorTest { - - @Test - public void shouldDetectValidBranch() { - final StoredNodeFactory storedNodeFactory = mock(StoredNodeFactory.class); - when(storedNodeFactory.createBranch(any(), any())) - .thenReturn( - new LeafNode( - Bytes.EMPTY, Bytes.of(0x00), storedNodeFactory, Function.identity())); - final ArrayList> children = new ArrayList<>(); - for (int i = 0; i < 16; i++) { - children.add(new StoredNode<>(storedNodeFactory, Bytes.EMPTY, Hash.ZERO)); - } - final BranchNode invalidBranchNode = - new BranchNode<>( - Bytes.EMPTY, - children, - Optional.of(Bytes.of(0x00)), - storedNodeFactory, - Function.identity()); - final SnapPutVisitor snapPutVisitor = - new SnapPutVisitor<>(storedNodeFactory, Bytes.EMPTY); - Node visit = - snapPutVisitor.visit(invalidBranchNode, Bytes.of(CompactEncoding.LEAF_TERMINATOR)); - Assertions.assertThat(visit.isHealNeeded()).isFalse(); - } - - @Test - public void shouldDetectBranchWithMissingChildren() { - final StoredNodeFactory storedNodeFactory = mock(StoredNodeFactory.class); - when(storedNodeFactory.createBranch(any(), any())) - .thenReturn(new MissingNode<>(Hash.ZERO, Bytes.EMPTY)); - final ArrayList> children = new ArrayList<>(); - for (int i = 0; i < 16; i++) { - children.add(new StoredNode<>(storedNodeFactory, Bytes.EMPTY, Hash.ZERO)); - } - final BranchNode invalidBranchNode = - new BranchNode<>( - Bytes.EMPTY, - children, - Optional.of(Bytes.of(0x00)), - storedNodeFactory, - Function.identity()); - final SnapPutVisitor snapPutVisitor = - new SnapPutVisitor<>(storedNodeFactory, Bytes.EMPTY); - Node visit = - snapPutVisitor.visit(invalidBranchNode, Bytes.of(CompactEncoding.LEAF_TERMINATOR)); - Assertions.assertThat(visit.isHealNeeded()).isTrue(); - } - - @Test - public void shouldDetectValidExtension() { - final StoredNodeFactory storedNodeFactory = mock(StoredNodeFactory.class); - when(storedNodeFactory.createBranch(any(), any())) - .thenReturn( - new LeafNode<>(Bytes.EMPTY, Bytes.of(0x00), storedNodeFactory, Function.identity())); - final ArrayList> children = new ArrayList<>(); - for (int i = 0; i < 16; i++) { - children.add(new StoredNode<>(storedNodeFactory, Bytes.EMPTY, Hash.ZERO)); - } - final BranchNode invalidBranchNode = - new BranchNode<>( - Bytes.EMPTY, - children, - Optional.of(Bytes.of(0x00)), - storedNodeFactory, - Function.identity()); - final SnapPutVisitor snapPutVisitor = - new SnapPutVisitor<>(storedNodeFactory, Bytes.EMPTY); - Node visit = - snapPutVisitor.visit(invalidBranchNode, Bytes.of(CompactEncoding.LEAF_TERMINATOR)); - Assertions.assertThat(visit.isHealNeeded()).isFalse(); - } - - @Test - public void shouldDetectExtensionWithMissingChildren() { - final StoredNodeFactory storedNodeFactory = mock(StoredNodeFactory.class); - when(storedNodeFactory.createBranch(anyByte(), any(), anyByte(), any())) - .thenReturn(new MissingNode<>(Hash.ZERO, Bytes.EMPTY)); - when(storedNodeFactory.createLeaf(any(), any())) - .thenReturn(new MissingNode<>(Hash.ZERO, Bytes.EMPTY)); - final ExtensionNode invalidBranchNode = - new ExtensionNode<>( - Bytes.of(0x00), - new StoredNode<>(storedNodeFactory, Bytes.EMPTY, Hash.ZERO), - storedNodeFactory); - final SnapPutVisitor snapPutVisitor = - new SnapPutVisitor<>(storedNodeFactory, Bytes.EMPTY); - Node visit = - snapPutVisitor.visit(invalidBranchNode, Bytes.of(CompactEncoding.LEAF_TERMINATOR)); - Assertions.assertThat(visit.isHealNeeded()).isTrue(); - } -} From e5c04463a15f5138b2daf2bf995bcd1ff3c34ea7 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Tue, 26 Mar 2024 11:01:01 -0400 Subject: [PATCH 17/18] also fail on cancelled and skipped Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 13 +++++++++++-- .github/workflows/pre-review.yml | 13 +++++++++++-- .github/workflows/reference-tests.yml | 14 +++++++++++--- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 0d241424944..74ba75c2f91 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -77,6 +77,15 @@ jobs: permissions: checks: write statuses: write + if: always() steps: - - name: consolidation - run: echo "consolidating statuses" + # Fail if any `needs` job was not a success. + # Along with `if: always()`, this allows this job to act as a single required status check for the entire workflow. + - name: Fail on workflow error + run: exit 1 + if: >- + ${{ + contains(needs.*.result, 'failure') + || contains(needs.*.result, 'cancelled') + || contains(needs.*.result, 'skipped') + }} \ No newline at end of file diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index e7dc683dea4..1fa7d3d31d8 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -135,6 +135,15 @@ jobs: permissions: checks: write statuses: write + if: always() steps: - - name: consolidation - run: echo "consolidating statuses" + # Fail if any `needs` job was not a success. + # Along with `if: always()`, this allows this job to act as a single required status check for the entire workflow. + - name: Fail on workflow error + run: exit 1 + if: >- + ${{ + contains(needs.*.result, 'failure') + || contains(needs.*.result, 'cancelled') + || contains(needs.*.result, 'skipped') + }} diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index c83022526e4..68310944af4 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -69,7 +69,15 @@ jobs: permissions: checks: write statuses: write + if: always() steps: - - name: consolidation - run: echo "consolidating statuses" - + # Fail if any `needs` job was not a success. + # Along with `if: always()`, this allows this job to act as a single required status check for the entire workflow. + - name: Fail on workflow error + run: exit 1 + if: >- + ${{ + contains(needs.*.result, 'failure') + || contains(needs.*.result, 'cancelled') + || contains(needs.*.result, 'skipped') + }} From bb647d14432f39177899d02d4ead5a51645dccff Mon Sep 17 00:00:00 2001 From: ahamlat Date: Tue, 26 Mar 2024 17:25:15 +0100 Subject: [PATCH 18/18] Modify the message when the selection of transactions is interrupted because it reached the maximum configured duration (#6814) Signed-off-by: Ameziane H --- .../blockcreation/txselection/BlockTransactionSelector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java index b37cff7e43a..c12061f4cbe 100644 --- a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java +++ b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/txselection/BlockTransactionSelector.java @@ -192,7 +192,7 @@ private void timeLimitedSelection() { isTimeout.set(true); } LOG.warn( - "Interrupting transaction selection since it is taking more than the max configured time of " + "Interrupting the selection of transactions for block inclusion as it exceeds the maximum configured duration of " + blockTxsSelectionMaxTime + "ms", e);