diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d1039228b8..3f0d8d0eb8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ From v1.2, Besu requires Java 11. Besu on Java 8 is no longer supported. In v1.2, we removed the entry-point script from our Docker image. Refer to the [migration guide](https://besu.hyperledger.org/en/latest/HowTo/Get-Started/Migration-Docker/) for information on options that were previously automatically added to the Besu command line. -### 1.3 RC +### 1.3 ### Breaking Change @@ -26,6 +26,11 @@ for information on options that were previously automatically added to the Besu - Added [`retesteth`](https://besu.hyperledger.org/en/latest/Reference/CLI/CLI-Subcommands/#retesteth) subcommand - Added [`debug_accountRange`](https://besu.hyperledger.org/en/latest/Reference/API-Methods/#debug_accountrange) JSON-RPC API method - Clarified purpose of [static nodes](https://besu.hyperledger.org/en/latest/HowTo/Find-and-Connect/Managing-Peers/#static-nodes) + - Added links [Kubernetes reference implementations](https://besu.hyperledger.org/en/latest/HowTo/Deploy/Kubernetes/) + - Added content about [access between private and public states](https://besu.hyperledger.org/en/latest/Concepts/Privacy/Privacy-Groups/#access-between-states) + - Added restriction that [account permissioning cannot be used with random key signing](https://besu.hyperledger.org/en/latest/HowTo/Use-Privacy/Sign-Privacy-Marker-Transactions/). + - Added high availability requirement for [private transaction manager](https://besu.hyperledger.org/en/latest/Concepts/Privacy/Privacy-Overview/#availability) (ie, Orion) + - Added [genesis file reference](https://besu.hyperledger.org/en/latest/Reference/Config-Items/) ### Technical Improvements diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyAcceptanceTestBase.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyAcceptanceTestBase.java index 3fbb116b274..dfa3c0fcc36 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyAcceptanceTestBase.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyAcceptanceTestBase.java @@ -22,6 +22,7 @@ import org.hyperledger.besu.tests.acceptance.dsl.privacy.contract.PrivateContractTransactions; import org.hyperledger.besu.tests.acceptance.dsl.privacy.transaction.PrivacyTransactions; import org.hyperledger.besu.tests.acceptance.dsl.transaction.contract.ContractTransactions; +import org.hyperledger.besu.tests.acceptance.dsl.transaction.eth.EthTransactions; import org.hyperledger.besu.tests.acceptance.dsl.transaction.net.NetTransactions; import org.junit.After; @@ -40,8 +41,10 @@ public class PrivacyAcceptanceTestBase { protected final PrivacyAccountResolver privacyAccountResolver; protected final ContractTransactions contractTransactions; protected final NetConditions net; + protected final EthTransactions ethTransactions; public PrivacyAcceptanceTestBase() { + ethTransactions = new EthTransactions(); net = new NetConditions(new NetTransactions()); privacyTransactions = new PrivacyTransactions(); privateContractVerifier = new PrivateContractVerifier(); diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/transaction/PrivDistributeTransactionTransaction.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/transaction/PrivDistributeTransactionTransaction.java new file mode 100644 index 00000000000..fad9a3dcabb --- /dev/null +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/transaction/PrivDistributeTransactionTransaction.java @@ -0,0 +1,40 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.tests.acceptance.dsl.privacy.transaction; + +import org.hyperledger.besu.tests.acceptance.dsl.transaction.NodeRequests; +import org.hyperledger.besu.tests.acceptance.dsl.transaction.Transaction; + +import java.io.IOException; + +public class PrivDistributeTransactionTransaction implements Transaction { + private String signedPrivateTransaction; + + public PrivDistributeTransactionTransaction(final String signedPrivateTransaction) { + this.signedPrivateTransaction = signedPrivateTransaction; + } + + @Override + public String execute(final NodeRequests node) { + try { + return node.privacy() + .privDistributeTransaction(signedPrivateTransaction) + .send() + .getTransactionKey(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/transaction/PrivacyTransactions.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/transaction/PrivacyTransactions.java index 7c742b91ae3..aa27b83e9cc 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/transaction/PrivacyTransactions.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/transaction/PrivacyTransactions.java @@ -34,4 +34,9 @@ public CreatePrivacyGroupTransaction createPrivacyGroup( public FindPrivacyGroupTransaction findPrivacyGroup(final List nodes) { return new FindPrivacyGroupTransaction(nodes); } + + public PrivDistributeTransactionTransaction privDistributeTransaction( + final String signedPrivateTransaction) { + return new PrivDistributeTransactionTransaction(signedPrivateTransaction); + } } diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/privacy/PrivacyRequestFactory.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/privacy/PrivacyRequestFactory.java index 573a86981f7..3635f823edf 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/privacy/PrivacyRequestFactory.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/privacy/PrivacyRequestFactory.java @@ -14,17 +14,41 @@ */ package org.hyperledger.besu.tests.acceptance.dsl.transaction.privacy; +import static java.util.Collections.singletonList; + import org.web3j.protocol.Web3jService; +import org.web3j.protocol.core.Request; +import org.web3j.protocol.core.Response; import org.web3j.protocol.pantheon.Pantheon; public class PrivacyRequestFactory { private final Pantheon besuClient; + private final Web3jService web3jService; public PrivacyRequestFactory(final Web3jService web3jService) { + this.web3jService = web3jService; this.besuClient = Pantheon.build(web3jService); } public Pantheon getBesuClient() { return besuClient; } + + public Request privDistributeTransaction( + final String signedPrivateTransaction) { + return new Request<>( + "priv_distributeRawTransaction", + singletonList(signedPrivateTransaction), + web3jService, + PrivDistributeTransactionResponse.class); + } + + public static class PrivDistributeTransactionResponse extends Response { + + public PrivDistributeTransactionResponse() {} + + public String getTransactionKey() { + return getResult(); + } + } } diff --git a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/web3j/privacy/PrivacyClusterAcceptanceTest.java b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/web3j/privacy/PrivacyClusterAcceptanceTest.java index 5f07136b089..2a7e19cd079 100644 --- a/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/web3j/privacy/PrivacyClusterAcceptanceTest.java +++ b/acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/web3j/privacy/PrivacyClusterAcceptanceTest.java @@ -14,20 +14,40 @@ */ package org.hyperledger.besu.tests.web3j.privacy; +import static org.assertj.core.api.Assertions.assertThat; + +import org.hyperledger.besu.enclave.Enclave; +import org.hyperledger.besu.enclave.types.ReceiveRequest; +import org.hyperledger.besu.enclave.types.ReceiveResponse; +import org.hyperledger.besu.ethereum.core.Address; import org.hyperledger.besu.tests.acceptance.dsl.privacy.PrivacyAcceptanceTestBase; import org.hyperledger.besu.tests.acceptance.dsl.privacy.PrivacyNode; import org.hyperledger.besu.tests.web3j.generated.EventEmitter; +import org.hyperledger.besu.util.bytes.BytesValue; +import org.hyperledger.besu.util.bytes.BytesValues; import java.math.BigInteger; +import java.util.Collections; import org.junit.Before; import org.junit.Test; +import org.web3j.crypto.Credentials; +import org.web3j.crypto.RawTransaction; +import org.web3j.crypto.TransactionEncoder; +import org.web3j.protocol.eea.crypto.PrivateTransactionEncoder; +import org.web3j.protocol.eea.crypto.RawPrivateTransaction; import org.web3j.protocol.pantheon.response.privacy.PrivateTransactionReceipt; +import org.web3j.utils.Base64String; +import org.web3j.utils.Numeric; +import org.web3j.utils.Restriction; public class PrivacyClusterAcceptanceTest extends PrivacyAcceptanceTestBase { private static final long POW_CHAIN_ID = 2018; + private static final String eventEmmitterDeployed = + "0x6080604052600436106100565763ffffffff7c01000000000000000000000000000000000000000000000000000000006000350416633fa4f245811461005b5780636057361d1461008257806367e404ce146100ae575b600080fd5b34801561006757600080fd5b506100706100ec565b60408051918252519081900360200190f35b34801561008e57600080fd5b506100ac600480360360208110156100a557600080fd5b50356100f2565b005b3480156100ba57600080fd5b506100c3610151565b6040805173ffffffffffffffffffffffffffffffffffffffff9092168252519081900360200190f35b60025490565b604080513381526020810183905281517fc9db20adedc6cf2b5d25252b101ab03e124902a73fcb12b753f3d1aaa2d8f9f5929181900390910190a16002556001805473ffffffffffffffffffffffffffffffffffffffff191633179055565b60015473ffffffffffffffffffffffffffffffffffffffff169056fea165627a7a72305820c7f729cb24e05c221f5aa913700793994656f233fe2ce3b9fd9a505ea17e8d8a0029"; + private PrivacyNode alice; private PrivacyNode bob; private PrivacyNode charlie; @@ -81,6 +101,78 @@ public void onlyAliceAndBobCanExecuteContract() { charlie.verify(privateTransactionVerifier.noPrivateTransactionReceipt(transactionHash)); } + @Test + public void aliceCanUsePrivDistributeTransaction() { + // Contract address is generated from sender address and transaction nonce + final String contractAddress = "0xebf56429e6500e84442467292183d4d621359838"; + + final RawPrivateTransaction rawPrivateTransaction = + RawPrivateTransaction.createContractTransaction( + BigInteger.ZERO, + BigInteger.ZERO, + BigInteger.ZERO, + Numeric.prependHexPrefix(EventEmitter.BINARY), + Base64String.wrap(alice.getEnclaveKey()), + Collections.singletonList(Base64String.wrap(bob.getEnclaveKey())), + Restriction.RESTRICTED); + + final String signedPrivateTransaction = + Numeric.toHexString( + PrivateTransactionEncoder.signMessage( + rawPrivateTransaction, + POW_CHAIN_ID, + Credentials.create(alice.getTransactionSigningKey()))); + final String transactionKey = + alice.execute(privacyTransactions.privDistributeTransaction(signedPrivateTransaction)); + + final Enclave aliceEnclave = new Enclave(alice.getOrion().clientUrl()); + final ReceiveResponse aliceRR = + aliceEnclave.receive( + new ReceiveRequest( + BytesValues.asBase64String(BytesValue.fromHexString(transactionKey)), + alice.getEnclaveKey())); + + final Enclave bobEnclave = new Enclave(bob.getOrion().clientUrl()); + final ReceiveResponse bobRR = + bobEnclave.receive( + new ReceiveRequest( + BytesValues.asBase64String(BytesValue.fromHexString(transactionKey)), + bob.getEnclaveKey())); + + assertThat(bobRR).isEqualToComparingFieldByField(aliceRR); + + final RawTransaction pmt = + RawTransaction.createTransaction( + BigInteger.ZERO, + BigInteger.valueOf(1000), + BigInteger.valueOf(65000), + Address.DEFAULT_PRIVACY.toString(), + transactionKey); + + final String signedPmt = + Numeric.toHexString( + TransactionEncoder.signMessage( + pmt, POW_CHAIN_ID, Credentials.create(alice.getTransactionSigningKey()))); + + final String transactionHash = alice.execute(ethTransactions.sendRawTransaction(signedPmt)); + + final PrivateTransactionReceipt expectedReceipt = + new PrivateTransactionReceipt( + contractAddress, + "0xfe3b557e8fb62b89f4916b721be55ceb828dbd73", + null, + eventEmmitterDeployed, + Collections.emptyList()); + + alice.verify( + privateTransactionVerifier.validPrivateTransactionReceipt( + transactionHash, expectedReceipt)); + + bob.verify( + privateTransactionVerifier.validPrivateTransactionReceipt( + transactionHash, expectedReceipt)); + } + @Test public void aliceCanDeployMultipleTimesInSingleGroup() { final String firstDeployedAddress = "0xebf56429e6500e84442467292183d4d621359838"; diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 29bad3e6da2..5fae5ba3a75 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -71,6 +71,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis; import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration; import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.Hash; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.core.Wei; @@ -130,6 +131,7 @@ import java.time.Clock; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -589,7 +591,7 @@ void setBannedNodeIds(final List values) { names = {"--pruning-blocks-retained"}, hidden = true, description = - "Number of recent blocks for which to keep entire world state (default: ${DEFAULT-VALUE})", + "Minimum number of recent blocks for which to keep entire world state (default: ${DEFAULT-VALUE})", arity = "1") private final Long pruningBlocksRetained = DEFAULT_PRUNING_BLOCKS_RETAINED; @@ -597,7 +599,7 @@ void setBannedNodeIds(final List values) { names = {"--pruning-block-confirmations"}, hidden = true, description = - "Number of confirmations on a block before marking begins (default: ${DEFAULT-VALUE})", + "Minimum number of confirmations on a block before marking begins (default: ${DEFAULT-VALUE})", arity = "1") private final Long pruningBlockConfirmations = DEFAULT_PRUNING_BLOCK_CONFIRMATIONS; @@ -645,6 +647,14 @@ void setBannedNodeIds(final List values) { "Enable passing the revert reason back through TransactionReceipts (default: ${DEFAULT-VALUE})") private final Boolean isRevertReasonEnabled = false; + @Option( + names = {"--required-blocks", "--required-block"}, + paramLabel = "BLOCK=HASH", + description = "Block number and hash peers are required to have.", + arity = "*", + split = ",") + private final Map requiredBlocks = new HashMap<>(); + @Option( names = {"--privacy-url"}, description = "The URL on which the enclave is running") @@ -835,6 +845,7 @@ private BesuCommand registerConverters() { commandLine.registerConverter(UInt256.class, (arg) -> UInt256.of(new BigInteger(arg))); commandLine.registerConverter(Wei.class, (arg) -> Wei.of(Long.parseUnsignedLong(arg))); commandLine.registerConverter(PositiveNumber.class, PositiveNumber::fromString); + commandLine.registerConverter(Hash.class, Hash::fromHexString); metricCategoryConverter.addCategories(BesuMetricCategory.class); metricCategoryConverter.addCategories(StandardMetricCategory.class); @@ -1085,7 +1096,8 @@ public BesuControllerBuilder getControllerBuilder() { .isPruningEnabled(isPruningEnabled) .pruningConfiguration(buildPruningConfiguration()) .genesisConfigOverrides(genesisConfigOverrides) - .targetGasLimit(targetGasLimit == null ? Optional.empty() : Optional.of(targetGasLimit)); + .targetGasLimit(targetGasLimit == null ? Optional.empty() : Optional.of(targetGasLimit)) + .requiredBlocks(requiredBlocks); } catch (final IOException e) { throw new ExecutionException(this.commandLine, "Invalid path", e); } diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index b786c89c6af..b4baacbf6c9 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.GenesisState; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; +import org.hyperledger.besu.ethereum.core.Hash; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.core.Synchronizer; @@ -34,6 +35,7 @@ import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.peervalidation.DaoForkPeerValidator; import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator; +import org.hyperledger.besu.ethereum.eth.peervalidation.RequiredBlocksPeerValidator; import org.hyperledger.besu.ethereum.eth.sync.DefaultSynchronizer; import org.hyperledger.besu.ethereum.eth.sync.SyncMode; import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration; @@ -72,9 +74,8 @@ public abstract class BesuControllerBuilder { private static final Logger LOG = LogManager.getLogger(); protected GenesisConfigFile genesisConfig; - protected SynchronizerConfiguration syncConfig; - protected EthProtocolManager ethProtocolManager; - protected EthProtocolConfiguration ethereumWireProtocolConfiguration; + SynchronizerConfiguration syncConfig; + EthProtocolConfiguration ethereumWireProtocolConfiguration; protected TransactionPoolConfiguration transactionPoolConfiguration; protected BigInteger networkId; protected MiningParameters miningParameters; @@ -82,14 +83,15 @@ public abstract class BesuControllerBuilder { protected PrivacyParameters privacyParameters; protected Path dataDirectory; protected Clock clock; - protected KeyPair nodeKeys; + KeyPair nodeKeys; protected boolean isRevertReasonEnabled; - protected GasLimitCalculator gasLimitCalculator; + GasLimitCalculator gasLimitCalculator; private StorageProvider storageProvider; private final List shutdownActions = new ArrayList<>(); private boolean isPruningEnabled; private PruningConfiguration pruningConfiguration; Map genesisConfigOverrides; + private Map requiredBlocks = Collections.emptyMap(); public BesuControllerBuilder storageProvider(final StorageProvider storageProvider) { this.storageProvider = storageProvider; @@ -187,6 +189,11 @@ public BesuControllerBuilder targetGasLimit(final Optional targetGasLim return this; } + public BesuControllerBuilder requiredBlocks(final Map requiredBlocks) { + this.requiredBlocks = requiredBlocks; + return this; + } + public BesuController build() { checkNotNull(genesisConfig, "Missing genesis config"); checkNotNull(syncConfig, "Missing sync config"); @@ -253,7 +260,7 @@ public BesuController build() { })); final boolean fastSyncEnabled = syncConfig.getSyncMode().equals(SyncMode.FAST); - ethProtocolManager = + final EthProtocolManager ethProtocolManager = createEthProtocolManager( protocolContext, fastSyncEnabled, createPeerValidators(protocolSchedule)); final SyncState syncState = @@ -335,7 +342,7 @@ protected SubProtocolConfiguration createSubProtocolConfiguration( return new SubProtocolConfiguration().withSubProtocol(EthProtocol.get(), ethProtocolManager); } - protected final void addShutdownAction(final Runnable action) { + final void addShutdownAction(final Runnable action) { shutdownActions.add(action); } @@ -372,7 +379,7 @@ protected EthProtocolManager createEthProtocolManager( ethereumWireProtocolConfiguration); } - protected List createPeerValidators(final ProtocolSchedule protocolSchedule) { + private List createPeerValidators(final ProtocolSchedule protocolSchedule) { final List validators = new ArrayList<>(); final OptionalLong daoBlock = @@ -383,6 +390,12 @@ protected List createPeerValidators(final ProtocolSchedule pro new DaoForkPeerValidator(protocolSchedule, metricsSystem, daoBlock.getAsLong())); } + for (final Map.Entry requiredBlock : requiredBlocks.entrySet()) { + validators.add( + new RequiredBlocksPeerValidator( + protocolSchedule, metricsSystem, requiredBlock.getKey(), requiredBlock.getValue())); + } + return validators; } } diff --git a/besu/src/main/java/org/hyperledger/besu/util/PlatformDetector.java b/besu/src/main/java/org/hyperledger/besu/util/PlatformDetector.java index e9cf06e80f3..a1b72424f86 100644 --- a/besu/src/main/java/org/hyperledger/besu/util/PlatformDetector.java +++ b/besu/src/main/java/org/hyperledger/besu/util/PlatformDetector.java @@ -186,6 +186,9 @@ static String normalizeVM(final String javaVendor, final String javaVmName) { if (javaVendor.contains("amazoncominc")) { return "corretto"; } + if (javaVmName.contains("openjdk")) { + return "openjdk"; + } return "-" + javaVendor + "-" + javaVmName; } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 29ca217a55c..4e71e52149a 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -44,6 +44,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.RpcApi; import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration; import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.Hash; import org.hyperledger.besu.ethereum.core.MiningParameters; import org.hyperledger.besu.ethereum.core.PrivacyParameters; import org.hyperledger.besu.ethereum.core.Wei; @@ -73,6 +74,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -2788,4 +2790,92 @@ public void targetGasLimitIsDisabledWhenNotSpecified() throws Exception { assertThat(targetGasLimitArg.getValue()).isEqualTo(Optional.empty()); } + + @Test + public void requiredBlocksSetWhenSpecified() { + final long blockNumber = 8675309L; + final String hash = "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"; + + parseCommand("--required-block=" + blockNumber + "=" + hash); + + @SuppressWarnings("unchecked") + final ArgumentCaptor> requiredBlocksArg = ArgumentCaptor.forClass(Map.class); + + verify(mockControllerBuilder).requiredBlocks(requiredBlocksArg.capture()); + verify(mockControllerBuilder).build(); + + assertThat(commandOutput.toString()).isEmpty(); + assertThat(commandErrorOutput.toString()).isEmpty(); + + assertThat(requiredBlocksArg.getValue()).containsOnlyKeys(blockNumber); + assertThat(requiredBlocksArg.getValue()) + .containsEntry(blockNumber, Hash.fromHexStringLenient(hash)); + } + + @Test + public void requiredBlocksEmptyWhenNotSpecified() { + parseCommand(); + + @SuppressWarnings("unchecked") + final ArgumentCaptor> requiredBlocksArg = ArgumentCaptor.forClass(Map.class); + + verify(mockControllerBuilder).requiredBlocks(requiredBlocksArg.capture()); + verify(mockControllerBuilder).build(); + + assertThat(commandOutput.toString()).isEmpty(); + assertThat(commandErrorOutput.toString()).isEmpty(); + + assertThat(requiredBlocksArg.getValue()).isEmpty(); + } + + @Test + public void requiredBlocksMulpleBlocksOneArg() { + final long block1 = 8675309L; + final long block2 = 5551212L; + final String hash1 = "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"; + final String hash2 = "0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + parseCommand("--required-block=" + block1 + "=" + hash1 + "," + block2 + "=" + hash2); + + @SuppressWarnings("unchecked") + final ArgumentCaptor> requiredBlocksArg = ArgumentCaptor.forClass(Map.class); + + verify(mockControllerBuilder).requiredBlocks(requiredBlocksArg.capture()); + verify(mockControllerBuilder).build(); + + assertThat(commandOutput.toString()).isEmpty(); + assertThat(commandErrorOutput.toString()).isEmpty(); + + assertThat(requiredBlocksArg.getValue()).containsOnlyKeys(block1, block2); + assertThat(requiredBlocksArg.getValue()) + .containsEntry(block1, Hash.fromHexStringLenient(hash1)); + assertThat(requiredBlocksArg.getValue()) + .containsEntry(block2, Hash.fromHexStringLenient(hash2)); + } + + @Test + public void requiredBlocksMulpleBlocksTwoArgs() { + final long block1 = 8675309L; + final long block2 = 5551212L; + final String hash1 = "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"; + final String hash2 = "0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + parseCommand( + "--required-block=" + block1 + "=" + hash1, "--required-block=" + block2 + "=" + hash2); + + @SuppressWarnings("unchecked") + final ArgumentCaptor> requiredBlocksArg = ArgumentCaptor.forClass(Map.class); + + verify(mockControllerBuilder).requiredBlocks(requiredBlocksArg.capture()); + verify(mockControllerBuilder).build(); + + assertThat(commandOutput.toString()).isEmpty(); + assertThat(commandErrorOutput.toString()).isEmpty(); + + assertThat(requiredBlocksArg.getValue()).containsOnlyKeys(block1, block2); + assertThat(requiredBlocksArg.getValue()) + .containsEntry(block1, Hash.fromHexStringLenient(hash1)); + assertThat(requiredBlocksArg.getValue()) + .containsEntry(block2, Hash.fromHexStringLenient(hash2)); + } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java index 0bf32d917b2..4de6e11bfd7 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java @@ -170,6 +170,7 @@ public void initMocks() throws Exception { when(mockControllerBuilder.pruningConfiguration(any())).thenReturn(mockControllerBuilder); when(mockControllerBuilder.genesisConfigOverrides(any())).thenReturn(mockControllerBuilder); when(mockControllerBuilder.targetGasLimit(any())).thenReturn(mockControllerBuilder); + when(mockControllerBuilder.requiredBlocks(any())).thenReturn(mockControllerBuilder); // doReturn used because of generic BesuController doReturn(mockController).when(mockControllerBuilder).build(); diff --git a/besu/src/test/resources/everything_config.toml b/besu/src/test/resources/everything_config.toml index df223091c88..d77d6bb25bd 100644 --- a/besu/src/test/resources/everything_config.toml +++ b/besu/src/test/resources/everything_config.toml @@ -31,6 +31,7 @@ max-peers=42 remote-connections-limit-enabled=true remote-connections-max-percentage=60 host-whitelist=["all"] +required-blocks=["8675309=123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"] # chain network="MAINNET" diff --git a/build.gradle b/build.gradle index 5b989b8af3f..0a9dff7ad2c 100644 --- a/build.gradle +++ b/build.gradle @@ -268,9 +268,26 @@ allprojects { task deploy() {} +task checkMavenCoordianteCollisions { + doLast { + def coordinates = [:] + getAllprojects().forEach { + if (it.properties.containsKey('publishing')) { + def coordinate = it.publishing?.publications[0].coordinates + if (coordinates.containsKey(coordinate)) { + throw new GradleException("Duplicate maven coordinates detected, ${coordinate} is used by " + + "both ${coordinates[coordinate]} and ${it.path}.\n" + + "Please add a `publishing` script block to one or both subprojects.") + } + coordinates[coordinate] = it.path + } + } + } +} + tasks.register('checkPluginAPIChanges', DefaultTask) { } checkPluginAPIChanges.dependsOn(':plugin-api:checkAPIChanges') -check.dependsOn('checkPluginAPIChanges') +check.dependsOn('checkPluginAPIChanges', 'checkMavenCoordianteCollisions') subprojects { diff --git a/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/worldstate/PrunerIntegrationTest.java b/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/worldstate/PrunerIntegrationTest.java index 4008ef9b667..053a4df3a38 100644 --- a/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/worldstate/PrunerIntegrationTest.java +++ b/ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/worldstate/PrunerIntegrationTest.java @@ -123,6 +123,10 @@ private void testPruner( generateBlockchainData(numBlockInCycle, accountsPerBlock); assertThat(pruner.getState()).isEqualByComparingTo(Pruner.State.IDLE); + // Restarting the Pruner shouldn't matter since we're idle + pruner.stop(); + pruner.start(); + // Collect the nodes we expect to keep final Set expectedNodes = new HashSet<>(); for (int i = fullyMarkedBlockNum; i <= blockchain.getChainHeadBlockNumber(); i++) { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPruner.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPruner.java index 951e265de7d..b240af77664 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPruner.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPruner.java @@ -97,14 +97,12 @@ public MarkSweepPruner( } public void prepare() { - worldStateStorage.removeNodeAddedListener(nodeAddedListenerId); // Just in case. - markStorage.clear(); - pendingMarks.clear(); - nodeAddedListenerId = worldStateStorage.addNodeAddedListener(this::markNodes); - } + // Optimization for the case where the previous cycle was interrupted (like the node was shut + // down). If the previous cycle was interrupted, there will be marks in the mark storage from + // last time, causing the first sweep to be smaller than it needs to be. + clearMarks(); - public void cleanup() { - worldStateStorage.removeNodeAddedListener(nodeAddedListenerId); + nodeAddedListenerId = worldStateStorage.addNodeAddedListener(this::markNodes); } public void mark(final Hash rootHash) { @@ -151,9 +149,18 @@ public void sweepBefore(final long markedBlockNumber) { // Sweep non-state-root nodes prunedNodeCount += worldStateStorage.prune(this::isMarked); sweptNodesCounter.inc(prunedNodeCount); + clearMarks(); + LOG.debug("Completed sweeping unused nodes"); + } + + public void cleanup() { worldStateStorage.removeNodeAddedListener(nodeAddedListenerId); + clearMarks(); + } + + public void clearMarks() { markStorage.clear(); - LOG.debug("Completed sweeping unused nodes"); + pendingMarks.clear(); } private boolean isMarked(final Bytes32 key) { @@ -190,7 +197,14 @@ private void processAccountState(final BytesValue value) { @VisibleForTesting void markNode(final Bytes32 hash) { - markNodes(Collections.singleton(hash)); + markedNodesCounter.inc(); + markLock.lock(); + try { + pendingMarks.add(hash); + maybeFlushPendingMarks(); + } finally { + markLock.unlock(); + } } private void markNodes(final Collection nodeHashes) { @@ -210,7 +224,7 @@ private void maybeFlushPendingMarks() { } } - void flushPendingMarks() { + private void flushPendingMarks() { markLock.lock(); try { final KeyValueStorageTransaction transaction = markStorage.startTransaction(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/Pruner.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/Pruner.java index fb8c7e5116f..aebd1988304 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/Pruner.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/Pruner.java @@ -35,6 +35,7 @@ public class Pruner { private final MarkSweepPruner pruningStrategy; private final Blockchain blockchain; private final ExecutorService executorService; + private Long blockAddedObserverId; private final long blocksRetained; private final AtomicReference state = new AtomicReference<>(State.IDLE); private volatile long markBlockNumber = 0; @@ -58,11 +59,14 @@ public Pruner( public void start() { LOG.info("Starting Pruner."); - blockchain.observeBlockAdded((event, blockchain) -> handleNewBlock(event)); + pruningStrategy.prepare(); + blockAddedObserverId = + blockchain.observeBlockAdded((event, blockchain) -> handleNewBlock(event)); } public void stop() throws InterruptedException { pruningStrategy.cleanup(); + blockchain.removeObserver(blockAddedObserverId); executorService.awaitTermination(10, TimeUnit.SECONDS); } @@ -73,7 +77,6 @@ private void handleNewBlock(final BlockAddedEvent event) { final long blockNumber = event.getBlock().getHeader().getNumber(); if (state.compareAndSet(State.IDLE, State.MARK_BLOCK_CONFIRMATIONS_AWAITING)) { - pruningStrategy.prepare(); markBlockNumber = blockNumber; } else if (blockNumber >= markBlockNumber + blockConfirmations && state.compareAndSet(State.MARK_BLOCK_CONFIRMATIONS_AWAITING, State.MARKING)) { @@ -87,7 +90,6 @@ private void handleNewBlock(final BlockAddedEvent event) { } private void mark(final BlockHeader header) { - markBlockNumber = header.getNumber(); final Hash stateRoot = header.getStateRoot(); LOG.debug( "Begin marking used nodes for pruning. Block number: {} State root: {}", @@ -117,6 +119,7 @@ private void execute(final Runnable action) { executorService.execute(action); } catch (final Throwable t) { LOG.error("Pruning failed", t); + pruningStrategy.cleanup(); state.set(State.IDLE); } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/peervalidation/AbstractPeerBlockValidator.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/peervalidation/AbstractPeerBlockValidator.java new file mode 100644 index 00000000000..c1315a6d98e --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/peervalidation/AbstractPeerBlockValidator.java @@ -0,0 +1,120 @@ +/* + * + * * Copyright 2019 ConsenSys AG. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.eth.peervalidation; + +import static com.google.common.base.Preconditions.checkArgument; + +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.eth.manager.EthContext; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.task.AbstractPeerTask; +import org.hyperledger.besu.ethereum.eth.manager.task.GetHeadersFromPeerByNumberTask; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.plugin.services.MetricsSystem; + +import java.time.Duration; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +abstract class AbstractPeerBlockValidator implements PeerValidator { + private static final Logger LOG = LogManager.getLogger(); + static long DEFAULT_CHAIN_HEIGHT_ESTIMATION_BUFFER = 10L; + + private final ProtocolSchedule protocolSchedule; + private final MetricsSystem metricsSystem; + + final long blockNumber; + // Wait for peer's chainhead to advance some distance beyond blockNumber before validating + private final long chainHeightEstimationBuffer; + + AbstractPeerBlockValidator( + final ProtocolSchedule protocolSchedule, + final MetricsSystem metricsSystem, + final long blockNumber, + final long chainHeightEstimationBuffer) { + checkArgument(chainHeightEstimationBuffer >= 0); + this.protocolSchedule = protocolSchedule; + this.metricsSystem = metricsSystem; + this.blockNumber = blockNumber; + this.chainHeightEstimationBuffer = chainHeightEstimationBuffer; + } + + public AbstractPeerBlockValidator( + final ProtocolSchedule protocolSchedule, + final MetricsSystem metricsSystem, + final long blockNumber) { + this(protocolSchedule, metricsSystem, blockNumber, DEFAULT_CHAIN_HEIGHT_ESTIMATION_BUFFER); + } + + @Override + public CompletableFuture validatePeer( + final EthContext ethContext, final EthPeer ethPeer) { + final AbstractPeerTask> getHeaderTask = + GetHeadersFromPeerByNumberTask.forSingleNumber( + protocolSchedule, ethContext, blockNumber, metricsSystem) + .setTimeout(Duration.ofSeconds(20)) + .assignPeer(ethPeer); + return getHeaderTask + .run() + .handle( + (res, err) -> { + if (err != null) { + // Mark peer as invalid on error + LOG.debug( + "Peer {} is invalid because required block block ({}) is unavailable: {}", + ethPeer, + blockNumber, + err.toString()); + return false; + } + final List headers = res.getResult(); + if (headers.size() == 0) { + // If no headers are returned, fail + LOG.debug( + "Peer {} is invalid because required block ({}) is unavailable.", + ethPeer, + blockNumber); + return false; + } + final BlockHeader header = headers.get(0); + return validateBlockHeader(ethPeer, header); + }); + } + + abstract boolean validateBlockHeader(EthPeer ethPeer, BlockHeader header); + + @Override + public boolean canBeValidated(final EthPeer ethPeer) { + return ethPeer.chainState().getEstimatedHeight() >= (blockNumber + chainHeightEstimationBuffer); + } + + @Override + public Duration nextValidationCheckTimeout(final EthPeer ethPeer) { + if (!ethPeer.chainState().hasEstimatedHeight()) { + return Duration.ofSeconds(30); + } + final long distanceToBlock = blockNumber - ethPeer.chainState().getEstimatedHeight(); + if (distanceToBlock < 100_000L) { + return Duration.ofMinutes(1); + } + // If the peer is trailing behind, give it some time to catch up before trying again. + return Duration.ofMinutes(10); + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/peervalidation/DaoForkPeerValidator.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/peervalidation/DaoForkPeerValidator.java index 35c46c6a153..cd7c18f02e7 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/peervalidation/DaoForkPeerValidator.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/peervalidation/DaoForkPeerValidator.java @@ -14,45 +14,24 @@ */ package org.hyperledger.besu.ethereum.eth.peervalidation; -import static com.google.common.base.Preconditions.checkArgument; - import org.hyperledger.besu.ethereum.core.BlockHeader; -import org.hyperledger.besu.ethereum.eth.manager.EthContext; import org.hyperledger.besu.ethereum.eth.manager.EthPeer; -import org.hyperledger.besu.ethereum.eth.manager.task.AbstractPeerTask; -import org.hyperledger.besu.ethereum.eth.manager.task.GetHeadersFromPeerByNumberTask; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderValidator; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.plugin.services.MetricsSystem; -import java.time.Duration; -import java.util.List; -import java.util.concurrent.CompletableFuture; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -public class DaoForkPeerValidator implements PeerValidator { +public class DaoForkPeerValidator extends AbstractPeerBlockValidator { private static final Logger LOG = LogManager.getLogger(); - private static long DEFAULT_CHAIN_HEIGHT_ESTIMATION_BUFFER = 10L; - - private final ProtocolSchedule protocolSchedule; - private final MetricsSystem metricsSystem; - - private final long daoBlockNumber; - // Wait for peer's chainhead to advance some distance beyond daoBlockNumber before validating - private final long chainHeightEstimationBuffer; DaoForkPeerValidator( final ProtocolSchedule protocolSchedule, final MetricsSystem metricsSystem, final long daoBlockNumber, final long chainHeightEstimationBuffer) { - checkArgument(chainHeightEstimationBuffer >= 0); - this.protocolSchedule = protocolSchedule; - this.metricsSystem = metricsSystem; - this.daoBlockNumber = daoBlockNumber; - this.chainHeightEstimationBuffer = chainHeightEstimationBuffer; + super(protocolSchedule, metricsSystem, daoBlockNumber, chainHeightEstimationBuffer); } public DaoForkPeerValidator( @@ -63,63 +42,11 @@ public DaoForkPeerValidator( } @Override - public CompletableFuture validatePeer( - final EthContext ethContext, final EthPeer ethPeer) { - AbstractPeerTask> getHeaderTask = - GetHeadersFromPeerByNumberTask.forSingleNumber( - protocolSchedule, ethContext, daoBlockNumber, metricsSystem) - .setTimeout(Duration.ofSeconds(20)) - .assignPeer(ethPeer); - return getHeaderTask - .run() - .handle( - (res, err) -> { - if (err != null) { - // Mark peer as invalid on error - LOG.debug( - "Peer {} is invalid because DAO block ({}) is unavailable: {}", - ethPeer, - daoBlockNumber, - err.toString()); - return false; - } - List headers = res.getResult(); - if (headers.size() == 0) { - // If no headers are returned, fail - LOG.debug( - "Peer {} is invalid because DAO block ({}) is unavailable.", - ethPeer, - daoBlockNumber); - return false; - } - BlockHeader header = headers.get(0); - boolean validDaoBlock = MainnetBlockHeaderValidator.validateHeaderForDaoFork(header); - if (!validDaoBlock) { - LOG.debug( - "Peer {} is invalid because DAO block ({}) is invalid.", - ethPeer, - daoBlockNumber); - } - return validDaoBlock; - }); - } - - @Override - public boolean canBeValidated(final EthPeer ethPeer) { - return ethPeer.chainState().getEstimatedHeight() - >= (daoBlockNumber + chainHeightEstimationBuffer); - } - - @Override - public Duration nextValidationCheckTimeout(final EthPeer ethPeer) { - if (!ethPeer.chainState().hasEstimatedHeight()) { - return Duration.ofSeconds(30); - } - long distanceToDaoBlock = daoBlockNumber - ethPeer.chainState().getEstimatedHeight(); - if (distanceToDaoBlock < 100_000L) { - return Duration.ofMinutes(1); + boolean validateBlockHeader(final EthPeer ethPeer, final BlockHeader header) { + final boolean validDaoBlock = MainnetBlockHeaderValidator.validateHeaderForDaoFork(header); + if (!validDaoBlock) { + LOG.debug("Peer {} is invalid because DAO block ({}) is invalid.", ethPeer, blockNumber); } - // If the peer is trailing behind, give it some time to catch up before trying again. - return Duration.ofMinutes(10); + return validDaoBlock; } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/peervalidation/RequiredBlocksPeerValidator.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/peervalidation/RequiredBlocksPeerValidator.java new file mode 100644 index 00000000000..6c0dfb0aeac --- /dev/null +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/peervalidation/RequiredBlocksPeerValidator.java @@ -0,0 +1,64 @@ +/* + * + * * Copyright ConsenSys AG. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.eth.peervalidation; + +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.plugin.services.MetricsSystem; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class RequiredBlocksPeerValidator extends AbstractPeerBlockValidator { + private static final Logger LOG = LogManager.getLogger(); + + private final Hash hash; + + RequiredBlocksPeerValidator( + final ProtocolSchedule protocolSchedule, + final MetricsSystem metricsSystem, + final long blockNumber, + final Hash hash, + final long chainHeightEstimationBuffer) { + super(protocolSchedule, metricsSystem, blockNumber, chainHeightEstimationBuffer); + this.hash = hash; + } + + public RequiredBlocksPeerValidator( + final ProtocolSchedule protocolSchedule, + final MetricsSystem metricsSystem, + final long blockNumber, + final Hash hash) { + this( + protocolSchedule, metricsSystem, blockNumber, hash, DEFAULT_CHAIN_HEIGHT_ESTIMATION_BUFFER); + } + + @Override + boolean validateBlockHeader(final EthPeer ethPeer, final BlockHeader header) { + final boolean validBlock = hash.equals(header.getHash()); + if (!validBlock) { + LOG.debug( + "Peer {} is invalid because required block ({}) does not match required hash ({}).", + ethPeer, + blockNumber, + hash); + } + return validBlock; + } +} diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java index ad9b013a734..51004654d8d 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/DefaultSynchronizer.java @@ -170,8 +170,8 @@ private void handleFastSyncResult(final FastSyncState result, final Throwable er } private void startFullSync() { - fullSyncDownloader.start(); maybePruner.ifPresent(Pruner::start); + fullSyncDownloader.start(); } @Override diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/peervalidation/AbstractPeerBlockValidatorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/peervalidation/AbstractPeerBlockValidatorTest.java new file mode 100644 index 00000000000..178723546a0 --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/peervalidation/AbstractPeerBlockValidatorTest.java @@ -0,0 +1,151 @@ +/* + * + * * Copyright 2019 ConsenSys AG. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.eth.peervalidation; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockDataGenerator; +import org.hyperledger.besu.ethereum.core.BlockDataGenerator.BlockOptions; +import org.hyperledger.besu.ethereum.eth.manager.DeterministicEthScheduler; +import org.hyperledger.besu.ethereum.eth.manager.EthPeer; +import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; +import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManagerTestUtil; +import org.hyperledger.besu.ethereum.eth.manager.RespondingEthPeer; +import org.hyperledger.besu.ethereum.eth.messages.BlockHeadersMessage; +import org.hyperledger.besu.ethereum.eth.messages.EthPV62; +import org.hyperledger.besu.ethereum.eth.messages.GetBlockHeadersMessage; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.junit.Test; + +public abstract class AbstractPeerBlockValidatorTest { + + abstract AbstractPeerBlockValidator createValidator(long blockNumber, long buffer); + + @Test + public void validatePeer_unresponsivePeer() { + final EthProtocolManager ethProtocolManager = + EthProtocolManagerTestUtil.create(DeterministicEthScheduler.TimeoutPolicy.ALWAYS_TIMEOUT); + final long blockNumber = 500; + + final PeerValidator validator = createValidator(blockNumber, 0); + + final RespondingEthPeer peer = + EthProtocolManagerTestUtil.createPeer(ethProtocolManager, blockNumber); + + final CompletableFuture result = + validator.validatePeer(ethProtocolManager.ethContext(), peer.getEthPeer()); + + // Request should timeout immediately + assertThat(result).isDone(); + assertThat(result).isCompletedWithValue(false); + } + + @Test + public void validatePeer_requestBlockFromPeerBeingTested() { + final EthProtocolManager ethProtocolManager = EthProtocolManagerTestUtil.create(); + final BlockDataGenerator gen = new BlockDataGenerator(1); + final long blockNumber = 500; + final Block block = gen.block(BlockOptions.create().setBlockNumber(blockNumber)); + + final PeerValidator validator = createValidator(blockNumber, 0); + + final int peerCount = 1000; + final List otherPeers = + Stream.generate( + () -> EthProtocolManagerTestUtil.createPeer(ethProtocolManager, blockNumber)) + .limit(peerCount) + .collect(Collectors.toList()); + final RespondingEthPeer targetPeer = + EthProtocolManagerTestUtil.createPeer(ethProtocolManager, blockNumber); + + final CompletableFuture result = + validator.validatePeer(ethProtocolManager.ethContext(), targetPeer.getEthPeer()); + + assertThat(result).isNotDone(); + + // Other peers should not receive request for target block + for (final RespondingEthPeer otherPeer : otherPeers) { + final AtomicBoolean blockRequestedForOtherPeer = respondToBlockRequest(otherPeer, block); + assertThat(blockRequestedForOtherPeer).isFalse(); + } + + // Target peer should receive request for target block + final AtomicBoolean blockRequested = respondToBlockRequest(targetPeer, block); + assertThat(blockRequested).isTrue(); + } + + @Test + public void canBeValidated() { + final BlockDataGenerator gen = new BlockDataGenerator(1); + final EthProtocolManager ethProtocolManager = + EthProtocolManagerTestUtil.create(DeterministicEthScheduler.TimeoutPolicy.ALWAYS_TIMEOUT); + final long blockNumber = 500; + final long buffer = 10; + + final PeerValidator validator = createValidator(blockNumber, buffer); + final EthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0).getEthPeer(); + + peer.chainState().update(gen.hash(), blockNumber - 10); + assertThat(validator.canBeValidated(peer)).isFalse(); + + peer.chainState().update(gen.hash(), blockNumber); + assertThat(validator.canBeValidated(peer)).isFalse(); + + peer.chainState().update(gen.hash(), blockNumber + buffer - 1); + assertThat(validator.canBeValidated(peer)).isFalse(); + + peer.chainState().update(gen.hash(), blockNumber + buffer); + assertThat(validator.canBeValidated(peer)).isTrue(); + + peer.chainState().update(gen.hash(), blockNumber + buffer + 10); + assertThat(validator.canBeValidated(peer)).isTrue(); + } + + AtomicBoolean respondToBlockRequest(final RespondingEthPeer peer, final Block block) { + final AtomicBoolean blockRequested = new AtomicBoolean(false); + + final RespondingEthPeer.Responder responder = + RespondingEthPeer.targetedResponder( + (cap, msg) -> { + if (msg.getCode() != EthPV62.GET_BLOCK_HEADERS) { + return false; + } + final GetBlockHeadersMessage headersRequest = GetBlockHeadersMessage.readFrom(msg); + final boolean isTargetedBlockRequest = + headersRequest.blockNumber().isPresent() + && headersRequest.blockNumber().getAsLong() == block.getHeader().getNumber(); + if (isTargetedBlockRequest) { + blockRequested.set(true); + } + return isTargetedBlockRequest; + }, + (cap, msg) -> BlockHeadersMessage.create(block.getHeader())); + + // Respond + peer.respond(responder); + + return blockRequested; + } +} diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/peervalidation/DaoForkPeerValidatorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/peervalidation/DaoForkPeerValidatorTest.java index 780db24f928..c5938fb9e45 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/peervalidation/DaoForkPeerValidatorTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/peervalidation/DaoForkPeerValidatorTest.java @@ -19,54 +19,52 @@ import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.core.BlockDataGenerator.BlockOptions; -import org.hyperledger.besu.ethereum.eth.manager.DeterministicEthScheduler; -import org.hyperledger.besu.ethereum.eth.manager.EthPeer; import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManagerTestUtil; import org.hyperledger.besu.ethereum.eth.manager.RespondingEthPeer; -import org.hyperledger.besu.ethereum.eth.messages.BlockHeadersMessage; -import org.hyperledger.besu.ethereum.eth.messages.EthPV62; -import org.hyperledger.besu.ethereum.eth.messages.GetBlockHeadersMessage; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderValidator; import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; import org.hyperledger.besu.util.bytes.BytesValue; -import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.junit.Test; -public class DaoForkPeerValidatorTest { +public class DaoForkPeerValidatorTest extends AbstractPeerBlockValidatorTest { + + @Override + AbstractPeerBlockValidator createValidator(final long blockNumber, final long buffer) { + return new DaoForkPeerValidator( + MainnetProtocolSchedule.create(), new NoOpMetricsSystem(), blockNumber, buffer); + } @Test public void validatePeer_responsivePeerOnRightSideOfFork() { - EthProtocolManager ethProtocolManager = EthProtocolManagerTestUtil.create(); - BlockDataGenerator gen = new BlockDataGenerator(1); - long daoBlockNumber = 500; - Block daoBlock = + final EthProtocolManager ethProtocolManager = EthProtocolManagerTestUtil.create(); + final BlockDataGenerator gen = new BlockDataGenerator(1); + final long daoBlockNumber = 500; + final Block daoBlock = gen.block( BlockOptions.create() .setBlockNumber(daoBlockNumber) .setExtraData(MainnetBlockHeaderValidator.DAO_EXTRA_DATA)); - PeerValidator validator = + final PeerValidator validator = new DaoForkPeerValidator( MainnetProtocolSchedule.create(), new NoOpMetricsSystem(), daoBlockNumber, 0); - RespondingEthPeer peer = + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, daoBlockNumber); - CompletableFuture result = + final CompletableFuture result = validator.validatePeer(ethProtocolManager.ethContext(), peer.getEthPeer()); assertThat(result).isNotDone(); // Send response for dao block - AtomicBoolean daoBlockRequested = respondToDaoBlockRequest(peer, daoBlock); + final AtomicBoolean daoBlockRequested = respondToBlockRequest(peer, daoBlock); assertThat(daoBlockRequested).isTrue(); assertThat(result).isDone(); @@ -75,149 +73,30 @@ public void validatePeer_responsivePeerOnRightSideOfFork() { @Test public void validatePeer_responsivePeerOnWrongSideOfFork() { - EthProtocolManager ethProtocolManager = EthProtocolManagerTestUtil.create(); - BlockDataGenerator gen = new BlockDataGenerator(1); - long daoBlockNumber = 500; - Block daoBlock = + final EthProtocolManager ethProtocolManager = EthProtocolManagerTestUtil.create(); + final BlockDataGenerator gen = new BlockDataGenerator(1); + final long daoBlockNumber = 500; + final Block daoBlock = gen.block( BlockOptions.create().setBlockNumber(daoBlockNumber).setExtraData(BytesValue.EMPTY)); - PeerValidator validator = + final PeerValidator validator = new DaoForkPeerValidator( MainnetProtocolSchedule.create(), new NoOpMetricsSystem(), daoBlockNumber, 0); - RespondingEthPeer peer = + final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, daoBlockNumber); - CompletableFuture result = + final CompletableFuture result = validator.validatePeer(ethProtocolManager.ethContext(), peer.getEthPeer()); assertThat(result).isNotDone(); // Send response for dao block - AtomicBoolean daoBlockRequested = respondToDaoBlockRequest(peer, daoBlock); + final AtomicBoolean daoBlockRequested = respondToBlockRequest(peer, daoBlock); assertThat(daoBlockRequested).isTrue(); assertThat(result).isDone(); assertThat(result).isCompletedWithValue(false); } - - @Test - public void validatePeer_unresponsivePeer() { - EthProtocolManager ethProtocolManager = - EthProtocolManagerTestUtil.create(DeterministicEthScheduler.TimeoutPolicy.ALWAYS_TIMEOUT); - long daoBlockNumber = 500; - - PeerValidator validator = - new DaoForkPeerValidator( - MainnetProtocolSchedule.create(), new NoOpMetricsSystem(), daoBlockNumber, 0); - - RespondingEthPeer peer = - EthProtocolManagerTestUtil.createPeer(ethProtocolManager, daoBlockNumber); - - CompletableFuture result = - validator.validatePeer(ethProtocolManager.ethContext(), peer.getEthPeer()); - - // Request should timeout immediately - assertThat(result).isDone(); - assertThat(result).isCompletedWithValue(false); - } - - @Test - public void validatePeer_requestBlockFromPeerBeingTested() { - EthProtocolManager ethProtocolManager = EthProtocolManagerTestUtil.create(); - BlockDataGenerator gen = new BlockDataGenerator(1); - long daoBlockNumber = 500; - Block daoBlock = - gen.block( - BlockOptions.create() - .setBlockNumber(daoBlockNumber) - .setExtraData(MainnetBlockHeaderValidator.DAO_EXTRA_DATA)); - - PeerValidator validator = - new DaoForkPeerValidator( - MainnetProtocolSchedule.create(), new NoOpMetricsSystem(), daoBlockNumber, 0); - - int peerCount = 1000; - List otherPeers = - Stream.generate( - () -> EthProtocolManagerTestUtil.createPeer(ethProtocolManager, daoBlockNumber)) - .limit(peerCount) - .collect(Collectors.toList()); - RespondingEthPeer targetPeer = - EthProtocolManagerTestUtil.createPeer(ethProtocolManager, daoBlockNumber); - - CompletableFuture result = - validator.validatePeer(ethProtocolManager.ethContext(), targetPeer.getEthPeer()); - - assertThat(result).isNotDone(); - - // Other peers should not receive request for dao block - for (RespondingEthPeer otherPeer : otherPeers) { - AtomicBoolean daoBlockRequestedForOtherPeer = respondToDaoBlockRequest(otherPeer, daoBlock); - assertThat(daoBlockRequestedForOtherPeer).isFalse(); - } - - // Target peer should receive request for dao block - final AtomicBoolean daoBlockRequested = respondToDaoBlockRequest(targetPeer, daoBlock); - assertThat(daoBlockRequested).isTrue(); - } - - @Test - public void canBeValidated() { - BlockDataGenerator gen = new BlockDataGenerator(1); - EthProtocolManager ethProtocolManager = - EthProtocolManagerTestUtil.create(DeterministicEthScheduler.TimeoutPolicy.ALWAYS_TIMEOUT); - long daoBlockNumber = 500; - long buffer = 10; - - PeerValidator validator = - new DaoForkPeerValidator( - MainnetProtocolSchedule.create(), new NoOpMetricsSystem(), daoBlockNumber, buffer); - - EthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, 0).getEthPeer(); - - peer.chainState().update(gen.hash(), daoBlockNumber - 10); - assertThat(validator.canBeValidated(peer)).isFalse(); - - peer.chainState().update(gen.hash(), daoBlockNumber); - assertThat(validator.canBeValidated(peer)).isFalse(); - - peer.chainState().update(gen.hash(), daoBlockNumber + buffer - 1); - assertThat(validator.canBeValidated(peer)).isFalse(); - - peer.chainState().update(gen.hash(), daoBlockNumber + buffer); - assertThat(validator.canBeValidated(peer)).isTrue(); - - peer.chainState().update(gen.hash(), daoBlockNumber + buffer + 10); - assertThat(validator.canBeValidated(peer)).isTrue(); - } - - private AtomicBoolean respondToDaoBlockRequest( - final RespondingEthPeer peer, final Block daoBlock) { - AtomicBoolean daoBlockRequested = new AtomicBoolean(false); - - RespondingEthPeer.Responder responder = - RespondingEthPeer.targetedResponder( - (cap, msg) -> { - if (msg.getCode() != EthPV62.GET_BLOCK_HEADERS) { - return false; - } - GetBlockHeadersMessage headersRequest = GetBlockHeadersMessage.readFrom(msg); - boolean isDaoBlockRequest = - headersRequest.blockNumber().isPresent() - && headersRequest.blockNumber().getAsLong() - == daoBlock.getHeader().getNumber(); - if (isDaoBlockRequest) { - daoBlockRequested.set(true); - } - return isDaoBlockRequest; - }, - (cap, msg) -> BlockHeadersMessage.create(daoBlock.getHeader())); - - // Respond - peer.respond(responder); - - return daoBlockRequested; - } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/peervalidation/RequiredBlocksPeerValidatorTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/peervalidation/RequiredBlocksPeerValidatorTest.java new file mode 100644 index 00000000000..af7cacb40d0 --- /dev/null +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/peervalidation/RequiredBlocksPeerValidatorTest.java @@ -0,0 +1,107 @@ +/* + * + * * Copyright ConsenSys AG. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * * specific language governing permissions and limitations under the License. + * * + * * SPDX-License-Identifier: Apache-2.0 + * + */ +package org.hyperledger.besu.ethereum.eth.peervalidation; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockDataGenerator; +import org.hyperledger.besu.ethereum.core.BlockDataGenerator.BlockOptions; +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManager; +import org.hyperledger.besu.ethereum.eth.manager.EthProtocolManagerTestUtil; +import org.hyperledger.besu.ethereum.eth.manager.RespondingEthPeer; +import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule; +import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.Test; + +public class RequiredBlocksPeerValidatorTest extends AbstractPeerBlockValidatorTest { + + @Override + AbstractPeerBlockValidator createValidator(final long blockNumber, final long buffer) { + return new RequiredBlocksPeerValidator( + MainnetProtocolSchedule.create(), new NoOpMetricsSystem(), blockNumber, Hash.ZERO, buffer); + } + + @Test + public void validatePeer_responsivePeerWithRequiredBlock() { + final EthProtocolManager ethProtocolManager = EthProtocolManagerTestUtil.create(); + final BlockDataGenerator gen = new BlockDataGenerator(1); + final long requiredBlockNumber = 500; + final Block requiredBlock = + gen.block(BlockOptions.create().setBlockNumber(requiredBlockNumber)); + + final PeerValidator validator = + new RequiredBlocksPeerValidator( + MainnetProtocolSchedule.create(), + new NoOpMetricsSystem(), + requiredBlockNumber, + requiredBlock.getHash(), + 0); + + final RespondingEthPeer peer = + EthProtocolManagerTestUtil.createPeer(ethProtocolManager, requiredBlockNumber); + + final CompletableFuture result = + validator.validatePeer(ethProtocolManager.ethContext(), peer.getEthPeer()); + + assertThat(result).isNotDone(); + + // Send response for block + final AtomicBoolean requiredBlockRequested = respondToBlockRequest(peer, requiredBlock); + + assertThat(requiredBlockRequested).isTrue(); + assertThat(result).isDone(); + assertThat(result).isCompletedWithValue(true); + } + + @Test + public void validatePeer_responsivePeerWithBadRequiredBlock() { + final EthProtocolManager ethProtocolManager = EthProtocolManagerTestUtil.create(); + final BlockDataGenerator gen = new BlockDataGenerator(1); + final long requiredBlockNumber = 500; + final Block requiredBlock = + gen.block(BlockOptions.create().setBlockNumber(requiredBlockNumber)); + + final PeerValidator validator = + new RequiredBlocksPeerValidator( + MainnetProtocolSchedule.create(), + new NoOpMetricsSystem(), + requiredBlockNumber, + Hash.ZERO, + 0); + + final RespondingEthPeer peer = + EthProtocolManagerTestUtil.createPeer(ethProtocolManager, requiredBlockNumber); + + final CompletableFuture result = + validator.validatePeer(ethProtocolManager.ethContext(), peer.getEthPeer()); + + assertThat(result).isNotDone(); + + // Send response for required block + final AtomicBoolean requiredBlockRequested = respondToBlockRequest(peer, requiredBlock); + + assertThat(requiredBlockRequested).isTrue(); + assertThat(result).isDone(); + assertThat(result).isCompletedWithValue(false); + } +}