Refactoring (#6635)

* Resolved conflicts

* max body size for http json rpc (#36)

* Format

* Self-recovery of TxPool max capacity

* fix ProcessedTransactionsDbCleanerTests

* Update txPool worst value if removal unsuccessful (#6702)

(cherry picked from commit f262cb2)

* Clearer error message for missing starting block state (#6672)

* Update txPool worst value if removal unsuccessful (#6702)

* Add Cancun hard-fork settings for Gnosis (#6709)

* Cancun spec and config for mainnet (#6679)

* Add Paris release spec class for completeness (#6633)

* 0.0.0.0 to +

---------

Co-authored-by: smartprogrammer <smartprogrammer@windowslive.com>
Co-authored-by: Marcin Sobczak <marcindsobczak@gmail.com>
Co-authored-by: Ben Adams <thundercat@illyriad.co.uk>
Co-authored-by: Ruben Buniatyan <rubo@users.noreply.github.com>
Co-authored-by: Ahmad Bitar <33181301+smartprogrammer93@users.noreply.github.com>
Co-authored-by: Kamil Chodoła <kamil@nethermind.io>
Co-authored-by: Lukasz Rozmej <lukasz.rozmej@gmail.com>

.github/workflows/hive-consensus-tests.yml

@@ -1,11 +1,11 @@
-name: 'Hive consensus tests' 
+name: 'Hive consensus tests'
 
 on:
   workflow_run:
     workflows: ["Publish Docker image"]
     branches: ["release/*"]
     types:
-      - completed  
+      - completed
   workflow_dispatch:
     inputs:
       parallelism:
@@ -40,7 +40,7 @@ jobs:
       run: |
         echo "ORG_NAME=${{ github.repository_owner }}" >> $GITHUB_ENV
         echo "REPO_NAME=${{ github.event.repository.name }}" >> $GITHUB_ENV
-        
+
     - name: Check if master or release branch
       id: check_conditions
       run: |
@@ -71,7 +71,7 @@ jobs:
               "dockerfile": "Dockerfile",
               "build-config": "release"
            }'
-        
+
     - name: Wait for Docker Build Action to complete
       if: steps.check_conditions.outputs.skip_docker_build != 'true'
       env:
@@ -107,7 +107,7 @@ jobs:
           dotnet run
           cat matrix.json
           echo "matrix=$(jq -c . matrix.json)" >> $GITHUB_OUTPUT
-          
+
   run_hive_tests:
     runs-on: ubuntu-latest
     needs: [generate_hive_consensus_tests, create_docker_image]
@@ -124,7 +124,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           path: nethermind
-          
+
       - name: Install Linux packages
         run: |
           sudo apt-get update
@@ -164,13 +164,12 @@ jobs:
         run: |
           chmod +x nethermind/scripts/hive-results.sh
           nethermind/scripts/hive-results.sh "hive/workspace/logs/*.json"
-      - name: Send results to dashboard
-        continue-on-error: true
-        uses: appleboy/scp-action@master
-        with:
-          host: ${{ secrets.HIVE_HOST }}
-          username: ${{ secrets.HIVE_USERNAME }}
-          key: ${{ secrets.HIVE_KEY }}
-          port: ${{ secrets.HIVE_PORT }}
-          source: hive/workspace/logs/*
-          target: ${{ secrets.HIVE_DIR }}/
+      # - name: Send results to dashboard
+      #   uses: appleboy/scp-action@master
+      #   with:
+      #     host: ${{ secrets.HIVE_HOST }}
+      #     username: ${{ secrets.HIVE_USERNAME }}
+      #     key: ${{ secrets.HIVE_KEY }}
+      #     port: ${{ secrets.HIVE_PORT }}
+      #     source: hive/workspace/logs/*
+      #     target: ${{ secrets.HIVE_DIR }}/

src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs

@@ -181,15 +181,13 @@ protected async Task<EthereumTestResult> RunTest(BlockchainTest test, Stopwatch?
             stopwatch?.Start();
             List<(Block Block, string ExpectedException)> correctRlp = DecodeRlps(test, failOnInvalidRlp);
 
-            if (test.GenesisRlp == null)
-            {
-                test.GenesisRlp = Rlp.Encode(new Block(JsonToEthereumTest.Convert(test.GenesisBlockHeader)));
-            }
+            test.GenesisRlp ??= Rlp.Encode(new Block(JsonToEthereumTest.Convert(test.GenesisBlockHeader)));
 
             Block genesisBlock = Rlp.Decode<Block>(test.GenesisRlp.Bytes);
             Assert.That(genesisBlock.Header.Hash, Is.EqualTo(new Hash256(test.GenesisBlockHeader.Hash)));
 
             ManualResetEvent genesisProcessed = new(false);
+
             blockTree.NewHeadBlock += (_, args) =>
             {
                 if (args.Block.Number == 0)
@@ -271,7 +269,6 @@ protected async Task<EthereumTestResult> RunTest(BlockchainTest test, Stopwatch?
                     {
                         Assert.That(suggestedBlock.Header.Hash, Is.EqualTo(new Hash256(testBlockJson.BlockHeader.Hash)));
 
-
                         for (int uncleIndex = 0; uncleIndex < suggestedBlock.Uncles.Length; uncleIndex++)
                         {
                             Assert.That(suggestedBlock.Uncles[uncleIndex].Hash, Is.EqualTo(new Hash256(testBlockJson.UncleHeaders[uncleIndex].Hash)));

src/Nethermind/Nethermind.AuRa.Test/AuRaSealValidatorTests.cs

@@ -144,7 +144,7 @@ TestCaseData GetTestCaseData(
         public (bool, object) validate_params(BlockHeader parentBlock, BlockHeader block, Action<AuRaParameters> modifyParameters, Repeat repeat, bool parentIsHead, bool isValidSealer)
         {
             _blockTree.Head.Returns(parentIsHead ? new Block(parentBlock) : new Block(Build.A.BlockHeader.WithNumber(parentBlock.Number - 1).TestObject));
-            _validSealerStrategy.IsValidSealer(Arg.Any<IList<Address>>(), block.Beneficiary, block.AuRaStep.Value).Returns(isValidSealer);
+            _validSealerStrategy.IsValidSealer(Arg.Any<IList<Address>>(), block.Beneficiary, block.AuRaStep.Value, out _).Returns(isValidSealer);
 
             object cause = null;
 

src/Nethermind/Nethermind.AuRa.Test/AuRaSealerTests.cs

@@ -60,15 +60,15 @@ public void Setup()
         public bool can_seal(long auRaStep, bool validSealer)
         {
             _auRaStepCalculator.CurrentStep.Returns(auRaStep);
-            _validSealerStrategy.IsValidSealer(Arg.Any<IList<Address>>(), _address, auRaStep).Returns(validSealer);
+            _validSealerStrategy.IsValidSealer(Arg.Any<IList<Address>>(), _address, auRaStep, out _).Returns(validSealer);
             return _auRaSealer.CanSeal(10, _blockTree.Head.Hash);
         }
 
         [Test]
         public async Task seal_can_recover_address()
         {
             _auRaStepCalculator.CurrentStep.Returns(11);
-            _validSealerStrategy.IsValidSealer(Arg.Any<IList<Address>>(), _address, 11).Returns(true);
+            _validSealerStrategy.IsValidSealer(Arg.Any<IList<Address>>(), _address, 11, out _).Returns(true);
             Block block = Build.A.Block.WithHeader(Build.A.BlockHeader.WithBeneficiary(_address).WithAura(11, null).TestObject).TestObject;
 
             block = await _auRaSealer.SealBlock(block, CancellationToken.None);

src/Nethermind/Nethermind.AuRa.Test/Validators/ListValidatorTests.cs

@@ -48,7 +48,7 @@ private static IEnumerable ValidateTestCases
 
         [TestCaseSource(nameof(ValidateTestCases))]
         public bool should_validate_correctly(Address address, long index) =>
-            _validSealerStrategy.IsValidSealer(GetListValidator(TestItem.AddressA, TestItem.AddressB).Validators, address, index);
+            _validSealerStrategy.IsValidSealer(GetListValidator(TestItem.AddressA, TestItem.AddressB).Validators, address, index, out _);
 
         [TestCase(1)]
         [TestCase(2)]

src/Nethermind/Nethermind.Blockchain.Test/BlockchainProcessorTests.cs

@@ -78,7 +78,7 @@ public Block[] Process(Hash256 newBranchStateRoot, List<Block> suggestedBlocks,
                     if (blockTracer != NullBlockTracer.Instance)
                     {
                         // this is for block reruns on failure for diag tracing
-                        throw new InvalidBlockException(suggestedBlocks[0]);
+                        throw new InvalidBlockException(suggestedBlocks[0], "wrong tracer");
                     }
 
                     _logger.Info($"Processing {suggestedBlocks.Last().ToString(Block.Format.Short)}");
@@ -96,7 +96,7 @@ public Block[] Process(Hash256 newBranchStateRoot, List<Block> suggestedBlocks,
                                 {
                                     _allowedToFail.Remove(hash);
                                     BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(suggestedBlocks.Last(), Array.Empty<TxReceipt>()));
-                                    throw new InvalidBlockException(suggestedBlock);
+                                    throw new InvalidBlockException(suggestedBlock, "allowed to fail");
                                 }
 
                                 notYet = true;

src/Nethermind/Nethermind.Blockchain/InvalidBlockException.cs

@@ -8,8 +8,11 @@ namespace Nethermind.Blockchain;
 
 public class InvalidBlockException : BlockchainException
 {
-    public InvalidBlockException(Block block, Exception? innerException = null)
-        : base($"Invalid block: {block}", innerException) => InvalidBlock = block;
+    public InvalidBlockException(Block block, string message, Exception? innerException = null)
+        : base($"Invalid block: {block} : {message}", innerException) => InvalidBlock = block.Header;
 
-    public Block InvalidBlock { get; }
+    public InvalidBlockException(BlockHeader block, string message, Exception? innerException = null)
+        : base($"Invalid block: {block} : {message}", innerException) => InvalidBlock = block;
+
+    public BlockHeader InvalidBlock { get; }
 }

src/Nethermind/Nethermind.Consensus.AuRa/AuRaBlockProcessor.cs

@@ -108,8 +108,9 @@ private void ValidateGasLimit(Block block)
             BlockHeader parentHeader = GetParentHeader(block);
             if (_gasLimitOverride?.IsGasLimitValid(parentHeader, block.GasLimit, out long? expectedGasLimit) == false)
             {
-                if (_logger.IsWarn) _logger.Warn($"Invalid gas limit for block {block.Number}, hash {block.Hash}, expected value from contract {expectedGasLimit}, but found {block.GasLimit}.");
-                throw new InvalidBlockException(block);
+                string reason = $"Invalid gas limit, expected value from contract {expectedGasLimit}, but found {block.GasLimit}";
+                if (_logger.IsWarn) _logger.Warn($"Proposed block is not valid {block.ToString(Block.Format.FullHashAndNumber)}. {reason}.");
+                throw new InvalidBlockException(block, reason);
             }
         }
 
@@ -121,8 +122,9 @@ private void ValidateTxs(Block block)
                 AddingTxEventArgs args = CheckTxPosdaoRules(new AddingTxEventArgs(i, tx, block, block.Transactions));
                 if (args.Action != TxAction.Add)
                 {
-                    if (_logger.IsWarn) _logger.Warn($"Proposed block is not valid {block.ToString(Block.Format.FullHashAndNumber)}. {tx.ToShortString()} doesn't have required permissions. Reason: {args.Reason}.");
-                    throw new InvalidBlockException(block);
+                    string reason = $"{tx.ToShortString()} doesn't have required permissions: {args.Reason}";
+                    if (_logger.IsWarn) _logger.Warn($"Proposed block is not valid {block.ToString(Block.Format.FullHashAndNumber)}. {reason}.");
+                    throw new InvalidBlockException(block, reason);
                 }
             }
         }

src/Nethermind/Nethermind.Consensus.AuRa/AuRaSealValidator.cs

@@ -75,9 +75,9 @@ public bool ValidateParams(BlockHeader parent, BlockHeader header, bool isUncle
                 // no worries we do this validation later before processing the block
                 if (parent.Hash == _blockTree.Head?.Hash)
                 {
-                    if (!_validSealerStrategy.IsValidSealer(_validatorStore.GetValidators(), header.Beneficiary, step))
+                    if (!_validSealerStrategy.IsValidSealer(_validatorStore.GetValidators(), header.Beneficiary, step, out Address expectedAddress))
                     {
-                        if (_logger.IsError) _logger.Error($"Block from incorrect proposer at block {header.ToString(BlockHeader.Format.FullHashAndNumber)}, step {step} from author {header.Beneficiary}.");
+                        if (_logger.IsError) _logger.Error($"Proposed block is not valid {header.ToString(BlockHeader.Format.FullHashAndNumber)}. Incorrect proposer at step {step}, expected {expectedAddress}, but found {header.Beneficiary}.");
                         return false;
                     }
                 }

src/Nethermind/Nethermind.Consensus.AuRa/AuRaSealer.cs

@@ -74,8 +74,8 @@ bool StepNotYetProduced(long step) => !_blockTree.Head.Header.AuRaStep.HasValue
 
             bool IsThisNodeTurn(long step)
             {
-                var validators = _validatorStore.GetValidators();
-                return _validSealerStrategy.IsValidSealer(validators, _signer.Address, step);
+                Address[] validators = _validatorStore.GetValidators();
+                return _validSealerStrategy.IsValidSealer(validators, _signer.Address, step, out _);
             }
 
             long currentStep = _auRaStepCalculator.CurrentStep;

src/Nethermind/Nethermind.Consensus.AuRa/Validators/AuRaValidatorBase.cs

@@ -49,11 +49,12 @@ public virtual void OnBlockProcessingStart(Block block, ProcessingOptions option
             if (!options.ContainsFlag(ProcessingOptions.ProducingBlock) && !block.IsGenesis)
             {
                 var auRaStep = block.Header.AuRaStep.Value;
-                if (!_validSealerStrategy.IsValidSealer(Validators, block.Beneficiary, auRaStep))
+                if (!_validSealerStrategy.IsValidSealer(Validators, block.Beneficiary, auRaStep, out Address expectedAddress))
                 {
-                    if (_logger.IsError) _logger.Error($"Block from incorrect proposer at block {block.ToString(Block.Format.FullHashAndNumber)}, step {auRaStep} from author {block.Beneficiary}.");
+                    string reason = $"Incorrect proposer at step {auRaStep}, expected {expectedAddress}, but found {block.Beneficiary}";
+                    if (_logger.IsError) _logger.Error($"Proposed block is not valid {block.ToString(Block.Format.FullHashAndNumber)}. {reason}.");
                     this.GetReportingValidator().ReportBenign(block.Beneficiary, block.Number, IReportingValidator.BenignCause.IncorrectProposer);
-                    throw new InvalidBlockException(block);
+                    throw new InvalidBlockException(block, reason);
                 }
             }
         }

src/Nethermind/Nethermind.Consensus.AuRa/Validators/IValidSealerStrategy.cs

@@ -14,7 +14,8 @@ public interface IValidSealerStrategy
         /// <param name="validators">Validators at given step.</param>
         /// <param name="address">Address to be checked if its a sealer at this step.</param>
         /// <param name="step">Step to be checked.</param>
+        /// <param name="expectedAddress">Address which should create block in current step.</param>
         /// <returns>'true' if <see cref="address"/> should seal a block at <see cref="step"/> for supplied <see cref="validators"/> collection. Otherwise 'false'.</returns>
-        bool IsValidSealer(IList<Address> validators, Address address, long step);
+        bool IsValidSealer(IList<Address> validators, Address address, long step, out Address expectedAddress);
     }
 }

src/Nethermind/Nethermind.Consensus.AuRa/Validators/ValidSealerStrategy.cs

@@ -9,6 +9,7 @@ namespace Nethermind.Consensus.AuRa.Validators
 {
     public class ValidSealerStrategy : IValidSealerStrategy
     {
-        public bool IsValidSealer(IList<Address> validators, Address address, long step) => validators.GetItemRoundRobin(step) == address;
+        public bool IsValidSealer(IList<Address> validators, Address address, long step, out Address expectedAddress) =>
+            (expectedAddress = validators.GetItemRoundRobin(step)) == address;
     }
 }

src/Nethermind/Nethermind.Consensus/Processing/BlockExtensions.cs

@@ -39,16 +39,11 @@ public static bool TrySetTransactions(this Block block, Transaction[] transactio
             return false;
         }
 
-        public static bool IsByNethermindNode(this Block block)
-        {
-            try
-            {
-                return Encoding.ASCII.GetString(block.ExtraData).Contains(BlocksConfig.DefaultExtraData, StringComparison.InvariantCultureIgnoreCase);
-            }
-            catch (Exception)
-            {
-                return false;
-            }
-        }
+        public static bool IsByNethermindNode(this Block block) => block.Header.IsByNethermindNode();
+
+        public static bool IsByNethermindNode(this BlockHeader block) =>
+            Ascii.IsValid(block.ExtraData)
+            && Encoding.ASCII.GetString(block.ExtraData ?? Array.Empty<byte>())
+                .Contains(BlocksConfig.DefaultExtraData, StringComparison.InvariantCultureIgnoreCase);
     }
 }

src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockProductionTransactionsExecutor.cs

@@ -109,13 +109,20 @@ protected TxAction ProcessTransaction(
                 }
                 else
                 {
-                    _transactionProcessor.ProcessTransaction(in blkCtx, currentTx, receiptsTracer, processingOptions, _stateProvider);
+                    TransactionResult result = _transactionProcessor.ProcessTransaction(in blkCtx, currentTx, receiptsTracer, processingOptions, _stateProvider);
 
-                    if (addToBlock)
+                    if (result)
                     {
-                        transactionsInBlock.Add(currentTx);
-                        _transactionProcessed?.Invoke(this,
-                            new TxProcessedEventArgs(index, currentTx, receiptsTracer.TxReceipts[index]));
+                        if (addToBlock)
+                        {
+                            transactionsInBlock.Add(currentTx);
+                            _transactionProcessed?.Invoke(this,
+                                new TxProcessedEventArgs(index, currentTx, receiptsTracer.TxReceipts[index]));
+                        }
+                    }
+                    else
+                    {
+                        args.Set(TxAction.Skip, result.Error!);
                     }
                 }
 

src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockValidationTransactionsExecutor.cs

@@ -2,13 +2,17 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
 using System.Linq;
+using Nethermind.Blockchain;
 using Nethermind.Core;
 using Nethermind.Core.Specs;
 using Nethermind.Evm;
 using Nethermind.Evm.Tracing;
 using Nethermind.Evm.TransactionProcessing;
 using Nethermind.State;
+using Metrics = Nethermind.Evm.Metrics;
 
 namespace Nethermind.Consensus.Processing
 {
@@ -34,7 +38,7 @@ public BlockValidationTransactionsExecutor(ITransactionProcessorAdapter transact
 
             public TxReceipt[] ProcessTransactions(Block block, ProcessingOptions processingOptions, BlockReceiptsTracer receiptsTracer, IReleaseSpec spec)
             {
-                Evm.Metrics.ResetBlockStats();
+                Metrics.ResetBlockStats();
                 BlockExecutionContext blkCtx = new(block.Header);
                 for (int i = 0; i < block.Transactions.Length; i++)
                 {
@@ -46,9 +50,17 @@ public TxReceipt[] ProcessTransactions(Block block, ProcessingOptions processing
 
             private void ProcessTransaction(in BlockExecutionContext blkCtx, Transaction currentTx, int index, BlockReceiptsTracer receiptsTracer, ProcessingOptions processingOptions)
             {
-                _transactionProcessor.ProcessTransaction(in blkCtx, currentTx, receiptsTracer, processingOptions, _stateProvider);
+                TransactionResult result = _transactionProcessor.ProcessTransaction(in blkCtx, currentTx, receiptsTracer, processingOptions, _stateProvider);
+                if (!result) ThrowInvalidBlockException(result, blkCtx.Header, currentTx, index);
                 TransactionProcessed?.Invoke(this, new TxProcessedEventArgs(index, currentTx, receiptsTracer.TxReceipts[index]));
             }
+
+            [DoesNotReturn]
+            [StackTraceHidden]
+            private void ThrowInvalidBlockException(TransactionResult result, BlockHeader header, Transaction currentTx, int index)
+            {
+                throw new InvalidBlockException(header, $"Transaction {currentTx.Hash} at index {index} failed with error {result.Error}");
+            }
         }
     }
 }

src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs

@@ -217,7 +217,7 @@ private void ValidateProcessedBlock(Block suggestedBlock, ProcessingOptions opti
         {
             if (_logger.IsWarn) _logger.Warn($"Processed block is not valid {suggestedBlock.ToString(Block.Format.FullHashAndNumber)}");
             if (_logger.IsWarn) _logger.Warn($"Suggested block TD: {suggestedBlock.TotalDifficulty}, Suggested block IsPostMerge {suggestedBlock.IsPostMerge}, Block TD: {block.TotalDifficulty}, Block IsPostMerge {block.IsPostMerge}");
-            throw new InvalidBlockException(suggestedBlock);
+            throw new InvalidBlockException(suggestedBlock, "invalid hash after block processing");
         }
     }