Wait for blob inclusion

src/Nethermind/Ethereum.Test.Base/GeneralTestBase.cs

@@ -89,8 +89,8 @@ protected EthereumTestResult RunTest(GeneralStateTest test, ITxTracer txTracer)
             header.MixHash = test.CurrentRandom;
 
             Stopwatch stopwatch = Stopwatch.StartNew();
-            var txValidator = new TxValidator((MainnetSpecProvider.Instance.ChainId));
-            var spec = specProvider.GetSpec((ForkActivation)test.CurrentNumber);
+            TxValidator? txValidator = new((MainnetSpecProvider.Instance.ChainId));
+            IReleaseSpec? spec = specProvider.GetSpec((ForkActivation)test.CurrentNumber);
             if (test.Transaction.ChainId == null)
                 test.Transaction.ChainId = MainnetSpecProvider.Instance.ChainId;
             bool isValid = txValidator.IsWellFormed(test.Transaction, spec);

src/Nethermind/Nethermind.AuRa.Test/Validators/ContractBasedValidatorTests.cs

@@ -7,7 +7,6 @@
 using FluentAssertions;
 using Nethermind.Abi;
 using Nethermind.Blockchain;
-using Nethermind.Blockchain.Find;
 using Nethermind.Blockchain.Receipts;
 using Nethermind.Consensus;
 using Nethermind.Consensus.AuRa;

src/Nethermind/Nethermind.Blockchain.Test/Producers/BlockProducerBaseTests.cs

@@ -17,7 +17,6 @@
 using Nethermind.State;
 using NSubstitute;
 using NUnit.Framework;
-using NUnit.Framework.Internal.Execution;
 
 namespace Nethermind.Blockchain.Test.Producers
 {

src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs

@@ -158,7 +158,7 @@ public static IEnumerable EnoughShardBlobTransactionsSelectedTestCases
                     tx.BlobVersionedHashes = new byte[1][];
                 });
                 maxTransactionsSelected.Transactions[1].BlobVersionedHashes =
-                    new byte[Eip4844Constants.MaxBlobsPerBlock - 1][];
+                    new byte[Eip4844Constants.MaxDataGasPerTransaction / Eip4844Constants.DataGasPerBlob - 1][];
                 maxTransactionsSelected.ExpectedSelectedTransactions.AddRange(
                     maxTransactionsSelected.Transactions.OrderBy(t => t.Nonce).Take(2));
                 yield return new TestCaseData(maxTransactionsSelected).SetName("Enough transactions selected");
@@ -168,7 +168,8 @@ public static IEnumerable EnoughShardBlobTransactionsSelectedTestCases
                 Transaction[] expectedSelectedTransactions =
                     enoughTransactionsSelected.Transactions.OrderBy(t => t.Nonce).ToArray();
                 expectedSelectedTransactions[0].Type = TxType.Blob;
-                expectedSelectedTransactions[0].BlobVersionedHashes = new byte[Eip4844Constants.MaxBlobsPerBlock][];
+                expectedSelectedTransactions[0].BlobVersionedHashes =
+                    new byte[Eip4844Constants.MaxDataGasPerTransaction / Eip4844Constants.DataGasPerBlob][];
                 expectedSelectedTransactions[1].Type = TxType.Blob;
                 expectedSelectedTransactions[1].BlobVersionedHashes = new byte[1][];
                 enoughTransactionsSelected.ExpectedSelectedTransactions.AddRange(

src/Nethermind/Nethermind.Blockchain.Test/Validators/ShardBlobBlockValidatorTests.cs

@@ -1,57 +1,105 @@
 // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
+using System.Collections.Generic;
+using System.Linq;
+using Nethermind.Blockchain.Find;
 using Nethermind.Consensus.Validators;
 using Nethermind.Core;
 using Nethermind.Core.Specs;
 using Nethermind.Core.Test.Builders;
 using Nethermind.Logging;
 using Nethermind.Specs.Forks;
 using Nethermind.Specs.Test;
+using NSubstitute;
 using NUnit.Framework;
 
 namespace Nethermind.Blockchain.Test.Validators;
 
 public class ShardBlobBlockValidatorTests
 {
-    [Test]
-    public void Not_null_ExcessDataGas_is_invalid_pre_cancun()
+    [TestCaseSource(nameof(DataGasFieldsPerForkTestCases))]
+    public static bool Data_gas_fields_should_be_set(IReleaseSpec spec, ulong? dataGasUsed, ulong? excessDataGas)
     {
-        ISpecProvider specProvider = new CustomSpecProvider(((ForkActivation)0, Shanghai.Instance));
-        BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, LimboLogs.Instance);
-        bool isValid = blockValidator.ValidateSuggestedBlock(Build.A.Block
-            .WithWithdrawalsRoot(TestItem.KeccakA)
-            .WithWithdrawals(TestItem.WithdrawalA_1Eth)
-            .WithExcessDataGas(1).TestObject);
-        Assert.False(isValid);
-    }
-
-    [Test]
-    public void Null_ExcessDataGas_is_invalid_post_cancun()
-    {
-        ISpecProvider specProvider = new CustomSpecProvider(((ForkActivation)0, Cancun.Instance));
-        BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, LimboLogs.Instance);
-        bool isValid = blockValidator.ValidateSuggestedBlock(Build.A.Block
+        ISpecProvider specProvider = new CustomSpecProvider(((ForkActivation)0, spec));
+        HeaderValidator headerValidator = new(Substitute.For<IBlockTree>(), Always.Valid, specProvider, TestLogManager.Instance);
+        BlockValidator blockValidator = new(Always.Valid, headerValidator, Always.Valid, specProvider, TestLogManager.Instance);
+        return blockValidator.ValidateSuggestedBlock(Build.A.Block
+            .WithDataGasUsed(dataGasUsed)
+            .WithExcessDataGas(excessDataGas)
             .WithWithdrawalsRoot(TestItem.KeccakA)
             .WithWithdrawals(TestItem.WithdrawalA_1Eth)
+            .WithParent(Build.A.BlockHeader.TestObject)
             .TestObject);
-        Assert.False(isValid);
     }
 
-    [TestCase(0, ExpectedResult = true)]
-    [TestCase(Eip4844Constants.MaxBlobsPerBlock - 1, ExpectedResult = true)]
-    [TestCase(Eip4844Constants.MaxBlobsPerBlock, ExpectedResult = true)]
-    [TestCase(Eip4844Constants.MaxBlobsPerBlock + 1, ExpectedResult = false)]
-    public bool Blobs_per_block_count_is_valid(int blobsCount)
+    [TestCase(0ul, ExpectedResult = true)]
+    [TestCase(Eip4844Constants.MaxDataGasPerBlock - Eip4844Constants.DataGasPerBlob, ExpectedResult = true)]
+    [TestCase(Eip4844Constants.MaxDataGasPerBlock, ExpectedResult = true)]
+    [TestCase(Eip4844Constants.MaxDataGasPerBlock + Eip4844Constants.DataGasPerBlob, ExpectedResult = false)]
+    public bool Blobs_per_block_count_is_valid(ulong dataGasUsed)
     {
         ISpecProvider specProvider = new CustomSpecProvider(((ForkActivation)0, Cancun.Instance));
-        BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, LimboLogs.Instance);
+        BlockValidator blockValidator = new(Always.Valid, Always.Valid, Always.Valid, specProvider, TestLogManager.Instance);
         return blockValidator.ValidateSuggestedBlock(
             Build.A.Block
                 .WithWithdrawalsRoot(TestItem.KeccakA)
                 .WithWithdrawals(TestItem.WithdrawalA_1Eth)
-                .WithExcessDataGas(1)
-                .WithTransactions(Build.A.Transaction.WithBlobVersionedHashes(blobsCount).TestObject)
+                .WithDataGasUsed(dataGasUsed)
+                .WithExcessDataGas(0)
+                .WithTransactions(Enumerable.Range(0, (int)(dataGasUsed / Eip4844Constants.DataGasPerBlob))
+                    .Select(i => Build.A.Transaction.WithType(TxType.Blob)
+                                                    .WithMaxFeePerDataGas(ulong.MaxValue)
+                                                    .WithBlobVersionedHashes(1).TestObject).ToArray())
                 .TestObject);
     }
+
+    public static IEnumerable<TestCaseData> DataGasFieldsPerForkTestCases
+    {
+        get
+        {
+            yield return new TestCaseData(Shanghai.Instance, null, null)
+            {
+                TestName = "Data gas fields are not set pre-Cancun",
+                ExpectedResult = true
+            };
+            yield return new TestCaseData(Shanghai.Instance, 0ul, null)
+            {
+                TestName = "DataGasUsed is set pre-Cancun",
+                ExpectedResult = false
+            };
+            yield return new TestCaseData(Shanghai.Instance, null, 0ul)
+            {
+                TestName = "ExcessDataGas is set pre-Cancun",
+                ExpectedResult = false
+            };
+            yield return new TestCaseData(Shanghai.Instance, 0ul, 0ul)
+            {
+                TestName = "Data gas fields are set pre-Cancun",
+                ExpectedResult = false
+            };
+
+
+            yield return new TestCaseData(Cancun.Instance, null, null)
+            {
+                TestName = "Data gas fields are not set post-Cancun",
+                ExpectedResult = false
+            };
+            yield return new TestCaseData(Cancun.Instance, 0ul, null)
+            {
+                TestName = "Just DataGasUsed is set post-Cancun",
+                ExpectedResult = false
+            };
+            yield return new TestCaseData(Cancun.Instance, null, 0ul)
+            {
+                TestName = "Just ExcessDataGas is set post-Cancun",
+                ExpectedResult = false
+            };
+            yield return new TestCaseData(Cancun.Instance, 0ul, 0ul)
+            {
+                TestName = "Data gas fields are set post-Cancun",
+                ExpectedResult = true
+            };
+        }
+    }
 }

src/Nethermind/Nethermind.Blockchain.Test/Validators/TxValidatorTests.cs

@@ -336,7 +336,7 @@ public void ShardBlobTransactions_should_have_destination_set()
             .WithChainId(TestBlockchainIds.ChainId)
             .SignedAndResolved().TestObject;
 
-        Transaction txtxWithTo = Build.A.Transaction
+        Transaction txWithTo = Build.A.Transaction
             .WithType(TxType.Blob)
             .WithTimestamp(ulong.MaxValue)
             .WithTo(TestItem.AddressA)
@@ -347,7 +347,7 @@ public void ShardBlobTransactions_should_have_destination_set()
             .SignedAndResolved().TestObject;
 
         Assert.That(txValidator.IsWellFormed(txWithoutTo, Cancun.Instance), Is.False);
-        Assert.That(txValidator.IsWellFormed(txtxWithTo, Cancun.Instance));
+        Assert.That(txValidator.IsWellFormed(txWithTo, Cancun.Instance));
     }
 
     [Timeout(Timeout.MaxTestTime)]
@@ -371,7 +371,6 @@ public bool MaxFeePerDataGas_should_be_set_for_blob_tx_only(TxType txType, bool
         return txValidator.IsWellFormed(tx, Cancun.Instance);
     }
 
-
     [TestCaseSource(nameof(BlobVersionedHashInvalidTestCases))]
     [TestCaseSource(nameof(BlobVersionedHashValidTestCases))]
     public bool BlobVersionedHash_should_be_correct(byte[] hash)
@@ -407,36 +406,41 @@ private static IEnumerable<TestCaseData> BlobVersionedHashInvalidTestCases
         {
             yield return new TestCaseData(null) { TestName = "Null hash", ExpectedResult = false };
             yield return new TestCaseData(MakeArray(0)) { TestName = "Empty hash", ExpectedResult = false };
-            yield return new TestCaseData(MakeArray(1, 1, 0))
+            yield return new TestCaseData(MakeArray(1, 1))
             {
                 TestName = "Correct version, incorrect length",
                 ExpectedResult = false
             };
-            yield return new TestCaseData(MakeArray(31, 1, 0))
+            yield return new TestCaseData(MakeArray(31, 1))
             {
                 TestName = "Correct version, incorrect length",
                 ExpectedResult = false
             };
-            yield return new TestCaseData(MakeArray(33, 1, 0))
+            yield return new TestCaseData(MakeArray(33, 1))
             {
                 TestName = "Correct version, incorrect length",
                 ExpectedResult = false
             };
-            yield return new TestCaseData(MakeArray(32, 0, 0))
+            yield return new TestCaseData(MakeArray(32, 0))
             {
                 TestName = "Incorrect version, correct length",
                 ExpectedResult = false
             };
-            yield return new TestCaseData(MakeArray(32, KzgPolynomialCommitments.KzgBlobHashVersionV1 - 1, 0))
+            yield return new TestCaseData(MakeArray(32, KzgPolynomialCommitments.KzgBlobHashVersionV1 - 1))
             {
                 TestName = "Incorrect version, correct length",
                 ExpectedResult = false
             };
-            yield return new TestCaseData(MakeArray(32, KzgPolynomialCommitments.KzgBlobHashVersionV1 + 1, 0))
+            yield return new TestCaseData(MakeArray(32, KzgPolynomialCommitments.KzgBlobHashVersionV1 + 1))
             {
                 TestName = "Incorrect version, correct length",
                 ExpectedResult = false
             };
+            yield return new TestCaseData(MakeArray(32, KzgPolynomialCommitments.KzgBlobHashVersionV1))
+            {
+                TestName = "Correct version, correct length",
+                ExpectedResult = true
+            };
         }
     }
 
@@ -494,19 +498,19 @@ static TransactionBuilder<Transaction> MakeTestObject(int blobCount = 1) => Buil
                 TestName = "More than minimum BlobVersionedHashes",
                 ExpectedResult = true
             };
-            yield return new TestCaseData(MakeTestObject(Eip4844Constants.MaxBlobsPerTransaction - 1)
+            yield return new TestCaseData(MakeTestObject((int)(Eip4844Constants.MaxDataGasPerBlock / Eip4844Constants.DataGasPerBlob - 1))
                 .SignedAndResolved().TestObject)
             {
                 TestName = "Less than maximum BlobVersionedHashes",
                 ExpectedResult = true
             };
-            yield return new TestCaseData(MakeTestObject(Eip4844Constants.MaxBlobsPerTransaction)
+            yield return new TestCaseData(MakeTestObject((int)(Eip4844Constants.MaxDataGasPerBlock / Eip4844Constants.DataGasPerBlob))
                 .SignedAndResolved().TestObject)
             {
                 TestName = "Maximum BlobVersionedHashes",
                 ExpectedResult = true
             };
-            yield return new TestCaseData(MakeTestObject(Eip4844Constants.MaxBlobsPerTransaction + 1)
+            yield return new TestCaseData(MakeTestObject((int)(Eip4844Constants.MaxDataGasPerBlock / Eip4844Constants.DataGasPerBlob + 1))
                 .SignedAndResolved().TestObject)
             {
                 TestName = "Too many BlobVersionedHashes",

src/Nethermind/Nethermind.Cli/Console/CliConsole.cs

@@ -25,19 +25,19 @@ public CliConsole()
     {
         _terminal = PrepareConsoleForTerminal();
 
-        System.Console.WriteLine("**********************************************");
-        System.Console.WriteLine();
-        System.Console.WriteLine("Nethermind CLI {0}", ProductInfo.Version);
-        System.Console.WriteLine("  https://github.com/NethermindEth/nethermind");
-        System.Console.WriteLine("  https://nethermind.readthedocs.io/en/latest/");
-        System.Console.WriteLine();
-        System.Console.WriteLine("powered by:");
-        System.Console.WriteLine("  https://github.com/sebastienros/jint");
-        System.Console.WriteLine("  https://github.com/tomakita/Colorful.Console");
-        System.Console.WriteLine("  https://github.com/tonerdo/readline");
-        System.Console.WriteLine();
-        System.Console.WriteLine("**********************************************");
-        System.Console.WriteLine();
+        //System.Console.WriteLine("**********************************************");
+        //System.Console.WriteLine();
+        //System.Console.WriteLine("Nethermind CLI {0}", ProductInfo.Version);
+        //System.Console.WriteLine("  https://github.com/NethermindEth/nethermind");
+        //System.Console.WriteLine("  https://nethermind.readthedocs.io/en/latest/");
+        //System.Console.WriteLine();
+        //System.Console.WriteLine("powered by:");
+        //System.Console.WriteLine("  https://github.com/sebastienros/jint");
+        //System.Console.WriteLine("  https://github.com/tomakita/Colorful.Console");
+        //System.Console.WriteLine("  https://github.com/tonerdo/readline");
+        //System.Console.WriteLine();
+        //System.Console.WriteLine("**********************************************");
+        //System.Console.WriteLine();
     }
 
     protected Terminal GetTerminal()

src/Nethermind/Nethermind.Cli/NodeManager.cs

@@ -63,7 +63,7 @@ public async Task<JsValue> PostJint(string method, params object[] parameters)
             {
                 if (_currentClient is null)
                 {
-                    _cliConsole.WriteErrorLine("[INTERNAL ERROR] JSON RPC client not set.");
+                    //_cliConsole.WriteErrorLine("[INTERNAL ERROR] JSON RPC client not set.");
                 }
                 else
                 {
@@ -115,7 +115,7 @@ public async Task<JsValue> PostJint(string method, params object[] parameters)
             {
                 if (_currentClient is null)
                 {
-                    _cliConsole.WriteErrorLine("[INTERNAL ERROR] JSON RPC client not set.");
+                    //_cliConsole.WriteErrorLine("[INTERNAL ERROR] JSON RPC client not set.");
                 }
                 else
                 {

src/Nethermind/Nethermind.Clique.Test/CliqueBlockProducerTests.cs

@@ -17,7 +17,6 @@
 using Nethermind.Consensus.Rewards;
 using Nethermind.Consensus.Transactions;
 using Nethermind.Consensus.Validators;
-using Nethermind.Consensus.Withdrawals;
 using Nethermind.Core;
 using Nethermind.Core.Crypto;
 using Nethermind.Core.Extensions;
@@ -37,7 +36,6 @@
 using Nethermind.Trie.Pruning;
 using Nethermind.TxPool;
 using NUnit.Framework;
-using BlockTree = Nethermind.Blockchain.BlockTree;
 using Nethermind.Config;
 
 namespace Nethermind.Clique.Test
@@ -123,7 +121,9 @@ public On CreateNode(PrivateKey privateKey, bool withGenesisAlreadyProcessed = f
                 _genesis.Header.Hash = _genesis.Header.CalculateHash();
                 _genesis3Validators.Header.Hash = _genesis3Validators.Header.CalculateHash();
 
-                TransactionProcessor transactionProcessor = new(goerliSpecProvider, stateProvider, new VirtualMachine(blockhashProvider, specProvider, nodeLogManager), nodeLogManager);
+                TransactionProcessor transactionProcessor = new(goerliSpecProvider, stateProvider,
+                    new VirtualMachine(blockhashProvider, specProvider, nodeLogManager),
+                    nodeLogManager);
                 BlockProcessor blockProcessor = new(
                     goerliSpecProvider,
                     Always.Valid,

src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Numerics;
 using Nethermind.Blockchain;
 using Nethermind.Blockchain.Receipts;
@@ -13,11 +14,13 @@
 using Nethermind.Core.Crypto;
 using Nethermind.Core.Specs;
 using Nethermind.Crypto;
+using Nethermind.Evm;
 using Nethermind.Evm.Tracing;
 using Nethermind.Int256;
 using Nethermind.Logging;
 using Nethermind.Specs.Forks;
 using Nethermind.State;
+using Metrics = Nethermind.Blockchain.Metrics;
 
 namespace Nethermind.Consensus.Processing;
 
@@ -221,8 +224,15 @@ protected virtual TxReceipt[] ProcessBlock(
 
         _receiptsTracer.SetOtherTracer(blockTracer);
         _receiptsTracer.StartNewBlockTrace(block);
+
         TxReceipt[] receipts = _blockTransactionsExecutor.ProcessTransactions(block, options, _receiptsTracer, spec);
 
+        if (spec.IsEip4844Enabled)
+        {
+            block.Header.DataGasUsed = DataGasCalculator.CalculateDataGas(
+                block.Transactions.Sum(tx => tx.BlobVersionedHashes?.Length ?? 0));
+        }
+
         block.Header.ReceiptsRoot = receipts.GetReceiptsRoot(spec, block.ReceiptsRoot);
         ApplyMinerRewards(block, blockTracer, spec);
         _withdrawalProcessor.ProcessWithdrawals(block, spec);
@@ -258,6 +268,7 @@ private Block PrepareBlockForProcessing(Block suggestedBlock)
             bh.GasLimit,
             bh.Timestamp,
             bh.ExtraData,
+            bh.DataGasUsed,
             bh.ExcessDataGas)
         {
             Bloom = Bloom.Empty,