Add support for eth68 (#4866)

* eth68 implementation

* test

* refactor serializer

* change typesSize back

* Tests and enable capability

* serializer tests

* Fix whitespaces

* fix whitespaces

* Refactor

* Fix serialization & peer drop

* RequestTransactions68

* Metrics & logs

* Reduce paket max size

* Calculate length properly

* fix tests & length

* huge transaction test

* fix spaces

* Make size calculated once per transaction

* Refactor & optimizations

* Fix hive tests

* Copyright fix

* ubuntu 20.04 workflow

* Revert "ubuntu 20.04 workflow"

This reverts commit 77701f6.

* Revert "Revert "ubuntu 20.04 workflow""

This reverts commit 648e156.

* Remove metrics & for loop

* Remove test & RequestTransactionsEth68

* EncodeList implementation

* Fix test

* Revert renaming

* Rename write

* Remove Request68

* Fix run-nethermind-tests.yml

* Fix run-nethermind-tests.yml

Co-authored-by: Marcin Sobczak <marcindsobczak@gmail.com>

src/Nethermind/Nethermind.AccountAbstraction/Data/UserOperationDecoder.cs

@@ -1,5 +1,5 @@
 // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
-// SPDX-License-Identifier: LGPL-3.0-only 
+// SPDX-License-Identifier: LGPL-3.0-only
 
 using Nethermind.AccountAbstraction.Network;
 using Nethermind.Core;

src/Nethermind/Nethermind.Core/Transaction.cs

@@ -65,6 +65,15 @@ public class Transaction
         /// <remarks>Used for sorting in edge cases.</remarks>
         public ulong PoolIndex { get; set; }
 
+        private int? _size = null;
+        /// <summary>
+        /// Encoded transaction length
+        /// </summary>
+        public int GetLength(ITransactionSizeCalculator sizeCalculator)
+        {
+            return _size ??= sizeCalculator.GetLength(this);
+        }
+
         public string ToShortString()
         {
             string gasPriceString =
@@ -114,7 +123,16 @@ public string ToString(string indent)
     public class GeneratedTransaction : Transaction { }
 
     /// <summary>
-    /// System transaction that is to be executed by the node without including in the block. 
+    /// System transaction that is to be executed by the node without including in the block.
     /// </summary>
     public class SystemTransaction : Transaction { }
+
+    /// <summary>
+    /// Used inside Transaction::GetSize to calculate encoded transaction size
+    /// </summary>
+    /// <remarks>Created because of cyclic dependencies between Core and Rlp modules</remarks>
+    public interface ITransactionSizeCalculator
+    {
+        int GetLength(Transaction tx);
+    }
 }

src/Nethermind/Nethermind.Crypto/KeccakRlpStream.cs

@@ -1,7 +1,9 @@
 // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
-// SPDX-License-Identifier: LGPL-3.0-only 
+// SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Runtime.InteropServices;
 using Nethermind.Core.Crypto;
 using Nethermind.Serialization.Rlp;
@@ -28,6 +30,11 @@ public override void Write(Span<byte> bytesToWrite)
             _keccakHash.Update(bytesToWrite, 0, bytesToWrite.Length);
         }
 
+        public override void Write(IReadOnlyList<byte> bytesToWrite)
+        {
+            _keccakHash.Update(bytesToWrite.ToArray(), 0, bytesToWrite.Count);
+        }
+
         public override void WriteByte(byte byteToWrite)
         {
             _keccakHash.Update(MemoryMarshal.CreateSpan(ref byteToWrite, 1), 0, 1);

src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs

@@ -207,8 +207,9 @@ await InitPeer().ContinueWith(initPeerTask =>
             // we can't add eth67 capability as default, because it needs snap protocol for syncing (GetNodeData is
             // no longer available). Eth67 should be added if snap is enabled OR sync is finished
             _api.ProtocolsManager!.AddSupportedCapability(new Capability(Protocol.Eth, 67));
+            _api.ProtocolsManager!.AddSupportedCapability(new Capability(Protocol.Eth, 68));
         }
-        else if (_logger.IsDebug) _logger.Debug("Skipped enabling eth67 capability");
+        else if (_logger.IsDebug) _logger.Debug("Skipped enabling eth67 & eth68 capabilities");
 
         if (_syncConfig.SnapSync && !stateSyncFinished)
         {

src/Nethermind/Nethermind.Network.Enr/NodeRecordSigner.cs

@@ -1,5 +1,5 @@
 // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
-// SPDX-License-Identifier: LGPL-3.0-only 
+// SPDX-License-Identifier: LGPL-3.0-only
 
 using System.Net;
 using Nethermind.Core.Crypto;

src/Nethermind/Nethermind.Network.Test/Builders/SerializationBuilder.cs

@@ -87,6 +87,12 @@ public SerializationBuilder WithEth66()
                 .With(new Network.P2P.Subprotocols.Eth.V66.Messages.ReceiptsMessageSerializer(new ReceiptsMessageSerializer(MainnetSpecProvider.Instance)));
         }
 
+        public SerializationBuilder WithEth68()
+        {
+            return WithEth66()
+                .With(new Network.P2P.Subprotocols.Eth.V68.Messages.NewPooledTransactionHashesMessageSerializer());
+        }
+
         public SerializationBuilder WithDiscovery(PrivateKey privateKey)
         {
             Ecdsa ecdsa = new();

src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V68/Eth68ProtocolHandlerTests.cs

@@ -0,0 +1,189 @@
+// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using System.Collections.Generic;
+using System.Net;
+using DotNetty.Buffers;
+using FluentAssertions;
+using Nethermind.Consensus;
+using Nethermind.Core;
+using Nethermind.Core.Crypto;
+using Nethermind.Core.Specs;
+using Nethermind.Core.Test.Builders;
+using Nethermind.Core.Timers;
+using Nethermind.Logging;
+using Nethermind.Network.P2P;
+using Nethermind.Network.P2P.Subprotocols;
+using Nethermind.Network.P2P.Subprotocols.Eth.V62.Messages;
+using Nethermind.Network.P2P.Subprotocols.Eth.V65;
+using Nethermind.Network.P2P.Subprotocols.Eth.V66.Messages;
+using Nethermind.Network.P2P.Subprotocols.Eth.V68;
+using Nethermind.Network.P2P.Subprotocols.Eth.V68.Messages;
+using Nethermind.Network.Rlpx;
+using Nethermind.Network.Test.Builders;
+using Nethermind.Serialization.Rlp;
+using Nethermind.Stats;
+using Nethermind.Stats.Model;
+using Nethermind.Synchronization;
+using Nethermind.TxPool;
+using NSubstitute;
+using NUnit.Framework;
+
+namespace Nethermind.Network.Test.P2P.Subprotocols.Eth.V68;
+
+public class Eth68ProtocolHandlerTests
+{
+    private ISession _session;
+    private IMessageSerializationService _svc;
+    private ISyncServer _syncManager;
+    private ITxPool _transactionPool;
+    private IPooledTxsRequestor _pooledTxsRequestor;
+    private IGossipPolicy _gossipPolicy;
+    private ISpecProvider _specProvider;
+    private Block _genesisBlock;
+    private Eth68ProtocolHandler _handler;
+
+    [SetUp]
+    public void Setup()
+    {
+        _svc = Build.A.SerializationService().WithEth68().TestObject;
+
+        NetworkDiagTracer.IsEnabled = true;
+
+        _session = Substitute.For<ISession>();
+        Node node = new(TestItem.PublicKeyA, new IPEndPoint(IPAddress.Broadcast, 30303));
+        _session.Node.Returns(node);
+        _syncManager = Substitute.For<ISyncServer>();
+        _transactionPool = Substitute.For<ITxPool>();
+        _pooledTxsRequestor = Substitute.For<IPooledTxsRequestor>();
+        _specProvider = Substitute.For<ISpecProvider>();
+        _gossipPolicy = Substitute.For<IGossipPolicy>();
+        _genesisBlock = Build.A.Block.Genesis.TestObject;
+        _syncManager.Head.Returns(_genesisBlock.Header);
+        _syncManager.Genesis.Returns(_genesisBlock.Header);
+        ITimerFactory timerFactory = Substitute.For<ITimerFactory>();
+        _handler = new Eth68ProtocolHandler(
+            _session,
+            _svc,
+            new NodeStatsManager(timerFactory, LimboLogs.Instance),
+            _syncManager,
+            _transactionPool,
+            _pooledTxsRequestor,
+            _gossipPolicy,
+            _specProvider,
+            LimboLogs.Instance);
+        _handler.Init();
+    }
+
+    [TearDown]
+    public void TearDown()
+    {
+        _handler.Dispose();
+    }
+
+    [Test]
+    public void Metadata_correct()
+    {
+        _handler.ProtocolCode.Should().Be("eth");
+        _handler.Name.Should().Be("eth68");
+        _handler.ProtocolVersion.Should().Be(68);
+        _handler.MessageIdSpaceSize.Should().Be(17);
+        _handler.IncludeInTxPool.Should().BeTrue();
+        _handler.ClientId.Should().Be(_session.Node?.ClientId);
+        _handler.HeadHash.Should().BeNull();
+        _handler.HeadNumber.Should().Be(0);
+    }
+
+    [TestCase(0)]
+    [TestCase(1)]
+    [TestCase(2)]
+    [TestCase(100)]
+    public void Can_handle_NewPooledTransactions_message(int txCount)
+    {
+        GenerateLists(txCount, out List<byte> types, out List<int> sizes, out List<Keccak> hashes);
+
+        var msg = new NewPooledTransactionHashesMessage68(types, sizes, hashes);
+
+        HandleIncomingStatusMessage();
+        HandleZeroMessage(msg, Eth68MessageCode.NewPooledTransactionHashes);
+        _pooledTxsRequestor.Received().RequestTransactionsEth66(Arg.Any<Action<GetPooledTransactionsMessage>>(),
+            Arg.Any<IReadOnlyList<Keccak>>());
+    }
+
+    [TestCase(true)]
+    [TestCase(false)]
+    public void Should_throw_when_sizes_doesnt_match(bool removeSize)
+    {
+        GenerateLists(4, out List<byte> types, out List<int> sizes, out List<Keccak> hashes);
+
+        if (removeSize)
+        {
+            sizes.RemoveAt(sizes.Count - 1);
+        }
+        else
+        {
+            types.RemoveAt(sizes.Count - 1);
+        }
+
+        var msg = new NewPooledTransactionHashesMessage68(types, sizes, hashes);
+
+        HandleIncomingStatusMessage();
+        Action action = () => HandleZeroMessage(msg, Eth68MessageCode.NewPooledTransactionHashes);
+        action.Should().Throw<SubprotocolException>();
+    }
+
+    [Test]
+    public void Should_process_huge_transaction()
+    {
+        Transaction tx = Build.A.Transaction.WithType(TxType.EIP1559).WithData(new byte[2 * 1024 * 1024])
+            .WithHash(TestItem.KeccakA).TestObject;
+
+        TxDecoder txDecoder = new();
+
+        var msg = new NewPooledTransactionHashesMessage68(new[] { (byte)tx.Type },
+            new[] { txDecoder.GetLength(tx, RlpBehaviors.None) }, new[] { tx.Hash });
+
+        HandleIncomingStatusMessage();
+
+        HandleZeroMessage(msg, Eth68MessageCode.NewPooledTransactionHashes);
+        _pooledTxsRequestor.Received().RequestTransactionsEth66(Arg.Any<Action<GetPooledTransactionsMessage>>(),
+            Arg.Any<IReadOnlyList<Keccak>>());
+    }
+
+    private void HandleIncomingStatusMessage()
+    {
+        var statusMsg = new StatusMessage();
+        statusMsg.GenesisHash = _genesisBlock.Hash;
+        statusMsg.BestHash = _genesisBlock.Hash;
+
+        IByteBuffer statusPacket = _svc.ZeroSerialize(statusMsg);
+        statusPacket.ReadByte();
+        _handler.HandleMessage(new ZeroPacket(statusPacket) { PacketType = 0 });
+    }
+
+    private void HandleZeroMessage<T>(T msg, byte messageCode) where T : MessageBase
+    {
+        IByteBuffer getBlockHeadersPacket = _svc.ZeroSerialize(msg);
+        getBlockHeadersPacket.ReadByte();
+        _handler.HandleMessage(new ZeroPacket(getBlockHeadersPacket) { PacketType = messageCode });
+    }
+
+    private void GenerateLists(int txCount, out List<byte> types, out List<int> sizes, out List<Keccak> hashes)
+    {
+        TxDecoder txDecoder = new();
+        types = new();
+        sizes = new();
+        hashes = new();
+
+        for (int i = 0; i < txCount; ++i)
+        {
+            Transaction tx = Build.A.Transaction.WithType((TxType)(i % 3)).WithData(new byte[i])
+                .WithHash(i % 2 == 0 ? TestItem.KeccakA : TestItem.KeccakB).TestObject;
+
+            types.Add((byte)tx.Type);
+            sizes.Add(txDecoder.GetLength(tx, RlpBehaviors.None));
+            hashes.Add(tx.Hash);
+        }
+    }
+}

src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V68/NewPooledTransactionHashesMessageSerializerTests.cs

@@ -0,0 +1,61 @@
+// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System.Linq;
+using Nethermind.Core;
+using Nethermind.Core.Crypto;
+using Nethermind.Core.Test.Builders;
+using Nethermind.Network.P2P.Subprotocols.Eth.V68.Messages;
+using NUnit.Framework;
+
+namespace Nethermind.Network.Test.P2P.Subprotocols.Eth.V68;
+
+[TestFixture, Parallelizable(ParallelScope.All)]
+public class NewPooledTransactionHashesMessageSerializerTests
+{
+    private static void Test(TxType[] types, int[] sizes, Keccak[] hashes, string expected = null)
+    {
+        NewPooledTransactionHashesMessage68 message = new(types.Select(t => (byte)t).ToList(), sizes, hashes);
+        NewPooledTransactionHashesMessageSerializer serializer = new();
+
+        SerializerTester.TestZero(serializer, message, expected);
+    }
+
+    [Test]
+    public void Roundtrip()
+    {
+        TxType[] types = { TxType.Legacy, TxType.AccessList, TxType.EIP1559 };
+        int[] sizes = { 5, 10, 1500 };
+        Keccak[] hashes = { TestItem.KeccakA, TestItem.KeccakB, TestItem.KeccakC };
+        Test(types, sizes, hashes);
+    }
+
+    [Test]
+    public void Empty_serialization()
+    {
+        TxType[] types = { };
+        int[] sizes = { };
+        Keccak[] hashes = { };
+        Test(types, sizes, hashes, "c380c0c0");
+    }
+
+    [Test]
+    public void Empty_hashes_serialization()
+    {
+        TxType[] types = { TxType.EIP1559 };
+        int[] sizes = { 10 };
+        Keccak[] hashes = { };
+        Test(types, sizes, hashes, "c402c10ac0");
+    }
+
+    [Test]
+    public void Non_empty_serialization()
+    {
+        TxType[] types = { TxType.AccessList };
+        int[] sizes = { 2 };
+        Keccak[] hashes = { TestItem.KeccakA };
+        Test(types, sizes, hashes,
+            "e5" + "01" + "c102" + "e1a0" + TestItem.KeccakA.ToString(false));
+    }
+
+}

src/Nethermind/Nethermind.Network/Metrics.cs

@@ -156,6 +156,12 @@ public static class Metrics
         [Description("Number of eth.65 NewPooledTransactionHashes messages sent")]
         public static long Eth65NewPooledTransactionHashesSent { get; set; }
 
+        [Description("Number of eth.68 NewPooledTransactionHashes messages received")]
+        public static long Eth68NewPooledTransactionHashesReceived { get; set; }
+
+        [Description("Number of eth.68 NewPooledTransactionHashes messages sent")]
+        public static long Eth68NewPooledTransactionHashesSent { get; set; }
+
         [Description("Number of eth.65 GetPooledTransactions messages received")]
         public static long Eth65GetPooledTransactionsReceived { get; set; }
 

src/Nethermind/Nethermind.Network/P2P/NettyRlpStream.cs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
+using System.Collections.Generic;
 using DotNetty.Buffers;
 using Nethermind.Serialization.Rlp;
 
@@ -31,6 +32,20 @@ public override void Write(Span<byte> bytesToWrite)
             _buffer.SetWriterIndex(newWriterIndex);
         }
 
+        public override void Write(IReadOnlyList<byte> bytesToWrite)
+        {
+            _buffer.EnsureWritable(bytesToWrite.Count, true);
+            Span<byte> target =
+                _buffer.Array.AsSpan(_buffer.ArrayOffset + _buffer.WriterIndex, bytesToWrite.Count);
+            for (int i = 0; i < bytesToWrite.Count; ++i)
+            {
+                target[i] = bytesToWrite[i];
+            }
+
+            int newWriterIndex = _buffer.WriterIndex + bytesToWrite.Count;
+            _buffer.SetWriterIndex(newWriterIndex);
+        }
+
         public override void WriteByte(byte byteToWrite)
         {
             _buffer.EnsureWritable(1, true);

src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/IPooledTxsRequestor.cs

@@ -1,5 +1,5 @@
 // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
-// SPDX-License-Identifier: LGPL-3.0-only 
+// SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
 using System.Collections.Generic;