Merge branch 'refactor/generic-send-request' into perf/limited-reques…

…t-size

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "gitbook/docs"]
 	path = gitbook/docs
 	url = https://github.com/NethermindEth/docs.git
+[submodule "src/Math.Gmp.Native"]
+	path = src/Math.Gmp.Native
+	url = https://github.com/NethermindEth/Math.Gmp.Native.git

CONTRIBUTING.md

@@ -44,7 +44,12 @@ Please do not:
 
 ## Branch Naming
 
-Branch names must follow `snake_case` pattern. Follow the pattern `feature/<name>` or `fix/<name>` `(folder/<name>)` when it is possible and add issue reference if applicable.
+Branch names must follow `snake_case` pattern. Follow the pattern `<projectIfAny>/<typeOfTheChange>/<issueNo>_<title>` when it is possible and add issue reference if applicable. For example:
+
+* feature/1234_issue_title
+* shanghai/feature/1234_issue_title
+* fix/2345_bug_title
+* shanghai/refactor/4567_title
 
 ## File Headers
 

Dockerfile

@@ -8,8 +8,10 @@ ARG BUILDPLATFORM
 COPY . .
 
 RUN if [ "$TARGETARCH" = "amd64" ] ; \
-    then dotnet publish src/Nethermind/Nethermind.Runner -r $TARGETOS-x64 -c release -o out ; \
-    else dotnet publish src/Nethermind/Nethermind.Runner -r $TARGETOS-$TARGETARCH -c release -o out ; \
+    then git submodule update --init src/Math.Gmp.Native && \
+    dotnet publish src/Nethermind/Nethermind.Runner -r $TARGETOS-x64 -c release -o out ; \
+    else git submodule update --init src/Math.Gmp.Native && \
+    dotnet publish src/Nethermind/Nethermind.Runner -r $TARGETOS-$TARGETARCH -c release -o out ; \
     fi
 
 FROM --platform=$TARGETPLATFORM mcr.microsoft.com/dotnet/aspnet:6.0-jammy

Dockerfile.dev

@@ -8,10 +8,12 @@ ARG BUILDPLATFORM
 COPY . .
 
 RUN if [ "$TARGETARCH" = "amd64" ] ; \
-    then dotnet tool install JetBrains.dotTrace.GlobalTools --version 2021.3.2 --tool-path /tmp/ && \
+    then git submodule update --init src/Math.Gmp.Native && \
+    dotnet tool install JetBrains.dotTrace.GlobalTools --version 2021.3.2 --tool-path /tmp/ && \
     dotnet add src/Nethermind/Nethermind.Runner package JetBrains.dotMemory.Console.$TARGETOS-x64 --version 2021.3.2 --package-directory /tmp/ && \
     dotnet publish src/Nethermind/Nethermind.Runner -r $TARGETOS-x64 -c release -o out ; \
-    else dotnet tool install JetBrains.dotTrace.GlobalTools --version 2021.3.2 --tool-path /tmp/ && \
+    else git submodule update --init src/Math.Gmp.Native && \
+    dotnet tool install JetBrains.dotTrace.GlobalTools --version 2021.3.2 --tool-path /tmp/ && \
     dotnet add src/Nethermind/Nethermind.Runner package JetBrains.dotMemory.Console.$TARGETOS-$TARGETARCH --version 2021.3.2 --package-directory /tmp/ && \
     dotnet publish src/Nethermind/Nethermind.Runner -r $TARGETOS-$TARGETARCH -c release -o out ; \
     fi

Dockerfile.gmp

@@ -0,0 +1,17 @@
+FROM debian@sha256:acf7795dc91df17e10effee064bd229580a9c34213b4dba578d64768af5d8c51 AS gmp
+WORKDIR /source
+
+RUN apt-get update && apt-get install -y mercurial autoconf automake libtool texinfo build-essential bison
+
+RUN hg clone https://gmplib.org/repo/gmp-6.2/ gmp && \
+    cd gmp && \
+    ./.bootstrap && \
+    ./configure --disable-static --enable-shared && \
+    make -j16
+
+RUN strip gmp/.libs/libgmp.so
+
+
+FROM debian@sha256:acf7795dc91df17e10effee064bd229580a9c34213b4dba578d64768af5d8c51 AS libgmp
+WORKDIR /nethermind
+COPY --from=gmp /source/gmp/.libs/libgmp.so .

Dockerfile.stateRunner

@@ -3,10 +3,11 @@ FROM mcr.microsoft.com/dotnet/sdk:6.0-alpine AS build
 COPY . .
 
 RUN apk add git && \
+ git submodule update --init src/Math.Gmp.Native && \
  dotnet publish src/Nethermind/Nethermind.State.Test.Runner -c release -o out
 
 FROM mcr.microsoft.com/dotnet/sdk:6.0-alpine
 
 COPY --from=build /out .
 
-ENTRYPOINT [ "./nethtest" ]
+ENTRYPOINT [ "./nethtest" ]

src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs

@@ -1,16 +1,16 @@
 //  Copyright (c) 2021 Demerzel Solutions Limited
 //  This file is part of the Nethermind library.
-// 
+//
 //  The Nethermind library is free software: you can redistribute it and/or modify
 //  it under the terms of the GNU Lesser General Public License as published by
 //  the Free Software Foundation, either version 3 of the License, or
 //  (at your option) any later version.
-// 
+//
 //  The Nethermind library is distributed in the hope that it will be useful,
 //  but WITHOUT ANY WARRANTY; without even the implied warranty of
 //  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 //  GNU Lesser General Public License for more details.
-// 
+//
 //  You should have received a copy of the GNU Lesser General Public License
 //  along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.
 
@@ -94,5 +94,8 @@ public interface ISyncConfig : IConfig
         [ConfigItem(Description = "[ONLY FOR MISSING RECEIPTS ISSUE] Turns on receipts validation that checks for ones that might be missing due to previous bug. It downloads them from network if needed." +
                                   "If used please check that PivotNumber is same as original used when syncing the node as its used as a cut-off point.", DefaultValue = "false")]
         public bool FixReceipts { get; set; }
+
+        [ConfigItem(Description = "Disable some optimization and run a more extensive sync. Useful for broken sync state but normally not needed", DefaultValue = "false")]
+        public bool StrictMode { get; set; }
     }
 }

src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs

@@ -56,6 +56,7 @@ public bool SynchronizationEnabled
         public bool WitnessProtocolEnabled { get; set; } = false;
         public bool SnapSync { get; set; } = false;
         public bool FixReceipts { get; set; } = false;
+        public bool StrictMode { get; set; } = false;
         public bool BlockGossipEnabled { get; set; } = true;
 
         public override string ToString()

src/Nethermind/Nethermind.Evm/Nethermind.Evm.csproj

@@ -10,12 +10,12 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\..\Math.Gmp.Native\MathGmp.Native\MathGmp.Native.csproj" />
     <ProjectReference Include="..\Nethermind.Core\Nethermind.Core.csproj" />
     <ProjectReference Include="..\Nethermind.Serialization.Rlp\Nethermind.Serialization.Rlp.csproj" />
     <ProjectReference Include="..\Nethermind.State\Nethermind.State.csproj" />
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="FastEnum" Version="1.7.0" />
-    <PackageReference Include="Nethermind.Gmp" Version="1.0.0" />
   </ItemGroup>
 </Project>

src/Nethermind/Nethermind.JsonRpc/JsonRpcLocalStats.cs

@@ -1,16 +1,16 @@
 //  Copyright (c) 2021 Demerzel Solutions Limited
 //  This file is part of the Nethermind library.
-// 
+//
 //  The Nethermind library is free software: you can redistribute it and/or modify
 //  it under the terms of the GNU Lesser General Public License as published by
 //  the Free Software Foundation, either version 3 of the License, or
 //  (at your option) any later version.
-// 
+//
 //  The Nethermind library is distributed in the hope that it will be useful,
 //  but WITHOUT ANY WARRANTY; without even the implied warranty of
 //  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 //  GNU Lesser General Public License for more details.
-// 
+//
 //  You should have received a copy of the GNU Lesser General Public License
 //  along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.
 
@@ -125,11 +125,11 @@ private void BuildReport()
 
             const string reportHeader = "method                                  | " +
                                         "successes | " +
-                                        " avg time | " +
-                                        " max time | " +
+                                        " avg time (µs) | " +
+                                        " max time (µs) | " +
                                         "   errors | " +
-                                        " avg time | " +
-                                        " max time |" +
+                                        " avg time (µs) | " +
+                                        " max time (µs) |" +
                                         " avg size |" +
                                         " total size |";
 
@@ -168,23 +168,21 @@ private void BuildReport()
 
         private void Swap()
         {
-            var temp = _currentStats;
-            _currentStats = _previousStats;
-            _previousStats = temp;
+            (_currentStats, _previousStats) = (_previousStats, _currentStats);
         }
 
         [Pure]
         private string PrepareReportLine(in string key, MethodStats methodStats)
         {
-            string reportLine = $"{key.PadRight(40)}| " +
-                                $"{methodStats.Successes.ToString().PadLeft(9)} | " +
-                                $"{methodStats.AvgTimeOfSuccesses.ToString("0", CultureInfo.InvariantCulture).PadLeft(9)} | " +
-                                $"{methodStats.MaxTimeOfSuccess.ToString(CultureInfo.InvariantCulture).PadLeft(9)} | " +
-                                $"{methodStats.Errors.ToString().PadLeft(9)} | " +
-                                $"{methodStats.AvgTimeOfErrors.ToString("0", CultureInfo.InvariantCulture).PadLeft(9)} | " +
-                                $"{methodStats.MaxTimeOfError.ToString(CultureInfo.InvariantCulture).PadLeft(9)} | " +
-                                $"{methodStats.AvgSize.ToString("0", CultureInfo.InvariantCulture).PadLeft(8)} | " +
-                                $"{methodStats.TotalSize.ToString("0", CultureInfo.InvariantCulture).PadLeft(10)} | ";
+            string reportLine = $"{key,-40}| " +
+                                $"{methodStats.Successes.ToString(),9} | " +
+                                $"{methodStats.AvgTimeOfSuccesses.ToString("0", CultureInfo.InvariantCulture),14} | " +
+                                $"{methodStats.MaxTimeOfSuccess.ToString(CultureInfo.InvariantCulture),14} | " +
+                                $"{methodStats.Errors.ToString(),9} | " +
+                                $"{methodStats.AvgTimeOfErrors.ToString("0", CultureInfo.InvariantCulture),14} | " +
+                                $"{methodStats.MaxTimeOfError.ToString(CultureInfo.InvariantCulture),14} | " +
+                                $"{methodStats.AvgSize.ToString("0", CultureInfo.InvariantCulture),8} | " +
+                                $"{methodStats.TotalSize.ToString("0", CultureInfo.InvariantCulture),10} | ";
 
             return reportLine;
         }

src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.DelayBlockImprovementContextFactory.cs

@@ -32,41 +32,42 @@ private class DelayBlockImprovementContextFactory : IBlockImprovementContextFact
     {
         private readonly IManualBlockProductionTrigger _productionTrigger;
         private readonly TimeSpan _timeout;
-        private readonly int _delay;
+        private readonly TimeSpan _delay;
 
-        public DelayBlockImprovementContextFactory(IManualBlockProductionTrigger productionTrigger, TimeSpan timeout, int delay)
+        public DelayBlockImprovementContextFactory(IManualBlockProductionTrigger productionTrigger, TimeSpan timeout, TimeSpan delay)
         {
             _productionTrigger = productionTrigger;
             _timeout = timeout;
             _delay = delay;
         }
 
-        public IBlockImprovementContext StartBlockImprovementContext(Block currentBestBlock, BlockHeader parentHeader, PayloadAttributes payloadAttributes) =>
-            new DelayBlockImprovementContext(currentBestBlock, _productionTrigger, _timeout, parentHeader, payloadAttributes, _delay);
+        public IBlockImprovementContext StartBlockImprovementContext(Block currentBestBlock, BlockHeader parentHeader, PayloadAttributes payloadAttributes, DateTimeOffset startDateTime) =>
+            new DelayBlockImprovementContext(currentBestBlock, _productionTrigger, _timeout, parentHeader, payloadAttributes, _delay, startDateTime);
     }
 
     private class DelayBlockImprovementContext : IBlockImprovementContext
     {
         private CancellationTokenSource? _cancellationTokenSource;
 
-        public DelayBlockImprovementContext(
-            Block currentBestBlock,
+        public DelayBlockImprovementContext(Block currentBestBlock,
             IManualBlockProductionTrigger blockProductionTrigger,
             TimeSpan timeout,
             BlockHeader parentHeader,
             PayloadAttributes payloadAttributes,
-            int delay)
+            TimeSpan delay,
+            DateTimeOffset startDateTime)
         {
             _cancellationTokenSource = new CancellationTokenSource(timeout);
             CurrentBestBlock = currentBestBlock;
+            StartDateTime = startDateTime;
             ImprovementTask = BuildBlock(blockProductionTrigger, parentHeader, payloadAttributes, delay, _cancellationTokenSource.Token);
         }
 
         private async Task<Block?> BuildBlock(
             IManualBlockProductionTrigger blockProductionTrigger,
             BlockHeader parentHeader,
             PayloadAttributes payloadAttributes,
-            int delay,
+            TimeSpan delay,
             CancellationToken cancellationToken)
         {
             Block? block = await blockProductionTrigger.BuildBlock(parentHeader, cancellationToken, NullBlockTracer.Instance, payloadAttributes);
@@ -80,11 +81,13 @@ public DelayBlockImprovementContext(
         }
 
         public Task<Block?> ImprovementTask { get; }
-
         public Block? CurrentBestBlock { get; private set; }
+        public bool Disposed { get; private set; }
+        public DateTimeOffset StartDateTime { get; }
 
         public void Dispose()
         {
+            Disposed = true;
             CancellationTokenExtensions.CancelDisposeAndClear(ref _cancellationTokenSource);
         }
     }

src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.HelperFunctions.cs

@@ -1,19 +1,19 @@
 //  Copyright (c) 2021 Demerzel Solutions Limited
 //  This file is part of the Nethermind library.
-// 
+//
 //  The Nethermind library is free software: you can redistribute it and/or modify
 //  it under the terms of the GNU Lesser General Public License as published by
 //  the Free Software Foundation, either version 3 of the License, or
 //  (at your option) any later version.
-// 
+//
 //  The Nethermind library is distributed in the hope that it will be useful,
 //  but WITHOUT ANY WARRANTY; without even the implied warranty of
 //  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 //  GNU Lesser General Public License for more details.
-// 
+//
 //  You should have received a copy of the GNU Lesser General Public License
 //  along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.
-// 
+//
 
 using System;
 using System.Collections.Generic;
@@ -59,12 +59,10 @@ private void AssertExecutionStatusChanged(IEngineRpcModule rpc, Keccak headBlock
         private (UInt256, UInt256) AddTransactions(MergeTestBlockchain chain, ExecutionPayloadV1 executePayloadRequest,
             PrivateKey from, Address to, uint count, int value, out BlockHeader parentHeader)
         {
-            Transaction[] transactions = BuildTransactions(chain, executePayloadRequest.ParentHash, from, to, count, value,
-                out Account accountFrom, out parentHeader);
+            Transaction[] transactions = BuildTransactions(chain, executePayloadRequest.ParentHash, from, to, count, value, out Account accountFrom, out parentHeader);
             executePayloadRequest.SetTransactions(transactions);
             UInt256 totalValue = ((int)(count * value)).GWei();
-            return (accountFrom.Balance - totalValue,
-                chain.StateReader.GetBalance(parentHeader.StateRoot!, to) + totalValue);
+            return (accountFrom.Balance - totalValue, chain.StateReader.GetBalance(parentHeader.StateRoot!, to) + totalValue);
         }
 
         private Transaction[] BuildTransactions(MergeTestBlockchain chain, Keccak parentHash, PrivateKey from,
@@ -82,11 +80,10 @@ Transaction BuildTransaction(uint index, Account senderAccount) =>
                     .TestObject;
 
             parentHeader = chain.BlockTree.FindHeader(parentHash, BlockTreeLookupOptions.None)!;
-            Account account = chain.StateReader.GetAccount(parentHeader.StateRoot!, @from.Address)!;
+            Account account = chain.StateReader.GetAccount(parentHeader.StateRoot!, from.Address)!;
             accountFrom = account;
 
-            return Enumerable.Range(0, (int)count)
-                .Select(i => BuildTransaction((uint)i, account)).ToArray();
+            return Enumerable.Range(0, (int)count).Select(i => BuildTransaction((uint)i, account)).ToArray();
         }
 
         private ExecutionPayloadV1 CreateParentBlockRequestOnHead(IBlockTree blockTree)
@@ -141,8 +138,7 @@ private static ExecutionPayloadV1[] CreateBlockRequestBranch(ExecutionPayloadV1
         private Block? RunForAllBlocksInBranch(IBlockTree blockTree, Keccak blockHash, Func<Block, bool> shouldStop,
             bool requireCanonical)
         {
-            BlockTreeLookupOptions options =
-                requireCanonical ? BlockTreeLookupOptions.RequireCanonical : BlockTreeLookupOptions.None;
+            BlockTreeLookupOptions options = requireCanonical ? BlockTreeLookupOptions.RequireCanonical : BlockTreeLookupOptions.None;
             Block? current = blockTree.FindBlock(blockHash, options);
             while (current is not null && !shouldStop(current))
             {
@@ -160,8 +156,7 @@ private static TestCaseData GetNewBlockRequestBadDataTestCase<T>(
             Action<ExecutionPayloadV1> wrongValueSetter = r => setter(r, wrongValue);
             return new TestCaseData(wrongValueSetter)
             {
-                TestName =
-                    $"executePayload_rejects_incorrect_{propertyAccess.GetName().ToLower()}({wrongValue?.ToString()})"
+                TestName = $"executePayload_rejects_incorrect_{propertyAccess.GetName().ToLower()}({wrongValue?.ToString()})"
             };
         }
 

src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.MockBlockImprovementContextFactory.cs

@@ -0,0 +1,52 @@
+//  Copyright (c) 2021 Demerzel Solutions Limited
+//  This file is part of the Nethermind library.
+//
+//  The Nethermind library is free software: you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as published by
+//  the Free Software Foundation, either version 3 of the License, or
+//  (at your option) any later version.
+//
+//  The Nethermind library is distributed in the hope that it will be useful,
+//  but WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//  GNU Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public License
+//  along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.
+//
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Nethermind.Consensus.Producers;
+using Nethermind.Core;
+using Nethermind.Core.Extensions;
+using Nethermind.Evm.Tracing;
+using Nethermind.Merge.Plugin.BlockProduction;
+
+namespace Nethermind.Merge.Plugin.Test;
+
+public partial class EngineModuleTests
+{
+    private class MockBlockImprovementContextFactory : IBlockImprovementContextFactory
+    {
+        public IBlockImprovementContext StartBlockImprovementContext(Block currentBestBlock, BlockHeader parentHeader, PayloadAttributes payloadAttributes, DateTimeOffset startDateTime) =>
+            new MockBlockImprovementContext(currentBestBlock, startDateTime);
+    }
+
+    private class MockBlockImprovementContext : IBlockImprovementContext
+    {
+        public MockBlockImprovementContext(Block currentBestBlock, DateTimeOffset startDateTime)
+        {
+            CurrentBestBlock = currentBestBlock;
+            StartDateTime = startDateTime;
+            ImprovementTask = Task.FromResult((Block?)currentBestBlock);
+        }
+
+        public void Dispose() => Disposed = true;
+        public Task<Block?> ImprovementTask { get; }
+        public Block? CurrentBestBlock { get; }
+        public bool Disposed { get; private set; }
+        public DateTimeOffset StartDateTime { get; }
+    }
+}