Read only memory (#2966)

* optimizations on memory loader * tets runner * remove meory over shared * fix output
NethermindEth · Apr 10, 2021 · 7f6fe55 · 7f6fe55
1 parent eee1e58
commit 7f6fe55
Show file tree

Hide file tree

Showing 68 changed files with 373 additions and 253 deletions.
diff --git a/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs b/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs
@@ -358,7 +358,7 @@ private List<string> RunAssertions(BlockchainTest test, Block headBlock, IStorag
 
                 if (accountState.Balance != balance)
                 {
-                    differences.Add($"{acountAddress} balance exp: {accountState.Balance}, actual: {balance}, diff: {balance - accountState.Balance}");
+                    differences.Add($"{acountAddress} balance exp: {accountState.Balance}, actual: {balance}, diff: {(balance > accountState.Balance ? balance - accountState.Balance : accountState.Balance - balance)}");
                 }
 
                 if (accountState.Nonce != nonce)

diff --git a/src/Nethermind/Ethereum.Test.Base/GeneralTestBase.cs b/src/Nethermind/Ethereum.Test.Base/GeneralTestBase.cs
@@ -60,7 +60,6 @@ protected EthereumTestResult RunTest(GeneralStateTest test)
         protected EthereumTestResult RunTest(GeneralStateTest test, ITxTracer txTracer)
         {
             TestContext.Write($"Running {test.Name} at {DateTime.UtcNow:HH:mm:ss.ffffff}");
-            Stopwatch stopwatch = Stopwatch.StartNew();
             Assert.IsNull(test.LoadFailure, "test data loading failure");
 
             IDb stateDb = new MemDb();
@@ -100,7 +99,9 @@ protected EthereumTestResult RunTest(GeneralStateTest test, ITxTracer txTracer)
             header.StateRoot = test.PostHash;
             header.Hash = Keccak.Compute("1");
 
+            Stopwatch stopwatch = Stopwatch.StartNew();
             transactionProcessor.Execute(test.Transaction, header, txTracer);
+            stopwatch.Stop();
 
             stateProvider.Commit(specProvider.GenesisSpec);
             stateProvider.CommitTree(1);

diff --git a/src/Nethermind/Ethereum.VM.Test/VMTestBase.cs b/src/Nethermind/Ethereum.VM.Test/VMTestBase.cs
@@ -205,8 +205,8 @@ protected void RunTest(VirtualMachineTest test)
                 return;
             }
 
-            Assert.True(Bytes.AreEqual(test.Out, substate.Output),
-                $"Exp: {test.Out.ToHexString(true)} != Actual: {substate.Output.ToHexString(true)}");
+            Assert.True(Bytes.AreEqual(test.Out, substate.Output.ToArray()),
+                $"Exp: {test.Out.ToHexString(true)} != Actual: {substate.Output.ToArray().ToHexString(true)}");
             Assert.AreEqual((long)test.Gas, state.GasAvailable, "gas available");
             foreach (KeyValuePair<Address, AccountState> accountState in test.Post)
             {

diff --git a/src/Nethermind/Nethermind.Core/Crypto/Keccak.cs b/src/Nethermind/Nethermind.Core/Crypto/Keccak.cs
@@ -135,7 +135,7 @@ public static Keccak Compute(byte[]? input)
         }
 
         [DebuggerStepThrough]
-        public static Keccak Compute(Span<byte> input)
+        public static Keccak Compute(ReadOnlySpan<byte> input)
         {
             if (input.Length == 0)
             {

diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs
@@ -381,7 +381,7 @@ public static Span<byte> ComputeHash(ReadOnlySpan<byte> input, int size = HASH_S
             return output;
         }
 
-        public static byte[] ComputeHashBytes(Span<byte> input, int size = HASH_SIZE)
+        public static byte[] ComputeHashBytes(ReadOnlySpan<byte> input, int size = HASH_SIZE)
         {
             var output = new byte[HASH_SIZE];
             ComputeHash(input, output);

diff --git a/src/Nethermind/Nethermind.Core/Extensions/ByteArrayExtensions.cs b/src/Nethermind/Nethermind.Core/Extensions/ByteArrayExtensions.cs
@@ -69,5 +69,19 @@ public static byte[] SliceWithZeroPaddingEmptyOnError(this byte[] bytes, int sta
             Buffer.BlockCopy(bytes, startIndex, slice, 0, copiedFragmentLength);
             return slice;
         }
+
+        public static byte[] SliceWithZeroPaddingEmptyOnError(this ReadOnlySpan<byte> bytes, int startIndex, int length)
+        {
+            int copiedFragmentLength = Math.Min(bytes.Length - startIndex, length);
+            if (copiedFragmentLength <= 0)
+            {
+                return Array.Empty<byte>();
+            }
+
+            byte[] slice = new byte[length];
+
+            bytes.Slice(startIndex, copiedFragmentLength).CopyTo(slice.AsSpan().Slice(0, copiedFragmentLength));
+            return slice;
+        }
     }
 }
diff --git a/src/Nethermind/Nethermind.Crypto/ShamatarLib.cs b/src/Nethermind/Nethermind.Crypto/ShamatarLib.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Buffers;
 using System.Runtime.InteropServices;
 
 namespace Nethermind.Crypto.Bls
@@ -30,7 +31,7 @@ private static extern unsafe uint eip2537_perform_operation(
             byte* error,
             ref int errorLength);
 
-        private static unsafe bool Bn256Op(byte operation, Span<byte> input, Span<byte> output)
+        private static unsafe bool Bn256Op(byte operation, ReadOnlySpan<byte> input, Span<byte> output)
         {
             int outputLength = output.Length;
             int errorLength = 256;
@@ -48,7 +49,7 @@ private static unsafe bool Bn256Op(byte operation, Span<byte> input, Span<byte>
             return externalCallResult == 0;
         }
 
-        private static unsafe bool BlsOp(byte operation, Span<byte> input, Span<byte> output)
+        private static unsafe bool BlsOp(byte operation, ReadOnlySpan<byte> input, Span<byte> output)
         {
             int outputLength = output.Length;
             int errorLength = 256;
@@ -66,64 +67,64 @@ private static unsafe bool BlsOp(byte operation, Span<byte> input, Span<byte> ou
             return externalCallResult == 0;
         }
 
-        public static bool Bn256Add(Span<byte> input, Span<byte> output)
+        public static bool Bn256Add(ReadOnlySpan<byte> input, Span<byte> output)
         {
             return Bn256Op(1, input, output);
         }
 
-        public static bool Bn256Mul(Span<byte> input, Span<byte> output)
+        public static bool Bn256Mul(ReadOnlySpan<byte> input, Span<byte> output)
         {
             return Bn256Op(2, input, output);
         }
 
-        public static bool Bn256Pairing(Span<byte> input, Span<byte> output)
+        public static bool Bn256Pairing(ReadOnlySpan<byte> input, Span<byte> output)
         {
             return Bn256Op(3, input, output);
         }
 
-        public static bool BlsG1Add(Span<byte> input, Span<byte> output)
+        public static bool BlsG1Add(ReadOnlySpan<byte> input, Span<byte> output)
         {
             return BlsOp(1, input, output);
         }
 
-        public static bool BlsG1Mul(Span<byte> input, Span<byte> output)
+        public static bool BlsG1Mul(ReadOnlySpan<byte> input, Span<byte> output)
         {
             return BlsOp(2, input, output);
         }
 
-        public static bool BlsG1MultiExp(Span<byte> input, Span<byte> output)
+        public static bool BlsG1MultiExp(ReadOnlySpan<byte> input, Span<byte> output)
         {
             return BlsOp(3, input, output);
         }
 
-        public static bool BlsG2Add(Span<byte> input, Span<byte> output)
+        public static bool BlsG2Add(ReadOnlySpan<byte> input, Span<byte> output)
         {
             return BlsOp(4, input, output);
         }
 
-        public static bool BlsG2Mul(Span<byte> input, Span<byte> output)
+        public static bool BlsG2Mul(ReadOnlySpan<byte> input, Span<byte> output)
         {
             return BlsOp(5, input, output);
         }
 
-        public static bool BlsG2MultiExp(Span<byte> input, Span<byte> output)
+        public static bool BlsG2MultiExp(ReadOnlySpan<byte> input, Span<byte> output)
         {
             return BlsOp(6, input, output);
         }
 
-        public static bool BlsPairing(Span<byte> input, Span<byte> output)
+        public static bool BlsPairing(ReadOnlySpan<byte> input, Span<byte> output)
         {
             return BlsOp(7, input, output);
         }
 
-        public static bool BlsMapToG1(Span<byte> input, Span<byte> output)
+        public static bool BlsMapToG1(ReadOnlySpan<byte> input, Span<byte> output)
         {
             return BlsOp(8, input, output);
         }
 
-        public static bool BlsMapToG2(Span<byte> input, Span<byte> output)
+        public static bool BlsMapToG2(ReadOnlySpan<byte> input, Span<byte> output)
         {
             return BlsOp(9, input, output);
         }
     }
-}
+}
diff --git a/src/Nethermind/Nethermind.Evm.Test/BlsAddG1PrecompileTests.cs b/src/Nethermind/Nethermind.Evm.Test/BlsAddG1PrecompileTests.cs
@@ -15,6 +15,7 @@
 //  along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.
 // 
 
+using System;
 using System.Collections.Generic;
 using FluentAssertions;
 using Nethermind.Core.Extensions;
@@ -34,16 +35,16 @@ public void Test()
             foreach (var (input, expectedResult) in Inputs)
             {
                 IPrecompile precompile = G1AddPrecompile.Instance;
-                (byte[] output, bool success) = precompile.Run(input, MuirGlacier.Instance);
-                output.Should().BeEquivalentTo(expectedResult);
+                (ReadOnlyMemory<byte> output, bool success) = precompile.Run(input, MuirGlacier.Instance);
+                output.ToArray().Should().BeEquivalentTo(expectedResult.ToArray());
                 success.Should().BeTrue();
             }
         }
 
         /// <summary>
         /// https://github.com/matter-labs/eip1962/tree/master/src/test/test_vectors/eip2537
         /// </summary>
-        private static readonly Dictionary<byte[], byte[]> Inputs = new()
+        private static readonly Dictionary<byte[], ReadOnlyMemory<byte>> Inputs = new()
         {
                 {Bytes.FromHexString("0000000000000000000000000000000012196c5a43d69224d8713389285f26b98f86ee910ab3dd668e413738282003cc5b7357af9a7af54bb713d62255e80f560000000000000000000000000000000006ba8102bfbeea4416b710c73e8cce3032c31c6269c44906f8ac4f7874ce99fb17559992486528963884ce429a992fee000000000000000000000000000000000001101098f5c39893765766af4512a0c74e1bb89bc7e6fdf14e3e7337d257cc0f94658179d83320b99f31ff94cd2bac0000000000000000000000000000000003e1a9f9f44ca2cdab4f43a1a3ee3470fdf90b2fc228eb3b709fcd72f014838ac82a6d797aeefed9a0804b22ed1ce8f7"), Bytes.FromHexString("000000000000000000000000000000001466e1373ae4a7e7ba885c5f0c3ccfa48cdb50661646ac6b779952f466ac9fc92730dcaed9be831cd1f8c4fefffd5209000000000000000000000000000000000c1fb750d2285d4ca0378e1e8cdbf6044151867c34a711b73ae818aee6dbe9e886f53d7928cc6ed9c851e0422f609b11")},
                 {Bytes.FromHexString("00000000000000000000000000000000117dbe419018f67844f6a5e1b78a1e597283ad7b8ee7ac5e58846f5a5fd68d0da99ce235a91db3ec1cf340fe6b7afcdb0000000000000000000000000000000013316f23de032d25e912ae8dc9b54c8dba1be7cecdbb9d2228d7e8f652011d46be79089dd0a6080a73c82256ce5e4ed2000000000000000000000000000000000441e7f7f96198e4c23bd5eb16f1a7f045dbc8c53219ab2bcea91d3a027e2dfe659feac64905f8b9add7e4bfc91bec2b0000000000000000000000000000000005fc51bb1b40c87cd4292d4b66f8ca5ce4ef9abd2b69d4464b4879064203bda7c9fc3f896a3844ebc713f7bb20951d95"), Bytes.FromHexString("0000000000000000000000000000000016b8ab56b45a9294466809b8e858c1ad15ad0d52cfcb62f8f5753dc94cee1de6efaaebce10701e3ec2ecaa9551024ea600000000000000000000000000000000124571eec37c0b1361023188d66ec17c1ec230d31b515e0e81e599ec19e40c8a7c8cdea9735bc3d8b4e37ca7e5dd71f6")},

diff --git a/src/Nethermind/Nethermind.Evm.Test/BlsAddG2PrecompileTests.cs b/src/Nethermind/Nethermind.Evm.Test/BlsAddG2PrecompileTests.cs
@@ -15,6 +15,7 @@
 //  along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.
 // 
 
+using System;
 using System.Collections.Generic;
 using FluentAssertions;
 using Nethermind.Core.Extensions;
@@ -34,16 +35,16 @@ public void Test()
             foreach (var (input, expectedResult) in Inputs)
             {
                 IPrecompile precompile = G2AddPrecompile.Instance;
-                (byte[] output, bool success) = precompile.Run(input, MuirGlacier.Instance);
-                output.Should().BeEquivalentTo(expectedResult);
+                (ReadOnlyMemory<byte> output, bool success) = precompile.Run(input, MuirGlacier.Instance);
+                output.ToArray().Should().BeEquivalentTo(expectedResult.ToArray());
                 success.Should().BeTrue();
             }
         }
 
         /// <summary>
         /// https://github.com/matter-labs/eip1962/tree/master/src/test/test_vectors/eip2537
         /// </summary>
-        private static readonly Dictionary<byte[], byte[]> Inputs = new()
+        private static readonly Dictionary<byte[], ReadOnlyMemory<byte>> Inputs = new()
         {
             {Bytes.FromHexString("00000000000000000000000000000000039b10ccd664da6f273ea134bb55ee48f09ba585a7e2bb95b5aec610631ac49810d5d616f67ba0147e6d1be476ea220e0000000000000000000000000000000000fbcdff4e48e07d1f73ec42fe7eb026f5c30407cfd2f22bbbfe5b2a09e8a7bb4884178cb6afd1c95f80e646929d30040000000000000000000000000000000001ed3b0e71acb0adbf44643374edbf4405af87cfc0507db7e8978889c6c3afbe9754d1182e98ac3060d64994d31ef576000000000000000000000000000000001681a2bf65b83be5a2ca50430949b6e2a099977482e9405b593f34d2ed877a3f0d1bddc37d0cec4d59d7df74b2b8f2df0000000000000000000000000000000017c9fcf0504e62d3553b2f089b64574150aa5117bd3d2e89a8c1ed59bb7f70fb83215975ef31976e757abf60a75a1d9f0000000000000000000000000000000008f5a53d704298fe0cfc955e020442874fe87d5c729c7126abbdcbed355eef6c8f07277bee6d49d56c4ebaf334848624000000000000000000000000000000001302dcc50c6ce4c28086f8e1b43f9f65543cf598be440123816765ab6bc93f62bceda80045fbcad8598d4f32d03ee8fa000000000000000000000000000000000bbb4eb37628d60b035a3e0c45c0ea8c4abef5a6ddc5625e0560097ef9caab208221062e81cd77ef72162923a1906a40"), Bytes.FromHexString("000000000000000000000000000000000a9b880c2c13da05bdeda62ea8f61e5fc2bf0b7aa5cc31eaf512bef7c5073d9e9927084b512e818dbf05eab697ba0661000000000000000000000000000000000b963b527aa3ec36813b108f2294115f732c878ac28551b5490615b436406773b5bb6a3f002be0e54db0bcebe40cb2e2000000000000000000000000000000000bd6e9060b42e36b57d88bc95b8b993da2d9d5acd95b73bad0509c2324212bcf7a94a46901932c0750535d00008a34f7000000000000000000000000000000000a374afd32bc3bb20c22a8864ce0dafe298bda17260b9d1d598a80830400c3fd4e8a8f677630eae5d4aa0a76a434e0ba")},
             {Bytes.FromHexString("0000000000000000000000000000000018c0ada6351b70661f053365deae56910798bd2ace6e2bf6ba4192d1a229967f6af6ca1c9a8a11ebc0a232344ee0f6d6000000000000000000000000000000000cc70a587f4652039d8117b6103858adcd9728f6aebe230578389a62da0042b7623b1c0436734f463cfdd187d20903240000000000000000000000000000000009f50bd7beedb23328818f9ffdafdb6da6a4dd80c5a9048ab8b154df3cad938ccede829f1156f769d9e149791e8e0cd900000000000000000000000000000000079ba50d2511631b20b6d6f3841e616e9d11b68ec3368cd60129d9d4787ab56c4e9145a38927e51c9cd6271d493d938800000000000000000000000000000000192fa5d8732ff9f38e0b1cf12eadfd2608f0c7a39aced7746837833ae253bb57ef9c0d98a4b69eeb2950901917e99d1e0000000000000000000000000000000009aeb10c372b5ef1010675c6a4762fda33636489c23b581c75220589afbc0cc46249f921eea02dd1b761e036ffdbae220000000000000000000000000000000002d225447600d49f932b9dd3ca1e6959697aa603e74d8666681a2dca8160c3857668ae074440366619eb8920256c4e4a00000000000000000000000000000000174882cdd3551e0ce6178861ff83e195fecbcffd53a67b6f10b4431e423e28a480327febe70276036f60bb9c99cf7633"), Bytes.FromHexString("000000000000000000000000000000001963e94d1501b6038de347037236c18a0a0c8cec677e48fc514e9fc9753a7d8dcf0acc4b3b64572cb571aebbe0b696640000000000000000000000000000000000d9739acc3a60f6dffb26f9b5f1fd114a21f2983deea192663c53e012b9f8e1cabd4942ad039badbd4745ddc0a26a91000000000000000000000000000000000b4206dcdb80d62195febb6773acab25fa2c09a2e4be9416ca019faeb72f1fad1dfdc51e8cea39b371a045b18947d40a00000000000000000000000000000000100758b888fa27e9258ddd5d83409e8aeac576874bc399b33b8bc50d77fce5358cb091d42f9a1b1ed09be3f200959989")},

diff --git a/src/Nethermind/Nethermind.Evm.Test/BlsMapToG1Tests.cs b/src/Nethermind/Nethermind.Evm.Test/BlsMapToG1Tests.cs
@@ -15,6 +15,7 @@
 //  along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.
 // 
 
+using System;
 using System.Collections.Generic;
 using FluentAssertions;
 using Nethermind.Core.Extensions;
@@ -34,17 +35,17 @@ public void Test()
             foreach (var (input, expectedResult) in Inputs)
             {
                 IPrecompile precompile = MapToG1Precompile.Instance;
-                (byte[] output, bool success) = precompile.Run(input, MuirGlacier.Instance);
+                (ReadOnlyMemory<byte> output, bool success) = precompile.Run(input, MuirGlacier.Instance);
 
-                output.Should().BeEquivalentTo(expectedResult);
+                output.ToArray().Should().BeEquivalentTo(expectedResult.ToArray());
                 success.Should().BeTrue();
             }
         }
 
         /// <summary>
         /// https://github.com/matter-labs/eip1962/tree/master/src/test/test_vectors/eip2537
         /// </summary>
-        private static readonly Dictionary<byte[], byte[]> Inputs = new()
+        private static readonly Dictionary<byte[], ReadOnlyMemory<byte>> Inputs = new()
         {
             {Bytes.FromHexString("0000000000000000000000000000000014406e5bfb9209256a3820879a29ac2f62d6aca82324bf3ae2aa7d3c54792043bd8c791fccdb080c1a52dc68b8b69350"), Bytes.FromHexString("000000000000000000000000000000000d7721bcdb7ce1047557776eb2659a444166dc6dd55c7ca6e240e21ae9aa18f529f04ac31d861b54faf3307692545db700000000000000000000000000000000108286acbdf4384f67659a8abe89e712a504cb3ce1cba07a716869025d60d499a00d1da8cdc92958918c222ea93d87f0")},
             {Bytes.FromHexString("000000000000000000000000000000000e885bb33996e12f07da69073e2c0cc880bc8eff26d2a724299eb12d54f4bcf26f4748bb020e80a7e3794a7b0e47a641"), Bytes.FromHexString("00000000000000000000000000000000191ba6e4c4dafa22c03d41b050fe8782629337641be21e0397dc2553eb8588318a21d30647182782dee7f62a22fd020c000000000000000000000000000000000a721510a67277eabed3f153bd91df0074e1cbd37ef65b85226b1ce4fb5346d943cf21c388f0c5edbc753888254c760a")},

diff --git a/src/Nethermind/Nethermind.Evm.Test/BlsMapToG2Tests.cs b/src/Nethermind/Nethermind.Evm.Test/BlsMapToG2Tests.cs
@@ -15,6 +15,7 @@
 //  along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.
 // 
 
+using System;
 using System.Collections.Generic;
 using FluentAssertions;
 using Nethermind.Core.Extensions;
@@ -34,17 +35,17 @@ public void Test()
             foreach (var (input, expectedResult) in Inputs)
             {
                 IPrecompile precompile = MapToG2Precompile.Instance;
-                (byte[] output, bool success) = precompile.Run(input, MuirGlacier.Instance);
+                (ReadOnlyMemory<byte> output, bool success) = precompile.Run(input, MuirGlacier.Instance);
 
-                output.Should().BeEquivalentTo(expectedResult);
+                output.ToArray().Should().BeEquivalentTo(expectedResult.ToArray());
                 success.Should().BeTrue();
             }
         }
 
         /// <summary>
         /// https://github.com/matter-labs/eip1962/tree/master/src/test/test_vectors/eip2537
         /// </summary>
-        private static readonly Dictionary<byte[], byte[]> Inputs = new()
+        private static readonly Dictionary<byte[], ReadOnlyMemory<byte>> Inputs = new()
         {
             {Bytes.FromHexString("0000000000000000000000000000000014406e5bfb9209256a3820879a29ac2f62d6aca82324bf3ae2aa7d3c54792043bd8c791fccdb080c1a52dc68b8b69350000000000000000000000000000000000e885bb33996e12f07da69073e2c0cc880bc8eff26d2a724299eb12d54f4bcf26f4748bb020e80a7e3794a7b0e47a641"), Bytes.FromHexString("000000000000000000000000000000000d029393d3a13ff5b26fe52bd8953768946c5510f9441f1136f1e938957882db6adbd7504177ee49281ecccba596f2bf000000000000000000000000000000001993f668fb1ae603aefbb1323000033fcb3b65d8ed3bf09c84c61e27704b745f540299a1872cd697ae45a5afd780f1d600000000000000000000000000000000079cb41060ef7a128d286c9ef8638689a49ca19da8672ea5c47b6ba6dbde193ee835d3b87a76a689966037c07159c10d0000000000000000000000000000000017c688ae9a8b59a7069c27f2d58dd2196cb414f4fb89da8510518a1142ab19d158badd1c3bad03408fafb1669903cd6c")},
             {Bytes.FromHexString("000000000000000000000000000000000ba1b6d79150bdc368a14157ebfe8b5f691cf657a6bbe30e79b6654691136577d2ef1b36bfb232e3336e7e4c9352a8ed000000000000000000000000000000000f12847f7787f439575031bcdb1f03cfb79f942f3a9709306e4bd5afc73d3f78fd1c1fef913f503c8cbab58453fb7df2"), Bytes.FromHexString("000000000000000000000000000000000a2bca68ca23f3f03c678140d87465b5b336dbd50926d1219fcc0def162280765fe1093c117d52483d3d8cdc7ab76529000000000000000000000000000000000fe83e3a958d6038569da6132bfa19f0e3dae3bee0d8a60e7cc33e4d7084a9e8c32fe31ec6e617277e2e450699eba1f80000000000000000000000000000000005602683f0ef231cc0b7c8c695765d7933f4efa7503ed9f2aa3c774284eabcdd32fd287b6a3539c9749f2e15b58f5cd50000000000000000000000000000000000b4f17de0db6e9d081723b613b23864c1eeae91b7cbda40ecd24823022aee7fc4068adc41947b97e17009fad9d0d4de")},