Leaf value collector

src/Nethermind/Nethermind.Core/Collections/IOwnedReadOnlyList.cs

@@ -20,3 +20,16 @@ public interface IOwnedReadOnlyList<T> : IReadOnlyList<T>, IDisposable
 {
     ReadOnlySpan<T> AsSpan();
 }
+
+public static class OwnedReadOnlyListExtensions
+{
+    public static void DisposeRecursive<T>(this IOwnedReadOnlyList<T> list) where T : IDisposable
+    {
+        for (int i = 0; i < list.Count; i++)
+        {
+            list[i]?.Dispose();
+        }
+
+        list.Dispose();
+    }
+}

src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Snap/Messages/StorageRangesMessageSerializerTests.cs

@@ -20,10 +20,10 @@ public class StorageRangesMessageSerializerTests
         [Test]
         public void Roundtrip_NoSlotsNoProofs()
         {
-            StorageRangeMessage msg = new()
+            using StorageRangeMessage msg = new()
             {
                 RequestId = MessageConstants.Random.NextLong(),
-                Slots = ArrayPoolList<PathWithStorageSlot[]>.Empty(),
+                Slots = ArrayPoolList<IOwnedReadOnlyList<PathWithStorageSlot>>.Empty(),
                 Proofs = ArrayPoolList<byte[]>.Empty()
             };
             StorageRangesMessageSerializer serializer = new();
@@ -34,28 +34,34 @@ public void Roundtrip_NoSlotsNoProofs()
         [Test]
         public void Roundtrip_OneProof()
         {
-            StorageRangeMessage msg = new()
+            using StorageRangeMessage msg = new()
             {
                 RequestId = MessageConstants.Random.NextLong(),
-                Slots = ArrayPoolList<PathWithStorageSlot[]>.Empty(),
+                Slots = ArrayPoolList<IOwnedReadOnlyList<PathWithStorageSlot>>.Empty(),
                 Proofs = new ArrayPoolList<byte[]>(2) { TestItem.RandomDataA }
             };
 
             StorageRangesMessageSerializer serializer = new();
 
             var serialized = serializer.Serialize(msg);
-            var deserialized = serializer.Deserialize(serialized);
+            using var deserialized = serializer.Deserialize(serialized);
 
             SerializerTester.TestZero(serializer, msg);
         }
 
         [Test]
         public void Roundtrip_OneSlot()
         {
-            StorageRangeMessage msg = new()
+            using StorageRangeMessage msg = new()
             {
                 RequestId = MessageConstants.Random.NextLong(),
-                Slots = new ArrayPoolList<PathWithStorageSlot[]>(1) { new PathWithStorageSlot[] { new PathWithStorageSlot(new Hash256("0x10d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"), TestItem.RandomDataA) } },
+                Slots = new ArrayPoolList<IOwnedReadOnlyList<PathWithStorageSlot>>(1)
+                {
+                    new ArrayPoolList<PathWithStorageSlot> (1)
+                    {
+                        new PathWithStorageSlot(new Hash256("0x10d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"), TestItem.RandomDataA)
+                    }
+                },
                 Proofs = ArrayPoolList<byte[]>.Empty()
             };
 
@@ -67,15 +73,15 @@ public void Roundtrip_OneSlot()
         [Test]
         public void Roundtrip_Many()
         {
-            StorageRangeMessage msg = new()
+            using StorageRangeMessage msg = new()
             {
                 RequestId = MessageConstants.Random.NextLong(),
-                Slots = new ArrayPoolList<PathWithStorageSlot[]>(2) {
-                    new PathWithStorageSlot[] {
+                Slots = new ArrayPoolList<IOwnedReadOnlyList<PathWithStorageSlot>>(2) {
+                    new ArrayPoolList<PathWithStorageSlot>(2) {
                         new PathWithStorageSlot(new Hash256("0x10d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"), Rlp.Encode(TestItem.RandomDataA).Bytes) ,
                         new PathWithStorageSlot(new Hash256("0x12d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"), Rlp.Encode(TestItem.RandomDataB).Bytes)
                     },
-                    new PathWithStorageSlot[] {
+                    new ArrayPoolList<PathWithStorageSlot>(2) {
                         new PathWithStorageSlot(new Hash256("0x21d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"), Rlp.Encode(TestItem.RandomDataB).Bytes) ,
                         new PathWithStorageSlot(new Hash256("0x22d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"), Rlp.Encode(TestItem.RandomDataC).Bytes)
                     }

src/Nethermind/Nethermind.Network/P2P/Subprotocols/Snap/Messages/StorageRangeMessage.cs

@@ -1,6 +1,7 @@
 // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
+using System.Data.Common;
 using Nethermind.Core.Collections;
 using Nethermind.State.Snap;
 
@@ -13,7 +14,7 @@ public class StorageRangeMessage : SnapMessageBase
         /// <summary>
         /// List of list of consecutive slots from the trie (one list per account)
         /// </summary>
-        public IOwnedReadOnlyList<PathWithStorageSlot[]> Slots { get; set; }
+        public IOwnedReadOnlyList<IOwnedReadOnlyList<PathWithStorageSlot>> Slots { get; set; }
 
         /// <summary>
         /// List of trie nodes proving the slot range
@@ -23,7 +24,7 @@ public class StorageRangeMessage : SnapMessageBase
         public override void Dispose()
         {
             base.Dispose();
-            Slots?.Dispose();
+            Slots?.DisposeRecursive();
             Proofs?.Dispose();
         }
     }

src/Nethermind/Nethermind.Network/P2P/Subprotocols/Snap/Messages/StorageRangesMessageSerializer.cs

@@ -3,6 +3,7 @@
 
 using System;
 using DotNetty.Buffers;
+using Nethermind.Core.Collections;
 using Nethermind.Core.Crypto;
 using Nethermind.Serialization.Rlp;
 using Nethermind.State.Snap;
@@ -12,13 +13,13 @@ namespace Nethermind.Network.P2P.Subprotocols.Snap.Messages
     public sealed class StorageRangesMessageSerializer : IZeroMessageSerializer<StorageRangeMessage>
     {
         private readonly Func<RlpStream, PathWithStorageSlot> _decodeSlot;
-        private readonly Func<RlpStream, PathWithStorageSlot[]> _decodeSlotArray;
+        private readonly Func<RlpStream, IOwnedReadOnlyList<PathWithStorageSlot>> _decodeSlotArray;
 
         public StorageRangesMessageSerializer()
         {
             // Capture closures once
             _decodeSlot = DecodeSlot;
-            _decodeSlotArray = s => s.DecodeArray(_decodeSlot);
+            _decodeSlotArray = s => s.DecodeArrayPoolList(_decodeSlot);
         }
 
         public void Serialize(IByteBuffer byteBuffer, StorageRangeMessage message)
@@ -44,9 +45,9 @@ public void Serialize(IByteBuffer byteBuffer, StorageRangeMessage message)
                 {
                     stream.StartSequence(accountSlotsLengths[i]);
 
-                    PathWithStorageSlot[] accountSlots = message.Slots[i];
+                    IOwnedReadOnlyList<PathWithStorageSlot> accountSlots = message.Slots[i];
 
-                    for (int j = 0; j < accountSlots.Length; j++)
+                    for (int j = 0; j < accountSlots.Count; j++)
                     {
                         var slot = accountSlots[j];
 

src/Nethermind/Nethermind.Network/P2P/Subprotocols/Snap/SnapProtocolHandler.cs

@@ -259,10 +259,10 @@ private StorageRangeMessage FulfillStorageRangeMessage(GetStorageRangeMessage ge
             if (SyncServer is null) return new StorageRangeMessage()
             {
                 Proofs = ArrayPoolList<byte[]>.Empty(),
-                Slots = ArrayPoolList<PathWithStorageSlot[]>.Empty(),
+                Slots = ArrayPoolList<IOwnedReadOnlyList<PathWithStorageSlot>>.Empty(),
             };
             StorageRange? storageRange = getStorageRangeMessage.StoragetRange;
-            (IOwnedReadOnlyList<PathWithStorageSlot[]>? ranges, IOwnedReadOnlyList<byte[]>? proofs) = SyncServer.GetStorageRanges(storageRange.RootHash, storageRange.Accounts,
+            (IOwnedReadOnlyList<IOwnedReadOnlyList<PathWithStorageSlot>>? ranges, IOwnedReadOnlyList<byte[]> proofs) = SyncServer.GetStorageRanges(storageRange.RootHash, storageRange.Accounts,
                 storageRange.StartingHash, storageRange.LimitHash, getStorageRangeMessage.ResponseBytes, cancellationToken);
             StorageRangeMessage? response = new() { Proofs = proofs, Slots = ranges };
             return response;

src/Nethermind/Nethermind.Serialization.Rlp/AccountDecoder.cs

@@ -14,6 +14,7 @@ public class AccountDecoder : IRlpObjectDecoder<Account?>, IRlpStreamDecoder<Acc
         private readonly bool _slimFormat;
 
         public static AccountDecoder Instance => new();
+        public static AccountDecoder Slim => new(slimFormat: true);
 
         public AccountDecoder() { }
 

src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromAccountRangesTests.cs

@@ -110,7 +110,7 @@ public void RecreateAccountStateFromOneRangeWithNonExistenceProof()
             MemDb db = new();
             DbProvider dbProvider = new();
             dbProvider.RegisterDb(DbNames.State, db);
-            ProgressTracker progressTracker = new(null, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
+            using ProgressTracker progressTracker = new(null, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
             SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance);
             AddRangeResult result = snapProvider.AddAccountRange(1, rootHash, Keccak.Zero, TestItem.Tree.AccountsWithPaths, firstProof!.Concat(lastProof!).ToArray());
 
@@ -151,7 +151,7 @@ public void RecreateAccountStateFromOneRangeWithoutProof()
             MemDb db = new();
             DbProvider dbProvider = new();
             dbProvider.RegisterDb(DbNames.State, db);
-            ProgressTracker progressTracker = new(null, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
+            using ProgressTracker progressTracker = new(null, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
             SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance);
             var result = snapProvider.AddAccountRange(1, rootHash, TestItem.Tree.AccountsWithPaths[0].Path, TestItem.Tree.AccountsWithPaths);
 
@@ -169,7 +169,7 @@ public void RecreateAccountStateFromMultipleRange()
             MemDb db = new();
             DbProvider dbProvider = new();
             dbProvider.RegisterDb(DbNames.State, db);
-            ProgressTracker progressTracker = new(null, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
+            using ProgressTracker progressTracker = new(null, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
             SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance);
 
             AccountProofCollector accountProofCollector = new(Keccak.Zero.Bytes);
@@ -219,7 +219,7 @@ public void MissingAccountFromRange()
             MemDb db = new();
             DbProvider dbProvider = new();
             dbProvider.RegisterDb(DbNames.State, db);
-            ProgressTracker progressTracker = new(null, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
+            using ProgressTracker progressTracker = new(null, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
             SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance);
 
             AccountProofCollector accountProofCollector = new(Keccak.Zero.Bytes);

src/Nethermind/Nethermind.State.Test/SnapSync/RecreateStateFromStorageRangesTests.cs

@@ -51,7 +51,7 @@ public void RecreateStorageStateFromOneRangeWithNonExistenceProof()
             MemDb db = new();
             DbProvider dbProvider = new();
             dbProvider.RegisterDb(DbNames.State, db);
-            ProgressTracker progressTracker = new(null, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
+            using ProgressTracker progressTracker = new(null, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
             SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance);
             var result = snapProvider.AddStorageRange(1, null, rootHash, Keccak.Zero, TestItem.Tree.SlotsWithPaths, proof!.StorageProofs![0].Proof!.Concat(proof!.StorageProofs![1].Proof!).ToArray());
 
@@ -70,7 +70,7 @@ public void RecreateAccountStateFromOneRangeWithExistenceProof()
             MemDb db = new();
             DbProvider dbProvider = new();
             dbProvider.RegisterDb(DbNames.State, db);
-            ProgressTracker progressTracker = new(null, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
+            using ProgressTracker progressTracker = new(null, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
             SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance);
             var result = snapProvider.AddStorageRange(1, null, rootHash, Keccak.Zero, TestItem.Tree.SlotsWithPaths, proof!.StorageProofs![0].Proof!.Concat(proof!.StorageProofs![1].Proof!).ToArray());
 
@@ -85,7 +85,7 @@ public void RecreateStorageStateFromOneRangeWithoutProof()
             MemDb db = new MemDb();
             DbProvider dbProvider = new();
             dbProvider.RegisterDb(DbNames.State, db);
-            ProgressTracker progressTracker = new(null, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
+            using ProgressTracker progressTracker = new(null, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
             SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance);
             var result = snapProvider.AddStorageRange(1, null, rootHash, TestItem.Tree.SlotsWithPaths[0].Path, TestItem.Tree.SlotsWithPaths);
 
@@ -101,7 +101,7 @@ public void RecreateAccountStateFromMultipleRange()
             MemDb db = new MemDb();
             DbProvider dbProvider = new();
             dbProvider.RegisterDb(DbNames.State, db);
-            ProgressTracker progressTracker = new(null, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
+            using ProgressTracker progressTracker = new(null, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
             SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance);
 
             AccountProofCollector accountProofCollector = new(TestItem.Tree.AccountAddress0.Bytes, new ValueHash256[] { Keccak.Zero, TestItem.Tree.SlotsWithPaths[1].Path });
@@ -136,7 +136,7 @@ public void MissingAccountFromRange()
             MemDb db = new MemDb();
             DbProvider dbProvider = new();
             dbProvider.RegisterDb(DbNames.State, db);
-            ProgressTracker progressTracker = new(null, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
+            using ProgressTracker progressTracker = new(null, dbProvider.GetDb<IDb>(DbNames.State), LimboLogs.Instance);
             SnapProvider snapProvider = new(progressTracker, dbProvider, LimboLogs.Instance);
 
             AccountProofCollector accountProofCollector = new(TestItem.Tree.AccountAddress0.Bytes, new ValueHash256[] { Keccak.Zero, TestItem.Tree.SlotsWithPaths[1].Path });

src/Nethermind/Nethermind.State/Snap/SlotsAndProofs.cs

@@ -8,12 +8,23 @@ namespace Nethermind.State.Snap
 {
     public class SlotsAndProofs : IDisposable
     {
-        public IOwnedReadOnlyList<PathWithStorageSlot[]> PathsAndSlots { get; set; }
+        public IOwnedReadOnlyList<IOwnedReadOnlyList<PathWithStorageSlot>> PathsAndSlots { get; set; }
         public IOwnedReadOnlyList<byte[]> Proofs { get; set; }
 
+        private bool _disposed = false;
+
         public void Dispose()
         {
-            PathsAndSlots?.Dispose();
+            if (_disposed) return;
+            _disposed = true;
+            if (PathsAndSlots != null)
+            {
+                foreach (IOwnedReadOnlyList<PathWithStorageSlot> pathWithStorageSlots in PathsAndSlots)
+                {
+                    pathWithStorageSlots?.Dispose();
+                }
+                PathsAndSlots?.Dispose();
+            }
             Proofs?.Dispose();
         }
     }