Refactor/extract block store from blocktree

src/Nethermind/Nethermind.Blockchain.Test/BlockTreeTests.cs

@@ -1306,7 +1306,6 @@ public void Cannot_insert_genesis()
             tree.SuggestBlock(genesis);
             Assert.Throws<InvalidOperationException>(() => tree.Insert(genesis));
             Assert.Throws<InvalidOperationException>(() => tree.Insert(genesis.Header));
-            Assert.Throws<InvalidOperationException>(() => tree.Insert(new[] { genesis }));
         }
 
         [Test, Timeout(Timeout.MaxTestTime)]
@@ -1337,33 +1336,6 @@ public void Should_set_zero_total_difficulty()
             tree.FindBlock(A.Hash, BlockTreeLookupOptions.None)!.TotalDifficulty.Should().Be(UInt256.Zero);
         }
 
-        [Test, Timeout(Timeout.MaxTestTime)]
-        public void Can_batch_insert_blocks()
-        {
-            MemDb blocksDb = new();
-            MemDb blockInfosDb = new();
-            MemDb headersDb = new();
-            MemDb metadataDb = new();
-
-            long pivotNumber = 5L;
-
-            SyncConfig syncConfig = new();
-            syncConfig.PivotNumber = pivotNumber.ToString();
-
-            BlockTree tree = new(blocksDb, headersDb, blockInfosDb, metadataDb, new ChainLevelInfoRepository(blockInfosDb), MainnetSpecProvider.Instance, NullBloomStorage.Instance, syncConfig, LimboLogs.Instance);
-            tree.SuggestBlock(Build.A.Block.Genesis.TestObject);
-
-            List<Block> blocks = new();
-            for (long i = 5; i > 0; i--)
-            {
-                Block block = Build.A.Block.WithNumber(i).WithTotalDifficulty(1L).TestObject;
-                tree.Insert(block.Header);
-                blocks.Add(block);
-            }
-
-            tree.Insert(blocks);
-        }
-
         [Test, Timeout(Timeout.MaxTestTime)]
         public void Inserts_blooms()
         {

src/Nethermind/Nethermind.Blockchain.Test/Blocks/BlockStoreTests.cs

@@ -0,0 +1,63 @@
+// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using FluentAssertions;
+using Nethermind.Blockchain.Blocks;
+using Nethermind.Core;
+using Nethermind.Core.Test;
+using Nethermind.Core.Test.Builders;
+using NUnit.Framework;
+
+namespace Nethermind.Blockchain.Test.Blocks;
+
+public class BlockStoreTests
+{
+    [TestCase(true)]
+    [TestCase(false)]
+    public void Test_can_insert_get_and_remove_blocks(bool cached)
+    {
+        TestMemDb db = new TestMemDb();
+        BlockStore store = new BlockStore(db);
+
+        Block block = Build.A.Block.WithNumber(1).TestObject;
+        store.Insert(block);
+
+        Block? retrieved = store.Get(block.Hash, cached);
+        retrieved.Should().BeEquivalentTo(block);
+
+        store.Delete(block.Hash);
+
+        store.Get(block.Hash, cached).Should().BeNull();
+    }
+
+    [Test]
+    public void Test_can_set_and_get_metadata()
+    {
+        TestMemDb db = new TestMemDb();
+        BlockStore store = new BlockStore(db);
+
+        byte[] key = new byte[] { 1, 2, 3 };
+        byte[] value = new byte[] { 4, 5, 6 };
+
+        store.SetMetadata(key, value);
+        store.GetMetadata(key).Should().BeEquivalentTo(value);
+    }
+
+    [Test]
+    public void Test_when_cached_does_not_touch_db_on_next_get()
+    {
+        TestMemDb db = new TestMemDb();
+        BlockStore store = new BlockStore(db);
+
+        Block block = Build.A.Block.WithNumber(1).TestObject;
+        store.Insert(block);
+
+        Block? retrieved = store.Get(block.Hash, true);
+        retrieved.Should().BeEquivalentTo(block);
+
+        db.Clear();
+
+        retrieved = store.Get(block.Hash, true);
+        retrieved.Should().BeEquivalentTo(block);
+    }
+}

src/Nethermind/Nethermind.Blockchain/BlockTree.cs

@@ -8,6 +8,7 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Nethermind.Blockchain.Blocks;
 using Nethermind.Blockchain.Find;
 using Nethermind.Blockchain.Synchronization;
 using Nethermind.Core;
@@ -30,33 +31,29 @@ namespace Nethermind.Blockchain
     public partial class BlockTree : IBlockTree
     {
         // there is not much logic in the addressing here
-        private const long LowestInsertedBodyNumberDbEntryAddress = 0;
+        private static readonly byte[] LowestInsertedBodyNumberDbEntryAddress = ((long)0).ToBigEndianByteArrayWithoutLeadingZeros();
         private static byte[] StateHeadHashDbEntryAddress = new byte[16];
         internal static Keccak DeletePointerAddressInDb = new(new BitArray(32 * 8, true).ToBytes());
 
         internal static Keccak HeadAddressInDb = Keccak.Zero;
 
         private const int CacheSize = 64;
 
-        private readonly LruCache<KeccakKey, Block>
-            _blockCache = new(CacheSize, CacheSize, "blocks");
-
         private readonly LruCache<KeccakKey, BlockHeader> _headerCache =
             new(CacheSize, CacheSize, "headers");
 
         private const int BestKnownSearchLimit = 256_000_000;
 
         private readonly object _batchInsertLock = new();
 
-        private readonly IDb _blockDb;
+        private readonly IBlockStore _blockStore;
         private readonly IDb _headerDb;
         private readonly IDb _blockInfoDb;
         private readonly IDb _metadataDb;
 
         private readonly LruCache<KeccakKey, Block> _invalidBlocks =
             new(128, 128, "invalid blocks");
 
-        private readonly BlockDecoder _blockDecoder = new();
         private readonly HeaderDecoder _headerDecoder = new();
         private readonly ILogger _logger;
         private readonly ISpecProvider _specProvider;
@@ -99,7 +96,7 @@ public long? LowestInsertedBodyNumber
                 _lowestInsertedReceiptBlock = value;
                 if (value.HasValue)
                 {
-                    _blockDb.Set(LowestInsertedBodyNumberDbEntryAddress, Rlp.Encode(value.Value).Bytes);
+                    _blockStore.SetMetadata(LowestInsertedBodyNumberDbEntryAddress, Rlp.Encode(value.Value).Bytes);
                 }
             }
         }
@@ -165,7 +162,7 @@ public BlockTree(
             ILogManager? logManager)
         {
             _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager));
-            _blockDb = blockDb ?? throw new ArgumentNullException(nameof(blockDb));
+            _blockStore = new BlockStore(blockDb ?? throw new ArgumentNullException(nameof(blockDb)));
             _headerDb = headerDb ?? throw new ArgumentNullException(nameof(headerDb));
             _blockInfoDb = blockInfoDb ?? throw new ArgumentNullException(nameof(blockInfoDb));
             _metadataDb = metadataDb ?? throw new ArgumentNullException(nameof(metadataDb));
@@ -252,7 +249,7 @@ public void RecalculateTreeLevels()
         private void LoadLowestInsertedBodyNumber()
         {
             LowestInsertedBodyNumber =
-                _blockDb.Get(LowestInsertedBodyNumberDbEntryAddress)?
+                _blockStore.GetMetadata(LowestInsertedBodyNumberDbEntryAddress)?
                     .AsRlpValueContext().DecodeLong();
         }
 
@@ -654,23 +651,12 @@ public AddBlockResult Insert(Block block, BlockTreeInsertBlockOptions insertBloc
                 return AddBlockResult.CannotAccept;
             }
 
-            if (block.Hash is null)
-            {
-                throw new InvalidOperationException("An attempt to store a block with a null hash.");
-            }
-
             if (block.Number == 0)
             {
                 throw new InvalidOperationException("Genesis block should not be inserted.");
             }
 
-
-            // if we carry Rlp from the network message all the way here then we could solve 4GB of allocations and some processing
-            // by avoiding encoding back to RLP here (allocations measured on a sample 3M blocks Goerli fast sync
-            using (NettyRlpStream newRlp = _blockDecoder.EncodeToNewNettyStream(block))
-            {
-                _blockDb.Set(block.Hash, newRlp.AsSpan());
-            }
+            _blockStore.Insert(block);
 
             bool saveHeader = (insertBlockOptions & BlockTreeInsertBlockOptions.SaveHeader) != 0;
             if (saveHeader)
@@ -681,26 +667,6 @@ public AddBlockResult Insert(Block block, BlockTreeInsertBlockOptions insertBloc
             return AddBlockResult.Added;
         }
 
-        public void Insert(IEnumerable<Block> blocks)
-        {
-            lock (_batchInsertLock)
-            {
-                // TODO: why is this commented out? why was it here in the first place? (2021-03-27)
-                // try
-                // {
-                //   _blockDb.StartBatch();
-                foreach (Block block in blocks)
-                {
-                    Insert(block);
-                }
-                // }
-                // finally
-                // {
-                //     _blockDb.CommitBatch();
-                // }
-            }
-        }
-
         private AddBlockResult Suggest(Block? block, BlockHeader header, BlockTreeSuggestOptions options = BlockTreeSuggestOptions.ShouldProcess)
         {
             bool shouldProcess = options.ContainsFlag(BlockTreeSuggestOptions.ShouldProcess);
@@ -757,8 +723,7 @@ private AddBlockResult Suggest(Block? block, BlockHeader header, BlockTreeSugges
                     throw new InvalidOperationException("An attempt to suggest block with a null hash.");
                 }
 
-                using NettyRlpStream newRlp = _blockDecoder.EncodeToNewNettyStream(block);
-                _blockDb.Set(block.Hash, newRlp.AsSpan());
+                _blockStore.Insert(block);
             }
 
             if (!isKnown)
@@ -1182,8 +1147,7 @@ private void DeleteBlocks(Keccak deletePointer)
                 }
 
                 if (_logger.IsInfo) _logger.Info($"Deleting invalid block {currentHash} at level {currentNumber}");
-                _blockCache.Delete(currentHash);
-                _blockDb.Delete(currentHash);
+                _blockStore.Delete(currentHash);
                 _headerCache.Delete(currentHash);
                 _headerDb.Delete(currentHash);
 
@@ -1272,7 +1236,7 @@ public void MarkChainAsProcessed(IReadOnlyList<Block> blocks)
                 Block block = blocks[i];
                 if (ShouldCache(block.Number))
                 {
-                    _blockCache.Set(block.Hash, blocks[i]);
+                    _blockStore.Cache(block);
                     _headerCache.Set(block.Hash, block.Header);
                 }
 
@@ -1346,7 +1310,7 @@ public void UpdateMainChain(IReadOnlyList<Block> blocks, bool wereProcessed, boo
                 Block block = blocks[i];
                 if (ShouldCache(block.Number))
                 {
-                    _blockCache.Set(block.Hash, blocks[i]);
+                    _blockStore.Cache(block);
                     _headerCache.Set(block.Hash, block.Header);
                 }
 
@@ -1791,7 +1755,7 @@ private bool ShouldCache(long number)
                 return null;
             }
 
-            Block block = _blockDb.Get(blockHash, _blockDecoder, _blockCache, false);
+            Block block = _blockStore.Get(blockHash, false);
             if (block is null)
             {
                 bool allowInvalid = (options & BlockTreeLookupOptions.AllowInvalid) == BlockTreeLookupOptions.AllowInvalid;
@@ -1845,7 +1809,7 @@ private bool ShouldCache(long number)
 
             if (block is not null && ShouldCache(block.Number))
             {
-                _blockCache.Set(blockHash, block);
+                _blockStore.Cache(block);
                 _headerCache.Set(blockHash, block.Header);
             }
 
@@ -2038,7 +2002,7 @@ public int DeleteChainSlice(in long startNumber, long? endNumber)
                     {
                         Keccak blockHash = blockInfo.BlockHash;
                         _blockInfoDb.Delete(blockHash);
-                        _blockDb.Delete(blockHash);
+                        _blockStore.Delete(blockHash);
                         _headerDb.Delete(blockHash);
                     }
                 }

src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs

@@ -0,0 +1,65 @@
+// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using Nethermind.Core;
+using Nethermind.Core.Caching;
+using Nethermind.Core.Crypto;
+using Nethermind.Db;
+using Nethermind.Serialization.Rlp;
+
+namespace Nethermind.Blockchain.Blocks;
+
+public class BlockStore : IBlockStore
+{
+    private readonly IDb _blockDb;
+    private readonly BlockDecoder _blockDecoder = new();
+    private const int CacheSize = 64;
+
+    private readonly LruCache<KeccakKey, Block>
+        _blockCache = new(CacheSize, CacheSize, "blocks");
+
+    public BlockStore(IDb blockDb)
+    {
+        _blockDb = blockDb;
+    }
+
+    public void SetMetadata(byte[] key, byte[] value)
+    {
+        _blockDb.Set(key, value);
+    }
+
+    public byte[]? GetMetadata(byte[] key)
+    {
+        return _blockDb.Get(key);
+    }
+
+    public void Insert(Block block)
+    {
+        if (block.Hash is null)
+        {
+            throw new InvalidOperationException("An attempt to store a block with a null hash.");
+        }
+
+        // if we carry Rlp from the network message all the way here then we could solve 4GB of allocations and some processing
+        // by avoiding encoding back to RLP here (allocations measured on a sample 3M blocks Goerli fast sync
+        using NettyRlpStream newRlp = _blockDecoder.EncodeToNewNettyStream(block);
+        _blockDb.Set(block.Hash, newRlp.AsSpan());
+    }
+
+    public void Delete(Keccak blockHash)
+    {
+        _blockDb.Delete(blockHash);
+        _blockCache.Delete(blockHash);
+    }
+
+    public Block? Get(Keccak blockHash, bool shouldCache)
+    {
+        return _blockDb.Get(blockHash, _blockDecoder, _blockCache, shouldCache);
+    }
+
+    public void Cache(Block block)
+    {
+        _blockCache.Set(block.Hash, block);
+    }
+}

src/Nethermind/Nethermind.Blockchain/Blocks/IBlockStore.cs

@@ -0,0 +1,24 @@
+// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using Nethermind.Core;
+using Nethermind.Core.Crypto;
+
+namespace Nethermind.Blockchain.Blocks;
+
+/// <summary>
+/// Raw block store. Does not know or care about blockchain or blocktree, only encoding/decoding to kv store.
+/// Generally you probably need IBlockTree instead of this.
+/// </summary>
+public interface IBlockStore
+{
+    void Insert(Block block);
+    void Delete(Keccak blockHash);
+    Block Get(Keccak blockHash, bool shouldCache = true);
+    void Cache(Block block);
+
+
+    // These two are used by blocktree. Try not to use them...
+    void SetMetadata(byte[] key, byte[] value);
+    byte[]? GetMetadata(byte[] key);
+}

src/Nethermind/Nethermind.Blockchain/IBlockTree.cs

@@ -80,8 +80,6 @@ public interface IBlockTree : IBlockFinder
         AddBlockResult Insert(Block block, BlockTreeInsertBlockOptions insertBlockOptions = BlockTreeInsertBlockOptions.None,
             BlockTreeInsertHeaderOptions insertHeaderOptions = BlockTreeInsertHeaderOptions.None);
 
-        void Insert(IEnumerable<Block> blocks);
-
         void UpdateHeadBlock(Keccak blockHash);
 
         /// <summary>