diff --git a/WalletWasabi.Backend/Global.cs b/WalletWasabi.Backend/Global.cs index 3b183259bc6..80e6c34f044 100644 --- a/WalletWasabi.Backend/Global.cs +++ b/WalletWasabi.Backend/Global.cs @@ -42,7 +42,7 @@ public Global(string dataDir, IRPCClient rpcClient, Config config) // Initialize index building var indexBuilderServiceDir = Path.Combine(DataDir, "IndexBuilderService"); - var indexFilePath = Path.Combine(indexBuilderServiceDir, $"Index{RpcClient.Network}.dat"); + var indexFilePath = Path.Combine(indexBuilderServiceDir, $"Index{RpcClient.Network}.sqlite"); IndexBuilderService = new(RpcClient, HostedServices.Get(), indexFilePath); } diff --git a/WalletWasabi.Tests/UnitTests/BitcoinCore/IndexBuilderServiceTests.cs b/WalletWasabi.Tests/UnitTests/BitcoinCore/IndexBuilderServiceTests.cs index 1b8f80f23fa..196a2fb695f 100644 --- a/WalletWasabi.Tests/UnitTests/BitcoinCore/IndexBuilderServiceTests.cs +++ b/WalletWasabi.Tests/UnitTests/BitcoinCore/IndexBuilderServiceTests.cs @@ -28,13 +28,14 @@ public async Task SegwitTaprootUnsynchronizedBitcoinNodeAsync() }), }; using var blockNotifier = new BlockNotifier(rpc); - var indexer = new IndexBuilderService(rpc, blockNotifier, "filters.txt"); + var indexer = new IndexBuilderService(rpc, blockNotifier, "filters.sqlite"); indexer.Synchronize(); await Task.Delay(TimeSpan.FromSeconds(1)); //// Assert.False(indexer.IsRunning); // <------------ ERROR: it should have stopped but there is a bug for RegTest - Assert.Throws(() => indexer.GetLastFilter()); // There are no filters + // There is only starting filter + Assert.True(indexer.GetLastFilter()?.Header.BlockHash.Equals(StartingFilters.GetStartingFilter(rpc.Network).Header.BlockHash)); } [Fact] @@ -55,13 +56,14 @@ public async Task SegwitTaprootStalledBitcoinNodeAsync() } }; using var blockNotifier = new BlockNotifier(rpc); - var indexer = new IndexBuilderService(rpc, blockNotifier, "filters.txt"); + var indexer = new IndexBuilderService(rpc, blockNotifier, "filters.sqlite"); indexer.Synchronize(); await Task.Delay(TimeSpan.FromSeconds(2)); Assert.True(indexer.IsRunning); // It is still working - Assert.Throws(() => indexer.GetLastFilter()); // There are no filters + // There is only starting filter + Assert.True(indexer.GetLastFilter()?.Header.BlockHash.Equals(StartingFilters.GetStartingFilter(rpc.Network).Header.BlockHash)); Assert.True(called > 1); } @@ -94,7 +96,7 @@ public async Task SegwitTaprootSynchronizingBitcoinNodeAsync() Assert.True(indexer.IsRunning); // It is still working var lastFilter = indexer.GetLastFilter(); - Assert.Equal(9, (int)lastFilter.Header.Height); + Assert.Equal(9, (int)lastFilter!.Header.Height); Assert.True(called > 1); } @@ -182,13 +184,14 @@ public async Task TaprootUnsynchronizedBitcoinNodeAsync() }), }; using var blockNotifier = new BlockNotifier(rpc); - var indexer = new IndexBuilderService(rpc, blockNotifier, "filters.txt"); + var indexer = new IndexBuilderService(rpc, blockNotifier, "filters.sqlite"); indexer.Synchronize(); await Task.Delay(TimeSpan.FromSeconds(1)); //// Assert.False(indexer.IsRunning); // <------------ ERROR: it should have stopped but there is a bug for RegTest - Assert.Throws(() => indexer.GetLastFilter()); // There are no filters + // There is only starting filter + Assert.True(indexer.GetLastFilter()?.Header.BlockHash.Equals(StartingFilters.GetStartingFilter(rpc.Network).Header.BlockHash)); } [Fact] @@ -209,13 +212,14 @@ public async Task TaprootStalledBitcoinNodeAsync() } }; using var blockNotifier = new BlockNotifier(rpc); - var indexer = new IndexBuilderService(rpc, blockNotifier, "filters.txt"); + var indexer = new IndexBuilderService(rpc, blockNotifier, "filters.sqlite"); indexer.Synchronize(); await Task.Delay(TimeSpan.FromSeconds(2)); Assert.True(indexer.IsRunning); // It is still working - Assert.Throws(() => indexer.GetLastFilter()); // There are no filters + // There is only starting filter + Assert.True(indexer.GetLastFilter()?.Header.BlockHash.Equals(StartingFilters.GetStartingFilter(rpc.Network).Header.BlockHash)); Assert.True(called > 1); } @@ -248,7 +252,7 @@ public async Task TaprootSynchronizingBitcoinNodeAsync() Assert.True(indexer.IsRunning); // It is still working var lastFilter = indexer.GetLastFilter(); - Assert.Equal(9, (int)lastFilter.Header.Height); + Assert.Equal(9, (int)lastFilter!.Header.Height); Assert.True(called > 1); } diff --git a/WalletWasabi/Blockchain/BlockFilters/IndexBuilderService.cs b/WalletWasabi/Blockchain/BlockFilters/IndexBuilderService.cs index 087a3b157b5..cf663557aa6 100644 --- a/WalletWasabi/Blockchain/BlockFilters/IndexBuilderService.cs +++ b/WalletWasabi/Blockchain/BlockFilters/IndexBuilderService.cs @@ -1,10 +1,10 @@ using NBitcoin; using System.Collections.Generic; -using System.Collections.Immutable; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.Data.Sqlite; using WalletWasabi.Backend.Models; using WalletWasabi.BitcoinCore.Rpc; using WalletWasabi.BitcoinCore.Rpc.Models; @@ -12,6 +12,7 @@ using WalletWasabi.Helpers; using WalletWasabi.Logging; using WalletWasabi.Models; +using WalletWasabi.Stores; namespace WalletWasabi.Blockchain.BlockFilters; @@ -41,31 +42,12 @@ public IndexBuilderService(IRPCClient rpc, BlockNotifier blockNotifier, string i IoHelpers.EnsureContainingDirectoryExists(IndexFilePath); - // Testing permissions. - using (var _ = File.Open(IndexFilePath, FileMode.OpenOrCreate, FileAccess.ReadWrite)) + if (RpcClient.Network == Network.RegTest && File.Exists(IndexFilePath)) { + File.Delete(IndexFilePath); // RegTest is not a global ledger, better to delete it. } - if (File.Exists(IndexFilePath)) - { - if (RpcClient.Network == Network.RegTest) - { - File.Delete(IndexFilePath); // RegTest is not a global ledger, better to delete it. - } - else - { - ImmutableList.Builder builder = ImmutableList.CreateBuilder(); - - foreach (var line in File.ReadAllLines(IndexFilePath)) - { - var filter = FilterModel.FromLine(line); - builder.Add(filter); - } - - Index = builder.ToImmutableList(); - } - } - + IndexStorage = CreateBlockFilterSqliteStorage(); BlockNotifier.OnBlock += BlockNotifier_OnBlock; } @@ -74,7 +56,7 @@ public IndexBuilderService(IRPCClient rpc, BlockNotifier blockNotifier, string i private IRPCClient RpcClient { get; } private BlockNotifier BlockNotifier { get; } private string IndexFilePath { get; } - private ImmutableList Index { get; set; } = ImmutableList.Empty; + private BlockFilterSqliteStorage IndexStorage { get; set; } /// Guards . private object IndexLock { get; } = new(); @@ -85,6 +67,21 @@ public IndexBuilderService(IRPCClient rpc, BlockNotifier blockNotifier, string i private RpcPubkeyType[] PubKeyTypes { get; } = [RpcPubkeyType.TxWitnessV0Keyhash, RpcPubkeyType.TxWitnessV1Taproot]; + private BlockFilterSqliteStorage CreateBlockFilterSqliteStorage() + { + try + { + return BlockFilterSqliteStorage.FromFile(dataSource: IndexFilePath, startingFilter: StartingFilters.GetStartingFilter(RpcClient.Network)); + } + catch (SqliteException ex) when (ex.SqliteExtendedErrorCode == 11) // 11 ~ SQLITE_CORRUPT error code + { + Logger.LogError($"Failed to open SQLite storage file because it's corrupted. Deleting the storage file '{IndexFilePath}'."); + + File.Delete(IndexFilePath); + throw; + } + } + public static GolombRiceFilter CreateDummyEmptyFilter(uint256 blockHash) { return new GolombRiceFilterBuilder() @@ -127,14 +124,11 @@ public void Synchronize() { SyncInfo syncInfo = await GetSyncInfoAsync().ConfigureAwait(false); - FilterModel? lastIndexFilter = null; + FilterModel? lastIndexFilter; lock (IndexLock) { - if (Index.Count != 0) - { - lastIndexFilter = Index[^1]; - } + lastIndexFilter = GetLastFilter(); } uint currentHeight; @@ -190,7 +184,7 @@ public void Synchronize() { Logger.LogWarning("Reorg observed on the network."); - await ReorgOneAsync().ConfigureAwait(false); + ReorgOne(); // Skip the current block. continue; @@ -201,11 +195,9 @@ public void Synchronize() var smartHeader = new SmartHeader(block.Hash, block.PrevBlockHash, nextHeight, block.BlockTime); var filterModel = new FilterModel(smartHeader, filter); - await File.AppendAllLinesAsync(IndexFilePath, new[] { filterModel.ToLine() }).ConfigureAwait(false); - lock (IndexLock) { - Index = Index.Add(filterModel); + IndexStorage.TryAppend(filterModel); } // If not close to the tip, just log debug. @@ -289,22 +281,15 @@ private static List