Skip to content

Commit

Permalink
Fix rocksdb test will hang sometimes (#4686)
Browse files Browse the repository at this point in the history 
* Add write limits or it will hang sometimes

* Fix test still hangs

* Add test to make sure write is faulted
  • Loading branch information
@asdacap @rubo
asdacap authored and rubo committed Oct 13, 2022
1 parent 41b377f commit 8f26979
Show file tree
Hide file tree
Showing 2 changed files with 65 additions and 22 deletions.
63 changes: 45 additions & 18 deletions src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ public class DbOnTheRocks : IDbWithSpan

private static readonly ConcurrentDictionary<string, RocksDb> _dbsByPath = new();

private bool _isDisposing;
private bool _isDisposed;

private readonly ConcurrentHashSet<IBatch> _currentBatches = new();
Expand Down Expand Up @@ -273,7 +274,7 @@ public byte[]? this[byte[] key]
{
get
{
if (_isDisposed)
if (_isDisposing)
{
throw new ObjectDisposedException($"Attempted to read form a disposed database {Name}");
}
Expand All @@ -283,7 +284,7 @@ public byte[]? this[byte[] key]
}
set
{
if (_isDisposed)
if (_isDisposing)
{
throw new ObjectDisposedException($"Attempted to write to a disposed database {Name}");
}
Expand All @@ -305,7 +306,7 @@ public byte[]? this[byte[] key]

public Span<byte> GetSpan(byte[] key)
{
if (_isDisposed)
if (_isDisposing)
{
throw new ObjectDisposedException($"Attempted to read form a disposed database {Name}");
}
Expand All @@ -319,7 +320,7 @@ public Span<byte> GetSpan(byte[] key)

public void Remove(byte[] key)
{
if (_isDisposed)
if (_isDisposing)
{
throw new ObjectDisposedException($"Attempted to delete form a disposed database {Name}");
}
Expand All @@ -329,7 +330,7 @@ public void Remove(byte[] key)

public IEnumerable<KeyValuePair<byte[], byte[]>> GetAll(bool ordered = false)
{
if (_isDisposed)
if (_isDisposing)
{
throw new ObjectDisposedException($"Attempted to create an iterator on a disposed database {Name}");
}
Expand All @@ -347,7 +348,7 @@ protected internal Iterator CreateIterator(bool ordered = false, ColumnFamilyHan

public IEnumerable<byte[]> GetAllValues(bool ordered = false)
{
if (_isDisposed)
if (_isDisposing)
{
throw new ObjectDisposedException($"Attempted to read form a disposed database {Name}");
}
Expand All @@ -370,7 +371,7 @@ internal IEnumerable<byte[]> GetAllValuesCore(Iterator iterator)

public IEnumerable<KeyValuePair<byte[], byte[]>> GetAllCore(Iterator iterator)
{
if (_isDisposed)
if (_isDisposing)
{
throw new ObjectDisposedException($"Attempted to read form a disposed database {Name}");
}
Expand All @@ -387,7 +388,7 @@ public IEnumerable<KeyValuePair<byte[], byte[]>> GetAllCore(Iterator iterator)

public bool KeyExists(byte[] key)
{
if (_isDisposed)
if (_isDisposing)
{
throw new ObjectDisposedException($"Attempted to read form a disposed database {Name}");
}
Expand All @@ -407,14 +408,15 @@ public IBatch StartBatch()
internal class RocksDbBatch : IBatch
{
private readonly DbOnTheRocks _dbOnTheRocks;
private bool _isDisposed;

internal readonly WriteBatch _rocksBatch;

public RocksDbBatch(DbOnTheRocks dbOnTheRocks)
{
_dbOnTheRocks = dbOnTheRocks;

if (_dbOnTheRocks._isDisposed)
if (_dbOnTheRocks._isDisposing)
{
throw new ObjectDisposedException($"Attempted to create a batch on a disposed database {_dbOnTheRocks.Name}");
}
Expand All @@ -429,16 +431,35 @@ public void Dispose()
throw new ObjectDisposedException($"Attempted to commit a batch on a disposed database {_dbOnTheRocks.Name}");
}

if (_isDisposed)
{
return;
}
_isDisposed = true;

_dbOnTheRocks._db.Write(_rocksBatch, _dbOnTheRocks.WriteOptions);
_dbOnTheRocks._currentBatches.TryRemove(this);
_rocksBatch.Dispose();
}

public byte[]? this[byte[] key]
{
get => _dbOnTheRocks[key];
get
{
if (_isDisposed)
{
throw new ObjectDisposedException($"Attempted to read a disposed batch {_dbOnTheRocks.Name}");
}

return _dbOnTheRocks[key];
}
set
{
if (_isDisposed)
{
throw new ObjectDisposedException($"Attempted to write a disposed batch {_dbOnTheRocks.Name}");
}

if (value == null)
{
_rocksBatch.Delete(key);
Expand All @@ -453,11 +474,16 @@ public byte[]? this[byte[] key]

public void Flush()
{
if (_isDisposed)
if (_isDisposing)
{
throw new ObjectDisposedException($"Attempted to flush a disposed database {Name}");
}

InnerFlush();
}

private void InnerFlush()
{
RocksDbSharp.Native.Instance.rocksdb_flush(_db.Handle, FlushOptions.DefaultFlushOptions.Handle);
}

Expand Down Expand Up @@ -529,13 +555,14 @@ private void ReleaseUnmanagedResources()

public void Dispose()
{
if (!_isDisposed)
{
if (_logger.IsInfo) _logger.Info($"Disposing DB {Name}");
Flush();
ReleaseUnmanagedResources();
_dbsByPath.Remove(_fullPath!, out _);
}
if (_isDisposing) return;
_isDisposing = true;

if (_logger.IsInfo) _logger.Info($"Disposing DB {Name}");
InnerFlush();
ReleaseUnmanagedResources();

_dbsByPath.Remove(_fullPath!, out _);

_isDisposed = true;
}
Expand Down
24 changes: 20 additions & 4 deletions src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.

using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
using Nethermind.Core;
Expand Down Expand Up @@ -62,25 +64,39 @@ public async Task Dispose_while_writing_does_not_cause_access_violation_exceptio
IDbConfig config = new DbConfig();
DbOnTheRocks db = new("testDispose1", GetRocksDbSettings("testDispose1", "TestDispose1"), config, LimboLogs.Instance);

CancellationTokenSource cancelSource = new();
ManualResetEventSlim firstWriteWait = new();
firstWriteWait.Reset();
bool writeCompleted = false;

Task task = new(() =>
{
while (true)
for (int i = 0; i < 10000; i++)
{
// ReSharper disable once AccessToDisposedClosure
db.Set(Keccak.Zero, new byte[] { 1, 2, 3 });
if (i == 0) firstWriteWait.Set();

if (cancelSource.IsCancellationRequested)
{
return;
}
}

// ReSharper disable once FunctionNeverReturns
writeCompleted = true;
});

task.Start();

await Task.Delay(100);
firstWriteWait.Wait(TimeSpan.FromSeconds(1)).Should().BeTrue();

db.Dispose();

await Task.Delay(100);

cancelSource.Cancel();
writeCompleted.Should().BeFalse();

task.IsFaulted.Should().BeTrue();
task.Dispose();
}

Expand Down

0 comments on commit 8f26979

Please sign in to comment.