Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

More pruning scenario #4295

Merged
merged 4 commits into from
Jul 20, 2022
Merged

More pruning scenario #4295

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ca051c9
More pruning scenario
asdacap Jul 18, 2022
dceb67a
More failed pruning scenario
asdacap Jul 19, 2022
ee66e68
Fix some test scenario
asdacap Jul 19, 2022
38d5aa5
Fix if statement
asdacap Jul 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions src/Nethermind/Nethermind.Trie.Test/Pruning/TestPruningStrategy.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,24 @@ namespace Nethermind.Trie.Test.Pruning
public class TestPruningStrategy : IPruningStrategy
{
private readonly bool _pruningEnabled;
private readonly bool _shouldPrune;
public TestPruningStrategy(bool pruningEnabled, bool shouldPrune = false)
{
_pruningEnabled = pruningEnabled;
_shouldPrune = shouldPrune;
ShouldPruneEnabled = shouldPrune;
}

public bool PruningEnabled => _pruningEnabled;
public bool ShouldPruneEnabled { get; set; }

public bool ShouldPrune(in long currentMemory) => ShouldPruneEnabled || (_pruningEnabled && _shouldPrune);
public int? WithMemoryLimit { get; set; }

public bool ShouldPrune(in long currentMemory)
{
if (!_pruningEnabled) return false;
if (ShouldPruneEnabled) return true;
if (WithMemoryLimit != null && currentMemory > WithMemoryLimit) return true;

return false;
}
}
}
157 changes: 146 additions & 11 deletions src/Nethermind/Nethermind.Trie.Test/PruningScenariosTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,16 @@ public PruningContext SetAccountBalance(int accountIndex, UInt256 balance)
return this;
}

public PruningContext SetManyAccountWithSameBalance(int startNum, int numOfAccount, UInt256 balance)
{
for (int i = 0; i < numOfAccount; i++)
{
this.SetAccountBalance(startNum + i, balance);
}
return this;
}


public PruningContext PruneOldBlock()
{
return this;
Expand All @@ -122,6 +132,12 @@ public PruningContext TurnOnPrune()
return this;
}

public PruningContext SetPruningMemoryLimit(int? memoryLimit)
{
_pruningStrategy.WithMemoryLimit = memoryLimit;
return this;
}

public PruningContext SetStorage(int accountIndex, int storageKey, int storageValue = 1)
{
_storageProvider.Set(
Expand Down Expand Up @@ -160,6 +176,13 @@ public PruningContext ReadAccountViaStateReader(int accountIndex)
return this;
}

public PruningContext CommitWithRandomChange()
{
SetAccountBalance(Random.Shared.Next(), (UInt256)Random.Shared.Next());
Commit();
return this;
}

public PruningContext Commit()
{
_storageProvider.Commit();
Expand Down Expand Up @@ -452,36 +475,148 @@ public void Persist_alternate_commitset()
{
Reorganization.MaxDepth = 3;

PruningContext.InMemoryAlwaysPrune
PruningContext.InMemory
.SetAccountBalance(1, 100)
.Commit()
.SetAccountBalance(2, 10)
.Commit()

.SaveBranchingPoint("revert_main")
.SetAccountBalance(1, 103)
.CreateAccount(3)
.SetStorage(3, 1, 999)
.Commit()
.SaveBranchingPoint("main")

// Storage is not set here, but commit set will commit this instead of block 3 in main as it appear later
.RestoreBranchingPoint("revert_main")
.SetAccountBalance(1, 103)
// We need this to get persisted
// Storage is not set here, but commit set will commit this instead of previous block 3
.RestoreBranchingPoint("revert_main")
.SetManyAccountWithSameBalance(100, 20, 1)
.Commit()
.RestoreBranchingPoint("main")

.Commit()
.Commit()
.Commit()

.RestoreBranchingPoint("main")
.SetPruningMemoryLimit(10000)
// First commit it should prune and persist alternate block 3, memory usage should go down from 16k to 1.2k
.Commit()

.SetManyAccountWithSameBalance(100, 2, 1)
.Commit()
.VerifyStorageValue(3, 1, 999)
.SetManyAccountWithSameBalance(100, 2, 2)
.Commit()
.VerifyStorageValue(3, 1, 999)
.SetManyAccountWithSameBalance(100, 2, 3)
.Commit()
.VerifyStorageValue(3, 1, 999)
.SetManyAccountWithSameBalance(100, 2, 4)
.Commit()
.VerifyStorageValue(3, 1, 999)
.SetManyAccountWithSameBalance(100, 2, 5)
.Commit()
.VerifyStorageValue(3, 1, 999)
.SetManyAccountWithSameBalance(100, 2, 6)
.Commit()
.VerifyStorageValue(3, 1, 999)
.SetManyAccountWithSameBalance(100, 2, 7)
.Commit()
.VerifyStorageValue(3, 1, 999)
.SetManyAccountWithSameBalance(100, 2, 8)
.Commit()
.VerifyStorageValue(3, 1, 999)
.SetManyAccountWithSameBalance(100, 2, 9)
.Commit()

// Storage root actually never got pruned even-though another parallel branch get persisted. This
// is because the condition `LastSeen < LastPersistedBlock` never turn to true.
.VerifyStorageValue(3, 1, 999);
}

[Test]
public void Persist_alternate_branch_commitset_of_length_2()
{
Reorganization.MaxDepth = 3;

PruningContext.InMemory
.SetAccountBalance(1, 100)
.Commit()
.SetAccountBalance(2, 10)
.Commit()

.SaveBranchingPoint("revert_main")

// Block 3
.CreateAccount(3)
.SetStorage(3, 1, 999)
.Commit()

// Block 4
.Commit()
.SaveBranchingPoint("main")

// Block 3 - 2
.RestoreBranchingPoint("revert_main")
.Commit()

// Block 4 - 2
.SetManyAccountWithSameBalance(100, 20, 1)
.Commit()
.RestoreBranchingPoint("main")

.Commit()
.Commit()
.Commit()

.TurnOnPrune()
.Commit()

.VerifyStorageValue(3, 1, 999);
}

[Test]
public void Persist_with_2_alternate_branch_consecutive_of_each_other()
{
Reorganization.MaxDepth = 3;

PruningContext.InMemory
.SetAccountBalance(1, 100)
.Commit()
.SetAccountBalance(4, 108)
.SetAccountBalance(2, 10)
.Commit()

.SaveBranchingPoint("revert_main")

// Block 3
.CreateAccount(3)
.SetStorage(3, 1, 999)
.Commit()
.SaveBranchingPoint("main")

// Block 3 - 2
.RestoreBranchingPoint("revert_main")
.Commit()

.RestoreBranchingPoint("main")

// Block 4
.SaveBranchingPoint("revert_main")
.Commit()
.SaveBranchingPoint("main")

.RestoreBranchingPoint("revert_main") // Go back to block 3
// Block 4 - 2
.SetStorage(3, 1, 1)
.Commit()
.RestoreBranchingPoint("main") // Go back to block 4 on main

.TurnOnPrune()
.Commit()
.Commit()
.Commit()
.Commit()

// Although the committed set is different, later commit set still refer to the storage root hash.
// When later set is committed, the storage root hash will be committed also, unless the alternate chain
// is longer than reorg depth, in which case, we have two parallel branch of length > reorg depth, which
// is not supported.
.VerifyStorageValue(3, 1, 999);
}

Expand Down
34 changes: 24 additions & 10 deletions src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -412,28 +412,42 @@ private bool CanPruneCacheFurther()
{
if (_logger.IsDebug) _logger.Debug("Elevated pruning starting");

BlockCommitSet? candidateSet = null;
while (_commitSetQueue.TryPeek(out BlockCommitSet? frontSet))
List<BlockCommitSet> toAddBack = new();

List<BlockCommitSet> candidateSets = new();
while (_commitSetQueue.TryDequeue(out BlockCommitSet? frontSet))
{
if (frontSet!.BlockNumber >= LatestCommittedBlockNumber - Reorganization.MaxDepth)
{
break;
toAddBack.Add(frontSet);
continue;
}

if (_commitSetQueue.TryDequeue(out frontSet))
if (candidateSets.Count > 0 && candidateSets[0].BlockNumber == frontSet.BlockNumber)
{
candidateSet = frontSet;
candidateSets.Add(frontSet);
}
else
else if (candidateSets.Count == 0 || frontSet.BlockNumber > candidateSets[0].BlockNumber)
{
break;
candidateSets = new();
candidateSets.Add(frontSet);
}
}

if (candidateSet is not null)
// TODO: Find a way to not have to re-add everything
foreach (BlockCommitSet blockCommitSet in toAddBack)
{
_commitSetQueue.Enqueue(blockCommitSet);
}

foreach (BlockCommitSet blockCommitSet in candidateSets)
{
if (_logger.IsDebug) _logger.Debug($"Elevated pruning for candidate {blockCommitSet.BlockNumber}");
Persist(blockCommitSet);
}

if (candidateSets.Count > 0)
{
if (_logger.IsDebug) _logger.Debug($"Elevated pruning for candidate {candidateSet.BlockNumber}");
Persist(candidateSet);
return true;
}

Expand Down