From 5a80b576c66d360e76cd9cbee4be6378826f7090 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Mon, 3 Feb 2025 11:38:05 +0200 Subject: [PATCH 1/4] add logging for trie goroutines and batch --- common/interface.go | 2 +- genesis/process/memoryComponents.go | 1 + .../benchmarks/loadFromTrie_test.go | 4 +-- .../state/stateTrie/stateTrie_test.go | 14 ++++----- .../state/stateTrieSync/stateTrieSync_test.go | 4 +-- state/accountsDB.go | 10 +++---- state/accountsDB_test.go | 30 +++++++++---------- .../storagePruningManager_test.go | 4 +-- state/syncer/baseAccountsSyncer.go | 3 +- state/trackableDataTrie/trackableDataTrie.go | 3 +- testscommon/trie/trieStub.go | 6 ++-- trie/baseIterator.go | 2 +- trie/branchNode_test.go | 6 ++-- trie/depthFirstSync_test.go | 4 +-- trie/doubleListSync_test.go | 4 +-- trie/extensionNode_test.go | 18 +++++------ trie/factory/trieCreator.go | 1 + trie/goroutinesManager.go | 7 +++++ trie/goroutinesManager_test.go | 24 +++++++-------- trie/leafNode_test.go | 12 ++++---- trie/patriciaMerkleTrie.go | 23 ++++++++------ trie/patriciaMerkleTrie_test.go | 14 ++++----- trie/sync_test.go | 2 +- trie/trieBatchManager/trieBatchManager.go | 22 ++++++++++++-- .../trieBatchManager/trieBatchManager_test.go | 16 +++++----- trie/trieChangesBatch/trieChangesBatch.go | 13 ++++++-- .../trieChangesBatch/trieChangesBatch_test.go | 16 +++++----- 27 files changed, 153 insertions(+), 112 deletions(-) diff --git a/common/interface.go b/common/interface.go index 1cf647bdc90..90d1aeb682e 100644 --- a/common/interface.go +++ b/common/interface.go @@ -42,7 +42,7 @@ type Trie interface { Delete(key []byte) RootHash() ([]byte, error) Commit(collector TrieHashesCollector) error - Recreate(options RootHashHolder) (Trie, error) + Recreate(options RootHashHolder, identifier string) (Trie, error) GetSerializedNodes([]byte, uint64) ([][]byte, uint64, error) GetSerializedNode([]byte) ([]byte, error) GetAllLeavesOnChannel(allLeavesChan *TrieIteratorChannels, ctx context.Context, rootHash []byte, keyBuilder KeyBuilder, trieLeafParser TrieLeafParser) error diff --git a/genesis/process/memoryComponents.go b/genesis/process/memoryComponents.go index cefa0631f59..24404271cb6 100644 --- a/genesis/process/memoryComponents.go +++ b/genesis/process/memoryComponents.go @@ -29,6 +29,7 @@ func createAccountAdapter( EnableEpochsHandler: enableEpochsHandler, MaxTrieLevelInMemory: maxTrieLevelInMemory, Throttler: trie.NewDisabledTrieGoRoutinesThrottler(), + Identifier: "main trie in memory", }) if err != nil { return nil, err diff --git a/integrationTests/benchmarks/loadFromTrie_test.go b/integrationTests/benchmarks/loadFromTrie_test.go index 21fcd3a37fa..520ddbe8bf0 100644 --- a/integrationTests/benchmarks/loadFromTrie_test.go +++ b/integrationTests/benchmarks/loadFromTrie_test.go @@ -70,7 +70,7 @@ func testTrieLoadTime(t *testing.T, numChildrenPerBranch int, numTries int, maxT func timeTrieRecreate(tries []*keyForTrie, depth int) { startTime := time.Now() for j := range tries { - _, _ = tries[j].tr.Recreate(holders.NewDefaultRootHashesHolder(tries[j].key)) + _, _ = tries[j].tr.Recreate(holders.NewDefaultRootHashesHolder(tries[j].key), "") } duration := time.Since(startTime) fmt.Printf("trie with depth %d, duration %d \n", depth, duration.Nanoseconds()/int64(len(tries))) @@ -117,7 +117,7 @@ func generateTriesWithMaxDepth( key := insertKeysIntoTrie(t, tr, numTrieLevels, numChildrenPerBranch) rootHash, _ := tr.RootHash() - collapsedTrie, _ := tr.Recreate(holders.NewDefaultRootHashesHolder(rootHash)) + collapsedTrie, _ := tr.Recreate(holders.NewDefaultRootHashesHolder(rootHash), "") if numTrieLevels == 1 { key = rootHash diff --git a/integrationTests/state/stateTrie/stateTrie_test.go b/integrationTests/state/stateTrie/stateTrie_test.go index a8d47468da9..cf99c61a698 100644 --- a/integrationTests/state/stateTrie/stateTrie_test.go +++ b/integrationTests/state/stateTrie/stateTrie_test.go @@ -283,7 +283,7 @@ func TestTrieDB_RecreateFromStorageShouldWork(t *testing.T) { err := tr1.Commit(hashesCollector.NewDisabledHashesCollector()) require.Nil(t, err) - tr2, err := tr1.Recreate(holders.NewDefaultRootHashesHolder(h1)) + tr2, err := tr1.Recreate(holders.NewDefaultRootHashesHolder(h1), "") require.Nil(t, err) valRecov, _, err := tr2.Get(key) @@ -1290,7 +1290,7 @@ func TestTrieDbPruning_GetDataTrieTrackerAfterPruning(t *testing.T) { func collapseTrie(state state.UserAccountHandler, t *testing.T) { stateRootHash := state.GetRootHash() stateTrie := state.DataTrie().(common.Trie) - stateNewTrie, _ := stateTrie.Recreate(holders.NewDefaultRootHashesHolder(stateRootHash)) + stateNewTrie, _ := stateTrie.Recreate(holders.NewDefaultRootHashesHolder(stateRootHash), "") require.NotNil(t, stateNewTrie) state.SetDataTrie(stateNewTrie) @@ -1692,12 +1692,12 @@ func checkTrieCanBeRecreated(tb testing.TB, node *integrationTests.TestProcessor stateTrie := node.TrieContainer.Get([]byte(dataRetriever.UserAccountsUnit.String())) roothash := node.BlockChain.GetCurrentBlockRootHash() - tr, err := stateTrie.Recreate(holders.NewDefaultRootHashesHolder(roothash)) + tr, err := stateTrie.Recreate(holders.NewDefaultRootHashesHolder(roothash), "") require.Nil(tb, err) require.NotNil(tb, tr) _, _, finalRoothash := node.BlockChain.GetFinalBlockInfo() - tr, err = stateTrie.Recreate(holders.NewDefaultRootHashesHolder(finalRoothash)) + tr, err = stateTrie.Recreate(holders.NewDefaultRootHashesHolder(finalRoothash), "") require.Nil(tb, err) require.NotNil(tb, tr) @@ -1709,7 +1709,7 @@ func checkTrieCanBeRecreated(tb testing.TB, node *integrationTests.TestProcessor err = integrationTests.TestMarshalizer.Unmarshal(hdr, hdrBytes) require.Nil(tb, err) - tr, err = stateTrie.Recreate(holders.NewDefaultRootHashesHolder(hdr.GetRootHash())) + tr, err = stateTrie.Recreate(holders.NewDefaultRootHashesHolder(hdr.GetRootHash()), "") require.Nil(tb, err) require.NotNil(tb, tr) } @@ -1860,7 +1860,7 @@ func testNodeStateSnapshotAndPruning( stateTrie := node.TrieContainer.Get([]byte(dataRetriever.UserAccountsUnit.String())) assert.Equal(t, 1, len(snapshotsRootHashes)) for i := range snapshotsRootHashes { - tr, err := stateTrie.Recreate(holders.NewDefaultRootHashesHolder(snapshotsRootHashes[i])) + tr, err := stateTrie.Recreate(holders.NewDefaultRootHashesHolder(snapshotsRootHashes[i]), "") require.Nil(t, err) require.NotNil(t, tr) } @@ -1868,7 +1868,7 @@ func testNodeStateSnapshotAndPruning( assert.Equal(t, 1, len(prunedRootHashes)) // if pruning is called for a root hash in a different epoch than the commit, then recreate trie should work for i := 0; i < len(prunedRootHashes)-1; i++ { - tr, err := stateTrie.Recreate(holders.NewDefaultRootHashesHolder(prunedRootHashes[i])) + tr, err := stateTrie.Recreate(holders.NewDefaultRootHashesHolder(prunedRootHashes[i]), "") require.Nil(t, tr) require.NotNil(t, err) } diff --git a/integrationTests/state/stateTrieSync/stateTrieSync_test.go b/integrationTests/state/stateTrieSync/stateTrieSync_test.go index 40d9eb37e71..4080993b72f 100644 --- a/integrationTests/state/stateTrieSync/stateTrieSync_test.go +++ b/integrationTests/state/stateTrieSync/stateTrieSync_test.go @@ -141,7 +141,7 @@ func testNodeRequestInterceptTrieNodesWithMessenger(t *testing.T, version int) { assert.Nil(t, err) cancel() - requesterTrie, err = requesterTrie.Recreate(holders.NewDefaultRootHashesHolder(rootHash)) + requesterTrie, err = requesterTrie.Recreate(holders.NewDefaultRootHashesHolder(rootHash), "") require.Nil(t, err) newRootHash, _ := requesterTrie.RootHash() @@ -563,7 +563,7 @@ func copyPartialState(t *testing.T, sourceNode, destinationNode *integrationTest func getDataTriesHashes(t *testing.T, tr common.Trie, dataTriesRootHashes [][]byte) [][]byte { hashes := make([][]byte, 0) for _, rh := range dataTriesRootHashes { - dt, err := tr.Recreate(holders.NewDefaultRootHashesHolder(rh)) + dt, err := tr.Recreate(holders.NewDefaultRootHashesHolder(rh), "") assert.Nil(t, err) dtHashes := GetAllHashes(t, dt, rh) diff --git a/state/accountsDB.go b/state/accountsDB.go index 61bc7911027..f859157ea67 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -437,7 +437,7 @@ func (adb *AccountsDB) loadDataTrieConcurrentSafe(accountHandler baseAccountHand } rootHashHolder := holders.NewDefaultRootHashesHolder(accountHandler.GetRootHash()) - dataTrie, err := mainTrie.Recreate(rootHashHolder) + dataTrie, err := mainTrie.Recreate(rootHashHolder, hex.EncodeToString(accountHandler.AddressBytes())) if err != nil { return fmt.Errorf("trie was not found for hash, rootHash = %s, err = %w", hex.EncodeToString(accountHandler.GetRootHash()), err) } @@ -878,7 +878,7 @@ func (adb *AccountsDB) recreateTrie(options common.RootHashHolder) error { adb.dataTries.Reset() adb.entries = make([]JournalEntry, 0) - newTrie, err := adb.mainTrie.Recreate(options) + newTrie, err := adb.mainTrie.Recreate(options, string(common.MainTrie)) if err != nil { return err } @@ -925,7 +925,7 @@ func (adb *AccountsDB) RecreateAllTries(rootHash []byte) (map[string]common.Trie userAccountRootHash := userAccount.GetRootHash() if len(userAccountRootHash) > 0 { rootHashHolder := holders.NewDefaultRootHashesHolder(userAccountRootHash) - dataTrie, errRecreate := mainTrie.Recreate(rootHashHolder) + dataTrie, errRecreate := mainTrie.Recreate(rootHashHolder, "") if errRecreate != nil { return nil, errRecreate } @@ -964,7 +964,7 @@ func getUserAccountFromBytes(accountFactory AccountFactory, marshaller marshal.M func (adb *AccountsDB) recreateMainTrie(rootHash []byte) (map[string]common.Trie, error) { rootHashHolder := holders.NewDefaultRootHashesHolder(rootHash) - recreatedTrie, err := adb.getMainTrie().Recreate(rootHashHolder) + recreatedTrie, err := adb.getMainTrie().Recreate(rootHashHolder, string(common.MainTrie)) if err != nil { return nil, err } @@ -978,7 +978,7 @@ func (adb *AccountsDB) recreateMainTrie(rootHash []byte) (map[string]common.Trie // GetTrie returns the trie that has the given rootHash func (adb *AccountsDB) GetTrie(rootHash []byte) (common.Trie, error) { rootHashHolder := holders.NewDefaultRootHashesHolder(rootHash) - return adb.getMainTrie().Recreate(rootHashHolder) + return adb.getMainTrie().Recreate(rootHashHolder, "") } // Journalize adds a new object to entries list. diff --git a/state/accountsDB_test.go b/state/accountsDB_test.go index 7e0f44aabd3..a8c83b50dfa 100644 --- a/state/accountsDB_test.go +++ b/state/accountsDB_test.go @@ -543,7 +543,7 @@ func TestAccountsDB_LoadAccountExistingShouldLoadDataTrie(t *testing.T) { } return nil, 0, nil }, - RecreateCalled: func(holder common.RootHashHolder) (d common.Trie, err error) { + RecreateCalled: func(holder common.RootHashHolder, _ string) (d common.Trie, err error) { return dataTrie, nil }, GetStorageManagerCalled: func() common.StorageManager { @@ -621,7 +621,7 @@ func TestAccountsDB_GetExistingAccountFoundShouldRetAccount(t *testing.T) { } return nil, 0, nil }, - RecreateCalled: func(root common.RootHashHolder) (d common.Trie, err error) { + RecreateCalled: func(root common.RootHashHolder, _ string) (d common.Trie, err error) { return dataTrie, nil }, GetStorageManagerCalled: func() common.StorageManager { @@ -844,7 +844,7 @@ func TestAccountsDB_LoadDataWithSomeValuesShouldWork(t *testing.T) { account := generateAccount() mockTrie := &trieMock.TrieStub{ - RecreateCalled: func(root common.RootHashHolder) (trie common.Trie, e error) { + RecreateCalled: func(root common.RootHashHolder, _ string) (trie common.Trie, e error) { if !bytes.Equal(root.GetRootHash(), rootHash) { return nil, errors.New("bad root hash") } @@ -889,7 +889,7 @@ func TestAccountsDB_CommitShouldCallCommitFromTrie(t *testing.T) { GetCalled: func(_ []byte) ([]byte, uint32, error) { return serializedAccount, 0, nil }, - RecreateCalled: func(root common.RootHashHolder) (trie common.Trie, err error) { + RecreateCalled: func(root common.RootHashHolder, _ string) (trie common.Trie, err error) { return &trieMock.TrieStub{ GetCalled: func(_ []byte) ([]byte, uint32, error) { return []byte("doge"), 0, nil @@ -937,7 +937,7 @@ func TestAccountsDB_RecreateTrieMalfunctionTrieShouldErr(t *testing.T) { return &storageManager.StorageManagerStub{} }, } - trieStub.RecreateCalled = func(_ common.RootHashHolder) (tree common.Trie, e error) { + trieStub.RecreateCalled = func(_ common.RootHashHolder, _ string) (tree common.Trie, e error) { wasCalled = true return nil, errExpected } @@ -959,7 +959,7 @@ func TestAccountsDB_RecreateTrieOutputsNilTrieShouldErr(t *testing.T) { return &storageManager.StorageManagerStub{} }, } - trieStub.RecreateCalled = func(_ common.RootHashHolder) (tree common.Trie, e error) { + trieStub.RecreateCalled = func(_ common.RootHashHolder, _ string) (tree common.Trie, e error) { wasCalled = true return nil, nil } @@ -981,7 +981,7 @@ func TestAccountsDB_RecreateTrieOkValsShouldWork(t *testing.T) { GetStorageManagerCalled: func() common.StorageManager { return &storageManager.StorageManagerStub{} }, - RecreateCalled: func(_ common.RootHashHolder) (common.Trie, error) { + RecreateCalled: func(_ common.RootHashHolder, _ string) (common.Trie, error) { wasCalled = true return &trieMock.TrieStub{}, nil }, @@ -1890,7 +1890,7 @@ func TestAccountsDB_PruningAndPruningCancellingOnTrieRollback(t *testing.T) { } for i := 0; i < len(rootHashes); i++ { - _, err := tr.Recreate(holders.NewDefaultRootHashesHolder(rootHashes[i])) + _, err := tr.Recreate(holders.NewDefaultRootHashesHolder(rootHashes[i]), "") assert.Nil(t, err) } @@ -1899,7 +1899,7 @@ func TestAccountsDB_PruningAndPruningCancellingOnTrieRollback(t *testing.T) { finalizeTrieState(t, 2, tr, adb, rootHashes) rollbackTrieState(t, 3, tr, adb, rootHashes) - _, err := tr.Recreate(holders.NewDefaultRootHashesHolder(rootHashes[2])) + _, err := tr.Recreate(holders.NewDefaultRootHashesHolder(rootHashes[2]), "") assert.Nil(t, err) } @@ -1908,7 +1908,7 @@ func finalizeTrieState(t *testing.T, index int, tr common.Trie, adb state.Accoun adb.CancelPrune(rootHashes[index], state.NewRoot) time.Sleep(trieDbOperationDelay) - _, err := tr.Recreate(holders.NewDefaultRootHashesHolder(rootHashes[index-1])) + _, err := tr.Recreate(holders.NewDefaultRootHashesHolder(rootHashes[index-1]), "") assert.NotNil(t, err) } @@ -1917,7 +1917,7 @@ func rollbackTrieState(t *testing.T, index int, tr common.Trie, adb state.Accoun adb.CancelPrune(rootHashes[index-1], state.OldRoot) time.Sleep(trieDbOperationDelay) - _, err := tr.Recreate(holders.NewDefaultRootHashesHolder(rootHashes[index])) + _, err := tr.Recreate(holders.NewDefaultRootHashesHolder(rootHashes[index]), "") assert.NotNil(t, err) } @@ -2063,7 +2063,7 @@ func TestAccountsDB_RecreateAllTries(t *testing.T) { return nil }, - RecreateCalled: func(root common.RootHashHolder) (common.Trie, error) { + RecreateCalled: func(root common.RootHashHolder, _ string) (common.Trie, error) { return &trieMock.TrieStub{}, nil }, GetStorageManagerCalled: func() common.StorageManager { @@ -2094,7 +2094,7 @@ func TestAccountsDB_RecreateAllTries(t *testing.T) { return nil }, - RecreateCalled: func(root common.RootHashHolder) (common.Trie, error) { + RecreateCalled: func(root common.RootHashHolder, _ string) (common.Trie, error) { return &trieMock.TrieStub{}, nil }, GetStorageManagerCalled: func() common.StorageManager { @@ -2266,7 +2266,7 @@ func TestAccountsDB_GetAccountFromBytes(t *testing.T) { }, } args.Trie = &trieMock.TrieStub{ - RecreateCalled: func(root common.RootHashHolder) (common.Trie, error) { + RecreateCalled: func(root common.RootHashHolder, _ string) (common.Trie, error) { assert.Equal(t, rootHash, root.GetRootHash()) return &trieMock.TrieStub{}, nil }, @@ -2299,7 +2299,7 @@ func TestAccountsDB_GetAccountFromBytesShouldLoadDataTrie(t *testing.T) { } return nil, 0, nil }, - RecreateCalled: func(root common.RootHashHolder) (d common.Trie, err error) { + RecreateCalled: func(root common.RootHashHolder, _ string) (d common.Trie, err error) { return dataTrie, nil }, GetStorageManagerCalled: func() common.StorageManager { diff --git a/state/storagePruningManager/storagePruningManager_test.go b/state/storagePruningManager/storagePruningManager_test.go index 489d716fb31..01a9411f488 100644 --- a/state/storagePruningManager/storagePruningManager_test.go +++ b/state/storagePruningManager/storagePruningManager_test.go @@ -2,7 +2,7 @@ package storagePruningManager import ( "testing" - + "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/holders" @@ -234,7 +234,7 @@ func TestAccountsDB_PruneAfterCancelPruneShouldFail(t *testing.T) { spm.CancelPrune(rootHash, state.OldRoot, trieStorage) spm.PruneTrie(rootHash, state.OldRoot, trieStorage, state.NewPruningHandler(state.EnableDataRemoval)) - newTr, err := tr.Recreate(holders.NewDefaultRootHashesHolder(rootHash)) + newTr, err := tr.Recreate(holders.NewDefaultRootHashesHolder(rootHash), "") assert.Nil(t, err) assert.NotNil(t, newTr) } diff --git a/state/syncer/baseAccountsSyncer.go b/state/syncer/baseAccountsSyncer.go index 634ebce0baf..a9d79a306d8 100644 --- a/state/syncer/baseAccountsSyncer.go +++ b/state/syncer/baseAccountsSyncer.go @@ -224,6 +224,7 @@ func (b *baseAccountsSyncer) GetSyncedTries() map[string]common.Trie { EnableEpochsHandler: b.enableEpochsHandler, MaxTrieLevelInMemory: b.maxTrieLevelInMemory, Throttler: trie.NewDisabledTrieGoRoutinesThrottler(), + Identifier: "base sync main trie ", } dataTrie, err := trie.NewTrie(trieArgs) if err != nil { @@ -235,7 +236,7 @@ func (b *baseAccountsSyncer) GetSyncedTries() map[string]common.Trie { clonedMap := make(map[string]common.Trie, len(b.dataTries)) for key := range b.dataTries { rootHashHolder := holders.NewDefaultRootHashesHolder([]byte(key)) - recreatedTrie, err = dataTrie.Recreate(rootHashHolder) + recreatedTrie, err = dataTrie.Recreate(rootHashHolder, "base sync data trie") if err != nil { log.Warn("error recreating trie in baseAccountsSyncer.GetSyncedTries", "roothash", []byte(key), "error", err) diff --git a/state/trackableDataTrie/trackableDataTrie.go b/state/trackableDataTrie/trackableDataTrie.go index 7994c0cbf9a..d483fec185b 100644 --- a/state/trackableDataTrie/trackableDataTrie.go +++ b/state/trackableDataTrie/trackableDataTrie.go @@ -1,6 +1,7 @@ package trackableDataTrie import ( + "encoding/hex" "fmt" "github.com/multiversx/mx-chain-core-go/core" @@ -216,7 +217,7 @@ func (tdt *trackableDataTrie) SaveDirtyData(mainTrie common.Trie) ([]core.TrieDa if check.IfNil(tdt.tr) { emptyRootHash := holders.NewDefaultRootHashesHolder(make([]byte, 0)) - newDataTrie, err := mainTrie.Recreate(emptyRootHash) + newDataTrie, err := mainTrie.Recreate(emptyRootHash, hex.EncodeToString(tdt.identifier)) if err != nil { return nil, err } diff --git a/testscommon/trie/trieStub.go b/testscommon/trie/trieStub.go index 07f5b2ea01e..fd8edad33bc 100644 --- a/testscommon/trie/trieStub.go +++ b/testscommon/trie/trieStub.go @@ -19,7 +19,7 @@ type TrieStub struct { DeleteCalled func(key []byte) RootCalled func() ([]byte, error) CommitCalled func(collector common.TrieHashesCollector) error - RecreateCalled func(options common.RootHashHolder) (common.Trie, error) + RecreateCalled func(options common.RootHashHolder, identifier string) (common.Trie, error) AppendToOldHashesCalled func([][]byte) GetSerializedNodesCalled func([]byte, uint64) ([][]byte, uint64, error) GetAllLeavesOnChannelCalled func(leavesChannels *common.TrieIteratorChannels, ctx context.Context, rootHash []byte, keyBuilder common.KeyBuilder, trieLeafParser common.TrieLeafParser) error @@ -130,9 +130,9 @@ func (ts *TrieStub) Commit(hc common.TrieHashesCollector) error { } // Recreate - -func (ts *TrieStub) Recreate(options common.RootHashHolder) (common.Trie, error) { +func (ts *TrieStub) Recreate(options common.RootHashHolder, identifier string) (common.Trie, error) { if ts.RecreateCalled != nil { - return ts.RecreateCalled(options) + return ts.RecreateCalled(options, identifier) } return nil, errNotImplemented diff --git a/trie/baseIterator.go b/trie/baseIterator.go index 2163a516748..367d083bdb2 100644 --- a/trie/baseIterator.go +++ b/trie/baseIterator.go @@ -18,7 +18,7 @@ func newBaseIterator(trie common.Trie, rootHash []byte) (*baseIterator, error) { return nil, ErrNilTrie } - trie, err := trie.Recreate(holders.NewDefaultRootHashesHolder(rootHash)) + trie, err := trie.Recreate(holders.NewDefaultRootHashesHolder(rootHash), "") if err != nil { return nil, err } diff --git a/trie/branchNode_test.go b/trie/branchNode_test.go index 37edc88096a..84370db537c 100644 --- a/trie/branchNode_test.go +++ b/trie/branchNode_test.go @@ -28,7 +28,7 @@ const initialModifiedHashesCapacity = 10 func getTestGoroutinesManager() common.TrieGoroutinesManager { th, _ := throttler.NewNumGoRoutinesThrottler(5) - goRoutinesManager, _ := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) + goRoutinesManager, _ := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{}), "") return goRoutinesManager } @@ -94,7 +94,7 @@ func newEmptyTrie() (*patriciaMerkleTrie, *trieStorageManager) { maxTrieLevelInMemory: 5, chanClose: make(chan struct{}), enableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, - batchManager: trieBatchManager.NewTrieBatchManager(), + batchManager: trieBatchManager.NewTrieBatchManager(""), goRoutinesManager: getTestGoroutinesManager(), RootManager: NewRootManager(), trieOperationInProgress: &atomic.Flag{}, @@ -891,7 +891,7 @@ func TestPatriciaMerkleTrie_CommitCollapsedDirtyTrieShouldWork(t *testing.T) { _ = tr.Update([]byte("zzz"), []byte("zzz")) _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() - collapsedTrie, _ := tr.recreate(rootHash, tr.trieStorage) + collapsedTrie, _ := tr.recreate(rootHash, "", tr.trieStorage) collapsedRoot := collapsedTrie.GetRootNode() collapsedTrie.Delete([]byte("zzz")) diff --git a/trie/depthFirstSync_test.go b/trie/depthFirstSync_test.go index ce993be6ca8..69f89738103 100644 --- a/trie/depthFirstSync_test.go +++ b/trie/depthFirstSync_test.go @@ -123,7 +123,7 @@ func TestDepthFirstTrieSyncer_StartSyncingNewTrieShouldWork(t *testing.T) { tsm, _ := arg.DB.(*trieStorageManager) db, _ := tsm.mainStorer.(storage.Persister) trie, _ := createInMemoryTrieFromDB(db) - trie, _ = trie.Recreate(holders.NewDefaultRootHashesHolder(roothash)) + trie, _ = trie.Recreate(holders.NewDefaultRootHashesHolder(roothash), "") require.False(t, check.IfNil(trie)) var val []byte @@ -200,7 +200,7 @@ func TestDepthFirstTrieSyncer_StartSyncingPartiallyFilledTrieShouldWork(t *testi tsm, _ := arg.DB.(*trieStorageManager) db, _ := tsm.mainStorer.(storage.Persister) trie, _ := createInMemoryTrieFromDB(db) - trie, _ = trie.Recreate(holders.NewDefaultRootHashesHolder(roothash)) + trie, _ = trie.Recreate(holders.NewDefaultRootHashesHolder(roothash), "") require.False(t, check.IfNil(trie)) var val []byte diff --git a/trie/doubleListSync_test.go b/trie/doubleListSync_test.go index 0bfb5754a8a..cd02489afd1 100644 --- a/trie/doubleListSync_test.go +++ b/trie/doubleListSync_test.go @@ -217,7 +217,7 @@ func TestDoubleListTrieSyncer_StartSyncingNewTrieShouldWork(t *testing.T) { tsm, _ := arg.DB.(*trieStorageManager) db, _ := tsm.mainStorer.(storage.Persister) trie, _ := createInMemoryTrieFromDB(db) - trie, _ = trie.Recreate(holders.NewDefaultRootHashesHolder(roothash)) + trie, _ = trie.Recreate(holders.NewDefaultRootHashesHolder(roothash), "") require.False(t, check.IfNil(trie)) var val []byte @@ -294,7 +294,7 @@ func TestDoubleListTrieSyncer_StartSyncingPartiallyFilledTrieShouldWork(t *testi tsm, _ := arg.DB.(*trieStorageManager) db, _ := tsm.mainStorer.(storage.Persister) trie, _ := createInMemoryTrieFromDB(db) - trie, _ = trie.Recreate(holders.NewDefaultRootHashesHolder(roothash)) + trie, _ = trie.Recreate(holders.NewDefaultRootHashesHolder(roothash), "") require.False(t, check.IfNil(trie)) var val []byte diff --git a/trie/extensionNode_test.go b/trie/extensionNode_test.go index cb03047d08b..2e8e7174f83 100644 --- a/trie/extensionNode_test.go +++ b/trie/extensionNode_test.go @@ -543,7 +543,7 @@ func TestExtensionNode_reduceNodeCollapsedNode(t *testing.T) { tr := initTrie() _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() - collapsedTrie, _ := tr.Recreate(holders.NewDefaultRootHashesHolder(rootHash)) + collapsedTrie, _ := tr.Recreate(holders.NewDefaultRootHashesHolder(rootHash), "") collapsedTrie.Delete([]byte("doe")) @@ -988,7 +988,7 @@ func TestExtensionNode_insertInSameEn(t *testing.T) { } th, _ := throttler.NewNumGoRoutinesThrottler(5) - goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) + goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{}), "") assert.Nil(t, err) modifiedHashes := common.NewModifiedHashesSlice(initialModifiedHashesCapacity) @@ -1014,7 +1014,7 @@ func TestExtensionNode_insertInSameEn(t *testing.T) { } th, _ := throttler.NewNumGoRoutinesThrottler(5) - goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) + goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{}), "") assert.Nil(t, err) modifiedHashes := common.NewModifiedHashesSlice(initialModifiedHashesCapacity) @@ -1052,7 +1052,7 @@ func TestExtensionNode_insertInNewBn(t *testing.T) { } th, _ := throttler.NewNumGoRoutinesThrottler(5) - goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) + goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{}), "") assert.Nil(t, err) modifiedHashes := common.NewModifiedHashesSlice(initialModifiedHashesCapacity) @@ -1088,7 +1088,7 @@ func TestExtensionNode_insertInNewBn(t *testing.T) { } th, _ := throttler.NewNumGoRoutinesThrottler(5) - goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) + goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{}), "") assert.Nil(t, err) modifiedHashes := common.NewModifiedHashesSlice(initialModifiedHashesCapacity) @@ -1123,7 +1123,7 @@ func TestExtensionNode_deleteBatch(t *testing.T) { } th, _ := throttler.NewNumGoRoutinesThrottler(5) - goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) + goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{}), "") assert.Nil(t, err) modifiedHashes := common.NewModifiedHashesSlice(initialModifiedHashesCapacity) @@ -1148,7 +1148,7 @@ func TestExtensionNode_deleteBatch(t *testing.T) { } th, _ := throttler.NewNumGoRoutinesThrottler(5) - goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) + goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{}), "") assert.Nil(t, err) modifiedHashes := common.NewModifiedHashesSlice(initialModifiedHashesCapacity) @@ -1174,7 +1174,7 @@ func TestExtensionNode_deleteBatch(t *testing.T) { } th, _ := throttler.NewNumGoRoutinesThrottler(5) - goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) + goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{}), "") assert.Nil(t, err) newEn := en.insert(data, goRoutinesManager, common.NewModifiedHashesSlice(initialModifiedHashesCapacity), nil) @@ -1214,7 +1214,7 @@ func TestExtensionNode_deleteBatch(t *testing.T) { } th, _ := throttler.NewNumGoRoutinesThrottler(5) - goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) + goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{}), "") assert.Nil(t, err) modifiedHashes := common.NewModifiedHashesSlice(initialModifiedHashesCapacity) diff --git a/trie/factory/trieCreator.go b/trie/factory/trieCreator.go index 9257d5e7529..73bcfc3cba5 100644 --- a/trie/factory/trieCreator.go +++ b/trie/factory/trieCreator.go @@ -93,6 +93,7 @@ func (tc *trieCreator) Create(args TrieCreateArgs) (common.StorageManager, commo EnableEpochsHandler: args.EnableEpochsHandler, MaxTrieLevelInMemory: args.MaxTrieLevelInMem, Throttler: goRoutinesThrottler, + Identifier: string(common.MainTrie), } newTrie, err := trie.NewTrie(trArgs) diff --git a/trie/goroutinesManager.go b/trie/goroutinesManager.go index 1932daa7f1c..72bc7d28559 100644 --- a/trie/goroutinesManager.go +++ b/trie/goroutinesManager.go @@ -15,6 +15,7 @@ type goroutinesManager struct { mutex sync.RWMutex canProcess bool + identifier string } // NewGoroutinesManager creates a new GoroutinesManager @@ -22,6 +23,7 @@ func NewGoroutinesManager( throttler core.Throttler, errorChannel common.BufferedErrChan, chanClose chan struct{}, + identifier string, ) (*goroutinesManager, error) { if check.IfNil(throttler) { return nil, ErrNilThrottler @@ -38,6 +40,7 @@ func NewGoroutinesManager( errorChannel: errorChannel, chanClose: chanClose, canProcess: true, + identifier: identifier, }, nil } @@ -45,6 +48,7 @@ func NewGoroutinesManager( func (gm *goroutinesManager) ShouldContinueProcessing() bool { select { case <-gm.chanClose: + log.Trace("goroutines manager closed", "identifier", gm.identifier) return false default: gm.mutex.RLock() @@ -63,6 +67,7 @@ func (gm *goroutinesManager) CanStartGoRoutine() bool { return false } + log.Trace("starting processing goroutine", "identifier", gm.identifier) gm.throttler.StartProcessing() return true } @@ -72,6 +77,7 @@ func (gm *goroutinesManager) EndGoRoutineProcessing() { gm.mutex.Lock() defer gm.mutex.Unlock() + log.Trace("ending processing goroutine", "identifier", gm.identifier) gm.throttler.EndProcessing() } @@ -94,6 +100,7 @@ func (gm *goroutinesManager) SetError(err error) { gm.mutex.Lock() defer gm.mutex.Unlock() + log.Trace("setting error", "identifier", gm.identifier, "error", err) gm.errorChannel.WriteInChanNonBlocking(err) gm.canProcess = false } diff --git a/trie/goroutinesManager_test.go b/trie/goroutinesManager_test.go index 02f7155ff9f..bfc7e30ac0a 100644 --- a/trie/goroutinesManager_test.go +++ b/trie/goroutinesManager_test.go @@ -15,28 +15,28 @@ func TestNewGoroutinesManager(t *testing.T) { t.Run("nil throttler", func(t *testing.T) { t.Parallel() - manager, err := NewGoroutinesManager(nil, nil, nil) + manager, err := NewGoroutinesManager(nil, nil, nil, "") assert.Nil(t, manager) assert.Equal(t, ErrNilThrottler, err) }) t.Run("nil error channel", func(t *testing.T) { t.Parallel() - manager, err := NewGoroutinesManager(&mock.ThrottlerStub{}, nil, nil) + manager, err := NewGoroutinesManager(&mock.ThrottlerStub{}, nil, nil, "") assert.Nil(t, manager) assert.Equal(t, ErrNilBufferedErrChan, err) }) t.Run("nil chan close", func(t *testing.T) { t.Parallel() - manager, err := NewGoroutinesManager(&mock.ThrottlerStub{}, errChan.NewErrChanWrapper(), nil) + manager, err := NewGoroutinesManager(&mock.ThrottlerStub{}, errChan.NewErrChanWrapper(), nil, "") assert.Nil(t, manager) assert.Equal(t, ErrNilChanClose, err) }) t.Run("should work", func(t *testing.T) { t.Parallel() - manager, err := NewGoroutinesManager(&mock.ThrottlerStub{}, errChan.NewErrChanWrapper(), make(chan struct{})) + manager, err := NewGoroutinesManager(&mock.ThrottlerStub{}, errChan.NewErrChanWrapper(), make(chan struct{}), "") assert.NotNil(t, manager) assert.Nil(t, err) assert.True(t, manager.canProcess) @@ -50,7 +50,7 @@ func TestGoroutinesManager_ShouldContinueProcessing(t *testing.T) { t.Parallel() closeChan := make(chan struct{}) - manager, _ := NewGoroutinesManager(&mock.ThrottlerStub{}, errChan.NewErrChanWrapper(), closeChan) + manager, _ := NewGoroutinesManager(&mock.ThrottlerStub{}, errChan.NewErrChanWrapper(), closeChan, "") close(closeChan) assert.False(t, manager.ShouldContinueProcessing()) @@ -59,7 +59,7 @@ func TestGoroutinesManager_ShouldContinueProcessing(t *testing.T) { t.Parallel() closeChan := make(chan struct{}) - manager, _ := NewGoroutinesManager(&mock.ThrottlerStub{}, errChan.NewErrChanWrapper(), closeChan) + manager, _ := NewGoroutinesManager(&mock.ThrottlerStub{}, errChan.NewErrChanWrapper(), closeChan, "") assert.True(t, manager.ShouldContinueProcessing()) }) @@ -86,7 +86,7 @@ func TestGoroutinesManager_CanStartGoRoutine(t *testing.T) { return false }, } - manager, _ := NewGoroutinesManager(throttler, errChan.NewErrChanWrapper(), make(chan struct{})) + manager, _ := NewGoroutinesManager(throttler, errChan.NewErrChanWrapper(), make(chan struct{}), "") assert.False(t, manager.CanStartGoRoutine()) }) @@ -102,7 +102,7 @@ func TestGoroutinesManager_CanStartGoRoutine(t *testing.T) { startProcessingCalled = true }, } - manager, _ := NewGoroutinesManager(throttler, errChan.NewErrChanWrapper(), make(chan struct{})) + manager, _ := NewGoroutinesManager(throttler, errChan.NewErrChanWrapper(), make(chan struct{}), "") assert.True(t, manager.CanStartGoRoutine()) assert.True(t, startProcessingCalled) @@ -118,7 +118,7 @@ func TestGoroutinesManager_EndGoRoutineProcessing(t *testing.T) { endProcessingCalled = true }, } - manager, _ := NewGoroutinesManager(throttler, errChan.NewErrChanWrapper(), make(chan struct{})) + manager, _ := NewGoroutinesManager(throttler, errChan.NewErrChanWrapper(), make(chan struct{}), "") manager.EndGoRoutineProcessing() assert.True(t, endProcessingCalled) } @@ -129,7 +129,7 @@ func TestGoroutinesManager_SetError(t *testing.T) { t.Run("should set error", func(t *testing.T) { expectedErr := errors.New("error") errCh := errChan.NewErrChanWrapper() - manager, _ := NewGoroutinesManager(&mock.ThrottlerStub{}, errCh, make(chan struct{})) + manager, _ := NewGoroutinesManager(&mock.ThrottlerStub{}, errCh, make(chan struct{}), "") err := errCh.ReadFromChanNonBlocking() assert.Nil(t, err) @@ -146,7 +146,7 @@ func TestGoroutinesManager_SetError(t *testing.T) { expectedErr := errors.New("error") anotherErr := errors.New("another error") errCh := errChan.NewErrChanWrapper() - manager, _ := NewGoroutinesManager(&mock.ThrottlerStub{}, errCh, make(chan struct{})) + manager, _ := NewGoroutinesManager(&mock.ThrottlerStub{}, errCh, make(chan struct{}), "") err := errCh.ReadFromChanNonBlocking() assert.Nil(t, err) @@ -165,7 +165,7 @@ func TestGoroutinesManager_GetError(t *testing.T) { expectedErr := errors.New("error") errCh := errChan.NewErrChanWrapper() - manager, _ := NewGoroutinesManager(&mock.ThrottlerStub{}, errCh, make(chan struct{})) + manager, _ := NewGoroutinesManager(&mock.ThrottlerStub{}, errCh, make(chan struct{}), "") err := manager.GetError() assert.Nil(t, err) diff --git a/trie/leafNode_test.go b/trie/leafNode_test.go index 77b00184262..c8e51f14769 100644 --- a/trie/leafNode_test.go +++ b/trie/leafNode_test.go @@ -661,7 +661,7 @@ func TestLeafNode_insertBatch(t *testing.T) { originalHash := ln.getHash() th, _ := throttler.NewNumGoRoutinesThrottler(5) - goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) + goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{}), "") assert.Nil(t, err) modifiedHashes := common.NewModifiedHashesSlice(initialModifiedHashesCapacity) @@ -682,7 +682,7 @@ func TestLeafNode_insertBatch(t *testing.T) { assert.False(t, ln.dirty) th, _ := throttler.NewNumGoRoutinesThrottler(5) - goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) + goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{}), "") assert.Nil(t, err) modifiedHashes := common.NewModifiedHashesSlice(initialModifiedHashesCapacity) @@ -707,7 +707,7 @@ func TestLeafNode_insertBatch(t *testing.T) { originalHash := ln.getHash() th, _ := throttler.NewNumGoRoutinesThrottler(5) - goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) + goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{}), "") assert.Nil(t, err) modifiedHashes := common.NewModifiedHashesSlice(initialModifiedHashesCapacity) @@ -738,7 +738,7 @@ func TestLeafNode_insertBatch(t *testing.T) { originalHash := ln.getHash() th, _ := throttler.NewNumGoRoutinesThrottler(5) - goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) + goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{}), "") assert.Nil(t, err) modifiedHashes := common.NewModifiedHashesSlice(initialModifiedHashesCapacity) @@ -770,7 +770,7 @@ func TestLeafNode_deleteBatch(t *testing.T) { originalHash := ln.getHash() th, _ := throttler.NewNumGoRoutinesThrottler(5) - goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) + goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{}), "") assert.Nil(t, err) modifiedHashes := common.NewModifiedHashesSlice(initialModifiedHashesCapacity) @@ -792,7 +792,7 @@ func TestLeafNode_deleteBatch(t *testing.T) { ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) th, _ := throttler.NewNumGoRoutinesThrottler(5) - goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) + goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{}), "") assert.Nil(t, err) modifiedHashes := common.NewModifiedHashesSlice(initialModifiedHashesCapacity) diff --git a/trie/patriciaMerkleTrie.go b/trie/patriciaMerkleTrie.go index d9d8c162e1e..c14e56c4b1a 100644 --- a/trie/patriciaMerkleTrie.go +++ b/trie/patriciaMerkleTrie.go @@ -51,6 +51,7 @@ type patriciaMerkleTrie struct { maxTrieLevelInMemory uint chanClose chan struct{} + identifier string } // TrieArgs is the arguments for creating a new trie @@ -61,6 +62,7 @@ type TrieArgs struct { EnableEpochsHandler common.EnableEpochsHandler MaxTrieLevelInMemory uint Throttler core.Throttler + Identifier string } // NewTrie creates a new Patricia Merkle Trie @@ -93,7 +95,7 @@ func NewTrie( } chanClose := make(chan struct{}) - goRoutinesManager, err := NewGoroutinesManager(args.Throttler, errChan.NewErrChanWrapper(), chanClose) + goRoutinesManager, err := NewGoroutinesManager(args.Throttler, errChan.NewErrChanWrapper(), chanClose, args.Identifier) if err != nil { return nil, err } @@ -107,11 +109,12 @@ func NewTrie( chanClose: chanClose, enableEpochsHandler: args.EnableEpochsHandler, trieNodeVersionVerifier: tnvv, - batchManager: trieBatchManager.NewTrieBatchManager(), + batchManager: trieBatchManager.NewTrieBatchManager(args.Identifier), goRoutinesManager: goRoutinesManager, trieOperationInProgress: &atomic.Flag{}, updateTrieMutex: sync.RWMutex{}, throttler: args.Throttler, + identifier: args.Identifier, }, nil } @@ -385,13 +388,13 @@ func (tr *patriciaMerkleTrie) Commit(hashesCollector common.TrieHashesCollector) } // Recreate returns a new trie, given the options -func (tr *patriciaMerkleTrie) Recreate(options common.RootHashHolder) (common.Trie, error) { +func (tr *patriciaMerkleTrie) Recreate(options common.RootHashHolder, identifier string) (common.Trie, error) { if check.IfNil(options) { return nil, ErrNilRootHashHolder } if !options.GetEpoch().HasValue { - return tr.recreate(options.GetRootHash(), tr.trieStorage) + return tr.recreate(options.GetRootHash(), identifier, tr.trieStorage) } tsmie, err := newTrieStorageManagerInEpoch(tr.trieStorage, options.GetEpoch().Value) @@ -399,10 +402,10 @@ func (tr *patriciaMerkleTrie) Recreate(options common.RootHashHolder) (common.Tr return nil, err } - return tr.recreate(options.GetRootHash(), tsmie) + return tr.recreate(options.GetRootHash(), identifier, tsmie) } -func (tr *patriciaMerkleTrie) recreate(root []byte, tsm common.StorageManager) (*patriciaMerkleTrie, error) { +func (tr *patriciaMerkleTrie) recreate(root []byte, identifier string, tsm common.StorageManager) (*patriciaMerkleTrie, error) { if common.IsEmptyTrie(root) { return NewTrie( TrieArgs{ @@ -412,11 +415,12 @@ func (tr *patriciaMerkleTrie) recreate(root []byte, tsm common.StorageManager) ( EnableEpochsHandler: tr.enableEpochsHandler, MaxTrieLevelInMemory: tr.maxTrieLevelInMemory, Throttler: tr.throttler, + Identifier: identifier, }, ) } - newTr, _, err := tr.recreateFromDb(root, tsm) + newTr, _, err := tr.recreateFromDb(root, identifier, tsm) if err != nil { if core.IsClosingError(err) { log.Debug("could not recreate", "rootHash", root, "error", err) @@ -455,7 +459,7 @@ func (tr *patriciaMerkleTrie) IsInterfaceNil() bool { return tr == nil } -func (tr *patriciaMerkleTrie) recreateFromDb(rootHash []byte, tsm common.StorageManager) (*patriciaMerkleTrie, snapshotNode, error) { +func (tr *patriciaMerkleTrie) recreateFromDb(rootHash []byte, identifier string, tsm common.StorageManager) (*patriciaMerkleTrie, snapshotNode, error) { newTr, err := NewTrie( TrieArgs{ tsm, @@ -464,6 +468,7 @@ func (tr *patriciaMerkleTrie) recreateFromDb(rootHash []byte, tsm common.Storage tr.enableEpochsHandler, tr.maxTrieLevelInMemory, tr.throttler, + identifier, }, ) if err != nil { @@ -578,7 +583,7 @@ func (tr *patriciaMerkleTrie) GetAllLeavesOnChannel( return ErrNilTrieLeafParser } - newTrie, err := tr.recreate(rootHash, tr.trieStorage) + newTrie, err := tr.recreate(rootHash, "", tr.trieStorage) if err != nil { close(leavesChannels.LeavesChan) leavesChannels.ErrChan.Close() diff --git a/trie/patriciaMerkleTrie_test.go b/trie/patriciaMerkleTrie_test.go index 7331414e498..a6f3743d4fc 100644 --- a/trie/patriciaMerkleTrie_test.go +++ b/trie/patriciaMerkleTrie_test.go @@ -421,7 +421,7 @@ func TestPatriciaMerkleTrie_Recreate(t *testing.T) { tr := initTrie() - newTr, err := tr.Recreate(nil) + newTr, err := tr.Recreate(nil, "") assert.Nil(t, newTr) assert.Equal(t, trie.ErrNilRootHashHolder, err) }) @@ -434,7 +434,7 @@ func TestPatriciaMerkleTrie_Recreate(t *testing.T) { _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHashHolder := holders.NewDefaultRootHashesHolder(rootHash) - newTr, err := tr.Recreate(rootHashHolder) + newTr, err := tr.Recreate(rootHashHolder, "") assert.Nil(t, err) assert.True(t, trie.IsBaseTrieStorageManager(newTr.GetStorageManager())) @@ -452,7 +452,7 @@ func TestPatriciaMerkleTrie_Recreate(t *testing.T) { HasValue: true, } rootHashHolder := holders.NewRootHashHolder(rootHash, optionalUint32) - newTr, err := tr.Recreate(rootHashHolder) + newTr, err := tr.Recreate(rootHashHolder, "") assert.Nil(t, err) assert.True(t, trie.IsTrieStorageManagerInEpoch(newTr.GetStorageManager())) @@ -464,7 +464,7 @@ func TestPatriciaMerkleTrie_RecreateWithInvalidRootHash(t *testing.T) { tr := initTrie() - newTr, err := tr.Recreate(holders.NewDefaultRootHashesHolder([]byte{})) + newTr, err := tr.Recreate(holders.NewDefaultRootHashesHolder([]byte{}), "") assert.Nil(t, err) root, _ := newTr.RootHash() assert.Equal(t, emptyTrieHash, root) @@ -988,7 +988,7 @@ func TestPatriciaMerkleTrie_ConcurrentOperations(t *testing.T) { HasValue: true, } rootHashHolder := holders.NewRootHashHolder(initialRootHash, epoch) - _, err := tr.Recreate(rootHashHolder) + _, err := tr.Recreate(rootHashHolder, "") assert.Nil(t, err) case 6: _, err := tr.GetSerializedNode(initialRootHash) @@ -1367,7 +1367,7 @@ func TestPatriciaMerkleTrie_CollectLeavesForMigration(t *testing.T) { addDefaultDataToTrie(tr) _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() - collapsedTrie, _ := tr.Recreate(holders.NewDefaultRootHashesHolder(rootHash)) + collapsedTrie, _ := tr.Recreate(holders.NewDefaultRootHashesHolder(rootHash), "") dtr := collapsedTrie.(dataTrie) dtm := &trieMock.DataTrieMigratorStub{ ConsumeStorageLoadGasCalled: func() bool { @@ -1783,7 +1783,7 @@ func TestPatriciaMerkleTrie_Get(t *testing.T) { // collapse the trie rootHash, _ := tr.RootHash() - tr, _ = tr.Recreate(holders.NewDefaultRootHashesHolder(rootHash)) + tr, _ = tr.Recreate(holders.NewDefaultRootHashesHolder(rootHash), "") for i := numTrieValues; i < numTrieValues+numBatchValues; i++ { _ = tr.Update([]byte("dog"+strconv.Itoa(i)), []byte("reindeer"+strconv.Itoa(i))) diff --git a/trie/sync_test.go b/trie/sync_test.go index edc935524b2..0a9fd5a79bd 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -3,7 +3,6 @@ package trie import ( "context" "errors" - "github.com/multiversx/mx-chain-go/state/hashesCollector" "sync" "testing" "time" @@ -12,6 +11,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-go/common" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" "github.com/multiversx/mx-chain-go/testscommon/marshallerMock" diff --git a/trie/trieBatchManager/trieBatchManager.go b/trie/trieBatchManager/trieBatchManager.go index 4ea3713eb76..efa43699cca 100644 --- a/trie/trieBatchManager/trieBatchManager.go +++ b/trie/trieBatchManager/trieBatchManager.go @@ -8,8 +8,11 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/trie/trieChangesBatch" + logger "github.com/multiversx/mx-chain-logger-go" ) +var log = logger.GetOrCreate("trieBatch") + // ErrTrieUpdateInProgress signals that a trie update is in progress var ErrTrieUpdateInProgress = errors.New("trie update is in progress") @@ -18,15 +21,17 @@ type trieBatchManager struct { tempBatch common.TrieBatcher isUpdateInProgress bool + identifier string mutex sync.RWMutex } // NewTrieBatchManager creates a new instance of trieBatchManager -func NewTrieBatchManager() *trieBatchManager { +func NewTrieBatchManager(identifier string) *trieBatchManager { return &trieBatchManager{ - currentBatch: trieChangesBatch.NewTrieChangesBatch(), + currentBatch: trieChangesBatch.NewTrieChangesBatch(identifier), tempBatch: nil, isUpdateInProgress: false, + identifier: identifier, } } @@ -39,9 +44,11 @@ func (t *trieBatchManager) MarkTrieUpdateInProgress() (common.TrieBatcher, error return nil, ErrTrieUpdateInProgress } + log.Debug("marking trie update in progress", "identifier", t.identifier) + t.isUpdateInProgress = true t.tempBatch = t.currentBatch - t.currentBatch = trieChangesBatch.NewTrieChangesBatch() + t.currentBatch = trieChangesBatch.NewTrieChangesBatch(t.identifier) return t.tempBatch, nil } @@ -51,6 +58,8 @@ func (t *trieBatchManager) MarkTrieUpdateCompleted() { t.mutex.Lock() defer t.mutex.Unlock() + log.Trace("marking trie update completed", "identifier", t.identifier) + t.isUpdateInProgress = false t.tempBatch = nil } @@ -60,6 +69,8 @@ func (t *trieBatchManager) Add(data core.TrieData) { t.mutex.RLock() defer t.mutex.RUnlock() + log.Trace("adding data to the current batch", "identifier", t.identifier, "key", data.Key, "value", data.Value) + t.currentBatch.Add(data) } @@ -70,16 +81,19 @@ func (t *trieBatchManager) Get(key []byte) ([]byte, bool) { val, isPresent := t.currentBatch.Get(key) if isPresent { + log.Trace("found data in the current batch", "identifier", t.identifier, "key", key, "value", val) return val, true } if t.isUpdateInProgress && !check.IfNil(t.tempBatch) { val, isPresent = t.tempBatch.Get(key) if isPresent { + log.Trace("found data in the temp batch", "identifier", t.identifier, "key", key, "value", val) return val, true } } + log.Trace("data not found in any batch", "identifier", t.identifier, "key", key) return nil, false } @@ -88,6 +102,8 @@ func (t *trieBatchManager) MarkForRemoval(key []byte) { t.mutex.RLock() defer t.mutex.RUnlock() + log.Trace("marking key for removal in the current batch", "identifier", t.identifier, "key", key) + t.currentBatch.MarkForRemoval(key) } diff --git a/trie/trieBatchManager/trieBatchManager_test.go b/trie/trieBatchManager/trieBatchManager_test.go index e253fda7b5b..ede19498a0d 100644 --- a/trie/trieBatchManager/trieBatchManager_test.go +++ b/trie/trieBatchManager/trieBatchManager_test.go @@ -11,7 +11,7 @@ import ( func TestNewTrieBatchManager(t *testing.T) { t.Parallel() - tbm := NewTrieBatchManager() + tbm := NewTrieBatchManager("") assert.False(t, check.IfNil(tbm)) assert.False(t, tbm.isUpdateInProgress) @@ -22,7 +22,7 @@ func TestNewTrieBatchManager(t *testing.T) { func TestTrieBatchManager_TrieUpdateInProgress(t *testing.T) { t.Parallel() - tbm := NewTrieBatchManager() + tbm := NewTrieBatchManager("") tbm.Add(core.TrieData{Key: []byte("key1")}) tbm.Add(core.TrieData{Key: []byte("key2")}) assert.False(t, tbm.isUpdateInProgress) @@ -49,7 +49,7 @@ func TestTrieBatchManager_TrieUpdateInProgress(t *testing.T) { func TestTrieBatchManager_AddUpdatesCurrentBatch(t *testing.T) { t.Parallel() - tbm := NewTrieBatchManager() + tbm := NewTrieBatchManager("") tbm.Add(core.TrieData{Key: []byte("key1")}) _, found := tbm.currentBatch.Get([]byte("key1")) assert.True(t, found) @@ -69,7 +69,7 @@ func TestTrieBatchManager_Get(t *testing.T) { t.Run("key exists in currentBatch", func(t *testing.T) { t.Parallel() - tbm := NewTrieBatchManager() + tbm := NewTrieBatchManager("") tbm.currentBatch.Add(core.TrieData{ Key: key, Value: value, @@ -82,7 +82,7 @@ func TestTrieBatchManager_Get(t *testing.T) { t.Run("check temp batch only if update in progress", func(t *testing.T) { t.Parallel() - tbm := NewTrieBatchManager() + tbm := NewTrieBatchManager("") tbm.currentBatch.Add(core.TrieData{ Key: key, Value: value, @@ -98,7 +98,7 @@ func TestTrieBatchManager_Get(t *testing.T) { t.Run("code does not panic if temp batch is nil", func(t *testing.T) { t.Parallel() - tbm := NewTrieBatchManager() + tbm := NewTrieBatchManager("") tbm.isUpdateInProgress = true tbm.tempBatch = nil @@ -109,7 +109,7 @@ func TestTrieBatchManager_Get(t *testing.T) { t.Run("key does not exist", func(t *testing.T) { t.Parallel() - tbm := NewTrieBatchManager() + tbm := NewTrieBatchManager("") data, found := tbm.Get(key) assert.False(t, found) @@ -121,7 +121,7 @@ func TestTrieBatchManager_MarkForRemovalUpdatesCurrentBatch(t *testing.T) { t.Parallel() key := []byte("key") - tbm := NewTrieBatchManager() + tbm := NewTrieBatchManager("") _, _ = tbm.MarkTrieUpdateInProgress() tbm.MarkForRemoval(key) diff --git a/trie/trieChangesBatch/trieChangesBatch.go b/trie/trieChangesBatch/trieChangesBatch.go index 2ea86e37ebb..62ec917fd29 100644 --- a/trie/trieChangesBatch/trieChangesBatch.go +++ b/trie/trieChangesBatch/trieChangesBatch.go @@ -6,20 +6,25 @@ import ( "sync" "github.com/multiversx/mx-chain-core-go/core" + logger "github.com/multiversx/mx-chain-logger-go" ) +var log = logger.GetOrCreate("trieBatch") + type trieChangesBatch struct { insertedData map[string]core.TrieData deletedKeys map[string]struct{} - mutex sync.RWMutex + identifier string + mutex sync.RWMutex } // NewTrieChangesBatch creates a new instance of trieChangesBatch -func NewTrieChangesBatch() *trieChangesBatch { +func NewTrieChangesBatch(identifier string) *trieChangesBatch { return &trieChangesBatch{ insertedData: make(map[string]core.TrieData), deletedKeys: make(map[string]struct{}), + identifier: identifier, } } @@ -77,6 +82,8 @@ func (t *trieChangesBatch) GetSortedDataForInsertion() []core.TrieData { data = append(data, t.insertedData[k]) } + log.Trace("sorted data for insertion", "identifier", t.identifier, "num insertions", len(data)) + return getSortedData(data) } @@ -90,6 +97,8 @@ func (t *trieChangesBatch) GetSortedDataForRemoval() []core.TrieData { data = append(data, core.TrieData{Key: []byte(k)}) } + log.Trace("sorted data for removal", "identifier", t.identifier, "num deletes", len(data)) + return getSortedData(data) } diff --git a/trie/trieChangesBatch/trieChangesBatch_test.go b/trie/trieChangesBatch/trieChangesBatch_test.go index 383c922fff6..00bfca4117a 100644 --- a/trie/trieChangesBatch/trieChangesBatch_test.go +++ b/trie/trieChangesBatch/trieChangesBatch_test.go @@ -11,7 +11,7 @@ import ( func TestNewTrieChangesBatch(t *testing.T) { t.Parallel() - tcb := NewTrieChangesBatch() + tcb := NewTrieChangesBatch("") assert.False(t, check.IfNil(tcb)) assert.Equal(t, 0, len(tcb.insertedData)) assert.Equal(t, 0, len(tcb.deletedKeys)) @@ -26,7 +26,7 @@ func TestTrieChangesBatch_Add(t *testing.T) { Version: core.NotSpecified, } - tcb := NewTrieChangesBatch() + tcb := NewTrieChangesBatch("") tcb.deletedKeys[string(dataForInsertion.Key)] = struct{}{} tcb.Add(dataForInsertion) @@ -40,7 +40,7 @@ func TestTrieChangesBatch_MarkForRemoval(t *testing.T) { keyForDeletion := []byte("keyForDeletion") - tcb := NewTrieChangesBatch() + tcb := NewTrieChangesBatch("") tcb.insertedData[string(keyForDeletion)] = core.TrieData{ Key: []byte("trieKey"), Value: []byte("trieValue"), @@ -63,7 +63,7 @@ func TestTrieChangesBatch_Get(t *testing.T) { key := []byte("key") value := []byte("value") - tcb := NewTrieChangesBatch() + tcb := NewTrieChangesBatch("") tcb.insertedData[string(key)] = core.TrieData{ Key: key, Value: value, @@ -78,7 +78,7 @@ func TestTrieChangesBatch_Get(t *testing.T) { t.Parallel() key := []byte("key") - tcb := NewTrieChangesBatch() + tcb := NewTrieChangesBatch("") tcb.deletedKeys[string(key)] = struct{}{} data, foundInBatch := tcb.Get(key) @@ -89,7 +89,7 @@ func TestTrieChangesBatch_Get(t *testing.T) { t.Parallel() key := []byte("key") - tcb := NewTrieChangesBatch() + tcb := NewTrieChangesBatch("") data, foundInBatch := tcb.Get(key) assert.False(t, foundInBatch) @@ -100,7 +100,7 @@ func TestTrieChangesBatch_Get(t *testing.T) { func TestTrieChangesBatch_GetSortedDataForInsertion(t *testing.T) { t.Parallel() - tcb := NewTrieChangesBatch() + tcb := NewTrieChangesBatch("") tcb.insertedData["key3"] = core.TrieData{Key: []byte("key3")} tcb.insertedData["key1"] = core.TrieData{Key: []byte("key1")} tcb.insertedData["key2"] = core.TrieData{Key: []byte("key2")} @@ -115,7 +115,7 @@ func TestTrieChangesBatch_GetSortedDataForInsertion(t *testing.T) { func TestTrieChangesBatch_GetSortedDataForRemoval(t *testing.T) { t.Parallel() - tcb := NewTrieChangesBatch() + tcb := NewTrieChangesBatch("") tcb.deletedKeys["key3"] = struct{}{} tcb.deletedKeys["key1"] = struct{}{} From eebace35a96f0c629622844dda36b4add6ce0d53 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Mon, 3 Feb 2025 11:56:30 +0200 Subject: [PATCH 2/4] small fix --- state/peerAccountsDB_test.go | 2 +- state/trackableDataTrie/trackableDataTrie_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/state/peerAccountsDB_test.go b/state/peerAccountsDB_test.go index 3e1f2322c24..9787cfe8a4b 100644 --- a/state/peerAccountsDB_test.go +++ b/state/peerAccountsDB_test.go @@ -163,7 +163,7 @@ func TestNewPeerAccountsDB_RecreateAllTries(t *testing.T) { GetStorageManagerCalled: func() common.StorageManager { return &storageManager.StorageManagerStub{} }, - RecreateCalled: func(_ common.RootHashHolder) (common.Trie, error) { + RecreateCalled: func(_ common.RootHashHolder, _ string) (common.Trie, error) { recreateCalled = true return nil, nil }, diff --git a/state/trackableDataTrie/trackableDataTrie_test.go b/state/trackableDataTrie/trackableDataTrie_test.go index dc04a55d8df..de5fc56923d 100644 --- a/state/trackableDataTrie/trackableDataTrie_test.go +++ b/state/trackableDataTrie/trackableDataTrie_test.go @@ -367,7 +367,7 @@ func TestTrackableDataTrie_SaveDirtyData(t *testing.T) { recreateCalled := false trie := &trieMock.TrieStub{ - RecreateCalled: func(root common.RootHashHolder) (common.Trie, error) { + RecreateCalled: func(root common.RootHashHolder, _ string) (common.Trie, error) { recreateCalled = true return &trieMock.TrieStub{ GetCalled: func(_ []byte) ([]byte, uint32, error) { From ad95ed27116791a108dfda96f41cdb541b9c3ed3 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Mon, 3 Feb 2025 15:21:25 +0200 Subject: [PATCH 3/4] fixes after review --- state/syncer/baseAccountsSyncer.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/state/syncer/baseAccountsSyncer.go b/state/syncer/baseAccountsSyncer.go index a9d79a306d8..2f1ee2a7dd3 100644 --- a/state/syncer/baseAccountsSyncer.go +++ b/state/syncer/baseAccountsSyncer.go @@ -18,6 +18,8 @@ import ( "github.com/multiversx/mx-chain-go/trie" ) +const syncTrieIdentifier = "base sync main trie" + type baseAccountsSyncer struct { hasher hashing.Hasher marshalizer marshal.Marshalizer @@ -224,7 +226,7 @@ func (b *baseAccountsSyncer) GetSyncedTries() map[string]common.Trie { EnableEpochsHandler: b.enableEpochsHandler, MaxTrieLevelInMemory: b.maxTrieLevelInMemory, Throttler: trie.NewDisabledTrieGoRoutinesThrottler(), - Identifier: "base sync main trie ", + Identifier: syncTrieIdentifier, } dataTrie, err := trie.NewTrie(trieArgs) if err != nil { From 7895a8da49bb75e1bb7c6dd73c38e5e416e4fce5 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Mon, 3 Feb 2025 16:00:00 +0200 Subject: [PATCH 4/4] fix after review --- genesis/process/memoryComponents.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/genesis/process/memoryComponents.go b/genesis/process/memoryComponents.go index 24404271cb6..46d058209f7 100644 --- a/genesis/process/memoryComponents.go +++ b/genesis/process/memoryComponents.go @@ -11,7 +11,10 @@ import ( "github.com/multiversx/mx-chain-go/trie" ) -const maxTrieLevelInMemory = uint(5) +const ( + maxTrieLevelInMemory = uint(5) + trieIdentifier = "main trie in memory" +) func createAccountAdapter( marshaller marshal.Marshalizer, @@ -29,7 +32,7 @@ func createAccountAdapter( EnableEpochsHandler: enableEpochsHandler, MaxTrieLevelInMemory: maxTrieLevelInMemory, Throttler: trie.NewDisabledTrieGoRoutinesThrottler(), - Identifier: "main trie in memory", + Identifier: trieIdentifier, }) if err != nil { return nil, err