Merge pull request #6723 from multiversx/refactor-trie-constructor

refactor trie constructor

cmd/node/config/config.toml

@@ -661,6 +661,7 @@
     PeerStatePruningEnabled = true
     MaxStateTrieLevelInMemory = 5
     MaxPeerTrieLevelInMemory = 5
+    MaxNumGoroutines = 100
     StateStatisticsEnabled = false
 
 [BlockSizeThrottleConfig]

config/config.go

@@ -311,6 +311,7 @@ type StateTriesConfig struct {
 	PeerStatePruningEnabled     bool
 	MaxStateTrieLevelInMemory   uint
 	MaxPeerTrieLevelInMemory    uint
+	MaxNumGoroutines            uint
 	StateStatisticsEnabled      bool
 }
 

epochStart/metachain/systemSCs_test.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/multiversx/mx-chain-core-go/core"
 	"github.com/multiversx/mx-chain-core-go/core/check"
+	"github.com/multiversx/mx-chain-core-go/core/throttler"
 	"github.com/multiversx/mx-chain-core-go/data/block"
 	"github.com/multiversx/mx-chain-core-go/data/rewardTx"
 	"github.com/multiversx/mx-chain-core-go/hashing"
@@ -747,7 +748,17 @@ func createAccountsDB(
 	trieStorageManager common.StorageManager,
 	enableEpochsHandler common.EnableEpochsHandler,
 ) *state.AccountsDB {
-	tr, _ := trie.NewTrie(trieStorageManager, marshaller, hasher, enableEpochsHandler, 5)
+	thr, _ := throttler.NewNumGoRoutinesThrottler(10)
+	trieArgs := trie.TrieArgs{
+		TrieStorage:          trieStorageManager,
+		Marshalizer:          marshaller,
+		Hasher:               hasher,
+		EnableEpochsHandler:  enableEpochsHandler,
+		MaxTrieLevelInMemory: 5,
+		Throttler:            thr,
+	}
+
+	tr, _ := trie.NewTrie(trieArgs)
 	ewlArgs := evictionWaitingList.MemoryEvictionWaitingListArgs{
 		RootHashesSize: 100,
 		HashesSize:     10000,

factory/api/apiResolverFactory.go

@@ -587,6 +587,7 @@ func createNewAccountsAdapterApi(args scQueryElementArgs, chainHandler data.Chai
 		Identifier:          dataRetriever.UserAccountsUnit.String(),
 		EnableEpochsHandler: args.coreComponents.EnableEpochsHandler(),
 		StatsCollector:      args.statusCoreComponents.StateStatsHandler(),
+		NumGoRoutines:       args.generalConfig.StateTriesConfig.MaxNumGoroutines,
 	}
 	trieStorageManager, merkleTrie, err := trFactory.Create(trieCreatorArgs)
 	if err != nil {

factory/processing/blockProcessorCreator_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/multiversx/mx-chain-core-go/core"
+	"github.com/multiversx/mx-chain-core-go/core/throttler"
 	"github.com/multiversx/mx-chain-core-go/hashing"
 	"github.com/multiversx/mx-chain-core-go/marshal"
 	"github.com/multiversx/mx-chain-go/common"
@@ -198,7 +199,16 @@ func createAccountAdapter(
 	trieStorage common.StorageManager,
 	handler common.EnableEpochsHandler,
 ) (state.AccountsAdapter, error) {
-	tr, err := trie.NewTrie(trieStorage, marshaller, hasher, handler, 5)
+	thr, _ := throttler.NewNumGoRoutinesThrottler(10)
+	trieArgs := trie.TrieArgs{
+		TrieStorage:          trieStorage,
+		Marshalizer:          marshaller,
+		Hasher:               hasher,
+		EnableEpochsHandler:  handler,
+		MaxTrieLevelInMemory: 5,
+		Throttler:            thr,
+	}
+	tr, err := trie.NewTrie(trieArgs)
 	if err != nil {
 		return nil, err
 	}

genesis/process/memoryComponents.go

@@ -21,7 +21,15 @@ func createAccountAdapter(
 	addressConverter core.PubkeyConverter,
 	enableEpochsHandler common.EnableEpochsHandler,
 ) (state.AccountsAdapter, error) {
-	tr, err := trie.NewTrie(trieStorage, marshaller, hasher, enableEpochsHandler, maxTrieLevelInMemory)
+	tr, err := trie.NewTrie(
+		trie.TrieArgs{
+			TrieStorage:          trieStorage,
+			Marshalizer:          marshaller,
+			Hasher:               hasher,
+			EnableEpochsHandler:  enableEpochsHandler,
+			MaxTrieLevelInMemory: maxTrieLevelInMemory,
+			Throttler:            trie.NewDisabledTrieGoRoutinesThrottler(),
+		})
 	if err != nil {
 		return nil, err
 	}

integrationTests/benchmarks/loadFromTrie_test.go

@@ -5,12 +5,13 @@ import (
 	"testing"
 	"time"
 
+	"github.com/multiversx/mx-chain-core-go/core/throttler"
 	"github.com/multiversx/mx-chain-core-go/hashing"
 	"github.com/multiversx/mx-chain-core-go/hashing/blake2b"
 	"github.com/multiversx/mx-chain-core-go/marshal"
 	"github.com/multiversx/mx-chain-go/common"
-	disabledStatistics "github.com/multiversx/mx-chain-go/common/statistics/disabled"
 	"github.com/multiversx/mx-chain-go/common/holders"
+	disabledStatistics "github.com/multiversx/mx-chain-go/common/statistics/disabled"
 	"github.com/multiversx/mx-chain-go/integrationTests"
 	"github.com/multiversx/mx-chain-go/state/hashesCollector"
 	"github.com/multiversx/mx-chain-go/storage"
@@ -101,8 +102,18 @@ func generateTriesWithMaxDepth(
 	hasher hashing.Hasher,
 ) []*keyForTrie {
 	tries := make([]*keyForTrie, numTries)
+	thr, _ := throttler.NewNumGoRoutinesThrottler(50)
+
 	for i := 0; i < numTries; i++ {
-		tr, _ := trie.NewTrie(storage, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 2)
+		trieArgs := trie.TrieArgs{
+			TrieStorage:          storage,
+			Marshalizer:          marshaller,
+			Hasher:               hasher,
+			EnableEpochsHandler:  &enableEpochsHandlerMock.EnableEpochsHandlerStub{},
+			MaxTrieLevelInMemory: 2,
+			Throttler:            thr,
+		}
+		tr, _ := trie.NewTrie(trieArgs)
 		key := insertKeysIntoTrie(t, tr, numTrieLevels, numChildrenPerBranch)
 
 		rootHash, _ := tr.RootHash()

integrationTests/longTests/storage/storage_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/multiversx/mx-chain-core-go/core/throttler"
 	"github.com/multiversx/mx-chain-core-go/hashing/blake2b"
 	"github.com/multiversx/mx-chain-core-go/marshal"
 	"github.com/multiversx/mx-chain-go/integrationTests"
@@ -118,7 +119,17 @@ func TestWriteContinuouslyInTree(t *testing.T) {
 	trieStorage, _ := trie.CreateTrieStorageManager(storageManagerArgs, options)
 
 	maxTrieLevelInMemory := uint(5)
-	tr, _ := trie.NewTrie(trieStorage, &marshal.JsonMarshalizer{}, blake2b.NewBlake2b(), &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory)
+	thr, _ := throttler.NewNumGoRoutinesThrottler(10)
+	trieArgs := trie.TrieArgs{
+		TrieStorage:          trieStorage,
+		Marshalizer:          &marshal.JsonMarshalizer{},
+		Hasher:               blake2b.NewBlake2b(),
+		EnableEpochsHandler:  &enableEpochsHandlerMock.EnableEpochsHandlerStub{},
+		MaxTrieLevelInMemory: maxTrieLevelInMemory,
+		Throttler:            thr,
+	}
+
+	tr, _ := trie.NewTrie(trieArgs)
 
 	defer func() {
 		_ = store.DestroyUnit()

integrationTests/state/stateTrie/stateTrie_test.go

@@ -273,8 +273,7 @@ func TestTrieDB_RecreateFromStorageShouldWork(t *testing.T) {
 	args.Hasher = hasher
 	trieStorage, _ := trie.NewTrieStorageManager(args)
 
-	maxTrieLevelInMemory := uint(5)
-	tr1, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory)
+	tr1, _ := trie.NewTrie(integrationTests.GetTrieArgs(trieStorage))
 
 	key := hasher.Compute("key")
 	value := hasher.Compute("value")
@@ -1061,8 +1060,8 @@ func createAccounts(
 	args := testStorage.GetStorageManagerArgs()
 	args.MainStorer = store
 	trieStorage, _ := trie.NewTrieStorageManager(args)
-	maxTrieLevelInMemory := uint(5)
-	tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory)
+
+	tr, _ := trie.NewTrie(integrationTests.GetTrieArgs(trieStorage))
 	spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10)
 	argsAccCreator := factory.ArgsAccountCreator{
 		Hasher:              integrationTests.TestHasher,
@@ -2708,8 +2707,7 @@ func createAccountsDBTestSetup() *state.AccountsDB {
 	args := testStorage.GetStorageManagerArgs()
 	args.GeneralConfig = generalCfg
 	trieStorage, _ := trie.NewTrieStorageManager(args)
-	maxTrieLevelInMemory := uint(5)
-	tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory)
+	tr, _ := trie.NewTrie(integrationTests.GetTrieArgs(trieStorage))
 	spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10)
 	argsAccCreator := factory.ArgsAccountCreator{
 		Hasher:              integrationTests.TestHasher,

integrationTests/state/stateTrieClose/stateTrieClose_test.go

@@ -13,7 +13,6 @@ import (
 	"github.com/multiversx/mx-chain-go/integrationTests"
 	"github.com/multiversx/mx-chain-go/state/hashesCollector"
 	"github.com/multiversx/mx-chain-go/state/parsers"
-	"github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock"
 	"github.com/multiversx/mx-chain-go/testscommon/goroutines"
 	"github.com/multiversx/mx-chain-go/testscommon/storage"
 	"github.com/multiversx/mx-chain-go/trie"
@@ -24,7 +23,7 @@ import (
 func TestPatriciaMerkleTrie_Close(t *testing.T) {
 	numLeavesToAdd := 200
 	trieStorage, _ := integrationTests.CreateTrieStorageManager(integrationTests.CreateMemUnit())
-	tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 5)
+	tr, _ := trie.NewTrie(integrationTests.GetTrieArgs(trieStorage))
 
 	for i := 0; i < numLeavesToAdd; i++ {
 		_ = tr.Update([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i)))

integrationTests/testInitializer.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/multiversx/mx-chain-core-go/core"
 	"github.com/multiversx/mx-chain-core-go/core/check"
+	"github.com/multiversx/mx-chain-core-go/core/throttler"
 	"github.com/multiversx/mx-chain-core-go/data"
 	dataBlock "github.com/multiversx/mx-chain-core-go/data/block"
 	"github.com/multiversx/mx-chain-core-go/data/transaction"
@@ -125,6 +126,20 @@ const (
 
 const defaultChancesSelection = 1
 
+// GetTrieArgs returns the trie arguments for the tests
+func GetTrieArgs(trieStorage common.StorageManager) trie.TrieArgs {
+	thr, _ := throttler.NewNumGoRoutinesThrottler(10)
+	maxTrieLevelInMemory := uint(5)
+	return trie.TrieArgs{
+		TrieStorage:          trieStorage,
+		Marshalizer:          TestMarshalizer,
+		Hasher:               TestHasher,
+		EnableEpochsHandler:  &enableEpochsHandlerMock.EnableEpochsHandlerStub{},
+		MaxTrieLevelInMemory: maxTrieLevelInMemory,
+		Throttler:            thr,
+	}
+}
+
 // GetConnectableAddress returns a non circuit, non windows default connectable address for provided messenger
 func GetConnectableAddress(mes p2p.Messenger) string {
 	for _, addr := range mes.Addresses() {
@@ -456,7 +471,9 @@ func CreateAccountsDBWithEnableEpochsHandler(
 	trieStorageManager common.StorageManager,
 	enableEpochsHandler common.EnableEpochsHandler,
 ) (*state.AccountsDB, common.Trie) {
-	tr, _ := trie.NewTrie(trieStorageManager, TestMarshalizer, TestHasher, enableEpochsHandler, maxTrieLevelInMemory)
+	trieArgs := GetTrieArgs(trieStorageManager)
+	trieArgs.EnableEpochsHandler = enableEpochsHandler
+	tr, _ := trie.NewTrie(trieArgs)
 
 	ewlArgs := evictionWaitingList.MemoryEvictionWaitingListArgs{
 		RootHashesSize: 100,
@@ -1078,7 +1095,16 @@ func CreateNewDefaultTrie() common.Trie {
 
 	trieStorage, _ := trie.NewTrieStorageManager(args)
 
-	tr, _ := trie.NewTrie(trieStorage, TestMarshalizer, TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory)
+	thr, _ := throttler.NewNumGoRoutinesThrottler(10)
+	trieArgs := trie.TrieArgs{
+		TrieStorage:          trieStorage,
+		Marshalizer:          TestMarshalizer,
+		Hasher:               TestHasher,
+		EnableEpochsHandler:  &enableEpochsHandlerMock.EnableEpochsHandlerStub{},
+		MaxTrieLevelInMemory: maxTrieLevelInMemory,
+		Throttler:            thr,
+	}
+	tr, _ := trie.NewTrie(trieArgs)
 	return tr
 }
 

integrationTests/vm/staking/componentsHolderCreator.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/multiversx/mx-chain-core-go/core"
 	"github.com/multiversx/mx-chain-core-go/core/nodetype"
+	"github.com/multiversx/mx-chain-core-go/core/throttler"
 	"github.com/multiversx/mx-chain-core-go/data"
 	"github.com/multiversx/mx-chain-core-go/data/block"
 	"github.com/multiversx/mx-chain-core-go/data/endProcess"
@@ -193,13 +194,16 @@ func createAccountsDB(
 	accountFactory state.AccountFactory,
 	trieStorageManager common.StorageManager,
 ) *state.AccountsDB {
-	tr, _ := trie.NewTrie(
-		trieStorageManager,
-		coreComponents.InternalMarshalizer(),
-		coreComponents.Hasher(),
-		coreComponents.EnableEpochsHandler(),
-		5,
-	)
+	th, _ := throttler.NewNumGoRoutinesThrottler(10)
+	trieArgs := trie.TrieArgs{
+		TrieStorage:          trieStorageManager,
+		Marshalizer:          coreComponents.InternalMarshalizer(),
+		Hasher:               coreComponents.Hasher(),
+		EnableEpochsHandler:  coreComponents.EnableEpochsHandler(),
+		MaxTrieLevelInMemory: 5,
+		Throttler:            th,
+	}
+	tr, _ := trie.NewTrie(trieArgs)
 
 	argsEvictionWaitingList := evictionWaitingList.MemoryEvictionWaitingListArgs{
 		RootHashesSize: 10,