diff --git a/cmd/node/config/config.toml b/cmd/node/config/config.toml index 6e1205d5f7e..4f008692e33 100644 --- a/cmd/node/config/config.toml +++ b/cmd/node/config/config.toml @@ -661,6 +661,7 @@ PeerStatePruningEnabled = true MaxStateTrieLevelInMemory = 5 MaxPeerTrieLevelInMemory = 5 + MaxNumGoroutines = 100 StateStatisticsEnabled = false [BlockSizeThrottleConfig] diff --git a/config/config.go b/config/config.go index 49ef257c341..8ad550cb5ad 100644 --- a/config/config.go +++ b/config/config.go @@ -311,6 +311,7 @@ type StateTriesConfig struct { PeerStatePruningEnabled bool MaxStateTrieLevelInMemory uint MaxPeerTrieLevelInMemory uint + MaxNumGoroutines uint StateStatisticsEnabled bool } diff --git a/epochStart/metachain/systemSCs_test.go b/epochStart/metachain/systemSCs_test.go index c97d3cdbdd6..79e88085bea 100644 --- a/epochStart/metachain/systemSCs_test.go +++ b/epochStart/metachain/systemSCs_test.go @@ -13,6 +13,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/rewardTx" "github.com/multiversx/mx-chain-core-go/hashing" @@ -747,7 +748,17 @@ func createAccountsDB( trieStorageManager common.StorageManager, enableEpochsHandler common.EnableEpochsHandler, ) *state.AccountsDB { - tr, _ := trie.NewTrie(trieStorageManager, marshaller, hasher, enableEpochsHandler, 5) + thr, _ := throttler.NewNumGoRoutinesThrottler(10) + trieArgs := trie.TrieArgs{ + TrieStorage: trieStorageManager, + Marshalizer: marshaller, + Hasher: hasher, + EnableEpochsHandler: enableEpochsHandler, + MaxTrieLevelInMemory: 5, + Throttler: thr, + } + + tr, _ := trie.NewTrie(trieArgs) ewlArgs := evictionWaitingList.MemoryEvictionWaitingListArgs{ RootHashesSize: 100, HashesSize: 10000, diff --git a/factory/api/apiResolverFactory.go b/factory/api/apiResolverFactory.go index 399ee4b1533..a2cf1329c8d 100644 --- a/factory/api/apiResolverFactory.go +++ b/factory/api/apiResolverFactory.go @@ -587,6 +587,7 @@ func createNewAccountsAdapterApi(args scQueryElementArgs, chainHandler data.Chai Identifier: dataRetriever.UserAccountsUnit.String(), EnableEpochsHandler: args.coreComponents.EnableEpochsHandler(), StatsCollector: args.statusCoreComponents.StateStatsHandler(), + NumGoRoutines: args.generalConfig.StateTriesConfig.MaxNumGoroutines, } trieStorageManager, merkleTrie, err := trFactory.Create(trieCreatorArgs) if err != nil { diff --git a/factory/processing/blockProcessorCreator_test.go b/factory/processing/blockProcessorCreator_test.go index 099fec4a82d..34d072df4f7 100644 --- a/factory/processing/blockProcessorCreator_test.go +++ b/factory/processing/blockProcessorCreator_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" @@ -198,7 +199,16 @@ func createAccountAdapter( trieStorage common.StorageManager, handler common.EnableEpochsHandler, ) (state.AccountsAdapter, error) { - tr, err := trie.NewTrie(trieStorage, marshaller, hasher, handler, 5) + thr, _ := throttler.NewNumGoRoutinesThrottler(10) + trieArgs := trie.TrieArgs{ + TrieStorage: trieStorage, + Marshalizer: marshaller, + Hasher: hasher, + EnableEpochsHandler: handler, + MaxTrieLevelInMemory: 5, + Throttler: thr, + } + tr, err := trie.NewTrie(trieArgs) if err != nil { return nil, err } diff --git a/genesis/process/memoryComponents.go b/genesis/process/memoryComponents.go index f996faa81ed..cefa0631f59 100644 --- a/genesis/process/memoryComponents.go +++ b/genesis/process/memoryComponents.go @@ -21,7 +21,15 @@ func createAccountAdapter( addressConverter core.PubkeyConverter, enableEpochsHandler common.EnableEpochsHandler, ) (state.AccountsAdapter, error) { - tr, err := trie.NewTrie(trieStorage, marshaller, hasher, enableEpochsHandler, maxTrieLevelInMemory) + tr, err := trie.NewTrie( + trie.TrieArgs{ + TrieStorage: trieStorage, + Marshalizer: marshaller, + Hasher: hasher, + EnableEpochsHandler: enableEpochsHandler, + MaxTrieLevelInMemory: maxTrieLevelInMemory, + Throttler: trie.NewDisabledTrieGoRoutinesThrottler(), + }) if err != nil { return nil, err } diff --git a/integrationTests/benchmarks/loadFromTrie_test.go b/integrationTests/benchmarks/loadFromTrie_test.go index f08548c5c94..21fcd3a37fa 100644 --- a/integrationTests/benchmarks/loadFromTrie_test.go +++ b/integrationTests/benchmarks/loadFromTrie_test.go @@ -5,12 +5,13 @@ import ( "testing" "time" + "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/hashing/blake2b" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" - disabledStatistics "github.com/multiversx/mx-chain-go/common/statistics/disabled" "github.com/multiversx/mx-chain-go/common/holders" + disabledStatistics "github.com/multiversx/mx-chain-go/common/statistics/disabled" "github.com/multiversx/mx-chain-go/integrationTests" "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/storage" @@ -101,8 +102,18 @@ func generateTriesWithMaxDepth( hasher hashing.Hasher, ) []*keyForTrie { tries := make([]*keyForTrie, numTries) + thr, _ := throttler.NewNumGoRoutinesThrottler(50) + for i := 0; i < numTries; i++ { - tr, _ := trie.NewTrie(storage, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 2) + trieArgs := trie.TrieArgs{ + TrieStorage: storage, + Marshalizer: marshaller, + Hasher: hasher, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + MaxTrieLevelInMemory: 2, + Throttler: thr, + } + tr, _ := trie.NewTrie(trieArgs) key := insertKeysIntoTrie(t, tr, numTrieLevels, numChildrenPerBranch) rootHash, _ := tr.RootHash() diff --git a/integrationTests/longTests/storage/storage_test.go b/integrationTests/longTests/storage/storage_test.go index e47fd932c52..23e4dae8c32 100644 --- a/integrationTests/longTests/storage/storage_test.go +++ b/integrationTests/longTests/storage/storage_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/hashing/blake2b" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/integrationTests" @@ -118,7 +119,17 @@ func TestWriteContinuouslyInTree(t *testing.T) { trieStorage, _ := trie.CreateTrieStorageManager(storageManagerArgs, options) maxTrieLevelInMemory := uint(5) - tr, _ := trie.NewTrie(trieStorage, &marshal.JsonMarshalizer{}, blake2b.NewBlake2b(), &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) + thr, _ := throttler.NewNumGoRoutinesThrottler(10) + trieArgs := trie.TrieArgs{ + TrieStorage: trieStorage, + Marshalizer: &marshal.JsonMarshalizer{}, + Hasher: blake2b.NewBlake2b(), + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + MaxTrieLevelInMemory: maxTrieLevelInMemory, + Throttler: thr, + } + + tr, _ := trie.NewTrie(trieArgs) defer func() { _ = store.DestroyUnit() diff --git a/integrationTests/state/stateTrie/stateTrie_test.go b/integrationTests/state/stateTrie/stateTrie_test.go index 7ea6cc2f362..a8d47468da9 100644 --- a/integrationTests/state/stateTrie/stateTrie_test.go +++ b/integrationTests/state/stateTrie/stateTrie_test.go @@ -273,8 +273,7 @@ func TestTrieDB_RecreateFromStorageShouldWork(t *testing.T) { args.Hasher = hasher trieStorage, _ := trie.NewTrieStorageManager(args) - maxTrieLevelInMemory := uint(5) - tr1, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) + tr1, _ := trie.NewTrie(integrationTests.GetTrieArgs(trieStorage)) key := hasher.Compute("key") value := hasher.Compute("value") @@ -1061,8 +1060,8 @@ func createAccounts( args := testStorage.GetStorageManagerArgs() args.MainStorer = store trieStorage, _ := trie.NewTrieStorageManager(args) - maxTrieLevelInMemory := uint(5) - tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) + + tr, _ := trie.NewTrie(integrationTests.GetTrieArgs(trieStorage)) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) argsAccCreator := factory.ArgsAccountCreator{ Hasher: integrationTests.TestHasher, @@ -2708,8 +2707,7 @@ func createAccountsDBTestSetup() *state.AccountsDB { args := testStorage.GetStorageManagerArgs() args.GeneralConfig = generalCfg trieStorage, _ := trie.NewTrieStorageManager(args) - maxTrieLevelInMemory := uint(5) - tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) + tr, _ := trie.NewTrie(integrationTests.GetTrieArgs(trieStorage)) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) argsAccCreator := factory.ArgsAccountCreator{ Hasher: integrationTests.TestHasher, diff --git a/integrationTests/state/stateTrieClose/stateTrieClose_test.go b/integrationTests/state/stateTrieClose/stateTrieClose_test.go index 0f6f4718ef7..3d5eba5bc2f 100644 --- a/integrationTests/state/stateTrieClose/stateTrieClose_test.go +++ b/integrationTests/state/stateTrieClose/stateTrieClose_test.go @@ -13,7 +13,6 @@ import ( "github.com/multiversx/mx-chain-go/integrationTests" "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/state/parsers" - "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/goroutines" "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" @@ -24,7 +23,7 @@ import ( func TestPatriciaMerkleTrie_Close(t *testing.T) { numLeavesToAdd := 200 trieStorage, _ := integrationTests.CreateTrieStorageManager(integrationTests.CreateMemUnit()) - tr, _ := trie.NewTrie(trieStorage, integrationTests.TestMarshalizer, integrationTests.TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 5) + tr, _ := trie.NewTrie(integrationTests.GetTrieArgs(trieStorage)) for i := 0; i < numLeavesToAdd; i++ { _ = tr.Update([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) diff --git a/integrationTests/testInitializer.go b/integrationTests/testInitializer.go index a7c6cdac3c3..4030ebf3027 100644 --- a/integrationTests/testInitializer.go +++ b/integrationTests/testInitializer.go @@ -15,6 +15,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/transaction" @@ -125,6 +126,20 @@ const ( const defaultChancesSelection = 1 +// GetTrieArgs returns the trie arguments for the tests +func GetTrieArgs(trieStorage common.StorageManager) trie.TrieArgs { + thr, _ := throttler.NewNumGoRoutinesThrottler(10) + maxTrieLevelInMemory := uint(5) + return trie.TrieArgs{ + TrieStorage: trieStorage, + Marshalizer: TestMarshalizer, + Hasher: TestHasher, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + MaxTrieLevelInMemory: maxTrieLevelInMemory, + Throttler: thr, + } +} + // GetConnectableAddress returns a non circuit, non windows default connectable address for provided messenger func GetConnectableAddress(mes p2p.Messenger) string { for _, addr := range mes.Addresses() { @@ -456,7 +471,9 @@ func CreateAccountsDBWithEnableEpochsHandler( trieStorageManager common.StorageManager, enableEpochsHandler common.EnableEpochsHandler, ) (*state.AccountsDB, common.Trie) { - tr, _ := trie.NewTrie(trieStorageManager, TestMarshalizer, TestHasher, enableEpochsHandler, maxTrieLevelInMemory) + trieArgs := GetTrieArgs(trieStorageManager) + trieArgs.EnableEpochsHandler = enableEpochsHandler + tr, _ := trie.NewTrie(trieArgs) ewlArgs := evictionWaitingList.MemoryEvictionWaitingListArgs{ RootHashesSize: 100, @@ -1078,7 +1095,16 @@ func CreateNewDefaultTrie() common.Trie { trieStorage, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(trieStorage, TestMarshalizer, TestHasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) + thr, _ := throttler.NewNumGoRoutinesThrottler(10) + trieArgs := trie.TrieArgs{ + TrieStorage: trieStorage, + Marshalizer: TestMarshalizer, + Hasher: TestHasher, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + MaxTrieLevelInMemory: maxTrieLevelInMemory, + Throttler: thr, + } + tr, _ := trie.NewTrie(trieArgs) return tr } diff --git a/integrationTests/vm/staking/componentsHolderCreator.go b/integrationTests/vm/staking/componentsHolderCreator.go index e3673b08ec7..c5d6f1642a3 100644 --- a/integrationTests/vm/staking/componentsHolderCreator.go +++ b/integrationTests/vm/staking/componentsHolderCreator.go @@ -5,6 +5,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/nodetype" + "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/endProcess" @@ -193,13 +194,16 @@ func createAccountsDB( accountFactory state.AccountFactory, trieStorageManager common.StorageManager, ) *state.AccountsDB { - tr, _ := trie.NewTrie( - trieStorageManager, - coreComponents.InternalMarshalizer(), - coreComponents.Hasher(), - coreComponents.EnableEpochsHandler(), - 5, - ) + th, _ := throttler.NewNumGoRoutinesThrottler(10) + trieArgs := trie.TrieArgs{ + TrieStorage: trieStorageManager, + Marshalizer: coreComponents.InternalMarshalizer(), + Hasher: coreComponents.Hasher(), + EnableEpochsHandler: coreComponents.EnableEpochsHandler(), + MaxTrieLevelInMemory: 5, + Throttler: th, + } + tr, _ := trie.NewTrie(trieArgs) argsEvictionWaitingList := evictionWaitingList.MemoryEvictionWaitingListArgs{ RootHashesSize: 10, diff --git a/state/accountsDB_test.go b/state/accountsDB_test.go index 878e543248e..7e0f44aabd3 100644 --- a/state/accountsDB_test.go +++ b/state/accountsDB_test.go @@ -17,6 +17,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/atomic" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/core/keyValStorage" + "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/errChan" @@ -85,6 +86,21 @@ func createMockAccountsDBArgs() state.ArgsAccountsDB { } } +func getDefaultTrieArgs(db common.BaseStorer) trie.TrieArgs { + args := storage.GetStorageManagerArgs() + args.MainStorer = db + trieStorage, _ := trie.NewTrieStorageManager(args) + th, _ := throttler.NewNumGoRoutinesThrottler(10) + return trie.TrieArgs{ + TrieStorage: trieStorage, + Marshalizer: &marshallerMock.MarshalizerMock{}, + Hasher: &hashingMocks.HasherMock{}, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + MaxTrieLevelInMemory: 5, + Throttler: th, + } +} + func createUserAcc(address []byte) state.UserAccountHandler { acc, _ := accounts.NewUserAccount(address, &trieMock.DataTrieTrackerStub{}, &trieMock.TrieLeafParserStub{}) return acc @@ -130,13 +146,10 @@ func getDefaultStateComponents( SnapshotsBufferLen: 10, SnapshotsGoroutineNum: 1, } - marshaller := &marshallerMock.MarshalizerMock{} - hasher := &hashingMocks.HasherMock{} - args := storage.GetStorageManagerArgs() - args.MainStorer = db - trieStorage, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(trieStorage, marshaller, hasher, enableEpochsHandler, 5) + trieArgs := getDefaultTrieArgs(db) + trieArgs.EnableEpochsHandler = enableEpochsHandler + tr, _ := trie.NewTrie(trieArgs) ewlArgs := evictionWaitingList.MemoryEvictionWaitingListArgs{ RootHashesSize: 100, HashesSize: 10000, @@ -144,15 +157,15 @@ func getDefaultStateComponents( ewl, _ := evictionWaitingList.NewMemoryEvictionWaitingList(ewlArgs) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, generalCfg.PruningBufferLen) argsAccCreator := factory.ArgsAccountCreator{ - Hasher: hasher, - Marshaller: marshaller, + Hasher: trieArgs.Hasher, + Marshaller: trieArgs.Marshalizer, EnableEpochsHandler: enableEpochsHandler, } accCreator, _ := factory.NewAccountCreator(argsAccCreator) snapshotsManager, _ := state.NewSnapshotsManager(state.ArgsNewSnapshotsManager{ ProcessingMode: common.Normal, - Marshaller: marshaller, + Marshaller: trieArgs.Marshalizer, AddressConverter: &testscommon.PubkeyConverterMock{}, ProcessStatusHandler: &testscommon.ProcessStatusHandlerStub{}, StateMetrics: &stateMock.StateMetricsStub{}, @@ -164,8 +177,8 @@ func getDefaultStateComponents( argsAccountsDB := state.ArgsAccountsDB{ Trie: tr, - Hasher: hasher, - Marshaller: marshaller, + Hasher: trieArgs.Hasher, + Marshaller: trieArgs.Marshalizer, AccountFactory: accCreator, StoragePruningManager: spm, AddressConverter: &testscommon.PubkeyConverterMock{}, @@ -173,7 +186,7 @@ func getDefaultStateComponents( } adb, _ := state.NewAccountsDB(argsAccountsDB) - return adb, tr, trieStorage + return adb, tr, trieArgs.TrieStorage } func TestNewAccountsDB(t *testing.T) { @@ -1774,22 +1787,19 @@ func TestAccountsDB_RemoveAccountAlsoRemovesCodeAndRevertsCorrectly(t *testing.T func TestAccountsDB_MainTrieAutomaticallyMarksCodeUpdatesForEviction(t *testing.T) { t.Parallel() - marshaller := &marshallerMock.MarshalizerMock{} - hasher := &hashingMocks.HasherMock{} + db := storage.GetStorageManagerArgs().MainStorer + trieArgs := getDefaultTrieArgs(db) + tr, _ := trie.NewTrie(trieArgs) ewl := stateMock.NewEvictionWaitingListMock(100) - args := storage.GetStorageManagerArgs() - tsm, _ := trie.NewTrieStorageManager(args) - maxTrieLevelInMemory := uint(5) - tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 5) argsAccountsDB := createMockAccountsDBArgs() argsAccountsDB.Trie = tr - argsAccountsDB.Hasher = hasher - argsAccountsDB.Marshaller = marshaller + argsAccountsDB.Hasher = trieArgs.Hasher + argsAccountsDB.Marshaller = trieArgs.Marshalizer argsAccCreator := factory.ArgsAccountCreator{ - Hasher: hasher, - Marshaller: marshaller, + Hasher: trieArgs.Hasher, + Marshaller: trieArgs.Marshalizer, EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, } argsAccountsDB.AccountFactory, _ = factory.NewAccountCreator(argsAccCreator) @@ -1971,21 +1981,17 @@ func modifyDataTries(t *testing.T, accountsAddresses [][]byte, adb *state.Accoun func TestAccountsDB_GetCode(t *testing.T) { t.Parallel() - maxTrieLevelInMemory := uint(5) - marshaller := &marshallerMock.MarshalizerMock{} - hasher := &hashingMocks.HasherMock{} - - args := storage.GetStorageManagerArgs() - tsm, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) + db := storage.GetStorageManagerArgs().MainStorer + trieArgs := getDefaultTrieArgs(db) + tr, _ := trie.NewTrie(trieArgs) spm := disabled.NewDisabledStoragePruningManager() argsAccountsDB := createMockAccountsDBArgs() argsAccountsDB.Trie = tr - argsAccountsDB.Hasher = hasher - argsAccountsDB.Marshaller = marshaller + argsAccountsDB.Hasher = trieArgs.Hasher + argsAccountsDB.Marshaller = trieArgs.Marshalizer argsAccCreator := factory.ArgsAccountCreator{ - Hasher: hasher, - Marshaller: marshaller, + Hasher: trieArgs.Hasher, + Marshaller: trieArgs.Marshalizer, EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, } argsAccountsDB.AccountFactory, _ = factory.NewAccountCreator(argsAccCreator) @@ -2009,7 +2015,7 @@ func TestAccountsDB_GetCode(t *testing.T) { err = adb.SaveAccount(userAcc) require.Nil(t, err) - codeHash := hasher.Compute(string(code)) + codeHash := trieArgs.Hasher.Compute(string(code)) retrievedCode := adb.GetCode(codeHash) assert.Equal(t, retrievedCode, code) @@ -2425,13 +2431,12 @@ func TestAccountsDB_NewAccountsDbStartsSnapshotAfterRestart(t *testing.T) { } func BenchmarkAccountsDb_GetCodeEntry(b *testing.B) { - maxTrieLevelInMemory := uint(5) marshaller := &marshallerMock.MarshalizerMock{} hasher := &hashingMocks.HasherMock{} args := storage.GetStorageManagerArgs() tsm, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory) + tr, _ := trie.NewTrie(getDefaultTrieArgs(tsm)) spm := disabled.NewDisabledStoragePruningManager() argsAccountsDB := createMockAccountsDBArgs() @@ -2751,19 +2756,19 @@ func TestAccountsDB_SaveKeyValAfterAccountIsReverted(t *testing.T) { func TestAccountsDB_RevertTxWhichMigratesDataRemovesMigratedData(t *testing.T) { t.Parallel() - marshaller := &marshallerMock.MarshalizerMock{} - hasher := &hashingMocks.HasherMock{} + db := storage.GetStorageManagerArgs().MainStorer enableEpochsHandler := enableEpochsHandlerMock.NewEnableEpochsHandlerStub() - tsm, _ := trie.NewTrieStorageManager(storage.GetStorageManagerArgs()) - tr, _ := trie.NewTrie(tsm, marshaller, hasher, enableEpochsHandler, uint(5)) + trieArgs := getDefaultTrieArgs(db) + trieArgs.EnableEpochsHandler = enableEpochsHandler + tr, _ := trie.NewTrie(trieArgs) spm := &stateMock.StoragePruningManagerStub{} argsAccountsDB := createMockAccountsDBArgs() argsAccountsDB.Trie = tr - argsAccountsDB.Hasher = hasher - argsAccountsDB.Marshaller = marshaller + argsAccountsDB.Hasher = trieArgs.Hasher + argsAccountsDB.Marshaller = trieArgs.Marshalizer argsAccCreator := factory.ArgsAccountCreator{ - Hasher: hasher, - Marshaller: marshaller, + Hasher: trieArgs.Hasher, + Marshaller: trieArgs.Marshalizer, EnableEpochsHandler: enableEpochsHandler, } argsAccountsDB.AccountFactory, _ = factory.NewAccountCreator(argsAccCreator) diff --git a/state/storagePruningManager/storagePruningManager_test.go b/state/storagePruningManager/storagePruningManager_test.go index 25bb2860b84..489d716fb31 100644 --- a/state/storagePruningManager/storagePruningManager_test.go +++ b/state/storagePruningManager/storagePruningManager_test.go @@ -2,7 +2,8 @@ package storagePruningManager import ( "testing" - + + "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/holders" "github.com/multiversx/mx-chain-go/common/statistics" @@ -32,7 +33,16 @@ func getDefaultTrieAndAccountsDbAndStoragePruningManager() (common.Trie, *state. hasher := &hashingMocks.HasherMock{} args := storage.GetStorageManagerArgs() trieStorage, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(trieStorage, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 5) + th, _ := throttler.NewNumGoRoutinesThrottler(10) + trieArgs := trie.TrieArgs{ + TrieStorage: trieStorage, + Marshalizer: marshaller, + Hasher: hasher, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + MaxTrieLevelInMemory: 5, + Throttler: th, + } + tr, _ := trie.NewTrie(trieArgs) ewlArgs := evictionWaitingList.MemoryEvictionWaitingListArgs{ RootHashesSize: 100, HashesSize: 10000, diff --git a/state/syncer/baseAccountsSyncer.go b/state/syncer/baseAccountsSyncer.go index 3cee93d7325..634ebce0baf 100644 --- a/state/syncer/baseAccountsSyncer.go +++ b/state/syncer/baseAccountsSyncer.go @@ -217,7 +217,15 @@ func (b *baseAccountsSyncer) GetSyncedTries() map[string]common.Trie { b.mutex.Lock() defer b.mutex.Unlock() - dataTrie, err := trie.NewTrie(b.trieStorageManager, b.marshalizer, b.hasher, b.enableEpochsHandler, b.maxTrieLevelInMemory) + trieArgs := trie.TrieArgs{ + TrieStorage: b.trieStorageManager, + Marshalizer: b.marshalizer, + Hasher: b.hasher, + EnableEpochsHandler: b.enableEpochsHandler, + MaxTrieLevelInMemory: b.maxTrieLevelInMemory, + Throttler: trie.NewDisabledTrieGoRoutinesThrottler(), + } + dataTrie, err := trie.NewTrie(trieArgs) if err != nil { log.Warn("error creating a new trie in baseAccountsSyncer.GetSyncedTries", "error", err) return make(map[string]common.Trie) diff --git a/state/syncer/userAccountSyncer_test.go b/state/syncer/userAccountSyncer_test.go index efdf1588717..6d39cf0712d 100644 --- a/state/syncer/userAccountSyncer_test.go +++ b/state/syncer/userAccountSyncer_test.go @@ -3,7 +3,8 @@ package syncer import ( "testing" "time" - + + "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-go/dataRetriever/mock" "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/testscommon" @@ -89,7 +90,16 @@ func TestUserAccountsSyncer_MissingDataTrieNodeFound(t *testing.T) { }, } - tr, _ := trie.NewTrie(tsm, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 5) + th, _ := throttler.NewNumGoRoutinesThrottler(10) + trieArgs := trie.TrieArgs{ + TrieStorage: tsm, + Marshalizer: args.Marshalizer, + Hasher: args.Hasher, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + MaxTrieLevelInMemory: 5, + Throttler: th, + } + tr, _ := trie.NewTrie(trieArgs) key := []byte("key") value := []byte("value") _ = tr.Update(key, value) diff --git a/state/syncer/userAccountsSyncer_test.go b/state/syncer/userAccountsSyncer_test.go index af30d51f989..fbd618f9741 100644 --- a/state/syncer/userAccountsSyncer_test.go +++ b/state/syncer/userAccountsSyncer_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" @@ -110,7 +111,13 @@ func getSerializedTrieNode( }, } - tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 5) + trieArgs := getDefaultTrieParameters() + trieArgs.TrieStorage = tsm + trieArgs.MaxTrieLevelInMemory = 5 + trieArgs.Marshalizer = marshaller + trieArgs.Hasher = hasher + + tr, _ := trie.NewTrie(trieArgs) _ = tr.Update(key, []byte("value")) _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) @@ -161,7 +168,7 @@ func TestUserAccountsSyncer_SyncAccounts(t *testing.T) { }) } -func getDefaultTrieParameters() (common.StorageManager, marshal.Marshalizer, hashing.Hasher, common.EnableEpochsHandler, uint) { +func getDefaultTrieParameters() trie.TrieArgs { marshalizer := &testscommon.ProtobufMarshalizerMock{} hasher := &testscommon.KeccakMock{} @@ -183,8 +190,15 @@ func getDefaultTrieParameters() (common.StorageManager, marshal.Marshalizer, has trieStorageManager, _ := trie.NewTrieStorageManager(args) maxTrieLevelInMemory := uint(1) - - return trieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory + th, _ := throttler.NewNumGoRoutinesThrottler(10) + return trie.TrieArgs{ + TrieStorage: trieStorageManager, + Marshalizer: args.Marshalizer, + Hasher: args.Hasher, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + MaxTrieLevelInMemory: maxTrieLevelInMemory, + Throttler: th, + } } func emptyTrie() common.Trie { @@ -236,7 +250,13 @@ func TestUserAccountsSyncer_SyncAccountDataTries(t *testing.T) { s, err := syncer.NewUserAccountsSyncer(args) require.Nil(t, err) - _, _ = trie.NewTrie(args.TrieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 5) + trieArgs := getDefaultTrieParameters() + trieArgs.TrieStorage = args.TrieStorageManager + trieArgs.MaxTrieLevelInMemory = 5 + trieArgs.Marshalizer = args.Marshalizer + trieArgs.Hasher = args.Hasher + + _, _ = trie.NewTrie(trieArgs) tr := emptyTrie() account, err := accounts.NewUserAccount(testscommon.TestPubKeyAlice, &trieMock.DataTrieTrackerStub{}, &trieMock.TrieLeafParserStub{}) @@ -293,7 +313,13 @@ func TestUserAccountsSyncer_SyncAccountDataTries(t *testing.T) { s, err := syncer.NewUserAccountsSyncer(args) require.Nil(t, err) - _, _ = trie.NewTrie(args.TrieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 5) + trieArgs := getDefaultTrieParameters() + trieArgs.TrieStorage = args.TrieStorageManager + trieArgs.MaxTrieLevelInMemory = 5 + trieArgs.Marshalizer = args.Marshalizer + trieArgs.Hasher = args.Hasher + + _, _ = trie.NewTrie(trieArgs) tr := emptyTrie() account, err := accounts.NewUserAccount(testscommon.TestPubKeyAlice, &trieMock.DataTrieTrackerStub{}, &trieMock.TrieLeafParserStub{}) @@ -360,7 +386,13 @@ func TestUserAccountsSyncer_MissingDataTrieNodeFound(t *testing.T) { }, } - tr, _ := trie.NewTrie(tsm, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 5) + trieArgs := getDefaultTrieParameters() + trieArgs.TrieStorage = tsm + trieArgs.MaxTrieLevelInMemory = 5 + trieArgs.Marshalizer = args.Marshalizer + trieArgs.Hasher = args.Hasher + + tr, _ := trie.NewTrie(trieArgs) key := []byte("key") value := []byte("value") _ = tr.Update(key, value) diff --git a/testscommon/components/components.go b/testscommon/components/components.go index 55b09fe99de..0d50ea0f79c 100644 --- a/testscommon/components/components.go +++ b/testscommon/components/components.go @@ -5,6 +5,7 @@ import ( "math/big" "testing" + "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/endProcess" "github.com/multiversx/mx-chain-core-go/data/outport" @@ -343,8 +344,18 @@ func GetStateFactoryArgs(coreComponents factory.CoreComponentsHolder, statusCore trieStorageManagers[dataRetriever.PeerAccountsUnit.String()] = storageManagerPeer triesHolder := state.NewDataTriesHolder() - trieUsers, _ := trie.NewTrie(storageManagerUser, coreComponents.InternalMarshalizer(), coreComponents.Hasher(), coreComponents.EnableEpochsHandler(), 5) - triePeers, _ := trie.NewTrie(storageManagerPeer, coreComponents.InternalMarshalizer(), coreComponents.Hasher(), coreComponents.EnableEpochsHandler(), 5) + th, _ := throttler.NewNumGoRoutinesThrottler(10) + trieArgs := trie.TrieArgs{ + TrieStorage: storageManagerUser, + Marshalizer: coreComponents.InternalMarshalizer(), + Hasher: coreComponents.Hasher(), + EnableEpochsHandler: coreComponents.EnableEpochsHandler(), + MaxTrieLevelInMemory: 5, + Throttler: th, + } + trieUsers, _ := trie.NewTrie(trieArgs) + trieArgs.TrieStorage = storageManagerPeer + triePeers, _ := trie.NewTrie(trieArgs) triesHolder.Put([]byte(dataRetriever.UserAccountsUnit.String()), trieUsers) triesHolder.Put([]byte(dataRetriever.PeerAccountsUnit.String()), triePeers) diff --git a/testscommon/integrationtests/factory.go b/testscommon/integrationtests/factory.go index 9acfa7c5e10..01930a1315b 100644 --- a/testscommon/integrationtests/factory.go +++ b/testscommon/integrationtests/factory.go @@ -1,6 +1,7 @@ package integrationtests import ( + "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/hashing/sha256" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" @@ -100,7 +101,16 @@ func CreateAccountsDB(db storage.Storer, enableEpochs common.EnableEpochsHandler trieStorage, _ := trie.NewTrieStorageManager(args) - tr, _ := trie.NewTrie(trieStorage, TestMarshalizer, TestHasher, enableEpochs, MaxTrieLevelInMemory) + th, _ := throttler.NewNumGoRoutinesThrottler(10) + trieArgs := trie.TrieArgs{ + TrieStorage: trieStorage, + Marshalizer: TestMarshalizer, + Hasher: TestHasher, + EnableEpochsHandler: enableEpochs, + MaxTrieLevelInMemory: MaxTrieLevelInMemory, + Throttler: th, + } + tr, _ := trie.NewTrie(trieArgs) spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10) argsAccCreator := accountFactory.ArgsAccountCreator{ diff --git a/trie/branchNode_test.go b/trie/branchNode_test.go index 285a56d2984..d643ca88461 100644 --- a/trie/branchNode_test.go +++ b/trie/branchNode_test.go @@ -86,6 +86,7 @@ func emptyDirtyBranchNode() *branchNode { func newEmptyTrie() (*patriciaMerkleTrie, *trieStorageManager) { args := GetDefaultTrieStorageManagerParameters() trieStorage, _ := NewTrieStorageManager(args) + thr, _ := throttler.NewNumGoRoutinesThrottler(10) tr := &patriciaMerkleTrie{ trieStorage: trieStorage, marshalizer: args.Marshalizer, @@ -97,6 +98,7 @@ func newEmptyTrie() (*patriciaMerkleTrie, *trieStorageManager) { goRoutinesManager: getTestGoroutinesManager(), RootManager: NewRootManager(), trieOperationInProgress: &atomic.Flag{}, + throttler: thr, } return tr, trieStorage diff --git a/trie/disabledTrieGoRoutinesThrottler.go b/trie/disabledTrieGoRoutinesThrottler.go new file mode 100644 index 00000000000..e4304fbbab9 --- /dev/null +++ b/trie/disabledTrieGoRoutinesThrottler.go @@ -0,0 +1,27 @@ +package trie + +type disabledTrieGoRoutinesThrottler struct { +} + +// NewDisabledTrieGoRoutinesThrottler returns a new instance of a disabledTrieGoRoutinesThrottler +func NewDisabledTrieGoRoutinesThrottler() *disabledTrieGoRoutinesThrottler { + return &disabledTrieGoRoutinesThrottler{} +} + +// CanProcess will always return false +func (d *disabledTrieGoRoutinesThrottler) CanProcess() bool { + return false +} + +// StartProcessing won't do anything +func (d *disabledTrieGoRoutinesThrottler) StartProcessing() { +} + +// EndProcessing won't do anything +func (d *disabledTrieGoRoutinesThrottler) EndProcessing() { +} + +// IsInterfaceNil returns true if there is no value under the interface +func (d *disabledTrieGoRoutinesThrottler) IsInterfaceNil() bool { + return d == nil +} diff --git a/trie/doubleListSync_test.go b/trie/doubleListSync_test.go index 6bd14ae9f43..0bfb5754a8a 100644 --- a/trie/doubleListSync_test.go +++ b/trie/doubleListSync_test.go @@ -10,6 +10,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/holders" "github.com/multiversx/mx-chain-go/state/hashesCollector" @@ -35,10 +36,22 @@ func createTrieStorageManager(store storage.Storer) (common.StorageManager, stor return tsm, store } +func getTrieArgs(tsm common.StorageManager) TrieArgs { + th, _ := throttler.NewNumGoRoutinesThrottler(10) + return TrieArgs{ + TrieStorage: tsm, + Marshalizer: marshalizer, + Hasher: hasherMock, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + MaxTrieLevelInMemory: 6, + Throttler: th, + } +} + func createInMemoryTrie() (common.Trie, storage.Storer) { memUnit := testscommon.CreateMemUnit() tsm, _ := createTrieStorageManager(memUnit) - tr, _ := NewTrie(tsm, marshalizer, hasherMock, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 6) + tr, _ := NewTrie(getTrieArgs(tsm)) return tr, memUnit } @@ -51,7 +64,7 @@ func createInMemoryTrieFromDB(db storage.Persister) (common.Trie, storage.Storer unit, _ := storageunit.NewStorageUnit(cache, db) tsm, _ := createTrieStorageManager(unit) - tr, _ := NewTrie(tsm, marshalizer, hasherMock, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 6) + tr, _ := NewTrie(getTrieArgs(tsm)) return tr, unit } diff --git a/trie/factory/trieCreator.go b/trie/factory/trieCreator.go index 198b33a0455..9257d5e7529 100644 --- a/trie/factory/trieCreator.go +++ b/trie/factory/trieCreator.go @@ -1,7 +1,9 @@ package factory import ( + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" @@ -22,6 +24,7 @@ type TrieCreateArgs struct { Identifier string EnableEpochsHandler common.EnableEpochsHandler StatsCollector common.StateStatisticsHandler + NumGoRoutines uint } type trieCreator struct { @@ -78,7 +81,21 @@ func (tc *trieCreator) Create(args TrieCreateArgs) (common.StorageManager, commo return nil, nil, err } - newTrie, err := trie.NewTrie(trieStorage, tc.marshalizer, tc.hasher, args.EnableEpochsHandler, args.MaxTrieLevelInMem) + goRoutinesThrottler, err := getGoRoutinesThrottler(args.NumGoRoutines) + if err != nil { + return nil, nil, err + } + + trArgs := trie.TrieArgs{ + TrieStorage: trieStorage, + Marshalizer: tc.marshalizer, + Hasher: tc.hasher, + EnableEpochsHandler: args.EnableEpochsHandler, + MaxTrieLevelInMemory: args.MaxTrieLevelInMem, + Throttler: goRoutinesThrottler, + } + + newTrie, err := trie.NewTrie(trArgs) if err != nil { return nil, nil, err } @@ -86,6 +103,14 @@ func (tc *trieCreator) Create(args TrieCreateArgs) (common.StorageManager, commo return trieStorage, newTrie, nil } +func getGoRoutinesThrottler(numGoroutines uint) (core.Throttler, error) { + if numGoroutines == 0 { + return trie.NewDisabledTrieGoRoutinesThrottler(), nil + } + + return throttler.NewNumGoRoutinesThrottler(int32(numGoroutines)) +} + // IsInterfaceNil returns true if there is no value under the interface func (tc *trieCreator) IsInterfaceNil() bool { return tc == nil @@ -123,6 +148,7 @@ func CreateTriesComponentsForShardId( Identifier: dataRetriever.UserAccountsUnit.String(), EnableEpochsHandler: coreComponentsHolder.EnableEpochsHandler(), StatsCollector: stateStatsHandler, + NumGoRoutines: generalConfig.StateTriesConfig.MaxNumGoroutines, } userStorageManager, userAccountTrie, err := trFactory.Create(args) if err != nil { @@ -149,6 +175,7 @@ func CreateTriesComponentsForShardId( Identifier: dataRetriever.PeerAccountsUnit.String(), EnableEpochsHandler: coreComponentsHolder.EnableEpochsHandler(), StatsCollector: stateStatsHandler, + NumGoRoutines: generalConfig.StateTriesConfig.MaxNumGoroutines, } peerStorageManager, peerAccountsTrie, err := trFactory.Create(args) if err != nil { diff --git a/trie/factory/trieCreator_test.go b/trie/factory/trieCreator_test.go index c4a716e2cc4..b0b0bf2c2cd 100644 --- a/trie/factory/trieCreator_test.go +++ b/trie/factory/trieCreator_test.go @@ -41,6 +41,7 @@ func getCreateArgs() factory.TrieCreateArgs { Identifier: dataRetriever.UserAccountsUnit.String(), EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, StatsCollector: disabled.NewStateStatistics(), + NumGoRoutines: 10, } } diff --git a/trie/patriciaMerkleTrie.go b/trie/patriciaMerkleTrie.go index 898c164ff9b..61f6d35be9d 100644 --- a/trie/patriciaMerkleTrie.go +++ b/trie/patriciaMerkleTrie.go @@ -10,7 +10,6 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/atomic" "github.com/multiversx/mx-chain-core-go/core/check" - "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" @@ -48,66 +47,71 @@ type patriciaMerkleTrie struct { goRoutinesManager common.TrieGoroutinesManager trieOperationInProgress *atomic.Flag updateTrieMutex sync.RWMutex + throttler core.Throttler maxTrieLevelInMemory uint chanClose chan struct{} } +// TrieArgs is the arguments for creating a new trie +type TrieArgs struct { + TrieStorage common.StorageManager + Marshalizer marshal.Marshalizer + Hasher hashing.Hasher + EnableEpochsHandler common.EnableEpochsHandler + MaxTrieLevelInMemory uint + Throttler core.Throttler +} + // NewTrie creates a new Patricia Merkle Trie func NewTrie( - trieStorage common.StorageManager, - msh marshal.Marshalizer, - hsh hashing.Hasher, - enableEpochsHandler common.EnableEpochsHandler, - maxTrieLevelInMemory uint, + args TrieArgs, ) (*patriciaMerkleTrie, error) { - if check.IfNil(trieStorage) { + if check.IfNil(args.TrieStorage) { return nil, ErrNilTrieStorage } - if check.IfNil(msh) { + if check.IfNil(args.Marshalizer) { return nil, ErrNilMarshalizer } - if check.IfNil(hsh) { + if check.IfNil(args.Hasher) { return nil, ErrNilHasher } - if check.IfNil(enableEpochsHandler) { + if check.IfNil(args.EnableEpochsHandler) { return nil, errors.ErrNilEnableEpochsHandler } - if maxTrieLevelInMemory == 0 { + if args.MaxTrieLevelInMemory == 0 { return nil, ErrInvalidLevelValue } - log.Trace("created new trie", "max trie level in memory", maxTrieLevelInMemory) - - tnvv, err := core.NewTrieNodeVersionVerifier(enableEpochsHandler) - if err != nil { - return nil, err + if check.IfNil(args.Throttler) { + return nil, ErrNilThrottler } + log.Trace("created new trie", "max trie level in memory", args.MaxTrieLevelInMemory) - // TODO give num goroutines from config as argument - trieThrottler, err := throttler.NewNumGoRoutinesThrottler(20) + tnvv, err := core.NewTrieNodeVersionVerifier(args.EnableEpochsHandler) if err != nil { return nil, err } chanClose := make(chan struct{}) - goRoutinesManager, err := NewGoroutinesManager(trieThrottler, errChan.NewErrChanWrapper(), chanClose) + goRoutinesManager, err := NewGoroutinesManager(args.Throttler, errChan.NewErrChanWrapper(), chanClose) if err != nil { return nil, err } return &patriciaMerkleTrie{ RootManager: NewRootManager(), - trieStorage: trieStorage, - marshalizer: msh, - hasher: hsh, - maxTrieLevelInMemory: maxTrieLevelInMemory, + trieStorage: args.TrieStorage, + marshalizer: args.Marshalizer, + hasher: args.Hasher, + maxTrieLevelInMemory: args.MaxTrieLevelInMemory, chanClose: chanClose, - enableEpochsHandler: enableEpochsHandler, + enableEpochsHandler: args.EnableEpochsHandler, trieNodeVersionVerifier: tnvv, batchManager: trieBatchManager.NewTrieBatchManager(), goRoutinesManager: goRoutinesManager, trieOperationInProgress: &atomic.Flag{}, updateTrieMutex: sync.RWMutex{}, + throttler: args.Throttler, }, nil } @@ -401,11 +405,14 @@ func (tr *patriciaMerkleTrie) Recreate(options common.RootHashHolder) (common.Tr func (tr *patriciaMerkleTrie) recreate(root []byte, tsm common.StorageManager) (*patriciaMerkleTrie, error) { if common.IsEmptyTrie(root) { return NewTrie( - tr.trieStorage, - tr.marshalizer, - tr.hasher, - tr.enableEpochsHandler, - tr.maxTrieLevelInMemory, + TrieArgs{ + TrieStorage: tr.trieStorage, + Marshalizer: tr.marshalizer, + Hasher: tr.hasher, + EnableEpochsHandler: tr.enableEpochsHandler, + MaxTrieLevelInMemory: tr.maxTrieLevelInMemory, + Throttler: tr.throttler, + }, ) } @@ -450,11 +457,14 @@ func (tr *patriciaMerkleTrie) IsInterfaceNil() bool { func (tr *patriciaMerkleTrie) recreateFromDb(rootHash []byte, tsm common.StorageManager) (*patriciaMerkleTrie, snapshotNode, error) { newTr, err := NewTrie( - tsm, - tr.marshalizer, - tr.hasher, - tr.enableEpochsHandler, - tr.maxTrieLevelInMemory, + TrieArgs{ + tsm, + tr.marshalizer, + tr.hasher, + tr.enableEpochsHandler, + tr.maxTrieLevelInMemory, + tr.throttler, + }, ) if err != nil { return nil, nil, err diff --git a/trie/patriciaMerkleTrie_test.go b/trie/patriciaMerkleTrie_test.go index 93818c41b71..7331414e498 100644 --- a/trie/patriciaMerkleTrie_test.go +++ b/trie/patriciaMerkleTrie_test.go @@ -14,9 +14,8 @@ import ( "time" "github.com/multiversx/mx-chain-core-go/core" - "github.com/multiversx/mx-chain-core-go/hashing" + "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/hashing/keccak" - "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/errChan" "github.com/multiversx/mx-chain-go/common/holders" @@ -43,18 +42,26 @@ func emptyTrie() common.Trie { } func emptyTrieWithCustomEnableEpochsHandler(handler common.EnableEpochsHandler) common.Trie { - storage, marshaller, hasher, _, maxTrieLevelInMem := getDefaultTrieParameters() - - tr, _ := trie.NewTrie(storage, marshaller, hasher, handler, maxTrieLevelInMem) + args := getDefaultTrieParameters() + args.EnableEpochsHandler = handler + tr, _ := trie.NewTrie(args) return tr } -func getDefaultTrieParameters() (common.StorageManager, marshal.Marshalizer, hashing.Hasher, common.EnableEpochsHandler, uint) { +func getDefaultTrieParameters() trie.TrieArgs { args := trie.GetDefaultTrieStorageManagerParameters() trieStorageManager, _ := trie.NewTrieStorageManager(args) maxTrieLevelInMemory := uint(1) - - return trieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, maxTrieLevelInMemory + th, _ := throttler.NewNumGoRoutinesThrottler(10) + + return trie.TrieArgs{ + TrieStorage: trieStorageManager, + Marshalizer: args.Marshalizer, + Hasher: args.Hasher, + EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, + MaxTrieLevelInMemory: maxTrieLevelInMemory, + Throttler: th, + } } func initTrieMultipleValues(nr int) (common.Trie, [][]byte) { @@ -88,8 +95,9 @@ func addDefaultDataToTrie(tr common.Trie) { func TestNewTrieWithNilTrieStorage(t *testing.T) { t.Parallel() - _, marshalizer, hasher, enableEpochsHandler, maxTrieLevelInMemory := getDefaultTrieParameters() - tr, err := trie.NewTrie(nil, marshalizer, hasher, enableEpochsHandler, maxTrieLevelInMemory) + args := getDefaultTrieParameters() + args.TrieStorage = nil + tr, err := trie.NewTrie(args) assert.Nil(t, tr) assert.Equal(t, trie.ErrNilTrieStorage, err) @@ -98,8 +106,9 @@ func TestNewTrieWithNilTrieStorage(t *testing.T) { func TestNewTrieWithNilMarshalizer(t *testing.T) { t.Parallel() - trieStorage, _, hasher, enableEpochsHandler, maxTrieLevelInMemory := getDefaultTrieParameters() - tr, err := trie.NewTrie(trieStorage, nil, hasher, enableEpochsHandler, maxTrieLevelInMemory) + args := getDefaultTrieParameters() + args.Marshalizer = nil + tr, err := trie.NewTrie(args) assert.Nil(t, tr) assert.Equal(t, trie.ErrNilMarshalizer, err) @@ -108,8 +117,9 @@ func TestNewTrieWithNilMarshalizer(t *testing.T) { func TestNewTrieWithNilHasher(t *testing.T) { t.Parallel() - trieStorage, marshalizer, _, enableEpochsHandler, maxTrieLevelInMemory := getDefaultTrieParameters() - tr, err := trie.NewTrie(trieStorage, marshalizer, nil, enableEpochsHandler, maxTrieLevelInMemory) + args := getDefaultTrieParameters() + args.Hasher = nil + tr, err := trie.NewTrie(args) assert.Nil(t, tr) assert.Equal(t, trie.ErrNilHasher, err) @@ -118,8 +128,9 @@ func TestNewTrieWithNilHasher(t *testing.T) { func TestNewTrieWithNilEnableEpochsHandler(t *testing.T) { t.Parallel() - trieStorage, marshalizer, hasher, _, maxTrieLevelInMemory := getDefaultTrieParameters() - tr, err := trie.NewTrie(trieStorage, marshalizer, hasher, nil, maxTrieLevelInMemory) + args := getDefaultTrieParameters() + args.EnableEpochsHandler = nil + tr, err := trie.NewTrie(args) assert.Nil(t, tr) assert.Equal(t, errorsCommon.ErrNilEnableEpochsHandler, err) @@ -128,13 +139,25 @@ func TestNewTrieWithNilEnableEpochsHandler(t *testing.T) { func TestNewTrieWithInvalidMaxTrieLevelInMemory(t *testing.T) { t.Parallel() - trieStorage, marshalizer, hasher, enableEpochsHandler, _ := getDefaultTrieParameters() - tr, err := trie.NewTrie(trieStorage, marshalizer, hasher, enableEpochsHandler, 0) + args := getDefaultTrieParameters() + args.MaxTrieLevelInMemory = 0 + tr, err := trie.NewTrie(args) assert.Nil(t, tr) assert.Equal(t, trie.ErrInvalidLevelValue, err) } +func TestNewTrieWithNilThrottler(t *testing.T) { + t.Parallel() + + args := getDefaultTrieParameters() + args.Throttler = nil + tr, err := trie.NewTrie(args) + + assert.Nil(t, tr) + assert.Equal(t, trie.ErrNilThrottler, err) +} + func TestPatriciaMerkleTree_Get(t *testing.T) { t.Parallel() @@ -1014,7 +1037,6 @@ func TestPatriciaMerkleTrie_ConcurrentOperations(t *testing.T) { func TestPatriciaMerkleTrie_GetSerializedNodesShouldSerializeTheCalls(t *testing.T) { t.Parallel() - args := trie.GetDefaultTrieStorageManagerParameters() numConcurrentCalls := int32(0) testTrieStorageManager := &storageManager.StorageManagerStub{ GetCalled: func(bytes []byte) ([]byte, error) { @@ -1030,7 +1052,11 @@ func TestPatriciaMerkleTrie_GetSerializedNodesShouldSerializeTheCalls(t *testing }, } - tr, _ := trie.NewTrie(testTrieStorageManager, args.Marshalizer, args.Hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 5) + trieArgs := getDefaultTrieParameters() + trieArgs.TrieStorage = testTrieStorageManager + trieArgs.MaxTrieLevelInMemory = 5 + + tr, _ := trie.NewTrie(trieArgs) numGoRoutines := 100 wg := sync.WaitGroup{} wg.Add(numGoRoutines) @@ -1461,13 +1487,14 @@ func TestPatriciaMerkleTrie_IsMigrated(t *testing.T) { t.Run("not migrated", func(t *testing.T) { t.Parallel() - tsm, marshaller, hasher, _, maxTrieInMem := getDefaultTrieParameters() + trieArgs := getDefaultTrieParameters() enableEpochs := &enableEpochsHandlerMock.EnableEpochsHandlerStub{ IsFlagEnabledCalled: func(flag core.EnableEpochFlag) bool { return flag == common.AutoBalanceDataTriesFlag }, } - tr, _ := trie.NewTrie(tsm, marshaller, hasher, enableEpochs, maxTrieInMem) + trieArgs.EnableEpochsHandler = enableEpochs + tr, _ := trie.NewTrie(trieArgs) _ = tr.Update([]byte("dog"), []byte("reindeer")) trie.ExecuteUpdatesFromBatch(tr) @@ -1479,13 +1506,14 @@ func TestPatriciaMerkleTrie_IsMigrated(t *testing.T) { t.Run("migrated", func(t *testing.T) { t.Parallel() - tsm, marshaller, hasher, _, maxTrieInMem := getDefaultTrieParameters() + trieArgs := getDefaultTrieParameters() enableEpochs := &enableEpochsHandlerMock.EnableEpochsHandlerStub{ IsFlagEnabledCalled: func(flag core.EnableEpochFlag) bool { return flag == common.AutoBalanceDataTriesFlag }, } - tr, _ := trie.NewTrie(tsm, marshaller, hasher, enableEpochs, maxTrieInMem) + trieArgs.EnableEpochsHandler = enableEpochs + tr, _ := trie.NewTrie(trieArgs) _ = tr.UpdateWithVersion([]byte("dog"), []byte("reindeer"), core.AutoBalanceEnabled) isMigrated, err := tr.IsMigratedToLatestVersion() diff --git a/update/factory/dataTrieFactory.go b/update/factory/dataTrieFactory.go index dcd83da1bd7..e4ee20e451c 100644 --- a/update/factory/dataTrieFactory.go +++ b/update/factory/dataTrieFactory.go @@ -142,7 +142,15 @@ func (d *dataTrieFactory) Create() (common.TriesHolder, error) { } func (d *dataTrieFactory) createAndAddOneTrie(shId uint32, accType genesis.Type, container common.TriesHolder) error { - dataTrie, err := trie.NewTrie(d.trieStorage, d.marshalizer, d.hasher, d.enableEpochsHandler, d.maxTrieLevelInMemory) + trieArgs := trie.TrieArgs{ + TrieStorage: d.trieStorage, + Marshalizer: d.marshalizer, + Hasher: d.hasher, + EnableEpochsHandler: d.enableEpochsHandler, + MaxTrieLevelInMemory: d.maxTrieLevelInMemory, + Throttler: trie.NewDisabledTrieGoRoutinesThrottler(), + } + dataTrie, err := trie.NewTrie(trieArgs) if err != nil { return err } diff --git a/update/genesis/import.go b/update/genesis/import.go index cea6fe5f0e9..80904f0c7e4 100644 --- a/update/genesis/import.go +++ b/update/genesis/import.go @@ -315,7 +315,15 @@ func (si *stateImport) getTrie(shardID uint32, accType Type) (common.Trie, error trieStorageManager = si.trieStorageManagers[dataRetriever.PeerAccountsUnit.String()] } - trieForShard, err := trie.NewTrie(trieStorageManager, si.marshalizer, si.hasher, si.enableEpochsHandler, maxTrieLevelInMemory) + trieArgs := trie.TrieArgs{ + TrieStorage: trieStorageManager, + Marshalizer: si.marshalizer, + Hasher: si.hasher, + EnableEpochsHandler: si.enableEpochsHandler, + MaxTrieLevelInMemory: maxTrieLevelInMemory, + Throttler: trie.NewDisabledTrieGoRoutinesThrottler(), + } + trieForShard, err := trie.NewTrie(trieArgs) if err != nil { return nil, err } @@ -347,7 +355,15 @@ func (si *stateImport) importDataTrie(identifier string, shID uint32, keys [][]b return fmt.Errorf("%w wanted a roothash", update.ErrWrongTypeAssertion) } - dataTrie, err := trie.NewTrie(si.trieStorageManagers[dataRetriever.UserAccountsUnit.String()], si.marshalizer, si.hasher, si.enableEpochsHandler, maxTrieLevelInMemory) + trieArgs := trie.TrieArgs{ + TrieStorage: si.trieStorageManagers[dataRetriever.UserAccountsUnit.String()], + Marshalizer: si.marshalizer, + Hasher: si.hasher, + EnableEpochsHandler: si.enableEpochsHandler, + MaxTrieLevelInMemory: maxTrieLevelInMemory, + Throttler: trie.NewDisabledTrieGoRoutinesThrottler(), + } + dataTrie, err := trie.NewTrie(trieArgs) if err != nil { return err }