Merge v1.4.0 feat optimise consensus sigcheck

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,9 +1,9 @@
-## Description of the reasoning behind the pull request (what feature was missing / how the problem was manifesting itself / what was the motive behind the refactoring)
+## Reasoning behind the pull request
 - 
 - 
 - 
 
-## Proposed Changes
+## Proposed changes
 - 
 - 
 - 

.github/release.yml

@@ -1,7 +1,7 @@
 changelog:
   exclude:
     labels:
-      - ignore-for-release
+      - ignore-for-release-notes
   categories:
     - title: Breaking Changes
       labels:

api/errors/errors.go

@@ -19,6 +19,9 @@ var ErrGetBalance = errors.New("get balance error")
 // ErrGetUsername signals an error in getting the username for an account
 var ErrGetUsername = errors.New("get username error")
 
+// ErrGetCodeHash signals an error in getting the code hash for an account
+var ErrGetCodeHash = errors.New("get code hash error")
+
 // ErrGetValueForKey signals an error in getting the value of a key for an account
 var ErrGetValueForKey = errors.New("get value for key error")
 

api/groups/addressGroup.go

@@ -20,6 +20,7 @@ const (
 	getAccountPath            = "/:address"
 	getBalancePath            = "/:address/balance"
 	getUsernamePath           = "/:address/username"
+	getCodeHashPath           = "/:address/code-hash"
 	getKeysPath               = "/:address/keys"
 	getKeyPath                = "/:address/key/:key"
 	getESDTTokensPath         = "/:address/esdt"
@@ -40,6 +41,7 @@ const (
 type addressFacadeHandler interface {
 	GetBalance(address string, options api.AccountQueryOptions) (*big.Int, api.BlockInfo, error)
 	GetUsername(address string, options api.AccountQueryOptions) (string, api.BlockInfo, error)
+	GetCodeHash(address string, options api.AccountQueryOptions) ([]byte, api.BlockInfo, error)
 	GetValueForKey(address string, key string, options api.AccountQueryOptions) (string, api.BlockInfo, error)
 	GetAccount(address string, options api.AccountQueryOptions) (api.AccountResponse, api.BlockInfo, error)
 	GetESDTData(address string, key string, nonce uint64, options api.AccountQueryOptions) (*esdt.ESDigitalToken, api.BlockInfo, error)
@@ -103,6 +105,11 @@ func NewAddressGroup(facade addressFacadeHandler) (*addressGroup, error) {
 			Method:  http.MethodGet,
 			Handler: ag.getUsername,
 		},
+		{
+			Path: 	 getCodeHashPath,
+			Method:  http.MethodGet,
+			Handler: ag.getCodeHash,
+		},
 		{
 			Path:    getKeyPath,
 			Method:  http.MethodGet,
@@ -219,6 +226,29 @@ func (ag *addressGroup) getUsername(c *gin.Context) {
 	shared.RespondWithSuccess(c, gin.H{"username": userName, "blockInfo": blockInfo})
 }
 
+// getCodeHash returns the code hash for the address parameter
+func (ag *addressGroup) getCodeHash(c *gin.Context) {
+	addr := c.Param("address")
+	if addr == "" {
+		shared.RespondWithValidationError(c, errors.ErrGetCodeHash, errors.ErrEmptyAddress)
+		return
+	}
+
+	options, err := parseAccountQueryOptions(c)
+	if err != nil {
+		shared.RespondWithValidationError(c, errors.ErrGetCodeHash, errors.ErrBadUrlParams)
+		return
+	}
+
+	codeHash, blockInfo, err := ag.getFacade().GetCodeHash(addr, options)
+	if err != nil {
+		shared.RespondWithInternalError(c, errors.ErrGetCodeHash, err)
+		return
+	}
+
+	shared.RespondWithSuccess(c, gin.H{"codeHash": codeHash, "blockInfo": blockInfo})
+}
+
 // getValueForKey returns the value for the given address and key
 func (ag *addressGroup) getValueForKey(c *gin.Context) {
 	addr := c.Param("address")

api/groups/addressGroup_test.go

@@ -1,6 +1,7 @@
 package groups_test
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -139,6 +140,16 @@ type usernameResponse struct {
 	Code  string               `json:"code"`
 }
 
+type codeHashResponseData struct {
+	CodeHash string `json:"codeHash"`
+}
+
+type codeHashResponse struct {
+	Data  codeHashResponseData `json:"data"`
+	Error string               `json:"error"`
+	Code  string               `json:"code"`
+}
+
 func TestNewAddressGroup(t *testing.T) {
 	t.Parallel()
 
@@ -395,6 +406,59 @@ func TestGetUsername_ShouldWork(t *testing.T) {
 	assert.Equal(t, testUsername, usernameResponseObj.Data.Username)
 }
 
+func TestGetCodeHash_NodeFailsShouldError(t *testing.T) {
+	t.Parallel()
+
+	testAddress := "address"
+	expectedErr := errors.New("expected error")
+	facade := mock.FacadeStub{
+		GetCodeHashCalled: func(_ string, _ api.AccountQueryOptions) ([]byte, api.BlockInfo, error) {
+			return nil, api.BlockInfo{}, expectedErr
+		},
+	}
+
+	addrGroup, err := groups.NewAddressGroup(&facade)
+	require.NoError(t, err)
+
+	ws := startWebServer(addrGroup, "address", getAddressRoutesConfig())
+
+	req, _ := http.NewRequest("GET", fmt.Sprintf("/address/%s/code-hash", testAddress), nil)
+	resp := httptest.NewRecorder()
+	ws.ServeHTTP(resp, req)
+
+	codeHashResponseObj := codeHashResponse{}
+	loadResponse(resp.Body, &codeHashResponseObj)
+	assert.Equal(t, http.StatusInternalServerError, resp.Code)
+	assert.True(t, strings.Contains(codeHashResponseObj.Error, expectedErr.Error()))
+}
+
+func TestGetCodeHash_ShouldWork(t *testing.T) {
+	t.Parallel()
+
+	testAddress := "address"
+	testCodeHash := []byte("value")
+	expectedResponseCodeHash := base64.StdEncoding.EncodeToString(testCodeHash)
+	facade := mock.FacadeStub{
+		GetCodeHashCalled: func(_ string, _ api.AccountQueryOptions) ([]byte, api.BlockInfo, error) {
+			return testCodeHash, api.BlockInfo{}, nil
+		},
+	}
+
+	addrGroup, err := groups.NewAddressGroup(&facade)
+	require.NoError(t, err)
+
+	ws := startWebServer(addrGroup, "address", getAddressRoutesConfig())
+
+	req, _ := http.NewRequest("GET", fmt.Sprintf("/address/%s/code-hash", testAddress), nil)
+	resp := httptest.NewRecorder()
+	ws.ServeHTTP(resp, req)
+
+	codeHashResponseObj := codeHashResponse{}
+	loadResponse(resp.Body, &codeHashResponseObj)
+	assert.Equal(t, http.StatusOK, resp.Code)
+	assert.Equal(t, expectedResponseCodeHash, codeHashResponseObj.Data.CodeHash)
+}
+
 func TestGetAccount_FailWhenFacadeStubGetAccountFails(t *testing.T) {
 	t.Parallel()
 
@@ -1034,6 +1098,7 @@ func getAddressRoutesConfig() config.ApiRoutesConfig {
 					{Name: "/:address", Open: true},
 					{Name: "/:address/balance", Open: true},
 					{Name: "/:address/username", Open: true},
+					{Name: "/:address/code-hash", Open: true},
 					{Name: "/:address/keys", Open: true},
 					{Name: "/:address/key/:key", Open: true},
 					{Name: "/:address/esdt", Open: true},

api/mock/facadeStub.go

@@ -42,6 +42,7 @@ type FacadeStub struct {
 	GetPeerInfoCalled                           func(pid string) ([]core.QueryP2PPeerInfo, error)
 	GetThrottlerForEndpointCalled               func(endpoint string) (core.Throttler, bool)
 	GetUsernameCalled                           func(address string, options api.AccountQueryOptions) (string, api.BlockInfo, error)
+	GetCodeHashCalled                           func(address string, options api.AccountQueryOptions) ([]byte, api.BlockInfo, error)
 	GetKeyValuePairsCalled                      func(address string, options api.AccountQueryOptions) (map[string]string, api.BlockInfo, error)
 	SimulateTransactionExecutionHandler         func(tx *transaction.Transaction) (*txSimData.SimulationResults, error)
 	GetESDTDataCalled                           func(address string, key string, nonce uint64, options api.AccountQueryOptions) (*esdt.ESDigitalToken, api.BlockInfo, error)
@@ -132,6 +133,15 @@ func (f *FacadeStub) GetUsername(address string, options api.AccountQueryOptions
 	return "", api.BlockInfo{}, nil
 }
 
+// GetCodeHash -
+func (f *FacadeStub) GetCodeHash(address string, options api.AccountQueryOptions) ([]byte, api.BlockInfo, error) {
+	if f.GetCodeHashCalled != nil {
+		return f.GetCodeHashCalled(address, options)
+	}
+
+	return nil, api.BlockInfo{}, nil
+}
+
 // GetThrottlerForEndpoint -
 func (f *FacadeStub) GetThrottlerForEndpoint(endpoint string) (core.Throttler, bool) {
 	if f.GetThrottlerForEndpointCalled != nil {

api/shared/interface.go

@@ -62,6 +62,7 @@ type GroupHandler interface {
 type FacadeHandler interface {
 	GetBalance(address string, options api.AccountQueryOptions) (*big.Int, api.BlockInfo, error)
 	GetUsername(address string, options api.AccountQueryOptions) (string, api.BlockInfo, error)
+	GetCodeHash(address string, options api.AccountQueryOptions) ([]byte, api.BlockInfo, error)
 	GetValueForKey(address string, key string, options api.AccountQueryOptions) (string, api.BlockInfo, error)
 	GetAccount(address string, options api.AccountQueryOptions) (api.AccountResponse, api.BlockInfo, error)
 	GetESDTData(address string, key string, nonce uint64, options api.AccountQueryOptions) (*esdt.ESDigitalToken, api.BlockInfo, error)

api/shared/logging/logging.go

@@ -8,6 +8,7 @@ import (
 )
 
 var log = logger.GetOrCreate("api/shared/logging")
+
 const thresholdMinAPICallDurationToLog = 200 * time.Millisecond
 
 // LogAPIActionDurationIfNeeded will log the duration of an action triggered by an API call if it's above a threshold

cmd/node/CLI.md

@@ -51,6 +51,7 @@ GLOBAL OPTIONS:
    --num-epochs-to-keep value             This flag represents the number of epochs which will kept in the databases. It is relevant only if the full archive flag is not set. (default: 2)
    --num-active-persisters value          This flag represents the number of databases (1 database = 1 epoch) which are kept open at a moment. It is relevant even if the node is full archive or not. (default: 2)
    --start-in-epoch                       Boolean option for enabling a node the fast bootstrap mechanism from the network.Should be enabled if data is not available in local disk.
+   --no-key                               Boolean flag for enabling the node to generate a signing key when it starts (if the validatorKey.pem file is present, setting this flag to true will overwrite the BLS key used by the node)
    --help, -h                             show help
    --version, -v                          print the version
 

cmd/node/config/api.toml

@@ -43,6 +43,9 @@
         # /address/:address/username will return the username of a given account
         { Name = "/:address/username", Open = true },
 
+        # /address/:address/code-hash will return the code hash of a given account
+        { Name = "/:address/code-hash", Open = true },
+
         # /address/:address/keys will return all the key-value pairs of a given account
         { Name = "/:address/keys", Open = true },
 

cmd/node/config/config.toml

@@ -443,6 +443,12 @@
     SizeInBytes = 26214400  # 25MB per each pair (metachain, destinationShard)
     Shards = 4
 
+[ValidatorInfoPool]
+    Name = "ValidatorInfoPool"
+    Capacity = 10000
+    SizeInBytes = 31457280  #30MB
+    Shards = 4
+
 #PublicKeyPeerId represents the main cache used to map Elrond block signing public keys to their associated peer id's.
 [PublicKeyPeerId]
     Name = "PublicKeyPeerId"
@@ -482,6 +488,10 @@
     TopRatedCacheCapacity = 5000
     BadRatedCacheCapacity = 5000
 
+[PoolsCleanersConfig]
+    MaxRoundsToKeepUnprocessedMiniBlocks = 300   # max number of rounds unprocessed miniblocks are kept in pool
+    MaxRoundsToKeepUnprocessedTransactions = 300 # max number of rounds unprocessed transactions are kept in pool
+
 [TrieSyncStorage]
     Capacity = 300000
     SizeInBytes = 104857600 #100MB
@@ -805,6 +815,11 @@
     [Debug.EpochStart]
         GoRoutineAnalyserEnabled = true
         ProcessDataTrieOnCommitEpoch = true
+    [Debug.Process]
+        Enabled = true
+        DebuggingLogLevel = "*:DEBUG,p2p:TRACE,debug:DEBUG,process:TRACE,intercept:TRACE"
+        GoRoutinesDump = true
+        PollingTimeInSeconds = 240 # 4 minutes
 
 [Health]
     IntervalVerifyMemoryInSeconds = 30

cmd/node/config/enableEpochs.toml

@@ -218,6 +218,21 @@
     # MiniBlockPartialExecutionEnableEpoch represents the epoch when mini block partial execution will be enabled
     MiniBlockPartialExecutionEnableEpoch = 3
 
+    # FixAsyncCallBackArgsListEnableEpoch represents the epoch when the async callback arguments lists fix will be enabled
+    FixAsyncCallBackArgsListEnableEpoch = 6
+
+    # SetSenderInEeiOutputTransferEnableEpoch represents the epoch when setting the sender in eei output transfers will be enabled
+    SetSenderInEeiOutputTransferEnableEpoch = 4
+
+    # RefactorPeersMiniBlocksEnableEpoch represents the epoch when refactor of the peers mini blocks will be enabled
+    RefactorPeersMiniBlocksEnableEpoch = 5
+
+    # BLSMultiSignerEnableEpoch represents the activation epoch for different types of BLS multi-signers
+    BLSMultiSignerEnableEpoch = [
+        { EnableEpoch = 0, Type = "no-KOSK"},
+        { EnableEpoch = 3, Type = "KOSK"}
+    ]
+
     # MaxNodesChangeEnableEpoch holds configuration for changing the maximum number of nodes and the enabling epoch
     MaxNodesChangeEnableEpoch = [
         { EpochEnable = 0, MaxNumNodes = 36, NodesToShufflePerShard = 4 },

cmd/node/config/enableRounds.toml

@@ -1,11 +1,12 @@
-# Since we do not have yet any round based enable flag, this file is empty for now.
-# Below, there is an example of how a new flag shall be defined. Please notice the following:
-# 1. It is mandatory that each flag is a separate table, e.g.: [ActivationDummyFlag1]
-# 2. It is mandatory that each table contains Name and Round keys ONLY
-# 3. It is NOT mandatory for table name to be the same as key Name, e.g.: [ActivationDummyFlag1] & Name = "Fix1" is fine
+# All activation rounds need to be declared as strings containing valid uint64 numbers. All values will be checked in the
+# enableRoundsHandler for correctness. The toml parser library can not correctly parse uint64, currently.
 
-# Example:
-#[ActivationDummyFlag1]
-#   Name = "Fix1"
-#   Round = 444
+# Each activation round definition contains an optional slice of strings. These are useful, for example, when we have
+# some exceptions to process in a different manner at a special round (say a block, a transaction, a smart contract result and such)
 
+[RoundActivations]
+    # RoundActivations.Example represents an example how to use the round activation
+    # mechanism. Replace this with the first real use case
+    [RoundActivations.Example]
+        Options = ["test string1", "test string 2"]
+        Round = "18446744073709551614"

cmd/node/config/external.toml

@@ -34,6 +34,9 @@
     # Password is used to authorize an observer to push event data
     Password = ""
 
+    # RequestTimeoutSec defines the timeout in seconds for the http client
+    RequestTimeoutSec = 60
+
 # CovalentConnector defines settings related to covalent indexer
 [CovalentConnector]
     # This flag shall only be used for observer nodes

cmd/node/flags.go

@@ -337,6 +337,13 @@ var (
 		Name:  "serialize-snapshots",
 		Usage: "Flag that will serialize `state snapshotting` and `processing`",
 	}
+
+	// noKey defines a flag that, if set, will generate every time when node starts a new signing key
+	noKey = cli.BoolFlag{
+		Name: "no-key",
+		Usage: "Boolean flag for enabling the node to generate a signing key when it starts (if the validatorKey.pem" +
+			" file is present, setting this flag to true will overwrite the BLS key used by the node)",
+	}
 )
 
 func getFlags() []cli.Flag {
@@ -389,6 +396,7 @@ func getFlags() []cli.Flag {
 		forceStartFromNetwork,
 		disableConsensusWatchdog,
 		serializeSnapshots,
+		noKey,
 	}
 }
 
@@ -414,6 +422,8 @@ func getFlagsConfig(ctx *cli.Context, log logger.Logger) *config.ContextFlagsCon
 	flagsConfig.ForceStartFromNetwork = ctx.GlobalBool(forceStartFromNetwork.Name)
 	flagsConfig.DisableConsensusWatchdog = ctx.GlobalBool(disableConsensusWatchdog.Name)
 	flagsConfig.SerializeSnapshots = ctx.GlobalBool(serializeSnapshots.Name)
+	flagsConfig.NoKeyProvided = ctx.GlobalBool(noKey.Name)
+
 	return flagsConfig
 }