Merge rc into feat refactor resolvers 2022.12.12

cmd/node/config/config.toml

@@ -914,25 +914,26 @@
     NumTotalPeers       = 3 # NumCrossShardPeers + num intra shard
     NumFullHistoryPeers = 3
 
+
 [HeartbeatV2]
     PeerAuthenticationTimeBetweenSendsInSec          = 600  # 10min TODO: change this for mainnet/devnet/testnet
     PeerAuthenticationTimeBetweenSendsWhenErrorInSec = 60   # 1min
-    PeerAuthenticationThresholdBetweenSends          = 0.1  # 10%
+    PeerAuthenticationTimeThresholdBetweenSends      = 0.1  # 10%   # max random threshold between messages, representing a percent of PeerAuthenticationTimeBetweenSendsInSec
     HeartbeatTimeBetweenSendsInSec                   = 60   # 1min
     HeartbeatTimeBetweenSendsWhenErrorInSec          = 60   # 1min
-    HeartbeatThresholdBetweenSends                   = 0.1  # 10%
-    HeartbeatExpiryTimespanInSec                     = 3600 # 1h
-    MinPeersThreshold                                = 0.8  # 80%
-    DelayBetweenRequestsInSec                        = 10   # 10sec
-    MaxTimeoutInSec                                  = 7200 # 2h
-    PeerShardTimeBetweenSendsInSec                   = 7200   # 2h
-    PeerShardThresholdBetweenSends                   = 0.1 # 10%
-    MaxMissingKeysInRequest                          = 1000
-    MaxDurationPeerUnresponsiveInSec                 = 900  # 15min
-    HideInactiveValidatorIntervalInSec               = 3600 # 1h
-    HardforkTimeBetweenSendsInSec                    = 60   # 1min
-    TimeBetweenConnectionsMetricsUpdateInSec         = 30   # 30sec
-    TimeToReadDirectConnectionsInSec                 = 15   # 15sec
+    HeartbeatTimeThresholdBetweenSends               = 0.1  # 10%   # max random threshold between messages, representing a percent of HeartbeatTimeBetweenSendsInSec
+    HeartbeatExpiryTimespanInSec                     = 3600 # 1h    # heartbeat message max age allowed
+    MinPeersThreshold                                = 0.8  # 80%   # min threshold of peers to consider enough peer auths received
+    DelayBetweenPeerAuthenticationRequestsInSec      = 10   # 10sec # delay between requests
+    PeerAuthenticationMaxTimeoutForRequestsInSec     = 7200 # 2h    # max timeout allowed to receive peer auth messages from peers
+    PeerShardTimeBetweenSendsInSec                   = 7200 # 2h
+    PeerShardTimeThresholdBetweenSends               = 0.1  # 10%   # max random threshold between messages, representing a percent of PeerShardTimeBetweenSendsInSec
+    MaxMissingKeysInRequest                          = 1000         # max number of missing keys allowed in a request
+    MaxDurationPeerUnresponsiveInSec                 = 900  # 15min # max duration after which a peer is considered inactive
+    HideInactiveValidatorIntervalInSec               = 3600 # 1h    # time that an inactive validator is returned through api
+    HardforkTimeBetweenSendsInSec                    = 60   # 1min  # time between hardfork messages
+    TimeBetweenConnectionsMetricsUpdateInSec         = 30   # 30sec # time between consecutive connections metrics updates
+    TimeToReadDirectConnectionsInSec                 = 15   # 15sec # time between consecutive peer shard mapper updates with direct connections
     [HeartbeatV2.HeartbeatPool]
         Name = "HeartbeatPool"
         Capacity = 1000

cmd/node/flags.go

@@ -247,7 +247,19 @@ var (
 	// workingDirectory defines a flag for the path for the working directory.
 	workingDirectory = cli.StringFlag{
 		Name:  "working-directory",
-		Usage: "This flag specifies the `directory` where the node will store databases, logs and statistics.",
+		Usage: "This flag specifies the `directory` where the node will store databases, logs and statistics if no other related flags are set.",
+		Value: "",
+	}
+	// dbDirectory defines a flag for the path for the db directory.
+	dbDirectory = cli.StringFlag{
+		Name:  "db-path",
+		Usage: "This flag specifies the `directory` where the node will store databases.",
+		Value: "",
+	}
+	// logsDirectory defines a flag for the path for the logs directory.
+	logsDirectory = cli.StringFlag{
+		Name:  "logs-path",
+		Usage: "This flag specifies the `directory` where the node will store logs.",
 		Value: "",
 	}
 
@@ -413,15 +425,18 @@ func getFlags() []cli.Flag {
 		serializeSnapshots,
 		noKey,
 		p2pKeyPemFile,
+		dbDirectory,
+		logsDirectory,
 		operationMode,
 	}
 }
 
 func getFlagsConfig(ctx *cli.Context, log logger.Logger) *config.ContextFlagsConfig {
 	flagsConfig := &config.ContextFlagsConfig{}
 
-	workingDir := ctx.GlobalString(workingDirectory.Name)
-	flagsConfig.WorkingDir = getWorkingDir(workingDir, log)
+	flagsConfig.WorkingDir = getWorkingDir(ctx, workingDirectory, log)
+	flagsConfig.DbDir = getCustomDirIfSet(ctx, dbDirectory, log)
+	flagsConfig.LogsDir = getCustomDirIfSet(ctx, logsDirectory, log)
 	flagsConfig.EnableGops = ctx.GlobalBool(gopsEn.Name)
 	flagsConfig.SaveLogFile = ctx.GlobalBool(logSaveFile.Name)
 	flagsConfig.EnableLogCorrelation = ctx.GlobalBool(logWithCorrelation.Name)
@@ -502,20 +517,32 @@ func applyFlags(ctx *cli.Context, cfgs *config.Configs, flagsConfig *config.Cont
 	return nil
 }
 
-func getWorkingDir(workingDir string, log logger.Logger) string {
+func getWorkingDir(ctx *cli.Context, cliFlag cli.StringFlag, log logger.Logger) string {
 	var err error
+
+	workingDir := ctx.GlobalString(cliFlag.Name)
 	if len(workingDir) == 0 {
 		workingDir, err = os.Getwd()
 		if err != nil {
 			log.LogIfError(err)
 			workingDir = ""
 		}
 	}
-	log.Trace("working directory", "path", workingDir)
+	log.Trace("working directory", "dirName", cliFlag.Name, "path", workingDir)
 
 	return workingDir
 }
 
+func getCustomDirIfSet(ctx *cli.Context, cliFlag cli.StringFlag, log logger.Logger) string {
+	dirStr := ctx.GlobalString(cliFlag.Name)
+
+	if len(dirStr) == 0 {
+		return getWorkingDir(ctx, workingDirectory, log)
+	}
+
+	return getWorkingDir(ctx, cliFlag, log)
+}
+
 func applyCompatibleConfigs(log logger.Logger, configs *config.Configs) error {
 	if configs.FlagsConfig.EnablePprof {
 		runtime.SetMutexProfileFraction(5)

cmd/node/main.go

@@ -253,7 +253,7 @@ func attachFileLogger(log logger.Logger, flagsConfig *config.ContextFlagsConfig)
 	var err error
 	if flagsConfig.SaveLogFile {
 		args := file.ArgsFileLogging{
-			WorkingDir:      flagsConfig.WorkingDir,
+			WorkingDir:      flagsConfig.LogsDir,
 			DefaultLogsPath: defaultLogsPath,
 			LogFilePrefix:   logFilePrefix,
 		}

common/constants.go

@@ -718,6 +718,13 @@ const InvalidMessageBlacklistDuration = time.Second * 3600
 // rating to a minimum threshold due to improper messages
 const PublicKeyBlacklistDuration = time.Second * 7200
 
+// WrongP2PMessageBlacklistDuration represents the time to keep a peer id in the blacklist if it sends a message that
+// do not follow this protocol
+const WrongP2PMessageBlacklistDuration = time.Second * 7200
+
+// InvalidSigningBlacklistDuration defines the time to keep a peer id in blacklist if it signs a message with invalid signature
+const InvalidSigningBlacklistDuration = time.Second * 7200
+
 // MaxWaitingTimeToReceiveRequestedItem represents the maximum waiting time in seconds needed to receive the requested items
 const MaxWaitingTimeToReceiveRequestedItem = 5 * time.Second
 

common/reflectcommon/structFieldsUpdate_test.go

@@ -99,10 +99,10 @@ func TestAdaptStructureValueBasedOnPath(t *testing.T) {
 	t.Run("should error when setting invalid float64", func(t *testing.T) {
 		t.Parallel()
 
-		path := "HeartbeatV2.PeerShardThresholdBetweenSends"
+		path := "HeartbeatV2.PeerShardTimeThresholdBetweenSends"
 		expectedNewValue := "invalid float64"
 		cfg := &config.Config{}
-		cfg.HeartbeatV2.PeerShardThresholdBetweenSends = 37.0
+		cfg.HeartbeatV2.PeerShardTimeThresholdBetweenSends = 37.0
 
 		err := AdaptStructureValueBasedOnPath(cfg, path, expectedNewValue)
 
@@ -279,15 +279,15 @@ func TestAdaptStructureValueBasedOnPath(t *testing.T) {
 	t.Run("should work and override float64 value", func(t *testing.T) {
 		t.Parallel()
 
-		path := "HeartbeatV2.PeerAuthenticationThresholdBetweenSends"
+		path := "HeartbeatV2.PeerAuthenticationTimeThresholdBetweenSends"
 		expectedNewValue := 38.0
 		cfg := &config.Config{}
-		cfg.HeartbeatV2.PeerAuthenticationThresholdBetweenSends = 37.0
+		cfg.HeartbeatV2.PeerAuthenticationTimeThresholdBetweenSends = 37.0
 
 		err := AdaptStructureValueBasedOnPath(cfg, path, fmt.Sprintf("%f", expectedNewValue))
 		require.NoError(t, err)
 
-		require.Equal(t, expectedNewValue, cfg.HeartbeatV2.PeerAuthenticationThresholdBetweenSends)
+		require.Equal(t, expectedNewValue, cfg.HeartbeatV2.PeerAuthenticationTimeThresholdBetweenSends)
 	})
 
 	t.Run("should work and override int value", func(t *testing.T) {

config/config.go

@@ -109,16 +109,16 @@ type SoftwareVersionConfig struct {
 type HeartbeatV2Config struct {
 	PeerAuthenticationTimeBetweenSendsInSec          int64
 	PeerAuthenticationTimeBetweenSendsWhenErrorInSec int64
-	PeerAuthenticationThresholdBetweenSends          float64
+	PeerAuthenticationTimeThresholdBetweenSends      float64
 	HeartbeatTimeBetweenSendsInSec                   int64
 	HeartbeatTimeBetweenSendsWhenErrorInSec          int64
-	HeartbeatThresholdBetweenSends                   float64
+	HeartbeatTimeThresholdBetweenSends               float64
 	HeartbeatExpiryTimespanInSec                     int64
 	MinPeersThreshold                                float32
-	DelayBetweenRequestsInSec                        int64
-	MaxTimeoutInSec                                  int64
+	DelayBetweenPeerAuthenticationRequestsInSec      int64
+	PeerAuthenticationMaxTimeoutForRequestsInSec     int64
 	PeerShardTimeBetweenSendsInSec                   int64
-	PeerShardThresholdBetweenSends                   float64
+	PeerShardTimeThresholdBetweenSends               float64
 	MaxMissingKeysInRequest                          uint32
 	MaxDurationPeerUnresponsiveInSec                 int64
 	HideInactiveValidatorIntervalInSec               int64

config/contextFlagsConfig.go

@@ -3,6 +3,8 @@ package config
 // ContextFlagsConfig will keep the values for the cli.Context flags
 type ContextFlagsConfig struct {
 	WorkingDir                   string
+	DbDir                        string
+	LogsDir                      string
 	EnableGops                   bool
 	SaveLogFile                  bool
 	EnableLogCorrelation         bool

consensus/blacklist/blacklist.go

@@ -0,0 +1,114 @@
+package blacklist
+
+import (
+	"context"
+	"time"
+
+	"github.com/ElrondNetwork/elrond-go-core/core"
+	"github.com/ElrondNetwork/elrond-go-core/core/check"
+	logger "github.com/ElrondNetwork/elrond-go-logger"
+	"github.com/ElrondNetwork/elrond-go/consensus/spos"
+)
+
+var log = logger.GetOrCreate("consensus/blacklist")
+
+var durationSweepP2PBlacklist = time.Second * 5
+
+// PeerBlackListArgs defines the arguments needed for peer blacklist component
+type PeerBlackListArgs struct {
+	PeerCacher spos.PeerBlackListCacher
+}
+
+type peerBlacklist struct {
+	peerCacher spos.PeerBlackListCacher
+	cancel     func()
+}
+
+// NewPeerBlacklist creates a new instance of peer blacklist
+func NewPeerBlacklist(args PeerBlackListArgs) (*peerBlacklist, error) {
+	err := checkArgs(args)
+	if err != nil {
+		return nil, err
+	}
+
+	pb := &peerBlacklist{
+		peerCacher: args.PeerCacher,
+	}
+
+	pb.startSweepingTimeCache()
+
+	return pb, nil
+}
+
+func checkArgs(args PeerBlackListArgs) error {
+	if check.IfNil(args.PeerCacher) {
+		return spos.ErrNilPeerBlacklistCacher
+	}
+
+	return nil
+}
+
+// IsPeerBlacklisted will check if specified peer is blacklisted
+func (pb *peerBlacklist) IsPeerBlacklisted(peer core.PeerID) bool {
+	return pb.peerCacher.Has(peer)
+}
+
+// BlacklistPeer will blacklist a peer for a certain amount of time
+func (pb *peerBlacklist) BlacklistPeer(peer core.PeerID, duration time.Duration) {
+	peerIsBlacklisted := pb.peerCacher.Has(peer)
+
+	err := pb.peerCacher.Upsert(peer, duration)
+	if err != nil {
+		log.Warn("error adding in blacklist",
+			"pid", peer.Pretty(),
+			"time", duration,
+			"error", "err",
+		)
+		return
+	}
+
+	if !peerIsBlacklisted {
+		log.Debug("blacklisted peer",
+			"pid", peer.Pretty(),
+			"time", duration,
+		)
+	}
+}
+
+// startSweepingTimeCache will trigger the sweeping cache goroutine
+func (pb *peerBlacklist) startSweepingTimeCache() {
+	var ctx context.Context
+	ctx, pb.cancel = context.WithCancel(context.Background())
+
+	go pb.startSweepingPeerCache(ctx)
+}
+
+func (pb *peerBlacklist) startSweepingPeerCache(ctx context.Context) {
+	timer := time.NewTimer(durationSweepP2PBlacklist)
+	defer timer.Stop()
+
+	for {
+		timer.Reset(durationSweepP2PBlacklist)
+
+		select {
+		case <-ctx.Done():
+			log.Debug("peerBlacklist's go routine is stopping...")
+			return
+		case <-timer.C:
+		}
+
+		pb.peerCacher.Sweep()
+	}
+}
+
+// Close will close the goroutine
+func (pb *peerBlacklist) Close() error {
+	pb.cancel()
+
+	return nil
+}
+
+// IsInterfaceNil returns true if there is no value under the interface
+func (pb *peerBlacklist) IsInterfaceNil() bool {
+	return pb == nil
+}