From 742c6325a85e85ed050208b020027d4a6a7ed883 Mon Sep 17 00:00:00 2001 From: benaadams Date: Thu, 1 Jun 2023 02:35:59 +0100 Subject: [PATCH] Less aggressive peer discovery --- .../Nethermind.Core/Crypto/PublicKey.cs | 13 ++++- .../Nethermind.Network.Stats/Model/Node.cs | 40 ++++++++------- .../NodeStatsLight.cs | 51 ++++++++----------- .../NodeStatsManager.cs | 30 +++-------- src/Nethermind/Nethermind.Network/Peer.cs | 11 +++- .../Nethermind.Network/PeerManager.cs | 41 ++++++++++----- src/Nethermind/Nethermind.Network/PeerPool.cs | 5 +- 7 files changed, 102 insertions(+), 89 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/PublicKey.cs b/src/Nethermind/Nethermind.Core/Crypto/PublicKey.cs index ca2f3ef87e2..40c77f6c4bc 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/PublicKey.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/PublicKey.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading; using Nethermind.Core.Extensions; @@ -96,7 +97,17 @@ public override bool Equals(object? obj) public override int GetHashCode() { - return MemoryMarshal.Read(Bytes); + byte[] bytes = Bytes; + long l0 = Unsafe.ReadUnaligned(ref MemoryMarshal.GetArrayDataReference(bytes)); + long l1 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long))); + long l2 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long) * 2)); + long l3 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long) * 3)); + long l4 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long) * 4)); + long l5 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long) * 5)); + long l6 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long) * 6)); + long l7 = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(bytes), sizeof(long) * 7)); + l0 ^= l1 ^ l2 ^ l3 ^ l4 ^ l5 ^ l6 ^ l7; + return (int)(l0 ^ (l0 >> 32)); } public override string ToString() diff --git a/src/Nethermind/Nethermind.Network.Stats/Model/Node.cs b/src/Nethermind/Nethermind.Network.Stats/Model/Node.cs index a3d505dd8e4..55b2cd204c0 100644 --- a/src/Nethermind/Nethermind.Network.Stats/Model/Node.cs +++ b/src/Nethermind/Nethermind.Network.Stats/Model/Node.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Collections.Generic; using System.Net; using Nethermind.Config; using Nethermind.Core.Crypto; @@ -12,11 +11,12 @@ namespace Nethermind.Stats.Model /// /// Represents a physical network node address and attributes that we assign to it (static, bootnode, trusted, etc.) /// - public class Node : IFormattable + public sealed class Node : IFormattable, IEquatable { private string _clientId; private string _paddedHost; private string _paddedPort; + private int _hashCode; /// /// Node public key - same as in enode. @@ -71,24 +71,23 @@ public string ClientId public string EthDetails { get; set; } public long CurrentReputation { get; set; } - public Node(PublicKey id, IPEndPoint address) - { - Id = id; - IdHash = Keccak.Compute(Id.PrefixedBytes); - SetIPEndPoint(address); - } - public Node(NetworkNode networkNode, bool isStatic = false) : this(networkNode.NodeId, networkNode.Host, networkNode.Port, isStatic) { } public Node(PublicKey id, string host, int port, bool isStatic = false) + : this(id, GetIPEndPoint(host, port), isStatic) + { + } + + public Node(PublicKey id, IPEndPoint address, bool isStatic = false) { Id = id; IdHash = Keccak.Compute(Id.PrefixedBytes); - SetIPEndPoint(host, port); + _hashCode = id.GetHashCode(); IsStatic = isStatic; + SetIPEndPoint(address); } private void SetIPEndPoint(IPEndPoint address) @@ -102,9 +101,9 @@ private void SetIPEndPoint(IPEndPoint address) _paddedPort = Port.ToString().PadLeft(5, ' '); } - private void SetIPEndPoint(string host, int port) + private static IPEndPoint GetIPEndPoint(string host, int port) { - SetIPEndPoint(new IPEndPoint(IPAddress.Parse(host), port)); + return new IPEndPoint(IPAddress.Parse(host), port); } public override bool Equals(object obj) @@ -122,7 +121,7 @@ public override bool Equals(object obj) return false; } - public override int GetHashCode() => HashCode.Combine(Id); + public override int GetHashCode() => _hashCode; public override string ToString() => ToString(Format.WithPublicKey); @@ -142,14 +141,19 @@ public string ToString(string format, IFormatProvider formatProvider) }; } + public bool Equals(Node other) + { + if (ReferenceEquals(this, other)) return true; + if (other is null) return false; + + return Id.Equals(other.Id); + } + public static bool operator ==(Node a, Node b) { - if (ReferenceEquals(a, null)) - { - return ReferenceEquals(b, null); - } + if (ReferenceEquals(a, b)) return true; - if (ReferenceEquals(b, null)) + if (ReferenceEquals(a, null) || ReferenceEquals(b, null)) { return false; } diff --git a/src/Nethermind/Nethermind.Network.Stats/NodeStatsLight.cs b/src/Nethermind/Nethermind.Network.Stats/NodeStatsLight.cs index 3b9d33300ee..d23610f4b24 100644 --- a/src/Nethermind/Nethermind.Network.Stats/NodeStatsLight.cs +++ b/src/Nethermind/Nethermind.Network.Stats/NodeStatsLight.cs @@ -3,6 +3,9 @@ using System; using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; + using FastEnumUtility; using Nethermind.Stats.Model; @@ -70,10 +73,7 @@ public NodeStatsLight(Node node, decimal latestSpeedWeight = 0.25m) private void Increment(NodeStatsEventType nodeStatsEventType) { - lock (_statCountersArray) - { - _statCountersArray[(int)nodeStatsEventType]++; - } + Interlocked.Increment(ref _statCountersArray[(int)nodeStatsEventType]); } public void AddNodeStatsEvent(NodeStatsEventType nodeStatsEventType) @@ -160,10 +160,7 @@ public void AddNodeStatsSyncEvent(NodeStatsEventType nodeStatsEventType) public bool DidEventHappen(NodeStatsEventType nodeStatsEventType) { - lock (_statCountersArray) - { - return _statCountersArray[(int)nodeStatsEventType] > 0; - } + return GetStat(nodeStatsEventType) > 0; } public void AddTransferSpeedCaptureEvent(TransferSpeedType transferSpeedType, long bytesPerMillisecond) @@ -276,11 +273,7 @@ private bool IsDelayedDueToFailedConnection() private int GetFailedConnectionDelay() { - int failedConnectionFailed; - lock (_statCountersArray) - { - failedConnectionFailed = _statCountersArray[(int)NodeStatsEventType.ConnectionFailed]; - } + int failedConnectionFailed = GetStat(NodeStatsEventType.ConnectionFailed); if (failedConnectionFailed == 0) { @@ -298,11 +291,7 @@ private int GetFailedConnectionDelay() private int GetDisconnectDelay() { int disconnectDelay; - int disconnectCount; - lock (_statCountersArray) - { - disconnectCount = _statCountersArray[(int)NodeStatsEventType.Disconnect]; - } + int disconnectCount = GetStat(NodeStatsEventType.Disconnect); if (disconnectCount == 0) { @@ -327,24 +316,26 @@ private long CalculateCurrentReputation() private bool HasDisconnectedOnce => _lastLocalDisconnect.HasValue || _lastRemoteDisconnect.HasValue; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private int GetStat(NodeStatsEventType nodeStatsEventType) + { + return Volatile.Read(ref _statCountersArray[(int)nodeStatsEventType]); + } + private long CalculateSessionReputation() { long discoveryReputation = 0; long rlpxReputation = 0; - lock (_statCountersArray) - { - - discoveryReputation += Math.Min(_statCountersArray[(int)NodeStatsEventType.DiscoveryPingIn], 10) * (_statCountersArray[(int)NodeStatsEventType.DiscoveryPingIn] == _statCountersArray[(int)NodeStatsEventType.DiscoveryPingOut] ? 2 : 1); - discoveryReputation += Math.Min(_statCountersArray[(int)NodeStatsEventType.DiscoveryNeighboursIn], 10) * 2; + discoveryReputation += Math.Min(GetStat(NodeStatsEventType.DiscoveryPingIn), 10) * (GetStat(NodeStatsEventType.DiscoveryPingIn) == GetStat(NodeStatsEventType.DiscoveryPingOut) ? 2 : 1); + discoveryReputation += Math.Min(GetStat(NodeStatsEventType.DiscoveryNeighboursIn), 10) * 2; - rlpxReputation += Math.Min(_statCountersArray[(int)NodeStatsEventType.P2PPingIn], 10) * (_statCountersArray[(int)NodeStatsEventType.P2PPingIn] == _statCountersArray[(int)NodeStatsEventType.P2PPingOut] ? 2 : 1); - rlpxReputation += _statCountersArray[(int)NodeStatsEventType.HandshakeCompleted] > 0 ? 10 : 0; - rlpxReputation += _statCountersArray[(int)NodeStatsEventType.P2PInitialized] > 0 ? 10 : 0; - rlpxReputation += _statCountersArray[(int)NodeStatsEventType.Eth62Initialized] > 0 ? 20 : 0; - rlpxReputation += _statCountersArray[(int)NodeStatsEventType.SyncStarted] > 0 ? 1000 : 0; - rlpxReputation += (rlpxReputation != 0 && !HasDisconnectedOnce) ? 1 : 0; - } + rlpxReputation += Math.Min(GetStat(NodeStatsEventType.P2PPingIn), 10) * (GetStat(NodeStatsEventType.P2PPingIn) == GetStat(NodeStatsEventType.P2PPingOut) ? 2 : 1); + rlpxReputation += GetStat(NodeStatsEventType.HandshakeCompleted) > 0 ? 10 : 0; + rlpxReputation += GetStat(NodeStatsEventType.P2PInitialized) > 0 ? 10 : 0; + rlpxReputation += GetStat(NodeStatsEventType.Eth62Initialized) > 0 ? 20 : 0; + rlpxReputation += GetStat(NodeStatsEventType.SyncStarted) > 0 ? 1000 : 0; + rlpxReputation += (rlpxReputation != 0 && !HasDisconnectedOnce) ? 1 : 0; if (HasDisconnectedOnce) { diff --git a/src/Nethermind/Nethermind.Network.Stats/NodeStatsManager.cs b/src/Nethermind/Nethermind.Network.Stats/NodeStatsManager.cs index ecaf3e58833..6a54998c138 100644 --- a/src/Nethermind/Nethermind.Network.Stats/NodeStatsManager.cs +++ b/src/Nethermind/Nethermind.Network.Stats/NodeStatsManager.cs @@ -4,10 +4,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.IO; using System.Linq; -using System.Timers; -using Nethermind.Core.Caching; using Nethermind.Core.Timers; using Nethermind.Logging; using Nethermind.Stats.Model; @@ -18,29 +15,12 @@ public class NodeStatsManager : INodeStatsManager, IDisposable { private class NodeComparer : IEqualityComparer { - public bool Equals(Node x, Node y) - { - if (ReferenceEquals(x, null)) - { - return ReferenceEquals(y, null); - } - - if (ReferenceEquals(y, null)) - { - return false; - } - - return x.Id == y.Id; - } - - public int GetHashCode(Node obj) - { - return obj?.GetHashCode() ?? 0; - } + public bool Equals(Node x, Node y) => ReferenceEquals(x, y) || x.Id == y.Id; + public int GetHashCode(Node obj) => obj.GetHashCode(); } private readonly ILogger _logger; - private readonly ConcurrentDictionary _nodeStats = new ConcurrentDictionary(new NodeComparer()); + private readonly ConcurrentDictionary _nodeStats = new(new NodeComparer()); private readonly ITimer _cleanupTimer; private readonly int _maxCount; @@ -56,6 +36,8 @@ public NodeStatsManager(ITimerFactory timerFactory, ILogManager logManager, int private void CleanupTimerOnElapsed(object sender, EventArgs e) { + _cleanupTimer.Stop(); + int deleteCount = _nodeStats.Count - _maxCount; if (deleteCount > 0) @@ -74,6 +56,8 @@ private void CleanupTimerOnElapsed(object sender, EventArgs e) if (_logger.IsDebug) _logger.Debug($"Removed {i} node stats."); } + + _cleanupTimer.Start(); } private INodeStats AddStats(Node node) diff --git a/src/Nethermind/Nethermind.Network/Peer.cs b/src/Nethermind/Nethermind.Network/Peer.cs index 4554798e881..d845eae140b 100644 --- a/src/Nethermind/Nethermind.Network/Peer.cs +++ b/src/Nethermind/Nethermind.Network/Peer.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using Nethermind.Network.P2P; +using Nethermind.Stats; using Nethermind.Stats.Model; namespace Nethermind.Network @@ -15,11 +16,15 @@ namespace Nethermind.Network /// The logic for choosing which session to drop has to be consistent between the two peers - we use the PublicKey /// comparison to choose the connection direction in the same way on both sides. /// - public class Peer + public sealed class Peer { - public Peer(Node node) + public Peer(Node node) : this(node, null) + { } + + public Peer(Node node, INodeStats stats) { Node = node; + Stats = stats; } public bool IsAwaitingConnection { get; set; } @@ -30,6 +35,8 @@ public Peer(Node node) /// public Node Node { get; } + internal INodeStats Stats { get; } + /// /// An incoming session to the Node which can be in one of many states. /// diff --git a/src/Nethermind/Nethermind.Network/PeerManager.cs b/src/Nethermind/Nethermind.Network/PeerManager.cs index d63416acc75..af3155806b7 100644 --- a/src/Nethermind/Nethermind.Network/PeerManager.cs +++ b/src/Nethermind/Nethermind.Network/PeerManager.cs @@ -239,14 +239,14 @@ private async Task RunPeerUpdateLoop() continue; } - Interlocked.Exchange(ref _tryCount, 0); - Interlocked.Exchange(ref _newActiveNodes, 0); - Interlocked.Exchange(ref _failedInitialConnect, 0); - Interlocked.Exchange(ref _connectionRounds, 0); + Volatile.Write(ref _tryCount, 0); + Volatile.Write(ref _newActiveNodes, 0); + Volatile.Write(ref _failedInitialConnect, 0); + Volatile.Write(ref _connectionRounds, 0); SelectAndRankCandidates(); List remainingCandidates = _currentSelection.Candidates; - if (!remainingCandidates.Any()) + if (remainingCandidates.Count == 0) { continue; } @@ -257,6 +257,7 @@ private async Task RunPeerUpdateLoop() } int currentPosition = 0; + long lastMs = Environment.TickCount64; while (true) { if (_cancellationTokenSource.IsCancellationRequested) @@ -288,9 +289,17 @@ private async Task RunPeerUpdateLoop() workerBlock.Complete(); // Wait for all messages to propagate through the network. - workerBlock.Completion.Wait(); + await workerBlock.Completion; Interlocked.Increment(ref _connectionRounds); + + long nowMs = Environment.TickCount64; + long diffMs = nowMs - lastMs; + if (diffMs < 50) + { + await Task.Delay(50 - (int)diffMs); + } + lastMs = nowMs; } if (_logger.IsTrace) @@ -352,6 +361,8 @@ private async Task RunPeerUpdateLoop() await Task.Delay(1000); } } + + _peerUpdateTimer.Start(); } } @@ -416,7 +427,7 @@ private void SelectAndRankCandidates() continue; } - (bool Result, NodeStatsEventType? DelayReason) delayResult = _stats.IsConnectionDelayed(preCandidate.Node); + (bool Result, NodeStatsEventType? DelayReason) delayResult = preCandidate.Stats.IsConnectionDelayed(); if (delayResult.Result) { if (delayResult.DelayReason == NodeStatsEventType.Disconnect) @@ -431,7 +442,7 @@ private void SelectAndRankCandidates() continue; } - if (_stats.FindCompatibilityValidationResult(preCandidate.Node).HasValue) + if (preCandidate.Stats.FailedCompatibilityValidation.HasValue) { _currentSelection.Incompatible.Add(preCandidate); continue; @@ -460,7 +471,11 @@ private void StartPeerUpdateLoop() if (_logger.IsDebug) _logger.Debug("Starting peer update timer"); _peerUpdateTimer = new Timer(_networkConfig.PeersUpdateInterval); - _peerUpdateTimer.Elapsed += (sender, e) => { _peerUpdateRequested.Set(); }; + _peerUpdateTimer.Elapsed += (sender, e) => + { + _peerUpdateTimer.Stop(); + _peerUpdateRequested.Set(); + }; _peerUpdateTimer.Start(); } @@ -726,7 +741,7 @@ private bool CanConnectToPeer(Peer peer) return false; } - (bool delayed, NodeStatsEventType? reason) = _stats.IsConnectionDelayed(peer.Node); + (bool delayed, NodeStatsEventType? reason) = peer.Stats.IsConnectionDelayed(); if (delayed) { if (_logger.IsTrace) _logger.Trace($"Not connecting peer: {peer} due forced connection delay. Reason: {reason}"); @@ -818,7 +833,7 @@ private void AddSession(ISession session, Peer peer) { if (newSessionIsIn) { - _stats.ReportHandshakeEvent(peer.Node, ConnectionDirection.In); + peer.Stats.AddNodeStatsHandshakeEvent(ConnectionDirection.In); peer.InSession = session; } else @@ -846,7 +861,7 @@ private void AddSession(ISession session, Peer peer) session.InitiateDisconnect(InitiateDisconnectReason.ReplacingSessionWithOppositeDirection, "same"); if (newSessionIsIn) { - _stats.ReportHandshakeEvent(peer.Node, ConnectionDirection.In); + peer.Stats.AddNodeStatsHandshakeEvent(ConnectionDirection.In); peer.InSession = session; } else @@ -958,7 +973,7 @@ private void OnHandshakeComplete(object sender, EventArgs args) return; } - _stats.ReportHandshakeEvent(peer.Node, ConnectionDirection.Out); + peer.Stats.AddNodeStatsHandshakeEvent(ConnectionDirection.Out); } if (_logger.IsTrace) _logger.Trace($"|NetworkTrace| {session} handshake initialized in peer manager"); diff --git a/src/Nethermind/Nethermind.Network/PeerPool.cs b/src/Nethermind/Nethermind.Network/PeerPool.cs index d20ed3a77bb..e9d1e1e03fa 100644 --- a/src/Nethermind/Nethermind.Network/PeerPool.cs +++ b/src/Nethermind/Nethermind.Network/PeerPool.cs @@ -95,7 +95,7 @@ private Peer CreateNew(PublicKey key, (Node Node, ConcurrentDictionary Statics) arg) { - Peer peer = new(new(arg.Node)); + Node node = new(arg.Node); + Peer peer = new(node, _stats.GetOrAdd(node)); PeerAdded?.Invoke(this, new PeerEventArgs(peer)); return peer;