From c65edee7b0c3ad4f7420127be199db508fdbd7b5 Mon Sep 17 00:00:00 2001 From: Andrei Rykov Date: Fri, 5 Jul 2024 13:07:17 +0200 Subject: [PATCH] added database filter for time difference - merge stable-24-2 (#6229) --- .../run/kikimr_services_initializers.cpp | 6 +- ydb/core/health_check/health_check.cpp | 81 ++++++++++--------- .../actors/interconnect/interconnect_common.h | 5 +- .../interconnect/interconnect_tcp_session.cpp | 7 +- 4 files changed, 57 insertions(+), 42 deletions(-) diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 53b794d0a48a..937ec56776b2 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -712,8 +712,10 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s data.Yellow ? NKikimrWhiteboard::EFlag::Yellow : data.Orange ? NKikimrWhiteboard::EFlag::Orange : data.Red ? NKikimrWhiteboard::EFlag::Red : NKikimrWhiteboard::EFlag())); - data.ActorSystem->Send(whiteboardId, new NNodeWhiteboard::TEvWhiteboard::TEvClockSkewUpdate( - data.PeerId, data.ClockSkew)); + if (data.ReportClockSkew) { + data.ActorSystem->Send(whiteboardId, new NNodeWhiteboard::TEvWhiteboard::TEvClockSkewUpdate( + data.PeerId, data.ClockSkew)); + } }; } diff --git a/ydb/core/health_check/health_check.cpp b/ydb/core/health_check/health_check.cpp index 07de22de1eaa..7ca1cb14e2f7 100644 --- a/ydb/core/health_check/health_check.cpp +++ b/ydb/core/health_check/health_check.cpp @@ -235,6 +235,7 @@ class TSelfCheckRequest : public TActorBootstrapped { ui64 StorageQuota; ui64 StorageUsage; TMaybeServerlessComputeResourcesMode ServerlessComputeResourcesMode; + TNodeId MaxTimeDifferenceNodeId = 0; }; struct TSelfCheckResult { @@ -813,10 +814,10 @@ class TSelfCheckRequest : public TActorBootstrapped { ReplyAndPassAway(); } - bool IsStaticNode(const TEvInterconnect::TNodeInfo& nodeInfo) const { + bool IsStaticNode(const TNodeId nodeId) const { TAppData* appData = AppData(); if (appData->DynamicNameserviceConfig) { - return nodeInfo.NodeId <= AppData()->DynamicNameserviceConfig->MaxStaticNodeId; + return nodeId <= AppData()->DynamicNameserviceConfig->MaxStaticNodeId; } else { return true; } @@ -827,7 +828,7 @@ class TSelfCheckRequest : public TActorBootstrapped { NodesInfo = ev->Release(); for (const auto& ni : NodesInfo->Nodes) { MergedNodeInfo[ni.NodeId] = ∋ - if (IsStaticNode(ni) && needComputeFromStaticNodes) { + if (IsStaticNode(ni.NodeId) && needComputeFromStaticNodes) { DatabaseState[DomainPath].ComputeNodeIds.push_back(ni.NodeId); RequestComputeNode(ni.NodeId); } @@ -1251,7 +1252,7 @@ class TSelfCheckRequest : public TActorBootstrapped { } } - void FillComputeNodeStatus(TDatabaseState& databaseState,TNodeId nodeId, Ydb::Monitoring::ComputeNodeStatus& computeNodeStatus, TSelfCheckContext context) { + void FillComputeNodeStatus(TDatabaseState& databaseState, TNodeId nodeId, Ydb::Monitoring::ComputeNodeStatus& computeNodeStatus, TSelfCheckContext context) { FillNodeInfo(nodeId, context.Location.mutable_compute()->mutable_node()); TSelfCheckContext rrContext(&context, "NODE_UPTIME"); @@ -1289,6 +1290,32 @@ class TSelfCheckRequest : public TActorBootstrapped { } loadAverageStatus.set_overall(laContext.GetOverallStatus()); } + + if (nodeSystemState.HasMaxClockSkewPeerId()) { + TNodeId peerId = nodeSystemState.GetMaxClockSkewPeerId(); + long timeDifferenceUs = nodeSystemState.GetMaxClockSkewWithPeerUs(); + TDuration timeDifferenceDuration = TDuration::MicroSeconds(abs(timeDifferenceUs)); + Ydb::Monitoring::StatusFlag::Status status; + if (timeDifferenceDuration > MAX_CLOCKSKEW_ORANGE_ISSUE_TIME) { + status = Ydb::Monitoring::StatusFlag::ORANGE; + } else if (timeDifferenceDuration > MAX_CLOCKSKEW_YELLOW_ISSUE_TIME) { + status = Ydb::Monitoring::StatusFlag::YELLOW; + } else { + status = Ydb::Monitoring::StatusFlag::GREEN; + } + + if (databaseState.MaxTimeDifferenceNodeId == nodeId) { + TSelfCheckContext tdContext(&context, "NODES_TIME_DIFFERENCE"); + if (status == Ydb::Monitoring::StatusFlag::GREEN) { + tdContext.ReportStatus(status); + } else { + tdContext.ReportStatus(status, TStringBuilder() << "Node is " + << timeDifferenceDuration.MilliSeconds() << " ms " + << (timeDifferenceUs > 0 ? "behind " : "ahead of ") + << "peer [" << peerId << "]", ETags::SyncState); + } + } + } } else { // context.ReportStatus(Ydb::Monitoring::StatusFlag::RED, // TStringBuilder() << "Compute node is not available", @@ -1320,12 +1347,24 @@ class TSelfCheckRequest : public TActorBootstrapped { if (systemStatus != Ydb::Monitoring::StatusFlag::GREEN && systemStatus != Ydb::Monitoring::StatusFlag::GREY) { context.ReportStatus(systemStatus, "Compute has issues with system tablets", ETags::ComputeState, {ETags::SystemTabletState}); } + long maxTimeDifferenceUs = 0; + for (TNodeId nodeId : *computeNodeIds) { + auto itNodeSystemState = MergedNodeSystemState.find(nodeId); + if (itNodeSystemState != MergedNodeSystemState.end()) { + if (std::count(computeNodeIds->begin(), computeNodeIds->end(), itNodeSystemState->second->GetMaxClockSkewPeerId()) > 0 + && abs(itNodeSystemState->second->GetMaxClockSkewWithPeerUs()) > maxTimeDifferenceUs) { + maxTimeDifferenceUs = abs(itNodeSystemState->second->GetMaxClockSkewWithPeerUs()); + databaseState.MaxTimeDifferenceNodeId = nodeId; + } + } + } for (TNodeId nodeId : *computeNodeIds) { auto& computeNode = *computeStatus.add_nodes(); FillComputeNodeStatus(databaseState, nodeId, computeNode, {&context, "COMPUTE_NODE"}); } context.ReportWithMaxChildStatus("Some nodes are restarting too often", ETags::ComputeState, {ETags::Uptime}); context.ReportWithMaxChildStatus("Compute is overloaded", ETags::ComputeState, {ETags::OverloadState}); + context.ReportWithMaxChildStatus("Database has time difference between nodes", ETags::ComputeState, {ETags::SyncState}); Ydb::Monitoring::StatusFlag::Status tabletsStatus = Ydb::Monitoring::StatusFlag::GREEN; computeNodeIds->push_back(0); // for tablets without node for (TNodeId nodeId : *computeNodeIds) { @@ -2072,39 +2111,6 @@ class TSelfCheckRequest : public TActorBootstrapped { const TDuration MAX_CLOCKSKEW_ORANGE_ISSUE_TIME = TDuration::MicroSeconds(25000); const TDuration MAX_CLOCKSKEW_YELLOW_ISSUE_TIME = TDuration::MicroSeconds(5000); - void FillNodesSyncStatus(TOverallStateContext& context) { - long maxClockSkewUs = 0; - TNodeId maxClockSkewPeerId = 0; - TNodeId maxClockSkewNodeId = 0; - for (auto& [nodeId, nodeSystemState] : MergedNodeSystemState) { - if (abs(nodeSystemState->GetMaxClockSkewWithPeerUs()) > maxClockSkewUs) { - maxClockSkewUs = abs(nodeSystemState->GetMaxClockSkewWithPeerUs()); - maxClockSkewPeerId = nodeSystemState->GetMaxClockSkewPeerId(); - maxClockSkewNodeId = nodeId; - } - } - if (!maxClockSkewNodeId) { - return; - } - - TSelfCheckResult syncContext; - syncContext.Type = "NODES_TIME_DIFFERENCE"; - FillNodeInfo(maxClockSkewNodeId, syncContext.Location.mutable_node()); - FillNodeInfo(maxClockSkewPeerId, syncContext.Location.mutable_peer()); - - TDuration maxClockSkewTime = TDuration::MicroSeconds(maxClockSkewUs); - if (maxClockSkewTime > MAX_CLOCKSKEW_ORANGE_ISSUE_TIME) { - syncContext.ReportStatus(Ydb::Monitoring::StatusFlag::ORANGE, TStringBuilder() << "The nodes have a time difference of " << maxClockSkewTime.MilliSeconds() << " ms", ETags::SyncState); - } else if (maxClockSkewTime > MAX_CLOCKSKEW_YELLOW_ISSUE_TIME) { - syncContext.ReportStatus(Ydb::Monitoring::StatusFlag::YELLOW, TStringBuilder() << "The nodes have a time difference of " << maxClockSkewTime.MilliSeconds() << " ms", ETags::SyncState); - } else { - syncContext.ReportStatus(Ydb::Monitoring::StatusFlag::GREEN); - } - - context.UpdateMaxStatus(syncContext.GetOverallStatus()); - context.AddIssues(syncContext.IssueRecords); - } - void FillResult(TOverallStateContext context) { if (IsSpecificDatabaseFilter()) { FillDatabaseResult(context, FilterDatabase, DatabaseState[FilterDatabase]); @@ -2113,7 +2119,6 @@ class TSelfCheckRequest : public TActorBootstrapped { FillDatabaseResult(context, path, state); } } - FillNodesSyncStatus(context); if (DatabaseState.empty()) { Ydb::Monitoring::DatabaseStatus& databaseStatus(*context.Result->add_database_status()); TSelfCheckResult tabletContext; diff --git a/ydb/library/actors/interconnect/interconnect_common.h b/ydb/library/actors/interconnect/interconnect_common.h index 29f32256838b..fc23f70b5315 100644 --- a/ydb/library/actors/interconnect/interconnect_common.h +++ b/ydb/library/actors/interconnect/interconnect_common.h @@ -74,8 +74,10 @@ namespace NActors { bool Orange; bool Red; i64 ClockSkew; + bool ReportClockSkew; - TWhiteboardSessionStatus(TActorSystem* actorSystem, ui32 peerId, const TString& peer, bool connected, bool green, bool yellow, bool orange, bool red, i64 clockSkew) + TWhiteboardSessionStatus(TActorSystem* actorSystem, ui32 peerId, const TString& peer, bool connected, + bool green, bool yellow, bool orange, bool red, i64 clockSkew, bool reportClockSkew) : ActorSystem(actorSystem) , PeerId(peerId) , Peer(peer) @@ -85,6 +87,7 @@ namespace NActors { , Orange(orange) , Red(red) , ClockSkew(clockSkew) + , ReportClockSkew(reportClockSkew) {} }; diff --git a/ydb/library/actors/interconnect/interconnect_tcp_session.cpp b/ydb/library/actors/interconnect/interconnect_tcp_session.cpp index e1f2bc735130..a5ecb2052137 100644 --- a/ydb/library/actors/interconnect/interconnect_tcp_session.cpp +++ b/ydb/library/actors/interconnect/interconnect_tcp_session.cpp @@ -999,6 +999,10 @@ namespace NActors { } while (false); } + // we need track clockskew only if it's one tenant nodes connection + // they have one scope in this case + bool reportClockSkew = Proxy->Common->LocalScopeId.first != 0 && Proxy->Common->LocalScopeId == Params.PeerScopeId; + callback({TlsActivationContext->ExecutorThread.ActorSystem, Proxy->PeerNodeId, Proxy->Metrics->GetHumanFriendlyPeerHostName(), @@ -1007,7 +1011,8 @@ namespace NActors { flagState == EFlag::YELLOW, flagState == EFlag::ORANGE, flagState == EFlag::RED, - ReceiveContext->ClockSkew_us.load()}); + ReceiveContext->ClockSkew_us.load(), + reportClockSkew}); } if (connected) {