From c1d5a308f837790b25dd959bd0c635e4d27d29b1 Mon Sep 17 00:00:00 2001 From: vporyadke Date: Mon, 12 Feb 2024 10:01:03 +0100 Subject: [PATCH] store history of changes made to hive via http (#1637) --- ydb/core/mind/hive/hive_schema.h | 12 +- ydb/core/mind/hive/monitoring.cpp | 281 ++++++++++++++---- ydb/core/mind/hive/monitoring.h | 23 ++ ydb/core/mind/hive/ya.make | 2 + ydb/core/tablet/tablet_monitoring_proxy.cpp | 11 +- ydb/library/actors/protos/actors.proto | 1 + .../flat_hive.schema | 47 +++ 7 files changed, 307 insertions(+), 70 deletions(-) create mode 100644 ydb/core/mind/hive/monitoring.h diff --git a/ydb/core/mind/hive/hive_schema.h b/ydb/core/mind/hive/hive_schema.h index eb8748834ae2..cd6d33fcac97 100644 --- a/ydb/core/mind/hive/hive_schema.h +++ b/ydb/core/mind/hive/hive_schema.h @@ -298,6 +298,15 @@ struct Schema : NIceDb::Schema { using TColumns = TableColumns; }; + struct OperationsLog : Table<21> { + struct Timestamp : Column<1, NScheme::NTypeIds::Uint64> {}; + struct User : Column<2, NScheme::NTypeIds::String> {}; + struct Operation : Column<3, NScheme::NTypeIds::String> {}; // JSON + + using TKey = TableKey; + using TColumns = TableColumns; + }; + using TTables = SchemaTables< State, Tablet, @@ -313,7 +322,8 @@ struct Schema : NIceDb::Schema { SubDomain, BlockedOwner, TabletOwners, - TabletAvailabilityRestrictions + TabletAvailabilityRestrictions, + OperationsLog >; using TSettings = SchemaSettings< ExecutorLogBatching, diff --git a/ydb/core/mind/hive/monitoring.cpp b/ydb/core/mind/hive/monitoring.cpp index 15e669d5de82..b428ff077684 100644 --- a/ydb/core/mind/hive/monitoring.cpp +++ b/ydb/core/mind/hive/monitoring.cpp @@ -8,10 +8,31 @@ #include "hive_transactions.h" #include "hive_schema.h" #include "hive_log.h" +#include "monitoring.h" namespace NKikimr { namespace NHive { +TLoggedMonTransaction::TLoggedMonTransaction(const NMon::TEvRemoteHttpInfo::TPtr& ev) { + const auto& query = ev->Get()->ExtendedQuery; + if (query) { + NACLib::TUserToken token(query->GetUserToken()); + User = token.GetUserSID(); + } +} + +bool TLoggedMonTransaction::Prepare(NIceDb::TNiceDb& db) { + Timestamp = TActivationContext::Now(); + auto rowset = db.Table().Key(Timestamp.MilliSeconds()).Select(); + return rowset.IsReady() && !rowset.HaveValue(); +} + +void TLoggedMonTransaction::WriteOperation(NIceDb::TNiceDb& db, const NJson::TJsonValue& op) { + TStringStream str; + NJson::WriteJson(&str, &op); + db.Table().Key(Timestamp.MilliSeconds()).Update(User, str.Str()); +} + class TTxMonEvent_DbState : public TTransactionBase { public: struct TTabletInfo { @@ -671,7 +692,7 @@ class TTxMonEvent_Resources : public TTransactionBase { } }; -class TTxMonEvent_Settings : public TTransactionBase { +class TTxMonEvent_Settings : public TTransactionBase, public TLoggedMonTransaction { public: const TActorId Source; TAutoPtr Event; @@ -679,19 +700,21 @@ class TTxMonEvent_Settings : public TTransactionBase { TTxMonEvent_Settings(const TActorId &source, NMon::TEvRemoteHttpInfo::TPtr& ev, TSelf *hive) : TBase(hive) + , TLoggedMonTransaction(ev) , Source(source) , Event(ev->Release()) {} TTxType GetTxType() const override { return NHive::TXTYPE_MON_SETTINGS; } - void UpdateConfig(NIceDb::TNiceDb& db, const TString& param, TSchemeIds::State compatibilityParam = TSchemeIds::State::DefaultState) { + void UpdateConfig(NIceDb::TNiceDb& db, const TString& param, NJson::TJsonValue& jsonLog, TSchemeIds::State compatibilityParam = TSchemeIds::State::DefaultState) { const auto& params(Event->Cgi()); if (params.contains(param)) { const TString& value = params.Get(param); const google::protobuf::Reflection* reflection = Self->DatabaseConfig.GetReflection(); const google::protobuf::FieldDescriptor* field = Self->DatabaseConfig.GetDescriptor()->FindFieldByName(param); if (reflection != nullptr && field != nullptr) { + jsonLog[param] = value; if (value.empty()) { reflection->ClearField(&Self->DatabaseConfig, field); // compatibility @@ -758,63 +781,70 @@ class TTxMonEvent_Settings : public TTransactionBase { const auto& params(Event->Cgi()); NIceDb::TNiceDb db(txc.DB); - UpdateConfig(db, "MaxTabletsScheduled", TSchemeIds::State::MaxTabletsScheduled); - UpdateConfig(db, "MaxBootBatchSize", TSchemeIds::State::MaxBootBatchSize); - UpdateConfig(db, "DrainInflight", TSchemeIds::State::DrainInflight); - UpdateConfig(db, "MaxResourceCPU", TSchemeIds::State::MaxResourceCPU); - UpdateConfig(db, "MaxResourceMemory", TSchemeIds::State::MaxResourceMemory); - UpdateConfig(db, "MaxResourceNetwork", TSchemeIds::State::MaxResourceNetwork); - UpdateConfig(db, "MaxResourceCounter", TSchemeIds::State::MaxResourceCounter); - UpdateConfig(db, "MinScatterToBalance", TSchemeIds::State::MinScatterToBalance); - UpdateConfig(db, "MinCPUScatterToBalance"); - UpdateConfig(db, "MinMemoryScatterToBalance"); - UpdateConfig(db, "MinNetworkScatterToBalance"); - UpdateConfig(db, "MinCounterScatterToBalance"); - UpdateConfig(db, "MaxNodeUsageToKick", TSchemeIds::State::MaxNodeUsageToKick); - UpdateConfig(db, "ResourceChangeReactionPeriod", TSchemeIds::State::ResourceChangeReactionPeriod); - UpdateConfig(db, "TabletKickCooldownPeriod", TSchemeIds::State::TabletKickCooldownPeriod); - UpdateConfig(db, "SpreadNeighbours", TSchemeIds::State::SpreadNeighbours); - UpdateConfig(db, "DefaultUnitIOPS", TSchemeIds::State::DefaultUnitIOPS); - UpdateConfig(db, "DefaultUnitThroughput", TSchemeIds::State::DefaultUnitThroughput); - UpdateConfig(db, "DefaultUnitSize", TSchemeIds::State::DefaultUnitSize); - UpdateConfig(db, "StorageOvercommit", TSchemeIds::State::StorageOvercommit); - UpdateConfig(db, "StorageBalanceStrategy", TSchemeIds::State::StorageBalanceStrategy); - UpdateConfig(db, "StorageSelectStrategy", TSchemeIds::State::StorageSelectStrategy); - UpdateConfig(db, "StorageSafeMode", TSchemeIds::State::StorageSafeMode); - UpdateConfig(db, "RequestSequenceSize", TSchemeIds::State::RequestSequenceSize); - UpdateConfig(db, "MinRequestSequenceSize", TSchemeIds::State::MinRequestSequenceSize); - UpdateConfig(db, "MaxRequestSequenceSize", TSchemeIds::State::MaxRequestSequenceSize); - UpdateConfig(db, "MetricsWindowSize", TSchemeIds::State::MetricsWindowSize); - UpdateConfig(db, "ResourceOvercommitment", TSchemeIds::State::ResourceOvercommitment); - UpdateConfig(db, "NodeBalanceStrategy"); - UpdateConfig(db, "TabletBalanceStrategy"); - UpdateConfig(db, "MinPeriodBetweenBalance"); - UpdateConfig(db, "BalancerInflight"); - UpdateConfig(db, "MaxMovementsOnAutoBalancer"); - UpdateConfig(db, "ContinueAutoBalancer"); - UpdateConfig(db, "MinPeriodBetweenEmergencyBalance"); - UpdateConfig(db, "EmergencyBalancerInflight"); - UpdateConfig(db, "MaxMovementsOnEmergencyBalancer"); - UpdateConfig(db, "ContinueEmergencyBalancer"); - UpdateConfig(db, "MinNodeUsageToBalance"); - UpdateConfig(db, "MinPeriodBetweenReassign"); - UpdateConfig(db, "NodeSelectStrategy"); - UpdateConfig(db, "CheckMoveExpediency"); - UpdateConfig(db, "SpaceUsagePenaltyThreshold"); - UpdateConfig(db, "SpaceUsagePenalty"); - UpdateConfig(db, "WarmUpBootWaitingPeriod"); - UpdateConfig(db, "MaxWarmUpPeriod"); - UpdateConfig(db, "WarmUpEnabled"); - UpdateConfig(db, "ObjectImbalanceToBalance"); - UpdateConfig(db, "ChannelBalanceStrategy"); - UpdateConfig(db, "MaxChannelHistorySize"); - UpdateConfig(db, "StorageInfoRefreshFrequency"); - UpdateConfig(db, "MinStorageScatterToBalance"); - UpdateConfig(db, "MinGroupUsageToBalance"); - UpdateConfig(db, "StorageBalancerInflight"); + if (!Prepare(db)) { + return false; + } + NJson::TJsonValue jsonOperation; + auto& configUpdates = jsonOperation["ConfigUpdates"]; + + UpdateConfig(db, "MaxTabletsScheduled", configUpdates, TSchemeIds::State::MaxTabletsScheduled); + UpdateConfig(db, "MaxBootBatchSize", configUpdates, TSchemeIds::State::MaxBootBatchSize); + UpdateConfig(db, "DrainInflight", configUpdates, TSchemeIds::State::DrainInflight); + UpdateConfig(db, "MaxResourceCPU", configUpdates, TSchemeIds::State::MaxResourceCPU); + UpdateConfig(db, "MaxResourceMemory", configUpdates, TSchemeIds::State::MaxResourceMemory); + UpdateConfig(db, "MaxResourceNetwork", configUpdates, TSchemeIds::State::MaxResourceNetwork); + UpdateConfig(db, "MaxResourceCounter", configUpdates, TSchemeIds::State::MaxResourceCounter); + UpdateConfig(db, "MinScatterToBalance", configUpdates, TSchemeIds::State::MinScatterToBalance); + UpdateConfig(db, "MinCPUScatterToBalance", configUpdates); + UpdateConfig(db, "MinMemoryScatterToBalance", configUpdates); + UpdateConfig(db, "MinNetworkScatterToBalance", configUpdates); + UpdateConfig(db, "MinCounterScatterToBalance", configUpdates); + UpdateConfig(db, "MaxNodeUsageToKick", configUpdates, TSchemeIds::State::MaxNodeUsageToKick); + UpdateConfig(db, "ResourceChangeReactionPeriod", configUpdates, TSchemeIds::State::ResourceChangeReactionPeriod); + UpdateConfig(db, "TabletKickCooldownPeriod", configUpdates, TSchemeIds::State::TabletKickCooldownPeriod); + UpdateConfig(db, "SpreadNeighbours", configUpdates, TSchemeIds::State::SpreadNeighbours); + UpdateConfig(db, "DefaultUnitIOPS", configUpdates, TSchemeIds::State::DefaultUnitIOPS); + UpdateConfig(db, "DefaultUnitThroughput", configUpdates, TSchemeIds::State::DefaultUnitThroughput); + UpdateConfig(db, "DefaultUnitSize", configUpdates, TSchemeIds::State::DefaultUnitSize); + UpdateConfig(db, "StorageOvercommit", configUpdates, TSchemeIds::State::StorageOvercommit); + UpdateConfig(db, "StorageBalanceStrategy", configUpdates, TSchemeIds::State::StorageBalanceStrategy); + UpdateConfig(db, "StorageSelectStrategy", configUpdates, TSchemeIds::State::StorageSelectStrategy); + UpdateConfig(db, "StorageSafeMode", configUpdates, TSchemeIds::State::StorageSafeMode); + UpdateConfig(db, "RequestSequenceSize", configUpdates, TSchemeIds::State::RequestSequenceSize); + UpdateConfig(db, "MinRequestSequenceSize", configUpdates, TSchemeIds::State::MinRequestSequenceSize); + UpdateConfig(db, "MaxRequestSequenceSize", configUpdates, TSchemeIds::State::MaxRequestSequenceSize); + UpdateConfig(db, "MetricsWindowSize", configUpdates, TSchemeIds::State::MetricsWindowSize); + UpdateConfig(db, "ResourceOvercommitment", configUpdates, TSchemeIds::State::ResourceOvercommitment); + UpdateConfig(db, "NodeBalanceStrategy", configUpdates); + UpdateConfig(db, "TabletBalanceStrategy", configUpdates); + UpdateConfig(db, "MinPeriodBetweenBalance", configUpdates); + UpdateConfig(db, "BalancerInflight", configUpdates); + UpdateConfig(db, "MaxMovementsOnAutoBalancer", configUpdates); + UpdateConfig(db, "ContinueAutoBalancer", configUpdates); + UpdateConfig(db, "MinPeriodBetweenEmergencyBalance", configUpdates); + UpdateConfig(db, "EmergencyBalancerInflight", configUpdates); + UpdateConfig(db, "MaxMovementsOnEmergencyBalancer", configUpdates); + UpdateConfig(db, "ContinueEmergencyBalancer", configUpdates); + UpdateConfig(db, "MinNodeUsageToBalance", configUpdates); + UpdateConfig(db, "MinPeriodBetweenReassign", configUpdates); + UpdateConfig(db, "NodeSelectStrategy", configUpdates); + UpdateConfig(db, "CheckMoveExpediency", configUpdates); + UpdateConfig(db, "SpaceUsagePenaltyThreshold", configUpdates); + UpdateConfig(db, "SpaceUsagePenalty", configUpdates); + UpdateConfig(db, "WarmUpBootWaitingPeriod", configUpdates); + UpdateConfig(db, "MaxWarmUpPeriod", configUpdates); + UpdateConfig(db, "WarmUpEnabled", configUpdates); + UpdateConfig(db, "ObjectImbalanceToBalance", configUpdates); + UpdateConfig(db, "ChannelBalanceStrategy", configUpdates); + UpdateConfig(db, "MaxChannelHistorySize", configUpdates); + UpdateConfig(db, "StorageInfoRefreshFrequency", configUpdates); + UpdateConfig(db, "MinStorageScatterToBalance", configUpdates); + UpdateConfig(db, "MinGroupUsageToBalance", configUpdates); + UpdateConfig(db, "StorageBalancerInflight", configUpdates); if (params.contains("BalancerIgnoreTabletTypes")) { - TVector tabletTypeNames = SplitString(params.Get("BalancerIgnoreTabletTypes"), ";"); + auto value = params.Get("BalancerIgnoreTabletTypes"); + TVector tabletTypeNames = SplitString(value, ";"); std::vector newTypeList; for (const auto& name : tabletTypeNames) { TTabletTypes::EType type = TTabletTypes::StrToType(Strip(name)); @@ -832,11 +862,13 @@ class TTxMonEvent_Settings : public TTransactionBase { field->Add(i); } ChangeRequest = true; + configUpdates["BalancerIgnoreTabletTypes"] = value; // Self->BalancerIgnoreTabletTypes will be replaced by Self->BuildCurrentConfig() } } if (params.contains("DefaultTabletLimit")) { + auto value = params.Get("DefaultTabletLimit"); auto tabletLimits = SplitString(params.Get("DefaultTabletLimit"), ";"); for (TStringBuf limit : tabletLimits) { TStringBuf tabletType = limit.NextTok(':'); @@ -851,6 +883,8 @@ class TTxMonEvent_Settings : public TTransactionBase { protoLimit->SetMaxCount(*maxCount); } + configUpdates["DefaultTabletLimit"] = value; + // Get rid of duplicates & default values google::protobuf::RepeatedPtrField cleanTabletLimits; auto* dirtyTabletLimits = Self->DatabaseConfig.MutableDefaultTabletLimit(); @@ -873,6 +907,7 @@ class TTxMonEvent_Settings : public TTransactionBase { db.Table().Key(TSchemeIds::State::DefaultState).Update(Self->DatabaseConfig); } if (params.contains("allowedMetrics")) { + auto& jsonAllowedMetrics = jsonOperation["AllowedMetricsUpdate"]; TVector allowedMetrics = SplitString(params.Get("allowedMetrics"), ";"); for (TStringBuf tabletAllowedMetrics : allowedMetrics) { TStringBuf tabletType = tabletAllowedMetrics.NextTok(':'); @@ -899,6 +934,20 @@ class TTxMonEvent_Settings : public TTransactionBase { } } if (changed) { + ChangeRequest = true; + NJson::TJsonValue jsonUpdate; + jsonUpdate["Type"] = tabletType; + const auto* descriptor = NKikimrTabletBase::TMetrics::descriptor(); + auto& jsonMetrics = jsonUpdate["AllowedMetrics"]; + for (auto metricNum : metrics) { + const auto* field = descriptor->FindFieldByNumber(metricNum); + if (field) { + jsonMetrics.AppendValue(field->name()); + } else { + jsonMetrics.AppendValue(metricNum); + } + } + jsonAllowedMetrics.AppendValue(std::move(jsonUpdate)); db.Table().Key(type).Update(metrics); Self->TabletTypeAllowedMetrics[type] = metrics; } @@ -906,6 +955,10 @@ class TTxMonEvent_Settings : public TTransactionBase { } } } + + if (ChangeRequest) { + WriteOperation(db, jsonOperation); + } return true; } @@ -1252,7 +1305,7 @@ class TTxMonEvent_Settings : public TTransactionBase { } }; -class TTxMonEvent_TabletAvailability : public TTransactionBase { +class TTxMonEvent_TabletAvailability : public TTransactionBase, public TLoggedMonTransaction { public: const TActorId Source; TAutoPtr Event; @@ -1262,6 +1315,7 @@ class TTxMonEvent_TabletAvailability : public TTransactionBase { TTxMonEvent_TabletAvailability(const TActorId &source, NMon::TEvRemoteHttpInfo::TPtr& ev, TSelf *hive) : TBase(hive) + , TLoggedMonTransaction(ev) , Source(source) , Event(ev->Release()) { @@ -1282,12 +1336,23 @@ class TTxMonEvent_TabletAvailability : public TTransactionBase { if (Node == nullptr) { return true; } + if (!Prepare(db)) { + return false; + } + + NJson::TJsonValue jsonOperation; + jsonOperation["NodeId"] = NodeId; + const auto& cgi = Event->Cgi(); auto changeType = ParseTabletType(cgi.Get("changetype")); auto maxCount = TryFromString(cgi.Get("maxcount")); auto resetType = ParseTabletType(cgi.Get("resettype")); if (changeType != TTabletTypes::TypeInvalid && maxCount) { ChangeRequest = true; + NJson::TJsonValue jsonUpdate; + jsonUpdate["Type"] = GetTabletTypeShortName(changeType); + jsonUpdate["MaxCount"] = *maxCount; + jsonOperation["TabletAvailability"].AppendValue(std::move(jsonUpdate)); db.Table().Key(NodeId, changeType).Update(*maxCount); Node->TabletAvailabilityRestrictions[changeType] = *maxCount; auto it = Node->TabletAvailability.find(changeType); @@ -1297,6 +1362,10 @@ class TTxMonEvent_TabletAvailability : public TTransactionBase { } if (resetType != TTabletTypes::TypeInvalid) { ChangeRequest = true; + NJson::TJsonValue jsonUpdate; + jsonUpdate["Type"] = GetTabletTypeShortName(resetType); + jsonUpdate["MaxCount"] = "[default]"; + jsonOperation["TabletAvailability"].AppendValue(std::move(jsonUpdate)); Node->TabletAvailabilityRestrictions.erase(resetType); auto it = Node->TabletAvailability.find(resetType); if (it != Node->TabletAvailability.end()) { @@ -1315,6 +1384,7 @@ class TTxMonEvent_TabletAvailability : public TTransactionBase { if (ChangeRequest) { Self->ObjectDistributions.RemoveNode(*Node); Self->ObjectDistributions.AddNode(*Node); + WriteOperation(db, jsonOperation); } return true; } @@ -1522,6 +1592,9 @@ class TTxMonEvent_Landing : public TTransactionBase { out << "
"; out << ""; out << "
"; + out << "
"; + out << ""; + out << "
"; out << ""; out << "
"; @@ -2422,15 +2495,16 @@ class TTxMonEvent_LandingData : public TTransactionBase { } }; -class TTxMonEvent_SetDown : public TTransactionBase { +class TTxMonEvent_SetDown : public TTransactionBase, public TLoggedMonTransaction { public: const TActorId Source; const TNodeId NodeId; const bool Down; TString Response; - TTxMonEvent_SetDown(const TActorId& source, TNodeId nodeId, bool down, TSelf* hive) + TTxMonEvent_SetDown(const TActorId& source, TNodeId nodeId, bool down, TSelf* hive, NMon::TEvRemoteHttpInfo::TPtr& ev) : TBase(hive) + , TLoggedMonTransaction(ev) , Source(source) , NodeId(nodeId) , Down(down) @@ -2440,10 +2514,17 @@ class TTxMonEvent_SetDown : public TTransactionBase { bool Execute(TTransactionContext& txc, const TActorContext&) override { NIceDb::TNiceDb db(txc.DB); + if (!Prepare(db)) { + return false; + } TNodeInfo* node = Self->FindNode(NodeId); if (node != nullptr) { node->SetDown(Down); db.Table().Key(NodeId).Update(NIceDb::TUpdate(Down)); + NJson::TJsonValue jsonOperation; + jsonOperation["NodeId"] = NodeId; + jsonOperation["Down"] = Down; + WriteOperation(db, jsonOperation); Response = "{\"NodeId\":" + ToString(NodeId) + ',' + "\"Down\":" + (Down ? "true" : "false") + "}"; } else { Response = "{\"Error\":\"Node " + ToString(NodeId) + " not found\"}"; @@ -2457,15 +2538,16 @@ class TTxMonEvent_SetDown : public TTransactionBase { } }; -class TTxMonEvent_SetFreeze : public TTransactionBase { +class TTxMonEvent_SetFreeze : public TTransactionBase, public TLoggedMonTransaction { public: const TActorId Source; const TNodeId NodeId; const bool Freeze; TString Response; - TTxMonEvent_SetFreeze(const TActorId& source, TNodeId nodeId, bool freeze, TSelf* hive) + TTxMonEvent_SetFreeze(const TActorId& source, TNodeId nodeId, bool freeze, TSelf* hive, NMon::TEvRemoteHttpInfo::TPtr& ev) : TBase(hive) + , TLoggedMonTransaction(ev) , Source(source) , NodeId(nodeId) , Freeze(freeze) @@ -2475,10 +2557,17 @@ class TTxMonEvent_SetFreeze : public TTransactionBase { bool Execute(TTransactionContext& txc, const TActorContext&) override { NIceDb::TNiceDb db(txc.DB); + if (!Prepare(db)) { + return false; + } TNodeInfo* node = Self->FindNode(NodeId); if (node != nullptr) { node->SetFreeze(Freeze); db.Table().Key(NodeId).Update(NIceDb::TUpdate(Freeze)); + NJson::TJsonValue jsonOperation; + jsonOperation["NodeId"] = NodeId; + jsonOperation["Freeze"] = Freeze; + WriteOperation(db, jsonOperation); Response = "{\"NodeId\":" + ToString(NodeId) + ',' + "\"Freeze\":" + (Freeze ? "true" : "false") + "}"; } else { Response = "{\"Error\":\"Node " + ToString(NodeId) + " not found\"}"; @@ -4193,7 +4282,68 @@ class TTxMonEvent_Subactors : public TTransactionBase { } }; +class TTxMonEvent_OperationsLog : public TTransactionBase { +public: + const TActorId Source; + THolder Event; + ui64 MaxCount = 100; + + TTxMonEvent_OperationsLog(const TActorId& source, NMon::TEvRemoteHttpInfo::TPtr& ev, TSelf* hive) + : TBase(hive) + , Source(source) + , Event(ev->Release()) + { + MaxCount = FromStringWithDefault(Event->Cgi().Get("max"), MaxCount); + } + bool Execute(TTransactionContext& txc, const TActorContext& ctx) override { + TStringStream out; + out << ""; + out << ""; + out << ""; + out << ""; + out << ""; + out << ""; + out << ""; + out << ""; + out << ""; + + NIceDb::TNiceDb db(txc.DB); + auto operationsRowset = db.Table().All().Reverse().Select(); + if (!operationsRowset.IsReady()) { + return false; + } + for (ui64 cnt = 0; !operationsRowset.EndOfSet() && cnt < MaxCount; ++cnt) { + TString user = operationsRowset.GetValue(); + out << ""; + out << ""; + out << ""; + out << ""; + out << ""; + if (!operationsRowset.Next()) { + return false; + } + } + out << ""; + out << "
TimestampUserDescription
" << TInstant::MilliSeconds(operationsRowset.GetValue()) << "" << (user.empty() ? "anonymous" : user.c_str()) << ""; + out << operationsRowset.GetValue(); + out << "
"; + out << ""; + ctx.Send(Source, new NMon::TEvRemoteHttpInfoRes(out.Str())); + return true; + } + + void Complete(const TActorContext&) override {} +}; void THive::CreateEvMonitoring(NMon::TEvRemoteHttpInfo::TPtr& ev, const TActorContext& ctx) { if (!ReadyForConnections) { return Execute(new TTxMonEvent_NotReady(ev->Sender, this), ctx); @@ -4211,11 +4361,11 @@ void THive::CreateEvMonitoring(NMon::TEvRemoteHttpInfo::TPtr& ev, const TActorCo return Execute(new TTxMonEvent_DbState(ev->Sender, this), ctx); if (page == "SetDown") { TNodeId nodeId = FromStringWithDefault(cgi.Get("node"), 0); - return Execute(new TTxMonEvent_SetDown(ev->Sender, nodeId, FromStringWithDefault(cgi.Get("down"), 0) != 0, this), ctx); + return Execute(new TTxMonEvent_SetDown(ev->Sender, nodeId, FromStringWithDefault(cgi.Get("down"), 0) != 0, this, ev), ctx); } if (page == "SetFreeze") { TNodeId nodeId = FromStringWithDefault(cgi.Get("node"), 0); - return Execute(new TTxMonEvent_SetFreeze(ev->Sender, nodeId, FromStringWithDefault(cgi.Get("freeze"), 0) != 0, this), ctx); + return Execute(new TTxMonEvent_SetFreeze(ev->Sender, nodeId, FromStringWithDefault(cgi.Get("freeze"), 0) != 0, this, ev), ctx); } if (page == "KickNode") { TNodeId nodeId = FromStringWithDefault(cgi.Get("node"), 0); @@ -4328,6 +4478,9 @@ void THive::CreateEvMonitoring(NMon::TEvRemoteHttpInfo::TPtr& ev, const TActorCo if (page == "TabletAvailability") { return Execute(new TTxMonEvent_TabletAvailability(ev->Sender, ev, this), ctx); } + if (page == "OperationsLog") { + return Execute(new TTxMonEvent_OperationsLog(ev->Sender, ev, this), ctx); + } return Execute(new TTxMonEvent_Landing(ev->Sender, ev, this), ctx); } diff --git a/ydb/core/mind/hive/monitoring.h b/ydb/core/mind/hive/monitoring.h new file mode 100644 index 000000000000..da1ceb4a23ae --- /dev/null +++ b/ydb/core/mind/hive/monitoring.h @@ -0,0 +1,23 @@ +#include +#include +#include +#include +#include + +namespace NKikimr { +namespace NHive { + +class TLoggedMonTransaction { +private: + TInstant Timestamp; + TString User; + +protected: + TLoggedMonTransaction(const NMon::TEvRemoteHttpInfo::TPtr& ev); + + bool Prepare(NIceDb::TNiceDb& db); + void WriteOperation(NIceDb::TNiceDb& db, const NJson::TJsonValue& op); +}; + +} +} diff --git a/ydb/core/mind/hive/ya.make b/ydb/core/mind/hive/ya.make index 4580b4909241..5d4132950cd9 100644 --- a/ydb/core/mind/hive/ya.make +++ b/ydb/core/mind/hive/ya.make @@ -25,6 +25,7 @@ SRCS( leader_tablet_info.h metrics.h monitoring.cpp + monitoring.h node_info.cpp node_info.h object_distribution.h @@ -82,6 +83,7 @@ SRCS( ) PEERDIR( + ydb/library/aclib ydb/library/actors/core ydb/library/actors/interconnect library/cpp/containers/ring_buffer diff --git a/ydb/core/tablet/tablet_monitoring_proxy.cpp b/ydb/core/tablet/tablet_monitoring_proxy.cpp index 30a7c445a89a..212e6a6c96ae 100644 --- a/ydb/core/tablet/tablet_monitoring_proxy.cpp +++ b/ydb/core/tablet/tablet_monitoring_proxy.cpp @@ -35,15 +35,15 @@ class TForwardingActor : public TActorBootstrapped { return NKikimrServices::TActivity::TABLET_FORWARDING_ACTOR; } - TForwardingActor(const TTabletMonitoringProxyConfig& config, ui64 targetTablet, bool forceFollower, const TActorId& sender, const NMonitoring::IMonHttpRequest& request) + TForwardingActor(const TTabletMonitoringProxyConfig& config, ui64 targetTablet, bool forceFollower, const TActorId& sender, const NMonitoring::IMonHttpRequest& request, const TString& userToken) : Config(config) , TargetTablet(targetTablet) , ForceFollower(forceFollower) , Sender(sender) - , Request(ConvertRequestToProtobuf(request)) + , Request(ConvertRequestToProtobuf(request, userToken)) {} - static NActorsProto::TRemoteHttpInfo ConvertRequestToProtobuf(const NMonitoring::IMonHttpRequest& request) { + static NActorsProto::TRemoteHttpInfo ConvertRequestToProtobuf(const NMonitoring::IMonHttpRequest& request, const TString& userToken) { NActorsProto::TRemoteHttpInfo pb; pb.SetMethod(request.GetMethod()); pb.SetPath(TString(request.GetPathInfo())); @@ -70,6 +70,7 @@ class TForwardingActor : public TActorBootstrapped { if (const auto& addr = request.GetRemoteAddr()) { pb.SetRemoteAddr(addr.data(), addr.size()); } + pb.SetUserToken(userToken); return pb; } @@ -279,7 +280,7 @@ TTabletMonitoringProxyActor::Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorCon const TString &tabletIdParam = cgi->Get("FollowerID"); const ui64 tabletId = TryParseTabletId(tabletIdParam); if (tabletId) { - ctx.ExecutorThread.RegisterActor(new TForwardingActor(Config, tabletId, true, ev->Sender, msg->Request)); + ctx.ExecutorThread.RegisterActor(new TForwardingActor(Config, tabletId, true, ev->Sender, msg->Request, msg->UserToken)); return; } } @@ -289,7 +290,7 @@ TTabletMonitoringProxyActor::Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorCon const TString &tabletIdParam = cgi->Get("TabletID"); const ui64 tabletId = TryParseTabletId(tabletIdParam); if (tabletId) { - ctx.ExecutorThread.RegisterActor(new TForwardingActor(Config, tabletId, false, ev->Sender, msg->Request)); + ctx.ExecutorThread.RegisterActor(new TForwardingActor(Config, tabletId, false, ev->Sender, msg->Request, msg->UserToken)); return; } } diff --git a/ydb/library/actors/protos/actors.proto b/ydb/library/actors/protos/actors.proto index 329eb998dfce..788698470aa0 100644 --- a/ydb/library/actors/protos/actors.proto +++ b/ydb/library/actors/protos/actors.proto @@ -34,6 +34,7 @@ message TRemoteHttpInfo { optional bytes PostContent = 8; repeated THeader Headers = 9; optional string RemoteAddr = 7; + optional string UserToken = 10; // for compatibility reasons (incorrect field types merged in 21-4) reserved 5; diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_hive_/flat_hive.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_hive_/flat_hive.schema index 25077ac2ce38..de353533371a 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_hive_/flat_hive.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_hive_/flat_hive.schema @@ -549,6 +549,53 @@ } } }, + { + "TableId": 21, + "TableName": "OperationsLog", + "TableKey": [ + 1 + ], + "ColumnsAdded": [ + { + "ColumnId": 1, + "ColumnName": "Timestamp", + "ColumnType": "Uint64" + }, + { + "ColumnId": 2, + "ColumnName": "User", + "ColumnType": "String" + }, + { + "ColumnId": 3, + "ColumnName": "Operation", + "ColumnType": "String" + } + ], + "ColumnsDropped": [], + "ColumnFamilies": { + "0": { + "Columns": [ + 1, + 2, + 3 + ], + "RoomID": 0, + "Codec": 0, + "InMemory": false, + "Cache": 0, + "Small": 4294967295, + "Large": 4294967295 + } + }, + "Rooms": { + "0": { + "Main": 1, + "Outer": 1, + "Blobs": 1 + } + } + }, { "TableId": 4, "TableName": "Node",