Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

store history of changes made to hive via http #1637

Merged
merged 4 commits into from
Feb 12, 2024
Merged

store history of changes made to hive via http #1637

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9db56be
store history of changes made to hive via http
vporyadke Feb 2, 2024
dad185b
switch to json represenation
vporyadke Feb 9, 2024
0025c6a
add new file
vporyadke Feb 9, 2024
2794150
remove dead code
vporyadke Feb 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion ydb/core/mind/hive/hive_schema.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,15 @@ struct Schema : NIceDb::Schema {
using TColumns = TableColumns<Node, TabletType, MaxCount>;
};

struct OperationsLog : Table<21> {
struct Timestamp : Column<1, NScheme::NTypeIds::Uint64> {};
struct User : Column<2, NScheme::NTypeIds::String> {};
struct Operation : Column<3, NScheme::NTypeIds::String> { using Type = NKikimrHive::TOperation; };
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think you actually need these complex protobuf specs just for logging purposes. And think about migrations and rollbacks - it could became a huge pain...
I think just a free text would be more than enough. (or json if you need some structure to parse)


using TKey = TableKey<Timestamp>;
using TColumns = TableColumns<Timestamp, User, Operation>;
};

using TTables = SchemaTables<
State,
Tablet,
Expand All @@ -313,7 +322,8 @@ struct Schema : NIceDb::Schema {
SubDomain,
BlockedOwner,
TabletOwners,
TabletAvailabilityRestrictions
TabletAvailabilityRestrictions,
OperationsLog
>;
using TSettings = SchemaSettings<
ExecutorLogBatching<true>,
Expand Down
356 changes: 292 additions & 64 deletions ydb/core/mind/hive/monitoring.cpp
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions ydb/core/mind/hive/ya.make
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ SRCS(
leader_tablet_info.h
metrics.h
monitoring.cpp
monitoring.h
node_info.cpp
node_info.h
object_distribution.h
Expand Down Expand Up @@ -82,6 +83,7 @@ SRCS(
)

PEERDIR(
ydb/library/aclib
ydb/library/actors/core
ydb/library/actors/interconnect
library/cpp/containers/ring_buffer
Expand Down
36 changes: 36 additions & 0 deletions ydb/core/protos/hive.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -567,3 +567,39 @@ message TEvUpdateDomainReply {
optional fixed64 Origin = 1;
optional uint64 TxId = 2;
}

message TConfigUpdate {
optional uint64 FieldNumber = 1;
optional string Value = 2;
}

message TConfigUpdates {
repeated TConfigUpdate ConfigUpdate = 1;
}

message TTabletAvailabilityRestriction {
optional NKikimrTabletBase.TTabletTypes.EType Type = 1;
optional uint64 MaxCount = 2 [default = 1000000];
}

message TNodeOperation {
optional uint32 NodeId = 1;
optional bool Down = 2;
optional bool Freeze = 3;
repeated TTabletAvailabilityRestriction TabletAvailability = 4;
}

message TAllowedMetricsUpdate {
optional NKikimrTabletBase.TTabletTypes.EType Type = 1;
repeated uint64 AllowedMetrics = 2;
}

message TAllowedMetricsUpdates {
repeated TAllowedMetricsUpdate AllowedMetricsUpdate = 1;
}

message TOperation {
optional TConfigUpdates ConfigUpdates = 1;
optional TNodeOperation NodeOperation = 2;
optional TAllowedMetricsUpdates AllowedMetricsUpdates = 3;
}
11 changes: 6 additions & 5 deletions ydb/core/tablet/tablet_monitoring_proxy.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,15 @@ class TForwardingActor : public TActorBootstrapped<TForwardingActor> {
return NKikimrServices::TActivity::TABLET_FORWARDING_ACTOR;
}

TForwardingActor(const TTabletMonitoringProxyConfig& config, ui64 targetTablet, bool forceFollower, const TActorId& sender, const NMonitoring::IMonHttpRequest& request)
TForwardingActor(const TTabletMonitoringProxyConfig& config, ui64 targetTablet, bool forceFollower, const TActorId& sender, const NMonitoring::IMonHttpRequest& request, const TString& userToken)
: Config(config)
, TargetTablet(targetTablet)
, ForceFollower(forceFollower)
, Sender(sender)
, Request(ConvertRequestToProtobuf(request))
, Request(ConvertRequestToProtobuf(request, userToken))
{}

static NActorsProto::TRemoteHttpInfo ConvertRequestToProtobuf(const NMonitoring::IMonHttpRequest& request) {
static NActorsProto::TRemoteHttpInfo ConvertRequestToProtobuf(const NMonitoring::IMonHttpRequest& request, const TString& userToken) {
NActorsProto::TRemoteHttpInfo pb;
pb.SetMethod(request.GetMethod());
pb.SetPath(TString(request.GetPathInfo()));
Expand All @@ -70,6 +70,7 @@ class TForwardingActor : public TActorBootstrapped<TForwardingActor> {
if (const auto& addr = request.GetRemoteAddr()) {
pb.SetRemoteAddr(addr.data(), addr.size());
}
pb.SetUserToken(userToken);
return pb;
}

Expand Down Expand Up @@ -279,7 +280,7 @@ TTabletMonitoringProxyActor::Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorCon
const TString &tabletIdParam = cgi->Get("FollowerID");
const ui64 tabletId = TryParseTabletId(tabletIdParam);
if (tabletId) {
ctx.ExecutorThread.RegisterActor(new TForwardingActor(Config, tabletId, true, ev->Sender, msg->Request));
ctx.ExecutorThread.RegisterActor(new TForwardingActor(Config, tabletId, true, ev->Sender, msg->Request, msg->UserToken));
return;
}
}
Expand All @@ -289,7 +290,7 @@ TTabletMonitoringProxyActor::Handle(NMon::TEvHttpInfo::TPtr &ev, const TActorCon
const TString &tabletIdParam = cgi->Get("TabletID");
const ui64 tabletId = TryParseTabletId(tabletIdParam);
if (tabletId) {
ctx.ExecutorThread.RegisterActor(new TForwardingActor(Config, tabletId, false, ev->Sender, msg->Request));
ctx.ExecutorThread.RegisterActor(new TForwardingActor(Config, tabletId, false, ev->Sender, msg->Request, msg->UserToken));
return;
}
}
Expand Down
1 change: 1 addition & 0 deletions ydb/library/actors/protos/actors.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ message TRemoteHttpInfo {
optional bytes PostContent = 8;
repeated THeader Headers = 9;
optional string RemoteAddr = 7;
optional string UserToken = 10;
adameat marked this conversation as resolved.
Show resolved Hide resolved

// for compatibility reasons (incorrect field types merged in 21-4)
reserved 5;
Expand Down
47 changes: 47 additions & 0 deletions ...ata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_hive_/flat_hive.schema
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -549,6 +549,53 @@
}
}
},
{
"TableId": 21,
"TableName": "OperationsLog",
"TableKey": [
1
],
"ColumnsAdded": [
{
"ColumnId": 1,
"ColumnName": "Timestamp",
"ColumnType": "Uint64"
},
{
"ColumnId": 2,
"ColumnName": "User",
"ColumnType": "String"
},
{
"ColumnId": 3,
"ColumnName": "Operation",
"ColumnType": "String"
}
],
"ColumnsDropped": [],
"ColumnFamilies": {
"0": {
"Columns": [
1,
2,
3
],
"RoomID": 0,
"Codec": 0,
"InMemory": false,
"Cache": 0,
"Small": 4294967295,
"Large": 4294967295
}
},
"Rooms": {
"0": {
"Main": 1,
"Outer": 1,
"Blobs": 1
}
}
},
{
"TableId": 4,
"TableName": "Node",
Expand Down
Loading