Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

storage healthcheck fixes (#7212) #7394

Merged
merged 2 commits into from
Aug 9, 2024
Merged

storage healthcheck fixes (#7212) #7394

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2092cd6
fix missing static group issue (#7212)
vporyadke Aug 1, 2024
39aead8
do not look at active/inactive pdisk status in healthchceck
vporyadke Aug 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions ydb/core/health_check/health_check.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ struct hash<NKikimrBlobStorage::TVSlotId> {
}

#define BLOG_CRIT(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::HEALTH, stream)
#define BLOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::HEALTH, stream)

namespace NKikimr {

Expand Down Expand Up @@ -643,7 +644,7 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
}

bool NeedWhiteboardInfoForGroup(TGroupId groupId) {
return !HaveAllBSControllerInfo() && IsStaticGroup(groupId);
return UnknownStaticGroups.contains(groupId) || (!HaveAllBSControllerInfo() && IsStaticGroup(groupId));
}

void Handle(TEvNodeWardenStorageConfig::TPtr ev) {
Expand Down Expand Up @@ -678,6 +679,7 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {

auto groupId = vDisk.GetVDiskID().GetGroupID();
if (NeedWhiteboardInfoForGroup(groupId)) {
BLOG_D("Requesting whiteboard for group " << groupId);
RequestStorageNode(vDisk.GetVDiskLocation().GetNodeID());
}
}
Expand Down Expand Up @@ -1308,6 +1310,7 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
// it should not be trusted
Ydb::Monitoring::StorageGroupStatus staticGroupStatus;
FillGroupStatus(0, staticGroupStatus, {nullptr});
BLOG_D("Static group status is " << staticGroupStatus.overall());
if (staticGroupStatus.overall() != Ydb::Monitoring::StatusFlag::GREEN) {
UnknownStaticGroups.emplace(0);
RequestStorageConfig();
Expand Down Expand Up @@ -1712,12 +1715,9 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
ETags::PDiskState);
}
switch (status->number()) {
case NKikimrBlobStorage::ACTIVE: {
context.ReportStatus(Ydb::Monitoring::StatusFlag::GREEN);
break;
}
case NKikimrBlobStorage::ACTIVE:
case NKikimrBlobStorage::INACTIVE: {
context.ReportStatus(Ydb::Monitoring::StatusFlag::YELLOW, "PDisk is inactive", ETags::PDiskState);
context.ReportStatus(Ydb::Monitoring::StatusFlag::GREEN);
break;
}
case NKikimrBlobStorage::FAULTY:
Expand Down Expand Up @@ -2169,7 +2169,7 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
context.OverallStatus = MinStatus(context.OverallStatus, Ydb::Monitoring::StatusFlag::YELLOW);
checker.ReportStatus(context);


BLOG_D("Group " << groupId << " has status " << context.GetOverallStatus());
storageGroupStatus.set_overall(context.GetOverallStatus());
}

Expand Down
47 changes: 39 additions & 8 deletions ydb/core/health_check/health_check_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,12 +163,12 @@ Y_UNIT_TEST_SUITE(THealthCheckTest) {
}

void AddVSlotsToSysViewResponse(NSysView::TEvSysView::TEvGetVSlotsResponse::TPtr* ev, size_t groupCount,
const TVector<NKikimrBlobStorage::EVDiskStatus>& vdiskStatuses) {
const TVector<NKikimrBlobStorage::EVDiskStatus>& vdiskStatuses, ui32 groupStartId = GROUP_START_ID) {
auto& record = (*ev)->Get()->Record;
auto entrySample = record.entries(0);
record.clear_entries();

auto groupId = GROUP_START_ID;
auto groupId = groupStartId;
const auto *descriptor = NKikimrBlobStorage::EVDiskStatus_descriptor();
for (size_t i = 0; i < groupCount; ++i) {
auto vslotId = VCARD_START_ID;
Expand Down Expand Up @@ -252,13 +252,13 @@ Y_UNIT_TEST_SUITE(THealthCheckTest) {
sPool->set_name(STORAGE_POOL_NAME);
};

void AddVSlotInVDiskStateResponse(TEvWhiteboard::TEvVDiskStateResponse::TPtr* ev, int groupCount, int vslotCount) {
void AddVSlotInVDiskStateResponse(TEvWhiteboard::TEvVDiskStateResponse::TPtr* ev, int groupCount, int vslotCount, ui32 groupStartId = GROUP_START_ID) {
auto& pbRecord = (*ev)->Get()->Record;

auto sample = pbRecord.vdiskstateinfo(0);
pbRecord.clear_vdiskstateinfo();

auto groupId = GROUP_START_ID;
auto groupId = groupStartId;
for (int i = 0; i < groupCount; i++) {
auto slotId = VCARD_START_ID;
for (int j = 0; j < vslotCount; j++) {
Expand All @@ -273,6 +273,12 @@ Y_UNIT_TEST_SUITE(THealthCheckTest) {
}
}

void ChangeGroupStateResponse(NNodeWhiteboard::TEvWhiteboard::TEvBSGroupStateResponse::TPtr* ev) {
for (auto& groupInfo : *(*ev)->Get()->Record.mutable_bsgroupstateinfo()) {
groupInfo.set_erasurespecies(NHealthCheck::TSelfCheckRequest::BLOCK_4_2);
}
}

void SetLongHostValue(TEvInterconnect::TEvNodesInfo::TPtr* ev) {
TString host(1000000, 'a');
auto& pbRecord = (*ev)->Get()->Nodes;
Expand Down Expand Up @@ -383,7 +389,7 @@ Y_UNIT_TEST_SUITE(THealthCheckTest) {
CheckHcResult(result, groupNumber, vdiscPerGroupNumber, isMergeRecords);
}

Ydb::Monitoring::SelfCheckResult RequestHcWithVdisks(const NKikimrBlobStorage::TGroupStatus::E groupStatus, const TVector<NKikimrBlobStorage::EVDiskStatus>& vdiskStatuses) {
Ydb::Monitoring::SelfCheckResult RequestHcWithVdisks(const NKikimrBlobStorage::TGroupStatus::E groupStatus, const TVector<NKikimrBlobStorage::EVDiskStatus>& vdiskStatuses, bool forStaticGroup = false) {
TPortManager tp;
ui16 port = tp.GetPort(2134);
ui16 grpcPort = tp.GetPort(2135);
Expand Down Expand Up @@ -418,7 +424,11 @@ Y_UNIT_TEST_SUITE(THealthCheckTest) {
}
case NSysView::TEvSysView::EvGetVSlotsResponse: {
auto* x = reinterpret_cast<NSysView::TEvSysView::TEvGetVSlotsResponse::TPtr*>(&ev);
AddVSlotsToSysViewResponse(x, 1, vdiskStatuses);
if (forStaticGroup) {
AddVSlotsToSysViewResponse(x, 1, vdiskStatuses, 0);
} else {
AddVSlotsToSysViewResponse(x, 1, vdiskStatuses);
}
break;
}
case NSysView::TEvSysView::EvGetGroupsResponse: {
Expand All @@ -431,6 +441,19 @@ Y_UNIT_TEST_SUITE(THealthCheckTest) {
AddStoragePoolsToSysViewResponse(x);
break;
}
case NNodeWhiteboard::TEvWhiteboard::EvVDiskStateResponse: {
auto *x = reinterpret_cast<NNodeWhiteboard::TEvWhiteboard::TEvVDiskStateResponse::TPtr*>(&ev);
if (forStaticGroup) {
AddVSlotInVDiskStateResponse(x, 1, vdiskStatuses.size(), 0);
} else {
AddVSlotInVDiskStateResponse(x, 1, vdiskStatuses.size());
}
break;
}
case NNodeWhiteboard::TEvWhiteboard::EvBSGroupStateResponse: {
auto* x = reinterpret_cast<NNodeWhiteboard::TEvWhiteboard::TEvBSGroupStateResponse::TPtr*>(&ev);
ChangeGroupStateResponse(x);
}
}

return TTestActorRuntime::EEventAction::PROCESS;
Expand All @@ -444,10 +467,12 @@ Y_UNIT_TEST_SUITE(THealthCheckTest) {
return runtime.GrabEdgeEvent<NHealthCheck::TEvSelfCheckResult>(handle)->Result;
}

void CheckHcResultHasIssuesWithStatus(Ydb::Monitoring::SelfCheckResult& result, const TString& type, const Ydb::Monitoring::StatusFlag::Status expectingStatus, ui32 total) {
void CheckHcResultHasIssuesWithStatus(Ydb::Monitoring::SelfCheckResult& result, const TString& type,
const Ydb::Monitoring::StatusFlag::Status expectingStatus, ui32 total,
std::string_view pool = "/Root:test") {
int issuesCount = 0;
for (const auto& issue_log : result.Getissue_log()) {
if (issue_log.type() == type && issue_log.location().storage().pool().name() == "/Root:test" && issue_log.status() == expectingStatus) {
if (issue_log.type() == type && issue_log.location().storage().pool().name() == pool && issue_log.status() == expectingStatus) {
issuesCount++;
}
}
Expand Down Expand Up @@ -589,6 +614,12 @@ Y_UNIT_TEST_SUITE(THealthCheckTest) {
CheckHcResultHasIssuesWithStatus(result, "STORAGE_GROUP", Ydb::Monitoring::StatusFlag::RED, 1);
}

Y_UNIT_TEST(StaticGroupIssue) {
auto result = RequestHcWithVdisks(NKikimrBlobStorage::TGroupStatus::PARTIAL, {NKikimrBlobStorage::ERROR}, /*forStatic*/ true);
Cerr << result.ShortDebugString() << Endl;
CheckHcResultHasIssuesWithStatus(result, "STORAGE_GROUP", Ydb::Monitoring::StatusFlag::YELLOW, 1, "static");
}

/* HC currently infers group status on its own, so it's never unknown
Y_UNIT_TEST(RedGroupIssueWhenUnknownGroupStatus) {
auto result = RequestHcWithVdisks(NKikimrBlobStorage::TGroupStatus::UNKNOWN, {});
Expand Down
Loading