From 17e9397f6a12140e453b44f81142ba52d8165d19 Mon Sep 17 00:00:00 2001 From: Semyon Danilov Date: Thu, 30 May 2024 13:05:27 +0000 Subject: [PATCH 1/2] Fix flaky PDiskRestartManyLogWrites --- .../blobstorage/pdisk/blobstorage_pdisk_actor.cpp | 2 +- ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp | 11 ++++++----- ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_env.h | 10 +++++++--- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp index a72a229a37b6..7e38473de491 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_actor.cpp @@ -1019,7 +1019,7 @@ class TPDiskActor : public TActorBootstrapped { SecureWipeBuffer((ui8*)ev->Get()->MainKey.Keys.data(), sizeof(NPDisk::TKey) * ev->Get()->MainKey.Keys.size()); LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::BS_PDISK, "PDiskId# " << PDisk->PDiskId - << " Going to restart PDisk since recieved TEvAskWardenRestartPDiskResult"); + << " Going to restart PDisk since received TEvAskWardenRestartPDiskResult"); const TActorIdentity& thisActorId = SelfId(); ui32 nodeId = thisActorId.NodeId(); diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp index d9dfd60a358d..35619f78b746 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp @@ -504,7 +504,7 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { vdisk.InitFull(); vdisk.SendEvLogSync(); - testCtx.StartPDiskRestart(); + testCtx.GracefulPDiskRestart(); vdisk.InitFull(); vdisk.SendEvLogSync(); @@ -512,7 +512,6 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { Y_UNIT_TEST(PDiskRestartManyLogWrites) { TActorTestContext testCtx({ false }); - testCtx.TestCtx.SectorMap->ImitateIoErrorProbability = 1e-4; const TVDiskID vDiskID(0, 1, 0, 0, 0); const auto evInitRes = testCtx.TestResponse( @@ -522,14 +521,16 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { //ui32 errors = 0; ui32 lsn = 2; TRcBuf logData = TRcBuf(PrepareData(4096)); + for (ui32 i = 0; i < 1000; ++i) { testCtx.Send(new NPDisk::TEvLog(evInitRes->PDiskParams->Owner, evInitRes->PDiskParams->OwnerRound, 0, logData, TLsnSeg(lsn, lsn), nullptr)); if (i == 100) { - testCtx.Send(new TEvBlobStorage::TEvAskWardenRestartPDiskResult(testCtx.GetPDisk()->PDiskId, testCtx.MainKey, true, nullptr)); + testCtx.GracefulPDiskRestart(false); } ++lsn; } + for (ui32 i = 0; i < 100;) { const auto logRes = testCtx.Recv(); i += logRes->Results.size(); @@ -929,7 +930,7 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { while (writeLog() == NKikimrProto::OK) {} UNIT_ASSERT_VALUES_EQUAL(writeLog(), NKikimrProto::OUT_OF_SPACE); - testCtx.StartPDiskRestart(); + testCtx.GracefulPDiskRestart(); vdisk.InitFull(); vdisk.SendEvLogSync(); @@ -944,7 +945,7 @@ Y_UNIT_TEST_SUITE(PDiskCompatibilityInfo) { THolder RestartPDisk(TActorTestContext& testCtx, ui32 pdiskId, TVDiskMock& vdisk, TCurrent* newInfo) { TCompatibilityInfoTest::Reset(newInfo); Y_UNUSED(pdiskId); - testCtx.StartPDiskRestart(); + testCtx.GracefulPDiskRestart(); testCtx.Send(new NPDisk::TEvYardInit(vdisk.OwnerRound.fetch_add(1), vdisk.VDiskID, testCtx.TestCtx.PDiskGuid)); return testCtx.Recv(); } diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_env.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_env.h index b2883cd29529..f28564e6f7d3 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_env.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut_env.h @@ -30,8 +30,8 @@ struct TActorTestContext { private: std::optional PDiskActor; - THolder Runtime; std::shared_ptr IoContext; + THolder Runtime; NPDisk::TPDisk *PDisk = nullptr; public: @@ -131,11 +131,15 @@ struct TActorTestContext { return PDisk; } - void StartPDiskRestart() { + void GracefulPDiskRestart(bool waitForRestart = true) { ui32 pdiskId = GetPDisk()->PDiskId; Send(new TEvBlobStorage::TEvAskWardenRestartPDiskResult(pdiskId, MainKey, true, nullptr)); - const auto evInitRes = Recv(); + + if (waitForRestart) { + const auto evInitRes = Recv(); + UNIT_ASSERT_VALUES_EQUAL(NKikimrProto::EReplyStatus::OK, evInitRes->Status); + } if (!Settings.UsePDiskMock) { TActorId wellKnownPDiskActorId = MakeBlobStoragePDiskID(PDiskActor->NodeId(), pdiskId); From 9ba9abbd01cc40923da34c1310ebb5984dd91fc0 Mon Sep 17 00:00:00 2001 From: Semyon Danilov Date: Thu, 30 May 2024 18:49:20 +0000 Subject: [PATCH 2/2] . --- ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp index 35619f78b746..9c9454e1bf55 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp @@ -518,7 +518,6 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { new NPDisk::TEvYardInit(2, vDiskID, testCtx.TestCtx.PDiskGuid), NKikimrProto::OK); - //ui32 errors = 0; ui32 lsn = 2; TRcBuf logData = TRcBuf(PrepareData(4096)); @@ -528,6 +527,10 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { if (i == 100) { testCtx.GracefulPDiskRestart(false); } + if (i == 600) { + const auto evInitRes = testCtx.Recv(); + UNIT_ASSERT_VALUES_EQUAL(NKikimrProto::EReplyStatus::OK, evInitRes->Status); + } ++lsn; } @@ -540,7 +543,6 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { Ctest << "TEvLogResult status is error" << Endl; } } - testCtx.Recv(); } Y_UNIT_TEST(CommitDeleteChunks) {