From 36b2aee15c4463422f8375310d9ad3f74e8d109d Mon Sep 17 00:00:00 2001 From: Sergey Belyakov Date: Tue, 4 Jun 2024 14:48:45 +0300 Subject: [PATCH] Improve cost metrics configuration (#5045) --- .../blobstorage/dsproxy/ut_fat/dsproxy_ut.cpp | 1 + .../nodewarden/node_warden_impl.cpp | 25 ++++++ .../blobstorage/nodewarden/node_warden_impl.h | 12 +++ .../nodewarden/node_warden_vdisk.cpp | 23 +++-- .../blobstorage/pdisk/blobstorage_pdisk.h | 8 +- .../pdisk/blobstorage_pdisk_impl.cpp | 8 +- .../pdisk/blobstorage_pdisk_params.cpp | 3 +- .../pdisk/blobstorage_pdisk_params.h | 4 +- .../blobstorage/pdisk/mock/pdisk_mock.cpp | 2 +- ydb/core/blobstorage/ut_blobstorage/lib/env.h | 86 +++++++++++++++++-- .../blobstorage/ut_blobstorage/monitoring.cpp | 29 ++++--- ydb/core/blobstorage/ut_group/main.cpp | 1 + ydb/core/blobstorage/ut_mirror3of4/main.cpp | 1 + ydb/core/blobstorage/ut_vdisk/lib/prepare.cpp | 1 + ydb/core/blobstorage/ut_vdisk2/env.h | 1 + .../vdisk/common/blobstorage_cost_tracker.cpp | 8 +- .../vdisk/common/blobstorage_cost_tracker.h | 23 +++-- .../blobstorage/vdisk/common/vdisk_config.cpp | 2 - .../blobstorage/vdisk/common/vdisk_config.h | 7 +- .../vdisk/common/vdisk_context.cpp | 7 +- .../blobstorage/vdisk/common/vdisk_context.h | 20 +++-- .../vdisk/common/vdisk_performance_params.cpp | 10 +++ .../vdisk/common/vdisk_performance_params.h | 20 +++++ .../vdisk/hullop/blobstorage_hullcompact.h | 8 +- .../localrecovery/localrecovery_public.cpp | 15 ++++ .../repl/blobstorage_hullreplwritesst_ut.cpp | 3 +- .../vdisk/scrub/scrub_actor_pdisk.cpp | 8 +- .../vdisk/skeleton/blobstorage_skeleton.cpp | 22 +++-- .../skeleton/blobstorage_skeletonfront.cpp | 15 +++- .../vdisk/skeleton/skeleton_oos_tracker.cpp | 4 +- ydb/core/protos/config.proto | 52 ++++++++--- 31 files changed, 332 insertions(+), 97 deletions(-) create mode 100644 ydb/core/blobstorage/vdisk/common/vdisk_performance_params.cpp create mode 100644 ydb/core/blobstorage/vdisk/common/vdisk_performance_params.h diff --git a/ydb/core/blobstorage/dsproxy/ut_fat/dsproxy_ut.cpp b/ydb/core/blobstorage/dsproxy/ut_fat/dsproxy_ut.cpp index abd56038b9a2..38b1698bed98 100644 --- a/ydb/core/blobstorage/dsproxy/ut_fat/dsproxy_ut.cpp +++ b/ydb/core/blobstorage/dsproxy/ut_fat/dsproxy_ut.cpp @@ -4252,6 +4252,7 @@ class TBlobStorageProxyTest: public TTestBase { vDiskConfig->GCOnlySynced = false; vDiskConfig->HullCompLevelRateThreshold = 0.1; vDiskConfig->SkeletonFrontQueueBackpressureCheckMsgId = false; + vDiskConfig->UseCostTracker = false; IActor* vDisk = CreateVDisk(vDiskConfig, bsInfo, counters); TActorSetupCmd vDiskSetup(vDisk, TMailboxType::Revolving, 0); diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp index ae71ba2387a1..deb7ebd977b1 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.cpp @@ -169,8 +169,33 @@ void TNodeWarden::Bootstrap() { DsProxyPerPoolCounters = new TDsProxyPerPoolCounters(AppData()->Counters); if (actorSystem && actorSystem->AppData() && actorSystem->AppData()->Icb) { +<<<<<<< HEAD actorSystem->AppData()->Icb->RegisterLocalControl(EnablePutBatching, "BlobStorage_EnablePutBatching"); actorSystem->AppData()->Icb->RegisterLocalControl(EnableVPatch, "BlobStorage_EnableVPatch"); +======= + const TIntrusivePtr& icb = actorSystem->AppData()->Icb; + + icb->RegisterLocalControl(EnablePutBatching, "BlobStorage_EnablePutBatching"); + icb->RegisterLocalControl(EnableVPatch, "BlobStorage_EnableVPatch"); + icb->RegisterSharedControl(EnableLocalSyncLogDataCutting, "VDiskControls.EnableLocalSyncLogDataCutting"); + icb->RegisterSharedControl(EnableSyncLogChunkCompressionHDD, "VDiskControls.EnableSyncLogChunkCompressionHDD"); + icb->RegisterSharedControl(EnableSyncLogChunkCompressionSSD, "VDiskControls.EnableSyncLogChunkCompressionSSD"); + icb->RegisterSharedControl(MaxSyncLogChunksInFlightHDD, "VDiskControls.MaxSyncLogChunksInFlightHDD"); + icb->RegisterSharedControl(MaxSyncLogChunksInFlightSSD, "VDiskControls.MaxSyncLogChunksInFlightSSD"); + + icb->RegisterSharedControl(CostMetricsParametersByMedia[NPDisk::DEVICE_TYPE_ROT].BurstThresholdNs, + "VDiskControls.BurstThresholdNsHDD"); + icb->RegisterSharedControl(CostMetricsParametersByMedia[NPDisk::DEVICE_TYPE_SSD].BurstThresholdNs, + "VDiskControls.BurstThresholdNsSSD"); + icb->RegisterSharedControl(CostMetricsParametersByMedia[NPDisk::DEVICE_TYPE_NVME].BurstThresholdNs, + "VDiskControls.BurstThresholdNsNVME"); + icb->RegisterSharedControl(CostMetricsParametersByMedia[NPDisk::DEVICE_TYPE_ROT].DiskTimeAvailableScale, + "VDiskControls.DiskTimeAvailableScaleHDD"); + icb->RegisterSharedControl(CostMetricsParametersByMedia[NPDisk::DEVICE_TYPE_SSD].DiskTimeAvailableScale, + "VDiskControls.DiskTimeAvailableScaleSSD"); + icb->RegisterSharedControl(CostMetricsParametersByMedia[NPDisk::DEVICE_TYPE_NVME].DiskTimeAvailableScale, + "VDiskControls.DiskTimeAvailableScaleNVME"); +>>>>>>> 208706d808... Improve cost metrics configuration (#5045) } // start replication broker diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.h b/ydb/core/blobstorage/nodewarden/node_warden_impl.h index 1cc803bed339..dbca86a4e7ec 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_impl.h +++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.h @@ -125,6 +125,8 @@ namespace NKikimr::NStorage { TReplQuoter::TPtr ReplNodeRequestQuoter; TReplQuoter::TPtr ReplNodeResponseQuoter; + TCostMetricsParametersByMedia CostMetricsParametersByMedia; + public: struct TGroupRecord; @@ -137,6 +139,16 @@ namespace NKikimr::NStorage { : Cfg(cfg) , EnablePutBatching(Cfg->FeatureFlags.GetEnablePutBatchingForBlobStorage(), false, true) , EnableVPatch(Cfg->FeatureFlags.GetEnableVPatch(), false, true) + , EnableLocalSyncLogDataCutting(0, 0, 1) + , EnableSyncLogChunkCompressionHDD(1, 0, 1) + , EnableSyncLogChunkCompressionSSD(0, 0, 1) + , MaxSyncLogChunksInFlightHDD(10, 1, 1024) + , MaxSyncLogChunksInFlightSSD(10, 1, 1024) + , CostMetricsParametersByMedia({ + TCostMetricsParameters{200}, + TCostMetricsParameters{50}, + TCostMetricsParameters{32}, + }) { Y_ABORT_UNLESS(Cfg->BlobStorageConfig.GetServiceSet().AvailabilityDomainsSize() <= 1); AvailDomainId = 1; diff --git a/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp b/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp index eeea89001a2f..fe5997d9f5d8 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp @@ -174,21 +174,20 @@ namespace NKikimr::NStorage { vdiskConfig->EnableVDiskCooldownTimeout = Cfg->EnableVDiskCooldownTimeout; vdiskConfig->ReplPausedAtStart = Cfg->VDiskReplPausedAtStart; vdiskConfig->EnableVPatch = EnableVPatch; - vdiskConfig->FeatureFlags = Cfg->FeatureFlags; - if (Cfg->BlobStorageConfig.HasCostMetricsSettings()) { - for (auto type : Cfg->BlobStorageConfig.GetCostMetricsSettings().GetVDiskTypes()) { - if (type.HasPDiskType() && deviceType == PDiskTypeToPDiskType(type.GetPDiskType())) { - if (type.HasBurstThresholdNs()) { - vdiskConfig->BurstThresholdNs = type.GetBurstThresholdNs(); - } - if (type.HasDiskTimeAvailableScale()) { - vdiskConfig->DiskTimeAvailableScale = type.GetDiskTimeAvailableScale(); - } - } - } + vdiskConfig->EnableLocalSyncLogDataCutting = EnableLocalSyncLogDataCutting; + if (deviceType == NPDisk::EDeviceType::DEVICE_TYPE_ROT) { + vdiskConfig->EnableSyncLogChunkCompression = EnableSyncLogChunkCompressionHDD; + vdiskConfig->MaxSyncLogChunksInFlight = MaxSyncLogChunksInFlightHDD; + } else { + vdiskConfig->EnableSyncLogChunkCompression = EnableSyncLogChunkCompressionSSD; + vdiskConfig->MaxSyncLogChunksInFlight = MaxSyncLogChunksInFlightSSD; } + vdiskConfig->CostMetricsParametersByMedia = CostMetricsParametersByMedia; + + vdiskConfig->FeatureFlags = Cfg->FeatureFlags; + // issue initial report to whiteboard before creating actor to avoid races Send(WhiteboardId, new NNodeWhiteboard::TEvWhiteboard::TEvVDiskStateUpdate(vdiskId, groupInfo->GetStoragePoolName(), vslotId.PDiskId, vslotId.VDiskSlotId, pdiskGuid, kind, donorMode, whiteboardInstanceGuid, std::move(donors))); diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h index 2c1654a83b36..f0a07545df36 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h @@ -173,7 +173,7 @@ struct TEvYardInitResult : public TEventLocal ownedChunks, - const TString &errorReason) + EDeviceType trueMediaType, const TString &errorReason) : Status(status) , StatusFlags(statusFlags) , PDiskParams(new TPDiskParams( @@ -196,8 +196,8 @@ struct TEvYardInitResult : public TEventLocal(), error.Str())); + GetStatusFlags(OwnerSystem, evYardInit.OwnerGroupType), TVector(), + Cfg->RetrieveDeviceType(), error.Str())); Mon.YardInit.CountResponse(); } @@ -1816,7 +1817,8 @@ bool TPDisk::YardInitForKnownVDisk(TYardInit &evYardInit, TOwner owner) { DriveModel.SeekTimeNs() / 1000ull, DriveModel.Speed(TDriveModel::OP_TYPE_READ), DriveModel.Speed(TDriveModel::OP_TYPE_WRITE), readBlockSize, writeBlockSize, DriveModel.BulkWriteBlockSize(), GetUserAccessibleChunkSize(), GetChunkAppendBlockSize(), owner, - ownerRound, GetStatusFlags(OwnerSystem, evYardInit.OwnerGroupType), ownedChunks, nullptr)); + ownerRound, GetStatusFlags(OwnerSystem, evYardInit.OwnerGroupType), ownedChunks, + Cfg->RetrieveDeviceType(), nullptr)); GetStartingPoints(owner, result->StartingPoints); ownerData.VDiskId = vDiskId; ownerData.CutLogId = evYardInit.CutLogId; @@ -1967,7 +1969,7 @@ void TPDisk::YardInitFinish(TYardInit &evYardInit) { DriveModel.Speed(TDriveModel::OP_TYPE_WRITE), readBlockSize, writeBlockSize, DriveModel.BulkWriteBlockSize(), GetUserAccessibleChunkSize(), GetChunkAppendBlockSize(), owner, ownerRound, GetStatusFlags(OwnerSystem, evYardInit.OwnerGroupType) | ui32(NKikimrBlobStorage::StatusNewOwner), TVector(), - nullptr)); + Cfg->RetrieveDeviceType(), nullptr)); GetStartingPoints(result->PDiskParams->Owner, result->StartingPoints); WriteSysLogRestorePoint(new TCompletionEventSender( this, evYardInit.Sender, result.Release(), Mon.YardInit.Results), evYardInit.ReqId, {}); diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_params.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_params.cpp index b8564ea0305e..4e7078f9a8d9 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_params.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_params.cpp @@ -11,7 +11,7 @@ namespace NKikimr { //////////////////////////////////////////////////////////////////////////// TPDiskParams::TPDiskParams(NPDisk::TOwner owner, ui64 ownerRound, ui32 chunkSize, ui32 appendBlockSize, ui64 seekTimeUs, ui64 readSpeedBps, ui64 writeSpeedBps, ui64 readBlockSize, - ui64 writeBlockSize, ui64 bulkWriteBlockSize) + ui64 writeBlockSize, ui64 bulkWriteBlockSize, NPDisk::EDeviceType trueMediaType) : Owner(owner) , OwnerRound(ownerRound) , ChunkSize(chunkSize) @@ -25,6 +25,7 @@ namespace NKikimr { , BulkWriteBlockSize(bulkWriteBlockSize) , PrefetchSizeBytes(CalculatePrefetchSizeBytes(seekTimeUs, readSpeedBps)) , GlueRequestDistanceBytes(CalculateGlueRequestDistanceBytes(seekTimeUs, readSpeedBps)) + , TrueMediaType(trueMediaType) { Y_DEBUG_ABORT_UNLESS(AppendBlockSize <= ChunkSize); } diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_params.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_params.h index 7e5017d87916..175bd3ec0ddd 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_params.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_params.h @@ -27,13 +27,15 @@ namespace NKikimr { const ui64 PrefetchSizeBytes; // Pdisk is expected to stream data of this size at 83% of max speed. const ui64 GlueRequestDistanceBytes; // It is faster to read unneeded data of this size than to seek over it. + const NPDisk::EDeviceType TrueMediaType; + static ui32 CalculateRecommendedReadSize(ui64 seekTimeUs, ui64 readSpeedBps, ui64 appendBlockSize); static ui64 CalculatePrefetchSizeBytes(ui64 seekTimeUs, ui64 readSpeedBps); static ui64 CalculateGlueRequestDistanceBytes(ui64 seekTimeUs, ui64 readSpeedBps); TPDiskParams(NPDisk::TOwner owner, ui64 ownerRound, ui32 chunkSize, ui32 appendBlockSize, ui64 seekTimeUs, ui64 readSpeedBps, ui64 writeSpeedBps, ui64 readBlockSize, - ui64 writeBlockSize, ui64 bulkWriteBlockSize); + ui64 writeBlockSize, ui64 bulkWriteBlockSize, NPDisk::EDeviceType trueMediaType); void OutputHtml(IOutputStream &str) const; TString ToString() const; }; diff --git a/ydb/core/blobstorage/pdisk/mock/pdisk_mock.cpp b/ydb/core/blobstorage/pdisk/mock/pdisk_mock.cpp index 74854853dbcd..ff6fc007ce9a 100644 --- a/ydb/core/blobstorage/pdisk/mock/pdisk_mock.cpp +++ b/ydb/core/blobstorage/pdisk/mock/pdisk_mock.cpp @@ -427,7 +427,7 @@ class TPDiskMockActor : public TActorBootstrapped { const ui64 bulkWriteBlockSize = 65536; res = std::make_unique(NKikimrProto::OK, seekTimeUs, readSpeedBps, writeSpeedBps, readBlockSize, writeBlockSize, bulkWriteBlockSize, Impl.ChunkSize, Impl.AppendBlockSize, ownerId, - owner->OwnerRound, GetStatusFlags(), std::move(ownedChunks), TString()); + owner->OwnerRound, GetStatusFlags(), std::move(ownedChunks), NPDisk::DEVICE_TYPE_NVME, TString()); res->StartingPoints = owner->StartingPoints; } else { res = std::make_unique(NKikimrProto::INVALID_ROUND, "invalid owner round"); diff --git a/ydb/core/blobstorage/ut_blobstorage/lib/env.h b/ydb/core/blobstorage/ut_blobstorage/lib/env.h index 913b8191016a..4914ed70a713 100644 --- a/ydb/core/blobstorage/ut_blobstorage/lib/env.h +++ b/ydb/core/blobstorage/ut_blobstorage/lib/env.h @@ -23,6 +23,10 @@ struct TEnvironmentSetup { std::set CommencedReplication; std::unordered_map Cache; + using TIcbControlKey = std::pair; // { nodeId, name } + + std::unordered_map IcbControls; + struct TSettings { const ui32 NodeCount = 9; const bool VDiskReplPausedAtStart = false; @@ -39,7 +43,12 @@ struct TEnvironmentSetup { const bool SuppressCompatibilityCheck = false; const TFeatureFlags FeatureFlags; const NPDisk::EDeviceType DiskType = NPDisk::EDeviceType::DEVICE_TYPE_NVME; +<<<<<<< HEAD const ui32 BurstThresholdNs = 0; +======= + const ui64 BurstThresholdNs = 0; + const ui32 MinHugeBlobInBytes = 0; +>>>>>>> 208706d808... Improve cost metrics configuration (#5045) const float DiskTimeAvailableScale = 1; }; @@ -326,14 +335,24 @@ struct TEnvironmentSetup { } config->FeatureFlags = Settings.FeatureFlags; - { - auto* type = config->BlobStorageConfig.MutableCostMetricsSettings()->AddVDiskTypes(); - type->SetPDiskType(NKikimrBlobStorage::EPDiskType::ROT); - if (Settings.BurstThresholdNs) { - type->SetBurstThresholdNs(Settings.BurstThresholdNs); - } - type->SetDiskTimeAvailableScale(Settings.DiskTimeAvailableScale); + TAppData* appData = Runtime->GetNode(nodeId)->AppData.get(); + +#define ADD_ICB_CONTROL(controlName, defaultVal, minVal, maxVal, currentValue) { \ + TControlWrapper control(defaultVal, minVal, maxVal); \ + appData->Icb->RegisterSharedControl(control, controlName); \ + control = currentValue; \ + IcbControls.insert({{nodeId, controlName}, std::move(control)}); \ + } + + if (Settings.BurstThresholdNs) { + ADD_ICB_CONTROL("VDiskControls.BurstThresholdNsHDD", 200'000'000, 1, 1'000'000'000'000, Settings.BurstThresholdNs); + ADD_ICB_CONTROL("VDiskControls.BurstThresholdNsSSD", 50'000'000, 1, 1'000'000'000'000, Settings.BurstThresholdNs); + ADD_ICB_CONTROL("VDiskControls.BurstThresholdNsNVME", 32'000'000, 1, 1'000'000'000'000, Settings.BurstThresholdNs); } + ADD_ICB_CONTROL("VDiskControls.DiskTimeAvailableScaleHDD", 1'000, 1, 1'000'000, std::round(Settings.DiskTimeAvailableScale * 1'000)); + ADD_ICB_CONTROL("VDiskControls.DiskTimeAvailableScaleSSD", 1'000, 1, 1'000'000, std::round(Settings.DiskTimeAvailableScale * 1'000)); + ADD_ICB_CONTROL("VDiskControls.DiskTimeAvailableScaleNVME", 1'000, 1, 1'000'000, std::round(Settings.DiskTimeAvailableScale * 1'000)); +#undef ADD_ICB_CONTROL warden.reset(CreateBSNodeWarden(config)); } @@ -641,6 +660,19 @@ struct TEnvironmentSetup { }); } +<<<<<<< HEAD +======= + void PutBlob(const ui32 groupId, const TLogoBlobID& blobId, const TString& part) { + TActorId edge = Runtime->AllocateEdgeActor(Settings.ControllerNodeId); + Runtime->WrapInActorContext(edge, [&] { + SendToBSProxy(edge, groupId, new TEvBlobStorage::TEvPut(blobId, part, TInstant::Max(), + NKikimrBlobStorage::TabletLog, TEvBlobStorage::TEvPut::TacticMaxThroughput)); + }); + auto res = WaitForEdgeActorEvent(edge); + Y_ABORT_UNLESS(res->Get()->Status == NKikimrProto::OK); + } + +>>>>>>> 208706d808... Improve cost metrics configuration (#5045) void CommenceReplication() { for (ui32 groupId : GetGroups()) { auto info = GetGroupInfo(groupId); @@ -831,4 +863,44 @@ struct TEnvironmentSetup { return SyncQueryFactory(actorId, [&] { return std::make_unique(args...); }); } + ui64 AggregateVDiskCounters(TString storagePool, ui32 nodesCount, ui32 groupSize, ui32 groupId, + const std::vector& pdiskLayout, TString subsystem, TString counter, bool derivative = false) { + ui64 ctr = 0; + + for (ui32 nodeId = 1; nodeId <= nodesCount; ++nodeId) { + auto* appData = Runtime->GetNode(nodeId)->AppData.get(); + for (ui32 i = 0; i < groupSize; ++i) { + TStringStream ss; + ss << LeftPad(i, 2, '0'); + TString orderNumber = ss.Str(); + ss.Clear(); + ss << LeftPad(pdiskLayout[i], 9, '0'); + TString pdisk = ss.Str(); + ctr += GetServiceCounters(appData->Counters, "vdisks")-> + GetSubgroup("storagePool", storagePool)-> + GetSubgroup("group", std::to_string(groupId))-> + GetSubgroup("orderNumber", orderNumber)-> + GetSubgroup("pdisk", pdisk)-> + GetSubgroup("media", "rot")-> + GetSubgroup("subsystem", subsystem)-> + GetCounter(counter, derivative)->Val(); + } + } + return ctr; + }; + + void SetIcbControl(ui32 nodeId, TString controlName, ui64 value) { + if (nodeId == 0) { + for (nodeId = 1; nodeId <= Settings.NodeCount; ++nodeId) { + auto it = IcbControls.find({nodeId, controlName}); + Y_ABORT_UNLESS(it != IcbControls.end()); + it->second = value; + } + } else { + auto it = IcbControls.find({nodeId, controlName}); + Y_ABORT_UNLESS(it != IcbControls.end()); + it->second = value; + } + } + }; diff --git a/ydb/core/blobstorage/ut_blobstorage/monitoring.cpp b/ydb/core/blobstorage/ut_blobstorage/monitoring.cpp index 3e550d349b1a..b2011f64927b 100644 --- a/ydb/core/blobstorage/ut_blobstorage/monitoring.cpp +++ b/ydb/core/blobstorage/ut_blobstorage/monitoring.cpp @@ -457,21 +457,22 @@ Y_UNIT_TEST_SUITE(BurstDetection) { } void TestDiskTimeAvailableScaling() { - auto measure = [](float scale) { - TBlobStorageGroupInfo::TTopology topology(TBlobStorageGroupType::ErasureNone, 1, 1, 1, true); - std::unique_ptr env; - ui32 groupSize; - TBlobStorageGroupType groupType; - ui32 groupId; - std::vector pdiskLayout; - SetupEnv(topology, env, groupSize, groupType, groupId, pdiskLayout, 0, scale); - - return AggregateVDiskCounters(env, env->StoragePoolName, groupSize, groupId, pdiskLayout, - "advancedCost", "DiskTimeAvailable"); - }; + TBlobStorageGroupInfo::TTopology topology(TBlobStorageGroupType::ErasureNone, 1, 1, 1, true); + std::unique_ptr env; + ui32 groupSize; + TBlobStorageGroupType groupType; + ui32 groupId; + std::vector pdiskLayout; + SetupEnv(topology, env, groupSize, groupType, groupId, pdiskLayout, 0, 1); + + i64 test1 = env->AggregateVDiskCounters(env->StoragePoolName, groupSize, groupSize, groupId, pdiskLayout, + "advancedCost", "DiskTimeAvailable"); + + env->SetIcbControl(0, "VDiskControls.DiskTimeAvailableScaleNVME", 2'000); + env->Sim(TDuration::Minutes(5)); - i64 test1 = measure(1); - i64 test2 = measure(2); + i64 test2 = env->AggregateVDiskCounters(env->StoragePoolName, groupSize, groupSize, groupId, pdiskLayout, + "advancedCost", "DiskTimeAvailable"); i64 delta = test1 * 2 - test2; diff --git a/ydb/core/blobstorage/ut_group/main.cpp b/ydb/core/blobstorage/ut_group/main.cpp index 81b1266c5bdc..a7e9e12a6702 100644 --- a/ydb/core/blobstorage/ut_group/main.cpp +++ b/ydb/core/blobstorage/ut_group/main.cpp @@ -417,6 +417,7 @@ class TTestEnv { TString()); auto vdiskConfig = AllVDiskKinds->MakeVDiskConfig(baseInfo); vdiskConfig->EnableVDiskCooldownTimeout = true; + vdiskConfig->UseCostTracker = false; auto counters = Counters->GetSubgroup("node", ToString(disk.NodeId))->GetSubgroup("vdisk", disk.VDiskId.ToString()); const TActorId& actorId = runtime.Register(CreateVDisk(vdiskConfig, Info, counters), TActorId(), 0, std::nullopt, disk.NodeId); runtime.RegisterService(disk.VDiskActorId, actorId); diff --git a/ydb/core/blobstorage/ut_mirror3of4/main.cpp b/ydb/core/blobstorage/ut_mirror3of4/main.cpp index b8f950d70ffe..b7fc225d63d0 100644 --- a/ydb/core/blobstorage/ut_mirror3of4/main.cpp +++ b/ydb/core/blobstorage/ut_mirror3of4/main.cpp @@ -273,6 +273,7 @@ class TTestEnv { "" ); const auto vcfg = MakeIntrusive(baseInfo); + vcfg->UseCostTracker = false; auto vdiskCounters = Counters->GetSubgroup("vdisk", ToString(i)); runtime.RegisterService(info.GetActorId(i), runtime.Register(CreateVDisk(vcfg, Info, vdiskCounters), TActorId(), 0, std::nullopt, pdisk.NodeId)); diff --git a/ydb/core/blobstorage/ut_vdisk/lib/prepare.cpp b/ydb/core/blobstorage/ut_vdisk/lib/prepare.cpp index a52190e897c9..7a3ef866095a 100644 --- a/ydb/core/blobstorage/ut_vdisk/lib/prepare.cpp +++ b/ydb/core/blobstorage/ut_vdisk/lib/prepare.cpp @@ -256,6 +256,7 @@ bool TDefaultVDiskSetup::SetUp(TAllVDisks::TVDiskInstance &vdisk, TAllPDisks *pd modifier(vdisk.Cfg.Get()); } vdisk.Cfg->RunRepl = runRepl; + vdisk.Cfg->UseCostTracker = false; return true; } diff --git a/ydb/core/blobstorage/ut_vdisk2/env.h b/ydb/core/blobstorage/ut_vdisk2/env.h index f1019a874082..608dc28b3965 100644 --- a/ydb/core/blobstorage/ut_vdisk2/env.h +++ b/ydb/core/blobstorage/ut_vdisk2/env.h @@ -112,6 +112,7 @@ namespace NKikimr { NPDisk::DEVICE_TYPE_SSD, VSlotId, NKikimrBlobStorage::TVDiskKind::Default, 1, "static"); VDiskConfig = AllVDiskKinds->MakeVDiskConfig(baseInfo); + VDiskConfig->UseCostTracker = false; // create and register actor std::unique_ptr vdisk(NKikimr::CreateVDisk(VDiskConfig, Info, Counters->GetSubgroup("subsystem", "vdisk"))); diff --git a/ydb/core/blobstorage/vdisk/common/blobstorage_cost_tracker.cpp b/ydb/core/blobstorage/vdisk/common/blobstorage_cost_tracker.cpp index f4a0ec15f4db..5bb82b231d6f 100644 --- a/ydb/core/blobstorage/vdisk/common/blobstorage_cost_tracker.cpp +++ b/ydb/core/blobstorage/vdisk/common/blobstorage_cost_tracker.cpp @@ -42,8 +42,8 @@ class TBsCostModelMirror3of4 : public TBsCostModelBase { }; TBsCostTracker::TBsCostTracker(const TBlobStorageGroupType& groupType, NPDisk::EDeviceType diskType, - const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters, ui64 burstThresholdNs, - float diskTimeAvailableScale) + const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters, + const TCostMetricsParameters& costMetricsParameters) : GroupType(groupType) , CostCounters(counters->GetSubgroup("subsystem", "advancedCost")) , UserDiskCost(CostCounters->GetCounter("UserDiskCost", true)) @@ -54,8 +54,10 @@ TBsCostTracker::TBsCostTracker(const TBlobStorageGroupType& groupType, NPDisk::E , DiskTimeAvailableCtr(CostCounters->GetCounter("DiskTimeAvailable", false)) , BucketCapacity(burstThresholdNs * diskTimeAvailableScale / GroupType.BlobSubgroupSize()) , Bucket(&DiskTimeAvailable, &BucketCapacity, nullptr, nullptr, nullptr, nullptr, true) - , DiskTimeAvailableScale(diskTimeAvailableScale) + , BurstThresholdNs(costMetricsParameters.BurstThresholdNs) + , DiskTimeAvailableScale(costMetricsParameters.DiskTimeAvailableScale) { + AtomicSet(BucketCapacity, GetDiskTimeAvailableScale() * BurstThresholdNs); BurstDetector.Initialize(CostCounters, "BurstDetector"); switch (GroupType.GetErasure()) { case TBlobStorageGroupType::ErasureMirror3dc: diff --git a/ydb/core/blobstorage/vdisk/common/blobstorage_cost_tracker.h b/ydb/core/blobstorage/vdisk/common/blobstorage_cost_tracker.h index 115392cc9ab5..6832df748826 100644 --- a/ydb/core/blobstorage/vdisk/common/blobstorage_cost_tracker.h +++ b/ydb/core/blobstorage/vdisk/common/blobstorage_cost_tracker.h @@ -4,6 +4,8 @@ #include "vdisk_costmodel.h" #include "vdisk_events.h" #include "vdisk_handle_class.h" +#include "vdisk_mongroups.h" +#include "vdisk_performance_params.h" #include #include @@ -319,18 +321,20 @@ class TBsCostTracker { ::NMonitoring::TDynamicCounters::TCounterPtr InternalDiskCost; ::NMonitoring::TDynamicCounters::TCounterPtr DiskTimeAvailableCtr; - TAtomic BucketCapacity; // 10^9 nsec + TAtomic BucketCapacity = 1'000'000'000; // 10^9 nsec TAtomic DiskTimeAvailable = 1'000'000'000; TBucketQuoter> Bucket; TLight BurstDetector; std::atomic SeqnoBurstDetector = 0; static constexpr ui32 ConcurrentHugeRequestsAllowed = 3; - float DiskTimeAvailableScale = 1; + + TMemorizableControlWrapper BurstThresholdNs; + TMemorizableControlWrapper DiskTimeAvailableScale; public: TBsCostTracker(const TBlobStorageGroupType& groupType, NPDisk::EDeviceType diskType, - const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters, ui64 burstThresholdNs, - float diskTimeAvailableScale); + const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters, + const TCostMetricsParameters& costMetricsParameters); template ui64 GetCost(const TEv& ev) const { @@ -351,12 +355,14 @@ class TBsCostTracker { } void CountRequest(ui64 cost) { + AtomicSet(BucketCapacity, GetDiskTimeAvailableScale() * BurstThresholdNs.Update(TAppData::TimeProvider->Now())); Bucket.UseAndFill(cost); BurstDetector.Set(!Bucket.IsAvail(), SeqnoBurstDetector.fetch_add(1)); } - void SetTimeAvailable(ui32 diskTimeAvailableNSec) { - ui64 diskTimeAvailable = diskTimeAvailableNSec * DiskTimeAvailableScale; + void SetTimeAvailable(ui64 diskTimeAvailableNSec) { + ui64 diskTimeAvailable = diskTimeAvailableNSec * GetDiskTimeAvailableScale(); + AtomicSet(DiskTimeAvailable, diskTimeAvailable); *DiskTimeAvailableCtr = diskTimeAvailable; } @@ -410,6 +416,11 @@ class TBsCostTracker { void CountPDiskResponse() { BurstDetector.Set(!Bucket.IsAvail(), SeqnoBurstDetector.fetch_add(1)); } + +private: + float GetDiskTimeAvailableScale() { + return 0.001 * DiskTimeAvailableScale.Update(TAppData::TimeProvider->Now()); + } }; } // NKikimr diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp index 18f5e5735aef..618b384591ce 100644 --- a/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp +++ b/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp @@ -120,8 +120,6 @@ namespace NKikimr { BarrierValidation = true; // switch by default on debug builds #endif - BurstThresholdNs = NPDisk::DevicePerformance.at(baseInfo.DeviceType).BurstThresholdNs; - DiskTimeAvailableScale = 1; } void TVDiskConfig::SetupHugeBytes() { diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_config.h b/ydb/core/blobstorage/vdisk/common/vdisk_config.h index a2d5bd28b191..4fed5f2e0ad8 100644 --- a/ydb/core/blobstorage/vdisk/common/vdisk_config.h +++ b/ydb/core/blobstorage/vdisk/common/vdisk_config.h @@ -1,5 +1,8 @@ #pragma once #include "defs.h" + +#include "vdisk_performance_params.h" + #include #include #include @@ -211,8 +214,8 @@ namespace NKikimr { TControlWrapper EnableVPatch = true; ///////////// COST METRICS SETTINGS //////////////// - ui64 BurstThresholdNs = 1'000'000'000; - float DiskTimeAvailableScale = 1; + bool UseCostTracker = true; + TCostMetricsParametersByMedia CostMetricsParametersByMedia; ///////////// FEATURE FLAGS //////////////////////// NKikimrConfig::TFeatureFlags FeatureFlags; diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_context.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_context.cpp index 7afc10652dfb..d1dcbccaec1a 100644 --- a/ydb/core/blobstorage/vdisk/common/vdisk_context.cpp +++ b/ydb/core/blobstorage/vdisk/common/vdisk_context.cpp @@ -30,9 +30,7 @@ namespace NKikimr { TReplQuoter::TPtr replPDiskReadQuoter, TReplQuoter::TPtr replPDiskWriteQuoter, TReplQuoter::TPtr replNodeRequestQuoter, - TReplQuoter::TPtr replNodeResponseQuoter, - ui64 burstThresholdNs, - float diskTimeAvailableScale) + TReplQuoter::TPtr replNodeResponseQuoter) : TBSProxyContext(vdiskCounters->GetSubgroup("subsystem", "memhull")) , VDiskActorId(vdiskActorId) , Top(std::move(top)) @@ -59,8 +57,7 @@ namespace NKikimr { , ReplPDiskWriteQuoter(std::move(replPDiskWriteQuoter)) , ReplNodeRequestQuoter(std::move(replNodeRequestQuoter)) , ReplNodeResponseQuoter(std::move(replNodeResponseQuoter)) - , CostTracker(std::make_shared(Top->GType, type, vdiskCounters, burstThresholdNs, - diskTimeAvailableScale)) + , CostTracker() , OutOfSpaceState(Top->GetTotalVDisksNum(), Top->GetOrderNumber(ShortSelfVDisk)) , CostMonGroup(vdiskCounters, "subsystem", "cost") , Logger(as ? ActorSystemLogger(as) : DevNullLogger()) diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_context.h b/ydb/core/blobstorage/vdisk/common/vdisk_context.h index cdf099729f32..e1ac1338599a 100644 --- a/ydb/core/blobstorage/vdisk/common/vdisk_context.h +++ b/ydb/core/blobstorage/vdisk/common/vdisk_context.h @@ -68,7 +68,7 @@ namespace NKikimr { TString LocalRecoveryErrorStr; std::unique_ptr CostModel; - std::shared_ptr CostTracker; + std::unique_ptr CostTracker; private: // Managing disk space @@ -100,9 +100,7 @@ namespace NKikimr { TReplQuoter::TPtr replPDiskReadQuoter = nullptr, TReplQuoter::TPtr replPDiskWriteQuoter = nullptr, TReplQuoter::TPtr replNodeRequestQuoter = nullptr, - TReplQuoter::TPtr replNodeResponseQuoter = nullptr, - ui64 burstThresholdNs = 1'000'000'000, - float diskTimeAvailableScale = 1); + TReplQuoter::TPtr replNodeResponseQuoter = nullptr); // The function checks response from PDisk. Normally, it's OK. // Other alternatives are: 1) shutdown; 2) FAIL @@ -176,7 +174,9 @@ namespace NKikimr { if (CostModel) { CostMonGroup.DefragCostNs() += CostModel->GetCost(ev); } - CostTracker->CountDefragRequest(ev); + if (CostTracker) { + CostTracker->CountDefragRequest(ev); + } } template @@ -184,7 +184,9 @@ namespace NKikimr { if (CostModel) { CostMonGroup.ScrubCostNs() += CostModel->GetCost(ev); } - CostTracker->CountScrubRequest(ev); + if (CostTracker) { + CostTracker->CountScrubRequest(ev); + } } template @@ -192,12 +194,14 @@ namespace NKikimr { if (CostModel) { CostMonGroup.CompactionCostNs() += CostModel->GetCost(ev); } - CostTracker->CountCompactionRequest(ev); + if (CostTracker) { + CostTracker->CountCompactionRequest(ev); + } } void UpdateCostModel(std::unique_ptr&& newCostModel) { CostModel = std::move(newCostModel); - if (CostModel) { + if (CostModel && CostTracker) { CostTracker->UpdateCostModel(*CostModel); } } diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_performance_params.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_performance_params.cpp new file mode 100644 index 000000000000..14bb5043763f --- /dev/null +++ b/ydb/core/blobstorage/vdisk/common/vdisk_performance_params.cpp @@ -0,0 +1,10 @@ +#include "vdisk_performance_params.h" + +namespace NKikimr { + +TCostMetricsParameters::TCostMetricsParameters(ui64 defaultBurstThresholdMs) + : BurstThresholdNs(defaultBurstThresholdMs * 1'000'000, 1, 1'000'000'000'000) + , DiskTimeAvailableScale(1'000, 1, 1'000'000) +{} + +} // namespace NKikimr diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_performance_params.h b/ydb/core/blobstorage/vdisk/common/vdisk_performance_params.h new file mode 100644 index 000000000000..76e50d42ba0c --- /dev/null +++ b/ydb/core/blobstorage/vdisk/common/vdisk_performance_params.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +#include "defs.h" + +#include +#include + +namespace NKikimr { + +struct TCostMetricsParameters { + TCostMetricsParameters(ui64 defaultBurstThresholdMs = 100); + TControlWrapper BurstThresholdNs; + TControlWrapper DiskTimeAvailableScale; +}; + +using TCostMetricsParametersByMedia = TStackVec; + +} // namespace NKikimr diff --git a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompact.h b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompact.h index c4ab51604470..a064cf4efbd5 100644 --- a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompact.h +++ b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompact.h @@ -170,7 +170,9 @@ namespace NKikimr { // the same logic for every yard response: apply response and restart main cycle void HandleYardResponse(NPDisk::TEvChunkReadResult::TPtr& ev, const TActorContext &ctx) { --PendingResponses; - HullCtx->VCtx->CostTracker->CountPDiskResponse(); + if (HullCtx->VCtx->CostTracker) { + HullCtx->VCtx->CostTracker->CountPDiskResponse(); + } if (ev->Get()->Status != NKikimrProto::CORRUPTED) { CHECK_PDISK_RESPONSE(HullCtx->VCtx, ev, ctx); } @@ -203,7 +205,9 @@ namespace NKikimr { void HandleYardResponse(NPDisk::TEvChunkWriteResult::TPtr& ev, const TActorContext &ctx) { --PendingResponses; - HullCtx->VCtx->CostTracker->CountPDiskResponse(); + if (HullCtx->VCtx->CostTracker) { + HullCtx->VCtx->CostTracker->CountPDiskResponse(); + } CHECK_PDISK_RESPONSE(HullCtx->VCtx, ev, ctx); if (FinalizeIfAborting(ctx)) { return; diff --git a/ydb/core/blobstorage/vdisk/localrecovery/localrecovery_public.cpp b/ydb/core/blobstorage/vdisk/localrecovery/localrecovery_public.cpp index e90483d6a6e3..b6d328d7030f 100644 --- a/ydb/core/blobstorage/vdisk/localrecovery/localrecovery_public.cpp +++ b/ydb/core/blobstorage/vdisk/localrecovery/localrecovery_public.cpp @@ -505,6 +505,21 @@ namespace NKikimr { LocRecCtx->HullDbRecovery = std::make_shared(hullCtx); LocRecCtx->HullCtx = hullCtx; + if (Config->UseCostTracker) { + NPDisk::EDeviceType trueMediaType = LocRecCtx->PDiskCtx->Dsk->TrueMediaType; + if (trueMediaType == NPDisk::DEVICE_TYPE_UNKNOWN) { + // Unable to resolve type from PDisk's properties, using type from VDisk config + trueMediaType = Config->BaseInfo.DeviceType; + } + if (trueMediaType != NPDisk::DEVICE_TYPE_UNKNOWN) { + LocRecCtx->HullCtx->VCtx->CostTracker.reset(new TBsCostTracker( + LocRecCtx->HullCtx->VCtx->Top->GType, trueMediaType, + LocRecCtx->HullCtx->VCtx->VDiskCounters, + Config->CostMetricsParametersByMedia[trueMediaType] + )); + } + } + // store reported owned chunks LocRecCtx->ReportedOwnedChunks = std::move(m->OwnedChunks); diff --git a/ydb/core/blobstorage/vdisk/repl/blobstorage_hullreplwritesst_ut.cpp b/ydb/core/blobstorage/vdisk/repl/blobstorage_hullreplwritesst_ut.cpp index f38d872c6ec0..c2131faba130 100644 --- a/ydb/core/blobstorage/vdisk/repl/blobstorage_hullreplwritesst_ut.cpp +++ b/ydb/core/blobstorage/vdisk/repl/blobstorage_hullreplwritesst_ut.cpp @@ -15,7 +15,8 @@ std::shared_ptr CreateReplCtx(TVector& vdisks, const TIntrus auto vctx = MakeIntrusive(TActorId(), info->PickTopology(), counters, TVDiskID(0, 1, 0, 0, 0), nullptr, NPDisk::DEVICE_TYPE_UNKNOWN); auto hugeBlobCtx = std::make_shared(512u << 10u, nullptr); - auto dsk = MakeIntrusive(ui8(1), 1u, 128u << 20, 4096u, 0u, 1000000000u, 1000000000u, 65536u, 65536u, 65536u); + auto dsk = MakeIntrusive(ui8(1), 1u, 128u << 20, 4096u, 0u, 1000000000u, 1000000000u, 65536u, 65536u, 65536u, + NPDisk::DEVICE_TYPE_UNKNOWN); auto pdiskCtx = std::make_shared(dsk, TActorId(), TString()); auto replCtx = std::make_shared( vctx, diff --git a/ydb/core/blobstorage/vdisk/scrub/scrub_actor_pdisk.cpp b/ydb/core/blobstorage/vdisk/scrub/scrub_actor_pdisk.cpp index e477ce788fc8..22e93924194c 100644 --- a/ydb/core/blobstorage/vdisk/scrub/scrub_actor_pdisk.cpp +++ b/ydb/core/blobstorage/vdisk/scrub/scrub_actor_pdisk.cpp @@ -11,7 +11,9 @@ namespace NKikimr { Send(ScrubCtx->PDiskCtx->PDiskId, msg.release()); CurrentState = TStringBuilder() << "reading data from " << part.ToString(); auto res = WaitForPDiskEvent(); - ScrubCtx->VCtx->CostTracker->CountPDiskResponse(); + if (ScrubCtx->VCtx->CostTracker) { + ScrubCtx->VCtx->CostTracker->CountPDiskResponse(); + } auto *m = res->Get(); Y_VERIFY_S(m->Status == NKikimrProto::OK || m->Status == NKikimrProto::CORRUPTED, "Status# " << NKikimrProto::EReplyStatus_Name(m->Status)); @@ -42,7 +44,9 @@ namespace NKikimr { Send(ScrubCtx->PDiskCtx->PDiskId, msg.release()); CurrentState = TStringBuilder() << "writing index to " << part.ToString(); auto res = WaitForPDiskEvent(); - ScrubCtx->VCtx->CostTracker->CountPDiskResponse(); + if (ScrubCtx->VCtx->CostTracker) { + ScrubCtx->VCtx->CostTracker->CountPDiskResponse(); + } Y_ABORT_UNLESS(res->Get()->Status == NKikimrProto::OK); // FIXME: good logic } diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp index 50e9868e55f6..a8f380b04f92 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp @@ -2153,13 +2153,21 @@ namespace NKikimr { TABLED() {str << "VDiskIncarnationGuid";} TABLED() {str << Db->GetVDiskIncarnationGuid(true);} } - TABLER() { - TABLED() {str << "BurstThresholdNs";} - TABLED() {str << Config->BurstThresholdNs;} - } - TABLER() { - TABLED() {str << "DiskTimeAvailableScale";} - TABLED() {str << Config->DiskTimeAvailableScale;} + + if (PDiskCtx && PDiskCtx->Dsk) { + NPDisk::EDeviceType trueMedia = PDiskCtx->Dsk->TrueMediaType; + TABLER() { + TABLED() {str << "TrueMediaType";} + TABLED() {str << NPDisk::DeviceTypeStr(trueMedia, true); } + } + TABLER() { + TABLED() {str << "BurstThresholdNs";} + TABLED() {str << (i64)Config->CostMetricsParametersByMedia[trueMedia].BurstThresholdNs;} + } + TABLER() { + TABLED() {str << "DiskTimeAvailableScale";} + TABLED() {str << 0.001 * Config->CostMetricsParametersByMedia[trueMedia].DiskTimeAvailableScale;} + } } } } diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp index f8c00b92ff3a..4ad862f3a66f 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp @@ -755,7 +755,7 @@ namespace NKikimr { VCtx = MakeIntrusive(ctx.SelfID, GInfo->PickTopology(), VDiskCounters, SelfVDiskId, ctx.ExecutorThread.ActorSystem, baseInfo.DeviceType, baseInfo.DonorMode, baseInfo.ReplPDiskReadQuoter, baseInfo.ReplPDiskWriteQuoter, baseInfo.ReplNodeRequestQuoter, - baseInfo.ReplNodeResponseQuoter, Config->BurstThresholdNs, Config->DiskTimeAvailableScale); + baseInfo.ReplNodeResponseQuoter); // create IntQueues IntQueueAsyncGets = std::make_unique( @@ -1233,7 +1233,7 @@ namespace NKikimr { void HandleRequestWithQoS(const TActorContext &ctx, TEventPtr &ev, const char *msgName, ui64 cost, TIntQueueClass &intQueue) { CheckEvent(ev, msgName); - const ui64 advancedCost = VCtx->CostTracker->GetCost(*ev->Get()); + const ui64 advancedCost = VCtx->CostTracker ? VCtx->CostTracker->GetCost(*ev->Get()) : 0; const ui32 recByteSize = ev->Get()->GetCachedByteSize(); auto &record = ev->Get()->Record; auto &msgQoS = *record.MutableMsgQoS(); @@ -1281,10 +1281,14 @@ namespace NKikimr { } else { if (clientId.GetType() == NBackpressure::EQueueClientType::DSProxy) { CostGroup.SkeletonFrontUserCostNs() += cost; - VCtx->CostTracker->CountUserCost(advancedCost); + if (VCtx->CostTracker) { + VCtx->CostTracker->CountUserCost(advancedCost); + } } else { CostGroup.SkeletonFrontInternalCostNs() += cost; - VCtx->CostTracker->CountInternalCost(advancedCost); + if (VCtx->CostTracker) { + VCtx->CostTracker->CountInternalCost(advancedCost); + } } } } @@ -1632,6 +1636,9 @@ namespace NKikimr { extQueue.Completed(ctx, msgCtx, event); TIntQueueClass &intQueue = GetIntQueue(msgCtx.IntQueueId); intQueue.Completed(ctx, msgCtx, *this, id); + if (VCtx->CostTracker) { + VCtx->CostTracker->CountPDiskResponse(); + } if (!ev->Get()->DoNotResend) { TActivationContext::Send(event.release()); } diff --git a/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_tracker.cpp b/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_tracker.cpp index d0d5715a01b9..1d0d5cccd045 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_tracker.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_tracker.cpp @@ -94,7 +94,9 @@ namespace NKikimr { if (msg->NumSlots > 0) { ui32 timeAvailable = 1'000'000'000 / msg->NumSlots; CostGroup.DiskTimeAvailableNs() = timeAvailable; - VCtx->CostTracker->SetTimeAvailable(timeAvailable); + if (VCtx->CostTracker) { + VCtx->CostTracker->SetTimeAvailable(timeAvailable); + } } Become(&TThis::WaitFunc); diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index c6cb5c8f8254..00079ebee234 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -281,17 +281,7 @@ message TBlobStorageConfig { optional TAutoconfigSettings AutoconfigSettings = 6; - message TCostMetricsConfig { - optional NKikimrBlobStorage.EPDiskType PDiskType = 1; - optional uint64 BurstThresholdNs = 2; - optional float DiskTimeAvailableScale = 3; - } - - message TCostMetricsSettings { - repeated TCostMetricsConfig VDiskTypes = 1; - }; - - optional TCostMetricsSettings CostMetricsSettings = 7; + reserved 7; // TCostMetricsSettings, moved to ICB } message TBlobStorageFormatConfig { @@ -1307,6 +1297,46 @@ message TImmediateControlsConfig { optional TKeyValue KeyValue = 1; } + message TVDiskControls { + // SyncLog Data cutter options, not merged to 24-1 + reserved 1; + reserved 2; + reserved 3; + reserved 4; + reserved 5; + + optional uint64 BurstThresholdNsHDD = 6 [(ControlOptions) = { + Description: "Minumum operation queue size that is considered a burst, setting for HDD", + MinValue: 1, + MaxValue: 1000000000000, + DefaultValue: 200000000 }]; + optional uint64 BurstThresholdNsSSD = 7 [(ControlOptions) = { + Description: "Minumum operation queue size that is considered a burst, setting for SSD", + MinValue: 1, + MaxValue: 1000000000000, + DefaultValue: 50000000 }]; + optional uint64 BurstThresholdNsNVME = 8 [(ControlOptions) = { + Description: "Minumum operation queue size that is considered a burst, setting for NVME", + MinValue: 1, + MaxValue: 1000000000000, + DefaultValue: 32000000 }]; + optional uint64 DiskTimeAvailableScaleHDD = 9 [(ControlOptions) = { + Description: "Scale coefficient DiskTimeAvailableScale metric, this parameter will be converted to float and divided by 1'000, setting for HDD", + MinValue: 1, + MaxValue: 1000000, + DefaultValue: 1000 }]; + optional uint64 DiskTimeAvailableScaleSSD = 10 [(ControlOptions) = { + Description: "Scale coefficient for DiskTimeAvailableScale metric, this parameter will be converted to float and divided by 1'000, setting for SSD", + MinValue: 1, + MaxValue: 1000000, + DefaultValue: 1000 }]; + optional uint64 DiskTimeAvailableScaleNVME = 11 [(ControlOptions) = { + Description: "Scale coefficient DiskTimeAvailableScale metric, this parameter will be converted to float and divided by 1'000, setting for NVME", + MinValue: 1, + MaxValue: 1000000, + DefaultValue: 1000 }]; + } + optional TDataShardControls DataShardControls = 1; optional TTxLimitControls TxLimitControls = 2; optional TCoordinatorControls CoordinatorControls = 3;