issue-1824: filestore tablet channel balancing by ApproximateFreeSpac…

…eShare (#1858)

cloud/blockstore/libs/storage/partition/part_actor_monitoring.cpp

@@ -124,6 +124,7 @@ void DumpChannels(
             TStringBuilder() << dataKind,
             state.CheckPermissions(c, EChannelPermission::UserWritesAllowed),
             state.CheckPermissions(c, EChannelPermission::SystemWritesAllowed),
+            state.GetFreeSpaceShare(c),
         });
     }
     NCloud::NStorage::DumpChannels(

cloud/blockstore/libs/storage/partition/part_state.cpp

@@ -1,5 +1,7 @@
 #include "part_state.h"
 
+#include <cloud/storage/core/libs/tablet/model/channels.h>
+
 #include <library/cpp/monlib/service/pages/templates.h>
 #include <library/cpp/protobuf/json/proto2json.h>
 
@@ -51,19 +53,6 @@ void DumpOperationState(IOutputStream& out, const TOperationState& op)
 
 ////////////////////////////////////////////////////////////////////////////////
 
-double Normalize(double x, double lo, double hi)
-{
-    if (x > hi) {
-        return 1;
-    }
-
-    if (x < lo) {
-        return 0;
-    }
-
-    return (x - lo) / (hi - lo);
-}
-
 double BPFeature(const TBackpressureFeatureConfig& c, double x)
 {
     auto nx = Normalize(x, c.InputThreshold, c.InputLimit);
@@ -248,6 +237,12 @@ bool TPartitionState::CheckPermissions(ui32 channel, EChannelPermissions permiss
     return ch ? ch->Permissions.HasFlags(permissions) : true;
 }
 
+double TPartitionState::GetFreeSpaceShare(ui32 channel) const
+{
+    const auto* ch = GetChannel(channel);
+    return ch ? ch->ApproximateFreeSpaceShare : 0;
+}
+
 bool TPartitionState::UpdateChannelFreeSpaceShare(ui32 channel, double share)
 {
     if (share) {
@@ -319,17 +314,14 @@ bool TPartitionState::CheckChannelFreeSpaceShare(ui32 channel) const
     const auto& fsc = FreeSpaceConfig;
     const auto* ch = GetChannel(channel);
 
-    if (!ch || !ch->ApproximateFreeSpaceShare) {
+    if (!ch) {
         return true;
     }
 
-    const auto fss = Normalize(
+    return NCloud::CheckChannelFreeSpaceShare(
         ch->ApproximateFreeSpaceShare,
         fsc.ChannelMinFreeSpace,
-        fsc.ChannelFreeSpaceThreshold
-    );
-
-    return RandomNumber<double>() < fss;
+        fsc.ChannelFreeSpaceThreshold);
 }
 
 bool TPartitionState::IsCompactionAllowed() const

cloud/blockstore/libs/storage/partition/part_state.h

@@ -441,6 +441,7 @@ class TPartitionState
 
     bool UpdatePermissions(ui32 channel, EChannelPermissions permissions);
     bool CheckPermissions(ui32 channel, EChannelPermissions permissions) const;
+    double GetFreeSpaceShare(ui32 channel) const;
     bool UpdateChannelFreeSpaceShare(ui32 channel, double share);
     bool CheckChannelFreeSpaceShare(ui32 channel) const;
     bool IsCompactionAllowed() const;

cloud/blockstore/libs/storage/partition2/part2_state.cpp

@@ -8,6 +8,7 @@
 #include <cloud/blockstore/libs/storage/partition2/model/rebase_logic.h>
 
 #include <cloud/storage/core/libs/common/format.h>
+#include <cloud/storage/core/libs/tablet/model/channels.h>
 
 #include <library/cpp/monlib/service/pages/templates.h>
 #include <library/cpp/protobuf/json/proto2json.h>
@@ -38,19 +39,6 @@ TJsonValue ToJson(const TOperationState& op)
 
 ////////////////////////////////////////////////////////////////////////////////
 
-double Normalize(double x, double lo, double hi)
-{
-    if (x > hi) {
-        return 1;
-    }
-
-    if (x < lo) {
-        return 0;
-    }
-
-    return (x - lo) / (hi - lo);
-}
-
 double BPFeature(const TBackpressureFeatureConfig& c, double x)
 {
     auto nx = Normalize(x, c.InputThreshold, c.InputLimit);
@@ -375,19 +363,14 @@ bool TPartitionState::CheckChannelFreeSpaceShare(ui32 channel) const
     const auto& fsc = FreeSpaceConfig;
     const auto* ch = GetChannel(channel);
 
-    if (!ch || !ch->ApproximateFreeSpaceShare) {
+    if (!ch) {
         return true;
     }
 
-    // fss will be something like O(exp(-t)), where t is time
-    // so fss(t) > 0 for any t and lim(fss) = 0 as t approaches +inf
-    const auto fss = Normalize(
+    return NCloud::CheckChannelFreeSpaceShare(
         ch->ApproximateFreeSpaceShare,
         fsc.ChannelMinFreeSpace,
-        fsc.ChannelFreeSpaceThreshold
-    );
-
-    return RandomNumber<double>() < fss;
+        fsc.ChannelFreeSpaceThreshold);
 }
 
 bool TPartitionState::IsCompactionAllowed() const

cloud/filestore/config/storage.proto

@@ -341,4 +341,9 @@ message TStorageConfig
         repeated TFilestoreAliasEntry Entries = 1;
     }
     optional TFilestoreAliases FilestoreAliases = 368;
+
+    // Channel free space threshold - used for write request balancing.
+    optional uint32 ChannelFreeSpaceThreshold = 369;
+    // Channel min free space - used for write request balancing.
+    optional uint32 ChannelMinFreeSpace = 370;
 }

cloud/filestore/libs/storage/core/config.cpp

@@ -181,6 +181,8 @@ using TAliases = NProto::TStorageConfig::TFilestoreAliases;
     xxx(BlobCompressionCodec,            TString,               "lz4"         )\
                                                                                \
     xxx(MaxZeroCompactionRangesToDeletePerTx,           ui32,      10000      )\
+    xxx(ChannelFreeSpaceThreshold,                      ui32,      25         )\
+    xxx(ChannelMinFreeSpace,                            ui32,      10         )\
 // FILESTORE_STORAGE_CONFIG
 
 #define FILESTORE_STORAGE_CONFIG_REF(xxx)                                      \

cloud/filestore/libs/storage/core/config.h

@@ -242,6 +242,9 @@ class TStorageConfig
 
     const NProto::TStorageConfig::TFilestoreAliases& GetFilestoreAliases() const;
     const TString* FindFileSystemIdByAlias(const TString& alias) const;
+
+    ui32 GetChannelFreeSpaceThreshold() const;
+    ui32 GetChannelMinFreeSpace() const;
 };
 
 }   // namespace NCloud::NFileStore::NStorage

cloud/filestore/libs/storage/service/service_actor_writedata.cpp

@@ -51,6 +51,7 @@ class TWriteDataActor final: public TActorBootstrapped<TWriteDataActor>
     TMaybe<TInFlightRequest> InFlightRequest;
     TVector<std::unique_ptr<TInFlightRequest>> InFlightBSRequests;
     TVector<ui32> StorageStatusFlags;
+    TVector<double> ApproximateFreeSpaceShares;
     const NCloud::NProto::EStorageMediaKind MediaKind;
 
 public:
@@ -175,6 +176,7 @@ class TWriteDataActor final: public TActorBootstrapped<TWriteDataActor>
         RequestInfo->CallContext->RequestType = EFileStoreRequest::WriteBlob;
         InFlightBSRequests.reserve(RemainingBlobsToWrite);
         StorageStatusFlags.resize(GenerateBlobIdsResponse.BlobsSize());
+        ApproximateFreeSpaceShares.resize(GenerateBlobIdsResponse.BlobsSize());
         for (const auto& blob: GenerateBlobIdsResponse.GetBlobs()) {
             NKikimr::TLogoBlobID blobId =
                 LogoBlobIDFromLogoBlobID(blob.GetBlobId());
@@ -259,6 +261,7 @@ class TWriteDataActor final: public TActorBootstrapped<TWriteDataActor>
             !InFlightBSRequests[blobIdx]->IsCompleted());
         InFlightBSRequests[blobIdx]->Complete(ctx.Now(), {});
         StorageStatusFlags[blobIdx] = msg->StatusFlags.Raw;
+        ApproximateFreeSpaceShares[blobIdx] = msg->ApproximateFreeSpaceShare;
 
         --RemainingBlobsToWrite;
         if (RemainingBlobsToWrite == 0) {
@@ -287,6 +290,11 @@ class TWriteDataActor final: public TActorBootstrapped<TWriteDataActor>
         for (const auto flags: StorageStatusFlags) {
             request->Record.AddStorageStatusFlags(flags);
         }
+        request->Record.MutableApproximateFreeSpaceShares()->Reserve(
+            ApproximateFreeSpaceShares.size());
+        for (const auto share: ApproximateFreeSpaceShares) {
+            request->Record.AddApproximateFreeSpaceShares(share);
+        }
 
         if (Range.Offset < BlobRange.Offset) {
             auto& unalignedHead = *request->Record.AddUnalignedDataRanges();

cloud/filestore/libs/storage/service/service_ut.cpp

@@ -2724,6 +2724,7 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest)
 
         NProtoPrivate::TAddDataRequest addData;
         using TFlags = NKikimr::TStorageStatusFlags;
+        const float freeSpaceShare = 0.22;
         env.GetRuntime().SetEventFilter(
             [&](auto& runtime, auto& event)
             {
@@ -2735,6 +2736,8 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest)
                             event->template Get<TEvBlobStorage::TEvPutResult>();
                         const_cast<TFlags&>(msg->StatusFlags).Raw |=
                             ui32(yellowFlag);
+                        const_cast<float&>(msg->ApproximateFreeSpaceShare) =
+                            freeSpaceShare;
                         break;
                     }
 
@@ -2753,6 +2756,10 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest)
         UNIT_ASSERT_VALUES_EQUAL(1, addData.StorageStatusFlagsSize());
         UNIT_ASSERT(NKikimr::TStorageStatusFlags(
             addData.GetStorageStatusFlags(0)).Check(yellowFlag));
+        UNIT_ASSERT_VALUES_EQUAL(1, addData.ApproximateFreeSpaceSharesSize());
+        UNIT_ASSERT_VALUES_EQUAL(
+            freeSpaceShare,
+            addData.GetApproximateFreeSpaceShares(0));
     }
 
     void ConfigureFollowers(

cloud/filestore/libs/storage/tablet/model/channels.cpp

@@ -1,5 +1,7 @@
 #include "channels.h"
 
+#include <cloud/storage/core/libs/tablet/model/channels.h>
+
 #include <util/generic/deque.h>
 #include <util/generic/vector.h>
 
@@ -18,6 +20,7 @@ struct TChannelMeta
     TString PoolKind;
     bool Writable = true;
     bool ToMove = false;
+    double FreeSpaceShare = 0;
 
     TChannelMeta() = default;
 
@@ -35,17 +38,35 @@ struct TChannelRegistry
     TVector<TChannelMeta*> ChannelMetas;
     ui32 ChannelIndex = 0;
 
-    const TChannelMeta* SelectChannel()
+    const TChannelMeta* SelectChannel(
+        double minFreeSpace,
+        double freeSpaceThreshold)
     {
+        const TChannelMeta* bestMeta = nullptr;
+        double bestSpaceShare = 0;
         for (ui32 i = 0; i < ChannelMetas.size(); ++i) {
             const auto* meta = ChannelMetas[ChannelIndex % ChannelMetas.size()];
             ++ChannelIndex;
-            if (meta->Writable) {
+            if (!meta->Writable) {
+                continue;
+            }
+
+            const bool ok = CheckChannelFreeSpaceShare(
+                meta->FreeSpaceShare,
+                minFreeSpace,
+                freeSpaceThreshold);
+
+            if (ok) {
                 return meta;
             }
+
+            if (meta->FreeSpaceShare > bestSpaceShare) {
+                bestMeta = meta;
+                bestSpaceShare = meta->FreeSpaceShare;
+            }
         }
 
-        return nullptr;
+        return bestMeta;
     }
 
     TVector<ui32> GetChannels() const
@@ -75,9 +96,15 @@ struct TChannels::TImpl
     TChannelsByDataKind ByDataKind;
 
     void AddChannel(ui32 channel, EChannelDataKind dataKind, TString poolKind);
-    void RegisterUnwritableChannel(ui32 channel);
-    void RegisterChannelToMove(ui32 channel);
-    TMaybe<ui32> SelectChannel(EChannelDataKind dataKind);
+    void UpdateChannelStats(
+        ui32 channel,
+        bool writable,
+        bool toMove,
+        double freeSpaceShare);
+    TMaybe<ui32> SelectChannel(
+        EChannelDataKind dataKind,
+        double minFreeSpace,
+        double freeSpaceThreshold);
 
     TVector<ui32> GetChannels(EChannelDataKind dataKind) const;
     TVector<ui32> GetUnwritableChannels() const;
@@ -107,18 +134,20 @@ void TChannels::TImpl::AddChannel(
     byDataKind.ChannelMetas.push_back(&AllChannels.back());
 }
 
-void TChannels::TImpl::RegisterUnwritableChannel(ui32 channel)
-{
-    Y_ABORT_UNLESS(channel < AllChannels.size());
-
-    AllChannels[channel].Writable = false;
-}
-
-void TChannels::TImpl::RegisterChannelToMove(ui32 channel)
+void TChannels::TImpl::UpdateChannelStats(
+    ui32 channel,
+    bool writable,
+    bool toMove,
+    double freeSpaceShare)
 {
     Y_ABORT_UNLESS(channel < AllChannels.size());
 
-    AllChannels[channel].ToMove = true;
+    AllChannels[channel].Writable = writable;
+    AllChannels[channel].ToMove = toMove;
+    // a value which is exactly 0 is equivalent to "no data"
+    if (freeSpaceShare != 0.) {
+        AllChannels[channel].FreeSpaceShare = freeSpaceShare;
+    }
 }
 
 TVector<ui32> TChannels::TImpl::GetChannels(EChannelDataKind dataKind) const
@@ -139,7 +168,8 @@ TVector<ui32> TChannels::TImpl::GetUnwritableChannels() const
     return result;
 }
 
-TVector<ui32> TChannels::TImpl::GetChannelsToMove(ui32 percentageThreshold) const
+TVector<ui32> TChannels::TImpl::GetChannelsToMove(
+    ui32 percentageThreshold) const
 {
     TVector<ui32> result;
 
@@ -169,6 +199,7 @@ TChannels::TImpl::MakeChannelMonInfos() const
             TStringBuilder() << meta.DataKind,
             meta.Writable,
             meta.Writable, // TODO: SystemWritable
+            meta.FreeSpaceShare,
         });
     }
 
@@ -188,10 +219,15 @@ TChannelsStats TChannels::TImpl::CalculateChannelsStats() const
     return stats;
 }
 
-TMaybe<ui32> TChannels::TImpl::SelectChannel(EChannelDataKind dataKind)
+TMaybe<ui32> TChannels::TImpl::SelectChannel(
+    EChannelDataKind dataKind,
+    double minFreeSpace,
+    double freeSpaceThreshold)
 {
     auto& byDataKind = ByDataKind[static_cast<ui32>(dataKind)];
-    if (const auto* meta = byDataKind.SelectChannel()) {
+    const auto* meta =
+        byDataKind.SelectChannel(minFreeSpace, freeSpaceThreshold);
+    if (meta) {
         return meta->Channel;
     }
 
@@ -226,14 +262,13 @@ void TChannels::AddChannel(
     GetImpl().AddChannel(channel, dataKind, std::move(poolKind));
 }
 
-void TChannels::RegisterUnwritableChannel(ui32 channel)
-{
-    GetImpl().RegisterUnwritableChannel(channel);
-}
-
-void TChannels::RegisterChannelToMove(ui32 channel)
+void TChannels::UpdateChannelStats(
+    ui32 channel,
+    bool writable,
+    bool toMove,
+    double freeSpaceShare)
 {
-    GetImpl().RegisterChannelToMove(channel);
+    GetImpl().UpdateChannelStats(channel, writable, toMove, freeSpaceShare);
 }
 
 TVector<ui32> TChannels::GetChannels(EChannelDataKind dataKind) const
@@ -261,9 +296,12 @@ TChannelsStats TChannels::CalculateChannelsStats() const
     return GetImpl().CalculateChannelsStats();
 }
 
-TMaybe<ui32> TChannels::SelectChannel(EChannelDataKind dataKind)
+TMaybe<ui32> TChannels::SelectChannel(
+    EChannelDataKind dataKind,
+    double minFreeSpace,
+    double freeSpaceThreshold)
 {
-    return GetImpl().SelectChannel(dataKind);
+    return GetImpl().SelectChannel(dataKind, minFreeSpace, freeSpaceThreshold);
 }
 
 ui32 TChannels::Size() const