YQL-18516 reduce grace join with spilling memory consumption (#5087)

ydb-platform · Jun 5, 2024 · cd070ab · cd070ab
1 parent 0c193f2
commit cd070ab
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 0 deletions.
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_grace_join.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_grace_join.cpp
@@ -894,9 +894,11 @@ EFetchResult ProcessSpilledData(TComputationContext&, NUdf::TUnboxedValue*const*
 
                 LeftPacker->TuplesBatchPacked = 0;
                 LeftPacker->TablePtr->ClearBucket(NextBucketToJoin); // Clear content of returned bucket
+                LeftPacker->TablePtr->ShrinkBucket(NextBucketToJoin);
 
                 RightPacker->TuplesBatchPacked = 0;
                 RightPacker->TablePtr->ClearBucket(NextBucketToJoin); // Clear content of returned bucket
+                RightPacker->TablePtr->ShrinkBucket(NextBucketToJoin);
 
                 JoinedTablePtr->Clear();
                 JoinedTablePtr->ResetIterator();

diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_grace_join_imp.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_grace_join_imp.cpp
@@ -1128,6 +1128,18 @@ void TTable::ClearBucket(ui64 bucket) {
     tbs.StringValuesTotalSize = 0;
 }
 
+void TTable::ShrinkBucket(ui64 bucket) {
+    TTableBucket & tb = TableBuckets[bucket];
+    tb.KeyIntVals.shrink_to_fit();
+    tb.DataIntVals.shrink_to_fit();
+    tb.StringsOffsets.shrink_to_fit();
+    tb.StringsValues.shrink_to_fit();
+    tb.InterfaceValues.shrink_to_fit();
+    tb.InterfaceOffsets.shrink_to_fit();
+    tb.JoinIds.shrink_to_fit();
+    tb.RightIds.shrink_to_fit();
+}
+
 void TTable::InitializeBucketSpillers(ISpiller::TPtr spiller) {
     for (size_t i = 0; i < NumberOfBuckets; ++i) {
         TableBucketsSpillers.emplace_back(spiller, 5_MB);

diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_grace_join_imp.h b/ydb/library/yql/minikql/comp_nodes/mkql_grace_join_imp.h
@@ -320,6 +320,9 @@ class TTable {
     // Clears all the data related to a single bucket
     void ClearBucket(ui64 bucket);
 
+    // Forces bucket to release the space used for underlying containers.
+    void ShrinkBucket(ui64 bucket);
+
     // Clears table content
     void Clear();