non partitioned lister has been sped up (ydb-platform#5870)

marsaly79 · Jun 26, 2024 · 84a22eb · 84a22eb
1 parent 35e8bdd
commit 84a22eb
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 10 deletions.
diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_datasource_type_ann.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_datasource_type_ann.cpp
@@ -22,12 +22,6 @@ bool ValidateS3PackedPaths(TPositionHandle pos, TStringBuf blob, bool isTextEnco
     try {
         TPathList paths;
         UnpackPathsList(blob, isTextEncoded, paths);
-        for (size_t i = 0; i < paths.size(); ++i) {
-            if (paths[i].Path.empty()) {
-                ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Expected non-empty path (index " << i << ")"));
-                return false;
-            }
-        }
     } catch (const std::exception& ex) {
         ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Failed to parse packed paths: " << ex.what()));
         return false;

diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_io_discovery.cpp
@@ -864,6 +864,10 @@ class TS3IODiscoveryTransformer : public TGraphTransformerBase {
                         entries.Directories.back().Path = req.S3Request.Pattern;
                         future = NThreading::MakeFuture<NS3Lister::TListResult>(std::move(entries));
                     } else {
+                        auto useRuntimeListing = State_->Configuration->UseRuntimeListing.Get().GetOrElse(false);
+                        if (useRuntimeListing && !req.Options.IsPartitionedDataset) {
+                            req.Options.MaxResultSet = 1;
+                        }
                         future = ListingStrategy_->List(req.S3Request, req.Options);
                     }
                     PendingRequests_[req] = future;

diff --git a/ydb/library/yql/providers/s3/range_helpers/file_tree_builder.cpp b/ydb/library/yql/providers/s3/range_helpers/file_tree_builder.cpp
@@ -8,15 +8,14 @@ void TFileTreeBuilder::AddPath(const TString& path, ui64 fileSize, bool isDirect
     for (size_t i = 0, size = parts.size(); i < size; ++i) {
         bool isSubDirectory = i != size - 1;
         if (!isSubDirectory) {
-            TPath& p = (*currentChildren)[TTreeKey{parts[i], isDirectory}];
+            TPath& p = (*currentChildren)[TTreeKey{path == "" ? "/" : parts[i], isDirectory}];
             Y_ABORT_UNLESS(p.FileSize == 0);
             Y_ABORT_UNLESS(!p.Read);
             p.FileSize = fileSize;
             p.Read = true;
         } else {
             TPath& p = (*currentChildren)[TTreeKey{parts[i], isSubDirectory}];
             currentChildren = &p.Children;
-
         }
     }
 }

diff --git a/ydb/library/yql/providers/s3/range_helpers/file_tree_builder_ut.cpp b/ydb/library/yql/providers/s3/range_helpers/file_tree_builder_ut.cpp
@@ -179,6 +179,36 @@ Y_UNIT_TEST_SUITE(S3FileTreeBuilderTest) {
         UNIT_ASSERT_VALUES_EQUAL(paths[1].IsDirectory, false);
         UNIT_ASSERT_VALUES_EQUAL(paths[1].PathIndex, 1);
     }
+
+    Y_UNIT_TEST(DeserializesRootSlash) {
+        TFileTreeBuilder b;
+        b.AddPath("/root/name/", 3, true);
+        b.AddPath("", 42, true);
+        b.AddPath("//", 42, true);
+
+        NS3::TRange range;
+        b.Save(&range);
+
+        TPathList paths;
+        ReadPathsList({}, MakeParams(range), {}, paths);
+
+        UNIT_ASSERT_VALUES_EQUAL(paths.size(), 3);
+
+        UNIT_ASSERT_VALUES_EQUAL(paths[0].Path, "//");
+        UNIT_ASSERT_VALUES_EQUAL(paths[0].Size, 42);
+        UNIT_ASSERT_VALUES_EQUAL(paths[0].IsDirectory, true);
+        UNIT_ASSERT_VALUES_EQUAL(paths[0].PathIndex, 0);
+
+        UNIT_ASSERT_VALUES_EQUAL(paths[1].Path, "/root/name/");
+        UNIT_ASSERT_VALUES_EQUAL(paths[1].Size, 3);
+        UNIT_ASSERT_VALUES_EQUAL(paths[1].IsDirectory, true);
+        UNIT_ASSERT_VALUES_EQUAL(paths[1].PathIndex, 1);
+
+        UNIT_ASSERT_VALUES_EQUAL(paths[2].Path, "");
+        UNIT_ASSERT_VALUES_EQUAL(paths[2].Size, 42);
+        UNIT_ASSERT_VALUES_EQUAL(paths[2].IsDirectory, true);
+        UNIT_ASSERT_VALUES_EQUAL(paths[2].PathIndex, 2);
+    }
 }
 
 } // namespace NYql::NS3Details
diff --git a/ydb/library/yql/providers/s3/range_helpers/path_list_reader.cpp b/ydb/library/yql/providers/s3/range_helpers/path_list_reader.cpp
@@ -22,10 +22,10 @@ static void BuildPathsFromTree(const google::protobuf::RepeatedPtrField<NYql::NS
     }
     for (const auto& path : children) {
         const size_t prevSize = currentPath.size();
-        currentPath += path.GetName();
+        currentPath += path.GetName() != "/" ? path.GetName() : "";
         if (path.GetRead()) {
             auto isDirectory = path.GetIsDirectory();
-            auto readPath = isDirectory ? currentPath + "/" : currentPath;
+            auto readPath = isDirectory && path.GetName() != "/" ? currentPath + "/" : currentPath;
             paths.emplace_back(TPath{readPath, path.GetSize(), path.GetIsDirectory(), nextPathIndex++});
         }
         BuildPathsFromTree(path.GetChildren(), paths, currentPath, currentDepth + 1, nextPathIndex);