[Issue #121]: fix listing objects. (#123)

Support listing more than 1000 object in a bucket.
pixelsdb · Oct 8, 2021 · 8f058f4 · 8f058f4
1 parent dde1be5
commit 8f058f4
Showing 1 changed file with 9 additions and 1 deletion.
diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/physical/impl/S3.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/physical/impl/S3.java
@@ -216,7 +216,15 @@ public List<Status> listStatus(String path) throws IOException
         ListObjectsV2Request request = ListObjectsV2Request.builder()
                 .bucket(p.bucket).build();
         ListObjectsV2Response response = s3.listObjectsV2(request);
-        List<S3Object> objects = response.contents();
+        List<S3Object> objects = new ArrayList<>(response.keyCount());
+        while (response.isTruncated())
+        {
+            objects.addAll(response.contents());
+            request = ListObjectsV2Request.builder().bucket(p.bucket)
+                    .continuationToken(response.nextContinuationToken()).build();
+            response = s3.listObjectsV2(request);
+        }
+        objects.addAll(response.contents());
         List<Status> statuses = new ArrayList<>();
         Path op = new Path(path);
         op.isBucket = false;