chore: update BucketCleaner to clean up folders and managed folders i…

…n addition to objects

.kokoro/build.sh

@@ -28,7 +28,7 @@ mvn -version
 echo ${JOB_TYPE}
 
 # attempt to install 3 times with exponential backoff (starting with 10 seconds)
-retry_with_backoff 3 10 \
+#retry_with_backoff 3 10 \
   mvn install -B -V -ntp \
     -DskipTests=true \
     -Dclirr.skip=true \

google-cloud-storage-control/pom.xml

@@ -81,11 +81,6 @@
             <artifactId>google-api-services-storage</artifactId>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>com.google.cloud</groupId>
-            <artifactId>google-cloud-storage</artifactId>
-            <scope>test</scope>
-        </dependency>
         <dependency>
             <groupId>com.google.auth</groupId>
             <artifactId>google-auth-library-oauth2-http</artifactId>

google-cloud-storage/pom.xml

@@ -240,6 +240,17 @@
       <scope>test</scope>
       <version>${pubsub-proto.version}</version>
     </dependency>
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-storage-control</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.api.grpc</groupId>
+      <artifactId>proto-google-cloud-storage-control-v2</artifactId>
+      <scope>test</scope>
+    </dependency>
+
     <dependency>
       <groupId>com.google.cloud</groupId>
       <artifactId>google-cloud-core</artifactId>

google-cloud-storage/src/test/java/com/google/cloud/storage/it/BucketCleaner.java

@@ -17,15 +17,27 @@
 package com.google.cloud.storage.it;
 
 import com.google.api.gax.paging.Page;
+import com.google.api.gax.rpc.ApiException;
+import com.google.api.gax.rpc.FailedPreconditionException;
 import com.google.cloud.storage.Blob;
 import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.Storage.BlobField;
 import com.google.cloud.storage.Storage.BlobListOption;
 import com.google.cloud.storage.Storage.BlobSourceOption;
 import com.google.cloud.storage.Storage.BucketSourceOption;
+import com.google.common.collect.ImmutableList;
+import com.google.storage.control.v2.BucketName;
+import com.google.storage.control.v2.Folder;
+import com.google.storage.control.v2.StorageControlClient;
+import com.google.storage.control.v2.StorageLayout;
+import com.google.storage.control.v2.StorageLayoutName;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 
 public final class BucketCleaner {
@@ -48,12 +60,9 @@ public static void doCleanup(String bucketName, Storage s) {
                           b.getName(),
                           s.delete(b.getBlobId(), BlobSourceOption.userProject(projectId))))
               .collect(Collectors.toList());
-      List<DeleteResult> failedDeletes =
-          deleteResults.stream().filter(r -> !r.success).collect(Collectors.toList());
-      failedDeletes.forEach(
-          r -> LOGGER.warning(String.format("Failed to delete object %s/%s", bucketName, r.name)));
+      boolean anyFailedObjectDeletes = getIfAnyFailedAndReport(bucketName, deleteResults, "object");
 
-      if (failedDeletes.isEmpty()) {
+      if (!anyFailedObjectDeletes) {
         s.delete(bucketName, BucketSourceOption.userProject(projectId));
       } else {
         LOGGER.warning("Unable to delete bucket due to previous failed object deletes");
@@ -64,6 +73,122 @@ public static void doCleanup(String bucketName, Storage s) {
     }
   }
 
+  public static void doCleanup(String bucketName, Storage s, StorageControlClient ctrl) {
+    LOGGER.warning("Starting bucket cleanup: " + bucketName);
+    String projectId = s.getOptions().getProjectId();
+    try {
+      // TODO: probe bucket existence, a bad test could have deleted the bucket
+      Page<Blob> page1 =
+          s.list(
+              bucketName,
+              BlobListOption.userProject(projectId),
+              BlobListOption.versions(true),
+              BlobListOption.fields(BlobField.NAME));
+
+      List<DeleteResult> objectResults =
+          StreamSupport.stream(page1.iterateAll().spliterator(), false)
+              .map(
+                  b ->
+                      new DeleteResult(
+                          b.getName(),
+                          s.delete(b.getBlobId(), BlobSourceOption.userProject(projectId))))
+              .collect(Collectors.toList());
+      boolean anyFailedObjectDelete = getIfAnyFailedAndReport(bucketName, objectResults, "object");
+      boolean anyFailedFolderDelete = false;
+      boolean anyFailedManagedFolderDelete = false;
+
+      if (!anyFailedObjectDelete) {
+        BucketName parent = BucketName.of("_", bucketName);
+        StorageLayout storageLayout =
+            ctrl.getStorageLayout(StorageLayoutName.of(parent.getProject(), parent.getBucket()));
+        List<DeleteResult> folderDeletes;
+        if (storageLayout.hasHierarchicalNamespace()
+            && storageLayout.getHierarchicalNamespace().getEnabled()) {
+          folderDeletes =
+              StreamSupport.stream(ctrl.listFolders(parent).iterateAll().spliterator(), false)
+                  .collect(Collectors.toList())
+                  .stream()
+                  .sorted(Collections.reverseOrder(Comparator.comparing(Folder::getName)))
+                  .map(
+                      folder -> {
+                        LOGGER.warning(String.format("folder = %s", folder.getName()));
+                        boolean success = true;
+                        try {
+                          ctrl.deleteFolder(folder.getName());
+                        } catch (ApiException e) {
+                          success = false;
+                        }
+                        return new DeleteResult(folder.getName(), success);
+                      })
+                  .collect(Collectors.toList());
+        } else {
+          folderDeletes = ImmutableList.of();
+        }
+
+        List<DeleteResult> managedFolderDeletes;
+        try {
+          managedFolderDeletes =
+              StreamSupport.stream(
+                      ctrl.listManagedFolders(parent).iterateAll().spliterator(), false)
+                  .map(
+                      managedFolder -> {
+                        LOGGER.warning(
+                            String.format("managedFolder = %s", managedFolder.getName()));
+                        boolean success = true;
+                        try {
+                          ctrl.deleteFolder(managedFolder.getName());
+                        } catch (ApiException e) {
+                          success = false;
+                        }
+                        return new DeleteResult(managedFolder.getName(), success);
+                      })
+                  .collect(Collectors.toList());
+        } catch (FailedPreconditionException fpe) {
+          // FAILED_PRECONDITION: Uniform bucket-level access is required to be enabled on the
+          //   bucket in order to perform this operation. Read more at
+          //   https://cloud.google.com/storage/docs/uniform-bucket-level-access
+          managedFolderDeletes = ImmutableList.of();
+        }
+
+        anyFailedFolderDelete = getIfAnyFailedAndReport(bucketName, folderDeletes, "folder");
+        anyFailedManagedFolderDelete =
+            getIfAnyFailedAndReport(bucketName, managedFolderDeletes, "managed folder");
+      }
+
+      List<String> failed =
+          Stream.of(
+                  anyFailedObjectDelete ? "object" : "",
+                  anyFailedFolderDelete ? "folder" : "",
+                  anyFailedManagedFolderDelete ? "managed folder" : "")
+              .filter(ss -> !ss.isEmpty())
+              .collect(Collectors.toList());
+
+      if (!anyFailedObjectDelete && !anyFailedFolderDelete && !anyFailedManagedFolderDelete) {
+        s.delete(bucketName, BucketSourceOption.userProject(projectId));
+      } else {
+        LOGGER.warning(
+            String.format(
+                "Unable to delete bucket %s due to previous failed %s deletes",
+                bucketName, failed));
+      }
+
+      LOGGER.warning("Bucket cleanup complete: " + bucketName);
+    } catch (Exception e) {
+      LOGGER.log(Level.SEVERE, e, () -> "Error during bucket cleanup.");
+    }
+  }
+
+  private static boolean getIfAnyFailedAndReport(
+      String bucketName, List<DeleteResult> deleteResults, String resourceType) {
+    List<DeleteResult> failedDeletes =
+        deleteResults.stream().filter(r -> !r.success).collect(Collectors.toList());
+    failedDeletes.forEach(
+        r ->
+            LOGGER.warning(
+                String.format("Failed to delete %s %s/%s", resourceType, bucketName, r.name)));
+    return !failedDeletes.isEmpty();
+  }
+
   private static final class DeleteResult {
     private final String name;
     private final boolean success;

google-cloud-storage/src/test/java/com/google/cloud/storage/it/TemporaryBucket.java

@@ -23,22 +23,26 @@
 import com.google.cloud.storage.Storage;
 import com.google.cloud.storage.conformance.retry.CleanupStrategy;
 import com.google.common.base.Preconditions;
+import com.google.storage.control.v2.StorageControlClient;
 import java.time.Duration;
 
 public final class TemporaryBucket implements AutoCloseable {
 
   private final BucketInfo bucket;
   private final Storage storage;
+  private final StorageControlClient ctrl;
   private final Duration cleanupTimeout;
   private final CleanupStrategy cleanupStrategy;
 
   private TemporaryBucket(
       BucketInfo bucket,
       Storage storage,
+      StorageControlClient ctrl,
       Duration cleanupTimeout,
       CleanupStrategy cleanupStrategy) {
     this.bucket = bucket;
     this.storage = storage;
+    this.ctrl = ctrl;
     this.cleanupTimeout = cleanupTimeout;
     this.cleanupStrategy = cleanupStrategy;
   }
@@ -51,7 +55,7 @@ public BucketInfo getBucket() {
   @Override
   public void close() throws Exception {
     if (cleanupStrategy == CleanupStrategy.ALWAYS) {
-      BucketCleaner.doCleanup(bucket.getName(), storage);
+      BucketCleaner.doCleanup(bucket.getName(), storage, ctrl);
     }
   }
 
@@ -65,6 +69,7 @@ public static final class Builder {
     private Duration cleanupTimeoutDuration;
     private BucketInfo bucketInfo;
     private Storage storage;
+    private StorageControlClient ctrl;
 
     private Builder() {
       this.cleanupStrategy = CleanupStrategy.ALWAYS;
@@ -91,14 +96,20 @@ public Builder setStorage(Storage storage) {
       return this;
     }
 
+    public Builder setStorageControl(StorageControlClient ctrl) {
+      this.ctrl = ctrl;
+      return this;
+    }
+
     public TemporaryBucket build() {
       Preconditions.checkArgument(
           cleanupStrategy != CleanupStrategy.ONLY_ON_SUCCESS, "Unable to detect success.");
       Storage s = requireNonNull(storage, "storage must be non null");
+      StorageControlClient c = requireNonNull(ctrl, "ctrl must be non null");
       Bucket b = s.create(requireNonNull(bucketInfo, "bucketInfo must be non null"));
 
       // intentionally drop from Bucket to BucketInfo to ensure not leaking the Storage instance
-      return new TemporaryBucket(b.asBucketInfo(), s, cleanupTimeoutDuration, cleanupStrategy);
+      return new TemporaryBucket(b.asBucketInfo(), s, c, cleanupTimeoutDuration, cleanupStrategy);
     }
   }
 }

samples/install-without-bom/pom.xml

@@ -22,6 +22,7 @@
     <maven.compiler.target>1.8</maven.compiler.target>
     <maven.compiler.source>1.8</maven.compiler.source>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <test>com.example.storage.ITVerboseBucketCleanupTest</test>
   </properties>
 
 
@@ -32,6 +33,13 @@
       <artifactId>google-cloud-storage</artifactId>
       <version>2.42.0</version>
     </dependency>
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-storage</artifactId>
+      <version>2.42.0</version>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>com.google.cloud</groupId>
       <artifactId>google-cloud-storage-control</artifactId>

samples/pom.xml

@@ -29,9 +29,9 @@
   </properties>
 
   <modules>
-    <module>install-without-bom</module>
+<!--    <module>install-without-bom</module>-->
     <module>snapshot</module>
-    <module>snippets</module>
+<!--    <module>snippets</module>-->
   </modules>
 
   <build>

samples/snapshot/pom.xml

@@ -22,6 +22,7 @@
     <maven.compiler.target>1.8</maven.compiler.target>
     <maven.compiler.source>1.8</maven.compiler.source>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <test>com.example.storage.ITVerboseBucketCleanupTest</test>
   </properties>
 
   <dependencies>
@@ -30,6 +31,13 @@
       <artifactId>google-cloud-storage</artifactId>
       <version>2.42.1-SNAPSHOT</version><!-- {x-version-update:google-cloud-storage:current} -->
     </dependency>
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-storage</artifactId>
+      <version>2.42.1-SNAPSHOT</version><!-- {x-version-update:google-cloud-storage:current} -->
+      <classifier>tests</classifier>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>com.google.cloud</groupId>
       <artifactId>google-cloud-storage-control</artifactId>