Fast removal of S3 Storage buckets with 10k-1 million objects (#893)

Author: michaelzhiluo

sky/cli.py

@@ -2010,7 +2010,7 @@ def storage_delete(all: bool, name: str):  # pylint: disable=redefined-builtin
         sky storage delete -a
     """
     if all:
-        click.echo('Deleting all storage objects...')
+        click.echo('Deleting all storage objects.')
         storages = global_user_state.get_storage()
         for row in storages:
             store_object = data.Storage(name=row['name'],
@@ -2023,7 +2023,7 @@ def storage_delete(all: bool, name: str):  # pylint: disable=redefined-builtin
             if handle is None:
                 click.echo(f'Storage name {n} not found.')
             else:
-                click.echo(f'Deleting storage object {n}...')
+                click.echo(f'Deleting storage object {n}.')
                 store_object = data.Storage(name=handle.storage_name,
                                             source=handle.source,
                                             sync_on_reconstruction=False)

sky/data/storage.py

@@ -737,8 +737,8 @@ def upload(self):
                 f'Upload failed for store {self.name}') from e
 
     def delete(self) -> None:
-        logger.info(f'Deleting S3 Bucket {self.name}')
-        return self._delete_s3_bucket(self.name)
+        self._delete_s3_bucket(self.name)
+        logger.info(f'Deleted S3 bucket {self.name}.')
 
     def get_handle(self) -> StorageHandle:
         return aws.resource('s3').Bucket(self.name)
@@ -941,15 +941,22 @@ def _delete_s3_bucket(self, bucket_name: str) -> None:
         Args:
           bucket_name: str; Name of bucket
         """
+        # Deleting objects is very slow programatically
+        # (i.e. bucket.objects.all().delete() is slow).
+        # In addition, standard delete operations (i.e. via `aws s3 rm`)
+        # are slow, since AWS puts deletion markers.
+        # https://stackoverflow.com/questions/49239351/why-is-it-so-much-slower-to-delete-objects-in-aws-s3-than-it-is-to-create-them
+        # The fastest way to delete is to run `aws s3 rb --force`,
+        # which removes the bucket by force.
+        remove_command = f'aws s3 rb s3://{bucket_name} --force'
         try:
-            s3 = aws.resource('s3')
-            bucket = s3.Bucket(bucket_name)
-            bucket.objects.all().delete()
-            bucket.delete()
-        except aws.client_exception() as e:
-            logger.error(f'Unable to delete S3 bucket {self.name}')
-            logger.error(e)
-            raise e
+            with backend_utils.safe_console_status(
+                    f'[bold cyan]Deleting [green]bucket {bucket_name}'):
+                subprocess.check_output(remove_command.split(' '))
+        except subprocess.CalledProcessError as e:
+            logger.error(e.output)
+            raise exceptions.StorageBucketDeleteError(
+                f'Failed to delete S3 bucket {bucket_name}.')
 
 
 class GcsStore(AbstractStore):

sky/exceptions.py

@@ -59,6 +59,11 @@ class StorageBucketGetError(StorageInitError):
     pass
 
 
+class StorageBucketDeleteError(StorageError):
+    # Error raised if attempt to delete an existing bucket fails.
+    pass
+
+
 class StorageUploadError(StorageError):
     # Error raised when bucket is successfully initialized, but upload fails,
     # either due to permissions, ctrl-c, or other reasons.