feat: support bucket soft-delete policies

gcs/bucket.py

@@ -44,6 +44,7 @@
         "encryption",
         "billing",
         "retention_policy",
+        "soft_delete_policy",
         "location_type",
         "iam_config",
         "rpo",
@@ -190,6 +191,25 @@
             },
         )
 
+    @classmethod
+    def __preprocess_rest_soft_delete_retention_duration(cls, rp):
+        # The JSON representation for a proto duration is a string in basically
+        # this format "%{seconds + nanos/1'000'000'000.0}s". For this field
+        # type the nanos should always be zero.
+        return f"{rp}s"
+
+    @classmethod
+    def __preprocess_rest_soft_delete_policy(cls, rp):
+        return testbench.common.rest_adjust(
+            rp,
+            {
+                "retentionDurationSeconds": lambda x: (
+                    "retentionDuration",
+                    Bucket.__preprocess_rest_soft_delete_retention_duration(x),
+                ),
+            },
+        )
+
     @classmethod
     def __preprocess_rest(cls, rest):
         rest = testbench.common.rest_adjust(
@@ -218,6 +238,10 @@
                     "retentionPolicy",
                     Bucket.__preprocess_rest_retention_policy(x),
                 ),
+                "softDeletePolicy": lambda x: (
+                    "softDeletePolicy",
+                    Bucket.__preprocess_rest_soft_delete_policy(x),
+                ),
             },
         )
         if rest.get("acl", None) is not None:
@@ -285,6 +309,26 @@
         ).hexdigest()
         metadata.etag = cls._metadata_etag(metadata)
 
+    @classmethod
+    def __validate_soft_delete_policy(cls, metadata, context):
+        if not metadata.HasField("soft_delete_policy"):
+            return
+        policy = metadata.soft_delete_policy
+        if policy.retention_duration.nanos != 0:
+            testbench.error.invalid(
+                "SoftDeletePolicy.retention_duration should not have nanoseconds",
+                context,
+            )
+        if policy.retention_duration.ToSeconds() < 7 * 86400:
+            testbench.error.invalid(
+                "SoftDeletePolicy.retention_duration should be at least 7 days", context
+            )
+        if policy.retention_duration.ToSeconds() >= 365 * 86400:
+            testbench.error.invalid(
+                "SoftDeletePolicy.retention_duration should be at less than a year",
+                context,
+            )
+
     @classmethod
     def init(cls, request, context):
         metadata = cls.__preprocess_rest(json.loads(request.data))
@@ -329,6 +373,11 @@
         metadata.iam_config.uniform_bucket_level_access.enabled = is_uniform
         metadata.bucket_id = testbench.common.bucket_name_from_proto(metadata.name)
         metadata.project = "projects/" + testbench.acl.PROJECT_NUMBER
+        if metadata.HasField("soft_delete_policy"):
+            metadata.soft_delete_policy.effective_time.FromDatetime(
+                datetime.datetime.now()
+            )
+        cls.__validate_soft_delete_policy(metadata, context)
         return (
             cls(metadata, {}, cls.__init_iam_policy(metadata, context)),
             testbench.common.extract_projection(request, default_projection, context),
@@ -364,6 +413,11 @@
         cls.__insert_predefined_default_object_acl(
             metadata, predefined_default_object_acl, context
         )
+        if metadata.HasField("soft_delete_policy"):
+            metadata.soft_delete_policy.effective_time.FromDatetime(
+                datetime.datetime.now()
+            )
+        cls.__validate_soft_delete_policy(metadata, context)
         return (cls(metadata, {}, cls.__init_iam_policy(metadata, context)), "noAcl")
 
     # === IAM === #
@@ -430,8 +484,12 @@
             update_mask = field_mask_pb2.FieldMask(paths=Bucket.modifiable_fields)
         update_mask.MergeMessage(source, self.metadata, True, True)
         now = datetime.datetime.now()
-        if "autoclass" in update_mask.paths:
+        if "autoclass" in update_mask.paths and source.HasField("autoclass"):
             self.metadata.autoclass.toggle_time.FromDatetime(now)
+        if "soft_delete_policy" in update_mask.paths and source.HasField(
+            "soft_delete_policy"
+        ):
+            self.metadata.soft_delete_policy.effective_time.FromDatetime(now)
         self.metadata.metageneration += 1
         self.metadata.update_time.FromDatetime(now)
         self.metadata.etag = Bucket._metadata_etag(self.metadata)
@@ -441,6 +499,7 @@
         assert context is None
         data = self.__preprocess_rest(json.loads(request.data))
         metadata = json_format.ParseDict(data, storage_pb2.Bucket())
+        Bucket.__validate_soft_delete_policy(metadata, context)
         self.__update_metadata(metadata, None)
         self.__insert_predefined_acl(
             metadata,
@@ -460,6 +519,7 @@
             testbench.common.rest_patch(self.rest(), json.loads(request.data))
         )
         metadata = json_format.ParseDict(rest, storage_pb2.Bucket())
+        Bucket.__validate_soft_delete_policy(metadata, context)
         self.__update_metadata(metadata, None)
         self.__insert_predefined_acl(
             metadata,

testbench/proto2rest.py

@@ -151,6 +151,25 @@ def __postprocess_rest_retention_policy(data):
     )
 
 
+def __postprocess_rest_soft_delete_policy_duration(data: str):
+    # The string is in the canonical JSON representation for proto durations,
+    # that is: "%{seconds + nanos/1'000'000'000}s", we are just going to
+    # ignore the nanos and return this as a string.
+    return str(int(data[:-1]))
+
+
+def __postprocess_rest_soft_delete_policy(data):
+    return testbench.common.rest_adjust(
+        data,
+        {
+            "retentionDuration": lambda x: (
+                "retentionDurationSeconds",
+                __postprocess_rest_soft_delete_policy_duration(x),
+            )
+        },
+    )
+
+
 def __postprocess_bucket_rest(data):
     bucket_id = testbench.common.bucket_name_from_proto(data["name"])
     data = testbench.common.rest_adjust(
@@ -185,6 +204,10 @@ def __postprocess_bucket_rest(data):
                 "retentionPolicy",
                 __postprocess_rest_retention_policy(x),
             ),
+            "softDeletePolicy": lambda x: (
+                "softDeletePolicy",
+                __postprocess_rest_soft_delete_policy(x),
+            ),
         },
     )
     data["kind"] = "storage#bucket"

tests/test_bucket.py

@@ -19,6 +19,8 @@
 import json
 import unittest
 
+import google.protobuf.duration_pb2 as duration_pb2
+import google.protobuf.timestamp_pb2 as timestamp_pb2
 from google.protobuf import json_format
 
 import gcs
@@ -538,6 +540,69 @@ def test_rpo(self):
         bucket.patch(request, None)
         self.assertEqual(bucket.metadata.rpo, "ASYNC_TURBO")
 
+    def test_soft_delete(self):
+        request = testbench.common.FakeRequest(
+            args={},
+            data=json.dumps(
+                {
+                    "name": "bucket",
+                    "softDeletePolicy": {"retentionDurationSeconds": 7 * 86400},
+                }
+            ),
+        )
+        bucket, _ = gcs.bucket.Bucket.init(request, None)
+        expected = duration_pb2.Duration()
+        expected.FromSeconds(7 * 86400)
+        self.assertEqual(
+            bucket.metadata.soft_delete_policy.retention_duration, expected
+        )
+        self.assertNotEqual(
+            bucket.metadata.soft_delete_policy.effective_time, timestamp_pb2.Timestamp()
+        )
+        request = testbench.common.FakeRequest(
+            args={"bucket": "bucket"},
+            data=json.dumps(
+                {"softDeletePolicy": {"retentionDurationSeconds": 30 * 86400}}
+            ),
+        )
+        bucket.patch(request, None)
+        expected = duration_pb2.Duration()
+        expected.FromSeconds(30 * 86400)
+        self.assertEqual(
+            bucket.metadata.soft_delete_policy.retention_duration, expected
+        )
+        self.assertNotEqual(
+            bucket.metadata.soft_delete_policy.effective_time, timestamp_pb2.Timestamp()
+        )
+
+    def test_soft_delete_too_short(self):
+        request = testbench.common.FakeRequest(
+            args={},
+            data=json.dumps(
+                {
+                    "name": "bucket",
+                    "softDeletePolicy": {"retentionDurationSeconds": 1 * 86400},
+                }
+            ),
+        )
+        with self.assertRaises(testbench.error.RestException) as rest:
+            bucket, _ = gcs.bucket.Bucket.init(request, None)
+        self.assertEqual(rest.exception.code, 400)
+
+    def test_soft_delete_too_long(self):
+        request = testbench.common.FakeRequest(
+            args={},
+            data=json.dumps(
+                {
+                    "name": "bucket",
+                    "softDeletePolicy": {"retentionDurationSeconds": 700 * 86400},
+                }
+            ),
+        )
+        with self.assertRaises(testbench.error.RestException) as rest:
+            bucket, _ = gcs.bucket.Bucket.init(request, None)
+        self.assertEqual(rest.exception.code, 400)
+
     def test_cdr(self):
         request = testbench.common.FakeRequest(
             args={},

tests/test_bucket_grpc.py

@@ -203,6 +203,58 @@ def test_init_grpc_retention_policy(self):
             rest.get("retentionPolicy", dict()), {"retentionPeriod": "86400"}
         )
 
+    def test_init_grpc_soft_delete_policy(self):
+        policy = storage_pb2.Bucket.SoftDeletePolicy()
+        policy.retention_duration.FromTimedelta(datetime.timedelta(seconds=30 * 86400))
+        request = storage_pb2.CreateBucketRequest(
+            parent="projects/_",
+            bucket_id="test-bucket-name",
+            bucket=storage_pb2.Bucket(
+                project="project/test-project",
+                soft_delete_policy=policy,
+            ),
+        )
+        context = unittest.mock.Mock()
+        bucket, _ = gcs.bucket.Bucket.init_grpc(request, context)
+        self.assertEqual(bucket.metadata.name, "projects/_/buckets/test-bucket-name")
+        self.assertEqual(bucket.metadata.bucket_id, "test-bucket-name")
+        self.assertEqual(
+            bucket.metadata.soft_delete_policy.retention_duration,
+            policy.retention_duration,
+        )
+        self.assertTrue(
+            bucket.metadata.soft_delete_policy.HasField("effective_time"),
+            msg=f"{bucket.metadata.soft_delete_policy}",
+        )
+        self.assertLess(0, bucket.metadata.metageneration)
+
+        rest = bucket.rest()
+        rest_policy = rest.get("softDeletePolicy", dict())
+        self.assertIn("retentionDurationSeconds", rest_policy)
+        self.assertIn("effectiveTime", rest_policy)
+        self.assertEqual(rest_policy["retentionDurationSeconds"], str(30 * 86400))
+
+    def test_init_grpc_soft_delete_policy_nanos_are_invalid(self):
+        policy = storage_pb2.Bucket.SoftDeletePolicy()
+        policy.retention_duration.FromTimedelta(
+            datetime.timedelta(seconds=30 * 86400, milliseconds=500)
+        )
+        request = storage_pb2.CreateBucketRequest(
+            parent="projects/_",
+            bucket_id="test-bucket-name",
+            bucket=storage_pb2.Bucket(
+                project="project/test-project",
+                soft_delete_policy=policy,
+            ),
+        )
+        context = unittest.mock.Mock()
+        context.abort.side_effect = TestBucketGrpc._raise_grpc_error
+        with self.assertRaises(Exception) as _:
+            _, _ = gcs.bucket.Bucket.init_grpc(request, context)
+        context.abort.assert_called_once_with(
+            grpc.StatusCode.INVALID_ARGUMENT, unittest.mock.ANY
+        )
+
 
 if __name__ == "__main__":
     unittest.main()