diff --git a/testbench/database.py b/testbench/database.py index 30cd0284..65412f11 100644 --- a/testbench/database.py +++ b/testbench/database.py @@ -606,6 +606,11 @@ def __validate_grpc_method_implemented_retry(self, method): "storage.default_object_acl.insert", "storage.default_object_acl.patch", "storage.default_object_acl.update", + "storage.hmacKey.create", + "storage.hmacKey.delete", + "storage.hmacKey.get", + "storage.hmacKey.list", + "storage.hmacKey.update", "storage.object_acl.get", "storage.object_acl.list", "storage.object_acl.delete", @@ -616,6 +621,7 @@ def __validate_grpc_method_implemented_retry(self, method): "storage.notifications.get", "storage.notifications.insert", "storage.notifications.list", + "storage.serviceaccount.get", } if method in not_supported_grpc_w_retry: testbench.error.unimplemented( diff --git a/testbench/grpc_server.py b/testbench/grpc_server.py index 6087ef37..0172fa7a 100644 --- a/testbench/grpc_server.py +++ b/testbench/grpc_server.py @@ -352,71 +352,6 @@ def UpdateBucket(self, request, context): bucket.metadata.update_time.FromDatetime(now) return bucket.metadata - def _notification_from_rest(self, rest, bucket_name): - # We need to make a copy before changing any values - rest = rest.copy() - rest.pop("kind") - rest["name"] = bucket_name + "/notificationConfigs/" + rest.pop("id") - rest["topic"] = "//pubsub.googleapis.com/" + rest["topic"] - return json_format.ParseDict(rest, storage_pb2.NotificationConfig()) - - def _decompose_notification_name(self, notification_name, context): - loc = notification_name.find("/notificationConfigs/") - if loc == -1: - testbench.error.invalid( - "GetNotificationConfig() malformed notification name [%s]" - % notification_name, - context, - ) - return (None, None) - bucket_name = notification_name[:loc] - notification_id = notification_name[loc + len("/notificationConfigs/") :] - return (bucket_name, notification_id) - - def DeleteNotificationConfig(self, request, context): - bucket_name, notification_id = self._decompose_notification_name( - request.name, context - ) - if bucket_name is None: - return None - bucket = self.db.get_bucket(bucket_name, context) - bucket.delete_notification(notification_id, context) - return empty_pb2.Empty() - - def GetNotificationConfig(self, request, context): - bucket_name, notification_id = self._decompose_notification_name( - request.name, context - ) - if bucket_name is None: - return None - bucket = self.db.get_bucket(bucket_name, context) - rest = bucket.get_notification(notification_id, context) - return self._notification_from_rest(rest, bucket_name) - - def CreateNotificationConfig(self, request, context): - pattern = "^//pubsub.googleapis.com/projects/[^/]+/topics/[^/]+$" - if re.match(pattern, request.notification_config.topic) is None: - return testbench.error.invalid( - "topic names must be in" - + " //pubsub.googleapis.com/projects/{project-identifier}/topics/{my-topic}" - + " format, got=%s" % request.notification_config.topic, - context, - ) - bucket = self.db.get_bucket(request.parent, context) - notification = json_format.MessageToDict(request.notification_config) - # Convert topic names to REST format - notification["topic"] = notification["topic"][len("//pubsub.googleapis.com/") :] - rest = bucket.insert_notification(json.dumps(notification), context) - return self._notification_from_rest(rest, request.parent) - - def ListNotificationConfigs(self, request, context): - bucket = self.db.get_bucket(request.parent, context) - items = bucket.list_notifications(context).get("items", []) - return storage_pb2.ListNotificationConfigsResponse( - notification_configs=[ - self._notification_from_rest(r, request.parent) for r in items - ] - ) @retry_test(method="storage.objects.compose") def ComposeObject(self, request, context): @@ -815,124 +750,6 @@ def QueryWriteStatus(self, request, context): return storage_pb2.QueryWriteStatusResponse(resource=upload.blob.metadata) return storage_pb2.QueryWriteStatusResponse(persisted_size=len(upload.media)) - @retry_test("storage.serviceaccount.get") - def GetServiceAccount(self, request, context): - if not request.project.startswith("projects/"): - return testbench.error.invalid( - "project name must start with projects/, got=%s" % request.project, - context, - ) - project_id = request.project[len("projects/") :] - project = self.db.get_project(project_id) - return storage_pb2.ServiceAccount(email_address=project.service_account_email()) - - def _hmac_key_metadata_from_rest(self, rest): - rest = rest.copy() - rest.pop("kind", None) - rest["project"] = "projects/" + rest.pop("projectId") - rest["create_time"] = rest.pop("timeCreated") - rest["update_time"] = rest.pop("updated") - return json_format.ParseDict(rest, storage_pb2.HmacKeyMetadata()) - - @retry_test(method="storage.hmacKey.create") - def CreateHmacKey(self, request, context): - if not request.project.startswith("projects/"): - return testbench.error.invalid( - "project name must start with projects/, got=%s" % request.project, - context, - ) - if request.service_account_email == "": - return testbench.error.invalid( - "service account email must be non-empty", context - ) - project_id = request.project[len("projects/") :] - project = self.db.get_project(project_id) - rest = project.insert_hmac_key(request.service_account_email) - return storage_pb2.CreateHmacKeyResponse( - secret_key_bytes=base64.b64decode(rest.get("secret").encode("utf-8")), - metadata=self._hmac_key_metadata_from_rest(rest.get("metadata")), - ) - - @retry_test(method="storage.hmacKey.delete") - def DeleteHmacKey(self, request, context): - if not request.project.startswith("projects/"): - return testbench.error.invalid( - "project name must start with projects/, got=%s" % request.project, - context, - ) - project_id = request.project[len("projects/") :] - project = self.db.get_project(project_id) - project.delete_hmac_key(request.access_id, context) - return empty_pb2.Empty() - - @retry_test("storage.hmacKey.get") - def GetHmacKey(self, request, context): - if not request.project.startswith("projects/"): - return testbench.error.invalid( - "project name must start with projects/, got=%s" % request.project, - context, - ) - project_id = request.project[len("projects/") :] - project = self.db.get_project(project_id) - rest = project.get_hmac_key(request.access_id, context) - return self._hmac_key_metadata_from_rest(rest) - - @retry_test("storage.hmacKey.list") - def ListHmacKeys(self, request, context): - if not request.project.startswith("projects/"): - return testbench.error.invalid( - "project name must start with projects/, got=%s" % request.project, - context, - ) - project_id = request.project[len("projects/") :] - project = self.db.get_project(project_id) - - items = [] - sa_email = request.service_account_email - if len(sa_email) != 0: - service_account = project.service_account(sa_email) - if service_account: - items = service_account.key_items() - else: - for sa in project.service_accounts.values(): - items.extend(sa.key_items()) - - state_filter = lambda x: x.get("state") != "DELETED" - if request.show_deleted_keys: - state_filter = lambda x: True - - return storage_pb2.ListHmacKeysResponse( - hmac_keys=[ - self._hmac_key_metadata_from_rest(i) for i in items if state_filter(i) - ] - ) - - @retry_test(method="storage.hmacKey.update") - def UpdateHmacKey(self, request, context): - if request.update_mask.paths == []: - return testbench.error.invalid( - "UpdateHmacKey() with an empty update mask", context - ) - if request.update_mask.paths != ["state"]: - return testbench.error.invalid( - "UpdateHmacKey() only the `state` field can be modified [%s]" - % ",".join(request.update_mask.paths), - context, - ) - project_id = request.hmac_key.project - if not project_id.startswith("projects/"): - return testbench.error.invalid( - "project name must start with projects/, got=%s" % project_id, context - ) - project_id = project_id[len("projects/") :] - project = self.db.get_project(project_id) - payload = {"state": request.hmac_key.state} - if request.hmac_key.etag != "": - payload["etag"] = request.hmac_key.etag - rest = project.update_hmac_key(request.hmac_key.access_id, payload, context) - return self._hmac_key_metadata_from_rest(rest) - - def run(port, database, echo_metadata=False): server = grpc.server(futures.ThreadPoolExecutor(max_workers=1)) storage_pb2_grpc.add_StorageServicer_to_server( diff --git a/tests/test_grpc_server.py b/tests/test_grpc_server.py index 3c178832..bd4e0a49 100755 --- a/tests/test_grpc_server.py +++ b/tests/test_grpc_server.py @@ -638,180 +638,6 @@ def test_update_bucket_soft_delete(self): grpc.StatusCode.INVALID_ARGUMENT, unittest.mock.ANY ) - def test_delete_notification(self): - context = unittest.mock.Mock() - create = self.grpc.CreateNotificationConfig( - storage_pb2.CreateNotificationConfigRequest( - parent="projects/_/buckets/bucket-name", - notification_config=storage_pb2.NotificationConfig( - topic="//pubsub.googleapis.com/projects/test-project-id/topics/test-topic", - custom_attributes={"key": "value"}, - payload_format="JSON_API_V1", - ), - ), - context, - ) - self.assertTrue( - create.name.startswith( - "projects/_/buckets/bucket-name/notificationConfigs/" - ), - msg=create, - ) - - context = unittest.mock.Mock() - _ = self.grpc.DeleteNotificationConfig( - storage_pb2.GetNotificationConfigRequest(name=create.name), context - ) - context.abort.assert_not_called() - - # The code depends on `context.abort()` raising an exception. - context = unittest.mock.Mock() - context.abort = unittest.mock.MagicMock() - context.abort.side_effect = grpc.RpcError() - with self.assertRaises(grpc.RpcError): - _ = self.grpc.GetNotificationConfig( - storage_pb2.GetNotificationConfigRequest(name=create.name), context - ) - context.abort.assert_called_once_with( - grpc.StatusCode.NOT_FOUND, unittest.mock.ANY - ) - - def test_delete_notification_malformed(self): - context = unittest.mock.Mock() - _ = self.grpc.DeleteNotificationConfig( - storage_pb2.GetNotificationConfigRequest(name=""), context - ) - context.abort.assert_called_once_with( - grpc.StatusCode.INVALID_ARGUMENT, unittest.mock.ANY - ) - - def test_get_notification(self): - context = unittest.mock.Mock() - topic_name = ( - "//pubsub.googleapis.com/projects/test-project-id/topics/test-topic" - ) - response = self.grpc.CreateNotificationConfig( - storage_pb2.CreateNotificationConfigRequest( - parent="projects/_/buckets/bucket-name", - notification_config=storage_pb2.NotificationConfig( - topic=topic_name, - custom_attributes={"key": "value"}, - payload_format="JSON_API_V1", - ), - ), - context, - ) - self.assertTrue( - response.name.startswith( - "projects/_/buckets/bucket-name/notificationConfigs/" - ), - msg=response, - ) - - context = unittest.mock.Mock() - get = self.grpc.GetNotificationConfig( - storage_pb2.GetNotificationConfigRequest(name=response.name), context - ) - self.assertEqual(get, response) - self.assertEqual(get.topic, topic_name) - - def test_get_notification_malformed(self): - context = unittest.mock.Mock() - _ = self.grpc.GetNotificationConfig( - storage_pb2.GetNotificationConfigRequest( - name="projects/_/buckets/bucket-name/" - ), - context, - ) - context.abort.assert_called_once_with( - grpc.StatusCode.INVALID_ARGUMENT, unittest.mock.ANY - ) - - def test_create_notification(self): - topic_name = ( - "//pubsub.googleapis.com/projects/test-project-id/topics/test-topic" - ) - context = unittest.mock.Mock() - response = self.grpc.CreateNotificationConfig( - storage_pb2.CreateNotificationConfigRequest( - parent="projects/_/buckets/bucket-name", - notification_config=storage_pb2.NotificationConfig( - topic=topic_name, - custom_attributes={"key": "value"}, - payload_format="JSON_API_V1", - ), - ), - context, - ) - self.assertTrue( - response.name.startswith( - "projects/_/buckets/bucket-name/notificationConfigs/" - ), - msg=response, - ) - self.assertEqual(response.topic, topic_name) - - # Invalid topic names return an error - invalid_topic_names = [ - "", - "//pubsub.googleapis.com", - "//pubsub.googleapis.com/", - "//pubsub.googleapis.com/projects", - "//pubsub.googleapis.com/projects/", - "//pubsub.googleapis.com/projects/test-project-id", - "//pubsub.googleapis.com/projects/test-project-id/", - "//pubsub.googleapis.com/projects/test-project-id/topics", - "//pubsub.googleapis.com/projects/test-project-id/topics/", - ] - for topic in invalid_topic_names: - context = unittest.mock.Mock() - context.abort = unittest.mock.MagicMock() - context.abort.side_effect = grpc.RpcError() - with self.assertRaises(grpc.RpcError): - _ = self.grpc.CreateNotificationConfig( - storage_pb2.CreateNotificationConfigRequest( - parent="projects/_/buckets/bucket-name", - notification_config=storage_pb2.NotificationConfig( - topic=topic, payload_format="JSON_API_V1" - ), - ), - context, - ) - context.abort.assert_called_once_with( - grpc.StatusCode.INVALID_ARGUMENT, unittest.mock.ANY - ) - - def test_list_notifications(self): - expected = set() - topics = [ - "//pubsub.googleapis.com/projects/test-project-id/topics/test-topic-1", - "//pubsub.googleapis.com/projects/test-project-id/topics/test-topic-2", - ] - for topic in topics: - context = unittest.mock.Mock() - response = self.grpc.CreateNotificationConfig( - storage_pb2.CreateNotificationConfigRequest( - parent="projects/_/buckets/bucket-name", - notification_config=storage_pb2.NotificationConfig( - topic=topic, - custom_attributes={"key": "value"}, - payload_format="JSON_API_V1", - ), - ), - context, - ) - expected.add(response.name) - - context = unittest.mock.Mock() - response = self.grpc.ListNotificationConfigs( - storage_pb2.ListNotificationConfigsRequest( - parent="projects/_/buckets/bucket-name" - ), - context, - ) - names = {n.name for n in response.notification_configs} - self.assertEqual(names, expected) - def test_compose_object(self): payloads = { "fox": b"The quick brown fox jumps over the lazy dog\n", @@ -1867,298 +1693,6 @@ def test_list_objects_trailing_delimiters(self): response_names = [o.name for o in response.objects] self.assertEqual(response_names, case["expected"], msg=case) - def test_get_service_account(self): - context = unittest.mock.Mock() - response = self.grpc.GetServiceAccount( - storage_pb2.GetServiceAccountRequest(project="projects/test-project"), - context, - ) - self.assertIn("@", response.email_address) - - def test_get_service_account_failure(self): - context = unittest.mock.Mock() - _ = self.grpc.GetServiceAccount( - storage_pb2.GetServiceAccountRequest(project="invalid-format"), - context, - ) - context.abort.assert_called_once_with( - grpc.StatusCode.INVALID_ARGUMENT, unittest.mock.ANY - ) - - def test_create_hmac_key(self): - sa_email = "test-sa@test-project-id.iam.gserviceaccount.com" - context = unittest.mock.Mock() - response = self.grpc.CreateHmacKey( - storage_pb2.CreateHmacKeyRequest( - project="projects/test-project-id", - service_account_email=sa_email, - ), - context, - ) - self.assertNotEqual(response.secret_key_bytes, "") - self.assertNotEqual(response.metadata.id, "") - self.assertNotEqual(response.metadata.access_id, "") - self.assertEqual(response.metadata.service_account_email, sa_email) - self.assertNotEqual(response.metadata.etag, "") - - context = unittest.mock.Mock() - _ = self.grpc.CreateHmacKey( - storage_pb2.CreateHmacKeyRequest( - project="invalid-project-name-format", - service_account_email=sa_email, - ), - context, - ) - context.abort.assert_called_once_with( - grpc.StatusCode.INVALID_ARGUMENT, unittest.mock.ANY - ) - - # Missing service account emails should fail - context = unittest.mock.Mock() - _ = self.grpc.CreateHmacKey( - storage_pb2.CreateHmacKeyRequest( - project="projects/test-project-id", - ), - context, - ) - context.abort.assert_called_once_with( - grpc.StatusCode.INVALID_ARGUMENT, unittest.mock.ANY - ) - - def test_delete_hmac_key(self): - sa_email = "test-sa@test-project-id.iam.gserviceaccount.com" - context = unittest.mock.Mock() - create = self.grpc.CreateHmacKey( - storage_pb2.CreateHmacKeyRequest( - project="projects/test-project-id", - service_account_email=sa_email, - ), - context, - ) - - metadata = storage_pb2.HmacKeyMetadata() - metadata.CopyFrom(create.metadata) - metadata.state = "INACTIVE" - - context = unittest.mock.Mock() - response = self.grpc.UpdateHmacKey( - storage_pb2.UpdateHmacKeyRequest( - hmac_key=metadata, update_mask=field_mask_pb2.FieldMask(paths=["state"]) - ), - context, - ) - self.assertEqual(response.state, "INACTIVE") - - context = unittest.mock.Mock() - _ = self.grpc.DeleteHmacKey( - storage_pb2.DeleteHmacKeyRequest( - access_id=metadata.access_id, project=metadata.project - ), - context, - ) - context.abort.assert_not_called() - - # The code depends on `context.abort()` raising an exception. - context.abort = unittest.mock.MagicMock() - context.abort.side_effect = grpc.RpcError() - with self.assertRaises(grpc.RpcError): - _ = self.grpc.GetHmacKey( - storage_pb2.GetHmacKeyRequest( - access_id=metadata.access_id, project="projects/test-project-id" - ), - context, - ) - context.abort.assert_called_once_with( - grpc.StatusCode.NOT_FOUND, unittest.mock.ANY - ) - - # Missing or malformed project id is an error - context = unittest.mock.Mock() - _ = self.grpc.DeleteHmacKey( - storage_pb2.DeleteHmacKeyRequest(access_id=metadata.access_id), - context, - ) - context.abort.assert_called_once_with( - grpc.StatusCode.INVALID_ARGUMENT, unittest.mock.ANY - ) - - def test_get_hmac_key(self): - sa_email = "test-sa@test-project-id.iam.gserviceaccount.com" - context = unittest.mock.Mock() - create = self.grpc.CreateHmacKey( - storage_pb2.CreateHmacKeyRequest( - project="projects/test-project-id", - service_account_email=sa_email, - ), - context, - ) - - context = unittest.mock.Mock() - response = self.grpc.GetHmacKey( - storage_pb2.GetHmacKeyRequest( - access_id=create.metadata.access_id, project="projects/test-project-id" - ), - context, - ) - self.assertEqual(response, create.metadata) - - # Missing or malformed project id is an error - context = unittest.mock.Mock() - _ = self.grpc.GetHmacKey( - storage_pb2.GetHmacKeyRequest(access_id=create.metadata.access_id), - context, - ) - context.abort.assert_called_once_with( - grpc.StatusCode.INVALID_ARGUMENT, unittest.mock.ANY - ) - - def test_list_hmac_keys(self): - # Create several keys for two different projects. - expected = [] - sa_emails = [ - "test-sa-1@test-project-id.iam.gserviceaccount.com", - "test-sa-2@test-project-id.iam.gserviceaccount.com", - ] - for sa in sa_emails: - for _ in [1, 2]: - context = unittest.mock.Mock() - create = self.grpc.CreateHmacKey( - storage_pb2.CreateHmacKeyRequest( - project="projects/test-project-id", - service_account_email=sa, - ), - context, - ) - expected.append(create.metadata) - - # First test without any filtering - context = unittest.mock.Mock() - response = self.grpc.ListHmacKeys( - storage_pb2.ListHmacKeysRequest(project="projects/test-project-id"), - context, - ) - expected_access_ids = {k.access_id for k in expected} - self.assertEqual({k.access_id for k in response.hmac_keys}, expected_access_ids) - - # Then only for one of the sa emails - context = unittest.mock.Mock() - sa_filter = sa_emails[0] - response = self.grpc.ListHmacKeys( - storage_pb2.ListHmacKeysRequest( - project="projects/test-project-id", service_account_email=sa_filter - ), - context, - ) - expected_access_ids = { - k.access_id for k in expected if k.service_account_email == sa_filter - } - self.assertEqual({k.access_id for k in response.hmac_keys}, expected_access_ids) - - # Include deleted accounts (there are none, but should work) - context = unittest.mock.Mock() - sa_filter = sa_emails[0] - response = self.grpc.ListHmacKeys( - storage_pb2.ListHmacKeysRequest( - project="projects/test-project-id", - service_account_email=sa_filter, - show_deleted_keys=True, - ), - context, - ) - expected_access_ids = { - k.access_id for k in expected if k.service_account_email == sa_filter - } - self.assertEqual({k.access_id for k in response.hmac_keys}, expected_access_ids) - - # Missing or malformed project id is an error - context = unittest.mock.Mock() - _ = self.grpc.ListHmacKeys(storage_pb2.ListHmacKeysRequest(), context) - context.abort.assert_called_once_with( - grpc.StatusCode.INVALID_ARGUMENT, unittest.mock.ANY - ) - - def test_update_hmac_key(self): - sa_email = "test-sa@test-project-id.iam.gserviceaccount.com" - context = unittest.mock.Mock() - create = self.grpc.CreateHmacKey( - storage_pb2.CreateHmacKeyRequest( - project="projects/test-project-id", - service_account_email=sa_email, - ), - context, - ) - - metadata = storage_pb2.HmacKeyMetadata() - metadata.CopyFrom(create.metadata) - metadata.state = "INACTIVE" - - context = unittest.mock.Mock() - response = self.grpc.UpdateHmacKey( - storage_pb2.UpdateHmacKeyRequest( - hmac_key=metadata, update_mask=field_mask_pb2.FieldMask(paths=["state"]) - ), - context, - ) - self.assertEqual(response.id, metadata.id) - self.assertEqual(response.access_id, metadata.access_id) - self.assertEqual(response.project, metadata.project) - self.assertEqual(response.service_account_email, metadata.service_account_email) - self.assertEqual(response.state, metadata.state) - self.assertEqual(response.create_time, metadata.create_time) - - # Verify a missing, empty, or invalid update mask returns an error - for mask in [[], ["not-state"], ["state", "and-more"]]: - context = unittest.mock.Mock() - _ = self.grpc.UpdateHmacKey( - storage_pb2.UpdateHmacKeyRequest( - hmac_key=metadata, update_mask=field_mask_pb2.FieldMask(paths=mask) - ), - context, - ) - context.abort.assert_called_once_with( - grpc.StatusCode.INVALID_ARGUMENT, unittest.mock.ANY - ) - - # An empty metadata attribute is also an error - context = unittest.mock.Mock() - _ = self.grpc.UpdateHmacKey( - storage_pb2.UpdateHmacKeyRequest( - hmac_key=storage_pb2.HmacKeyMetadata(), - update_mask=field_mask_pb2.FieldMask(paths=["state"]), - ), - context, - ) - context.abort.assert_called_once_with( - grpc.StatusCode.INVALID_ARGUMENT, unittest.mock.ANY - ) - - def test_update_hmac_key_bad_etag(self): - sa_email = "test-sa@test-project-id.iam.gserviceaccount.com" - context = unittest.mock.Mock() - create = self.grpc.CreateHmacKey( - storage_pb2.CreateHmacKeyRequest( - project="projects/test-project-id", - service_account_email=sa_email, - ), - context, - ) - - metadata = storage_pb2.HmacKeyMetadata() - metadata.CopyFrom(create.metadata) - metadata.state = "INACTIVE" - metadata.etag = "test-only-invalid" - - context = unittest.mock.Mock() - response = self.grpc.UpdateHmacKey( - storage_pb2.UpdateHmacKeyRequest( - hmac_key=metadata, update_mask=field_mask_pb2.FieldMask(paths=["state"]) - ), - context, - ) - context.abort.assert_called_once_with( - grpc.StatusCode.FAILED_PRECONDITION, unittest.mock.ANY - ) - def test_run(self): port, server = testbench.grpc_server.run(0, self.db) self.assertNotEqual(port, 0)