From 1a85239c2a002b660eb21cd9c524056fabf1c9ca Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Tue, 19 Nov 2019 16:46:21 +0200 Subject: [PATCH 01/20] Update test with mock --- pubsub/cloud-client/quickstart/sub_test.py | 41 +++++++++++----------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/pubsub/cloud-client/quickstart/sub_test.py b/pubsub/cloud-client/quickstart/sub_test.py index 476139a02642..5ba292f9be3f 100644 --- a/pubsub/cloud-client/quickstart/sub_test.py +++ b/pubsub/cloud-client/quickstart/sub_test.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import mock import os import pytest @@ -68,31 +69,31 @@ def _publish_messages(topic_path): publish_future.result() -def _sub_timeout(project_id, subscription_name): - # This is an exactly copy of `sub.py` except - # StreamingPullFuture.result() will time out after 10s. - client = pubsub_v1.SubscriberClient() - subscription_path = client.subscription_path( - project_id, subscription_name) +def test_sub(monkeypatch, topic_path, subscription_path, capsys): - def callback(message): - print('Received message {} of message ID {}\n'.format( - message, message.message_id)) - message.ack() - print('Acknowledged message {}\n'.format(message.message_id)) + real_client = pubsub_v1.SubscriberClient() + mock_client = mock.Mock(spec=pubsub_v1.SubscriberClient, + wraps=real_client) - streaming_pull_future = client.subscribe( - subscription_path, callback=callback) - print('Listening for messages on {}..\n'.format(subscription_path)) + # Attributes on mock_client_constructor uses the corresponding + # attributes on pubsub_v1.SubscriberClient. + mock_client_constructor = mock.create_autospec(pubsub_v1.SubscriberClient) + mock_client_constructor.return_value = mock_client - try: - streaming_pull_future.result(timeout=10) - except: # noqa - streaming_pull_future.cancel() + monkeypatch.setattr(pubsub_v1, 'SubscriberClient', mock_client_constructor) + def mock_subscribe(subscription_path, callback=None): + real_future = real_client.subscribe(subscription_path, + callback=callback) + mock_future = mock.Mock(spec=real_future, wraps=real_future) -def test_sub(monkeypatch, topic_path, subscription_path, capsys): - monkeypatch.setattr(sub, 'sub', _sub_timeout) + def mock_result(): + return real_future.result(timeout=10) + + mock_future.result.side_effect = mock_result + return mock_future + + mock_client.subscribe.side_effect = mock_subscribe _publish_messages(topic_path) From 666b6bd4e96e9ed859c8ec9a70377dd4c10a0131 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Mon, 2 Dec 2019 11:20:45 -0800 Subject: [PATCH 02/20] clean up resources after tests --- pubsub/cloud-client/subscriber_test.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pubsub/cloud-client/subscriber_test.py b/pubsub/cloud-client/subscriber_test.py index 4c5fd61223db..3fc5278a416f 100644 --- a/pubsub/cloud-client/subscriber_test.py +++ b/pubsub/cloud-client/subscriber_test.py @@ -185,6 +185,20 @@ def new_sleep(period): return mock.patch('time.sleep', new=new_sleep) +def _to_delete(): + publisher_client = pubsub_v1.PublisherClient() + subscriber_client = pubsub_v1.SubscriberClient() + resources = [TOPIC, SUBSCRIPTION_TWO, SUBSCRIPTION_THREE] + + for item in resources: + if 'subscription-test-topic' in item: + publisher_client.delete_topic( + 'projects/{}/topics/{}'.format(PROJECT, item)) + if 'subscription-test-subscription' in item: + subscriber_client.delete_subscription( + 'projects/{}/subscriptions/{}'.format(PROJECT, item)) + + def test_receive(publisher_client, topic, subscription_two, capsys): _publish_messages(publisher_client, topic) @@ -249,3 +263,6 @@ def test_receive_synchronously_with_lease( out, _ = capsys.readouterr() assert 'Done.' in out + + # Clean up resources after all the tests. + _to_delete() From 0cccbf847cd4807f0e0723a4962f65fa26ee5b0b Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Mon, 2 Dec 2019 13:25:42 -0800 Subject: [PATCH 03/20] update test for async message reception --- pubsub/cloud-client/subscriber_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pubsub/cloud-client/subscriber_test.py b/pubsub/cloud-client/subscriber_test.py index 3fc5278a416f..2e81bcfbef67 100644 --- a/pubsub/cloud-client/subscriber_test.py +++ b/pubsub/cloud-client/subscriber_test.py @@ -209,7 +209,7 @@ def test_receive(publisher_client, topic, subscription_two, capsys): out, _ = capsys.readouterr() assert 'Listening' in out assert subscription_two in out - assert 'Message 1' in out + assert 'Message' in out def test_receive_with_custom_attributes( @@ -241,7 +241,7 @@ def test_receive_with_flow_control( out, _ = capsys.readouterr() assert 'Listening' in out assert subscription_two in out - assert 'Message 1' in out + assert 'Message' in out def test_receive_synchronously( From 96c66fe5220fd6b7ef34ef1db4afb3376072dd52 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Mon, 2 Dec 2019 13:35:44 -0800 Subject: [PATCH 04/20] clean up resources for all tests --- pubsub/cloud-client/iam_test.py | 12 ++++++++++++ pubsub/cloud-client/publisher_test.py | 8 ++++++++ pubsub/cloud-client/quickstart_test.py | 12 ++++++++++++ 3 files changed, 32 insertions(+) diff --git a/pubsub/cloud-client/iam_test.py b/pubsub/cloud-client/iam_test.py index 8a524c35a061..51ae99795828 100644 --- a/pubsub/cloud-client/iam_test.py +++ b/pubsub/cloud-client/iam_test.py @@ -63,6 +63,15 @@ def subscription(subscriber_client, topic): yield subscription_path +def _to_delete(): + publisher_client = pubsub_v1.PublisherClient() + subscriber_client = pubsub_v1.SubscriberClient() + publisher_client.delete_topic( + 'projects/{}/topics/{}'.format(PROJECT, TOPIC)) + subscriber_client.delete_subscription( + 'projects/{}/subscriptions/{}'.format(PROJECT, SUBSCRIPTION)) + + def test_get_topic_policy(topic, capsys): iam.get_topic_policy(PROJECT, TOPIC) @@ -109,3 +118,6 @@ def test_check_subscription_permissions(subscription, capsys): assert subscription in out assert 'pubsub.subscriptions.consume' in out + + # Clean up resources. + _to_delete() diff --git a/pubsub/cloud-client/publisher_test.py b/pubsub/cloud-client/publisher_test.py index 5e550abd641d..bffb1436daa3 100644 --- a/pubsub/cloud-client/publisher_test.py +++ b/pubsub/cloud-client/publisher_test.py @@ -56,6 +56,12 @@ def new_sleep(period): return mock.patch('time.sleep', new=new_sleep) +def _to_delete(): + publisher_client = pubsub_v1.PublisherClient() + publisher_client.delete_topic('projects/{}/topics/{}'.format( + PROJECT, TOPIC)) + + def test_list(client, topic, capsys): @eventually_consistent.call def _(): @@ -127,3 +133,5 @@ def test_publish_with_futures(topic, capsys): out, _ = capsys.readouterr() assert 'Published' in out + + _to_delete() diff --git a/pubsub/cloud-client/quickstart_test.py b/pubsub/cloud-client/quickstart_test.py index d318b260c63c..85f0949bc0ca 100644 --- a/pubsub/cloud-client/quickstart_test.py +++ b/pubsub/cloud-client/quickstart_test.py @@ -61,6 +61,15 @@ def subscription(subscriber_client, topic): yield SUBSCRIPTION +def _to_delete(): + publisher_client = pubsub_v1.PublisherClient() + subscriber_client = pubsub_v1.SubscriberClient() + publisher_client.delete_topic( + 'projects/{}/topics/{}'.format(PROJECT, TOPIC)) + subscriber_client.delete_subscription( + 'projects/{}/subscriptions/{}'.format(PROJECT, SUBSCRIPTION)) + + def test_end_to_end(topic, subscription, capsys): quickstart.end_to_end(PROJECT, topic, subscription, N) @@ -69,3 +78,6 @@ def test_end_to_end(topic, subscription, capsys): assert "Received all messages" in out assert "Publish time lapsed" in out assert "Subscribe time lapsed" in out + + # Clean up resources. + _to_delete() From 3acf66ad83686c95c26ceb72c9658428ae35fe14 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Mon, 2 Dec 2019 13:44:18 -0800 Subject: [PATCH 05/20] use unique resource names avoid test failures --- pubsub/cloud-client/iam_test.py | 6 ++++-- pubsub/cloud-client/publisher_test.py | 4 +++- pubsub/cloud-client/quickstart_test.py | 6 ++++-- pubsub/cloud-client/subscriber_test.py | 10 ++++++---- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/pubsub/cloud-client/iam_test.py b/pubsub/cloud-client/iam_test.py index 51ae99795828..6eeedfb5ec07 100644 --- a/pubsub/cloud-client/iam_test.py +++ b/pubsub/cloud-client/iam_test.py @@ -13,15 +13,17 @@ # limitations under the License. import os +import uuid from google.cloud import pubsub_v1 import pytest import iam +UUID = uuid.uuid4().hex PROJECT = os.environ['GCLOUD_PROJECT'] -TOPIC = 'iam-test-topic' -SUBSCRIPTION = 'iam-test-subscription' +TOPIC = 'iam-test-topic-' + UUID +SUBSCRIPTION = 'iam-test-subscription-' + UUID @pytest.fixture(scope='module') diff --git a/pubsub/cloud-client/publisher_test.py b/pubsub/cloud-client/publisher_test.py index bffb1436daa3..91d9e7601f39 100644 --- a/pubsub/cloud-client/publisher_test.py +++ b/pubsub/cloud-client/publisher_test.py @@ -14,6 +14,7 @@ import os import time +import uuid from gcp_devrel.testing import eventually_consistent from google.cloud import pubsub_v1 @@ -22,8 +23,9 @@ import publisher +UUID = uuid.uuid4().hex PROJECT = os.environ['GCLOUD_PROJECT'] -TOPIC = 'publisher-test-topic' +TOPIC = 'publisher-test-topic-' + UUID @pytest.fixture diff --git a/pubsub/cloud-client/quickstart_test.py b/pubsub/cloud-client/quickstart_test.py index 85f0949bc0ca..4d628e493cd8 100644 --- a/pubsub/cloud-client/quickstart_test.py +++ b/pubsub/cloud-client/quickstart_test.py @@ -15,14 +15,16 @@ # limitations under the License. import os +import uuid from google.cloud import pubsub_v1 import pytest import quickstart +UUID = uuid.uuid4().hex PROJECT = os.environ['GCLOUD_PROJECT'] -TOPIC = 'end-to-end-test-topic' -SUBSCRIPTION = 'end-to-end-test-topic-sub' +TOPIC = 'end-to-end-test-topic-' + UUID +SUBSCRIPTION = 'end-to-end-test-topic-sub-' + UUID N = 10 diff --git a/pubsub/cloud-client/subscriber_test.py b/pubsub/cloud-client/subscriber_test.py index 2e81bcfbef67..4576f90cfdbb 100644 --- a/pubsub/cloud-client/subscriber_test.py +++ b/pubsub/cloud-client/subscriber_test.py @@ -14,6 +14,7 @@ import os import time +import uuid from gcp_devrel.testing import eventually_consistent from google.cloud import pubsub_v1 @@ -22,11 +23,12 @@ import subscriber +UUID = uuid.uuid4().hex PROJECT = os.environ['GCLOUD_PROJECT'] -TOPIC = 'subscription-test-topic' -SUBSCRIPTION_ONE = 'subscription-test-subscription-one' -SUBSCRIPTION_TWO = 'subscription-test-subscription-two' -SUBSCRIPTION_THREE = 'subscription-test-subscription-three' +TOPIC = 'subscription-test-topic-' + UUID +SUBSCRIPTION_ONE = 'subscription-test-subscription-one-' + UUID +SUBSCRIPTION_TWO = 'subscription-test-subscription-two-' + UUID +SUBSCRIPTION_THREE = 'subscription-test-subscription-three-' + UUID ENDPOINT = 'https://{}.appspot.com/push'.format(PROJECT) NEW_ENDPOINT = 'https://{}.appspot.com/push2'.format(PROJECT) From 5eba4fa4d08e57e5c00665491cc5a23ba022a114 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Mon, 2 Dec 2019 14:19:38 -0800 Subject: [PATCH 06/20] also delete subscriptions in cleanup phase --- pubsub/streaming-analytics/PubSubToGCS_test.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pubsub/streaming-analytics/PubSubToGCS_test.py b/pubsub/streaming-analytics/PubSubToGCS_test.py index 644cf0865d06..0cc4356e8ec3 100644 --- a/pubsub/streaming-analytics/PubSubToGCS_test.py +++ b/pubsub/streaming-analytics/PubSubToGCS_test.py @@ -26,8 +26,8 @@ PROJECT = os.environ['GCLOUD_PROJECT'] BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] -TOPIC = 'test-topic' -UUID = uuid.uuid4().hex +UUID = uuid.uuid1().hex +TOPIC = 'test-topic-' + UUID @pytest.fixture @@ -95,6 +95,10 @@ def test_run(publisher_client, topic_path): files = gcs_client.list_prefix('gs://{}/pubsub/{}'.format(BUCKET, UUID)) assert len(files) > 0 - # Clean up. Delete topic. Delete files. + # Clean up. Delete subscription. Delete topic. Delete GCS files. + subscriber_client = pubsub_v1.SubscriberClient() + subscriptions = publisher_client.list_topic_subscriptions(topic_path) + for subscription in subscriptions: + subscriber_client.delete_subscription(subscription) publisher_client.delete_topic(topic_path) gcs_client.delete_batch(list(files)) From 475a57d27bff3637339fc22d3ae8f121ae3ac0ae Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Fri, 6 Dec 2019 10:56:26 -0800 Subject: [PATCH 07/20] tim's suggestions --- pubsub/cloud-client/iam_test.py | 14 +++-------- pubsub/cloud-client/quickstart/pub_test.py | 12 ++-------- pubsub/cloud-client/quickstart/sub_test.py | 27 +++++++++------------- pubsub/cloud-client/quickstart_test.py | 14 +++-------- 4 files changed, 19 insertions(+), 48 deletions(-) diff --git a/pubsub/cloud-client/iam_test.py b/pubsub/cloud-client/iam_test.py index 6eeedfb5ec07..2223fa4672e1 100644 --- a/pubsub/cloud-client/iam_test.py +++ b/pubsub/cloud-client/iam_test.py @@ -44,6 +44,8 @@ def topic(publisher_client): yield topic_path + publisher_client.delete_topic(topic_path) + @pytest.fixture(scope='module') def subscriber_client(): @@ -64,14 +66,7 @@ def subscription(subscriber_client, topic): yield subscription_path - -def _to_delete(): - publisher_client = pubsub_v1.PublisherClient() - subscriber_client = pubsub_v1.SubscriberClient() - publisher_client.delete_topic( - 'projects/{}/topics/{}'.format(PROJECT, TOPIC)) - subscriber_client.delete_subscription( - 'projects/{}/subscriptions/{}'.format(PROJECT, SUBSCRIPTION)) + subscriber_client.delete_subscription(subscription_path) def test_get_topic_policy(topic, capsys): @@ -120,6 +115,3 @@ def test_check_subscription_permissions(subscription, capsys): assert subscription in out assert 'pubsub.subscriptions.consume' in out - - # Clean up resources. - _to_delete() diff --git a/pubsub/cloud-client/quickstart/pub_test.py b/pubsub/cloud-client/quickstart/pub_test.py index 09443364a3f6..0751ad59363d 100644 --- a/pubsub/cloud-client/quickstart/pub_test.py +++ b/pubsub/cloud-client/quickstart/pub_test.py @@ -42,20 +42,12 @@ def topic(publisher_client): yield TOPIC + publisher_client.delete_topic(topic_path) -@pytest.fixture -def to_delete(publisher_client): - doomed = [] - yield doomed - for item in doomed: - publisher_client.delete_topic(item) - -def test_pub(publisher_client, topic, to_delete, capsys): +def test_pub(publisher_client, topic, capsys): pub.pub(PROJECT, topic) - to_delete.append('projects/{}/topics/{}'.format(PROJECT, TOPIC)) - out, _ = capsys.readouterr() assert "Published message b'Hello, World!'" in out diff --git a/pubsub/cloud-client/quickstart/sub_test.py b/pubsub/cloud-client/quickstart/sub_test.py index 5ba292f9be3f..7df20a437a35 100644 --- a/pubsub/cloud-client/quickstart/sub_test.py +++ b/pubsub/cloud-client/quickstart/sub_test.py @@ -17,6 +17,7 @@ import mock import os import pytest +import uuid from google.api_core.exceptions import AlreadyExists from google.cloud import pubsub_v1 @@ -24,9 +25,10 @@ import sub +UUID = uuid.uuid4().hex PROJECT = os.environ['GCLOUD_PROJECT'] -TOPIC = 'quickstart-sub-test-topic' -SUBSCRIPTION = 'quickstart-sub-test-topic-sub' +TOPIC = 'quickstart-sub-test-topic-' + UUID +SUBSCRIPTION = 'quickstart-sub-test-topic-sub-' + UUID publisher_client = pubsub_v1.PublisherClient() subscriber_client = pubsub_v1.SubscriberClient() @@ -38,9 +40,11 @@ def topic_path(): try: topic = publisher_client.create_topic(topic_path) - return topic.name + yield topic.name except AlreadyExists: - return topic_path + yield topic_path + + publisher_client.delete_topic(topic_path) @pytest.fixture(scope='module') @@ -51,17 +55,11 @@ def subscription_path(topic_path): try: subscription = subscriber_client.create_subscription( subscription_path, topic_path) - return subscription.name + yield subscription.name except AlreadyExists: - return subscription_path - + yield subscription_path -def _to_delete(resource_paths): - for item in resource_paths: - if 'topics' in item: - publisher_client.delete_topic(item) - if 'subscriptions' in item: - subscriber_client.delete_subscription(item) + subscriber_client.delete_subscription(subscription_path) def _publish_messages(topic_path): @@ -99,9 +97,6 @@ def mock_result(): sub.sub(PROJECT, SUBSCRIPTION) - # Clean up resources. - _to_delete([topic_path, subscription_path]) - out, _ = capsys.readouterr() assert "Received message" in out assert "Acknowledged message" in out diff --git a/pubsub/cloud-client/quickstart_test.py b/pubsub/cloud-client/quickstart_test.py index 4d628e493cd8..c22e2172a6ab 100644 --- a/pubsub/cloud-client/quickstart_test.py +++ b/pubsub/cloud-client/quickstart_test.py @@ -44,6 +44,8 @@ def topic(publisher_client): yield TOPIC + publisher_client.delete_topic(topic_path) + @pytest.fixture(scope='module') def subscriber_client(): @@ -62,14 +64,7 @@ def subscription(subscriber_client, topic): yield SUBSCRIPTION - -def _to_delete(): - publisher_client = pubsub_v1.PublisherClient() - subscriber_client = pubsub_v1.SubscriberClient() - publisher_client.delete_topic( - 'projects/{}/topics/{}'.format(PROJECT, TOPIC)) - subscriber_client.delete_subscription( - 'projects/{}/subscriptions/{}'.format(PROJECT, SUBSCRIPTION)) + subscriber_client.delete_subscription(subscription_path) def test_end_to_end(topic, subscription, capsys): @@ -80,6 +75,3 @@ def test_end_to_end(topic, subscription, capsys): assert "Received all messages" in out assert "Publish time lapsed" in out assert "Subscribe time lapsed" in out - - # Clean up resources. - _to_delete() From bca6eef72253592db2ab604bd9aec8bcc98f0d76 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Fri, 6 Dec 2019 11:37:09 -0800 Subject: [PATCH 08/20] unique topic name --- pubsub/cloud-client/quickstart/pub_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pubsub/cloud-client/quickstart/pub_test.py b/pubsub/cloud-client/quickstart/pub_test.py index 0751ad59363d..db6d876b2483 100644 --- a/pubsub/cloud-client/quickstart/pub_test.py +++ b/pubsub/cloud-client/quickstart/pub_test.py @@ -16,14 +16,16 @@ import os import pytest +import uuid from google.api_core.exceptions import AlreadyExists from google.cloud import pubsub_v1 import pub +UUID = uuid.uuid4().hex PROJECT = os.environ['GCLOUD_PROJECT'] -TOPIC = 'quickstart-pub-test-topic' +TOPIC = 'quickstart-pub-test-topic-' + UUID @pytest.fixture(scope='module') From 06d0ce68185ee66d4026e4f99432fff664cfacda Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Fri, 6 Dec 2019 13:21:47 -0800 Subject: [PATCH 09/20] reformat --- pubsub/cloud-client/iam.py | 109 ++++--- pubsub/cloud-client/iam_test.py | 27 +- pubsub/cloud-client/publisher.py | 161 +++++------ pubsub/cloud-client/publisher_test.py | 23 +- pubsub/cloud-client/quickstart.py | 38 ++- pubsub/cloud-client/quickstart/pub.py | 27 +- pubsub/cloud-client/quickstart/pub_test.py | 8 +- pubsub/cloud-client/quickstart/sub.py | 26 +- pubsub/cloud-client/quickstart/sub_test.py | 26 +- pubsub/cloud-client/quickstart_test.py | 17 +- pubsub/cloud-client/subscriber.py | 271 +++++++++--------- pubsub/cloud-client/subscriber_test.py | 120 ++++---- pubsub/streaming-analytics/PubSubToGCS.py | 86 +++--- .../streaming-analytics/PubSubToGCS_test.py | 38 ++- 14 files changed, 481 insertions(+), 496 deletions(-) diff --git a/pubsub/cloud-client/iam.py b/pubsub/cloud-client/iam.py index f9865ed3934e..44e52e576295 100644 --- a/pubsub/cloud-client/iam.py +++ b/pubsub/cloud-client/iam.py @@ -34,9 +34,9 @@ def get_topic_policy(project, topic_name): policy = client.get_iam_policy(topic_path) - print('Policy for topic {}:'.format(topic_path)) + print("Policy for topic {}:".format(topic_path)) for binding in policy.bindings: - print('Role: {}, Members: {}'.format(binding.role, binding.members)) + print("Role: {}, Members: {}".format(binding.role, binding.members)) # [END pubsub_get_topic_policy] @@ -48,9 +48,9 @@ def get_subscription_policy(project, subscription_name): policy = client.get_iam_policy(subscription_path) - print('Policy for subscription {}:'.format(subscription_path)) + print("Policy for subscription {}:".format(subscription_path)) for binding in policy.bindings: - print('Role: {}, Members: {}'.format(binding.role, binding.members)) + print("Role: {}, Members: {}".format(binding.role, binding.members)) # [END pubsub_get_subscription_policy] @@ -63,20 +63,17 @@ def set_topic_policy(project, topic_name): policy = client.get_iam_policy(topic_path) # Add all users as viewers. - policy.bindings.add( - role='roles/pubsub.viewer', - members=['allUsers']) + policy.bindings.add(role="roles/pubsub.viewer", members=["allUsers"]) # Add a group as a publisher. policy.bindings.add( - role='roles/pubsub.publisher', - members=['group:cloud-logs@google.com']) + role="roles/pubsub.publisher", members=["group:cloud-logs@google.com"] + ) # Set the policy policy = client.set_iam_policy(topic_path, policy) - print('IAM policy for topic {} set: {}'.format( - topic_name, policy)) + print("IAM policy for topic {} set: {}".format(topic_name, policy)) # [END pubsub_set_topic_policy] @@ -89,20 +86,15 @@ def set_subscription_policy(project, subscription_name): policy = client.get_iam_policy(subscription_path) # Add all users as viewers. - policy.bindings.add( - role='roles/pubsub.viewer', - members=['allUsers']) + policy.bindings.add(role="roles/pubsub.viewer", members=["allUsers"]) # Add a group as an editor. - policy.bindings.add( - role='roles/editor', - members=['group:cloud-logs@google.com']) + policy.bindings.add(role="roles/editor", members=["group:cloud-logs@google.com"]) # Set the policy policy = client.set_iam_policy(subscription_path, policy) - print('IAM policy for subscription {} set: {}'.format( - subscription_name, policy)) + print("IAM policy for subscription {} set: {}".format(subscription_name, policy)) # [END pubsub_set_subscription_policy] @@ -112,16 +104,13 @@ def check_topic_permissions(project, topic_name): client = pubsub_v1.PublisherClient() topic_path = client.topic_path(project, topic_name) - permissions_to_check = [ - 'pubsub.topics.publish', - 'pubsub.topics.update' - ] + permissions_to_check = ["pubsub.topics.publish", "pubsub.topics.update"] - allowed_permissions = client.test_iam_permissions( - topic_path, permissions_to_check) + allowed_permissions = client.test_iam_permissions(topic_path, permissions_to_check) - print('Allowed permissions for topic {}: {}'.format( - topic_path, allowed_permissions)) + print( + "Allowed permissions for topic {}: {}".format(topic_path, allowed_permissions) + ) # [END pubsub_test_topic_permissions] @@ -132,63 +121,71 @@ def check_subscription_permissions(project, subscription_name): subscription_path = client.subscription_path(project, subscription_name) permissions_to_check = [ - 'pubsub.subscriptions.consume', - 'pubsub.subscriptions.update' + "pubsub.subscriptions.consume", + "pubsub.subscriptions.update", ] allowed_permissions = client.test_iam_permissions( - subscription_path, permissions_to_check) + subscription_path, permissions_to_check + ) - print('Allowed permissions for subscription {}: {}'.format( - subscription_path, allowed_permissions)) + print( + "Allowed permissions for subscription {}: {}".format( + subscription_path, allowed_permissions + ) + ) # [END pubsub_test_subscription_permissions] -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) - parser.add_argument('project', help='Your Google Cloud project ID') + parser.add_argument("project", help="Your Google Cloud project ID") - subparsers = parser.add_subparsers(dest='command') + subparsers = parser.add_subparsers(dest="command") get_topic_policy_parser = subparsers.add_parser( - 'get-topic-policy', help=get_topic_policy.__doc__) - get_topic_policy_parser.add_argument('topic_name') + "get-topic-policy", help=get_topic_policy.__doc__ + ) + get_topic_policy_parser.add_argument("topic_name") get_subscription_policy_parser = subparsers.add_parser( - 'get-subscription-policy', help=get_subscription_policy.__doc__) - get_subscription_policy_parser.add_argument('subscription_name') + "get-subscription-policy", help=get_subscription_policy.__doc__ + ) + get_subscription_policy_parser.add_argument("subscription_name") set_topic_policy_parser = subparsers.add_parser( - 'set-topic-policy', help=set_topic_policy.__doc__) - set_topic_policy_parser.add_argument('topic_name') + "set-topic-policy", help=set_topic_policy.__doc__ + ) + set_topic_policy_parser.add_argument("topic_name") set_subscription_policy_parser = subparsers.add_parser( - 'set-subscription-policy', help=set_subscription_policy.__doc__) - set_subscription_policy_parser.add_argument('subscription_name') + "set-subscription-policy", help=set_subscription_policy.__doc__ + ) + set_subscription_policy_parser.add_argument("subscription_name") check_topic_permissions_parser = subparsers.add_parser( - 'check-topic-permissions', help=check_topic_permissions.__doc__) - check_topic_permissions_parser.add_argument('topic_name') + "check-topic-permissions", help=check_topic_permissions.__doc__ + ) + check_topic_permissions_parser.add_argument("topic_name") check_subscription_permissions_parser = subparsers.add_parser( - 'check-subscription-permissions', - help=check_subscription_permissions.__doc__) - check_subscription_permissions_parser.add_argument('subscription_name') + "check-subscription-permissions", help=check_subscription_permissions.__doc__ + ) + check_subscription_permissions_parser.add_argument("subscription_name") args = parser.parse_args() - if args.command == 'get-topic-policy': + if args.command == "get-topic-policy": get_topic_policy(args.project, args.topic_name) - elif args.command == 'get-subscription-policy': + elif args.command == "get-subscription-policy": get_subscription_policy(args.project, args.subscription_name) - elif args.command == 'set-topic-policy': + elif args.command == "set-topic-policy": set_topic_policy(args.project, args.topic_name) - elif args.command == 'set-subscription-policy': + elif args.command == "set-subscription-policy": set_subscription_policy(args.project, args.subscription_name) - elif args.command == 'check-topic-permissions': + elif args.command == "check-topic-permissions": check_topic_permissions(args.project, args.topic_name) - elif args.command == 'check-subscription-permissions': + elif args.command == "check-subscription-permissions": check_subscription_permissions(args.project, args.subscription_name) diff --git a/pubsub/cloud-client/iam_test.py b/pubsub/cloud-client/iam_test.py index 2223fa4672e1..ac62bbfc4f31 100644 --- a/pubsub/cloud-client/iam_test.py +++ b/pubsub/cloud-client/iam_test.py @@ -21,17 +21,17 @@ import iam UUID = uuid.uuid4().hex -PROJECT = os.environ['GCLOUD_PROJECT'] -TOPIC = 'iam-test-topic-' + UUID -SUBSCRIPTION = 'iam-test-subscription-' + UUID +PROJECT = os.environ["GCLOUD_PROJECT"] +TOPIC = "iam-test-topic-" + UUID +SUBSCRIPTION = "iam-test-subscription-" + UUID -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def publisher_client(): yield pubsub_v1.PublisherClient() -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def topic(publisher_client): topic_path = publisher_client.topic_path(PROJECT, TOPIC) @@ -47,15 +47,14 @@ def topic(publisher_client): publisher_client.delete_topic(topic_path) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def subscriber_client(): yield pubsub_v1.SubscriberClient() @pytest.fixture def subscription(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION) try: subscriber_client.delete_subscription(subscription_path) @@ -87,16 +86,16 @@ def test_set_topic_policy(publisher_client, topic): iam.set_topic_policy(PROJECT, TOPIC) policy = publisher_client.get_iam_policy(topic) - assert 'roles/pubsub.publisher' in str(policy) - assert 'allUsers' in str(policy) + assert "roles/pubsub.publisher" in str(policy) + assert "allUsers" in str(policy) def test_set_subscription_policy(subscriber_client, subscription): iam.set_subscription_policy(PROJECT, SUBSCRIPTION) policy = subscriber_client.get_iam_policy(subscription) - assert 'roles/pubsub.viewer' in str(policy) - assert 'allUsers' in str(policy) + assert "roles/pubsub.viewer" in str(policy) + assert "allUsers" in str(policy) def test_check_topic_permissions(topic, capsys): @@ -105,7 +104,7 @@ def test_check_topic_permissions(topic, capsys): out, _ = capsys.readouterr() assert topic in out - assert 'pubsub.topics.publish' in out + assert "pubsub.topics.publish" in out def test_check_subscription_permissions(subscription, capsys): @@ -114,4 +113,4 @@ def test_check_subscription_permissions(subscription, capsys): out, _ = capsys.readouterr() assert subscription in out - assert 'pubsub.subscriptions.consume' in out + assert "pubsub.subscriptions.consume" in out diff --git a/pubsub/cloud-client/publisher.py b/pubsub/cloud-client/publisher.py index 490c903b2c1b..0fd23cdba694 100644 --- a/pubsub/cloud-client/publisher.py +++ b/pubsub/cloud-client/publisher.py @@ -53,7 +53,7 @@ def create_topic(project_id, topic_name): topic = publisher.create_topic(topic_path) - print('Topic created: {}'.format(topic)) + print("Topic created: {}".format(topic)) # [END pubsub_quickstart_create_topic] # [END pubsub_create_topic] @@ -71,7 +71,7 @@ def delete_topic(project_id, topic_name): publisher.delete_topic(topic_path) - print('Topic deleted: {}'.format(topic_path)) + print("Topic deleted: {}".format(topic_path)) # [END pubsub_delete_topic] @@ -90,14 +90,14 @@ def publish_messages(project_id, topic_name): topic_path = publisher.topic_path(project_id, topic_name) for n in range(1, 10): - data = u'Message number {}'.format(n) + data = u"Message number {}".format(n) # Data must be a bytestring - data = data.encode('utf-8') + data = data.encode("utf-8") # When you publish a message, the client returns a future. future = publisher.publish(topic_path, data=data) print(future.result()) - print('Published messages.') + print("Published messages.") # [END pubsub_quickstart_publisher] # [END pubsub_publish] @@ -115,16 +115,16 @@ def publish_messages_with_custom_attributes(project_id, topic_name): topic_path = publisher.topic_path(project_id, topic_name) for n in range(1, 10): - data = u'Message number {}'.format(n) + data = u"Message number {}".format(n) # Data must be a bytestring - data = data.encode('utf-8') + data = data.encode("utf-8") # Add two attributes, origin and username, to the message future = publisher.publish( - topic_path, data, origin='python-sample', username='gcp' + topic_path, data, origin="python-sample", username="gcp" ) print(future.result()) - print('Published messages with custom attributes.') + print("Published messages with custom attributes.") # [END pubsub_publish_custom_attributes] @@ -141,14 +141,14 @@ def publish_messages_with_futures(project_id, topic_name): topic_path = publisher.topic_path(project_id, topic_name) for n in range(1, 10): - data = u'Message number {}'.format(n) + data = u"Message number {}".format(n) # Data must be a bytestring - data = data.encode('utf-8') + data = data.encode("utf-8") # When you publish a message, the client returns a future. future = publisher.publish(topic_path, data=data) print(future.result()) - print('Published messages with futures.') + print("Published messages with futures.") # [END pubsub_publisher_concurrency_control] @@ -173,7 +173,7 @@ def callback(f): print(f.result()) futures.pop(data) except: # noqa - print('Please handle {} for {}.'.format(f.exception(), data)) + print("Please handle {} for {}.".format(f.exception(), data)) return callback @@ -182,7 +182,7 @@ def callback(f): futures.update({data: None}) # When you publish a message, the client returns a future. future = publisher.publish( - topic_path, data=data.encode('utf-8') # data must be a bytestring. + topic_path, data=data.encode("utf-8") # data must be a bytestring. ) futures[data] = future # Publish failures shall be handled in the callback function. @@ -192,7 +192,7 @@ def callback(f): while futures: time.sleep(5) - print('Published message with error handler.') + print("Published message with error handler.") # [END pubsub_publish_messages_error_handler] @@ -207,20 +207,19 @@ def publish_messages_with_batch_settings(project_id, topic_name): # Configure the batch to publish as soon as there is one kilobyte # of data or one second has passed. batch_settings = pubsub_v1.types.BatchSettings( - max_bytes=1024, # One kilobyte - max_latency=1, # One second + max_bytes=1024, max_latency=1 # One kilobyte # One second ) publisher = pubsub_v1.PublisherClient(batch_settings) topic_path = publisher.topic_path(project_id, topic_name) for n in range(1, 10): - data = u'Message number {}'.format(n) + data = u"Message number {}".format(n) # Data must be a bytestring - data = data.encode('utf-8') + data = data.encode("utf-8") future = publisher.publish(topic_path, data=data) print(future.result()) - print('Published messages with batch settings.') + print("Published messages with batch settings.") # [END pubsub_publisher_batch_settings] @@ -234,34 +233,34 @@ def publish_messages_with_retry_settings(project_id, topic_name): # Configure the retry settings. Defaults will be overwritten. retry_settings = { - 'interfaces': { - 'google.pubsub.v1.Publisher': { - 'retry_codes': { - 'publish': [ - 'ABORTED', - 'CANCELLED', - 'DEADLINE_EXCEEDED', - 'INTERNAL', - 'RESOURCE_EXHAUSTED', - 'UNAVAILABLE', - 'UNKNOWN', + "interfaces": { + "google.pubsub.v1.Publisher": { + "retry_codes": { + "publish": [ + "ABORTED", + "CANCELLED", + "DEADLINE_EXCEEDED", + "INTERNAL", + "RESOURCE_EXHAUSTED", + "UNAVAILABLE", + "UNKNOWN", ] }, - 'retry_params': { - 'messaging': { - 'initial_retry_delay_millis': 100, # default: 100 - 'retry_delay_multiplier': 1.3, # default: 1.3 - 'max_retry_delay_millis': 60000, # default: 60000 - 'initial_rpc_timeout_millis': 5000, # default: 25000 - 'rpc_timeout_multiplier': 1.0, # default: 1.0 - 'max_rpc_timeout_millis': 600000, # default: 30000 - 'total_timeout_millis': 600000, # default: 600000 + "retry_params": { + "messaging": { + "initial_retry_delay_millis": 100, # default: 100 + "retry_delay_multiplier": 1.3, # default: 1.3 + "max_retry_delay_millis": 60000, # default: 60000 + "initial_rpc_timeout_millis": 5000, # default: 25000 + "rpc_timeout_multiplier": 1.0, # default: 1.0 + "max_rpc_timeout_millis": 600000, # default: 30000 + "total_timeout_millis": 600000, # default: 600000 } }, - 'methods': { - 'Publish': { - 'retry_codes_name': 'publish', - 'retry_params_name': 'messaging', + "methods": { + "Publish": { + "retry_codes_name": "publish", + "retry_params_name": "messaging", } }, } @@ -272,85 +271,77 @@ def publish_messages_with_retry_settings(project_id, topic_name): topic_path = publisher.topic_path(project_id, topic_name) for n in range(1, 10): - data = u'Message number {}'.format(n) + data = u"Message number {}".format(n) # Data must be a bytestring - data = data.encode('utf-8') + data = data.encode("utf-8") future = publisher.publish(topic_path, data=data) print(future.result()) - print('Published messages with retry settings.') + print("Published messages with retry settings.") # [END pubsub_publisher_retry_settings] if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) - parser.add_argument('project_id', help='Your Google Cloud project ID') + parser.add_argument("project_id", help="Your Google Cloud project ID") - subparsers = parser.add_subparsers(dest='command') - subparsers.add_parser('list', help=list_topics.__doc__) + subparsers = parser.add_subparsers(dest="command") + subparsers.add_parser("list", help=list_topics.__doc__) - create_parser = subparsers.add_parser('create', - help=create_topic.__doc__) - create_parser.add_argument('topic_name') + create_parser = subparsers.add_parser("create", help=create_topic.__doc__) + create_parser.add_argument("topic_name") - delete_parser = subparsers.add_parser('delete', - help=delete_topic.__doc__) - delete_parser.add_argument('topic_name') + delete_parser = subparsers.add_parser("delete", help=delete_topic.__doc__) + delete_parser.add_argument("topic_name") - publish_parser = subparsers.add_parser('publish', - help=publish_messages.__doc__) - publish_parser.add_argument('topic_name') + publish_parser = subparsers.add_parser("publish", help=publish_messages.__doc__) + publish_parser.add_argument("topic_name") publish_with_custom_attributes_parser = subparsers.add_parser( - 'publish-with-custom-attributes', + "publish-with-custom-attributes", help=publish_messages_with_custom_attributes.__doc__, ) - publish_with_custom_attributes_parser.add_argument('topic_name') + publish_with_custom_attributes_parser.add_argument("topic_name") publish_with_futures_parser = subparsers.add_parser( - 'publish-with-futures', help=publish_messages_with_futures.__doc__ + "publish-with-futures", help=publish_messages_with_futures.__doc__ ) - publish_with_futures_parser.add_argument('topic_name') + publish_with_futures_parser.add_argument("topic_name") publish_with_error_handler_parser = subparsers.add_parser( - 'publish-with-error-handler', - help=publish_messages_with_error_handler.__doc__ + "publish-with-error-handler", help=publish_messages_with_error_handler.__doc__ ) - publish_with_error_handler_parser.add_argument('topic_name') + publish_with_error_handler_parser.add_argument("topic_name") publish_with_batch_settings_parser = subparsers.add_parser( - 'publish-with-batch-settings', - help=publish_messages_with_batch_settings.__doc__ + "publish-with-batch-settings", help=publish_messages_with_batch_settings.__doc__ ) - publish_with_batch_settings_parser.add_argument('topic_name') + publish_with_batch_settings_parser.add_argument("topic_name") publish_with_retry_settings_parser = subparsers.add_parser( - 'publish-with-retry-settings', - help=publish_messages_with_retry_settings.__doc__ + "publish-with-retry-settings", help=publish_messages_with_retry_settings.__doc__ ) - publish_with_retry_settings_parser.add_argument('topic_name') + publish_with_retry_settings_parser.add_argument("topic_name") args = parser.parse_args() - if args.command == 'list': + if args.command == "list": list_topics(args.project_id) - elif args.command == 'create': + elif args.command == "create": create_topic(args.project_id, args.topic_name) - elif args.command == 'delete': + elif args.command == "delete": delete_topic(args.project_id, args.topic_name) - elif args.command == 'publish': + elif args.command == "publish": publish_messages(args.project_id, args.topic_name) - elif args.command == 'publish-with-custom-attributes': - publish_messages_with_custom_attributes(args.project_id, - args.topic_name) - elif args.command == 'publish-with-futures': + elif args.command == "publish-with-custom-attributes": + publish_messages_with_custom_attributes(args.project_id, args.topic_name) + elif args.command == "publish-with-futures": publish_messages_with_futures(args.project_id, args.topic_name) - elif args.command == 'publish-with-error-handler': + elif args.command == "publish-with-error-handler": publish_messages_with_error_handler(args.project_id, args.topic_name) - elif args.command == 'publish-with-batch-settings': + elif args.command == "publish-with-batch-settings": publish_messages_with_batch_settings(args.project_id, args.topic_name) - elif args.command == 'publish-with-retry-settings': + elif args.command == "publish-with-retry-settings": publish_messages_with_retry_settings(args.project_id, args.topic_name) diff --git a/pubsub/cloud-client/publisher_test.py b/pubsub/cloud-client/publisher_test.py index 91d9e7601f39..a868732790d5 100644 --- a/pubsub/cloud-client/publisher_test.py +++ b/pubsub/cloud-client/publisher_test.py @@ -24,8 +24,8 @@ import publisher UUID = uuid.uuid4().hex -PROJECT = os.environ['GCLOUD_PROJECT'] -TOPIC = 'publisher-test-topic-' + UUID +PROJECT = os.environ["GCLOUD_PROJECT"] +TOPIC = "publisher-test-topic-" + UUID @pytest.fixture @@ -51,17 +51,16 @@ def _make_sleep_patch(): def new_sleep(period): if period == 60: real_sleep(5) - raise RuntimeError('sigil') + raise RuntimeError("sigil") else: real_sleep(period) - return mock.patch('time.sleep', new=new_sleep) + return mock.patch("time.sleep", new=new_sleep) def _to_delete(): publisher_client = pubsub_v1.PublisherClient() - publisher_client.delete_topic('projects/{}/topics/{}'.format( - PROJECT, TOPIC)) + publisher_client.delete_topic("projects/{}/topics/{}".format(PROJECT, TOPIC)) def test_list(client, topic, capsys): @@ -99,41 +98,41 @@ def test_publish(topic, capsys): publisher.publish_messages(PROJECT, TOPIC) out, _ = capsys.readouterr() - assert 'Published' in out + assert "Published" in out def test_publish_with_custom_attributes(topic, capsys): publisher.publish_messages_with_custom_attributes(PROJECT, TOPIC) out, _ = capsys.readouterr() - assert 'Published' in out + assert "Published" in out def test_publish_with_batch_settings(topic, capsys): publisher.publish_messages_with_batch_settings(PROJECT, TOPIC) out, _ = capsys.readouterr() - assert 'Published' in out + assert "Published" in out def test_publish_with_retry_settings(topic, capsys): publisher.publish_messages_with_retry_settings(PROJECT, TOPIC) out, _ = capsys.readouterr() - assert 'Published' in out + assert "Published" in out def test_publish_with_error_handler(topic, capsys): publisher.publish_messages_with_error_handler(PROJECT, TOPIC) out, _ = capsys.readouterr() - assert 'Published' in out + assert "Published" in out def test_publish_with_futures(topic, capsys): publisher.publish_messages_with_futures(PROJECT, TOPIC) out, _ = capsys.readouterr() - assert 'Published' in out + assert "Published" in out _to_delete() diff --git a/pubsub/cloud-client/quickstart.py b/pubsub/cloud-client/quickstart.py index f48d085e06b5..2a534165ce77 100644 --- a/pubsub/cloud-client/quickstart.py +++ b/pubsub/cloud-client/quickstart.py @@ -38,35 +38,33 @@ def end_to_end(project_id, topic_name, subscription_name, num_messages): # The `subscription_path` method creates a fully qualified identifier # in the form `projects/{project_id}/subscriptions/{subscription_name}` - subscription_path = subscriber.subscription_path( - project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_name) # Create the topic. topic = publisher.create_topic(topic_path) - print('\nTopic created: {}'.format(topic.name)) + print("\nTopic created: {}".format(topic.name)) # Create a subscription. - subscription = subscriber.create_subscription( - subscription_path, topic_path) - print('\nSubscription created: {}\n'.format(subscription.name)) + subscription = subscriber.create_subscription(subscription_path, topic_path) + print("\nSubscription created: {}\n".format(subscription.name)) publish_begin = time.time() # Publish messages. for n in range(num_messages): - data = u'Message number {}'.format(n) + data = u"Message number {}".format(n) # Data must be a bytestring - data = data.encode('utf-8') + data = data.encode("utf-8") # When you publish a message, the client returns a future. future = publisher.publish(topic_path, data=data) - print('Published {} of message ID {}.'.format(data, future.result())) + print("Published {} of message ID {}.".format(data, future.result())) publish_time = time.time() - publish_begin messages = set() def callback(message): - print('Received message: {}'.format(message)) + print("Received message: {}".format(message)) # Unacknowledged messages will be sent again. message.ack() messages.add(message) @@ -76,7 +74,7 @@ def callback(message): # Receive messages. The subscriber is nonblocking. subscriber.subscribe(subscription_path, callback=callback) - print('\nListening for messages on {}...\n'.format(subscription_path)) + print("\nListening for messages on {}...\n".format(subscription_path)) while True: if len(messages) == num_messages: @@ -87,22 +85,20 @@ def callback(message): break else: # Sleeps the thread at 50Hz to save on resources. - time.sleep(1. / 50) + time.sleep(1.0 / 50) # [END pubsub_end_to_end] -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) - parser.add_argument('project_id', help='Your Google Cloud project ID') - parser.add_argument('topic_name', help='Your topic name') - parser.add_argument('subscription_name', help='Your subscription name') - parser.add_argument('num_msgs', type=int, help='Number of test messages') + parser.add_argument("project_id", help="Your Google Cloud project ID") + parser.add_argument("topic_name", help="Your topic name") + parser.add_argument("subscription_name", help="Your subscription name") + parser.add_argument("num_msgs", type=int, help="Number of test messages") args = parser.parse_args() - end_to_end(args.project_id, args.topic_name, args.subscription_name, - args.num_msgs) + end_to_end(args.project_id, args.topic_name, args.subscription_name, args.num_msgs) diff --git a/pubsub/cloud-client/quickstart/pub.py b/pubsub/cloud-client/quickstart/pub.py index e340eb4f36ec..e14a8c04bf10 100644 --- a/pubsub/cloud-client/quickstart/pub.py +++ b/pubsub/cloud-client/quickstart/pub.py @@ -17,22 +17,32 @@ # [START pubsub_quickstart_pub_all] import argparse import time + # [START pubsub_quickstart_pub_deps] from google.cloud import pubsub_v1 + # [END pubsub_quickstart_pub_deps] def get_callback(api_future, data, ref): """Wrap message data in the context of the callback function.""" + def callback(api_future): try: - print("Published message {} now has message ID {}".format( - data, api_future.result())) + print( + "Published message {} now has message ID {}".format( + data, api_future.result() + ) + ) ref["num_messages"] += 1 except Exception: - print("A problem occurred when publishing {}: {}\n".format( - data, api_future.exception())) + print( + "A problem occurred when publishing {}: {}\n".format( + data, api_future.exception() + ) + ) raise + return callback @@ -63,13 +73,12 @@ def pub(project_id, topic_name): print("Published {} message(s).".format(ref["num_messages"])) -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) - parser.add_argument('project_id', help='Google Cloud project ID') - parser.add_argument('topic_name', help='Pub/Sub topic name') + parser.add_argument("project_id", help="Google Cloud project ID") + parser.add_argument("topic_name", help="Pub/Sub topic name") args = parser.parse_args() diff --git a/pubsub/cloud-client/quickstart/pub_test.py b/pubsub/cloud-client/quickstart/pub_test.py index db6d876b2483..b92a2ceaaeb7 100644 --- a/pubsub/cloud-client/quickstart/pub_test.py +++ b/pubsub/cloud-client/quickstart/pub_test.py @@ -24,16 +24,16 @@ import pub UUID = uuid.uuid4().hex -PROJECT = os.environ['GCLOUD_PROJECT'] -TOPIC = 'quickstart-pub-test-topic-' + UUID +PROJECT = os.environ["GCLOUD_PROJECT"] +TOPIC = "quickstart-pub-test-topic-" + UUID -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def publisher_client(): yield pubsub_v1.PublisherClient() -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def topic(publisher_client): topic_path = publisher_client.topic_path(PROJECT, TOPIC) diff --git a/pubsub/cloud-client/quickstart/sub.py b/pubsub/cloud-client/quickstart/sub.py index e39f14105b1a..5113c91b3255 100644 --- a/pubsub/cloud-client/quickstart/sub.py +++ b/pubsub/cloud-client/quickstart/sub.py @@ -16,8 +16,10 @@ # [START pubsub_quickstart_sub_all] import argparse + # [START pubsub_quickstart_sub_deps] from google.cloud import pubsub_v1 + # [END pubsub_quickstart_sub_deps] @@ -29,19 +31,18 @@ def sub(project_id, subscription_name): # [END pubsub_quickstart_sub_client] # Create a fully qualified identifier in the form of # `projects/{project_id}/subscriptions/{subscription_name}` - subscription_path = client.subscription_path( - project_id, subscription_name) + subscription_path = client.subscription_path(project_id, subscription_name) def callback(message): - print('Received message {} of message ID {}\n'.format( - message, message.message_id)) + print( + "Received message {} of message ID {}\n".format(message, message.message_id) + ) # Acknowledge the message. Unack'ed messages will be redelivered. message.ack() - print('Acknowledged message {}\n'.format(message.message_id)) + print("Acknowledged message {}\n".format(message.message_id)) - streaming_pull_future = client.subscribe( - subscription_path, callback=callback) - print('Listening for messages on {}..\n'.format(subscription_path)) + streaming_pull_future = client.subscribe(subscription_path, callback=callback) + print("Listening for messages on {}..\n".format(subscription_path)) # Calling result() on StreamingPullFuture keeps the main thread from # exiting while messages get processed in the callbacks. @@ -51,13 +52,12 @@ def callback(message): streaming_pull_future.cancel() -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) - parser.add_argument('project_id', help='Google Cloud project ID') - parser.add_argument('subscription_name', help='Pub/Sub subscription name') + parser.add_argument("project_id", help="Google Cloud project ID") + parser.add_argument("subscription_name", help="Pub/Sub subscription name") args = parser.parse_args() diff --git a/pubsub/cloud-client/quickstart/sub_test.py b/pubsub/cloud-client/quickstart/sub_test.py index 7df20a437a35..584b6dae1099 100644 --- a/pubsub/cloud-client/quickstart/sub_test.py +++ b/pubsub/cloud-client/quickstart/sub_test.py @@ -26,15 +26,15 @@ UUID = uuid.uuid4().hex -PROJECT = os.environ['GCLOUD_PROJECT'] -TOPIC = 'quickstart-sub-test-topic-' + UUID -SUBSCRIPTION = 'quickstart-sub-test-topic-sub-' + UUID +PROJECT = os.environ["GCLOUD_PROJECT"] +TOPIC = "quickstart-sub-test-topic-" + UUID +SUBSCRIPTION = "quickstart-sub-test-topic-sub-" + UUID publisher_client = pubsub_v1.PublisherClient() subscriber_client = pubsub_v1.SubscriberClient() -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def topic_path(): topic_path = publisher_client.topic_path(PROJECT, TOPIC) @@ -47,14 +47,14 @@ def topic_path(): publisher_client.delete_topic(topic_path) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def subscription_path(topic_path): - subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION) try: subscription = subscriber_client.create_subscription( - subscription_path, topic_path) + subscription_path, topic_path + ) yield subscription.name except AlreadyExists: yield subscription_path @@ -63,26 +63,24 @@ def subscription_path(topic_path): def _publish_messages(topic_path): - publish_future = publisher_client.publish(topic_path, data=b'Hello World!') + publish_future = publisher_client.publish(topic_path, data=b"Hello World!") publish_future.result() def test_sub(monkeypatch, topic_path, subscription_path, capsys): real_client = pubsub_v1.SubscriberClient() - mock_client = mock.Mock(spec=pubsub_v1.SubscriberClient, - wraps=real_client) + mock_client = mock.Mock(spec=pubsub_v1.SubscriberClient, wraps=real_client) # Attributes on mock_client_constructor uses the corresponding # attributes on pubsub_v1.SubscriberClient. mock_client_constructor = mock.create_autospec(pubsub_v1.SubscriberClient) mock_client_constructor.return_value = mock_client - monkeypatch.setattr(pubsub_v1, 'SubscriberClient', mock_client_constructor) + monkeypatch.setattr(pubsub_v1, "SubscriberClient", mock_client_constructor) def mock_subscribe(subscription_path, callback=None): - real_future = real_client.subscribe(subscription_path, - callback=callback) + real_future = real_client.subscribe(subscription_path, callback=callback) mock_future = mock.Mock(spec=real_future, wraps=real_future) def mock_result(): diff --git a/pubsub/cloud-client/quickstart_test.py b/pubsub/cloud-client/quickstart_test.py index c22e2172a6ab..e3a567a7787d 100644 --- a/pubsub/cloud-client/quickstart_test.py +++ b/pubsub/cloud-client/quickstart_test.py @@ -22,18 +22,18 @@ import quickstart UUID = uuid.uuid4().hex -PROJECT = os.environ['GCLOUD_PROJECT'] -TOPIC = 'end-to-end-test-topic-' + UUID -SUBSCRIPTION = 'end-to-end-test-topic-sub-' + UUID +PROJECT = os.environ["GCLOUD_PROJECT"] +TOPIC = "end-to-end-test-topic-" + UUID +SUBSCRIPTION = "end-to-end-test-topic-sub-" + UUID N = 10 -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def publisher_client(): yield pubsub_v1.PublisherClient() -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def topic(publisher_client): topic_path = publisher_client.topic_path(PROJECT, TOPIC) @@ -47,15 +47,14 @@ def topic(publisher_client): publisher_client.delete_topic(topic_path) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def subscriber_client(): yield pubsub_v1.SubscriberClient() -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def subscription(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION) try: subscriber_client.delete_subscription(subscription_path) diff --git a/pubsub/cloud-client/subscriber.py b/pubsub/cloud-client/subscriber.py index 3bbad0ead1b0..e07e02281589 100644 --- a/pubsub/cloud-client/subscriber.py +++ b/pubsub/cloud-client/subscriber.py @@ -66,20 +66,15 @@ def create_subscription(project_id, topic_name, subscription_name): subscriber = pubsub_v1.SubscriberClient() topic_path = subscriber.topic_path(project_id, topic_name) - subscription_path = subscriber.subscription_path( - project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_name) - subscription = subscriber.create_subscription( - subscription_path, topic_path) + subscription = subscriber.create_subscription(subscription_path, topic_path) - print('Subscription created: {}'.format(subscription)) + print("Subscription created: {}".format(subscription)) # [END pubsub_create_pull_subscription] -def create_push_subscription(project_id, - topic_name, - subscription_name, - endpoint): +def create_push_subscription(project_id, topic_name, subscription_name, endpoint): """Create a new push subscription on the given topic.""" # [START pubsub_create_push_subscription] from google.cloud import pubsub_v1 @@ -91,17 +86,16 @@ def create_push_subscription(project_id, subscriber = pubsub_v1.SubscriberClient() topic_path = subscriber.topic_path(project_id, topic_name) - subscription_path = subscriber.subscription_path( - project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_name) - push_config = pubsub_v1.types.PushConfig( - push_endpoint=endpoint) + push_config = pubsub_v1.types.PushConfig(push_endpoint=endpoint) subscription = subscriber.create_subscription( - subscription_path, topic_path, push_config) + subscription_path, topic_path, push_config + ) - print('Push subscription created: {}'.format(subscription)) - print('Endpoint for subscription is: {}'.format(endpoint)) + print("Push subscription created: {}".format(subscription)) + print("Endpoint for subscription is: {}".format(endpoint)) # [END pubsub_create_push_subscription] @@ -114,12 +108,11 @@ def delete_subscription(project_id, subscription_name): # TODO subscription_name = "Your Pub/Sub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_name) subscriber.delete_subscription(subscription_path) - print('Subscription deleted: {}'.format(subscription_path)) + print("Subscription deleted: {}".format(subscription_path)) # [END pubsub_delete_subscription] @@ -138,28 +131,21 @@ def update_subscription(project_id, subscription_name, endpoint): # TODO endpoint = "https://my-test-project.appspot.com/push" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_name) - push_config = pubsub_v1.types.PushConfig( - push_endpoint=endpoint) + push_config = pubsub_v1.types.PushConfig(push_endpoint=endpoint) subscription = pubsub_v1.types.Subscription( - name=subscription_path, - push_config=push_config) + name=subscription_path, push_config=push_config + ) - update_mask = { - 'paths': { - 'push_config', - } - } + update_mask = {"paths": {"push_config"}} subscriber.update_subscription(subscription, update_mask) result = subscriber.get_subscription(subscription_path) - print('Subscription updated: {}'.format(subscription_path)) - print('New endpoint for subscription is: {}'.format( - result.push_config)) + print("Subscription updated: {}".format(subscription_path)) + print("New endpoint for subscription is: {}".format(result.push_config)) # [END pubsub_update_push_configuration] @@ -177,18 +163,17 @@ def receive_messages(project_id, subscription_name): subscriber = pubsub_v1.SubscriberClient() # The `subscription_path` method creates a fully qualified identifier # in the form `projects/{project_id}/subscriptions/{subscription_name}` - subscription_path = subscriber.subscription_path( - project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_name) def callback(message): - print('Received message: {}'.format(message)) + print("Received message: {}".format(message)) message.ack() subscriber.subscribe(subscription_path, callback=callback) # The subscriber is non-blocking. We must keep the main thread from # exiting to allow it to process messages asynchronously in the background. - print('Listening for messages on {}'.format(subscription_path)) + print("Listening for messages on {}".format(subscription_path)) while True: time.sleep(60) # [END pubsub_subscriber_async_pull] @@ -207,23 +192,22 @@ def receive_messages_with_custom_attributes(project_id, subscription_name): # TODO subscription_name = "Your Pub/Sub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_name) def callback(message): - print('Received message: {}'.format(message.data)) + print("Received message: {}".format(message.data)) if message.attributes: - print('Attributes:') + print("Attributes:") for key in message.attributes: value = message.attributes.get(key) - print('{}: {}'.format(key, value)) + print("{}: {}".format(key, value)) message.ack() subscriber.subscribe(subscription_path, callback=callback) # The subscriber is non-blocking, so we must keep the main thread from # exiting to allow it to process messages in the background. - print('Listening for messages on {}'.format(subscription_path)) + print("Listening for messages on {}".format(subscription_path)) while True: time.sleep(60) # [END pubsub_subscriber_async_pull_custom_attributes] @@ -241,21 +225,21 @@ def receive_messages_with_flow_control(project_id, subscription_name): # TODO subscription_name = "Your Pub/Sub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_name) def callback(message): - print('Received message: {}'.format(message.data)) + print("Received message: {}".format(message.data)) message.ack() # Limit the subscriber to only have ten outstanding messages at a time. flow_control = pubsub_v1.types.FlowControl(max_messages=10) subscriber.subscribe( - subscription_path, callback=callback, flow_control=flow_control) + subscription_path, callback=callback, flow_control=flow_control + ) # The subscriber is non-blocking, so we must keep the main thread from # exiting to allow it to process messages in the background. - print('Listening for messages on {}'.format(subscription_path)) + print("Listening for messages on {}".format(subscription_path)) while True: time.sleep(60) # [END pubsub_subscriber_flow_settings] @@ -270,8 +254,7 @@ def synchronous_pull(project_id, subscription_name): # TODO subscription_name = "Your Pub/Sub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_name) NUM_MESSAGES = 3 @@ -286,8 +269,11 @@ def synchronous_pull(project_id, subscription_name): # Acknowledges the received messages so they will not be sent again. subscriber.acknowledge(subscription_path, ack_ids) - print('Received and acknowledged {} messages. Done.'.format( - len(response.received_messages))) + print( + "Received and acknowledged {} messages. Done.".format( + len(response.received_messages) + ) + ) # [END pubsub_subscriber_sync_pull] @@ -305,8 +291,7 @@ def synchronous_pull_with_lease_management(project_id, subscription_name): # TODO subscription_name = "Your Pub/Sub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_name) NUM_MESSAGES = 2 ACK_DEADLINE = 30 @@ -322,8 +307,11 @@ def synchronous_pull_with_lease_management(project_id, subscription_name): def worker(msg): """Simulates a long-running process.""" RUN_TIME = random.randint(1, 60) - logger.info('{}: Running {} for {}s'.format( - time.strftime("%X", time.gmtime()), msg.message.data, RUN_TIME)) + logger.info( + "{}: Running {} for {}s".format( + time.strftime("%X", time.gmtime()), msg.message.data, RUN_TIME + ) + ) time.sleep(RUN_TIME) # `processes` stores process as key and ack id and message as values. @@ -342,26 +330,33 @@ def worker(msg): if process.is_alive(): # `ack_deadline_seconds` must be between 10 to 600. subscriber.modify_ack_deadline( - subscription_path, - [ack_id], - ack_deadline_seconds=ACK_DEADLINE) - logger.info('{}: Reset ack deadline for {} for {}s'.format( - time.strftime("%X", time.gmtime()), - msg_data, ACK_DEADLINE)) + subscription_path, [ack_id], ack_deadline_seconds=ACK_DEADLINE + ) + logger.info( + "{}: Reset ack deadline for {} for {}s".format( + time.strftime("%X", time.gmtime()), msg_data, ACK_DEADLINE + ) + ) # If the processs is finished, acknowledges using `ack_id`. else: subscriber.acknowledge(subscription_path, [ack_id]) - logger.info("{}: Acknowledged {}".format( - time.strftime("%X", time.gmtime()), msg_data)) + logger.info( + "{}: Acknowledged {}".format( + time.strftime("%X", time.gmtime()), msg_data + ) + ) processes.pop(process) # If there are still processes running, sleeps the thread. if processes: time.sleep(SLEEP_TIME) - print('Received and acknowledged {} messages. Done.'.format( - len(response.received_messages))) + print( + "Received and acknowledged {} messages. Done.".format( + len(response.received_messages) + ) + ) # [END pubsub_subscriber_sync_pull_with_lease] @@ -374,11 +369,10 @@ def listen_for_errors(project_id, subscription_name): # TODO subscription_name = "Your Pubsub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, subscription_name) + subscription_path = subscriber.subscription_path(project_id, subscription_name) def callback(message): - print('Received message: {}'.format(message)) + print("Received message: {}".format(message)) message.ack() future = subscriber.subscribe(subscription_path, callback=callback) @@ -390,109 +384,102 @@ def callback(message): future.result(timeout=30) except Exception as e: print( - 'Listening for messages on {} threw an Exception: {}.'.format( - subscription_name, e)) + "Listening for messages on {} threw an Exception: {}.".format( + subscription_name, e + ) + ) # [END pubsub_subscriber_error_listener] -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) - parser.add_argument('project_id', help='Your Google Cloud project ID') + parser.add_argument("project_id", help="Your Google Cloud project ID") - subparsers = parser.add_subparsers(dest='command') + subparsers = parser.add_subparsers(dest="command") list_in_topic_parser = subparsers.add_parser( - 'list_in_topic', help=list_subscriptions_in_topic.__doc__) - list_in_topic_parser.add_argument('topic_name') + "list_in_topic", help=list_subscriptions_in_topic.__doc__ + ) + list_in_topic_parser.add_argument("topic_name") list_in_project_parser = subparsers.add_parser( - 'list_in_project', help=list_subscriptions_in_project.__doc__) + "list_in_project", help=list_subscriptions_in_project.__doc__ + ) - create_parser = subparsers.add_parser( - 'create', help=create_subscription.__doc__) - create_parser.add_argument('topic_name') - create_parser.add_argument('subscription_name') + create_parser = subparsers.add_parser("create", help=create_subscription.__doc__) + create_parser.add_argument("topic_name") + create_parser.add_argument("subscription_name") create_push_parser = subparsers.add_parser( - 'create-push', help=create_push_subscription.__doc__) - create_push_parser.add_argument('topic_name') - create_push_parser.add_argument('subscription_name') - create_push_parser.add_argument('endpoint') + "create-push", help=create_push_subscription.__doc__ + ) + create_push_parser.add_argument("topic_name") + create_push_parser.add_argument("subscription_name") + create_push_parser.add_argument("endpoint") - delete_parser = subparsers.add_parser( - 'delete', help=delete_subscription.__doc__) - delete_parser.add_argument('subscription_name') + delete_parser = subparsers.add_parser("delete", help=delete_subscription.__doc__) + delete_parser.add_argument("subscription_name") - update_parser = subparsers.add_parser( - 'update', help=update_subscription.__doc__) - update_parser.add_argument('subscription_name') - update_parser.add_argument('endpoint') + update_parser = subparsers.add_parser("update", help=update_subscription.__doc__) + update_parser.add_argument("subscription_name") + update_parser.add_argument("endpoint") - receive_parser = subparsers.add_parser( - 'receive', help=receive_messages.__doc__) - receive_parser.add_argument('subscription_name') + receive_parser = subparsers.add_parser("receive", help=receive_messages.__doc__) + receive_parser.add_argument("subscription_name") receive_with_custom_attributes_parser = subparsers.add_parser( - 'receive-custom-attributes', - help=receive_messages_with_custom_attributes.__doc__) - receive_with_custom_attributes_parser.add_argument('subscription_name') + "receive-custom-attributes", + help=receive_messages_with_custom_attributes.__doc__, + ) + receive_with_custom_attributes_parser.add_argument("subscription_name") receive_with_flow_control_parser = subparsers.add_parser( - 'receive-flow-control', - help=receive_messages_with_flow_control.__doc__) - receive_with_flow_control_parser.add_argument('subscription_name') + "receive-flow-control", help=receive_messages_with_flow_control.__doc__ + ) + receive_with_flow_control_parser.add_argument("subscription_name") synchronous_pull_parser = subparsers.add_parser( - 'receive-synchronously', - help=synchronous_pull.__doc__) - synchronous_pull_parser.add_argument('subscription_name') + "receive-synchronously", help=synchronous_pull.__doc__ + ) + synchronous_pull_parser.add_argument("subscription_name") synchronous_pull_with_lease_management_parser = subparsers.add_parser( - 'receive-synchronously-with-lease', - help=synchronous_pull_with_lease_management.__doc__) - synchronous_pull_with_lease_management_parser.add_argument( - 'subscription_name') + "receive-synchronously-with-lease", + help=synchronous_pull_with_lease_management.__doc__, + ) + synchronous_pull_with_lease_management_parser.add_argument("subscription_name") listen_for_errors_parser = subparsers.add_parser( - 'listen_for_errors', help=listen_for_errors.__doc__) - listen_for_errors_parser.add_argument('subscription_name') + "listen_for_errors", help=listen_for_errors.__doc__ + ) + listen_for_errors_parser.add_argument("subscription_name") args = parser.parse_args() - if args.command == 'list_in_topic': + if args.command == "list_in_topic": list_subscriptions_in_topic(args.project_id, args.topic_name) - elif args.command == 'list_in_project': + elif args.command == "list_in_project": list_subscriptions_in_project(args.project_id) - elif args.command == 'create': - create_subscription( - args.project_id, args.topic_name, args.subscription_name) - elif args.command == 'create-push': + elif args.command == "create": + create_subscription(args.project_id, args.topic_name, args.subscription_name) + elif args.command == "create-push": create_push_subscription( - args.project_id, - args.topic_name, - args.subscription_name, - args.endpoint) - elif args.command == 'delete': - delete_subscription( - args.project_id, args.subscription_name) - elif args.command == 'update': - update_subscription( - args.project_id, args.subscription_name, args.endpoint) - elif args.command == 'receive': + args.project_id, args.topic_name, args.subscription_name, args.endpoint + ) + elif args.command == "delete": + delete_subscription(args.project_id, args.subscription_name) + elif args.command == "update": + update_subscription(args.project_id, args.subscription_name, args.endpoint) + elif args.command == "receive": receive_messages(args.project_id, args.subscription_name) - elif args.command == 'receive-custom-attributes': - receive_messages_with_custom_attributes( - args.project_id, args.subscription_name) - elif args.command == 'receive-flow-control': - receive_messages_with_flow_control( - args.project_id, args.subscription_name) - elif args.command == 'receive-synchronously': - synchronous_pull( - args.project_id, args.subscription_name) - elif args.command == 'receive-synchronously-with-lease': - synchronous_pull_with_lease_management( - args.project_id, args.subscription_name) - elif args.command == 'listen_for_errors': + elif args.command == "receive-custom-attributes": + receive_messages_with_custom_attributes(args.project_id, args.subscription_name) + elif args.command == "receive-flow-control": + receive_messages_with_flow_control(args.project_id, args.subscription_name) + elif args.command == "receive-synchronously": + synchronous_pull(args.project_id, args.subscription_name) + elif args.command == "receive-synchronously-with-lease": + synchronous_pull_with_lease_management(args.project_id, args.subscription_name) + elif args.command == "listen_for_errors": listen_for_errors(args.project_id, args.subscription_name) diff --git a/pubsub/cloud-client/subscriber_test.py b/pubsub/cloud-client/subscriber_test.py index 4576f90cfdbb..63d659dbcb6f 100644 --- a/pubsub/cloud-client/subscriber_test.py +++ b/pubsub/cloud-client/subscriber_test.py @@ -24,21 +24,21 @@ import subscriber UUID = uuid.uuid4().hex -PROJECT = os.environ['GCLOUD_PROJECT'] -TOPIC = 'subscription-test-topic-' + UUID -SUBSCRIPTION_ONE = 'subscription-test-subscription-one-' + UUID -SUBSCRIPTION_TWO = 'subscription-test-subscription-two-' + UUID -SUBSCRIPTION_THREE = 'subscription-test-subscription-three-' + UUID -ENDPOINT = 'https://{}.appspot.com/push'.format(PROJECT) -NEW_ENDPOINT = 'https://{}.appspot.com/push2'.format(PROJECT) +PROJECT = os.environ["GCLOUD_PROJECT"] +TOPIC = "subscription-test-topic-" + UUID +SUBSCRIPTION_ONE = "subscription-test-subscription-one-" + UUID +SUBSCRIPTION_TWO = "subscription-test-subscription-two-" + UUID +SUBSCRIPTION_THREE = "subscription-test-subscription-three-" + UUID +ENDPOINT = "https://{}.appspot.com/push".format(PROJECT) +NEW_ENDPOINT = "https://{}.appspot.com/push2".format(PROJECT) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def publisher_client(): yield pubsub_v1.PublisherClient() -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def topic(publisher_client): topic_path = publisher_client.topic_path(PROJECT, TOPIC) @@ -50,49 +50,43 @@ def topic(publisher_client): yield response.name -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def subscriber_client(): yield pubsub_v1.SubscriberClient() -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def subscription_one(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_ONE) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_ONE) try: response = subscriber_client.get_subscription(subscription_path) except: # noqa - response = subscriber_client.create_subscription( - subscription_path, topic=topic) + response = subscriber_client.create_subscription(subscription_path, topic=topic) yield response.name -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def subscription_two(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_TWO) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_TWO) try: response = subscriber_client.get_subscription(subscription_path) except: # noqa - response = subscriber_client.create_subscription( - subscription_path, topic=topic) + response = subscriber_client.create_subscription(subscription_path, topic=topic) yield response.name -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def subscription_three(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_THREE) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_THREE) try: response = subscriber_client.get_subscription(subscription_path) except: # noqa - response = subscriber_client.create_subscription( - subscription_path, topic=topic) + response = subscriber_client.create_subscription(subscription_path, topic=topic) yield response.name @@ -114,8 +108,7 @@ def _(): def test_create(subscriber_client): - subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_ONE) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_ONE) try: subscriber_client.delete_subscription(subscription_path) @@ -130,15 +123,13 @@ def _(): def test_create_push(subscriber_client): - subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION_ONE) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_ONE) try: subscriber_client.delete_subscription(subscription_path) except Exception: pass - subscriber.create_push_subscription( - PROJECT, TOPIC, SUBSCRIPTION_ONE, ENDPOINT) + subscriber.create_push_subscription(PROJECT, TOPIC, SUBSCRIPTION_ONE, ENDPOINT) @eventually_consistent.call def _(): @@ -149,7 +140,7 @@ def test_update(subscriber_client, subscription_one, capsys): subscriber.update_subscription(PROJECT, SUBSCRIPTION_ONE, NEW_ENDPOINT) out, _ = capsys.readouterr() - assert 'Subscription updated' in out + assert "Subscription updated" in out def test_delete(subscriber_client, subscription_one): @@ -163,14 +154,14 @@ def _(): def _publish_messages(publisher_client, topic): for n in range(5): - data = u'Message {}'.format(n).encode('utf-8') + data = u"Message {}".format(n).encode("utf-8") future = publisher_client.publish(topic, data=data) future.result() def _publish_messages_with_custom_attributes(publisher_client, topic): - data = u'Test message'.encode('utf-8') - future = publisher_client.publish(topic, data=data, origin='python-sample') + data = u"Test message".encode("utf-8") + future = publisher_client.publish(topic, data=data, origin="python-sample") future.result() @@ -180,11 +171,11 @@ def _make_sleep_patch(): def new_sleep(period): if period == 60: real_sleep(5) - raise RuntimeError('sigil') + raise RuntimeError("sigil") else: real_sleep(period) - return mock.patch('time.sleep', new=new_sleep) + return mock.patch("time.sleep", new=new_sleep) def _to_delete(): @@ -193,78 +184,77 @@ def _to_delete(): resources = [TOPIC, SUBSCRIPTION_TWO, SUBSCRIPTION_THREE] for item in resources: - if 'subscription-test-topic' in item: - publisher_client.delete_topic( - 'projects/{}/topics/{}'.format(PROJECT, item)) - if 'subscription-test-subscription' in item: + if "subscription-test-topic" in item: + publisher_client.delete_topic("projects/{}/topics/{}".format(PROJECT, item)) + if "subscription-test-subscription" in item: subscriber_client.delete_subscription( - 'projects/{}/subscriptions/{}'.format(PROJECT, item)) + "projects/{}/subscriptions/{}".format(PROJECT, item) + ) def test_receive(publisher_client, topic, subscription_two, capsys): _publish_messages(publisher_client, topic) with _make_sleep_patch(): - with pytest.raises(RuntimeError, match='sigil'): + with pytest.raises(RuntimeError, match="sigil"): subscriber.receive_messages(PROJECT, SUBSCRIPTION_TWO) out, _ = capsys.readouterr() - assert 'Listening' in out + assert "Listening" in out assert subscription_two in out - assert 'Message' in out + assert "Message" in out def test_receive_with_custom_attributes( - publisher_client, topic, subscription_two, capsys): + publisher_client, topic, subscription_two, capsys +): _publish_messages_with_custom_attributes(publisher_client, topic) with _make_sleep_patch(): - with pytest.raises(RuntimeError, match='sigil'): + with pytest.raises(RuntimeError, match="sigil"): subscriber.receive_messages_with_custom_attributes( - PROJECT, SUBSCRIPTION_TWO) + PROJECT, SUBSCRIPTION_TWO + ) out, _ = capsys.readouterr() - assert 'Test message' in out - assert 'origin' in out - assert 'python-sample' in out + assert "Test message" in out + assert "origin" in out + assert "python-sample" in out -def test_receive_with_flow_control( - publisher_client, topic, subscription_two, capsys): +def test_receive_with_flow_control(publisher_client, topic, subscription_two, capsys): _publish_messages(publisher_client, topic) with _make_sleep_patch(): - with pytest.raises(RuntimeError, match='sigil'): - subscriber.receive_messages_with_flow_control( - PROJECT, SUBSCRIPTION_TWO) + with pytest.raises(RuntimeError, match="sigil"): + subscriber.receive_messages_with_flow_control(PROJECT, SUBSCRIPTION_TWO) out, _ = capsys.readouterr() - assert 'Listening' in out + assert "Listening" in out assert subscription_two in out - assert 'Message' in out + assert "Message" in out -def test_receive_synchronously( - publisher_client, topic, subscription_three, capsys): +def test_receive_synchronously(publisher_client, topic, subscription_three, capsys): _publish_messages(publisher_client, topic) subscriber.synchronous_pull(PROJECT, SUBSCRIPTION_THREE) out, _ = capsys.readouterr() - assert 'Done.' in out + assert "Done." in out def test_receive_synchronously_with_lease( - publisher_client, topic, subscription_three, capsys): + publisher_client, topic, subscription_three, capsys +): _publish_messages(publisher_client, topic) - subscriber.synchronous_pull_with_lease_management( - PROJECT, SUBSCRIPTION_THREE) + subscriber.synchronous_pull_with_lease_management(PROJECT, SUBSCRIPTION_THREE) out, _ = capsys.readouterr() - assert 'Done.' in out + assert "Done." in out # Clean up resources after all the tests. _to_delete() diff --git a/pubsub/streaming-analytics/PubSubToGCS.py b/pubsub/streaming-analytics/PubSubToGCS.py index f003e3362234..3546edc558c2 100644 --- a/pubsub/streaming-analytics/PubSubToGCS.py +++ b/pubsub/streaming-analytics/PubSubToGCS.py @@ -34,24 +34,25 @@ def __init__(self, window_size): self.window_size = int(window_size * 60) def expand(self, pcoll): - return (pcoll - # Assigns window info to each Pub/Sub message based on its - # publish timestamp. - | 'Window into Fixed Intervals' >> beam.WindowInto( - window.FixedWindows(self.window_size)) - | 'Add timestamps to messages' >> (beam.ParDo(AddTimestamps())) - # Use a dummy key to group the elements in the same window. - # Note that all the elements in one window must fit into memory - # for this. If the windowed elements do not fit into memory, - # please consider using `beam.util.BatchElements`. - # https://beam.apache.org/releases/pydoc/current/apache_beam.transforms.util.html#apache_beam.transforms.util.BatchElements - | 'Add Dummy Key' >> beam.Map(lambda elem: (None, elem)) - | 'Groupby' >> beam.GroupByKey() - | 'Abandon Dummy Key' >> beam.MapTuple(lambda _, val: val)) + return ( + pcoll + # Assigns window info to each Pub/Sub message based on its + # publish timestamp. + | "Window into Fixed Intervals" + >> beam.WindowInto(window.FixedWindows(self.window_size)) + | "Add timestamps to messages" >> (beam.ParDo(AddTimestamps())) + # Use a dummy key to group the elements in the same window. + # Note that all the elements in one window must fit into memory + # for this. If the windowed elements do not fit into memory, + # please consider using `beam.util.BatchElements`. + # https://beam.apache.org/releases/pydoc/current/apache_beam.transforms.util.html#apache_beam.transforms.util.BatchElements + | "Add Dummy Key" >> beam.Map(lambda elem: (None, elem)) + | "Groupby" >> beam.GroupByKey() + | "Abandon Dummy Key" >> beam.MapTuple(lambda _, val: val) + ) class AddTimestamps(beam.DoFn): - def process(self, element, publish_time=beam.DoFn.TimestampParam): """Processes each incoming windowed element by extracting the Pub/Sub message and its publish timestamp into a dictionary. `publish_time` @@ -60,61 +61,70 @@ def process(self, element, publish_time=beam.DoFn.TimestampParam): """ yield { - 'message_body': element.decode('utf-8'), - 'publish_time': datetime.datetime.utcfromtimestamp( - float(publish_time)).strftime("%Y-%m-%d %H:%M:%S.%f"), + "message_body": element.decode("utf-8"), + "publish_time": datetime.datetime.utcfromtimestamp( + float(publish_time) + ).strftime("%Y-%m-%d %H:%M:%S.%f"), } class WriteBatchesToGCS(beam.DoFn): - def __init__(self, output_path): self.output_path = output_path def process(self, batch, window=beam.DoFn.WindowParam): """Write one batch per file to a Google Cloud Storage bucket. """ - ts_format = '%H:%M' + ts_format = "%H:%M" window_start = window.start.to_utc_datetime().strftime(ts_format) window_end = window.end.to_utc_datetime().strftime(ts_format) - filename = '-'.join([self.output_path, window_start, window_end]) + filename = "-".join([self.output_path, window_start, window_end]) - with beam.io.gcp.gcsio.GcsIO().open(filename=filename, mode='w') as f: + with beam.io.gcp.gcsio.GcsIO().open(filename=filename, mode="w") as f: for element in batch: - f.write('{}\n'.format(json.dumps(element)).encode('utf-8')) + f.write("{}\n".format(json.dumps(element)).encode("utf-8")) def run(input_topic, output_path, window_size=1.0, pipeline_args=None): # `save_main_session` is set to true because some DoFn's rely on # globally imported modules. pipeline_options = PipelineOptions( - pipeline_args, streaming=True, save_main_session=True) + pipeline_args, streaming=True, save_main_session=True + ) with beam.Pipeline(options=pipeline_options) as pipeline: - (pipeline - | 'Read PubSub Messages' >> beam.io.ReadFromPubSub(topic=input_topic) - | 'Window into' >> GroupWindowsIntoBatches(window_size) - | 'Write to GCS' >> beam.ParDo(WriteBatchesToGCS(output_path))) + ( + pipeline + | "Read PubSub Messages" >> beam.io.ReadFromPubSub(topic=input_topic) + | "Window into" >> GroupWindowsIntoBatches(window_size) + | "Write to GCS" >> beam.ParDo(WriteBatchesToGCS(output_path)) + ) -if __name__ == '__main__': # noqa +if __name__ == "__main__": # noqa logging.getLogger().setLevel(logging.INFO) parser = argparse.ArgumentParser() parser.add_argument( - '--input_topic', - help='The Cloud Pub/Sub topic to read from.\n' - '"projects//topics/".') + "--input_topic", + help="The Cloud Pub/Sub topic to read from.\n" + '"projects//topics/".', + ) parser.add_argument( - '--window_size', + "--window_size", type=float, default=1.0, - help='Output file\'s window size in number of minutes.') + help="Output file's window size in number of minutes.", + ) parser.add_argument( - '--output_path', - help='GCS Path of the output file including filename prefix.') + "--output_path", help="GCS Path of the output file including filename prefix." + ) known_args, pipeline_args = parser.parse_known_args() - run(known_args.input_topic, known_args.output_path, known_args.window_size, - pipeline_args) + run( + known_args.input_topic, + known_args.output_path, + known_args.window_size, + pipeline_args, + ) # [END pubsub_to_gcs] diff --git a/pubsub/streaming-analytics/PubSubToGCS_test.py b/pubsub/streaming-analytics/PubSubToGCS_test.py index 0cc4356e8ec3..d59097be5a68 100644 --- a/pubsub/streaming-analytics/PubSubToGCS_test.py +++ b/pubsub/streaming-analytics/PubSubToGCS_test.py @@ -24,10 +24,10 @@ from google.cloud import pubsub_v1 -PROJECT = os.environ['GCLOUD_PROJECT'] -BUCKET = os.environ['CLOUD_STORAGE_BUCKET'] +PROJECT = os.environ["GCLOUD_PROJECT"] +BUCKET = os.environ["CLOUD_STORAGE_BUCKET"] UUID = uuid.uuid1().hex -TOPIC = 'test-topic-' + UUID +TOPIC = "test-topic-" + UUID @pytest.fixture @@ -51,7 +51,8 @@ def topic_path(publisher_client): def _infinite_publish_job(publisher_client, topic_path): while True: future = publisher_client.publish( - topic_path, data='Hello World!'.encode('utf-8')) + topic_path, data="Hello World!".encode("utf-8") + ) future.result() time.sleep(10) @@ -63,19 +64,28 @@ def test_run(publisher_client, topic_path): # Use one process to publish messages to a topic. publish_process = mp.Process( - target=lambda: _infinite_publish_job(publisher_client, topic_path)) + target=lambda: _infinite_publish_job(publisher_client, topic_path) + ) # Use another process to run the streaming pipeline that should write one # file to GCS every minute (according to the default window size). pipeline_process = mp.Process( - target=lambda: sp.call([ - 'python', 'PubSubToGCS.py', - '--project', PROJECT, - '--runner', 'DirectRunner', - '--temp_location', tempfile.mkdtemp(), - '--input_topic', topic_path, - '--output_path', 'gs://{}/pubsub/{}/output'.format(BUCKET, UUID), - ]) + target=lambda: sp.call( + [ + "python", + "PubSubToGCS.py", + "--project", + PROJECT, + "--runner", + "DirectRunner", + "--temp_location", + tempfile.mkdtemp(), + "--input_topic", + topic_path, + "--output_path", + "gs://{}/pubsub/{}/output".format(BUCKET, UUID), + ] + ) ) publish_process.start() @@ -92,7 +102,7 @@ def test_run(publisher_client, topic_path): # Check for output files on GCS. gcs_client = beam.io.gcp.gcsio.GcsIO() # This returns a dictionary. - files = gcs_client.list_prefix('gs://{}/pubsub/{}'.format(BUCKET, UUID)) + files = gcs_client.list_prefix("gs://{}/pubsub/{}".format(BUCKET, UUID)) assert len(files) > 0 # Clean up. Delete subscription. Delete topic. Delete GCS files. From 3242686e736843417c4a374f033edb2e361c3c4a Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Fri, 6 Dec 2019 13:40:01 -0800 Subject: [PATCH 10/20] lint --- pubsub/cloud-client/iam.py | 23 +++- pubsub/cloud-client/iam_test.py | 4 +- pubsub/cloud-client/publisher.py | 22 +++- pubsub/cloud-client/publisher_test.py | 4 +- pubsub/cloud-client/quickstart.py | 20 +++- pubsub/cloud-client/quickstart/pub.py | 3 +- pubsub/cloud-client/quickstart/sub.py | 13 +- pubsub/cloud-client/quickstart/sub_test.py | 10 +- pubsub/cloud-client/quickstart_test.py | 5 +- pubsub/cloud-client/subscriber.py | 132 ++++++++++++++++----- pubsub/cloud-client/subscriber_test.py | 80 ++++++++++--- pubsub/streaming-analytics/PubSubToGCS.py | 7 +- 12 files changed, 254 insertions(+), 69 deletions(-) diff --git a/pubsub/cloud-client/iam.py b/pubsub/cloud-client/iam.py index 44e52e576295..80c3073c0ecc 100644 --- a/pubsub/cloud-client/iam.py +++ b/pubsub/cloud-client/iam.py @@ -89,12 +89,16 @@ def set_subscription_policy(project, subscription_name): policy.bindings.add(role="roles/pubsub.viewer", members=["allUsers"]) # Add a group as an editor. - policy.bindings.add(role="roles/editor", members=["group:cloud-logs@google.com"]) + policy.bindings.add( + role="roles/editor", + members=["group:cloud-logs@google.com"] + ) # Set the policy policy = client.set_iam_policy(subscription_path, policy) - print("IAM policy for subscription {} set: {}".format(subscription_name, policy)) + print("IAM policy for subscription {} set: {}".format( + subscription_name, policy)) # [END pubsub_set_subscription_policy] @@ -106,10 +110,15 @@ def check_topic_permissions(project, topic_name): permissions_to_check = ["pubsub.topics.publish", "pubsub.topics.update"] - allowed_permissions = client.test_iam_permissions(topic_path, permissions_to_check) + allowed_permissions = client.test_iam_permissions( + topic_path, + permissions_to_check + ) print( - "Allowed permissions for topic {}: {}".format(topic_path, allowed_permissions) + "Allowed permissions for topic {}: {}".format( + topic_path, + allowed_permissions) ) # [END pubsub_test_topic_permissions] @@ -139,7 +148,8 @@ def check_subscription_permissions(project, subscription_name): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument("project", help="Your Google Cloud project ID") @@ -171,7 +181,8 @@ def check_subscription_permissions(project, subscription_name): check_topic_permissions_parser.add_argument("topic_name") check_subscription_permissions_parser = subparsers.add_parser( - "check-subscription-permissions", help=check_subscription_permissions.__doc__ + "check-subscription-permissions", + help=check_subscription_permissions.__doc__ ) check_subscription_permissions_parser.add_argument("subscription_name") diff --git a/pubsub/cloud-client/iam_test.py b/pubsub/cloud-client/iam_test.py index ac62bbfc4f31..2b019f9ea16f 100644 --- a/pubsub/cloud-client/iam_test.py +++ b/pubsub/cloud-client/iam_test.py @@ -54,7 +54,9 @@ def subscriber_client(): @pytest.fixture def subscription(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION) + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION + ) try: subscriber_client.delete_subscription(subscription_path) diff --git a/pubsub/cloud-client/publisher.py b/pubsub/cloud-client/publisher.py index 0fd23cdba694..043e5731e2fe 100644 --- a/pubsub/cloud-client/publisher.py +++ b/pubsub/cloud-client/publisher.py @@ -283,7 +283,8 @@ def publish_messages_with_retry_settings(project_id, topic_name): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument("project_id", help="Your Google Cloud project ID") @@ -296,7 +297,10 @@ def publish_messages_with_retry_settings(project_id, topic_name): delete_parser = subparsers.add_parser("delete", help=delete_topic.__doc__) delete_parser.add_argument("topic_name") - publish_parser = subparsers.add_parser("publish", help=publish_messages.__doc__) + publish_parser = subparsers.add_parser( + "publish", + help=publish_messages.__doc__ + ) publish_parser.add_argument("topic_name") publish_with_custom_attributes_parser = subparsers.add_parser( @@ -311,17 +315,20 @@ def publish_messages_with_retry_settings(project_id, topic_name): publish_with_futures_parser.add_argument("topic_name") publish_with_error_handler_parser = subparsers.add_parser( - "publish-with-error-handler", help=publish_messages_with_error_handler.__doc__ + "publish-with-error-handler", + help=publish_messages_with_error_handler.__doc__ ) publish_with_error_handler_parser.add_argument("topic_name") publish_with_batch_settings_parser = subparsers.add_parser( - "publish-with-batch-settings", help=publish_messages_with_batch_settings.__doc__ + "publish-with-batch-settings", + help=publish_messages_with_batch_settings.__doc__ ) publish_with_batch_settings_parser.add_argument("topic_name") publish_with_retry_settings_parser = subparsers.add_parser( - "publish-with-retry-settings", help=publish_messages_with_retry_settings.__doc__ + "publish-with-retry-settings", + help=publish_messages_with_retry_settings.__doc__ ) publish_with_retry_settings_parser.add_argument("topic_name") @@ -336,7 +343,10 @@ def publish_messages_with_retry_settings(project_id, topic_name): elif args.command == "publish": publish_messages(args.project_id, args.topic_name) elif args.command == "publish-with-custom-attributes": - publish_messages_with_custom_attributes(args.project_id, args.topic_name) + publish_messages_with_custom_attributes( + args.project_id, + args.topic_name + ) elif args.command == "publish-with-futures": publish_messages_with_futures(args.project_id, args.topic_name) elif args.command == "publish-with-error-handler": diff --git a/pubsub/cloud-client/publisher_test.py b/pubsub/cloud-client/publisher_test.py index a868732790d5..125fae3c06b9 100644 --- a/pubsub/cloud-client/publisher_test.py +++ b/pubsub/cloud-client/publisher_test.py @@ -60,7 +60,9 @@ def new_sleep(period): def _to_delete(): publisher_client = pubsub_v1.PublisherClient() - publisher_client.delete_topic("projects/{}/topics/{}".format(PROJECT, TOPIC)) + publisher_client.delete_topic( + "projects/{}/topics/{}".format(PROJECT, TOPIC) + ) def test_list(client, topic, capsys): diff --git a/pubsub/cloud-client/quickstart.py b/pubsub/cloud-client/quickstart.py index 2a534165ce77..79ddacaed4e0 100644 --- a/pubsub/cloud-client/quickstart.py +++ b/pubsub/cloud-client/quickstart.py @@ -38,14 +38,20 @@ def end_to_end(project_id, topic_name, subscription_name, num_messages): # The `subscription_path` method creates a fully qualified identifier # in the form `projects/{project_id}/subscriptions/{subscription_name}` - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, + subscription_name + ) # Create the topic. topic = publisher.create_topic(topic_path) print("\nTopic created: {}".format(topic.name)) # Create a subscription. - subscription = subscriber.create_subscription(subscription_path, topic_path) + subscription = subscriber.create_subscription( + subscription_path, + topic_path + ) print("\nSubscription created: {}\n".format(subscription.name)) publish_begin = time.time() @@ -92,7 +98,8 @@ def callback(message): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument("project_id", help="Your Google Cloud project ID") parser.add_argument("topic_name", help="Your topic name") @@ -101,4 +108,9 @@ def callback(message): args = parser.parse_args() - end_to_end(args.project_id, args.topic_name, args.subscription_name, args.num_msgs) + end_to_end( + args.project_id, + args.topic_name, + args.subscription_name, + args.num_msgs + ) diff --git a/pubsub/cloud-client/quickstart/pub.py b/pubsub/cloud-client/quickstart/pub.py index e14a8c04bf10..ad48503e5b8e 100644 --- a/pubsub/cloud-client/quickstart/pub.py +++ b/pubsub/cloud-client/quickstart/pub.py @@ -75,7 +75,8 @@ def pub(project_id, topic_name): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument("project_id", help="Google Cloud project ID") parser.add_argument("topic_name", help="Pub/Sub topic name") diff --git a/pubsub/cloud-client/quickstart/sub.py b/pubsub/cloud-client/quickstart/sub.py index 5113c91b3255..d26d108a6fc9 100644 --- a/pubsub/cloud-client/quickstart/sub.py +++ b/pubsub/cloud-client/quickstart/sub.py @@ -35,13 +35,19 @@ def sub(project_id, subscription_name): def callback(message): print( - "Received message {} of message ID {}\n".format(message, message.message_id) + "Received message {} of message ID {}\n".format( + message, + message.message_id + ) ) # Acknowledge the message. Unack'ed messages will be redelivered. message.ack() print("Acknowledged message {}\n".format(message.message_id)) - streaming_pull_future = client.subscribe(subscription_path, callback=callback) + streaming_pull_future = client.subscribe( + subscription_path, + callback=callback + ) print("Listening for messages on {}..\n".format(subscription_path)) # Calling result() on StreamingPullFuture keeps the main thread from @@ -54,7 +60,8 @@ def callback(message): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument("project_id", help="Google Cloud project ID") parser.add_argument("subscription_name", help="Pub/Sub subscription name") diff --git a/pubsub/cloud-client/quickstart/sub_test.py b/pubsub/cloud-client/quickstart/sub_test.py index 584b6dae1099..376d8078e90e 100644 --- a/pubsub/cloud-client/quickstart/sub_test.py +++ b/pubsub/cloud-client/quickstart/sub_test.py @@ -49,7 +49,10 @@ def topic_path(): @pytest.fixture(scope="module") def subscription_path(topic_path): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION) + subscription_path = subscriber_client.subscription_path( + PROJECT, + SUBSCRIPTION + ) try: subscription = subscriber_client.create_subscription( @@ -80,7 +83,10 @@ def test_sub(monkeypatch, topic_path, subscription_path, capsys): monkeypatch.setattr(pubsub_v1, "SubscriberClient", mock_client_constructor) def mock_subscribe(subscription_path, callback=None): - real_future = real_client.subscribe(subscription_path, callback=callback) + real_future = real_client.subscribe( + subscription_path, + callback=callback + ) mock_future = mock.Mock(spec=real_future, wraps=real_future) def mock_result(): diff --git a/pubsub/cloud-client/quickstart_test.py b/pubsub/cloud-client/quickstart_test.py index e3a567a7787d..30a459017830 100644 --- a/pubsub/cloud-client/quickstart_test.py +++ b/pubsub/cloud-client/quickstart_test.py @@ -54,7 +54,10 @@ def subscriber_client(): @pytest.fixture(scope="module") def subscription(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION) + subscription_path = subscriber_client.subscription_path( + PROJECT, + SUBSCRIPTION + ) try: subscriber_client.delete_subscription(subscription_path) diff --git a/pubsub/cloud-client/subscriber.py b/pubsub/cloud-client/subscriber.py index e07e02281589..8c084eb64e45 100644 --- a/pubsub/cloud-client/subscriber.py +++ b/pubsub/cloud-client/subscriber.py @@ -66,15 +66,26 @@ def create_subscription(project_id, topic_name, subscription_name): subscriber = pubsub_v1.SubscriberClient() topic_path = subscriber.topic_path(project_id, topic_name) - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, + subscription_name + ) - subscription = subscriber.create_subscription(subscription_path, topic_path) + subscription = subscriber.create_subscription( + subscription_path, + topic_path + ) print("Subscription created: {}".format(subscription)) # [END pubsub_create_pull_subscription] -def create_push_subscription(project_id, topic_name, subscription_name, endpoint): +def create_push_subscription( + project_id, + topic_name, + subscription_name, + endpoint +): """Create a new push subscription on the given topic.""" # [START pubsub_create_push_subscription] from google.cloud import pubsub_v1 @@ -86,7 +97,10 @@ def create_push_subscription(project_id, topic_name, subscription_name, endpoint subscriber = pubsub_v1.SubscriberClient() topic_path = subscriber.topic_path(project_id, topic_name) - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, + subscription_name + ) push_config = pubsub_v1.types.PushConfig(push_endpoint=endpoint) @@ -108,7 +122,10 @@ def delete_subscription(project_id, subscription_name): # TODO subscription_name = "Your Pub/Sub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, + subscription_name + ) subscriber.delete_subscription(subscription_path) @@ -131,7 +148,10 @@ def update_subscription(project_id, subscription_name, endpoint): # TODO endpoint = "https://my-test-project.appspot.com/push" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, + subscription_name + ) push_config = pubsub_v1.types.PushConfig(push_endpoint=endpoint) @@ -163,7 +183,10 @@ def receive_messages(project_id, subscription_name): subscriber = pubsub_v1.SubscriberClient() # The `subscription_path` method creates a fully qualified identifier # in the form `projects/{project_id}/subscriptions/{subscription_name}` - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, + subscription_name + ) def callback(message): print("Received message: {}".format(message)) @@ -192,7 +215,10 @@ def receive_messages_with_custom_attributes(project_id, subscription_name): # TODO subscription_name = "Your Pub/Sub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, + subscription_name + ) def callback(message): print("Received message: {}".format(message.data)) @@ -225,7 +251,10 @@ def receive_messages_with_flow_control(project_id, subscription_name): # TODO subscription_name = "Your Pub/Sub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, + subscription_name + ) def callback(message): print("Received message: {}".format(message.data)) @@ -254,7 +283,10 @@ def synchronous_pull(project_id, subscription_name): # TODO subscription_name = "Your Pub/Sub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, + subscription_name + ) NUM_MESSAGES = 3 @@ -291,7 +323,10 @@ def synchronous_pull_with_lease_management(project_id, subscription_name): # TODO subscription_name = "Your Pub/Sub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, + subscription_name + ) NUM_MESSAGES = 2 ACK_DEADLINE = 30 @@ -330,11 +365,15 @@ def worker(msg): if process.is_alive(): # `ack_deadline_seconds` must be between 10 to 600. subscriber.modify_ack_deadline( - subscription_path, [ack_id], ack_deadline_seconds=ACK_DEADLINE + subscription_path, + [ack_id], + ack_deadline_seconds=ACK_DEADLINE ) logger.info( "{}: Reset ack deadline for {} for {}s".format( - time.strftime("%X", time.gmtime()), msg_data, ACK_DEADLINE + time.strftime("%X", time.gmtime()), + msg_data, + ACK_DEADLINE ) ) @@ -369,7 +408,10 @@ def listen_for_errors(project_id, subscription_name): # TODO subscription_name = "Your Pubsub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, + subscription_name + ) def callback(message): print("Received message: {}".format(message)) @@ -393,7 +435,8 @@ def callback(message): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument("project_id", help="Your Google Cloud project ID") @@ -407,7 +450,10 @@ def callback(message): "list_in_project", help=list_subscriptions_in_project.__doc__ ) - create_parser = subparsers.add_parser("create", help=create_subscription.__doc__) + create_parser = subparsers.add_parser( + "create", + help=create_subscription.__doc__ + ) create_parser.add_argument("topic_name") create_parser.add_argument("subscription_name") @@ -418,14 +464,23 @@ def callback(message): create_push_parser.add_argument("subscription_name") create_push_parser.add_argument("endpoint") - delete_parser = subparsers.add_parser("delete", help=delete_subscription.__doc__) + delete_parser = subparsers.add_parser( + "delete", + help=delete_subscription.__doc__ + ) delete_parser.add_argument("subscription_name") - update_parser = subparsers.add_parser("update", help=update_subscription.__doc__) + update_parser = subparsers.add_parser( + "update", + help=update_subscription.__doc__ + ) update_parser.add_argument("subscription_name") update_parser.add_argument("endpoint") - receive_parser = subparsers.add_parser("receive", help=receive_messages.__doc__) + receive_parser = subparsers.add_parser( + "receive", + help=receive_messages.__doc__ + ) receive_parser.add_argument("subscription_name") receive_with_custom_attributes_parser = subparsers.add_parser( @@ -435,7 +490,8 @@ def callback(message): receive_with_custom_attributes_parser.add_argument("subscription_name") receive_with_flow_control_parser = subparsers.add_parser( - "receive-flow-control", help=receive_messages_with_flow_control.__doc__ + "receive-flow-control", + help=receive_messages_with_flow_control.__doc__ ) receive_with_flow_control_parser.add_argument("subscription_name") @@ -448,7 +504,9 @@ def callback(message): "receive-synchronously-with-lease", help=synchronous_pull_with_lease_management.__doc__, ) - synchronous_pull_with_lease_management_parser.add_argument("subscription_name") + synchronous_pull_with_lease_management_parser.add_argument( + "subscription_name" + ) listen_for_errors_parser = subparsers.add_parser( "listen_for_errors", help=listen_for_errors.__doc__ @@ -462,24 +520,44 @@ def callback(message): elif args.command == "list_in_project": list_subscriptions_in_project(args.project_id) elif args.command == "create": - create_subscription(args.project_id, args.topic_name, args.subscription_name) + create_subscription( + args.project_id, + args.topic_name, + args.subscription_name + ) elif args.command == "create-push": create_push_subscription( - args.project_id, args.topic_name, args.subscription_name, args.endpoint + args.project_id, + args.topic_name, + args.subscription_name, + args.endpoint ) elif args.command == "delete": delete_subscription(args.project_id, args.subscription_name) elif args.command == "update": - update_subscription(args.project_id, args.subscription_name, args.endpoint) + update_subscription( + args.project_id, + args.subscription_name, + args.endpoint + ) elif args.command == "receive": receive_messages(args.project_id, args.subscription_name) elif args.command == "receive-custom-attributes": - receive_messages_with_custom_attributes(args.project_id, args.subscription_name) + receive_messages_with_custom_attributes( + args.project_id, + args.subscription_name + ) elif args.command == "receive-flow-control": - receive_messages_with_flow_control(args.project_id, args.subscription_name) + receive_messages_with_flow_control( + args.project_id, + args.subscription_name + ) elif args.command == "receive-synchronously": synchronous_pull(args.project_id, args.subscription_name) elif args.command == "receive-synchronously-with-lease": - synchronous_pull_with_lease_management(args.project_id, args.subscription_name) + synchronous_pull_with_lease_management( + args.project_id, + args.subscription_name + ) elif args.command == "listen_for_errors": listen_for_errors(args.project_id, args.subscription_name) diff --git a/pubsub/cloud-client/subscriber_test.py b/pubsub/cloud-client/subscriber_test.py index 63d659dbcb6f..41fabd373559 100644 --- a/pubsub/cloud-client/subscriber_test.py +++ b/pubsub/cloud-client/subscriber_test.py @@ -57,36 +57,54 @@ def subscriber_client(): @pytest.fixture(scope="module") def subscription_one(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_ONE) + subscription_path = subscriber_client.subscription_path( + PROJECT, + SUBSCRIPTION_ONE + ) try: response = subscriber_client.get_subscription(subscription_path) except: # noqa - response = subscriber_client.create_subscription(subscription_path, topic=topic) + response = subscriber_client.create_subscription( + subscription_path, + topic=topic + ) yield response.name @pytest.fixture(scope="module") def subscription_two(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_TWO) + subscription_path = subscriber_client.subscription_path( + PROJECT, + SUBSCRIPTION_TWO + ) try: response = subscriber_client.get_subscription(subscription_path) except: # noqa - response = subscriber_client.create_subscription(subscription_path, topic=topic) + response = subscriber_client.create_subscription( + subscription_path, + topic=topic + ) yield response.name @pytest.fixture(scope="module") def subscription_three(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_THREE) + subscription_path = subscriber_client.subscription_path( + PROJECT, + SUBSCRIPTION_THREE + ) try: response = subscriber_client.get_subscription(subscription_path) except: # noqa - response = subscriber_client.create_subscription(subscription_path, topic=topic) + response = subscriber_client.create_subscription( + subscription_path, + topic=topic + ) yield response.name @@ -108,7 +126,10 @@ def _(): def test_create(subscriber_client): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_ONE) + subscription_path = subscriber_client.subscription_path( + PROJECT, + SUBSCRIPTION_ONE + ) try: subscriber_client.delete_subscription(subscription_path) @@ -123,13 +144,21 @@ def _(): def test_create_push(subscriber_client): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_ONE) + subscription_path = subscriber_client.subscription_path( + PROJECT, + SUBSCRIPTION_ONE + ) try: subscriber_client.delete_subscription(subscription_path) except Exception: pass - subscriber.create_push_subscription(PROJECT, TOPIC, SUBSCRIPTION_ONE, ENDPOINT) + subscriber.create_push_subscription( + PROJECT, + TOPIC, + SUBSCRIPTION_ONE, + ENDPOINT + ) @eventually_consistent.call def _(): @@ -185,7 +214,9 @@ def _to_delete(): for item in resources: if "subscription-test-topic" in item: - publisher_client.delete_topic("projects/{}/topics/{}".format(PROJECT, item)) + publisher_client.delete_topic( + "projects/{}/topics/{}".format(PROJECT, item) + ) if "subscription-test-subscription" in item: subscriber_client.delete_subscription( "projects/{}/subscriptions/{}".format(PROJECT, item) @@ -223,13 +254,21 @@ def test_receive_with_custom_attributes( assert "python-sample" in out -def test_receive_with_flow_control(publisher_client, topic, subscription_two, capsys): +def test_receive_with_flow_control( + publisher_client, + topic, + subscription_two, + capsys +): _publish_messages(publisher_client, topic) with _make_sleep_patch(): with pytest.raises(RuntimeError, match="sigil"): - subscriber.receive_messages_with_flow_control(PROJECT, SUBSCRIPTION_TWO) + subscriber.receive_messages_with_flow_control( + PROJECT, + SUBSCRIPTION_TWO + ) out, _ = capsys.readouterr() assert "Listening" in out @@ -237,7 +276,12 @@ def test_receive_with_flow_control(publisher_client, topic, subscription_two, ca assert "Message" in out -def test_receive_synchronously(publisher_client, topic, subscription_three, capsys): +def test_receive_synchronously( + publisher_client, + topic, + subscription_three, + capsys +): _publish_messages(publisher_client, topic) subscriber.synchronous_pull(PROJECT, SUBSCRIPTION_THREE) @@ -247,11 +291,17 @@ def test_receive_synchronously(publisher_client, topic, subscription_three, caps def test_receive_synchronously_with_lease( - publisher_client, topic, subscription_three, capsys + publisher_client, + topic, + subscription_three, + capsys ): _publish_messages(publisher_client, topic) - subscriber.synchronous_pull_with_lease_management(PROJECT, SUBSCRIPTION_THREE) + subscriber.synchronous_pull_with_lease_management( + PROJECT, + SUBSCRIPTION_THREE + ) out, _ = capsys.readouterr() assert "Done." in out diff --git a/pubsub/streaming-analytics/PubSubToGCS.py b/pubsub/streaming-analytics/PubSubToGCS.py index 3546edc558c2..f414fdf7705f 100644 --- a/pubsub/streaming-analytics/PubSubToGCS.py +++ b/pubsub/streaming-analytics/PubSubToGCS.py @@ -95,7 +95,9 @@ def run(input_topic, output_path, window_size=1.0, pipeline_args=None): with beam.Pipeline(options=pipeline_options) as pipeline: ( pipeline - | "Read PubSub Messages" >> beam.io.ReadFromPubSub(topic=input_topic) + | "Read PubSub Messages" >> beam.io.ReadFromPubSub( + topic=input_topic + ) | "Window into" >> GroupWindowsIntoBatches(window_size) | "Write to GCS" >> beam.ParDo(WriteBatchesToGCS(output_path)) ) @@ -117,7 +119,8 @@ def run(input_topic, output_path, window_size=1.0, pipeline_args=None): help="Output file's window size in number of minutes.", ) parser.add_argument( - "--output_path", help="GCS Path of the output file including filename prefix." + "--output_path", + help="GCS Path of the output file including filename prefix." ) known_args, pipeline_args = parser.parse_known_args() From 8670eff59258c06bf934a2a609ce609f1a62859e Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Fri, 6 Dec 2019 13:47:36 -0800 Subject: [PATCH 11/20] lint --- pubsub/cloud-client/iam.py | 23 +--- pubsub/cloud-client/iam_test.py | 4 +- pubsub/cloud-client/publisher.py | 22 +--- pubsub/cloud-client/publisher_test.py | 4 +- pubsub/cloud-client/quickstart.py | 20 +--- pubsub/cloud-client/quickstart/pub.py | 3 +- pubsub/cloud-client/quickstart/sub.py | 13 +- pubsub/cloud-client/quickstart/sub_test.py | 10 +- pubsub/cloud-client/quickstart_test.py | 5 +- pubsub/cloud-client/subscriber.py | 132 +++++---------------- pubsub/cloud-client/subscriber_test.py | 80 +++---------- pubsub/streaming-analytics/PubSubToGCS.py | 7 +- 12 files changed, 69 insertions(+), 254 deletions(-) diff --git a/pubsub/cloud-client/iam.py b/pubsub/cloud-client/iam.py index 80c3073c0ecc..44e52e576295 100644 --- a/pubsub/cloud-client/iam.py +++ b/pubsub/cloud-client/iam.py @@ -89,16 +89,12 @@ def set_subscription_policy(project, subscription_name): policy.bindings.add(role="roles/pubsub.viewer", members=["allUsers"]) # Add a group as an editor. - policy.bindings.add( - role="roles/editor", - members=["group:cloud-logs@google.com"] - ) + policy.bindings.add(role="roles/editor", members=["group:cloud-logs@google.com"]) # Set the policy policy = client.set_iam_policy(subscription_path, policy) - print("IAM policy for subscription {} set: {}".format( - subscription_name, policy)) + print("IAM policy for subscription {} set: {}".format(subscription_name, policy)) # [END pubsub_set_subscription_policy] @@ -110,15 +106,10 @@ def check_topic_permissions(project, topic_name): permissions_to_check = ["pubsub.topics.publish", "pubsub.topics.update"] - allowed_permissions = client.test_iam_permissions( - topic_path, - permissions_to_check - ) + allowed_permissions = client.test_iam_permissions(topic_path, permissions_to_check) print( - "Allowed permissions for topic {}: {}".format( - topic_path, - allowed_permissions) + "Allowed permissions for topic {}: {}".format(topic_path, allowed_permissions) ) # [END pubsub_test_topic_permissions] @@ -148,8 +139,7 @@ def check_subscription_permissions(project, subscription_name): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument("project", help="Your Google Cloud project ID") @@ -181,8 +171,7 @@ def check_subscription_permissions(project, subscription_name): check_topic_permissions_parser.add_argument("topic_name") check_subscription_permissions_parser = subparsers.add_parser( - "check-subscription-permissions", - help=check_subscription_permissions.__doc__ + "check-subscription-permissions", help=check_subscription_permissions.__doc__ ) check_subscription_permissions_parser.add_argument("subscription_name") diff --git a/pubsub/cloud-client/iam_test.py b/pubsub/cloud-client/iam_test.py index 2b019f9ea16f..ac62bbfc4f31 100644 --- a/pubsub/cloud-client/iam_test.py +++ b/pubsub/cloud-client/iam_test.py @@ -54,9 +54,7 @@ def subscriber_client(): @pytest.fixture def subscription(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path( - PROJECT, SUBSCRIPTION - ) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION) try: subscriber_client.delete_subscription(subscription_path) diff --git a/pubsub/cloud-client/publisher.py b/pubsub/cloud-client/publisher.py index 043e5731e2fe..0fd23cdba694 100644 --- a/pubsub/cloud-client/publisher.py +++ b/pubsub/cloud-client/publisher.py @@ -283,8 +283,7 @@ def publish_messages_with_retry_settings(project_id, topic_name): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument("project_id", help="Your Google Cloud project ID") @@ -297,10 +296,7 @@ def publish_messages_with_retry_settings(project_id, topic_name): delete_parser = subparsers.add_parser("delete", help=delete_topic.__doc__) delete_parser.add_argument("topic_name") - publish_parser = subparsers.add_parser( - "publish", - help=publish_messages.__doc__ - ) + publish_parser = subparsers.add_parser("publish", help=publish_messages.__doc__) publish_parser.add_argument("topic_name") publish_with_custom_attributes_parser = subparsers.add_parser( @@ -315,20 +311,17 @@ def publish_messages_with_retry_settings(project_id, topic_name): publish_with_futures_parser.add_argument("topic_name") publish_with_error_handler_parser = subparsers.add_parser( - "publish-with-error-handler", - help=publish_messages_with_error_handler.__doc__ + "publish-with-error-handler", help=publish_messages_with_error_handler.__doc__ ) publish_with_error_handler_parser.add_argument("topic_name") publish_with_batch_settings_parser = subparsers.add_parser( - "publish-with-batch-settings", - help=publish_messages_with_batch_settings.__doc__ + "publish-with-batch-settings", help=publish_messages_with_batch_settings.__doc__ ) publish_with_batch_settings_parser.add_argument("topic_name") publish_with_retry_settings_parser = subparsers.add_parser( - "publish-with-retry-settings", - help=publish_messages_with_retry_settings.__doc__ + "publish-with-retry-settings", help=publish_messages_with_retry_settings.__doc__ ) publish_with_retry_settings_parser.add_argument("topic_name") @@ -343,10 +336,7 @@ def publish_messages_with_retry_settings(project_id, topic_name): elif args.command == "publish": publish_messages(args.project_id, args.topic_name) elif args.command == "publish-with-custom-attributes": - publish_messages_with_custom_attributes( - args.project_id, - args.topic_name - ) + publish_messages_with_custom_attributes(args.project_id, args.topic_name) elif args.command == "publish-with-futures": publish_messages_with_futures(args.project_id, args.topic_name) elif args.command == "publish-with-error-handler": diff --git a/pubsub/cloud-client/publisher_test.py b/pubsub/cloud-client/publisher_test.py index 125fae3c06b9..a868732790d5 100644 --- a/pubsub/cloud-client/publisher_test.py +++ b/pubsub/cloud-client/publisher_test.py @@ -60,9 +60,7 @@ def new_sleep(period): def _to_delete(): publisher_client = pubsub_v1.PublisherClient() - publisher_client.delete_topic( - "projects/{}/topics/{}".format(PROJECT, TOPIC) - ) + publisher_client.delete_topic("projects/{}/topics/{}".format(PROJECT, TOPIC)) def test_list(client, topic, capsys): diff --git a/pubsub/cloud-client/quickstart.py b/pubsub/cloud-client/quickstart.py index 79ddacaed4e0..2a534165ce77 100644 --- a/pubsub/cloud-client/quickstart.py +++ b/pubsub/cloud-client/quickstart.py @@ -38,20 +38,14 @@ def end_to_end(project_id, topic_name, subscription_name, num_messages): # The `subscription_path` method creates a fully qualified identifier # in the form `projects/{project_id}/subscriptions/{subscription_name}` - subscription_path = subscriber.subscription_path( - project_id, - subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) # Create the topic. topic = publisher.create_topic(topic_path) print("\nTopic created: {}".format(topic.name)) # Create a subscription. - subscription = subscriber.create_subscription( - subscription_path, - topic_path - ) + subscription = subscriber.create_subscription(subscription_path, topic_path) print("\nSubscription created: {}\n".format(subscription.name)) publish_begin = time.time() @@ -98,8 +92,7 @@ def callback(message): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument("project_id", help="Your Google Cloud project ID") parser.add_argument("topic_name", help="Your topic name") @@ -108,9 +101,4 @@ def callback(message): args = parser.parse_args() - end_to_end( - args.project_id, - args.topic_name, - args.subscription_name, - args.num_msgs - ) + end_to_end(args.project_id, args.topic_name, args.subscription_name, args.num_msgs) diff --git a/pubsub/cloud-client/quickstart/pub.py b/pubsub/cloud-client/quickstart/pub.py index ad48503e5b8e..e14a8c04bf10 100644 --- a/pubsub/cloud-client/quickstart/pub.py +++ b/pubsub/cloud-client/quickstart/pub.py @@ -75,8 +75,7 @@ def pub(project_id, topic_name): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument("project_id", help="Google Cloud project ID") parser.add_argument("topic_name", help="Pub/Sub topic name") diff --git a/pubsub/cloud-client/quickstart/sub.py b/pubsub/cloud-client/quickstart/sub.py index d26d108a6fc9..5113c91b3255 100644 --- a/pubsub/cloud-client/quickstart/sub.py +++ b/pubsub/cloud-client/quickstart/sub.py @@ -35,19 +35,13 @@ def sub(project_id, subscription_name): def callback(message): print( - "Received message {} of message ID {}\n".format( - message, - message.message_id - ) + "Received message {} of message ID {}\n".format(message, message.message_id) ) # Acknowledge the message. Unack'ed messages will be redelivered. message.ack() print("Acknowledged message {}\n".format(message.message_id)) - streaming_pull_future = client.subscribe( - subscription_path, - callback=callback - ) + streaming_pull_future = client.subscribe(subscription_path, callback=callback) print("Listening for messages on {}..\n".format(subscription_path)) # Calling result() on StreamingPullFuture keeps the main thread from @@ -60,8 +54,7 @@ def callback(message): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument("project_id", help="Google Cloud project ID") parser.add_argument("subscription_name", help="Pub/Sub subscription name") diff --git a/pubsub/cloud-client/quickstart/sub_test.py b/pubsub/cloud-client/quickstart/sub_test.py index 376d8078e90e..584b6dae1099 100644 --- a/pubsub/cloud-client/quickstart/sub_test.py +++ b/pubsub/cloud-client/quickstart/sub_test.py @@ -49,10 +49,7 @@ def topic_path(): @pytest.fixture(scope="module") def subscription_path(topic_path): - subscription_path = subscriber_client.subscription_path( - PROJECT, - SUBSCRIPTION - ) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION) try: subscription = subscriber_client.create_subscription( @@ -83,10 +80,7 @@ def test_sub(monkeypatch, topic_path, subscription_path, capsys): monkeypatch.setattr(pubsub_v1, "SubscriberClient", mock_client_constructor) def mock_subscribe(subscription_path, callback=None): - real_future = real_client.subscribe( - subscription_path, - callback=callback - ) + real_future = real_client.subscribe(subscription_path, callback=callback) mock_future = mock.Mock(spec=real_future, wraps=real_future) def mock_result(): diff --git a/pubsub/cloud-client/quickstart_test.py b/pubsub/cloud-client/quickstart_test.py index 30a459017830..e3a567a7787d 100644 --- a/pubsub/cloud-client/quickstart_test.py +++ b/pubsub/cloud-client/quickstart_test.py @@ -54,10 +54,7 @@ def subscriber_client(): @pytest.fixture(scope="module") def subscription(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path( - PROJECT, - SUBSCRIPTION - ) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION) try: subscriber_client.delete_subscription(subscription_path) diff --git a/pubsub/cloud-client/subscriber.py b/pubsub/cloud-client/subscriber.py index 8c084eb64e45..e07e02281589 100644 --- a/pubsub/cloud-client/subscriber.py +++ b/pubsub/cloud-client/subscriber.py @@ -66,26 +66,15 @@ def create_subscription(project_id, topic_name, subscription_name): subscriber = pubsub_v1.SubscriberClient() topic_path = subscriber.topic_path(project_id, topic_name) - subscription_path = subscriber.subscription_path( - project_id, - subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) - subscription = subscriber.create_subscription( - subscription_path, - topic_path - ) + subscription = subscriber.create_subscription(subscription_path, topic_path) print("Subscription created: {}".format(subscription)) # [END pubsub_create_pull_subscription] -def create_push_subscription( - project_id, - topic_name, - subscription_name, - endpoint -): +def create_push_subscription(project_id, topic_name, subscription_name, endpoint): """Create a new push subscription on the given topic.""" # [START pubsub_create_push_subscription] from google.cloud import pubsub_v1 @@ -97,10 +86,7 @@ def create_push_subscription( subscriber = pubsub_v1.SubscriberClient() topic_path = subscriber.topic_path(project_id, topic_name) - subscription_path = subscriber.subscription_path( - project_id, - subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) push_config = pubsub_v1.types.PushConfig(push_endpoint=endpoint) @@ -122,10 +108,7 @@ def delete_subscription(project_id, subscription_name): # TODO subscription_name = "Your Pub/Sub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, - subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) subscriber.delete_subscription(subscription_path) @@ -148,10 +131,7 @@ def update_subscription(project_id, subscription_name, endpoint): # TODO endpoint = "https://my-test-project.appspot.com/push" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, - subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) push_config = pubsub_v1.types.PushConfig(push_endpoint=endpoint) @@ -183,10 +163,7 @@ def receive_messages(project_id, subscription_name): subscriber = pubsub_v1.SubscriberClient() # The `subscription_path` method creates a fully qualified identifier # in the form `projects/{project_id}/subscriptions/{subscription_name}` - subscription_path = subscriber.subscription_path( - project_id, - subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) def callback(message): print("Received message: {}".format(message)) @@ -215,10 +192,7 @@ def receive_messages_with_custom_attributes(project_id, subscription_name): # TODO subscription_name = "Your Pub/Sub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, - subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) def callback(message): print("Received message: {}".format(message.data)) @@ -251,10 +225,7 @@ def receive_messages_with_flow_control(project_id, subscription_name): # TODO subscription_name = "Your Pub/Sub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, - subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) def callback(message): print("Received message: {}".format(message.data)) @@ -283,10 +254,7 @@ def synchronous_pull(project_id, subscription_name): # TODO subscription_name = "Your Pub/Sub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, - subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) NUM_MESSAGES = 3 @@ -323,10 +291,7 @@ def synchronous_pull_with_lease_management(project_id, subscription_name): # TODO subscription_name = "Your Pub/Sub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, - subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) NUM_MESSAGES = 2 ACK_DEADLINE = 30 @@ -365,15 +330,11 @@ def worker(msg): if process.is_alive(): # `ack_deadline_seconds` must be between 10 to 600. subscriber.modify_ack_deadline( - subscription_path, - [ack_id], - ack_deadline_seconds=ACK_DEADLINE + subscription_path, [ack_id], ack_deadline_seconds=ACK_DEADLINE ) logger.info( "{}: Reset ack deadline for {} for {}s".format( - time.strftime("%X", time.gmtime()), - msg_data, - ACK_DEADLINE + time.strftime("%X", time.gmtime()), msg_data, ACK_DEADLINE ) ) @@ -408,10 +369,7 @@ def listen_for_errors(project_id, subscription_name): # TODO subscription_name = "Your Pubsub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path( - project_id, - subscription_name - ) + subscription_path = subscriber.subscription_path(project_id, subscription_name) def callback(message): print("Received message: {}".format(message)) @@ -435,8 +393,7 @@ def callback(message): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument("project_id", help="Your Google Cloud project ID") @@ -450,10 +407,7 @@ def callback(message): "list_in_project", help=list_subscriptions_in_project.__doc__ ) - create_parser = subparsers.add_parser( - "create", - help=create_subscription.__doc__ - ) + create_parser = subparsers.add_parser("create", help=create_subscription.__doc__) create_parser.add_argument("topic_name") create_parser.add_argument("subscription_name") @@ -464,23 +418,14 @@ def callback(message): create_push_parser.add_argument("subscription_name") create_push_parser.add_argument("endpoint") - delete_parser = subparsers.add_parser( - "delete", - help=delete_subscription.__doc__ - ) + delete_parser = subparsers.add_parser("delete", help=delete_subscription.__doc__) delete_parser.add_argument("subscription_name") - update_parser = subparsers.add_parser( - "update", - help=update_subscription.__doc__ - ) + update_parser = subparsers.add_parser("update", help=update_subscription.__doc__) update_parser.add_argument("subscription_name") update_parser.add_argument("endpoint") - receive_parser = subparsers.add_parser( - "receive", - help=receive_messages.__doc__ - ) + receive_parser = subparsers.add_parser("receive", help=receive_messages.__doc__) receive_parser.add_argument("subscription_name") receive_with_custom_attributes_parser = subparsers.add_parser( @@ -490,8 +435,7 @@ def callback(message): receive_with_custom_attributes_parser.add_argument("subscription_name") receive_with_flow_control_parser = subparsers.add_parser( - "receive-flow-control", - help=receive_messages_with_flow_control.__doc__ + "receive-flow-control", help=receive_messages_with_flow_control.__doc__ ) receive_with_flow_control_parser.add_argument("subscription_name") @@ -504,9 +448,7 @@ def callback(message): "receive-synchronously-with-lease", help=synchronous_pull_with_lease_management.__doc__, ) - synchronous_pull_with_lease_management_parser.add_argument( - "subscription_name" - ) + synchronous_pull_with_lease_management_parser.add_argument("subscription_name") listen_for_errors_parser = subparsers.add_parser( "listen_for_errors", help=listen_for_errors.__doc__ @@ -520,44 +462,24 @@ def callback(message): elif args.command == "list_in_project": list_subscriptions_in_project(args.project_id) elif args.command == "create": - create_subscription( - args.project_id, - args.topic_name, - args.subscription_name - ) + create_subscription(args.project_id, args.topic_name, args.subscription_name) elif args.command == "create-push": create_push_subscription( - args.project_id, - args.topic_name, - args.subscription_name, - args.endpoint + args.project_id, args.topic_name, args.subscription_name, args.endpoint ) elif args.command == "delete": delete_subscription(args.project_id, args.subscription_name) elif args.command == "update": - update_subscription( - args.project_id, - args.subscription_name, - args.endpoint - ) + update_subscription(args.project_id, args.subscription_name, args.endpoint) elif args.command == "receive": receive_messages(args.project_id, args.subscription_name) elif args.command == "receive-custom-attributes": - receive_messages_with_custom_attributes( - args.project_id, - args.subscription_name - ) + receive_messages_with_custom_attributes(args.project_id, args.subscription_name) elif args.command == "receive-flow-control": - receive_messages_with_flow_control( - args.project_id, - args.subscription_name - ) + receive_messages_with_flow_control(args.project_id, args.subscription_name) elif args.command == "receive-synchronously": synchronous_pull(args.project_id, args.subscription_name) elif args.command == "receive-synchronously-with-lease": - synchronous_pull_with_lease_management( - args.project_id, - args.subscription_name - ) + synchronous_pull_with_lease_management(args.project_id, args.subscription_name) elif args.command == "listen_for_errors": listen_for_errors(args.project_id, args.subscription_name) diff --git a/pubsub/cloud-client/subscriber_test.py b/pubsub/cloud-client/subscriber_test.py index 41fabd373559..63d659dbcb6f 100644 --- a/pubsub/cloud-client/subscriber_test.py +++ b/pubsub/cloud-client/subscriber_test.py @@ -57,54 +57,36 @@ def subscriber_client(): @pytest.fixture(scope="module") def subscription_one(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path( - PROJECT, - SUBSCRIPTION_ONE - ) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_ONE) try: response = subscriber_client.get_subscription(subscription_path) except: # noqa - response = subscriber_client.create_subscription( - subscription_path, - topic=topic - ) + response = subscriber_client.create_subscription(subscription_path, topic=topic) yield response.name @pytest.fixture(scope="module") def subscription_two(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path( - PROJECT, - SUBSCRIPTION_TWO - ) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_TWO) try: response = subscriber_client.get_subscription(subscription_path) except: # noqa - response = subscriber_client.create_subscription( - subscription_path, - topic=topic - ) + response = subscriber_client.create_subscription(subscription_path, topic=topic) yield response.name @pytest.fixture(scope="module") def subscription_three(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path( - PROJECT, - SUBSCRIPTION_THREE - ) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_THREE) try: response = subscriber_client.get_subscription(subscription_path) except: # noqa - response = subscriber_client.create_subscription( - subscription_path, - topic=topic - ) + response = subscriber_client.create_subscription(subscription_path, topic=topic) yield response.name @@ -126,10 +108,7 @@ def _(): def test_create(subscriber_client): - subscription_path = subscriber_client.subscription_path( - PROJECT, - SUBSCRIPTION_ONE - ) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_ONE) try: subscriber_client.delete_subscription(subscription_path) @@ -144,21 +123,13 @@ def _(): def test_create_push(subscriber_client): - subscription_path = subscriber_client.subscription_path( - PROJECT, - SUBSCRIPTION_ONE - ) + subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_ONE) try: subscriber_client.delete_subscription(subscription_path) except Exception: pass - subscriber.create_push_subscription( - PROJECT, - TOPIC, - SUBSCRIPTION_ONE, - ENDPOINT - ) + subscriber.create_push_subscription(PROJECT, TOPIC, SUBSCRIPTION_ONE, ENDPOINT) @eventually_consistent.call def _(): @@ -214,9 +185,7 @@ def _to_delete(): for item in resources: if "subscription-test-topic" in item: - publisher_client.delete_topic( - "projects/{}/topics/{}".format(PROJECT, item) - ) + publisher_client.delete_topic("projects/{}/topics/{}".format(PROJECT, item)) if "subscription-test-subscription" in item: subscriber_client.delete_subscription( "projects/{}/subscriptions/{}".format(PROJECT, item) @@ -254,21 +223,13 @@ def test_receive_with_custom_attributes( assert "python-sample" in out -def test_receive_with_flow_control( - publisher_client, - topic, - subscription_two, - capsys -): +def test_receive_with_flow_control(publisher_client, topic, subscription_two, capsys): _publish_messages(publisher_client, topic) with _make_sleep_patch(): with pytest.raises(RuntimeError, match="sigil"): - subscriber.receive_messages_with_flow_control( - PROJECT, - SUBSCRIPTION_TWO - ) + subscriber.receive_messages_with_flow_control(PROJECT, SUBSCRIPTION_TWO) out, _ = capsys.readouterr() assert "Listening" in out @@ -276,12 +237,7 @@ def test_receive_with_flow_control( assert "Message" in out -def test_receive_synchronously( - publisher_client, - topic, - subscription_three, - capsys -): +def test_receive_synchronously(publisher_client, topic, subscription_three, capsys): _publish_messages(publisher_client, topic) subscriber.synchronous_pull(PROJECT, SUBSCRIPTION_THREE) @@ -291,17 +247,11 @@ def test_receive_synchronously( def test_receive_synchronously_with_lease( - publisher_client, - topic, - subscription_three, - capsys + publisher_client, topic, subscription_three, capsys ): _publish_messages(publisher_client, topic) - subscriber.synchronous_pull_with_lease_management( - PROJECT, - SUBSCRIPTION_THREE - ) + subscriber.synchronous_pull_with_lease_management(PROJECT, SUBSCRIPTION_THREE) out, _ = capsys.readouterr() assert "Done." in out diff --git a/pubsub/streaming-analytics/PubSubToGCS.py b/pubsub/streaming-analytics/PubSubToGCS.py index f414fdf7705f..3546edc558c2 100644 --- a/pubsub/streaming-analytics/PubSubToGCS.py +++ b/pubsub/streaming-analytics/PubSubToGCS.py @@ -95,9 +95,7 @@ def run(input_topic, output_path, window_size=1.0, pipeline_args=None): with beam.Pipeline(options=pipeline_options) as pipeline: ( pipeline - | "Read PubSub Messages" >> beam.io.ReadFromPubSub( - topic=input_topic - ) + | "Read PubSub Messages" >> beam.io.ReadFromPubSub(topic=input_topic) | "Window into" >> GroupWindowsIntoBatches(window_size) | "Write to GCS" >> beam.ParDo(WriteBatchesToGCS(output_path)) ) @@ -119,8 +117,7 @@ def run(input_topic, output_path, window_size=1.0, pipeline_args=None): help="Output file's window size in number of minutes.", ) parser.add_argument( - "--output_path", - help="GCS Path of the output file including filename prefix." + "--output_path", help="GCS Path of the output file including filename prefix." ) known_args, pipeline_args = parser.parse_known_args() From ffa76bb6a93f7bce38eeb372ed94f7208b88da43 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Fri, 6 Dec 2019 14:35:11 -0800 Subject: [PATCH 12/20] update assert to remove bytestring notation --- pubsub/cloud-client/quickstart/pub_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pubsub/cloud-client/quickstart/pub_test.py b/pubsub/cloud-client/quickstart/pub_test.py index b92a2ceaaeb7..b9a6f807f37d 100644 --- a/pubsub/cloud-client/quickstart/pub_test.py +++ b/pubsub/cloud-client/quickstart/pub_test.py @@ -52,4 +52,4 @@ def test_pub(publisher_client, topic, capsys): out, _ = capsys.readouterr() - assert "Published message b'Hello, World!'" in out + assert "Hello, World!" in out From e877d548b444670beef6b35219b3ee42ba345064 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Fri, 6 Dec 2019 14:35:37 -0800 Subject: [PATCH 13/20] move cleanup code in fixture --- pubsub/streaming-analytics/PubSubToGCS_test.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pubsub/streaming-analytics/PubSubToGCS_test.py b/pubsub/streaming-analytics/PubSubToGCS_test.py index d59097be5a68..e465bddc37fc 100644 --- a/pubsub/streaming-analytics/PubSubToGCS_test.py +++ b/pubsub/streaming-analytics/PubSubToGCS_test.py @@ -47,6 +47,12 @@ def topic_path(publisher_client): response = publisher_client.create_topic(topic_path) yield response.name + subscriber_client = pubsub_v1.SubscriberClient() + subscriptions = publisher_client.list_topic_subscriptions(topic_path) + for subscription in subscriptions: + subscriber_client.delete_subscription(subscription) + publisher_client.delete_topic(topic_path) + def _infinite_publish_job(publisher_client, topic_path): while True: @@ -105,10 +111,5 @@ def test_run(publisher_client, topic_path): files = gcs_client.list_prefix("gs://{}/pubsub/{}".format(BUCKET, UUID)) assert len(files) > 0 - # Clean up. Delete subscription. Delete topic. Delete GCS files. - subscriber_client = pubsub_v1.SubscriberClient() - subscriptions = publisher_client.list_topic_subscriptions(topic_path) - for subscription in subscriptions: - subscriber_client.delete_subscription(subscription) - publisher_client.delete_topic(topic_path) + # Clean up. gcs_client.delete_batch(list(files)) From 6fb596c77431574ff635c806a5dbc1ab659f626a Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Mon, 9 Dec 2019 16:53:28 -0800 Subject: [PATCH 14/20] rewrite PubSubToGCS test using dataflow testing module --- .../streaming-analytics/PubSubToGCS_test.py | 138 +++++++----------- pubsub/streaming-analytics/requirements.txt | 3 +- 2 files changed, 53 insertions(+), 88 deletions(-) diff --git a/pubsub/streaming-analytics/PubSubToGCS_test.py b/pubsub/streaming-analytics/PubSubToGCS_test.py index e465bddc37fc..239a14e4959c 100644 --- a/pubsub/streaming-analytics/PubSubToGCS_test.py +++ b/pubsub/streaming-analytics/PubSubToGCS_test.py @@ -12,104 +12,68 @@ # See the License for the specific language governing permissions and # limitations under the License. -import multiprocessing as mp +import logging import os -import pytest -import subprocess as sp -import tempfile -import time +import unittest import uuid import apache_beam as beam -from google.cloud import pubsub_v1 +from apache_beam.testing.test_pipeline import TestPipeline +from apache_beam.testing.test_stream import TestStream +from apache_beam.testing.test_utils import TempDir +from apache_beam.transforms.window import TimestampedValue +from apache_beam.testing.util import assert_that +from PubSubToGCS import * PROJECT = os.environ["GCLOUD_PROJECT"] BUCKET = os.environ["CLOUD_STORAGE_BUCKET"] UUID = uuid.uuid1().hex -TOPIC = "test-topic-" + UUID -@pytest.fixture -def publisher_client(): - yield pubsub_v1.PublisherClient() +class PubSubToGCSTest(unittest.TestCase): + def test_pubsub_to_gcs(self): - -@pytest.fixture -def topic_path(publisher_client): - topic_path = publisher_client.topic_path(PROJECT, TOPIC) - - try: - publisher_client.delete_topic(topic_path) - except Exception: - pass - - response = publisher_client.create_topic(topic_path) - yield response.name - - subscriber_client = pubsub_v1.SubscriberClient() - subscriptions = publisher_client.list_topic_subscriptions(topic_path) - for subscription in subscriptions: - subscriber_client.delete_subscription(subscription) - publisher_client.delete_topic(topic_path) - - -def _infinite_publish_job(publisher_client, topic_path): - while True: - future = publisher_client.publish( - topic_path, data="Hello World!".encode("utf-8") + pipeline_options = PipelineOptions( + project=PROJECT, + runner="DirectRunner", + temp_location=TempDir().get_path(), + streaming=True, + save_main_session=True, ) - future.result() - time.sleep(10) - - -def test_run(publisher_client, topic_path): - """This is an integration test that runs `PubSubToGCS.py` in its entirety. - It checks for output files on GCS. - """ - - # Use one process to publish messages to a topic. - publish_process = mp.Process( - target=lambda: _infinite_publish_job(publisher_client, topic_path) - ) - - # Use another process to run the streaming pipeline that should write one - # file to GCS every minute (according to the default window size). - pipeline_process = mp.Process( - target=lambda: sp.call( - [ - "python", - "PubSubToGCS.py", - "--project", - PROJECT, - "--runner", - "DirectRunner", - "--temp_location", - tempfile.mkdtemp(), - "--input_topic", - topic_path, - "--output_path", - "gs://{}/pubsub/{}/output".format(BUCKET, UUID), - ] - ) - ) - - publish_process.start() - pipeline_process.start() - - # Times out the streaming pipeline after 90 seconds. - pipeline_process.join(timeout=90) - # Immediately kills the publish process after the pipeline shuts down. - publish_process.join(timeout=0) - - pipeline_process.terminate() - publish_process.terminate() - - # Check for output files on GCS. - gcs_client = beam.io.gcp.gcsio.GcsIO() - # This returns a dictionary. - files = gcs_client.list_prefix("gs://{}/pubsub/{}".format(BUCKET, UUID)) - assert len(files) > 0 - # Clean up. - gcs_client.delete_batch(list(files)) + with TestPipeline(options=pipeline_options) as p: + + mocked_pubsub_stream = ( + TestStream() + .advance_watermark_to(0) + .advance_processing_time(30) + .add_elements([TimestampedValue(b"a", 1575937195)]) + .advance_processing_time(30) + .add_elements([TimestampedValue(b"b", 1575937225)]) + .advance_processing_time(30) + .add_elements([TimestampedValue(b"c", 1575937255)]) + .advance_watermark_to_infinity() + ) + + output_path = "gs://{}/pubsub/{}/output".format(BUCKET, UUID) + + _ = ( + p + | mocked_pubsub_stream + | "Window into" >> GroupWindowsIntoBatches(1) + | "Write to GCS" >> beam.ParDo(WriteBatchesToGCS(output_path)) + ) + + # Check for output files on GCS. + gcs_client = beam.io.gcp.gcsio.GcsIO() + files = gcs_client.list_prefix("gs://{}/pubsub/{}".format(BUCKET, UUID)) + assert len(files) > 0 + + # Clean up. + gcs_client.delete_batch(list(files)) + + +if __name__ == "__main__": + logging.getLogger().setLevel(logging.INFO) + unittest.main() diff --git a/pubsub/streaming-analytics/requirements.txt b/pubsub/streaming-analytics/requirements.txt index d1ce1adeb94a..451f429f5bde 100644 --- a/pubsub/streaming-analytics/requirements.txt +++ b/pubsub/streaming-analytics/requirements.txt @@ -1 +1,2 @@ -apache-beam[gcp]==2.15.0 +apache-beam[gcp]==2.16.0 +nose==1.3.7 From 85a3d98ea3316eb5286449a2413813c67d489de9 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Mon, 9 Dec 2019 17:00:41 -0800 Subject: [PATCH 15/20] lint --- pubsub/streaming-analytics/PubSubToGCS_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pubsub/streaming-analytics/PubSubToGCS_test.py b/pubsub/streaming-analytics/PubSubToGCS_test.py index 239a14e4959c..cbccfe70cf5e 100644 --- a/pubsub/streaming-analytics/PubSubToGCS_test.py +++ b/pubsub/streaming-analytics/PubSubToGCS_test.py @@ -24,7 +24,7 @@ from apache_beam.transforms.window import TimestampedValue from apache_beam.testing.util import assert_that -from PubSubToGCS import * +import PubSubToGCS PROJECT = os.environ["GCLOUD_PROJECT"] BUCKET = os.environ["CLOUD_STORAGE_BUCKET"] @@ -61,8 +61,8 @@ def test_pubsub_to_gcs(self): _ = ( p | mocked_pubsub_stream - | "Window into" >> GroupWindowsIntoBatches(1) - | "Write to GCS" >> beam.ParDo(WriteBatchesToGCS(output_path)) + | PubSubToGCS.GroupWindowsIntoBatches(1) + | beam.ParDo(PubSubToGCS.WriteBatchesToGCS(output_path)) ) # Check for output files on GCS. From 9ef38f50d524024c5b7e63f19d77e9bd12706cb7 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Mon, 9 Dec 2019 17:02:58 -0800 Subject: [PATCH 16/20] missing import --- pubsub/streaming-analytics/PubSubToGCS_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pubsub/streaming-analytics/PubSubToGCS_test.py b/pubsub/streaming-analytics/PubSubToGCS_test.py index cbccfe70cf5e..2b5dfeeb150d 100644 --- a/pubsub/streaming-analytics/PubSubToGCS_test.py +++ b/pubsub/streaming-analytics/PubSubToGCS_test.py @@ -18,6 +18,7 @@ import uuid import apache_beam as beam +from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.test_stream import TestStream from apache_beam.testing.test_utils import TempDir From a4d9a2662a273572274d1ef670191b67c5c842f5 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Mon, 9 Dec 2019 17:13:33 -0800 Subject: [PATCH 17/20] redundant import --- pubsub/streaming-analytics/PubSubToGCS_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pubsub/streaming-analytics/PubSubToGCS_test.py b/pubsub/streaming-analytics/PubSubToGCS_test.py index 2b5dfeeb150d..ed8e549c0271 100644 --- a/pubsub/streaming-analytics/PubSubToGCS_test.py +++ b/pubsub/streaming-analytics/PubSubToGCS_test.py @@ -23,7 +23,6 @@ from apache_beam.testing.test_stream import TestStream from apache_beam.testing.test_utils import TempDir from apache_beam.transforms.window import TimestampedValue -from apache_beam.testing.util import assert_that import PubSubToGCS From b4a0120550cc7e8725845476fe24facecda8a8eb Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Tue, 10 Dec 2019 12:11:02 -0800 Subject: [PATCH 18/20] david's suggestions --- pubsub/streaming-analytics/PubSubToGCS.py | 2 +- .../streaming-analytics/PubSubToGCS_test.py | 80 +++++++------------ 2 files changed, 32 insertions(+), 50 deletions(-) diff --git a/pubsub/streaming-analytics/PubSubToGCS.py b/pubsub/streaming-analytics/PubSubToGCS.py index 3546edc558c2..b698823de5c8 100644 --- a/pubsub/streaming-analytics/PubSubToGCS.py +++ b/pubsub/streaming-analytics/PubSubToGCS.py @@ -40,7 +40,7 @@ def expand(self, pcoll): # publish timestamp. | "Window into Fixed Intervals" >> beam.WindowInto(window.FixedWindows(self.window_size)) - | "Add timestamps to messages" >> (beam.ParDo(AddTimestamps())) + | "Add timestamps to messages" >> beam.ParDo(AddTimestamps()) # Use a dummy key to group the elements in the same window. # Note that all the elements in one window must fit into memory # for this. If the windowed elements do not fit into memory, diff --git a/pubsub/streaming-analytics/PubSubToGCS_test.py b/pubsub/streaming-analytics/PubSubToGCS_test.py index ed8e549c0271..7799e2081d8b 100644 --- a/pubsub/streaming-analytics/PubSubToGCS_test.py +++ b/pubsub/streaming-analytics/PubSubToGCS_test.py @@ -12,13 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging +import mock import os -import unittest import uuid import apache_beam as beam -from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.test_stream import TestStream from apache_beam.testing.test_utils import TempDir @@ -31,49 +29,33 @@ UUID = uuid.uuid1().hex -class PubSubToGCSTest(unittest.TestCase): - def test_pubsub_to_gcs(self): - - pipeline_options = PipelineOptions( - project=PROJECT, - runner="DirectRunner", - temp_location=TempDir().get_path(), - streaming=True, - save_main_session=True, - ) - - with TestPipeline(options=pipeline_options) as p: - - mocked_pubsub_stream = ( - TestStream() - .advance_watermark_to(0) - .advance_processing_time(30) - .add_elements([TimestampedValue(b"a", 1575937195)]) - .advance_processing_time(30) - .add_elements([TimestampedValue(b"b", 1575937225)]) - .advance_processing_time(30) - .add_elements([TimestampedValue(b"c", 1575937255)]) - .advance_watermark_to_infinity() - ) - - output_path = "gs://{}/pubsub/{}/output".format(BUCKET, UUID) - - _ = ( - p - | mocked_pubsub_stream - | PubSubToGCS.GroupWindowsIntoBatches(1) - | beam.ParDo(PubSubToGCS.WriteBatchesToGCS(output_path)) - ) - - # Check for output files on GCS. - gcs_client = beam.io.gcp.gcsio.GcsIO() - files = gcs_client.list_prefix("gs://{}/pubsub/{}".format(BUCKET, UUID)) - assert len(files) > 0 - - # Clean up. - gcs_client.delete_batch(list(files)) - - -if __name__ == "__main__": - logging.getLogger().setLevel(logging.INFO) - unittest.main() +@mock.patch("apache_beam.Pipeline", TestPipeline) +@mock.patch( + "apache_beam.io.ReadFromPubSub", + lambda topic: ( + TestStream() + .advance_watermark_to(0) + .advance_processing_time(30) + .add_elements([TimestampedValue(b"a", 1575937195)]) + .advance_processing_time(30) + .add_elements([TimestampedValue(b"b", 1575937225)]) + .advance_processing_time(30) + .add_elements([TimestampedValue(b"c", 1575937255)]) + .advance_watermark_to_infinity() + ), +) +def test_pubsub_to_gcs(): + PubSubToGCS.run( + input_topic="unused", # mocked by TestStream + output_path="gs://{}/pubsub/{}/output".format(BUCKET, UUID), + window_size=1, # 1 minute + pipeline_args=["--project", PROJECT, "--temp_location", TempDir().get_path()], + ) + + # Check for output files on GCS. + gcs_client = beam.io.gcp.gcsio.GcsIO() + files = gcs_client.list_prefix("gs://{}/pubsub/{}".format(BUCKET, UUID)) + assert len(files) > 0 + + # Clean up. + gcs_client.delete_batch(list(files)) From 4fadbfc8e732cdbc3a775c58f450051f4aff8880 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Tue, 10 Dec 2019 13:48:07 -0800 Subject: [PATCH 19/20] update requirements.txt --- pubsub/streaming-analytics/requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pubsub/streaming-analytics/requirements.txt b/pubsub/streaming-analytics/requirements.txt index 451f429f5bde..7ffca51187b5 100644 --- a/pubsub/streaming-analytics/requirements.txt +++ b/pubsub/streaming-analytics/requirements.txt @@ -1,2 +1 @@ -apache-beam[gcp]==2.16.0 -nose==1.3.7 +apache-beam[gcp,test]==2.16.0 From 7cf569338a5253175a1070db3a12fabdfa351e2e Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Tue, 10 Dec 2019 15:59:54 -0800 Subject: [PATCH 20/20] black -l 79 --- pubsub/cloud-client/iam.py | 24 +++- pubsub/cloud-client/iam_test.py | 4 +- pubsub/cloud-client/publisher.py | 20 +++- pubsub/cloud-client/publisher_test.py | 4 +- pubsub/cloud-client/quickstart.py | 15 ++- pubsub/cloud-client/quickstart/pub.py | 3 +- pubsub/cloud-client/quickstart/sub.py | 11 +- pubsub/cloud-client/quickstart/sub_test.py | 8 +- pubsub/cloud-client/quickstart_test.py | 4 +- pubsub/cloud-client/subscriber.py | 104 +++++++++++++----- pubsub/cloud-client/subscriber_test.py | 56 +++++++--- pubsub/streaming-analytics/PubSubToGCS.py | 6 +- .../streaming-analytics/PubSubToGCS_test.py | 7 +- 13 files changed, 198 insertions(+), 68 deletions(-) diff --git a/pubsub/cloud-client/iam.py b/pubsub/cloud-client/iam.py index 44e52e576295..f014ce749022 100644 --- a/pubsub/cloud-client/iam.py +++ b/pubsub/cloud-client/iam.py @@ -89,12 +89,18 @@ def set_subscription_policy(project, subscription_name): policy.bindings.add(role="roles/pubsub.viewer", members=["allUsers"]) # Add a group as an editor. - policy.bindings.add(role="roles/editor", members=["group:cloud-logs@google.com"]) + policy.bindings.add( + role="roles/editor", members=["group:cloud-logs@google.com"] + ) # Set the policy policy = client.set_iam_policy(subscription_path, policy) - print("IAM policy for subscription {} set: {}".format(subscription_name, policy)) + print( + "IAM policy for subscription {} set: {}".format( + subscription_name, policy + ) + ) # [END pubsub_set_subscription_policy] @@ -106,10 +112,14 @@ def check_topic_permissions(project, topic_name): permissions_to_check = ["pubsub.topics.publish", "pubsub.topics.update"] - allowed_permissions = client.test_iam_permissions(topic_path, permissions_to_check) + allowed_permissions = client.test_iam_permissions( + topic_path, permissions_to_check + ) print( - "Allowed permissions for topic {}: {}".format(topic_path, allowed_permissions) + "Allowed permissions for topic {}: {}".format( + topic_path, allowed_permissions + ) ) # [END pubsub_test_topic_permissions] @@ -139,7 +149,8 @@ def check_subscription_permissions(project, subscription_name): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("project", help="Your Google Cloud project ID") @@ -171,7 +182,8 @@ def check_subscription_permissions(project, subscription_name): check_topic_permissions_parser.add_argument("topic_name") check_subscription_permissions_parser = subparsers.add_parser( - "check-subscription-permissions", help=check_subscription_permissions.__doc__ + "check-subscription-permissions", + help=check_subscription_permissions.__doc__, ) check_subscription_permissions_parser.add_argument("subscription_name") diff --git a/pubsub/cloud-client/iam_test.py b/pubsub/cloud-client/iam_test.py index ac62bbfc4f31..2b019f9ea16f 100644 --- a/pubsub/cloud-client/iam_test.py +++ b/pubsub/cloud-client/iam_test.py @@ -54,7 +54,9 @@ def subscriber_client(): @pytest.fixture def subscription(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION) + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION + ) try: subscriber_client.delete_subscription(subscription_path) diff --git a/pubsub/cloud-client/publisher.py b/pubsub/cloud-client/publisher.py index 0fd23cdba694..d227baab9584 100644 --- a/pubsub/cloud-client/publisher.py +++ b/pubsub/cloud-client/publisher.py @@ -283,7 +283,8 @@ def publish_messages_with_retry_settings(project_id, topic_name): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("project_id", help="Your Google Cloud project ID") @@ -296,7 +297,9 @@ def publish_messages_with_retry_settings(project_id, topic_name): delete_parser = subparsers.add_parser("delete", help=delete_topic.__doc__) delete_parser.add_argument("topic_name") - publish_parser = subparsers.add_parser("publish", help=publish_messages.__doc__) + publish_parser = subparsers.add_parser( + "publish", help=publish_messages.__doc__ + ) publish_parser.add_argument("topic_name") publish_with_custom_attributes_parser = subparsers.add_parser( @@ -311,17 +314,20 @@ def publish_messages_with_retry_settings(project_id, topic_name): publish_with_futures_parser.add_argument("topic_name") publish_with_error_handler_parser = subparsers.add_parser( - "publish-with-error-handler", help=publish_messages_with_error_handler.__doc__ + "publish-with-error-handler", + help=publish_messages_with_error_handler.__doc__, ) publish_with_error_handler_parser.add_argument("topic_name") publish_with_batch_settings_parser = subparsers.add_parser( - "publish-with-batch-settings", help=publish_messages_with_batch_settings.__doc__ + "publish-with-batch-settings", + help=publish_messages_with_batch_settings.__doc__, ) publish_with_batch_settings_parser.add_argument("topic_name") publish_with_retry_settings_parser = subparsers.add_parser( - "publish-with-retry-settings", help=publish_messages_with_retry_settings.__doc__ + "publish-with-retry-settings", + help=publish_messages_with_retry_settings.__doc__, ) publish_with_retry_settings_parser.add_argument("topic_name") @@ -336,7 +342,9 @@ def publish_messages_with_retry_settings(project_id, topic_name): elif args.command == "publish": publish_messages(args.project_id, args.topic_name) elif args.command == "publish-with-custom-attributes": - publish_messages_with_custom_attributes(args.project_id, args.topic_name) + publish_messages_with_custom_attributes( + args.project_id, args.topic_name + ) elif args.command == "publish-with-futures": publish_messages_with_futures(args.project_id, args.topic_name) elif args.command == "publish-with-error-handler": diff --git a/pubsub/cloud-client/publisher_test.py b/pubsub/cloud-client/publisher_test.py index a868732790d5..125fae3c06b9 100644 --- a/pubsub/cloud-client/publisher_test.py +++ b/pubsub/cloud-client/publisher_test.py @@ -60,7 +60,9 @@ def new_sleep(period): def _to_delete(): publisher_client = pubsub_v1.PublisherClient() - publisher_client.delete_topic("projects/{}/topics/{}".format(PROJECT, TOPIC)) + publisher_client.delete_topic( + "projects/{}/topics/{}".format(PROJECT, TOPIC) + ) def test_list(client, topic, capsys): diff --git a/pubsub/cloud-client/quickstart.py b/pubsub/cloud-client/quickstart.py index 2a534165ce77..d01105885cb8 100644 --- a/pubsub/cloud-client/quickstart.py +++ b/pubsub/cloud-client/quickstart.py @@ -38,14 +38,18 @@ def end_to_end(project_id, topic_name, subscription_name, num_messages): # The `subscription_path` method creates a fully qualified identifier # in the form `projects/{project_id}/subscriptions/{subscription_name}` - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, subscription_name + ) # Create the topic. topic = publisher.create_topic(topic_path) print("\nTopic created: {}".format(topic.name)) # Create a subscription. - subscription = subscriber.create_subscription(subscription_path, topic_path) + subscription = subscriber.create_subscription( + subscription_path, topic_path + ) print("\nSubscription created: {}\n".format(subscription.name)) publish_begin = time.time() @@ -92,7 +96,8 @@ def callback(message): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("project_id", help="Your Google Cloud project ID") parser.add_argument("topic_name", help="Your topic name") @@ -101,4 +106,6 @@ def callback(message): args = parser.parse_args() - end_to_end(args.project_id, args.topic_name, args.subscription_name, args.num_msgs) + end_to_end( + args.project_id, args.topic_name, args.subscription_name, args.num_msgs + ) diff --git a/pubsub/cloud-client/quickstart/pub.py b/pubsub/cloud-client/quickstart/pub.py index e14a8c04bf10..a3f8087ecd15 100644 --- a/pubsub/cloud-client/quickstart/pub.py +++ b/pubsub/cloud-client/quickstart/pub.py @@ -75,7 +75,8 @@ def pub(project_id, topic_name): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("project_id", help="Google Cloud project ID") parser.add_argument("topic_name", help="Pub/Sub topic name") diff --git a/pubsub/cloud-client/quickstart/sub.py b/pubsub/cloud-client/quickstart/sub.py index 5113c91b3255..5791af14d799 100644 --- a/pubsub/cloud-client/quickstart/sub.py +++ b/pubsub/cloud-client/quickstart/sub.py @@ -35,13 +35,17 @@ def sub(project_id, subscription_name): def callback(message): print( - "Received message {} of message ID {}\n".format(message, message.message_id) + "Received message {} of message ID {}\n".format( + message, message.message_id + ) ) # Acknowledge the message. Unack'ed messages will be redelivered. message.ack() print("Acknowledged message {}\n".format(message.message_id)) - streaming_pull_future = client.subscribe(subscription_path, callback=callback) + streaming_pull_future = client.subscribe( + subscription_path, callback=callback + ) print("Listening for messages on {}..\n".format(subscription_path)) # Calling result() on StreamingPullFuture keeps the main thread from @@ -54,7 +58,8 @@ def callback(message): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("project_id", help="Google Cloud project ID") parser.add_argument("subscription_name", help="Pub/Sub subscription name") diff --git a/pubsub/cloud-client/quickstart/sub_test.py b/pubsub/cloud-client/quickstart/sub_test.py index 584b6dae1099..07edfad7c4d2 100644 --- a/pubsub/cloud-client/quickstart/sub_test.py +++ b/pubsub/cloud-client/quickstart/sub_test.py @@ -49,7 +49,9 @@ def topic_path(): @pytest.fixture(scope="module") def subscription_path(topic_path): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION) + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION + ) try: subscription = subscriber_client.create_subscription( @@ -80,7 +82,9 @@ def test_sub(monkeypatch, topic_path, subscription_path, capsys): monkeypatch.setattr(pubsub_v1, "SubscriberClient", mock_client_constructor) def mock_subscribe(subscription_path, callback=None): - real_future = real_client.subscribe(subscription_path, callback=callback) + real_future = real_client.subscribe( + subscription_path, callback=callback + ) mock_future = mock.Mock(spec=real_future, wraps=real_future) def mock_result(): diff --git a/pubsub/cloud-client/quickstart_test.py b/pubsub/cloud-client/quickstart_test.py index e3a567a7787d..6a1d4aae1b5f 100644 --- a/pubsub/cloud-client/quickstart_test.py +++ b/pubsub/cloud-client/quickstart_test.py @@ -54,7 +54,9 @@ def subscriber_client(): @pytest.fixture(scope="module") def subscription(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION) + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION + ) try: subscriber_client.delete_subscription(subscription_path) diff --git a/pubsub/cloud-client/subscriber.py b/pubsub/cloud-client/subscriber.py index e07e02281589..ea1cc9ff9e72 100644 --- a/pubsub/cloud-client/subscriber.py +++ b/pubsub/cloud-client/subscriber.py @@ -66,15 +66,21 @@ def create_subscription(project_id, topic_name, subscription_name): subscriber = pubsub_v1.SubscriberClient() topic_path = subscriber.topic_path(project_id, topic_name) - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, subscription_name + ) - subscription = subscriber.create_subscription(subscription_path, topic_path) + subscription = subscriber.create_subscription( + subscription_path, topic_path + ) print("Subscription created: {}".format(subscription)) # [END pubsub_create_pull_subscription] -def create_push_subscription(project_id, topic_name, subscription_name, endpoint): +def create_push_subscription( + project_id, topic_name, subscription_name, endpoint +): """Create a new push subscription on the given topic.""" # [START pubsub_create_push_subscription] from google.cloud import pubsub_v1 @@ -86,7 +92,9 @@ def create_push_subscription(project_id, topic_name, subscription_name, endpoint subscriber = pubsub_v1.SubscriberClient() topic_path = subscriber.topic_path(project_id, topic_name) - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, subscription_name + ) push_config = pubsub_v1.types.PushConfig(push_endpoint=endpoint) @@ -108,7 +116,9 @@ def delete_subscription(project_id, subscription_name): # TODO subscription_name = "Your Pub/Sub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, subscription_name + ) subscriber.delete_subscription(subscription_path) @@ -131,7 +141,9 @@ def update_subscription(project_id, subscription_name, endpoint): # TODO endpoint = "https://my-test-project.appspot.com/push" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, subscription_name + ) push_config = pubsub_v1.types.PushConfig(push_endpoint=endpoint) @@ -163,7 +175,9 @@ def receive_messages(project_id, subscription_name): subscriber = pubsub_v1.SubscriberClient() # The `subscription_path` method creates a fully qualified identifier # in the form `projects/{project_id}/subscriptions/{subscription_name}` - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, subscription_name + ) def callback(message): print("Received message: {}".format(message)) @@ -192,7 +206,9 @@ def receive_messages_with_custom_attributes(project_id, subscription_name): # TODO subscription_name = "Your Pub/Sub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, subscription_name + ) def callback(message): print("Received message: {}".format(message.data)) @@ -225,7 +241,9 @@ def receive_messages_with_flow_control(project_id, subscription_name): # TODO subscription_name = "Your Pub/Sub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, subscription_name + ) def callback(message): print("Received message: {}".format(message.data)) @@ -254,7 +272,9 @@ def synchronous_pull(project_id, subscription_name): # TODO subscription_name = "Your Pub/Sub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, subscription_name + ) NUM_MESSAGES = 3 @@ -291,7 +311,9 @@ def synchronous_pull_with_lease_management(project_id, subscription_name): # TODO subscription_name = "Your Pub/Sub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, subscription_name + ) NUM_MESSAGES = 2 ACK_DEADLINE = 30 @@ -330,11 +352,15 @@ def worker(msg): if process.is_alive(): # `ack_deadline_seconds` must be between 10 to 600. subscriber.modify_ack_deadline( - subscription_path, [ack_id], ack_deadline_seconds=ACK_DEADLINE + subscription_path, + [ack_id], + ack_deadline_seconds=ACK_DEADLINE, ) logger.info( "{}: Reset ack deadline for {} for {}s".format( - time.strftime("%X", time.gmtime()), msg_data, ACK_DEADLINE + time.strftime("%X", time.gmtime()), + msg_data, + ACK_DEADLINE, ) ) @@ -369,7 +395,9 @@ def listen_for_errors(project_id, subscription_name): # TODO subscription_name = "Your Pubsub subscription name" subscriber = pubsub_v1.SubscriberClient() - subscription_path = subscriber.subscription_path(project_id, subscription_name) + subscription_path = subscriber.subscription_path( + project_id, subscription_name + ) def callback(message): print("Received message: {}".format(message)) @@ -393,7 +421,8 @@ def callback(message): if __name__ == "__main__": parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("project_id", help="Your Google Cloud project ID") @@ -407,7 +436,9 @@ def callback(message): "list_in_project", help=list_subscriptions_in_project.__doc__ ) - create_parser = subparsers.add_parser("create", help=create_subscription.__doc__) + create_parser = subparsers.add_parser( + "create", help=create_subscription.__doc__ + ) create_parser.add_argument("topic_name") create_parser.add_argument("subscription_name") @@ -418,14 +449,20 @@ def callback(message): create_push_parser.add_argument("subscription_name") create_push_parser.add_argument("endpoint") - delete_parser = subparsers.add_parser("delete", help=delete_subscription.__doc__) + delete_parser = subparsers.add_parser( + "delete", help=delete_subscription.__doc__ + ) delete_parser.add_argument("subscription_name") - update_parser = subparsers.add_parser("update", help=update_subscription.__doc__) + update_parser = subparsers.add_parser( + "update", help=update_subscription.__doc__ + ) update_parser.add_argument("subscription_name") update_parser.add_argument("endpoint") - receive_parser = subparsers.add_parser("receive", help=receive_messages.__doc__) + receive_parser = subparsers.add_parser( + "receive", help=receive_messages.__doc__ + ) receive_parser.add_argument("subscription_name") receive_with_custom_attributes_parser = subparsers.add_parser( @@ -448,7 +485,9 @@ def callback(message): "receive-synchronously-with-lease", help=synchronous_pull_with_lease_management.__doc__, ) - synchronous_pull_with_lease_management_parser.add_argument("subscription_name") + synchronous_pull_with_lease_management_parser.add_argument( + "subscription_name" + ) listen_for_errors_parser = subparsers.add_parser( "listen_for_errors", help=listen_for_errors.__doc__ @@ -462,24 +501,37 @@ def callback(message): elif args.command == "list_in_project": list_subscriptions_in_project(args.project_id) elif args.command == "create": - create_subscription(args.project_id, args.topic_name, args.subscription_name) + create_subscription( + args.project_id, args.topic_name, args.subscription_name + ) elif args.command == "create-push": create_push_subscription( - args.project_id, args.topic_name, args.subscription_name, args.endpoint + args.project_id, + args.topic_name, + args.subscription_name, + args.endpoint, ) elif args.command == "delete": delete_subscription(args.project_id, args.subscription_name) elif args.command == "update": - update_subscription(args.project_id, args.subscription_name, args.endpoint) + update_subscription( + args.project_id, args.subscription_name, args.endpoint + ) elif args.command == "receive": receive_messages(args.project_id, args.subscription_name) elif args.command == "receive-custom-attributes": - receive_messages_with_custom_attributes(args.project_id, args.subscription_name) + receive_messages_with_custom_attributes( + args.project_id, args.subscription_name + ) elif args.command == "receive-flow-control": - receive_messages_with_flow_control(args.project_id, args.subscription_name) + receive_messages_with_flow_control( + args.project_id, args.subscription_name + ) elif args.command == "receive-synchronously": synchronous_pull(args.project_id, args.subscription_name) elif args.command == "receive-synchronously-with-lease": - synchronous_pull_with_lease_management(args.project_id, args.subscription_name) + synchronous_pull_with_lease_management( + args.project_id, args.subscription_name + ) elif args.command == "listen_for_errors": listen_for_errors(args.project_id, args.subscription_name) diff --git a/pubsub/cloud-client/subscriber_test.py b/pubsub/cloud-client/subscriber_test.py index 63d659dbcb6f..0645c0738e1c 100644 --- a/pubsub/cloud-client/subscriber_test.py +++ b/pubsub/cloud-client/subscriber_test.py @@ -57,36 +57,48 @@ def subscriber_client(): @pytest.fixture(scope="module") def subscription_one(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_ONE) + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION_ONE + ) try: response = subscriber_client.get_subscription(subscription_path) except: # noqa - response = subscriber_client.create_subscription(subscription_path, topic=topic) + response = subscriber_client.create_subscription( + subscription_path, topic=topic + ) yield response.name @pytest.fixture(scope="module") def subscription_two(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_TWO) + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION_TWO + ) try: response = subscriber_client.get_subscription(subscription_path) except: # noqa - response = subscriber_client.create_subscription(subscription_path, topic=topic) + response = subscriber_client.create_subscription( + subscription_path, topic=topic + ) yield response.name @pytest.fixture(scope="module") def subscription_three(subscriber_client, topic): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_THREE) + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION_THREE + ) try: response = subscriber_client.get_subscription(subscription_path) except: # noqa - response = subscriber_client.create_subscription(subscription_path, topic=topic) + response = subscriber_client.create_subscription( + subscription_path, topic=topic + ) yield response.name @@ -108,7 +120,9 @@ def _(): def test_create(subscriber_client): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_ONE) + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION_ONE + ) try: subscriber_client.delete_subscription(subscription_path) @@ -123,13 +137,17 @@ def _(): def test_create_push(subscriber_client): - subscription_path = subscriber_client.subscription_path(PROJECT, SUBSCRIPTION_ONE) + subscription_path = subscriber_client.subscription_path( + PROJECT, SUBSCRIPTION_ONE + ) try: subscriber_client.delete_subscription(subscription_path) except Exception: pass - subscriber.create_push_subscription(PROJECT, TOPIC, SUBSCRIPTION_ONE, ENDPOINT) + subscriber.create_push_subscription( + PROJECT, TOPIC, SUBSCRIPTION_ONE, ENDPOINT + ) @eventually_consistent.call def _(): @@ -185,7 +203,9 @@ def _to_delete(): for item in resources: if "subscription-test-topic" in item: - publisher_client.delete_topic("projects/{}/topics/{}".format(PROJECT, item)) + publisher_client.delete_topic( + "projects/{}/topics/{}".format(PROJECT, item) + ) if "subscription-test-subscription" in item: subscriber_client.delete_subscription( "projects/{}/subscriptions/{}".format(PROJECT, item) @@ -223,13 +243,17 @@ def test_receive_with_custom_attributes( assert "python-sample" in out -def test_receive_with_flow_control(publisher_client, topic, subscription_two, capsys): +def test_receive_with_flow_control( + publisher_client, topic, subscription_two, capsys +): _publish_messages(publisher_client, topic) with _make_sleep_patch(): with pytest.raises(RuntimeError, match="sigil"): - subscriber.receive_messages_with_flow_control(PROJECT, SUBSCRIPTION_TWO) + subscriber.receive_messages_with_flow_control( + PROJECT, SUBSCRIPTION_TWO + ) out, _ = capsys.readouterr() assert "Listening" in out @@ -237,7 +261,9 @@ def test_receive_with_flow_control(publisher_client, topic, subscription_two, ca assert "Message" in out -def test_receive_synchronously(publisher_client, topic, subscription_three, capsys): +def test_receive_synchronously( + publisher_client, topic, subscription_three, capsys +): _publish_messages(publisher_client, topic) subscriber.synchronous_pull(PROJECT, SUBSCRIPTION_THREE) @@ -251,7 +277,9 @@ def test_receive_synchronously_with_lease( ): _publish_messages(publisher_client, topic) - subscriber.synchronous_pull_with_lease_management(PROJECT, SUBSCRIPTION_THREE) + subscriber.synchronous_pull_with_lease_management( + PROJECT, SUBSCRIPTION_THREE + ) out, _ = capsys.readouterr() assert "Done." in out diff --git a/pubsub/streaming-analytics/PubSubToGCS.py b/pubsub/streaming-analytics/PubSubToGCS.py index b698823de5c8..a3c8179b21e9 100644 --- a/pubsub/streaming-analytics/PubSubToGCS.py +++ b/pubsub/streaming-analytics/PubSubToGCS.py @@ -95,7 +95,8 @@ def run(input_topic, output_path, window_size=1.0, pipeline_args=None): with beam.Pipeline(options=pipeline_options) as pipeline: ( pipeline - | "Read PubSub Messages" >> beam.io.ReadFromPubSub(topic=input_topic) + | "Read PubSub Messages" + >> beam.io.ReadFromPubSub(topic=input_topic) | "Window into" >> GroupWindowsIntoBatches(window_size) | "Write to GCS" >> beam.ParDo(WriteBatchesToGCS(output_path)) ) @@ -117,7 +118,8 @@ def run(input_topic, output_path, window_size=1.0, pipeline_args=None): help="Output file's window size in number of minutes.", ) parser.add_argument( - "--output_path", help="GCS Path of the output file including filename prefix." + "--output_path", + help="GCS Path of the output file including filename prefix.", ) known_args, pipeline_args = parser.parse_known_args() diff --git a/pubsub/streaming-analytics/PubSubToGCS_test.py b/pubsub/streaming-analytics/PubSubToGCS_test.py index 7799e2081d8b..d39fb568184e 100644 --- a/pubsub/streaming-analytics/PubSubToGCS_test.py +++ b/pubsub/streaming-analytics/PubSubToGCS_test.py @@ -49,7 +49,12 @@ def test_pubsub_to_gcs(): input_topic="unused", # mocked by TestStream output_path="gs://{}/pubsub/{}/output".format(BUCKET, UUID), window_size=1, # 1 minute - pipeline_args=["--project", PROJECT, "--temp_location", TempDir().get_path()], + pipeline_args=[ + "--project", + PROJECT, + "--temp_location", + TempDir().get_path(), + ], ) # Check for output files on GCS.