From 26270d511073ede0a083360af692dad89e5c3eaf Mon Sep 17 00:00:00 2001 From: Averi Kitsch Date: Tue, 6 Aug 2019 11:15:06 -0700 Subject: [PATCH] Add Migration Guide Snippets for Cloud Tasks [(#2316)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/2316) * Migration guide * remove app and update migraitonn * snippets for migration guide - tests added * lint * remove print statements * Styling changes * Travis trigger --- samples/appengine/flexible/tasks/snippets.py | 285 ++++++++++++++++++ .../appengine/flexible/tasks/snippets_test.py | 110 +++++++ 2 files changed, 395 insertions(+) create mode 100644 samples/appengine/flexible/tasks/snippets.py create mode 100644 samples/appengine/flexible/tasks/snippets_test.py diff --git a/samples/appengine/flexible/tasks/snippets.py b/samples/appengine/flexible/tasks/snippets.py new file mode 100644 index 00000000..0c96f96d --- /dev/null +++ b/samples/appengine/flexible/tasks/snippets.py @@ -0,0 +1,285 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import tasks + + +def create_queue(project, location, queue_blue_name, queue_red_name): + # [START taskqueues_using_yaml] + client = tasks.CloudTasksClient() + + # TODO(developer): Uncomment these lines and replace with your values. + # project = 'my-project-id' + # location = 'us- central1' + # queue_blue_name = 'queue-blue' + # queue_red_name = 'queue-red' + + parent = client.location_path(project, location) + + queue_blue = { + 'name': client.queue_path(project, location, queue_blue_name), + 'rate_limits': { + 'max_dispatches_per_second': 5 + }, + 'app_engine_routing_override': { + 'version': 'v2', + 'service': 'task-module' + } + } + + queue_red = { + 'name': client.queue_path(project, location, queue_red_name), + 'rate_limits': { + 'max_dispatches_per_second': 1 + } + } + + queues = [queue_blue, queue_red] + for queue in queues: + response = client.create_queue(parent, queue) + print(response) + # [END taskqueues_using_yaml] + return response + + +def update_queue(project, location, queue): + # [START taskqueues_processing_rate] + client = tasks.CloudTasksClient() + + # TODO(developer): Uncomment these lines and replace with your values. + # project = 'my-project-id' + # location = 'us- central1' + # queue = 'queue-blue' + + # Get queue object + queue_path = client.queue_path(project, location, queue) + queue = client.get_queue(queue_path) + + # Update queue object + queue.rate_limits.max_dispatches_per_second = 20 + queue.rate_limits.max_concurrent_dispatches = 10 + + response = client.update_queue(queue) + print(response) + # [END taskqueues_processing_rate] + return response + + +def create_task(project, location, queue): + # [START taskqueues_new_task] + client = tasks.CloudTasksClient() + + # TODO(developer): Uncomment these lines and replace with your values. + # project = 'my-project-id' + # location = 'us- central1' + # queue = 'default' + amount = 10 + + parent = client.queue_path(project, location, queue) + + task = { + 'app_engine_http_request': { + 'http_method': 'POST', + 'relative_uri': '/update_counter', + 'app_engine_routing': { + 'service': 'worker' + }, + 'body': str(amount).encode() + } + } + + response = client.create_task(parent, task) + eta = response.schedule_time.ToDatetime().strftime("%m/%d/%Y, %H:%M:%S") + print('Task {} enqueued, ETA {}.'.format(response.name, eta)) + # [END taskqueues_new_task] + return response + + +def create_tasks_with_data(project, location, queue): + # [START taskqueues_passing_data] + import json + client = tasks.CloudTasksClient() + + # TODO(developer): Uncomment these lines and replace with your values. + # project = 'my-project-id' + # location = 'us- central1' + # queue = 'default' + + parent = client.queue_path(project, location, queue) + + task1 = { + 'app_engine_http_request': { + 'http_method': 'POST', + 'relative_uri': '/update_counter?key=blue', + 'app_engine_routing': { + 'service': 'worker' + } + } + } + + task2 = { + 'app_engine_http_request': { + 'http_method': 'POST', + 'relative_uri': '/update_counter', + 'app_engine_routing': { + 'service': 'worker' + }, + 'headers': { + 'Content-Type': 'application/json' + }, + 'body': json.dumps({'key': 'blue'}).encode() + } + } + + response = client.create_task(parent, task1) + print(response) + response = client.create_task(parent, task2) + print(response) + # [END taskqueues_passing_data] + return response + + +def create_task_with_name(project, location, queue, task_name): + # [START taskqueues_naming_tasks] + client = tasks.CloudTasksClient() + + # TODO(developer): Uncomment these lines and replace with your values. + # project = 'my-project-id' + # location = 'us- central1' + # queue = 'default' + # task_name = 'first-try' + + parent = client.queue_path(project, location, queue) + + task = { + 'name': client.task_path(project, location, queue, task_name), + 'app_engine_http_request': { + 'http_method': 'GET', + 'relative_uri': '/url/path' + } + } + response = client.create_task(parent, task) + print(response) + # [END taskqueues_naming_tasks] + return response + + +def delete_task(project, location, queue): + # [START taskqueues_setup] + client = tasks.CloudTasksClient() + + # TODO(developer): Uncomment these lines and replace with your values. + # project = 'my-project-id' + # location = 'us- central1' + # queue = 'queue1' + # [START taskqueues_setup] + + # [START taskqueues_deleting_tasks] + task_path = client.task_path(project, location, queue, 'foo') + response = client.delete_task(task_path) + # [END taskqueues_deleting_tasks] + + # [START taskqueues_purging_tasks] + queue_path = client.queue_path(project, location, queue) + response = client.purge_queue(queue_path) + # [END taskqueues_purging_tasks] + + # [START taskqueues_pause_queue] + queue_path = client.queue_path(project, location, queue) + response = client.pause_queue(queue_path) + # [END taskqueues_pause_queues] + return response + + +def delete_queue(project, location, queue): + client = tasks.CloudTasksClient() + + # TODO(developer): Uncomment these lines and replace with your values. + # project = 'my-project-id' + # location = 'us- central1' + # queue = 'queue1' + + # [START taskqueues_deleting_queues] + queue_path = client.queue_path(project, location, queue) + response = client.delete_queue(queue_path) + # [END taskqueues_deleting_queues] + return response + + +def retry_task(project, location, fooqueue, barqueue, bazqueue): + # [START taskqueues_retrying_tasks] + from google.protobuf import duration_pb2 + + client = tasks.CloudTasksClient() + + # TODO(developer): Uncomment these lines and replace with your values. + # project = 'my-project-id' + # location = 'us- central1' + # fooqueue = 'fooqueue' + # barqueue = 'barqueue' + # bazqueue = 'bazqueue' + + parent = client.location_path(project, location) + + max_retry = duration_pb2.Duration() + max_retry.seconds = 2*60*60*24 + + foo = { + 'name': client.queue_path(project, location, fooqueue), + 'rate_limits': { + 'max_dispatches_per_second': 1 + }, + 'retry_config': { + 'max_attempts': 7, + 'max_retry_duration': max_retry + } + } + + min = duration_pb2.Duration() + min.seconds = 10 + + max = duration_pb2.Duration() + max.seconds = 200 + + bar = { + 'name': client.queue_path(project, location, barqueue), + 'rate_limits': { + 'max_dispatches_per_second': 1 + }, + 'retry_config': { + 'min_backoff': min, + 'max_backoff': max, + 'max_doublings': 0 + } + } + + max.seconds = 300 + baz = { + 'name': client.queue_path(project, location, bazqueue), + 'rate_limits': { + 'max_dispatches_per_second': 1 + }, + 'retry_config': { + 'min_backoff': min, + 'max_backoff': max, + 'max_doublings': 3 + } + } + + queues = [foo, bar, baz] + for queue in queues: + response = client.create_queue(parent, queue) + print(response) + # [END taskqueues_retrying_tasks] + return response diff --git a/samples/appengine/flexible/tasks/snippets_test.py b/samples/appengine/flexible/tasks/snippets_test.py new file mode 100644 index 00000000..8923c670 --- /dev/null +++ b/samples/appengine/flexible/tasks/snippets_test.py @@ -0,0 +1,110 @@ +# Copyright 2019 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest +import uuid + +import snippets + +TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') +TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') +QUEUE_NAME_1 = "queue-{}".format(uuid.uuid4()) +QUEUE_NAME_2 = "queue-{}".format(uuid.uuid4()) + + +@pytest.mark.order1 +def test_create_queue(): + name = "projects/{}/locations/{}/queues/{}".format( + TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_2) + result = snippets.create_queue( + TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1, QUEUE_NAME_2) + assert name in result.name + + +@pytest.mark.order2 +def test_update_queue(): + name = "projects/{}/locations/{}/queues/{}".format( + TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1) + result = snippets.update_queue( + TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1) + assert name in result.name + + +@pytest.mark.order3 +def test_create_task(): + name = "projects/{}/locations/{}/queues/{}".format( + TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1) + result = snippets.create_task( + TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1) + assert name in result.name + + +@pytest.mark.order4 +def test_create_task_with_data(): + name = "projects/{}/locations/{}/queues/{}".format( + TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1) + result = snippets.create_tasks_with_data( + TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1) + assert name in result.name + + +@pytest.mark.order5 +def test_create_task_with_name(): + name = "projects/{}/locations/{}/queues/{}".format( + TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1) + result = snippets.create_task_with_name( + TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1, 'foo') + assert name in result.name + + +@pytest.mark.order6 +def test_delete_task(): + name = "projects/{}/locations/{}/queues/{}".format( + TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1) + result = snippets.delete_task( + TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1) + assert name in result.name + + +@pytest.mark.order7 +def test_delete_queue(): + result = snippets.delete_queue( + TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1) + assert result is None + + result = snippets.delete_queue( + TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_2) + assert result is None + + +@pytest.mark.order8 +def test_retry_task(): + QUEUE_SIZE = 3 + QUEUE_NAME = [] + for i in range(QUEUE_SIZE): + QUEUE_NAME.append("queue-{}".format(uuid.uuid4())) + + name = "projects/{}/locations/{}/queues/{}".format( + TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME[2]) + result = snippets.retry_task( + TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME[0], QUEUE_NAME[1], + QUEUE_NAME[2]) + assert name in result.name + + for i in range(QUEUE_SIZE): + snippets.delete_queue( + project=TEST_PROJECT_ID, + location=TEST_LOCATION, + queue=QUEUE_NAME[i])