From 87c721f345550ee86989d0bea1c54d081d4ee571 Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Fri, 11 Aug 2017 15:22:21 -0700 Subject: [PATCH 1/7] Add samples for Cloud Tasks --- appengine/flexible/tasks/README.md | 21 +++ .../flexible/tasks/appengine_queues/README.md | 133 ++++++++++++++++ .../flexible/tasks/appengine_queues/app.yaml | 6 + .../app_engine_queue_snippets.py | 144 ++++++++++++++++++ .../app_engine_queue_snippets_test.py | 46 ++++++ .../flexible/tasks/appengine_queues/main.py | 96 ++++++++++++ .../tasks/appengine_queues/main_test.py | 43 ++++++ .../tasks/appengine_queues/queue.yaml | 4 + .../appengine_queues/requirements-test.txt | 2 + .../tasks/appengine_queues/requirements.txt | 4 + .../flexible/tasks/pull_queues/README.md | 94 ++++++++++++ .../tasks/pull_queues/pull_queue_snippets.py | 140 +++++++++++++++++ .../pull_queues/pull_queue_snippets_test.py | 45 ++++++ .../flexible/tasks/pull_queues/queue.yaml | 3 + .../tasks/pull_queues/requirements.txt | 1 + 15 files changed, 782 insertions(+) create mode 100644 appengine/flexible/tasks/README.md create mode 100644 appengine/flexible/tasks/appengine_queues/README.md create mode 100644 appengine/flexible/tasks/appengine_queues/app.yaml create mode 100644 appengine/flexible/tasks/appengine_queues/app_engine_queue_snippets.py create mode 100644 appengine/flexible/tasks/appengine_queues/app_engine_queue_snippets_test.py create mode 100644 appengine/flexible/tasks/appengine_queues/main.py create mode 100644 appengine/flexible/tasks/appengine_queues/main_test.py create mode 100644 appengine/flexible/tasks/appengine_queues/queue.yaml create mode 100644 appengine/flexible/tasks/appengine_queues/requirements-test.txt create mode 100644 appengine/flexible/tasks/appengine_queues/requirements.txt create mode 100644 appengine/flexible/tasks/pull_queues/README.md create mode 100644 appengine/flexible/tasks/pull_queues/pull_queue_snippets.py create mode 100644 appengine/flexible/tasks/pull_queues/pull_queue_snippets_test.py create mode 100644 appengine/flexible/tasks/pull_queues/queue.yaml create mode 100644 appengine/flexible/tasks/pull_queues/requirements.txt diff --git a/appengine/flexible/tasks/README.md b/appengine/flexible/tasks/README.md new file mode 100644 index 000000000000..f08d7ea88166 --- /dev/null +++ b/appengine/flexible/tasks/README.md @@ -0,0 +1,21 @@ +# Google Cloud Tasks Samples + +Sample program for interacting with the Google Cloud Tasks API. + +The `pull_queues` directory contains command line samples for listing queues, creating +tasks, pulling tasks, and acknowledging them. + +The `appengine_queues` directory contains an App Engine app and command line +samples for creating tasks to be pushed to the App Engine app. + +See the respective READMEs for detailed instructions. + +## Contributing changes + +Contributions are not accepted during Alpha. + +## Licensing + +* See [LICENSE](LICENSE) + + diff --git a/appengine/flexible/tasks/appengine_queues/README.md b/appengine/flexible/tasks/appengine_queues/README.md new file mode 100644 index 000000000000..3c1d445e147d --- /dev/null +++ b/appengine/flexible/tasks/appengine_queues/README.md @@ -0,0 +1,133 @@ +# Google Cloud Tasks App Engine Queue Samples + +Sample command-line program for interacting with the Cloud Tasks API +using App Engine queues. + +App Engine queues push tasks to an App Engine HTTP target. This directory +contains both the App Engine app to deploy, as well as the snippets to run +locally to push tasks to it, which could also be called on App Engine. + +`app_engine_queue_snippets.py` is a simple command-line program to create tasks +to be pushed to the App Engine app. + +`main.py` is the main App Engine app. This app serves as an endpoint to receive +App Engine task attempts. + +`app.yaml` configures the App Engine app. + + +## Prerequisites to run locally: + +The samples require a Python environment with +[pip](https://pypi.python.org/pypi/pip) installed. +[virtualenv](https://virtualenv.readthedocs.org/en/latest/) is also recommended. + +All samples require a Google Cloud Project whitelisted for the Cloud Tasks API. + +* Enable the Cloud Tasks API +* Enable the Cloud Datastore API (used to demonstrate storing payloads) + +To create a project and enable the API, go to the [Google Developers +Console](https://console.developer.google.com). You must also create an API key. +This can be done under API Manager -> Credentials. + +To install the Python application dependencies, run the following commands: + + * pip install -r requirements.txt + +## Authentication + +To set up authentication locally, download the +[Cloud SDK](https://cloud.google.com/sdk), and run + + gcloud auth application-default login + +On App Engine, authentication credentials will be automatically detected. + +On Compute Engine and Container Engine, authenticatino credentials will be +automatically detected, but the instances must have been created with the +necessary scopes. + +In any other environment, for example Compute Engine instance without the +necessary scopes, you should set `GOOGLE_APPLICATION_CREDENTIALS` environment +variable to a JSON key file for a service account. + +See the [authentication guide](https://cloud.google.com/docs/authentication) +for more information. + +## Creating a queue + +Queues can not currently be created by the API. To create a queue using the +Cloud SDK, use the provided queue.yaml: + + gcloud app deploy queue.yaml + +## Deploying the App Engine app + +First, vendor the dependencies into the project: + + pip install -r requirements.txt + +Next, deploy the App Engine app + + gcloud app deploy + +Verify the index page is serving: + + gcloud app browse + +The App Engine app serves as a target for the push requests. It has an +endpoint `/set_payload` that that stores the payload from the HTTP POST data in +Cloud Datastore. The payload can be accessed in your browser at the + `/get_payload` endpoint with a GET request. + +## Running the Samples + +The project ID must be specified either as a command line argument using +`--project-id`, or by editing `DEFAULT_PROJECT_ID` within `task_snippets.py`. + +Set the environment variables: + + export PROJECT_ID=my-project-id + export LOCATION_ID=us-central1 + export QUEUE_ID=my-appengine-queue # From queue.yaml + +View all queues: + + python app_engine_queue_snippets.py --api_key=$API_KEY list-queues --project_id=$PROJECT_ID --location_id=$LOCATION_ID + +Set the queue name as an environment variable: + + export QUEUE_NAME=projects/$PROJECT_ID/locations/$LOCATION_ID/queues/$QUEUE_ID + +Create a task, targeted at the `set_payload` endpoint with a payload specified: + + python app_engine_queue_snippets.py --api_key=$API_KEY create-task --queue_name=$QUEUE_NAME --payload=hello + +Now view that the payload was received and verify the count and payload: + + http://your-app-id.appspot.com/get_payload + +Create a task that will be scheduled for a few seconds in the future using +the `--in_seconds` flag: + + python app_engine_queue_snippets.py --api_key=$API_KEY create-task --queue_name=$QUEUE_NAME --payload=hello --in_seconds=30 + +Since `--in_seconds` was set to 30, it will take 30 seconds for the new +payload to be pushed to the `/get_payload` endpoint, which can then be viewed at: + + http://your-app-id.appspot.com/get_payload + +It might also be helpful to view the request logs of your App Engine app: + + https://console.cloud.google.com/logs + +## Testing the Samples + +Install the testing dependencies: + + pip install -r requirements-test.txt + +Run pytest: + + pytest diff --git a/appengine/flexible/tasks/appengine_queues/app.yaml b/appengine/flexible/tasks/appengine_queues/app.yaml new file mode 100644 index 000000000000..e5ac514e8b6e --- /dev/null +++ b/appengine/flexible/tasks/appengine_queues/app.yaml @@ -0,0 +1,6 @@ +runtime: python +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 diff --git a/appengine/flexible/tasks/appengine_queues/app_engine_queue_snippets.py b/appengine/flexible/tasks/appengine_queues/app_engine_queue_snippets.py new file mode 100644 index 000000000000..09e3b52949a9 --- /dev/null +++ b/appengine/flexible/tasks/appengine_queues/app_engine_queue_snippets.py @@ -0,0 +1,144 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import base64 +import datetime + +from googleapiclient import discovery + + +def format_rfc3339(datetime_instance): + """Format a datetime per RFC 3339.""" + return datetime_instance.isoformat("T") + "Z" + + +def get_seconds_from_now_rfc3339(seconds): + """Return seconds from the current time as a RFC 3339 string.""" + d = datetime.datetime.utcnow() + datetime.timedelta(seconds=seconds) + return format_rfc3339(d) + + +def list_queues(api_key, project_id, location_id): + """List the queues in the location.""" + client = get_client(api_key) + parent = 'projects/{}/locations/{}'.format(project_id, location_id) + queues = [] + next_page_token = None + + while True: + response = client.projects().locations().queues().list( + parent=parent, pageToken=next_page_token).execute() + queues += response['queues'] + if next_page_token is None: + break + + print('Listing queues for location {}'.format(location_id)) + + for queue in response['queues']: + print queue['name'] + return response + + +def create_task(api_key, queue_name, payload=None, in_seconds=None): + """Create a task for a given queue with an arbitrary payload.""" + client = get_client(api_key) + + url = '/set_payload' + task = { + 'task': { + 'app_engine_task_target': { + 'http_method': 'POST', + 'relative_url': url + } + } + } + + if payload is not None: + task['task']['app_engine_task_target']['payload'] = base64.b64encode( + payload) + + if in_seconds is not None: + scheduled_time = get_seconds_from_now_rfc3339(int(in_seconds)) + task['task']['schedule_time'] = scheduled_time + + print('Sending task {}'.format(task)) + + response = client.projects().locations().queues().tasks().create( + parent=queue_name, body=task).execute() + + # By default CreateTaskRequest.responseView is BASIC, so not all + # information is retrieved by default because some data, such as payloads, + # might be desirable to return only when needed because of its large size + # or because of the sensitivity of data that it contains. + print('Created task {}'.format(response['name'])) + return response + + +def get_client(api_key): + """Build an authenticated http client.""" + DISCOVERY_URL = 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2&key={}'.format( + api_key) + client = discovery.build('cloudtasks', 'v2beta2', + discoveryServiceUrl=DISCOVERY_URL) + return client + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + parser.add_argument('--api_key', help='API Key', required=True) + + subparsers = parser.add_subparsers(dest='command') + + list_queues_parser = subparsers.add_parser( + 'list-queues', + help=list_queues.__doc__) + + list_queues_parser.add_argument( + '--project_id', + help='Project ID you want to access.', + required=True) + + list_queues_parser.add_argument( + '--location_id', + help='Location of the queues.', + required=True) + + create_task_parser = subparsers.add_parser( + 'create-task', + help=create_task.__doc__) + create_task_parser.add_argument( + '--queue_name', + help='Fully qualified name of the queue to add the task to.' + ) + + create_task_parser.add_argument( + '--payload', + help='Optional payload to attach to the push queue.' + ) + + create_task_parser.add_argument( + '--in_seconds', + help='The number of seconds from now to schedule task attempt.' + ) + + args = parser.parse_args() + + if args.command == 'list-queues': + list_queues(args.api_key, args.project_id, args.location_id) + if args.command == 'create-task': + create_task(args.api_key, args.queue_name, args.payload, args.in_seconds) diff --git a/appengine/flexible/tasks/appengine_queues/app_engine_queue_snippets_test.py b/appengine/flexible/tasks/appengine_queues/app_engine_queue_snippets_test.py new file mode 100644 index 000000000000..a48973f7a1f6 --- /dev/null +++ b/appengine/flexible/tasks/appengine_queues/app_engine_queue_snippets_test.py @@ -0,0 +1,46 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +import app_engine_queue_snippets + +TEST_PROJECT_ID = 'mock-project' +TEST_LOCATION = 'us-central1' +API_KEY = 'mock-api-key' +TEST_QUEUE_NAME = 'projects/{}/locations/{}/queues/{}'.format( + TEST_PROJECT_ID, TEST_LOCATION, 'my-push-queue') + + +@mock.patch('app_engine_queue_snippets.get_client') +def test_list_queues(get_client): + locations = get_client.return_value.projects.return_value.locations + queues = locations.return_value.queues + execute_function = queues.return_value.list.return_value.execute + execute_function.return_value = {'name': 'task_name', 'queues': []} + app_engine_queue_snippets.list_queues(API_KEY, TEST_PROJECT_ID, + TEST_LOCATION) + assert execute_function.called + + +@mock.patch('app_engine_queue_snippets.get_client') +def test_create_task(get_client): + projects = get_client.return_value.projects.return_value + locations = projects.locations.return_value + create_function = locations.queues.return_value.tasks.return_value.create + execute_function = create_function.return_value.execute + execute_function.return_value = {'name': 'task_name'} + app_engine_queue_snippets.create_task(API_KEY, TEST_QUEUE_NAME) + assert execute_function.called + diff --git a/appengine/flexible/tasks/appengine_queues/main.py b/appengine/flexible/tasks/appengine_queues/main.py new file mode 100644 index 000000000000..4a87de7b4c4f --- /dev/null +++ b/appengine/flexible/tasks/appengine_queues/main.py @@ -0,0 +1,96 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""App Engine app to serve as an endpoint for App Engine queue samples.""" + +from google.cloud import datastore +from flask import Flask, request + +app = Flask(__name__) +client = datastore.Client() + + +def get_payload_from_datastore(): + payload_key = client.key('Payload', 1) + payload_entity = client.get(payload_key) + if payload_entity is None: + return None + return payload_entity['value'] + + +def get_counter_from_datastore(): + counter_key = client.key('Counter', 1) + counter_entity = client.get(counter_key) + if counter_entity is None: + return 0 + return counter_entity['counter'] + + +def update_payload(request_data): + """Sets the payload value entity in Cloud Datastore.""" + payload_key = client.key('Payload', 1) + payload_entity = datastore.Entity(payload_key) + + if request_data: + payload_entity['value'] = request_data + client.put(payload_entity) + + payload_entity = client.get(payload_key) + return payload_entity['value'] + + +def increment_counter(): + """Increments a counter value in Cloud Datastore.""" + counter_key = client.key('Counter', 1) + counter_entity = client.get(counter_key) + if counter_entity is None: + counter_entity = datastore.Entity(counter_key) + counter_entity['counter'] = 0 + client.put(counter_entity) + + counter_entity['counter'] += 1 + client.put(counter_entity) + return counter_entity['counter'] + + +@app.route('/set_payload', methods=['POST']) +def set_payload(): + """Main endpoint to demonstrate Cloud Tasks App Engine queue features.""" + counter = increment_counter() + payload = update_payload(request.data) + + return 'Counter value is now {}, payload value is now {}'.format( + counter, payload) + + +@app.route('/get_payload') +def get_payload(): + """Main endpoint to demonstrate Cloud Tasks App Engine queue features.""" + counter = get_counter_from_datastore() + payload = get_payload_from_datastore() + + return 'Counter value is now {}, payload value is now {}'.format( + counter, payload) + + +@app.route('/') +def hello(): + """Basic index to verify app is serving.""" + return 'Hello World!' + + +if __name__ == '__main__': + # This is used when running locally. Gunicorn is used to run the + # application on Google App Engine. See entrypoint in app.yaml. + app.run(host='127.0.0.1', port=8080, debug=True) diff --git a/appengine/flexible/tasks/appengine_queues/main_test.py b/appengine/flexible/tasks/appengine_queues/main_test.py new file mode 100644 index 000000000000..2b07228304ac --- /dev/null +++ b/appengine/flexible/tasks/appengine_queues/main_test.py @@ -0,0 +1,43 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import pytest + + +@pytest.fixture +def app(): + import main + main.app.testing = True + return main.app.test_client() + + +def test_index(app): + r = app.get('/') + assert r.status_code == 200 + + +@mock.patch('main.update_payload') +@mock.patch('main.increment_counter') +def test_set_payload(increment_counter, update_payload, app): + payload = 'hello' + + r = app.post('/set_payload', payload) + assert r.status_code == 200 + + assert increment_counter.called + assert update_payload.called + + r = app.get('/get_payload') + assert r.status_code == 200 diff --git a/appengine/flexible/tasks/appengine_queues/queue.yaml b/appengine/flexible/tasks/appengine_queues/queue.yaml new file mode 100644 index 000000000000..96c6d0f1e3f7 --- /dev/null +++ b/appengine/flexible/tasks/appengine_queues/queue.yaml @@ -0,0 +1,4 @@ +queue: +- name: my-appengine-queue + target: default + rate: 1/s diff --git a/appengine/flexible/tasks/appengine_queues/requirements-test.txt b/appengine/flexible/tasks/appengine_queues/requirements-test.txt new file mode 100644 index 000000000000..63daca40d019 --- /dev/null +++ b/appengine/flexible/tasks/appengine_queues/requirements-test.txt @@ -0,0 +1,2 @@ +pytest=3.0.3 +mock=2.0.0 diff --git a/appengine/flexible/tasks/appengine_queues/requirements.txt b/appengine/flexible/tasks/appengine_queues/requirements.txt new file mode 100644 index 000000000000..046a5b5a3d03 --- /dev/null +++ b/appengine/flexible/tasks/appengine_queues/requirements.txt @@ -0,0 +1,4 @@ +Flask==0.11.1 +google-api-python-client==1.6.0 +google-cloud-datastore==0.22.0 +gunicorn==19.6.0 diff --git a/appengine/flexible/tasks/pull_queues/README.md b/appengine/flexible/tasks/pull_queues/README.md new file mode 100644 index 000000000000..fcac4b30f7d6 --- /dev/null +++ b/appengine/flexible/tasks/pull_queues/README.md @@ -0,0 +1,94 @@ +# Google Cloud Tasks Pull Queue Samples + +Sample command-line program for interacting with the Google Cloud Tasks API +using pull queues. + +Pull queues let you add tasks to a queue, then programatically remove and +interact with them. Tasks can be added or processed in any environment, +such as on Google App Engine or Google Compute Engine. + +`pull_queue_snippets.py` is a simple command-line program to demonstrate listing queues, + creating tasks, and pulling and acknowledging tasks. + +## Prerequisites to run locally: + +The samples require a Python environment with [pip](https://pypi.python.org/pypi/pip) installed. +[virtualenv](https://virtualenv.readthedocs.org/en/latest/) is also recommended. + +All samples require a Google Cloud Project whitelisted for the Cloud Tasks API. The Cloud Tasks +API must also be enabled. To create a project and enable the API, go to the [Google Developers +Console](https://console.developer.google.com). You must also create an API key. This can be +done under API Manager -> Credentials. + +To install the Python application dependencies, run the following commands: + + * pip install -r requirements.txt + + +## Authentication + +To set up authentication locally, download the [Cloud SDK](https://cloud.google.com/sdk), and run + + gcloud auth application-default login + +On App Engine, authentication credentials will be automatically detected. + +On Compute Engine and Container Engine, authentication credentials will be +automatically detected, but the instances must have been created with the +necessary scopes. + +In any other environment, for example Compute Engine instance without the +necessary scopes, you should set `GOOGLE_APPLICATION_CREDENTIALS` environment +variable to a JSON key file for a service account. + +See the [authentication guide](https://cloud.google.com/docs/authentication) +for more information. + +## Creating a queue + +Queues can not currently be created by the API. To create the queue using the Cloud SDK, use +the provided queue.yaml: + + gcloud app deploy queue.yaml + +## Running the Samples + +The project ID must be specified either as a command line argument using `--project-id`, or by +editing `DEFAULT_PROJECT_ID` within `task_snippets.py`. + +Set the environment variables: + + export API_KEY=your-api-key + export PROJECT_ID=my-project-id + export LOCATION_ID=us-central1 + export QUEUE_ID=my-pull-queue # From queue.yaml + export QUEUE_NAME=projects/$PROJECT_ID/locations/$LOCATION_ID/queues/$QUEUE_ID + +View all queues: + + python pull_queue_snippets.py --api_key=$API_KEY list-queues --project_id=$PROJECT_ID --location_id=$LOCATION_ID + +Create a task for a queue: + + python pull_queue_snippets.py --api_key=$API_KEY create-task --queue_name=$QUEUE_NAME + +Pull and acknowledge a task: + + python pull_queue_snippets.py --api_key=$API_KEY pull-and-ack-task --queue_name=$QUEUE_NAME + +Note that usually, there would be a processing step in between pulling a task and acknowledging it. + +## Testing the Samples + +Install pytest: + + pip install pytest + +Set the `GOOGLE_CLOUD_PROJECT` and `API_KEY` environment variable to your project ID. + + export GOOGLE_CLOUD_PROJECT=my-project-id + export API_KEY=my-api-key + +Run pytest: + + pytest diff --git a/appengine/flexible/tasks/pull_queues/pull_queue_snippets.py b/appengine/flexible/tasks/pull_queues/pull_queue_snippets.py new file mode 100644 index 000000000000..c76843afe907 --- /dev/null +++ b/appengine/flexible/tasks/pull_queues/pull_queue_snippets.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sample command-line program for interacting with the Cloud Tasks API. + +Please note that Cloud Tasks is currently in alpha. + +See README.md for instructions on setting up your development environment +and running the scripts. +""" + +import argparse +import base64 + +from googleapiclient import discovery + + +def list_queues(api_key, project_id, location_id): + """List the queues in the location.""" + client = get_client(api_key) + parent = 'projects/{}/locations/{}'.format(project_id, location_id) + queues = [] + next_page_token = None + + while True: + response = client.projects().locations( + ).queues().list(parent=parent,pageToken=next_page_token).execute() + queues += response['queues'] + if next_page_token is None: + break + + print('Listing queues for location {}'.format(location_id)) + + for queue in response['queues']: + print queue['name'] + return response + + +def create_task(api_key, queue_name): + """Create a task for a given queue with an arbitrary payload.""" + client = get_client(api_key) + payload = 'a message for the recipient' + task = { + 'task': { + 'pull_task_target': { + 'payload': base64.b64encode(payload) + } + } + } + response = client.projects().locations().queues().tasks().create( + parent=queue_name, body=task).execute() + print('Created task {}'.format(response['name'])) + return response + + +def pull_task(api_key, queue_name): + """Pull a single task from a given queue and lease it for 10 minutes.""" + client = get_client(api_key) + duration_seconds = '600s' + pull_options = { + 'max_tasks': 1, + 'leaseDuration': duration_seconds, + 'responseView': 'FULL' + } + response = client.projects().locations().queues().tasks().pull( + name=queue_name, body=pull_options).execute() + print('Pulled task {}'.format(response)) + return response['tasks'][0] + + +def acknowledge_task(api_key, task): + """Acknowledge a given task.""" + client = get_client(api_key) + body = {'scheduleTime': task['scheduleTime']} + client.projects().locations().queues().tasks().acknowledge( + name=task['name'], body=body).execute() + print('Acknowledged task {}'.format(task['name'])) + + +def get_client(api_key): + """Build an authenticated http client.""" + DISCOVERY_URL = 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2&key={}'.format( + api_key) + client = discovery.build('cloudtasks', 'v2beta2', + discoveryServiceUrl=DISCOVERY_URL) + return client + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + subparsers = parser.add_subparsers(dest='command') + parser.add_argument('--api_key', help='API Key', required=True) + + list_queues_parser = subparsers.add_parser( + 'list-queues', + help=list_queues.__doc__) + + list_queues_parser.add_argument( + '--project_id', + help='Project ID you want to access.', + required=True) + list_queues_parser.add_argument('--location_id', + help='Location of the queues.', + required=True) + + create_task_parser = subparsers.add_parser('create-task', + help=create_task.__doc__) + create_task_parser.add_argument( + '--queue_name', + help='Fully qualified name of the queue to add the task to.') + + pull_and_ack_parser = subparsers.add_parser('pull-and-ack-task', + help=create_task.__doc__) + pull_and_ack_parser.add_argument('--queue_name', + help='Fully qualified name of the queue to add the task to.') + + args = parser.parse_args() + + if args.command == 'list-queues': + list_queues(args.api_key, args.project_id, args.location_id) + if args.command == 'create-task': + create_task(args.api_key, args.queue_name) + if args.command == 'pull-and-ack-task': + task = pull_task(args.api_key, args.queue_name) + acknowledge_task(args.api_key, task) diff --git a/appengine/flexible/tasks/pull_queues/pull_queue_snippets_test.py b/appengine/flexible/tasks/pull_queues/pull_queue_snippets_test.py new file mode 100644 index 000000000000..8800ab12fb55 --- /dev/null +++ b/appengine/flexible/tasks/pull_queues/pull_queue_snippets_test.py @@ -0,0 +1,45 @@ +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pull_queue_snippets + +TEST_PROJECT_ID = os.getenv('GOOGLE_CLOUD_PROJECT') +TEST_LOCATION = 'us-central1' +API_KEY = os.getenv('API_KEY') + + +def test_list_queues(): + result = pull_queue_snippets.list_queues( + API_KEY, TEST_PROJECT_ID, TEST_LOCATION) + assert len(result['queues']) > 0 + + +def test_create_task(): + result = pull_queue_snippets.list_queues( + API_KEY, TEST_PROJECT_ID, TEST_LOCATION) + queue_name = result['queues'][0]['name'] + result = pull_queue_snippets.create_task(API_KEY, queue_name) + assert queue_name in result['name'] + + +def test_pull_and_ack_task(): + result = pull_queue_snippets.list_queues( + API_KEY, TEST_PROJECT_ID, TEST_LOCATION) + queue_name = result['queues'][0]['name'] + pull_queue_snippets.create_task(API_KEY, queue_name) + task = pull_queue_snippets.pull_task(API_KEY, queue_name) + pull_queue_snippets.acknowledge_task(API_KEY, task) + diff --git a/appengine/flexible/tasks/pull_queues/queue.yaml b/appengine/flexible/tasks/pull_queues/queue.yaml new file mode 100644 index 000000000000..08c1503b4953 --- /dev/null +++ b/appengine/flexible/tasks/pull_queues/queue.yaml @@ -0,0 +1,3 @@ +queue: +- name: my-pull-queue + mode: pull diff --git a/appengine/flexible/tasks/pull_queues/requirements.txt b/appengine/flexible/tasks/pull_queues/requirements.txt new file mode 100644 index 000000000000..b7233cdf335b --- /dev/null +++ b/appengine/flexible/tasks/pull_queues/requirements.txt @@ -0,0 +1 @@ +google-api-python-client==1.6.0 From 507a54b9e42c5a09b7342788d07fb37a0fa7a69b Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Mon, 28 Aug 2017 17:08:45 -0700 Subject: [PATCH 2/7] Respond to tasks sample review --- appengine/flexible/tasks/README.md | 108 ++++++++++++-- .../tasks/{appengine_queues => }/app.yaml | 0 .../app_engine_queue_snippets.py | 19 +-- .../app_engine_queue_snippets_test.py | 0 .../flexible/tasks/appengine_queues/README.md | 133 ------------------ .../appengine_queues/requirements-test.txt | 2 - .../tasks/{appengine_queues => }/main.py | 11 +- .../tasks/{appengine_queues => }/main_test.py | 0 .../tasks/{appengine_queues => }/queue.yaml | 0 .../{appengine_queues => }/requirements.txt | 0 .../tasks/pull_queues => tasks}/README.md | 28 +--- .../pull_queue_snippets.py | 29 ++-- .../pull_queue_snippets_test.py | 0 .../tasks/pull_queues => tasks}/queue.yaml | 0 .../pull_queues => tasks}/requirements.txt | 0 15 files changed, 132 insertions(+), 198 deletions(-) rename appengine/flexible/tasks/{appengine_queues => }/app.yaml (100%) rename appengine/flexible/tasks/{appengine_queues => }/app_engine_queue_snippets.py (90%) rename appengine/flexible/tasks/{appengine_queues => }/app_engine_queue_snippets_test.py (100%) delete mode 100644 appengine/flexible/tasks/appengine_queues/README.md delete mode 100644 appengine/flexible/tasks/appengine_queues/requirements-test.txt rename appengine/flexible/tasks/{appengine_queues => }/main.py (91%) rename appengine/flexible/tasks/{appengine_queues => }/main_test.py (100%) rename appengine/flexible/tasks/{appengine_queues => }/queue.yaml (100%) rename appengine/flexible/tasks/{appengine_queues => }/requirements.txt (100%) rename {appengine/flexible/tasks/pull_queues => tasks}/README.md (72%) rename {appengine/flexible/tasks/pull_queues => tasks}/pull_queue_snippets.py (83%) rename {appengine/flexible/tasks/pull_queues => tasks}/pull_queue_snippets_test.py (100%) rename {appengine/flexible/tasks/pull_queues => tasks}/queue.yaml (100%) rename {appengine/flexible/tasks/pull_queues => tasks}/requirements.txt (100%) diff --git a/appengine/flexible/tasks/README.md b/appengine/flexible/tasks/README.md index f08d7ea88166..61d6387b3024 100644 --- a/appengine/flexible/tasks/README.md +++ b/appengine/flexible/tasks/README.md @@ -1,21 +1,107 @@ -# Google Cloud Tasks Samples +# Google Cloud Tasks App Engine Queue Samples -Sample program for interacting with the Google Cloud Tasks API. +Sample command-line program for interacting with the Cloud Tasks API +using App Engine queues. -The `pull_queues` directory contains command line samples for listing queues, creating -tasks, pulling tasks, and acknowledging them. +App Engine queues push tasks to an App Engine HTTP target. This directory +contains both the App Engine app to deploy, as well as the snippets to run +locally to push tasks to it, which could also be called on App Engine. -The `appengine_queues` directory contains an App Engine app and command line -samples for creating tasks to be pushed to the App Engine app. +`app_engine_queue_snippets.py` is a simple command-line program to create tasks +to be pushed to the App Engine app. -See the respective READMEs for detailed instructions. +`main.py` is the main App Engine app. This app serves as an endpoint to receive +App Engine task attempts. -## Contributing changes +`app.yaml` configures the App Engine app. -Contributions are not accepted during Alpha. -## Licensing +## Prerequisites to run locally: -* See [LICENSE](LICENSE) +Please refer to [Setting Up a Python Development Environment](https://cloud.google.com/python/setup). +## Authentication +To set up authentication locally, download the +[Cloud SDK](https://cloud.google.com/sdk), and run + + gcloud auth application-default login + +On App Engine, authentication credentials will be automatically detected. + +On Compute Engine and Container Engine, authentication credentials will be +automatically detected, but the instances must have been created with the +necessary scopes. + +In any other environment, for example Compute Engine instance without the +necessary scopes, you should set `GOOGLE_APPLICATION_CREDENTIALS` environment +variable to a JSON key file for a service account. + +See the [authentication guide](https://cloud.google.com/docs/authentication) +for more information. + +## Creating a queue + +To create a queue using the Cloud SDK, use the provided queue.yaml: + + gcloud app deploy queue.yaml + +## Deploying the App Engine app + +First, vendor the dependencies into the project: + + pip install -r requirements.txt + +Next, deploy the App Engine app + + gcloud app deploy + +Verify the index page is serving: + + gcloud app browse + +The App Engine app serves as a target for the push requests. It has an +endpoint `/set_payload` that that stores the payload from the HTTP POST data in +Cloud Datastore. The payload can be accessed in your browser at the + `/get_payload` endpoint with a GET request. + +## Running the Samples + +The project ID must be specified either as a command line argument using +`--project-id`, or by editing `DEFAULT_PROJECT_ID` within `task_snippets.py`. + +Set the environment variables: + + export PROJECT_ID=my-project-id + export LOCATION_ID=us-central1 + export QUEUE_ID=my-appengine-queue # From queue.yaml + +View all queues: + + python app_engine_queue_snippets.py --api_key=$API_KEY list-queues --project_id=$PROJECT_ID --location_id=$LOCATION_ID + +Set the queue name as an environment variable: + + export QUEUE_NAME=projects/$PROJECT_ID/locations/$LOCATION_ID/queues/$QUEUE_ID + +Create a task, targeted at the `set_payload` endpoint with a payload specified: + + python app_engine_queue_snippets.py --api_key=$API_KEY create-task --queue_name=$QUEUE_NAME --payload=hello + +Now view that the payload was received and verify the count and payload: + + http://your-app-id.appspot.com/get_payload + +Create a task that will be scheduled for a few seconds in the future using +the `--in_seconds` flag: + + python app_engine_queue_snippets.py --api_key=$API_KEY create-task --queue_name=$QUEUE_NAME --payload=hello --in_seconds=30 + +Since `--in_seconds` was set to 30, it will take 30 seconds for the new +payload to be pushed to the `/get_payload` endpoint, which can then be viewed at: + + http://your-app-id.appspot.com/get_payload + +It might also be helpful to view the request logs of your App Engine app: + + https://console.cloud.google.com/logs diff --git a/appengine/flexible/tasks/appengine_queues/app.yaml b/appengine/flexible/tasks/app.yaml similarity index 100% rename from appengine/flexible/tasks/appengine_queues/app.yaml rename to appengine/flexible/tasks/app.yaml diff --git a/appengine/flexible/tasks/appengine_queues/app_engine_queue_snippets.py b/appengine/flexible/tasks/app_engine_queue_snippets.py similarity index 90% rename from appengine/flexible/tasks/appengine_queues/app_engine_queue_snippets.py rename to appengine/flexible/tasks/app_engine_queue_snippets.py index 09e3b52949a9..23121af8f2ba 100644 --- a/appengine/flexible/tasks/appengine_queues/app_engine_queue_snippets.py +++ b/appengine/flexible/tasks/app_engine_queue_snippets.py @@ -12,16 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import argparse import base64 import datetime +import pprint from googleapiclient import discovery def format_rfc3339(datetime_instance): """Format a datetime per RFC 3339.""" - return datetime_instance.isoformat("T") + "Z" + return datetime_instance.isoformat('T') + 'Z' def get_seconds_from_now_rfc3339(seconds): @@ -47,7 +50,7 @@ def list_queues(api_key, project_id, location_id): print('Listing queues for location {}'.format(location_id)) for queue in response['queues']: - print queue['name'] + print(queue['name']) return response @@ -56,7 +59,7 @@ def create_task(api_key, queue_name, payload=None, in_seconds=None): client = get_client(api_key) url = '/set_payload' - task = { + body = { 'task': { 'app_engine_task_target': { 'http_method': 'POST', @@ -66,17 +69,17 @@ def create_task(api_key, queue_name, payload=None, in_seconds=None): } if payload is not None: - task['task']['app_engine_task_target']['payload'] = base64.b64encode( + body['task']['app_engine_task_target']['payload'] = base64.b64encode( payload) if in_seconds is not None: - scheduled_time = get_seconds_from_now_rfc3339(int(in_seconds)) - task['task']['schedule_time'] = scheduled_time + scheduled_time = get_seconds_from_now_rfc3339(in_seconds) + body['task']['schedule_time'] = scheduled_time - print('Sending task {}'.format(task)) + print('Sending task {}'.format(pprint.pformat(body))) response = client.projects().locations().queues().tasks().create( - parent=queue_name, body=task).execute() + parent=queue_name, body=body).execute() # By default CreateTaskRequest.responseView is BASIC, so not all # information is retrieved by default because some data, such as payloads, diff --git a/appengine/flexible/tasks/appengine_queues/app_engine_queue_snippets_test.py b/appengine/flexible/tasks/app_engine_queue_snippets_test.py similarity index 100% rename from appengine/flexible/tasks/appengine_queues/app_engine_queue_snippets_test.py rename to appengine/flexible/tasks/app_engine_queue_snippets_test.py diff --git a/appengine/flexible/tasks/appengine_queues/README.md b/appengine/flexible/tasks/appengine_queues/README.md deleted file mode 100644 index 3c1d445e147d..000000000000 --- a/appengine/flexible/tasks/appengine_queues/README.md +++ /dev/null @@ -1,133 +0,0 @@ -# Google Cloud Tasks App Engine Queue Samples - -Sample command-line program for interacting with the Cloud Tasks API -using App Engine queues. - -App Engine queues push tasks to an App Engine HTTP target. This directory -contains both the App Engine app to deploy, as well as the snippets to run -locally to push tasks to it, which could also be called on App Engine. - -`app_engine_queue_snippets.py` is a simple command-line program to create tasks -to be pushed to the App Engine app. - -`main.py` is the main App Engine app. This app serves as an endpoint to receive -App Engine task attempts. - -`app.yaml` configures the App Engine app. - - -## Prerequisites to run locally: - -The samples require a Python environment with -[pip](https://pypi.python.org/pypi/pip) installed. -[virtualenv](https://virtualenv.readthedocs.org/en/latest/) is also recommended. - -All samples require a Google Cloud Project whitelisted for the Cloud Tasks API. - -* Enable the Cloud Tasks API -* Enable the Cloud Datastore API (used to demonstrate storing payloads) - -To create a project and enable the API, go to the [Google Developers -Console](https://console.developer.google.com). You must also create an API key. -This can be done under API Manager -> Credentials. - -To install the Python application dependencies, run the following commands: - - * pip install -r requirements.txt - -## Authentication - -To set up authentication locally, download the -[Cloud SDK](https://cloud.google.com/sdk), and run - - gcloud auth application-default login - -On App Engine, authentication credentials will be automatically detected. - -On Compute Engine and Container Engine, authenticatino credentials will be -automatically detected, but the instances must have been created with the -necessary scopes. - -In any other environment, for example Compute Engine instance without the -necessary scopes, you should set `GOOGLE_APPLICATION_CREDENTIALS` environment -variable to a JSON key file for a service account. - -See the [authentication guide](https://cloud.google.com/docs/authentication) -for more information. - -## Creating a queue - -Queues can not currently be created by the API. To create a queue using the -Cloud SDK, use the provided queue.yaml: - - gcloud app deploy queue.yaml - -## Deploying the App Engine app - -First, vendor the dependencies into the project: - - pip install -r requirements.txt - -Next, deploy the App Engine app - - gcloud app deploy - -Verify the index page is serving: - - gcloud app browse - -The App Engine app serves as a target for the push requests. It has an -endpoint `/set_payload` that that stores the payload from the HTTP POST data in -Cloud Datastore. The payload can be accessed in your browser at the - `/get_payload` endpoint with a GET request. - -## Running the Samples - -The project ID must be specified either as a command line argument using -`--project-id`, or by editing `DEFAULT_PROJECT_ID` within `task_snippets.py`. - -Set the environment variables: - - export PROJECT_ID=my-project-id - export LOCATION_ID=us-central1 - export QUEUE_ID=my-appengine-queue # From queue.yaml - -View all queues: - - python app_engine_queue_snippets.py --api_key=$API_KEY list-queues --project_id=$PROJECT_ID --location_id=$LOCATION_ID - -Set the queue name as an environment variable: - - export QUEUE_NAME=projects/$PROJECT_ID/locations/$LOCATION_ID/queues/$QUEUE_ID - -Create a task, targeted at the `set_payload` endpoint with a payload specified: - - python app_engine_queue_snippets.py --api_key=$API_KEY create-task --queue_name=$QUEUE_NAME --payload=hello - -Now view that the payload was received and verify the count and payload: - - http://your-app-id.appspot.com/get_payload - -Create a task that will be scheduled for a few seconds in the future using -the `--in_seconds` flag: - - python app_engine_queue_snippets.py --api_key=$API_KEY create-task --queue_name=$QUEUE_NAME --payload=hello --in_seconds=30 - -Since `--in_seconds` was set to 30, it will take 30 seconds for the new -payload to be pushed to the `/get_payload` endpoint, which can then be viewed at: - - http://your-app-id.appspot.com/get_payload - -It might also be helpful to view the request logs of your App Engine app: - - https://console.cloud.google.com/logs - -## Testing the Samples - -Install the testing dependencies: - - pip install -r requirements-test.txt - -Run pytest: - - pytest diff --git a/appengine/flexible/tasks/appengine_queues/requirements-test.txt b/appengine/flexible/tasks/appengine_queues/requirements-test.txt deleted file mode 100644 index 63daca40d019..000000000000 --- a/appengine/flexible/tasks/appengine_queues/requirements-test.txt +++ /dev/null @@ -1,2 +0,0 @@ -pytest=3.0.3 -mock=2.0.0 diff --git a/appengine/flexible/tasks/appengine_queues/main.py b/appengine/flexible/tasks/main.py similarity index 91% rename from appengine/flexible/tasks/appengine_queues/main.py rename to appengine/flexible/tasks/main.py index 4a87de7b4c4f..53153299a85c 100644 --- a/appengine/flexible/tasks/appengine_queues/main.py +++ b/appengine/flexible/tasks/main.py @@ -20,9 +20,12 @@ app = Flask(__name__) client = datastore.Client() +PAYLOAD_KEY_NAME = 'Payload' +COUNTER_KEY_NAME = 'Counter' + def get_payload_from_datastore(): - payload_key = client.key('Payload', 1) + payload_key = client.key(PAYLOAD_KEY_NAME, 1) payload_entity = client.get(payload_key) if payload_entity is None: return None @@ -30,7 +33,7 @@ def get_payload_from_datastore(): def get_counter_from_datastore(): - counter_key = client.key('Counter', 1) + counter_key = client.key(COUNTER_KEY_NAME, 1) counter_entity = client.get(counter_key) if counter_entity is None: return 0 @@ -39,7 +42,7 @@ def get_counter_from_datastore(): def update_payload(request_data): """Sets the payload value entity in Cloud Datastore.""" - payload_key = client.key('Payload', 1) + payload_key = client.key(PAYLOAD_KEY_NAME, 1) payload_entity = datastore.Entity(payload_key) if request_data: @@ -52,7 +55,7 @@ def update_payload(request_data): def increment_counter(): """Increments a counter value in Cloud Datastore.""" - counter_key = client.key('Counter', 1) + counter_key = client.key(COUNTER_KEY_NAME, 1) counter_entity = client.get(counter_key) if counter_entity is None: counter_entity = datastore.Entity(counter_key) diff --git a/appengine/flexible/tasks/appengine_queues/main_test.py b/appengine/flexible/tasks/main_test.py similarity index 100% rename from appengine/flexible/tasks/appengine_queues/main_test.py rename to appengine/flexible/tasks/main_test.py diff --git a/appengine/flexible/tasks/appengine_queues/queue.yaml b/appengine/flexible/tasks/queue.yaml similarity index 100% rename from appengine/flexible/tasks/appengine_queues/queue.yaml rename to appengine/flexible/tasks/queue.yaml diff --git a/appengine/flexible/tasks/appengine_queues/requirements.txt b/appengine/flexible/tasks/requirements.txt similarity index 100% rename from appengine/flexible/tasks/appengine_queues/requirements.txt rename to appengine/flexible/tasks/requirements.txt diff --git a/appengine/flexible/tasks/pull_queues/README.md b/tasks/README.md similarity index 72% rename from appengine/flexible/tasks/pull_queues/README.md rename to tasks/README.md index fcac4b30f7d6..2aaf6f59a850 100644 --- a/appengine/flexible/tasks/pull_queues/README.md +++ b/tasks/README.md @@ -12,18 +12,7 @@ such as on Google App Engine or Google Compute Engine. ## Prerequisites to run locally: -The samples require a Python environment with [pip](https://pypi.python.org/pypi/pip) installed. -[virtualenv](https://virtualenv.readthedocs.org/en/latest/) is also recommended. - -All samples require a Google Cloud Project whitelisted for the Cloud Tasks API. The Cloud Tasks -API must also be enabled. To create a project and enable the API, go to the [Google Developers -Console](https://console.developer.google.com). You must also create an API key. This can be -done under API Manager -> Credentials. - -To install the Python application dependencies, run the following commands: - - * pip install -r requirements.txt - +Please refer to [Setting Up a Python Development Environment](https://cloud.google.com/python/setup). ## Authentication @@ -77,18 +66,3 @@ Pull and acknowledge a task: python pull_queue_snippets.py --api_key=$API_KEY pull-and-ack-task --queue_name=$QUEUE_NAME Note that usually, there would be a processing step in between pulling a task and acknowledging it. - -## Testing the Samples - -Install pytest: - - pip install pytest - -Set the `GOOGLE_CLOUD_PROJECT` and `API_KEY` environment variable to your project ID. - - export GOOGLE_CLOUD_PROJECT=my-project-id - export API_KEY=my-api-key - -Run pytest: - - pytest diff --git a/appengine/flexible/tasks/pull_queues/pull_queue_snippets.py b/tasks/pull_queue_snippets.py similarity index 83% rename from appengine/flexible/tasks/pull_queues/pull_queue_snippets.py rename to tasks/pull_queue_snippets.py index c76843afe907..d4615dc3f53e 100644 --- a/appengine/flexible/tasks/pull_queues/pull_queue_snippets.py +++ b/tasks/pull_queue_snippets.py @@ -16,8 +16,6 @@ """Sample command-line program for interacting with the Cloud Tasks API. -Please note that Cloud Tasks is currently in alpha. - See README.md for instructions on setting up your development environment and running the scripts. """ @@ -100,8 +98,9 @@ def get_client(api_key): if __name__ == '__main__': - parser = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) subparsers = parser.add_subparsers(dest='command') parser.add_argument('--api_key', help='API Key', required=True) @@ -114,20 +113,24 @@ def get_client(api_key): '--project_id', help='Project ID you want to access.', required=True) - list_queues_parser.add_argument('--location_id', - help='Location of the queues.', - required=True) + list_queues_parser.add_argument( + '--location_id', + help='Location of the queues.', + required=True) - create_task_parser = subparsers.add_parser('create-task', - help=create_task.__doc__) + create_task_parser = subparsers.add_parser( + 'create-task', + help=create_task.__doc__) create_task_parser.add_argument( '--queue_name', help='Fully qualified name of the queue to add the task to.') - pull_and_ack_parser = subparsers.add_parser('pull-and-ack-task', - help=create_task.__doc__) - pull_and_ack_parser.add_argument('--queue_name', - help='Fully qualified name of the queue to add the task to.') + pull_and_ack_parser = subparsers.add_parser( + 'pull-and-ack-task', + help=create_task.__doc__) + pull_and_ack_parser.add_argument( + '--queue_name', + help='Fully qualified name of the queue to add the task to.') args = parser.parse_args() diff --git a/appengine/flexible/tasks/pull_queues/pull_queue_snippets_test.py b/tasks/pull_queue_snippets_test.py similarity index 100% rename from appengine/flexible/tasks/pull_queues/pull_queue_snippets_test.py rename to tasks/pull_queue_snippets_test.py diff --git a/appengine/flexible/tasks/pull_queues/queue.yaml b/tasks/queue.yaml similarity index 100% rename from appengine/flexible/tasks/pull_queues/queue.yaml rename to tasks/queue.yaml diff --git a/appengine/flexible/tasks/pull_queues/requirements.txt b/tasks/requirements.txt similarity index 100% rename from appengine/flexible/tasks/pull_queues/requirements.txt rename to tasks/requirements.txt From 1a43939ce9aa2463178dbd57e19be4aec946f160 Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Wed, 30 Aug 2017 17:04:56 -0700 Subject: [PATCH 3/7] Update app engine queues samples --- appengine/flexible/tasks/README.md | 64 ++++---- .../tasks/app_engine_queue_snippets.py | 147 ------------------ .../tasks/create_app_engine_queue_task.py | 112 +++++++++++++ ...y => create_app_engine_queue_task_test.py} | 24 +-- appengine/flexible/tasks/main.py | 74 +-------- appengine/flexible/tasks/main_test.py | 17 +- appengine/flexible/tasks/queue.yaml | 4 - 7 files changed, 166 insertions(+), 276 deletions(-) delete mode 100644 appengine/flexible/tasks/app_engine_queue_snippets.py create mode 100644 appengine/flexible/tasks/create_app_engine_queue_task.py rename appengine/flexible/tasks/{app_engine_queue_snippets_test.py => create_app_engine_queue_task_test.py} (55%) delete mode 100644 appengine/flexible/tasks/queue.yaml diff --git a/appengine/flexible/tasks/README.md b/appengine/flexible/tasks/README.md index 61d6387b3024..d7b4f48e36d4 100644 --- a/appengine/flexible/tasks/README.md +++ b/appengine/flexible/tasks/README.md @@ -33,7 +33,7 @@ On Compute Engine and Container Engine, authentication credentials will be automatically detected, but the instances must have been created with the necessary scopes. -In any other environment, for example Compute Engine instance without the +In any other environment, for example a Compute Engine instance without the necessary scopes, you should set `GOOGLE_APPLICATION_CREDENTIALS` environment variable to a JSON key file for a service account. @@ -42,17 +42,18 @@ for more information. ## Creating a queue -To create a queue using the Cloud SDK, use the provided queue.yaml: +To create a queue using the Cloud SDK, use the following gcloud command: - gcloud app deploy queue.yaml + gcloud alpha tasks queues create-app-engine-queue "my-appengine-queue" -## Deploying the App Engine app - -First, vendor the dependencies into the project: +Note: A newly created queue will route to the default App Engine service and +version unless configured to do otherwise. Read the online help for the +`create-app-engine-queue` or the `update-app-engine-queue` commands to learn +about routing overrides for App Engine queues. - pip install -r requirements.txt +## Deploying the App Engine app -Next, deploy the App Engine app +Deploy the App Engine app with gcloud: gcloud app deploy @@ -61,47 +62,44 @@ Verify the index page is serving: gcloud app browse The App Engine app serves as a target for the push requests. It has an -endpoint `/set_payload` that that stores the payload from the HTTP POST data in -Cloud Datastore. The payload can be accessed in your browser at the - `/get_payload` endpoint with a GET request. +endpoint `/log_payload` that reads the payload (i.e., the request body) of the +HTTP POST request and logs it. The log output can be viewed with: + + gcloud app logs read ## Running the Samples The project ID must be specified either as a command line argument using `--project-id`, or by editing `DEFAULT_PROJECT_ID` within `task_snippets.py`. -Set the environment variables: +Set environment variables: + +First, your project ID: export PROJECT_ID=my-project-id - export LOCATION_ID=us-central1 - export QUEUE_ID=my-appengine-queue # From queue.yaml -View all queues: +Then the queue ID, as specified at queue creation time. Queue IDs already +created can be listed with `gcloud alpha tasks queue list`. - python app_engine_queue_snippets.py --api_key=$API_KEY list-queues --project_id=$PROJECT_ID --location_id=$LOCATION_ID + export QUEUE_ID=my-appengine-queue -Set the queue name as an environment variable: +And finally the location ID, which can be discovered with +`gcloud alpha tasks queue describe $QUEUE_ID`, with the location embedded in the +"name" value (for instance, if the name is +"projects/my-project/locations/us-central1/queues/my-appengine-queue", then the +location is "us-central1"). - export QUEUE_NAME=projects/$PROJECT_ID/locations/$LOCATION_ID/queues/$QUEUE_ID + export LOCATION_ID=us-central1 Create a task, targeted at the `set_payload` endpoint with a payload specified: - python app_engine_queue_snippets.py --api_key=$API_KEY create-task --queue_name=$QUEUE_NAME --payload=hello - -Now view that the payload was received and verify the count and payload: - - http://your-app-id.appspot.com/get_payload - -Create a task that will be scheduled for a few seconds in the future using -the `--in_seconds` flag: - - python app_engine_queue_snippets.py --api_key=$API_KEY create-task --queue_name=$QUEUE_NAME --payload=hello --in_seconds=30 + python create_app_engine_queue_task.py --project=$PROJECT_ID --queue=QUEUE_ID --location=LOCATION_ID --payload=hello -Since `--in_seconds` was set to 30, it will take 30 seconds for the new -payload to be pushed to the `/get_payload` endpoint, which can then be viewed at: +Now view that the payload was received and verify the payload: - http://your-app-id.appspot.com/get_payload + gcloud app logs read -It might also be helpful to view the request logs of your App Engine app: +Create a task that will be scheduled for a time in the future using the +`--in_seconds` flag: - https://console.cloud.google.com/logs + python create_app_engine_queue_task.py --project=$PROJECT_ID --queue=QUEUE_ID --location=LOCATION_ID --payload=hello --in_seconds=30 diff --git a/appengine/flexible/tasks/app_engine_queue_snippets.py b/appengine/flexible/tasks/app_engine_queue_snippets.py deleted file mode 100644 index 23121af8f2ba..000000000000 --- a/appengine/flexible/tasks/app_engine_queue_snippets.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import argparse -import base64 -import datetime -import pprint - -from googleapiclient import discovery - - -def format_rfc3339(datetime_instance): - """Format a datetime per RFC 3339.""" - return datetime_instance.isoformat('T') + 'Z' - - -def get_seconds_from_now_rfc3339(seconds): - """Return seconds from the current time as a RFC 3339 string.""" - d = datetime.datetime.utcnow() + datetime.timedelta(seconds=seconds) - return format_rfc3339(d) - - -def list_queues(api_key, project_id, location_id): - """List the queues in the location.""" - client = get_client(api_key) - parent = 'projects/{}/locations/{}'.format(project_id, location_id) - queues = [] - next_page_token = None - - while True: - response = client.projects().locations().queues().list( - parent=parent, pageToken=next_page_token).execute() - queues += response['queues'] - if next_page_token is None: - break - - print('Listing queues for location {}'.format(location_id)) - - for queue in response['queues']: - print(queue['name']) - return response - - -def create_task(api_key, queue_name, payload=None, in_seconds=None): - """Create a task for a given queue with an arbitrary payload.""" - client = get_client(api_key) - - url = '/set_payload' - body = { - 'task': { - 'app_engine_task_target': { - 'http_method': 'POST', - 'relative_url': url - } - } - } - - if payload is not None: - body['task']['app_engine_task_target']['payload'] = base64.b64encode( - payload) - - if in_seconds is not None: - scheduled_time = get_seconds_from_now_rfc3339(in_seconds) - body['task']['schedule_time'] = scheduled_time - - print('Sending task {}'.format(pprint.pformat(body))) - - response = client.projects().locations().queues().tasks().create( - parent=queue_name, body=body).execute() - - # By default CreateTaskRequest.responseView is BASIC, so not all - # information is retrieved by default because some data, such as payloads, - # might be desirable to return only when needed because of its large size - # or because of the sensitivity of data that it contains. - print('Created task {}'.format(response['name'])) - return response - - -def get_client(api_key): - """Build an authenticated http client.""" - DISCOVERY_URL = 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2&key={}'.format( - api_key) - client = discovery.build('cloudtasks', 'v2beta2', - discoveryServiceUrl=DISCOVERY_URL) - return client - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - - parser.add_argument('--api_key', help='API Key', required=True) - - subparsers = parser.add_subparsers(dest='command') - - list_queues_parser = subparsers.add_parser( - 'list-queues', - help=list_queues.__doc__) - - list_queues_parser.add_argument( - '--project_id', - help='Project ID you want to access.', - required=True) - - list_queues_parser.add_argument( - '--location_id', - help='Location of the queues.', - required=True) - - create_task_parser = subparsers.add_parser( - 'create-task', - help=create_task.__doc__) - create_task_parser.add_argument( - '--queue_name', - help='Fully qualified name of the queue to add the task to.' - ) - - create_task_parser.add_argument( - '--payload', - help='Optional payload to attach to the push queue.' - ) - - create_task_parser.add_argument( - '--in_seconds', - help='The number of seconds from now to schedule task attempt.' - ) - - args = parser.parse_args() - - if args.command == 'list-queues': - list_queues(args.api_key, args.project_id, args.location_id) - if args.command == 'create-task': - create_task(args.api_key, args.queue_name, args.payload, args.in_seconds) diff --git a/appengine/flexible/tasks/create_app_engine_queue_task.py b/appengine/flexible/tasks/create_app_engine_queue_task.py new file mode 100644 index 000000000000..f3cbda53b80d --- /dev/null +++ b/appengine/flexible/tasks/create_app_engine_queue_task.py @@ -0,0 +1,112 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import argparse +import base64 +import datetime +import pprint + +from googleapiclient import discovery + + +def seconds_from_now_to_rfc3339_datetime(seconds): + """Return an RFC 3339 datetime string for a number of seconds from now.""" + d = datetime.datetime.utcnow() + datetime.timedelta(seconds=seconds) + return d.isoformat('T') + 'Z' + + +def create_task(project, queue, location, payload=None, in_seconds=None): + """Create a task for a given queue with an arbitrary payload.""" + client = get_client() + + url = '/log_payload' + body = { + 'task': { + 'app_engine_task_target': { + 'http_method': 'POST', + 'relative_url': url + } + } + } + + if payload is not None: + # Payload is a string (unicode), so + body['task']['app_engine_task_target']['payload'] = base64.b64encode( + payload.encode()).decode() + + if in_seconds is not None: + scheduled_time = seconds_from_now_to_rfc3339_datetime(in_seconds) + body['task']['schedule_time'] = scheduled_time + + queue_name = 'projects/{}/locations/{}/queues/{}'.format( + project, location, queue) + + print('Sending task {}'.format(pprint.pformat(body))) + + response = client.projects().locations().queues().tasks().create( + parent=queue_name, body=body).execute() + + # By default CreateTaskRequest.responseView is BASIC, so not all + # information is retrieved by default because some data, such as payloads, + # might be desirable to return only when needed because of its large size + # or because of the sensitivity of data that it contains. + print('Created task {}'.format(response['name'])) + return response + + +def get_client(): + """Build an http client.""" + DISCOVERY_URL = 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2' + client = discovery.build('cloudtasks', 'v2beta2', + discoveryServiceUrl=DISCOVERY_URL) + return client + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=create_task.__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + parser.add_argument( + '--project', + help='Project of the queue to add the task to.' + ) + + parser.add_argument( + '--queue', + help='ID (short name) of the queue to add the task to.' + ) + + parser.add_argument( + '--location', + help='Location of the queue to add the task to.' + ) + + parser.add_argument( + '--payload', + help='Optional payload to attach to the push queue.' + ) + + parser.add_argument( + '--in_seconds', + help='The number of seconds from now to schedule task attempt.' + ) + + args = parser.parse_args() + + create_task( + args.project, args.queue, args.location, + args.payload, args.in_seconds) diff --git a/appengine/flexible/tasks/app_engine_queue_snippets_test.py b/appengine/flexible/tasks/create_app_engine_queue_task_test.py similarity index 55% rename from appengine/flexible/tasks/app_engine_queue_snippets_test.py rename to appengine/flexible/tasks/create_app_engine_queue_task_test.py index a48973f7a1f6..0c83077580d2 100644 --- a/appengine/flexible/tasks/app_engine_queue_snippets_test.py +++ b/appengine/flexible/tasks/create_app_engine_queue_task_test.py @@ -14,33 +14,19 @@ import mock -import app_engine_queue_snippets +import create_app_engine_queue_task -TEST_PROJECT_ID = 'mock-project' +TEST_PROJECT = 'mock-project' TEST_LOCATION = 'us-central1' -API_KEY = 'mock-api-key' -TEST_QUEUE_NAME = 'projects/{}/locations/{}/queues/{}'.format( - TEST_PROJECT_ID, TEST_LOCATION, 'my-push-queue') +TEST_QUEUE = 'my-appengine-queue' -@mock.patch('app_engine_queue_snippets.get_client') -def test_list_queues(get_client): - locations = get_client.return_value.projects.return_value.locations - queues = locations.return_value.queues - execute_function = queues.return_value.list.return_value.execute - execute_function.return_value = {'name': 'task_name', 'queues': []} - app_engine_queue_snippets.list_queues(API_KEY, TEST_PROJECT_ID, - TEST_LOCATION) - assert execute_function.called - - -@mock.patch('app_engine_queue_snippets.get_client') +@mock.patch('create_app_engine_queue_task.get_client') def test_create_task(get_client): projects = get_client.return_value.projects.return_value locations = projects.locations.return_value create_function = locations.queues.return_value.tasks.return_value.create execute_function = create_function.return_value.execute execute_function.return_value = {'name': 'task_name'} - app_engine_queue_snippets.create_task(API_KEY, TEST_QUEUE_NAME) + create_app_engine_queue_task.create_task(TEST_PROJECT, TEST_QUEUE, TEST_LOCATION) assert execute_function.called - diff --git a/appengine/flexible/tasks/main.py b/appengine/flexible/tasks/main.py index 53153299a85c..c2e20439fdc4 100644 --- a/appengine/flexible/tasks/main.py +++ b/appengine/flexible/tasks/main.py @@ -14,77 +14,19 @@ """App Engine app to serve as an endpoint for App Engine queue samples.""" -from google.cloud import datastore +import logging + from flask import Flask, request app = Flask(__name__) -client = datastore.Client() - -PAYLOAD_KEY_NAME = 'Payload' -COUNTER_KEY_NAME = 'Counter' - - -def get_payload_from_datastore(): - payload_key = client.key(PAYLOAD_KEY_NAME, 1) - payload_entity = client.get(payload_key) - if payload_entity is None: - return None - return payload_entity['value'] - - -def get_counter_from_datastore(): - counter_key = client.key(COUNTER_KEY_NAME, 1) - counter_entity = client.get(counter_key) - if counter_entity is None: - return 0 - return counter_entity['counter'] - - -def update_payload(request_data): - """Sets the payload value entity in Cloud Datastore.""" - payload_key = client.key(PAYLOAD_KEY_NAME, 1) - payload_entity = datastore.Entity(payload_key) - - if request_data: - payload_entity['value'] = request_data - client.put(payload_entity) - - payload_entity = client.get(payload_key) - return payload_entity['value'] - - -def increment_counter(): - """Increments a counter value in Cloud Datastore.""" - counter_key = client.key(COUNTER_KEY_NAME, 1) - counter_entity = client.get(counter_key) - if counter_entity is None: - counter_entity = datastore.Entity(counter_key) - counter_entity['counter'] = 0 - client.put(counter_entity) - - counter_entity['counter'] += 1 - client.put(counter_entity) - return counter_entity['counter'] - - -@app.route('/set_payload', methods=['POST']) -def set_payload(): - """Main endpoint to demonstrate Cloud Tasks App Engine queue features.""" - counter = increment_counter() - payload = update_payload(request.data) - - return 'Counter value is now {}, payload value is now {}'.format( - counter, payload) - -@app.route('/get_payload') -def get_payload(): - """Main endpoint to demonstrate Cloud Tasks App Engine queue features.""" - counter = get_counter_from_datastore() - payload = get_payload_from_datastore() - return 'Counter value is now {}, payload value is now {}'.format( - counter, payload) +@app.route('/log_payload', methods=['POST']) +def log_payload(): + """Log the request payload.""" + payload = request.data or "empty payload" + logging.warn(payload) + return 'Logged request payload: {}'.format(payload) @app.route('/') diff --git a/appengine/flexible/tasks/main_test.py b/appengine/flexible/tasks/main_test.py index 2b07228304ac..d81529c5227e 100644 --- a/appengine/flexible/tasks/main_test.py +++ b/appengine/flexible/tasks/main_test.py @@ -28,16 +28,19 @@ def test_index(app): assert r.status_code == 200 -@mock.patch('main.update_payload') -@mock.patch('main.increment_counter') -def test_set_payload(increment_counter, update_payload, app): +@mock.patch('logging.warn') +def test_log_payload(logging_mock, app): payload = 'hello' - r = app.post('/set_payload', payload) + r = app.post('/log_payload', payload) assert r.status_code == 200 - assert increment_counter.called - assert update_payload.called + assert logging_mock.called - r = app.get('/get_payload') + +@mock.patch('logging.warn') +def test_empty_payload(logging_mock, app): + r = app.post('/log_payload') assert r.status_code == 200 + + assert logging_mock.called diff --git a/appengine/flexible/tasks/queue.yaml b/appengine/flexible/tasks/queue.yaml deleted file mode 100644 index 96c6d0f1e3f7..000000000000 --- a/appengine/flexible/tasks/queue.yaml +++ /dev/null @@ -1,4 +0,0 @@ -queue: -- name: my-appengine-queue - target: default - rate: 1/s From ee920fb0eb22b8990393b1a59d0f3e29e8ca8d97 Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Tue, 12 Sep 2017 16:43:16 -0700 Subject: [PATCH 4/7] Address review feedback --- appengine/flexible/tasks/README.md | 14 +++++++------- .../tasks/create_app_engine_queue_task.py | 19 ++++++++----------- .../create_app_engine_queue_task_test.py | 6 +++--- tasks/pull_queue_snippets.py | 11 +++++++---- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/appengine/flexible/tasks/README.md b/appengine/flexible/tasks/README.md index d7b4f48e36d4..b00fe20b140e 100644 --- a/appengine/flexible/tasks/README.md +++ b/appengine/flexible/tasks/README.md @@ -44,7 +44,7 @@ for more information. To create a queue using the Cloud SDK, use the following gcloud command: - gcloud alpha tasks queues create-app-engine-queue "my-appengine-queue" + gcloud alpha tasks queues create-app-engine-queue my-appengine-queue Note: A newly created queue will route to the default App Engine service and version unless configured to do otherwise. Read the online help for the @@ -79,21 +79,21 @@ First, your project ID: export PROJECT_ID=my-project-id Then the queue ID, as specified at queue creation time. Queue IDs already -created can be listed with `gcloud alpha tasks queue list`. +created can be listed with `gcloud alpha tasks queues list`. export QUEUE_ID=my-appengine-queue And finally the location ID, which can be discovered with -`gcloud alpha tasks queue describe $QUEUE_ID`, with the location embedded in the -"name" value (for instance, if the name is +`gcloud alpha tasks queues describe $QUEUE_ID`, with the location embedded in +the "name" value (for instance, if the name is "projects/my-project/locations/us-central1/queues/my-appengine-queue", then the location is "us-central1"). export LOCATION_ID=us-central1 -Create a task, targeted at the `set_payload` endpoint with a payload specified: +Create a task, targeted at the `log_payload` endpoint, with a payload specified: - python create_app_engine_queue_task.py --project=$PROJECT_ID --queue=QUEUE_ID --location=LOCATION_ID --payload=hello + python create_app_engine_queue_task.py --project=$PROJECT_ID --queue=$QUEUE_ID --location=$LOCATION_ID --payload=hello Now view that the payload was received and verify the payload: @@ -102,4 +102,4 @@ Now view that the payload was received and verify the payload: Create a task that will be scheduled for a time in the future using the `--in_seconds` flag: - python create_app_engine_queue_task.py --project=$PROJECT_ID --queue=QUEUE_ID --location=LOCATION_ID --payload=hello --in_seconds=30 + python create_app_engine_queue_task.py --project=$PROJECT_ID --queue=$QUEUE_ID --location=$LOCATION_ID --payload=hello --in_seconds=30 diff --git a/appengine/flexible/tasks/create_app_engine_queue_task.py b/appengine/flexible/tasks/create_app_engine_queue_task.py index f3cbda53b80d..df36ccfd1249 100644 --- a/appengine/flexible/tasks/create_app_engine_queue_task.py +++ b/appengine/flexible/tasks/create_app_engine_queue_task.py @@ -17,7 +17,7 @@ import argparse import base64 import datetime -import pprint +import json from googleapiclient import discovery @@ -30,7 +30,12 @@ def seconds_from_now_to_rfc3339_datetime(seconds): def create_task(project, queue, location, payload=None, in_seconds=None): """Create a task for a given queue with an arbitrary payload.""" - client = get_client() + + # Create a client. + DISCOVERY_URL = ( + 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2') + client = discovery.build( + 'cloudtasks', 'v2beta2', discoveryServiceUrl=DISCOVERY_URL) url = '/log_payload' body = { @@ -54,7 +59,7 @@ def create_task(project, queue, location, payload=None, in_seconds=None): queue_name = 'projects/{}/locations/{}/queues/{}'.format( project, location, queue) - print('Sending task {}'.format(pprint.pformat(body))) + print('Sending task {}'.format(json.dumps(body))) response = client.projects().locations().queues().tasks().create( parent=queue_name, body=body).execute() @@ -67,14 +72,6 @@ def create_task(project, queue, location, payload=None, in_seconds=None): return response -def get_client(): - """Build an http client.""" - DISCOVERY_URL = 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2' - client = discovery.build('cloudtasks', 'v2beta2', - discoveryServiceUrl=DISCOVERY_URL) - return client - - if __name__ == '__main__': parser = argparse.ArgumentParser( description=create_task.__doc__, diff --git a/appengine/flexible/tasks/create_app_engine_queue_task_test.py b/appengine/flexible/tasks/create_app_engine_queue_task_test.py index 0c83077580d2..48f6157b284d 100644 --- a/appengine/flexible/tasks/create_app_engine_queue_task_test.py +++ b/appengine/flexible/tasks/create_app_engine_queue_task_test.py @@ -21,9 +21,9 @@ TEST_QUEUE = 'my-appengine-queue' -@mock.patch('create_app_engine_queue_task.get_client') -def test_create_task(get_client): - projects = get_client.return_value.projects.return_value +@mock.patch('googleapiclient.discovery.build') +def test_create_task(build): + projects = build.return_value.projects.return_value locations = projects.locations.return_value create_function = locations.queues.return_value.tasks.return_value.create execute_function = create_function.return_value.execute diff --git a/tasks/pull_queue_snippets.py b/tasks/pull_queue_snippets.py index d4615dc3f53e..107bea5182a2 100644 --- a/tasks/pull_queue_snippets.py +++ b/tasks/pull_queue_snippets.py @@ -35,7 +35,7 @@ def list_queues(api_key, project_id, location_id): while True: response = client.projects().locations( - ).queues().list(parent=parent,pageToken=next_page_token).execute() + ).queues().list(parent=parent, pageToken=next_page_token).execute() queues += response['queues'] if next_page_token is None: break @@ -90,10 +90,13 @@ def acknowledge_task(api_key, task): def get_client(api_key): """Build an authenticated http client.""" - DISCOVERY_URL = 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2&key={}'.format( - api_key) + discovery_url = ( + 'https://cloudtasks.googleapis.com/' + '$discovery/rest?version=v2beta2&key={}'.format( + api_key) + ) client = discovery.build('cloudtasks', 'v2beta2', - discoveryServiceUrl=DISCOVERY_URL) + discoveryServiceUrl=discovery_url) return client From fa4270ccbb5b6d9b200b537111c31389ccb229c0 Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Thu, 14 Sep 2017 13:57:48 -0700 Subject: [PATCH 5/7] Address review issues and convert pull queue sample to not use API key auth --- appengine/flexible/tasks/README.md | 19 ++--------- tasks/README.md | 18 ++-------- tasks/pull_queue_snippets.py | 54 +++++++++++++++--------------- tasks/pull_queue_snippets_test.py | 16 ++++----- 4 files changed, 38 insertions(+), 69 deletions(-) diff --git a/appengine/flexible/tasks/README.md b/appengine/flexible/tasks/README.md index b00fe20b140e..90e64a6c3a29 100644 --- a/appengine/flexible/tasks/README.md +++ b/appengine/flexible/tasks/README.md @@ -22,23 +22,8 @@ Please refer to [Setting Up a Python Development Environment](https://cloud.goog ## Authentication -To set up authentication locally, download the -[Cloud SDK](https://cloud.google.com/sdk), and run - - gcloud auth application-default login - -On App Engine, authentication credentials will be automatically detected. - -On Compute Engine and Container Engine, authentication credentials will be -automatically detected, but the instances must have been created with the -necessary scopes. - -In any other environment, for example a Compute Engine instance without the -necessary scopes, you should set `GOOGLE_APPLICATION_CREDENTIALS` environment -variable to a JSON key file for a service account. - -See the [authentication guide](https://cloud.google.com/docs/authentication) -for more information. +To set up authentication, please refer to our +[authentication getting started guide](https://cloud.google.com/docs/authentication/getting-started). ## Creating a queue diff --git a/tasks/README.md b/tasks/README.md index 2aaf6f59a850..87f73b14fa52 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -16,22 +16,8 @@ Please refer to [Setting Up a Python Development Environment](https://cloud.goog ## Authentication -To set up authentication locally, download the [Cloud SDK](https://cloud.google.com/sdk), and run - - gcloud auth application-default login - -On App Engine, authentication credentials will be automatically detected. - -On Compute Engine and Container Engine, authentication credentials will be -automatically detected, but the instances must have been created with the -necessary scopes. - -In any other environment, for example Compute Engine instance without the -necessary scopes, you should set `GOOGLE_APPLICATION_CREDENTIALS` environment -variable to a JSON key file for a service account. - -See the [authentication guide](https://cloud.google.com/docs/authentication) -for more information. +To set up authentication, please refer to our +[authentication getting started guide](https://cloud.google.com/docs/authentication/getting-started). ## Creating a queue diff --git a/tasks/pull_queue_snippets.py b/tasks/pull_queue_snippets.py index 107bea5182a2..b78b6cef67d2 100644 --- a/tasks/pull_queue_snippets.py +++ b/tasks/pull_queue_snippets.py @@ -26,16 +26,20 @@ from googleapiclient import discovery -def list_queues(api_key, project_id, location_id): +def list_queues(project_id, location_id): """List the queues in the location.""" - client = get_client(api_key) + DISCOVERY_URL = ( + 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2') + client = discovery.build('cloudtasks', 'v2beta2', + discoveryServiceUrl=discovery_url) parent = 'projects/{}/locations/{}'.format(project_id, location_id) queues = [] next_page_token = None while True: - response = client.projects().locations( - ).queues().list(parent=parent, pageToken=next_page_token).execute() + queues_api = client.projects().locations().queues() + response = queues_api.list( + parent=parent, pageToken=next_page_token).execute() queues += response['queues'] if next_page_token is None: break @@ -47,9 +51,12 @@ def list_queues(api_key, project_id, location_id): return response -def create_task(api_key, queue_name): +def create_task(queue_name): """Create a task for a given queue with an arbitrary payload.""" - client = get_client(api_key) + DISCOVERY_URL = ( + 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2') + client = discovery.build('cloudtasks', 'v2beta2', + discoveryServiceUrl=discovery_url) payload = 'a message for the recipient' task = { 'task': { @@ -64,9 +71,12 @@ def create_task(api_key, queue_name): return response -def pull_task(api_key, queue_name): +def pull_task(queue_name): """Pull a single task from a given queue and lease it for 10 minutes.""" - client = get_client(api_key) + DISCOVERY_URL = ( + 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2') + client = discovery.build('cloudtasks', 'v2beta2', + discoveryServiceUrl=discovery_url) duration_seconds = '600s' pull_options = { 'max_tasks': 1, @@ -79,34 +89,24 @@ def pull_task(api_key, queue_name): return response['tasks'][0] -def acknowledge_task(api_key, task): +def acknowledge_task(task): """Acknowledge a given task.""" - client = get_client(api_key) + DISCOVERY_URL = ( + 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2') + client = discovery.build('cloudtasks', 'v2beta2', + discoveryServiceUrl=discovery_url) body = {'scheduleTime': task['scheduleTime']} client.projects().locations().queues().tasks().acknowledge( name=task['name'], body=body).execute() print('Acknowledged task {}'.format(task['name'])) -def get_client(api_key): - """Build an authenticated http client.""" - discovery_url = ( - 'https://cloudtasks.googleapis.com/' - '$discovery/rest?version=v2beta2&key={}'.format( - api_key) - ) - client = discovery.build('cloudtasks', 'v2beta2', - discoveryServiceUrl=discovery_url) - return client - - if __name__ == '__main__': parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) subparsers = parser.add_subparsers(dest='command') - parser.add_argument('--api_key', help='API Key', required=True) list_queues_parser = subparsers.add_parser( 'list-queues', @@ -138,9 +138,9 @@ def get_client(api_key): args = parser.parse_args() if args.command == 'list-queues': - list_queues(args.api_key, args.project_id, args.location_id) + list_queues(args.project_id, args.location_id) if args.command == 'create-task': - create_task(args.api_key, args.queue_name) + create_task(args.queue_name) if args.command == 'pull-and-ack-task': - task = pull_task(args.api_key, args.queue_name) - acknowledge_task(args.api_key, task) + task = pull_task(args.queue_name) + acknowledge_task(task) diff --git a/tasks/pull_queue_snippets_test.py b/tasks/pull_queue_snippets_test.py index 8800ab12fb55..f3d27028e679 100644 --- a/tasks/pull_queue_snippets_test.py +++ b/tasks/pull_queue_snippets_test.py @@ -18,28 +18,26 @@ TEST_PROJECT_ID = os.getenv('GOOGLE_CLOUD_PROJECT') TEST_LOCATION = 'us-central1' -API_KEY = os.getenv('API_KEY') def test_list_queues(): result = pull_queue_snippets.list_queues( - API_KEY, TEST_PROJECT_ID, TEST_LOCATION) + TEST_PROJECT_ID, TEST_LOCATION) assert len(result['queues']) > 0 def test_create_task(): result = pull_queue_snippets.list_queues( - API_KEY, TEST_PROJECT_ID, TEST_LOCATION) + TEST_PROJECT_ID, TEST_LOCATION) queue_name = result['queues'][0]['name'] - result = pull_queue_snippets.create_task(API_KEY, queue_name) + result = pull_queue_snippets.create_task(queue_name) assert queue_name in result['name'] def test_pull_and_ack_task(): result = pull_queue_snippets.list_queues( - API_KEY, TEST_PROJECT_ID, TEST_LOCATION) + TEST_PROJECT_ID, TEST_LOCATION) queue_name = result['queues'][0]['name'] - pull_queue_snippets.create_task(API_KEY, queue_name) - task = pull_queue_snippets.pull_task(API_KEY, queue_name) - pull_queue_snippets.acknowledge_task(API_KEY, task) - + pull_queue_snippets.create_task(queue_name) + task = pull_queue_snippets.pull_task(queue_name) + pull_queue_snippets.acknowledge_task(task) From 4d1f228dc56260481e74d5c02d0aa2698a21aa3d Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Thu, 14 Sep 2017 14:35:52 -0700 Subject: [PATCH 6/7] Reform pull queues to match appengine queues changes to auth, command line input, readme --- appengine/flexible/tasks/README.md | 3 - tasks/README.md | 33 +++++---- tasks/pull_queue_snippets.py | 103 ++++++++++++++--------------- tasks/pull_queue_snippets_test.py | 26 +++----- 4 files changed, 76 insertions(+), 89 deletions(-) diff --git a/appengine/flexible/tasks/README.md b/appengine/flexible/tasks/README.md index 90e64a6c3a29..51e8ae6c0749 100644 --- a/appengine/flexible/tasks/README.md +++ b/appengine/flexible/tasks/README.md @@ -54,9 +54,6 @@ HTTP POST request and logs it. The log output can be viewed with: ## Running the Samples -The project ID must be specified either as a command line argument using -`--project-id`, or by editing `DEFAULT_PROJECT_ID` within `task_snippets.py`. - Set environment variables: First, your project ID: diff --git a/tasks/README.md b/tasks/README.md index 87f73b14fa52..012c56efad51 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -21,34 +21,39 @@ To set up authentication, please refer to our ## Creating a queue -Queues can not currently be created by the API. To create the queue using the Cloud SDK, use -the provided queue.yaml: +To create a queue using the Cloud SDK, use the following gcloud command: - gcloud app deploy queue.yaml + gcloud alpha tasks queues create-pull-queue my-pull-queue ## Running the Samples -The project ID must be specified either as a command line argument using `--project-id`, or by -editing `DEFAULT_PROJECT_ID` within `task_snippets.py`. - Set the environment variables: - export API_KEY=your-api-key +Set environment variables: + +First, your project ID: + export PROJECT_ID=my-project-id - export LOCATION_ID=us-central1 - export QUEUE_ID=my-pull-queue # From queue.yaml - export QUEUE_NAME=projects/$PROJECT_ID/locations/$LOCATION_ID/queues/$QUEUE_ID -View all queues: +Then the queue ID, as specified at queue creation time. Queue IDs already +created can be listed with `gcloud alpha tasks queues list`. - python pull_queue_snippets.py --api_key=$API_KEY list-queues --project_id=$PROJECT_ID --location_id=$LOCATION_ID + export QUEUE_ID=my-pull-queue + +And finally the location ID, which can be discovered with +`gcloud alpha tasks queues describe $QUEUE_ID`, with the location embedded in +the "name" value (for instance, if the name is +"projects/my-project/locations/us-central1/queues/my-pull-queue", then the +location is "us-central1"). + + export LOCATION_ID=us-central1 Create a task for a queue: - python pull_queue_snippets.py --api_key=$API_KEY create-task --queue_name=$QUEUE_NAME + python pull_queue_snippets.py create-task --project=$PROJECT_ID --queue=$QUEUE_ID --location=$LOCATION_ID Pull and acknowledge a task: - python pull_queue_snippets.py --api_key=$API_KEY pull-and-ack-task --queue_name=$QUEUE_NAME + python pull_queue_snippets.py pull-and-ack-task --project=$PROJECT_ID --queue=$QUEUE_ID --location=$LOCATION_ID Note that usually, there would be a processing step in between pulling a task and acknowledging it. diff --git a/tasks/pull_queue_snippets.py b/tasks/pull_queue_snippets.py index b78b6cef67d2..8d898a58a0a1 100644 --- a/tasks/pull_queue_snippets.py +++ b/tasks/pull_queue_snippets.py @@ -26,37 +26,14 @@ from googleapiclient import discovery -def list_queues(project_id, location_id): - """List the queues in the location.""" - DISCOVERY_URL = ( - 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2') - client = discovery.build('cloudtasks', 'v2beta2', - discoveryServiceUrl=discovery_url) - parent = 'projects/{}/locations/{}'.format(project_id, location_id) - queues = [] - next_page_token = None - - while True: - queues_api = client.projects().locations().queues() - response = queues_api.list( - parent=parent, pageToken=next_page_token).execute() - queues += response['queues'] - if next_page_token is None: - break - - print('Listing queues for location {}'.format(location_id)) - - for queue in response['queues']: - print queue['name'] - return response - - -def create_task(queue_name): +def create_task(project, queue, location): """Create a task for a given queue with an arbitrary payload.""" + DISCOVERY_URL = ( 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2') - client = discovery.build('cloudtasks', 'v2beta2', - discoveryServiceUrl=discovery_url) + client = discovery.build( + 'cloudtasks', 'v2beta2', discoveryServiceUrl=DISCOVERY_URL) + payload = 'a message for the recipient' task = { 'task': { @@ -65,39 +42,54 @@ def create_task(queue_name): } } } + + queue_name = 'projects/{}/locations/{}/queues/{}'.format( + project, location, queue) + response = client.projects().locations().queues().tasks().create( parent=queue_name, body=task).execute() + print('Created task {}'.format(response['name'])) return response -def pull_task(queue_name): +def pull_task(project, queue, location): """Pull a single task from a given queue and lease it for 10 minutes.""" + DISCOVERY_URL = ( 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2') - client = discovery.build('cloudtasks', 'v2beta2', - discoveryServiceUrl=discovery_url) + client = discovery.build( + 'cloudtasks', 'v2beta2', discoveryServiceUrl=DISCOVERY_URL) + duration_seconds = '600s' pull_options = { 'max_tasks': 1, 'leaseDuration': duration_seconds, 'responseView': 'FULL' } + + queue_name = 'projects/{}/locations/{}/queues/{}'.format( + project, location, queue) + response = client.projects().locations().queues().tasks().pull( name=queue_name, body=pull_options).execute() + print('Pulled task {}'.format(response)) return response['tasks'][0] def acknowledge_task(task): """Acknowledge a given task.""" + DISCOVERY_URL = ( 'https://cloudtasks.googleapis.com/$discovery/rest?version=v2beta2') - client = discovery.build('cloudtasks', 'v2beta2', - discoveryServiceUrl=discovery_url) + client = discovery.build( + 'cloudtasks', 'v2beta2', discoveryServiceUrl=DISCOVERY_URL) + body = {'scheduleTime': task['scheduleTime']} client.projects().locations().queues().tasks().acknowledge( name=task['name'], body=body).execute() + print('Acknowledged task {}'.format(task['name'])) @@ -108,39 +100,42 @@ def acknowledge_task(task): subparsers = parser.add_subparsers(dest='command') - list_queues_parser = subparsers.add_parser( - 'list-queues', - help=list_queues.__doc__) - - list_queues_parser.add_argument( - '--project_id', - help='Project ID you want to access.', - required=True) - list_queues_parser.add_argument( - '--location_id', - help='Location of the queues.', - required=True) - create_task_parser = subparsers.add_parser( 'create-task', help=create_task.__doc__) create_task_parser.add_argument( - '--queue_name', - help='Fully qualified name of the queue to add the task to.') + '--project', + help='Project of the queue to add the task to.' + ) + create_task_parser.add_argument( + '--queue', + help='ID (short name) of the queue to add the task to.' + ) + create_task_parser.add_argument( + '--location', + help='Location of the queue to add the task to.' + ) pull_and_ack_parser = subparsers.add_parser( 'pull-and-ack-task', help=create_task.__doc__) pull_and_ack_parser.add_argument( - '--queue_name', - help='Fully qualified name of the queue to add the task to.') + '--project', + help='Project of the queue to pull the task from.' + ) + pull_and_ack_parser.add_argument( + '--queue', + help='ID (short name) of the queue to pull the task from.' + ) + pull_and_ack_parser.add_argument( + '--location', + help='Location of the queue to pull the task from.' + ) args = parser.parse_args() - if args.command == 'list-queues': - list_queues(args.project_id, args.location_id) if args.command == 'create-task': - create_task(args.queue_name) + create_task(args.project, args.queue, args.location) if args.command == 'pull-and-ack-task': - task = pull_task(args.queue_name) + task = pull_task(args.project, args.queue, args.location) acknowledge_task(task) diff --git a/tasks/pull_queue_snippets_test.py b/tasks/pull_queue_snippets_test.py index f3d27028e679..68d3a1a4f469 100644 --- a/tasks/pull_queue_snippets_test.py +++ b/tasks/pull_queue_snippets_test.py @@ -16,28 +16,18 @@ import pull_queue_snippets -TEST_PROJECT_ID = os.getenv('GOOGLE_CLOUD_PROJECT') -TEST_LOCATION = 'us-central1' - - -def test_list_queues(): - result = pull_queue_snippets.list_queues( - TEST_PROJECT_ID, TEST_LOCATION) - assert len(result['queues']) > 0 +TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') +TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') +TEST_QUEUE_NAME = os.getenv('TEST_QUEUE_NAME', 'test-queue') def test_create_task(): - result = pull_queue_snippets.list_queues( - TEST_PROJECT_ID, TEST_LOCATION) - queue_name = result['queues'][0]['name'] - result = pull_queue_snippets.create_task(queue_name) - assert queue_name in result['name'] + result = pull_queue_snippets.create_task( + TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) + assert TEST_QUEUE_NAME in result['name'] def test_pull_and_ack_task(): - result = pull_queue_snippets.list_queues( - TEST_PROJECT_ID, TEST_LOCATION) - queue_name = result['queues'][0]['name'] - pull_queue_snippets.create_task(queue_name) - task = pull_queue_snippets.pull_task(queue_name) + pull_queue_snippets.create_task(TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) + task = pull_queue_snippets.pull_task(TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) pull_queue_snippets.acknowledge_task(task) From 5e7a37938260fd94c6b96cd5e4b568a3cbd59650 Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Thu, 14 Sep 2017 14:40:42 -0700 Subject: [PATCH 7/7] flake8 and fix comment --- appengine/flexible/tasks/create_app_engine_queue_task.py | 3 ++- .../flexible/tasks/create_app_engine_queue_task_test.py | 3 ++- tasks/pull_queue_snippets_test.py | 6 ++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/appengine/flexible/tasks/create_app_engine_queue_task.py b/appengine/flexible/tasks/create_app_engine_queue_task.py index df36ccfd1249..63d365b72253 100644 --- a/appengine/flexible/tasks/create_app_engine_queue_task.py +++ b/appengine/flexible/tasks/create_app_engine_queue_task.py @@ -48,7 +48,8 @@ def create_task(project, queue, location, payload=None, in_seconds=None): } if payload is not None: - # Payload is a string (unicode), so + # Payload is a string (unicode), and must be encoded for base64. + # The finished request body is JSON, which requires unicode. body['task']['app_engine_task_target']['payload'] = base64.b64encode( payload.encode()).decode() diff --git a/appengine/flexible/tasks/create_app_engine_queue_task_test.py b/appengine/flexible/tasks/create_app_engine_queue_task_test.py index 48f6157b284d..491f30017022 100644 --- a/appengine/flexible/tasks/create_app_engine_queue_task_test.py +++ b/appengine/flexible/tasks/create_app_engine_queue_task_test.py @@ -28,5 +28,6 @@ def test_create_task(build): create_function = locations.queues.return_value.tasks.return_value.create execute_function = create_function.return_value.execute execute_function.return_value = {'name': 'task_name'} - create_app_engine_queue_task.create_task(TEST_PROJECT, TEST_QUEUE, TEST_LOCATION) + create_app_engine_queue_task.create_task( + TEST_PROJECT, TEST_QUEUE, TEST_LOCATION) assert execute_function.called diff --git a/tasks/pull_queue_snippets_test.py b/tasks/pull_queue_snippets_test.py index 68d3a1a4f469..6905fdb3abc4 100644 --- a/tasks/pull_queue_snippets_test.py +++ b/tasks/pull_queue_snippets_test.py @@ -28,6 +28,8 @@ def test_create_task(): def test_pull_and_ack_task(): - pull_queue_snippets.create_task(TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) - task = pull_queue_snippets.pull_task(TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) + pull_queue_snippets.create_task( + TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) + task = pull_queue_snippets.pull_task( + TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) pull_queue_snippets.acknowledge_task(task)