diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index f94e9a878d54c..926f678584c74 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -168,6 +168,9 @@ Running Regression Tests `docs `__ for explanation on how to get a private key. + DJH: THIS PART NEEDS TO BE UPDATED AFTER DISCUSSION OF IMPLICIT ENVIRON + USE IN PRODUCTION CODE. + - Examples of these can be found in ``regression/local_test_setup.sample``. We recommend copying this to ``regression/local_test_setup``, editing the values and sourcing them into your environment:: diff --git a/gcloud/datastore/__init__.py b/gcloud/datastore/__init__.py index fee5a98b47b9d..608774c29ddd8 100644 --- a/gcloud/datastore/__init__.py +++ b/gcloud/datastore/__init__.py @@ -32,11 +32,38 @@ which represents a lookup or search over the rows in the datastore. """ +import os + __version__ = '0.1.2' SCOPE = ('https://www.googleapis.com/auth/datastore ', 'https://www.googleapis.com/auth/userinfo.email') """The scope required for authenticating as a Cloud Datastore consumer.""" +DATASET = None +"""Module global which allows users to only optionally use a dataset.""" + + +def get_local_dataset_settings(): + """Determines auth settings from local enviroment. + + Currently only supports enviroment variables but will implicitly + support App Engine, Compute Engine and other environments in + the future. + + Local environment variables used are: + - GCLOUD_DATASET_ID + - GCLOUD_CLIENT_EMAIL + - GCLOUD_KEY_FILE + """ + local_dataset_settings = ( + os.getenv('GCLOUD_DATASET_ID'), + os.getenv('GCLOUD_CLIENT_EMAIL'), + os.getenv('GCLOUD_KEY_FILE'), + ) + if None in local_dataset_settings: + return None + else: + return local_dataset_settings def get_connection(client_email, private_key_path): @@ -102,3 +129,73 @@ def get_dataset(dataset_id, client_email, private_key_path): """ connection = get_connection(client_email, private_key_path) return connection.dataset(dataset_id) + + +def _require_dataset(): + """Convenience method to ensure DATASET is set. + + :raises: :class:`EnvironmentError` if DATASET is not set. + """ + if DATASET is None: + raise EnvironmentError('Dataset could not be implied.') + + +def get_entity(key): + """Retrieves entity from implicit dataset, along with its attributes. + + :type key: :class:`gcloud.datastore.key.Key` + :param key: The name of the item to retrieve. + + :rtype: :class:`gcloud.datastore.entity.Entity` or ``None`` + :return: The requested entity, or ``None`` if there was no match found. + """ + _require_dataset() + return DATASET.get_entity(key) + + +def get_entities(keys): + """Retrieves entities from implied dataset, along with their attributes. + + :type keys: list of :class:`gcloud.datastore.key.Key` + :param keys: The name of the item to retrieve. + + :rtype: list of :class:`gcloud.datastore.entity.Entity` + :return: The requested entities. + """ + _require_dataset() + return DATASET.get_entities(keys) + + +def allocate_ids(incomplete_key, num_ids): + """Allocates a list of IDs from a partial key. + + :type incomplete_key: A :class:`gcloud.datastore.key.Key` + :param incomplete_key: The partial key to use as base for allocated IDs. + + :type num_ids: A :class:`int`. + :param num_ids: The number of IDs to allocate. + + :rtype: list of :class:`gcloud.datastore.key.Key` + :return: The (complete) keys allocated with `incomplete_key` as root. + """ + _require_dataset() + + if not incomplete_key.is_partial(): + raise ValueError(('Key is not partial.', incomplete_key)) + + incomplete_key_pb = incomplete_key.to_protobuf() + incomplete_key_pbs = [incomplete_key_pb] * num_ids + + allocated_key_pbs = DATASET.connection().allocate_ids( + DATASET.id(), incomplete_key_pbs) + allocated_ids = [allocated_key_pb.path_element[-1].id + for allocated_key_pb in allocated_key_pbs] + return [incomplete_key.id(allocated_id) + for allocated_id in allocated_ids] + + +# Set DATASET if it can be implied from the environment. +LOCAL_DATASET_SETTINGS = get_local_dataset_settings() +if LOCAL_DATASET_SETTINGS is not None: + DATASET = get_dataset(*LOCAL_DATASET_SETTINGS) +del LOCAL_DATASET_SETTINGS diff --git a/gcloud/datastore/dataset.py b/gcloud/datastore/dataset.py index cb5484abbf38f..d4d8ae3ddeea9 100644 --- a/gcloud/datastore/dataset.py +++ b/gcloud/datastore/dataset.py @@ -106,7 +106,7 @@ def get_entity(self, key): """Retrieves entity from the dataset, along with its attributes. :type key: :class:`gcloud.datastore.key.Key` - :param item_name: The name of the item to retrieve. + :param key: The name of the item to retrieve. :rtype: :class:`gcloud.datastore.entity.Entity` or ``None`` :return: The requested entity, or ``None`` if there was no match found. @@ -118,8 +118,8 @@ def get_entity(self, key): def get_entities(self, keys): """Retrieves entities from the dataset, along with their attributes. - :type key: list of :class:`gcloud.datastore.key.Key` - :param item_name: The name of the item to retrieve. + :type keys: list of :class:`gcloud.datastore.key.Key` + :param keys: The name of the item to retrieve. :rtype: list of :class:`gcloud.datastore.entity.Entity` :return: The requested entities. diff --git a/gcloud/datastore/entity.py b/gcloud/datastore/entity.py index 0d75a5c9b4138..52c996da6c333 100644 --- a/gcloud/datastore/entity.py +++ b/gcloud/datastore/entity.py @@ -15,6 +15,7 @@ delete or persist the data stored on the entity. """ +import gcloud.datastore from gcloud.datastore import datastore_v1_pb2 as datastore_pb from gcloud.datastore.key import Key @@ -73,6 +74,8 @@ class Entity(dict): def __init__(self, dataset=None, kind=None): super(Entity, self).__init__() self._dataset = dataset + if self._dataset is None: + self._dataset = gcloud.datastore.DATASET if kind: self._key = Key().kind(kind) else: diff --git a/gcloud/datastore/query.py b/gcloud/datastore/query.py index c3a02615ffae7..73bb7309a2089 100644 --- a/gcloud/datastore/query.py +++ b/gcloud/datastore/query.py @@ -2,6 +2,7 @@ import base64 +import gcloud.datastore from gcloud.datastore import datastore_v1_pb2 as datastore_pb from gcloud.datastore import helpers from gcloud.datastore.key import Key @@ -56,6 +57,8 @@ class Query(object): def __init__(self, kind=None, dataset=None, namespace=None): self._dataset = dataset + if self._dataset is None: + self._dataset = gcloud.datastore.DATASET self._namespace = namespace self._pb = datastore_pb.Query() self._cursor = None diff --git a/gcloud/datastore/test___init__.py b/gcloud/datastore/test___init__.py index 1475631090663..9bf218401c4cd 100644 --- a/gcloud/datastore/test___init__.py +++ b/gcloud/datastore/test___init__.py @@ -33,6 +33,50 @@ def test_it(self): self.assertEqual(client._called_with, expected_called_with) +class Test_get_local_dataset_settings(unittest2.TestCase): + + def _callFUT(self): + from gcloud.datastore import get_local_dataset_settings + return get_local_dataset_settings() + + def _test_with_environ(self, environ, expected_result): + import os + from gcloud._testing import _Monkey + + def custom_getenv(key): + return environ.get(key) + + with _Monkey(os, getenv=custom_getenv): + result = self._callFUT() + + self.assertEqual(result, expected_result) + + def test_all_set(self): + # Fake auth variables. + DATASET = 'dataset' + CLIENT_EMAIL = 'phred@example.com' + TEMP_PATH = 'fakepath' + + # Make a custom getenv function to Monkey. + VALUES = { + 'GCLOUD_DATASET_ID': DATASET, + 'GCLOUD_CLIENT_EMAIL': CLIENT_EMAIL, + 'GCLOUD_KEY_FILE': TEMP_PATH, + } + expected_result = (DATASET, CLIENT_EMAIL, TEMP_PATH) + self._test_with_environ(VALUES, expected_result) + + def test_partial_set(self): + # Fake auth variables. + DATASET = 'dataset' + + # Make a custom getenv function to Monkey. + VALUES = { + 'GCLOUD_DATASET_ID': DATASET, + } + self._test_with_environ(VALUES, None) + + class Test_get_dataset(unittest2.TestCase): def _callFUT(self, dataset_id, client_email, private_key_path): @@ -66,3 +110,160 @@ def test_it(self): 'scope': SCOPE, } self.assertEqual(client._called_with, expected_called_with) + + +class Test_implicit_behavior(unittest2.TestCase): + + def test__require_dataset(self): + import gcloud.datastore + original_dataset = gcloud.datastore.DATASET + + try: + gcloud.datastore.DATASET = None + self.assertRaises(EnvironmentError, + gcloud.datastore._require_dataset) + gcloud.datastore.DATASET = object() + self.assertEqual(gcloud.datastore._require_dataset(), None) + finally: + gcloud.datastore.DATASET = original_dataset + + def test_get_entity(self): + import gcloud.datastore + from gcloud.datastore.test_entity import _Dataset + from gcloud._testing import _Monkey + + CUSTOM_DATASET = _Dataset() + DUMMY_KEY = object() + DUMMY_VAL = object() + CUSTOM_DATASET[DUMMY_KEY] = DUMMY_VAL + with _Monkey(gcloud.datastore, DATASET=CUSTOM_DATASET): + result = gcloud.datastore.get_entity(DUMMY_KEY) + self.assertTrue(result is DUMMY_VAL) + + def test_get_entities(self): + import gcloud.datastore + from gcloud.datastore.test_entity import _Dataset + from gcloud._testing import _Monkey + + class _ExtendedDataset(_Dataset): + def get_entities(self, keys): + return [self.get(key) for key in keys] + + CUSTOM_DATASET = _ExtendedDataset() + DUMMY_KEYS = [object(), object()] + DUMMY_VALS = [object(), object()] + for key, val in zip(DUMMY_KEYS, DUMMY_VALS): + CUSTOM_DATASET[key] = val + + with _Monkey(gcloud.datastore, DATASET=CUSTOM_DATASET): + result = gcloud.datastore.get_entities(DUMMY_KEYS) + self.assertTrue(result == DUMMY_VALS) + + def test_allocate_ids(self): + import gcloud.datastore + from gcloud.datastore.test_entity import _Connection + from gcloud.datastore.test_entity import _DATASET_ID + from gcloud.datastore.test_entity import _Dataset + from gcloud.datastore.test_entity import _Key + from gcloud._testing import _Monkey + + class _PathElementProto(object): + COUNTER = 0 + + def __init__(self): + _PathElementProto.COUNTER += 1 + self.id = _PathElementProto.COUNTER + + class _KeyProto(object): + + def __init__(self): + self.path_element = [_PathElementProto()] + + class _ExtendedKey(_Key): + def id(self, id_to_set): + self._called_id = id_to_set + return id_to_set + + INCOMPLETE_KEY = _ExtendedKey() + INCOMPLETE_KEY._key = _KeyProto() + INCOMPLETE_KEY._partial = True + NUM_IDS = 2 + + class _ExtendedConnection(_Connection): + def allocate_ids(self, dataset_id, key_pbs): + self._called_dataset_id = dataset_id + self._called_key_pbs = key_pbs + return key_pbs + + CUSTOM_CONNECTION = _ExtendedConnection() + CUSTOM_DATASET = _Dataset(connection=CUSTOM_CONNECTION) + with _Monkey(gcloud.datastore, DATASET=CUSTOM_DATASET): + result = gcloud.datastore.allocate_ids(INCOMPLETE_KEY, NUM_IDS) + + self.assertEqual(_PathElementProto.COUNTER, 1) + self.assertEqual(result, [1, 1]) + self.assertEqual(CUSTOM_CONNECTION._called_dataset_id, _DATASET_ID) + self.assertEqual(len(CUSTOM_CONNECTION._called_key_pbs), 2) + key_paths = [key_pb.path_element[-1].id + for key_pb in CUSTOM_CONNECTION._called_key_pbs] + self.assertEqual(key_paths, [1, 1]) + + def test_allocate_ids_with_complete(self): + import gcloud.datastore + from gcloud.datastore.test_entity import _Connection + from gcloud.datastore.test_entity import _Dataset + from gcloud.datastore.test_entity import _Key + from gcloud._testing import _Monkey + + COMPLETE_KEY = _Key() + NUM_IDS = 2 + CUSTOM_CONNECTION = _Connection() + CUSTOM_DATASET = _Dataset(connection=CUSTOM_CONNECTION) + with _Monkey(gcloud.datastore, DATASET=CUSTOM_DATASET): + self.assertRaises(ValueError, gcloud.datastore.allocate_ids, + COMPLETE_KEY, NUM_IDS) + + def test_set_DATASET(self): + import os + import tempfile + from gcloud import credentials + from gcloud.test_credentials import _Client + from gcloud._testing import _Monkey + + # Make custom client for doing auth. + client = _Client() + + # Fake auth variables. + CLIENT_EMAIL = 'phred@example.com' + PRIVATE_KEY = 'SEEkR1t' + DATASET = 'dataset' + + # Write the fake key to a temp file. + TEMP_PATH = tempfile.mktemp() + with open(TEMP_PATH, 'w') as file_obj: + file_obj.write(PRIVATE_KEY) + file_obj.flush() + + # Make a custom getenv function to Monkey. + VALUES = { + 'GCLOUD_DATASET_ID': DATASET, + 'GCLOUD_CLIENT_EMAIL': CLIENT_EMAIL, + 'GCLOUD_KEY_FILE': TEMP_PATH, + } + + def custom_getenv(key): + return VALUES.get(key) + + # Perform the import again with our test patches. + with _Monkey(credentials, client=client): + with _Monkey(os, getenv=custom_getenv): + import gcloud.datastore + reload(gcloud.datastore) + + # Check that the DATASET was correctly implied from the environ. + implicit_dataset = gcloud.datastore.DATASET + self.assertEqual(implicit_dataset.id(), DATASET) + # Check that the credentials on the implicit DATASET was set on the + # fake client. + credentials = implicit_dataset.connection().credentials + self.assertTrue(credentials is client._signed) diff --git a/gcloud/datastore/test_entity.py b/gcloud/datastore/test_entity.py index d53f32e6e7d07..2a6d8d6ed5ead 100644 --- a/gcloud/datastore/test_entity.py +++ b/gcloud/datastore/test_entity.py @@ -9,6 +9,9 @@ class TestEntity(unittest2.TestCase): def _getTargetClass(self): + import gcloud.datastore + gcloud.datastore.DATASET = None + from gcloud.datastore.entity import Entity return Entity diff --git a/gcloud/datastore/test_helpers.py b/gcloud/datastore/test_helpers.py index a845c6c920f75..de6e1e9d3828a 100644 --- a/gcloud/datastore/test_helpers.py +++ b/gcloud/datastore/test_helpers.py @@ -6,6 +6,9 @@ class Test_entity_from_protobuf(unittest2.TestCase): _MARKER = object() def _callFUT(self, val, dataset=_MARKER): + import gcloud.datastore + gcloud.datastore.DATASET = None + from gcloud.datastore.helpers import entity_from_protobuf if dataset is self._MARKER: diff --git a/gcloud/datastore/test_query.py b/gcloud/datastore/test_query.py index 09bc7b3277d02..0d8cbf2d3c1b4 100644 --- a/gcloud/datastore/test_query.py +++ b/gcloud/datastore/test_query.py @@ -4,6 +4,9 @@ class TestQuery(unittest2.TestCase): def _getTargetClass(self): + import gcloud.datastore + gcloud.datastore.DATASET = None + from gcloud.datastore.query import Query return Query diff --git a/gcloud/datastore/test_transaction.py b/gcloud/datastore/test_transaction.py index 82eba83ff56d0..db4e437e3759f 100644 --- a/gcloud/datastore/test_transaction.py +++ b/gcloud/datastore/test_transaction.py @@ -24,6 +24,15 @@ def test_ctor(self): self.assertEqual(len(xact._auto_id_entities), 0) self.assertTrue(xact.connection() is connection) + def test_ctor_with_env(self): + SENTINEL_VAL = object() + + import gcloud.datastore + gcloud.datastore.DATASET = SENTINEL_VAL + + transaction = self._makeOne(dataset=None) + self.assertEqual(transaction.dataset(), SENTINEL_VAL) + def test_add_auto_id_entity(self): entity = _Entity() _DATASET = 'DATASET' diff --git a/gcloud/datastore/transaction.py b/gcloud/datastore/transaction.py index fd11941cbea92..54fe3faad5912 100644 --- a/gcloud/datastore/transaction.py +++ b/gcloud/datastore/transaction.py @@ -1,5 +1,6 @@ """Create / interact with gcloud datastore transactions.""" +import gcloud.datastore from gcloud.datastore import datastore_v1_pb2 as datastore_pb from gcloud.datastore import helpers @@ -124,8 +125,10 @@ class Transaction(object): :param dataset: The dataset to which this :class:`Transaction` belongs. """ - def __init__(self, dataset): + def __init__(self, dataset=None): self._dataset = dataset + if self._dataset is None: + self._dataset = gcloud.datastore.DATASET self._id = None self._mutation = datastore_pb.Mutation() self._auto_id_entities = [] diff --git a/pylintrc_default b/pylintrc_default index 84293654d3aa8..b17c93f0c51d5 100644 --- a/pylintrc_default +++ b/pylintrc_default @@ -24,7 +24,7 @@ ignore = datastore_v1_pb2.py [MESSAGES CONTROL] disable = I, protected-access, maybe-no-member, no-member, redefined-builtin, star-args, missing-format-attribute, - similarities, arguments-differ + similarities, arguments-differ, cyclic-import [REPORTS] reports = no diff --git a/regression/clear_datastore.py b/regression/clear_datastore.py new file mode 100644 index 0000000000000..19d294e43b531 --- /dev/null +++ b/regression/clear_datastore.py @@ -0,0 +1,57 @@ +"""Script to populate datastore with regression test data.""" + + +# This assumes the command is being run via tox hence the +# repository root is the current directory. +from regression import regression_utils + + +FETCH_MAX = 20 +ALL_KINDS = [ + 'Character', + 'Company', + 'Kind', + 'Person', + 'Post', +] + + +def remove_kind(dataset, kind): + dataset_id = dataset.id() + connection = dataset.connection() + + with dataset.transaction(): + query = dataset.query(kind=kind).limit( + FETCH_MAX).projection(['__key__']) + results = [] + more_results = True + while more_results: + # Make new query. + if query._cursor is not None: + query = query.with_cursor(query._cursor) + + curr_results = query.fetch() + results.extend(curr_results) + + more_results = len(curr_results) == FETCH_MAX + + # Now that we have all results, we seek to delete. + key_pbs = [entity.key().to_protobuf() for entity in results] + connection.delete_entities(dataset_id, key_pbs) + + +def remove_all_entities(): + print 'This command will remove all entities for the following kinds:' + print '\n'.join(['- ' + val for val in ALL_KINDS]) + response = raw_input('Is this OK [y/n]? ') + if response.lower() != 'y': + print 'Doing nothing.' + return + + dataset = regression_utils.get_dataset() + for kind in ALL_KINDS: + remove_kind(dataset, kind) + + +if __name__ == '__main__': + remove_all_entities() diff --git a/regression/datastore.py b/regression/datastore.py index a6213d6e85427..dbfb54874b5bb 100644 --- a/regression/datastore.py +++ b/regression/datastore.py @@ -6,20 +6,15 @@ # This assumes the command is being run via tox hence the # repository root is the current directory. from regression import populate_datastore -from regression import regression_utils class TestDatastore(unittest2.TestCase): - @classmethod - def setUpClass(cls): - cls.dataset = regression_utils.get_dataset() - def setUp(self): self.case_entities_to_delete = [] def tearDown(self): - with self.dataset.transaction(): + with datastore.transaction.Transaction(): for entity in self.case_entities_to_delete: entity.delete() @@ -28,17 +23,12 @@ class TestDatastoreAllocateIDs(TestDatastore): def test_allocate_ids(self): incomplete_key = datastore.key.Key(path=[{'kind': 'Kind'}]) - incomplete_key_pb = incomplete_key.to_protobuf() - incomplete_key_pbs = [incomplete_key_pb] * 10 - - connection = self.dataset.connection() - allocated_key_pbs = connection.allocate_ids(self.dataset.id(), - incomplete_key_pbs) - allocated_keys = [datastore.helpers.key_from_protobuf(key_pb) - for key_pb in allocated_key_pbs] - self.assertEqual(len(allocated_keys), 10) + num_ids = 10 + allocated_keys = datastore.allocate_ids(incomplete_key, num_ids) + self.assertEqual(len(allocated_keys), num_ids) for key in allocated_keys: - self.assertFalse(key.is_partial()) + self.assertEqual(key.name(), None) + self.assertNotEqual(key.id(), None) class TestDatastoreSave(TestDatastore): @@ -53,8 +43,8 @@ def _get_post(self, name=None, key_id=None, post_content=None): 'wordCount': 400, 'rating': 5.0, } - # Create an entity with the given content in our dataset. - entity = self.dataset.entity(kind='Post') + # Create an entity with the given content. + entity = datastore.entity.Entity(kind='Post') entity.update(post_content) # Update the entity key. @@ -79,7 +69,7 @@ def _generic_test_post(self, name=None, key_id=None): self.assertEqual(entity.key().name(), name) if key_id is not None: self.assertEqual(entity.key().id(), key_id) - retrieved_entity = self.dataset.get_entity(entity.key()) + retrieved_entity = datastore.get_entity(entity.key()) # Check the keys are the same. self.assertEqual(retrieved_entity.key().path(), entity.key().path()) self.assertEqual(retrieved_entity.key().namespace(), @@ -100,7 +90,7 @@ def test_post_with_generated_id(self): self._generic_test_post() def test_save_multiple(self): - with self.dataset.transaction(): + with datastore.transaction.Transaction(): entity1 = self._get_post() entity1.save() # Register entity to be deleted. @@ -121,11 +111,11 @@ def test_save_multiple(self): self.case_entities_to_delete.append(entity2) keys = [entity1.key(), entity2.key()] - matches = self.dataset.get_entities(keys) + matches = datastore.get_entities(keys) self.assertEqual(len(matches), 2) def test_empty_kind(self): - posts = self.dataset.query('Post').limit(2).fetch() + posts = datastore.query.Query(kind='Post').limit(2).fetch() self.assertEqual(posts, []) @@ -133,14 +123,14 @@ class TestDatastoreSaveKeys(TestDatastore): def test_save_key_self_reference(self): key = datastore.key.Key.from_path('Person', 'name') - entity = self.dataset.entity(kind=None).key(key) + entity = datastore.entity.Entity(kind=None).key(key) entity['fullName'] = u'Full name' entity['linkedTo'] = key # Self reference. entity.save() self.case_entities_to_delete.append(entity) - query = self.dataset.query('Person').filter( + query = datastore.query.Query(kind='Person').filter( 'linkedTo =', key).limit(2) stored_persons = query.fetch() @@ -162,7 +152,8 @@ def setUpClass(cls): path=[populate_datastore.ANCESTOR]) def _base_query(self): - return self.dataset.query('Character').ancestor(self.ANCESTOR_KEY) + return datastore.query.Query(kind='Character').ancestor( + self.ANCESTOR_KEY) def test_limit_queries(self): limit = 5 @@ -332,17 +323,17 @@ class TestDatastoreTransaction(TestDatastore): def test_transaction(self): key = datastore.key.Key.from_path('Company', 'Google') - entity = self.dataset.entity(kind=None).key(key) + entity = datastore.entity.Entity(kind=None).key(key) entity['url'] = u'www.google.com' - with self.dataset.transaction(): - retrieved_entity = self.dataset.get_entity(key) + with datastore.transaction.Transaction(): + retrieved_entity = datastore.get_entity(key) if retrieved_entity is None: entity.save() self.case_entities_to_delete.append(entity) # This will always return after the transaction. - retrieved_entity = self.dataset.get_entity(key) + retrieved_entity = datastore.get_entity(key) retrieved_dict = dict(retrieved_entity.items()) entity_dict = dict(entity.items()) self.assertEqual(retrieved_dict, entity_dict)