From 19c9d9620174db18ca8dad2ca71cf2d51f619d00 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Wed, 10 Jan 2018 16:54:57 -0800 Subject: [PATCH] Revert "Revert "api_core: Make PageIterator.item_to_value public. (#4702)" (#4731)" This reverts commit b0dc7bc17d8bb3ab56928f28096846c55cb79893. --- api_core/google/api_core/page_iterator.py | 21 +++++++++++++++++---- api_core/tests/unit/test_page_iterator.py | 10 +++++----- bigquery/tests/unit/test_table.py | 2 +- datastore/google/cloud/datastore/query.py | 2 +- datastore/tests/unit/test_query.py | 2 -- spanner/google/cloud/spanner_v1/client.py | 4 ++-- spanner/google/cloud/spanner_v1/instance.py | 2 +- 7 files changed, 27 insertions(+), 16 deletions(-) diff --git a/api_core/google/api_core/page_iterator.py b/api_core/google/api_core/page_iterator.py index 3d2a55b7173a..0cf03a619358 100644 --- a/api_core/google/api_core/page_iterator.py +++ b/api_core/google/api_core/page_iterator.py @@ -158,12 +158,25 @@ def __init__(self, client, item_to_value=_item_to_value_identity, page_token=None, max_results=None): self._started = False self.client = client - self._item_to_value = item_to_value + """Optional[Any]: The client that created this iterator.""" + self.item_to_value = item_to_value + """Callable[Iterator, Any]: Callable to convert an item from the type + in the raw API response into the native object. Will be called with + the iterator and a + single item. + """ self.max_results = max_results + """int: The maximum number of results to fetch.""" + # The attributes below will change over the life of the iterator. self.page_number = 0 + """int: The current page of results.""" self.next_page_token = page_token + """str: The token for the next page of results. If this is set before + the iterator starts, it effectively offsets the iterator to a + specific starting point.""" self.num_results = 0 + """int: The total number of results fetched so far.""" @property def pages(self): @@ -335,7 +348,7 @@ def _next_page(self): if self._has_next_page(): response = self._get_next_page_response() items = response.get(self._items_key, ()) - page = Page(self, items, self._item_to_value) + page = Page(self, items, self.item_to_value) self._page_start(self, page, response) self.next_page_token = response.get(self._next_token) return page @@ -428,7 +441,7 @@ def _next_page(self): """ try: items = six.next(self._gax_page_iter) - page = Page(self, items, self._item_to_value) + page = Page(self, items, self.item_to_value) self.next_page_token = self._gax_page_iter.page_token or None return page except StopIteration: @@ -500,7 +513,7 @@ def _next_page(self): self.next_page_token = getattr(response, self._response_token_field) items = getattr(response, self._items_field) - page = Page(self, items, self._item_to_value) + page = Page(self, items, self.item_to_value) return page diff --git a/api_core/tests/unit/test_page_iterator.py b/api_core/tests/unit/test_page_iterator.py index 5defb765b3bb..172be0318b9c 100644 --- a/api_core/tests/unit/test_page_iterator.py +++ b/api_core/tests/unit/test_page_iterator.py @@ -88,7 +88,7 @@ def test_constructor(self): assert not iterator._started assert iterator.client is client - assert iterator._item_to_value == item_to_value + assert iterator.item_to_value == item_to_value assert iterator.max_results == max_results # Changing attributes. assert iterator.page_number == 0 @@ -214,7 +214,7 @@ def test_constructor(self): assert not iterator._started assert iterator.client is client assert iterator.path == path - assert iterator._item_to_value is mock.sentinel.item_to_value + assert iterator.item_to_value is mock.sentinel.item_to_value assert iterator._items_key == 'items' assert iterator.max_results is None assert iterator.extra_params == {} @@ -419,10 +419,10 @@ def test_constructor(self): assert not iterator._started assert iterator.client is client assert iterator.max_results is None + assert iterator.item_to_value is page_iterator._item_to_value_identity assert iterator._method == mock.sentinel.method assert iterator._request == mock.sentinel.request assert iterator._items_field == items_field - assert iterator._item_to_value is page_iterator._item_to_value_identity assert (iterator._request_token_field == page_iterator.GRPCIterator._DEFAULT_REQUEST_TOKEN_FIELD) assert (iterator._response_token_field == @@ -446,10 +446,10 @@ def test_constructor_options(self): assert iterator.client is client assert iterator.max_results == 42 + assert iterator.item_to_value is mock.sentinel.item_to_value assert iterator._method == mock.sentinel.method assert iterator._request == mock.sentinel.request assert iterator._items_field == items_field - assert iterator._item_to_value is mock.sentinel.item_to_value assert iterator._request_token_field == request_field assert iterator._response_token_field == response_field @@ -517,7 +517,7 @@ def test_constructor(self): assert not iterator._started assert iterator.client is client - assert iterator._item_to_value is item_to_value + assert iterator.item_to_value is item_to_value assert iterator.max_results == max_results assert iterator._gax_page_iter is page_iter # Changing attributes. diff --git a/bigquery/tests/unit/test_table.py b/bigquery/tests/unit/test_table.py index 8770256236cb..faa7c3cb7a5c 100644 --- a/bigquery/tests/unit/test_table.py +++ b/bigquery/tests/unit/test_table.py @@ -941,7 +941,7 @@ def test_constructor(self): self.assertFalse(iterator._started) self.assertIs(iterator.client, client) self.assertEqual(iterator.path, path) - self.assertIs(iterator._item_to_value, _item_to_row) + self.assertIs(iterator.item_to_value, _item_to_row) self.assertEqual(iterator._items_key, 'rows') self.assertIsNone(iterator.max_results) self.assertEqual(iterator.extra_params, {}) diff --git a/datastore/google/cloud/datastore/query.py b/datastore/google/cloud/datastore/query.py index 5d968562f421..6cb0810cd6d2 100644 --- a/datastore/google/cloud/datastore/query.py +++ b/datastore/google/cloud/datastore/query.py @@ -515,7 +515,7 @@ def _next_page(self): query=query_pb, ) entity_pbs = self._process_query_results(response_pb) - return page_iterator.Page(self, entity_pbs, self._item_to_value) + return page_iterator.Page(self, entity_pbs, self.item_to_value) def _pb_from_query(query): diff --git a/datastore/tests/unit/test_query.py b/datastore/tests/unit/test_query.py index 70945895ba99..3a7aa9300a56 100644 --- a/datastore/tests/unit/test_query.py +++ b/datastore/tests/unit/test_query.py @@ -361,7 +361,6 @@ def test_constructor_defaults(self): self.assertFalse(iterator._started) self.assertIs(iterator.client, client) - self.assertIsNotNone(iterator._item_to_value) self.assertIsNone(iterator.max_results) self.assertEqual(iterator.page_number, 0) self.assertIsNone(iterator.next_page_token,) @@ -384,7 +383,6 @@ def test_constructor_explicit(self): self.assertFalse(iterator._started) self.assertIs(iterator.client, client) - self.assertIsNotNone(iterator._item_to_value) self.assertEqual(iterator.max_results, limit) self.assertEqual(iterator.page_number, 0) self.assertEqual(iterator.next_page_token, start_cursor) diff --git a/spanner/google/cloud/spanner_v1/client.py b/spanner/google/cloud/spanner_v1/client.py index 7bc2f28c92a8..b879e9e16463 100644 --- a/spanner/google/cloud/spanner_v1/client.py +++ b/spanner/google/cloud/spanner_v1/client.py @@ -205,7 +205,7 @@ def list_instance_configs(self, page_size=None, page_token=None): page_iter = self.instance_admin_api.list_instance_configs( path, page_size=page_size, metadata=metadata) page_iter.next_page_token = page_token - page_iter._item_to_value = _item_to_instance_config + page_iter.item_to_value = _item_to_instance_config return page_iter def instance(self, instance_id, @@ -265,7 +265,7 @@ def list_instances(self, filter_='', page_size=None, page_token=None): path = 'projects/%s' % (self.project,) page_iter = self.instance_admin_api.list_instances( path, page_size=page_size, metadata=metadata) - page_iter._item_to_value = self._item_to_instance + page_iter.item_to_value = self._item_to_instance page_iter.next_page_token = page_token return page_iter diff --git a/spanner/google/cloud/spanner_v1/instance.py b/spanner/google/cloud/spanner_v1/instance.py index daaacd817600..6f20704874fd 100644 --- a/spanner/google/cloud/spanner_v1/instance.py +++ b/spanner/google/cloud/spanner_v1/instance.py @@ -349,7 +349,7 @@ def list_databases(self, page_size=None, page_token=None): page_iter = self._client.database_admin_api.list_databases( self.name, page_size=page_size, metadata=metadata) page_iter.next_page_token = page_token - page_iter._item_to_value = self._item_to_database + page_iter.item_to_value = self._item_to_database return page_iter def _item_to_database(self, iterator, database_pb):