Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[remo] Support offset in the cache #1

Closed
wants to merge 4 commits into from
Closed
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions perceval/backends/mozilla/remo.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,8 @@ def fetch(self, offset=REMO_DEFAULT_OFFSET, category='events'):
current_offset = offset

self._purge_cache_queue()
# Add to the cache the offset so it can be used to recover from cache
self._push_cache_queue(offset)

for raw_items in self.client.get_items(category, offset):
self._push_cache_queue(raw_items)
Expand All @@ -140,6 +142,7 @@ def fetch(self, offset=REMO_DEFAULT_OFFSET, category='events'):

logger.info("Total number of events: %i (%i total, %i offset)", nitems, titems, offset)

@remo_metadata
@metadata
def fetch_from_cache(self):
"""Fetch the items from the cache.
Expand All @@ -159,12 +162,18 @@ def fetch_from_cache(self):
nitems = 0

for item in cache_items:
if type(item) is int:
# offset from a new execution results in the cache
offset = item
item = next(cache_items)
data = json.loads(item)
# The raw_data is always a list of items or an item
if 'count' in data:
# It is a list
continue
else:
data['offset'] = offset
offset += 1
yield data
nitems += 1

Expand Down
7 changes: 7 additions & 0 deletions tests/test_remo.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,8 @@ def __check_events_contents(self, items):
self.assertEqual(items[0]['updated_on'], 1339326000.0)
self.assertEqual(items[0]['category'], 'event')
self.assertEqual(items[0]['tag'], MOZILLA_REPS_SERVER_URL)
self.assertEqual(items[0]['offset'], 0)
self.assertEqual(items[3]['offset'], 3)

def __check_activities_contents(self, items):
self.assertEqual(items[0]['data']['location'], 'Bhopal, Madhya Pradesh, India')
Expand All @@ -164,6 +166,8 @@ def __check_activities_contents(self, items):
self.assertEqual(items[0]['updated_on'], 1478304000.0)
self.assertEqual(items[0]['category'], 'activity')
self.assertEqual(items[0]['tag'], MOZILLA_REPS_SERVER_URL)
self.assertEqual(items[0]['offset'], 0)
self.assertEqual(items[3]['offset'], 3)

def __check_users_contents(self, items):
self.assertEqual(items[0]['data']['city'], 'Makati City')
Expand All @@ -173,6 +177,8 @@ def __check_users_contents(self, items):
self.assertEqual(items[0]['updated_on'], 1306886400.0)
self.assertEqual(items[0]['category'], 'user')
self.assertEqual(items[0]['tag'], MOZILLA_REPS_SERVER_URL)
self.assertEqual(items[0]['offset'], 0)
self.assertEqual(items[3]['offset'], 3)

@httpretty.activate
def __test_fetch(self, category='events'):
Expand Down Expand Up @@ -295,6 +301,7 @@ def __test_fetch_from_cache(self, category):
self.assertEqual(len(cached_items), len(items))
for i in range(0,len(items)):
self.assertDictEqual(cached_items[i]['data'], items[i]['data'])
self.assertEqual(cached_items[i]['offset'], items[i]['offset'])

def test_fetch_from_cache_events(self):
self.__test_fetch_from_cache('events')
Expand Down