From 1fa2de38f0512e37fb3e61b84e55998b95f0ed30 Mon Sep 17 00:00:00 2001 From: Evgeny Demchenko Date: Wed, 19 Oct 2016 15:31:27 +0800 Subject: [PATCH 01/10] Improve style (imports and whitespace, PEP8) --- after/movie_reviews.py | 43 +++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/after/movie_reviews.py b/after/movie_reviews.py index 80e95fb..60ee2b1 100644 --- a/after/movie_reviews.py +++ b/after/movie_reviews.py @@ -11,38 +11,37 @@ -h --help Show this screen. --version Show version. """ - - +import logging +import os import requests + from docopt import docopt -from TwitterSearch import * from dateutil import parser from imdbpie import Imdb -import logging -import os +from TwitterSearch import TwitterSearch, TwitterSearchOrder, TwitterSearchException + def main(title): reviews = [] # Search tweets ts = TwitterSearch( - consumer_key = os.environ.get('TWITTER_CONSUMER_KEY'), - consumer_secret = os.environ.get('TWITTER_CONSUMER_SECRET'), - access_token = os.environ.get('TWITTER_ACCESS_TOKEN'), - access_token_secret = os.environ.get('TWITTER_TOKEN_SECRET') + consumer_key=os.environ.get('TWITTER_CONSUMER_KEY'), + consumer_secret=os.environ.get('TWITTER_CONSUMER_SECRET'), + access_token=os.environ.get('TWITTER_ACCESS_TOKEN'), + access_token_secret=os.environ.get('TWITTER_TOKEN_SECRET') ) try: ts.connect() - tso = TwitterSearchOrder() # create a TwitterSearchOrder object - tso.setKeywords(['#' + title + 'Movie']) # let's define all words we would like to have a look for - tso.setLanguage('en') # we want to see German tweets only - tso.setIncludeEntities(False) # and don't give us all those entity information + tso = TwitterSearchOrder() + tso.setKeywords(['#' + title + 'Movie']) + tso.setLanguage('en') + tso.setIncludeEntities(False) - # add tweets to reviews list results = ts.getSearchResults(tso) - except TwitterSearchException as e: # take care of all those ugly errors if there are some + except TwitterSearchException as e: logging.exception(str(e)) ts.cleanUp() else: @@ -101,18 +100,24 @@ def main(title): # Sort reviews by date reviews.sort(cmp=_cmprev) - # Print reviews + # Display reviews for review in reviews: - print('(%s) @%s: %s [Source: %s]' % ( review['date'].strftime('%Y-%m-%d'), review['author'], review['summary'], review['source'] ) ) + print('(%s) @%s: %s [Source: %s]' % ( + review['date'].strftime('%Y-%m-%d'), + review['author'], + review['summary'], + review['source'])) + def _cmprev(r1, r2): - if r1['date']>r2['date']: + if r1['date'] > r2['date']: return -1 - elif r1['date']' in arguments: From 80fc3d613a54e6702f5be4b6a6c2931e9f1312bd Mon Sep 17 00:00:00 2001 From: Evgeny Demchenko Date: Wed, 19 Oct 2016 15:39:18 +0800 Subject: [PATCH 02/10] Improve form (names) --- after/movie_reviews.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/after/movie_reviews.py b/after/movie_reviews.py index 60ee2b1..61c84dc 100644 --- a/after/movie_reviews.py +++ b/after/movie_reviews.py @@ -98,7 +98,7 @@ def main(title): count += 1 # Sort reviews by date - reviews.sort(cmp=_cmprev) + reviews.sort(cmp=_sort_by_date_desc) # Display reviews for review in reviews: @@ -109,13 +109,16 @@ def main(title): review['source'])) -def _cmprev(r1, r2): - if r1['date'] > r2['date']: +def _sort_by_date_desc(first, second): + first_date = first['date'] + second_date = second['date'] + + if first_date > second_date: return -1 - elif r1['date'] < r2['date']: + elif first_date < second_date: return 1 - else: - return 0 + + return 0 if __name__ == '__main__': From 31564964fe6de895d8a4412ae8ac2f855c98bb2e Mon Sep 17 00:00:00 2001 From: Evgeny Demchenko Date: Wed, 19 Oct 2016 15:47:22 +0800 Subject: [PATCH 03/10] Added TwitterReviews adapter --- after/backends/__init__.py | 4 ++ after/backends/twitter.py | 87 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 after/backends/__init__.py create mode 100644 after/backends/twitter.py diff --git a/after/backends/__init__.py b/after/backends/__init__.py new file mode 100644 index 0000000..03a396f --- /dev/null +++ b/after/backends/__init__.py @@ -0,0 +1,4 @@ +from .twitter import TwitterReviews + + +__all__ = ['TwitterReviews'] diff --git a/after/backends/twitter.py b/after/backends/twitter.py new file mode 100644 index 0000000..44a95db --- /dev/null +++ b/after/backends/twitter.py @@ -0,0 +1,87 @@ +import logging +import os +from dateutil import parser + +from TwitterSearch import TwitterSearch, TwitterSearchOrder, TwitterSearchException + + +class TwitterReviews(object): + def __init__(self, movie, limit=10, language='en'): + self.movie = movie + self.limit = 10 + self.language = language + self.client = TwitterSearch( + consumer_key=os.environ.get('TWITTER_CONSUMER_KEY'), + consumer_secret=os.environ.get('TWITTER_CONSUMER_SECRET'), + access_token=os.environ.get('TWITTER_ACCESS_TOKEN'), + access_token_secret=os.environ.get('TWITTER_TOKEN_SECRET') + ) + + def __enter__(self): + self.client.connect() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type == TwitterSearchException: + logging.exception(str(exc_val)) + self.client.cleanUp() + self.client.disconnect() + + @property + def reviews(self): + return Reviews(self._get_results(), limit=self.limit) + + def _prepare_request(self): + tso = TwitterSearchOrder() + tso.setKeywords(self._get_keywords()) + tso.setLanguage(self.language) + tso.setIncludeEntities(False) + return tso + + def _get_keywords(self): + return ['#' + self.movie + 'Movie'] + + def _get_results(self): + request = self._prepare_request() + return self.client.getSearchResults(request) + + +class Reviews(object): + def __init__(self, tweets, limit=10): + self.limit = limit + self.tweets = tweets + + def __len__(self): + size = self.tweets.getSize() + return size if size < self.limit else self.limit + + def __getitem__(self, item): + if item >= len(self): + raise IndexError + tweet = self.tweets.getTweetByIndex(item) + return Review(tweet) + + +class Review(object): + def __init__(self, review): + self.review = review + + @property + def author(self): + return self.review.getUserName() + + @property + def summary(self): + return self.review.getText() + + @property + def text(self): + return self.review.getText() + + @property + def date(self): + return parser.parse(self.review.getCreatedDate(), ignoretz=True) + + @property + def source(self): + return 'Twitter' From d0a40a40a9bfccfe76e61a46a9fedeee63cd7fb9 Mon Sep 17 00:00:00 2001 From: Evgeny Demchenko Date: Wed, 19 Oct 2016 15:52:43 +0800 Subject: [PATCH 04/10] Switch to TwitterReviews backend --- after/movie_reviews.py | 40 +++++----------------------------------- 1 file changed, 5 insertions(+), 35 deletions(-) diff --git a/after/movie_reviews.py b/after/movie_reviews.py index 61c84dc..ba40fd9 100644 --- a/after/movie_reviews.py +++ b/after/movie_reviews.py @@ -18,46 +18,16 @@ from docopt import docopt from dateutil import parser from imdbpie import Imdb -from TwitterSearch import TwitterSearch, TwitterSearchOrder, TwitterSearchException + +from backends import TwitterReviews def main(title): reviews = [] - # Search tweets - ts = TwitterSearch( - consumer_key=os.environ.get('TWITTER_CONSUMER_KEY'), - consumer_secret=os.environ.get('TWITTER_CONSUMER_SECRET'), - access_token=os.environ.get('TWITTER_ACCESS_TOKEN'), - access_token_secret=os.environ.get('TWITTER_TOKEN_SECRET') - ) - try: - ts.connect() - - tso = TwitterSearchOrder() - tso.setKeywords(['#' + title + 'Movie']) - tso.setLanguage('en') - tso.setIncludeEntities(False) - - results = ts.getSearchResults(tso) - - except TwitterSearchException as e: - logging.exception(str(e)) - ts.cleanUp() - else: - for offset in range(results.getSize()): - if offset > 9: - break - tweet = results.getTweetByIndex(offset) - reviews.append({ - 'author': tweet.getUserName(), - 'summary': tweet.getText(), - 'text': tweet.getText(), - 'date': parser.parse(tweet.getCreatedDate(), ignoretz=True), - 'source': 'Twitter' - }) - finally: - ts.disconnect() + with TwitterReviews(title) as reviews_backend: + for review in reviews_backend.reviews: + reviews.append(review) # Search Imdb imdb = Imdb() From aefc54ea186640a7c9115e2f3b4b6c73fe226d90 Mon Sep 17 00:00:00 2001 From: Evgeny Demchenko Date: Wed, 19 Oct 2016 16:26:15 +0800 Subject: [PATCH 05/10] Added IMDBReviews adapter --- after/backends/__init__.py | 3 +- after/backends/imdb.py | 58 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 after/backends/imdb.py diff --git a/after/backends/__init__.py b/after/backends/__init__.py index 03a396f..87921c8 100644 --- a/after/backends/__init__.py +++ b/after/backends/__init__.py @@ -1,4 +1,5 @@ from .twitter import TwitterReviews +from .imdb import IMDBReviews -__all__ = ['TwitterReviews'] +__all__ = ['TwitterReviews', 'IMDBReviews'] diff --git a/after/backends/imdb.py b/after/backends/imdb.py new file mode 100644 index 0000000..94c8df6 --- /dev/null +++ b/after/backends/imdb.py @@ -0,0 +1,58 @@ +import logging + +from dateutil import parser +from imdbpie import Imdb + + +class IMDBReviews(object): + def __init__(self, movie, limit=10, language='en'): + self.movie = movie + self.limit = limit + self.language = language + self.client = Imdb() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_val: + logging.exception(str(exc_val)) + + @property + def reviews(self): + return [Review(result) for result in self._get_results()] + + def _get_results(self): + try: + response = self.client.search_for_title(self.movie)[0] + title_id = response['imdb_id'] + response = self.client.get_title_reviews(title_id, max_results=10) + return response + except IndexError as e: + logging.exception(str(e)) + return [] + + +class Review(object): + def __init__(self, review): + self.review = review + + @property + def author(self): + return self.review.username + + @property + def summary(self): + return self.review.summary + + @property + def text(self): + return self.review.text + + @property + def date(self): + return parser.parse(self.review.date, ignoretz=True) + + @property + def source(self): + return 'IMDB' From 23888d992e77400abecc0bcfa342afc6a446f077 Mon Sep 17 00:00:00 2001 From: Evgeny Demchenko Date: Wed, 19 Oct 2016 16:27:28 +0800 Subject: [PATCH 06/10] Use IMDBReviews adapter --- after/movie_reviews.py | 29 +++++------------------------ 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/after/movie_reviews.py b/after/movie_reviews.py index ba40fd9..b9a9667 100644 --- a/after/movie_reviews.py +++ b/after/movie_reviews.py @@ -11,41 +11,22 @@ -h --help Show this screen. --version Show version. """ -import logging import os import requests from docopt import docopt from dateutil import parser -from imdbpie import Imdb -from backends import TwitterReviews +from backends import TwitterReviews, IMDBReviews def main(title): reviews = [] - with TwitterReviews(title) as reviews_backend: - for review in reviews_backend.reviews: - reviews.append(review) - - # Search Imdb - imdb = Imdb() - try: - response = imdb.search_for_title(title)[0] - title_id = response['imdb_id'] - response = imdb.get_title_reviews(title_id, max_results=10) - except IndexError as e: - logging.exception(str(e)) - else: - for review in response: - reviews.append({ - 'author': review.username, - 'summary': review.summary, - 'text': review.text, - 'date': parser.parse(review.date, ignoretz=True), - 'source': 'IMDB' - }) + for backend_class in (TwitterReviews, IMDBReviews): + with backend_class(title) as reviews_backend: + for review in reviews_backend.reviews: + reviews.append(review) # Search NYTimes url = "https://api.nytimes.com/svc/movies/v2/reviews/search.json" From 4a314cdf78ccc97efd407c437bf760a7acb0cf2a Mon Sep 17 00:00:00 2001 From: Evgeny Demchenko Date: Wed, 19 Oct 2016 16:33:46 +0800 Subject: [PATCH 07/10] Added BaseReview class --- after/backends/base.py | 30 ++++++++++++++++++++++++++++++ after/backends/imdb.py | 7 +++---- after/backends/twitter.py | 9 ++++----- 3 files changed, 37 insertions(+), 9 deletions(-) create mode 100644 after/backends/base.py diff --git a/after/backends/base.py b/after/backends/base.py new file mode 100644 index 0000000..8d808c9 --- /dev/null +++ b/after/backends/base.py @@ -0,0 +1,30 @@ +class BaseReview(object): + def __init__(self, review): + self.review = review + + @property + def author(self): + raise NotImplementedError + + @property + def summary(self): + raise NotImplementedError + + @property + def text(self): + raise NotImplementedError + + @property + def date(self): + raise NotImplementedError + + @property + def source(self): + raise NotImplementedError + + def display(self): + print '(%s) @%s: %s [Source: %s]' % ( + self.date.strftime('%Y-%m-%d'), + self.author, + self.summary, + self.source) diff --git a/after/backends/imdb.py b/after/backends/imdb.py index 94c8df6..ced075a 100644 --- a/after/backends/imdb.py +++ b/after/backends/imdb.py @@ -3,6 +3,8 @@ from dateutil import parser from imdbpie import Imdb +from .base import BaseReview + class IMDBReviews(object): def __init__(self, movie, limit=10, language='en'): @@ -33,10 +35,7 @@ def _get_results(self): return [] -class Review(object): - def __init__(self, review): - self.review = review - +class Review(BaseReview): @property def author(self): return self.review.username diff --git a/after/backends/twitter.py b/after/backends/twitter.py index 44a95db..7d75a03 100644 --- a/after/backends/twitter.py +++ b/after/backends/twitter.py @@ -1,9 +1,11 @@ import logging import os -from dateutil import parser +from dateutil import parser from TwitterSearch import TwitterSearch, TwitterSearchOrder, TwitterSearchException +from .base import BaseReview + class TwitterReviews(object): def __init__(self, movie, limit=10, language='en'): @@ -62,10 +64,7 @@ def __getitem__(self, item): return Review(tweet) -class Review(object): - def __init__(self, review): - self.review = review - +class Review(BaseReview): @property def author(self): return self.review.getUserName() From 801f26ce08f4aa82622c3aec22a5971f165576f2 Mon Sep 17 00:00:00 2001 From: Evgeny Demchenko Date: Wed, 19 Oct 2016 16:41:40 +0800 Subject: [PATCH 08/10] Added NYTimesReviews backend --- after/backends/__init__.py | 3 +- after/backends/ny_times.py | 60 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 after/backends/ny_times.py diff --git a/after/backends/__init__.py b/after/backends/__init__.py index 87921c8..05abcc9 100644 --- a/after/backends/__init__.py +++ b/after/backends/__init__.py @@ -1,5 +1,6 @@ from .twitter import TwitterReviews from .imdb import IMDBReviews +from .ny_times import NYTimesReviews -__all__ = ['TwitterReviews', 'IMDBReviews'] +__all__ = ['TwitterReviews', 'IMDBReviews', 'NYTimesReviews'] diff --git a/after/backends/ny_times.py b/after/backends/ny_times.py new file mode 100644 index 0000000..ba71799 --- /dev/null +++ b/after/backends/ny_times.py @@ -0,0 +1,60 @@ +import logging +import os + +import requests +from dateutil import parser + +from .base import BaseReview + + +class NYTimesReviews(object): + def __init__(self, movie, limit=10, language='en'): + self.movie = movie + self.limit = 10 + self.language = language + self.url = 'https://api.nytimes.com/svc/movies/v2/reviews/search.json' + self.api_key = os.environ.get('NY_TIMES_API_KEY') + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_val: + logging.exception(str(exc_val)) + + @property + def reviews(self): + return [Review(result) for result in self._get_results()] + + def _get_results(self): + response = requests.get(self.url, self._prepare_request_data()) + return response.json()['results'] + + def _prepare_request_data(self): + data = { + 'query': self.movie, + 'api-key': self.api_key + } + return data + + +class Review(BaseReview): + @property + def author(self): + return self.review['byline'] + + @property + def summary(self): + return self.review['headline'] + + @property + def text(self): + return self.review['summary_short'] + + @property + def date(self): + return parser.parse(self.review['date_updated'], ignoretz=True) + + @property + def source(self): + return 'NYTimes' From 49079b6d83f9219193b606c7f1c20f2218ba8a13 Mon Sep 17 00:00:00 2001 From: Evgeny Demchenko Date: Wed, 19 Oct 2016 16:42:45 +0800 Subject: [PATCH 09/10] Switch to NYTimesReviews adapter --- after/movie_reviews.py | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/after/movie_reviews.py b/after/movie_reviews.py index b9a9667..265bb40 100644 --- a/after/movie_reviews.py +++ b/after/movie_reviews.py @@ -17,37 +17,17 @@ from docopt import docopt from dateutil import parser -from backends import TwitterReviews, IMDBReviews +from backends import TwitterReviews, IMDBReviews, NYTimesReviews def main(title): reviews = [] - for backend_class in (TwitterReviews, IMDBReviews): + for backend_class in (TwitterReviews, IMDBReviews, NYTimesReviews): with backend_class(title) as reviews_backend: for review in reviews_backend.reviews: reviews.append(review) - # Search NYTimes - url = "https://api.nytimes.com/svc/movies/v2/reviews/search.json" - data = { - 'query': title, - 'api-key': os.environ.get('NY_TIMES_API_KEY') - } - response = requests.get(url, data) - count = 0 - for review in response.json()['results']: - if count > 9: - break - reviews.append({ - 'author': review['byline'], - 'summary': review['headline'], - 'text': review['summary_short'], - 'date': parser.parse(review['date_updated'], ignoretz=True), - 'source': 'NYTimes' - }) - count += 1 - # Sort reviews by date reviews.sort(cmp=_sort_by_date_desc) From fa75a8f06b9e239babd1b096e7c1aaa0d32c650f Mon Sep 17 00:00:00 2001 From: Evgeny Demchenko Date: Wed, 19 Oct 2016 16:46:46 +0800 Subject: [PATCH 10/10] Improve sorting and add get_reviews service --- after/movie_reviews.py | 37 +++---------------------------------- after/services.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 34 deletions(-) create mode 100644 after/services.py diff --git a/after/movie_reviews.py b/after/movie_reviews.py index 265bb40..5813d66 100644 --- a/after/movie_reviews.py +++ b/after/movie_reviews.py @@ -11,45 +11,14 @@ -h --help Show this screen. --version Show version. """ -import os -import requests - from docopt import docopt -from dateutil import parser -from backends import TwitterReviews, IMDBReviews, NYTimesReviews +from services import get_reviews def main(title): - reviews = [] - - for backend_class in (TwitterReviews, IMDBReviews, NYTimesReviews): - with backend_class(title) as reviews_backend: - for review in reviews_backend.reviews: - reviews.append(review) - - # Sort reviews by date - reviews.sort(cmp=_sort_by_date_desc) - - # Display reviews - for review in reviews: - print('(%s) @%s: %s [Source: %s]' % ( - review['date'].strftime('%Y-%m-%d'), - review['author'], - review['summary'], - review['source'])) - - -def _sort_by_date_desc(first, second): - first_date = first['date'] - second_date = second['date'] - - if first_date > second_date: - return -1 - elif first_date < second_date: - return 1 - - return 0 + for review in get_reviews(title): + review.display() if __name__ == '__main__': diff --git a/after/services.py b/after/services.py new file mode 100644 index 0000000..b3b5ce3 --- /dev/null +++ b/after/services.py @@ -0,0 +1,15 @@ +from backends import TwitterReviews, IMDBReviews, NYTimesReviews + + +def get_reviews(title): + reviews = [] + for backend_class in (TwitterReviews, IMDBReviews, NYTimesReviews): + with backend_class(title) as reviews_backend: + for review in reviews_backend.reviews: + reviews.append(review) + + return reversed(sorted(reviews, key=_get_review_date)) + + +def _get_review_date(review): + return review.date \ No newline at end of file