Skip to content

Commit

Permalink
use requests_toolbelt.MultipartEncoder to be able to upload large fil…
Browse files Browse the repository at this point in the history
…es (>RAM)
  • Loading branch information
grunichev committed Jan 16, 2018
1 parent c04d475 commit fdc0523
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 1 deletion.
1 change: 1 addition & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ six
oauthlib
requests
requests_oauthlib
requests-toolbelt
python-librsync
coveralls
coverage
8 changes: 7 additions & 1 deletion smartfile/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import requests
from requests.exceptions import RequestException
from requests_toolbelt import MultipartEncoder

from smartfile.errors import APIError
from smartfile.errors import RequestError
Expand Down Expand Up @@ -79,6 +80,7 @@ def _request(self, method, endpoint, id=None, **kwargs):
raise RequestError('Invalid method %s' % method)
# Find files, separate them out to correct kwarg for requests.
data = kwargs.get('data')
mpe_data = None
if data:
files = {}
for name, value in list(data.items()):
Expand All @@ -87,7 +89,9 @@ def _request(self, method, endpoint, id=None, **kwargs):
if hasattr(value, 'read') or isinstance(value, tuple):
files[name] = data.pop(name)
if files:
kwargs.setdefault('files', {}).update(files)
mpe_data = MultipartEncoder(fields=files)
kwargs['data'] = mpe_data
# kwargs.setdefault('files', {}).update(files)
path = ['api', self.version, endpoint]
# If we received an ID, append it to the path.
if id:
Expand All @@ -102,6 +106,8 @@ def _request(self, method, endpoint, id=None, **kwargs):
# Add our user agent.
kwargs.setdefault('headers', {}).setdefault('User-Agent',
HTTP_USER_AGENT)
if mpe_data:
kwargs['headers']['content-type'] = mpe_data.content_type
# Now try the request, if we get throttled, sleep and try again.
trys, retrys = 0, 3
while True:
Expand Down

0 comments on commit fdc0523

Please sign in to comment.