From fdc0523f67825958b70b6e12b55f52d59bc1598c Mon Sep 17 00:00:00 2001 From: Alexey Grunichev Date: Tue, 16 Jan 2018 13:30:52 +0300 Subject: [PATCH] use requests_toolbelt.MultipartEncoder to be able to upload large files (>RAM) --- requirements.txt | 1 + smartfile/__init__.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3faf34c..7b5aa22 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ six oauthlib requests requests_oauthlib +requests-toolbelt python-librsync coveralls coverage diff --git a/smartfile/__init__.py b/smartfile/__init__.py index 958e9fa..dbd7291 100644 --- a/smartfile/__init__.py +++ b/smartfile/__init__.py @@ -14,6 +14,7 @@ import requests from requests.exceptions import RequestException +from requests_toolbelt import MultipartEncoder from smartfile.errors import APIError from smartfile.errors import RequestError @@ -79,6 +80,7 @@ def _request(self, method, endpoint, id=None, **kwargs): raise RequestError('Invalid method %s' % method) # Find files, separate them out to correct kwarg for requests. data = kwargs.get('data') + mpe_data = None if data: files = {} for name, value in list(data.items()): @@ -87,7 +89,9 @@ def _request(self, method, endpoint, id=None, **kwargs): if hasattr(value, 'read') or isinstance(value, tuple): files[name] = data.pop(name) if files: - kwargs.setdefault('files', {}).update(files) + mpe_data = MultipartEncoder(fields=files) + kwargs['data'] = mpe_data + # kwargs.setdefault('files', {}).update(files) path = ['api', self.version, endpoint] # If we received an ID, append it to the path. if id: @@ -102,6 +106,8 @@ def _request(self, method, endpoint, id=None, **kwargs): # Add our user agent. kwargs.setdefault('headers', {}).setdefault('User-Agent', HTTP_USER_AGENT) + if mpe_data: + kwargs['headers']['content-type'] = mpe_data.content_type # Now try the request, if we get throttled, sleep and try again. trys, retrys = 0, 3 while True: