From f2b0bd9047a00a085343c16b7773b912aab458e6 Mon Sep 17 00:00:00 2001
From: Ori Hoch <ori@uumpa.com>
Date: Tue, 2 Oct 2018 23:36:06 +0300
Subject: [PATCH] support modifying the resource download request

---
 ckanext/xloader/interfaces.py | 13 +++++++++++++
 ckanext/xloader/jobs.py       |  9 +++++++--
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/ckanext/xloader/interfaces.py b/ckanext/xloader/interfaces.py
index cdecf52a..d32a0b16 100644
--- a/ckanext/xloader/interfaces.py
+++ b/ckanext/xloader/interfaces.py
@@ -10,6 +10,19 @@ class IXloader(Interface):
     The before_submit function, when implemented
     """
 
+    def modify_download_request(self, url, resource, api_key, headers):
+        """ Can be used to modify the http download request.
+        The headers parameter is a dict which should be modified directly.
+
+        Return value should be the modified (or unmodified) url.
+
+        :param url: The download url
+        :param resource: The dict representation of the resource to be downloaded
+        :param api_key: The CKAN api key, in case authorization header needs to be modified
+        :param headers: The http request headers dict, to be modified directly
+        """
+        return url
+
     def can_upload(self, resource_id):
         """ This call when implemented can be used to stop the processing of
         the xloader submit function. This method will not be called if
diff --git a/ckanext/xloader/jobs.py b/ckanext/xloader/jobs.py
index 199cace1..1a44f341 100644
--- a/ckanext/xloader/jobs.py
+++ b/ckanext/xloader/jobs.py
@@ -14,6 +14,7 @@
 import sqlalchemy as sa
 
 from ckan.plugins.toolkit import get_action
+import ckan.plugins as p
 try:
     from ckan.plugins.toolkit import config
 except ImportError:
@@ -22,6 +23,7 @@
 
 import loader
 import db
+import interfaces as xloader_interfaces
 from job_exceptions import JobError, HTTPError
 
 if config.get('ckanext.xloader.ssl_verify') in ['False', 'FALSE', '0', False, 0]:
@@ -35,7 +37,6 @@
 CHUNK_SIZE = 16 * 1024  # 16kb
 DOWNLOAD_TIMEOUT = 30
 
-
 # 'api_key': user['apikey'],
 # 'job_type': 'push_to_datastore',
 # 'result_url': callback_url,
@@ -167,8 +168,12 @@ def xloader_data_into_datastore_(input, job_dict):
             headers['Authorization'] = api_key
 
         def get_url():
+            _url = url
+            for plugin in p.PluginImplementations(xloader_interfaces.IXloader):
+                if hasattr(plugin, 'modify_download_request'):
+                    _url = plugin.modify_download_request(_url, resource, api_key, headers)
             return requests.get(
-                url,
+                _url,
                 headers=headers,
                 timeout=DOWNLOAD_TIMEOUT,
                 verify=SSL_VERIFY,