From 6a9d95f1ac67ea7b091a35da6ad05766c2740fb6 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 22 Jul 2015 17:36:46 +0300 Subject: [PATCH 1/3] Fix: if you change the result object, python runner wouldn't return any results --- redash/query_runner/python.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/redash/query_runner/python.py b/redash/query_runner/python.py index 5891d1ca5b..f155b8e946 100644 --- a/redash/query_runner/python.py +++ b/redash/query_runner/python.py @@ -52,14 +52,12 @@ def annotate_query(cls): return False def __init__(self, configuration_json): - global ALLOWED_MODULES - super(Python, self).__init__(configuration_json) self.syntax = "python" self._allowed_modules = {} - self._result = { "rows" : [], "columns" : [], "log" : [] } + self._script_locals = { "result" : { "rows" : [], "columns" : [], "log" : [] } } self._enable_print_log = True if self.configuration.get("allowedImportModules", None): @@ -166,7 +164,6 @@ def run_query(self, query): safe_builtins["_getiter_"] = self.custom_get_iter safe_builtins["_print_"] = CustomPrint(weakref.ref(self)) - script_locals = { "result" : self._result } restricted_globals = dict(__builtins__=safe_builtins) restricted_globals["get_query_result"] = self.get_query_result @@ -187,9 +184,9 @@ def run_query(self, query): # One option is to use ETA with Celery + timeouts on workers # And replacement of worker process every X requests handled. - exec(code) in restricted_globals, script_locals + exec(code) in restricted_globals, self._script_locals - json_data = json.dumps(self._result) + json_data = json.dumps(self._script_locals['result']) except KeyboardInterrupt: error = "Query cancelled by user." json_data = None From 21f33462d54e494a22eba340436532513ce52e72 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 22 Jul 2015 17:42:51 +0300 Subject: [PATCH 2/3] Anoter try in removing optipng from build --- Makefile | 2 +- circle.yml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 36dce3219b..b3ee4da8c0 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ deps: pack: sed -ri "s/^__version__ = '([0-9.]*)'/__version__ = '$(FULL_VERSION)'/" redash/__init__.py - tar -zcv -f $(FILENAME) --exclude=".git*" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" --exclude="rd_ui/node_modules" --exclude="rd_ui/dist/bower_components" --exclude="rd_ui/app" * + tar -zcv -f $(FILENAME) --exclude optipng* --exclude=".git*" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" --exclude="rd_ui/node_modules" --exclude="rd_ui/dist/bower_components" --exclude="rd_ui/app" * upload: python bin/release_manager.py $(CIRCLE_SHA1) $(BASE_VERSION) $(FILENAME) diff --git a/circle.yml b/circle.yml index 25e08def2a..b47040fb0a 100644 --- a/circle.yml +++ b/circle.yml @@ -10,8 +10,6 @@ dependencies: - wget http://downloads.sourceforge.net/project/optipng/OptiPNG/optipng-0.7.5/optipng-0.7.5.tar.gz - tar xvf optipng-0.7.5.tar.gz - cd optipng-0.7.5; ./configure; make; sudo checkinstall -y; - - rm -rf optipng-0.7.5 - - rm optipng-0.7.5.tar.gz - make deps - pip install -r dev_requirements.txt - pip install -r requirements.txt From 07b88d0b53189cf777c252029d361ad00c05fe9e Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 22 Jul 2015 17:56:49 +0300 Subject: [PATCH 3/3] Fix: log results were lost --- Makefile | 2 +- redash/query_runner/python.py | 35 +++++++++++++++++++---------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index b3ee4da8c0..d8c23047ff 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ deps: pack: sed -ri "s/^__version__ = '([0-9.]*)'/__version__ = '$(FULL_VERSION)'/" redash/__init__.py - tar -zcv -f $(FILENAME) --exclude optipng* --exclude=".git*" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" --exclude="rd_ui/node_modules" --exclude="rd_ui/dist/bower_components" --exclude="rd_ui/app" * + tar -zcv -f $(FILENAME) --exclude="optipng*" --exclude=".git*" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" --exclude="rd_ui/node_modules" --exclude="rd_ui/dist/bower_components" --exclude="rd_ui/app" * upload: python bin/release_manager.py $(CIRCLE_SHA1) $(BASE_VERSION) $(FILENAME) diff --git a/redash/query_runner/python.py b/redash/query_runner/python.py index f155b8e946..5832249789 100644 --- a/redash/query_runner/python.py +++ b/redash/query_runner/python.py @@ -14,23 +14,30 @@ from RestrictedPython import compile_restricted from RestrictedPython.Guards import safe_builtins + class CustomPrint(object): """ CustomPrint redirect "print" calls to be sent as "log" on the result object """ - def __init__(self, python_runner): - self._python_runner = python_runner + def __init__(self): + self.enabled = True + self.lines = [] def write(self, text): - if self._python_runner()._enable_print_log: + if self.enabled: if text and text.strip(): log_line = "[{0}] {1}".format(datetime.datetime.utcnow().isoformat(), text) - self._python_runner()._result["log"].append(log_line) + self.lines.append(log_line) + + def enable(self): + self.enabled = True + + def disable(self): + self.enabled = False def __call__(self): return self class Python(BaseQueryRunner): - @classmethod def configuration_schema(cls): return { @@ -59,6 +66,7 @@ def __init__(self, configuration_json): self._allowed_modules = {} self._script_locals = { "result" : { "rows" : [], "columns" : [], "log" : [] } } self._enable_print_log = True + self._custom_print = CustomPrint() if self.configuration.get("allowedImportModules", None): for item in self.configuration["allowedImportModules"].split(","): @@ -90,12 +98,6 @@ def custom_get_item(self, obj, key): def custom_get_iter(self, obj): return iter(obj) - def disable_print_log(self): - self._enable_print_log = False - - def enable_print_log(self): - self._enable_print_log = True - def add_result_column(self, result, column_name, friendly_name, column_type): """ Helper function to add columns inside a Python script running in re:dash in an easier way """ if column_type not in SUPPORTED_COLUMN_TYPES: @@ -162,16 +164,15 @@ def run_query(self, query): safe_builtins["setattr"] = setattr safe_builtins["_getitem_"] = self.custom_get_item safe_builtins["_getiter_"] = self.custom_get_iter - safe_builtins["_print_"] = CustomPrint(weakref.ref(self)) - + safe_builtins["_print_"] = self._custom_print restricted_globals = dict(__builtins__=safe_builtins) restricted_globals["get_query_result"] = self.get_query_result restricted_globals["execute_query"] = self.execute_query restricted_globals["add_result_column"] = self.add_result_column restricted_globals["add_result_row"] = self.add_result_row - restricted_globals["disable_print_log"] = self.disable_print_log - restricted_globals["enable_print_log"] = self.enable_print_log + restricted_globals["disable_print_log"] = self._custom_print.disable + restricted_globals["enable_print_log"] = self._custom_print.enable restricted_globals["TYPE_DATETIME"] = TYPE_DATETIME restricted_globals["TYPE_BOOLEAN"] = TYPE_BOOLEAN @@ -186,7 +187,9 @@ def run_query(self, query): exec(code) in restricted_globals, self._script_locals - json_data = json.dumps(self._script_locals['result']) + result = self._script_locals['result'] + result['log'] = self._custom_print.lines + json_data = json.dumps(result) except KeyboardInterrupt: error = "Query cancelled by user." json_data = None