From fa19b1ddc87295283cafc1aab536c5d44e03ed1c Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 23 Mar 2015 12:37:14 +0200 Subject: [PATCH 01/12] Endpoint to return data source schema --- redash/controllers.py | 13 +++++++++++-- redash/query_runner/__init__.py | 3 +++ redash/query_runner/pg.py | 25 +++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/redash/controllers.py b/redash/controllers.py index 1f57fe8ad9..e377f95830 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -22,7 +22,7 @@ from redash.tasks import QueryTask, record_event from redash.cache import headers as cache_headers from redash.permissions import require_permission -from redash.query_runner import query_runners, validate_configuration +from redash.query_runner import query_runners, validate_configuration, get_query_runner @app.route('/ping', methods=['GET']) @@ -219,10 +219,19 @@ def post(self): return datasource.to_dict() - api.add_resource(DataSourceListAPI, '/api/data_sources', endpoint='data_sources') +class DataSourceSchemaAPI(BaseResource): + def get(self, data_source_id): + data_source = models.DataSource.get_by_id(data_source_id) + query_runner = get_query_runner(data_source.type, data_source.options) + schema = query_runner.get_schema() + + return schema + +api.add_resource(DataSourceSchemaAPI, '/api/data_sources//schema') + class DashboardRecentAPI(BaseResource): def get(self): return [d.to_dict() for d in models.Dashboard.recent(current_user.id).limit(20)] diff --git a/redash/query_runner/__init__.py b/redash/query_runner/__init__.py index eaf5693a57..2408119f08 100644 --- a/redash/query_runner/__init__.py +++ b/redash/query_runner/__init__.py @@ -57,6 +57,9 @@ def configuration_schema(cls): def run_query(self, query): raise NotImplementedError() + def get_schema(self): + return {} + @classmethod def to_dict(cls): return { diff --git a/redash/query_runner/pg.py b/redash/query_runner/pg.py index 811d52a2dc..4f2c57e898 100644 --- a/redash/query_runner/pg.py +++ b/redash/query_runner/pg.py @@ -1,3 +1,4 @@ +from collections import defaultdict import json import logging import psycopg2 @@ -83,6 +84,30 @@ def __init__(self, configuration_json): self.connection_string = " ".join(values) + def get_schema(self): + query = """ + SELECT table_schema, table_name, column_name, data_type + FROM information_schema.columns + WHERE table_schema NOT IN ('pg_catalog', 'information_schema'); + """ + + results, error = self.run_query(query) + + if error is not None: + raise Exception("Failed getting schema.") + + results = json.loads(results) + + schema = defaultdict(list) + for row in results['rows']: + if row['table_schema'] != 'public': + table_name = '{}.{}'.format(row['table_schema'], row['table_name']) + else: + table_name = row['table_name'] + schema[table_name].append({'name': row['column_name'], 'type': row['data_type']}) + + return schema + def run_query(self, query): connection = psycopg2.connect(self.connection_string, async=True) _wait(connection) From d487ec915320cf94cc699974f73ae2e452e325d8 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 23 Mar 2015 14:17:03 +0200 Subject: [PATCH 02/12] Upgrade codemirror to latest version --- rd_ui/app/scripts/directives/query_directives.js | 16 +++++++++------- rd_ui/app/views/query.html | 2 +- rd_ui/bower.json | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/rd_ui/app/scripts/directives/query_directives.js b/rd_ui/app/scripts/directives/query_directives.js index 3554ca04f3..1be8aab572 100644 --- a/rd_ui/app/scripts/directives/query_directives.js +++ b/rd_ui/app/scripts/directives/query_directives.js @@ -29,7 +29,7 @@ restrict: 'E', template: '\ Show Source\ + ng-href="/queries/{{query.id}}/source#{{selectedTab}}">Show Source\ \ Hide Source\ @@ -68,19 +68,21 @@ template: '', - link: function($scope) { - $scope.editorOptions = { + link: { + pre: function ($scope) { + $scope.editorOptions = { mode: 'text/x-sql', lineWrapping: true, lineNumbers: true, readOnly: false, matchBrackets: true, autoCloseBrackets: true - }; + }; - $scope.$watch('lock', function(locked) { - $scope.editorOptions.readOnly = locked ? 'nocursor' : false; - }); + $scope.$watch('lock', function (locked) { + $scope.editorOptions.readOnly = locked ? 'nocursor' : false; + }); + }, } } } diff --git a/rd_ui/app/views/query.html b/rd_ui/app/views/query.html index 0c0775195e..02cc915c62 100644 --- a/rd_ui/app/views/query.html +++ b/rd_ui/app/views/query.html @@ -80,7 +80,7 @@

-
+

diff --git a/rd_ui/bower.json b/rd_ui/bower.json index d26c5fe60a..5be77330fa 100644 --- a/rd_ui/bower.json +++ b/rd_ui/bower.json @@ -12,7 +12,7 @@ "es5-shim": "2.0.8", "angular-moment": "0.2.0", "moment": "2.1.0", - "angular-ui-codemirror": "0.0.5", + "angular-ui-codemirror": "0.2.3", "highcharts": "3.0.10", "underscore": "1.5.1", "pivottable": "~1.1.1", From c5b7fe53215528db6f919224c775b656b03192cf Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 23 Mar 2015 16:08:11 +0200 Subject: [PATCH 03/12] Use codemirror directly without ui-codemirror --- rd_ui/app/index.html | 4 +- rd_ui/app/scripts/app.js | 1 - .../scripts/directives/query_directives.js | 39 +++++++++++++++---- rd_ui/bower.json | 2 +- 4 files changed, 35 insertions(+), 11 deletions(-) diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index 0ac1007ce1..3145c4c2c8 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -18,6 +18,7 @@ + @@ -105,9 +106,10 @@ + + - diff --git a/rd_ui/app/scripts/app.js b/rd_ui/app/scripts/app.js index 11f2ab5a35..63a63c8bac 100644 --- a/rd_ui/app/scripts/app.js +++ b/rd_ui/app/scripts/app.js @@ -6,7 +6,6 @@ angular.module('redash', [ 'redash.services', 'redash.renderers', 'redash.visualization', - 'ui.codemirror', 'highchart', 'ui.select2', 'angular-growl', diff --git a/rd_ui/app/scripts/directives/query_directives.js b/rd_ui/app/scripts/directives/query_directives.js index 1be8aab572..b8e37369a9 100644 --- a/rd_ui/app/scripts/directives/query_directives.js +++ b/rd_ui/app/scripts/directives/query_directives.js @@ -65,26 +65,49 @@ 'query': '=', 'lock': '=' }, - template: '', + template: '', link: { - pre: function ($scope) { + pre: function ($scope, element) { + var textarea = element.children()[0]; $scope.editorOptions = { mode: 'text/x-sql', lineWrapping: true, lineNumbers: true, readOnly: false, matchBrackets: true, - autoCloseBrackets: true + autoCloseBrackets: true, + extraKeys: {"Ctrl-Space": "autocomplete"} }; + CodeMirror.commands.autocomplete = function(cm) { + CodeMirror.showHint(cm, CodeMirror.hint.anyword); + }; + + var codemirror = CodeMirror.fromTextArea(textarea, $scope.editorOptions); + + codemirror.on('change', function(instance) { + var newValue = instance.getValue(); + + if (newValue !== $scope.query.query) { + $scope.$evalAsync(function() { + $scope.query.query = newValue; + }); + } + }); + + $scope.$watch('query.query', function (newValue, oldValue) { + if ($scope.query.query !== codemirror.getValue()) { + codemirror.setValue($scope.query.query); + } + }); + $scope.$watch('lock', function (locked) { - $scope.editorOptions.readOnly = locked ? 'nocursor' : false; + var readOnly = locked ? 'nocursor' : false; + codemirror.setOption('readOnly', readOnly); }); - }, + } } - } + }; } function queryFormatter($http) { diff --git a/rd_ui/bower.json b/rd_ui/bower.json index 5be77330fa..97d6dde597 100644 --- a/rd_ui/bower.json +++ b/rd_ui/bower.json @@ -12,7 +12,7 @@ "es5-shim": "2.0.8", "angular-moment": "0.2.0", "moment": "2.1.0", - "angular-ui-codemirror": "0.2.3", + "codemirror": "4.8.0", "highcharts": "3.0.10", "underscore": "1.5.1", "pivottable": "~1.1.1", From a54119f4a2d71262ad3b4945cbc503f1a54c3bce Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 23 Mar 2015 19:15:45 +0200 Subject: [PATCH 04/12] Show schema along side the query --- rd_ui/app/scripts/controllers/query_view.js | 18 ++++++++++++++ .../scripts/directives/query_directives.js | 15 ++++++++---- rd_ui/app/scripts/services/resources.js | 7 +++++- rd_ui/app/views/query.html | 24 ++++++++++++------- redash/query_runner/__init__.py | 2 +- redash/query_runner/pg.py | 12 ++++++---- 6 files changed, 60 insertions(+), 18 deletions(-) diff --git a/rd_ui/app/scripts/controllers/query_view.js b/rd_ui/app/scripts/controllers/query_view.js index e158767c33..1bd7ac871e 100644 --- a/rd_ui/app/scripts/controllers/query_view.js +++ b/rd_ui/app/scripts/controllers/query_view.js @@ -19,6 +19,22 @@ } $scope.query = $route.current.locals.query; + + var updateSchema = function() { + $scope.hasSchema = false; + $scope.editorSize = "col-md-12"; + var dataSourceId = $scope.query.data_source_id || $scope.dataSources[0].id; + $scope.schema = DataSource.getSchema({id: dataSourceId}, function(data) { + if (data && data.length > 0) { + $scope.editorSize = "col-md-9"; + $scope.hasSchema = true; + } else { + $scope.hasSchema = false; + $scope.editorSize = "col-md-12"; + } + }); + } + Events.record(currentUser, 'view', 'query', $scope.query.id); getQueryResult(); $scope.queryExecuting = false; @@ -27,6 +43,7 @@ $scope.canViewSource = currentUser.hasPermission('view_source'); $scope.dataSources = DataSource.get(function(dataSources) { + updateSchema(); $scope.query.data_source_id = $scope.query.data_source_id || dataSources[0].id; }); @@ -126,6 +143,7 @@ }); } + updateSchema(); $scope.executeQuery(); }; diff --git a/rd_ui/app/scripts/directives/query_directives.js b/rd_ui/app/scripts/directives/query_directives.js index b8e37369a9..30d4614958 100644 --- a/rd_ui/app/scripts/directives/query_directives.js +++ b/rd_ui/app/scripts/directives/query_directives.js @@ -69,7 +69,7 @@ link: { pre: function ($scope, element) { var textarea = element.children()[0]; - $scope.editorOptions = { + var editorOptions = { mode: 'text/x-sql', lineWrapping: true, lineNumbers: true, @@ -80,10 +80,17 @@ }; CodeMirror.commands.autocomplete = function(cm) { - CodeMirror.showHint(cm, CodeMirror.hint.anyword); + var hinter = function(editor, options) { + var hints = CodeMirror.hint.anyword(editor, options); +// hints.list.push('select', 'from', 'where'); + return hints; + }; + +// CodeMirror.showHint(cm, CodeMirror.hint.anyword); + CodeMirror.showHint(cm, hinter); }; - var codemirror = CodeMirror.fromTextArea(textarea, $scope.editorOptions); + var codemirror = CodeMirror.fromTextArea(textarea, editorOptions); codemirror.on('change', function(instance) { var newValue = instance.getValue(); @@ -95,7 +102,7 @@ } }); - $scope.$watch('query.query', function (newValue, oldValue) { + $scope.$watch('query.query', function () { if ($scope.query.query !== codemirror.getValue()) { codemirror.setValue($scope.query.query); } diff --git a/rd_ui/app/scripts/services/resources.js b/rd_ui/app/scripts/services/resources.js index d6afa1a33d..5e15a8919d 100644 --- a/rd_ui/app/scripts/services/resources.js +++ b/rd_ui/app/scripts/services/resources.js @@ -480,7 +480,12 @@ var DataSource = function ($resource) { - var DataSourceResource = $resource('/api/data_sources/:id', {id: '@id'}, {'get': {'method': 'GET', 'cache': true, 'isArray': true}}); + var actions = { + 'get': {'method': 'GET', 'cache': true, 'isArray': true}, + 'getSchema': {'method': 'GET', 'cache': true, 'isArray': true, 'url': '/api/data_sources/:id/schema'} + }; + + var DataSourceResource = $resource('/api/data_sources/:id', {id: '@id'}, actions); return DataSourceResource; } diff --git a/rd_ui/app/views/query.html b/rd_ui/app/views/query.html index 02cc915c62..eb9d70258f 100644 --- a/rd_ui/app/views/query.html +++ b/rd_ui/app/views/query.html @@ -59,9 +59,9 @@

-
-
-
+
+
+

-
- -

-
- +
+
+ +
+
+
+ {{obj}} +
{{table.name}}
+
{{column}}
+
+
+
+
diff --git a/redash/query_runner/__init__.py b/redash/query_runner/__init__.py index 2408119f08..dde3b769e9 100644 --- a/redash/query_runner/__init__.py +++ b/redash/query_runner/__init__.py @@ -58,7 +58,7 @@ def run_query(self, query): raise NotImplementedError() def get_schema(self): - return {} + return [] @classmethod def to_dict(cls): diff --git a/redash/query_runner/pg.py b/redash/query_runner/pg.py index 4f2c57e898..da4059b658 100644 --- a/redash/query_runner/pg.py +++ b/redash/query_runner/pg.py @@ -86,7 +86,7 @@ def __init__(self, configuration_json): def get_schema(self): query = """ - SELECT table_schema, table_name, column_name, data_type + SELECT table_schema, table_name, column_name FROM information_schema.columns WHERE table_schema NOT IN ('pg_catalog', 'information_schema'); """ @@ -98,15 +98,19 @@ def get_schema(self): results = json.loads(results) - schema = defaultdict(list) + schema = {} for row in results['rows']: if row['table_schema'] != 'public': table_name = '{}.{}'.format(row['table_schema'], row['table_name']) else: table_name = row['table_name'] - schema[table_name].append({'name': row['column_name'], 'type': row['data_type']}) - return schema + if table_name not in schema: + schema[table_name] = {'name': table_name, 'columns': []} + + schema[table_name]['columns'].append(row['column_name']) + + return schema.values() def run_query(self, query): connection = psycopg2.connect(self.connection_string, async=True) From 1fe4f291f21eccc655e1efa6edeaf59dffc02ec6 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 30 Mar 2015 11:05:20 +0300 Subject: [PATCH 05/12] Flush test redis db after each test --- tests/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index 786bce8068..56abba3070 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -7,7 +7,9 @@ 'threadlocals': True } -from redash import models +settings.REDIS_URL = "redis://localhost:6379/5" + +from redash import models, redis_connection logging.getLogger('peewee').setLevel(logging.INFO) @@ -20,6 +22,7 @@ def setUp(self): def tearDown(self): models.db.close_db(None) models.create_db(False, True) + redis_connection.flushdb() def assertResponseEqual(self, expected, actual): for k, v in expected.iteritems(): From e3cc3ef9a41d470f8049cc0451da4b9ac77e4696 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 30 Mar 2015 11:06:15 +0300 Subject: [PATCH 06/12] Move schema fetching to DataSource + tests --- redash/cache.py | 3 --- redash/controllers.py | 5 ++--- redash/models.py | 20 +++++++++++++++++++- redash/query_runner/pg.py | 1 - redash/tasks.py | 11 +++++++++++ redash/worker.py | 4 ++++ tests/factories.py | 2 +- tests/test_models.py | 39 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 76 insertions(+), 9 deletions(-) diff --git a/redash/cache.py b/redash/cache.py index 56b8174b92..b853cc5a00 100644 --- a/redash/cache.py +++ b/redash/cache.py @@ -1,6 +1,3 @@ -from flask import make_response -from functools import update_wrapper - ONE_YEAR = 60 * 60 * 24 * 365.25 headers = { diff --git a/redash/controllers.py b/redash/controllers.py index e377f95830..ba14986619 100644 --- a/redash/controllers.py +++ b/redash/controllers.py @@ -22,7 +22,7 @@ from redash.tasks import QueryTask, record_event from redash.cache import headers as cache_headers from redash.permissions import require_permission -from redash.query_runner import query_runners, validate_configuration, get_query_runner +from redash.query_runner import query_runners, validate_configuration @app.route('/ping', methods=['GET']) @@ -225,8 +225,7 @@ def post(self): class DataSourceSchemaAPI(BaseResource): def get(self, data_source_id): data_source = models.DataSource.get_by_id(data_source_id) - query_runner = get_query_runner(data_source.type, data_source.options) - schema = query_runner.get_schema() + schema = data_source.get_schema() return schema diff --git a/redash/models.py b/redash/models.py index 9e2602c6bf..467ea7e216 100644 --- a/redash/models.py +++ b/redash/models.py @@ -13,7 +13,8 @@ from flask.ext.login import UserMixin, AnonymousUserMixin import psycopg2 -from redash import utils, settings +from redash import utils, settings, redis_connection +from redash.query_runner import get_query_runner class Database(object): @@ -241,6 +242,23 @@ def to_dict(self): 'type': self.type } + def get_schema(self, refresh=False): + key = "data_source:schema:{}".format(self.id) + + cache = None + if not refresh: + cache = redis_connection.get(key) + + if cache is None: + query_runner = get_query_runner(self.type, self.options) + schema = query_runner.get_schema() + + redis_connection.set(key, json.dumps(schema)) + else: + schema = json.loads(cache) + + return schema + @classmethod def all(cls): return cls.select().order_by(cls.id.asc()) diff --git a/redash/query_runner/pg.py b/redash/query_runner/pg.py index da4059b658..1393773100 100644 --- a/redash/query_runner/pg.py +++ b/redash/query_runner/pg.py @@ -1,4 +1,3 @@ -from collections import defaultdict import json import logging import psycopg2 diff --git a/redash/tasks.py b/redash/tasks.py index f224b635a9..cbca8d8350 100644 --- a/redash/tasks.py +++ b/redash/tasks.py @@ -218,6 +218,17 @@ def cleanup_query_results(): logger.info("Deleted %d unused query results out of total of %d." % (deleted_count, total_unused_query_results)) +@celery.task(base=BaseTask) +def refresh_schemas(): + """ + Refershs the datasources schema. + """ + + for ds in models.DataSource.all(): + logger.info("Refreshing schema for: {}".format(ds.name)) + ds.get_schema(refresh=True) + + @celery.task(bind=True, base=BaseTask, track_started=True) def execute_query(self, query, data_source_id): # TODO: maybe this should be a class? diff --git a/redash/worker.py b/redash/worker.py index e9ea0098ea..f52f5b851d 100644 --- a/redash/worker.py +++ b/redash/worker.py @@ -15,6 +15,10 @@ 'cleanup_tasks': { 'task': 'redash.tasks.cleanup_tasks', 'schedule': timedelta(minutes=5) + }, + 'refresh_schemas': { + 'task': 'redash.tasks.refresh_schemas', + 'schedule': timedelta(minutes=30) } } diff --git a/tests/factories.py b/tests/factories.py index ba2d59d4eb..879ac77a73 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -47,7 +47,7 @@ def __call__(self): data_source_factory = ModelFactory(redash.models.DataSource, name='Test', type='pg', - options='') + options='{"dbname": "test"}') dashboard_factory = ModelFactory(redash.models.Dashboard, diff --git a/tests/test_models.py b/tests/test_models.py index ea4e9f3887..5002888520 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -2,10 +2,12 @@ import datetime import json from unittest import TestCase +import mock from tests import BaseTestCase from redash import models from factories import dashboard_factory, query_factory, data_source_factory, query_result_factory, user_factory, widget_factory from redash.utils import gen_query_hash +from redash import query_runner class DashboardTest(BaseTestCase): @@ -195,6 +197,43 @@ def test_removes_scheduling(self): self.assertEqual(None, query.schedule) +class DataSourceTest(BaseTestCase): + def test_get_schema(self): + return_value = "{}" + with mock.patch('redash.query_runner.pg.PostgreSQL.get_schema') as patched_get_schema: + patched_get_schema.return_value = return_value + + ds = data_source_factory.create() + schema = ds.get_schema() + + self.assertEqual(return_value, schema) + + def test_get_schema_uses_cache(self): + return_value = "{}" + with mock.patch('redash.query_runner.pg.PostgreSQL.get_schema') as patched_get_schema: + patched_get_schema.return_value = return_value + + ds = data_source_factory.create() + ds.get_schema() + schema = ds.get_schema() + + self.assertEqual(return_value, schema) + self.assertEqual(patched_get_schema.call_count, 1) + + def test_get_schema_skips_cache_with_refresh_true(self): + return_value = "{}" + with mock.patch('redash.query_runner.pg.PostgreSQL.get_schema') as patched_get_schema: + patched_get_schema.return_value = return_value + + ds = data_source_factory.create() + ds.get_schema() + new_return_value = '{"new":true}' + patched_get_schema.return_value = new_return_value + schema = ds.get_schema(refresh=True) + + self.assertEqual(new_return_value, schema) + self.assertEqual(patched_get_schema.call_count, 2) + class QueryResultTest(BaseTestCase): def setUp(self): From 6ff6bdad9f7544be103e798838c12509411a2098 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Mon, 30 Mar 2015 11:08:57 +0300 Subject: [PATCH 07/12] Use the correct redis connection in tests --- tests/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 56abba3070..64a35d67d6 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,14 +1,16 @@ +import os +os.environ['REDASH_REDIS_URL'] = "redis://localhost:6379/5" + import logging from unittest import TestCase import datetime from redash import settings + settings.DATABASE_CONFIG = { 'name': 'circle_test', 'threadlocals': True } -settings.REDIS_URL = "redis://localhost:6379/5" - from redash import models, redis_connection logging.getLogger('peewee').setLevel(logging.INFO) From cb29d87b6376938a76774e2dc01ff5ebcf2e9cd9 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 2 Apr 2015 15:40:43 +0300 Subject: [PATCH 08/12] Improve formatting of schema browser --- rd_ui/app/scripts/controllers/query_view.js | 7 ++++++- rd_ui/app/styles/redash.css | 14 +++++++++++++- rd_ui/app/views/query.html | 20 ++++++++++++-------- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/rd_ui/app/scripts/controllers/query_view.js b/rd_ui/app/scripts/controllers/query_view.js index 1bd7ac871e..4eb048a5b0 100644 --- a/rd_ui/app/scripts/controllers/query_view.js +++ b/rd_ui/app/scripts/controllers/query_view.js @@ -24,8 +24,13 @@ $scope.hasSchema = false; $scope.editorSize = "col-md-12"; var dataSourceId = $scope.query.data_source_id || $scope.dataSources[0].id; - $scope.schema = DataSource.getSchema({id: dataSourceId}, function(data) { + DataSource.getSchema({id: dataSourceId}, function(data) { if (data && data.length > 0) { + $scope.schema = data; + _.each(data, function(table) { + table.collapsed = true; + }); + $scope.editorSize = "col-md-9"; $scope.hasSchema = true; } else { diff --git a/rd_ui/app/styles/redash.css b/rd_ui/app/styles/redash.css index 82efd820e4..14ccf57ed7 100644 --- a/rd_ui/app/styles/redash.css +++ b/rd_ui/app/styles/redash.css @@ -156,7 +156,7 @@ li.widget:hover { /* CodeMirror */ .CodeMirror { border: 1px solid #eee; - height: auto; + /*height: auto;*/ min-height: 300px; margin-bottom: 10px; } @@ -308,6 +308,18 @@ counter-renderer counter-name { height: 100%; } +.schema-browser { + height: 300px; + overflow: scroll; +} + +div.table-name { + overflow: scroll; + text-overflow: ellipsis; + white-space: nowrap; + cursor: pointer; +} + /* bootstrap's hidden-xs class adds display:block when not hidden use this class when you need to keep the original display value diff --git a/rd_ui/app/views/query.html b/rd_ui/app/views/query.html index eb9d70258f..b85a9675c2 100644 --- a/rd_ui/app/views/query.html +++ b/rd_ui/app/views/query.html @@ -87,17 +87,21 @@

-
+
- {{obj}} -
{{table.name}}
-
{{column}}
+
+ {{table.name}} +
+
+
{{column}}
+
-
-

+ +
+

@@ -105,7 +109,7 @@

Created By {{query.user.name}}

-

+

Last Modified By {{query.last_modified_by.name}} @@ -198,7 +202,7 @@

Query Archive

× - +
    • From 5ab3d4a40d60baa1ffd5d4438d4889a8673167c0 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 2 Apr 2015 16:12:33 +0300 Subject: [PATCH 09/12] Basic autocomplete functionality --- .../scripts/directives/query_directives.js | 26 +++++++++++++++++-- rd_ui/app/views/query.html | 2 +- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/rd_ui/app/scripts/directives/query_directives.js b/rd_ui/app/scripts/directives/query_directives.js index 30d4614958..07c3a47848 100644 --- a/rd_ui/app/scripts/directives/query_directives.js +++ b/rd_ui/app/scripts/directives/query_directives.js @@ -63,7 +63,8 @@ restrict: 'E', scope: { 'query': '=', - 'lock': '=' + 'lock': '=', + 'schema': '=' }, template: '', link: { @@ -79,10 +80,17 @@ extraKeys: {"Ctrl-Space": "autocomplete"} }; + var additionalHints = []; + CodeMirror.commands.autocomplete = function(cm) { var hinter = function(editor, options) { var hints = CodeMirror.hint.anyword(editor, options); -// hints.list.push('select', 'from', 'where'); + var cur = editor.getCursor(), token = editor.getTokenAt(cur).string; + + hints.list = _.union(hints.list, _.filter(additionalHints, function (h) { + return h.search(token) === 0; + })); + return hints; }; @@ -108,6 +116,20 @@ } }); + $scope.$watch('schema', function (schema) { + if (schema) { + var keywords = []; + _.each(schema, function (table) { + keywords.push(table.name); + _.each(table.columns, function (c) { + keywords.push(c); + }); + }); + + additionalHints = _.unique(keywords); + } + }); + $scope.$watch('lock', function (locked) { var readOnly = locked ? 'nocursor' : false; codemirror.setOption('readOnly', readOnly); diff --git a/rd_ui/app/views/query.html b/rd_ui/app/views/query.html index b85a9675c2..6029c32c16 100644 --- a/rd_ui/app/views/query.html +++ b/rd_ui/app/views/query.html @@ -79,7 +79,7 @@

    - +

    From edc1622cf5687295aacb694f5b231c62c745c9f1 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 2 Apr 2015 16:55:52 +0300 Subject: [PATCH 10/12] Schema support for MySQL --- redash/query_runner/mysql.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/redash/query_runner/mysql.py b/redash/query_runner/mysql.py index 8a89553882..1623bef945 100644 --- a/redash/query_runner/mysql.py +++ b/redash/query_runner/mysql.py @@ -44,6 +44,41 @@ def enabled(cls): def __init__(self, configuration_json): super(Mysql, self).__init__(configuration_json) + def get_schema(self): + query = """ + SELECT col.table_schema, + col.table_name, + col.column_name + FROM `information_schema`.`columns` col + INNER JOIN + (SELECT table_schema, + TABLE_NAME + FROM information_schema.tables + WHERE table_type <> 'SYSTEM VIEW' AND table_schema NOT IN ('performance_schema', 'mysql')) tables ON tables.table_schema = col.table_schema + AND tables.TABLE_NAME = col.TABLE_NAME; + """ + + results, error = self.run_query(query) + + if error is not None: + raise Exception("Failed getting schema.") + + results = json.loads(results) + + schema = {} + for row in results['rows']: + if row['table_schema'] != self.configuration['db']: + table_name = '{}.{}'.format(row['table_schema'], row['table_name']) + else: + table_name = row['table_name'] + + if table_name not in schema: + schema[table_name] = {'name': table_name, 'columns': []} + + schema[table_name]['columns'].append(row['column_name']) + + return schema.values() + def run_query(self, query): import MySQLdb From e675690cc66082b2ac4261a5b24fd451cc0f6373 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 2 Apr 2015 16:56:00 +0300 Subject: [PATCH 11/12] Sort schema by name --- redash/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redash/models.py b/redash/models.py index 467ea7e216..e897506e39 100644 --- a/redash/models.py +++ b/redash/models.py @@ -251,7 +251,7 @@ def get_schema(self, refresh=False): if cache is None: query_runner = get_query_runner(self.type, self.options) - schema = query_runner.get_schema() + schema = sorted(query_runner.get_schema(), key=lambda t: t['name']) redis_connection.set(key, json.dumps(schema)) else: From e3c5da5bc52f34b070f6c498f67726393c617abb Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Thu, 2 Apr 2015 17:05:16 +0300 Subject: [PATCH 12/12] Fix tests to use correct data --- tests/test_models.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_models.py b/tests/test_models.py index 5002888520..b9b2076ad1 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -199,7 +199,8 @@ def test_removes_scheduling(self): class DataSourceTest(BaseTestCase): def test_get_schema(self): - return_value = "{}" + return_value = [{'name': 'table', 'columns': []}] + with mock.patch('redash.query_runner.pg.PostgreSQL.get_schema') as patched_get_schema: patched_get_schema.return_value = return_value @@ -209,7 +210,7 @@ def test_get_schema(self): self.assertEqual(return_value, schema) def test_get_schema_uses_cache(self): - return_value = "{}" + return_value = [{'name': 'table', 'columns': []}] with mock.patch('redash.query_runner.pg.PostgreSQL.get_schema') as patched_get_schema: patched_get_schema.return_value = return_value @@ -221,13 +222,13 @@ def test_get_schema_uses_cache(self): self.assertEqual(patched_get_schema.call_count, 1) def test_get_schema_skips_cache_with_refresh_true(self): - return_value = "{}" + return_value = [{'name': 'table', 'columns': []}] with mock.patch('redash.query_runner.pg.PostgreSQL.get_schema') as patched_get_schema: patched_get_schema.return_value = return_value ds = data_source_factory.create() ds.get_schema() - new_return_value = '{"new":true}' + new_return_value = [{'name': 'new_table', 'columns': []}] patched_get_schema.return_value = new_return_value schema = ds.get_schema(refresh=True)