@@ -97,7 +109,7 @@
Created By
{{query.user.name}}
-
+
Last Modified By
{{query.last_modified_by.name}}
@@ -190,7 +202,7 @@
Query Archive
×
-
+
diff --git a/rd_ui/bower.json b/rd_ui/bower.json
index d26c5fe60a..97d6dde597 100644
--- a/rd_ui/bower.json
+++ b/rd_ui/bower.json
@@ -12,7 +12,7 @@
"es5-shim": "2.0.8",
"angular-moment": "0.2.0",
"moment": "2.1.0",
- "angular-ui-codemirror": "0.0.5",
+ "codemirror": "4.8.0",
"highcharts": "3.0.10",
"underscore": "1.5.1",
"pivottable": "~1.1.1",
diff --git a/redash/cache.py b/redash/cache.py
index 56b8174b92..b853cc5a00 100644
--- a/redash/cache.py
+++ b/redash/cache.py
@@ -1,6 +1,3 @@
-from flask import make_response
-from functools import update_wrapper
-
ONE_YEAR = 60 * 60 * 24 * 365.25
headers = {
diff --git a/redash/controllers.py b/redash/controllers.py
index 1f57fe8ad9..ba14986619 100644
--- a/redash/controllers.py
+++ b/redash/controllers.py
@@ -219,10 +219,18 @@ def post(self):
return datasource.to_dict()
-
api.add_resource(DataSourceListAPI, '/api/data_sources', endpoint='data_sources')
+class DataSourceSchemaAPI(BaseResource):
+ def get(self, data_source_id):
+ data_source = models.DataSource.get_by_id(data_source_id)
+ schema = data_source.get_schema()
+
+ return schema
+
+api.add_resource(DataSourceSchemaAPI, '/api/data_sources/
/schema')
+
class DashboardRecentAPI(BaseResource):
def get(self):
return [d.to_dict() for d in models.Dashboard.recent(current_user.id).limit(20)]
diff --git a/redash/models.py b/redash/models.py
index 9e2602c6bf..e897506e39 100644
--- a/redash/models.py
+++ b/redash/models.py
@@ -13,7 +13,8 @@
from flask.ext.login import UserMixin, AnonymousUserMixin
import psycopg2
-from redash import utils, settings
+from redash import utils, settings, redis_connection
+from redash.query_runner import get_query_runner
class Database(object):
@@ -241,6 +242,23 @@ def to_dict(self):
'type': self.type
}
+ def get_schema(self, refresh=False):
+ key = "data_source:schema:{}".format(self.id)
+
+ cache = None
+ if not refresh:
+ cache = redis_connection.get(key)
+
+ if cache is None:
+ query_runner = get_query_runner(self.type, self.options)
+ schema = sorted(query_runner.get_schema(), key=lambda t: t['name'])
+
+ redis_connection.set(key, json.dumps(schema))
+ else:
+ schema = json.loads(cache)
+
+ return schema
+
@classmethod
def all(cls):
return cls.select().order_by(cls.id.asc())
diff --git a/redash/query_runner/__init__.py b/redash/query_runner/__init__.py
index eaf5693a57..dde3b769e9 100644
--- a/redash/query_runner/__init__.py
+++ b/redash/query_runner/__init__.py
@@ -57,6 +57,9 @@ def configuration_schema(cls):
def run_query(self, query):
raise NotImplementedError()
+ def get_schema(self):
+ return []
+
@classmethod
def to_dict(cls):
return {
diff --git a/redash/query_runner/mysql.py b/redash/query_runner/mysql.py
index 8a89553882..1623bef945 100644
--- a/redash/query_runner/mysql.py
+++ b/redash/query_runner/mysql.py
@@ -44,6 +44,41 @@ def enabled(cls):
def __init__(self, configuration_json):
super(Mysql, self).__init__(configuration_json)
+ def get_schema(self):
+ query = """
+ SELECT col.table_schema,
+ col.table_name,
+ col.column_name
+ FROM `information_schema`.`columns` col
+ INNER JOIN
+ (SELECT table_schema,
+ TABLE_NAME
+ FROM information_schema.tables
+ WHERE table_type <> 'SYSTEM VIEW' AND table_schema NOT IN ('performance_schema', 'mysql')) tables ON tables.table_schema = col.table_schema
+ AND tables.TABLE_NAME = col.TABLE_NAME;
+ """
+
+ results, error = self.run_query(query)
+
+ if error is not None:
+ raise Exception("Failed getting schema.")
+
+ results = json.loads(results)
+
+ schema = {}
+ for row in results['rows']:
+ if row['table_schema'] != self.configuration['db']:
+ table_name = '{}.{}'.format(row['table_schema'], row['table_name'])
+ else:
+ table_name = row['table_name']
+
+ if table_name not in schema:
+ schema[table_name] = {'name': table_name, 'columns': []}
+
+ schema[table_name]['columns'].append(row['column_name'])
+
+ return schema.values()
+
def run_query(self, query):
import MySQLdb
diff --git a/redash/query_runner/pg.py b/redash/query_runner/pg.py
index 811d52a2dc..1393773100 100644
--- a/redash/query_runner/pg.py
+++ b/redash/query_runner/pg.py
@@ -83,6 +83,34 @@ def __init__(self, configuration_json):
self.connection_string = " ".join(values)
+ def get_schema(self):
+ query = """
+ SELECT table_schema, table_name, column_name
+ FROM information_schema.columns
+ WHERE table_schema NOT IN ('pg_catalog', 'information_schema');
+ """
+
+ results, error = self.run_query(query)
+
+ if error is not None:
+ raise Exception("Failed getting schema.")
+
+ results = json.loads(results)
+
+ schema = {}
+ for row in results['rows']:
+ if row['table_schema'] != 'public':
+ table_name = '{}.{}'.format(row['table_schema'], row['table_name'])
+ else:
+ table_name = row['table_name']
+
+ if table_name not in schema:
+ schema[table_name] = {'name': table_name, 'columns': []}
+
+ schema[table_name]['columns'].append(row['column_name'])
+
+ return schema.values()
+
def run_query(self, query):
connection = psycopg2.connect(self.connection_string, async=True)
_wait(connection)
diff --git a/redash/tasks.py b/redash/tasks.py
index f224b635a9..cbca8d8350 100644
--- a/redash/tasks.py
+++ b/redash/tasks.py
@@ -218,6 +218,17 @@ def cleanup_query_results():
logger.info("Deleted %d unused query results out of total of %d." % (deleted_count, total_unused_query_results))
+@celery.task(base=BaseTask)
+def refresh_schemas():
+ """
+ Refershs the datasources schema.
+ """
+
+ for ds in models.DataSource.all():
+ logger.info("Refreshing schema for: {}".format(ds.name))
+ ds.get_schema(refresh=True)
+
+
@celery.task(bind=True, base=BaseTask, track_started=True)
def execute_query(self, query, data_source_id):
# TODO: maybe this should be a class?
diff --git a/redash/worker.py b/redash/worker.py
index e9ea0098ea..f52f5b851d 100644
--- a/redash/worker.py
+++ b/redash/worker.py
@@ -15,6 +15,10 @@
'cleanup_tasks': {
'task': 'redash.tasks.cleanup_tasks',
'schedule': timedelta(minutes=5)
+ },
+ 'refresh_schemas': {
+ 'task': 'redash.tasks.refresh_schemas',
+ 'schedule': timedelta(minutes=30)
}
}
diff --git a/tests/__init__.py b/tests/__init__.py
index 786bce8068..64a35d67d6 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -1,13 +1,17 @@
+import os
+os.environ['REDASH_REDIS_URL'] = "redis://localhost:6379/5"
+
import logging
from unittest import TestCase
import datetime
from redash import settings
+
settings.DATABASE_CONFIG = {
'name': 'circle_test',
'threadlocals': True
}
-from redash import models
+from redash import models, redis_connection
logging.getLogger('peewee').setLevel(logging.INFO)
@@ -20,6 +24,7 @@ def setUp(self):
def tearDown(self):
models.db.close_db(None)
models.create_db(False, True)
+ redis_connection.flushdb()
def assertResponseEqual(self, expected, actual):
for k, v in expected.iteritems():
diff --git a/tests/factories.py b/tests/factories.py
index ba2d59d4eb..879ac77a73 100644
--- a/tests/factories.py
+++ b/tests/factories.py
@@ -47,7 +47,7 @@ def __call__(self):
data_source_factory = ModelFactory(redash.models.DataSource,
name='Test',
type='pg',
- options='')
+ options='{"dbname": "test"}')
dashboard_factory = ModelFactory(redash.models.Dashboard,
diff --git a/tests/test_models.py b/tests/test_models.py
index ea4e9f3887..b9b2076ad1 100644
--- a/tests/test_models.py
+++ b/tests/test_models.py
@@ -2,10 +2,12 @@
import datetime
import json
from unittest import TestCase
+import mock
from tests import BaseTestCase
from redash import models
from factories import dashboard_factory, query_factory, data_source_factory, query_result_factory, user_factory, widget_factory
from redash.utils import gen_query_hash
+from redash import query_runner
class DashboardTest(BaseTestCase):
@@ -195,6 +197,44 @@ def test_removes_scheduling(self):
self.assertEqual(None, query.schedule)
+class DataSourceTest(BaseTestCase):
+ def test_get_schema(self):
+ return_value = [{'name': 'table', 'columns': []}]
+
+ with mock.patch('redash.query_runner.pg.PostgreSQL.get_schema') as patched_get_schema:
+ patched_get_schema.return_value = return_value
+
+ ds = data_source_factory.create()
+ schema = ds.get_schema()
+
+ self.assertEqual(return_value, schema)
+
+ def test_get_schema_uses_cache(self):
+ return_value = [{'name': 'table', 'columns': []}]
+ with mock.patch('redash.query_runner.pg.PostgreSQL.get_schema') as patched_get_schema:
+ patched_get_schema.return_value = return_value
+
+ ds = data_source_factory.create()
+ ds.get_schema()
+ schema = ds.get_schema()
+
+ self.assertEqual(return_value, schema)
+ self.assertEqual(patched_get_schema.call_count, 1)
+
+ def test_get_schema_skips_cache_with_refresh_true(self):
+ return_value = [{'name': 'table', 'columns': []}]
+ with mock.patch('redash.query_runner.pg.PostgreSQL.get_schema') as patched_get_schema:
+ patched_get_schema.return_value = return_value
+
+ ds = data_source_factory.create()
+ ds.get_schema()
+ new_return_value = [{'name': 'new_table', 'columns': []}]
+ patched_get_schema.return_value = new_return_value
+ schema = ds.get_schema(refresh=True)
+
+ self.assertEqual(new_return_value, schema)
+ self.assertEqual(patched_get_schema.call_count, 2)
+
class QueryResultTest(BaseTestCase):
def setUp(self):