From a33e4799eb97c777c28e3e37135107fa10fcb295 Mon Sep 17 00:00:00 2001 From: Allen Short Date: Wed, 12 Apr 2017 22:02:42 +0000 Subject: [PATCH 01/32] In docker-entrypoint ensure tables exist --- bin/docker-entrypoint | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint index 0d45eb5482..1bed803efd 100755 --- a/bin/docker-entrypoint +++ b/bin/docker-entrypoint @@ -2,6 +2,7 @@ set -e worker() { + /app/manage.py db upgrade WORKERS_COUNT=${WORKERS_COUNT:-2} QUEUES=${QUEUES:-queries,scheduled_queries,celery} @@ -10,6 +11,7 @@ worker() { } scheduler() { + /app/manage.py db upgrade WORKERS_COUNT=${WORKERS_COUNT:-1} QUEUES=${QUEUES:-celery} @@ -19,6 +21,7 @@ scheduler() { } server() { + /app/manage.py db upgrade exec /usr/local/bin/gunicorn -b 0.0.0.0:5000 --name redash -w${REDASH_WEB_WORKERS:-4} redash.wsgi:app } From 0bd626db1d357f0b7ca1fe20bfd90037b3476a77 Mon Sep 17 00:00:00 2001 From: Alison Date: Fri, 11 Aug 2017 09:34:35 -0500 Subject: [PATCH 02/32] upgrade node and npm in dockerfile --- Dockerfile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7ed5ad950a..1482776101 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,9 +10,19 @@ RUN pip install -r requirements.txt -r requirements_dev.txt RUN if [ "x$skip_ds_deps" = "x" ] ; then pip install -r requirements_all_ds.txt ; else echo "Skipping pip install -r requirements_all_ds.txt" ; fi COPY . ./ +RUN npm install && npm run bundle && npm run build && rm -rf node_modules + +# Upgrade node to LTS 6.11.2 +RUN cd ~ +RUN wget https://nodejs.org/download/release/v6.11.2/node-v6.11.2-linux-x64.tar.gz +RUN sudo tar --strip-components 1 -xzvf node-v* -C /usr/local + +# Upgrade npm +RUN npm upgrade npm + RUN npm install && npm run bundle && npm run build && rm -rf node_modules RUN chown -R redash /app USER redash ENTRYPOINT ["/app/bin/docker-entrypoint"] -CMD ["server"] \ No newline at end of file +CMD ["server"] From 18695a90a8cad87ce84059a55a5d89e58bdaee33 Mon Sep 17 00:00:00 2001 From: Blake Imsland Date: Wed, 19 Apr 2017 10:23:14 -0700 Subject: [PATCH 03/32] Update Circle CI for our workflow - Use new master / rc release release strategy (#440) - Migrate Circle CI 2.0 (#488, #502) --- .circleci/config.yml | 85 +++++++++++++++++++++++++----------------- bin/dockerflow-version | 13 +++++++ 2 files changed, 64 insertions(+), 34 deletions(-) create mode 100755 bin/dockerflow-version diff --git a/.circleci/config.yml b/.circleci/config.yml index ac7e5bb0d1..4265eccf57 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,7 +39,7 @@ jobs: name: Copy Test Results command: | mkdir -p /tmp/test-results/unit-tests - docker cp tests:/app/coverage.xml ./coverage.xml + docker cp tests:/app/coverage.xml ./coverage.xml docker cp tests:/app/junit.xml /tmp/test-results/unit-tests/results.xml - store_test_results: path: /tmp/test-results @@ -76,29 +76,41 @@ jobs: - run: name: Execute Cypress tests command: npm run cypress run-ci - build-tarball: + deploy-master: docker: - - image: circleci/node:8 + - image: circleci/buildpack-deps:xenial steps: + - setup_remote_docker - checkout - - run: sudo apt install python-pip - - run: npm install - - run: .circleci/update_version - - run: npm run bundle - - run: npm run build - - run: .circleci/pack - - store_artifacts: - path: /tmp/artifacts/ - build-docker-image: + - run: ./bin/dockerflow-version "master" + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker build -t $DOCKERHUB_REPO:master . + - run: docker push $DOCKERHUB_REPO:master + deploy-rc: + docker: + - image: circleci/buildpack-deps:xenial + steps: + - setup_remote_docker + - checkout + - run: ./bin/dockerflow-version "rc" + - run: docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: docker build -t $DOCKERHUB_REPO:rc . + - run: docker push $DOCKERHUB_REPO:rc + deploy-milestone: docker: - image: circleci/buildpack-deps:xenial steps: - setup_remote_docker - checkout - - run: .circleci/update_version + - run: ./bin/dockerflow-version "$CIRCLE_TAG" - run: docker login -u $DOCKER_USER -p $DOCKER_PASS - - run: docker build -t redash/redash:$(.circleci/docker_tag) . - - run: docker push redash/redash:$(.circleci/docker_tag) + - run: docker build -t $DOCKERHUB_REPO:$CIRCLE_TAG . + - run: docker push $DOCKERHUB_REPO:$CIRCLE_TAG + # Create alias from tag to "latest" + - run: docker tag $DOCKERHUB_REPO:$CIRCLE_TAG $DOCKERHUB_REPO:latest + - run: docker push $DOCKERHUB_REPO:latest + + workflows: version: 2 build: @@ -108,22 +120,27 @@ workflows: - backend-unit-tests - frontend-unit-tests - frontend-e2e-tests - - build-tarball: - requires: - - backend-unit-tests - filters: - tags: - only: /v[0-9]+(\.[0-9\-a-z]+)*/ - branches: - only: - - master - - /release\/.*/ - - build-docker-image: - requires: - - backend-unit-tests - filters: - branches: - only: - - master - - preview-build - - /release\/.*/ + - deploy-master: + requires: + - backend-unit-tests + filters: + branches: + only: + - master + + - deploy-rc: + requires: + - backend-unit-tests + filters: + branches: + only: + - release + + - deploy-milestone: + requires: + - backend-unit-tests + filters: + tags: + only: /^m[0-9]+(\.[0-9]+)?$/ + branches: + ignore: /.*/ diff --git a/bin/dockerflow-version b/bin/dockerflow-version new file mode 100755 index 0000000000..027d61971f --- /dev/null +++ b/bin/dockerflow-version @@ -0,0 +1,13 @@ +#!/bin/bash + +set -eo pipefail + +VERSION="$1" + +printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ + "$CIRCLE_SHA1" \ + "$VERSION" \ + "$CIRCLE_PROJECT_USERNAME" \ + "$CIRCLE_PROJECT_REPONAME" \ + "$CIRCLE_BUILD_URL" \ +> version.json From 86f5c2fb23057d6b25a1750a93e22cc381ea1320 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Wed, 25 Oct 2017 19:32:37 +0200 Subject: [PATCH 04/32] Pin PyAthena dependency to 1.2.0. --- requirements_all_ds.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_all_ds.txt b/requirements_all_ds.txt index c2af9ebd7a..7e9c51686e 100644 --- a/requirements_all_ds.txt +++ b/requirements_all_ds.txt @@ -19,7 +19,7 @@ cassandra-driver==3.11.0 memsql==2.16.0 atsd_client==2.0.12 simple_salesforce==0.72.2 -PyAthena>=1.0.0 +PyAthena>=1.2.0 pymapd>=0.2.1 qds-sdk>=1.9.6 ibm-db>=2.0.9 From 1fc3098eee379e11ca3b6d8a3e13b6b7f8e0ba83 Mon Sep 17 00:00:00 2001 From: Allen Short Date: Thu, 14 Sep 2017 06:23:44 +0000 Subject: [PATCH 05/32] Switch to PyMySQL for MySQL 5.7 support --- redash/query_runner/mysql.py | 8 ++++---- requirements_all_ds.txt | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/redash/query_runner/mysql.py b/redash/query_runner/mysql.py index bfd6e7198e..9c674aea4a 100644 --- a/redash/query_runner/mysql.py +++ b/redash/query_runner/mysql.py @@ -90,7 +90,7 @@ def name(cls): @classmethod def enabled(cls): try: - import MySQLdb + import pymysql except ImportError: return False @@ -126,11 +126,11 @@ def _get_tables(self, schema): return schema.values() def run_query(self, query, user): - import MySQLdb + import pymysql connection = None try: - connection = MySQLdb.connect(host=self.configuration.get('host', ''), + connection = pymysql.connect(host=self.configuration.get('host', ''), user=self.configuration.get('user', ''), passwd=self.configuration.get('passwd', ''), db=self.configuration['db'], @@ -160,7 +160,7 @@ def run_query(self, query, user): error = "No data was returned." cursor.close() - except MySQLdb.Error as e: + except pymysql.Error as e: json_data = None error = e.args[1] except KeyboardInterrupt: diff --git a/requirements_all_ds.txt b/requirements_all_ds.txt index 7e9c51686e..79e9d0e5e9 100644 --- a/requirements_all_ds.txt +++ b/requirements_all_ds.txt @@ -2,7 +2,7 @@ google-api-python-client==1.5.1 gspread==0.6.2 impyla==0.10.0 influxdb==2.7.1 -MySQL-python==1.2.5 +PyMySQL==0.7.11 oauth2client==3.0.0 pyhive==0.5.1 pymongo[tls,srv]==3.6.1 From 57f11a704fe054958daf53aa38ff84d61f7e07c0 Mon Sep 17 00:00:00 2001 From: Allen Short Date: Fri, 10 Feb 2017 00:49:30 -0600 Subject: [PATCH 06/32] Don't execute query when changing data sources (fixes #29) --- client/app/pages/queries/query.html | 2 +- client/app/pages/queries/view.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/client/app/pages/queries/query.html b/client/app/pages/queries/query.html index 2e841a7baf..8353ec5c0e 100644 --- a/client/app/pages/queries/query.html +++ b/client/app/pages/queries/query.html @@ -221,7 +221,7 @@

-
+

Log Information:

diff --git a/client/app/pages/queries/view.js b/client/app/pages/queries/view.js index df0b8ed829..fc7cc866dc 100644 --- a/client/app/pages/queries/view.js +++ b/client/app/pages/queries/view.js @@ -121,6 +121,7 @@ function QueryViewCtrl( Notifications.getPermissions(); }; + $scope.dataSourceChanged = false; $scope.selectedTab = DEFAULT_TAB; $scope.currentUser = currentUser; $scope.dataSource = {}; @@ -321,7 +322,7 @@ function QueryViewCtrl( $scope.dataSource = find($scope.dataSources, ds => ds.id === $scope.query.data_source_id); getSchema(); - $scope.executeQuery(); + $scope.dataSourceChanged = true; }; $scope.setVisualizationTab = (visualization) => { From 6fd67edfd146445fc4aeb501c57f1834d082e856 Mon Sep 17 00:00:00 2001 From: Allen Short Date: Thu, 2 Mar 2017 16:43:57 -0600 Subject: [PATCH 07/32] Add `schedule_until` field to queries, to allow expiry (re #15) --- .../components/queries/schedule-dialog.html | 4 +++ .../app/components/queries/schedule-dialog.js | 12 ++++++++ client/app/services/query.js | 4 +++ migrations/versions/eb2f788f997e_.py | 27 ++++++++++++++++++ redash/handlers/queries.py | 2 ++ redash/models.py | 5 +++- redash/serializers.py | 1 + tests/test_models.py | 28 +++++++++++++++++++ 8 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 migrations/versions/eb2f788f997e_.py diff --git a/client/app/components/queries/schedule-dialog.html b/client/app/components/queries/schedule-dialog.html index 8f1ab21541..f9344238a1 100644 --- a/client/app/components/queries/schedule-dialog.html +++ b/client/app/components/queries/schedule-dialog.html @@ -15,4 +15,8 @@

Refresh Schedule

+
diff --git a/client/app/components/queries/schedule-dialog.js b/client/app/components/queries/schedule-dialog.js index bf0c2edd24..b71cdff67f 100644 --- a/client/app/components/queries/schedule-dialog.js +++ b/client/app/components/queries/schedule-dialog.js @@ -103,6 +103,17 @@ function queryRefreshSelect(clientConfig, Policy) { }; } +function scheduleUntil() { + return { + restrict: 'E', + scope: { + query: '=', + saveQuery: '=', + }, + template: '', + }; +} + const ScheduleForm = { controller() { this.query = this.resolve.query; @@ -125,6 +136,7 @@ const ScheduleForm = { export default function init(ngModule) { ngModule.directive('queryTimePicker', queryTimePicker); ngModule.directive('queryRefreshSelect', queryRefreshSelect); + ngModule.directive('scheduleUntil', scheduleUntil); ngModule.component('scheduleDialog', ScheduleForm); } diff --git a/client/app/services/query.js b/client/app/services/query.js index dfa73a2186..b78e261a40 100644 --- a/client/app/services/query.js +++ b/client/app/services/query.js @@ -404,6 +404,10 @@ function QueryResource( .format('HH:mm'); }; + Query.prototype.hasScheduleExpiry = function hasScheduleExpiry() { + return (this.schedule && this.schedule_until); + }; + Query.prototype.hasResult = function hasResult() { return !!(this.latest_query_data || this.latest_query_data_id); }; diff --git a/migrations/versions/eb2f788f997e_.py b/migrations/versions/eb2f788f997e_.py new file mode 100644 index 0000000000..71fd2bd5b3 --- /dev/null +++ b/migrations/versions/eb2f788f997e_.py @@ -0,0 +1,27 @@ +"""Add 'schedule_until' column to queries. + +Revision ID: eb2f788f997e +Revises: d1eae8b9893e +Create Date: 2017-03-02 12:20:00.029066 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'eb2f788f997e' +down_revision = 'd1eae8b9893e' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + 'queries', + sa.Column('schedule_until', sa.DateTime(timezone=True), nullable=True)) + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('queries', 'schedule_until') diff --git a/redash/handlers/queries.py b/redash/handlers/queries.py index 7f33a52843..7d333fdd03 100644 --- a/redash/handlers/queries.py +++ b/redash/handlers/queries.py @@ -113,6 +113,7 @@ def post(self): :json string query: Query text :>json string query_hash: Hash of query text :>json string schedule: Schedule interval, in seconds, for repeated execution of this query + :json string api_key: Key for public access to this query's results. :>json boolean is_archived: Whether this query is displayed in indexes and search results or not. :>json boolean is_draft: Whether this query is a draft or not diff --git a/redash/models.py b/redash/models.py index d8d1904346..850dbf7596 100644 --- a/redash/models.py +++ b/redash/models.py @@ -893,6 +893,7 @@ class Query(ChangeTrackingMixin, TimestampMixin, BelongsToOrgMixin, db.Model): is_draft = Column(db.Boolean, default=True, index=True) schedule = Column(db.String(10), nullable=True) schedule_failures = Column(db.Integer, default=0) + schedule_until = Column(db.DateTime(True), nullable=True) visualizations = db.relationship("Visualization", cascade="all, delete-orphan") options = Column(MutableDict.as_mutable(PseudoJSON), default={}) search_vector = Column(TSVectorType('id', 'name', 'description', 'query', @@ -1016,7 +1017,9 @@ def by_user(cls, user): def outdated_queries(cls): queries = (db.session.query(Query) .options(joinedload(Query.latest_query_data).load_only('retrieved_at')) - .filter(Query.schedule != None) + .filter(Query.schedule != None, + (Query.schedule_until == None) | + (Query.schedule_until > db.func.now())) .order_by(Query.id)) now = utils.utcnow() diff --git a/redash/serializers.py b/redash/serializers.py index d809a1f73e..279806e93a 100644 --- a/redash/serializers.py +++ b/redash/serializers.py @@ -91,6 +91,7 @@ def serialize_query(query, with_stats=False, with_visualizations=False, with_use 'query': query.query_text, 'query_hash': query.query_hash, 'schedule': query.schedule, + 'schedule_until': query.schedule_until, 'api_key': query.api_key, 'is_archived': query.is_archived, 'is_draft': query.is_draft, diff --git a/tests/test_models.py b/tests/test_models.py index 5ccf6e4af0..68fe2a5224 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -192,6 +192,34 @@ def test_failure_extends_schedule(self): query_result.retrieved_at = utcnow() - datetime.timedelta(minutes=17) self.assertEqual(list(models.Query.outdated_queries()), [query]) + def test_schedule_until_after(self): + """ + Queries with non-null ``schedule_until`` are not reported by + Query.outdated_queries() after the given time is past. + """ + three_hours_ago = utcnow() - datetime.timedelta(hours=3) + two_hours_ago = utcnow() - datetime.timedelta(hours=2) + query = self.factory.create_query(schedule="3600", schedule_until=three_hours_ago) + query_result = self.factory.create_query_result(query=query.query_text, retrieved_at=two_hours_ago) + query.latest_query_data = query_result + + queries = models.Query.outdated_queries() + self.assertNotIn(query, queries) + + def test_schedule_until_before(self): + """ + Queries with non-null ``schedule_until`` are reported by + Query.outdated_queries() before the given time is past. + """ + one_hour_from_now = utcnow() + datetime.timedelta(hours=1) + two_hours_ago = utcnow() - datetime.timedelta(hours=2) + query = self.factory.create_query(schedule="3600", schedule_until=one_hour_from_now) + query_result = self.factory.create_query_result(query=query.query_text, retrieved_at=two_hours_ago) + query.latest_query_data = query_result + + queries = models.Query.outdated_queries() + self.assertIn(query, queries) + class QueryArchiveTest(BaseTestCase): def setUp(self): From dd470bf5a073c5e725a917036ba475573a0d252e Mon Sep 17 00:00:00 2001 From: Davor Spasovski Date: Mon, 6 Feb 2017 13:39:46 -0500 Subject: [PATCH 08/32] Add compare query version support (re #7) --- .../pages/queries/compare-query-dialog.css | 54 ++++++++++++++++ .../pages/queries/compare-query-dialog.html | 33 ++++++++++ .../app/pages/queries/compare-query-dialog.js | 63 +++++++++++++++++++ client/app/pages/queries/query.html | 3 + client/app/pages/queries/view.js | 15 +++++ package.json | 1 + redash/handlers/api.py | 4 +- redash/handlers/dashboards.py | 1 + redash/handlers/queries.py | 15 +++++ redash/models.py | 40 +++++++----- tests/handlers/test_queries.py | 26 ++++++++ tests/models/test_changes.py | 13 +--- tests/test_models.py | 3 +- 13 files changed, 241 insertions(+), 30 deletions(-) create mode 100644 client/app/pages/queries/compare-query-dialog.css create mode 100644 client/app/pages/queries/compare-query-dialog.html create mode 100644 client/app/pages/queries/compare-query-dialog.js diff --git a/client/app/pages/queries/compare-query-dialog.css b/client/app/pages/queries/compare-query-dialog.css new file mode 100644 index 0000000000..ce2d01370e --- /dev/null +++ b/client/app/pages/queries/compare-query-dialog.css @@ -0,0 +1,54 @@ +/* Compare Query Version container */ +/* Offers slight visual improvement (alignment) to modern UAs */ +.compare-query-version { + display: flex; + justify-content: space-between; + align-items: center; +} + +.diff-removed { + background-color: rgba(208, 2, 27, 0.3); +} + +.diff-added { + background-color: rgba(65, 117, 5, 0.3); +} + +.query-diff-container span { + display: inline-block; + border-radius: 3px; + line-height: 20px; + vertical-align: middle; + margin: 0 5px 0 0; +} + +.query-diff-container > div:not(.compare-query-version-controls) { + float: left; + width: calc(50% - 5px); + margin: 0 10px 0 0; +} + +.compare-query-version { + background-color: #f5f5f5; + padding: 5px; + border: 1px solid #ccc; + margin-right: 15px; + border-radius: 3px; +} + +.diff-content { + border: 1px solid #ccc; + background-color: #f5f5f5; + border-radius: 3px; + padding: 15px; +} + +.query-diff-container > div:last-child { + margin: 0; +} + +.compare-query-version-controls { + display: flex; + align-items: center; + margin-bottom: 25px; +} diff --git a/client/app/pages/queries/compare-query-dialog.html b/client/app/pages/queries/compare-query-dialog.html new file mode 100644 index 0000000000..5214046055 --- /dev/null +++ b/client/app/pages/queries/compare-query-dialog.html @@ -0,0 +1,33 @@ +
+ +

Compare Query

+
+
+
+
+ Compare current version to + +
+ +
+
+
Current Version {{$ctrl.currentQuery.version}}
+
+

+ + {{part.value}} + +

+
+
+
+
Previous Version {{$ctrl.previousQueryVersion + 1}}
+
+

{{$ctrl.previousQuery}}

+
+
+
diff --git a/client/app/pages/queries/compare-query-dialog.js b/client/app/pages/queries/compare-query-dialog.js new file mode 100644 index 0000000000..fb4338971a --- /dev/null +++ b/client/app/pages/queries/compare-query-dialog.js @@ -0,0 +1,63 @@ +import * as jsDiff from 'diff'; +import template from './compare-query-dialog.html'; +import './compare-query-dialog.css'; + +const CompareQueryDialog = { + controller: ['clientConfig', '$http', function doCompare(clientConfig, $http) { + this.currentQuery = this.resolve.query; + + this.previousQuery = ''; + this.currentDiff = []; + this.previousDiff = []; + this.versions = []; + this.previousQueryVersion = this.currentQuery.version - 2; // due to 0-indexed versions[] + + this.compareQueries = (isInitialLoad) => { + if (!isInitialLoad) { + this.previousQueryVersion = document.getElementById('version-choice').value - 1; // due to 0-indexed versions[] + } + + this.previousQuery = this.versions[this.previousQueryVersion].change.query.current; + this.currentDiff = jsDiff.diffChars(this.previousQuery, this.currentQuery.query); + document.querySelector('.compare-query-revert-wrapper').classList.remove('hidden'); + }; + + this.revertQuery = () => { + this.resolve.query.query = this.previousQuery; + this.resolve.saveQuery(); + + // Close modal. + this.dismiss(); + }; + + $http.get(`/api/queries/${this.currentQuery.id}/version`).then((response) => { + this.versions = response.data; + + const compare = (a, b) => { + if (a.object_version < b.object_version) { + return -1; + } else if (a.object_version > b.object_version) { + return 1; + } + return 0; + }; + + this.versions.sort(compare); + this.compareQueries(true); + }); + }], + scope: { + query: '=', + saveQuery: '<', + }, + bindings: { + resolve: '<', + close: '&', + dismiss: '&', + }, + template, +}; + +export default function (ngModule) { + ngModule.component('compareQueryDialog', CompareQueryDialog); +} diff --git a/client/app/pages/queries/query.html b/client/app/pages/queries/query.html index 8353ec5c0e..1b504cb87a 100644 --- a/client/app/pages/queries/query.html +++ b/client/app/pages/queries/query.html @@ -69,6 +69,9 @@

  • Show API Key
    • +
  • + Query Versions +

    • diff --git a/client/app/pages/queries/view.js b/client/app/pages/queries/view.js index fc7cc866dc..29d2899899 100644 --- a/client/app/pages/queries/view.js +++ b/client/app/pages/queries/view.js @@ -354,6 +354,21 @@ function QueryViewCtrl( }); }; + $scope.compareQueryVersion = () => { + if (!$scope.query.query) { + return; + } + + $uibModal.open({ + windowClass: 'modal-xl', + component: 'compareQueryDialog', + resolve: { + query: $scope.query, + saveQuery: () => $scope.saveQuery, + }, + }); + }; + $scope.$watch('query.name', () => { Title.set($scope.query.name); }); diff --git a/package.json b/package.json index 565de8fd4d..3c6557af31 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "d3": "^3.5.17", "d3-cloud": "^1.2.4", "debug": "^3.1.0", + "diff": "^3.3.0", "font-awesome": "^4.7.0", "gridstack": "^0.3.0", "jquery": "^3.2.1", diff --git a/redash/handlers/api.py b/redash/handlers/api.py index f8ef199857..87c6ccc14a 100644 --- a/redash/handlers/api.py +++ b/redash/handlers/api.py @@ -9,7 +9,7 @@ from redash.handlers.dashboards import DashboardListResource, DashboardResource, DashboardShareResource, PublicDashboardResource from redash.handlers.data_sources import DataSourceTypeListResource, DataSourceListResource, DataSourceSchemaResource, DataSourceResource, DataSourcePauseResource, DataSourceTestResource from redash.handlers.events import EventsResource -from redash.handlers.queries import QueryForkResource, QueryRefreshResource, QueryListResource, QueryRecentResource, QuerySearchResource, QueryResource, MyQueriesResource +from redash.handlers.queries import QueryForkResource, QueryRefreshResource, QueryListResource, QueryRecentResource, QuerySearchResource, QueryResource, MyQueriesResource, QueryVersionListResource, ChangeResource from redash.handlers.query_results import QueryResultListResource, QueryResultResource, JobResource from redash.handlers.users import UserResource, UserListResource, UserInviteResource, UserResetPasswordResource, UserDisableResource from redash.handlers.visualizations import VisualizationListResource @@ -84,6 +84,8 @@ def json_representation(data, code, headers=None): api.add_org_resource(QueryRefreshResource, '/api/queries//refresh', endpoint='query_refresh') api.add_org_resource(QueryResource, '/api/queries/', endpoint='query') api.add_org_resource(QueryForkResource, '/api/queries//fork', endpoint='query_fork') +api.add_org_resource(QueryVersionListResource, '/api/queries//version', endpoint='query_versions') +api.add_org_resource(ChangeResource, '/api/changes/', endpoint='changes') api.add_org_resource(ObjectPermissionsListResource, '/api///acl', endpoint='object_permissions') api.add_org_resource(CheckPermissionResource, '/api///acl/', endpoint='check_permissions') diff --git a/redash/handlers/dashboards.py b/redash/handlers/dashboards.py index 5739fc872a..19da96550b 100644 --- a/redash/handlers/dashboards.py +++ b/redash/handlers/dashboards.py @@ -104,6 +104,7 @@ def post(self): user=self.current_user, is_draft=True, layout='[]') + dashboard.record_changes(changed_by=self.current_user) models.db.session.add(dashboard) models.db.session.commit() return serialize_dashboard(dashboard) diff --git a/redash/handlers/queries.py b/redash/handlers/queries.py index 7d333fdd03..c934f2cff8 100644 --- a/redash/handlers/queries.py +++ b/redash/handlers/queries.py @@ -152,6 +152,7 @@ def post(self): query_def['org'] = self.current_org query_def['is_draft'] = True query = models.Query.create(**query_def) + query.record_changes(changed_by=self.current_user) models.db.session.add(query) models.db.session.commit() @@ -301,6 +302,7 @@ def post(self, query_id): try: self.update_model(query, query_def) + query.record_changes(self.current_user) models.db.session.commit() except StaleDataError: abort(409) @@ -405,3 +407,16 @@ def get(self): for name, count in tags ] } + + +class QueryVersionListResource(BaseResource): + @require_permission('view_query') + def get(self, query_id): + results = models.Change.list_versions(models.Query.get_by_id(query_id)) + return [q.to_dict() for q in results] + + +class ChangeResource(BaseResource): + @require_permission('view_query') + def get(self, change_id): + return models.Change.query.get(change_id).to_dict() diff --git a/redash/models.py b/redash/models.py index 850dbf7596..731757241d 100644 --- a/redash/models.py +++ b/redash/models.py @@ -207,10 +207,6 @@ class ChangeTrackingMixin(object): skipped_fields = ('id', 'created_at', 'updated_at', 'version') _clean_values = None - def __init__(self, *a, **kw): - super(ChangeTrackingMixin, self).__init__(*a, **kw) - self.record_changes(self.user) - def prep_cleanvalues(self): self.__dict__['_clean_values'] = {} for attr in inspect(self.__class__).column_attrs: @@ -221,10 +217,10 @@ def prep_cleanvalues(self): def __setattr__(self, key, value): if self._clean_values is None: self.prep_cleanvalues() - for attr in inspect(self.__class__).column_attrs: - col, = attr.columns - previous = getattr(self, attr.key, None) - self._clean_values[col.name] = previous + + if key in inspect(self.__class__).column_attrs: + previous = getattr(self, key, None) + self._clean_values[key] = previous super(ChangeTrackingMixin, self).__setattr__(key, value) @@ -235,13 +231,19 @@ def record_changes(self, changed_by): for attr in inspect(self.__class__).column_attrs: col, = attr.columns if attr.key not in self.skipped_fields: - changes[col.name] = {'previous': self._clean_values[col.name], - 'current': getattr(self, attr.key)} + prev = self._clean_values[col.name] + current = getattr(self, attr.key) + if prev != current: + changes[col.name] = {'previous': prev, 'current': current} - db.session.add(Change(object=self, - object_version=self.version, - user=changed_by, - change=changes)) + if changes: + self.version = (self.version or 0) + 1 + change = Change(object=self, + object_version=self.version, + user=changed_by, + change=changes) + db.session.add(change) + return change class BelongsToOrgMixin(object): @@ -872,7 +874,7 @@ def should_schedule_next(previous_iteration, now, schedule, failures): @python_2_unicode_compatible class Query(ChangeTrackingMixin, TimestampMixin, BelongsToOrgMixin, db.Model): id = Column(db.Integer, primary_key=True) - version = Column(db.Integer, default=1) + version = Column(db.Integer, default=0) org_id = Column(db.Integer, db.ForeignKey('organizations.id')) org = db.relationship(Organization, backref="queries") data_source_id = Column(db.Integer, db.ForeignKey("data_sources.id"), nullable=True) @@ -1084,6 +1086,7 @@ def fork(self, user): kwargs = {a: getattr(self, a) for a in forked_list} forked_query = Query.create(name=u'Copy of (#{}) {}'.format(self.id, self.name), user=user, **kwargs) + forked_query.record_changes(changed_by=user) for v in self.visualizations: if v.type == 'TABLE': @@ -1257,7 +1260,6 @@ def to_dict(self, full=True): 'id': self.id, 'object_id': self.object_id, 'object_type': self.object_type, - 'change_type': self.change_type, 'object_version': self.object_version, 'change': self.change, 'created_at': self.created_at @@ -1277,6 +1279,12 @@ def last_change(cls, obj): cls.object_type == obj.__class__.__tablename__).order_by( cls.object_version.desc()).first() + @classmethod + def list_versions(cls, query): + return cls.query.filter( + cls.object_id == query.id, + cls.object_type == 'queries') + class Alert(TimestampMixin, db.Model): UNKNOWN_STATE = 'unknown' diff --git a/tests/handlers/test_queries.py b/tests/handlers/test_queries.py index 8e2352553e..93dfae83d0 100644 --- a/tests/handlers/test_queries.py +++ b/tests/handlers/test_queries.py @@ -259,3 +259,29 @@ def test_format_sql_query(self): self.assertEqual(rv.json['query'], expected) + +class ChangeResourceTests(BaseTestCase): + def test_list(self): + query = self.factory.create_query() + query.name = 'version A' + query.record_changes(self.factory.user) + query.name = 'version B' + query.record_changes(self.factory.user) + rv = self.make_request('get', '/api/queries/{0}/version'.format(query.id)) + self.assertEquals(rv.status_code, 200) + self.assertEquals(len(rv.json), 2) + self.assertEquals(rv.json[0]['change']['name']['current'], 'version A') + self.assertEquals(rv.json[1]['change']['name']['current'], 'version B') + + def test_get(self): + query = self.factory.create_query() + query.name = 'version A' + ch1 = query.record_changes(self.factory.user) + query.name = 'version B' + ch2 = query.record_changes(self.factory.user) + rv1 = self.make_request('get', '/api/changes/' + str(ch1.id)) + self.assertEqual(rv1.status_code, 200) + self.assertEqual(rv1.json['change']['name']['current'], 'version A') + rv2 = self.make_request('get', '/api/changes/' + str(ch2.id)) + self.assertEqual(rv2.status_code, 200) + self.assertEqual(rv2.json['change']['name']['current'], 'version B') diff --git a/tests/models/test_changes.py b/tests/models/test_changes.py index 124e17a30d..3d7c7496e8 100644 --- a/tests/models/test_changes.py +++ b/tests/models/test_changes.py @@ -56,23 +56,12 @@ def test_properly_log_modification(self): obj.record_changes(changed_by=self.factory.user) obj.name = 'Query 2' obj.description = 'description' - db.session.flush() obj.record_changes(changed_by=self.factory.user) change = Change.last_change(obj) self.assertIsNotNone(change) - # TODO: https://github.com/getredash/redash/issues/1550 - # self.assertEqual(change.object_version, 2) + self.assertEqual(change.object_version, 2) self.assertEqual(change.object_version, obj.version) self.assertIn('name', change.change) self.assertIn('description', change.change) - - def test_logs_create_method(self): - q = Query(name='Query', description='', query_text='', - user=self.factory.user, data_source=self.factory.data_source, - org=self.factory.org) - change = Change.last_change(q) - - self.assertIsNotNone(change) - self.assertEqual(q.user, change.user) diff --git a/tests/test_models.py b/tests/test_models.py index 68fe2a5224..fac35733db 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -180,7 +180,8 @@ def test_failure_extends_schedule(self): Execution failures recorded for a query result in exponential backoff for scheduling future execution. """ - query = self.factory.create_query(schedule="60", schedule_failures=4) + query = self.factory.create_query(schedule="60") + query.schedule_failures = 4 retrieved_at = utcnow() - datetime.timedelta(minutes=16) query_result = self.factory.create_query_result( retrieved_at=retrieved_at, query_text=query.query_text, From bc78aa58630b0ee8c9e9739fb624eb3c6576bb8e Mon Sep 17 00:00:00 2001 From: Allen Short Date: Mon, 9 Jul 2018 13:20:36 -0500 Subject: [PATCH 09/32] Filter tables from schema browser (re #31) --- client/app/components/queries/schema-browser.html | 12 +++++++++++- client/app/components/queries/schema-browser.js | 14 ++++++++++++++ client/app/pages/queries/query.html | 2 +- redash/query_runner/__init__.py | 6 ++++++ redash/query_runner/athena.py | 6 ++++++ redash/query_runner/axibase_tsd.py | 6 ++++++ redash/query_runner/big_query.py | 6 ++++++ redash/query_runner/cass.py | 6 ++++++ redash/query_runner/clickhouse.py | 6 ++++++ redash/query_runner/dynamodb_sql.py | 6 ++++++ redash/query_runner/elasticsearch.py | 6 ++++++ redash/query_runner/google_analytics.py | 6 ++++++ redash/query_runner/google_spreadsheets.py | 6 ++++++ redash/query_runner/graphite.py | 6 ++++++ redash/query_runner/hive_ds.py | 12 +++++++++--- redash/query_runner/impala_ds.py | 6 ++++++ redash/query_runner/influx_db.py | 6 ++++++ redash/query_runner/memsql_ds.py | 6 ++++++ redash/query_runner/mongodb.py | 6 ++++++ redash/query_runner/mssql.py | 6 ++++++ redash/query_runner/mysql.py | 6 ++++++ redash/query_runner/oracle.py | 6 ++++++ redash/query_runner/pg.py | 6 ++++++ redash/query_runner/presto.py | 6 ++++++ redash/query_runner/python.py | 6 ++++++ redash/query_runner/salesforce.py | 6 ++++++ redash/query_runner/script.py | 6 ++++++ redash/query_runner/snowflake.py | 6 ++++++ redash/query_runner/sqlite.py | 6 ++++++ redash/query_runner/treasuredata.py | 6 ++++++ redash/query_runner/vertica.py | 8 +++++++- tests/test_cli.py | 2 +- 32 files changed, 199 insertions(+), 7 deletions(-) diff --git a/client/app/components/queries/schema-browser.html b/client/app/components/queries/schema-browser.html index 6e3f518059..fe7e26669e 100644 --- a/client/app/components/queries/schema-browser.html +++ b/client/app/components/queries/schema-browser.html @@ -6,10 +6,20 @@ ng-click="$ctrl.onRefresh()"> + +
    -
    +
    diff --git a/client/app/components/queries/schema-browser.js b/client/app/components/queries/schema-browser.js index cb92fbd1df..b40cb0c68e 100644 --- a/client/app/components/queries/schema-browser.js +++ b/client/app/components/queries/schema-browser.js @@ -3,6 +3,9 @@ import template from './schema-browser.html'; function SchemaBrowserCtrl($rootScope, $scope) { 'ngInject'; + this.versionToggle = false; + this.versionFilter = 'abcdefghijklmnop'; + this.showTable = (table) => { table.collapsed = !table.collapsed; $scope.$broadcast('vsRepeatTrigger'); @@ -21,6 +24,15 @@ function SchemaBrowserCtrl($rootScope, $scope) { this.isEmpty = function isEmpty() { return this.schema === undefined || this.schema.length === 0; }; + this.flipToggleVersionedTables = (versionToggle, toggleString) => { + if (versionToggle === false) { + this.versionToggle = true; + this.versionFilter = toggleString; + } else { + this.versionToggle = false; + this.versionFilter = 'abcdefghijklmnop'; + } + }; this.itemSelected = ($event, hierarchy) => { $rootScope.$broadcast('query-editor.command', 'paste', hierarchy.join('.')); @@ -44,7 +56,9 @@ function SchemaBrowserCtrl($rootScope, $scope) { const SchemaBrowser = { bindings: { schema: '<', + tabletogglestring: '<', onRefresh: '&', + flipToggleVersionedTables: '&', }, controller: SchemaBrowserCtrl, template, diff --git a/client/app/pages/queries/query.html b/client/app/pages/queries/query.html index 1b504cb87a..b5357d9e1c 100644 --- a/client/app/pages/queries/query.html +++ b/client/app/pages/queries/query.html @@ -96,7 +96,7 @@

    - +
     
    diff --git a/redash/query_runner/__init__.py b/redash/query_runner/__init__.py index 73d6c49368..4a2ac1fc56 100644 --- a/redash/query_runner/__init__.py +++ b/redash/query_runner/__init__.py @@ -168,6 +168,12 @@ def configuration_schema(cls): 'type': 'string', 'title': cls.password_title, }, + 'toggle_table_string': { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, }, 'secret': ['password'] } diff --git a/redash/query_runner/athena.py b/redash/query_runner/athena.py index e7f1bb4ad5..eb2f619ad3 100644 --- a/redash/query_runner/athena.py +++ b/redash/query_runner/athena.py @@ -78,6 +78,12 @@ def configuration_schema(cls): 'type': 'boolean', 'title': 'Use Glue Data Catalog', }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } }, 'required': ['region', 's3_staging_dir'], 'order': ['region', 'aws_access_key', 'aws_secret_key', 's3_staging_dir', 'schema'], diff --git a/redash/query_runner/axibase_tsd.py b/redash/query_runner/axibase_tsd.py index 78f533fdbf..4514f886d5 100644 --- a/redash/query_runner/axibase_tsd.py +++ b/redash/query_runner/axibase_tsd.py @@ -132,6 +132,12 @@ def configuration_schema(cls): 'trust_certificate': { 'type': 'boolean', 'title': 'Trust SSL Certificate' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, 'required': ['username', 'password', 'hostname', 'protocol', 'port'], diff --git a/redash/query_runner/big_query.py b/redash/query_runner/big_query.py index 594d79d203..b1f3b3b678 100644 --- a/redash/query_runner/big_query.py +++ b/redash/query_runner/big_query.py @@ -124,6 +124,12 @@ def configuration_schema(cls): 'maximumBillingTier': { "type": "number", "title": "Maximum Billing Tier" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, 'required': ['jsonKeyFile', 'projectId'], diff --git a/redash/query_runner/cass.py b/redash/query_runner/cass.py index 0f0c72ff66..11550dd181 100644 --- a/redash/query_runner/cass.py +++ b/redash/query_runner/cass.py @@ -61,6 +61,12 @@ def configuration_schema(cls): 'type': 'number', 'title': 'Timeout', 'default': 10 + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, 'required': ['keyspace', 'host'] diff --git a/redash/query_runner/clickhouse.py b/redash/query_runner/clickhouse.py index a51328531a..86d9e0c7b3 100644 --- a/redash/query_runner/clickhouse.py +++ b/redash/query_runner/clickhouse.py @@ -36,6 +36,12 @@ def configuration_schema(cls): "type": "number", "title": "Request Timeout", "default": 30 + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, "required": ["dbname"], diff --git a/redash/query_runner/dynamodb_sql.py b/redash/query_runner/dynamodb_sql.py index 5f7c8f09d8..32cf84669d 100644 --- a/redash/query_runner/dynamodb_sql.py +++ b/redash/query_runner/dynamodb_sql.py @@ -46,6 +46,12 @@ def configuration_schema(cls): }, "secret_key": { "type": "string", + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, "required": ["access_key", "secret_key"], diff --git a/redash/query_runner/elasticsearch.py b/redash/query_runner/elasticsearch.py index e9327e504a..99203ea43d 100644 --- a/redash/query_runner/elasticsearch.py +++ b/redash/query_runner/elasticsearch.py @@ -62,6 +62,12 @@ def configuration_schema(cls): 'basic_auth_password': { 'type': 'string', 'title': 'Basic Auth Password' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, "secret": ["basic_auth_password"], diff --git a/redash/query_runner/google_analytics.py b/redash/query_runner/google_analytics.py index 71be522015..117205a763 100644 --- a/redash/query_runner/google_analytics.py +++ b/redash/query_runner/google_analytics.py @@ -102,6 +102,12 @@ def configuration_schema(cls): 'jsonKeyFile': { "type": "string", 'title': 'JSON Key File' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, 'required': ['jsonKeyFile'], diff --git a/redash/query_runner/google_spreadsheets.py b/redash/query_runner/google_spreadsheets.py index 620fe770a1..8ac65d0be9 100644 --- a/redash/query_runner/google_spreadsheets.py +++ b/redash/query_runner/google_spreadsheets.py @@ -168,6 +168,12 @@ def configuration_schema(cls): 'jsonKeyFile': { "type": "string", 'title': 'JSON Key File' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, 'required': ['jsonKeyFile'], diff --git a/redash/query_runner/graphite.py b/redash/query_runner/graphite.py index 6b394e81ec..ec0068dbb6 100644 --- a/redash/query_runner/graphite.py +++ b/redash/query_runner/graphite.py @@ -43,6 +43,12 @@ def configuration_schema(cls): 'verify': { 'type': 'boolean', 'title': 'Verify SSL certificate' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, 'required': ['url'], diff --git a/redash/query_runner/hive_ds.py b/redash/query_runner/hive_ds.py index b3c78bf431..6d8ec67c3e 100644 --- a/redash/query_runner/hive_ds.py +++ b/redash/query_runner/hive_ds.py @@ -55,6 +55,12 @@ def configuration_schema(cls): "username": { "type": "string" }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + } }, "order": ["host", "port", "database", "username"], "required": ["host"] @@ -98,14 +104,14 @@ def _get_connection(self): database=self.configuration.get('database', 'default'), username=self.configuration.get('username', None), ) - + return connection def run_query(self, query, user): connection = None try: - connection = self._get_connection() + connection = self._get_connection() cursor = connection.cursor() cursor.execute(query) @@ -214,7 +220,7 @@ def _get_connection(self): # create connection connection = hive.connect(thrift_transport=transport) - + return connection diff --git a/redash/query_runner/impala_ds.py b/redash/query_runner/impala_ds.py index 5b8b590777..b57bb0b7e6 100644 --- a/redash/query_runner/impala_ds.py +++ b/redash/query_runner/impala_ds.py @@ -64,6 +64,12 @@ def configuration_schema(cls): }, "timeout": { "type": "number" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, "required": ["host"], diff --git a/redash/query_runner/influx_db.py b/redash/query_runner/influx_db.py index 47f3a4201f..aee41318b8 100644 --- a/redash/query_runner/influx_db.py +++ b/redash/query_runner/influx_db.py @@ -57,6 +57,12 @@ def configuration_schema(cls): 'properties': { 'url': { 'type': 'string' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, 'required': ['url'] diff --git a/redash/query_runner/memsql_ds.py b/redash/query_runner/memsql_ds.py index bbec2836d4..b573b529ff 100644 --- a/redash/query_runner/memsql_ds.py +++ b/redash/query_runner/memsql_ds.py @@ -55,6 +55,12 @@ def configuration_schema(cls): }, "password": { "type": "string" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, diff --git a/redash/query_runner/mongodb.py b/redash/query_runner/mongodb.py index bfe40f485e..6b814faade 100644 --- a/redash/query_runner/mongodb.py +++ b/redash/query_runner/mongodb.py @@ -134,6 +134,12 @@ def configuration_schema(cls): 'type': 'string', 'title': 'Replica Set Name' }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, }, 'required': ['connectionString', 'dbName'] } diff --git a/redash/query_runner/mssql.py b/redash/query_runner/mssql.py index 007aa825b6..0cf25567a9 100644 --- a/redash/query_runner/mssql.py +++ b/redash/query_runner/mssql.py @@ -60,6 +60,12 @@ def configuration_schema(cls): "db": { "type": "string", "title": "Database Name" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, "required": ["db"], diff --git a/redash/query_runner/mysql.py b/redash/query_runner/mysql.py index 9c674aea4a..9815100b4e 100644 --- a/redash/query_runner/mysql.py +++ b/redash/query_runner/mysql.py @@ -54,6 +54,12 @@ def configuration_schema(cls): 'port': { 'type': 'number', 'default': 3306, + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, "order": ['host', 'port', 'user', 'passwd', 'db'], diff --git a/redash/query_runner/oracle.py b/redash/query_runner/oracle.py index eff9250042..8979ebb11b 100644 --- a/redash/query_runner/oracle.py +++ b/redash/query_runner/oracle.py @@ -63,6 +63,12 @@ def configuration_schema(cls): "servicename": { "type": "string", "title": "DSN Service Name" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, "required": ["servicename", "user", "password", "host", "port"], diff --git a/redash/query_runner/pg.py b/redash/query_runner/pg.py index 96aa03c07d..454f77040f 100644 --- a/redash/query_runner/pg.py +++ b/redash/query_runner/pg.py @@ -74,6 +74,12 @@ def configuration_schema(cls): "type": "string", "title": "SSL Mode", "default": "prefer" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, "order": ['host', 'port', 'user', 'password'], diff --git a/redash/query_runner/presto.py b/redash/query_runner/presto.py index 975ea70c07..a84e9f0106 100644 --- a/redash/query_runner/presto.py +++ b/redash/query_runner/presto.py @@ -56,6 +56,12 @@ def configuration_schema(cls): 'username': { 'type': 'string' }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, }, 'order': ['host', 'protocol', 'port', 'username', 'schema', 'catalog'], 'required': ['host'] diff --git a/redash/query_runner/python.py b/redash/query_runner/python.py index f6cc2fbcd9..4fdf0de626 100644 --- a/redash/query_runner/python.py +++ b/redash/query_runner/python.py @@ -55,6 +55,12 @@ def configuration_schema(cls): }, 'additionalModulesPaths': { 'type': 'string' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, } diff --git a/redash/query_runner/salesforce.py b/redash/query_runner/salesforce.py index 527f1e26ec..7222028fd0 100644 --- a/redash/query_runner/salesforce.py +++ b/redash/query_runner/salesforce.py @@ -81,6 +81,12 @@ def configuration_schema(cls): "type": "string", "title": "Salesforce API Version", "default": DEFAULT_API_VERSION + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, "required": ["username", "password", "token"], diff --git a/redash/query_runner/script.py b/redash/query_runner/script.py index 38e3ae62c5..1a4b80bdfd 100644 --- a/redash/query_runner/script.py +++ b/redash/query_runner/script.py @@ -49,6 +49,12 @@ def configuration_schema(cls): 'shell': { 'type': 'boolean', 'title': 'Execute command through the shell' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, 'required': ['path'] diff --git a/redash/query_runner/snowflake.py b/redash/query_runner/snowflake.py index 3bf2bd64aa..21fddf2af3 100644 --- a/redash/query_runner/snowflake.py +++ b/redash/query_runner/snowflake.py @@ -45,6 +45,12 @@ def configuration_schema(cls): }, "database": { "type": "string" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, "required": ["user", "password", "account", "database", "warehouse"], diff --git a/redash/query_runner/sqlite.py b/redash/query_runner/sqlite.py index c1933d81e6..9f02315e60 100644 --- a/redash/query_runner/sqlite.py +++ b/redash/query_runner/sqlite.py @@ -21,6 +21,12 @@ def configuration_schema(cls): "dbpath": { "type": "string", "title": "Database Path" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, "required": ["dbpath"], diff --git a/redash/query_runner/treasuredata.py b/redash/query_runner/treasuredata.py index 5e3673ed78..52ee2029c1 100644 --- a/redash/query_runner/treasuredata.py +++ b/redash/query_runner/treasuredata.py @@ -58,6 +58,12 @@ def configuration_schema(cls): 'type': 'boolean', 'title': 'Auto Schema Retrieval', 'default': False + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." } }, 'required': ['apikey','db'] diff --git a/redash/query_runner/vertica.py b/redash/query_runner/vertica.py index 92ab864c1a..92633eb23e 100644 --- a/redash/query_runner/vertica.py +++ b/redash/query_runner/vertica.py @@ -60,6 +60,12 @@ def configuration_schema(cls): "type": "number", "title": "Connection Timeout" }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, }, 'required': ['database'], 'order': ['host', 'port', 'user', 'password', 'database', 'read_timeout', 'connection_timeout'], @@ -117,7 +123,7 @@ def run_query(self, query, user): 'database': self.configuration.get('database', ''), 'read_timeout': self.configuration.get('read_timeout', 600) } - + if self.configuration.get('connection_timeout'): conn_info['connection_timeout'] = self.configuration.get('connection_timeout') diff --git a/tests/test_cli.py b/tests/test_cli.py index 3fb016f099..fa5e081a5b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -16,7 +16,7 @@ def test_interactive_new(self): result = runner.invoke( manager, ['ds', 'new'], - input="test\n%s\n\n\nexample.com\n\n\ntestdb\n" % (pg_i,)) + input="test\n%s\n\n\n\n\nexample.com\n\n\ntestdb\n" % (pg_i,)) self.assertFalse(result.exception) self.assertEqual(result.exit_code, 0) self.assertEqual(DataSource.query.count(), 1) From 42688c2b5baa80b6d348029708a5722ebf792327 Mon Sep 17 00:00:00 2001 From: Alison Date: Thu, 22 Jun 2017 10:02:11 -0500 Subject: [PATCH 10/32] give warning/error msg on inaccurate graph config (re #57) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I tried to make `JSON.stringify(this.visualization.options.columnMapping)` a variable to avoid repeating it, but if I make it a `let` the linter throws an error and if I make it a `const` then it doesn’t change with the UI and the logic doesn’t work. :( updated based on PR comments --- .../edit-visualization-dialog.css | 5 +++++ .../edit-visualization-dialog.html | 10 ++++++++- .../edit-visualization-dialog.js | 21 +++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 client/app/visualizations/edit-visualization-dialog.css diff --git a/client/app/visualizations/edit-visualization-dialog.css b/client/app/visualizations/edit-visualization-dialog.css new file mode 100644 index 0000000000..3e84b755b2 --- /dev/null +++ b/client/app/visualizations/edit-visualization-dialog.css @@ -0,0 +1,5 @@ +/* Edit Visualization Dialog specific CSS */ + +.slight-padding { + padding: 5px; +} \ No newline at end of file diff --git a/client/app/visualizations/edit-visualization-dialog.html b/client/app/visualizations/edit-visualization-dialog.html index 28791ee2ca..4d9b531b5c 100644 --- a/client/app/visualizations/edit-visualization-dialog.html +++ b/client/app/visualizations/edit-visualization-dialog.html @@ -34,10 +34,18 @@

    Visualization Editor

    +
    +
    +
    +
    +
    +
    +
    +
    - +
    diff --git a/client/app/visualizations/edit-visualization-dialog.js b/client/app/visualizations/edit-visualization-dialog.js index c3d61b6b03..1d695de21d 100644 --- a/client/app/visualizations/edit-visualization-dialog.js +++ b/client/app/visualizations/edit-visualization-dialog.js @@ -1,6 +1,7 @@ import { map } from 'lodash'; import { copy } from 'angular'; import template from './edit-visualization-dialog.html'; +import './edit-visualization-dialog.css'; const EditVisualizationDialog = { template, @@ -21,6 +22,8 @@ const EditVisualizationDialog = { // Don't allow to change type after creating visualization this.canChangeType = !(this.visualization && this.visualization.id); + this.warning_three_column_groupby = 'You have more than 2 columns in your result set. To ensure the chart is accurate, please do one of the following:
    • Change the SQL query to give 2 result columns. You can CONCAT() columns together if you wish.
    • Select column(s) to group by.
    '; + this.warning_three_column_stacking = 'You have more than 2 columns in your result set. You may wish to make the Stacking option equal to `Enabled` or `Percent`.'; this.newVisualization = () => ({ type: Visualization.defaultVisualization.type, @@ -46,6 +49,24 @@ const EditVisualizationDialog = { } }; + this.has3plusColumnsFunction = () => { + let has3plusColumns = false; + if ((JSON.stringify(this.visualization.options.columnMapping).match(/,/g) || []).length > 2) { + has3plusColumns = true; + } + return has3plusColumns; + }; + + this.disableSubmit = () => { + if (this.visualization.options.globalSeriesType === 'column' + && this.has3plusColumnsFunction() + && !JSON.stringify(this.visualization.options.columnMapping).includes('"":') + && JSON.stringify(this.visualization.options.columnMapping).includes('unused')) { + return true; + } + return false; + }; + this.submit = () => { if (this.visualization.id) { Events.record('update', 'visualization', this.visualization.id, { type: this.visualization.type }); From 424b9f8dca2238d3973589bbfa34f59b45c904d7 Mon Sep 17 00:00:00 2001 From: Alison Date: Fri, 28 Jul 2017 06:36:19 -0500 Subject: [PATCH 11/32] Add column type info to query runners (re #152, #23) --- redash/query_runner/athena.py | 4 ++-- redash/query_runner/mysql.py | 5 +++-- redash/query_runner/pg.py | 5 ++++- redash/query_runner/presto.py | 4 ++-- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/redash/query_runner/athena.py b/redash/query_runner/athena.py index eb2f619ad3..b549d8e97c 100644 --- a/redash/query_runner/athena.py +++ b/redash/query_runner/athena.py @@ -147,7 +147,7 @@ def get_schema(self, get_stats=False): schema = {} query = """ - SELECT table_schema, table_name, column_name + SELECT table_schema, table_name, column_name, data_type as column_type FROM information_schema.columns WHERE table_schema NOT IN ('information_schema') """ @@ -161,7 +161,7 @@ def get_schema(self, get_stats=False): table_name = '{0}.{1}'.format(row['table_schema'], row['table_name']) if table_name not in schema: schema[table_name] = {'name': table_name, 'columns': []} - schema[table_name]['columns'].append(row['column_name']) + schema[table_name]['columns'].append(row['column_name'] + ' (' + row['column_type'] + ')') return schema.values() diff --git a/redash/query_runner/mysql.py b/redash/query_runner/mysql.py index 9815100b4e..3f5f9d310b 100644 --- a/redash/query_runner/mysql.py +++ b/redash/query_runner/mysql.py @@ -106,7 +106,8 @@ def _get_tables(self, schema): query = """ SELECT col.table_schema as table_schema, col.table_name as table_name, - col.column_name as column_name + col.column_name as column_name, + col.column_type as column_type FROM `information_schema`.`columns` col WHERE col.table_schema NOT IN ('information_schema', 'performance_schema', 'mysql', 'sys'); """ @@ -127,7 +128,7 @@ def _get_tables(self, schema): if table_name not in schema: schema[table_name] = {'name': table_name, 'columns': []} - schema[table_name]['columns'].append(row['column_name']) + schema[table_name]['columns'].append(row['column_name'] + ' (' + row['column_type'] + ')') return schema.values() diff --git a/redash/query_runner/pg.py b/redash/query_runner/pg.py index 454f77040f..356a4d773c 100644 --- a/redash/query_runner/pg.py +++ b/redash/query_runner/pg.py @@ -108,7 +108,7 @@ def _get_definitions(self, schema, query): if table_name not in schema: schema[table_name] = {'name': table_name, 'columns': []} - schema[table_name]['columns'].append(row['column_name']) + schema[table_name]['columns'].append(row['column_name'] + ' (' + row['column_type'] + ')') def _get_tables(self, schema): ''' @@ -128,6 +128,7 @@ def _get_tables(self, schema): query = """ SELECT s.nspname as table_schema, c.relname as table_name, + t.typname as column_type, a.attname as column_name FROM pg_class c JOIN pg_namespace s @@ -137,6 +138,8 @@ def _get_tables(self, schema): ON a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped + JOIN pg_type t + ON c.reltype = t.oid WHERE c.relkind IN ('r', 'v', 'm', 'f', 'p') """ diff --git a/redash/query_runner/presto.py b/redash/query_runner/presto.py index a84e9f0106..5d84b1e803 100644 --- a/redash/query_runner/presto.py +++ b/redash/query_runner/presto.py @@ -78,7 +78,7 @@ def type(cls): def get_schema(self, get_stats=False): schema = {} query = """ - SELECT table_schema, table_name, column_name + SELECT table_schema, table_name, column_name, data_type as column_type FROM information_schema.columns WHERE table_schema NOT IN ('pg_catalog', 'information_schema') """ @@ -96,7 +96,7 @@ def get_schema(self, get_stats=False): if table_name not in schema: schema[table_name] = {'name': table_name, 'columns': []} - schema[table_name]['columns'].append(row['column_name']) + schema[table_name]['columns'].append(row['column_name'] + ' (' + row['column_type'] + ')') return schema.values() From 25a5f943578db38c4761af3868381b94dacd75bc Mon Sep 17 00:00:00 2001 From: Alison Date: Fri, 11 Aug 2017 20:36:18 -0500 Subject: [PATCH 12/32] Add last_active_at column to users page (re #155) --- client/app/pages/users/list.html | 7 +++++++ redash/models.py | 2 ++ 2 files changed, 9 insertions(+) diff --git a/client/app/pages/users/list.html b/client/app/pages/users/list.html index bef079d785..00bd49a536 100644 --- a/client/app/pages/users/list.html +++ b/client/app/pages/users/list.html @@ -46,6 +46,10 @@ Joined + + Last Active At + + @@ -62,6 +66,9 @@ + + +
    diff --git a/redash/models.py b/redash/models.py index 731757241d..9b0122d57b 100644 --- a/redash/models.py +++ b/redash/models.py @@ -486,6 +486,8 @@ def to_dict(self, with_api_key=False): if with_api_key: d['api_key'] = self.api_key + d['last_active_at'] = Event.query.filter(Event.user_id == self.id).with_entities(Event.created_at).order_by(Event.created_at.desc()).first() + return d def is_api_user(self): From d8c8eea0260cb989b7308a9457a02039d2a0f6fe Mon Sep 17 00:00:00 2001 From: Allen Short Date: Sat, 9 Dec 2017 05:48:56 +0000 Subject: [PATCH 13/32] Add partition key marker to Athena and Presto columns (re #185) --- redash/query_runner/athena.py | 14 ++++++++++++-- redash/query_runner/presto.py | 17 +++++++++++++++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/redash/query_runner/athena.py b/redash/query_runner/athena.py index b549d8e97c..b9a9944956 100644 --- a/redash/query_runner/athena.py +++ b/redash/query_runner/athena.py @@ -147,9 +147,10 @@ def get_schema(self, get_stats=False): schema = {} query = """ - SELECT table_schema, table_name, column_name, data_type as column_type + SELECT table_schema, table_name, column_name, data_type as column_type, comment as extra_info FROM information_schema.columns WHERE table_schema NOT IN ('information_schema') + ORDER BY 1, 5 DESC """ results, error = self.run_query(query, None) @@ -161,7 +162,16 @@ def get_schema(self, get_stats=False): table_name = '{0}.{1}'.format(row['table_schema'], row['table_name']) if table_name not in schema: schema[table_name] = {'name': table_name, 'columns': []} - schema[table_name]['columns'].append(row['column_name'] + ' (' + row['column_type'] + ')') + + if row['extra_info'] == 'Partition Key': + schema[table_name]['columns'].append('[P] ' + row['column_name'] + ' (' + row['column_type'] + ')') + elif row['column_type'] == 'integer' or row['column_type'] == 'varchar' or row['column_type'] == 'timestamp' or row['column_type'] == 'boolean' or row['column_type'] == 'bigint': + schema[table_name]['columns'].append(row['column_name'] + ' (' + row['column_type'] + ')') + elif row['column_type'][0:2] == 'row' or row['column_type'][0:2] == 'map' or row['column_type'][0:2] == 'arr': + schema[table_name]['columns'].append(row['column_name'] + ' (row or map or array)') + else: + schema[table_name]['columns'].append(row['column_name']) + return schema.values() diff --git a/redash/query_runner/presto.py b/redash/query_runner/presto.py index 5d84b1e803..570d003cd7 100644 --- a/redash/query_runner/presto.py +++ b/redash/query_runner/presto.py @@ -1,3 +1,5 @@ +from markupsafe import escape + from redash.query_runner import * from redash.utils import json_dumps, json_loads @@ -78,9 +80,10 @@ def type(cls): def get_schema(self, get_stats=False): schema = {} query = """ - SELECT table_schema, table_name, column_name, data_type as column_type + SELECT table_schema, table_name, column_name, data_type as column_type, extra_info FROM information_schema.columns WHERE table_schema NOT IN ('pg_catalog', 'information_schema') + ORDER BY 1, 5 DESC """ results, error = self.run_query(query, None) @@ -96,7 +99,14 @@ def get_schema(self, get_stats=False): if table_name not in schema: schema[table_name] = {'name': table_name, 'columns': []} - schema[table_name]['columns'].append(row['column_name'] + ' (' + row['column_type'] + ')') + if row['extra_info'] == 'partition key': + schema[table_name]['columns'].append('[P] ' + row['column_name'] + ' (' + row['column_type'] + ')') + elif row['column_type'] == 'integer' or row['column_type'] == 'varchar' or row['column_type'] == 'timestamp' or row['column_type'] == 'boolean' or row['column_type'] == 'bigint': + schema[table_name]['columns'].append(row['column_name'] + ' (' + row['column_type'] + ')') + elif row['column_type'][0:2] == 'row' or row['column_type'][0:2] == 'map' or row['column_type'][0:2] == 'arr': + schema[table_name]['columns'].append(row['column_name'] + ' (row or map or array)') + else: + schema[table_name]['columns'].append(row['column_name']) return schema.values() @@ -117,6 +127,9 @@ def run_query(self, query, user): column_tuples = [(i[0], PRESTO_TYPES_MAPPING.get(i[1], None)) for i in cursor.description] columns = self.fetch_columns(column_tuples) rows = [dict(zip(([c['name'] for c in columns]), r)) for i, r in enumerate(cursor.fetchall())] + for row in rows: + for field in row: + field = escape(field) data = {'columns': columns, 'rows': rows} json_data = json_dumps(data) error = None From 894128db6dca26823bdefdb87740be59ebfc1186 Mon Sep 17 00:00:00 2001 From: Allen Short Date: Wed, 6 Sep 2017 20:29:50 +0000 Subject: [PATCH 14/32] Run queries with no cached result in public dashboards (re #220) --- redash/serializers.py | 18 ++++++++++++++---- tests/handlers/test_embed.py | 12 ++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/redash/serializers.py b/redash/serializers.py index 279806e93a..75325c341f 100644 --- a/redash/serializers.py +++ b/redash/serializers.py @@ -8,6 +8,7 @@ from flask_login import current_user from redash import models +from redash.handlers.query_results import run_query_sync from redash.permissions import has_access, view_only from redash.utils import json_loads @@ -22,8 +23,16 @@ def public_widget(widget): 'created_at': widget.created_at } - if widget.visualization and widget.visualization.id: - query_data = models.QueryResult.query.get(widget.visualization.query_rel.latest_query_data_id).to_dict() + if (widget.visualization and + widget.visualization.id and + widget.visualization.query_rel is not None): + q = widget.visualization.query_rel + # make sure the widget's query has a latest_query_data_id that is + # not null so public dashboards work + if q.latest_query_data_id is None: + run_query_sync(q.data_source, {}, q.query_text) + + query_data = q.latest_query_data.to_dict() res['visualization'] = { 'type': widget.visualization.type, 'name': widget.visualization.name, @@ -32,9 +41,10 @@ def public_widget(widget): 'updated_at': widget.visualization.updated_at, 'created_at': widget.visualization.created_at, 'query': { + 'id': q.id, 'query': ' ', # workaround, as otherwise the query data won't be loaded. - 'name': widget.visualization.query_rel.name, - 'description': widget.visualization.query_rel.description, + 'name': q.name, + 'description': q.description, 'options': {}, 'latest_query_data': query_data } diff --git a/tests/handlers/test_embed.py b/tests/handlers/test_embed.py index 18f119d786..905a6f8672 100644 --- a/tests/handlers/test_embed.py +++ b/tests/handlers/test_embed.py @@ -1,5 +1,8 @@ +import mock + from tests import BaseTestCase from redash.models import db +from redash.query_runner.pg import PostgreSQL class TestEmbedVisualization(BaseTestCase): @@ -97,6 +100,15 @@ def test_inactive_token(self): res = self.make_request('get', '/api/dashboards/public/{}'.format(api_key.api_key), user=False, is_json=False) self.assertEqual(res.status_code, 404) + def test_dashboard_widgets(self): + dashboard = self.factory.create_dashboard() + w1 = self.factory.create_widget(dashboard=dashboard) + w2 = self.factory.create_widget(dashboard=dashboard, visualization=None, text="a text box") + api_key = self.factory.create_api_key(object=dashboard) + with mock.patch.object(PostgreSQL, "run_query") as qr: + qr.return_value = ("[1, 2]", None) + res = self.make_request('get', '/api/dashboards/public/{}'.format(api_key.api_key), user=False, is_json=False) + self.assertEqual(res.status_code, 200) # Not relevant for now, as tokens in api_keys table are only created for dashboards. Once this changes, we should # add this test. # def test_token_doesnt_belong_to_dashboard(self): From 2e4a17da481604a7e3152761e9cea8e2f9274e54 Mon Sep 17 00:00:00 2001 From: Allen Short Date: Wed, 27 Sep 2017 21:18:32 +0000 Subject: [PATCH 15/32] secure cookies, add X-Content-Type-Options header (bug 1371613) --- redash/__init__.py | 5 +++++ redash/settings/__init__.py | 1 + 2 files changed, 6 insertions(+) diff --git a/redash/__init__.py b/redash/__init__.py index 942550908c..963e6fa6d8 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -127,6 +127,11 @@ def create_app(load_admin=True): app.config['SQLALCHEMY_DATABASE_URI'] = settings.SQLALCHEMY_DATABASE_URI app.config.update(settings.all_settings()) + def set_response_headers(response): + response.headers['X-Content-Type-Options'] = 'nosniff' + return response + + app.after_request(set_response_headers) provision_app(app) db.init_app(app) migrate.init_app(app, db) diff --git a/redash/settings/__init__.py b/redash/settings/__init__.py index ef23e5e8e3..b14663b101 100644 --- a/redash/settings/__init__.py +++ b/redash/settings/__init__.py @@ -15,6 +15,7 @@ def all_settings(): return settings +SESSION_COOKIE_SECURE = True REDIS_URL = os.environ.get('REDASH_REDIS_URL', os.environ.get('REDIS_URL', "redis://localhost:6379/0")) PROXIES_COUNT = int(os.environ.get('REDASH_PROXIES_COUNT', "1")) From 7327d9eb2ef378bc65fc1de7613cb7a7fc02ab4a Mon Sep 17 00:00:00 2001 From: Allen Short Date: Tue, 12 Dec 2017 04:47:08 +0000 Subject: [PATCH 16/32] Merge mozilla schema updates with schema from master --- migrations/versions/40384fa03dd1_.py | 40 ++++++++++++++++++++++++++++ migrations/versions/58f810489c47_.py | 28 +++++++++++++++++++ migrations/versions/f9571a5ab4f3_.py | 28 +++++++++++++++++++ migrations/versions/fbc0849e2674_.py | 26 ++++++++++++++++++ 4 files changed, 122 insertions(+) create mode 100644 migrations/versions/40384fa03dd1_.py create mode 100644 migrations/versions/58f810489c47_.py create mode 100644 migrations/versions/f9571a5ab4f3_.py create mode 100644 migrations/versions/fbc0849e2674_.py diff --git a/migrations/versions/40384fa03dd1_.py b/migrations/versions/40384fa03dd1_.py new file mode 100644 index 0000000000..f2c53711c0 --- /dev/null +++ b/migrations/versions/40384fa03dd1_.py @@ -0,0 +1,40 @@ +"""Upgrade 'data_scanned' column to form used in upstream + +Revision ID: 40384fa03dd1 +Revises: 58f810489c47 +Create Date: 2018-01-18 18:44:04.917081 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.sql.expression import func, cast + +# revision identifiers, used by Alembic. +revision = '40384fa03dd1' +down_revision = 'fbc0849e2674' +branch_labels = None +depends_on = None + + +def upgrade(): + qr = sa.sql.table('query_results', + sa.sql.column('data_scanned', sa.String), + sa.sql.column('data', sa.String)) + op.execute( + qr.update() + .where(qr.c.data_scanned != '') + .where(qr.c.data_scanned != 'error') + .where(qr.c.data_scanned != 'N/A') + .values(data=cast( + func.jsonb_set(cast(qr.c.data, JSONB), + '{metadata}', + cast('{"data_scanned": ' + + qr.c.data_scanned + '}', + JSONB)), + sa.String))) + op.drop_column('query_results', 'data_scanned') + + +def downgrade(): + op.add_column('query_results', sa.Column('data_scanned', sa.String(length=255), nullable=True)) diff --git a/migrations/versions/58f810489c47_.py b/migrations/versions/58f810489c47_.py new file mode 100644 index 0000000000..1ed4190288 --- /dev/null +++ b/migrations/versions/58f810489c47_.py @@ -0,0 +1,28 @@ +"""add 'data_scanned' column to query_results + +Revision ID: 58f810489c47 +Revises: eb2f788f997e +Create Date: 2017-06-25 21:24:54.942119 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '58f810489c47' +down_revision = 'eb2f788f997e' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('query_results', sa.Column('data_scanned', sa.String(length=255), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('query_results', 'data_scanned') + # ### end Alembic commands ### diff --git a/migrations/versions/f9571a5ab4f3_.py b/migrations/versions/f9571a5ab4f3_.py new file mode 100644 index 0000000000..da1ba02d6d --- /dev/null +++ b/migrations/versions/f9571a5ab4f3_.py @@ -0,0 +1,28 @@ +"""Rename 'image_url' to 'profile_image_url' + + a revision was changed after we pulled it from upstream in m12, so it had to + be fixed here. + + +Revision ID: f9571a5ab4f3 +Revises: 40384fa03dd1 +Create Date: 2018-01-18 18:04:07.943843 +""" +from alembic import op + + +# revision identifiers, used by Alembic. +revision = 'f9571a5ab4f3' +down_revision = '40384fa03dd1' +branch_labels = None +depends_on = None + + +def upgrade(): + # Upstream changed the column name in migration revision 7671dca4e604 -- + # see git revision 62e5e3892603502c5f3a6da277c33c73510b8819 + op.alter_column('users', 'image_url', new_column_name='profile_image_url') + + +def downgrade(): + op.alter_column('users', 'profile_image_url', new_column_name='image_url') diff --git a/migrations/versions/fbc0849e2674_.py b/migrations/versions/fbc0849e2674_.py new file mode 100644 index 0000000000..6195141496 --- /dev/null +++ b/migrations/versions/fbc0849e2674_.py @@ -0,0 +1,26 @@ +""" +Merge upstream fulltext search + +This formerly merged the fulltext search changes (6b5be7e0a0ef, 5ec5c84ba61e) +with upstream's 7671dca4e604 - but then those changes moved in the revision +graph to be direct descendants of that upstream revision, so the merge point +has been moved. + +Revision ID: fbc0849e2674 +Revises: 6b5be7e0a0ef, eb2f788f997e +Create Date: 2017-12-12 04:45:34.360587 +""" + +# revision identifiers, used by Alembic. +revision = 'fbc0849e2674' +down_revision = ('6b5be7e0a0ef', '58f810489c47') +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass From b0c5bd6fc8b2da8c6e58beb0ebc42a0869752cf8 Mon Sep 17 00:00:00 2001 From: Allen Short Date: Wed, 14 Feb 2018 17:52:43 +0000 Subject: [PATCH 17/32] merge upstream db changes --- migrations/versions/15041b7085fe_.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 migrations/versions/15041b7085fe_.py diff --git a/migrations/versions/15041b7085fe_.py b/migrations/versions/15041b7085fe_.py new file mode 100644 index 0000000000..fcb10aa78f --- /dev/null +++ b/migrations/versions/15041b7085fe_.py @@ -0,0 +1,24 @@ +"""empty message + +Revision ID: 15041b7085fe +Revises: f9571a5ab4f3, 969126bd800f +Create Date: 2018-02-14 17:52:17.138127 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '15041b7085fe' +down_revision = ('f9571a5ab4f3', '969126bd800f') +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass From bf9a5531572b9b15c2e77c5a8ba409739952022d Mon Sep 17 00:00:00 2001 From: Allen Short Date: Wed, 21 Mar 2018 20:38:48 +0000 Subject: [PATCH 18/32] properly rollback failed db commits --- redash/handlers/dashboards.py | 5 +++++ redash/handlers/data_sources.py | 2 ++ redash/handlers/users.py | 3 ++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/redash/handlers/dashboards.py b/redash/handlers/dashboards.py index 19da96550b..ffa47a41b4 100644 --- a/redash/handlers/dashboards.py +++ b/redash/handlers/dashboards.py @@ -10,6 +10,7 @@ from redash.permissions import (can_modify, require_admin_or_owner, require_object_modify_permission, require_permission) +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm.exc import StaleDataError @@ -198,7 +199,11 @@ def post(self, dashboard_slug): try: models.db.session.commit() except StaleDataError: + models.db.session.rollback() abort(409) + except IntegrityError: + models.db.session.rollback() + abort(400) result = serialize_dashboard(dashboard, with_widgets=True, user=self.current_user) diff --git a/redash/handlers/data_sources.py b/redash/handlers/data_sources.py index 65532ee509..a13854fd8d 100644 --- a/redash/handlers/data_sources.py +++ b/redash/handlers/data_sources.py @@ -55,6 +55,7 @@ def post(self, data_source_id): try: models.db.session.commit() except IntegrityError as e: + models.db.session.rollback() if req['name'] in e.message: abort(400, message="Data source with the name {} already exists.".format(req['name'])) @@ -130,6 +131,7 @@ def post(self): models.db.session.commit() except IntegrityError as e: + models.db.session.rollback() if req['name'] in e.message: abort(400, message="Data source with the name {} already exists.".format(req['name'])) diff --git a/redash/handlers/users.py b/redash/handlers/users.py index e7244de4b8..a008da9d57 100644 --- a/redash/handlers/users.py +++ b/redash/handlers/users.py @@ -105,6 +105,7 @@ def post(self): models.db.session.add(user) models.db.session.commit() except IntegrityError as e: + models.db.session.rollback() if "email" in e.message: abort(400, message='Email already taken.') abort(500) @@ -199,7 +200,7 @@ def post(self, user_id): message = "Email already taken." else: message = "Error updating record" - + models.db.session.rollback() abort(400, message=message) self.record_event({ From 84e6973402d26fcdab2320996037da11b4cd66f1 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Wed, 28 Feb 2018 21:14:30 +0100 Subject: [PATCH 19/32] Install redash-stmo. In the long run we'll be able to install additional dependencies by having an own Dockerfile to build images based on the Redash image but that installs additional Python dependencies. But until we have a fork with lots of changes ourselves we need to do it this way. Redash-stmo contains the ability to hook up our own Dockerflow library. Refs #13 Refs #37 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 8ad41077df..bbde299daa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,3 +54,4 @@ disposable-email-domains # Uncomment the requirement for ldap3 if using ldap. # It is not included by default because of the GPL license conflict. # ldap3==2.2.4 +redash-stmo>=2018.4.0 From c266c95663b97effaebfa0444bc7c21bbb5d32b4 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 23 Mar 2018 05:45:39 +0100 Subject: [PATCH 20/32] Extend the Remote User Auth backend with REMOTE_GROUPS ability (#311) Extend the Remote User Auth backend with the ability to pass remote user groups via a configurable request header similar to the REMOTE_USER header. Refs #37. If enabled the feature allows checks the header value against a configured list of group names, including the ability to use UNIX shell-style wildcards. --- redash/authentication/remote_user_auth.py | 15 +++++++++++++++ redash/settings/__init__.py | 7 +++++++ redash/settings/helpers.py | 2 +- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/redash/authentication/remote_user_auth.py b/redash/authentication/remote_user_auth.py index 77002e9324..61dfd793d0 100644 --- a/redash/authentication/remote_user_auth.py +++ b/redash/authentication/remote_user_auth.py @@ -31,6 +31,21 @@ def login(org_slug=None): logger.error("Cannot use remote user for login when it's not provided in the request (looked in headers['" + settings.REMOTE_USER_HEADER + "'])") return redirect(url_for('redash.index', next=next_path, org_slug=org_slug)) + # Check if there is a header of user groups and if yes + # check it against a list of allowed user groups from the settings + if settings.REMOTE_GROUPS_ENABLED: + remote_groups = settings.set_from_string( + request.headers.get(settings.REMOTE_GROUPS_HEADER) or '' + ) + allowed_groups = settings.REMOTE_GROUPS_ALLOWED + if not allowed_groups.intersection(remote_groups): + logger.error( + "User groups provided in the %s header are not " + "matching the allowed groups.", + settings.REMOTE_GROUPS_HEADER + ) + return redirect(url_for('redash.index', next=next_path)) + logger.info("Logging in " + email + " via remote user") user = create_and_login_user(current_org, email, email) diff --git a/redash/settings/__init__.py b/redash/settings/__init__.py index b14663b101..605ba37013 100644 --- a/redash/settings/__init__.py +++ b/redash/settings/__init__.py @@ -84,6 +84,13 @@ def all_settings(): REMOTE_USER_LOGIN_ENABLED = parse_boolean(os.environ.get("REDASH_REMOTE_USER_LOGIN_ENABLED", "false")) REMOTE_USER_HEADER = os.environ.get("REDASH_REMOTE_USER_HEADER", "X-Forwarded-Remote-User") +# When enabled this will match the given remote groups request header with a +# configured list of allowed user groups using UNIX shell-style wildcards such +# as * and ?. +REMOTE_GROUPS_ENABLED = parse_boolean(os.environ.get("REDASH_REMOTE_GROUPS_ENABLED", "false")) +REMOTE_GROUPS_HEADER = os.environ.get("REDASH_REMOTE_GROUPS_HEADER", "X-Forwarded-Remote-Groups") +REMOTE_GROUPS_ALLOWED = set_from_string(os.environ.get("REDASH_REMOTE_GROUPS_ALLOWED", "")) + # If the organization setting auth_password_login_enabled is not false, then users will still be # able to login through Redash instead of the LDAP server LDAP_LOGIN_ENABLED = parse_boolean(os.environ.get('REDASH_LDAP_LOGIN_ENABLED', 'false')) diff --git a/redash/settings/helpers.py b/redash/settings/helpers.py index 98946d81e4..4d6f84185b 100644 --- a/redash/settings/helpers.py +++ b/redash/settings/helpers.py @@ -11,7 +11,7 @@ def array_from_string(s): if "" in array: array.remove("") - return array + return [item.strip() for item in array] def set_from_string(s): From c9f672b2fde2feb6bac9b8847b6b87d26441cab8 Mon Sep 17 00:00:00 2001 From: Allen Short Date: Tue, 16 Jan 2018 22:06:32 +0000 Subject: [PATCH 21/32] Unique names for query parameters (re #164) --- client/app/services/query.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/client/app/services/query.js b/client/app/services/query.js index b78e261a40..5f4c69e5f3 100644 --- a/client/app/services/query.js +++ b/client/app/services/query.js @@ -146,7 +146,7 @@ class Parameter { }; } return { - [`p_${this.name}`]: this.value, + [`p_${this.name}_${this.queryId}`]: this.value, }; } @@ -158,7 +158,7 @@ class Parameter { this.setValue([query[keyStart], query[keyEnd]]); } } else { - const key = `p_${this.name}`; + const key = `p_${this.name}_${this.queryId}`; if (has(query, key)) { this.setValue(query[key]); } @@ -221,7 +221,9 @@ class Parameters { }); const parameterExists = p => includes(parameterNames, p.name); - this.query.options.parameters = this.query.options.parameters.filter(parameterExists).map(p => new Parameter(p)); + this.query.options.parameters = this.query.options.parameters + .filter(parameterExists) + .map(p => new Parameter(Object.assign({ queryId: this.query.id }, p))); } initFromQueryString(query) { @@ -486,7 +488,7 @@ function QueryResource( params += '&'; } - params += `p_${encodeURIComponent(name)}=${encodeURIComponent(value)}`; + params += `p_${encodeURIComponent(name)}_${this.id}=${encodeURIComponent(value)}`; }); } From 3798b628e47aa4d7f791f6daf1c73d013bb0f357 Mon Sep 17 00:00:00 2001 From: Allen Short Date: Tue, 27 Mar 2018 20:58:26 +0000 Subject: [PATCH 22/32] Aggregate query results (re #35) (#339) --- .../components/queries/schedule-dialog.html | 3 + .../app/components/queries/schedule-dialog.js | 13 ++- client/app/pages/queries/view.js | 1 + client/app/services/query-result.js | 10 +++ client/app/services/query.js | 6 +- migrations/versions/9d7678c47452_.py | 34 ++++++++ redash/handlers/api.py | 3 +- redash/handlers/queries.py | 3 + redash/handlers/query_results.py | 27 +++++++ redash/models.py | 53 +++++++++++- redash/serializers.py | 1 + redash/tasks/queries.py | 1 + tests/factories.py | 7 +- tests/handlers/test_queries.py | 80 +++++++++++++++++++ tests/test_models.py | 64 +++++++++++++-- 15 files changed, 293 insertions(+), 13 deletions(-) create mode 100644 migrations/versions/9d7678c47452_.py diff --git a/client/app/components/queries/schedule-dialog.html b/client/app/components/queries/schedule-dialog.html index f9344238a1..aca492cdfe 100644 --- a/client/app/components/queries/schedule-dialog.html +++ b/client/app/components/queries/schedule-dialog.html @@ -19,4 +19,7 @@

    Refresh Schedule

    Stop scheduling at date/time (format yyyy-MM-ddTHH:mm:ss, like 2016-12-28T14:57:00): +
    diff --git a/client/app/components/queries/schedule-dialog.js b/client/app/components/queries/schedule-dialog.js index b71cdff67f..4e1de82915 100644 --- a/client/app/components/queries/schedule-dialog.js +++ b/client/app/components/queries/schedule-dialog.js @@ -114,11 +114,21 @@ function scheduleUntil() { }; } +function scheduleKeepResults() { + return { + restrict: 'E', + scope: { + query: '=', + saveQuery: '=', + }, + template: '', + }; +} + const ScheduleForm = { controller() { this.query = this.resolve.query; this.saveQuery = this.resolve.saveQuery; - if (this.query.hasDailySchedule()) { this.refreshType = 'daily'; } else { @@ -137,6 +147,7 @@ export default function init(ngModule) { ngModule.directive('queryTimePicker', queryTimePicker); ngModule.directive('queryRefreshSelect', queryRefreshSelect); ngModule.directive('scheduleUntil', scheduleUntil); + ngModule.directive('scheduleKeepResults', scheduleKeepResults); ngModule.component('scheduleDialog', ScheduleForm); } diff --git a/client/app/pages/queries/view.js b/client/app/pages/queries/view.js index 29d2899899..dc996df012 100644 --- a/client/app/pages/queries/view.js +++ b/client/app/pages/queries/view.js @@ -209,6 +209,7 @@ function QueryViewCtrl( } else { request = pick($scope.query, [ 'schedule', + 'schedule_resultset_size', 'query', 'id', 'description', diff --git a/client/app/services/query-result.js b/client/app/services/query-result.js index 1ccb31439c..9cb6bac99e 100644 --- a/client/app/services/query-result.js +++ b/client/app/services/query-result.js @@ -52,6 +52,7 @@ function addPointToSeries(point, seriesCollection, seriesName) { function QueryResultService($resource, $timeout, $q, QueryResultError) { const QueryResultResource = $resource('api/query_results/:id', { id: '@id' }, { post: { method: 'POST' } }); + const QueryResultSetResource = $resource('api/queries/:id/resultset', { id: '@id' }); const Job = $resource('api/jobs/:id', { id: '@id' }); const statuses = { 1: 'waiting', @@ -461,6 +462,15 @@ function QueryResultService($resource, $timeout, $q, QueryResultError) { return queryResult; } + static getResultSet(queryId) { + const queryResult = new QueryResult(); + + QueryResultSetResource.get({ id: queryId }, (response) => { + queryResult.update(response); + }); + + return queryResult; + } loadResult(tryCount) { this.isLoadingResult = true; QueryResultResource.get( diff --git a/client/app/services/query.js b/client/app/services/query.js index 5f4c69e5f3..add63edde8 100644 --- a/client/app/services/query.js +++ b/client/app/services/query.js @@ -451,7 +451,11 @@ function QueryResource( this.latest_query_data_id = null; } - if (this.latest_query_data && maxAge !== 0) { + if (this.schedule_resultset_size) { + if (!this.queryResult) { + this.queryResult = QueryResult.getResultSet(this.id); + } + } else if (this.latest_query_data && maxAge !== 0) { if (!this.queryResult) { this.queryResult = new QueryResult({ query_result: this.latest_query_data, diff --git a/migrations/versions/9d7678c47452_.py b/migrations/versions/9d7678c47452_.py new file mode 100644 index 0000000000..d351153c87 --- /dev/null +++ b/migrations/versions/9d7678c47452_.py @@ -0,0 +1,34 @@ +"""Incremental query results aggregation + +Revision ID: 9d7678c47452 +Revises: 15041b7085fe +Create Date: 2018-03-08 04:36:12.802199 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '9d7678c47452' +down_revision = '15041b7085fe' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table('query_resultsets', + sa.Column('query_id', sa.Integer(), nullable=False), + sa.Column('result_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['query_id'], ['queries.id'], ), + sa.ForeignKeyConstraint(['result_id'], ['query_results.id'], ), + sa.PrimaryKeyConstraint('query_id', 'result_id') + ) + op.add_column(u'queries', sa.Column('schedule_resultset_size', sa.Integer(), nullable=True)) +1 + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column(u'queries', 'schedule_resultset_size') + op.drop_table('query_resultsets') + # ### end Alembic commands ### diff --git a/redash/handlers/api.py b/redash/handlers/api.py index 87c6ccc14a..49884e46e0 100644 --- a/redash/handlers/api.py +++ b/redash/handlers/api.py @@ -10,7 +10,7 @@ from redash.handlers.data_sources import DataSourceTypeListResource, DataSourceListResource, DataSourceSchemaResource, DataSourceResource, DataSourcePauseResource, DataSourceTestResource from redash.handlers.events import EventsResource from redash.handlers.queries import QueryForkResource, QueryRefreshResource, QueryListResource, QueryRecentResource, QuerySearchResource, QueryResource, MyQueriesResource, QueryVersionListResource, ChangeResource -from redash.handlers.query_results import QueryResultListResource, QueryResultResource, JobResource +from redash.handlers.query_results import QueryResultListResource, QueryResultResource, JobResource, QueryResultSetResource from redash.handlers.users import UserResource, UserListResource, UserInviteResource, UserResetPasswordResource, UserDisableResource from redash.handlers.visualizations import VisualizationListResource from redash.handlers.visualizations import VisualizationResource @@ -84,6 +84,7 @@ def json_representation(data, code, headers=None): api.add_org_resource(QueryRefreshResource, '/api/queries//refresh', endpoint='query_refresh') api.add_org_resource(QueryResource, '/api/queries/', endpoint='query') api.add_org_resource(QueryForkResource, '/api/queries//fork', endpoint='query_fork') +api.add_org_resource(QueryResultSetResource, '/api/queries//resultset', endpoint='query_aggregate_results') api.add_org_resource(QueryVersionListResource, '/api/queries//version', endpoint='query_versions') api.add_org_resource(ChangeResource, '/api/changes/', endpoint='changes') diff --git a/redash/handlers/queries.py b/redash/handlers/queries.py index c934f2cff8..db39f872c2 100644 --- a/redash/handlers/queries.py +++ b/redash/handlers/queries.py @@ -114,6 +114,7 @@ def post(self): : 0: + q.query_results.append(query_result) query_ids = [q.id for q in queries] logging.info("Updated %s queries with result (%s).", len(query_ids), query_hash) @@ -883,6 +887,7 @@ class Query(ChangeTrackingMixin, TimestampMixin, BelongsToOrgMixin, db.Model): data_source = db.relationship(DataSource, backref='queries') latest_query_data_id = Column(db.Integer, db.ForeignKey("query_results.id"), nullable=True) latest_query_data = db.relationship(QueryResult) + query_results = db.relationship("QueryResult", secondary="query_resultsets") name = Column(db.String(255)) description = Column(db.String(4096), nullable=True) query_text = Column("query", db.Text) @@ -898,6 +903,7 @@ class Query(ChangeTrackingMixin, TimestampMixin, BelongsToOrgMixin, db.Model): schedule = Column(db.String(10), nullable=True) schedule_failures = Column(db.Integer, default=0) schedule_until = Column(db.DateTime(True), nullable=True) + schedule_resultset_size = Column(db.Integer, nullable=True) visualizations = db.relationship("Visualization", cascade="all, delete-orphan") options = Column(MutableDict.as_mutable(PseudoJSON), default={}) search_vector = Column(TSVectorType('id', 'name', 'description', 'query', @@ -1050,6 +1056,37 @@ def search(cls, term, group_ids, user_id=None, include_drafts=False, limit=None) # sort the result using the weight as defined in the search vector column return all_queries.search(term, sort=True).limit(limit) + @classmethod + def delete_stale_resultsets(cls): + delete_count = 0 + texts = [c[0] for c in db.session.query(Query.query_text) + .filter(Query.schedule_resultset_size != None).distinct()] + for text in texts: + queries = (Query.query.filter(Query.query_text == text, + Query.schedule_resultset_size != None) + .order_by(Query.schedule_resultset_size.desc())) + # Multiple queries with the same text may request multiple result sets + # be kept. We start with the one that keeps the most, and delete both + # the unneeded bridge rows and result sets. + first_query = queries.first() + if first_query is not None and first_query.schedule_resultset_size: + resultsets = QueryResultSet.query.filter(QueryResultSet.query_rel == first_query).order_by(QueryResultSet.result_id) + resultset_count = resultsets.count() + if resultset_count > first_query.schedule_resultset_size: + n_to_delete = resultset_count - first_query.schedule_resultset_size + r_ids = [r.result_id for r in resultsets][:n_to_delete] + QueryResultSet.query.filter(QueryResultSet.result_id.in_(r_ids)).delete(synchronize_session=False) + delete_count += QueryResult.query.filter(QueryResult.id.in_(r_ids)).delete(synchronize_session=False) + # By this point there are no stale result sets left. + # Delete unneeded bridge rows for the remaining queries. + for q in queries[1:]: + resultsets = db.session.query(QueryResultSet.result_id).filter(QueryResultSet.query_rel == q).order_by(QueryResultSet.result_id) + n_to_delete = resultsets.count() - q.schedule_resultset_size + if n_to_delete > 0: + stale_r = QueryResultSet.query.filter(QueryResultSet.result_id.in_(resultsets.limit(n_to_delete).subquery())) + stale_r.delete(synchronize_session=False) + return delete_count + @classmethod def search_by_user(cls, term, user, limit=None): return cls.by_user(user).search(term, sort=True).limit(limit) @@ -1131,6 +1168,16 @@ def __repr__(self): return '' % (self.id, self.name or 'untitled') +class QueryResultSet(db.Model): + query_id = Column(db.Integer, db.ForeignKey("queries.id"), + primary_key=True) + query_rel = db.relationship(Query) + result_id = Column(db.Integer, db.ForeignKey("query_results.id"), + primary_key=True) + result = db.relationship(QueryResult) + __tablename__ = 'query_resultsets' + + @vectorizer(db.Integer) def integer_vectorizer(column): return db.func.cast(column, db.Text) diff --git a/redash/serializers.py b/redash/serializers.py index 75325c341f..eb4f203b2b 100644 --- a/redash/serializers.py +++ b/redash/serializers.py @@ -102,6 +102,7 @@ def serialize_query(query, with_stats=False, with_visualizations=False, with_use 'query_hash': query.query_hash, 'schedule': query.schedule, 'schedule_until': query.schedule_until, + 'schedule_resultset_size': query.schedule_resultset_size, 'api_key': query.api_key, 'is_archived': query.is_archived, 'is_draft': query.is_draft, diff --git a/redash/tasks/queries.py b/redash/tasks/queries.py index 4f44c3b854..abc967959b 100644 --- a/redash/tasks/queries.py +++ b/redash/tasks/queries.py @@ -354,6 +354,7 @@ def cleanup_query_results(): deleted_count = models.QueryResult.query.filter( models.QueryResult.id.in_(unused_query_results.subquery()) ).delete(synchronize_session=False) + deleted_count += models.Query.delete_stale_resultsets() models.db.session.commit() logger.info("Deleted %d unused query results.", deleted_count) diff --git a/tests/factories.py b/tests/factories.py index 0b56ac016d..2ffc6349fe 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -111,7 +111,9 @@ def __call__(self): query_hash=gen_query_hash('SELECT 1'), data_source=data_source_factory.create, org_id=1) - +query_resultset_factory = ModelFactory(redash.models.QueryResultSet, + query_rel=query_factory.create, + result=query_result_factory.create) visualization_factory = ModelFactory(redash.models.Visualization, type='CHART', query_rel=query_factory.create, @@ -297,6 +299,9 @@ def create_query_result(self, **kwargs): return query_result_factory.create(**args) + def create_query_resultset(self, **kwargs): + return query_resultset_factory.create(**kwargs) + def create_visualization(self, **kwargs): args = { 'query_rel': self.create_query() diff --git a/tests/handlers/test_queries.py b/tests/handlers/test_queries.py index 93dfae83d0..135d29c69a 100644 --- a/tests/handlers/test_queries.py +++ b/tests/handlers/test_queries.py @@ -1,3 +1,5 @@ +import json + from tests import BaseTestCase from redash import models from redash.models import db @@ -285,3 +287,81 @@ def test_get(self): rv2 = self.make_request('get', '/api/changes/' + str(ch2.id)) self.assertEqual(rv2.status_code, 200) self.assertEqual(rv2.json['change']['name']['current'], 'version B') + + +class AggregateResultsTests(BaseTestCase): + def test_aggregate(self): + qtxt = "SELECT x FROM mytable;" + q = self.factory.create_query(query_text=qtxt, schedule_resultset_size=3) + qr0 = self.factory.create_query_result( + query_text=qtxt, + data=json.dumps({'columns': ['name', 'color'], + 'rows': [{'name': 'eve', 'color': 'grue'}, + {'name': 'mallory', 'color': 'bleen'}]})) + qr1 = self.factory.create_query_result( + query_text=qtxt, + data=json.dumps({'columns': ['name', 'color'], + 'rows': [{'name': 'bob', 'color': 'green'}, + {'name': 'fred', 'color': 'blue'}]})) + qr2 = self.factory.create_query_result( + query_text=qtxt, + data=json.dumps({'columns': ['name', 'color'], + 'rows': [{'name': 'alice', 'color': 'red'}, + {'name': 'eddie', 'color': 'orange'}]})) + qr3 = self.factory.create_query_result( + query_text=qtxt, + data=json.dumps({'columns': ['name', 'color'], + 'rows': [{'name': 'dave', 'color': 'yellow'}, + {'name': 'carol', 'color': 'taupe'}]})) + for qr in (qr0, qr1, qr2, qr3): + self.factory.create_query_resultset(query_rel=q, result=qr) + rv = self.make_request('get', '/api/queries/{}/resultset'.format(q.id)) + self.assertEqual(rv.status_code, 200) + self.assertEqual(rv.json['query_result']['data'], + {'columns': ['name', 'color'], + 'rows': [ + {'name': 'bob', 'color': 'green'}, + {'name': 'fred', 'color': 'blue'}, + {'name': 'alice', 'color': 'red'}, + {'name': 'eddie', 'color': 'orange'}, + {'name': 'dave', 'color': 'yellow'}, + {'name': 'carol', 'color': 'taupe'} + ]}) + + def test_underfilled_aggregate(self): + qtxt = "SELECT x FROM mytable;" + q = self.factory.create_query(query_text=qtxt, + schedule_resultset_size=3) + qr1 = self.factory.create_query_result( + query_text=qtxt, + data=json.dumps({'columns': ['name', 'color'], + 'rows': [{'name': 'bob', 'color': 'green'}, + {'name': 'fred', 'color': 'blue'}]})) + qr2 = self.factory.create_query_result( + query_text=qtxt, + data=json.dumps({'columns': ['name', 'color'], + 'rows': [{'name': 'alice', 'color': 'red'}, + {'name': 'eddie', 'color': 'orange'}]})) + for qr in (qr1, qr2): + self.factory.create_query_resultset(query_rel=q, result=qr) + rv = self.make_request('get', '/api/queries/{}/resultset'.format(q.id)) + self.assertEqual(rv.status_code, 200) + self.assertEqual(rv.json['query_result']['data'], + {'columns': ['name', 'color'], + 'rows': [ + {'name': 'bob', 'color': 'green'}, + {'name': 'fred', 'color': 'blue'}, + {'name': 'alice', 'color': 'red'}, + {'name': 'eddie', 'color': 'orange'} + ]}) + + def test_no_aggregate(self): + qtxt = "SELECT x FROM mytable;" + q = self.factory.create_query(query_text=qtxt) + self.factory.create_query_result( + query_text=qtxt, + data=json.dumps({'columns': ['name', 'color'], + 'rows': [{'name': 'eve', 'color': 'grue'}, + {'name': 'mallory', 'color': 'bleen'}]})) + rv = self.make_request('get', '/api/queries/{}/resultset'.format(q.id)) + self.assertEqual(rv.status_code, 404) diff --git a/tests/test_models.py b/tests/test_models.py index fac35733db..f521a138f4 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -278,22 +278,74 @@ def test_deletes_alerts(self): class TestUnusedQueryResults(BaseTestCase): def test_returns_only_unused_query_results(self): two_weeks_ago = utcnow() - datetime.timedelta(days=14) - qr = self.factory.create_query_result() - query = self.factory.create_query(latest_query_data=qr) + qt = "SELECT 1" + qr = self.factory.create_query_result(query_text=qt, retrieved_at=two_weeks_ago) + query = self.factory.create_query(query_text=qt, latest_query_data=qr) + unused_qr = self.factory.create_query_result(query_text=qt, retrieved_at=two_weeks_ago) db.session.flush() - unused_qr = self.factory.create_query_result(retrieved_at=two_weeks_ago) self.assertIn((unused_qr.id,), models.QueryResult.unused()) self.assertNotIn((qr.id,), list(models.QueryResult.unused())) def test_returns_only_over_a_week_old_results(self): two_weeks_ago = utcnow() - datetime.timedelta(days=14) - unused_qr = self.factory.create_query_result(retrieved_at=two_weeks_ago) + qt = "SELECT 1" + unused_qr = self.factory.create_query_result(query_text=qt, retrieved_at=two_weeks_ago) db.session.flush() - new_unused_qr = self.factory.create_query_result() - + new_unused_qr = self.factory.create_query_result(query_text=qt) self.assertIn((unused_qr.id,), models.QueryResult.unused()) self.assertNotIn((new_unused_qr.id,), models.QueryResult.unused()) + def test_doesnt_return_live_incremental_results(self): + two_weeks_ago = utcnow() - datetime.timedelta(days=14) + qt = "SELECT 1" + qrs = [self.factory.create_query_result(query_text=qt, retrieved_at=two_weeks_ago) + for _ in range(5)] + q = self.factory.create_query(query_text=qt, latest_query_data=qrs[0], + schedule_resultset_size=3) + for qr in qrs: + self.factory.create_query_resultset(query_rel=q, result=qr) + db.session.flush() + self.assertEqual([], list(models.QueryResult.unused())) + + def test_deletes_stale_resultsets(self): + qt = "SELECT 17" + query = self.factory.create_query(query_text=qt, + schedule_resultset_size=5) + for _ in range(10): + r = self.factory.create_query_result(query_text=qt) + self.factory.create_query_resultset(query_rel=query, result=r) + qt2 = "SELECT 100" + query2 = self.factory.create_query(query_text=qt2, schedule_resultset_size=5) + for _ in range(10): + r = self.factory.create_query_result(query_text=qt2) + self.factory.create_query_resultset(query_rel=query2, result=r) + db.session.flush() + self.assertEqual(models.QueryResultSet.query.count(), 20) + self.assertEqual(models.Query.delete_stale_resultsets(), 10) + self.assertEqual(models.QueryResultSet.query.count(), 10) + + def test_deletes_stale_resultsets_with_dupe_queries(self): + qt = "SELECT 17" + query = self.factory.create_query(query_text=qt, + schedule_resultset_size=5) + for _ in range(10): + r = self.factory.create_query_result(query_text=qt) + self.factory.create_query_resultset(query_rel=query, result=r) + query2 = self.factory.create_query(query_text=qt, + schedule_resultset_size=3) + for _ in range(10): + self.factory.create_query_result(query_text=qt) + self.factory.create_query_resultset(query_rel=query2) + qt2 = "SELECT 100" + query3 = self.factory.create_query(query_text=qt2, schedule_resultset_size=5) + for _ in range(10): + r = self.factory.create_query_result(query_text=qt2) + self.factory.create_query_resultset(query_rel=query3, result=r) + db.session.flush() + self.assertEqual(models.QueryResultSet.query.count(), 30) + self.assertEqual(models.Query.delete_stale_resultsets(), 10) + self.assertEqual(models.QueryResultSet.query.count(), 13) + class TestQueryAll(BaseTestCase): def test_returns_only_queries_in_given_groups(self): From 3e0e2c0497c7c719e891046f5a0c3c86259f600d Mon Sep 17 00:00:00 2001 From: Jason Thomas Date: Wed, 11 Apr 2018 11:37:18 -0400 Subject: [PATCH 23/32] Updates to docker-entrypoint for worker and scheduler (#364) * Use --max-tasks-per-child as per celery documentation * Set --max-memory-per-child to 1/4th of total system memory * Split exec command over multiple lines * Fix memory variable typo --- bin/docker-entrypoint | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint index 1bed803efd..a91be66fc8 100755 --- a/bin/docker-entrypoint +++ b/bin/docker-entrypoint @@ -5,9 +5,13 @@ worker() { /app/manage.py db upgrade WORKERS_COUNT=${WORKERS_COUNT:-2} QUEUES=${QUEUES:-queries,scheduled_queries,celery} + MAX_MEMORY=$(($(/usr/bin/awk '/MemTotal/ {print $2}' /proc/meminfo)/4)) echo "Starting $WORKERS_COUNT workers for queues: $QUEUES..." - exec /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo --maxtasksperchild=10 -Ofair + exec /usr/local/bin/celery worker --app=redash.worker -c$WORKERS_COUNT -Q$QUEUES -linfo \ + --max-tasks-per-child=10 \ + --max-memory-per-child=$MAX_MEMORY \ + -Ofair } scheduler() { @@ -17,7 +21,9 @@ scheduler() { echo "Starting scheduler and $WORKERS_COUNT workers for queues: $QUEUES..." - exec /usr/local/bin/celery worker --app=redash.worker --beat -c$WORKERS_COUNT -Q$QUEUES -linfo --maxtasksperchild=10 -Ofair + exec /usr/local/bin/celery worker --app=redash.worker --beat -c$WORKERS_COUNT -Q$QUEUES -linfo \ + --max-tasks-per-child=10 \ + -Ofair } server() { From ab309d63762f12a955468b37f7e480ab896c49ee Mon Sep 17 00:00:00 2001 From: Marina Samuel Date: Fri, 11 May 2018 16:02:26 -0400 Subject: [PATCH 24/32] Closes #396: Integration with Flower. --- docker-compose.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index e01be5bfd1..18ae07bf1c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,3 +43,13 @@ services: - "15432:5432" command: "postgres -c fsync=off -c full_page_writes=off -c synchronous_commit=OFF" restart: unless-stopped + flower: + image: mher/flower:latest + command: flower + environment: + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + ports: + - "5555:5555" + links: + - redis From 9bb44f0b88be9467e5d730076364c65f3151ee0a Mon Sep 17 00:00:00 2001 From: Allen Short Date: Thu, 28 Jun 2018 11:10:12 -0500 Subject: [PATCH 25/32] merge upstream db changes --- migrations/versions/2ba47e9812b1_.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 migrations/versions/2ba47e9812b1_.py diff --git a/migrations/versions/2ba47e9812b1_.py b/migrations/versions/2ba47e9812b1_.py new file mode 100644 index 0000000000..93d0f59268 --- /dev/null +++ b/migrations/versions/2ba47e9812b1_.py @@ -0,0 +1,24 @@ +"""empty message + +Revision ID: 2ba47e9812b1 +Revises: 71477dadd6ef, 9d7678c47452 +Create Date: 2018-07-25 16:09:54.769289 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '2ba47e9812b1' +down_revision = ('71477dadd6ef', '9d7678c47452', ) +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass From f20588289419143ee3aec462672e8bdf7fa9d4be Mon Sep 17 00:00:00 2001 From: Marina Samuel Date: Fri, 27 Apr 2018 10:31:01 -0400 Subject: [PATCH 26/32] Add data source health monitoring via an extension. Refs: #379, #415 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bbde299daa..acde253066 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,4 +54,4 @@ disposable-email-domains # Uncomment the requirement for ldap3 if using ldap. # It is not included by default because of the GPL license conflict. # ldap3==2.2.4 -redash-stmo>=2018.4.0 +redash-stmo>=2018.8.1 From 37f8aab1242970acad59f7b19d4dc82e697486e8 Mon Sep 17 00:00:00 2001 From: Allen Short Date: Fri, 9 Nov 2018 12:19:50 -0600 Subject: [PATCH 27/32] Use production build of react when deployed (fixes #606) --- webpack.config.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/webpack.config.js b/webpack.config.js index b476c7abf6..934d94a3af 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -226,6 +226,14 @@ if (process.env.DEV_SERVER_HOST) { config.devServer.host = process.env.DEV_SERVER_HOST; } +if (isProduction) { + config.plugins.push( + new webpack.DefinePlugin({ + 'process.env.NODE_ENV': JSON.stringify('production') + }) + ); +} + if (process.env.BUNDLE_ANALYZER) { config.plugins.push(new BundleAnalyzerPlugin()); } From 84f1d064655ae61c134a35f72b3f73f122048edb Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 14 Dec 2018 11:09:02 +0100 Subject: [PATCH 28/32] Various merge fixes. --- redash/handlers/query_results.py | 2 +- redash/query_runner/db2.py | 4 ++-- redash/query_runner/kylin.py | 5 ++--- redash/query_runner/rockset.py | 4 ++-- redash/serializers.py | 4 +++- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/redash/handlers/query_results.py b/redash/handlers/query_results.py index 9ff1f5ed2f..a453b080c5 100644 --- a/redash/handlers/query_results.py +++ b/redash/handlers/query_results.py @@ -159,7 +159,7 @@ def get(self, query_id=None, filetype='json'): for r in results: aggregate_result['data']['rows'].extend(r['data']['rows']) - data = json.dumps({'query_result': aggregate_result}, cls=utils.JSONEncoder) + data = json_dumps({'query_result': aggregate_result}) headers = {'Content-Type': "application/json"} return make_response(data, 200, headers) diff --git a/redash/query_runner/db2.py b/redash/query_runner/db2.py index 3253cee0b9..7413189459 100644 --- a/redash/query_runner/db2.py +++ b/redash/query_runner/db2.py @@ -3,7 +3,7 @@ import logging from redash.query_runner import * -from redash.utils import JSONEncoder +from redash.utils import json_dumps logger = logging.getLogger(__name__) @@ -129,7 +129,7 @@ def run_query(self, query, user): data = {'columns': columns, 'rows': rows} error = None - json_data = json.dumps(data, cls=JSONEncoder) + json_data = json_dumps(data) else: error = 'Query completed but it returned no data.' json_data = None diff --git a/redash/query_runner/kylin.py b/redash/query_runner/kylin.py index a9f5d1fdb4..261fa3f5e0 100644 --- a/redash/query_runner/kylin.py +++ b/redash/query_runner/kylin.py @@ -1,12 +1,11 @@ import os -import json import logging import requests from requests.auth import HTTPBasicAuth from redash import settings from redash.query_runner import * -from redash.utils import JSONEncoder +from redash.utils import json_dumps logger = logging.getLogger(__name__) @@ -102,7 +101,7 @@ def run_query(self, query, user): columns = self.get_columns(data['columnMetas']) rows = self.get_rows(columns, data['results']) - return json.dumps({'columns': columns, 'rows': rows}), None + return json_dumps({'columns': columns, 'rows': rows}), None def get_schema(self, get_stats=False): url = self.configuration['url'] diff --git a/redash/query_runner/rockset.py b/redash/query_runner/rockset.py index 5d0d30d99d..8b0abe7c83 100644 --- a/redash/query_runner/rockset.py +++ b/redash/query_runner/rockset.py @@ -1,7 +1,7 @@ import requests import os from redash.query_runner import * -from redash.utils import JSONEncoder +from redash.utils import json_dumps import json @@ -96,7 +96,7 @@ def run_query(self, query, user): columns = [] for k in rows[0]: columns.append({'name': k, 'friendly_name': k, 'type': _get_type(rows[0][k])}) - data = json.dumps({'columns': columns, 'rows': rows}, cls=JSONEncoder) + data = json_dumps({'columns': columns, 'rows': rows}) return data, None diff --git a/redash/serializers.py b/redash/serializers.py index eb4f203b2b..b275dff4d6 100644 --- a/redash/serializers.py +++ b/redash/serializers.py @@ -8,7 +8,6 @@ from flask_login import current_user from redash import models -from redash.handlers.query_results import run_query_sync from redash.permissions import has_access, view_only from redash.utils import json_loads @@ -30,6 +29,9 @@ def public_widget(widget): # make sure the widget's query has a latest_query_data_id that is # not null so public dashboards work if q.latest_query_data_id is None: + # this import is inline since it triggers a circular + # import otherwise + from redash.handlers.query_results import run_query_sync run_query_sync(q.data_source, {}, q.query_text) query_data = q.latest_query_data.to_dict() From 018172aee23492276cf52f8c00a0af11860db2ca Mon Sep 17 00:00:00 2001 From: Allen Short Date: Tue, 8 Nov 2016 16:15:35 -0600 Subject: [PATCH 29/32] Enable documentation links and versions of data sources (re #6). Refs #537, #553. Co-authored-by: Marina Samuel Co-authored-by: Allen Short --- client/app/assets/less/redash/query.less | 6 ++ client/app/pages/data-sources/list.html | 2 +- client/app/pages/queries/query.html | 2 + redash/handlers/api.py | 2 +- redash/models.py | 2 + redash/query_runner/__init__.py | 52 +++++---- redash/query_runner/big_query.py | 83 +++++++-------- redash/query_runner/cass.py | 75 ++++++------- redash/query_runner/dynamodb_sql.py | 39 +++---- redash/query_runner/elasticsearch.py | 41 ++++---- redash/query_runner/google_spreadsheets.py | 25 ++--- redash/query_runner/graphite.py | 2 +- redash/query_runner/hive_ds.py | 41 ++++---- redash/query_runner/impala_ds.py | 67 ++++++------ redash/query_runner/influx_db.py | 23 ++-- redash/query_runner/mongodb.py | 42 ++++---- redash/query_runner/mssql.py | 73 ++++++------- redash/query_runner/mysql.py | 57 +++++----- redash/query_runner/oracle.py | 50 ++++----- redash/query_runner/pg.py | 117 +++++++++++---------- redash/query_runner/presto.py | 51 +++++---- redash/query_runner/python.py | 32 +++--- redash/query_runner/script.py | 34 +++--- redash/query_runner/sqlite.py | 25 ++--- redash/query_runner/treasuredata.py | 53 +++++----- redash/query_runner/vertica.py | 67 ++++++------ requirements.txt | 2 +- tests/handlers/test_data_sources.py | 7 +- 28 files changed, 563 insertions(+), 509 deletions(-) diff --git a/client/app/assets/less/redash/query.less b/client/app/assets/less/redash/query.less index 20791a8c47..1418f52187 100644 --- a/client/app/assets/less/redash/query.less +++ b/client/app/assets/less/redash/query.less @@ -457,6 +457,7 @@ a.label-tag { .datasource-small { visibility: hidden; + display: none !important; } .query-fullscreen .query-metadata__mobile { @@ -579,6 +580,11 @@ nav .rg-bottom { display: none; } + .datasource-small { + visibility: visible; + display: inline-block !important; + } + .query-fullscreen { flex-direction: column; overflow: hidden; diff --git a/client/app/pages/data-sources/list.html b/client/app/pages/data-sources/list.html index 56af90e071..fd23dfc516 100644 --- a/client/app/pages/data-sources/list.html +++ b/client/app/pages/data-sources/list.html @@ -9,7 +9,7 @@
    {{dataSource.name}} -

    {{dataSource.name}}

    +

    {{dataSource.name}}

    - {{dataSource.type_name}}
    diff --git a/client/app/pages/queries/query.html b/client/app/pages/queries/query.html index b5357d9e1c..e4eff25bd2 100644 --- a/client/app/pages/queries/query.html +++ b/client/app/pages/queries/query.html @@ -93,6 +93,8 @@

    {{ds.name}} + +
    diff --git a/redash/handlers/api.py b/redash/handlers/api.py index 49884e46e0..4518bffcce 100644 --- a/redash/handlers/api.py +++ b/redash/handlers/api.py @@ -6,7 +6,7 @@ from redash.handlers.base import org_scoped_rule from redash.handlers.permissions import ObjectPermissionsListResource, CheckPermissionResource from redash.handlers.alerts import AlertResource, AlertListResource, AlertSubscriptionListResource, AlertSubscriptionResource -from redash.handlers.dashboards import DashboardListResource, DashboardResource, DashboardShareResource, PublicDashboardResource +from redash.handlers.dashboards import DashboardListResource, DashboardResource, DashboardShareResource, PublicDashboardResource from redash.handlers.data_sources import DataSourceTypeListResource, DataSourceListResource, DataSourceSchemaResource, DataSourceResource, DataSourcePauseResource, DataSourceTestResource from redash.handlers.events import EventsResource from redash.handlers.queries import QueryForkResource, QueryRefreshResource, QueryListResource, QueryRecentResource, QuerySearchResource, QueryResource, MyQueriesResource, QueryVersionListResource, ChangeResource diff --git a/redash/models.py b/redash/models.py index ec5592d2a3..3d0cc01464 100644 --- a/redash/models.py +++ b/redash/models.py @@ -686,6 +686,8 @@ def add_group(self, group, view_only=False): db.session.add(dsg) return dsg + setattr(self, 'data_source_groups', dsg) + def remove_group(self, group): db.session.query(DataSourceGroup).filter( DataSourceGroup.group == group, diff --git a/redash/query_runner/__init__.py b/redash/query_runner/__init__.py index 4a2ac1fc56..411bb65aea 100644 --- a/redash/query_runner/__init__.py +++ b/redash/query_runner/__init__.py @@ -51,6 +51,7 @@ class NotSupported(Exception): class BaseQueryRunner(object): noop_query = None + configuration_properties = None def __init__(self, configuration): self.syntax = 'sql' @@ -76,6 +77,12 @@ def annotate_query(cls): def configuration_schema(cls): return {} + @classmethod + def add_configuration_property(cls, property, value): + if cls.configuration_properties is None: + raise NotImplementedError() + cls.configuration_properties[property] = value + def test_connection(self): if self.noop_query is None: raise NotImplementedError() @@ -150,31 +157,36 @@ class BaseHTTPQueryRunner(BaseQueryRunner): url_title = 'URL base path' username_title = 'HTTP Basic Auth Username' password_title = 'HTTP Basic Auth Password' + configuration_properties = { + 'url': { + 'type': 'string', + 'title': url_title, + }, + 'username': { + 'type': 'string', + 'title': username_title, + }, + 'password': { + 'type': 'string', + 'title': password_title, + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": ( + "This string will be used to toggle visibility of " + "tables in the schema browser when editing a query " + "in order to remove non-useful tables from sight." + ), + }, + } @classmethod def configuration_schema(cls): schema = { 'type': 'object', - 'properties': { - 'url': { - 'type': 'string', - 'title': cls.url_title, - }, - 'username': { - 'type': 'string', - 'title': cls.username_title, - }, - 'password': { - 'type': 'string', - 'title': cls.password_title, - }, - 'toggle_table_string': { - "type": "string", - "title": "Toggle Table String", - "default": "_v", - "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - }, - }, + 'properties': cls.configuration_properties, 'secret': ['password'] } diff --git a/redash/query_runner/big_query.py b/redash/query_runner/big_query.py index b1f3b3b678..f4eeaf47b8 100644 --- a/redash/query_runner/big_query.py +++ b/redash/query_runner/big_query.py @@ -82,6 +82,47 @@ def _get_query_results(jobs, project_id, location, job_id, start_index): class BigQuery(BaseQueryRunner): noop_query = "SELECT 1" + configuration_properties = { + 'projectId': { + 'type': 'string', + 'title': 'Project ID' + }, + 'jsonKeyFile': { + "type": "string", + 'title': 'JSON Key File' + }, + 'totalMBytesProcessedLimit': { + "type": "number", + 'title': 'Scanned Data Limit (MB)' + }, + 'userDefinedFunctionResourceUri': { + "type": "string", + 'title': 'UDF Source URIs (i.e. gs://bucket/date_utils.js, gs://bucket/string_utils.js )' + }, + 'useStandardSql': { + "type": "boolean", + 'title': "Use Standard SQL (Beta)", + }, + 'location': { + "type": "string", + "title": "Processing Location", + "default": "US", + }, + 'loadSchema': { + "type": "boolean", + "title": "Load Schema" + }, + 'maximumBillingTier': { + "type": "number", + "title": "Maximum Billing Tier" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, + } @classmethod def enabled(cls): @@ -91,47 +132,7 @@ def enabled(cls): def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'projectId': { - 'type': 'string', - 'title': 'Project ID' - }, - 'jsonKeyFile': { - "type": "string", - 'title': 'JSON Key File' - }, - 'totalMBytesProcessedLimit': { - "type": "number", - 'title': 'Scanned Data Limit (MB)' - }, - 'userDefinedFunctionResourceUri': { - "type": "string", - 'title': 'UDF Source URIs (i.e. gs://bucket/date_utils.js, gs://bucket/string_utils.js )' - }, - 'useStandardSql': { - "type": "boolean", - 'title': "Use Standard SQL", - "default": True, - }, - 'location': { - "type": "string", - "title": "Processing Location", - }, - 'loadSchema': { - "type": "boolean", - "title": "Load Schema" - }, - 'maximumBillingTier': { - "type": "number", - "title": "Maximum Billing Tier" - }, - "toggle_table_string": { - "type": "string", - "title": "Toggle Table String", - "default": "_v", - "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - } - }, + 'properties': cls.configuration_properties, 'required': ['jsonKeyFile', 'projectId'], "order": ['projectId', 'jsonKeyFile', 'loadSchema', 'useStandardSql', 'location', 'totalMBytesProcessedLimit', 'maximumBillingTier', 'userDefinedFunctionResourceUri'], 'secret': ['jsonKeyFile'] diff --git a/redash/query_runner/cass.py b/redash/query_runner/cass.py index 11550dd181..e59f8d0ce2 100644 --- a/redash/query_runner/cass.py +++ b/redash/query_runner/cass.py @@ -23,6 +23,43 @@ def default(self, o): class Cassandra(BaseQueryRunner): noop_query = "SELECT dateof(now()) FROM system.local" + configuration_properties = { + 'host': { + 'type': 'string', + }, + 'port': { + 'type': 'number', + 'default': 9042, + }, + 'keyspace': { + 'type': 'string', + 'title': 'Keyspace name' + }, + 'username': { + 'type': 'string', + 'title': 'Username' + }, + 'password': { + 'type': 'string', + 'title': 'Password' + }, + 'protocol': { + 'type': 'number', + 'title': 'Protocol Version', + 'default': 3 + }, + 'timeout': { + 'type': 'number', + 'title': 'Timeout', + 'default': 10 + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, + } @classmethod def enabled(cls): @@ -32,43 +69,7 @@ def enabled(cls): def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'host': { - 'type': 'string', - }, - 'port': { - 'type': 'number', - 'default': 9042, - }, - 'keyspace': { - 'type': 'string', - 'title': 'Keyspace name' - }, - 'username': { - 'type': 'string', - 'title': 'Username' - }, - 'password': { - 'type': 'string', - 'title': 'Password' - }, - 'protocol': { - 'type': 'number', - 'title': 'Protocol Version', - 'default': 3 - }, - 'timeout': { - 'type': 'number', - 'title': 'Timeout', - 'default': 10 - }, - "toggle_table_string": { - "type": "string", - "title": "Toggle Table String", - "default": "_v", - "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - } - }, + 'properties': cls.configuration_properties, 'required': ['keyspace', 'host'] } diff --git a/redash/query_runner/dynamodb_sql.py b/redash/query_runner/dynamodb_sql.py index 32cf84669d..3623e6a6f0 100644 --- a/redash/query_runner/dynamodb_sql.py +++ b/redash/query_runner/dynamodb_sql.py @@ -32,28 +32,31 @@ class DynamoDBSQL(BaseSQLQueryRunner): + noop_query = "SELECT 1" + configuration_properties = { + "region": { + "type": "string", + "default": "us-east-1" + }, + "access_key": { + "type": "string", + }, + "secret_key": { + "type": "string", + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, + } + @classmethod def configuration_schema(cls): return { "type": "object", - "properties": { - "region": { - "type": "string", - "default": "us-east-1" - }, - "access_key": { - "type": "string", - }, - "secret_key": { - "type": "string", - }, - "toggle_table_string": { - "type": "string", - "title": "Toggle Table String", - "default": "_v", - "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - } - }, + "properties": cls.configuration_properties, "required": ["access_key", "secret_key"], "secret": ["secret_key"] } diff --git a/redash/query_runner/elasticsearch.py b/redash/query_runner/elasticsearch.py index 99203ea43d..9b7817f2fb 100644 --- a/redash/query_runner/elasticsearch.py +++ b/redash/query_runner/elasticsearch.py @@ -45,31 +45,32 @@ class BaseElasticSearch(BaseQueryRunner): DEBUG_ENABLED = False + configuration_properties = { + 'server': { + 'type': 'string', + 'title': 'Base URL' + }, + 'basic_auth_user': { + 'type': 'string', + 'title': 'Basic Auth User' + }, + 'basic_auth_password': { + 'type': 'string', + 'title': 'Basic Auth Password' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, + } @classmethod def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'server': { - 'type': 'string', - 'title': 'Base URL' - }, - 'basic_auth_user': { - 'type': 'string', - 'title': 'Basic Auth User' - }, - 'basic_auth_password': { - 'type': 'string', - 'title': 'Basic Auth Password' - }, - "toggle_table_string": { - "type": "string", - "title": "Toggle Table String", - "default": "_v", - "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - } - }, + 'properties': cls.configuration_properties, "secret": ["basic_auth_password"], "required": ["server"] } diff --git a/redash/query_runner/google_spreadsheets.py b/redash/query_runner/google_spreadsheets.py index 8ac65d0be9..0af6fb484b 100644 --- a/redash/query_runner/google_spreadsheets.py +++ b/redash/query_runner/google_spreadsheets.py @@ -147,6 +147,18 @@ def request(self, *args, **kwargs): class GoogleSpreadsheet(BaseQueryRunner): + configuration_properties = { + 'jsonKeyFile': { + "type": "string", + 'title': 'JSON Key File' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, + } @classmethod def annotate_query(cls): @@ -164,18 +176,7 @@ def enabled(cls): def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'jsonKeyFile': { - "type": "string", - 'title': 'JSON Key File' - }, - "toggle_table_string": { - "type": "string", - "title": "Toggle Table String", - "default": "_v", - "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - } - }, + 'properties': cls.configuration_properties, 'required': ['jsonKeyFile'], 'secret': ['jsonKeyFile'] } diff --git a/redash/query_runner/graphite.py b/redash/query_runner/graphite.py index ec0068dbb6..98e5ddd514 100644 --- a/redash/query_runner/graphite.py +++ b/redash/query_runner/graphite.py @@ -49,7 +49,7 @@ def configuration_schema(cls): "title": "Toggle Table String", "default": "_v", "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - } + }, }, 'required': ['url'], 'secret': ['password'] diff --git a/redash/query_runner/hive_ds.py b/redash/query_runner/hive_ds.py index 6d8ec67c3e..cab6ff1d96 100644 --- a/redash/query_runner/hive_ds.py +++ b/redash/query_runner/hive_ds.py @@ -37,31 +37,32 @@ class Hive(BaseSQLQueryRunner): noop_query = "SELECT 1" + configuration_properties = { + "host": { + "type": "string" + }, + "port": { + "type": "number" + }, + "database": { + "type": "string" + }, + "username": { + "type": "string" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, + } @classmethod def configuration_schema(cls): return { "type": "object", - "properties": { - "host": { - "type": "string" - }, - "port": { - "type": "number" - }, - "database": { - "type": "string" - }, - "username": { - "type": "string" - }, - "toggle_table_string": { - "type": "string", - "title": "Toggle Table String", - "default": "_v", - "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - } - }, + "properties": cls.configuration_properties, "order": ["host", "port", "database", "username"], "required": ["host"] } diff --git a/redash/query_runner/impala_ds.py b/redash/query_runner/impala_ds.py index b57bb0b7e6..111d39b4ae 100644 --- a/redash/query_runner/impala_ds.py +++ b/redash/query_runner/impala_ds.py @@ -34,44 +34,45 @@ class Impala(BaseSQLQueryRunner): noop_query = "show schemas" + configuration_properties = { + "host": { + "type": "string" + }, + "port": { + "type": "number" + }, + "protocol": { + "type": "string", + "title": "Please specify beeswax or hiveserver2" + }, + "database": { + "type": "string" + }, + "use_ldap": { + "type": "boolean" + }, + "ldap_user": { + "type": "string" + }, + "ldap_password": { + "type": "string" + }, + "timeout": { + "type": "number" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, + } @classmethod def configuration_schema(cls): return { "type": "object", - "properties": { - "host": { - "type": "string" - }, - "port": { - "type": "number" - }, - "protocol": { - "type": "string", - "title": "Please specify beeswax or hiveserver2" - }, - "database": { - "type": "string" - }, - "use_ldap": { - "type": "boolean" - }, - "ldap_user": { - "type": "string" - }, - "ldap_password": { - "type": "string" - }, - "timeout": { - "type": "number" - }, - "toggle_table_string": { - "type": "string", - "title": "Toggle Table String", - "default": "_v", - "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - } - }, + "properties": cls.configuration_properties, "required": ["host"], "secret": ["ldap_password"] } diff --git a/redash/query_runner/influx_db.py b/redash/query_runner/influx_db.py index aee41318b8..d3351312c1 100644 --- a/redash/query_runner/influx_db.py +++ b/redash/query_runner/influx_db.py @@ -49,22 +49,23 @@ def _transform_result(results): class InfluxDB(BaseQueryRunner): noop_query = "show measurements limit 1" + configuration_properties = { + 'url': { + 'type': 'string' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, + } @classmethod def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'url': { - 'type': 'string' - }, - "toggle_table_string": { - "type": "string", - "title": "Toggle Table String", - "default": "_v", - "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - } - }, + 'properties': cls.configuration_properties, 'required': ['url'] } diff --git a/redash/query_runner/mongodb.py b/redash/query_runner/mongodb.py index 6b814faade..72db989843 100644 --- a/redash/query_runner/mongodb.py +++ b/redash/query_runner/mongodb.py @@ -117,30 +117,32 @@ def parse_results(results): class MongoDB(BaseQueryRunner): + configuration_properties = { + 'connectionString': { + 'type': 'string', + 'title': 'Connection String' + }, + 'dbName': { + 'type': 'string', + 'title': "Database Name" + }, + 'replicaSetName': { + 'type': 'string', + 'title': 'Replica Set Name' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, + } + @classmethod def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'connectionString': { - 'type': 'string', - 'title': 'Connection String' - }, - 'dbName': { - 'type': 'string', - 'title': "Database Name" - }, - 'replicaSetName': { - 'type': 'string', - 'title': 'Replica Set Name' - }, - "toggle_table_string": { - "type": "string", - "title": "Toggle Table String", - "default": "_v", - "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - }, - }, + 'properties': cls.configuration_properties, 'required': ['connectionString', 'dbName'] } diff --git a/redash/query_runner/mssql.py b/redash/query_runner/mssql.py index 0cf25567a9..b2c188d112 100644 --- a/redash/query_runner/mssql.py +++ b/redash/query_runner/mssql.py @@ -27,47 +27,48 @@ class SqlServer(BaseSQLQueryRunner): noop_query = "SELECT 1" + configuration_properties = { + "user": { + "type": "string" + }, + "password": { + "type": "string" + }, + "server": { + "type": "string", + "default": "127.0.0.1" + }, + "port": { + "type": "number", + "default": 1433 + }, + "tds_version": { + "type": "string", + "default": "7.0", + "title": "TDS Version" + }, + "charset": { + "type": "string", + "default": "UTF-8", + "title": "Character Set" + }, + "db": { + "type": "string", + "title": "Database Name" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, + } @classmethod def configuration_schema(cls): return { "type": "object", - "properties": { - "user": { - "type": "string" - }, - "password": { - "type": "string" - }, - "server": { - "type": "string", - "default": "127.0.0.1" - }, - "port": { - "type": "number", - "default": 1433 - }, - "tds_version": { - "type": "string", - "default": "7.0", - "title": "TDS Version" - }, - "charset": { - "type": "string", - "default": "UTF-8", - "title": "Character Set" - }, - "db": { - "type": "string", - "title": "Database Name" - }, - "toggle_table_string": { - "type": "string", - "title": "Toggle Table String", - "default": "_v", - "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - } - }, + "properties": cls.configuration_properties, "required": ["db"], "secret": ["password"] } diff --git a/redash/query_runner/mysql.py b/redash/query_runner/mysql.py index 3f5f9d310b..18ce41f72a 100644 --- a/redash/query_runner/mysql.py +++ b/redash/query_runner/mysql.py @@ -28,6 +28,33 @@ class Mysql(BaseSQLQueryRunner): noop_query = "SELECT 1" + configuration_properties = { + 'host': { + 'type': 'string', + 'default': '127.0.0.1' + }, + 'user': { + 'type': 'string' + }, + 'passwd': { + 'type': 'string', + 'title': 'Password' + }, + 'db': { + 'type': 'string', + 'title': 'Database name' + }, + 'port': { + 'type': 'number', + 'default': 3306, + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, + } @classmethod def configuration_schema(cls): @@ -35,33 +62,7 @@ def configuration_schema(cls): schema = { 'type': 'object', - 'properties': { - 'host': { - 'type': 'string', - 'default': '127.0.0.1' - }, - 'user': { - 'type': 'string' - }, - 'passwd': { - 'type': 'string', - 'title': 'Password' - }, - 'db': { - 'type': 'string', - 'title': 'Database name' - }, - 'port': { - 'type': 'number', - 'default': 3306, - }, - "toggle_table_string": { - "type": "string", - "title": "Toggle Table String", - "default": "_v", - "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - } - }, + 'properties': cls.configuration_properties, "order": ['host', 'port', 'user', 'passwd', 'db'], 'required': ['db'], 'secret': ['passwd'] @@ -84,7 +85,7 @@ def configuration_schema(cls): 'ssl_key': { 'type': 'string', 'title': 'Path to private key file (SSL)' - } + }, }) return schema diff --git a/redash/query_runner/oracle.py b/redash/query_runner/oracle.py index 8979ebb11b..7acb9f0038 100644 --- a/redash/query_runner/oracle.py +++ b/redash/query_runner/oracle.py @@ -29,8 +29,33 @@ logger = logging.getLogger(__name__) + class Oracle(BaseSQLQueryRunner): noop_query = "SELECT 1 FROM dual" + configuration_properties = { + "user": { + "type": "string" + }, + "password": { + "type": "string" + }, + "host": { + "type": "string" + }, + "port": { + "type": "number" + }, + "servicename": { + "type": "string", + "title": "DSN Service Name" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, + } @classmethod def get_col_type(cls, col_type, scale): @@ -47,30 +72,7 @@ def enabled(cls): def configuration_schema(cls): return { "type": "object", - "properties": { - "user": { - "type": "string" - }, - "password": { - "type": "string" - }, - "host": { - "type": "string" - }, - "port": { - "type": "number" - }, - "servicename": { - "type": "string", - "title": "DSN Service Name" - }, - "toggle_table_string": { - "type": "string", - "title": "Toggle Table String", - "default": "_v", - "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - } - }, + "properties": cls.configuration_properties, "required": ["servicename", "user", "password", "host", "port"], "secret": ["password"] } diff --git a/redash/query_runner/pg.py b/redash/query_runner/pg.py index 356a4d773c..1590166ae4 100644 --- a/redash/query_runner/pg.py +++ b/redash/query_runner/pg.py @@ -46,42 +46,43 @@ def _wait(conn, timeout=None): class PostgreSQL(BaseSQLQueryRunner): noop_query = "SELECT 1" + configuration_properties = { + "user": { + "type": "string" + }, + "password": { + "type": "string" + }, + "host": { + "type": "string", + "default": "127.0.0.1" + }, + "port": { + "type": "number", + "default": 5432 + }, + "dbname": { + "type": "string", + "title": "Database Name" + }, + "sslmode": { + "type": "string", + "title": "SSL Mode", + "default": "prefer" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, + } @classmethod def configuration_schema(cls): return { "type": "object", - "properties": { - "user": { - "type": "string" - }, - "password": { - "type": "string" - }, - "host": { - "type": "string", - "default": "127.0.0.1" - }, - "port": { - "type": "number", - "default": 5432 - }, - "dbname": { - "type": "string", - "title": "Database Name" - }, - "sslmode": { - "type": "string", - "title": "SSL Mode", - "default": "prefer" - }, - "toggle_table_string": { - "type": "string", - "title": "Toggle Table String", - "default": "_v", - "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - } - }, + "properties": cls.configuration_properties, "order": ['host', 'port', 'user', 'password'], "required": ["dbname"], "secret": ["password"] @@ -195,6 +196,36 @@ def run_query(self, query, user): class Redshift(PostgreSQL): + configuration_properties = { + "user": { + "type": "string" + }, + "password": { + "type": "string" + }, + "host": { + "type": "string" + }, + "port": { + "type": "number" + }, + "dbname": { + "type": "string", + "title": "Database Name" + }, + "sslmode": { + "type": "string", + "title": "SSL Mode", + "default": "prefer" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, + } + @classmethod def type(cls): return "redshift" @@ -218,29 +249,7 @@ def configuration_schema(cls): return { "type": "object", - "properties": { - "user": { - "type": "string" - }, - "password": { - "type": "string" - }, - "host": { - "type": "string" - }, - "port": { - "type": "number" - }, - "dbname": { - "type": "string", - "title": "Database Name" - }, - "sslmode": { - "type": "string", - "title": "SSL Mode", - "default": "prefer" - } - }, + "properties": cls.configuration_properties, "order": ['host', 'port', 'user', 'password'], "required": ["dbname", "user", "password", "host", "port"], "secret": ["password"] diff --git a/redash/query_runner/presto.py b/redash/query_runner/presto.py index 570d003cd7..631b384fdd 100644 --- a/redash/query_runner/presto.py +++ b/redash/query_runner/presto.py @@ -33,38 +33,35 @@ class Presto(BaseQueryRunner): noop_query = 'SHOW TABLES' + configuration_properties = { + 'host': { + 'type': 'string' + }, + 'port': { + 'type': 'number' + }, + 'schema': { + 'type': 'string' + }, + 'catalog': { + 'type': 'string' + }, + 'username': { + 'type': 'string' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, + } @classmethod def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'host': { - 'type': 'string' - }, - 'protocol': { - 'type': 'string', - 'default': 'http' - }, - 'port': { - 'type': 'number' - }, - 'schema': { - 'type': 'string' - }, - 'catalog': { - 'type': 'string' - }, - 'username': { - 'type': 'string' - }, - "toggle_table_string": { - "type": "string", - "title": "Toggle Table String", - "default": "_v", - "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - }, - }, + 'properties': cls.configuration_properties, 'order': ['host', 'protocol', 'port', 'username', 'schema', 'catalog'], 'required': ['host'] } diff --git a/redash/query_runner/python.py b/redash/query_runner/python.py index 4fdf0de626..9b29128c2f 100644 --- a/redash/query_runner/python.py +++ b/redash/query_runner/python.py @@ -44,25 +44,27 @@ class Python(BaseQueryRunner): 'tuple', 'set', 'list', 'dict', 'bool', ) + configuration_properties = { + 'allowedImportModules': { + 'type': 'string', + 'title': 'Modules to import prior to running the script' + }, + 'additionalModulesPaths': { + 'type': 'string' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, + } + @classmethod def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'allowedImportModules': { - 'type': 'string', - 'title': 'Modules to import prior to running the script' - }, - 'additionalModulesPaths': { - 'type': 'string' - }, - "toggle_table_string": { - "type": "string", - "title": "Toggle Table String", - "default": "_v", - "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - } - }, + 'properties': cls.configuration_properties } @classmethod diff --git a/redash/query_runner/script.py b/redash/query_runner/script.py index 1a4b80bdfd..808d1024a2 100644 --- a/redash/query_runner/script.py +++ b/redash/query_runner/script.py @@ -29,6 +29,23 @@ def run_script(script, shell): class Script(BaseQueryRunner): + configuration_properties = { + 'path': { + 'type': 'string', + 'title': 'Scripts path' + }, + 'shell': { + 'type': 'boolean', + 'title': 'Execute command through the shell' + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, + } + @classmethod def annotate_query(cls): return False @@ -41,22 +58,7 @@ def enabled(cls): def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'path': { - 'type': 'string', - 'title': 'Scripts path' - }, - 'shell': { - 'type': 'boolean', - 'title': 'Execute command through the shell' - }, - "toggle_table_string": { - "type": "string", - "title": "Toggle Table String", - "default": "_v", - "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - } - }, + 'properties': cls.configuration_properties, 'required': ['path'] } diff --git a/redash/query_runner/sqlite.py b/redash/query_runner/sqlite.py index 9f02315e60..79c4f9c3e4 100644 --- a/redash/query_runner/sqlite.py +++ b/redash/query_runner/sqlite.py @@ -12,23 +12,24 @@ class Sqlite(BaseSQLQueryRunner): noop_query = "pragma quick_check" + configuration_properties = { + "dbpath": { + "type": "string", + "title": "Database Path" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, + } @classmethod def configuration_schema(cls): return { "type": "object", - "properties": { - "dbpath": { - "type": "string", - "title": "Database Path" - }, - "toggle_table_string": { - "type": "string", - "title": "Toggle Table String", - "default": "_v", - "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - } - }, + "properties": cls.configuration_properties, "required": ["dbpath"], } diff --git a/redash/query_runner/treasuredata.py b/redash/query_runner/treasuredata.py index 52ee2029c1..5321706801 100644 --- a/redash/query_runner/treasuredata.py +++ b/redash/query_runner/treasuredata.py @@ -35,37 +35,38 @@ class TreasureData(BaseQueryRunner): noop_query = "SELECT 1" + configuration_properties = { + 'endpoint': { + 'type': 'string' + }, + 'apikey': { + 'type': 'string' + }, + 'type': { + 'type': 'string' + }, + 'db': { + 'type': 'string', + 'title': 'Database Name' + }, + 'get_schema': { + 'type': 'boolean', + 'title': 'Auto Schema Retrieval', + 'default': False + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, + } @classmethod def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'endpoint': { - 'type': 'string' - }, - 'apikey': { - 'type': 'string' - }, - 'type': { - 'type': 'string' - }, - 'db': { - 'type': 'string', - 'title': 'Database Name' - }, - 'get_schema': { - 'type': 'boolean', - 'title': 'Auto Schema Retrieval', - 'default': False - }, - "toggle_table_string": { - "type": "string", - "title": "Toggle Table String", - "default": "_v", - "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - } - }, + 'properties': cls.configuration_properties, 'required': ['apikey','db'] } diff --git a/redash/query_runner/vertica.py b/redash/query_runner/vertica.py index 92633eb23e..6bffece1ea 100644 --- a/redash/query_runner/vertica.py +++ b/redash/query_runner/vertica.py @@ -29,44 +29,45 @@ class Vertica(BaseSQLQueryRunner): noop_query = "SELECT 1" + configuration_properties = { + 'host': { + 'type': 'string' + }, + 'user': { + 'type': 'string' + }, + 'password': { + 'type': 'string', + 'title': 'Password' + }, + 'database': { + 'type': 'string', + 'title': 'Database name' + }, + "port": { + "type": "number" + }, + "read_timeout": { + "type": "number", + "title": "Read Timeout" + }, + "connection_timeout": { + "type": "number", + "title": "Connection Timeout" + }, + "toggle_table_string": { + "type": "string", + "title": "Toggle Table String", + "default": "_v", + "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." + }, + } @classmethod def configuration_schema(cls): return { 'type': 'object', - 'properties': { - 'host': { - 'type': 'string' - }, - 'user': { - 'type': 'string' - }, - 'password': { - 'type': 'string', - 'title': 'Password' - }, - 'database': { - 'type': 'string', - 'title': 'Database name' - }, - "port": { - "type": "number" - }, - "read_timeout": { - "type": "number", - "title": "Read Timeout" - }, - "connection_timeout": { - "type": "number", - "title": "Connection Timeout" - }, - "toggle_table_string": { - "type": "string", - "title": "Toggle Table String", - "default": "_v", - "info": "This string will be used to toggle visibility of tables in the schema browser when editing a query in order to remove non-useful tables from sight." - }, - }, + 'properties': cls.configuration_properties, 'required': ['database'], 'order': ['host', 'port', 'user', 'password', 'database', 'read_timeout', 'connection_timeout'], 'secret': ['password'] diff --git a/requirements.txt b/requirements.txt index acde253066..132e0352d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,4 +54,4 @@ disposable-email-domains # Uncomment the requirement for ldap3 if using ldap. # It is not included by default because of the GPL license conflict. # ldap3==2.2.4 -redash-stmo>=2018.8.1 +redash-stmo>=2018.9.1 diff --git a/tests/handlers/test_data_sources.py b/tests/handlers/test_data_sources.py index f07a2b3719..4590056fd4 100644 --- a/tests/handlers/test_data_sources.py +++ b/tests/handlers/test_data_sources.py @@ -60,7 +60,8 @@ def test_updates_data_source(self): new_name = 'New Name' new_options = {"dbname": "newdb"} rv = self.make_request('post', self.path, - data={'name': new_name, 'type': 'pg', 'options': new_options}, + data={'name': new_name, 'type': 'pg', 'options': new_options, + 'doc_url': None}, user=admin) self.assertEqual(rv.status_code, 200) @@ -101,7 +102,9 @@ def test_returns_400_when_configuration_invalid(self): def test_creates_data_source(self): admin = self.factory.create_admin() rv = self.make_request('post', '/api/data_sources', - data={'name': 'DS 1', 'type': 'pg', 'options': {"dbname": "redash"}}, user=admin) + data={'name': 'DS 1', 'type': 'pg', + 'options': {"dbname": "redash"}, + 'doc_url': None}, user=admin) self.assertEqual(rv.status_code, 200) From 3eacea13574710011de2763a7822e8f2d4b18b9a Mon Sep 17 00:00:00 2001 From: Marina Samuel Date: Tue, 27 Nov 2018 10:18:34 -0500 Subject: [PATCH 30/32] Use entry_point.module_name instead of entry_point.name for looking up the bundles. --- bin/bundle-extensions | 9 ++++++--- requirements.txt | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/bin/bundle-extensions b/bin/bundle-extensions index 8416aab776..56173f4c5f 100755 --- a/bin/bundle-extensions +++ b/bin/bundle-extensions @@ -22,9 +22,12 @@ os.environ["EXTENSIONS_DIRECTORY"] = EXTENSIONS_RELATIVE_PATH for entry_point in iter_entry_points('redash.extensions'): # This is where the frontend code for an extension lives # inside of its package. - content_folder_relative = os.path.join( - entry_point.name, 'bundle') - (root_module, _) = os.path.splitext(entry_point.module_name) + + split_module_path = entry_point.module_name.split(os.extsep) + root_module = split_module_path.pop(0) + + content_folder_relative = os.path.join(os.path.join( + *split_module_path), 'bundle') if not resource_isdir(root_module, content_folder_relative): continue diff --git a/requirements.txt b/requirements.txt index 132e0352d8..450d069d56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,4 +54,4 @@ disposable-email-domains # Uncomment the requirement for ldap3 if using ldap. # It is not included by default because of the GPL license conflict. # ldap3==2.2.4 -redash-stmo>=2018.9.1 +redash-stmo>=2018.12.0 From 59bac93766bbd6369ba70c90c4e59a81e7febabe Mon Sep 17 00:00:00 2001 From: Allen Short Date: Wed, 19 Dec 2018 14:07:20 -0600 Subject: [PATCH 31/32] hotfix for #451 --- client/app/services/query.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/app/services/query.js b/client/app/services/query.js index add63edde8..b53ca25149 100644 --- a/client/app/services/query.js +++ b/client/app/services/query.js @@ -451,7 +451,7 @@ function QueryResource( this.latest_query_data_id = null; } - if (this.schedule_resultset_size) { + if (this.schedule_resultset_size > 1) { if (!this.queryResult) { this.queryResult = QueryResult.getResultSet(this.id); } From 0f46b14d850a5cfa893c035d0ec66e465b338e07 Mon Sep 17 00:00:00 2001 From: Marina Samuel Date: Wed, 10 Oct 2018 11:56:40 -0400 Subject: [PATCH 32/32] Closes #427: Migrate schedule_until into new json schedule field. --- migrations/versions/b8a479422596_.py | 74 ++++++++++++++++++++++++++++ redash/models.py | 1 - redash/serializers.py | 1 - 3 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 migrations/versions/b8a479422596_.py diff --git a/migrations/versions/b8a479422596_.py b/migrations/versions/b8a479422596_.py new file mode 100644 index 0000000000..d838ab0e07 --- /dev/null +++ b/migrations/versions/b8a479422596_.py @@ -0,0 +1,74 @@ +""" +Migrate schedule_until to schedule.until + +Revision ID: b8a479422596 +Revises: 73beceabb948 +Create Date: 2018-10-10 14:53:20.042470 + +""" +from datetime import datetime +from alembic import op +import sqlalchemy as sa +from sqlalchemy.sql import table + +from redash.models import MutableDict, PseudoJSON + + +# revision identifiers, used by Alembic. +revision = 'b8a479422596' +down_revision = '73beceabb948' +branch_labels = None +depends_on = None + + +def upgrade(): + queries = table( + 'queries', + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('schedule', MutableDict.as_mutable(PseudoJSON)), + sa.Column('schedule_until', sa.DateTime(True), nullable=True)) + + conn = op.get_bind() + for query in conn.execute(queries.select()): + if query.schedule_until is None: + continue + + schedule_json = query.schedule + if schedule_json is None: + schedule_json = { + 'interval': None, + 'day_of_week': None, + 'time': None + } + schedule_json['until'] = query.schedule_until.strftime('%Y-%m-%d') + + conn.execute( + queries + .update() + .where(queries.c.id == query.id) + .values(schedule=MutableDict(schedule_json))) + + op.drop_column('queries', 'schedule_until') + + +def downgrade(): + op.add_column('queries', sa.Column('schedule_until', sa.DateTime(True), nullable=True)) + + queries = table( + 'queries', + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('schedule', MutableDict.as_mutable(PseudoJSON)), + sa.Column('schedule_until', sa.DateTime(True), nullable=True)) + + conn = op.get_bind() + for query in conn.execute(queries.select()): + if query.schedule is None or query.schedule['until'] is None: + continue + + scheduleUntil = datetime.strptime(query.schedule['until'], '%Y-%m-%d') + + conn.execute( + queries + .update() + .where(queries.c.id == query.id) + .values(schedule_until=scheduleUntil)) diff --git a/redash/models.py b/redash/models.py index 3d0cc01464..80dab05206 100644 --- a/redash/models.py +++ b/redash/models.py @@ -904,7 +904,6 @@ class Query(ChangeTrackingMixin, TimestampMixin, BelongsToOrgMixin, db.Model): is_draft = Column(db.Boolean, default=True, index=True) schedule = Column(db.String(10), nullable=True) schedule_failures = Column(db.Integer, default=0) - schedule_until = Column(db.DateTime(True), nullable=True) schedule_resultset_size = Column(db.Integer, nullable=True) visualizations = db.relationship("Visualization", cascade="all, delete-orphan") options = Column(MutableDict.as_mutable(PseudoJSON), default={}) diff --git a/redash/serializers.py b/redash/serializers.py index b275dff4d6..84f1274813 100644 --- a/redash/serializers.py +++ b/redash/serializers.py @@ -103,7 +103,6 @@ def serialize_query(query, with_stats=False, with_visualizations=False, with_use 'query': query.query_text, 'query_hash': query.query_hash, 'schedule': query.schedule, - 'schedule_until': query.schedule_until, 'schedule_resultset_size': query.schedule_resultset_size, 'api_key': query.api_key, 'is_archived': query.is_archived,