Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Scheduling queries from SQL Lab (#7416) #7446

Merged
merged 11 commits into from
May 3, 2019
78 changes: 78 additions & 0 deletions docs/installation.rst
Original file line number Diff line number Diff line change
Expand Up @@ -811,6 +811,84 @@ in this dictionary are made available for users to use in their SQL.
'my_crazy_macro': lambda x: x*2,
}

**Scheduling queries**

You can optionally allow your users to schedule queries directly in SQL Lab.
This is done by addding extra metadata to saved queries, which are then picked
up by an external scheduled (like [Apache Airflow](https://airflow.apache.org/)).

To allow scheduled queries, add the following to your `config.py`:

.. code-block:: python

FEATURE_FLAGS = {
# Configuration for scheduling queries from SQL Lab. This information is
# collected when the user clicks "Schedule query", and saved into the `extra`
# field of saved queries.
# See: https://github.com/mozilla-services/react-jsonschema-form
'SCHEDULED_QUERIES': {
'JSONSCHEMA': {
'title': 'Schedule',
'description': (
'In order to schedule a query, you need to specify when it '
'should start running, when it should stop running, and how '
'often it should run. You can also optionally specify '
'dependencies that should be met before the query is '
'executed. Please read the documentation for best practices '
'and more information on how to specify dependencies.'
),
'type': 'object',
'properties': {
'output_table': {
'type': 'string',
'title': 'Output table name',
},
'start_date': {
'type': 'string',
'format': 'date-time',
'title': 'Start date',
},
'end_date': {
'type': 'string',
'format': 'date-time',
'title': 'End date',
},
'schedule_interval': {
'type': 'string',
'title': 'Schedule interval',
},
'dependencies': {
'type': 'array',
'title': 'Dependencies',
'items': {
'type': 'string',
},
},
},
},
'UISCHEMA': {
'schedule_interval': {
'ui:placeholder': '@daily, @weekly, etc.',
},
'dependencies': {
'ui:help': (
'Check the documentation for the correct format when '
'defining dependencies.'
),
},
},
},
}

This feature flag is based on [react-jsonschema-form](https://github.com/mozilla-services/react-jsonschema-form),
and will add a button called "Schedule Query" to SQL Lab. When the button is
clicked, a modal will show up where the user can add the metadata required for
scheduling the query.

This information can then be retrieved from the endpoint `/savedqueryviewapi/api/read`
and used to schedule the queries that have `scheduled_queries` in their JSON
metadata. For schedulers other than Airflow, additional fields can be easily
added to the configuration file above.

Celery Flower
-------------
Expand Down
46 changes: 42 additions & 4 deletions superset/assets/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions superset/assets/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@
"react-dom": "^16.4.1",
"react-gravatar": "^2.6.1",
"react-hot-loader": "^4.3.6",
"react-jsonschema-form": "^1.2.0",
"react-map-gl": "^4.0.10",
"react-markdown": "^3.3.0",
"react-redux": "^5.0.2",
Expand Down
10 changes: 9 additions & 1 deletion superset/assets/src/SqlLab/components/QueryAutoRefresh.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,18 @@ class QueryAutoRefresh extends React.PureComponent {
const { queries, queriesLastUpdate } = this.props;
const now = new Date().getTime();

// due to a race condition, queries can be marked as successful before the
// results key is set; this is a workaround until we fix the underlying
// problem
const isQueryRunning = q => (
['running', 'started', 'pending', 'fetching'].indexOf(q.state) >= 0 ||
(q.state === 'success' && q.resultsKey === null)
Copy link

@graceguo-supercat graceguo-supercat May 16, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@DiggidyDave this logic is problematic. Sql Lab can run async and synchronous queries. For synchronous queries, it will not have a resultsKey.

the new logic will make extra polling happen when user has at least 1 synchronous query (and < 6 hours old). For synchronous queries, results are returned from sql_json request, polling should not happen.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, my bad. I'll fix it.

);

return (
queriesLastUpdate > 0 &&
Object.values(queries).some(
q => ['running', 'started', 'pending', 'fetching'].indexOf(q.state) >= 0 &&
q => isQueryRunning(q) &&
now - q.startDttm < MAX_QUERY_AGE_TO_POLL,
)
);
Expand Down
109 changes: 109 additions & 0 deletions superset/assets/src/SqlLab/components/ScheduleQueryButton.jsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import React from 'react';
import PropTypes from 'prop-types';
import Form from 'react-jsonschema-form';
import { t } from '@superset-ui/translation';

import Button from '../../components/Button';
import ModalTrigger from '../../components/ModalTrigger';

const propTypes = {
defaultLabel: PropTypes.string,
sql: PropTypes.string.isRequired,
schema: PropTypes.string.isRequired,
dbId: PropTypes.number.isRequired,
animation: PropTypes.bool,
onSchedule: PropTypes.func,
};
const defaultProps = {
defaultLabel: t('Undefined'),
animation: true,
onSchedule: () => {},
};

class ScheduleQueryButton extends React.PureComponent {
constructor(props) {
super(props);
this.state = {
description: '',
label: props.defaultLabel,
showSchedule: false,
};
this.toggleSchedule = this.toggleSchedule.bind(this);
this.onSchedule = this.onSchedule.bind(this);
this.onCancel = this.onCancel.bind(this);
this.onLabelChange = this.onLabelChange.bind(this);
this.onDescriptionChange = this.onDescriptionChange.bind(this);
}
onSchedule({ formData }) {
const query = {
label: this.state.label,
description: this.state.description,
db_id: this.props.dbId,
schema: this.props.schema,
sql: this.props.sql,
extra_json: JSON.stringify({ schedule_info: formData }),
};
this.props.onSchedule(query);
this.saveModal.close();
}
onCancel() {
this.saveModal.close();
}
onLabelChange(e) {
this.setState({ label: e.target.value });
}
onDescriptionChange(e) {
this.setState({ description: e.target.value });
}
toggleSchedule(e) {
this.setState({ target: e.target, showSchedule: !this.state.showSchedule });
}
renderModalBody() {
return (
<Form
schema={window.featureFlags.SCHEDULED_QUERIES.JSONSCHEMA}
uiSchema={window.featureFlags.SCHEDULED_QUERIES.UISCHEMA}
onSubmit={this.onSchedule}
/>
);
}
render() {
return (
<span className="ScheduleQueryButton">
<ModalTrigger
ref={(ref) => { this.saveModal = ref; }}
modalTitle={t('Schedule Query')}
modalBody={this.renderModalBody()}
triggerNode={
<Button bsSize="small" className="toggleSchedule" onClick={this.toggleSchedule}>
<i className="fa fa-calendar" /> {t('Schedule Query')}
</Button>
}
bsSize="medium"
/>
</span>
);
}
}
ScheduleQueryButton.propTypes = propTypes;
ScheduleQueryButton.defaultProps = defaultProps;

export default ScheduleQueryButton;
14 changes: 14 additions & 0 deletions superset/assets/src/SqlLab/components/SqlEditor.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,15 @@ import LimitControl from './LimitControl';
import TemplateParamsEditor from './TemplateParamsEditor';
import SouthPane from './SouthPane';
import SaveQuery from './SaveQuery';
import ScheduleQueryButton from './ScheduleQueryButton';
import ShareSqlLabQuery from './ShareSqlLabQuery';
import Timer from '../../components/Timer';
import Hotkeys from '../../components/Hotkeys';
import SqlEditorLeftBar from './SqlEditorLeftBar';
import AceEditorWrapper from './AceEditorWrapper';
import { STATE_BSSTYLE_MAP } from '../constants';
import RunQueryActionButton from './RunQueryActionButton';
import { FeatureFlag, isFeatureEnabled } from '../../featureFlags';

const SQL_EDITOR_PADDING = 10;
const SQL_TOOLBAR_HEIGHT = 51;
Expand Down Expand Up @@ -313,6 +315,18 @@ class SqlEditor extends React.PureComponent {
sql={this.state.sql}
/>
</span>
{isFeatureEnabled(FeatureFlag.SCHEDULED_QUERIES) &&
<span className="m-r-5">
<ScheduleQueryButton
defaultLabel={qe.title}
sql={qe.sql}
className="m-r-5"
onSchedule={this.props.actions.saveQuery}
schema={qe.schema}
dbId={qe.dbId}
/>
</span>
}
<span className="m-r-5">
<SaveQuery
defaultLabel={qe.title}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ export default class FilterableTable extends PureComponent {

headerRenderer({ dataKey, label, sortBy, sortDirection }) {
return (
<div>
<div className="header-style">
{label}
{sortBy === dataKey &&
<SortIndicator sortDirection={sortDirection} />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,3 +72,8 @@
}
.even-row { background: #f2f2f2; }
.odd-row { background: #ffffff; }
.header-style {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
3 changes: 2 additions & 1 deletion superset/assets/src/featureFlags.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ export enum FeatureFlag {
SCOPED_FILTER = 'SCOPED_FILTER',
OMNIBAR = 'OMNIBAR',
CLIENT_CACHE = 'CLIENT_CACHE',
SCHEDULED_QUERIES = 'SCHEDULED_QUERIES',
}

export type FeatureFlagMap = {
Expand All @@ -39,5 +40,5 @@ export function initFeatureFlags(featureFlags: FeatureFlagMap) {
}

export function isFeatureEnabled(feature: FeatureFlag) {
return !!window.featureFlags[feature];
return window && window.featureFlags && !!window.featureFlags[feature];
}
Loading