From 35221f83577f964eae1d3809b54fe020e75827d5 Mon Sep 17 00:00:00 2001 From: Marina Samuel Date: Mon, 25 Mar 2019 15:01:59 -0400 Subject: [PATCH] Schema Viewer Drawer (getredash#3291) * Process extra column metadata for a few sql-based data sources. * Add Table and Column metadata tables. * Periodically update table and column schema tables in a celery task. * Fetching schema returns data from table and column metadata tables. * Add tests for backend changes. * Front-end shows extra table metadata and uses new schema response. * Delete datasource schema data when deleting a data source. * Process and store data source schema when a data source is first created or after a migration. * Tables should have a unique name per datasource. * Addressing review comments. * Update migration file for mixins. * Appease PEP8 * Upgrade migration file for rebase. * Cascade delete. * Adding org_id * Remove redundant column and table prefixes. * Non-existing tables and columns should be filtered out on the server side not client side. * Fetching table samples should be optional and should happen in a separate task per table. * Allow users to force a schema refresh. * Use updated_at to help prune old schema metadata periodically. * Using settings.SCHEMAS_REFRESH_QUEUE * fix for getredash#2426 test * more stable test_interactive_new * Closes #927, #928: Schema refresh improvements. * Closes #934, #935: Remove type from schema browser and don't show empty example column in schema drawer (#936) * Speed up schema fetch requests with fewer postgres queries. * Add column metadata to Athena glue processing. * Fix bug assuming 'metadata' exists for every table. * Closes #939: Persisted, existing table metadata should be updated. * Sample processing should be rate-limited. * Add cli command for refreshing data samples. * Schema refreshes should not overwrite column 'example' field. * refresh_samples() should filter tables_to_sample on the datasource's id being sampled * Correctly wrap long text in schema drawer. Schema Improvements Part 2: Add data source config options. Adding BigQuery schema drawer with data types and samples. Add empty migration to replace the removed schedule_until migration Add merge migration. Co-authored-by: Alison --- client/app/assets/less/ant.less | 1 + client/app/assets/less/inc/base.less | 4 + client/app/assets/less/inc/popover.less | 4 +- .../app/assets/less/inc/schema-browser.less | 14 +- client/app/assets/less/redash/query.less | 14 + .../dynamic-form/dynamicFormHelper.js | 8 + client/app/components/keywordBuilder.js | 8 +- client/app/components/proptypes.js | 17 +- client/app/components/queries/SchemaData.jsx | 144 ++++++++ .../components/queries/schema-browser.html | 17 +- .../app/components/queries/schema-browser.js | 23 ++ .../app/pages/data-sources/EditDataSource.jsx | 21 +- .../schema-table-components/EditableTable.jsx | 83 +++++ .../schema-table-components/QueryListItem.jsx | 40 ++ .../QuerySearchDialog.jsx | 106 ++++++ .../SampleQueryList.jsx | 85 +++++ .../schema-table-components/SchemaTable.jsx | 274 ++++++++++++++ .../TableVisibilityCheckbox.jsx | 24 ++ .../schema-table-components/schema-table.css | 24 ++ client/app/pages/queries/query.html | 3 + client/app/services/data-source.js | 13 + docker-compose.yml | 1 + migrations/versions/118aa16f565b_.py | 38 ++ migrations/versions/151a4c333e96_.py | 24 ++ .../171aaafb2d52_add_more_db_indexes.py | 101 +++++ migrations/versions/280daa582976_.py | 59 +++ migrations/versions/6adb92e75691_.py | 27 ++ migrations/versions/ba150362b02e_.py | 26 ++ migrations/versions/cf135a57332e_.py | 32 ++ migrations/versions/eb2f788f997e_.py | 23 ++ redash/cli/data_sources.py | 34 +- redash/handlers/data_sources.py | 32 +- redash/models/__init__.py | 144 ++++++-- redash/query_runner/__init__.py | 20 + redash/query_runner/athena.py | 25 +- redash/query_runner/big_query.py | 117 +++++- redash/query_runner/mysql.py | 10 +- redash/query_runner/pg.py | 27 +- redash/query_runner/presto.py | 9 +- redash/schedule.py | 2 + redash/schema.py | 183 +++++++++ redash/serializers/__init__.py | 82 +++- redash/settings/__init__.py | 22 +- redash/tasks/__init__.py | 4 + redash/tasks/queries/__init__.py | 2 + redash/tasks/queries/maintenance.py | 177 +++++++-- redash/tasks/queries/samples.py | 128 +++++++ tests/factories.py | 15 + tests/handlers/test_data_sources.py | 43 +++ tests/models/test_data_sources.py | 51 --- tests/query_runner/test_athena.py | 27 +- tests/query_runner/test_bigquery.py | 54 +++ tests/query_runner/test_get_schema_format.py | 70 ++++ tests/query_runner/test_pg.py | 15 +- tests/tasks/test_queries.py | 24 +- tests/tasks/test_refresh_schemas.py | 349 +++++++++++++++++- tests/test_cli.py | 15 +- 57 files changed, 2773 insertions(+), 166 deletions(-) create mode 100644 client/app/components/queries/SchemaData.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/EditableTable.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/QueryListItem.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/QuerySearchDialog.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/SampleQueryList.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/SchemaTable.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/TableVisibilityCheckbox.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/schema-table.css create mode 100644 migrations/versions/118aa16f565b_.py create mode 100644 migrations/versions/151a4c333e96_.py create mode 100644 migrations/versions/171aaafb2d52_add_more_db_indexes.py create mode 100644 migrations/versions/280daa582976_.py create mode 100644 migrations/versions/6adb92e75691_.py create mode 100644 migrations/versions/ba150362b02e_.py create mode 100644 migrations/versions/cf135a57332e_.py create mode 100644 migrations/versions/eb2f788f997e_.py create mode 100644 redash/schema.py create mode 100644 redash/tasks/queries/samples.py create mode 100644 tests/query_runner/test_bigquery.py create mode 100644 tests/query_runner/test_get_schema_format.py diff --git a/client/app/assets/less/ant.less b/client/app/assets/less/ant.less index 864bf8ee44..e6ebaf8356 100644 --- a/client/app/assets/less/ant.less +++ b/client/app/assets/less/ant.less @@ -14,6 +14,7 @@ @import '~antd/lib/radio/style/index'; @import '~antd/lib/time-picker/style/index'; @import '~antd/lib/pagination/style/index'; +@import '~antd/lib/drawer/style/index'; @import '~antd/lib/table/style/index'; @import '~antd/lib/popover/style/index'; @import '~antd/lib/icon/style/index'; diff --git a/client/app/assets/less/inc/base.less b/client/app/assets/less/inc/base.less index b9e3cda7ea..40ce7e0ef0 100755 --- a/client/app/assets/less/inc/base.less +++ b/client/app/assets/less/inc/base.less @@ -131,6 +131,10 @@ strong { transition: height 0s, width 0s !important; } +.admin-schema-editor { + padding: 50px 0; +} + // Ace Editor .ace_editor { border: 1px solid fade(@redash-gray, 15%) !important; diff --git a/client/app/assets/less/inc/popover.less b/client/app/assets/less/inc/popover.less index 5fcad7089b..c687a089a2 100755 --- a/client/app/assets/less/inc/popover.less +++ b/client/app/assets/less/inc/popover.less @@ -1,5 +1,7 @@ .popover { box-shadow: fade(@redash-gray, 25%) 0px 0px 15px 0px; + color: #000000; + z-index: 1000000001; // So that it can popover a dropdown menu } .popover-title { @@ -19,4 +21,4 @@ p { margin-bottom: 0; } -} \ No newline at end of file +} diff --git a/client/app/assets/less/inc/schema-browser.less b/client/app/assets/less/inc/schema-browser.less index 0034391086..d547a78790 100644 --- a/client/app/assets/less/inc/schema-browser.less +++ b/client/app/assets/less/inc/schema-browser.less @@ -7,14 +7,14 @@ div.table-name { border-radius: @redash-radius; position: relative; - .copy-to-editor { + .copy-to-editor, .info { display: none; } &:hover { background: fade(@redash-gray, 10%); - .copy-to-editor { + .copy-to-editor, .info { display: flex; } } @@ -36,7 +36,7 @@ div.table-name { background: transparent; } - .copy-to-editor { + .copy-to-editor, .info { color: fade(@redash-gray, 90%); cursor: pointer; position: absolute; @@ -49,6 +49,10 @@ div.table-name { justify-content: center; } + .info { + right: 20px + } + .table-open { padding: 0 22px 0 26px; overflow: hidden; @@ -56,14 +60,14 @@ div.table-name { white-space: nowrap; position: relative; - .copy-to-editor { + .copy-to-editor, .info { display: none; } &:hover { background: fade(@redash-gray, 10%); - .copy-to-editor { + .copy-to-editor, .info { display: flex; } } diff --git a/client/app/assets/less/redash/query.less b/client/app/assets/less/redash/query.less index d48fb9bd1f..0d75dd610a 100644 --- a/client/app/assets/less/redash/query.less +++ b/client/app/assets/less/redash/query.less @@ -698,3 +698,17 @@ nav .rg-bottom { } } } + +.ui-select-choices-row .info { + display: none; +} + +.ui-select-choices-row { + &:hover { + .info { + cursor: pointer; + width: 20px; + display: inline; + } + } +} diff --git a/client/app/components/dynamic-form/dynamicFormHelper.js b/client/app/components/dynamic-form/dynamicFormHelper.js index 7976db6dc5..317d3d0be7 100644 --- a/client/app/components/dynamic-form/dynamicFormHelper.js +++ b/client/app/components/dynamic-form/dynamicFormHelper.js @@ -100,6 +100,13 @@ function getFields(type = {}, target = { options: {} }) { placeholder: `My ${type.name}`, autoFocus: isNewTarget, }, + { + name: "description", + title: "Description", + type: "text", + required: false, + initialValue: target.description, + }, ...orderedInputs(configurationSchema.properties, configurationSchema.order, target.options), ]; @@ -108,6 +115,7 @@ function getFields(type = {}, target = { options: {} }) { function updateTargetWithValues(target, values) { target.name = values.name; + target.description = values.description; Object.keys(values).forEach(key => { if (key !== "name") { target.options[key] = values[key]; diff --git a/client/app/components/keywordBuilder.js b/client/app/components/keywordBuilder.js index 3265c6fa4b..dc301b4e65 100644 --- a/client/app/components/keywordBuilder.js +++ b/client/app/components/keywordBuilder.js @@ -4,9 +4,9 @@ function buildTableColumnKeywords(table) { const keywords = []; table.columns.forEach(column => { keywords.push({ - caption: column, - name: `${table.name}.${column}`, - value: `${table.name}.${column}`, + caption: column.name, + name: `${table.name}.${column.name}`, + value: `${table.name}.${column.name}`, score: 100, meta: "Column", className: "completion", @@ -29,7 +29,7 @@ function buildKeywordsFromSchema(schema) { }); tableColumnKeywords[table.name] = buildTableColumnKeywords(table); table.columns.forEach(c => { - columnKeywords[c] = "Column"; + columnKeywords[c.name] = "Column"; }); }); diff --git a/client/app/components/proptypes.js b/client/app/components/proptypes.js index 355b912da4..1d815b50c6 100644 --- a/client/app/components/proptypes.js +++ b/client/app/components/proptypes.js @@ -11,8 +11,16 @@ export const DataSource = PropTypes.shape({ type_name: PropTypes.string, }); +export const DataSourceMetadata = PropTypes.shape({ + key: PropTypes.number, + name: PropTypes.string, + type: PropTypes.string, + example: PropTypes.string, + description: PropTypes.string, +}); + export const Table = PropTypes.shape({ - columns: PropTypes.arrayOf(PropTypes.string).isRequired, + columns: PropTypes.arrayOf(PropTypes.object).isRequired, }); export const Schema = PropTypes.arrayOf(Table); @@ -31,6 +39,13 @@ export const RefreshScheduleDefault = { until: null, }; +export const TableMetadata = PropTypes.shape({ + key: PropTypes.number.isRequired, + name: PropTypes.string.isRequired, + description: PropTypes.string, + visible: PropTypes.bool.isRequired, +}); + export const Field = PropTypes.shape({ name: PropTypes.string.isRequired, title: PropTypes.string, diff --git a/client/app/components/queries/SchemaData.jsx b/client/app/components/queries/SchemaData.jsx new file mode 100644 index 0000000000..148a305b43 --- /dev/null +++ b/client/app/components/queries/SchemaData.jsx @@ -0,0 +1,144 @@ +import React from "react"; +import PropTypes from "prop-types"; +import { react2angular } from "react2angular"; +import Drawer from "antd/lib/drawer"; +import Table from "antd/lib/table"; + +import { DataSourceMetadata, Query } from "@/components/proptypes"; + +function textWrapRenderer(text) { + return
{text}
; +} + +class SchemaData extends React.PureComponent { + static propTypes = { + show: PropTypes.bool.isRequired, + onClose: PropTypes.func.isRequired, + tableName: PropTypes.string, + tableDescription: PropTypes.string, + sampleQueries: PropTypes.arrayOf(Query), + tableMetadata: PropTypes.arrayOf(DataSourceMetadata), + }; + + static defaultProps = { + tableName: "", + tableDescription: "", + tableMetadata: [], + sampleQueries: [], + }; + + render() { + const tableDataColumns = [ + { + title: "Metadata", + dataIndex: "metadata", + width: 400, + key: "metadata", + }, + { + title: "Value", + dataIndex: "value", + width: 400, + key: "value", + render: text => { + if (typeof text === "string") { + return text; + } + return ( + + ); + }, + }, + ]; + + const columnDataColumns = [ + { + title: "Column Name", + dataIndex: "name", + width: 400, + key: "name", + render: textWrapRenderer, + }, + { + title: "Column Type", + dataIndex: "type", + width: 400, + key: "type", + render: textWrapRenderer, + }, + ]; + + const hasDescription = this.props.tableMetadata.some(columnMetadata => columnMetadata.description); + + const hasExample = this.props.tableMetadata.some(columnMetadata => columnMetadata.example); + + if (hasDescription) { + columnDataColumns.push({ + title: "Description", + dataIndex: "description", + width: 400, + key: "description", + render: textWrapRenderer, + }); + } + + if (hasExample) { + columnDataColumns.push({ + title: "Example", + dataIndex: "example", + width: 400, + key: "example", + render: textWrapRenderer, + }); + } + const tableData = [ + { + metadata: "Table Description", + value: this.props.tableDescription || "N/A", + }, + { + metadata: "Sample Usage", + value: this.props.sampleQueries.length > 0 ? this.props.sampleQueries : "N/A", + }, + ]; + + return ( + +

{this.props.tableName}

+
+
Table Data
+ +
+
Column Data
+
+ + ); + } +} + +export default function init(ngModule) { + ngModule.component("schemaData", react2angular(SchemaData, null, [])); +} + +init.init = true; diff --git a/client/app/components/queries/schema-browser.html b/client/app/components/queries/schema-browser.html index fe7e26669e..804264e352 100644 --- a/client/app/components/queries/schema-browser.html +++ b/client/app/components/queries/schema-browser.html @@ -19,22 +19,33 @@
-
+
{{table.name}} ({{table.size}}) +
-
{{column}} +
+ {{column.name}} + ng-click="$ctrl.itemSelected($event, [column.name])">
+
diff --git a/client/app/components/queries/schema-browser.js b/client/app/components/queries/schema-browser.js index a41b5a3cf1..70d429e925 100644 --- a/client/app/components/queries/schema-browser.js +++ b/client/app/components/queries/schema-browser.js @@ -11,6 +11,21 @@ function SchemaBrowserCtrl($rootScope, $scope) { $scope.$broadcast("vsRepeatTrigger"); }; + $scope.showSchemaInfo = false; + $scope.openSchemaInfo = ($event, table) => { + $scope.tableName = table.name; + $scope.tableDescription = table.description; + $scope.tableMetadata = table.columns; + $scope.sampleQueries = Object.values(table.sample_queries); + $scope.showSchemaInfo = true; + $event.stopPropagation(); + }; + $scope.closeSchemaInfo = () => { + $scope.$apply(() => { + $scope.showSchemaInfo = false; + }); + }; + this.getSize = table => { let size = 22; @@ -34,6 +49,14 @@ function SchemaBrowserCtrl($rootScope, $scope) { } }; + this.itemExists = item => { + if ("visible" in item) { + return item.visible; + } else { + return false; + } + }; + this.itemSelected = ($event, hierarchy) => { $rootScope.$broadcast("query-editor.command", "paste", hierarchy.join(".")); $event.preventDefault(); diff --git a/client/app/pages/data-sources/EditDataSource.jsx b/client/app/pages/data-sources/EditDataSource.jsx index b3716c6af1..0408aecb3f 100644 --- a/client/app/pages/data-sources/EditDataSource.jsx +++ b/client/app/pages/data-sources/EditDataSource.jsx @@ -10,6 +10,7 @@ import notification from "@/services/notification"; import PromiseRejectionError from "@/lib/promise-rejection-error"; import LoadingState from "@/components/items-list/components/LoadingState"; import DynamicForm from "@/components/dynamic-form/DynamicForm"; +import SchemaTable from "@/pages/data-sources/schema-table-components/SchemaTable"; import helper from "@/components/dynamic-form/dynamicFormHelper"; import HelpTrigger, { TYPES as HELP_TRIGGER_TYPES } from "@/components/HelpTrigger"; import wrapSettingsTab from "@/components/SettingsWrapper"; @@ -27,6 +28,7 @@ class EditDataSource extends React.Component { dataSource: null, type: null, loading: true, + schema: null, }; componentDidMount() { @@ -34,7 +36,15 @@ class EditDataSource extends React.Component { .$promise.then(dataSource => { const { type } = dataSource; this.setState({ dataSource }); - DataSource.types(types => this.setState({ type: find(types, { type }), loading: false })); + const typesPromise = DataSource.types().$promise; + const schemaPromise = DataSource.schema({ id: $route.current.params.dataSourceId }).$promise; + + typesPromise.then(types => this.setState({ type: find(types, { type }) })); + schemaPromise.then(data => this.setState({ schema: data.schema })); + + Promise.all([typesPromise, schemaPromise]).then(() => { + this.setState({ loading: false }); + }); }) .catch(error => { // ANGULAR_REMOVE_ME This code is related to Angular's HTTP services @@ -84,6 +94,12 @@ class EditDataSource extends React.Component { }); }; + updateSchema = (schema, tableId, columnId) => { + const { dataSource } = this.state; + const data = { tableId, columnId, schema }; + DataSource.updateSchema({ id: dataSource.id }, data); + }; + testConnection = callback => { const { dataSource } = this.state; DataSource.test( @@ -138,6 +154,9 @@ class EditDataSource extends React.Component {
+
+ +
); } diff --git a/client/app/pages/data-sources/schema-table-components/EditableTable.jsx b/client/app/pages/data-sources/schema-table-components/EditableTable.jsx new file mode 100644 index 0000000000..4150b7802b --- /dev/null +++ b/client/app/pages/data-sources/schema-table-components/EditableTable.jsx @@ -0,0 +1,83 @@ +import React from "react"; +import Form from "antd/lib/form"; +import Input from "antd/lib/input"; +import PropTypes from "prop-types"; +import { TableMetadata } from "@/components/proptypes"; +import TableVisibilityCheckbox from "./TableVisibilityCheckbox"; +import SampleQueryList from "./SampleQueryList"; + +import "./schema-table.css"; + +const FormItem = Form.Item; +const { TextArea } = Input; +export const EditableContext = React.createContext(); + +// eslint-disable-next-line react/prop-types +const EditableRow = ({ form, index, ...props }) => ( + + + +); + +export const EditableFormRow = Form.create()(EditableRow); + +export class EditableCell extends React.Component { + static propTypes = { + dataIndex: PropTypes.string, + input_type: PropTypes.string, + editing: PropTypes.bool, + record: TableMetadata, + }; + + static defaultProps = { + dataIndex: undefined, + input_type: undefined, + editing: false, + record: {}, + }; + + constructor(props) { + super(props); + this.state = { + visible: this.props.record ? this.props.record.visible : false, + }; + } + + onChange = () => { + this.setState(prevState => ({ visible: !prevState.visible })); + }; + + getInput = () => { + if (this.props.input_type === "visible") { + return ; + } else if (this.props.input_type === "sample_queries") { + return ; + } + return