Merge pull request apache#29 from john-bodley/john-bodley-cherry-pick…

…-4551-4655

John bodley cherry pick 4551 4655

superset/assets/javascripts/SqlLab/components/SqlEditor.jsx

@@ -241,6 +241,7 @@ class SqlEditor extends React.PureComponent {
             >
               <SqlEditorLeftBar
                 height={height}
+                database={this.props.database}
                 queryEditor={this.props.queryEditor}
                 tables={this.props.tables}
                 actions={this.props.actions}

superset/assets/javascripts/SqlLab/components/SqlEditorLeftBar.jsx

@@ -16,6 +16,7 @@ const propTypes = {
   height: PropTypes.number.isRequired,
   tables: PropTypes.array,
   actions: PropTypes.object,
+  database: PropTypes.object,
 };
 
 const defaultProps = {
@@ -139,6 +140,14 @@ class SqlEditorLeftBar extends React.PureComponent {
   render() {
     const shouldShowReset = window.location.search === '?reset=1';
     const tableMetaDataHeight = this.props.height - 130; // 130 is the height of the selects above
+    let tableSelectPlaceholder;
+    let tableSelectDisabled = false;
+    if (this.props.database && this.props.database.allow_multi_schema_metadata_fetch) {
+      tableSelectPlaceholder = t('Type to search ...');
+    } else {
+      tableSelectPlaceholder = t('Select table ');
+      tableSelectDisabled = true;
+    }
     return (
       <div className="clearfix sql-toolbar">
         <div>
@@ -186,7 +195,6 @@ class SqlEditorLeftBar extends React.PureComponent {
               name="select-table"
               ref="selectTable"
               isLoading={this.state.tableLoading}
-              value={this.state.tableName}
               placeholder={t('Add a table (%s)', this.state.tableOptions.length)}
               autosize={false}
               onChange={this.changeTable.bind(this)}
@@ -199,8 +207,8 @@ class SqlEditorLeftBar extends React.PureComponent {
               async
               name="async-select-table"
               ref="selectTable"
-              value={this.state.tableName}
-              placeholder={t('Type to search ...')}
+              placeholder={tableSelectPlaceholder}
+              disabled={tableSelectDisabled}
               autosize={false}
               onChange={this.changeTable.bind(this)}
               loadOptions={this.getTableNamesBySubStr.bind(this)}

superset/assets/spec/javascripts/sqllab/SqlEditorLeftBar_spec.jsx

@@ -23,6 +23,7 @@ describe('SqlEditorLeftBar', () => {
     },
     tables: [table],
     queryEditor: defaultQueryEditor,
+    database: {},
     height: 0,
   };
 

superset/assets/spec/javascripts/sqllab/TabbedSqlEditors_spec.jsx

@@ -51,6 +51,7 @@ describe('TabbedSqlEditors', () => {
     tabHistory: initialState.tabHistory,
     editorHeight: '',
     getHeight: () => ('100px'),
+    database: {},
   };
   const getWrapper = () => (
     shallow(<TabbedSqlEditors {...mockedProps} />, {

superset/cli.py

@@ -166,10 +166,9 @@ def load_examples(load_test_data):
 )
 @manager.option(
     '-m', '--merge',
-    help=(
-        "Specify using 'merge' property during operation. "
-        'Default value is False '
-    ),
+    action='store_true',
+    help="Specify using 'merge' property during operation.",
+    default=False,
 )
 def refresh_druid(datasource, merge):
     """Refresh druid datasources"""

superset/connectors/druid/models.py

@@ -29,7 +29,7 @@
 from six import string_types
 import sqlalchemy as sa
 from sqlalchemy import (
-    Boolean, Column, DateTime, ForeignKey, Integer, or_, String, Text, UniqueConstraint,
+    Boolean, Column, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint,
 )
 from sqlalchemy.orm import backref, relationship
 
@@ -199,33 +199,31 @@ def refresh(self, datasource_names, merge_flag, refreshAll):
                 col_objs_list = (
                     session.query(DruidColumn)
                     .filter(DruidColumn.datasource_id == datasource.id)
-                    .filter(or_(DruidColumn.column_name == col for col in cols))
+                    .filter(DruidColumn.column_name.in_(cols.keys()))
                 )
                 col_objs = {col.column_name: col for col in col_objs_list}
                 for col in cols:
                     if col == '__time':  # skip the time column
                         continue
-                    col_obj = col_objs.get(col, None)
+                    col_obj = col_objs.get(col)
                     if not col_obj:
                         col_obj = DruidColumn(
                             datasource_id=datasource.id,
                             column_name=col)
                         with session.no_autoflush:
                             session.add(col_obj)
-                    datatype = cols[col]['type']
-                    if datatype == 'STRING':
+                    col_obj.type = cols[col]['type']
+                    col_obj.datasource = datasource
+                    if col_obj.type == 'STRING':
                         col_obj.groupby = True
                         col_obj.filterable = True
-                    if datatype == 'hyperUnique' or datatype == 'thetaSketch':
+                    if col_obj.type == 'hyperUnique' or col_obj.type == 'thetaSketch':
                         col_obj.count_distinct = True
-                    # Allow sum/min/max for long or double
-                    if datatype == 'LONG' or datatype == 'DOUBLE':
+                    if col_obj.is_num:
                         col_obj.sum = True
                         col_obj.min = True
                         col_obj.max = True
-                    col_obj.type = datatype
-                    col_obj.datasource = datasource
-                datasource.generate_metrics_for(col_objs_list)
+                datasource.refresh_metrics()
         session.commit()
 
     @property
@@ -360,21 +358,24 @@ def get_metrics(self):
                 )
         return metrics
 
-    def generate_metrics(self):
-        """Generate metrics based on the column metadata"""
+    def refresh_metrics(self):
+        """Refresh metrics based on the column metadata"""
         metrics = self.get_metrics()
         dbmetrics = (
             db.session.query(DruidMetric)
             .filter(DruidMetric.datasource_id == self.datasource_id)
-            .filter(or_(
-                DruidMetric.metric_name == m for m in metrics
-            ))
+            .filter(DruidMetric.metric_name.in_(metrics.keys()))
         )
         dbmetrics = {metric.metric_name: metric for metric in dbmetrics}
         for metric in metrics.values():
-            metric.datasource_id = self.datasource_id
-            if not dbmetrics.get(metric.metric_name, None):
-                db.session.add(metric)
+            dbmetric = dbmetrics.get(metric.metric_name)
+            if dbmetric:
+                for attr in ['json', 'metric_type', 'verbose_name']:
+                    setattr(dbmetric, attr, getattr(metric, attr))
+            else:
+                with db.session.no_autoflush:
+                    metric.datasource_id = self.datasource_id
+                    db.session.add(metric)
 
     @classmethod
     def import_obj(cls, i_column):
@@ -651,24 +652,9 @@ def latest_metadata(self):
         if segment_metadata:
             return segment_metadata[-1]['columns']
 
-    def generate_metrics(self):
-        self.generate_metrics_for(self.columns)
-
-    def generate_metrics_for(self, columns):
-        metrics = {}
-        for col in columns:
-            metrics.update(col.get_metrics())
-        dbmetrics = (
-            db.session.query(DruidMetric)
-            .filter(DruidMetric.datasource_id == self.id)
-            .filter(or_(DruidMetric.metric_name == m for m in metrics))
-        )
-        dbmetrics = {metric.metric_name: metric for metric in dbmetrics}
-        for metric in metrics.values():
-            metric.datasource_id = self.id
-            if not dbmetrics.get(metric.metric_name, None):
-                with db.session.no_autoflush:
-                    db.session.add(metric)
+    def refresh_metrics(self):
+        for col in self.columns:
+            col.refresh_metrics()
 
     @classmethod
     def sync_to_db_from_config(
@@ -701,7 +687,7 @@ def sync_to_db_from_config(
         col_objs = (
             session.query(DruidColumn)
             .filter(DruidColumn.datasource_id == datasource.id)
-            .filter(or_(DruidColumn.column_name == dim for dim in dimensions))
+            .filter(DruidColumn.column_name.in_(dimensions))
         )
         col_objs = {col.column_name: col for col in col_objs}
         for dim in dimensions:
@@ -721,8 +707,9 @@ def sync_to_db_from_config(
         metric_objs = (
             session.query(DruidMetric)
             .filter(DruidMetric.datasource_id == datasource.id)
-            .filter(or_(DruidMetric.metric_name == spec['name']
-                    for spec in druid_config['metrics_spec']))
+            .filter(DruidMetric.metric_name.in_(
+                spec['name'] for spec in druid_config['metrics_spec']
+            ))
         )
         metric_objs = {metric.metric_name: metric for metric in metric_objs}
         for metric_spec in druid_config['metrics_spec']:

superset/connectors/druid/views.py

@@ -91,7 +91,7 @@ def pre_update(self, col):
                     .format(dimension_spec['outputName'], col.column_name))
 
     def post_update(self, col):
-        col.generate_metrics()
+        col.refresh_metrics()
 
     def post_add(self, col):
         self.post_update(col)
@@ -277,7 +277,7 @@ def pre_add(self, datasource):
                     datasource.full_name))
 
     def post_add(self, datasource):
-        datasource.generate_metrics()
+        datasource.refresh_metrics()
         security.merge_perm(sm, 'datasource_access', datasource.get_perm())
         if datasource.schema:
             security.merge_perm(sm, 'schema_access', datasource.schema_perm)

superset/migrations/versions/e68c4473c581_allow_multi_schema_metadata_fetch.py

@@ -0,0 +1,30 @@
+"""allow_multi_schema_metadata_fetch
+
+Revision ID: e68c4473c581
+Revises: e866bd2d4976
+Create Date: 2018-03-06 12:24:30.896293
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'e68c4473c581'
+down_revision = 'e866bd2d4976'
+
+
+def upgrade():
+
+    op.add_column(
+        'dbs',
+        sa.Column(
+            'allow_multi_schema_metadata_fetch',
+            sa.Boolean(),
+            nullable=True,
+            default=True,
+        ),
+    )
+
+
+def downgrade():
+    op.drop_column('dbs', 'allow_multi_schema_metadata_fetch')

superset/migrations/versions/f231d82b9b26_.py

@@ -0,0 +1,72 @@
+"""empty message
+
+Revision ID: f231d82b9b26
+Revises: e68c4473c581
+Create Date: 2018-03-20 19:47:54.991259
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'f231d82b9b26'
+down_revision = 'e68c4473c581'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.exc import OperationalError
+
+from superset.utils import generic_find_uq_constraint_name
+
+conv = {
+    'uq': 'uq_%(table_name)s_%(column_0_name)s',
+}
+
+names = {
+    'columns': 'column_name',
+    'metrics': 'metric_name',
+}
+
+bind = op.get_bind()
+insp = sa.engine.reflection.Inspector.from_engine(bind)
+
+
+def upgrade():
+
+    # Reduce the size of the metric_name column for constraint viability.
+    with op.batch_alter_table('metrics', naming_convention=conv) as batch_op:
+        batch_op.alter_column(
+            'metric_name',
+            existing_type=sa.String(length=512),
+            type_=sa.String(length=255),
+            existing_nullable=True,
+        )
+
+    # Add the missing uniqueness constraints.
+    for table, column in names.items():
+        with op.batch_alter_table(table, naming_convention=conv) as batch_op:
+            batch_op.create_unique_constraint(
+                'uq_{}_{}'.format(table, column),
+                [column, 'datasource_id'],
+            )
+
+def downgrade():
+
+    # Restore the size of the metric_name column.
+    with op.batch_alter_table('metrics', naming_convention=conv) as batch_op:
+        batch_op.alter_column(
+            'metric_name',
+            existing_type=sa.String(length=255),
+            type_=sa.String(length=512),
+            existing_nullable=True,
+        )
+
+    # Remove the previous missing uniqueness constraints.
+    for table, column in names.items():
+        with op.batch_alter_table(table, naming_convention=conv) as batch_op:
+            batch_op.drop_constraint(
+                    generic_find_uq_constraint_name(
+                        table,
+                        {column, 'datasource_id'},
+                        insp,
+                    ) or 'uq_{}_{}'.format(table, column),
+                    type_='unique',
+                )

superset/models/core.py

@@ -567,6 +567,7 @@ class Database(Model, AuditMixinNullable, ImportMixin):
     allow_ctas = Column(Boolean, default=False)
     allow_dml = Column(Boolean, default=False)
     force_ctas_schema = Column(String(250))
+    allow_multi_schema_metadata_fetch = Column(Boolean, default=True)
     extra = Column(Text, default=textwrap.dedent("""\
     {
         "metadata_params": {},
@@ -593,6 +594,8 @@ def data(self):
         return {
             'name': self.database_name,
             'backend': self.backend,
+            'allow_multi_schema_metadata_fetch':
+                self.allow_multi_schema_metadata_fetch,
         }
 
     @property
@@ -733,6 +736,8 @@ def inspector(self):
 
     def all_table_names(self, schema=None, force=False):
         if not schema:
+            if not self.allow_multi_schema_metadata_fetch:
+                return []
             tables_dict = self.db_engine_spec.fetch_result_sets(
                 self, 'table', force=force)
             return tables_dict.get('', [])
@@ -741,6 +746,8 @@ def all_table_names(self, schema=None, force=False):
 
     def all_view_names(self, schema=None, force=False):
         if not schema:
+            if not self.allow_multi_schema_metadata_fetch:
+                return []
             views_dict = self.db_engine_spec.fetch_result_sets(
                 self, 'view', force=force)
             return views_dict.get('', [])

superset/views/core.py

@@ -199,7 +199,9 @@ class DatabaseView(SupersetModelView, DeleteMixin, YamlExportMixin):  # noqa
     add_columns = [
         'database_name', 'sqlalchemy_uri', 'cache_timeout', 'extra',
         'expose_in_sqllab', 'allow_run_sync', 'allow_run_async',
-        'allow_ctas', 'allow_dml', 'force_ctas_schema', 'impersonate_user']
+        'allow_ctas', 'allow_dml', 'force_ctas_schema', 'impersonate_user',
+        'allow_multi_schema_metadata_fetch',
+    ]
     search_exclude_columns = (
         'password', 'tables', 'created_by', 'changed_by', 'queries',
         'saved_queries')
@@ -258,6 +260,10 @@ class DatabaseView(SupersetModelView, DeleteMixin, YamlExportMixin):  # noqa
             'If Hive and hive.server2.enable.doAs is enabled, will run the queries as '
             'service account, but impersonate the currently logged on user '
             'via hive.server2.proxy.user property.'),
+        'allow_multi_schema_metadata_fetch': _(
+            'Allow SQL Lab to fetch a list of all tables and all views across '
+            'all database schemas. For large data warehouse with thousands of '
+            'tables, this can be expensive and put strain on the system.'),
     }
     label_columns = {
         'expose_in_sqllab': _('Expose in SQL Lab'),
@@ -314,6 +320,7 @@ class DatabaseAsync(DatabaseView):
         'id', 'database_name',
         'expose_in_sqllab', 'allow_ctas', 'force_ctas_schema',
         'allow_run_async', 'allow_run_sync', 'allow_dml',
+        'allow_multi_schema_metadata_fetch',
     ]