Hide nested fields across Kibana apps (#55278) (#56030)

* Hide nested fields across Kibana apps

* Filter out nested fields in TSVB

* Fix import paths

* Hide nested fields in timelion autocomplete

* Fix remaining map places

* Filter out nested fields in graph

* Fix remaining map places

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

src/legacy/core_plugins/timelion/public/services/arg_value_suggestions.ts

@@ -20,6 +20,7 @@
 import { get } from 'lodash';
 import { TimelionFunctionArgs } from '../../common/types';
 import { getIndexPatterns, getSavedObjectsClient } from './plugin_services';
+import { isNestedField } from '../../../../../../src/plugins/data/public';
 
 export interface Location {
   min: number;
@@ -120,7 +121,8 @@ export function getArgValueSuggestions() {
             return (
               field.aggregatable &&
               'number' === field.type &&
-              containsFieldName(valueSplit[1], field)
+              containsFieldName(valueSplit[1], field) &&
+              !isNestedField(field)
             );
           })
           .map(field => {
@@ -138,7 +140,8 @@ export function getArgValueSuggestions() {
             return (
               field.aggregatable &&
               ['number', 'boolean', 'date', 'ip', 'string'].includes(field.type) &&
-              containsFieldName(partial, field)
+              containsFieldName(partial, field) &&
+              !isNestedField(field)
             );
           })
           .map(field => {
@@ -153,7 +156,9 @@ export function getArgValueSuggestions() {
 
         return indexPattern.fields
           .filter(field => {
-            return 'date' === field.type && containsFieldName(partial, field);
+            return (
+              'date' === field.type && containsFieldName(partial, field) && !isNestedField(field)
+            );
           })
           .map(field => {
             return { name: field.name };

src/legacy/core_plugins/vis_type_timeseries/server/lib/get_fields.js

@@ -19,6 +19,7 @@
 import { SearchStrategiesRegister } from './search_strategies/search_strategies_register';
 import { uniq } from 'lodash';
 import { getIndexPatternObject } from './vis_data/helpers/get_index_pattern';
+import { isNestedField } from '../../../../../plugins/data/server';
 
 export async function getFields(req) {
   const indexPattern = req.query.index;
@@ -30,7 +31,7 @@ export async function getFields(req) {
 
   const fields = (
     await searchStrategy.getFieldsForWildcard(req, indexPatternString, capabilities)
-  ).filter(field => field.aggregatable);
+  ).filter(field => field.aggregatable && !isNestedField(field));
 
   return uniq(fields, field => field.name);
 }

src/legacy/ui/public/agg_types/param_types/field.ts

@@ -26,6 +26,7 @@ import { BaseParamType } from './base';
 import { toastNotifications } from '../../notify';
 import { propFilter } from '../filter';
 import { Field, IFieldList } from '../../../../../plugins/data/public';
+import { isNestedField } from '../../../../../plugins/data/public';
 
 const filterByType = propFilter('type');
 
@@ -116,7 +117,7 @@ export class FieldParamType extends BaseParamType {
       const { onlyAggregatable, scriptable, filterFieldTypes } = this;
 
       if (
-        (onlyAggregatable && (!field.aggregatable || field.subType?.nested)) ||
+        (onlyAggregatable && (!field.aggregatable || isNestedField(field))) ||
         (!scriptable && field.scripted)
       ) {
         return false;

src/plugins/data/common/index_patterns/fields/index.ts

@@ -18,4 +18,4 @@
  */
 
 export * from './types';
-export { isFilterable } from './utils';
+export { isFilterable, isNestedField } from './utils';

src/plugins/data/common/index_patterns/fields/utils.ts

@@ -28,3 +28,7 @@ export function isFilterable(field: IFieldType): boolean {
     Boolean(field.searchable && filterableTypes.includes(field.type))
   );
 }
+
+export function isNestedField(field: IFieldType): boolean {
+  return !!field.subType?.nested;
+}

src/plugins/data/public/index.ts

@@ -92,6 +92,7 @@ export {
   getKbnTypeNames,
   // utils
   parseInterval,
+  isNestedField,
 } from '../common';
 
 // Export plugin after all other imports

src/plugins/data/server/index.ts

@@ -49,6 +49,7 @@ export {
   TimeRange,
   // utils
   parseInterval,
+  isNestedField,
 } from '../common';
 
 /**

x-pack/legacy/plugins/graph/public/services/persistence/deserialize.ts

@@ -24,7 +24,7 @@ import {
   colorChoices,
   iconChoicesByClass,
 } from '../../helpers/style_choices';
-import { IndexPattern } from '../../../../../../../src/plugins/data/public';
+import { IndexPattern, isNestedField } from '../../../../../../../src/plugins/data/public';
 
 const defaultAdvancedSettings: AdvancedSettings = {
   useSignificance: true,
@@ -80,7 +80,7 @@ export function mapFields(indexPattern: IndexPattern): WorkspaceField[] {
 
   return indexPattern
     .getNonScriptedFields()
-    .filter(field => !blockedFieldNames.includes(field.name))
+    .filter(field => !blockedFieldNames.includes(field.name) && !isNestedField(field))
     .map((field, index) => ({
       name: field.name,
       hopSize: defaultHopSize,

x-pack/legacy/plugins/lens/public/indexpattern_plugin/loader.ts

@@ -21,6 +21,7 @@ import { updateLayerIndexPattern } from './state_helpers';
 import { DateRange, ExistingFields } from '../../common/types';
 import { BASE_API_URL } from '../../common';
 import { documentField } from './document_field';
+import { isNestedField, IFieldType } from '../../../../../../src/plugins/data/public';
 
 interface SavedIndexPatternAttributes extends SavedObjectAttributes {
   title: string;
@@ -270,9 +271,9 @@ function fromSavedObject(
     id,
     type,
     title: attributes.title,
-    fields: (JSON.parse(attributes.fields) as IndexPatternField[])
-      .filter(({ aggregatable, scripted }) => !!aggregatable || !!scripted)
-      .concat(documentField),
+    fields: (JSON.parse(attributes.fields) as IFieldType[])
+      .filter(field => !isNestedField(field) && (!!field.aggregatable || !!field.scripted))
+      .concat(documentField) as IndexPatternField[],
     typeMeta: attributes.typeMeta
       ? (JSON.parse(attributes.typeMeta) as SavedRestrictionsInfo)
       : undefined,

x-pack/legacy/plugins/maps/public/connected_components/gis_map/view.js

@@ -15,6 +15,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiCallOut } from '@elastic/eui';
 import { ExitFullScreenButton } from 'ui/exit_full_screen';
 import { getIndexPatternsFromIds } from '../../index_pattern_util';
 import { ES_GEO_FIELD_TYPE } from '../../../common/constants';
+import { isNestedField } from '../../../../../../../src/plugins/data/public';
 import { i18n } from '@kbn/i18n';
 import uuid from 'uuid/v4';
 
@@ -80,8 +81,9 @@ export class GisMap extends Component {
       indexPatterns.forEach(indexPattern => {
         indexPattern.fields.forEach(field => {
           if (
-            field.type === ES_GEO_FIELD_TYPE.GEO_POINT ||
-            field.type === ES_GEO_FIELD_TYPE.GEO_SHAPE
+            !isNestedField(field) &&
+            (field.type === ES_GEO_FIELD_TYPE.GEO_POINT ||
+              field.type === ES_GEO_FIELD_TYPE.GEO_SHAPE)
           ) {
             geoFields.push({
               geoFieldName: field.name,

x-pack/legacy/plugins/maps/public/connected_components/layer_panel/join_editor/resources/join.js

@@ -13,6 +13,7 @@ import { MetricsExpression } from './metrics_expression';
 import { WhereExpression } from './where_expression';
 import { GlobalFilterCheckbox } from '../../../../components/global_filter_checkbox';
 
+import { isNestedField } from '../../../../../../../../../src/plugins/data/public';
 import { indexPatternService } from '../../../../kibana_services';
 
 const getIndexPatternId = props => {
@@ -88,7 +89,7 @@ export class Join extends Component {
     }
 
     this.setState({
-      rightFields: indexPattern.fields,
+      rightFields: indexPattern.fields.filter(field => !isNestedField(field)),
       indexPattern,
     });
   }

x-pack/legacy/plugins/maps/public/connected_components/layer_panel/join_editor/resources/join_expression.js

@@ -23,6 +23,7 @@ import { getTermsFields } from '../../../../index_pattern_util';
 import { indexPatternService } from '../../../../kibana_services';
 
 import { npStart } from 'ui/new_platform';
+import { isNestedField } from '../../../../../../../../../src/plugins/data/public';
 const { IndexPatternSelect } = npStart.plugins.data.ui;
 
 export class JoinExpression extends Component {
@@ -134,7 +135,7 @@ export class JoinExpression extends Component {
     }
 
     const filterStringOrNumberFields = field => {
-      return field.type === 'string' || field.type === 'number';
+      return (field.type === 'string' && !isNestedField(field)) || field.type === 'number';
     };
 
     return (

x-pack/legacy/plugins/maps/public/index_pattern_util.js

@@ -5,6 +5,7 @@
  */
 
 import { indexPatternService } from './kibana_services';
+import { isNestedField } from '../../../../../src/plugins/data/public';
 
 export async function getIndexPatternsFromIds(indexPatternIds = []) {
   const promises = [];
@@ -20,7 +21,11 @@ export async function getIndexPatternsFromIds(indexPatternIds = []) {
 
 export function getTermsFields(fields) {
   return fields.filter(field => {
-    return field.aggregatable && ['number', 'boolean', 'date', 'ip', 'string'].includes(field.type);
+    return (
+      field.aggregatable &&
+      !isNestedField(field) &&
+      ['number', 'boolean', 'date', 'ip', 'string'].includes(field.type)
+    );
   });
 }
 
@@ -29,6 +34,6 @@ export function getSourceFields(fields) {
   return fields.filter(field => {
     // Multi fields are not stored in _source and only exist in index.
     const isMultiField = field.subType && field.subType.multi;
-    return !isMultiField;
+    return !isMultiField && !isNestedField(field);
   });
 }

x-pack/legacy/plugins/maps/public/layers/fields/es_doc_field.js

@@ -7,13 +7,15 @@
 import { AbstractField } from './field';
 import { ESTooltipProperty } from '../tooltips/es_tooltip_property';
 import { COLOR_PALETTE_MAX_SIZE } from '../../../common/constants';
+import { isNestedField } from '../../../../../../../src/plugins/data/public';
 
 export class ESDocField extends AbstractField {
   static type = 'ES_DOC';
 
   async _getField() {
     const indexPattern = await this._source.getIndexPattern();
-    return indexPattern.fields.getByName(this._fieldName);
+    const field = indexPattern.fields.getByName(this._fieldName);
+    return isNestedField(field) ? undefined : field;
   }
 
   async createTooltipProperty(value) {

x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/create_source_editor.js

@@ -16,6 +16,7 @@ import { i18n } from '@kbn/i18n';
 
 import { EuiFormRow, EuiComboBox, EuiSpacer } from '@elastic/eui';
 import { ES_GEO_FIELD_TYPE } from '../../../../common/constants';
+import { isNestedField } from '../../../../../../../../src/plugins/data/public';
 
 import { npStart } from 'ui/new_platform';
 const { IndexPatternSelect } = npStart.plugins.data.ui;
@@ -115,7 +116,9 @@ export class CreateSourceEditor extends Component {
     });
 
     //make default selection
-    const geoFields = indexPattern.fields.filter(filterGeoField);
+    const geoFields = indexPattern.fields
+      .filter(field => !isNestedField(field))
+      .filter(filterGeoField);
     if (geoFields[0]) {
       this._onGeoFieldSelect(geoFields[0].name);
     }
@@ -171,7 +174,11 @@ export class CreateSourceEditor extends Component {
           value={this.state.geoField}
           onChange={this._onGeoFieldSelect}
           filterField={filterGeoField}
-          fields={this.state.indexPattern ? this.state.indexPattern.fields : undefined}
+          fields={
+            this.state.indexPattern
+              ? this.state.indexPattern.fields.filter(field => !isNestedField(field))
+              : undefined
+          }
         />
       </EuiFormRow>
     );

x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/update_source_editor.js

@@ -14,6 +14,7 @@ import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { EuiPanel, EuiSpacer, EuiTitle } from '@elastic/eui';
 import { isMetricCountable } from '../../util/is_metric_countable';
+import { isNestedField } from '../../../../../../../../src/plugins/data/public';
 
 export class UpdateSourceEditor extends Component {
   state = {
@@ -51,7 +52,7 @@ export class UpdateSourceEditor extends Component {
       return;
     }
 
-    this.setState({ fields: indexPattern.fields });
+    this.setState({ fields: indexPattern.fields.filter(field => !isNestedField(field)) });
   }
 
   _onMetricsChange = metrics => {

x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/create_source_editor.js

@@ -15,6 +15,7 @@ import { FormattedMessage } from '@kbn/i18n/react';
 
 import { EuiFormRow, EuiCallOut } from '@elastic/eui';
 import { ES_GEO_FIELD_TYPE } from '../../../../common/constants';
+import { isNestedField } from '../../../../../../../../src/plugins/data/public';
 
 import { npStart } from 'ui/new_platform';
 const { IndexPatternSelect } = npStart.plugins.data.ui;
@@ -91,7 +92,9 @@ export class CreateSourceEditor extends Component {
       return;
     }
 
-    const geoFields = indexPattern.fields.filter(filterGeoField);
+    const geoFields = indexPattern.fields
+      .filter(field => !isNestedField(field))
+      .filter(filterGeoField);
 
     this.setState({
       isLoadingIndexPattern: false,
@@ -163,7 +166,11 @@ export class CreateSourceEditor extends Component {
             value={this.state.destGeoField}
             onChange={this._onDestGeoSelect}
             filterField={filterGeoField}
-            fields={this.state.indexPattern ? this.state.indexPattern.fields : undefined}
+            fields={
+              this.state.indexPattern
+                ? this.state.indexPattern.fields.filter(field => !isNestedField(field))
+                : undefined
+            }
           />
         </EuiFormRow>
       </Fragment>

x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/es_pew_pew_source.js

@@ -25,6 +25,7 @@ import { AggConfigs } from 'ui/agg_types';
 import { AbstractESAggSource } from '../es_agg_source';
 import { DynamicStyleProperty } from '../../styles/vector/properties/dynamic_style_property';
 import { COLOR_GRADIENTS } from '../../styles/color_utils';
+import { isNestedField } from '../../../../../../../../src/plugins/data/public';
 
 const MAX_GEOTILE_LEVEL = 29;
 
@@ -228,7 +229,8 @@ export class ESPewPewSource extends AbstractESAggSource {
 
   async _getGeoField() {
     const indexPattern = await this.getIndexPattern();
-    const geoField = indexPattern.fields.getByName(this._descriptor.destGeoField);
+    const field = indexPattern.fields.getByName(this._descriptor.destGeoField);
+    const geoField = isNestedField(field) ? undefined : field;
     if (!geoField) {
       throw new Error(
         i18n.translate('xpack.maps.source.esSource.noGeoFieldErrorMessage', {

x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/update_source_editor.js

@@ -11,6 +11,7 @@ import { indexPatternService } from '../../../kibana_services';
 import { i18n } from '@kbn/i18n';
 import { EuiPanel, EuiTitle, EuiSpacer } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n/react';
+import { isNestedField } from '../../../../../../../../src/plugins/data/public';
 
 export class UpdateSourceEditor extends Component {
   state = {
@@ -48,7 +49,7 @@ export class UpdateSourceEditor extends Component {
       return;
     }
 
-    this.setState({ fields: indexPattern.fields });
+    this.setState({ fields: indexPattern.fields.filter(field => !isNestedField(field)) });
   }
 
   _onMetricsChange = metrics => {

x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/create_source_editor.js

@@ -21,6 +21,7 @@ import {
   DEFAULT_MAX_RESULT_WINDOW,
 } from '../../../../common/constants';
 import { DEFAULT_FILTER_BY_MAP_BOUNDS } from './constants';
+import { isNestedField } from '../../../../../../../../src/plugins/data/public';
 
 import { npStart } from 'ui/new_platform';
 const { IndexPatternSelect } = npStart.plugins.data.ui;
@@ -124,7 +125,9 @@ export class CreateSourceEditor extends Component {
     });
 
     //make default selection
-    const geoFields = indexPattern.fields.filter(filterGeoField);
+    const geoFields = indexPattern.fields
+      .filter(field => !isNestedField(field))
+      .filter(filterGeoField);
     if (geoFields[0]) {
       this.onGeoFieldSelect(geoFields[0].name);
     }
@@ -178,7 +181,11 @@ export class CreateSourceEditor extends Component {
           value={this.state.geoField}
           onChange={this.onGeoFieldSelect}
           filterField={filterGeoField}
-          fields={this.state.indexPattern ? this.state.indexPattern.fields : undefined}
+          fields={
+            this.state.indexPattern
+              ? this.state.indexPattern.fields.filter(field => !isNestedField(field))
+              : undefined
+          }
         />
       </EuiFormRow>
     );