From 2c52ac28cbfc1062ee486f5e5134d453fe7771e3 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Mon, 17 Jan 2022 22:42:09 +0300 Subject: [PATCH] [Lens] Provide formula helper to simplify integration of Lens instances (#122371) * [Lens] Provide formula helper to simplify integration of Lens instances Closes: #103055 * remove generateFormulaColumns from start contract * upsertFormulaColumn * add upsertFormulaColumn to start contract * add integration with embedded_lens_examples * upsert -> insertOrReplace * add support of overriding operations * add docs * fix TS issues * fix some comments * fix PR comments * fix PR comments * fix CI * Update x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula_helper.ts Co-authored-by: Marco Liberati * Update x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula_helper.ts Co-authored-by: Marco Liberati * remove useEffect * move baseLayer part into getLensAttributes * introduce stateHelperApi * Map -> WeakMap * remove [params.operations] from params Co-authored-by: Marco Liberati Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../embedded_lens_example/public/app.tsx | 297 +++++++++--------- .../embedded_lens_example/public/mount.tsx | 27 +- x-pack/plugins/lens/public/async_services.ts | 2 + x-pack/plugins/lens/public/index.ts | 1 + .../dimension_panel/format_selector.tsx | 9 +- .../public/indexpattern_datasource/index.ts | 2 + .../public/indexpattern_datasource/loader.ts | 150 ++++----- .../operations/definitions/column_types.ts | 9 +- .../formula/editor/formula_editor.tsx | 98 ++++-- .../definitions/formula/formula.test.tsx | 136 +++++--- .../definitions/formula/formula.tsx | 29 +- .../formula/formula_public_api.test.ts | 106 +++++++ .../definitions/formula/formula_public_api.ts | 75 +++++ .../operations/definitions/formula/index.ts | 4 +- .../operations/definitions/formula/parse.ts | 141 ++++++--- .../operations/index.ts | 1 + .../operations/layer_helpers.ts | 11 +- .../public/indexpattern_datasource/types.ts | 4 + .../lens/public/mocks/lens_plugin_mock.tsx | 6 + x-pack/plugins/lens/public/plugin.ts | 16 + 20 files changed, 728 insertions(+), 396 deletions(-) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula_public_api.test.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula_public_api.ts diff --git a/x-pack/examples/embedded_lens_example/public/app.tsx b/x-pack/examples/embedded_lens_example/public/app.tsx index f1b683f2430f7..510d9469c7878 100644 --- a/x-pack/examples/embedded_lens_example/public/app.tsx +++ b/x-pack/examples/embedded_lens_example/public/app.tsx @@ -17,37 +17,32 @@ import { EuiPageHeader, EuiPageHeaderSection, EuiTitle, - EuiCallOut, } from '@elastic/eui'; -import { IndexPattern } from 'src/plugins/data/public'; -import { CoreStart } from 'kibana/public'; -import { ViewMode } from '../../../../src/plugins/embeddable/public'; -import { + +import type { DataView } from 'src/plugins/data_views/public'; +import type { CoreStart } from 'kibana/public'; +import type { StartDependencies } from './plugin'; +import type { TypedLensByValueInput, PersistedIndexPatternLayer, XYState, LensEmbeddableInput, + FormulaPublicApi, DateHistogramIndexPatternColumn, } from '../../../plugins/lens/public'; -import { StartDependencies } from './plugin'; + +import { ViewMode } from '../../../../src/plugins/embeddable/public'; // Generate a Lens state based on some app-specific input parameters. // `TypedLensByValueInput` can be used for type-safety - it uses the same interfaces as Lens-internal code. function getLensAttributes( - defaultIndexPattern: IndexPattern, - color: string + color: string, + dataView: DataView, + formula: FormulaPublicApi ): TypedLensByValueInput['attributes'] { - const dataLayer: PersistedIndexPatternLayer = { - columnOrder: ['col1', 'col2'], + const baseLayer: PersistedIndexPatternLayer = { + columnOrder: ['col1'], columns: { - col2: { - dataType: 'number', - isBucketed: false, - label: 'Count of records', - operationType: 'count', - scale: 'ratio', - sourceField: 'Records', - }, col1: { dataType: 'date', isBucketed: true, @@ -55,11 +50,18 @@ function getLensAttributes( operationType: 'date_histogram', params: { interval: 'auto' }, scale: 'interval', - sourceField: defaultIndexPattern.timeFieldName!, + sourceField: dataView.timeFieldName!, } as DateHistogramIndexPatternColumn, }, }; + const dataLayer = formula.insertOrReplaceFormulaColumn( + 'col2', + { formula: 'count()' }, + baseLayer, + dataView + ); + const xyConfig: XYState = { axisTitlesVisibilitySettings: { x: true, yLeft: true, yRight: true }, fittingFunction: 'None', @@ -85,12 +87,12 @@ function getLensAttributes( title: 'Prefilled from example app', references: [ { - id: defaultIndexPattern.id!, + id: dataView.id!, name: 'indexpattern-datasource-current-indexpattern', type: 'index-pattern', }, { - id: defaultIndexPattern.id!, + id: dataView.id!, name: 'indexpattern-datasource-layer-layer1', type: 'index-pattern', }, @@ -99,7 +101,7 @@ function getLensAttributes( datasourceStates: { indexpattern: { layers: { - layer1: dataLayer, + layer1: dataLayer!, }, }, }, @@ -113,19 +115,22 @@ function getLensAttributes( export const App = (props: { core: CoreStart; plugins: StartDependencies; - defaultIndexPattern: IndexPattern | null; + defaultDataView: DataView; + formula: FormulaPublicApi; }) => { const [color, setColor] = useState('green'); const [isLoading, setIsLoading] = useState(false); const [isSaveModalVisible, setIsSaveModalVisible] = useState(false); - const LensComponent = props.plugins.lens.EmbeddableComponent; - const LensSaveModalComponent = props.plugins.lens.SaveModalComponent; - const [time, setTime] = useState({ from: 'now-5d', to: 'now', }); + const LensComponent = props.plugins.lens.EmbeddableComponent; + const LensSaveModalComponent = props.plugins.lens.SaveModalComponent; + + const attributes = getLensAttributes(color, props.defaultDataView, props.formula); + return ( @@ -147,138 +152,122 @@ export const App = (props: { the series which causes Lens to re-render. The Edit button will take the current configuration and navigate to a prefilled editor.

- {props.defaultIndexPattern && props.defaultIndexPattern.isTimeBased() ? ( - <> - - - { - // eslint-disable-next-line no-bitwise - const newColor = '#' + ((Math.random() * 0xffffff) << 0).toString(16); - setColor(newColor); - }} - > - Change color - - - - { - props.plugins.lens.navigateToPrefilledEditor( - { - id: '', - timeRange: time, - attributes: getLensAttributes(props.defaultIndexPattern!, color), - }, - { - openInNewTab: true, - } - ); - // eslint-disable-next-line no-bitwise - const newColor = '#' + ((Math.random() * 0xffffff) << 0).toString(16); - setColor(newColor); - }} - > - Edit in Lens (new tab) - - - - { - props.plugins.lens.navigateToPrefilledEditor( - { - id: '', - timeRange: time, - attributes: getLensAttributes(props.defaultIndexPattern!, color), - }, - { - openInNewTab: false, - } - ); - }} - > - Edit in Lens (same tab) - - - - { - setIsSaveModalVisible(true); - }} - > - Save Visualization - - - - { - setTime({ - from: '2015-09-18T06:31:44.000Z', - to: '2015-09-23T18:31:44.000Z', - }); - }} - > - Change time range - - - - { - setIsLoading(val); + + + + { + // eslint-disable-next-line no-bitwise + const newColor = '#' + ((Math.random() * 0xffffff) << 0).toString(16); + setColor(newColor); }} - onBrushEnd={({ range }) => { - setTime({ - from: new Date(range[0]).toISOString(), - to: new Date(range[1]).toISOString(), - }); + > + Change color + + + + { + props.plugins.lens.navigateToPrefilledEditor( + { + id: '', + timeRange: time, + attributes, + }, + { + openInNewTab: true, + } + ); + // eslint-disable-next-line no-bitwise + const newColor = '#' + ((Math.random() * 0xffffff) << 0).toString(16); + setColor(newColor); + }} + > + Edit in Lens (new tab) + + + + { + props.plugins.lens.navigateToPrefilledEditor( + { + id: '', + timeRange: time, + attributes, + }, + { + openInNewTab: false, + } + ); }} - onFilter={(_data) => { - // call back event for on filter event + > + Edit in Lens (same tab) + + + + { + setIsSaveModalVisible(true); }} - onTableRowClick={(_data) => { - // call back event for on table row click event + > + Save Visualization + + + + { + setTime({ + from: '2015-09-18T06:31:44.000Z', + to: '2015-09-23T18:31:44.000Z', + }); }} - viewMode={ViewMode.VIEW} - /> - {isSaveModalVisible && ( - {}} - onClose={() => setIsSaveModalVisible(false)} - /> - )} - - ) : ( - -

This demo only works if your default index pattern is set and time based

- + > + Change time range + + + + { + setIsLoading(val); + }} + onBrushEnd={({ range }) => { + setTime({ + from: new Date(range[0]).toISOString(), + to: new Date(range[1]).toISOString(), + }); + }} + onFilter={(_data) => { + // call back event for on filter event + }} + onTableRowClick={(_data) => { + // call back event for on table row click event + }} + viewMode={ViewMode.VIEW} + /> + {isSaveModalVisible && ( + {}} + onClose={() => setIsSaveModalVisible(false)} + /> )} diff --git a/x-pack/examples/embedded_lens_example/public/mount.tsx b/x-pack/examples/embedded_lens_example/public/mount.tsx index 58ec363223270..e438b6946b8b6 100644 --- a/x-pack/examples/embedded_lens_example/public/mount.tsx +++ b/x-pack/examples/embedded_lens_example/public/mount.tsx @@ -7,8 +7,10 @@ import * as React from 'react'; import { render, unmountComponentAtNode } from 'react-dom'; -import { CoreSetup, AppMountParameters } from 'kibana/public'; -import { StartDependencies } from './plugin'; +import { EuiCallOut } from '@elastic/eui'; + +import type { CoreSetup, AppMountParameters } from 'kibana/public'; +import type { StartDependencies } from './plugin'; export const mount = (coreSetup: CoreSetup) => @@ -16,20 +18,27 @@ export const mount = const [core, plugins] = await coreSetup.getStartServices(); const { App } = await import('./app'); - const deps = { - core, - plugins, - }; - - const defaultIndexPattern = await plugins.data.indexPatterns.getDefault(); + const defaultDataView = await plugins.data.indexPatterns.getDefault(); + const { formula } = await plugins.lens.stateHelperApi(); const i18nCore = core.i18n; const reactElement = ( - + {defaultDataView && defaultDataView.isTimeBased() ? ( + + ) : ( + +

This demo only works if your default index pattern is set and time based

+ + )} ); + render(reactElement, element); return () => unmountComponentAtNode(element); }; diff --git a/x-pack/plugins/lens/public/async_services.ts b/x-pack/plugins/lens/public/async_services.ts index bbb4faf55e1e9..09b434c648418 100644 --- a/x-pack/plugins/lens/public/async_services.ts +++ b/x-pack/plugins/lens/public/async_services.ts @@ -28,6 +28,8 @@ export * from './visualizations/gauge/gauge_visualization'; export * from './visualizations/gauge'; export * from './indexpattern_datasource/indexpattern'; +export { createFormulaPublicApi } from './indexpattern_datasource/operations/definitions/formula/formula_public_api'; + export * from './indexpattern_datasource'; export * from './editor_frame_service/editor_frame'; diff --git a/x-pack/plugins/lens/public/index.ts b/x-pack/plugins/lens/public/index.ts index 29dce6f0d1090..3e622d8ac9312 100644 --- a/x-pack/plugins/lens/public/index.ts +++ b/x-pack/plugins/lens/public/index.ts @@ -55,6 +55,7 @@ export type { FormulaIndexPatternColumn, MathIndexPatternColumn, OverallSumIndexPatternColumn, + FormulaPublicApi, } from './indexpattern_datasource/types'; export type { LensEmbeddableInput } from './embeddable'; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/format_selector.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/format_selector.tsx index dd3185b3c7990..efe7966870531 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/format_selector.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/format_selector.tsx @@ -9,6 +9,7 @@ import React, { useCallback, useMemo, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiFormRow, EuiComboBox, EuiSpacer, EuiRange } from '@elastic/eui'; import { GenericIndexPatternColumn } from '../indexpattern'; +import { isColumnFormatted } from '../operations/definitions/helpers'; const supportedFormats: Record = { number: { @@ -55,11 +56,9 @@ const RANGE_MAX = 15; export function FormatSelector(props: FormatSelectorProps) { const { selectedColumn, onChange } = props; - - const currentFormat = - 'params' in selectedColumn && selectedColumn.params && 'format' in selectedColumn.params - ? selectedColumn.params.format - : undefined; + const currentFormat = isColumnFormatted(selectedColumn) + ? selectedColumn.params?.format + : undefined; const [decimals, setDecimals] = useState(currentFormat?.params?.decimals ?? 2); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/index.ts index 386cd7a58ae01..4301540e5bf7e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/index.ts @@ -21,6 +21,8 @@ import type { FieldFormatsSetup, } from '../../../../../src/plugins/field_formats/public'; +export type { PersistedIndexPatternLayer, IndexPattern, FormulaPublicApi } from './types'; + export interface IndexPatternDatasourceSetupPlugins { expressions: ExpressionsSetup; fieldFormats: FieldFormatsSetup; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts b/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts index e1a15b87e5f5c..c61569539bec8 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts @@ -6,9 +6,11 @@ */ import { uniq, mapValues, difference } from 'lodash'; -import { IStorageWrapper } from 'src/plugins/kibana_utils/public'; -import { HttpSetup, SavedObjectReference } from 'kibana/public'; -import { InitializationOptions, StateSetter } from '../types'; +import type { IStorageWrapper } from 'src/plugins/kibana_utils/public'; +import type { DataView } from 'src/plugins/data_views/public'; +import type { HttpSetup, SavedObjectReference } from 'kibana/public'; +import type { InitializationOptions, StateSetter } from '../types'; + import { IndexPattern, IndexPatternRef, @@ -17,6 +19,7 @@ import { IndexPatternField, IndexPatternLayer, } from './types'; + import { updateLayerIndexPattern, translateToOperationName } from './operations'; import { DateRange, ExistingFields } from '../../common/types'; import { BASE_API_URL } from '../../common'; @@ -35,6 +38,72 @@ type SetState = StateSetter; type IndexPatternsService = Pick; type ErrorHandler = (err: Error) => void; +export function convertDataViewIntoLensIndexPattern(dataView: DataView): IndexPattern { + const newFields = dataView.fields + .filter( + (field) => + !indexPatternsUtils.isNestedField(field) && (!!field.aggregatable || !!field.scripted) + ) + .map((field): IndexPatternField => { + // Convert the getters on the index pattern service into plain JSON + const base = { + name: field.name, + displayName: field.displayName, + type: field.type, + aggregatable: field.aggregatable, + searchable: field.searchable, + meta: dataView.metaFields.includes(field.name), + esTypes: field.esTypes, + scripted: field.scripted, + runtime: Boolean(field.runtimeField), + }; + + // Simplifies tests by hiding optional properties instead of undefined + return base.scripted + ? { + ...base, + lang: field.lang, + script: field.script, + } + : base; + }) + .concat(documentField); + + const { typeMeta, title, timeFieldName, fieldFormatMap } = dataView; + if (typeMeta?.aggs) { + const aggs = Object.keys(typeMeta.aggs); + newFields.forEach((field, index) => { + const restrictionsObj: IndexPatternField['aggregationRestrictions'] = {}; + aggs.forEach((agg) => { + const restriction = typeMeta.aggs && typeMeta.aggs[agg] && typeMeta.aggs[agg][field.name]; + if (restriction) { + restrictionsObj[translateToOperationName(agg)] = restriction; + } + }); + if (Object.keys(restrictionsObj).length) { + newFields[index] = { ...field, aggregationRestrictions: restrictionsObj }; + } + }); + } + + return { + id: dataView.id!, // id exists for sure because we got index patterns by id + title, + timeFieldName, + fieldFormatMap: + fieldFormatMap && + Object.fromEntries( + Object.entries(fieldFormatMap).map(([id, format]) => [ + id, + 'toJSON' in format ? format.toJSON() : format, + ]) + ), + fields: newFields, + getFieldByName: getFieldByNameFactory(newFields), + hasRestrictions: !!typeMeta?.aggs, + }; +} + export async function loadIndexPatterns({ indexPatternsService, patterns, @@ -79,77 +148,10 @@ export async function loadIndexPatterns({ } const indexPatternsObject = indexPatterns.reduce( - (acc, indexPattern) => { - const newFields = indexPattern.fields - .filter( - (field) => - !indexPatternsUtils.isNestedField(field) && (!!field.aggregatable || !!field.scripted) - ) - .map((field): IndexPatternField => { - // Convert the getters on the index pattern service into plain JSON - const base = { - name: field.name, - displayName: field.displayName, - type: field.type, - aggregatable: field.aggregatable, - searchable: field.searchable, - meta: indexPattern.metaFields.includes(field.name), - esTypes: field.esTypes, - scripted: field.scripted, - runtime: Boolean(field.runtimeField), - }; - - // Simplifies tests by hiding optional properties instead of undefined - return base.scripted - ? { - ...base, - lang: field.lang, - script: field.script, - } - : base; - }) - .concat(documentField); - - const { typeMeta, title, timeFieldName, fieldFormatMap } = indexPattern; - if (typeMeta?.aggs) { - const aggs = Object.keys(typeMeta.aggs); - newFields.forEach((field, index) => { - const restrictionsObj: IndexPatternField['aggregationRestrictions'] = {}; - aggs.forEach((agg) => { - const restriction = - typeMeta.aggs && typeMeta.aggs[agg] && typeMeta.aggs[agg][field.name]; - if (restriction) { - restrictionsObj[translateToOperationName(agg)] = restriction; - } - }); - if (Object.keys(restrictionsObj).length) { - newFields[index] = { ...field, aggregationRestrictions: restrictionsObj }; - } - }); - } - - const currentIndexPattern: IndexPattern = { - id: indexPattern.id!, // id exists for sure because we got index patterns by id - title, - timeFieldName, - fieldFormatMap: - fieldFormatMap && - Object.fromEntries( - Object.entries(fieldFormatMap).map(([id, format]) => [ - id, - 'toJSON' in format ? format.toJSON() : format, - ]) - ), - fields: newFields, - getFieldByName: getFieldByNameFactory(newFields), - hasRestrictions: !!typeMeta?.aggs, - }; - - return { - [currentIndexPattern.id]: currentIndexPattern, - ...acc, - }; - }, + (acc, indexPattern) => ({ + [indexPattern.id!]: convertDataViewIntoLensIndexPattern(indexPattern), + ...acc, + }), { ...cache } ); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/column_types.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/column_types.ts index bd5b816cd8917..2b11d182eeed0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/column_types.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/column_types.ts @@ -20,8 +20,7 @@ export interface BaseIndexPatternColumn extends Operation { } // Formatting can optionally be added to any column -// export interface FormattedIndexPatternColumn extends BaseIndexPatternColumn { -export type FormattedIndexPatternColumn = BaseIndexPatternColumn & { +export interface FormattedIndexPatternColumn extends BaseIndexPatternColumn { params?: { format?: { id: string; @@ -30,15 +29,13 @@ export type FormattedIndexPatternColumn = BaseIndexPatternColumn & { }; }; }; -}; +} export interface FieldBasedIndexPatternColumn extends BaseIndexPatternColumn { sourceField: string; } -export interface ReferenceBasedIndexPatternColumn - extends BaseIndexPatternColumn, - FormattedIndexPatternColumn { +export interface ReferenceBasedIndexPatternColumn extends FormattedIndexPatternColumn { references: string[]; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx index fc69ea1d869f1..62a681ac3d604 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_editor.tsx @@ -45,7 +45,7 @@ import { trackUiEvent } from '../../../../../lens_ui_telemetry'; import './formula.scss'; import { FormulaIndexPatternColumn } from '../formula'; -import { regenerateLayerFromAst } from '../parse'; +import { insertOrReplaceFormulaColumn } from '../parse'; import { filterByVisibleOperation } from '../util'; import { getColumnTimeShiftWarnings, getDateHistogramInterval } from '../../../../time_shift_utils'; @@ -151,16 +151,24 @@ export function FormulaEditor({ setIsCloseable(true); // If the text is not synced, update the column. if (text !== currentColumn.params.formula) { - updateLayer((prevLayer) => { - return regenerateLayerFromAst( - text || '', - prevLayer, - columnId, - currentColumn, - indexPattern, - operationDefinitionMap - ).newLayer; - }); + updateLayer( + (prevLayer) => + insertOrReplaceFormulaColumn( + columnId, + { + ...currentColumn, + params: { + ...currentColumn.params, + formula: text || '', + }, + }, + prevLayer, + { + indexPattern, + operations: operationDefinitionMap, + } + ).layer + ); } }); @@ -173,15 +181,23 @@ export function FormulaEditor({ monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); if (currentColumn.params.formula) { // Only submit if valid - const { newLayer } = regenerateLayerFromAst( - text || '', - layer, - columnId, - currentColumn, - indexPattern, - operationDefinitionMap + updateLayer( + insertOrReplaceFormulaColumn( + columnId, + { + ...currentColumn, + params: { + ...currentColumn.params, + formula: text || '', + }, + }, + layer, + { + indexPattern, + operations: operationDefinitionMap, + } + ).layer ); - updateLayer(newLayer); } return; @@ -215,14 +231,21 @@ export function FormulaEditor({ // If the formula is already broken, show the latest error message in the workspace if (currentColumn.params.formula !== text) { updateLayer( - regenerateLayerFromAst( - text || '', - layer, + insertOrReplaceFormulaColumn( columnId, - currentColumn, - indexPattern, - visibleOperationsMap - ).newLayer + { + ...currentColumn, + params: { + ...currentColumn.params, + formula: text || '', + }, + }, + layer, + { + indexPattern, + operations: operationDefinitionMap, + } + ).layer ); } } @@ -270,14 +293,25 @@ export function FormulaEditor({ monaco.editor.setModelMarkers(editorModel.current, 'LENS', []); // Only submit if valid - const { newLayer, locations } = regenerateLayerFromAst( - text || '', - layer, + const { + layer: newLayer, + meta: { locations }, + } = insertOrReplaceFormulaColumn( columnId, - currentColumn, - indexPattern, - visibleOperationsMap + { + ...currentColumn, + params: { + ...currentColumn.params, + formula: text || '', + }, + }, + layer, + { + indexPattern, + operations: operationDefinitionMap, + } ); + updateLayer(newLayer); const managedColumns = getManagedColumnsFrom(columnId, newLayer.columns); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index 2babd87768e32..d1561e93aa807 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -8,7 +8,7 @@ import { createMockedIndexPattern } from '../../../mocks'; import { formulaOperation, GenericOperationDefinition, GenericIndexPatternColumn } from '../index'; import { FormulaIndexPatternColumn } from './formula'; -import { regenerateLayerFromAst } from './parse'; +import { insertOrReplaceFormulaColumn } from './parse'; import type { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../../types'; import { tinymathFunctions } from './util'; import { TermsIndexPatternColumn } from '../terms'; @@ -424,25 +424,36 @@ describe('formula', () => { }); }); - describe('regenerateLayerFromAst()', () => { + describe('insertOrReplaceFormulaColumn()', () => { let indexPattern: IndexPattern; let currentColumn: FormulaIndexPatternColumn; function testIsBrokenFormula( formula: string, - columnParams: Partial> = {} + partialColumn: Partial> = {} ) { - const mergedColumn = { ...currentColumn, ...columnParams }; + const mergedColumn = { + ...currentColumn, + ...partialColumn, + }; const mergedLayer = { ...layer, columns: { ...layer.columns, col1: mergedColumn } }; + expect( - regenerateLayerFromAst( - formula, - mergedLayer, + insertOrReplaceFormulaColumn( 'col1', - mergedColumn, - indexPattern, - operationDefinitionMap - ).newLayer + { + ...mergedColumn, + params: { + ...mergedColumn.params, + formula, + }, + }, + mergedLayer, + { + indexPattern, + operations: operationDefinitionMap, + } + ).layer ).toEqual({ ...mergedLayer, columns: { @@ -475,14 +486,21 @@ describe('formula', () => { it('should mutate the layer with new columns for valid formula expressions', () => { expect( - regenerateLayerFromAst( - 'average(bytes)', - layer, + insertOrReplaceFormulaColumn( 'col1', - currentColumn, - indexPattern, - operationDefinitionMap - ).newLayer + { + ...currentColumn, + params: { + ...currentColumn.params, + formula: 'average(bytes)', + }, + }, + layer, + { + indexPattern, + operations: operationDefinitionMap, + } + ).layer ).toEqual({ ...layer, columnOrder: ['col1X0', 'col1'], @@ -514,14 +532,21 @@ describe('formula', () => { it('should create a valid formula expression for numeric literals', () => { expect( - regenerateLayerFromAst( - '0', - layer, + insertOrReplaceFormulaColumn( 'col1', - currentColumn, - indexPattern, - operationDefinitionMap - ).newLayer + { + ...currentColumn, + params: { + ...currentColumn.params, + formula: '0', + }, + }, + layer, + { + indexPattern, + operations: operationDefinitionMap, + } + ).layer ).toEqual({ ...layer, columnOrder: ['col1X0', 'col1'], @@ -672,14 +697,21 @@ describe('formula', () => { it('returns the locations of each function', () => { expect( - regenerateLayerFromAst( - 'moving_average(average(bytes), window=7) + count()', - layer, + insertOrReplaceFormulaColumn( 'col1', - currentColumn, - indexPattern, - operationDefinitionMap - ).locations + { + ...currentColumn, + params: { + ...currentColumn.params, + formula: 'moving_average(average(bytes), window=7) + count()', + }, + }, + layer, + { + indexPattern, + operations: operationDefinitionMap, + } + ).meta.locations ).toEqual({ col1X0: { min: 15, max: 29 }, col1X1: { min: 0, max: 41 }, @@ -693,14 +725,22 @@ describe('formula', () => { const mergedLayer = { ...layer, columns: { ...layer.columns, col1: mergedColumn } }; const formula = 'moving_average(average(bytes), window=7) + count()'; - const { newLayer } = regenerateLayerFromAst( - formula, - mergedLayer, + const { layer: newLayer } = insertOrReplaceFormulaColumn( 'col1', - mergedColumn, - indexPattern, - operationDefinitionMap + { + ...mergedColumn, + params: { + ...mergedColumn.params, + formula, + }, + }, + mergedLayer, + { + indexPattern, + operations: operationDefinitionMap, + } ); + // average and math are not filterable in the mocks expect(newLayer.columns).toEqual( expect.objectContaining({ @@ -737,14 +777,22 @@ describe('formula', () => { const mergedLayer = { ...layer, columns: { ...layer.columns, col1: mergedColumn } }; const formula = `moving_average(average(bytes), window=7, kql='${innerFilter}') + count(kql='${innerFilter}')`; - const { newLayer } = regenerateLayerFromAst( - formula, - mergedLayer, + const { layer: newLayer } = insertOrReplaceFormulaColumn( 'col1', - mergedColumn, - indexPattern, - operationDefinitionMap + { + ...mergedColumn, + params: { + ...mergedColumn.params, + formula, + }, + }, + mergedLayer, + { + indexPattern, + operations: operationDefinitionMap, + } ); + // average and math are not filterable in the mocks expect(newLayer.columns).toEqual( expect.objectContaining({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx index 15c49a7336c7e..ce0d03a232e28 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx @@ -6,12 +6,12 @@ */ import { i18n } from '@kbn/i18n'; -import type { OperationDefinition } from '../index'; +import type { BaseIndexPatternColumn, OperationDefinition } from '../index'; import type { ReferenceBasedIndexPatternColumn } from '../column_types'; import type { IndexPattern } from '../../../types'; import { runASTValidation, tryToParse } from './validation'; import { WrappedFormulaEditor } from './editor'; -import { regenerateLayerFromAst } from './parse'; +import { insertOrReplaceFormulaColumn } from './parse'; import { generateFormula } from './generate'; import { filterByVisibleOperation } from './util'; import { getManagedColumnsFrom } from '../../layer_helpers'; @@ -36,6 +36,12 @@ export interface FormulaIndexPatternColumn extends ReferenceBasedIndexPatternCol }; } +export function isFormulaIndexPatternColumn( + column: BaseIndexPatternColumn +): column is FormulaIndexPatternColumn { + return 'params' in column && 'formula' in (column as FormulaIndexPatternColumn).params; +} + export const formulaOperation: OperationDefinition = { type: 'formula', @@ -150,22 +156,11 @@ export const formulaOperation: OperationDefinition ({ + insertOrReplaceFormulaColumn: jest.fn().mockReturnValue({}), +})); + +jest.mock('../../../loader', () => ({ + convertDataViewIntoLensIndexPattern: jest.fn((v) => v), +})); + +const getBaseLayer = (): PersistedIndexPatternLayer => ({ + columnOrder: ['col1'], + columns: { + col1: { + dataType: 'date', + isBucketed: true, + label: '@timestamp', + operationType: 'date_histogram', + params: { interval: 'auto' }, + scale: 'interval', + } as DateHistogramIndexPatternColumn, + }, +}); + +describe('createFormulaPublicApi', () => { + let publicApiHelper: FormulaPublicApi; + let dataView: DataView; + + beforeEach(() => { + publicApiHelper = createFormulaPublicApi(); + dataView = {} as DataView; + + jest.clearAllMocks(); + }); + + test('should use cache for caching lens index patterns', () => { + const baseLayer = getBaseLayer(); + + publicApiHelper.insertOrReplaceFormulaColumn( + 'col', + { formula: 'count()' }, + baseLayer, + dataView + ); + + publicApiHelper.insertOrReplaceFormulaColumn( + 'col', + { formula: 'count()' }, + baseLayer, + dataView + ); + + expect(convertDataViewIntoLensIndexPattern).toHaveBeenCalledTimes(1); + }); + + test('should execute insertOrReplaceFormulaColumn with valid arguments', () => { + const baseLayer = getBaseLayer(); + + publicApiHelper.insertOrReplaceFormulaColumn( + 'col', + { formula: 'count()' }, + baseLayer, + dataView + ); + + expect(insertOrReplaceFormulaColumn).toHaveBeenCalledWith( + 'col', + { + customLabel: false, + dataType: 'number', + isBucketed: false, + label: 'count()', + operationType: 'formula', + params: { formula: 'count()' }, + references: [], + }, + { + columnOrder: ['col1'], + columns: { + col1: { + dataType: 'date', + isBucketed: true, + label: '@timestamp', + operationType: 'date_histogram', + params: { interval: 'auto' }, + scale: 'interval', + }, + }, + indexPatternId: undefined, + }, + { indexPattern: {} } + ); + }); +}); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula_public_api.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula_public_api.ts new file mode 100644 index 0000000000000..63255ad2bf9dc --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula_public_api.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IndexPattern, PersistedIndexPatternLayer } from '../../../types'; +import type { DataView } from '../../../../../../../../src/plugins/data_views/public'; + +import { insertOrReplaceFormulaColumn } from './parse'; +import { convertDataViewIntoLensIndexPattern } from '../../../loader'; + +/** @public **/ +export interface FormulaPublicApi { + /** + * Method which Lens consumer can import and given a formula string, + * return a parsed result as list of columns to use as Embeddable attributes. + * + * @param id - Formula column id + * @param column.formula - String representation of a formula + * @param [column.label] - Custom formula label + * @param layer - The layer to which the formula columns will be added + * @param dataView - The dataView instance + * + * See `x-pack/examples/embedded_lens_example` for exemplary usage. + */ + insertOrReplaceFormulaColumn: ( + id: string, + column: { + formula: string; + label?: string; + }, + layer: PersistedIndexPatternLayer, + dataView: DataView + ) => PersistedIndexPatternLayer | undefined; +} + +/** @public **/ +export const createFormulaPublicApi = (): FormulaPublicApi => { + const cache: WeakMap = new WeakMap(); + + const getCachedLensIndexPattern = (dataView: DataView): IndexPattern => { + const cachedIndexPattern = cache.get(dataView); + if (cachedIndexPattern) { + return cachedIndexPattern; + } + const indexPattern = convertDataViewIntoLensIndexPattern(dataView); + cache.set(dataView, indexPattern); + return indexPattern; + }; + + return { + insertOrReplaceFormulaColumn: (id, { formula, label }, layer, dataView) => { + const indexPattern = getCachedLensIndexPattern(dataView); + + return insertOrReplaceFormulaColumn( + id, + { + label: label ?? formula, + customLabel: Boolean(label), + operationType: 'formula', + dataType: 'number', + references: [], + isBucketed: false, + params: { + formula, + }, + }, + { ...layer, indexPatternId: indexPattern.id }, + { indexPattern } + ).layer; + }, + }; +}; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts index 5ff0c4e2d4bd7..cbe6efba1b859 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/index.ts @@ -7,6 +7,8 @@ export type { FormulaIndexPatternColumn } from './formula'; export { formulaOperation } from './formula'; -export { regenerateLayerFromAst } from './parse'; + +export { insertOrReplaceFormulaColumn } from './parse'; + export type { MathIndexPatternColumn } from './math'; export { mathOperation } from './math'; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts index ee245cc06bff9..a3b61429fb0bf 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts @@ -8,10 +8,11 @@ import { i18n } from '@kbn/i18n'; import { isObject } from 'lodash'; import type { TinymathAST, TinymathVariable, TinymathLocation } from '@kbn/tinymath'; -import type { +import { OperationDefinition, GenericOperationDefinition, GenericIndexPatternColumn, + operationDefinitionMap, } from '../index'; import type { IndexPattern, IndexPatternLayer } from '../../../types'; import { mathOperation } from './math'; @@ -24,10 +25,11 @@ import { groupArgsByType, mergeWithGlobalFilter, } from './util'; -import type { FormulaIndexPatternColumn } from './formula'; +import { FormulaIndexPatternColumn, isFormulaIndexPatternColumn } from './formula'; import { getColumnOrder } from '../../layer_helpers'; -function getManagedId(mainId: string, index: number) { +/** @internal **/ +export function getManagedId(mainId: string, index: number) { return `${mainId}X${index}`; } @@ -36,21 +38,15 @@ function parseAndExtract( layer: IndexPatternLayer, columnId: string, indexPattern: IndexPattern, - operationDefinitionMap: Record, + operations: Record, label?: string ) { - const { root, error } = tryToParse(text, operationDefinitionMap); + const { root, error } = tryToParse(text, operations); if (error || root == null) { return { extracted: [], isValid: false }; } // before extracting the data run the validation task and throw if invalid - const errors = runASTValidation( - root, - layer, - indexPattern, - operationDefinitionMap, - layer.columns[columnId] - ); + const errors = runASTValidation(root, layer, indexPattern, operations, layer.columns[columnId]); if (errors.length) { return { extracted: [], isValid: false }; } @@ -59,7 +55,7 @@ function parseAndExtract( */ const extracted = extractColumns( columnId, - operationDefinitionMap, + operations, root, layer, indexPattern, @@ -201,63 +197,116 @@ function extractColumns( return columns; } -export function regenerateLayerFromAst( - text: string, +interface ExpandColumnProperties { + indexPattern: IndexPattern; + operations?: Record; +} + +const getEmptyColumnsWithFormulaMeta = (): { + columns: Record; + meta: { + locations: Record; + }; +} => ({ + columns: {}, + meta: { + locations: {}, + }, +}); + +function generateFormulaColumns( + id: string, + column: FormulaIndexPatternColumn, layer: IndexPatternLayer, - columnId: string, - currentColumn: FormulaIndexPatternColumn, - indexPattern: IndexPattern, - operationDefinitionMap: Record + { indexPattern, operations = operationDefinitionMap }: ExpandColumnProperties ) { + const { columns, meta } = getEmptyColumnsWithFormulaMeta(); + const formula = column.params.formula || ''; + const { extracted, isValid } = parseAndExtract( - text, + formula, layer, - columnId, + id, indexPattern, - filterByVisibleOperation(operationDefinitionMap), - currentColumn.customLabel ? currentColumn.label : undefined + filterByVisibleOperation(operations), + column.customLabel ? column.label : undefined ); - const columns = { ...layer.columns }; - - const locations: Record = {}; + extracted.forEach(({ column: extractedColumn, location }, index) => { + const managedId = getManagedId(id, index); + columns[managedId] = extractedColumn; - Object.keys(columns).forEach((k) => { - if (k.startsWith(columnId)) { - delete columns[k]; + if (location) { + meta.locations[managedId] = location; } }); - extracted.forEach(({ column, location }, index) => { - columns[getManagedId(columnId, index)] = column; - if (location) locations[getManagedId(columnId, index)] = location; - }); - - columns[columnId] = { - ...currentColumn, - label: !currentColumn.customLabel - ? text ?? + columns[id] = { + ...column, + label: !column.customLabel + ? formula ?? i18n.translate('xpack.lens.indexPattern.formulaLabel', { defaultMessage: 'Formula', }) - : currentColumn.label, + : column.label, + references: !isValid ? [] : [getManagedId(id, extracted.length - 1)], params: { - ...currentColumn.params, - formula: text, + ...column.params, + formula, isFormulaBroken: !isValid, }, - references: !isValid ? [] : [getManagedId(columnId, extracted.length - 1)], } as FormulaIndexPatternColumn; + return { columns, meta }; +} + +/** @internal **/ +export function insertOrReplaceFormulaColumn( + id: string, + column: FormulaIndexPatternColumn, + baseLayer: IndexPatternLayer, + params: ExpandColumnProperties +) { + const layer = { + ...baseLayer, + columns: { + ...baseLayer.columns, + [id]: { + ...column, + }, + }, + }; + + const { columns: updatedColumns, meta } = Object.entries(layer.columns).reduce( + (acc, [currentColumnId, currentColumn]) => { + if (currentColumnId.startsWith(id)) { + if (currentColumnId === id && isFormulaIndexPatternColumn(currentColumn)) { + const formulaColumns = generateFormulaColumns( + currentColumnId, + currentColumn, + layer, + params + ); + acc.columns = { ...acc.columns, ...formulaColumns.columns }; + acc.meta = { ...acc.meta, ...formulaColumns.meta }; + } + } else { + acc.columns[currentColumnId] = { ...currentColumn }; + } + return acc; + }, + getEmptyColumnsWithFormulaMeta() + ); + return { - newLayer: { + layer: { ...layer, - columns, + columns: updatedColumns, columnOrder: getColumnOrder({ ...layer, - columns, + columns: updatedColumns, }), }, - locations, + meta, }; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/index.ts index b9d675716c788..4474effc8c8c8 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/index.ts @@ -8,6 +8,7 @@ export * from './operations'; export * from './layer_helpers'; export * from './time_scale_utils'; + export type { OperationType, BaseIndexPatternColumn, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 289161c9d3e37..dda1b16bc6c7b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -36,7 +36,7 @@ import { ReferenceBasedIndexPatternColumn, BaseIndexPatternColumn, } from './definitions/column_types'; -import { FormulaIndexPatternColumn, regenerateLayerFromAst } from './definitions/formula'; +import { FormulaIndexPatternColumn, insertOrReplaceFormulaColumn } from './definitions/formula'; import type { TimeScaleUnit } from '../../../common/expressions'; import { isColumnOfType } from './definitions/helpers'; @@ -533,14 +533,9 @@ export function replaceColumn({ try { newLayer = newColumn.params.formula - ? regenerateLayerFromAst( - newColumn.params.formula, - basicLayer, - columnId, - newColumn, + ? insertOrReplaceFormulaColumn(columnId, newColumn, basicLayer, { indexPattern, - operationDefinitionMap - ).newLayer + }).layer : basicLayer; } catch (e) { newLayer = basicLayer; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/types.ts b/x-pack/plugins/lens/public/indexpattern_datasource/types.ts index a0d43c5523c5b..08786b181f3e7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/types.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/types.ts @@ -38,10 +38,13 @@ export type { OverallSumIndexPatternColumn, } from './operations'; +export type { FormulaPublicApi } from './operations/definitions/formula/formula_public_api'; + export type DraggedField = DragDropIdentifier & { field: IndexPatternField; indexPatternId: string; }; + export interface IndexPattern { id: string; fields: IndexPatternField[]; @@ -79,6 +82,7 @@ export interface IndexPatternPersistedState { } export type PersistedIndexPatternLayer = Omit; + export interface IndexPatternPrivateState { currentIndexPatternId: string; layers: Record; diff --git a/x-pack/plugins/lens/public/mocks/lens_plugin_mock.tsx b/x-pack/plugins/lens/public/mocks/lens_plugin_mock.tsx index a92533a89ba67..4e713872c5a67 100644 --- a/x-pack/plugins/lens/public/mocks/lens_plugin_mock.tsx +++ b/x-pack/plugins/lens/public/mocks/lens_plugin_mock.tsx @@ -25,6 +25,12 @@ export const lensPluginMock = { getXyVisTypes: jest .fn() .mockReturnValue(new Promise((resolve) => resolve(visualizationTypes))), + + stateHelperApi: jest.fn().mockResolvedValue({ + formula: { + insertOrReplaceFormulaColumn: jest.fn(), + }, + }), }; return startContract; }, diff --git a/x-pack/plugins/lens/public/plugin.ts b/x-pack/plugins/lens/public/plugin.ts index 6b26753b41880..decd9d8c69510 100644 --- a/x-pack/plugins/lens/public/plugin.ts +++ b/x-pack/plugins/lens/public/plugin.ts @@ -39,6 +39,7 @@ import { IndexPatternFieldEditorStart } from '../../../../src/plugins/data_view_ import type { IndexPatternDatasource as IndexPatternDatasourceType, IndexPatternDatasourceSetupPlugins, + FormulaPublicApi, } from './indexpattern_datasource'; import type { XyVisualization as XyVisualizationType, @@ -160,6 +161,13 @@ export interface LensPublicStart { * Method which returns xy VisualizationTypes array keeping this async as to not impact page load bundle */ getXyVisTypes: () => Promise; + + /** + * API which returns state helpers keeping this async as to not impact page load bundle + */ + stateHelperApi: () => Promise<{ + formula: FormulaPublicApi; + }>; } export class LensPlugin { @@ -387,6 +395,14 @@ export class LensPlugin { const { visualizationTypes } = await import('./xy_visualization/types'); return visualizationTypes; }, + + stateHelperApi: async () => { + const { createFormulaPublicApi } = await import('./async_services'); + + return { + formula: createFormulaPublicApi(), + }; + }, }; }