From db10c1bebc09de27d3100a9ef208f29328b9ee41 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 28 Apr 2021 18:02:18 +0200 Subject: [PATCH 01/31] make basic time offset work --- .../data/common/search/aggs/agg_config.ts | 8 + .../data/common/search/aggs/agg_configs.ts | 230 ++++++++++++++---- .../data/common/search/aggs/metrics/avg_fn.ts | 7 + .../data/common/search/aggs/metrics/count.ts | 7 +- .../common/search/aggs/metrics/count_fn.ts | 7 + .../data/common/search/aggs/metrics/max_fn.ts | 7 + .../common/search/aggs/metrics/median_fn.ts | 7 + .../search/aggs/metrics/metric_agg_type.ts | 10 +- .../data/common/search/aggs/metrics/min_fn.ts | 7 + .../aggs/metrics/single_percentile_fn.ts | 7 + .../data/common/search/aggs/metrics/sum_fn.ts | 7 + .../common/search/aggs/metrics/top_hit_fn.ts | 7 + src/plugins/data/common/search/aggs/types.ts | 1 + .../expressions/esaggs/request_handler.ts | 68 +++++- .../search/search_source/search_source.ts | 33 ++- .../data/common/search/tabify/tabify.ts | 2 +- .../editor_frame/frame_layout.tsx | 2 +- .../indexpattern_datasource/to_expression.ts | 3 + 18 files changed, 355 insertions(+), 65 deletions(-) diff --git a/src/plugins/data/common/search/aggs/agg_config.ts b/src/plugins/data/common/search/aggs/agg_config.ts index 283d276a22904b..42dc52c59d822c 100644 --- a/src/plugins/data/common/search/aggs/agg_config.ts +++ b/src/plugins/data/common/search/aggs/agg_config.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import moment from 'moment'; import _ from 'lodash'; import { i18n } from '@kbn/i18n'; import { Assign, Ensure } from '@kbn/utility-types'; @@ -172,6 +173,13 @@ export class AggConfig { return _.get(this.params, key); } + getTimeShift(): undefined | moment.Duration { + const rawTimeShift = this.getParam('timeShift'); + if (!rawTimeShift) return undefined; + const [, amount, unit] = rawTimeShift.match(/(\d+)(\w)/); + return moment.duration(Number(amount), unit); + } + write(aggs?: IAggConfigs) { return writeParams(this.type.params, this, aggs); } diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index 2932ef7325aed8..eeb370765a77ba 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -6,11 +6,13 @@ * Side Public License, v 1. */ -import _ from 'lodash'; +import moment from 'moment'; +import _, { cloneDeep, isArray } from 'lodash'; import { i18n } from '@kbn/i18n'; import { Assign } from '@kbn/utility-types'; +import { Aggregate, FiltersAggregate, FiltersBucketItem } from '@elastic/elasticsearch/api/types'; -import { ISearchOptions, ISearchSource } from 'src/plugins/data/public'; +import { IEsSearchResponse, ISearchOptions, ISearchSource } from 'src/plugins/data/public'; import { AggConfig, AggConfigSerialized, IAggConfig } from './agg_config'; import { IAggType } from './agg_type'; import { AggTypesRegistryStart } from './agg_types_registry'; @@ -183,6 +185,9 @@ export class AggConfigs { let dslLvlCursor: Record; let nestedMetrics: Array<{ config: AggConfig; dsl: Record }> | []; + const timeShifts = this.getTimeShifts(); + const hasTimeShifts = Object.keys(timeShifts).length > 0; + if (this.hierarchical) { // collect all metrics, and filter out the ones that we won't be copying nestedMetrics = this.aggs @@ -196,57 +201,104 @@ export class AggConfigs { }; }); } - this.getRequestAggs() - .filter((config: AggConfig) => !config.type.hasNoDsl) - .forEach((config: AggConfig, i: number, list) => { - if (!dslLvlCursor) { - // start at the top level - dslLvlCursor = dslTopLvl; - } else { - const prevConfig: AggConfig = list[i - 1]; - const prevDsl = dslLvlCursor[prevConfig.id]; + this.getRequestAggs().forEach((config: AggConfig, i: number, list) => { + if (!dslLvlCursor) { + // start at the top level + dslLvlCursor = dslTopLvl; + } else { + const prevConfig: AggConfig = list[i - 1]; + const prevDsl = dslLvlCursor[prevConfig.id]; + + // advance the cursor and nest under the previous agg, or + // put it on the same level if the previous agg doesn't accept + // sub aggs + dslLvlCursor = prevDsl?.aggs || dslLvlCursor; + } - // advance the cursor and nest under the previous agg, or - // put it on the same level if the previous agg doesn't accept - // sub aggs - dslLvlCursor = prevDsl?.aggs || dslLvlCursor; - } + if (hasTimeShifts) { + dslLvlCursor = this.insertTimeShiftSplit(config, timeShifts, dslLvlCursor); + } - const dsl = config.type.hasNoDslParams - ? config.toDsl(this) - : (dslLvlCursor[config.id] = config.toDsl(this)); - let subAggs: any; + if (config.type.hasNoDsl) { + return; + } - parseParentAggs(dslLvlCursor, dsl); + const dsl = config.type.hasNoDslParams + ? config.toDsl(this) + : (dslLvlCursor[config.id] = config.toDsl(this)); + let subAggs: any; - if (config.type.type === AggGroupNames.Buckets && i < list.length - 1) { - // buckets that are not the last item in the list accept sub-aggs - subAggs = dsl.aggs || (dsl.aggs = {}); - } + parseParentAggs(dslLvlCursor, dsl); - if (subAggs) { - _.each(subAggs, (agg) => { - parseParentAggs(subAggs, agg); - }); - } - if (subAggs && nestedMetrics) { - nestedMetrics.forEach((agg: any) => { - subAggs[agg.config.id] = agg.dsl; - // if a nested metric agg has parent aggs, we have to add them to every level of the tree - // to make sure "bucket_path" references in the nested metric agg itself are still working - if (agg.dsl.parentAggs) { - Object.entries(agg.dsl.parentAggs).forEach(([parentAggId, parentAgg]) => { - subAggs[parentAggId] = parentAgg; - }); - } - }); - } - }); + if (config.type.type === AggGroupNames.Buckets && i < list.length - 1) { + // buckets that are not the last item in the list accept sub-aggs + subAggs = dsl.aggs || (dsl.aggs = {}); + } + + if (subAggs) { + _.each(subAggs, (agg) => { + parseParentAggs(subAggs, agg); + }); + } + if (subAggs && nestedMetrics) { + nestedMetrics.forEach((agg: any) => { + subAggs[agg.config.id] = agg.dsl; + // if a nested metric agg has parent aggs, we have to add them to every level of the tree + // to make sure "bucket_path" references in the nested metric agg itself are still working + if (agg.dsl.parentAggs) { + Object.entries(agg.dsl.parentAggs).forEach(([parentAggId, parentAgg]) => { + subAggs[parentAggId] = parentAgg; + }); + } + }); + } + }); removeParentAggs(dslTopLvl); return dslTopLvl; } + private insertTimeShiftSplit( + config: AggConfig, + timeShifts: Record, + dslLvlCursor: Record + ) { + if (this.isTimeShiftSplitAgg(config)) { + const filters: Record = {}; + // TODO validate this + const timeField = this.timeFields![0]; + filters['0'] = { + range: { + [timeField]: { + // only works if there is a time range + gte: this.timeRange!.from, + lte: this.timeRange!.to, + }, + }, + }; + Object.entries(timeShifts).forEach(([key, shift]) => { + filters[key] = { + range: { + [timeField]: { + // only works if there is a time range + gte: moment(this.timeRange!.from).subtract(shift).toISOString(), + lte: moment(this.timeRange!.to).subtract(shift).toISOString(), + }, + }, + }; + }); + dslLvlCursor.time_offset_split = { + filters: { + filters, + }, + aggs: {}, + }; + + dslLvlCursor = dslLvlCursor.time_offset_split.aggs; + } + return dslLvlCursor; + } + getAll() { return [...this.aggs]; } @@ -289,6 +341,98 @@ export class AggConfigs { ); } + getTimeShifts(): Record { + const timeShifts: Record = {}; + this.getAll() + .filter((agg) => agg.schema === 'metric') + .map((agg) => agg.getTimeShift()) + .forEach((timeShift) => { + if (timeShift) { + timeShifts[String(timeShift.asMilliseconds())] = timeShift; + } + }); + return timeShifts; + } + + isTimeShiftSplitAgg(aggConfig: AggConfig) { + // TODO - probably abstract this in an optional flag on the agg config instead of hard-coding the relationship here + const hasDateHistograms = this.getAll().some((agg) => agg.type.name === 'date_histogram'); + // the first date histogram or the first metric (if there are no date histograms) is the level to add the time shift filter split + return ( + (hasDateHistograms && aggConfig.type.name === 'date_histogram') || + this.byType('metrics')[0] === aggConfig + ); + } + + postFlightTransform(response: IEsSearchResponse) { + const timeShifts = this.getTimeShifts(); + if (Object.keys(timeShifts).length === 0) { + return response; + } + const transformedRawResponse = cloneDeep(response.rawResponse); + const aggCursor = transformedRawResponse.aggregations!; + + // TODO: Add a "context" param here which keeps track of at which level of the response we are + const mergeObject = (target: any, source: any, shiftKey: string) => { + Object.entries(source).forEach(([key, val]) => { + if (typeof val !== 'object') { + if (key === 'doc_count') { + target[`doc_count_${shiftKey}`] = val; + } else { + target[key] = val; + } + } else { + const agg = this.byId(key); + if (agg && agg.type.type === 'metrics') { + const timeShift = agg.getTimeShift(); + if (!timeShift || String(timeShift.asMilliseconds()) !== shiftKey) { + // this is a metric from another time shift, do not copy over + return; + } + } + if (!target[key]) { + target[key] = {}; + } + mergeObject(target[key], source[key], shiftKey); + } + }); + }; + const transformTimeShift = (cursor: Record): undefined => { + if (cursor.time_offset_split) { + const timeShiftedBuckets = (cursor.time_offset_split as FiltersAggregate).buckets as Record< + string, + FiltersBucketItem + >; + const subTree = timeShiftedBuckets['0']; + // TODO pull in metrics from other buckets + Object.entries(timeShifts).forEach(([key, shift]) => { + mergeObject(subTree, timeShiftedBuckets[key], key); + }); + + delete cursor.time_offset_split; + Object.assign(cursor, subTree); + return; + } + // recurse deeper into the response object + Object.keys(cursor).forEach((subAggId) => { + const subAgg = cursor[subAggId]; + if (typeof subAgg !== 'object' || !('buckets' in subAgg)) { + return; + } + if (isArray(subAgg.buckets)) { + subAgg.buckets.forEach(transformTimeShift); + } else { + Object.values(subAgg.buckets).forEach(transformTimeShift); + } + }); + }; + transformTimeShift(aggCursor); + return { + ...response, + rawResponse: transformedRawResponse, + }; + } + getRequestAggById(id: string) { return this.aggs.find((agg: AggConfig) => agg.id === id); } diff --git a/src/plugins/data/common/search/aggs/metrics/avg_fn.ts b/src/plugins/data/common/search/aggs/metrics/avg_fn.ts index 253013238d10e4..6b329a8bcfa998 100644 --- a/src/plugins/data/common/search/aggs/metrics/avg_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/avg_fn.ts @@ -62,6 +62,13 @@ export const aggAvg = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/count.ts b/src/plugins/data/common/search/aggs/metrics/count.ts index 8a10d7edb3f835..fac1751290f70d 100644 --- a/src/plugins/data/common/search/aggs/metrics/count.ts +++ b/src/plugins/data/common/search/aggs/metrics/count.ts @@ -31,7 +31,12 @@ export const getCountMetricAgg = () => }; }, getValue(agg, bucket) { - return bucket.doc_count; + const timeShift = agg.getTimeShift(); + if (!timeShift) { + return bucket.doc_count; + } else { + return bucket[`doc_count_${timeShift.asMilliseconds()}`]; + } }, isScalable() { return true; diff --git a/src/plugins/data/common/search/aggs/metrics/count_fn.ts b/src/plugins/data/common/search/aggs/metrics/count_fn.ts index 40c87db57eedca..7ca49078ee901e 100644 --- a/src/plugins/data/common/search/aggs/metrics/count_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/count_fn.ts @@ -54,6 +54,13 @@ export const aggCount = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/max_fn.ts b/src/plugins/data/common/search/aggs/metrics/max_fn.ts index 7eac992680737d..f42312a37ecd68 100644 --- a/src/plugins/data/common/search/aggs/metrics/max_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/max_fn.ts @@ -62,6 +62,13 @@ export const aggMax = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/median_fn.ts b/src/plugins/data/common/search/aggs/metrics/median_fn.ts index 1c0afd81a63c4b..9145e088d83a97 100644 --- a/src/plugins/data/common/search/aggs/metrics/median_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/median_fn.ts @@ -67,6 +67,13 @@ export const aggMedian = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts b/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts index 3ebb771413665d..45bc571c37515e 100644 --- a/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts +++ b/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts @@ -11,7 +11,7 @@ import { AggType, AggTypeConfig } from '../agg_type'; import { AggParamType } from '../param_types/agg'; import { AggConfig } from '../agg_config'; import { METRIC_TYPES } from './metric_agg_types'; -import { FieldTypes } from '../param_types'; +import { BaseParamType, FieldTypes } from '../param_types'; export interface IMetricAggConfig extends AggConfig { type: InstanceType; @@ -47,6 +47,14 @@ export class MetricAggType) { super(config); + this.params.push( + new BaseParamType({ + name: 'timeShift', + type: 'string', + write: () => {}, + }) as MetricAggParam + ); + this.getValue = config.getValue || ((agg, bucket) => { diff --git a/src/plugins/data/common/search/aggs/metrics/min_fn.ts b/src/plugins/data/common/search/aggs/metrics/min_fn.ts index 6dfbac1ecb8b4d..36b576081ac6e0 100644 --- a/src/plugins/data/common/search/aggs/metrics/min_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/min_fn.ts @@ -62,6 +62,13 @@ export const aggMin = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/single_percentile_fn.ts b/src/plugins/data/common/search/aggs/metrics/single_percentile_fn.ts index e7ef22c6faeee6..b11ed2da8cc76f 100644 --- a/src/plugins/data/common/search/aggs/metrics/single_percentile_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/single_percentile_fn.ts @@ -74,6 +74,13 @@ export const aggSinglePercentile = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/sum_fn.ts b/src/plugins/data/common/search/aggs/metrics/sum_fn.ts index d0175e0c8fafe4..6e795ada786697 100644 --- a/src/plugins/data/common/search/aggs/metrics/sum_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/sum_fn.ts @@ -62,6 +62,13 @@ export const aggSum = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/top_hit_fn.ts b/src/plugins/data/common/search/aggs/metrics/top_hit_fn.ts index 85b54f16954937..9204ca88846c25 100644 --- a/src/plugins/data/common/search/aggs/metrics/top_hit_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/top_hit_fn.ts @@ -94,6 +94,13 @@ export const aggTopHit = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/types.ts b/src/plugins/data/common/search/aggs/types.ts index 675be2323b93e8..c0eb0c6c241a94 100644 --- a/src/plugins/data/common/search/aggs/types.ts +++ b/src/plugins/data/common/search/aggs/types.ts @@ -132,6 +132,7 @@ export type AggsStart = Assign 0 ? timeFields : defaultTimeFields; + aggs.setTimeRange(timeRange as TimeRange); - aggs.setTimeFields(timeFields); + aggs.setTimeFields(allTimeFields); // For now we need to mirror the history of the passed search source, since // the request inspector wouldn't work otherwise. @@ -90,19 +97,60 @@ export const handleRequest = async ({ return aggs.onSearchRequestStart(paramSearchSource, options); }); - // If timeFields have been specified, use the specified ones, otherwise use primary time field of index - // pattern if it's available. - const defaultTimeField = indexPattern?.getTimeField?.(); - const defaultTimeFields = defaultTimeField ? [defaultTimeField.name] : []; - const allTimeFields = timeFields && timeFields.length > 0 ? timeFields : defaultTimeFields; - // If a timeRange has been specified and we had at least one timeField available, create range // filters for that those time fields if (timeRange && allTimeFields.length > 0) { timeFilterSearchSource.setField('filter', () => { - return allTimeFields - .map((fieldName) => getTime(indexPattern, timeRange, { fieldName, forceNow })) - .filter(isRangeFilter); + const timeShifts = aggs.getTimeShifts(); + const hasTimeShift = Object.values(aggs.getTimeShifts()).length > 0; + if (!hasTimeShift) { + return allTimeFields + .map((fieldName) => getTime(indexPattern, timeRange, { fieldName, forceNow })) + .filter(isRangeFilter); + } + return [ + { + meta: { index: indexPattern?.id, params: {}, alias: '', disabled: false, negate: false }, + query: { + bool: { + should: [ + ...Object.entries(timeShifts).map(([, shift]) => { + return { + bool: { + filter: allTimeFields + .map((fieldName) => + getTime( + indexPattern, + { + from: moment(timeRange.from).subtract(shift).toISOString(), + to: moment(timeRange.to).subtract(shift).toISOString(), + }, + { fieldName, forceNow } + ) + ) + .filter(isRangeFilter) + .map((filter) => ({ + range: filter.range, + })), + }, + }; + }), + { + bool: { + filter: allTimeFields + .map((fieldName) => getTime(indexPattern, timeRange, { fieldName, forceNow })) + .filter(isRangeFilter) + .map((filter) => ({ + range: filter.range, + })), + }, + }, + ], + minimum_should_match: 1, + }, + }, + }, + ]; }); } diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index 585126e1184d22..2a7f4fad83bb5f 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -75,7 +75,13 @@ import { estypes } from '@elastic/elasticsearch'; import { normalizeSortRequest } from './normalize_sort_request'; import { fieldWildcardFilter } from '../../../../kibana_utils/common'; import { IIndexPattern, IndexPattern, IndexPatternField } from '../../index_patterns'; -import { AggConfigs, ES_SEARCH_STRATEGY, ISearchGeneric, ISearchOptions } from '../..'; +import { + AggConfigs, + ES_SEARCH_STRATEGY, + IEsSearchResponse, + ISearchGeneric, + ISearchOptions, +} from '../..'; import type { ISearchSource, SearchFieldValue, @@ -408,6 +414,15 @@ export class SearchSource { } } + private postFlightTransform(response: IEsSearchResponse) { + const aggs = this.getField('aggs'); + if (aggs instanceof AggConfigs) { + return aggs.postFlightTransform(response); + } else { + return response; + } + } + private async fetchOthers(response: estypes.SearchResponse, options: ISearchOptions) { const aggs = this.getField('aggs'); if (aggs instanceof AggConfigs) { @@ -445,24 +460,26 @@ export class SearchSource { if (isErrorResponse(response)) { obs.error(response); } else if (isPartialResponse(response)) { - obs.next(response); + obs.next(this.postFlightTransform(response)); } else { if (!this.hasPostFlightRequests()) { - obs.next(response); + obs.next(this.postFlightTransform(response)); obs.complete(); } else { // Treat the complete response as partial, then run the postFlightRequests. obs.next({ - ...response, + ...this.postFlightTransform(response), isPartial: true, isRunning: true, }); const sub = from(this.fetchOthers(response.rawResponse, options)).subscribe({ next: (responseWithOther) => { - obs.next({ - ...response, - rawResponse: responseWithOther, - }); + obs.next( + this.postFlightTransform({ + ...response, + rawResponse: responseWithOther!, + }) + ); }, error: (e) => { obs.error(e); diff --git a/src/plugins/data/common/search/tabify/tabify.ts b/src/plugins/data/common/search/tabify/tabify.ts index 4a8972d4384c23..4ae8a192bc5746 100644 --- a/src/plugins/data/common/search/tabify/tabify.ts +++ b/src/plugins/data/common/search/tabify/tabify.ts @@ -139,7 +139,7 @@ export function tabifyAggResponse( const write = new TabbedAggResponseWriter(aggConfigs, respOpts || {}); const topLevelBucket: AggResponseBucket = { ...esResponse.aggregations, - doc_count: esResponse.hits?.total, + doc_count: esResponse.aggregations.doc_count || esResponse.hits?.total, }; collectBucket(aggConfigs, write, topLevelBucket, '', 1); diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx index a54901a2a2fe1d..3c959693d45aef 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx @@ -45,7 +45,7 @@ export function FrameLayout(props: FrameLayoutProps) { {props.workspacePanel} - {props.suggestionsPanel} + {/* {props.suggestionsPanel} */}
( 'aggFilteredMetric', From 4c4ad720fa2ca442e56b0b342138bb5d31bc52aa Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 29 Apr 2021 17:59:07 +0200 Subject: [PATCH 02/31] stabilize response merging and start building UI --- .../data/common/search/aggs/agg_configs.ts | 97 ++++++++++--- .../data/common/search/aggs/buckets/terms.ts | 45 ++++++ .../data/common/search/tabify/tabify.ts | 2 +- .../dimension_panel/time_shift.tsx | 131 ++++++++++++++++++ .../definitions/calculations/counter_rate.tsx | 2 + .../calculations/cumulative_sum.tsx | 2 + .../definitions/calculations/differences.tsx | 2 + .../calculations/moving_average.tsx | 2 + .../operations/definitions/cardinality.tsx | 2 + .../operations/definitions/column_types.ts | 1 + .../operations/definitions/count.tsx | 2 + .../operations/definitions/index.ts | 1 + .../operations/definitions/last_value.tsx | 2 + .../operations/definitions/metrics.tsx | 2 + .../operations/definitions/percentile.tsx | 2 + 15 files changed, 271 insertions(+), 24 deletions(-) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index eeb370765a77ba..b5ab9ab9b891ed 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -10,7 +10,12 @@ import moment from 'moment'; import _, { cloneDeep, isArray } from 'lodash'; import { i18n } from '@kbn/i18n'; import { Assign } from '@kbn/utility-types'; -import { Aggregate, FiltersAggregate, FiltersBucketItem } from '@elastic/elasticsearch/api/types'; +import { + Aggregate, + Bucket, + FiltersAggregate, + FiltersBucketItem, +} from '@elastic/elasticsearch/api/types'; import { IEsSearchResponse, ISearchOptions, ISearchSource } from 'src/plugins/data/public'; import { AggConfig, AggConfigSerialized, IAggConfig } from './agg_config'; @@ -50,6 +55,8 @@ export interface AggConfigsOptions { export type CreateAggConfigParams = Assign; +type GenericBucket = Bucket & { [property: string]: Aggregate }; + /** * @name AggConfigs * @@ -354,6 +361,19 @@ export class AggConfigs { return timeShifts; } + hasTimeShifts(): boolean { + const timeShifts: Record = {}; + this.getAll() + .filter((agg) => agg.schema === 'metric') + .map((agg) => agg.getTimeShift()) + .forEach((timeShift) => { + if (timeShift) { + timeShifts[String(timeShift.asMilliseconds())] = timeShift; + } + }); + return Object.keys(timeShifts).length > 0; + } + isTimeShiftSplitAgg(aggConfig: AggConfig) { // TODO - probably abstract this in an optional flag on the agg config instead of hard-coding the relationship here const hasDateHistograms = this.getAll().some((agg) => agg.type.name === 'date_histogram'); @@ -372,41 +392,70 @@ export class AggConfigs { const transformedRawResponse = cloneDeep(response.rawResponse); const aggCursor = transformedRawResponse.aggregations!; - // TODO: Add a "context" param here which keeps track of at which level of the response we are - const mergeObject = (target: any, source: any, shiftKey: string) => { + const bucketAggs = this.aggs.filter((agg) => agg.type.type === AggGroupNames.Buckets); + + const mergeAggLevel = ( + target: GenericBucket, + source: GenericBucket, + shiftKey: string, + aggIndex: number + ) => { Object.entries(source).forEach(([key, val]) => { - if (typeof val !== 'object') { - if (key === 'doc_count') { - target[`doc_count_${shiftKey}`] = val; - } else { - target[key] = val; - } + // copy over doc count into special key + if (typeof val === 'number' && key === 'doc_count') { + target[`doc_count_${shiftKey}`] = val; + } else if (typeof val !== 'object') { + // other meta keys not of interest + return; } else { + // a sub-agg const agg = this.byId(key); if (agg && agg.type.type === 'metrics') { const timeShift = agg.getTimeShift(); - if (!timeShift || String(timeShift.asMilliseconds()) !== shiftKey) { - // this is a metric from another time shift, do not copy over - return; + if (timeShift && String(timeShift.asMilliseconds()) === shiftKey) { + // this is a metric from the current time shift, copy it over + target[key] = source[key]; + } + } else if (agg === bucketAggs[aggIndex]) { + // expected next bucket sub agg + const subAggregate = val as Aggregate; + const baseSubAggregate = target[key] as Aggregate; + // only supported bucket formats in agg configs are array of buckets and record of buckets for filters + const buckets = ('buckets' in subAggregate ? subAggregate.buckets : undefined) as + | GenericBucket[] + | Record + | undefined; + const baseBuckets = ('buckets' in baseSubAggregate + ? baseSubAggregate.buckets + : undefined) as GenericBucket[] | Record | undefined; + // merge + if (isArray(buckets) && isArray(baseBuckets)) { + buckets.forEach((bucket, index) => + mergeAggLevel(baseBuckets[index], bucket, shiftKey, aggIndex + 1) + ); + } else if (baseBuckets && buckets && !isArray(baseBuckets)) { + Object.entries(buckets).forEach(([bucketKey, bucket]) => + mergeAggLevel(baseBuckets[bucketKey], bucket, shiftKey, aggIndex + 1) + ); } } - if (!target[key]) { - target[key] = {}; - } - mergeObject(target[key], source[key], shiftKey); } }); }; - const transformTimeShift = (cursor: Record): undefined => { + const transformTimeShift = (cursor: Record, aggIndex: number): undefined => { if (cursor.time_offset_split) { const timeShiftedBuckets = (cursor.time_offset_split as FiltersAggregate).buckets as Record< string, FiltersBucketItem >; const subTree = timeShiftedBuckets['0']; - // TODO pull in metrics from other buckets - Object.entries(timeShifts).forEach(([key, shift]) => { - mergeObject(subTree, timeShiftedBuckets[key], key); + Object.keys(timeShifts).forEach((key) => { + mergeAggLevel( + subTree as GenericBucket, + timeShiftedBuckets[key] as GenericBucket, + key, + aggIndex + ); }); delete cursor.time_offset_split; @@ -420,13 +469,15 @@ export class AggConfigs { return; } if (isArray(subAgg.buckets)) { - subAgg.buckets.forEach(transformTimeShift); + subAgg.buckets.forEach((bucket) => transformTimeShift(bucket, aggIndex + 1)); } else { - Object.values(subAgg.buckets).forEach(transformTimeShift); + Object.values(subAgg.buckets).forEach((bucket) => + transformTimeShift(bucket, aggIndex + 1) + ); } }); }; - transformTimeShift(aggCursor); + transformTimeShift(aggCursor, 0); return { ...response, rawResponse: transformedRawResponse, diff --git a/src/plugins/data/common/search/aggs/buckets/terms.ts b/src/plugins/data/common/search/aggs/buckets/terms.ts index 1b876051d009b7..44088ecb49db6c 100644 --- a/src/plugins/data/common/search/aggs/buckets/terms.ts +++ b/src/plugins/data/common/search/aggs/buckets/terms.ts @@ -9,6 +9,7 @@ import { noop } from 'lodash'; import { i18n } from '@kbn/i18n'; +import moment from 'moment'; import { BucketAggType, IBucketAggConfig } from './bucket_agg_type'; import { BUCKET_TYPES } from './bucket_agg_types'; import { createFilterTerms } from './create_filter/terms'; @@ -179,6 +180,50 @@ export const getTermsBucketAgg = () => return; } + if (aggs?.hasTimeShifts()) { + const shift = orderAgg.getTimeShift(); + orderAgg = aggs.createAggConfig( + { + type: 'filtered_metric', + id: 'shifted-order', + params: { + customBucket: aggs + .createAggConfig( + { + type: 'filter', + id: 'shift', + params: { + filter: { + language: 'lucene', + query: { + range: { + [aggs.timeFields![0]]: { + from: moment(aggs.timeRange!.from) + .subtract(shift || 0) + .toISOString(), + to: moment(aggs.timeRange!.to) + .subtract(shift || 0) + .toISOString(), + }, + }, + }, + }, + }, + }, + { + addToAggConfigs: false, + } + ) + .serialize(), + customMetric: orderAgg.serialize(), + }, + enabled: false, + }, + { + addToAggConfigs: false, + } + ); + } if (orderAgg.type.name === 'count') { order._count = dir; return; diff --git a/src/plugins/data/common/search/tabify/tabify.ts b/src/plugins/data/common/search/tabify/tabify.ts index 4ae8a192bc5746..a4d9551da75d50 100644 --- a/src/plugins/data/common/search/tabify/tabify.ts +++ b/src/plugins/data/common/search/tabify/tabify.ts @@ -139,7 +139,7 @@ export function tabifyAggResponse( const write = new TabbedAggResponseWriter(aggConfigs, respOpts || {}); const topLevelBucket: AggResponseBucket = { ...esResponse.aggregations, - doc_count: esResponse.aggregations.doc_count || esResponse.hits?.total, + doc_count: esResponse.aggregations?.doc_count || esResponse.hits?.total, }; collectBucket(aggConfigs, write, topLevelBucket, '', 1); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx new file mode 100644 index 00000000000000..65bc23b4eb1cad --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiButtonIcon, EuiLink, EuiPanel, EuiPopover } from '@elastic/eui'; +import { EuiFormRow, EuiFlexItem, EuiFlexGroup } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React, { useState } from 'react'; +import { Query } from 'src/plugins/data/public'; +import { IndexPatternColumn, operationDefinitionMap } from '../operations'; +import { isQueryValid } from '../operations/definitions/filters'; +import { QueryInput } from '../query_input'; +import { IndexPattern, IndexPatternLayer } from '../types'; + +// to do: get the language from uiSettings +export const defaultFilter: Query = { + query: '', + language: 'kuery', +}; + +export function setFilter(columnId: string, layer: IndexPatternLayer, query: Query | undefined) { + return { + ...layer, + columns: { + ...layer.columns, + [columnId]: { + ...layer.columns[columnId], + filter: query, + }, + }, + }; +} + +export function Filtering({ + selectedColumn, + columnId, + layer, + updateLayer, + indexPattern, + isInitiallyOpen, +}: { + selectedColumn: IndexPatternColumn; + indexPattern: IndexPattern; + columnId: string; + layer: IndexPatternLayer; + updateLayer: (newLayer: IndexPatternLayer) => void; + isInitiallyOpen: boolean; +}) { + const [filterPopoverOpen, setFilterPopoverOpen] = useState(isInitiallyOpen); + const selectedOperation = operationDefinitionMap[selectedColumn.operationType]; + if (!selectedOperation.filterable || !selectedColumn.filter) { + return null; + } + + const isInvalid = !isQueryValid(selectedColumn.filter, indexPattern); + + return ( + + + + { + setFilterPopoverOpen(false); + }} + anchorClassName="eui-fullWidth" + panelClassName="lnsIndexPatternDimensionEditor__filtersEditor" + button={ + + + {/* Empty for spacing */} + + { + setFilterPopoverOpen(!filterPopoverOpen); + }} + color={isInvalid ? 'danger' : 'text'} + title={i18n.translate('xpack.lens.indexPattern.filterBy.clickToEdit', { + defaultMessage: 'Click to edit', + })} + > + {selectedColumn.filter.query || + i18n.translate('xpack.lens.indexPattern.filterBy.emptyFilterQuery', { + defaultMessage: '(empty)', + })} + + + + { + updateLayer(setFilter(columnId, layer, undefined)); + }} + iconType="cross" + /> + + + + } + > + { + updateLayer(setFilter(columnId, layer, newQuery)); + }} + isInvalid={false} + onSubmit={() => {}} + /> + + + + + ); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx index c57f70ba1b58b1..c3c6dd92f1781b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx @@ -92,6 +92,7 @@ export const counterRateOperation: OperationDefinition< scale: 'ratio', references: referenceIds, timeScale, + timeShift: previousColumn?.timeShift, filter: previousColumn?.filter, params: getFormatFromPreviousColumn(previousColumn), }; @@ -118,4 +119,5 @@ export const counterRateOperation: OperationDefinition< }, timeScalingMode: 'mandatory', filterable: true, + shiftable: true, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx index 7cec1fa0d4bbc9..aefd6737f1bea9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx @@ -86,6 +86,7 @@ export const cumulativeSumOperation: OperationDefinition< isBucketed: false, scale: 'ratio', filter: previousColumn?.filter, + timeShift: previousColumn?.timeShift, references: referenceIds, params: getFormatFromPreviousColumn(previousColumn), }; @@ -111,4 +112,5 @@ export const cumulativeSumOperation: OperationDefinition< )?.join(', '); }, filterable: true, + shiftable: true, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx index bef3fbc2e48aed..2143dde0cb44a4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx @@ -82,6 +82,7 @@ export const derivativeOperation: OperationDefinition< references: referenceIds, timeScale: previousColumn?.timeScale, filter: previousColumn?.filter, + timeShift: previousColumn?.timeShift, params: getFormatFromPreviousColumn(previousColumn), }; }, @@ -108,4 +109,5 @@ export const derivativeOperation: OperationDefinition< }, timeScalingMode: 'optional', filterable: true, + shiftable: true, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx index 46cc64c2bc5184..72dbcc983f539d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx @@ -90,6 +90,7 @@ export const movingAverageOperation: OperationDefinition< references: referenceIds, timeScale: previousColumn?.timeScale, filter: previousColumn?.filter, + timeShift: previousColumn?.timeShift, params: { window: 5, ...getFormatFromPreviousColumn(previousColumn), @@ -121,6 +122,7 @@ export const movingAverageOperation: OperationDefinition< }, timeScalingMode: 'optional', filterable: true, + shiftable: true, }; function MovingAverageParamEditor({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx index fa1691ba9a78e5..0195304542385f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx @@ -71,6 +71,7 @@ export const cardinalityOperation: OperationDefinition ofName(getSafeName(column.sourceField, indexPattern)), buildColumn({ field, previousColumn }) { return { @@ -81,6 +82,7 @@ export const cardinalityOperation: OperationDefinition { */ timeScalingMode?: TimeScalingMode; filterable?: boolean; + shiftable?: boolean; getHelpMessage?: (props: HelpProps) => React.ReactNode; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx index 4f5c897fb5378b..d10c2fdbd73f6f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx @@ -162,6 +162,7 @@ export const lastValueOperation: OperationDefinition { return buildExpressionFunction('aggTopHit', { id: columnId, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx index 20580634d12e6b..fa9b5876c566f9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx @@ -99,6 +99,7 @@ function buildMetricOperation>({ scale: 'ratio', timeScale: optionalTimeScaling ? previousColumn?.timeScale : undefined, filter: previousColumn?.filter, + timeShift: previousColumn?.timeShift, params: getFormatFromPreviousColumn(previousColumn), } as T), onFieldChange: (oldColumn, field) => { @@ -119,6 +120,7 @@ function buildMetricOperation>({ getErrorMessage: (layer, columnId, indexPattern) => getInvalidFieldMessage(layer.columns[columnId] as FieldBasedIndexPatternColumn, indexPattern), filterable: true, + shiftable: true, } as OperationDefinition; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx index dd0f3b978da5fc..12172c407dda99 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx @@ -52,6 +52,7 @@ export const percentileOperation: OperationDefinition { if (supportedFieldTypes.includes(fieldType) && aggregatable && !aggregationRestrictions) { return { @@ -88,6 +89,7 @@ export const percentileOperation: OperationDefinition Date: Fri, 30 Apr 2021 14:29:55 +0200 Subject: [PATCH 03/31] continue building time shift --- .../data/common/search/aggs/agg_config.ts | 37 ++- .../data/common/search/aggs/agg_configs.ts | 219 ++++++++++++----- .../data/common/search/aggs/agg_type.ts | 23 +- .../search/aggs/buckets/bucket_agg_type.ts | 27 ++- .../search/aggs/buckets/date_histogram.ts | 11 + .../search/aggs/metrics/cardinality_fn.ts | 7 + .../search/aggs/metrics/metric_agg_type.ts | 5 + .../expressions/esaggs/request_handler.ts | 62 +---- .../public/components/agg_params_helper.ts | 3 + .../dimension_panel/dimension_editor.tsx | 28 +++ .../dimension_panel/time_shift.tsx | 222 ++++++++++++------ .../operations/definitions/cardinality.tsx | 1 + .../operations/definitions/count.tsx | 1 + .../operations/definitions/last_value.tsx | 1 + .../operations/definitions/metrics.tsx | 1 + .../operations/definitions/percentile.tsx | 1 + .../operations/layer_helpers.ts | 1 + .../indexpattern_datasource/to_expression.ts | 19 +- 18 files changed, 463 insertions(+), 206 deletions(-) diff --git a/src/plugins/data/common/search/aggs/agg_config.ts b/src/plugins/data/common/search/aggs/agg_config.ts index 42dc52c59d822c..38bc552bddbe0b 100644 --- a/src/plugins/data/common/search/aggs/agg_config.ts +++ b/src/plugins/data/common/search/aggs/agg_config.ts @@ -20,7 +20,7 @@ import { import { IAggType } from './agg_type'; import { writeParams } from './agg_params'; -import { IAggConfigs } from './agg_configs'; +import { GenericBucket, IAggConfigs } from './agg_configs'; type State = string | number | boolean | null | undefined | SerializableState; @@ -173,13 +173,48 @@ export class AggConfig { return _.get(this.params, key); } + hasTimeShift(): boolean { + return Boolean(this.getParam('timeShift')); + } + getTimeShift(): undefined | moment.Duration { const rawTimeShift = this.getParam('timeShift'); if (!rawTimeShift) return undefined; + if (rawTimeShift === 'previous') { + const timeShiftInterval = this.aggConfigs.getTimeShiftInterval(); + if (timeShiftInterval) { + return timeShiftInterval; + } else if (!this.aggConfigs.timeRange) { + return; + } + return moment.duration( + moment(this.aggConfigs.timeRange.to).diff(this.aggConfigs.timeRange.from) + ); + } const [, amount, unit] = rawTimeShift.match(/(\d+)(\w)/); return moment.duration(Number(amount), unit); } + getShiftedKey(key: string | number, timeShift: moment.Duration): string | number { + return this.type.getShiftedKey(this, key, timeShift); + } + + getTimeShiftInterval(): undefined | moment.Duration { + return this.type.getTimeShiftInterval(this); + } + + splitForTimeShift(aggs: IAggConfigs) { + return this.type.splitForTimeShift(this, aggs); + } + + orderBuckets(a: GenericBucket, b: GenericBucket): number { + if (this.type.orderBuckets) { + return this.type.orderBuckets(this, a, b); + } else { + return 0; + } + } + write(aggs?: IAggConfigs) { return writeParams(this.type.params, this, aggs); } diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index b5ab9ab9b891ed..afd537321e9039 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -15,15 +15,21 @@ import { Bucket, FiltersAggregate, FiltersBucketItem, + MultiBucketAggregate, } from '@elastic/elasticsearch/api/types'; -import { IEsSearchResponse, ISearchOptions, ISearchSource } from 'src/plugins/data/public'; +import { + IEsSearchResponse, + ISearchOptions, + ISearchSource, + RangeFilter, +} from 'src/plugins/data/public'; import { AggConfig, AggConfigSerialized, IAggConfig } from './agg_config'; import { IAggType } from './agg_type'; import { AggTypesRegistryStart } from './agg_types_registry'; import { AggGroupNames } from './agg_groups'; import { IndexPattern } from '../../index_patterns/index_patterns/index_pattern'; -import { TimeRange } from '../../../common'; +import { TimeRange, getTime, isRangeFilter } from '../../../common'; function removeParentAggs(obj: any) { for (const prop in obj) { @@ -55,7 +61,7 @@ export interface AggConfigsOptions { export type CreateAggConfigParams = Assign; -type GenericBucket = Bucket & { [property: string]: Aggregate }; +export type GenericBucket = Bucket & { [property: string]: Aggregate }; /** * @name AggConfigs @@ -270,40 +276,50 @@ export class AggConfigs { timeShifts: Record, dslLvlCursor: Record ) { - if (this.isTimeShiftSplitAgg(config)) { - const filters: Record = {}; - // TODO validate this - const timeField = this.timeFields![0]; - filters['0'] = { + if (!config.splitForTimeShift(this)) { + return dslLvlCursor; + } + if (!this.timeFields || this.timeFields.length < 1) { + throw new Error('Time shift can only be used with configured time field'); + } + if (!this.timeRange) { + throw new Error('Time shift can only be used with configured time range'); + } + const timeRange = this.timeRange; + const filters: Record = {}; + const timeField = this.timeFields[0]; + filters['0'] = { + range: { + [timeField]: { + // only works if there is a time range + gte: timeRange.from, + lte: timeRange.to, + }, + }, + }; + const timeShiftInterval = config.getTimeShiftInterval(); + Object.entries(timeShifts).forEach(([key, shift]) => { + if (timeShiftInterval && timeShiftInterval.asMilliseconds() > shift.asMilliseconds()) { + throw new Error('All time shifts need to be larger than underlying date interval'); + } + filters[key] = { range: { [timeField]: { // only works if there is a time range - gte: this.timeRange!.from, - lte: this.timeRange!.to, + gte: moment(timeRange.from).subtract(shift).toISOString(), + lte: moment(timeRange.to).subtract(shift).toISOString(), }, }, }; - Object.entries(timeShifts).forEach(([key, shift]) => { - filters[key] = { - range: { - [timeField]: { - // only works if there is a time range - gte: moment(this.timeRange!.from).subtract(shift).toISOString(), - lte: moment(this.timeRange!.to).subtract(shift).toISOString(), - }, - }, - }; - }); - dslLvlCursor.time_offset_split = { - filters: { - filters, - }, - aggs: {}, - }; + }); + dslLvlCursor.time_offset_split = { + filters: { + filters, + }, + aggs: {}, + }; - dslLvlCursor = dslLvlCursor.time_offset_split.aggs; - } - return dslLvlCursor; + return dslLvlCursor.time_offset_split.aggs; } getAll() { @@ -361,27 +377,81 @@ export class AggConfigs { return timeShifts; } + getTimeShiftInterval(): moment.Duration | undefined { + const splitAgg = this.getAll().find((agg) => agg.splitForTimeShift(this)); + return splitAgg?.getTimeShiftInterval(); + } + hasTimeShifts(): boolean { - const timeShifts: Record = {}; - this.getAll() - .filter((agg) => agg.schema === 'metric') - .map((agg) => agg.getTimeShift()) - .forEach((timeShift) => { - if (timeShift) { - timeShifts[String(timeShift.asMilliseconds())] = timeShift; - } - }); - return Object.keys(timeShifts).length > 0; + return this.getAll().some((agg) => agg.hasTimeShift()); } - isTimeShiftSplitAgg(aggConfig: AggConfig) { - // TODO - probably abstract this in an optional flag on the agg config instead of hard-coding the relationship here - const hasDateHistograms = this.getAll().some((agg) => agg.type.name === 'date_histogram'); - // the first date histogram or the first metric (if there are no date histograms) is the level to add the time shift filter split - return ( - (hasDateHistograms && aggConfig.type.name === 'date_histogram') || - this.byType('metrics')[0] === aggConfig - ); + getSearchSourceTimeFilter(forceNow?: Date) { + if (!this.timeFields || !this.timeRange) { + return []; + } + const timeRange = this.timeRange; + const timeFields = this.timeFields; + const timeShifts = this.getTimeShifts(); + const hasTimeShift = Object.values(this.getTimeShifts()).length > 0; + if (!hasTimeShift) { + return this.timeFields + .map((fieldName) => getTime(this.indexPattern, timeRange, { fieldName, forceNow })) + .filter(isRangeFilter); + } + return [ + { + meta: { + index: this.indexPattern?.id, + params: {}, + alias: '', + disabled: false, + negate: false, + }, + query: { + bool: { + should: [ + ...Object.entries(timeShifts).map(([, shift]) => { + return { + bool: { + filter: timeFields + .map( + (fieldName) => + [ + getTime(this.indexPattern, timeRange, { fieldName, forceNow }), + fieldName, + ] as [RangeFilter | undefined, string] + ) + .filter(([filter]) => isRangeFilter(filter)) + .map(([filter, field]) => ({ + range: { + [field]: { + gte: moment(filter?.range[field].gte).subtract(shift).toISOString(), + lte: moment(filter?.range[field].lte).subtract(shift).toISOString(), + }, + }, + })), + }, + }; + }), + { + bool: { + filter: timeFields + .map((fieldName) => + getTime(this.indexPattern, timeRange, { fieldName, forceNow }) + ) + .filter(isRangeFilter) + .map((filter) => ({ + range: filter.range, + })), + }, + }, + ], + minimum_should_match: 1, + }, + }, + }, + ]; } postFlightTransform(response: IEsSearchResponse) { @@ -397,46 +467,71 @@ export class AggConfigs { const mergeAggLevel = ( target: GenericBucket, source: GenericBucket, - shiftKey: string, + shift: moment.Duration, aggIndex: number ) => { Object.entries(source).forEach(([key, val]) => { // copy over doc count into special key if (typeof val === 'number' && key === 'doc_count') { - target[`doc_count_${shiftKey}`] = val; + target[`doc_count_${shift.asMilliseconds()}`] = val; } else if (typeof val !== 'object') { // other meta keys not of interest return; } else { // a sub-agg const agg = this.byId(key); - if (agg && agg.type.type === 'metrics') { + if (agg && agg.type.type === AggGroupNames.Metrics) { const timeShift = agg.getTimeShift(); - if (timeShift && String(timeShift.asMilliseconds()) === shiftKey) { + if (timeShift && timeShift.asMilliseconds() === shift.asMilliseconds()) { // this is a metric from the current time shift, copy it over target[key] = source[key]; } } else if (agg === bucketAggs[aggIndex]) { // expected next bucket sub agg const subAggregate = val as Aggregate; - const baseSubAggregate = target[key] as Aggregate; - // only supported bucket formats in agg configs are array of buckets and record of buckets for filters const buckets = ('buckets' in subAggregate ? subAggregate.buckets : undefined) as | GenericBucket[] | Record | undefined; + if (!target[key]) { + // sub aggregate only exists in shifted branch, not in base branch - create dummy aggregate + // which will be filled with shifted data + target[key] = { + buckets: isArray(buckets) ? [] : {}, + }; + } + const baseSubAggregate = target[key] as Aggregate; + // only supported bucket formats in agg configs are array of buckets and record of buckets for filters const baseBuckets = ('buckets' in baseSubAggregate ? baseSubAggregate.buckets : undefined) as GenericBucket[] | Record | undefined; // merge if (isArray(buckets) && isArray(baseBuckets)) { - buckets.forEach((bucket, index) => - mergeAggLevel(baseBuckets[index], bucket, shiftKey, aggIndex + 1) - ); + const baseBucketMap: Record = {}; + baseBuckets.forEach((bucket) => { + baseBucketMap[String(bucket.key)] = bucket; + }); + buckets.forEach((bucket) => { + const bucketKey = agg.getShiftedKey(bucket.key, shift); + // if a bucket is missing in the map, create an empty one + if (!baseBucketMap[bucketKey]) { + baseBucketMap[String(bucketKey)] = { + key: bucketKey, + } as GenericBucket; + } + mergeAggLevel(baseBucketMap[bucketKey], bucket, shift, aggIndex + 1); + }); + (baseSubAggregate as MultiBucketAggregate).buckets = Object.values( + baseBucketMap + ).sort(agg.orderBuckets.bind(agg)); } else if (baseBuckets && buckets && !isArray(baseBuckets)) { - Object.entries(buckets).forEach(([bucketKey, bucket]) => - mergeAggLevel(baseBuckets[bucketKey], bucket, shiftKey, aggIndex + 1) - ); + Object.entries(buckets).forEach(([bucketKey, bucket]) => { + // if a bucket is missing in the base response, create an empty one + if (!baseBuckets[bucketKey]) { + baseBuckets[bucketKey] = {} as GenericBucket; + } + mergeAggLevel(baseBuckets[bucketKey], bucket, shift, aggIndex + 1); + }); } } } @@ -449,11 +544,11 @@ export class AggConfigs { FiltersBucketItem >; const subTree = timeShiftedBuckets['0']; - Object.keys(timeShifts).forEach((key) => { + Object.entries(timeShifts).forEach(([key, shift]) => { mergeAggLevel( subTree as GenericBucket, timeShiftedBuckets[key] as GenericBucket, - key, + shift, aggIndex ); }); diff --git a/src/plugins/data/common/search/aggs/agg_type.ts b/src/plugins/data/common/search/aggs/agg_type.ts index f0f3912bf64fea..4d30d6a2116501 100644 --- a/src/plugins/data/common/search/aggs/agg_type.ts +++ b/src/plugins/data/common/search/aggs/agg_type.ts @@ -8,6 +8,7 @@ import { constant, noop, identity } from 'lodash'; import { i18n } from '@kbn/i18n'; +import moment from 'moment'; import { ISearchSource } from 'src/plugins/data/public'; import { DatatableColumnType, SerializedFieldFormat } from 'src/plugins/expressions/common'; @@ -16,7 +17,7 @@ import type { RequestAdapter } from 'src/plugins/inspector/common'; import { estypes } from '@elastic/elasticsearch'; import { initParams } from './agg_params'; import { AggConfig } from './agg_config'; -import { IAggConfigs } from './agg_configs'; +import { GenericBucket, IAggConfigs } from './agg_configs'; import { BaseParamType } from './param_types/base'; import { AggParamType } from './param_types/agg'; @@ -215,6 +216,26 @@ export class AggType< return agg.id; }; + getShiftedKey( + agg: TAggConfig, + key: string | number, + timeShift: moment.Duration + ): string | number { + return key; + } + + splitForTimeShift(agg: TAggConfig, aggs: IAggConfigs) { + return false; + } + + getTimeShiftInterval(agg: TAggConfig): undefined | moment.Duration { + return undefined; + } + + orderBuckets(agg: TAggConfig, a: GenericBucket, b: GenericBucket): number { + return Number(a.key) - Number(b.key); + } + /** * Generic AggType Constructor * diff --git a/src/plugins/data/common/search/aggs/buckets/bucket_agg_type.ts b/src/plugins/data/common/search/aggs/buckets/bucket_agg_type.ts index e9ed3799b90cf2..1626a89e9a3318 100644 --- a/src/plugins/data/common/search/aggs/buckets/bucket_agg_type.ts +++ b/src/plugins/data/common/search/aggs/buckets/bucket_agg_type.ts @@ -6,8 +6,9 @@ * Side Public License, v 1. */ +import moment from 'moment'; import { IAggConfig } from '../agg_config'; -import { KBN_FIELD_TYPES } from '../../../../common'; +import { GenericBucket, IAggConfigs, KBN_FIELD_TYPES } from '../../../../common'; import { AggType, AggTypeConfig } from '../agg_type'; import { AggParamType } from '../param_types/agg'; @@ -26,6 +27,14 @@ const bucketType = 'buckets'; interface BucketAggTypeConfig extends AggTypeConfig> { getKey?: (bucket: any, key: any, agg: IAggConfig) => any; + getShiftedKey?: ( + agg: TBucketAggConfig, + key: string | number, + timeShift: moment.Duration + ) => string | number; + orderBuckets?(agg: TBucketAggConfig, a: GenericBucket, b: GenericBucket): number; + splitForTimeShift?(agg: TBucketAggConfig, aggs: IAggConfigs): boolean; + getTimeShiftInterval?(agg: TBucketAggConfig): undefined | moment.Duration; } export class BucketAggType extends AggType< @@ -43,6 +52,22 @@ export class BucketAggType { return key || bucket.key; }); + + if (config.getShiftedKey) { + this.getShiftedKey = config.getShiftedKey; + } + + if (config.orderBuckets) { + this.orderBuckets = config.orderBuckets; + } + + if (config.getTimeShiftInterval) { + this.getTimeShiftInterval = config.getTimeShiftInterval; + } + + if (config.splitForTimeShift) { + this.splitForTimeShift = config.splitForTimeShift; + } } } diff --git a/src/plugins/data/common/search/aggs/buckets/date_histogram.ts b/src/plugins/data/common/search/aggs/buckets/date_histogram.ts index 4a83ae38d34db9..4cbf6562487b2c 100644 --- a/src/plugins/data/common/search/aggs/buckets/date_histogram.ts +++ b/src/plugins/data/common/search/aggs/buckets/date_histogram.ts @@ -135,6 +135,17 @@ export const getDateHistogramBucketAgg = ({ }, }; }, + getShiftedKey(agg, key, timeShift) { + return moment(key).add(timeShift).valueOf(); + }, + splitForTimeShift(agg, aggs) { + return aggs.hasTimeShifts() && Boolean(aggs.timeFields?.includes(agg.fieldName())); + }, + getTimeShiftInterval(agg) { + const { useNormalizedEsInterval } = agg.params; + const interval = agg.buckets.getInterval(useNormalizedEsInterval); + return interval; + }, params: [ { name: 'field', diff --git a/src/plugins/data/common/search/aggs/metrics/cardinality_fn.ts b/src/plugins/data/common/search/aggs/metrics/cardinality_fn.ts index ee0f72e01e1def..fe559b71094ecb 100644 --- a/src/plugins/data/common/search/aggs/metrics/cardinality_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/cardinality_fn.ts @@ -67,6 +67,13 @@ export const aggCardinality = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts b/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts index 45bc571c37515e..da71383dd564c0 100644 --- a/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts +++ b/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts @@ -12,6 +12,7 @@ import { AggParamType } from '../param_types/agg'; import { AggConfig } from '../agg_config'; import { METRIC_TYPES } from './metric_agg_types'; import { BaseParamType, FieldTypes } from '../param_types'; +import { AggGroupNames } from '../agg_groups'; export interface IMetricAggConfig extends AggConfig { type: InstanceType; @@ -77,6 +78,10 @@ export class MetricAggType false); + + // split at this point if there are time shifts and this is the first metric + this.splitForTimeShift = (agg, aggs) => + aggs.hasTimeShifts() && aggs.byType(AggGroupNames.Metrics)[0] === agg; } } diff --git a/src/plugins/data/common/search/expressions/esaggs/request_handler.ts b/src/plugins/data/common/search/expressions/esaggs/request_handler.ts index cf20e75f12a537..1eefd71d9e212b 100644 --- a/src/plugins/data/common/search/expressions/esaggs/request_handler.ts +++ b/src/plugins/data/common/search/expressions/esaggs/request_handler.ts @@ -7,18 +7,9 @@ */ import { i18n } from '@kbn/i18n'; -import moment from 'moment'; import { Adapters } from 'src/plugins/inspector/common'; -import { - calculateBounds, - Filter, - getTime, - IndexPattern, - isRangeFilter, - Query, - TimeRange, -} from '../../../../common'; +import { calculateBounds, Filter, IndexPattern, Query, TimeRange } from '../../../../common'; import { IAggConfigs } from '../../aggs'; import { ISearchStartSearchSource } from '../../search_source'; @@ -101,56 +92,7 @@ export const handleRequest = async ({ // filters for that those time fields if (timeRange && allTimeFields.length > 0) { timeFilterSearchSource.setField('filter', () => { - const timeShifts = aggs.getTimeShifts(); - const hasTimeShift = Object.values(aggs.getTimeShifts()).length > 0; - if (!hasTimeShift) { - return allTimeFields - .map((fieldName) => getTime(indexPattern, timeRange, { fieldName, forceNow })) - .filter(isRangeFilter); - } - return [ - { - meta: { index: indexPattern?.id, params: {}, alias: '', disabled: false, negate: false }, - query: { - bool: { - should: [ - ...Object.entries(timeShifts).map(([, shift]) => { - return { - bool: { - filter: allTimeFields - .map((fieldName) => - getTime( - indexPattern, - { - from: moment(timeRange.from).subtract(shift).toISOString(), - to: moment(timeRange.to).subtract(shift).toISOString(), - }, - { fieldName, forceNow } - ) - ) - .filter(isRangeFilter) - .map((filter) => ({ - range: filter.range, - })), - }, - }; - }), - { - bool: { - filter: allTimeFields - .map((fieldName) => getTime(indexPattern, timeRange, { fieldName, forceNow })) - .filter(isRangeFilter) - .map((filter) => ({ - range: filter.range, - })), - }, - }, - ], - minimum_should_match: 1, - }, - }, - }, - ]; + return aggs.getSearchSourceTimeFilter(forceNow); }); } diff --git a/src/plugins/vis_default_editor/public/components/agg_params_helper.ts b/src/plugins/vis_default_editor/public/components/agg_params_helper.ts index fcd97de56fc655..55db880a839322 100644 --- a/src/plugins/vis_default_editor/public/components/agg_params_helper.ts +++ b/src/plugins/vis_default_editor/public/components/agg_params_helper.ts @@ -72,6 +72,9 @@ function getAggParamsToRender({ if (hideCustomLabel && param.name === 'customLabel') { return; } + if (param.name === 'timeShift') { + return; + } // if field param exists, compute allowed fields if (param.type === 'field') { let availableFields: IndexPatternField[] = (param as IFieldParamType).getAvailableFields(agg); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 2c503a7bd69674..ae565720e098e9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -45,6 +45,7 @@ import { ReferenceEditor } from './reference_editor'; import { setTimeScaling, TimeScaling } from './time_scaling'; import { defaultFilter, Filtering, setFilter } from './filtering'; import { AdvancedOptions } from './advanced_options'; +import { setTimeShift, TimeShift } from './time_shift'; const operationPanels = getOperationDisplay(); @@ -160,6 +161,7 @@ export function DimensionEditor(props: DimensionEditorProps) { }, [fieldByOperation, operationWithoutField]); const [filterByOpenInitially, setFilterByOpenInitally] = useState(false); + const [timeShiftedFocused, setTimeShiftFocused] = useState(false); // Operations are compatible if they match inputs. They are always compatible in // the empty state. Field-based operations are not compatible with field-less operations. @@ -519,6 +521,32 @@ export function DimensionEditor(props: DimensionEditorProps) { /> ) : null, }, + { + title: i18n.translate('xpack.lens.indexPattern.timeShift.label', { + defaultMessage: 'Shift in time', + }), + dataTestSubj: 'indexPattern-time-shift-enable', + onClick: () => { + setTimeShiftFocused(true); + setStateWrapper(setTimeShift(columnId, state.layers[layerId], '')); + }, + showInPopover: Boolean( + operationDefinitionMap[selectedColumn.operationType].shiftable && + selectedColumn.timeShift === undefined + ), + inlineElement: + operationDefinitionMap[selectedColumn.operationType].shiftable && + selectedColumn.timeShift !== undefined ? ( + + ) : null, + }, ]} /> )} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index 65bc23b4eb1cad..c81d9e97f2dfd5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -5,14 +5,13 @@ * 2.0. */ -import { EuiButtonIcon, EuiLink, EuiPanel, EuiPopover } from '@elastic/eui'; +import { EuiButtonIcon } from '@elastic/eui'; import { EuiFormRow, EuiFlexItem, EuiFlexGroup } from '@elastic/eui'; +import { EuiComboBox } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import React, { useState } from 'react'; +import React from 'react'; import { Query } from 'src/plugins/data/public'; import { IndexPatternColumn, operationDefinitionMap } from '../operations'; -import { isQueryValid } from '../operations/definitions/filters'; -import { QueryInput } from '../query_input'; import { IndexPattern, IndexPatternLayer } from '../types'; // to do: get the language from uiSettings @@ -21,111 +20,178 @@ export const defaultFilter: Query = { language: 'kuery', }; -export function setFilter(columnId: string, layer: IndexPatternLayer, query: Query | undefined) { +export function setTimeShift( + columnId: string, + layer: IndexPatternLayer, + timeShift: string | undefined +) { return { ...layer, columns: { ...layer.columns, [columnId]: { ...layer.columns[columnId], - filter: query, + timeShift, }, }, }; } -export function Filtering({ +const timeShiftOptions = [ + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.hour', { defaultMessage: '1 hour' }), + value: '1h', + }, + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.3hours', { + defaultMessage: '3 hours', + }), + value: '3h', + }, + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.6hours', { + defaultMessage: '6 hours', + }), + value: '6h', + }, + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.12hours', { + defaultMessage: '12 hours', + }), + value: '12h', + }, + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.day', { defaultMessage: '1 day' }), + value: '1d', + }, + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.week', { defaultMessage: '1 week' }), + value: '1w', + }, + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.month', { defaultMessage: '1 month' }), + value: '1M', + }, + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.3months', { + defaultMessage: '3 months', + }), + value: '3M', + }, + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.6months', { + defaultMessage: '6 months', + }), + value: '6M', + }, + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.year', { defaultMessage: '1 year' }), + value: '1y', + }, + { + label: i18n.translate('xpack.lens.indexPattern.timeShift.previous', { + defaultMessage: 'Previous', + }), + value: 'previous', + }, +]; + +export function TimeShift({ selectedColumn, columnId, layer, updateLayer, indexPattern, - isInitiallyOpen, + isFocused, }: { selectedColumn: IndexPatternColumn; indexPattern: IndexPattern; columnId: string; layer: IndexPatternLayer; updateLayer: (newLayer: IndexPatternLayer) => void; - isInitiallyOpen: boolean; + isFocused: boolean; }) { - const [filterPopoverOpen, setFilterPopoverOpen] = useState(isInitiallyOpen); const selectedOperation = operationDefinitionMap[selectedColumn.operationType]; - if (!selectedOperation.filterable || !selectedColumn.filter) { + if (!selectedOperation.shiftable || selectedColumn.timeShift === undefined) { return null; } - const isInvalid = !isQueryValid(selectedColumn.filter, indexPattern); + const timeShift = selectedColumn.timeShift; + + function getSelectedOption() { + if (timeShift === '') return []; + const goodPick = timeShiftOptions.filter(({ value }) => value === timeShift); + if (goodPick.length > 0) return goodPick; + return [ + { + value: timeShift, + label: timeShift, + }, + ]; + } return ( - { + if (r && isFocused) { + const timeShiftInput = r.querySelector('[data-test-subj="comboBoxSearchInput"]'); + if (timeShiftInput instanceof HTMLInputElement) { + timeShiftInput.focus(); + } + } + }} > - - - { - setFilterPopoverOpen(false); - }} - anchorClassName="eui-fullWidth" - panelClassName="lnsIndexPatternDimensionEditor__filtersEditor" - button={ - - - {/* Empty for spacing */} - - { - setFilterPopoverOpen(!filterPopoverOpen); - }} - color={isInvalid ? 'danger' : 'text'} - title={i18n.translate('xpack.lens.indexPattern.filterBy.clickToEdit', { - defaultMessage: 'Click to edit', - })} - > - {selectedColumn.filter.query || - i18n.translate('xpack.lens.indexPattern.filterBy.emptyFilterQuery', { - defaultMessage: '(empty)', - })} - - - - { - updateLayer(setFilter(columnId, layer, undefined)); - }} - iconType="cross" - /> - - - - } - > - { - updateLayer(setFilter(columnId, layer, newQuery)); + + + + { + updateLayer(setTimeShift(columnId, layer, val)); + }} + onChange={(choices) => { + if (choices.length === 0) { + updateLayer(setTimeShift(columnId, layer, '')); + return; + } + + const choice = choices[0].value as string; + updateLayer(setTimeShift(columnId, layer, choice)); }} - isInvalid={false} - onSubmit={() => {}} /> - - - - + + {selectedOperation.timeScalingMode === 'optional' && ( + + { + updateLayer(setTimeShift(columnId, layer, undefined)); + }} + iconType="cross" + /> + + )} + + + ); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx index 0195304542385f..26318b643369ec 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx @@ -92,6 +92,7 @@ export const cardinalityOperation: OperationDefinition { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx index a3272064bd6024..6790019803343a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx @@ -83,6 +83,7 @@ export const countOperation: OperationDefinition { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx index d10c2fdbd73f6f..5f4f07e97afddd 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx @@ -181,6 +181,7 @@ export const lastValueOperation: OperationDefinition>({ enabled: true, schema: 'metric', field: column.sourceField, + timeShift: column.timeShift, }).toAst(); }, getErrorMessage: (layer, columnId, indexPattern) => diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx index 12172c407dda99..8bc99f078fe378 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx @@ -112,6 +112,7 @@ export const percentileOperation: OperationDefinition { + const referencedColumn = columns[referenceColumnId]; + const referenceDef = operationDefinitionMap[column.operationType]; + if (referenceDef.shiftable) { + columns[referenceColumnId] = { ...referencedColumn, timeShift: column.timeShift }; + } + }); + } }); const columnEntries = columnOrder.map((colId) => [colId, columns[colId]] as const); @@ -84,9 +100,6 @@ function getExpressionForLayer( layer, uiSettings ); - if (col.label === 'shifted') { - aggAst.arguments.timeShift = ['1h']; - } if (wrapInFilter) { aggAst = buildExpressionFunction( 'aggFilteredMetric', From fa12d2e0889fdd938596d97e4a648af3dadd0365 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 30 Apr 2021 18:33:12 +0200 Subject: [PATCH 04/31] add better error handling --- .../data/common/search/aggs/agg_config.ts | 10 +- .../data/common/search/aggs/utils/index.ts | 1 + .../search/aggs/utils/parse_time_shift.ts | 28 +++++ .../dimension_panel/dimension_editor.tsx | 2 + .../dimension_panel/time_shift.tsx | 103 ++++++++++++++--- .../operations/definitions/date_histogram.tsx | 107 ++++++++++++++++-- .../operations/definitions/index.ts | 2 + .../operations/definitions/terms/index.tsx | 32 +++++- .../operations/layer_helpers.ts | 2 + .../public/indexpattern_datasource/utils.ts | 8 +- 10 files changed, 260 insertions(+), 35 deletions(-) create mode 100644 src/plugins/data/common/search/aggs/utils/parse_time_shift.ts diff --git a/src/plugins/data/common/search/aggs/agg_config.ts b/src/plugins/data/common/search/aggs/agg_config.ts index 38bc552bddbe0b..5c34e17fb5c0b0 100644 --- a/src/plugins/data/common/search/aggs/agg_config.ts +++ b/src/plugins/data/common/search/aggs/agg_config.ts @@ -21,6 +21,7 @@ import { import { IAggType } from './agg_type'; import { writeParams } from './agg_params'; import { GenericBucket, IAggConfigs } from './agg_configs'; +import { parseTimeShift } from './utils'; type State = string | number | boolean | null | undefined | SerializableState; @@ -180,7 +181,11 @@ export class AggConfig { getTimeShift(): undefined | moment.Duration { const rawTimeShift = this.getParam('timeShift'); if (!rawTimeShift) return undefined; - if (rawTimeShift === 'previous') { + const parsedTimeShift = parseTimeShift(rawTimeShift); + if (parsedTimeShift === 'invalid') { + throw new Error(`could not parse time shift ${rawTimeShift}`); + } + if (parsedTimeShift === 'previous') { const timeShiftInterval = this.aggConfigs.getTimeShiftInterval(); if (timeShiftInterval) { return timeShiftInterval; @@ -191,8 +196,7 @@ export class AggConfig { moment(this.aggConfigs.timeRange.to).diff(this.aggConfigs.timeRange.from) ); } - const [, amount, unit] = rawTimeShift.match(/(\d+)(\w)/); - return moment.duration(Number(amount), unit); + return parsedTimeShift; } getShiftedKey(key: string | number, timeShift: moment.Duration): string | number { diff --git a/src/plugins/data/common/search/aggs/utils/index.ts b/src/plugins/data/common/search/aggs/utils/index.ts index c92653e8432336..1f2d0a308f0dd2 100644 --- a/src/plugins/data/common/search/aggs/utils/index.ts +++ b/src/plugins/data/common/search/aggs/utils/index.ts @@ -15,3 +15,4 @@ export * from './ipv4_address'; export * from './prop_filter'; export * from './to_angular_json'; export * from './infer_time_zone'; +export * from './parse_time_shift'; diff --git a/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts b/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts new file mode 100644 index 00000000000000..e874f489b13f2b --- /dev/null +++ b/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import moment from 'moment'; + +const allowedUnits = ['s', 'm', 'h', 'd', 'w', 'M', 'y'] as const; +type AllowedUnit = typeof allowedUnits[number]; + +/** + * This method parses a string into a time shift duration. + * If parsing fails, 'invalid' is returned. + * Allowed values are the string 'previous' and an integer followed by the units s,m,h,d,w,M,y + * */ +export const parseTimeShift = (val: string): moment.Duration | 'previous' | 'invalid' => { + if (val === 'previous') { + return 'previous'; + } + const [, amount, unit] = val.match(/(\d+)(\w)/) || []; + const parsedAmount = Number(amount); + if (Number.isNaN(parsedAmount) || !allowedUnits.includes(unit as AllowedUnit)) { + return 'invalid'; + } + return moment.duration(Number(amount), unit as AllowedUnit); +}; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index ae565720e098e9..73fdb6fe99ec82 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -544,6 +544,8 @@ export function DimensionEditor(props: DimensionEditorProps) { layer={state.layers[layerId]} updateLayer={setStateWrapper} isFocused={timeShiftedFocused} + activeData={props.activeData} + layerId={layerId} /> ) : null, }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index c81d9e97f2dfd5..8db0af5a71f0e1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -9,10 +9,13 @@ import { EuiButtonIcon } from '@elastic/eui'; import { EuiFormRow, EuiFlexItem, EuiFlexGroup } from '@elastic/eui'; import { EuiComboBox } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import React from 'react'; +import React, { useEffect, useState } from 'react'; import { Query } from 'src/plugins/data/public'; +import { search } from '../../../../../../src/plugins/data/public'; +import { parseTimeShift } from '../../../../../../src/plugins/data/common'; import { IndexPatternColumn, operationDefinitionMap } from '../operations'; import { IndexPattern, IndexPatternLayer } from '../types'; +import { IndexPatternDimensionEditorProps } from './dimension_panel'; // to do: get the language from uiSettings export const defaultFilter: Query = { @@ -39,53 +42,63 @@ export function setTimeShift( const timeShiftOptions = [ { - label: i18n.translate('xpack.lens.indexPattern.timeShift.hour', { defaultMessage: '1 hour' }), + label: i18n.translate('xpack.lens.indexPattern.timeShift.hour', { + defaultMessage: '1 hour (1h)', + }), value: '1h', }, { label: i18n.translate('xpack.lens.indexPattern.timeShift.3hours', { - defaultMessage: '3 hours', + defaultMessage: '3 hours (3h)', }), value: '3h', }, { label: i18n.translate('xpack.lens.indexPattern.timeShift.6hours', { - defaultMessage: '6 hours', + defaultMessage: '6 hours (6h)', }), value: '6h', }, { label: i18n.translate('xpack.lens.indexPattern.timeShift.12hours', { - defaultMessage: '12 hours', + defaultMessage: '12 hours (12h)', }), value: '12h', }, { - label: i18n.translate('xpack.lens.indexPattern.timeShift.day', { defaultMessage: '1 day' }), + label: i18n.translate('xpack.lens.indexPattern.timeShift.day', { + defaultMessage: '1 day (1d)', + }), value: '1d', }, { - label: i18n.translate('xpack.lens.indexPattern.timeShift.week', { defaultMessage: '1 week' }), + label: i18n.translate('xpack.lens.indexPattern.timeShift.week', { + defaultMessage: '1 week (1w)', + }), value: '1w', }, { - label: i18n.translate('xpack.lens.indexPattern.timeShift.month', { defaultMessage: '1 month' }), + label: i18n.translate('xpack.lens.indexPattern.timeShift.month', { + defaultMessage: '1 month (1M)', + }), value: '1M', }, { label: i18n.translate('xpack.lens.indexPattern.timeShift.3months', { - defaultMessage: '3 months', + defaultMessage: '3 months (3M)', }), value: '3M', }, { label: i18n.translate('xpack.lens.indexPattern.timeShift.6months', { - defaultMessage: '6 months', + defaultMessage: '6 months (6M)', }), value: '6M', }, { - label: i18n.translate('xpack.lens.indexPattern.timeShift.year', { defaultMessage: '1 year' }), + label: i18n.translate('xpack.lens.indexPattern.timeShift.year', { + defaultMessage: '1 year (1y)', + }), value: '1y', }, { @@ -103,6 +116,8 @@ export function TimeShift({ updateLayer, indexPattern, isFocused, + activeData, + layerId, }: { selectedColumn: IndexPatternColumn; indexPattern: IndexPattern; @@ -110,22 +125,52 @@ export function TimeShift({ layer: IndexPatternLayer; updateLayer: (newLayer: IndexPatternLayer) => void; isFocused: boolean; + activeData: IndexPatternDimensionEditorProps['activeData']; + layerId: string; }) { + const [localValue, setLocalValue] = useState(selectedColumn.timeShift); + useEffect(() => { + setLocalValue(selectedColumn.timeShift); + }, [selectedColumn.timeShift]); const selectedOperation = operationDefinitionMap[selectedColumn.operationType]; if (!selectedOperation.shiftable || selectedColumn.timeShift === undefined) { return null; } - const timeShift = selectedColumn.timeShift; + let dateHistogramInterval: null | moment.Duration = null; + const dateHistogramColumn = layer.columnOrder.find( + (colId) => layer.columns[colId].operationType === 'date_histogram' + ); + if (dateHistogramColumn && activeData && activeData[layerId] && activeData[layerId]) { + const column = activeData[layerId].columns.find((col) => col.id === dateHistogramColumn); + if (column) { + dateHistogramInterval = search.aggs.parseInterval( + search.aggs.getDateHistogramMetaDataByDatatableColumn(column)?.interval || '' + ); + } + } + + function isValueTooSmall(parsedValue: ReturnType) { + return ( + dateHistogramInterval && + parsedValue && + typeof parsedValue === 'object' && + parsedValue.asMilliseconds() < dateHistogramInterval.asMilliseconds() + ); + } + + const parsedLocalValue = localValue && parseTimeShift(localValue); + const localValueTooSmall = parsedLocalValue && isValueTooSmall(parsedLocalValue); + const isLocalValueInvalid = Boolean(parsedLocalValue === 'invalid' || localValueTooSmall); function getSelectedOption() { - if (timeShift === '') return []; - const goodPick = timeShiftOptions.filter(({ value }) => value === timeShift); + if (!localValue) return []; + const goodPick = timeShiftOptions.filter(({ value }) => value === localValue); if (goodPick.length > 0) return goodPick; return [ { - value: timeShift, - label: timeShift, + value: localValue, + label: localValue, }, ]; } @@ -147,6 +192,17 @@ export function TimeShift({ label={i18n.translate('xpack.lens.indexPattern.timeShift.label', { defaultMessage: 'Time shift', })} + helpText={i18n.translate('xpack.lens.indexPattern.timeShift.help', { + defaultMessage: 'Time shift is specified by a number followed by a time unit', + })} + error={ + localValueTooSmall && + i18n.translate('xpack.lens.indexPattern.timeShift.tooSmallHelp', { + defaultMessage: + 'Time shift has to be larger than the date histogram interval. Either increase time shift or specify smaller interval in date histogram', + }) + } + isInvalid={isLocalValueInvalid} > @@ -161,8 +217,14 @@ export function TimeShift({ options={timeShiftOptions} selectedOptions={getSelectedOption()} singleSelection={{ asPlainText: true }} + isInvalid={isLocalValueInvalid} onCreateOption={(val) => { - updateLayer(setTimeShift(columnId, layer, val)); + const parsedVal = parseTimeShift(val); + if (parsedVal !== 'invalid' && !isValueTooSmall(parsedVal)) { + updateLayer(setTimeShift(columnId, layer, val)); + } else { + setLocalValue(val); + } }} onChange={(choices) => { if (choices.length === 0) { @@ -171,7 +233,12 @@ export function TimeShift({ } const choice = choices[0].value as string; - updateLayer(setTimeShift(columnId, layer, choice)); + const parsedVal = parseTimeShift(choice); + if (parsedVal !== 'invalid' && !isValueTooSmall(parsedVal)) { + updateLayer(setTimeShift(columnId, layer, choice)); + } else { + setLocalValue(choice); + } }} /> diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx index bd7a270fd7ad80..c045b1a34d3a54 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useState } from 'react'; +import React, { useEffect, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -22,8 +22,10 @@ import { EuiSwitchEvent, EuiTextColor, } from '@elastic/eui'; +import moment from 'moment'; +import { parseTimeShift } from '../../../../../../../src/plugins/data/common'; import { updateColumnParam } from '../layer_helpers'; -import { OperationDefinition } from './index'; +import { OperationDefinition, ParamEditorProps } from './index'; import { FieldBasedIndexPatternColumn } from './column_types'; import { AggFunctionsMapping, @@ -35,6 +37,7 @@ import { import { buildExpressionFunction } from '../../../../../../../src/plugins/expressions/public'; import { getInvalidFieldMessage, getSafeName } from './helpers'; import { HelpPopover, HelpPopoverButton } from '../../help_popover'; +import { IndexPatternLayer } from '../../types'; const { isValidInterval } = search.aggs; const autoInterval = 'auto'; @@ -48,6 +51,28 @@ export interface DateHistogramIndexPatternColumn extends FieldBasedIndexPatternC }; } +function getMultipleDateHistogramsErrorMessage(layer: IndexPatternLayer, columnId: string) { + const usesTimeShift = Object.values(layer.columns).some( + (col) => col.timeShift && col.timeShift !== '' + ); + if (!usesTimeShift) { + return undefined; + } + const dateHistograms = layer.columnOrder.filter( + (colId) => layer.columns[colId].operationType === 'date_histogram' + ); + if (dateHistograms.length < 2) { + return undefined; + } + return i18n.translate('xpack.lens.indexPattern.multipleDateHistogramsError', { + defaultMessage: + '"{dimensionLabel}" is not the only date histogram. When using time shifts, make sure to only use one date histogram.', + values: { + dimensionLabel: layer.columns[columnId].label, + }, + }); +} + export const dateHistogramOperation: OperationDefinition< DateHistogramIndexPatternColumn, 'field' @@ -59,7 +84,13 @@ export const dateHistogramOperation: OperationDefinition< input: 'field', priority: 5, // Highest priority level used getErrorMessage: (layer, columnId, indexPattern) => - getInvalidFieldMessage(layer.columns[columnId] as FieldBasedIndexPatternColumn, indexPattern), + [ + ...(getInvalidFieldMessage( + layer.columns[columnId] as FieldBasedIndexPatternColumn, + indexPattern + ) || []), + getMultipleDateHistogramsErrorMessage(layer, columnId) || '', + ].filter(Boolean), getHelpMessage: (props) => , getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type }) => { if ( @@ -149,12 +180,43 @@ export const dateHistogramOperation: OperationDefinition< extended_bounds: JSON.stringify({}), }).toAst(); }, - paramEditor: ({ layer, columnId, currentColumn, updateLayer, dateRange, data, indexPattern }) => { + paramEditor: function ParamEditor({ + layer, + columnId, + currentColumn, + updateLayer, + dateRange, + data, + indexPattern, + activeData, + }: ParamEditorProps) { + const [localValue, setLocalValue] = useState(currentColumn?.params.interval); + useEffect(() => { + setLocalValue(currentColumn?.params.interval); + }, [currentColumn?.params.interval]); + + function isIntervalTooSmall(interval: moment.Duration | null) { + if (!interval) { + return false; + } + const timeShifts = Object.values(layer.columns) + .filter((col) => col.timeShift && col.timeShift !== 'previous') + .map((col) => col.timeShift); + return timeShifts.some((shift) => { + if (!shift) return false; + const parsedShift = parseTimeShift(shift); + return ( + typeof parsedShift === 'object' && + parsedShift.asMilliseconds() < interval.asMilliseconds() + ); + }); + } + const field = currentColumn && indexPattern.getFieldByName(currentColumn.sourceField); const intervalIsRestricted = field!.aggregationRestrictions && field!.aggregationRestrictions.date_histogram; - const interval = parseInterval(currentColumn.params.interval); + const interval = parseInterval(localValue); // We force the interval value to 1 if it's empty, since that is the ES behavior, // and the isValidInterval function doesn't handle the empty case properly. Fixing @@ -172,13 +234,24 @@ export const dateHistogramOperation: OperationDefinition< updateLayer(updateColumnParam({ layer, columnId, paramName: 'interval', value })); } - const setInterval = (newInterval: typeof interval) => { + const getIntervalAsDuration = (newInterval: typeof interval) => { const isCalendarInterval = calendarOnlyIntervals.has(newInterval.unit); const value = `${isCalendarInterval ? '1' : newInterval.value}${newInterval.unit || 'd'}`; + return search.aggs.parseInterval(value); + }; - updateLayer(updateColumnParam({ layer, columnId, paramName: 'interval', value })); + const setInterval = (newInterval: typeof interval, storeLocally: boolean = false) => { + const isCalendarInterval = calendarOnlyIntervals.has(newInterval.unit); + const value = `${isCalendarInterval ? '1' : newInterval.value}${newInterval.unit || 'd'}`; + + if (storeLocally) { + setLocalValue(value); + } else { + updateLayer(updateColumnParam({ layer, columnId, paramName: 'interval', value })); + } }; + const currentIntervalTooSmall = isIntervalTooSmall(getIntervalAsDuration(interval)); return ( <> {!intervalIsRestricted && ( @@ -200,6 +273,14 @@ export const dateHistogramOperation: OperationDefinition< })} fullWidth display="rowCompressed" + isInvalid={currentIntervalTooSmall} + error={ + currentIntervalTooSmall && + i18n.translate('xpack.lens.indexPattern.intervalTooSmallError', { + defaultMessage: + 'Interval is larger than configured time shift - make sure to keep date histogram interval smaller than time shifts', + }) + } > {intervalIsRestricted ? ( { - setInterval({ + const newInterval = { ...interval, value: e.target.value, - }); + }; + const tooSmall = isIntervalTooSmall(getIntervalAsDuration(newInterval)); + setInterval(newInterval, tooSmall); }} /> @@ -237,10 +320,12 @@ export const dateHistogramOperation: OperationDefinition< data-test-subj="lensDateHistogramUnit" value={interval.unit} onChange={(e) => { - setInterval({ + const newInterval = { ...interval, unit: e.target.value, - }); + }; + const tooSmall = isIntervalTooSmall(getIntervalAsDuration(newInterval)); + setInterval(newInterval, tooSmall); }} isInvalid={!isValid} options={[ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 8fb249501081b5..05b746e636edb9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -43,6 +43,7 @@ import { DateRange } from '../../../../common'; import { ExpressionAstFunction } from '../../../../../../../src/plugins/expressions/public'; import { DataPublicPluginStart } from '../../../../../../../src/plugins/data/public'; import { RangeIndexPatternColumn, rangeOperation } from './ranges'; +import { IndexPatternDimensionEditorProps } from '../../dimension_panel'; /** * A union type of all available column types. If a column is of an unknown type somewhere @@ -147,6 +148,7 @@ export interface ParamEditorProps { http: HttpSetup; dateRange: DateRange; data: DataPublicPluginStart; + activeData?: IndexPatternDimensionEditorProps['activeData']; } export interface HelpProps { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx index d226fe6f2a7452..d6d3ea77af4ab1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx @@ -50,6 +50,30 @@ function isSortableByColumn(layer: IndexPatternLayer, columnId: string) { ); } +function getInvalidNestingOrderMessage(layer: IndexPatternLayer, columnId: string) { + const usesTimeShift = Object.values(layer.columns).some( + (col) => col.timeShift && col.timeShift !== '' + ); + if (!usesTimeShift) { + return undefined; + } + const dateHistogramParent = layer.columnOrder + .slice(0, layer.columnOrder.indexOf(columnId)) + .find((colId) => layer.columns[colId].operationType === 'date_histogram'); + + if (!dateHistogramParent) { + return undefined; + } + return i18n.translate('xpack.lens.indexPattern.termsInShiftedDateHistogramError', { + defaultMessage: + 'Date histogram "{dateLabel}" is grouped by before "{dimensionLabel}". When using time shifts, make sure to group by top values first.', + values: { + dateLabel: layer.columns[dateHistogramParent].label, + dimensionLabel: layer.columns[columnId].label, + }, + }); +} + const DEFAULT_SIZE = 3; const supportedTypes = new Set(['string', 'boolean', 'number', 'ip']); @@ -90,7 +114,13 @@ export const termsOperation: OperationDefinition - getInvalidFieldMessage(layer.columns[columnId] as FieldBasedIndexPatternColumn, indexPattern), + [ + ...(getInvalidFieldMessage( + layer.columns[columnId] as FieldBasedIndexPatternColumn, + indexPattern + ) || []), + getInvalidNestingOrderMessage(layer, columnId) || '', + ].filter(Boolean), isTransferable: (column, newIndexPattern) => { const newField = newIndexPattern.getFieldByName(column.sourceField); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 9868359e1cf70a..f7989fa8180ffb 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -981,6 +981,8 @@ export function updateLayerIndexPattern( * - All columns have complete references * - All column references are valid * - All prerequisites are met + * - If timeshift is used, terms go before date histogram + * - If timeshift is used, only a single date histogram can be used */ export function getErrorMessages( layer: IndexPatternLayer, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts index 19c37da5bf2a93..43e386cff10a3e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts @@ -61,9 +61,13 @@ export function isColumnInvalid( 'references' in column && Boolean(getReferencesErrors(layer, column, indexPattern).filter(Boolean).length); - return ( - !!operationDefinition.getErrorMessage?.(layer, columnId, indexPattern) || referencesHaveErrors + const operationErrorMessages = operationDefinition.getErrorMessage?.( + layer, + columnId, + indexPattern ); + + return (operationErrorMessages && operationErrorMessages.length > 0) || referencesHaveErrors; } function getReferencesErrors( From 64e208acb131cf9f8b773df2180cf6f196e13977 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 3 May 2021 10:52:33 +0200 Subject: [PATCH 05/31] add better error handling --- .../data/common/search/aggs/agg_configs.ts | 23 ++++++- .../data/common/search/aggs/utils/index.ts | 1 + .../common/search/aggs/utils/shift_error.ts | 20 ++++++ .../editor_frame/config_panel/layer_panel.tsx | 2 + .../editor_frame/editor_frame.tsx | 2 + .../editor_frame/state_management.ts | 8 +++ .../workspace_panel/workspace_panel.tsx | 43 +++++++++++- .../dimension_panel/dimension_editor.tsx | 2 + .../dimension_panel/dimension_panel.tsx | 5 +- .../dimension_panel/time_shift.tsx | 69 ++++++++++++++++--- .../indexpattern_datasource/indexpattern.tsx | 15 ++-- .../operations/definitions/index.ts | 32 +++++++-- .../operations/definitions/terms/index.tsx | 56 +++++++++++---- .../operations/layer_helpers.ts | 36 ++++++++-- .../public/indexpattern_datasource/utils.ts | 13 +++- x-pack/plugins/lens/public/types.ts | 18 ++++- 16 files changed, 293 insertions(+), 52 deletions(-) create mode 100644 src/plugins/data/common/search/aggs/utils/shift_error.ts diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index afd537321e9039..24fe5070514db8 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -30,6 +30,7 @@ import { AggTypesRegistryStart } from './agg_types_registry'; import { AggGroupNames } from './agg_groups'; import { IndexPattern } from '../../index_patterns/index_patterns/index_pattern'; import { TimeRange, getTime, isRangeFilter } from '../../../common'; +import { ShiftError } from './utils'; function removeParentAggs(obj: any) { for (const prop in obj) { @@ -300,7 +301,27 @@ export class AggConfigs { const timeShiftInterval = config.getTimeShiftInterval(); Object.entries(timeShifts).forEach(([key, shift]) => { if (timeShiftInterval && timeShiftInterval.asMilliseconds() > shift.asMilliseconds()) { - throw new Error('All time shifts need to be larger than underlying date interval'); + const aggIds = this.getAll() + .filter((agg) => agg.getTimeShift()?.asMilliseconds() === shift.asMilliseconds()) + .map((agg) => agg.id); + throw new ShiftError( + `All time shifts need to be larger than underlying date interval of ${timeShiftInterval.humanize()}`, + aggIds, + 'tooSmall' + ); + } + if ( + timeShiftInterval && + !Number.isInteger(shift.asMilliseconds() / timeShiftInterval.asMilliseconds()) + ) { + const aggIds = this.getAll() + .filter((agg) => agg.getTimeShift()?.asMilliseconds() === shift.asMilliseconds()) + .map((agg) => agg.id); + throw new ShiftError( + `All time shifts need to be a multiple of the underlying interval of ${timeShiftInterval.humanize()}`, + aggIds, + 'notAMultiple' + ); } filters[key] = { range: { diff --git a/src/plugins/data/common/search/aggs/utils/index.ts b/src/plugins/data/common/search/aggs/utils/index.ts index 1f2d0a308f0dd2..8649686df37364 100644 --- a/src/plugins/data/common/search/aggs/utils/index.ts +++ b/src/plugins/data/common/search/aggs/utils/index.ts @@ -16,3 +16,4 @@ export * from './prop_filter'; export * from './to_angular_json'; export * from './infer_time_zone'; export * from './parse_time_shift'; +export * from './shift_error'; diff --git a/src/plugins/data/common/search/aggs/utils/shift_error.ts b/src/plugins/data/common/search/aggs/utils/shift_error.ts new file mode 100644 index 00000000000000..49f4f3714f5e92 --- /dev/null +++ b/src/plugins/data/common/search/aggs/utils/shift_error.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/** + * Configuration error of time shift + */ +export class ShiftError extends Error { + constructor( + message: string, + public aggIds: string[], + public reason: 'tooSmall' | 'notAMultiple' + ) { + super(message); + } +} diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx index cf3c9099d4b0dd..3b6e2b2a3eb02c 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx @@ -357,6 +357,7 @@ export function LayerPanel( columnId: accessorConfig.columnId, groupId: group.groupId, filterOperations: group.filterOperations, + runtimeError: props.framePublicAPI.runtimeError, }} /> @@ -433,6 +434,7 @@ export function LayerPanel( hideGrouping: activeGroup.hideGrouping, filterOperations: activeGroup.filterOperations, dimensionGroups: groups, + runtimeError: props.framePublicAPI.runtimeError, setState: ( newState: unknown, { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx index 362787ea91c4fb..642ef76da4f365 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx @@ -110,6 +110,7 @@ export function EditorFrame(props: EditorFrameProps) { const framePublicAPI: FramePublicAPI = { datasourceLayers, activeData: state.activeData, + runtimeError: state.lastRuntimeError, dateRange: props.dateRange, query: props.query, filters: props.filters, @@ -366,6 +367,7 @@ export function EditorFrame(props: EditorFrameProps) { plugins={props.plugins} visualizeTriggerFieldContext={visualizeTriggerFieldContext} getSuggestionForField={getSuggestionForField} + hasRuntimeError={Boolean(state.lastRuntimeError)} /> ) } diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts index 53aba0d6f3f6c1..d49ee4cec46f3d 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts @@ -24,6 +24,7 @@ export interface EditorFrameState extends PreviewState { stagedPreview?: PreviewState; activeDatasourceId: string | null; activeData?: TableInspectorAdapter; + lastRuntimeError?: Error; } export type Action = @@ -31,6 +32,10 @@ export type Action = type: 'RESET'; state: EditorFrameState; } + | { + type: 'RUNTIME_ERROR'; + error: Error; + } | { type: 'UPDATE_TITLE'; title: string; @@ -144,11 +149,14 @@ export const reducer = (state: EditorFrameState, action: Action): EditorFrameSta return action.state; case 'UPDATE_TITLE': return { ...state, title: action.title }; + case 'RUNTIME_ERROR': + return { ...state, lastRuntimeError: action.error }; case 'UPDATE_STATE': return action.updater(state); case 'UPDATE_ACTIVE_DATA': return { ...state, + lastRuntimeError: undefined, activeData: { ...action.tables }, }; case 'UPDATE_LAYER': diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx index a31146e5004349..389475d07d8313 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx @@ -18,6 +18,7 @@ import { EuiButtonEmpty, EuiLink, EuiPageContentBody, + EuiButton, } from '@elastic/eui'; import { CoreStart, ApplicationStart } from 'kibana/public'; import { @@ -74,12 +75,17 @@ export interface WorkspacePanelProps { core: CoreStart; plugins: { uiActions?: UiActionsStart; data: DataPublicPluginStart }; title?: string; + hasRuntimeError?: boolean; visualizeTriggerFieldContext?: VisualizeFieldContext; getSuggestionForField: (field: DragDropIdentifier) => Suggestion | undefined; } interface WorkspaceState { - expressionBuildError?: Array<{ shortMessage: string; longMessage: string }>; + expressionBuildError?: Array<{ + shortMessage: string; + longMessage: string; + fixAction?: { label: string; newState: unknown }; + }>; expandError: boolean; } @@ -127,6 +133,7 @@ export const InnerWorkspacePanel = React.memo(function InnerWorkspacePanel({ title, visualizeTriggerFieldContext, suggestionForDraggedField, + hasRuntimeError, }: Omit & { suggestionForDraggedField: Suggestion | undefined; }) { @@ -334,6 +341,8 @@ export const InnerWorkspacePanel = React.memo(function InnerWorkspacePanel({ localState={{ ...localState, configurationValidationError, missingRefsErrors }} ExpressionRendererComponent={ExpressionRendererComponent} application={core.application} + hasRuntimeError={hasRuntimeError} + activeDatasourceId={activeDatasourceId} /> ); }; @@ -377,6 +386,8 @@ export const VisualizationWrapper = ({ ExpressionRendererComponent, dispatch, application, + hasRuntimeError, + activeDatasourceId, }: { expression: string | null | undefined; framePublicAPI: FramePublicAPI; @@ -385,11 +396,17 @@ export const VisualizationWrapper = ({ dispatch: (action: Action) => void; setLocalState: (dispatch: (prevState: WorkspaceState) => WorkspaceState) => void; localState: WorkspaceState & { - configurationValidationError?: Array<{ shortMessage: string; longMessage: string }>; + configurationValidationError?: Array<{ + shortMessage: string; + longMessage: string; + fixAction?: { label: string; newState: unknown }; + }>; missingRefsErrors?: Array<{ shortMessage: string; longMessage: string }>; }; ExpressionRendererComponent: ReactExpressionRendererType; application: ApplicationStart; + hasRuntimeError?: boolean; + activeDatasourceId: string | null; }) => { const context: ExecutionContextSearch = useMemo( () => ({ @@ -467,6 +484,19 @@ export const VisualizationWrapper = ({

{localState.configurationValidationError[0].longMessage}

+ {localState.configurationValidationError[0].fixAction && activeDatasourceId && ( + { + dispatch({ + type: 'UPDATE_DATASOURCE_STATE', + datasourceId: activeDatasourceId, + updater: localState.configurationValidationError?.[0].fixAction?.newState, + }); + }} + > + {localState.configurationValidationError[0].fixAction.label} + + )} {showExtraErrors} @@ -526,6 +556,7 @@ export const VisualizationWrapper = ({ } if (localState.expressionBuildError?.length) { + const firstError = localState.expressionBuildError[0]; return ( @@ -539,7 +570,7 @@ export const VisualizationWrapper = ({ />

-

{localState.expressionBuildError[0].longMessage}

+

{firstError.longMessage}

} iconColor="danger" @@ -562,6 +593,12 @@ export const VisualizationWrapper = ({ onData$={onData$} renderMode="edit" renderError={(errorMessage?: string | null, error?: ExpressionRenderError | null) => { + if (!hasRuntimeError && error?.original) { + dispatch({ + type: 'RUNTIME_ERROR', + error: error.original, + }); + } const errorsFromRequest = getOriginalRequestErrorMessages(error); const visibleErrorMessages = errorsFromRequest.length ? errorsFromRequest diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 73fdb6fe99ec82..c5f653fa22cfa2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -121,6 +121,7 @@ export function DimensionEditor(props: DimensionEditorProps) { hideGrouping, dateRange, dimensionGroups, + runtimeError, } = props; const services = { data: props.data, @@ -546,6 +547,7 @@ export function DimensionEditor(props: DimensionEditorProps) { isFocused={timeShiftedFocused} activeData={props.activeData} layerId={layerId} + runtimeError={runtimeError} /> ) : null, }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.tsx index 3965f992805b5d..ccf5f53d6c9d5f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.tsx @@ -44,14 +44,15 @@ function wrapOnDot(str?: string) { export const IndexPatternDimensionTriggerComponent = function IndexPatternDimensionTrigger( props: IndexPatternDimensionTriggerProps ) { + const runtimeError = props.runtimeError; const layerId = props.layerId; const layer = props.state.layers[layerId]; const currentIndexPattern = props.state.indexPatterns[layer.indexPatternId]; const { columnId, uniqueLabel } = props; const currentColumnHasErrors = useMemo( - () => isColumnInvalid(layer, columnId, currentIndexPattern), - [layer, columnId, currentIndexPattern] + () => isColumnInvalid(layer, columnId, currentIndexPattern, runtimeError), + [layer, columnId, currentIndexPattern, runtimeError] ); const selectedColumn: IndexPatternColumn | null = layer.columns[props.columnId] ?? null; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index 8db0af5a71f0e1..b9052e4b9e40f6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -12,7 +12,7 @@ import { i18n } from '@kbn/i18n'; import React, { useEffect, useState } from 'react'; import { Query } from 'src/plugins/data/public'; import { search } from '../../../../../../src/plugins/data/public'; -import { parseTimeShift } from '../../../../../../src/plugins/data/common'; +import { parseTimeShift, ShiftError } from '../../../../../../src/plugins/data/common'; import { IndexPatternColumn, operationDefinitionMap } from '../operations'; import { IndexPattern, IndexPatternLayer } from '../types'; import { IndexPatternDimensionEditorProps } from './dimension_panel'; @@ -118,6 +118,7 @@ export function TimeShift({ isFocused, activeData, layerId, + runtimeError, }: { selectedColumn: IndexPatternColumn; indexPattern: IndexPattern; @@ -127,6 +128,7 @@ export function TimeShift({ isFocused: boolean; activeData: IndexPatternDimensionEditorProps['activeData']; layerId: string; + runtimeError?: Error | undefined; }) { const [localValue, setLocalValue] = useState(selectedColumn.timeShift); useEffect(() => { @@ -159,9 +161,35 @@ export function TimeShift({ ); } + function isValueNotMultiple(parsedValue: ReturnType) { + return ( + dateHistogramInterval && + parsedValue && + typeof parsedValue === 'object' && + !Number.isInteger(parsedValue.asMilliseconds() / dateHistogramInterval.asMilliseconds()) + ); + } + + function getRuntimeErrorReason() { + if (!(runtimeError instanceof ShiftError)) { + return undefined; + } + if (!runtimeError.aggIds.includes(columnId)) { + return undefined; + } + return runtimeError.reason; + } + const parsedLocalValue = localValue && parseTimeShift(localValue); - const localValueTooSmall = parsedLocalValue && isValueTooSmall(parsedLocalValue); - const isLocalValueInvalid = Boolean(parsedLocalValue === 'invalid' || localValueTooSmall); + const localValueTooSmall = + (parsedLocalValue && isValueTooSmall(parsedLocalValue)) || + getRuntimeErrorReason() === 'tooSmall'; + const localValueNotMultiple = + (parsedLocalValue && isValueNotMultiple(parsedLocalValue)) || + getRuntimeErrorReason() === 'notAMultiple'; + const isLocalValueInvalid = Boolean( + parsedLocalValue === 'invalid' || localValueTooSmall || localValueNotMultiple + ); function getSelectedOption() { if (!localValue) return []; @@ -196,11 +224,17 @@ export function TimeShift({ defaultMessage: 'Time shift is specified by a number followed by a time unit', })} error={ - localValueTooSmall && - i18n.translate('xpack.lens.indexPattern.timeShift.tooSmallHelp', { - defaultMessage: - 'Time shift has to be larger than the date histogram interval. Either increase time shift or specify smaller interval in date histogram', - }) + localValueTooSmall + ? i18n.translate('xpack.lens.indexPattern.timeShift.tooSmallHelp', { + defaultMessage: + 'Time shift has to be larger than the date histogram interval. Either increase time shift or specify smaller interval in date histogram', + }) + : localValueNotMultiple + ? i18n.translate('xpack.lens.indexPattern.timeShift.noMultipleHelp', { + defaultMessage: + 'Time shift has to be a multiple of the date histogram interval. Either adjust time shift or date histogram interval', + }) + : undefined } isInvalid={isLocalValueInvalid} > @@ -214,13 +248,22 @@ export function TimeShift({ placeholder={i18n.translate('xpack.lens.indexPattern.timeShiftPlaceholder', { defaultMessage: 'Time shift (e.g. 1d)', })} - options={timeShiftOptions} + options={timeShiftOptions.filter(({ value }) => { + const parsedValue = parseTimeShift(value); + return ( + parsedValue && !isValueTooSmall(parsedValue) && !isValueNotMultiple(parsedValue) + ); + })} selectedOptions={getSelectedOption()} singleSelection={{ asPlainText: true }} isInvalid={isLocalValueInvalid} onCreateOption={(val) => { const parsedVal = parseTimeShift(val); - if (parsedVal !== 'invalid' && !isValueTooSmall(parsedVal)) { + if ( + parsedVal !== 'invalid' && + !isValueTooSmall(parsedVal) && + !isValueNotMultiple(parsedVal) + ) { updateLayer(setTimeShift(columnId, layer, val)); } else { setLocalValue(val); @@ -234,7 +277,11 @@ export function TimeShift({ const choice = choices[0].value as string; const parsedVal = parseTimeShift(choice); - if (parsedVal !== 'invalid' && !isValueTooSmall(parsedVal)) { + if ( + parsedVal !== 'invalid' && + !isValueTooSmall(parsedVal) && + !isValueNotMultiple(parsedVal) + ) { updateLayer(setTimeShift(columnId, layer, choice)); } else { setLocalValue(choice); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index 81eb46e8167155..dfb0d74dd7cb5a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -374,13 +374,14 @@ export function getIndexPatternDatasource({ } // Forward the indexpattern as well, as it is required by some operationType checks - const layerErrors = Object.values(state.layers).map((layer) => - (getErrorMessages(layer, state.indexPatterns[layer.indexPatternId]) ?? []).map( - (message) => ({ - shortMessage: '', // Not displayed currently - longMessage: message, - }) - ) + const layerErrors = Object.entries(state.layers).map(([layerId, layer]) => + ( + getErrorMessages(layer, state.indexPatterns[layer.indexPatternId], state, layerId) ?? [] + ).map((message) => ({ + shortMessage: '', // Not displayed currently + longMessage: typeof message === 'string' ? message : message.message, + fixAction: typeof message === 'object' ? message.fixAction : undefined, + })) ); // Single layer case, no need to explain more diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 05b746e636edb9..ace51624c5c713 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -38,7 +38,12 @@ import { countOperation, CountIndexPatternColumn } from './count'; import { lastValueOperation, LastValueIndexPatternColumn } from './last_value'; import { OperationMetadata } from '../../../types'; import type { BaseIndexPatternColumn, ReferenceBasedIndexPatternColumn } from './column_types'; -import { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../types'; +import { + IndexPattern, + IndexPatternField, + IndexPatternLayer, + IndexPatternPrivateState, +} from '../../types'; import { DateRange } from '../../../../common'; import { ExpressionAstFunction } from '../../../../../../../src/plugins/expressions/public'; import { DataPublicPluginStart } from '../../../../../../../src/plugins/data/public'; @@ -222,8 +227,16 @@ interface BaseOperationDefinitionProps { getErrorMessage?: ( layer: IndexPatternLayer, columnId: string, - indexPattern: IndexPattern - ) => string[] | undefined; + indexPattern: IndexPattern, + // TODO - remove these again and remap the state on a higher level + state?: IndexPatternPrivateState, + layerId?: string + ) => + | Array< + | string + | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } + > + | undefined; /* * Flag whether this operation can be scaled by time unit if a date histogram is available. @@ -320,11 +333,18 @@ interface FieldBasedOperationDefinition { * - Requires a date histogram operation somewhere before it in order * - Missing references */ - getErrorMessage: ( + getErrorMessage?: ( layer: IndexPatternLayer, columnId: string, - indexPattern: IndexPattern - ) => string[] | undefined; + indexPattern: IndexPattern, + state?: IndexPatternPrivateState, + layerId?: string + ) => + | Array< + | string + | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } + > + | undefined; } export interface RequiredReference { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx index d6d3ea77af4ab1..77c76398017c4a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx @@ -24,7 +24,7 @@ import { OperationDefinition } from '../index'; import { FieldBasedIndexPatternColumn } from '../column_types'; import { ValuesInput } from './values_input'; import { getInvalidFieldMessage } from '../helpers'; -import type { IndexPatternLayer } from '../../../types'; +import type { IndexPatternLayer, IndexPatternPrivateState } from '../../../types'; function ofName(name?: string) { return i18n.translate('xpack.lens.indexPattern.termsOf', { @@ -50,7 +50,12 @@ function isSortableByColumn(layer: IndexPatternLayer, columnId: string) { ); } -function getInvalidNestingOrderMessage(layer: IndexPatternLayer, columnId: string) { +function getInvalidNestingOrderMessage( + layer: IndexPatternLayer, + columnId: string, + state?: IndexPatternPrivateState, + layerId?: string +) { const usesTimeShift = Object.values(layer.columns).some( (col) => col.timeShift && col.timeShift !== '' ); @@ -64,14 +69,41 @@ function getInvalidNestingOrderMessage(layer: IndexPatternLayer, columnId: strin if (!dateHistogramParent) { return undefined; } - return i18n.translate('xpack.lens.indexPattern.termsInShiftedDateHistogramError', { - defaultMessage: - 'Date histogram "{dateLabel}" is grouped by before "{dimensionLabel}". When using time shifts, make sure to group by top values first.', - values: { - dateLabel: layer.columns[dateHistogramParent].label, - dimensionLabel: layer.columns[columnId].label, - }, - }); + return { + message: i18n.translate('xpack.lens.indexPattern.termsInShiftedDateHistogramError', { + defaultMessage: + 'Date histogram "{dateLabel}" is grouped by before "{dimensionLabel}". When using time shifts, make sure to group by top values first.', + values: { + dateLabel: layer.columns[dateHistogramParent].label, + dimensionLabel: layer.columns[columnId].label, + }, + }), + fixAction: + state && layerId + ? { + label: i18n.translate( + 'xpack.lens.indexPattern.termsInShiftedDateHistogramFixActionLabel', + { + defaultMessage: 'Reorder dimensions', + } + ), + newState: { + ...state, + layers: { + ...state.layers, + [layerId]: { + ...layer, + columns: { + ...layer.columns, + [dateHistogramParent]: layer.columns[columnId], + [columnId]: layer.columns[dateHistogramParent], + }, + }, + }, + }, + } + : undefined, + }; } const DEFAULT_SIZE = 3; @@ -113,13 +145,13 @@ export const termsOperation: OperationDefinition + getErrorMessage: (layer, columnId, indexPattern, state, layerId) => [ ...(getInvalidFieldMessage( layer.columns[columnId] as FieldBasedIndexPatternColumn, indexPattern ) || []), - getInvalidNestingOrderMessage(layer, columnId) || '', + getInvalidNestingOrderMessage(layer, columnId, state, layerId) || '', ].filter(Boolean), isTransferable: (column, newIndexPattern) => { const newField = newIndexPattern.getFieldByName(column.sourceField); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index f7989fa8180ffb..8ef6831890cbd9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -14,7 +14,12 @@ import { IndexPatternColumn, RequiredReference, } from './definitions'; -import type { IndexPattern, IndexPatternField, IndexPatternLayer } from '../types'; +import type { + IndexPattern, + IndexPatternField, + IndexPatternLayer, + IndexPatternPrivateState, +} from '../types'; import { getSortScoreByPriority } from './operations'; import { generateId } from '../../id_generator'; import { ReferenceBasedIndexPatternColumn } from './definitions/column_types'; @@ -986,17 +991,36 @@ export function updateLayerIndexPattern( */ export function getErrorMessages( layer: IndexPatternLayer, - indexPattern: IndexPattern -): string[] | undefined { - const errors: string[] = Object.entries(layer.columns) + indexPattern: IndexPattern, + state: IndexPatternPrivateState, + layerId: string +): + | Array< + | string + | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } + > + | undefined { + const errors: Array< + string | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } + > = Object.entries(layer.columns) .flatMap(([columnId, column]) => { const def = operationDefinitionMap[column.operationType]; if (def.getErrorMessage) { - return def.getErrorMessage(layer, columnId, indexPattern); + return def.getErrorMessage(layer, columnId, indexPattern, state, layerId); } }) // remove the undefined values - .filter((v: string | undefined): v is string => v != null); + .filter( + ( + v: + | string + | undefined + | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } + ): v is + | string + | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } => + v != null + ); return errors.length ? errors : undefined; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts index 43e386cff10a3e..3734f18eb8a59b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { ShiftError } from '../../../../../src/plugins/data/common'; import { DataType } from '../types'; import { IndexPattern, IndexPatternLayer, DraggedField } from './types'; import type { @@ -49,7 +50,8 @@ export function isDraggedField(fieldCandidate: unknown): fieldCandidate is Dragg export function isColumnInvalid( layer: IndexPatternLayer, columnId: string, - indexPattern: IndexPattern + indexPattern: IndexPattern, + runtimeError?: Error ) { const column: IndexPatternColumn | undefined = layer.columns[columnId]; if (!column) return; @@ -67,7 +69,14 @@ export function isColumnInvalid( indexPattern ); - return (operationErrorMessages && operationErrorMessages.length > 0) || referencesHaveErrors; + const hasTimeShiftError = + runtimeError instanceof ShiftError && runtimeError.aggIds.includes(columnId); + + return ( + (operationErrorMessages && operationErrorMessages.length > 0) || + referencesHaveErrors || + hasTimeShiftError + ); } function getReferencesErrors( diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 94b4433a825510..2a7e261f668718 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -236,7 +236,13 @@ export interface Datasource { getErrorMessages: ( state: T, layersGroups?: Record - ) => Array<{ shortMessage: string; longMessage: string }> | undefined; + ) => + | Array<{ + shortMessage: string; + longMessage: string; + fixAction?: { label: string; newState: T }; + }> + | undefined; /** * uniqueLabels of dimensions exposed for aria-labels of dragged dimensions */ @@ -289,6 +295,7 @@ export type DatasourceDimensionProps = SharedDimensionProps & { onRemove?: (accessor: string) => void; state: T; activeData?: Record; + runtimeError?: Error; }; // The only way a visualization has to restrict the query building @@ -310,6 +317,7 @@ export interface DatasourceLayerPanelProps { state: T; setState: StateSetter; activeData?: Record; + runtimeError?: Error; } export interface DraggedOperation extends DraggingIdentifier { @@ -510,6 +518,7 @@ export interface FramePublicAPI { * If accessing, make sure to check whether expected columns actually exist. */ activeData?: Record; + runtimeError?: Error; dateRange: DateRange; query: Query; @@ -682,7 +691,12 @@ export interface Visualization { getErrorMessages: ( state: T, datasourceLayers?: Record - ) => Array<{ shortMessage: string; longMessage: string }> | undefined; + ) => + | Array<{ + shortMessage: string; + longMessage: string; + }> + | undefined; /** * The frame calls this function to display warnings about visualization From 5b389ce92131080d2516347332684bcdfb67aefd Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 3 May 2021 15:40:08 +0200 Subject: [PATCH 06/31] fix broken metric time shift --- .../data/common/search/aggs/metrics/metric_agg_type.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts b/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts index da71383dd564c0..9186ba8f70cfc3 100644 --- a/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts +++ b/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts @@ -81,7 +81,9 @@ export class MetricAggType - aggs.hasTimeShifts() && aggs.byType(AggGroupNames.Metrics)[0] === agg; + aggs.hasTimeShifts() && + aggs.byType(AggGroupNames.Metrics)[0] === agg && + !aggs.byType(AggGroupNames.Buckets).some((bucketAgg) => bucketAgg.splitForTimeShift(aggs)); } } From 20940a7a26412e87a1349704a85beaf8dd97a5f9 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 5 May 2021 09:00:48 +0200 Subject: [PATCH 07/31] experimental fix action --- .../workspace_panel/workspace_panel.tsx | 9 ++- .../dimension_panel/time_shift.tsx | 44 ++++------ .../indexpattern_datasource/indexpattern.tsx | 8 +- .../operations/definitions/index.ts | 16 +++- .../operations/definitions/terms/index.tsx | 81 ++++++++++++++++++- .../operations/layer_helpers.ts | 46 +++++++++-- .../public/indexpattern_datasource/utils.ts | 10 +-- x-pack/plugins/lens/public/types.ts | 2 +- 8 files changed, 163 insertions(+), 53 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx index 389475d07d8313..d563cd05ffd56c 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx @@ -84,7 +84,7 @@ interface WorkspaceState { expressionBuildError?: Array<{ shortMessage: string; longMessage: string; - fixAction?: { label: string; newState: unknown }; + fixAction?: { label: string; newState: () => Promise }; }>; expandError: boolean; } @@ -399,7 +399,7 @@ export const VisualizationWrapper = ({ configurationValidationError?: Array<{ shortMessage: string; longMessage: string; - fixAction?: { label: string; newState: unknown }; + fixAction?: { label: string; newState: () => Promise }; }>; missingRefsErrors?: Array<{ shortMessage: string; longMessage: string }>; }; @@ -486,11 +486,12 @@ export const VisualizationWrapper = ({

{localState.configurationValidationError[0].fixAction && activeDatasourceId && ( { + onClick={async () => { + const newState = await localState.configurationValidationError?.[0].fixAction?.newState(); dispatch({ type: 'UPDATE_DATASOURCE_STATE', datasourceId: activeDatasourceId, - updater: localState.configurationValidationError?.[0].fixAction?.newState, + updater: newState, }); }} > diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index b9052e4b9e40f6..d4e58ee77b73b2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -227,12 +227,12 @@ export function TimeShift({ localValueTooSmall ? i18n.translate('xpack.lens.indexPattern.timeShift.tooSmallHelp', { defaultMessage: - 'Time shift has to be larger than the date histogram interval. Either increase time shift or specify smaller interval in date histogram', + 'Time shift should to be larger than the date histogram interval. Either increase time shift or specify smaller interval in date histogram', }) : localValueNotMultiple ? i18n.translate('xpack.lens.indexPattern.timeShift.noMultipleHelp', { defaultMessage: - 'Time shift has to be a multiple of the date histogram interval. Either adjust time shift or date histogram interval', + 'Time shift should be a multiple of the date histogram interval. Either adjust time shift or date histogram interval', }) : undefined } @@ -259,11 +259,7 @@ export function TimeShift({ isInvalid={isLocalValueInvalid} onCreateOption={(val) => { const parsedVal = parseTimeShift(val); - if ( - parsedVal !== 'invalid' && - !isValueTooSmall(parsedVal) && - !isValueNotMultiple(parsedVal) - ) { + if (parsedVal !== 'invalid') { updateLayer(setTimeShift(columnId, layer, val)); } else { setLocalValue(val); @@ -277,11 +273,7 @@ export function TimeShift({ const choice = choices[0].value as string; const parsedVal = parseTimeShift(choice); - if ( - parsedVal !== 'invalid' && - !isValueTooSmall(parsedVal) && - !isValueNotMultiple(parsedVal) - ) { + if (parsedVal !== 'invalid') { updateLayer(setTimeShift(columnId, layer, choice)); } else { setLocalValue(choice); @@ -289,21 +281,19 @@ export function TimeShift({ }} /> - {selectedOperation.timeScalingMode === 'optional' && ( - - { - updateLayer(setTimeShift(columnId, layer, undefined)); - }} - iconType="cross" - /> - - )} + + { + updateLayer(setTimeShift(columnId, layer, undefined)); + }} + iconType="cross" + /> + diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index dfb0d74dd7cb5a..8ed6c2c3126f4e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -376,7 +376,13 @@ export function getIndexPatternDatasource({ // Forward the indexpattern as well, as it is required by some operationType checks const layerErrors = Object.entries(state.layers).map(([layerId, layer]) => ( - getErrorMessages(layer, state.indexPatterns[layer.indexPatternId], state, layerId) ?? [] + getErrorMessages( + layer, + state.indexPatterns[layer.indexPatternId], + state, + layerId, + core.http + ) ?? [] ).map((message) => ({ shortMessage: '', // Not displayed currently longMessage: typeof message === 'string' ? message : message.message, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index ace51624c5c713..c89ada8c7a6703 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -234,7 +234,13 @@ interface BaseOperationDefinitionProps { ) => | Array< | string - | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } + | { + message: string; + fixAction?: { + label: string; + newState: (http: HttpSetup) => Promise; + }; + } > | undefined; @@ -342,7 +348,13 @@ interface FieldBasedOperationDefinition { ) => | Array< | string - | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } + | { + message: string; + fixAction?: { + label: string; + newState: (http: HttpSetup) => Promise; + }; + } > | undefined; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx index 77c76398017c4a..8b0822a93bc303 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx @@ -16,11 +16,14 @@ import { EuiAccordion, EuiIconTip, } from '@elastic/eui'; +import { uniq } from 'lodash'; +import { HttpSetup } from 'kibana/public'; +import { FieldStatsResponse } from '../../../../../common'; import { AggFunctionsMapping } from '../../../../../../../../src/plugins/data/public'; import { buildExpressionFunction } from '../../../../../../../../src/plugins/expressions/public'; import { updateColumnParam, isReferenced } from '../../layer_helpers'; import { DataType } from '../../../../types'; -import { OperationDefinition } from '../index'; +import { FiltersIndexPatternColumn, OperationDefinition } from '../index'; import { FieldBasedIndexPatternColumn } from '../column_types'; import { ValuesInput } from './values_input'; import { getInvalidFieldMessage } from '../helpers'; @@ -50,6 +53,77 @@ function isSortableByColumn(layer: IndexPatternLayer, columnId: string) { ); } +function getDisallowedTermsMessage( + layer: IndexPatternLayer, + columnId: string, + state?: IndexPatternPrivateState, + layerId?: string +) { + const hasMultipleShifts = + uniq(Object.values(layer.columns).map((col) => col.timeShift !== '')).length > 1; + if (!hasMultipleShifts) { + return undefined; + } + const dateHistogramParent = layer.columnOrder + .slice(0, layer.columnOrder.indexOf(columnId)) + .find((colId) => layer.columns[colId].operationType === 'date_histogram'); + + if (!dateHistogramParent) { + return undefined; + } + return { + message: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShifts', { + defaultMessage: + "Can't use multiple time shifts in a single layer together with dynamic top values. Either use the same time shift for all metrics or use filters instead of top values.", + }), + fixAction: + state && layerId + ? { + label: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShiftsFixActionLabel', { + defaultMessage: 'Pin top values', + }), + newState: async (http: HttpSetup) => { + const indexPattern = state.indexPatterns[layer.indexPatternId]; + const fieldName = (layer.columns[columnId] as TermsIndexPatternColumn).sourceField; + const response: FieldStatsResponse = await http.post( + `/api/lens/index_stats/${indexPattern.id}/field`, + { + body: JSON.stringify({ + fieldName, + }), + } + ); + return { + ...state, + layers: { + ...state.layers, + [layerId]: { + ...layer, + columns: { + ...layer.columns, + [columnId]: { + ...layer.columns[columnId], + operationType: 'filters', + params: { + filters: response.topValues?.buckets.map(({ key }) => ({ + input: { + query: `${fieldName}: "${key}"`, + language: 'kuery', + }, + label: '', + })), + }, + } as FiltersIndexPatternColumn, + }, + }, + }, + }; + }, + } + : undefined, + }; +} + function getInvalidNestingOrderMessage( layer: IndexPatternLayer, columnId: string, @@ -87,7 +161,7 @@ function getInvalidNestingOrderMessage( defaultMessage: 'Reorder dimensions', } ), - newState: { + newState: async () => ({ ...state, layers: { ...state.layers, @@ -100,7 +174,7 @@ function getInvalidNestingOrderMessage( }, }, }, - }, + }), } : undefined, }; @@ -152,6 +226,7 @@ export const termsOperation: OperationDefinition { const newField = newIndexPattern.getFieldByName(column.sourceField); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 8ef6831890cbd9..8544bc7c2f1080 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -6,6 +6,7 @@ */ import _, { partition } from 'lodash'; +import { HttpSetup } from 'kibana/public'; import type { OperationMetadata, VisualizationDimensionGroupConfig } from '../../types'; import { operationDefinitionMap, @@ -993,15 +994,29 @@ export function getErrorMessages( layer: IndexPatternLayer, indexPattern: IndexPattern, state: IndexPatternPrivateState, - layerId: string + layerId: string, + http: HttpSetup ): | Array< | string - | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } + | { + message: string; + fixAction?: { + label: string; + newState: (http: HttpSetup) => Promise; + }; + } > | undefined { const errors: Array< - string | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } + | string + | { + message: string; + fixAction?: { + label: string; + newState: () => Promise; + }; + } > = Object.entries(layer.columns) .flatMap(([columnId, column]) => { const def = operationDefinitionMap[column.operationType]; @@ -1009,17 +1024,36 @@ export function getErrorMessages( return def.getErrorMessage(layer, columnId, indexPattern, state, layerId); } }) + .map((errorMessage) => { + if (typeof errorMessage !== 'object') { + return errorMessage; + } + return { + ...errorMessage, + fixAction: errorMessage.fixAction + ? { + ...errorMessage.fixAction, + newState: () => errorMessage.fixAction!.newState(http), + } + : undefined, + }; + }) // remove the undefined values .filter( ( v: | string | undefined - | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } + | { + message: string; + fixAction?: { label: string; newState: () => Promise }; + } ): v is | string - | { message: string; fixAction?: { label: string; newState: IndexPatternPrivateState } } => - v != null + | { + message: string; + fixAction?: { label: string; newState: () => Promise }; + } => v != null ); return errors.length ? errors : undefined; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts index 3734f18eb8a59b..179f8b2f1c5f28 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { ShiftError } from '../../../../../src/plugins/data/common'; import { DataType } from '../types'; import { IndexPattern, IndexPatternLayer, DraggedField } from './types'; import type { @@ -69,14 +68,7 @@ export function isColumnInvalid( indexPattern ); - const hasTimeShiftError = - runtimeError instanceof ShiftError && runtimeError.aggIds.includes(columnId); - - return ( - (operationErrorMessages && operationErrorMessages.length > 0) || - referencesHaveErrors || - hasTimeShiftError - ); + return (operationErrorMessages && operationErrorMessages.length > 0) || referencesHaveErrors; } function getReferencesErrors( diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 2a7e261f668718..01ac323d88b337 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -240,7 +240,7 @@ export interface Datasource { | Array<{ shortMessage: string; longMessage: string; - fixAction?: { label: string; newState: T }; + fixAction?: { label: string; newState: () => Promise }; }> | undefined; /** From 5603bb922f45394ad94f3d9bde3231f4ccb3baf5 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 5 May 2021 09:33:29 +0200 Subject: [PATCH 08/31] cleanup --- .../data/common/search/aggs/agg_configs.ts | 25 --------- .../data/common/search/aggs/utils/index.ts | 1 - .../common/search/aggs/utils/shift_error.ts | 20 -------- .../workspace_panel/workspace_panel.tsx | 8 +-- .../indexpattern_datasource/indexpattern.tsx | 2 +- .../operations/definitions/index.ts | 14 +++-- .../operations/definitions/terms/index.tsx | 51 ++++++++++++------- .../operations/layer_helpers.ts | 51 ++++++++----------- x-pack/plugins/lens/public/types.ts | 6 ++- 9 files changed, 75 insertions(+), 103 deletions(-) delete mode 100644 src/plugins/data/common/search/aggs/utils/shift_error.ts diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index 24fe5070514db8..8dbde8b9825866 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -30,7 +30,6 @@ import { AggTypesRegistryStart } from './agg_types_registry'; import { AggGroupNames } from './agg_groups'; import { IndexPattern } from '../../index_patterns/index_patterns/index_pattern'; import { TimeRange, getTime, isRangeFilter } from '../../../common'; -import { ShiftError } from './utils'; function removeParentAggs(obj: any) { for (const prop in obj) { @@ -298,31 +297,7 @@ export class AggConfigs { }, }, }; - const timeShiftInterval = config.getTimeShiftInterval(); Object.entries(timeShifts).forEach(([key, shift]) => { - if (timeShiftInterval && timeShiftInterval.asMilliseconds() > shift.asMilliseconds()) { - const aggIds = this.getAll() - .filter((agg) => agg.getTimeShift()?.asMilliseconds() === shift.asMilliseconds()) - .map((agg) => agg.id); - throw new ShiftError( - `All time shifts need to be larger than underlying date interval of ${timeShiftInterval.humanize()}`, - aggIds, - 'tooSmall' - ); - } - if ( - timeShiftInterval && - !Number.isInteger(shift.asMilliseconds() / timeShiftInterval.asMilliseconds()) - ) { - const aggIds = this.getAll() - .filter((agg) => agg.getTimeShift()?.asMilliseconds() === shift.asMilliseconds()) - .map((agg) => agg.id); - throw new ShiftError( - `All time shifts need to be a multiple of the underlying interval of ${timeShiftInterval.humanize()}`, - aggIds, - 'notAMultiple' - ); - } filters[key] = { range: { [timeField]: { diff --git a/src/plugins/data/common/search/aggs/utils/index.ts b/src/plugins/data/common/search/aggs/utils/index.ts index 8649686df37364..1f2d0a308f0dd2 100644 --- a/src/plugins/data/common/search/aggs/utils/index.ts +++ b/src/plugins/data/common/search/aggs/utils/index.ts @@ -16,4 +16,3 @@ export * from './prop_filter'; export * from './to_angular_json'; export * from './infer_time_zone'; export * from './parse_time_shift'; -export * from './shift_error'; diff --git a/src/plugins/data/common/search/aggs/utils/shift_error.ts b/src/plugins/data/common/search/aggs/utils/shift_error.ts deleted file mode 100644 index 49f4f3714f5e92..00000000000000 --- a/src/plugins/data/common/search/aggs/utils/shift_error.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -/** - * Configuration error of time shift - */ -export class ShiftError extends Error { - constructor( - message: string, - public aggIds: string[], - public reason: 'tooSmall' | 'notAMultiple' - ) { - super(message); - } -} diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx index d563cd05ffd56c..830fff3acddde6 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx @@ -84,7 +84,7 @@ interface WorkspaceState { expressionBuildError?: Array<{ shortMessage: string; longMessage: string; - fixAction?: { label: string; newState: () => Promise }; + fixAction?: { label: string; newState: (framePublicAPI: FramePublicAPI) => Promise }; }>; expandError: boolean; } @@ -399,7 +399,7 @@ export const VisualizationWrapper = ({ configurationValidationError?: Array<{ shortMessage: string; longMessage: string; - fixAction?: { label: string; newState: () => Promise }; + fixAction?: { label: string; newState: (framePublicAPI: FramePublicAPI) => Promise }; }>; missingRefsErrors?: Array<{ shortMessage: string; longMessage: string }>; }; @@ -487,7 +487,9 @@ export const VisualizationWrapper = ({ {localState.configurationValidationError[0].fixAction && activeDatasourceId && ( { - const newState = await localState.configurationValidationError?.[0].fixAction?.newState(); + const newState = await localState.configurationValidationError?.[0].fixAction?.newState( + framePublicAPI + ); dispatch({ type: 'UPDATE_DATASOURCE_STATE', datasourceId: activeDatasourceId, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index 8ed6c2c3126f4e..01dae7df64c83a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -381,7 +381,7 @@ export function getIndexPatternDatasource({ state.indexPatterns[layer.indexPatternId], state, layerId, - core.http + core ) ?? [] ).map((message) => ({ shortMessage: '', // Not displayed currently diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index c89ada8c7a6703..9421433eaf780b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { IUiSettingsClient, SavedObjectsClientContract, HttpSetup } from 'kibana/public'; +import { IUiSettingsClient, SavedObjectsClientContract, HttpSetup, CoreStart } from 'kibana/public'; import { IStorageWrapper } from 'src/plugins/kibana_utils/public'; import { termsOperation, TermsIndexPatternColumn } from './terms'; import { filtersOperation, FiltersIndexPatternColumn } from './filters'; @@ -36,7 +36,7 @@ import { } from './calculations'; import { countOperation, CountIndexPatternColumn } from './count'; import { lastValueOperation, LastValueIndexPatternColumn } from './last_value'; -import { OperationMetadata } from '../../../types'; +import { FramePublicAPI, OperationMetadata } from '../../../types'; import type { BaseIndexPatternColumn, ReferenceBasedIndexPatternColumn } from './column_types'; import { IndexPattern, @@ -238,7 +238,10 @@ interface BaseOperationDefinitionProps { message: string; fixAction?: { label: string; - newState: (http: HttpSetup) => Promise; + newState: ( + core: CoreStart, + frame: FramePublicAPI + ) => Promise; }; } > @@ -352,7 +355,10 @@ interface FieldBasedOperationDefinition { message: string; fixAction?: { label: string; - newState: (http: HttpSetup) => Promise; + newState: ( + core: CoreStart, + frame: FramePublicAPI + ) => Promise; }; } > diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx index 8b0822a93bc303..9f42ba4daaf476 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx @@ -17,13 +17,17 @@ import { EuiIconTip, } from '@elastic/eui'; import { uniq } from 'lodash'; -import { HttpSetup } from 'kibana/public'; +import { CoreStart } from 'kibana/public'; import { FieldStatsResponse } from '../../../../../common'; -import { AggFunctionsMapping } from '../../../../../../../../src/plugins/data/public'; +import { + AggFunctionsMapping, + esQuery, + IIndexPattern, +} from '../../../../../../../../src/plugins/data/public'; import { buildExpressionFunction } from '../../../../../../../../src/plugins/expressions/public'; import { updateColumnParam, isReferenced } from '../../layer_helpers'; -import { DataType } from '../../../../types'; -import { FiltersIndexPatternColumn, OperationDefinition } from '../index'; +import { DataType, FramePublicAPI } from '../../../../types'; +import { FiltersIndexPatternColumn, OperationDefinition, operationDefinitionMap } from '../index'; import { FieldBasedIndexPatternColumn } from '../column_types'; import { ValuesInput } from './values_input'; import { getInvalidFieldMessage } from '../helpers'; @@ -60,17 +64,14 @@ function getDisallowedTermsMessage( layerId?: string ) { const hasMultipleShifts = - uniq(Object.values(layer.columns).map((col) => col.timeShift !== '')).length > 1; + uniq( + Object.values(layer.columns) + .filter((col) => operationDefinitionMap[col.operationType].shiftable) + .map((col) => col.timeShift || '') + ).length > 1; if (!hasMultipleShifts) { return undefined; } - const dateHistogramParent = layer.columnOrder - .slice(0, layer.columnOrder.indexOf(columnId)) - .find((colId) => layer.columns[colId].operationType === 'date_histogram'); - - if (!dateHistogramParent) { - return undefined; - } return { message: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShifts', { defaultMessage: @@ -80,16 +81,24 @@ function getDisallowedTermsMessage( state && layerId ? { label: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShiftsFixActionLabel', { - defaultMessage: 'Pin top values', + defaultMessage: 'Pin current top values', }), - newState: async (http: HttpSetup) => { + newState: async (core: CoreStart, frame: FramePublicAPI) => { const indexPattern = state.indexPatterns[layer.indexPatternId]; const fieldName = (layer.columns[columnId] as TermsIndexPatternColumn).sourceField; - const response: FieldStatsResponse = await http.post( + const response: FieldStatsResponse = await core.http.post( `/api/lens/index_stats/${indexPattern.id}/field`, { body: JSON.stringify({ fieldName, + dslQuery: esQuery.buildEsQuery( + indexPattern as IIndexPattern, + frame.query, + frame.filters, + esQuery.getEsQueryConfig(core.uiSettings) + ), + fromDate: frame.dateRange.fromDate, + toDate: frame.dateRange.toDate, }), } ); @@ -102,7 +111,15 @@ function getDisallowedTermsMessage( columns: { ...layer.columns, [columnId]: { - ...layer.columns[columnId], + label: i18n.translate('xpack.lens.indexPattern.pinnedTopValuesLabel', { + defaultMessage: 'Pinned top values of {field}', + values: { + field: fieldName, + }, + }), + customLabel: true, + isBucketed: layer.columns[columnId].isBucketed, + dataType: 'string', operationType: 'filters', params: { filters: response.topValues?.buckets.map(({ key }) => ({ @@ -110,7 +127,7 @@ function getDisallowedTermsMessage( query: `${fieldName}: "${key}"`, language: 'kuery', }, - label: '', + label: String(key), })), }, } as FiltersIndexPatternColumn, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 787139c75a328e..ffac563eeeea8a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -6,8 +6,12 @@ */ import _, { partition } from 'lodash'; -import { HttpSetup } from 'kibana/public'; -import type { OperationMetadata, VisualizationDimensionGroupConfig } from '../../types'; +import { CoreStart } from 'kibana/public'; +import type { + FramePublicAPI, + OperationMetadata, + VisualizationDimensionGroupConfig, +} from '../../types'; import { operationDefinitionMap, operationDefinitions, @@ -1075,7 +1079,7 @@ export function getErrorMessages( indexPattern: IndexPattern, state: IndexPatternPrivateState, layerId: string, - http: HttpSetup + core: CoreStart ): | Array< | string @@ -1083,21 +1087,12 @@ export function getErrorMessages( message: string; fixAction?: { label: string; - newState: (http: HttpSetup) => Promise; + newState: (frame: FramePublicAPI) => Promise; }; } > | undefined { - const errors: Array< - | string - | { - message: string; - fixAction?: { - label: string; - newState: () => Promise; - }; - } - > = Object.entries(layer.columns) + const errors = Object.entries(layer.columns) .flatMap(([columnId, column]) => { const def = operationDefinitionMap[column.operationType]; if (def.getErrorMessage) { @@ -1113,28 +1108,22 @@ export function getErrorMessages( fixAction: errorMessage.fixAction ? { ...errorMessage.fixAction, - newState: () => errorMessage.fixAction!.newState(http), + newState: (frame: FramePublicAPI) => errorMessage.fixAction!.newState(core, frame), } : undefined, }; }) // remove the undefined values - .filter( - ( - v: - | string - | undefined - | { - message: string; - fixAction?: { label: string; newState: () => Promise }; - } - ): v is - | string - | { - message: string; - fixAction?: { label: string; newState: () => Promise }; - } => v != null - ); + .filter((v) => v != null) as Array< + | string + | { + message: string; + fixAction?: { + label: string; + newState: (framePublicAPI: FramePublicAPI) => Promise; + }; + } + >; return errors.length ? errors : undefined; } diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 01ac323d88b337..5516f16e0bbaed 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -235,7 +235,11 @@ export interface Datasource { getPublicAPI: (props: PublicAPIProps) => DatasourcePublicAPI; getErrorMessages: ( state: T, - layersGroups?: Record + layersGroups?: Record, + dateRange?: { + fromDate: string; + toDate: string; + } ) => | Array<{ shortMessage: string; From 8f6b95c369895f98dc198523d647da0641a6b823 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 5 May 2021 09:54:55 +0200 Subject: [PATCH 09/31] remove current runtime error thingy --- .../workspace_panel/workspace_panel.tsx | 11 ---------- .../dimension_panel/dimension_editor.tsx | 2 -- .../dimension_panel/dimension_panel.tsx | 5 ++--- .../dimension_panel/time_shift.tsx | 22 +++---------------- .../public/indexpattern_datasource/utils.ts | 3 +-- x-pack/plugins/lens/public/types.ts | 3 --- 6 files changed, 6 insertions(+), 40 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx index 830fff3acddde6..585ac73bfea9b6 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx @@ -75,7 +75,6 @@ export interface WorkspacePanelProps { core: CoreStart; plugins: { uiActions?: UiActionsStart; data: DataPublicPluginStart }; title?: string; - hasRuntimeError?: boolean; visualizeTriggerFieldContext?: VisualizeFieldContext; getSuggestionForField: (field: DragDropIdentifier) => Suggestion | undefined; } @@ -133,7 +132,6 @@ export const InnerWorkspacePanel = React.memo(function InnerWorkspacePanel({ title, visualizeTriggerFieldContext, suggestionForDraggedField, - hasRuntimeError, }: Omit & { suggestionForDraggedField: Suggestion | undefined; }) { @@ -341,7 +339,6 @@ export const InnerWorkspacePanel = React.memo(function InnerWorkspacePanel({ localState={{ ...localState, configurationValidationError, missingRefsErrors }} ExpressionRendererComponent={ExpressionRendererComponent} application={core.application} - hasRuntimeError={hasRuntimeError} activeDatasourceId={activeDatasourceId} /> ); @@ -386,7 +383,6 @@ export const VisualizationWrapper = ({ ExpressionRendererComponent, dispatch, application, - hasRuntimeError, activeDatasourceId, }: { expression: string | null | undefined; @@ -405,7 +401,6 @@ export const VisualizationWrapper = ({ }; ExpressionRendererComponent: ReactExpressionRendererType; application: ApplicationStart; - hasRuntimeError?: boolean; activeDatasourceId: string | null; }) => { const context: ExecutionContextSearch = useMemo( @@ -596,12 +591,6 @@ export const VisualizationWrapper = ({ onData$={onData$} renderMode="edit" renderError={(errorMessage?: string | null, error?: ExpressionRenderError | null) => { - if (!hasRuntimeError && error?.original) { - dispatch({ - type: 'RUNTIME_ERROR', - error: error.original, - }); - } const errorsFromRequest = getOriginalRequestErrorMessages(error); const visibleErrorMessages = errorsFromRequest.length ? errorsFromRequest diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index e51c4f2c02f1b0..a469924fac0543 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -120,7 +120,6 @@ export function DimensionEditor(props: DimensionEditorProps) { hideGrouping, dateRange, dimensionGroups, - runtimeError, } = props; const services = { data: props.data, @@ -537,7 +536,6 @@ export function DimensionEditor(props: DimensionEditorProps) { isFocused={timeShiftedFocused} activeData={props.activeData} layerId={layerId} - runtimeError={runtimeError} /> ) : null, }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.tsx index ccf5f53d6c9d5f..3965f992805b5d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.tsx @@ -44,15 +44,14 @@ function wrapOnDot(str?: string) { export const IndexPatternDimensionTriggerComponent = function IndexPatternDimensionTrigger( props: IndexPatternDimensionTriggerProps ) { - const runtimeError = props.runtimeError; const layerId = props.layerId; const layer = props.state.layers[layerId]; const currentIndexPattern = props.state.indexPatterns[layer.indexPatternId]; const { columnId, uniqueLabel } = props; const currentColumnHasErrors = useMemo( - () => isColumnInvalid(layer, columnId, currentIndexPattern, runtimeError), - [layer, columnId, currentIndexPattern, runtimeError] + () => isColumnInvalid(layer, columnId, currentIndexPattern), + [layer, columnId, currentIndexPattern] ); const selectedColumn: IndexPatternColumn | null = layer.columns[props.columnId] ?? null; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index d4e58ee77b73b2..534051ac02c162 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -12,7 +12,7 @@ import { i18n } from '@kbn/i18n'; import React, { useEffect, useState } from 'react'; import { Query } from 'src/plugins/data/public'; import { search } from '../../../../../../src/plugins/data/public'; -import { parseTimeShift, ShiftError } from '../../../../../../src/plugins/data/common'; +import { parseTimeShift } from '../../../../../../src/plugins/data/common'; import { IndexPatternColumn, operationDefinitionMap } from '../operations'; import { IndexPattern, IndexPatternLayer } from '../types'; import { IndexPatternDimensionEditorProps } from './dimension_panel'; @@ -118,7 +118,6 @@ export function TimeShift({ isFocused, activeData, layerId, - runtimeError, }: { selectedColumn: IndexPatternColumn; indexPattern: IndexPattern; @@ -128,7 +127,6 @@ export function TimeShift({ isFocused: boolean; activeData: IndexPatternDimensionEditorProps['activeData']; layerId: string; - runtimeError?: Error | undefined; }) { const [localValue, setLocalValue] = useState(selectedColumn.timeShift); useEffect(() => { @@ -170,23 +168,9 @@ export function TimeShift({ ); } - function getRuntimeErrorReason() { - if (!(runtimeError instanceof ShiftError)) { - return undefined; - } - if (!runtimeError.aggIds.includes(columnId)) { - return undefined; - } - return runtimeError.reason; - } - const parsedLocalValue = localValue && parseTimeShift(localValue); - const localValueTooSmall = - (parsedLocalValue && isValueTooSmall(parsedLocalValue)) || - getRuntimeErrorReason() === 'tooSmall'; - const localValueNotMultiple = - (parsedLocalValue && isValueNotMultiple(parsedLocalValue)) || - getRuntimeErrorReason() === 'notAMultiple'; + const localValueTooSmall = parsedLocalValue && isValueTooSmall(parsedLocalValue); + const localValueNotMultiple = parsedLocalValue && isValueNotMultiple(parsedLocalValue); const isLocalValueInvalid = Boolean( parsedLocalValue === 'invalid' || localValueTooSmall || localValueNotMultiple ); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts index 179f8b2f1c5f28..43e386cff10a3e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts @@ -49,8 +49,7 @@ export function isDraggedField(fieldCandidate: unknown): fieldCandidate is Dragg export function isColumnInvalid( layer: IndexPatternLayer, columnId: string, - indexPattern: IndexPattern, - runtimeError?: Error + indexPattern: IndexPattern ) { const column: IndexPatternColumn | undefined = layer.columns[columnId]; if (!column) return; diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 5516f16e0bbaed..4289366c2d3990 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -299,7 +299,6 @@ export type DatasourceDimensionProps = SharedDimensionProps & { onRemove?: (accessor: string) => void; state: T; activeData?: Record; - runtimeError?: Error; }; // The only way a visualization has to restrict the query building @@ -321,7 +320,6 @@ export interface DatasourceLayerPanelProps { state: T; setState: StateSetter; activeData?: Record; - runtimeError?: Error; } export interface DraggedOperation extends DraggingIdentifier { @@ -522,7 +520,6 @@ export interface FramePublicAPI { * If accessing, make sure to check whether expected columns actually exist. */ activeData?: Record; - runtimeError?: Error; dateRange: DateRange; query: Query; From 0d2621e7b0e3764cd0ef9276a98320249d8c5a20 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 5 May 2021 12:00:41 +0200 Subject: [PATCH 10/31] remove leftover runtime error stuff --- .../editor_frame/config_panel/layer_panel.tsx | 2 -- .../editor_frame_service/editor_frame/editor_frame.tsx | 2 -- .../editor_frame_service/editor_frame/state_management.ts | 8 -------- 3 files changed, 12 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx index 3b6e2b2a3eb02c..cf3c9099d4b0dd 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx @@ -357,7 +357,6 @@ export function LayerPanel( columnId: accessorConfig.columnId, groupId: group.groupId, filterOperations: group.filterOperations, - runtimeError: props.framePublicAPI.runtimeError, }} /> @@ -434,7 +433,6 @@ export function LayerPanel( hideGrouping: activeGroup.hideGrouping, filterOperations: activeGroup.filterOperations, dimensionGroups: groups, - runtimeError: props.framePublicAPI.runtimeError, setState: ( newState: unknown, { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx index 642ef76da4f365..362787ea91c4fb 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx @@ -110,7 +110,6 @@ export function EditorFrame(props: EditorFrameProps) { const framePublicAPI: FramePublicAPI = { datasourceLayers, activeData: state.activeData, - runtimeError: state.lastRuntimeError, dateRange: props.dateRange, query: props.query, filters: props.filters, @@ -367,7 +366,6 @@ export function EditorFrame(props: EditorFrameProps) { plugins={props.plugins} visualizeTriggerFieldContext={visualizeTriggerFieldContext} getSuggestionForField={getSuggestionForField} - hasRuntimeError={Boolean(state.lastRuntimeError)} /> ) } diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts index d49ee4cec46f3d..53aba0d6f3f6c1 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_management.ts @@ -24,7 +24,6 @@ export interface EditorFrameState extends PreviewState { stagedPreview?: PreviewState; activeDatasourceId: string | null; activeData?: TableInspectorAdapter; - lastRuntimeError?: Error; } export type Action = @@ -32,10 +31,6 @@ export type Action = type: 'RESET'; state: EditorFrameState; } - | { - type: 'RUNTIME_ERROR'; - error: Error; - } | { type: 'UPDATE_TITLE'; title: string; @@ -149,14 +144,11 @@ export const reducer = (state: EditorFrameState, action: Action): EditorFrameSta return action.state; case 'UPDATE_TITLE': return { ...state, title: action.title }; - case 'RUNTIME_ERROR': - return { ...state, lastRuntimeError: action.error }; case 'UPDATE_STATE': return action.updater(state); case 'UPDATE_ACTIVE_DATA': return { ...state, - lastRuntimeError: undefined, activeData: { ...action.tables }, }; case 'UPDATE_LAYER': From 3439d351c28dbbc576724b95115a0f297904234e Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 6 May 2021 18:08:50 +0200 Subject: [PATCH 11/31] fix some casest push --- .../data/common/search/aggs/agg_configs.ts | 118 ++++++------ .../buckets/_terms_other_bucket_helper.ts | 2 +- .../data/common/search/aggs/buckets/terms.ts | 8 +- .../operations/definitions/index.ts | 26 +-- .../operations/definitions/terms/index.tsx | 170 ++++++------------ .../operations/layer_helpers.ts | 10 +- 6 files changed, 124 insertions(+), 210 deletions(-) diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index 8dbde8b9825866..180113b1b31d86 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -199,9 +199,12 @@ export class AggConfigs { let nestedMetrics: Array<{ config: AggConfig; dsl: Record }> | []; const timeShifts = this.getTimeShifts(); - const hasTimeShifts = Object.keys(timeShifts).length > 0; + const hasMultipleTimeShifts = Object.keys(timeShifts).length > 1; if (this.hierarchical) { + if (hasMultipleTimeShifts) { + throw new Error('Multiple time shifts not supported for hierarchical metrics'); + } // collect all metrics, and filter out the ones that we won't be copying nestedMetrics = this.aggs .filter(function (agg) { @@ -228,7 +231,7 @@ export class AggConfigs { dslLvlCursor = prevDsl?.aggs || dslLvlCursor; } - if (hasTimeShifts) { + if (hasMultipleTimeShifts) { dslLvlCursor = this.insertTimeShiftSplit(config, timeShifts, dslLvlCursor); } @@ -288,15 +291,6 @@ export class AggConfigs { const timeRange = this.timeRange; const filters: Record = {}; const timeField = this.timeFields[0]; - filters['0'] = { - range: { - [timeField]: { - // only works if there is a time range - gte: timeRange.from, - lte: timeRange.to, - }, - }, - }; Object.entries(timeShifts).forEach(([key, shift]) => { filters[key] = { range: { @@ -368,6 +362,8 @@ export class AggConfigs { .forEach((timeShift) => { if (timeShift) { timeShifts[String(timeShift.asMilliseconds())] = timeShift; + } else { + timeShifts[0] = moment.duration(0); } }); return timeShifts; @@ -389,8 +385,7 @@ export class AggConfigs { const timeRange = this.timeRange; const timeFields = this.timeFields; const timeShifts = this.getTimeShifts(); - const hasTimeShift = Object.values(this.getTimeShifts()).length > 0; - if (!hasTimeShift) { + if (!this.hasTimeShifts()) { return this.timeFields .map((fieldName) => getTime(this.indexPattern, timeRange, { fieldName, forceNow })) .filter(isRangeFilter); @@ -406,43 +401,29 @@ export class AggConfigs { }, query: { bool: { - should: [ - ...Object.entries(timeShifts).map(([, shift]) => { - return { - bool: { - filter: timeFields - .map( - (fieldName) => - [ - getTime(this.indexPattern, timeRange, { fieldName, forceNow }), - fieldName, - ] as [RangeFilter | undefined, string] - ) - .filter(([filter]) => isRangeFilter(filter)) - .map(([filter, field]) => ({ - range: { - [field]: { - gte: moment(filter?.range[field].gte).subtract(shift).toISOString(), - lte: moment(filter?.range[field].lte).subtract(shift).toISOString(), - }, - }, - })), - }, - }; - }), - { + should: Object.entries(timeShifts).map(([, shift]) => { + return { bool: { filter: timeFields - .map((fieldName) => - getTime(this.indexPattern, timeRange, { fieldName, forceNow }) + .map( + (fieldName) => + [ + getTime(this.indexPattern, timeRange, { fieldName, forceNow }), + fieldName, + ] as [RangeFilter | undefined, string] ) - .filter(isRangeFilter) - .map((filter) => ({ - range: filter.range, + .filter(([filter]) => isRangeFilter(filter)) + .map(([filter, field]) => ({ + range: { + [field]: { + gte: moment(filter?.range[field].gte).subtract(shift).toISOString(), + lte: moment(filter?.range[field].lte).subtract(shift).toISOString(), + }, + }, })), }, - }, - ], + }; + }), minimum_should_match: 1, }, }, @@ -451,10 +432,11 @@ export class AggConfigs { } postFlightTransform(response: IEsSearchResponse) { - const timeShifts = this.getTimeShifts(); - if (Object.keys(timeShifts).length === 0) { + if (!this.hasTimeShifts()) { return response; } + const timeShifts = this.getTimeShifts(); + const hasMultipleTimeShifts = Object.keys(timeShifts).length > 1; const transformedRawResponse = cloneDeep(response.rawResponse); const aggCursor = transformedRawResponse.aggregations!; @@ -534,23 +516,33 @@ export class AggConfigs { }); }; const transformTimeShift = (cursor: Record, aggIndex: number): undefined => { - if (cursor.time_offset_split) { - const timeShiftedBuckets = (cursor.time_offset_split as FiltersAggregate).buckets as Record< - string, - FiltersBucketItem - >; - const subTree = timeShiftedBuckets['0']; - Object.entries(timeShifts).forEach(([key, shift]) => { - mergeAggLevel( - subTree as GenericBucket, - timeShiftedBuckets[key] as GenericBucket, - shift, - aggIndex - ); - }); + const shouldSplit = bucketAggs[aggIndex].splitForTimeShift(this); + if (shouldSplit) { + // multiple time shifts caused a filters agg in the tree we have to merge + if (hasMultipleTimeShifts && cursor.time_offset_split) { + const timeShiftedBuckets = (cursor.time_offset_split as FiltersAggregate) + .buckets as Record; + const subTree = timeShiftedBuckets['0'] || {}; + Object.entries(timeShifts) + .filter(([key]) => key !== '0') + .forEach(([key, shift]) => { + mergeAggLevel( + subTree as GenericBucket, + timeShiftedBuckets[key] as GenericBucket, + shift, + aggIndex + ); + }); - delete cursor.time_offset_split; - Object.assign(cursor, subTree); + delete cursor.time_offset_split; + Object.assign(cursor, subTree); + } else { + // otherwise we have to "merge" a single level to shift all keys + const [[, shift]] = Object.entries(timeShifts); + const subTree = {}; + mergeAggLevel(subTree, cursor, shift, aggIndex); + Object.assign(cursor, subTree); + } return; } // recurse deeper into the response object diff --git a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts index 6230ae897b1702..372d487bcf7a39 100644 --- a/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts +++ b/src/plugins/data/common/search/aggs/buckets/_terms_other_bucket_helper.ts @@ -166,7 +166,7 @@ export const buildOtherBucketAgg = ( key: string ) => { // make sure there are actually results for the buckets - if (aggregations[aggId].buckets.length < 1) { + if (aggregations[aggId]?.buckets.length < 1) { noAggBucketResults = true; return; } diff --git a/src/plugins/data/common/search/aggs/buckets/terms.ts b/src/plugins/data/common/search/aggs/buckets/terms.ts index 44088ecb49db6c..7356d434463581 100644 --- a/src/plugins/data/common/search/aggs/buckets/terms.ts +++ b/src/plugins/data/common/search/aggs/buckets/terms.ts @@ -180,12 +180,12 @@ export const getTermsBucketAgg = () => return; } - if (aggs?.hasTimeShifts()) { + if (aggs?.hasTimeShifts() && aggs.timeRange) { const shift = orderAgg.getTimeShift(); orderAgg = aggs.createAggConfig( { type: 'filtered_metric', - id: 'shifted-order', + id: orderAgg.id, params: { customBucket: aggs .createAggConfig( @@ -198,10 +198,10 @@ export const getTermsBucketAgg = () => query: { range: { [aggs.timeFields![0]]: { - from: moment(aggs.timeRange!.from) + gte: moment(aggs.timeRange.from) .subtract(shift || 0) .toISOString(), - to: moment(aggs.timeRange!.to) + lte: moment(aggs.timeRange.to) .subtract(shift || 0) .toISOString(), }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 9421433eaf780b..cefe4f1940e16f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -38,12 +38,7 @@ import { countOperation, CountIndexPatternColumn } from './count'; import { lastValueOperation, LastValueIndexPatternColumn } from './last_value'; import { FramePublicAPI, OperationMetadata } from '../../../types'; import type { BaseIndexPatternColumn, ReferenceBasedIndexPatternColumn } from './column_types'; -import { - IndexPattern, - IndexPatternField, - IndexPatternLayer, - IndexPatternPrivateState, -} from '../../types'; +import { IndexPattern, IndexPatternField, IndexPatternLayer } from '../../types'; import { DateRange } from '../../../../common'; import { ExpressionAstFunction } from '../../../../../../../src/plugins/expressions/public'; import { DataPublicPluginStart } from '../../../../../../../src/plugins/data/public'; @@ -227,10 +222,7 @@ interface BaseOperationDefinitionProps { getErrorMessage?: ( layer: IndexPatternLayer, columnId: string, - indexPattern: IndexPattern, - // TODO - remove these again and remap the state on a higher level - state?: IndexPatternPrivateState, - layerId?: string + indexPattern: IndexPattern ) => | Array< | string @@ -238,10 +230,7 @@ interface BaseOperationDefinitionProps { message: string; fixAction?: { label: string; - newState: ( - core: CoreStart, - frame: FramePublicAPI - ) => Promise; + newState: (core: CoreStart, frame: FramePublicAPI) => Promise; }; } > @@ -345,9 +334,7 @@ interface FieldBasedOperationDefinition { getErrorMessage?: ( layer: IndexPatternLayer, columnId: string, - indexPattern: IndexPattern, - state?: IndexPatternPrivateState, - layerId?: string + indexPattern: IndexPattern ) => | Array< | string @@ -355,10 +342,7 @@ interface FieldBasedOperationDefinition { message: string; fixAction?: { label: string; - newState: ( - core: CoreStart, - frame: FramePublicAPI - ) => Promise; + newState: (core: CoreStart, frame: FramePublicAPI) => Promise; }; } > diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx index 9f42ba4daaf476..78b3a13c60e5eb 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx @@ -31,7 +31,7 @@ import { FiltersIndexPatternColumn, OperationDefinition, operationDefinitionMap import { FieldBasedIndexPatternColumn } from '../column_types'; import { ValuesInput } from './values_input'; import { getInvalidFieldMessage } from '../helpers'; -import type { IndexPatternLayer, IndexPatternPrivateState } from '../../../types'; +import type { IndexPatternLayer, IndexPattern } from '../../../types'; function ofName(name?: string) { return i18n.translate('xpack.lens.indexPattern.termsOf', { @@ -60,8 +60,7 @@ function isSortableByColumn(layer: IndexPatternLayer, columnId: string) { function getDisallowedTermsMessage( layer: IndexPatternLayer, columnId: string, - state?: IndexPatternPrivateState, - layerId?: string + indexPattern: IndexPattern ) { const hasMultipleShifts = uniq( @@ -77,123 +76,57 @@ function getDisallowedTermsMessage( defaultMessage: "Can't use multiple time shifts in a single layer together with dynamic top values. Either use the same time shift for all metrics or use filters instead of top values.", }), - fixAction: - state && layerId - ? { - label: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShiftsFixActionLabel', { - defaultMessage: 'Pin current top values', + fixAction: { + label: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShiftsFixActionLabel', { + defaultMessage: 'Pin current top values', + }), + newState: async (core: CoreStart, frame: FramePublicAPI) => { + const fieldName = (layer.columns[columnId] as TermsIndexPatternColumn).sourceField; + const response: FieldStatsResponse = await core.http.post( + `/api/lens/index_stats/${indexPattern.id}/field`, + { + body: JSON.stringify({ + fieldName, + dslQuery: esQuery.buildEsQuery( + indexPattern as IIndexPattern, + frame.query, + frame.filters, + esQuery.getEsQueryConfig(core.uiSettings) + ), + fromDate: frame.dateRange.fromDate, + toDate: frame.dateRange.toDate, }), - newState: async (core: CoreStart, frame: FramePublicAPI) => { - const indexPattern = state.indexPatterns[layer.indexPatternId]; - const fieldName = (layer.columns[columnId] as TermsIndexPatternColumn).sourceField; - const response: FieldStatsResponse = await core.http.post( - `/api/lens/index_stats/${indexPattern.id}/field`, - { - body: JSON.stringify({ - fieldName, - dslQuery: esQuery.buildEsQuery( - indexPattern as IIndexPattern, - frame.query, - frame.filters, - esQuery.getEsQueryConfig(core.uiSettings) - ), - fromDate: frame.dateRange.fromDate, - toDate: frame.dateRange.toDate, - }), - } - ); - return { - ...state, - layers: { - ...state.layers, - [layerId]: { - ...layer, - columns: { - ...layer.columns, - [columnId]: { - label: i18n.translate('xpack.lens.indexPattern.pinnedTopValuesLabel', { - defaultMessage: 'Pinned top values of {field}', - values: { - field: fieldName, - }, - }), - customLabel: true, - isBucketed: layer.columns[columnId].isBucketed, - dataType: 'string', - operationType: 'filters', - params: { - filters: response.topValues?.buckets.map(({ key }) => ({ - input: { - query: `${fieldName}: "${key}"`, - language: 'kuery', - }, - label: String(key), - })), - }, - } as FiltersIndexPatternColumn, - }, - }, - }, - }; - }, } - : undefined, - }; -} - -function getInvalidNestingOrderMessage( - layer: IndexPatternLayer, - columnId: string, - state?: IndexPatternPrivateState, - layerId?: string -) { - const usesTimeShift = Object.values(layer.columns).some( - (col) => col.timeShift && col.timeShift !== '' - ); - if (!usesTimeShift) { - return undefined; - } - const dateHistogramParent = layer.columnOrder - .slice(0, layer.columnOrder.indexOf(columnId)) - .find((colId) => layer.columns[colId].operationType === 'date_histogram'); - - if (!dateHistogramParent) { - return undefined; - } - return { - message: i18n.translate('xpack.lens.indexPattern.termsInShiftedDateHistogramError', { - defaultMessage: - 'Date histogram "{dateLabel}" is grouped by before "{dimensionLabel}". When using time shifts, make sure to group by top values first.', - values: { - dateLabel: layer.columns[dateHistogramParent].label, - dimensionLabel: layer.columns[columnId].label, - }, - }), - fixAction: - state && layerId - ? { - label: i18n.translate( - 'xpack.lens.indexPattern.termsInShiftedDateHistogramFixActionLabel', - { - defaultMessage: 'Reorder dimensions', - } - ), - newState: async () => ({ - ...state, - layers: { - ...state.layers, - [layerId]: { - ...layer, - columns: { - ...layer.columns, - [dateHistogramParent]: layer.columns[columnId], - [columnId]: layer.columns[dateHistogramParent], - }, + ); + return { + ...layer, + columns: { + ...layer.columns, + [columnId]: { + label: i18n.translate('xpack.lens.indexPattern.pinnedTopValuesLabel', { + defaultMessage: 'Pinned top values of {field}', + values: { + field: fieldName, }, + }), + customLabel: true, + isBucketed: layer.columns[columnId].isBucketed, + dataType: 'string', + operationType: 'filters', + params: { + filters: response.topValues?.buckets.map(({ key }) => ({ + input: { + query: `${fieldName}: "${key}"`, + language: 'kuery', + }, + label: String(key), + })), }, - }), - } - : undefined, + } as FiltersIndexPatternColumn, + }, + }; + }, + }, }; } @@ -236,14 +169,13 @@ export const termsOperation: OperationDefinition + getErrorMessage: (layer, columnId, indexPattern) => [ ...(getInvalidFieldMessage( layer.columns[columnId] as FieldBasedIndexPatternColumn, indexPattern ) || []), - getInvalidNestingOrderMessage(layer, columnId, state, layerId) || '', - getDisallowedTermsMessage(layer, columnId, state, layerId) || '', + getDisallowedTermsMessage(layer, columnId, indexPattern) || '', ].filter(Boolean), isTransferable: (column, newIndexPattern) => { const newField = newIndexPattern.getFieldByName(column.sourceField); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index ffac563eeeea8a..29428116708665 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -1096,7 +1096,7 @@ export function getErrorMessages( .flatMap(([columnId, column]) => { const def = operationDefinitionMap[column.operationType]; if (def.getErrorMessage) { - return def.getErrorMessage(layer, columnId, indexPattern, state, layerId); + return def.getErrorMessage(layer, columnId, indexPattern); } }) .map((errorMessage) => { @@ -1108,7 +1108,13 @@ export function getErrorMessages( fixAction: errorMessage.fixAction ? { ...errorMessage.fixAction, - newState: (frame: FramePublicAPI) => errorMessage.fixAction!.newState(core, frame), + newState: async (frame: FramePublicAPI) => ({ + ...state, + layers: { + ...state.layers, + [layerId]: await errorMessage.fixAction!.newState(core, frame), + }, + }), } : undefined, }; From abeed2c157bf1f56e5d0080b9db9f98008fabfb9 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 11 May 2021 17:58:31 +0200 Subject: [PATCH 12/31] add tests and fix some bugs --- ...gins-data-public.aggconfig.gettimeshift.md | 15 + ...gins-data-public.aggconfig.hastimeshift.md | 15 + ...na-plugin-plugins-data-public.aggconfig.md | 2 + ...ic.aggconfigs.getsearchsourcetimefilter.md | 72 ++++ ...-public.aggconfigs.gettimeshiftinterval.md | 15 + ...ns-data-public.aggconfigs.gettimeshifts.md | 15 + ...ns-data-public.aggconfigs.hastimeshifts.md | 15 + ...a-plugin-plugins-data-public.aggconfigs.md | 5 + ...a-public.aggconfigs.postflighttransform.md | 22 ++ .../data/common/search/aggs/agg_config.ts | 22 +- .../common/search/aggs/agg_configs.test.ts | 338 ++++++++++++++++++ .../data/common/search/aggs/agg_configs.ts | 73 ++-- .../data/common/search/aggs/agg_type.ts | 19 +- .../search/aggs/buckets/bucket_agg_type.ts | 16 + .../common/search/aggs/metrics/avg_fn.test.ts | 1 + .../search/aggs/metrics/bucket_avg_fn.test.ts | 3 + .../search/aggs/metrics/bucket_avg_fn.ts | 7 + .../search/aggs/metrics/bucket_max_fn.test.ts | 3 + .../search/aggs/metrics/bucket_max_fn.ts | 7 + .../search/aggs/metrics/bucket_min_fn.test.ts | 3 + .../search/aggs/metrics/bucket_min_fn.ts | 7 + .../search/aggs/metrics/bucket_sum_fn.test.ts | 3 + .../search/aggs/metrics/bucket_sum_fn.ts | 7 + .../aggs/metrics/cardinality_fn.test.ts | 1 + .../search/aggs/metrics/count_fn.test.ts | 1 + .../aggs/metrics/cumulative_sum_fn.test.ts | 4 + .../search/aggs/metrics/cumulative_sum_fn.ts | 7 + .../search/aggs/metrics/derivative_fn.test.ts | 4 + .../search/aggs/metrics/derivative_fn.ts | 7 + .../aggs/metrics/filtered_metric_fn.test.ts | 3 + .../search/aggs/metrics/filtered_metric_fn.ts | 7 + .../search/aggs/metrics/geo_bounds_fn.test.ts | 1 + .../search/aggs/metrics/geo_bounds_fn.ts | 7 + .../aggs/metrics/geo_centroid_fn.test.ts | 1 + .../search/aggs/metrics/geo_centroid_fn.ts | 7 + .../common/search/aggs/metrics/max_fn.test.ts | 1 + .../data/common/search/aggs/metrics/median.ts | 2 +- .../search/aggs/metrics/median_fn.test.ts | 1 + .../search/aggs/metrics/metric_agg_type.ts | 4 +- .../common/search/aggs/metrics/min_fn.test.ts | 1 + .../aggs/metrics/percentile_ranks_fn.test.ts | 2 + .../aggs/metrics/percentile_ranks_fn.ts | 7 + .../aggs/metrics/percentiles_fn.test.ts | 2 + .../search/aggs/metrics/percentiles_fn.ts | 7 + .../aggs/metrics/serial_diff_fn.test.ts | 4 + .../search/aggs/metrics/serial_diff_fn.ts | 7 + .../aggs/metrics/std_deviation_fn.test.ts | 1 + .../search/aggs/metrics/std_deviation_fn.ts | 7 + .../common/search/aggs/metrics/sum_fn.test.ts | 1 + .../search/aggs/metrics/top_hit_fn.test.ts | 2 + src/plugins/data/public/public.api.md | 43 +++ src/plugins/data/server/server.api.md | 4 + .../snapshots/session/combined_test0.json | 1 - .../snapshots/session/combined_test1.json | 1 - .../snapshots/session/combined_test2.json | 1 - .../snapshots/session/combined_test3.json | 1 - .../snapshots/session/final_output_test.json | 1 - .../snapshots/session/metric_all_data.json | 1 - .../session/metric_invalid_data.json | 1 - .../session/metric_multi_metric_data.json | 1 - .../session/metric_percentage_mode.json | 1 - .../session/metric_single_metric_data.json | 1 - .../snapshots/session/partial_test_1.json | 1 - .../snapshots/session/partial_test_2.json | 1 - .../snapshots/session/partial_test_3.json | 1 - .../snapshots/session/step_output_test0.json | 1 - .../snapshots/session/step_output_test1.json | 1 - .../snapshots/session/step_output_test2.json | 1 - .../snapshots/session/step_output_test3.json | 1 - .../snapshots/session/tagcloud_all_data.json | 1 - .../snapshots/session/tagcloud_fontsize.json | 1 - .../session/tagcloud_invalid_data.json | 1 - .../session/tagcloud_metric_data.json | 1 - .../snapshots/session/tagcloud_options.json | 1 - .../run_pipeline/esaggs_timeshift.ts | 198 ++++++++++ .../test_suites/run_pipeline/index.ts | 1 + .../editor_frame/frame_layout.tsx | 2 +- .../workspace_panel/workspace_panel.tsx | 1 + .../dimension_panel/dimension_editor.tsx | 6 +- .../dimension_panel/dimension_panel.test.tsx | 194 ++++++++++ .../dimension_panel/time_shift.tsx | 22 +- .../indexpattern.test.ts | 37 ++ .../operations/definitions/date_histogram.tsx | 58 +-- .../definitions/terms/terms.test.tsx | 105 +++++- .../indexpattern_datasource/to_expression.ts | 2 +- x-pack/test/functional/apps/lens/index.ts | 1 + .../test/functional/apps/lens/time_shift.ts | 68 ++++ .../test/functional/page_objects/lens_page.ts | 19 + 88 files changed, 1404 insertions(+), 163 deletions(-) create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.gettimeshift.md create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.hastimeshift.md create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.getsearchsourcetimefilter.md create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.gettimeshiftinterval.md create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.gettimeshifts.md create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.hastimeshifts.md create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.postflighttransform.md delete mode 100644 test/interpreter_functional/snapshots/session/combined_test0.json delete mode 100644 test/interpreter_functional/snapshots/session/combined_test1.json delete mode 100644 test/interpreter_functional/snapshots/session/combined_test2.json delete mode 100644 test/interpreter_functional/snapshots/session/combined_test3.json delete mode 100644 test/interpreter_functional/snapshots/session/final_output_test.json delete mode 100644 test/interpreter_functional/snapshots/session/metric_all_data.json delete mode 100644 test/interpreter_functional/snapshots/session/metric_invalid_data.json delete mode 100644 test/interpreter_functional/snapshots/session/metric_multi_metric_data.json delete mode 100644 test/interpreter_functional/snapshots/session/metric_percentage_mode.json delete mode 100644 test/interpreter_functional/snapshots/session/metric_single_metric_data.json delete mode 100644 test/interpreter_functional/snapshots/session/partial_test_1.json delete mode 100644 test/interpreter_functional/snapshots/session/partial_test_2.json delete mode 100644 test/interpreter_functional/snapshots/session/partial_test_3.json delete mode 100644 test/interpreter_functional/snapshots/session/step_output_test0.json delete mode 100644 test/interpreter_functional/snapshots/session/step_output_test1.json delete mode 100644 test/interpreter_functional/snapshots/session/step_output_test2.json delete mode 100644 test/interpreter_functional/snapshots/session/step_output_test3.json delete mode 100644 test/interpreter_functional/snapshots/session/tagcloud_all_data.json delete mode 100644 test/interpreter_functional/snapshots/session/tagcloud_fontsize.json delete mode 100644 test/interpreter_functional/snapshots/session/tagcloud_invalid_data.json delete mode 100644 test/interpreter_functional/snapshots/session/tagcloud_metric_data.json delete mode 100644 test/interpreter_functional/snapshots/session/tagcloud_options.json create mode 100644 test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts create mode 100644 x-pack/test/functional/apps/lens/time_shift.ts diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.gettimeshift.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.gettimeshift.md new file mode 100644 index 00000000000000..de0d41286c0bbb --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.gettimeshift.md @@ -0,0 +1,15 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [AggConfig](./kibana-plugin-plugins-data-public.aggconfig.md) > [getTimeShift](./kibana-plugin-plugins-data-public.aggconfig.gettimeshift.md) + +## AggConfig.getTimeShift() method + +Signature: + +```typescript +getTimeShift(): undefined | moment.Duration; +``` +Returns: + +`undefined | moment.Duration` + diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.hastimeshift.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.hastimeshift.md new file mode 100644 index 00000000000000..024b0766ffd7b0 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.hastimeshift.md @@ -0,0 +1,15 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [AggConfig](./kibana-plugin-plugins-data-public.aggconfig.md) > [hasTimeShift](./kibana-plugin-plugins-data-public.aggconfig.hastimeshift.md) + +## AggConfig.hasTimeShift() method + +Signature: + +```typescript +hasTimeShift(): boolean; +``` +Returns: + +`boolean` + diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.md index d4a8eddf51cfcd..a96626d1a485d7 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfig.md @@ -46,8 +46,10 @@ export declare class AggConfig | [getRequestAggs()](./kibana-plugin-plugins-data-public.aggconfig.getrequestaggs.md) | | | | [getResponseAggs()](./kibana-plugin-plugins-data-public.aggconfig.getresponseaggs.md) | | | | [getTimeRange()](./kibana-plugin-plugins-data-public.aggconfig.gettimerange.md) | | | +| [getTimeShift()](./kibana-plugin-plugins-data-public.aggconfig.gettimeshift.md) | | | | [getValue(bucket)](./kibana-plugin-plugins-data-public.aggconfig.getvalue.md) | | | | [getValueBucketPath()](./kibana-plugin-plugins-data-public.aggconfig.getvaluebucketpath.md) | | Returns the bucket path containing the main value the agg will produce (e.g. for sum of bytes it will point to the sum, for median it will point to the 50 percentile in the percentile multi value bucket) | +| [hasTimeShift()](./kibana-plugin-plugins-data-public.aggconfig.hastimeshift.md) | | | | [isFilterable()](./kibana-plugin-plugins-data-public.aggconfig.isfilterable.md) | | | | [makeLabel(percentageMode)](./kibana-plugin-plugins-data-public.aggconfig.makelabel.md) | | | | [nextId(list)](./kibana-plugin-plugins-data-public.aggconfig.nextid.md) | static | Calculate the next id based on the ids in this list {array} list - a list of objects with id properties | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.getsearchsourcetimefilter.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.getsearchsourcetimefilter.md new file mode 100644 index 00000000000000..1f8bc1300a0a86 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.getsearchsourcetimefilter.md @@ -0,0 +1,72 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) > [getSearchSourceTimeFilter](./kibana-plugin-plugins-data-public.aggconfigs.getsearchsourcetimefilter.md) + +## AggConfigs.getSearchSourceTimeFilter() method + +Signature: + +```typescript +getSearchSourceTimeFilter(forceNow?: Date): RangeFilter[] | { + meta: { + index: string | undefined; + params: {}; + alias: string; + disabled: boolean; + negate: boolean; + }; + query: { + bool: { + should: { + bool: { + filter: { + range: { + [x: string]: { + gte: string; + lte: string; + }; + }; + }[]; + }; + }[]; + minimum_should_match: number; + }; + }; + }[]; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| forceNow | Date | | + +Returns: + +`RangeFilter[] | { + meta: { + index: string | undefined; + params: {}; + alias: string; + disabled: boolean; + negate: boolean; + }; + query: { + bool: { + should: { + bool: { + filter: { + range: { + [x: string]: { + gte: string; + lte: string; + }; + }; + }[]; + }; + }[]; + minimum_should_match: number; + }; + }; + }[]` + diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.gettimeshiftinterval.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.gettimeshiftinterval.md new file mode 100644 index 00000000000000..d15ccbc5dc0a1c --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.gettimeshiftinterval.md @@ -0,0 +1,15 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) > [getTimeShiftInterval](./kibana-plugin-plugins-data-public.aggconfigs.gettimeshiftinterval.md) + +## AggConfigs.getTimeShiftInterval() method + +Signature: + +```typescript +getTimeShiftInterval(): moment.Duration | undefined; +``` +Returns: + +`moment.Duration | undefined` + diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.gettimeshifts.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.gettimeshifts.md new file mode 100644 index 00000000000000..44ab25cf30eb2b --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.gettimeshifts.md @@ -0,0 +1,15 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) > [getTimeShifts](./kibana-plugin-plugins-data-public.aggconfigs.gettimeshifts.md) + +## AggConfigs.getTimeShifts() method + +Signature: + +```typescript +getTimeShifts(): Record; +``` +Returns: + +`Record` + diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.hastimeshifts.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.hastimeshifts.md new file mode 100644 index 00000000000000..db31e549666b45 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.hastimeshifts.md @@ -0,0 +1,15 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) > [hasTimeShifts](./kibana-plugin-plugins-data-public.aggconfigs.hastimeshifts.md) + +## AggConfigs.hasTimeShifts() method + +Signature: + +```typescript +hasTimeShifts(): boolean; +``` +Returns: + +`boolean` + diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md index 02e9a63d95ba37..ca191fdcae9a53 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md @@ -43,8 +43,13 @@ export declare class AggConfigs | [getRequestAggs()](./kibana-plugin-plugins-data-public.aggconfigs.getrequestaggs.md) | | | | [getResponseAggById(id)](./kibana-plugin-plugins-data-public.aggconfigs.getresponseaggbyid.md) | | Find a response agg by it's id. This may be an agg in the aggConfigs, or one created specifically for a response value | | [getResponseAggs()](./kibana-plugin-plugins-data-public.aggconfigs.getresponseaggs.md) | | Gets the AggConfigs (and possibly ResponseAggConfigs) that represent the values that will be produced when all aggs are run.With multi-value metric aggs it is possible for a single agg request to result in multiple agg values, which is why the length of a vis' responseValuesAggs may be different than the vis' aggs {array\[AggConfig\]} | +| [getSearchSourceTimeFilter(forceNow)](./kibana-plugin-plugins-data-public.aggconfigs.getsearchsourcetimefilter.md) | | | +| [getTimeShiftInterval()](./kibana-plugin-plugins-data-public.aggconfigs.gettimeshiftinterval.md) | | | +| [getTimeShifts()](./kibana-plugin-plugins-data-public.aggconfigs.gettimeshifts.md) | | | +| [hasTimeShifts()](./kibana-plugin-plugins-data-public.aggconfigs.hastimeshifts.md) | | | | [jsonDataEquals(aggConfigs)](./kibana-plugin-plugins-data-public.aggconfigs.jsondataequals.md) | | Data-by-data comparison of this Aggregation Ignores the non-array indexes | | [onSearchRequestStart(searchSource, options)](./kibana-plugin-plugins-data-public.aggconfigs.onsearchrequeststart.md) | | | +| [postFlightTransform(response)](./kibana-plugin-plugins-data-public.aggconfigs.postflighttransform.md) | | | | [setTimeFields(timeFields)](./kibana-plugin-plugins-data-public.aggconfigs.settimefields.md) | | | | [setTimeRange(timeRange)](./kibana-plugin-plugins-data-public.aggconfigs.settimerange.md) | | | | [toDsl()](./kibana-plugin-plugins-data-public.aggconfigs.todsl.md) | | | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.postflighttransform.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.postflighttransform.md new file mode 100644 index 00000000000000..b34fda40a30895 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.postflighttransform.md @@ -0,0 +1,22 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) > [postFlightTransform](./kibana-plugin-plugins-data-public.aggconfigs.postflighttransform.md) + +## AggConfigs.postFlightTransform() method + +Signature: + +```typescript +postFlightTransform(response: IEsSearchResponse): IEsSearchResponse; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| response | IEsSearchResponse<any> | | + +Returns: + +`IEsSearchResponse` + diff --git a/src/plugins/data/common/search/aggs/agg_config.ts b/src/plugins/data/common/search/aggs/agg_config.ts index 5c34e17fb5c0b0..3c83b5bdf6084b 100644 --- a/src/plugins/data/common/search/aggs/agg_config.ts +++ b/src/plugins/data/common/search/aggs/agg_config.ts @@ -20,7 +20,7 @@ import { import { IAggType } from './agg_type'; import { writeParams } from './agg_params'; -import { GenericBucket, IAggConfigs } from './agg_configs'; +import { IAggConfigs } from './agg_configs'; import { parseTimeShift } from './utils'; type State = string | number | boolean | null | undefined | SerializableState; @@ -199,26 +199,6 @@ export class AggConfig { return parsedTimeShift; } - getShiftedKey(key: string | number, timeShift: moment.Duration): string | number { - return this.type.getShiftedKey(this, key, timeShift); - } - - getTimeShiftInterval(): undefined | moment.Duration { - return this.type.getTimeShiftInterval(this); - } - - splitForTimeShift(aggs: IAggConfigs) { - return this.type.splitForTimeShift(this, aggs); - } - - orderBuckets(a: GenericBucket, b: GenericBucket): number { - if (this.type.orderBuckets) { - return this.type.orderBuckets(this, a, b); - } else { - return 0; - } - } - write(aggs?: IAggConfigs) { return writeParams(this.type.params, this, aggs); } diff --git a/src/plugins/data/common/search/aggs/agg_configs.test.ts b/src/plugins/data/common/search/aggs/agg_configs.test.ts index 28102544ae0553..d4980447c5468f 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.test.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.test.ts @@ -14,6 +14,7 @@ import { mockAggTypesRegistry } from './test_helpers'; import type { IndexPatternField } from '../../index_patterns'; import { IndexPattern } from '../../index_patterns/index_patterns/index_pattern'; import { stubIndexPattern, stubIndexPatternWithFields } from '../../../common/stubs'; +import { IEsSearchResponse } from '..'; describe('AggConfigs', () => { let indexPattern: IndexPattern; @@ -332,6 +333,101 @@ describe('AggConfigs', () => { }); }); + it('inserts a time split filters agg if there are multiple time shifts', () => { + const configStates = [ + { + enabled: true, + type: 'terms', + schema: 'segment', + params: { field: 'clientip', size: 10 }, + }, + { enabled: true, type: 'avg', schema: 'metric', params: { field: 'bytes' } }, + { + enabled: true, + type: 'sum', + schema: 'metric', + params: { field: 'bytes', timeShift: '1d' }, + }, + ]; + + const ac = new AggConfigs(indexPattern, configStates, { typesRegistry }); + ac.timeFields = ['timestamp']; + ac.timeRange = { + from: '2021-05-05T00:00:00.000Z', + to: '2021-05-10T00:00:00.000Z', + }; + const dsl = ac.toDsl(); + + const terms = ac.byName('terms')[0]; + const avg = ac.byName('avg')[0]; + const sum = ac.byName('sum')[0]; + + expect(dsl[terms.id].aggs.time_offset_split.filters.filters).toMatchInlineSnapshot(` + Object { + "0": Object { + "range": Object { + "timestamp": Object { + "gte": "2021-05-05T00:00:00.000Z", + "lte": "2021-05-10T00:00:00.000Z", + }, + }, + }, + "86400000": Object { + "range": Object { + "timestamp": Object { + "gte": "2021-05-04T00:00:00.000Z", + "lte": "2021-05-09T00:00:00.000Z", + }, + }, + }, + } + `); + expect(dsl[terms.id].aggs.time_offset_split.aggs).toHaveProperty(avg.id); + expect(dsl[terms.id].aggs.time_offset_split.aggs).toHaveProperty(sum.id); + }); + + it('does not insert a time split if there is a single time shift', () => { + const configStates = [ + { + enabled: true, + type: 'terms', + schema: 'segment', + params: { field: 'clientip', size: 10 }, + }, + { + enabled: true, + type: 'avg', + schema: 'metric', + params: { + field: 'bytes', + timeShift: '1d', + }, + }, + { + enabled: true, + type: 'sum', + schema: 'metric', + params: { field: 'bytes', timeShift: '1d' }, + }, + ]; + + const ac = new AggConfigs(indexPattern, configStates, { typesRegistry }); + ac.timeFields = ['timestamp']; + ac.timeRange = { + from: '2021-05-05T00:00:00.000Z', + to: '2021-05-10T00:00:00.000Z', + }; + const dsl = ac.toDsl(); + + const terms = ac.byName('terms')[0]; + const avg = ac.byName('avg')[0]; + const sum = ac.byName('sum')[0]; + + expect(dsl[terms.id].aggs).not.toHaveProperty('time_offset_split'); + expect(dsl[terms.id].aggs).toHaveProperty(avg.id); + expect(dsl[terms.id].aggs).toHaveProperty(sum.id); + }); + it('writes multiple metric aggregations at every level if the vis is hierarchical', () => { const configStates = [ { enabled: true, type: 'terms', schema: 'segment', params: { field: 'bytes', orderBy: 1 } }, @@ -426,4 +522,246 @@ describe('AggConfigs', () => { ); }); }); + + describe('#postFlightTransform', () => { + it('merges together splitted responses for multiple shifts', () => { + indexPattern = stubIndexPattern as IndexPattern; + indexPattern.fields.getByName = (name) => (({ name } as unknown) as IndexPatternField); + const configStates = [ + { + enabled: true, + type: 'terms', + schema: 'segment', + params: { field: 'clientip', size: 10 }, + }, + { + enabled: true, + type: 'date_histogram', + schema: 'segment', + params: { field: '@timestamp', interval: '1d' }, + }, + { + enabled: true, + type: 'avg', + schema: 'metric', + params: { + field: 'bytes', + timeShift: '1d', + }, + }, + { + enabled: true, + type: 'sum', + schema: 'metric', + params: { field: 'bytes' }, + }, + ]; + + const ac = new AggConfigs(indexPattern, configStates, { typesRegistry }); + ac.timeFields = ['@timestamp']; + ac.timeRange = { + from: '2021-05-05T00:00:00.000Z', + to: '2021-05-10T00:00:00.000Z', + }; + // 1 terms bucket (A), with 2 date buckets (7th and 8th of May) + // the bucket keys of the shifted time range will be shifted forward + const response = { + rawResponse: { + aggregations: { + '1': { + buckets: [ + { + key: 'A', + time_offset_split: { + buckets: { + '0': { + 2: { + buckets: [ + { + // 2021-05-07 + key: 1620345600000, + 3: { + value: 1.1, + }, + 4: { + value: 2.2, + }, + }, + { + // 2021-05-08 + key: 1620432000000, + doc_count: 26, + 3: { + value: 3.3, + }, + 4: { + value: 4.4, + }, + }, + ], + }, + }, + '86400000': { + 2: { + buckets: [ + { + // 2021-05-07 + key: 1620345600000, + doc_count: 13, + 3: { + value: 5.5, + }, + 4: { + value: 6.6, + }, + }, + { + // 2021-05-08 + key: 1620432000000, + 3: { + value: 7.7, + }, + 4: { + value: 8.8, + }, + }, + ], + }, + }, + }, + }, + }, + ], + }, + }, + }, + }; + const mergedResponse = ac.postFlightTransform( + (response as unknown) as IEsSearchResponse + ); + expect(mergedResponse.rawResponse).toEqual({ + aggregations: { + '1': { + buckets: [ + { + '2': { + buckets: [ + { + '4': { + value: 2.2, + }, + // 2021-05-07 + key: 1620345600000, + }, + { + '3': { + value: 5.5, + }, + '4': { + value: 4.4, + }, + doc_count: 26, + doc_count_86400000: 13, + // 2021-05-08 + key: 1620432000000, + }, + { + '3': { + value: 7.7, + }, + // 2021-05-09 + key: 1620518400000, + }, + ], + }, + key: 'A', + }, + ], + }, + }, + }); + }); + + it('shifts date histogram keys and renames doc_count properties for single shift', () => { + indexPattern = stubIndexPattern as IndexPattern; + indexPattern.fields.getByName = (name) => (({ name } as unknown) as IndexPatternField); + const configStates = [ + { + enabled: true, + type: 'date_histogram', + schema: 'segment', + params: { field: '@timestamp', interval: '1d' }, + }, + { + enabled: true, + type: 'avg', + schema: 'metric', + params: { + field: 'bytes', + timeShift: '1d', + }, + }, + ]; + + const ac = new AggConfigs(indexPattern, configStates, { typesRegistry }); + ac.timeFields = ['@timestamp']; + ac.timeRange = { + from: '2021-05-05T00:00:00.000Z', + to: '2021-05-10T00:00:00.000Z', + }; + const response = { + rawResponse: { + aggregations: { + '1': { + buckets: [ + { + // 2021-05-07 + key: 1620345600000, + doc_count: 26, + 2: { + value: 1.1, + }, + }, + { + // 2021-05-08 + key: 1620432000000, + doc_count: 27, + 2: { + value: 2.2, + }, + }, + ], + }, + }, + }, + }; + const mergedResponse = ac.postFlightTransform( + (response as unknown) as IEsSearchResponse + ); + expect(mergedResponse.rawResponse).toEqual({ + aggregations: { + '1': { + buckets: [ + { + '2': { + value: 1.1, + }, + doc_count_86400000: 26, + // 2021-05-08 + key: 1620432000000, + }, + { + '2': { + value: 2.2, + }, + doc_count_86400000: 27, + // 2021-05-09 + key: 1620518400000, + }, + ], + }, + }, + }); + }); + }); }); diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index 180113b1b31d86..bab7fab5f2248d 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -30,6 +30,7 @@ import { AggTypesRegistryStart } from './agg_types_registry'; import { AggGroupNames } from './agg_groups'; import { IndexPattern } from '../../index_patterns/index_patterns/index_pattern'; import { TimeRange, getTime, isRangeFilter } from '../../../common'; +import { IBucketAggConfig } from './buckets'; function removeParentAggs(obj: any) { for (const prop in obj) { @@ -217,7 +218,13 @@ export class AggConfigs { }; }); } - this.getRequestAggs().forEach((config: AggConfig, i: number, list) => { + const requestAggs = this.getRequestAggs(); + const aggsWithDsl = requestAggs.filter((agg) => !agg.type.hasNoDsl).length; + const timeSplitIndex = this.getAll().findIndex( + (config) => 'splitForTimeShift' in config.type && config.type.splitForTimeShift(config, this) + ); + + requestAggs.forEach((config: AggConfig, i: number, list) => { if (!dslLvlCursor) { // start at the top level dslLvlCursor = dslTopLvl; @@ -246,8 +253,11 @@ export class AggConfigs { parseParentAggs(dslLvlCursor, dsl); - if (config.type.type === AggGroupNames.Buckets && i < list.length - 1) { - // buckets that are not the last item in the list accept sub-aggs + if ( + config.type.type === AggGroupNames.Buckets && + (i < aggsWithDsl - 1 || timeSplitIndex > i) + ) { + // buckets that are not the last item in the list of dsl producing aggs or have a time split coming up accept sub-aggs subAggs = dsl.aggs || (dsl.aggs = {}); } @@ -279,7 +289,7 @@ export class AggConfigs { timeShifts: Record, dslLvlCursor: Record ) { - if (!config.splitForTimeShift(this)) { + if ('splitForTimeShift' in config.type && !config.type.splitForTimeShift(config, this)) { return dslLvlCursor; } if (!this.timeFields || this.timeFields.length < 1) { @@ -370,8 +380,10 @@ export class AggConfigs { } getTimeShiftInterval(): moment.Duration | undefined { - const splitAgg = this.getAll().find((agg) => agg.splitForTimeShift(this)); - return splitAgg?.getTimeShiftInterval(); + const splitAgg = (this.getAll().filter( + (agg) => agg.type.type === AggGroupNames.Buckets + ) as IBucketAggConfig[]).find((agg) => agg.type.splitForTimeShift(agg, this)); + return splitAgg?.type.getTimeShiftInterval(splitAgg); } hasTimeShifts(): boolean { @@ -438,9 +450,16 @@ export class AggConfigs { const timeShifts = this.getTimeShifts(); const hasMultipleTimeShifts = Object.keys(timeShifts).length > 1; const transformedRawResponse = cloneDeep(response.rawResponse); + if (!transformedRawResponse.aggregations) { + transformedRawResponse.aggregations = { + doc_count: response.rawResponse.hits?.total as Aggregate, + }; + } const aggCursor = transformedRawResponse.aggregations!; - const bucketAggs = this.aggs.filter((agg) => agg.type.type === AggGroupNames.Buckets); + const bucketAggs = this.aggs.filter( + (agg) => agg.type.type === AggGroupNames.Buckets + ) as IBucketAggConfig[]; const mergeAggLevel = ( target: GenericBucket, @@ -451,7 +470,11 @@ export class AggConfigs { Object.entries(source).forEach(([key, val]) => { // copy over doc count into special key if (typeof val === 'number' && key === 'doc_count') { - target[`doc_count_${shift.asMilliseconds()}`] = val; + if (shift.asMilliseconds() === 0) { + target.doc_count = val; + } else { + target[`doc_count_${shift.asMilliseconds()}`] = val; + } } else if (typeof val !== 'object') { // other meta keys not of interest return; @@ -460,11 +483,15 @@ export class AggConfigs { const agg = this.byId(key); if (agg && agg.type.type === AggGroupNames.Metrics) { const timeShift = agg.getTimeShift(); - if (timeShift && timeShift.asMilliseconds() === shift.asMilliseconds()) { + if ( + (timeShift && timeShift.asMilliseconds() === shift.asMilliseconds()) || + (shift.asMilliseconds() === 0 && !timeShift) + ) { // this is a metric from the current time shift, copy it over target[key] = source[key]; } - } else if (agg === bucketAggs[aggIndex]) { + } else if (agg && agg === bucketAggs[aggIndex]) { + const bucketAgg = agg as IBucketAggConfig; // expected next bucket sub agg const subAggregate = val as Aggregate; const buckets = ('buckets' in subAggregate ? subAggregate.buckets : undefined) as @@ -490,7 +517,7 @@ export class AggConfigs { baseBucketMap[String(bucket.key)] = bucket; }); buckets.forEach((bucket) => { - const bucketKey = agg.getShiftedKey(bucket.key, shift); + const bucketKey = bucketAgg.type.getShiftedKey(bucketAgg, bucket.key, shift); // if a bucket is missing in the map, create an empty one if (!baseBucketMap[bucketKey]) { baseBucketMap[String(bucketKey)] = { @@ -501,7 +528,7 @@ export class AggConfigs { }); (baseSubAggregate as MultiBucketAggregate).buckets = Object.values( baseBucketMap - ).sort(agg.orderBuckets.bind(agg)); + ).sort((a, b) => bucketAgg.type.orderBuckets(bucketAgg, a, b)); } else if (baseBuckets && buckets && !isArray(baseBuckets)) { Object.entries(buckets).forEach(([bucketKey, bucket]) => { // if a bucket is missing in the base response, create an empty one @@ -516,23 +543,21 @@ export class AggConfigs { }); }; const transformTimeShift = (cursor: Record, aggIndex: number): undefined => { - const shouldSplit = bucketAggs[aggIndex].splitForTimeShift(this); + const shouldSplit = this.aggs[aggIndex].type.splitForTimeShift(this.aggs[aggIndex], this); if (shouldSplit) { // multiple time shifts caused a filters agg in the tree we have to merge if (hasMultipleTimeShifts && cursor.time_offset_split) { const timeShiftedBuckets = (cursor.time_offset_split as FiltersAggregate) .buckets as Record; - const subTree = timeShiftedBuckets['0'] || {}; - Object.entries(timeShifts) - .filter(([key]) => key !== '0') - .forEach(([key, shift]) => { - mergeAggLevel( - subTree as GenericBucket, - timeShiftedBuckets[key] as GenericBucket, - shift, - aggIndex - ); - }); + const subTree = {}; + Object.entries(timeShifts).forEach(([key, shift]) => { + mergeAggLevel( + subTree as GenericBucket, + timeShiftedBuckets[key] as GenericBucket, + shift, + aggIndex + ); + }); delete cursor.time_offset_split; Object.assign(cursor, subTree); diff --git a/src/plugins/data/common/search/aggs/agg_type.ts b/src/plugins/data/common/search/aggs/agg_type.ts index 4d30d6a2116501..48ce54bbd61bdb 100644 --- a/src/plugins/data/common/search/aggs/agg_type.ts +++ b/src/plugins/data/common/search/aggs/agg_type.ts @@ -8,7 +8,6 @@ import { constant, noop, identity } from 'lodash'; import { i18n } from '@kbn/i18n'; -import moment from 'moment'; import { ISearchSource } from 'src/plugins/data/public'; import { DatatableColumnType, SerializedFieldFormat } from 'src/plugins/expressions/common'; @@ -17,7 +16,7 @@ import type { RequestAdapter } from 'src/plugins/inspector/common'; import { estypes } from '@elastic/elasticsearch'; import { initParams } from './agg_params'; import { AggConfig } from './agg_config'; -import { GenericBucket, IAggConfigs } from './agg_configs'; +import { IAggConfigs } from './agg_configs'; import { BaseParamType } from './param_types/base'; import { AggParamType } from './param_types/agg'; @@ -216,26 +215,10 @@ export class AggType< return agg.id; }; - getShiftedKey( - agg: TAggConfig, - key: string | number, - timeShift: moment.Duration - ): string | number { - return key; - } - splitForTimeShift(agg: TAggConfig, aggs: IAggConfigs) { return false; } - getTimeShiftInterval(agg: TAggConfig): undefined | moment.Duration { - return undefined; - } - - orderBuckets(agg: TAggConfig, a: GenericBucket, b: GenericBucket): number { - return Number(a.key) - Number(b.key); - } - /** * Generic AggType Constructor * diff --git a/src/plugins/data/common/search/aggs/buckets/bucket_agg_type.ts b/src/plugins/data/common/search/aggs/buckets/bucket_agg_type.ts index 1626a89e9a3318..d44e634a00fe6b 100644 --- a/src/plugins/data/common/search/aggs/buckets/bucket_agg_type.ts +++ b/src/plugins/data/common/search/aggs/buckets/bucket_agg_type.ts @@ -44,6 +44,22 @@ export class BucketAggType any; type = bucketType; + getShiftedKey( + agg: TBucketAggConfig, + key: string | number, + timeShift: moment.Duration + ): string | number { + return key; + } + + getTimeShiftInterval(agg: TBucketAggConfig): undefined | moment.Duration { + return undefined; + } + + orderBuckets(agg: TBucketAggConfig, a: GenericBucket, b: GenericBucket): number { + return Number(a.key) - Number(b.key); + } + constructor(config: BucketAggTypeConfig) { super(config); diff --git a/src/plugins/data/common/search/aggs/metrics/avg_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/avg_fn.test.ts index 05a6e9eeff7d74..0b794617fb96ed 100644 --- a/src/plugins/data/common/search/aggs/metrics/avg_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/avg_fn.test.ts @@ -27,6 +27,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "avg", diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.test.ts index 33f20b9a40dc26..ac214c1a1591ce 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.test.ts @@ -29,6 +29,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "customMetric": undefined, "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "avg_bucket", @@ -42,11 +43,13 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "customMetric": undefined, "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "avg_bucket", }, "json": undefined, + "timeShift": undefined, } `); }); diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.ts b/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.ts index b79e57207ebd81..b9374fb94720ad 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.ts @@ -77,6 +77,13 @@ export const aggBucketAvg = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.test.ts index 35b765ec0e075c..e6db7665a68ddc 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.test.ts @@ -29,6 +29,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "customMetric": undefined, "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "max_bucket", @@ -42,11 +43,13 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "customMetric": undefined, "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "max_bucket", }, "json": undefined, + "timeShift": undefined, } `); }); diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.ts b/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.ts index e12a592448334d..f50042bc1ca105 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.ts @@ -77,6 +77,13 @@ export const aggBucketMax = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.test.ts index 49346036ce6492..22ec55506fe901 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.test.ts @@ -29,6 +29,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "customMetric": undefined, "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "min_bucket", @@ -42,11 +43,13 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "customMetric": undefined, "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "min_bucket", }, "json": undefined, + "timeShift": undefined, } `); }); diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.ts b/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.ts index ece5c07c6e5f86..41bec5c5746c3b 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.ts @@ -77,6 +77,13 @@ export const aggBucketMin = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.test.ts index 0f5c84a477b06b..0e3370cec14e5d 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.test.ts @@ -29,6 +29,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "customMetric": undefined, "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "sum_bucket", @@ -42,11 +43,13 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "customMetric": undefined, "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "sum_bucket", }, "json": undefined, + "timeShift": undefined, } `); }); diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.ts b/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.ts index 5fe0ee75bfe38a..b56dba3b96ba10 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.ts @@ -77,6 +77,13 @@ export const aggBucketSum = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/cardinality_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/cardinality_fn.test.ts index 8b235edacb59a0..08d64e599d8a9e 100644 --- a/src/plugins/data/common/search/aggs/metrics/cardinality_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/cardinality_fn.test.ts @@ -27,6 +27,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "cardinality", diff --git a/src/plugins/data/common/search/aggs/metrics/count_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/count_fn.test.ts index 047b9bbd8517f2..c6736c5b69f7d4 100644 --- a/src/plugins/data/common/search/aggs/metrics/count_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/count_fn.test.ts @@ -23,6 +23,7 @@ describe('agg_expression_functions', () => { "id": undefined, "params": Object { "customLabel": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "count", diff --git a/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.test.ts index 5eb6d2b7804420..f311ab35a8d0df 100644 --- a/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.test.ts @@ -29,6 +29,7 @@ describe('agg_expression_functions', () => { "customMetric": undefined, "json": undefined, "metricAgg": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "cumulative_sum", @@ -54,6 +55,7 @@ describe('agg_expression_functions', () => { "customMetric": undefined, "json": undefined, "metricAgg": "sum", + "timeShift": undefined, }, "schema": undefined, "type": "cumulative_sum", @@ -81,12 +83,14 @@ describe('agg_expression_functions', () => { "customMetric": undefined, "json": undefined, "metricAgg": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "cumulative_sum", }, "json": undefined, "metricAgg": undefined, + "timeShift": undefined, } `); }); diff --git a/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.ts b/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.ts index cba4de1ad11aec..88c1fceb11f720 100644 --- a/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.ts @@ -81,6 +81,13 @@ export const aggCumulativeSum = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/derivative_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/derivative_fn.test.ts index 1eaca811a2481f..3e4fc838dd398a 100644 --- a/src/plugins/data/common/search/aggs/metrics/derivative_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/derivative_fn.test.ts @@ -29,6 +29,7 @@ describe('agg_expression_functions', () => { "customMetric": undefined, "json": undefined, "metricAgg": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "derivative", @@ -54,6 +55,7 @@ describe('agg_expression_functions', () => { "customMetric": undefined, "json": undefined, "metricAgg": "sum", + "timeShift": undefined, }, "schema": undefined, "type": "derivative", @@ -81,12 +83,14 @@ describe('agg_expression_functions', () => { "customMetric": undefined, "json": undefined, "metricAgg": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "derivative", }, "json": undefined, "metricAgg": undefined, + "timeShift": undefined, } `); }); diff --git a/src/plugins/data/common/search/aggs/metrics/derivative_fn.ts b/src/plugins/data/common/search/aggs/metrics/derivative_fn.ts index e27179c7209ade..38985b78cb378d 100644 --- a/src/plugins/data/common/search/aggs/metrics/derivative_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/derivative_fn.ts @@ -81,6 +81,13 @@ export const aggDerivative = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.test.ts index 22e97fe18b604c..d1ce6ff4639035 100644 --- a/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.test.ts @@ -28,6 +28,7 @@ describe('agg_expression_functions', () => { "customBucket": undefined, "customLabel": undefined, "customMetric": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "filtered_metric", @@ -40,10 +41,12 @@ describe('agg_expression_functions', () => { "customBucket": undefined, "customLabel": undefined, "customMetric": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "filtered_metric", }, + "timeShift": undefined, } `); }); diff --git a/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.ts b/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.ts index 6a7ff5fa5fd40e..7b14fd1c45ed27 100644 --- a/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.ts @@ -72,6 +72,13 @@ export const aggFilteredMetric = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.test.ts index c48233e84404c6..50b5f5b60376b6 100644 --- a/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.test.ts @@ -27,6 +27,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "geo_bounds", diff --git a/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.ts b/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.ts index 19d2dabc843dd9..c7915eb12c5b25 100644 --- a/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.ts @@ -67,6 +67,13 @@ export const aggGeoBounds = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.test.ts index e984df13527ca9..889ed29c63ee14 100644 --- a/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.test.ts @@ -27,6 +27,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "geo_centroid", diff --git a/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.ts b/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.ts index 1cc11c345e9ba0..df1b5182092fd2 100644 --- a/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.ts @@ -67,6 +67,13 @@ export const aggGeoCentroid = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/max_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/max_fn.test.ts index d94e01927c851b..021c5aac69e102 100644 --- a/src/plugins/data/common/search/aggs/metrics/max_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/max_fn.test.ts @@ -27,6 +27,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "max", diff --git a/src/plugins/data/common/search/aggs/metrics/median.ts b/src/plugins/data/common/search/aggs/metrics/median.ts index bad4c7baf173f6..4fdb1ce6b7d81c 100644 --- a/src/plugins/data/common/search/aggs/metrics/median.ts +++ b/src/plugins/data/common/search/aggs/metrics/median.ts @@ -46,7 +46,7 @@ export const getMedianMetricAgg = () => { { name: 'percents', default: [50], shouldShow: () => false, serialize: () => undefined }, ], getValue(agg, bucket) { - return bucket[agg.id].values['50.0']; + return bucket[agg.id]?.values['50.0']; }, }); }; diff --git a/src/plugins/data/common/search/aggs/metrics/median_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/median_fn.test.ts index e70520b743e179..7ff7f18cdbc02c 100644 --- a/src/plugins/data/common/search/aggs/metrics/median_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/median_fn.test.ts @@ -27,6 +27,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "median", diff --git a/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts b/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts index 9186ba8f70cfc3..6ddb0fdd9410d4 100644 --- a/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts +++ b/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts @@ -83,7 +83,9 @@ export class MetricAggType aggs.hasTimeShifts() && aggs.byType(AggGroupNames.Metrics)[0] === agg && - !aggs.byType(AggGroupNames.Buckets).some((bucketAgg) => bucketAgg.splitForTimeShift(aggs)); + !aggs + .byType(AggGroupNames.Buckets) + .some((bucketAgg) => bucketAgg.type.splitForTimeShift(bucketAgg, aggs)); } } diff --git a/src/plugins/data/common/search/aggs/metrics/min_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/min_fn.test.ts index ea2d2cd23edaea..fee4b28882408d 100644 --- a/src/plugins/data/common/search/aggs/metrics/min_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/min_fn.test.ts @@ -27,6 +27,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "min", diff --git a/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.test.ts index 9328597b24cfaa..873765374c80a6 100644 --- a/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.test.ts @@ -27,6 +27,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, "values": undefined, }, "schema": undefined, @@ -51,6 +52,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, "values": Array [ 1, 2, diff --git a/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.ts b/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.ts index 7929a01c0b5893..7bbf8c460682b6 100644 --- a/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.ts @@ -74,6 +74,13 @@ export const aggPercentileRanks = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/percentiles_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/percentiles_fn.test.ts index 0d71df240d1226..468da036cea88f 100644 --- a/src/plugins/data/common/search/aggs/metrics/percentiles_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/percentiles_fn.test.ts @@ -28,6 +28,7 @@ describe('agg_expression_functions', () => { "field": "machine.os.keyword", "json": undefined, "percents": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "percentiles", @@ -56,6 +57,7 @@ describe('agg_expression_functions', () => { 2, 3, ], + "timeShift": undefined, }, "schema": undefined, "type": "percentiles", diff --git a/src/plugins/data/common/search/aggs/metrics/percentiles_fn.ts b/src/plugins/data/common/search/aggs/metrics/percentiles_fn.ts index fa5120dfc3b97f..8c9c765d068e28 100644 --- a/src/plugins/data/common/search/aggs/metrics/percentiles_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/percentiles_fn.ts @@ -74,6 +74,13 @@ export const aggPercentiles = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.test.ts index 065ef8021cbda2..aa73d5c44dd7fc 100644 --- a/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.test.ts @@ -29,6 +29,7 @@ describe('agg_expression_functions', () => { "customMetric": undefined, "json": undefined, "metricAgg": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "serial_diff", @@ -54,6 +55,7 @@ describe('agg_expression_functions', () => { "customMetric": undefined, "json": undefined, "metricAgg": "sum", + "timeShift": undefined, }, "schema": undefined, "type": "serial_diff", @@ -81,12 +83,14 @@ describe('agg_expression_functions', () => { "customMetric": undefined, "json": undefined, "metricAgg": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "serial_diff", }, "json": undefined, "metricAgg": undefined, + "timeShift": undefined, } `); }); diff --git a/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.ts b/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.ts index 925d85774c7ad6..b481fa56a38ce9 100644 --- a/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.ts @@ -81,6 +81,13 @@ export const aggSerialDiff = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.test.ts index 9aaf82e65812b8..849987695dc7c4 100644 --- a/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.test.ts @@ -27,6 +27,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "std_dev", diff --git a/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.ts b/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.ts index 80787a3383c6b5..7ba0c5b789dbf8 100644 --- a/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.ts @@ -67,6 +67,13 @@ export const aggStdDeviation = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/src/plugins/data/common/search/aggs/metrics/sum_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/sum_fn.test.ts index e19fc072e1cd98..f4d4fb5451dcda 100644 --- a/src/plugins/data/common/search/aggs/metrics/sum_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/sum_fn.test.ts @@ -27,6 +27,7 @@ describe('agg_expression_functions', () => { "customLabel": undefined, "field": "machine.os.keyword", "json": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "sum", diff --git a/src/plugins/data/common/search/aggs/metrics/top_hit_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/top_hit_fn.test.ts index e9d6a619a9cd6b..2f8ef74b5c2f0c 100644 --- a/src/plugins/data/common/search/aggs/metrics/top_hit_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/top_hit_fn.test.ts @@ -32,6 +32,7 @@ describe('agg_expression_functions', () => { "size": undefined, "sortField": undefined, "sortOrder": undefined, + "timeShift": undefined, }, "schema": undefined, "type": "top_hits", @@ -64,6 +65,7 @@ describe('agg_expression_functions', () => { "size": 6, "sortField": "_score", "sortOrder": "asc", + "timeShift": undefined, }, "schema": "whatever", "type": "top_hits", diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index d4f0ccfe810c6f..16ed4d19e2ed98 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -7,11 +7,13 @@ import { $Values } from '@kbn/utility-types'; import { Action } from 'history'; import { Adapters as Adapters_2 } from 'src/plugins/inspector/common'; +import { Aggregate } from '@elastic/elasticsearch/api/types'; import { ApiResponse } from '@elastic/elasticsearch/lib/Transport'; import { ApplicationStart } from 'kibana/public'; import { Assign } from '@kbn/utility-types'; import { BfetchPublicSetup } from 'src/plugins/bfetch/public'; import Boom from '@hapi/boom'; +import { Bucket } from '@elastic/elasticsearch/api/types'; import { ConfigDeprecationProvider } from '@kbn/config'; import { CoreSetup } from 'src/core/public'; import { CoreSetup as CoreSetup_2 } from 'kibana/public'; @@ -46,6 +48,7 @@ import { Href } from 'history'; import { HttpSetup } from 'kibana/public'; import { IAggConfigs as IAggConfigs_2 } from 'src/plugins/data/public'; import { IconType } from '@elastic/eui'; +import { IEsSearchResponse as IEsSearchResponse_2 } from 'src/plugins/data/public'; import { IncomingHttpHeaders } from 'http'; import { InjectedIntl } from '@kbn/i18n/react'; import { ISearchOptions as ISearchOptions_2 } from 'src/plugins/data/public'; @@ -74,6 +77,7 @@ import * as PropTypes from 'prop-types'; import { PublicContract } from '@kbn/utility-types'; import { PublicMethodsOf } from '@kbn/utility-types'; import { PublicUiSettingsParams } from 'src/core/server/types'; +import { RangeFilter as RangeFilter_2 } from 'src/plugins/data/public'; import React from 'react'; import * as React_3 from 'react'; import { RecursiveReadonly } from '@kbn/utility-types'; @@ -152,9 +156,13 @@ export class AggConfig { // (undocumented) getTimeRange(): import("../../../public").TimeRange | undefined; // (undocumented) + getTimeShift(): undefined | moment.Duration; + // (undocumented) getValue(bucket: any): any; getValueBucketPath(): string; // (undocumented) + hasTimeShift(): boolean; + // (undocumented) id: string; // (undocumented) isFilterable(): boolean; @@ -253,6 +261,39 @@ export class AggConfigs { getResponseAggById(id: string): AggConfig | undefined; getResponseAggs(): AggConfig[]; // (undocumented) + getSearchSourceTimeFilter(forceNow?: Date): RangeFilter_2[] | { + meta: { + index: string | undefined; + params: {}; + alias: string; + disabled: boolean; + negate: boolean; + }; + query: { + bool: { + should: { + bool: { + filter: { + range: { + [x: string]: { + gte: string; + lte: string; + }; + }; + }[]; + }; + }[]; + minimum_should_match: number; + }; + }; + }[]; + // (undocumented) + getTimeShiftInterval(): moment.Duration | undefined; + // (undocumented) + getTimeShifts(): Record; + // (undocumented) + hasTimeShifts(): boolean; + // (undocumented) hierarchical?: boolean; // (undocumented) indexPattern: IndexPattern; @@ -260,6 +301,8 @@ export class AggConfigs { // (undocumented) onSearchRequestStart(searchSource: ISearchSource_2, options?: ISearchOptions_2): Promise<[unknown, unknown, unknown, unknown, unknown, unknown, unknown, unknown, unknown, unknown]>; // (undocumented) + postFlightTransform(response: IEsSearchResponse_2): IEsSearchResponse_2; + // (undocumented) setTimeFields(timeFields: string[] | undefined): void; // (undocumented) setTimeRange(timeRange: TimeRange): void; diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index e66eaab672e1c1..f9f155db5751e1 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -6,8 +6,10 @@ import { $Values } from '@kbn/utility-types'; import { Adapters } from 'src/plugins/inspector/common'; +import { Aggregate } from '@elastic/elasticsearch/api/types'; import { Assign } from '@kbn/utility-types'; import { BfetchServerSetup } from 'src/plugins/bfetch/server'; +import { Bucket } from '@elastic/elasticsearch/api/types'; import { ConfigDeprecationProvider } from '@kbn/config'; import { CoreSetup } from 'src/core/server'; import { CoreSetup as CoreSetup_2 } from 'kibana/server'; @@ -32,6 +34,7 @@ import { ExpressionsServerSetup } from 'src/plugins/expressions/server'; import { ExpressionValueBoxed } from 'src/plugins/expressions/common'; import { FormatFactory as FormatFactory_2 } from 'src/plugins/data/common/field_formats/utils'; import { IAggConfigs as IAggConfigs_2 } from 'src/plugins/data/public'; +import { IEsSearchResponse as IEsSearchResponse_2 } from 'src/plugins/data/public'; import { IScopedClusterClient } from 'src/core/server'; import { ISearchOptions as ISearchOptions_2 } from 'src/plugins/data/public'; import { ISearchSource } from 'src/plugins/data/public'; @@ -52,6 +55,7 @@ import { Plugin as Plugin_2 } from 'src/core/server'; import { Plugin as Plugin_3 } from 'kibana/server'; import { PluginInitializerContext as PluginInitializerContext_2 } from 'src/core/server'; import { PublicMethodsOf } from '@kbn/utility-types'; +import { RangeFilter } from 'src/plugins/data/public'; import { RecursiveReadonly } from '@kbn/utility-types'; import { RequestAdapter } from 'src/plugins/inspector/common'; import { RequestHandlerContext } from 'src/core/server'; diff --git a/test/interpreter_functional/snapshots/session/combined_test0.json b/test/interpreter_functional/snapshots/session/combined_test0.json deleted file mode 100644 index 8f00d72df8ab3a..00000000000000 --- a/test/interpreter_functional/snapshots/session/combined_test0.json +++ /dev/null @@ -1 +0,0 @@ -{"filters":[],"query":[],"timeRange":null,"type":"kibana_context"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/combined_test1.json b/test/interpreter_functional/snapshots/session/combined_test1.json deleted file mode 100644 index 8f00d72df8ab3a..00000000000000 --- a/test/interpreter_functional/snapshots/session/combined_test1.json +++ /dev/null @@ -1 +0,0 @@ -{"filters":[],"query":[],"timeRange":null,"type":"kibana_context"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/combined_test2.json b/test/interpreter_functional/snapshots/session/combined_test2.json deleted file mode 100644 index 4870694e6adbce..00000000000000 --- a/test/interpreter_functional/snapshots/session/combined_test2.json +++ /dev/null @@ -1 +0,0 @@ -{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/combined_test3.json b/test/interpreter_functional/snapshots/session/combined_test3.json deleted file mode 100644 index 64b1052552c8ff..00000000000000 --- a/test/interpreter_functional/snapshots/session/combined_test3.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/final_output_test.json b/test/interpreter_functional/snapshots/session/final_output_test.json deleted file mode 100644 index 64b1052552c8ff..00000000000000 --- a/test/interpreter_functional/snapshots/session/final_output_test.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/metric_all_data.json b/test/interpreter_functional/snapshots/session/metric_all_data.json deleted file mode 100644 index 0e1e5a723373f8..00000000000000 --- a/test/interpreter_functional/snapshots/session/metric_all_data.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":2,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"},{"id":"col-2-1","meta":{"field":"bytes","index":"logstash-*","params":{"id":"bytes","params":null},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{"field":"bytes"},"schema":"metric","type":"max"},"type":"number"},"name":"Max bytes"}],"rows":[{"col-0-2":"200","col-1-1":12891,"col-2-1":19986},{"col-0-2":"404","col-1-1":696,"col-2-1":19881},{"col-0-2":"503","col-1-1":417,"col-2-1":0}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/metric_invalid_data.json b/test/interpreter_functional/snapshots/session/metric_invalid_data.json deleted file mode 100644 index c7b4a0325dc912..00000000000000 --- a/test/interpreter_functional/snapshots/session/metric_invalid_data.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"metrics":[{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[],"meta":{},"rows":[],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/metric_multi_metric_data.json b/test/interpreter_functional/snapshots/session/metric_multi_metric_data.json deleted file mode 100644 index fc8622a818decb..00000000000000 --- a/test/interpreter_functional/snapshots/session/metric_multi_metric_data.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"metrics":[{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"},{"id":"col-2-1","meta":{"field":"bytes","index":"logstash-*","params":{"id":"bytes","params":null},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{"field":"bytes"},"schema":"metric","type":"max"},"type":"number"},"name":"Max bytes"}],"rows":[{"col-0-2":"200","col-1-1":12891,"col-2-1":19986},{"col-0-2":"404","col-1-1":696,"col-2-1":19881},{"col-0-2":"503","col-1-1":417,"col-2-1":0}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/metric_percentage_mode.json b/test/interpreter_functional/snapshots/session/metric_percentage_mode.json deleted file mode 100644 index 95c011f9259b9f..00000000000000 --- a/test/interpreter_functional/snapshots/session/metric_percentage_mode.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"metrics":[{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":1000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":true,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"},{"id":"col-2-1","meta":{"field":"bytes","index":"logstash-*","params":{"id":"bytes","params":null},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{"field":"bytes"},"schema":"metric","type":"max"},"type":"number"},"name":"Max bytes"}],"rows":[{"col-0-2":"200","col-1-1":12891,"col-2-1":19986},{"col-0-2":"404","col-1-1":696,"col-2-1":19881},{"col-0-2":"503","col-1-1":417,"col-2-1":0}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/metric_single_metric_data.json b/test/interpreter_functional/snapshots/session/metric_single_metric_data.json deleted file mode 100644 index f4a8cd1f14e18a..00000000000000 --- a/test/interpreter_functional/snapshots/session/metric_single_metric_data.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"metrics":[{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"},{"id":"col-2-1","meta":{"field":"bytes","index":"logstash-*","params":{"id":"bytes","params":null},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{"field":"bytes"},"schema":"metric","type":"max"},"type":"number"},"name":"Max bytes"}],"rows":[{"col-0-2":"200","col-1-1":12891,"col-2-1":19986},{"col-0-2":"404","col-1-1":696,"col-2-1":19881},{"col-0-2":"503","col-1-1":417,"col-2-1":0}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/partial_test_1.json b/test/interpreter_functional/snapshots/session/partial_test_1.json deleted file mode 100644 index 14c8428c6d432a..00000000000000 --- a/test/interpreter_functional/snapshots/session/partial_test_1.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/partial_test_2.json b/test/interpreter_functional/snapshots/session/partial_test_2.json deleted file mode 100644 index 64b1052552c8ff..00000000000000 --- a/test/interpreter_functional/snapshots/session/partial_test_2.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/partial_test_3.json b/test/interpreter_functional/snapshots/session/partial_test_3.json deleted file mode 100644 index e011b69de20227..00000000000000 --- a/test/interpreter_functional/snapshots/session/partial_test_3.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"region_map_vis","type":"render","value":{"visConfig":{"addTooltip":true,"bucket":{"accessor":0},"colorSchema":"Yellow to Red","isDisplayWarning":true,"legendPosition":"bottomright","mapCenter":[0,0],"mapZoom":2,"metric":{"accessor":1,"format":{"id":"number"}},"outlineWeight":1,"selectedJoinField":{},"selectedLayer":{},"showAllShapes":true,"wms":{}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"region_map"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/step_output_test0.json b/test/interpreter_functional/snapshots/session/step_output_test0.json deleted file mode 100644 index 8f00d72df8ab3a..00000000000000 --- a/test/interpreter_functional/snapshots/session/step_output_test0.json +++ /dev/null @@ -1 +0,0 @@ -{"filters":[],"query":[],"timeRange":null,"type":"kibana_context"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/step_output_test1.json b/test/interpreter_functional/snapshots/session/step_output_test1.json deleted file mode 100644 index 8f00d72df8ab3a..00000000000000 --- a/test/interpreter_functional/snapshots/session/step_output_test1.json +++ /dev/null @@ -1 +0,0 @@ -{"filters":[],"query":[],"timeRange":null,"type":"kibana_context"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/step_output_test2.json b/test/interpreter_functional/snapshots/session/step_output_test2.json deleted file mode 100644 index 4870694e6adbce..00000000000000 --- a/test/interpreter_functional/snapshots/session/step_output_test2.json +++ /dev/null @@ -1 +0,0 @@ -{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/step_output_test3.json b/test/interpreter_functional/snapshots/session/step_output_test3.json deleted file mode 100644 index 64b1052552c8ff..00000000000000 --- a/test/interpreter_functional/snapshots/session/step_output_test3.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/tagcloud_all_data.json b/test/interpreter_functional/snapshots/session/tagcloud_all_data.json deleted file mode 100644 index 073fca760b9a24..00000000000000 --- a/test/interpreter_functional/snapshots/session/tagcloud_all_data.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/tagcloud_fontsize.json b/test/interpreter_functional/snapshots/session/tagcloud_fontsize.json deleted file mode 100644 index 93f8d8a27d2334..00000000000000 --- a/test/interpreter_functional/snapshots/session/tagcloud_fontsize.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":40,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":20,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/tagcloud_invalid_data.json b/test/interpreter_functional/snapshots/session/tagcloud_invalid_data.json deleted file mode 100644 index 0c50947beca975..00000000000000 --- a/test/interpreter_functional/snapshots/session/tagcloud_invalid_data.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[],"meta":{},"rows":[],"type":"datatable"},"visParams":{"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/tagcloud_metric_data.json b/test/interpreter_functional/snapshots/session/tagcloud_metric_data.json deleted file mode 100644 index e8c47efdbe622b..00000000000000 --- a/test/interpreter_functional/snapshots/session/tagcloud_metric_data.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/tagcloud_options.json b/test/interpreter_functional/snapshots/session/tagcloud_options.json deleted file mode 100644 index 38683082975f8f..00000000000000 --- a/test/interpreter_functional/snapshots/session/tagcloud_options.json +++ /dev/null @@ -1 +0,0 @@ -{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"multiple","scale":"log","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts b/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts new file mode 100644 index 00000000000000..7971a7b8ee13a8 --- /dev/null +++ b/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts @@ -0,0 +1,198 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import expect from '@kbn/expect'; +import { Datatable } from 'src/plugins/expressions'; +import { ExpectExpression, expectExpressionProvider } from './helpers'; +import { FtrProviderContext } from '../../../functional/ftr_provider_context'; + +function getCell(esaggsResult: any, row: number, column: number): unknown | undefined { + const columnId = esaggsResult?.columns[column]?.id; + if (!columnId) { + return; + } + return esaggsResult?.rows[row]?.[columnId]; +} + +function checkShift(rows: Datatable['rows'], columns: Datatable['columns'], metricIndex = 1) { + rows.shift(); + rows.pop(); + rows.forEach((_, index) => { + if (index < rows.length - 1) { + expect(getCell({ rows, columns }, index, metricIndex + 1)).to.be( + getCell({ rows, columns }, index + 1, metricIndex) + ); + } + }); +} + +export default function ({ + getService, + updateBaselines, +}: FtrProviderContext & { updateBaselines: boolean }) { + let expectExpression: ExpectExpression; + + describe('esaggs timeshift tests', () => { + before(() => { + expectExpression = expectExpressionProvider({ getService, updateBaselines }); + }); + + const timeRange = { + from: '2015-09-21T00:00:00Z', + to: '2015-09-22T00:00:00Z', + }; + + it('shifts single metric', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggCount id="1" enabled=true schema="metric" timeShift="1d"} + `; + const result = await expectExpression('esaggs_shift_single_metric', expression).getResponse(); + expect(getCell(result, 0, 0)).to.be(4763); + }); + + it('shifts multiple metrics', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggCount id="1" enabled=true schema="metric" timeShift="12h"} + aggs={aggCount id="2" enabled=true schema="metric" timeShift="1d"} + aggs={aggCount id="3" enabled=true schema="metric"} + `; + const result = await expectExpression('esaggs_shift_multi_metric', expression).getResponse(); + expect(getCell(result, 0, 0)).to.be(4629); + expect(getCell(result, 0, 1)).to.be(4763); + expect(getCell(result, 0, 2)).to.be(4618); + }); + + it('shifts date histogram', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggDateHistogram id="1" enabled=true schema="bucket" field="@timestamp" interval="1h"} + aggs={aggAvg id="2" field="bytes" enabled=true schema="metric" timeShift="1h"} + aggs={aggAvg id="3" field="bytes" enabled=true schema="metric"} + `; + const result: Datatable = await expectExpression( + 'esaggs_shift_date_histogram', + expression + ).getResponse(); + expect(result.rows.length).to.be(25); + checkShift(result.rows, result.columns); + }); + + it('shifts terms', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggTerms id="1" field="geo.src" size="3" enabled=true schema="bucket" orderAgg={aggCount id="order" enabled=true schema="metric"} otherBucket=true} + aggs={aggAvg id="2" field="bytes" enabled=true schema="metric" timeShift="1d"} + aggs={aggAvg id="3" field="bytes" enabled=true schema="metric"} + `; + const result: Datatable = await expectExpression( + 'esaggs_shift_terms', + expression + ).getResponse(); + expect(result.rows).to.eql([ + { + 'col-0-1': 'CN', + 'col-1-2': 40, + 'col-2-3': 5806.404352806415, + }, + { + 'col-0-1': 'IN', + 'col-1-2': 7901, + 'col-2-3': 5838.315923566879, + }, + { + 'col-0-1': 'US', + 'col-1-2': 7440, + 'col-2-3': 5614.142857142857, + }, + { + 'col-0-1': '__other__', + 'col-1-2': 5766.575645756458, + 'col-2-3': 5742.1265576323985, + }, + ]); + }); + + it('shifts histogram', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggHistogram id="1" field="bytes" interval=5000 enabled=true schema="bucket"} + aggs={aggCount id="2" enabled=true schema="metric"} + aggs={aggCount id="3" enabled=true schema="metric" timeShift="6h"} + `; + const result: Datatable = await expectExpression( + 'esaggs_shift_histogram', + expression + ).getResponse(); + expect(result.rows).to.eql([ + { + 'col-0-1': 0, + 'col-1-2': 2020, + 'col-2-3': 2036, + }, + { + 'col-0-1': 5000, + 'col-1-2': 2360, + 'col-2-3': 2358, + }, + { + 'col-0-1': 10000, + 'col-1-2': 126, + 'col-2-3': 127, + }, + { + 'col-0-1': 15000, + 'col-1-2': 112, + 'col-2-3': 108, + }, + ]); + }); + + it('shifts pipeline aggs', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggBucketSum id="1" enabled=true schema="metric" customBucket={aggTerms id="2" enabled="true" schema="bucket" field="geo.src" size="3"} customMetric={aggCount id="4" enabled="true" schema="metric"}} + aggs={aggBucketSum id="5" enabled=true schema="metric" timeShift="1d" customBucket={aggTerms id="6" enabled="true" schema="bucket" field="geo.src" size="3"} customMetric={aggCount id="7" enabled="true" schema="metric"}} + `; + const result: Datatable = await expectExpression( + 'esaggs_shift_pipeline_aggs', + expression + ).getResponse(); + expect(getCell(result, 0, 0)).to.be(2050); + expect(getCell(result, 0, 1)).to.be(2053); + }); + + it('metrics at all levels should work for single shift', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} metricsAtAllLevels=true + aggs={aggCount id="1" enabled=true schema="metric" timeShift="1d"} + `; + const result = await expectExpression('esaggs_shift_single_metric', expression).getResponse(); + expect(getCell(result, 0, 0)).to.be(4763); + }); + + it('metrics at all levels should fail for multiple shifts', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} metricsAtAllLevels=true + aggs={aggCount id="1" enabled=true schema="metric" timeShift="1d"} + aggs={aggCount id="2" enabled=true schema="metric"} + `; + const result = await expectExpression('esaggs_shift_single_metric', expression).getResponse(); + expect(result.type).to.be('error'); + }); + }); +} diff --git a/test/interpreter_functional/test_suites/run_pipeline/index.ts b/test/interpreter_functional/test_suites/run_pipeline/index.ts index c33a87a93b9038..18d20c97be81e9 100644 --- a/test/interpreter_functional/test_suites/run_pipeline/index.ts +++ b/test/interpreter_functional/test_suites/run_pipeline/index.ts @@ -36,5 +36,6 @@ export default function ({ getService, getPageObjects, loadTestFile }: FtrProvid loadTestFile(require.resolve('./tag_cloud')); loadTestFile(require.resolve('./metric')); loadTestFile(require.resolve('./esaggs')); + loadTestFile(require.resolve('./esaggs_timeshift')); }); } diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx index 3c959693d45aef..a54901a2a2fe1d 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/frame_layout.tsx @@ -45,7 +45,7 @@ export function FrameLayout(props: FrameLayoutProps) { {props.workspacePanel} - {/* {props.suggestionsPanel} */} + {props.suggestionsPanel}
{localState.configurationValidationError[0].fixAction && activeDatasourceId && ( { const newState = await localState.configurationValidationError?.[0].fixAction?.newState( framePublicAPI diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index a469924fac0543..55ab5bee747773 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -522,7 +522,11 @@ export function DimensionEditor(props: DimensionEditorProps) { }, showInPopover: Boolean( operationDefinitionMap[selectedColumn.operationType].shiftable && - selectedColumn.timeShift === undefined + selectedColumn.timeShift === undefined && + (currentIndexPattern.timeFieldName || + Object.values(state.layers[layerId].columns).some( + (col) => col.operationType === 'date_histogram' + )) ), inlineElement: operationDefinitionMap[selectedColumn.operationType].shiftable && diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx index f80b12aecabdef..4b84c9af31f35e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx @@ -34,6 +34,7 @@ import { getFieldByNameFactory } from '../pure_helpers'; import { DimensionEditor } from './dimension_editor'; import { AdvancedOptions } from './advanced_options'; import { Filtering } from './filtering'; +import { TimeShift } from './time_shift'; jest.mock('../loader'); jest.mock('../query_input', () => ({ @@ -1291,6 +1292,199 @@ describe('IndexPatternDimensionEditorPanel', () => { }); }); + describe('time shift', () => { + function getProps(colOverrides: Partial) { + return { + ...defaultProps, + state: getStateWithColumns({ + datecolumn: { + dataType: 'date', + isBucketed: true, + label: '', + customLabel: true, + operationType: 'date_histogram', + sourceField: 'ts', + params: { + interval: '1d', + }, + }, + col2: { + dataType: 'number', + isBucketed: false, + label: 'Count of records', + operationType: 'count', + sourceField: 'Records', + ...colOverrides, + } as IndexPatternColumn, + }), + columnId: 'col2', + }; + } + + it('should not show custom options if time scaling is not available', () => { + const props = getProps({ + operationType: 'terms', + sourceField: 'bytes', + }); + wrapper = shallow( + + ); + expect( + wrapper + .find(DimensionEditor) + .dive() + .find(AdvancedOptions) + .dive() + .find('[data-test-subj="indexPattern-time-shift-enable"]') + ).toHaveLength(0); + }); + + it('should show custom options if time shift is available', () => { + wrapper = shallow(); + expect( + wrapper + .find(DimensionEditor) + .dive() + .find(AdvancedOptions) + .dive() + .find('[data-test-subj="indexPattern-time-shift-enable"]') + ).toHaveLength(1); + }); + + it('should show current time shift if set', () => { + wrapper = mount(); + expect(wrapper.find(TimeShift).find(EuiComboBox).prop('selectedOptions')[0].value).toEqual( + '1d' + ); + }); + + it('should allow to set time shift initially', () => { + const props = getProps({}); + wrapper = shallow(); + wrapper + .find(DimensionEditor) + .dive() + .find(AdvancedOptions) + .dive() + .find('[data-test-subj="indexPattern-time-shift-enable"]') + .prop('onClick')!({} as MouseEvent); + expect(props.setState).toHaveBeenCalledWith( + { + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeShift: '', + }), + }, + }, + }, + }, + { shouldRemoveDimension: false, shouldReplaceDimension: true } + ); + }); + + it('should carry over time shift to other operation if possible', () => { + const props = getProps({ + timeShift: '1d', + sourceField: 'bytes', + operationType: 'sum', + label: 'Sum of bytes per hour', + }); + wrapper = mount(); + wrapper + .find('button[data-test-subj="lns-indexPatternDimension-count incompatible"]') + .simulate('click'); + expect(props.setState).toHaveBeenCalledWith( + { + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeShift: '1d', + }), + }, + }, + }, + }, + { shouldRemoveDimension: false, shouldReplaceDimension: true } + ); + }); + + it('should allow to change time shift', () => { + const props = getProps({ + timeShift: '1d', + }); + wrapper = mount(); + wrapper.find(TimeShift).find(EuiComboBox).prop('onCreateOption')!('1h', []); + expect(props.setState).toHaveBeenCalledWith( + { + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeShift: '1h', + }), + }, + }, + }, + }, + { shouldRemoveDimension: false, shouldReplaceDimension: true } + ); + }); + + it('should allow to time shift', () => { + const props = getProps({ + timeShift: '1h', + }); + wrapper = mount(); + wrapper + .find('[data-test-subj="indexPattern-time-shift-remove"]') + .find(EuiButtonIcon) + .prop('onClick')!( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + {} as any + ); + expect(props.setState).toHaveBeenCalledWith( + { + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeShift: undefined, + }), + }, + }, + }, + }, + { shouldRemoveDimension: false, shouldReplaceDimension: true } + ); + }); + }); + describe('filtering', () => { function getProps(colOverrides: Partial) { return { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index 534051ac02c162..6271152f403cc4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -141,6 +141,9 @@ export function TimeShift({ const dateHistogramColumn = layer.columnOrder.find( (colId) => layer.columns[colId].operationType === 'date_histogram' ); + if (!dateHistogramColumn && !indexPattern.timeFieldName) { + return null; + } if (dateHistogramColumn && activeData && activeData[layerId] && activeData[layerId]) { const column = activeData[layerId].columns.find((col) => col.id === dateHistogramColumn); if (column) { @@ -169,11 +172,7 @@ export function TimeShift({ } const parsedLocalValue = localValue && parseTimeShift(localValue); - const localValueTooSmall = parsedLocalValue && isValueTooSmall(parsedLocalValue); - const localValueNotMultiple = parsedLocalValue && isValueNotMultiple(parsedLocalValue); - const isLocalValueInvalid = Boolean( - parsedLocalValue === 'invalid' || localValueTooSmall || localValueNotMultiple - ); + const isLocalValueInvalid = Boolean(parsedLocalValue === 'invalid'); function getSelectedOption() { if (!localValue) return []; @@ -207,19 +206,6 @@ export function TimeShift({ helpText={i18n.translate('xpack.lens.indexPattern.timeShift.help', { defaultMessage: 'Time shift is specified by a number followed by a time unit', })} - error={ - localValueTooSmall - ? i18n.translate('xpack.lens.indexPattern.timeShift.tooSmallHelp', { - defaultMessage: - 'Time shift should to be larger than the date histogram interval. Either increase time shift or specify smaller interval in date histogram', - }) - : localValueNotMultiple - ? i18n.translate('xpack.lens.indexPattern.timeShift.noMultipleHelp', { - defaultMessage: - 'Time shift should be a multiple of the date histogram interval. Either adjust time shift or date histogram interval', - }) - : undefined - } isInvalid={isLocalValueInvalid} > diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index c291c7ab3eac08..d91055d01ade90 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -474,6 +474,43 @@ describe('IndexPattern Data Source', () => { expect(ast.chain[0].arguments.timeFields).toEqual(['timestamp', 'another_datefield']); }); + it('should pass time shift parameter to metric agg functions', async () => { + const queryBaseState: IndexPatternBaseState = { + currentIndexPatternId: '1', + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col2', 'col1'], + columns: { + col1: { + label: 'Count of records', + dataType: 'number', + isBucketed: false, + sourceField: 'Records', + operationType: 'count', + timeShift: '1d', + }, + col2: { + label: 'Date', + dataType: 'date', + isBucketed: true, + operationType: 'date_histogram', + sourceField: 'timestamp', + params: { + interval: 'auto', + }, + }, + }, + }, + }, + }; + + const state = enrichBaseState(queryBaseState); + + const ast = indexPatternDatasource.toExpression(state, 'first') as Ast; + expect((ast.chain[0].arguments.aggs[1] as Ast).chain[0].arguments.timeShift).toEqual(['1d']); + }); + it('should wrap filtered metrics in filtered metric aggregation', async () => { const queryBaseState: IndexPatternBaseState = { currentIndexPatternId: '1', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx index c045b1a34d3a54..e784b00c7e10f5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useEffect, useState } from 'react'; +import React, { useState } from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -22,8 +22,6 @@ import { EuiSwitchEvent, EuiTextColor, } from '@elastic/eui'; -import moment from 'moment'; -import { parseTimeShift } from '../../../../../../../src/plugins/data/common'; import { updateColumnParam } from '../layer_helpers'; import { OperationDefinition, ParamEditorProps } from './index'; import { FieldBasedIndexPatternColumn } from './column_types'; @@ -188,35 +186,12 @@ export const dateHistogramOperation: OperationDefinition< dateRange, data, indexPattern, - activeData, }: ParamEditorProps) { - const [localValue, setLocalValue] = useState(currentColumn?.params.interval); - useEffect(() => { - setLocalValue(currentColumn?.params.interval); - }, [currentColumn?.params.interval]); - - function isIntervalTooSmall(interval: moment.Duration | null) { - if (!interval) { - return false; - } - const timeShifts = Object.values(layer.columns) - .filter((col) => col.timeShift && col.timeShift !== 'previous') - .map((col) => col.timeShift); - return timeShifts.some((shift) => { - if (!shift) return false; - const parsedShift = parseTimeShift(shift); - return ( - typeof parsedShift === 'object' && - parsedShift.asMilliseconds() < interval.asMilliseconds() - ); - }); - } - const field = currentColumn && indexPattern.getFieldByName(currentColumn.sourceField); const intervalIsRestricted = field!.aggregationRestrictions && field!.aggregationRestrictions.date_histogram; - const interval = parseInterval(localValue); + const interval = parseInterval(currentColumn.params.interval); // We force the interval value to 1 if it's empty, since that is the ES behavior, // and the isValidInterval function doesn't handle the empty case properly. Fixing @@ -234,24 +209,13 @@ export const dateHistogramOperation: OperationDefinition< updateLayer(updateColumnParam({ layer, columnId, paramName: 'interval', value })); } - const getIntervalAsDuration = (newInterval: typeof interval) => { - const isCalendarInterval = calendarOnlyIntervals.has(newInterval.unit); - const value = `${isCalendarInterval ? '1' : newInterval.value}${newInterval.unit || 'd'}`; - return search.aggs.parseInterval(value); - }; - - const setInterval = (newInterval: typeof interval, storeLocally: boolean = false) => { + const setInterval = (newInterval: typeof interval) => { const isCalendarInterval = calendarOnlyIntervals.has(newInterval.unit); const value = `${isCalendarInterval ? '1' : newInterval.value}${newInterval.unit || 'd'}`; - if (storeLocally) { - setLocalValue(value); - } else { - updateLayer(updateColumnParam({ layer, columnId, paramName: 'interval', value })); - } + updateLayer(updateColumnParam({ layer, columnId, paramName: 'interval', value })); }; - const currentIntervalTooSmall = isIntervalTooSmall(getIntervalAsDuration(interval)); return ( <> {!intervalIsRestricted && ( @@ -273,14 +237,6 @@ export const dateHistogramOperation: OperationDefinition< })} fullWidth display="rowCompressed" - isInvalid={currentIntervalTooSmall} - error={ - currentIntervalTooSmall && - i18n.translate('xpack.lens.indexPattern.intervalTooSmallError', { - defaultMessage: - 'Interval is larger than configured time shift - make sure to keep date histogram interval smaller than time shifts', - }) - } > {intervalIsRestricted ? ( @@ -324,8 +279,7 @@ export const dateHistogramOperation: OperationDefinition< ...interval, unit: e.target.value, }; - const tooSmall = isIntervalTooSmall(getIntervalAsDuration(newInterval)); - setInterval(newInterval, tooSmall); + setInterval(newInterval); }} isInvalid={!isValid} options={[ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx index 2e7307f6a2ec4a..3e127ffe906e32 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx @@ -9,7 +9,12 @@ import React from 'react'; import { act } from 'react-dom/test-utils'; import { shallow, mount } from 'enzyme'; import { EuiFieldNumber, EuiSelect, EuiSwitch, EuiSwitchEvent } from '@elastic/eui'; -import type { IUiSettingsClient, SavedObjectsClientContract, HttpSetup } from 'kibana/public'; +import type { + IUiSettingsClient, + SavedObjectsClientContract, + HttpSetup, + CoreStart, +} from 'kibana/public'; import type { IStorageWrapper } from 'src/plugins/kibana_utils/public'; import { dataPluginMock } from '../../../../../../../../src/plugins/data/public/mocks'; import { createMockedIndexPattern } from '../../../mocks'; @@ -17,6 +22,7 @@ import { ValuesInput } from './values_input'; import type { TermsIndexPatternColumn } from '.'; import { termsOperation } from '../index'; import { IndexPattern, IndexPatternLayer } from '../../../types'; +import { FramePublicAPI } from '../../../../types'; const uiSettingsMock = {} as IUiSettingsClient; @@ -983,8 +989,8 @@ describe('terms', () => { indexPatternId: '', }; }); - it('returns undefined if sourceField exists in index pattern', () => { - expect(termsOperation.getErrorMessage!(layer, 'col1', indexPattern)).toEqual(undefined); + it('returns empty array', () => { + expect(termsOperation.getErrorMessage!(layer, 'col1', indexPattern)).toEqual([]); }); it('returns error message if the sourceField does not exist in index pattern', () => { layer = { @@ -1000,5 +1006,98 @@ describe('terms', () => { 'Field notExisting was not found', ]); }); + + describe('time shift error', () => { + beforeEach(() => { + layer = { + ...layer, + columnOrder: ['col1', 'col2', 'col3'], + columns: { + ...layer.columns, + col2: { + dataType: 'number', + isBucketed: false, + operationType: 'count', + label: 'Count', + sourceField: 'document', + }, + col3: { + dataType: 'number', + isBucketed: false, + operationType: 'count', + label: 'Count', + sourceField: 'document', + timeShift: '1d', + }, + }, + }; + }); + it('returns error message if two time shifts are used together with terms', () => { + expect(termsOperation.getErrorMessage!(layer, 'col1', indexPattern)).toEqual([ + expect.objectContaining({ + message: + "Can't use multiple time shifts in a single layer together with dynamic top values. Either use the same time shift for all metrics or use filters instead of top values.", + }), + ]); + }); + it('returns fix action which calls field information endpoint and creates a pinned top values', async () => { + const errorMessage = termsOperation.getErrorMessage!(layer, 'col1', indexPattern)![0]; + const fixAction = (typeof errorMessage === 'object' + ? errorMessage.fixAction!.newState + : undefined)!; + const coreMock = ({ + uiSettings: { + get: () => undefined, + }, + http: { + post: jest.fn(() => + Promise.resolve({ + topValues: { + buckets: [ + { + key: 'A', + }, + { + key: 'B', + }, + ], + }, + }) + ), + }, + } as unknown) as CoreStart; + const newLayer = await fixAction(coreMock, ({ + query: { language: 'kuery', query: 'a: b' }, + filters: [], + dateRange: { + fromDate: '2020', + toDate: '2021', + }, + } as unknown) as FramePublicAPI); + expect(newLayer.columns.col1).toEqual( + expect.objectContaining({ + operationType: 'filters', + params: { + filters: [ + { + input: { + language: 'kuery', + query: 'bytes: "A"', + }, + label: 'A', + }, + { + input: { + language: 'kuery', + query: 'bytes: "B"', + }, + label: 'B', + }, + ], + }, + }) + ); + }); + }); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts index a3e2a6e567dedc..6d020d21d016c0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts @@ -61,7 +61,7 @@ function getExpressionForLayer( rootDef.input === 'fullReference' && column.timeShift ) { - // inherit filter to all referenced operations + // inherit time shift to all referenced operations column.references.forEach((referenceColumnId) => { const referencedColumn = columns[referenceColumnId]; const referenceDef = operationDefinitionMap[column.operationType]; diff --git a/x-pack/test/functional/apps/lens/index.ts b/x-pack/test/functional/apps/lens/index.ts index bfb0aad7177f4d..773dfc8c48729b 100644 --- a/x-pack/test/functional/apps/lens/index.ts +++ b/x-pack/test/functional/apps/lens/index.ts @@ -36,6 +36,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./persistent_context')); loadTestFile(require.resolve('./colors')); loadTestFile(require.resolve('./chart_data')); + loadTestFile(require.resolve('./time_shift')); loadTestFile(require.resolve('./drag_and_drop')); loadTestFile(require.resolve('./lens_reporting')); loadTestFile(require.resolve('./lens_tagging')); diff --git a/x-pack/test/functional/apps/lens/time_shift.ts b/x-pack/test/functional/apps/lens/time_shift.ts new file mode 100644 index 00000000000000..770ab3cd2c2d6b --- /dev/null +++ b/x-pack/test/functional/apps/lens/time_shift.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const PageObjects = getPageObjects(['visualize', 'lens', 'common', 'header']); + + describe('time shift', () => { + it('should able to configure a shifted metric', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + await PageObjects.lens.switchToVisualization('lnsDatatable'); + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_rows > lns-empty-dimension', + operation: 'date_histogram', + field: '@timestamp', + }); + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_metrics > lns-empty-dimension', + operation: 'median', + field: 'bytes', + }); + await PageObjects.lens.openDimensionEditor('lnsDatatable_metrics > lns-dimensionTrigger'); + await PageObjects.lens.enableTimeShift(); + await PageObjects.lens.setTimeShift('6h'); + + await PageObjects.lens.waitForVisualization(); + expect(await PageObjects.lens.getDatatableCellText(0, 1)).to.eql('5,994'); + }); + + it('should able to configure a regular metric next to a shifted metric', async () => { + await PageObjects.lens.closeDimensionEditor(); + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_metrics > lns-empty-dimension', + operation: 'average', + field: 'bytes', + }); + + await PageObjects.lens.waitForVisualization(); + + expect(await PageObjects.lens.getDatatableCellText(2, 1)).to.eql('5,994'); + expect(await PageObjects.lens.getDatatableCellText(2, 2)).to.eql('5,722.622'); + }); + + it('should show an error if terms is used and provide a fix action', async () => { + await PageObjects.lens.configureDimension({ + dimension: 'lnsDatatable_rows > lns-empty-dimension', + operation: 'terms', + field: 'ip', + }); + + expect(await PageObjects.lens.hasFixAction()).to.be(true); + await PageObjects.lens.useFixAction(); + + expect(await PageObjects.lens.getDatatableCellText(2, 2)).to.eql('6,976'); + expect(await PageObjects.lens.getDatatableCellText(2, 3)).to.eql('4,182.5'); + + expect(await PageObjects.lens.getDatatableHeaderText(0)).to.eql('Pinned top values of ip'); + }); + }); +} diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index 100ed8e079d379..ffa86d31612176 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -350,6 +350,25 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont }); }, + async enableTimeShift() { + await testSubjects.click('indexPattern-advanced-popover'); + await retry.try(async () => { + await testSubjects.click('indexPattern-time-shift-enable'); + }); + }, + + async setTimeShift(shift: string) { + await comboBox.setCustom('indexPattern-dimension-time-shift', shift); + }, + + async hasFixAction() { + return await testSubjects.exists('errorFixAction'); + }, + + async useFixAction() { + await testSubjects.click('errorFixAction'); + }, + // closes the dimension editor flyout async closeDimensionEditor() { await retry.try(async () => { From 89dd546c85176fa50916960713c07281e28cb29c Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 11 May 2021 18:34:02 +0200 Subject: [PATCH 13/31] add back session snapshots --- .../interpreter_functional/snapshots/session/combined_test0.json | 1 + .../interpreter_functional/snapshots/session/combined_test1.json | 1 + .../interpreter_functional/snapshots/session/combined_test2.json | 1 + .../interpreter_functional/snapshots/session/combined_test3.json | 1 + .../snapshots/session/final_output_test.json | 1 + .../snapshots/session/metric_all_data.json | 1 + .../snapshots/session/metric_invalid_data.json | 1 + .../snapshots/session/metric_multi_metric_data.json | 1 + .../snapshots/session/metric_percentage_mode.json | 1 + .../snapshots/session/metric_single_metric_data.json | 1 + .../interpreter_functional/snapshots/session/partial_test_1.json | 1 + .../interpreter_functional/snapshots/session/partial_test_2.json | 1 + .../interpreter_functional/snapshots/session/partial_test_3.json | 1 + .../snapshots/session/step_output_test0.json | 1 + .../snapshots/session/step_output_test1.json | 1 + .../snapshots/session/step_output_test2.json | 1 + .../snapshots/session/step_output_test3.json | 1 + .../snapshots/session/tagcloud_all_data.json | 1 + .../snapshots/session/tagcloud_fontsize.json | 1 + .../snapshots/session/tagcloud_invalid_data.json | 1 + .../snapshots/session/tagcloud_metric_data.json | 1 + .../snapshots/session/tagcloud_options.json | 1 + 22 files changed, 22 insertions(+) create mode 100644 test/interpreter_functional/snapshots/session/combined_test0.json create mode 100644 test/interpreter_functional/snapshots/session/combined_test1.json create mode 100644 test/interpreter_functional/snapshots/session/combined_test2.json create mode 100644 test/interpreter_functional/snapshots/session/combined_test3.json create mode 100644 test/interpreter_functional/snapshots/session/final_output_test.json create mode 100644 test/interpreter_functional/snapshots/session/metric_all_data.json create mode 100644 test/interpreter_functional/snapshots/session/metric_invalid_data.json create mode 100644 test/interpreter_functional/snapshots/session/metric_multi_metric_data.json create mode 100644 test/interpreter_functional/snapshots/session/metric_percentage_mode.json create mode 100644 test/interpreter_functional/snapshots/session/metric_single_metric_data.json create mode 100644 test/interpreter_functional/snapshots/session/partial_test_1.json create mode 100644 test/interpreter_functional/snapshots/session/partial_test_2.json create mode 100644 test/interpreter_functional/snapshots/session/partial_test_3.json create mode 100644 test/interpreter_functional/snapshots/session/step_output_test0.json create mode 100644 test/interpreter_functional/snapshots/session/step_output_test1.json create mode 100644 test/interpreter_functional/snapshots/session/step_output_test2.json create mode 100644 test/interpreter_functional/snapshots/session/step_output_test3.json create mode 100644 test/interpreter_functional/snapshots/session/tagcloud_all_data.json create mode 100644 test/interpreter_functional/snapshots/session/tagcloud_fontsize.json create mode 100644 test/interpreter_functional/snapshots/session/tagcloud_invalid_data.json create mode 100644 test/interpreter_functional/snapshots/session/tagcloud_metric_data.json create mode 100644 test/interpreter_functional/snapshots/session/tagcloud_options.json diff --git a/test/interpreter_functional/snapshots/session/combined_test0.json b/test/interpreter_functional/snapshots/session/combined_test0.json new file mode 100644 index 00000000000000..8f00d72df8ab3a --- /dev/null +++ b/test/interpreter_functional/snapshots/session/combined_test0.json @@ -0,0 +1 @@ +{"filters":[],"query":[],"timeRange":null,"type":"kibana_context"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/combined_test1.json b/test/interpreter_functional/snapshots/session/combined_test1.json new file mode 100644 index 00000000000000..8f00d72df8ab3a --- /dev/null +++ b/test/interpreter_functional/snapshots/session/combined_test1.json @@ -0,0 +1 @@ +{"filters":[],"query":[],"timeRange":null,"type":"kibana_context"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/combined_test2.json b/test/interpreter_functional/snapshots/session/combined_test2.json new file mode 100644 index 00000000000000..4870694e6adbce --- /dev/null +++ b/test/interpreter_functional/snapshots/session/combined_test2.json @@ -0,0 +1 @@ +{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/combined_test3.json b/test/interpreter_functional/snapshots/session/combined_test3.json new file mode 100644 index 00000000000000..64b1052552c8ff --- /dev/null +++ b/test/interpreter_functional/snapshots/session/combined_test3.json @@ -0,0 +1 @@ +{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/final_output_test.json b/test/interpreter_functional/snapshots/session/final_output_test.json new file mode 100644 index 00000000000000..64b1052552c8ff --- /dev/null +++ b/test/interpreter_functional/snapshots/session/final_output_test.json @@ -0,0 +1 @@ +{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/metric_all_data.json b/test/interpreter_functional/snapshots/session/metric_all_data.json new file mode 100644 index 00000000000000..0e1e5a723373f8 --- /dev/null +++ b/test/interpreter_functional/snapshots/session/metric_all_data.json @@ -0,0 +1 @@ +{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":2,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"},{"id":"col-2-1","meta":{"field":"bytes","index":"logstash-*","params":{"id":"bytes","params":null},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{"field":"bytes"},"schema":"metric","type":"max"},"type":"number"},"name":"Max bytes"}],"rows":[{"col-0-2":"200","col-1-1":12891,"col-2-1":19986},{"col-0-2":"404","col-1-1":696,"col-2-1":19881},{"col-0-2":"503","col-1-1":417,"col-2-1":0}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/metric_invalid_data.json b/test/interpreter_functional/snapshots/session/metric_invalid_data.json new file mode 100644 index 00000000000000..c7b4a0325dc912 --- /dev/null +++ b/test/interpreter_functional/snapshots/session/metric_invalid_data.json @@ -0,0 +1 @@ +{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"metrics":[{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[],"meta":{},"rows":[],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/metric_multi_metric_data.json b/test/interpreter_functional/snapshots/session/metric_multi_metric_data.json new file mode 100644 index 00000000000000..fc8622a818decb --- /dev/null +++ b/test/interpreter_functional/snapshots/session/metric_multi_metric_data.json @@ -0,0 +1 @@ +{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"metrics":[{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"},{"id":"col-2-1","meta":{"field":"bytes","index":"logstash-*","params":{"id":"bytes","params":null},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{"field":"bytes"},"schema":"metric","type":"max"},"type":"number"},"name":"Max bytes"}],"rows":[{"col-0-2":"200","col-1-1":12891,"col-2-1":19986},{"col-0-2":"404","col-1-1":696,"col-2-1":19881},{"col-0-2":"503","col-1-1":417,"col-2-1":0}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/metric_percentage_mode.json b/test/interpreter_functional/snapshots/session/metric_percentage_mode.json new file mode 100644 index 00000000000000..95c011f9259b9f --- /dev/null +++ b/test/interpreter_functional/snapshots/session/metric_percentage_mode.json @@ -0,0 +1 @@ +{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"metrics":[{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":1000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":true,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"},{"id":"col-2-1","meta":{"field":"bytes","index":"logstash-*","params":{"id":"bytes","params":null},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{"field":"bytes"},"schema":"metric","type":"max"},"type":"number"},"name":"Max bytes"}],"rows":[{"col-0-2":"200","col-1-1":12891,"col-2-1":19986},{"col-0-2":"404","col-1-1":696,"col-2-1":19881},{"col-0-2":"503","col-1-1":417,"col-2-1":0}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/metric_single_metric_data.json b/test/interpreter_functional/snapshots/session/metric_single_metric_data.json new file mode 100644 index 00000000000000..f4a8cd1f14e18a --- /dev/null +++ b/test/interpreter_functional/snapshots/session/metric_single_metric_data.json @@ -0,0 +1 @@ +{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"metrics":[{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"},{"id":"col-2-1","meta":{"field":"bytes","index":"logstash-*","params":{"id":"bytes","params":null},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{"field":"bytes"},"schema":"metric","type":"max"},"type":"number"},"name":"Max bytes"}],"rows":[{"col-0-2":"200","col-1-1":12891,"col-2-1":19986},{"col-0-2":"404","col-1-1":696,"col-2-1":19881},{"col-0-2":"503","col-1-1":417,"col-2-1":0}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/partial_test_1.json b/test/interpreter_functional/snapshots/session/partial_test_1.json new file mode 100644 index 00000000000000..14c8428c6d432a --- /dev/null +++ b/test/interpreter_functional/snapshots/session/partial_test_1.json @@ -0,0 +1 @@ +{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/partial_test_2.json b/test/interpreter_functional/snapshots/session/partial_test_2.json new file mode 100644 index 00000000000000..64b1052552c8ff --- /dev/null +++ b/test/interpreter_functional/snapshots/session/partial_test_2.json @@ -0,0 +1 @@ +{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/partial_test_3.json b/test/interpreter_functional/snapshots/session/partial_test_3.json new file mode 100644 index 00000000000000..e011b69de20227 --- /dev/null +++ b/test/interpreter_functional/snapshots/session/partial_test_3.json @@ -0,0 +1 @@ +{"as":"region_map_vis","type":"render","value":{"visConfig":{"addTooltip":true,"bucket":{"accessor":0},"colorSchema":"Yellow to Red","isDisplayWarning":true,"legendPosition":"bottomright","mapCenter":[0,0],"mapZoom":2,"metric":{"accessor":1,"format":{"id":"number"}},"outlineWeight":1,"selectedJoinField":{},"selectedLayer":{},"showAllShapes":true,"wms":{}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"region_map"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/step_output_test0.json b/test/interpreter_functional/snapshots/session/step_output_test0.json new file mode 100644 index 00000000000000..8f00d72df8ab3a --- /dev/null +++ b/test/interpreter_functional/snapshots/session/step_output_test0.json @@ -0,0 +1 @@ +{"filters":[],"query":[],"timeRange":null,"type":"kibana_context"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/step_output_test1.json b/test/interpreter_functional/snapshots/session/step_output_test1.json new file mode 100644 index 00000000000000..8f00d72df8ab3a --- /dev/null +++ b/test/interpreter_functional/snapshots/session/step_output_test1.json @@ -0,0 +1 @@ +{"filters":[],"query":[],"timeRange":null,"type":"kibana_context"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/step_output_test2.json b/test/interpreter_functional/snapshots/session/step_output_test2.json new file mode 100644 index 00000000000000..4870694e6adbce --- /dev/null +++ b/test/interpreter_functional/snapshots/session/step_output_test2.json @@ -0,0 +1 @@ +{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/step_output_test3.json b/test/interpreter_functional/snapshots/session/step_output_test3.json new file mode 100644 index 00000000000000..64b1052552c8ff --- /dev/null +++ b/test/interpreter_functional/snapshots/session/step_output_test3.json @@ -0,0 +1 @@ +{"as":"metric_vis","type":"render","value":{"visConfig":{"dimensions":{"bucket":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"metrics":[{"accessor":1,"format":{"id":"number","params":{}},"type":"vis_dimension"}]},"metric":{"colorSchema":"Green to Red","colorsRange":[{"from":0,"to":10000,"type":"range"}],"invertColors":false,"labels":{"show":true},"metricColorMode":"None","percentageMode":false,"style":{"bgColor":false,"bgFill":"#000","fontSize":60,"labelColor":false,"subText":""},"useRanges":false}},"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visType":"metric"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/tagcloud_all_data.json b/test/interpreter_functional/snapshots/session/tagcloud_all_data.json new file mode 100644 index 00000000000000..073fca760b9a24 --- /dev/null +++ b/test/interpreter_functional/snapshots/session/tagcloud_all_data.json @@ -0,0 +1 @@ +{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/tagcloud_fontsize.json b/test/interpreter_functional/snapshots/session/tagcloud_fontsize.json new file mode 100644 index 00000000000000..93f8d8a27d2334 --- /dev/null +++ b/test/interpreter_functional/snapshots/session/tagcloud_fontsize.json @@ -0,0 +1 @@ +{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":40,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":20,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/tagcloud_invalid_data.json b/test/interpreter_functional/snapshots/session/tagcloud_invalid_data.json new file mode 100644 index 00000000000000..0c50947beca975 --- /dev/null +++ b/test/interpreter_functional/snapshots/session/tagcloud_invalid_data.json @@ -0,0 +1 @@ +{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[],"meta":{},"rows":[],"type":"datatable"},"visParams":{"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/tagcloud_metric_data.json b/test/interpreter_functional/snapshots/session/tagcloud_metric_data.json new file mode 100644 index 00000000000000..e8c47efdbe622b --- /dev/null +++ b/test/interpreter_functional/snapshots/session/tagcloud_metric_data.json @@ -0,0 +1 @@ +{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"single","scale":"linear","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file diff --git a/test/interpreter_functional/snapshots/session/tagcloud_options.json b/test/interpreter_functional/snapshots/session/tagcloud_options.json new file mode 100644 index 00000000000000..38683082975f8f --- /dev/null +++ b/test/interpreter_functional/snapshots/session/tagcloud_options.json @@ -0,0 +1 @@ +{"as":"tagloud_vis","type":"render","value":{"visData":{"columns":[{"id":"col-0-2","meta":{"field":"response.raw","index":"logstash-*","params":{"id":"terms","params":{"id":"string","missingBucketLabel":"Missing","otherBucketLabel":"Other"}},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"2","indexPatternId":"logstash-*","params":{"field":"response.raw","missingBucket":false,"missingBucketLabel":"Missing","order":"desc","orderBy":"1","otherBucket":false,"otherBucketLabel":"Other","size":4},"schema":"segment","type":"terms"},"type":"string"},"name":"response.raw: Descending"},{"id":"col-1-1","meta":{"field":null,"index":"logstash-*","params":{"id":"number"},"source":"esaggs","sourceParams":{"appliedTimeRange":null,"enabled":true,"id":"1","indexPatternId":"logstash-*","params":{},"schema":"metric","type":"count"},"type":"number"},"name":"Count"}],"rows":[{"col-0-2":"200","col-1-1":12891},{"col-0-2":"404","col-1-1":696},{"col-0-2":"503","col-1-1":417}],"type":"datatable"},"visParams":{"bucket":{"accessor":1,"format":{"id":"string","params":{}},"type":"vis_dimension"},"maxFontSize":72,"metric":{"accessor":0,"format":{"id":"string","params":{}},"type":"vis_dimension"},"minFontSize":18,"orientation":"multiple","scale":"log","showLabel":true},"visType":"tagcloud"}} \ No newline at end of file From bcc0cd199c32e677977683d18da45447c50f8a1b Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 12 May 2021 09:37:11 +0200 Subject: [PATCH 14/31] fix some bugs --- .../data/common/search/aggs/agg_configs.ts | 5 ++++- .../data/common/search/aggs/metrics/count.ts | 2 +- .../search/aggs/metrics/filtered_metric.ts | 2 +- .../dimension_panel/time_shift.tsx | 2 +- .../operations/layer_helpers.test.ts | 20 +++++++++++++++---- .../indexpattern_datasource/to_expression.ts | 1 + 6 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index bab7fab5f2248d..bdfbc8c90884e2 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -457,6 +457,7 @@ export class AggConfigs { } const aggCursor = transformedRawResponse.aggregations!; + const requestAggs = this.getRequestAggs(); const bucketAggs = this.aggs.filter( (agg) => agg.type.type === AggGroupNames.Buckets ) as IBucketAggConfig[]; @@ -480,7 +481,9 @@ export class AggConfigs { return; } else { // a sub-agg - const agg = this.byId(key); + const agg = requestAggs.find( + (requestAgg) => key.substr(0, String(requestAgg.id).length) === requestAgg.id + ); if (agg && agg.type.type === AggGroupNames.Metrics) { const timeShift = agg.getTimeShift(); if ( diff --git a/src/plugins/data/common/search/aggs/metrics/count.ts b/src/plugins/data/common/search/aggs/metrics/count.ts index fac1751290f70d..7e93fae7e77993 100644 --- a/src/plugins/data/common/search/aggs/metrics/count.ts +++ b/src/plugins/data/common/search/aggs/metrics/count.ts @@ -35,7 +35,7 @@ export const getCountMetricAgg = () => if (!timeShift) { return bucket.doc_count; } else { - return bucket[`doc_count_${timeShift.asMilliseconds()}`]; + return bucket[`doc_count_${timeShift.asMilliseconds()}`] ?? bucket.doc_count; } }, isScalable() { diff --git a/src/plugins/data/common/search/aggs/metrics/filtered_metric.ts b/src/plugins/data/common/search/aggs/metrics/filtered_metric.ts index aa2417bbf84156..00f47d31b0398d 100644 --- a/src/plugins/data/common/search/aggs/metrics/filtered_metric.ts +++ b/src/plugins/data/common/search/aggs/metrics/filtered_metric.ts @@ -42,7 +42,7 @@ export const getFilteredMetricAgg = () => { getValue(agg, bucket) { const customMetric = agg.getParam('customMetric'); const customBucket = agg.getParam('customBucket'); - return customMetric.getValue(bucket[customBucket.id]); + return bucket && bucket[customBucket.id] && customMetric.getValue(bucket[customBucket.id]); }, getValueBucketPath(agg) { const customBucket = agg.getParam('customBucket'); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index 6271152f403cc4..4d721ba13468af 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -201,7 +201,7 @@ export function TimeShift({ display="columnCompressed" fullWidth label={i18n.translate('xpack.lens.indexPattern.timeShift.label', { - defaultMessage: 'Time shift', + defaultMessage: 'Shift in time', })} helpText={i18n.translate('xpack.lens.indexPattern.timeShift.help', { defaultMessage: 'Time shift is specified by a number followed by a time unit', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts index c506e800d6d01c..581f9db0d40221 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts @@ -2378,7 +2378,10 @@ describe('state_helpers', () => { col1: { operationType: 'average' }, }, }, - indexPattern + indexPattern, + {}, + '1', + {} ); expect(mock).toHaveBeenCalled(); expect(errors).toHaveLength(1); @@ -2397,7 +2400,10 @@ describe('state_helpers', () => { { operationType: 'testReference', references: [] }, }, }, - indexPattern + indexPattern, + {}, + '1', + {} ); expect(mock).toHaveBeenCalled(); expect(errors).toHaveLength(1); @@ -2426,7 +2432,10 @@ describe('state_helpers', () => { col1: { operationType: 'testIncompleteReference' }, }, }, - indexPattern + indexPattern, + {}, + '1', + {} ); expect(savedRef).toHaveBeenCalled(); expect(incompleteRef).not.toHaveBeenCalled(); @@ -2448,7 +2457,10 @@ describe('state_helpers', () => { { operationType: 'testReference', references: [] }, }, }, - indexPattern + indexPattern, + {}, + '1', + {} ); expect(mock).toHaveBeenCalledWith( { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts index 6d020d21d016c0..c045777e47aa9e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts @@ -116,6 +116,7 @@ function getExpressionForLayer( }), ]), customMetric: buildExpression({ type: 'expression', chain: [aggAst] }), + timeShift: col.timeShift, } ).toAst(); } From e6f112db48283582fefdbfad64009e3c4ac5c400 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 12 May 2021 10:18:12 +0200 Subject: [PATCH 15/31] fix terms sorting --- src/plugins/data/common/search/aggs/buckets/terms.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/plugins/data/common/search/aggs/buckets/terms.ts b/src/plugins/data/common/search/aggs/buckets/terms.ts index 7356d434463581..b9329bcb25af37 100644 --- a/src/plugins/data/common/search/aggs/buckets/terms.ts +++ b/src/plugins/data/common/search/aggs/buckets/terms.ts @@ -180,7 +180,11 @@ export const getTermsBucketAgg = () => return; } - if (aggs?.hasTimeShifts() && aggs.timeRange) { + if ( + aggs?.hasTimeShifts() && + Object.keys(aggs?.getTimeShifts()).length > 1 && + aggs.timeRange + ) { const shift = orderAgg.getTimeShift(); orderAgg = aggs.createAggConfig( { From 34a2208bfcd1ad408881e58d136b8d6a06f9bbe2 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 17 May 2021 17:26:16 +0200 Subject: [PATCH 16/31] fix bug and add tests --- .../data/common/search/aggs/metrics/count.ts | 2 +- .../search/aggs/metrics/moving_avg_fn.ts | 7 + .../run_pipeline/esaggs_timeshift.ts | 177 +++++++++++++++++- .../operations/definitions/cardinality.tsx | 2 +- .../operations/definitions/count.tsx | 2 +- .../operations/definitions/last_value.tsx | 2 +- .../operations/definitions/metrics.tsx | 2 +- .../operations/definitions/percentile.tsx | 2 +- 8 files changed, 188 insertions(+), 8 deletions(-) diff --git a/src/plugins/data/common/search/aggs/metrics/count.ts b/src/plugins/data/common/search/aggs/metrics/count.ts index 7e93fae7e77993..fac1751290f70d 100644 --- a/src/plugins/data/common/search/aggs/metrics/count.ts +++ b/src/plugins/data/common/search/aggs/metrics/count.ts @@ -35,7 +35,7 @@ export const getCountMetricAgg = () => if (!timeShift) { return bucket.doc_count; } else { - return bucket[`doc_count_${timeShift.asMilliseconds()}`] ?? bucket.doc_count; + return bucket[`doc_count_${timeShift.asMilliseconds()}`]; } }, isScalable() { diff --git a/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.ts b/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.ts index 667c585226a528..63169b4d659bb9 100644 --- a/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.ts @@ -94,6 +94,13 @@ export const aggMovingAvg = (): FunctionDefinition => ({ defaultMessage: 'Represents a custom label for this aggregation', }), }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + }), + }, }, fn: (input, args) => { const { id, enabled, schema, ...rest } = args; diff --git a/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts b/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts index 7971a7b8ee13a8..13cfcde9305f7a 100644 --- a/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts +++ b/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts @@ -71,6 +71,40 @@ export default function ({ expect(getCell(result, 0, 2)).to.be(4618); }); + it('shifts single percentile', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggSinglePercentile id="1" enabled=true schema="metric" field="bytes" percentile=95} + aggs={aggSinglePercentile id="2" enabled=true schema="metric" field="bytes" percentile=95 timeShift="1d"} + `; + const result = await expectExpression( + 'esaggs_shift_single_percentile', + expression + ).getResponse(); + // percentile is not stable + expect(getCell(result, 0, 0)).to.be.within(10400, 10500); + expect(getCell(result, 0, 1)).to.be.within(10500, 10600); + }); + + it('shifts multiple percentiles', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggPercentiles id="1" enabled=true schema="metric" field="bytes" percents=5 percents=95} + aggs={aggPercentiles id="2" enabled=true schema="metric" field="bytes" percents=5 percents=95 timeShift="1d"} + `; + const result = await expectExpression( + 'esaggs_shift_multi_percentile', + expression + ).getResponse(); + // percentile is not stable + expect(getCell(result, 0, 0)).to.be.within(300, 400); + expect(getCell(result, 0, 1)).to.be.within(10400, 10500); + expect(getCell(result, 0, 2)).to.be.within(200, 300); + expect(getCell(result, 0, 3)).to.be.within(10500, 10600); + }); + it('shifts date histogram', async () => { const expression = ` kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} @@ -87,6 +121,53 @@ export default function ({ checkShift(result.rows, result.columns); }); + it('shifts filtered metrics', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggDateHistogram id="1" enabled=true schema="bucket" field="@timestamp" interval="1h"} + aggs={aggFilteredMetric + id="2" + customBucket={aggFilter + id="2-filter" + enabled=true + schema="bucket" + filter='{"language":"kuery","query":"geo.src:US"}' + } + customMetric={aggAvg id="3" + field="bytes" + enabled=true + schema="metric" + } + enabled=true + schema="metric" + timeShift="1h" + } + aggs={aggFilteredMetric + id="4" + customBucket={aggFilter + id="4-filter" + enabled=true + schema="bucket" + filter='{"language":"kuery","query":"geo.src:US"}' + } + customMetric={aggAvg id="5" + field="bytes" + enabled=true + schema="metric" + } + enabled=true + schema="metric" + } + `; + const result: Datatable = await expectExpression( + 'esaggs_shift_filtered_metrics', + expression + ).getResponse(); + expect(result.rows.length).to.be(25); + checkShift(result.rows, result.columns); + }); + it('shifts terms', async () => { const expression = ` kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} @@ -123,6 +204,47 @@ export default function ({ ]); }); + it('shifts filters', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggFilters id="1" filters='[{"input":{"query":"geo.src:\\"US\\" ","language":"kuery"},"label":""},{"input":{"query":"geo.src: \\"CN\\"","language":"kuery"},"label":""}]'} + aggs={aggFilters id="2" filters='[{"input":{"query":"geo.dest:\\"US\\" ","language":"kuery"},"label":""},{"input":{"query":"geo.dest: \\"CN\\"","language":"kuery"},"label":""}]'} + aggs={aggAvg id="3" field="bytes" enabled=true schema="metric" timeShift="2h"} + aggs={aggAvg id="4" field="bytes" enabled=true schema="metric"} + `; + const result: Datatable = await expectExpression( + 'esaggs_shift_filters', + expression + ).getResponse(); + expect(result.rows).to.eql([ + { + 'col-0-1': 'geo.src:"US" ', + 'col-1-2': 'geo.dest:"US" ', + 'col-2-3': 5956.9, + 'col-3-4': 5956.9, + }, + { + 'col-0-1': 'geo.src:"US" ', + 'col-1-2': 'geo.dest: "CN"', + 'col-2-3': 5127.854838709677, + 'col-3-4': 5085.746031746032, + }, + { + 'col-0-1': 'geo.src: "CN"', + 'col-1-2': 'geo.dest:"US" ', + 'col-2-3': 5648.25, + 'col-3-4': 5643.793650793651, + }, + { + 'col-0-1': 'geo.src: "CN"', + 'col-1-2': 'geo.dest: "CN"', + 'col-2-3': 5842.858823529412, + 'col-3-4': 5842.858823529412, + }, + ]); + }); + it('shifts histogram', async () => { const expression = ` kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} @@ -159,7 +281,7 @@ export default function ({ ]); }); - it('shifts pipeline aggs', async () => { + it('shifts sibling pipeline aggs', async () => { const expression = ` kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} | esaggs index={indexPatternLoad id='logstash-*'} @@ -167,13 +289,64 @@ export default function ({ aggs={aggBucketSum id="5" enabled=true schema="metric" timeShift="1d" customBucket={aggTerms id="6" enabled="true" schema="bucket" field="geo.src" size="3"} customMetric={aggCount id="7" enabled="true" schema="metric"}} `; const result: Datatable = await expectExpression( - 'esaggs_shift_pipeline_aggs', + 'esaggs_shift_sibling_pipeline_aggs', expression ).getResponse(); expect(getCell(result, 0, 0)).to.be(2050); expect(getCell(result, 0, 1)).to.be(2053); }); + it('shifts parent pipeline aggs', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggDateHistogram id="1" enabled=true schema="bucket" field="@timestamp" interval="3h" min_doc_count=0} + aggs={aggMovingAvg id="2" enabled=true schema="metric" metricAgg="custom" window=5 script="MovingFunctions.unweightedAvg(values)" timeShift="3h" customMetric={aggCount id="2-metric" enabled="true" schema="metric"}} + `; + const result: Datatable = await expectExpression( + 'esaggs_shift_parent_pipeline_aggs', + expression + ).getResponse(); + expect(result.rows).to.eql([ + { + 'col-0-1': 1442791800000, + 'col-1-2': null, + }, + { + 'col-0-1': 1442802600000, + 'col-1-2': 30, + }, + { + 'col-0-1': 1442813400000, + 'col-1-2': 30.5, + }, + { + 'col-0-1': 1442824200000, + 'col-1-2': 69.66666666666667, + }, + { + 'col-0-1': 1442835000000, + 'col-1-2': 198.5, + }, + { + 'col-0-1': 1442845800000, + 'col-1-2': 415.6, + }, + { + 'col-0-1': 1442856600000, + 'col-1-2': 702.2, + }, + { + 'col-0-1': 1442867400000, + 'col-1-2': 859.8, + }, + { + 'col-0-1': 1442878200000, + 'col-1-2': 878.4, + }, + ]); + }); + it('metrics at all levels should work for single shift', async () => { const expression = ` kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx index 2e4d083155a945..f53d6136ce2b6e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx @@ -101,7 +101,7 @@ export const cardinalityOperation: OperationDefinition { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx index 0557235a5d9ff9..24d34940eccdd8 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx @@ -83,7 +83,7 @@ export const countOperation: OperationDefinition { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx index 9acbfa15433076..a2003204f534cc 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx @@ -186,7 +186,7 @@ export const lastValueOperation: OperationDefinition>({ enabled: true, schema: 'metric', field: column.sourceField, - timeShift: column.timeShift, + timeShift: column.filter ? undefined : column.timeShift, }).toAst(); }, getErrorMessage: (layer, columnId, indexPattern) => diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx index 925534afd12511..100a3f8ca57b34 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx @@ -115,7 +115,7 @@ export const percentileOperation: OperationDefinition Date: Mon, 17 May 2021 19:41:28 +0200 Subject: [PATCH 17/31] address comments --- .../common/search/aggs/agg_configs.test.ts | 8 + .../data/common/search/aggs/agg_configs.ts | 191 +------- .../data/common/search/aggs/metrics/avg_fn.ts | 2 +- .../search/aggs/metrics/bucket_avg_fn.ts | 2 +- .../search/aggs/metrics/bucket_max_fn.ts | 2 +- .../search/aggs/metrics/bucket_min_fn.ts | 2 +- .../search/aggs/metrics/bucket_sum_fn.ts | 2 +- .../search/aggs/metrics/cardinality_fn.ts | 2 +- .../common/search/aggs/metrics/count_fn.ts | 2 +- .../search/aggs/metrics/cumulative_sum_fn.ts | 2 +- .../search/aggs/metrics/derivative_fn.ts | 2 +- .../search/aggs/metrics/filtered_metric_fn.ts | 2 +- .../search/aggs/metrics/geo_bounds_fn.ts | 2 +- .../search/aggs/metrics/geo_centroid_fn.ts | 2 +- .../data/common/search/aggs/metrics/max_fn.ts | 2 +- .../common/search/aggs/metrics/median_fn.ts | 2 +- .../data/common/search/aggs/metrics/min_fn.ts | 2 +- .../search/aggs/metrics/moving_avg_fn.test.ts | 4 + .../search/aggs/metrics/moving_avg_fn.ts | 2 +- .../aggs/metrics/percentile_ranks_fn.ts | 2 +- .../search/aggs/metrics/percentiles_fn.ts | 2 +- .../search/aggs/metrics/serial_diff_fn.ts | 2 +- .../aggs/metrics/single_percentile_fn.ts | 2 +- .../search/aggs/metrics/std_deviation_fn.ts | 2 +- .../data/common/search/aggs/metrics/sum_fn.ts | 2 +- .../common/search/aggs/metrics/top_hit_fn.ts | 2 +- .../search/aggs/utils/parse_time_shift.ts | 2 +- .../common/search/aggs/utils/time_splits.ts | 447 ++++++++++++++++++ .../expressions/esaggs/request_handler.ts | 1 + .../dimension_panel/dimension_panel.test.tsx | 21 +- .../operations/definitions/index.ts | 12 +- .../operations/definitions/terms/index.tsx | 70 ++- .../operations/layer_helpers.ts | 2 +- .../plugins/lens/server/routes/field_stats.ts | 10 +- 34 files changed, 573 insertions(+), 241 deletions(-) create mode 100644 src/plugins/data/common/search/aggs/utils/time_splits.ts diff --git a/src/plugins/data/common/search/aggs/agg_configs.test.ts b/src/plugins/data/common/search/aggs/agg_configs.test.ts index d4980447c5468f..72ea64791fa5b3 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.test.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.test.ts @@ -349,6 +349,14 @@ describe('AggConfigs', () => { params: { field: 'bytes', timeShift: '1d' }, }, ]; + indexPattern.fields.push({ + name: 'timestamp', + type: 'date', + esTypes: ['date'], + aggregatable: true, + filterable: true, + searchable: true, + } as IndexPatternField); const ac = new AggConfigs(indexPattern, configStates, { typesRegistry }); ac.timeFields = ['timestamp']; diff --git a/src/plugins/data/common/search/aggs/agg_configs.ts b/src/plugins/data/common/search/aggs/agg_configs.ts index bdfbc8c90884e2..6f8a8d38a4a286 100644 --- a/src/plugins/data/common/search/aggs/agg_configs.ts +++ b/src/plugins/data/common/search/aggs/agg_configs.ts @@ -7,16 +7,10 @@ */ import moment from 'moment'; -import _, { cloneDeep, isArray } from 'lodash'; +import _, { cloneDeep } from 'lodash'; import { i18n } from '@kbn/i18n'; import { Assign } from '@kbn/utility-types'; -import { - Aggregate, - Bucket, - FiltersAggregate, - FiltersBucketItem, - MultiBucketAggregate, -} from '@elastic/elasticsearch/api/types'; +import { Aggregate, Bucket } from '@elastic/elasticsearch/api/types'; import { IEsSearchResponse, @@ -31,6 +25,7 @@ import { AggGroupNames } from './agg_groups'; import { IndexPattern } from '../../index_patterns/index_patterns/index_pattern'; import { TimeRange, getTime, isRangeFilter } from '../../../common'; import { IBucketAggConfig } from './buckets'; +import { insertTimeShiftSplit, mergeTimeShifts } from './utils/time_splits'; function removeParentAggs(obj: any) { for (const prop in obj) { @@ -82,6 +77,7 @@ export class AggConfigs { public indexPattern: IndexPattern; public timeRange?: TimeRange; public timeFields?: string[]; + public forceNow?: Date; public hierarchical?: boolean = false; private readonly typesRegistry: AggTypesRegistryStart; @@ -108,6 +104,10 @@ export class AggConfigs { this.timeFields = timeFields; } + setForceNow(now: Date | undefined) { + this.forceNow = now; + } + setTimeRange(timeRange: TimeRange) { this.timeRange = timeRange; @@ -239,7 +239,7 @@ export class AggConfigs { } if (hasMultipleTimeShifts) { - dslLvlCursor = this.insertTimeShiftSplit(config, timeShifts, dslLvlCursor); + dslLvlCursor = insertTimeShiftSplit(this, config, timeShifts, dslLvlCursor); } if (config.type.hasNoDsl) { @@ -284,44 +284,6 @@ export class AggConfigs { return dslTopLvl; } - private insertTimeShiftSplit( - config: AggConfig, - timeShifts: Record, - dslLvlCursor: Record - ) { - if ('splitForTimeShift' in config.type && !config.type.splitForTimeShift(config, this)) { - return dslLvlCursor; - } - if (!this.timeFields || this.timeFields.length < 1) { - throw new Error('Time shift can only be used with configured time field'); - } - if (!this.timeRange) { - throw new Error('Time shift can only be used with configured time range'); - } - const timeRange = this.timeRange; - const filters: Record = {}; - const timeField = this.timeFields[0]; - Object.entries(timeShifts).forEach(([key, shift]) => { - filters[key] = { - range: { - [timeField]: { - // only works if there is a time range - gte: moment(timeRange.from).subtract(shift).toISOString(), - lte: moment(timeRange.to).subtract(shift).toISOString(), - }, - }, - }; - }); - dslLvlCursor.time_offset_split = { - filters: { - filters, - }, - aggs: {}, - }; - - return dslLvlCursor.time_offset_split.aggs; - } - getAll() { return [...this.aggs]; } @@ -447,8 +409,6 @@ export class AggConfigs { if (!this.hasTimeShifts()) { return response; } - const timeShifts = this.getTimeShifts(); - const hasMultipleTimeShifts = Object.keys(timeShifts).length > 1; const transformedRawResponse = cloneDeep(response.rawResponse); if (!transformedRawResponse.aggregations) { transformedRawResponse.aggregations = { @@ -457,138 +417,7 @@ export class AggConfigs { } const aggCursor = transformedRawResponse.aggregations!; - const requestAggs = this.getRequestAggs(); - const bucketAggs = this.aggs.filter( - (agg) => agg.type.type === AggGroupNames.Buckets - ) as IBucketAggConfig[]; - - const mergeAggLevel = ( - target: GenericBucket, - source: GenericBucket, - shift: moment.Duration, - aggIndex: number - ) => { - Object.entries(source).forEach(([key, val]) => { - // copy over doc count into special key - if (typeof val === 'number' && key === 'doc_count') { - if (shift.asMilliseconds() === 0) { - target.doc_count = val; - } else { - target[`doc_count_${shift.asMilliseconds()}`] = val; - } - } else if (typeof val !== 'object') { - // other meta keys not of interest - return; - } else { - // a sub-agg - const agg = requestAggs.find( - (requestAgg) => key.substr(0, String(requestAgg.id).length) === requestAgg.id - ); - if (agg && agg.type.type === AggGroupNames.Metrics) { - const timeShift = agg.getTimeShift(); - if ( - (timeShift && timeShift.asMilliseconds() === shift.asMilliseconds()) || - (shift.asMilliseconds() === 0 && !timeShift) - ) { - // this is a metric from the current time shift, copy it over - target[key] = source[key]; - } - } else if (agg && agg === bucketAggs[aggIndex]) { - const bucketAgg = agg as IBucketAggConfig; - // expected next bucket sub agg - const subAggregate = val as Aggregate; - const buckets = ('buckets' in subAggregate ? subAggregate.buckets : undefined) as - | GenericBucket[] - | Record - | undefined; - if (!target[key]) { - // sub aggregate only exists in shifted branch, not in base branch - create dummy aggregate - // which will be filled with shifted data - target[key] = { - buckets: isArray(buckets) ? [] : {}, - }; - } - const baseSubAggregate = target[key] as Aggregate; - // only supported bucket formats in agg configs are array of buckets and record of buckets for filters - const baseBuckets = ('buckets' in baseSubAggregate - ? baseSubAggregate.buckets - : undefined) as GenericBucket[] | Record | undefined; - // merge - if (isArray(buckets) && isArray(baseBuckets)) { - const baseBucketMap: Record = {}; - baseBuckets.forEach((bucket) => { - baseBucketMap[String(bucket.key)] = bucket; - }); - buckets.forEach((bucket) => { - const bucketKey = bucketAgg.type.getShiftedKey(bucketAgg, bucket.key, shift); - // if a bucket is missing in the map, create an empty one - if (!baseBucketMap[bucketKey]) { - baseBucketMap[String(bucketKey)] = { - key: bucketKey, - } as GenericBucket; - } - mergeAggLevel(baseBucketMap[bucketKey], bucket, shift, aggIndex + 1); - }); - (baseSubAggregate as MultiBucketAggregate).buckets = Object.values( - baseBucketMap - ).sort((a, b) => bucketAgg.type.orderBuckets(bucketAgg, a, b)); - } else if (baseBuckets && buckets && !isArray(baseBuckets)) { - Object.entries(buckets).forEach(([bucketKey, bucket]) => { - // if a bucket is missing in the base response, create an empty one - if (!baseBuckets[bucketKey]) { - baseBuckets[bucketKey] = {} as GenericBucket; - } - mergeAggLevel(baseBuckets[bucketKey], bucket, shift, aggIndex + 1); - }); - } - } - } - }); - }; - const transformTimeShift = (cursor: Record, aggIndex: number): undefined => { - const shouldSplit = this.aggs[aggIndex].type.splitForTimeShift(this.aggs[aggIndex], this); - if (shouldSplit) { - // multiple time shifts caused a filters agg in the tree we have to merge - if (hasMultipleTimeShifts && cursor.time_offset_split) { - const timeShiftedBuckets = (cursor.time_offset_split as FiltersAggregate) - .buckets as Record; - const subTree = {}; - Object.entries(timeShifts).forEach(([key, shift]) => { - mergeAggLevel( - subTree as GenericBucket, - timeShiftedBuckets[key] as GenericBucket, - shift, - aggIndex - ); - }); - - delete cursor.time_offset_split; - Object.assign(cursor, subTree); - } else { - // otherwise we have to "merge" a single level to shift all keys - const [[, shift]] = Object.entries(timeShifts); - const subTree = {}; - mergeAggLevel(subTree, cursor, shift, aggIndex); - Object.assign(cursor, subTree); - } - return; - } - // recurse deeper into the response object - Object.keys(cursor).forEach((subAggId) => { - const subAgg = cursor[subAggId]; - if (typeof subAgg !== 'object' || !('buckets' in subAgg)) { - return; - } - if (isArray(subAgg.buckets)) { - subAgg.buckets.forEach((bucket) => transformTimeShift(bucket, aggIndex + 1)); - } else { - Object.values(subAgg.buckets).forEach((bucket) => - transformTimeShift(bucket, aggIndex + 1) - ); - } - }); - }; - transformTimeShift(aggCursor, 0); + mergeTimeShifts(this, aggCursor); return { ...response, rawResponse: transformedRawResponse, diff --git a/src/plugins/data/common/search/aggs/metrics/avg_fn.ts b/src/plugins/data/common/search/aggs/metrics/avg_fn.ts index 6b329a8bcfa998..e32de6cd0a83f3 100644 --- a/src/plugins/data/common/search/aggs/metrics/avg_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/avg_fn.ts @@ -66,7 +66,7 @@ export const aggAvg = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.ts b/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.ts index b9374fb94720ad..a980f6ac555a24 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_avg_fn.ts @@ -81,7 +81,7 @@ export const aggBucketAvg = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.ts b/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.ts index f50042bc1ca105..0d3e8a5e7f878e 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_max_fn.ts @@ -81,7 +81,7 @@ export const aggBucketMax = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.ts b/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.ts index 41bec5c5746c3b..3b6c32595909a9 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_min_fn.ts @@ -81,7 +81,7 @@ export const aggBucketMin = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.ts b/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.ts index b56dba3b96ba10..ae3502bbc25883 100644 --- a/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/bucket_sum_fn.ts @@ -81,7 +81,7 @@ export const aggBucketSum = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/cardinality_fn.ts b/src/plugins/data/common/search/aggs/metrics/cardinality_fn.ts index fe559b71094ecb..89006761407f74 100644 --- a/src/plugins/data/common/search/aggs/metrics/cardinality_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/cardinality_fn.ts @@ -71,7 +71,7 @@ export const aggCardinality = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/count_fn.ts b/src/plugins/data/common/search/aggs/metrics/count_fn.ts index 7ca49078ee901e..a3a4bcc16a3913 100644 --- a/src/plugins/data/common/search/aggs/metrics/count_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/count_fn.ts @@ -58,7 +58,7 @@ export const aggCount = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.ts b/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.ts index 88c1fceb11f720..5cdbcfe8575853 100644 --- a/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/cumulative_sum_fn.ts @@ -85,7 +85,7 @@ export const aggCumulativeSum = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/derivative_fn.ts b/src/plugins/data/common/search/aggs/metrics/derivative_fn.ts index 38985b78cb378d..8bfe808aede8e9 100644 --- a/src/plugins/data/common/search/aggs/metrics/derivative_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/derivative_fn.ts @@ -85,7 +85,7 @@ export const aggDerivative = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.ts b/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.ts index 7b14fd1c45ed27..0b3d3acd3a603f 100644 --- a/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/filtered_metric_fn.ts @@ -76,7 +76,7 @@ export const aggFilteredMetric = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.ts b/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.ts index c7915eb12c5b25..b2cfad1805b9f5 100644 --- a/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/geo_bounds_fn.ts @@ -71,7 +71,7 @@ export const aggGeoBounds = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.ts b/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.ts index df1b5182092fd2..9215f7afb4c6d1 100644 --- a/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/geo_centroid_fn.ts @@ -71,7 +71,7 @@ export const aggGeoCentroid = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/max_fn.ts b/src/plugins/data/common/search/aggs/metrics/max_fn.ts index f42312a37ecd68..7a1d8ad22fb7ed 100644 --- a/src/plugins/data/common/search/aggs/metrics/max_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/max_fn.ts @@ -66,7 +66,7 @@ export const aggMax = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/median_fn.ts b/src/plugins/data/common/search/aggs/metrics/median_fn.ts index 9145e088d83a97..a9537e1f99ca41 100644 --- a/src/plugins/data/common/search/aggs/metrics/median_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/median_fn.ts @@ -71,7 +71,7 @@ export const aggMedian = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/min_fn.ts b/src/plugins/data/common/search/aggs/metrics/min_fn.ts index 36b576081ac6e0..a97834f310a49e 100644 --- a/src/plugins/data/common/search/aggs/metrics/min_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/min_fn.ts @@ -66,7 +66,7 @@ export const aggMin = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.test.ts index bde90c563afc11..645519a6683761 100644 --- a/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.test.ts +++ b/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.test.ts @@ -30,6 +30,7 @@ describe('agg_expression_functions', () => { "json": undefined, "metricAgg": undefined, "script": undefined, + "timeShift": undefined, "window": undefined, }, "schema": undefined, @@ -59,6 +60,7 @@ describe('agg_expression_functions', () => { "json": undefined, "metricAgg": "sum", "script": "test", + "timeShift": undefined, "window": 10, }, "schema": undefined, @@ -88,6 +90,7 @@ describe('agg_expression_functions', () => { "json": undefined, "metricAgg": undefined, "script": undefined, + "timeShift": undefined, "window": undefined, }, "schema": undefined, @@ -96,6 +99,7 @@ describe('agg_expression_functions', () => { "json": undefined, "metricAgg": undefined, "script": undefined, + "timeShift": undefined, "window": undefined, } `); diff --git a/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.ts b/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.ts index 63169b4d659bb9..1637dad561c375 100644 --- a/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/moving_avg_fn.ts @@ -98,7 +98,7 @@ export const aggMovingAvg = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.ts b/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.ts index 7bbf8c460682b6..60a2882fcec581 100644 --- a/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/percentile_ranks_fn.ts @@ -78,7 +78,7 @@ export const aggPercentileRanks = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/percentiles_fn.ts b/src/plugins/data/common/search/aggs/metrics/percentiles_fn.ts index 8c9c765d068e28..1a746a86cbcd57 100644 --- a/src/plugins/data/common/search/aggs/metrics/percentiles_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/percentiles_fn.ts @@ -78,7 +78,7 @@ export const aggPercentiles = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.ts b/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.ts index b481fa56a38ce9..8460cb891f1e4c 100644 --- a/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/serial_diff_fn.ts @@ -85,7 +85,7 @@ export const aggSerialDiff = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/single_percentile_fn.ts b/src/plugins/data/common/search/aggs/metrics/single_percentile_fn.ts index b11ed2da8cc76f..edf69031c31ace 100644 --- a/src/plugins/data/common/search/aggs/metrics/single_percentile_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/single_percentile_fn.ts @@ -78,7 +78,7 @@ export const aggSinglePercentile = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.ts b/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.ts index 7ba0c5b789dbf8..c181065d2416e7 100644 --- a/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/std_deviation_fn.ts @@ -71,7 +71,7 @@ export const aggStdDeviation = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/sum_fn.ts b/src/plugins/data/common/search/aggs/metrics/sum_fn.ts index 6e795ada786697..d8e03d28bb12a7 100644 --- a/src/plugins/data/common/search/aggs/metrics/sum_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/sum_fn.ts @@ -66,7 +66,7 @@ export const aggSum = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/metrics/top_hit_fn.ts b/src/plugins/data/common/search/aggs/metrics/top_hit_fn.ts index 9204ca88846c25..bc20f19253eec7 100644 --- a/src/plugins/data/common/search/aggs/metrics/top_hit_fn.ts +++ b/src/plugins/data/common/search/aggs/metrics/top_hit_fn.ts @@ -98,7 +98,7 @@ export const aggTopHit = (): FunctionDefinition => ({ types: ['string'], help: i18n.translate('data.search.aggs.metrics.timeShift.help', { defaultMessage: - 'Specifies whether the time range of documents used for the metric should be shifted by the specified amount', + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', }), }, }, diff --git a/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts b/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts index e874f489b13f2b..256addac915209 100644 --- a/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts +++ b/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts @@ -19,7 +19,7 @@ export const parseTimeShift = (val: string): moment.Duration | 'previous' | 'inv if (val === 'previous') { return 'previous'; } - const [, amount, unit] = val.match(/(\d+)(\w)/) || []; + const [, amount, unit] = val.match(/^(\d+)(\w)$/) || []; const parsedAmount = Number(amount); if (Number.isNaN(parsedAmount) || !allowedUnits.includes(unit as AllowedUnit)) { return 'invalid'; diff --git a/src/plugins/data/common/search/aggs/utils/time_splits.ts b/src/plugins/data/common/search/aggs/utils/time_splits.ts new file mode 100644 index 00000000000000..4ac47efaea3476 --- /dev/null +++ b/src/plugins/data/common/search/aggs/utils/time_splits.ts @@ -0,0 +1,447 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import moment from 'moment'; +import _, { isArray } from 'lodash'; +import { + Aggregate, + FiltersAggregate, + FiltersBucketItem, + MultiBucketAggregate, +} from '@elastic/elasticsearch/api/types'; + +import { AggGroupNames } from '../agg_groups'; +import { GenericBucket, AggConfigs, getTime, AggConfig } from '../../../../common'; +import { IBucketAggConfig } from '../buckets'; + +/** + * This function will transform an ES response containg a time split (using a filters aggregation before the metrics or date histogram aggregation), + * merging together all branches for the different time ranges into a single response structure which can be tabified into a single table. + * + * If there is just a single time shift, there are no separate branches per time range - in this case only the date histogram keys are shifted by the + * configured amount of time. + * + * To do this, the following steps are taken: + * * Traverse the response tree, tracking the current agg config + * * Once the node which would contain the time split object is found, merge all separate time range buckets into a single layer of buckets of the parent agg + * * Recursively repeat this process for all nested sub-buckets + * + * Example input: + * ``` + * "aggregations" : { + "product" : { + "buckets" : [ + { + "key" : "Product A", + "doc_count" : 512, + "first_year" : { + "doc_count" : 418, + "overall_revenue" : { + "value" : 2163634.0 + } + }, + "time_offset_split" : { + "buckets" : { + "-1y" : { + "doc_count" : 420, + "year" : { + "buckets" : [ + { + "key_as_string" : "2018", + "doc_count" : 81, + "revenue" : { + "value" : 505124.0 + } + }, + { + "key_as_string" : "2019", + "doc_count" : 65, + "revenue" : { + "value" : 363058.0 + } + } + ] + } + }, + "regular" : { + "doc_count" : 418, + "year" : { + "buckets" : [ + { + "key_as_string" : "2019", + "doc_count" : 65, + "revenue" : { + "value" : 363058.0 + } + }, + { + "key_as_string" : "2020", + "doc_count" : 84, + "revenue" : { + "value" : 392924.0 + } + } + ] + } + } + } + } + }, + { + "key" : "Product B", + "doc_count" : 248, + "first_year" : { + "doc_count" : 215, + "overall_revenue" : { + "value" : 1315547.0 + } + }, + "time_offset_split" : { + "buckets" : { + "-1y" : { + "doc_count" : 211, + "year" : { + "buckets" : [ + { + "key_as_string" : "2018", + "key" : 1618963200000, + "doc_count" : 28, + "revenue" : { + "value" : 156543.0 + } + }, + // ... + * ``` + * + * Example output: + * ``` + * "aggregations" : { + "product" : { + "buckets" : [ + { + "key" : "Product A", + "doc_count" : 512, + "first_year" : { + "doc_count" : 418, + "overall_revenue" : { + "value" : 2163634.0 + } + }, + "year" : { + "buckets" : [ + { + "key_as_string" : "2019", + "doc_count" : 81, + "revenue_regular" : { + "value" : 505124.0 + }, + "revenue_-1y" : { + "value" : 302736.0 + } + }, + { + "key_as_string" : "2020", + "doc_count" : 78, + "revenue_regular" : { + "value" : 392924.0 + }, + "revenue_-1y" : { + "value" : 363058.0 + }, + } + // ... + * ``` + * + * + * @param aggConfigs The agg configs instance + * @param aggCursor The root aggregations object from the response which will be mutated in place + */ +export function mergeTimeShifts(aggConfigs: AggConfigs, aggCursor: Record) { + const timeShifts = aggConfigs.getTimeShifts(); + const hasMultipleTimeShifts = Object.keys(timeShifts).length > 1; + const requestAggs = aggConfigs.getRequestAggs(); + const bucketAggs = aggConfigs.aggs.filter( + (agg) => agg.type.type === AggGroupNames.Buckets + ) as IBucketAggConfig[]; + const mergeAggLevel = ( + target: GenericBucket, + source: GenericBucket, + shift: moment.Duration, + aggIndex: number + ) => { + Object.entries(source).forEach(([key, val]) => { + // copy over doc count into special key + if (typeof val === 'number' && key === 'doc_count') { + if (shift.asMilliseconds() === 0) { + target.doc_count = val; + } else { + target[`doc_count_${shift.asMilliseconds()}`] = val; + } + } else if (typeof val !== 'object') { + // other meta keys not of interest + return; + } else { + // a sub-agg + const agg = requestAggs.find((requestAgg) => key.indexOf(requestAgg.id) === 0); + if (agg && agg.type.type === AggGroupNames.Metrics) { + const timeShift = agg.getTimeShift(); + if ( + (timeShift && timeShift.asMilliseconds() === shift.asMilliseconds()) || + (shift.asMilliseconds() === 0 && !timeShift) + ) { + // this is a metric from the current time shift, copy it over + target[key] = source[key]; + } + } else if (agg && agg === bucketAggs[aggIndex]) { + const bucketAgg = agg as IBucketAggConfig; + // expected next bucket sub agg + const subAggregate = val as Aggregate; + const buckets = ('buckets' in subAggregate ? subAggregate.buckets : undefined) as + | GenericBucket[] + | Record + | undefined; + if (!target[key]) { + // sub aggregate only exists in shifted branch, not in base branch - create dummy aggregate + // which will be filled with shifted data + target[key] = { + buckets: isArray(buckets) ? [] : {}, + }; + } + const baseSubAggregate = target[key] as Aggregate; + // only supported bucket formats in agg configs are array of buckets and record of buckets for filters + const baseBuckets = ('buckets' in baseSubAggregate + ? baseSubAggregate.buckets + : undefined) as GenericBucket[] | Record | undefined; + // merge + if (isArray(buckets) && isArray(baseBuckets)) { + const baseBucketMap: Record = {}; + baseBuckets.forEach((bucket) => { + baseBucketMap[String(bucket.key)] = bucket; + }); + buckets.forEach((bucket) => { + const bucketKey = bucketAgg.type.getShiftedKey(bucketAgg, bucket.key, shift); + // if a bucket is missing in the map, create an empty one + if (!baseBucketMap[bucketKey]) { + baseBucketMap[String(bucketKey)] = { + key: bucketKey, + } as GenericBucket; + } + mergeAggLevel(baseBucketMap[bucketKey], bucket, shift, aggIndex + 1); + }); + (baseSubAggregate as MultiBucketAggregate).buckets = Object.values( + baseBucketMap + ).sort((a, b) => bucketAgg.type.orderBuckets(bucketAgg, a, b)); + } else if (baseBuckets && buckets && !isArray(baseBuckets)) { + Object.entries(buckets).forEach(([bucketKey, bucket]) => { + // if a bucket is missing in the base response, create an empty one + if (!baseBuckets[bucketKey]) { + baseBuckets[bucketKey] = {} as GenericBucket; + } + mergeAggLevel(baseBuckets[bucketKey], bucket, shift, aggIndex + 1); + }); + } + } + } + }); + }; + const transformTimeShift = (cursor: Record, aggIndex: number): undefined => { + const shouldSplit = aggConfigs.aggs[aggIndex].type.splitForTimeShift( + aggConfigs.aggs[aggIndex], + aggConfigs + ); + if (shouldSplit) { + // multiple time shifts caused a filters agg in the tree we have to merge + if (hasMultipleTimeShifts && cursor.time_offset_split) { + const timeShiftedBuckets = (cursor.time_offset_split as FiltersAggregate).buckets as Record< + string, + FiltersBucketItem + >; + const subTree = {}; + Object.entries(timeShifts).forEach(([key, shift]) => { + mergeAggLevel( + subTree as GenericBucket, + timeShiftedBuckets[key] as GenericBucket, + shift, + aggIndex + ); + }); + + delete cursor.time_offset_split; + Object.assign(cursor, subTree); + } else { + // otherwise we have to "merge" a single level to shift all keys + const [[, shift]] = Object.entries(timeShifts); + const subTree = {}; + mergeAggLevel(subTree, cursor, shift, aggIndex); + Object.assign(cursor, subTree); + } + return; + } + // recurse deeper into the response object + Object.keys(cursor).forEach((subAggId) => { + const subAgg = cursor[subAggId]; + if (typeof subAgg !== 'object' || !('buckets' in subAgg)) { + return; + } + if (isArray(subAgg.buckets)) { + subAgg.buckets.forEach((bucket) => transformTimeShift(bucket, aggIndex + 1)); + } else { + Object.values(subAgg.buckets).forEach((bucket) => transformTimeShift(bucket, aggIndex + 1)); + } + }); + }; + transformTimeShift(aggCursor, 0); +} + +/** + * Inserts a filters aggregation into the aggregation tree which splits buckets to fetch data for all time ranges + * configured in metric aggregations. + * + * The current agg config can implement `splitForTimeShift` to force insertion of the time split filters aggregation + * before the dsl of the agg config (date histogram and metrics aggregations do this) + * + * Example aggregation tree without time split: + * ``` + * "aggs": { + "product": { + "terms": { + "field": "product", + "size": 3, + "order": { "overall_revenue": "desc" } + }, + "aggs": { + "overall_revenue": { + "sum": { + "field": "sales" + } + }, + "year": { + "date_histogram": { + "field": "timestamp", + "interval": "year" + }, + "aggs": { + "revenue": { + "sum": { + "field": "sales" + } + } + } + // ... + * ``` + * + * Same aggregation tree with inserted time split: + * ``` + * "aggs": { + "product": { + "terms": { + "field": "product", + "size": 3, + "order": { "first_year>overall_revenue": "desc" } + }, + "aggs": { + "first_year": { + "filter": { + "range": { + "timestamp": { + "gte": "2019", + "lte": "2020" + } + } + }, + "aggs": { + "overall_revenue": { + "sum": { + "field": "sales" + } + } + } + }, + "time_offset_split": { + "filters": { + "filters": { + "regular": { + "range": { + "timestamp": { + "gte": "2019", + "lte": "2020" + } + } + }, + "-1y": { + "range": { + "timestamp": { + "gte": "2018", + "lte": "2019" + } + } + } + } + }, + "aggs": { + "year": { + "date_histogram": { + "field": "timestamp", + "interval": "year" + }, + "aggs": { + "revenue": { + "sum": { + "field": "sales" + } + } + } + } + } + } + } + * ``` + */ +export function insertTimeShiftSplit( + aggConfigs: AggConfigs, + config: AggConfig, + timeShifts: Record, + dslLvlCursor: Record +) { + if ('splitForTimeShift' in config.type && !config.type.splitForTimeShift(config, aggConfigs)) { + return dslLvlCursor; + } + if (!aggConfigs.timeFields || aggConfigs.timeFields.length < 1) { + throw new Error('Time shift can only be used with configured time field'); + } + if (!aggConfigs.timeRange) { + throw new Error('Time shift can only be used with configured time range'); + } + const timeRange = aggConfigs.timeRange; + const filters: Record = {}; + const timeField = aggConfigs.timeFields[0]; + Object.entries(timeShifts).forEach(([key, shift]) => { + const timeFilter = getTime(aggConfigs.indexPattern, timeRange, { + fieldName: timeField, + forceNow: aggConfigs.forceNow, + }); + if (timeFilter) { + filters[key] = { + range: { + [timeField]: { + gte: moment(timeFilter.range[timeField].gte).subtract(shift).toISOString(), + lte: moment(timeFilter.range[timeField].lte).subtract(shift).toISOString(), + }, + }, + }; + } + }); + dslLvlCursor.time_offset_split = { + filters: { + filters, + }, + aggs: {}, + }; + + return dslLvlCursor.time_offset_split.aggs; +} diff --git a/src/plugins/data/common/search/expressions/esaggs/request_handler.ts b/src/plugins/data/common/search/expressions/esaggs/request_handler.ts index 1eefd71d9e212b..61193c52a5e74b 100644 --- a/src/plugins/data/common/search/expressions/esaggs/request_handler.ts +++ b/src/plugins/data/common/search/expressions/esaggs/request_handler.ts @@ -69,6 +69,7 @@ export const handleRequest = async ({ const allTimeFields = timeFields && timeFields.length > 0 ? timeFields : defaultTimeFields; aggs.setTimeRange(timeRange as TimeRange); + aggs.setForceNow(forceNow); aggs.setTimeFields(allTimeFields); // For now we need to mirror the history of the passed search source, since diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx index 407e14b1f96233..631e8ce31512d4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx @@ -34,6 +34,8 @@ import { DateHistogramIndexPatternColumn } from '../operations/definitions/date_ import { getFieldByNameFactory } from '../pure_helpers'; import { Filtering } from './filtering'; import { TimeShift } from './time_shift'; +import { DimensionEditor } from './dimension_editor'; +import { AdvancedOptions } from './advanced_options'; jest.mock('../loader'); jest.mock('../query_input', () => ({ @@ -1333,11 +1335,20 @@ describe('IndexPatternDimensionEditorPanel', () => { }; } - it('should not show custom options if time scaling is not available', () => { - const props = getProps({ - operationType: 'terms', - sourceField: 'bytes', - }); + it('should not show custom options if time shift is not available', () => { + const props = { + ...defaultProps, + state: getStateWithColumns({ + col2: { + dataType: 'number', + isBucketed: false, + label: 'Count of records', + operationType: 'count', + sourceField: 'Records', + } as IndexPatternColumn, + }), + columnId: 'col2', + }; wrapper = shallow( { message: string; fixAction?: { label: string; - newState: (core: CoreStart, frame: FramePublicAPI) => Promise; + newState: ( + core: CoreStart, + frame: FramePublicAPI, + layerId: string + ) => Promise; }; } > @@ -389,7 +393,11 @@ interface FieldBasedOperationDefinition { message: string; fixAction?: { label: string; - newState: (core: CoreStart, frame: FramePublicAPI) => Promise; + newState: ( + core: CoreStart, + frame: FramePublicAPI, + layerId: string + ) => Promise; }; } > diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx index 78b3a13c60e5eb..651cf556b09d89 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx @@ -32,6 +32,7 @@ import { FieldBasedIndexPatternColumn } from '../column_types'; import { ValuesInput } from './values_input'; import { getInvalidFieldMessage } from '../helpers'; import type { IndexPatternLayer, IndexPattern } from '../../../types'; +import { defaultLabel } from '../filters'; function ofName(name?: string) { return i18n.translate('xpack.lens.indexPattern.termsOf', { @@ -80,24 +81,34 @@ function getDisallowedTermsMessage( label: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShiftsFixActionLabel', { defaultMessage: 'Pin current top values', }), - newState: async (core: CoreStart, frame: FramePublicAPI) => { - const fieldName = (layer.columns[columnId] as TermsIndexPatternColumn).sourceField; - const response: FieldStatsResponse = await core.http.post( - `/api/lens/index_stats/${indexPattern.id}/field`, - { - body: JSON.stringify({ - fieldName, - dslQuery: esQuery.buildEsQuery( - indexPattern as IIndexPattern, - frame.query, - frame.filters, - esQuery.getEsQueryConfig(core.uiSettings) - ), - fromDate: frame.dateRange.fromDate, - toDate: frame.dateRange.toDate, - }), - } + newState: async (core: CoreStart, frame: FramePublicAPI, layerId: string) => { + const currentColumn = layer.columns[columnId] as TermsIndexPatternColumn; + const fieldName = currentColumn.sourceField; + let currentTerms = uniq( + frame.activeData?.[layerId].rows + .map((row) => row[columnId] as string) + .filter((term) => typeof term === 'string' && term !== '__other__') || [] ); + if (currentTerms.length === 0) { + const response: FieldStatsResponse = await core.http.post( + `/api/lens/index_stats/${indexPattern.id}/field`, + { + body: JSON.stringify({ + fieldName, + dslQuery: esQuery.buildEsQuery( + indexPattern as IIndexPattern, + frame.query, + frame.filters, + esQuery.getEsQueryConfig(core.uiSettings) + ), + fromDate: frame.dateRange.fromDate, + toDate: frame.dateRange.toDate, + size: currentColumn.params.size, + }), + } + ); + currentTerms = response.topValues?.buckets.map(({ key }) => String(key)) || []; + } return { ...layer, columns: { @@ -114,13 +125,24 @@ function getDisallowedTermsMessage( dataType: 'string', operationType: 'filters', params: { - filters: response.topValues?.buckets.map(({ key }) => ({ - input: { - query: `${fieldName}: "${key}"`, - language: 'kuery', - }, - label: String(key), - })), + filters: + currentTerms.length > 0 + ? currentTerms.map((term) => ({ + input: { + query: `${fieldName}: "${term}"`, + language: 'kuery', + }, + label: term, + })) + : [ + { + input: { + query: '*', + language: 'kuery', + }, + label: defaultLabel, + }, + ], }, } as FiltersIndexPatternColumn, }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 42de955730ba04..6545b02ad422b6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -1187,7 +1187,7 @@ export function getErrorMessages( ...state, layers: { ...state.layers, - [layerId]: await errorMessage.fixAction!.newState(core, frame), + [layerId]: await errorMessage.fixAction!.newState(core, frame, layerId), }, }), } diff --git a/x-pack/plugins/lens/server/routes/field_stats.ts b/x-pack/plugins/lens/server/routes/field_stats.ts index 6cddd2c60f4165..6b7e197a4d5617 100644 --- a/x-pack/plugins/lens/server/routes/field_stats.ts +++ b/x-pack/plugins/lens/server/routes/field_stats.ts @@ -31,6 +31,7 @@ export async function initFieldsRoute(setup: CoreSetup) { fromDate: schema.string(), toDate: schema.string(), fieldName: schema.string(), + size: schema.maybe(schema.number()), }, { unknowns: 'allow' } ), @@ -38,7 +39,7 @@ export async function initFieldsRoute(setup: CoreSetup) { }, async (context, req, res) => { const requestClient = context.core.elasticsearch.client.asCurrentUser; - const { fromDate, toDate, fieldName, dslQuery } = req.body; + const { fromDate, toDate, fieldName, dslQuery, size } = req.body; const [{ savedObjects, elasticsearch }, { data }] = await setup.getStartServices(); const savedObjectsClient = savedObjects.getScopedClient(req); @@ -112,7 +113,7 @@ export async function initFieldsRoute(setup: CoreSetup) { } return res.ok({ - body: await getStringSamples(search, field), + body: await getStringSamples(search, field, size), }); } catch (e) { if (e instanceof SavedObjectNotFound) { @@ -245,7 +246,8 @@ export async function getNumberHistogram( export async function getStringSamples( aggSearchWithBody: (aggs: Record) => unknown, - field: IFieldType + field: IFieldType, + size = 10 ): Promise { const fieldRef = getFieldRef(field); @@ -257,7 +259,7 @@ export async function getStringSamples( top_values: { terms: { ...fieldRef, - size: 10, + size, }, }, }, From 50844cd75d775916192ce9ed9f9163ab404cc4bc Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 17 May 2021 19:56:49 +0200 Subject: [PATCH 18/31] fix test --- x-pack/test/functional/apps/lens/time_shift.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/test/functional/apps/lens/time_shift.ts b/x-pack/test/functional/apps/lens/time_shift.ts index 770ab3cd2c2d6b..9ddd2c9daf58d3 100644 --- a/x-pack/test/functional/apps/lens/time_shift.ts +++ b/x-pack/test/functional/apps/lens/time_shift.ts @@ -59,8 +59,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(await PageObjects.lens.hasFixAction()).to.be(true); await PageObjects.lens.useFixAction(); - expect(await PageObjects.lens.getDatatableCellText(2, 2)).to.eql('6,976'); - expect(await PageObjects.lens.getDatatableCellText(2, 3)).to.eql('4,182.5'); + expect(await PageObjects.lens.getDatatableCellText(2, 2)).to.eql('5,541.5'); + expect(await PageObjects.lens.getDatatableCellText(2, 3)).to.eql('3,628'); expect(await PageObjects.lens.getDatatableHeaderText(0)).to.eql('Pinned top values of ip'); }); From fc5f171fc6d894f5239ba411bb15caae7f3348fb Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 17 May 2021 20:52:14 +0200 Subject: [PATCH 19/31] fix types and docs --- ...plugins-data-public.aggconfigs.forcenow.md | 11 ++++++++++ ...a-plugin-plugins-data-public.aggconfigs.md | 2 ++ ...gins-data-public.aggconfigs.setforcenow.md | 22 +++++++++++++++++++ src/plugins/data/public/public.api.md | 4 ++++ .../definitions/terms/terms.test.tsx | 20 ++++++++++------- 5 files changed, 51 insertions(+), 8 deletions(-) create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.forcenow.md create mode 100644 docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.setforcenow.md diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.forcenow.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.forcenow.md new file mode 100644 index 00000000000000..8040c2939e2e45 --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.forcenow.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) > [forceNow](./kibana-plugin-plugins-data-public.aggconfigs.forcenow.md) + +## AggConfigs.forceNow property + +Signature: + +```typescript +forceNow?: Date; +``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md index ca191fdcae9a53..45333b6767cace 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.md @@ -22,6 +22,7 @@ export declare class AggConfigs | --- | --- | --- | --- | | [aggs](./kibana-plugin-plugins-data-public.aggconfigs.aggs.md) | | IAggConfig[] | | | [createAggConfig](./kibana-plugin-plugins-data-public.aggconfigs.createaggconfig.md) | | <T extends AggConfig = AggConfig>(params: CreateAggConfigParams, { addToAggConfigs }?: {
addToAggConfigs?: boolean | undefined;
}) => T | | +| [forceNow](./kibana-plugin-plugins-data-public.aggconfigs.forcenow.md) | | Date | | | [hierarchical](./kibana-plugin-plugins-data-public.aggconfigs.hierarchical.md) | | boolean | | | [indexPattern](./kibana-plugin-plugins-data-public.aggconfigs.indexpattern.md) | | IndexPattern | | | [timeFields](./kibana-plugin-plugins-data-public.aggconfigs.timefields.md) | | string[] | | @@ -50,6 +51,7 @@ export declare class AggConfigs | [jsonDataEquals(aggConfigs)](./kibana-plugin-plugins-data-public.aggconfigs.jsondataequals.md) | | Data-by-data comparison of this Aggregation Ignores the non-array indexes | | [onSearchRequestStart(searchSource, options)](./kibana-plugin-plugins-data-public.aggconfigs.onsearchrequeststart.md) | | | | [postFlightTransform(response)](./kibana-plugin-plugins-data-public.aggconfigs.postflighttransform.md) | | | +| [setForceNow(now)](./kibana-plugin-plugins-data-public.aggconfigs.setforcenow.md) | | | | [setTimeFields(timeFields)](./kibana-plugin-plugins-data-public.aggconfigs.settimefields.md) | | | | [setTimeRange(timeRange)](./kibana-plugin-plugins-data-public.aggconfigs.settimerange.md) | | | | [toDsl()](./kibana-plugin-plugins-data-public.aggconfigs.todsl.md) | | | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.setforcenow.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.setforcenow.md new file mode 100644 index 00000000000000..60a1bfe0872faf --- /dev/null +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs.setforcenow.md @@ -0,0 +1,22 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) > [AggConfigs](./kibana-plugin-plugins-data-public.aggconfigs.md) > [setForceNow](./kibana-plugin-plugins-data-public.aggconfigs.setforcenow.md) + +## AggConfigs.setForceNow() method + +Signature: + +```typescript +setForceNow(now: Date | undefined): void; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| now | Date | undefined | | + +Returns: + +`void` + diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 14ced4e3346e55..b68b7725bfff5b 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -253,6 +253,8 @@ export class AggConfigs { addToAggConfigs?: boolean | undefined; }) => T; // (undocumented) + forceNow?: Date; + // (undocumented) getAll(): AggConfig[]; // (undocumented) getRequestAggById(id: string): AggConfig | undefined; @@ -303,6 +305,8 @@ export class AggConfigs { // (undocumented) postFlightTransform(response: IEsSearchResponse_2): IEsSearchResponse_2; // (undocumented) + setForceNow(now: Date | undefined): void; + // (undocumented) setTimeFields(timeFields: string[] | undefined): void; // (undocumented) setTimeRange(timeRange: TimeRange): void; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx index 7c751e6db7ef1d..3baf004cf01bda 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx @@ -1067,14 +1067,18 @@ describe('terms', () => { ), }, } as unknown) as CoreStart; - const newLayer = await fixAction(coreMock, ({ - query: { language: 'kuery', query: 'a: b' }, - filters: [], - dateRange: { - fromDate: '2020', - toDate: '2021', - }, - } as unknown) as FramePublicAPI); + const newLayer = await fixAction( + coreMock, + ({ + query: { language: 'kuery', query: 'a: b' }, + filters: [], + dateRange: { + fromDate: '2020', + toDate: '2021', + }, + } as unknown) as FramePublicAPI, + 'first' + ); expect(newLayer.columns.col1).toEqual( expect.objectContaining({ operationType: 'filters', From 16b5ee21d860ecbeda07696781a3a408e67a756b Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 18 May 2021 12:34:26 +0200 Subject: [PATCH 20/31] increase accepted span for percentiles --- .../test_suites/run_pipeline/esaggs_timeshift.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts b/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts index 13cfcde9305f7a..c750602f735bd9 100644 --- a/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts +++ b/test/interpreter_functional/test_suites/run_pipeline/esaggs_timeshift.ts @@ -83,8 +83,8 @@ export default function ({ expression ).getResponse(); // percentile is not stable - expect(getCell(result, 0, 0)).to.be.within(10400, 10500); - expect(getCell(result, 0, 1)).to.be.within(10500, 10600); + expect(getCell(result, 0, 0)).to.be.within(10000, 20000); + expect(getCell(result, 0, 1)).to.be.within(10000, 20000); }); it('shifts multiple percentiles', async () => { @@ -99,10 +99,10 @@ export default function ({ expression ).getResponse(); // percentile is not stable - expect(getCell(result, 0, 0)).to.be.within(300, 400); - expect(getCell(result, 0, 1)).to.be.within(10400, 10500); - expect(getCell(result, 0, 2)).to.be.within(200, 300); - expect(getCell(result, 0, 3)).to.be.within(10500, 10600); + expect(getCell(result, 0, 0)).to.be.within(100, 1000); + expect(getCell(result, 0, 1)).to.be.within(10000, 20000); + expect(getCell(result, 0, 2)).to.be.within(100, 1000); + expect(getCell(result, 0, 3)).to.be.within(10000, 20000); }); it('shifts date histogram', async () => { From ef024fe00d2f7bd7fd54f3aa2472879e07682f47 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 19 May 2021 20:00:13 +0200 Subject: [PATCH 21/31] add warnings --- .../workspace_panel_wrapper.tsx | 17 ++- .../dimension_panel/time_shift.tsx | 105 ++++++++++++++- .../indexpattern.test.ts | 124 +++++++++++++++++- .../indexpattern_datasource/indexpattern.tsx | 2 + x-pack/plugins/lens/public/types.ts | 4 + 5 files changed, 246 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.tsx index 85f7601d8fb292..ec12e9e4002039 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel_wrapper.tsx @@ -60,9 +60,20 @@ export function WorkspacePanelWrapper({ }, [dispatch, activeVisualization] ); - const warningMessages = - activeVisualization?.getWarningMessages && - activeVisualization.getWarningMessages(visualizationState, framePublicAPI); + const warningMessages: React.ReactNode[] = []; + if (activeVisualization?.getWarningMessages) { + warningMessages.push( + ...(activeVisualization.getWarningMessages(visualizationState, framePublicAPI) || []) + ); + } + Object.entries(datasourceStates).forEach(([datasourceId, datasourceState]) => { + const datasource = datasourceMap[datasourceId]; + if (!datasourceState.isLoading && datasource.getWarningMessages) { + warningMessages.push( + ...(datasource.getWarningMessages(datasourceState.state, framePublicAPI) || []) + ); + } + }); return ( <>
diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index 4d721ba13468af..02eb3bd00e980d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -8,14 +8,17 @@ import { EuiButtonIcon } from '@elastic/eui'; import { EuiFormRow, EuiFlexItem, EuiFlexGroup } from '@elastic/eui'; import { EuiComboBox } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { uniq } from 'lodash'; import { i18n } from '@kbn/i18n'; import React, { useEffect, useState } from 'react'; import { Query } from 'src/plugins/data/public'; import { search } from '../../../../../../src/plugins/data/public'; import { parseTimeShift } from '../../../../../../src/plugins/data/common'; import { IndexPatternColumn, operationDefinitionMap } from '../operations'; -import { IndexPattern, IndexPatternLayer } from '../types'; +import { IndexPattern, IndexPatternLayer, IndexPatternPrivateState } from '../types'; import { IndexPatternDimensionEditorProps } from './dimension_panel'; +import { FramePublicAPI } from '../../types'; // to do: get the language from uiSettings export const defaultFilter: Query = { @@ -173,6 +176,8 @@ export function TimeShift({ const parsedLocalValue = localValue && parseTimeShift(localValue); const isLocalValueInvalid = Boolean(parsedLocalValue === 'invalid'); + const localValueTooSmall = parsedLocalValue && isValueTooSmall(parsedLocalValue); + const localValueNotMultiple = parsedLocalValue && isValueNotMultiple(parsedLocalValue); function getSelectedOption() { if (!localValue) return []; @@ -206,7 +211,19 @@ export function TimeShift({ helpText={i18n.translate('xpack.lens.indexPattern.timeShift.help', { defaultMessage: 'Time shift is specified by a number followed by a time unit', })} - isInvalid={isLocalValueInvalid} + error={ + (localValueTooSmall && + i18n.translate('xpack.lens.indexPattern.timeShift.tooSmallHelp', { + defaultMessage: + 'Time shift should to be larger than the date histogram interval. Either increase time shift or specify smaller interval in date histogram', + })) || + (localValueNotMultiple && + i18n.translate('xpack.lens.indexPattern.timeShift.noMultipleHelp', { + defaultMessage: + 'Time shift should be a multiple of the date histogram interval. Either adjust time shift or date histogram interval', + })) + } + isInvalid={Boolean(isLocalValueInvalid || localValueTooSmall || localValueNotMultiple)} > @@ -269,3 +286,87 @@ export function TimeShift({
); } + +export function getTimeShiftWarningMessages( + state: IndexPatternPrivateState, + { activeData }: FramePublicAPI +) { + if (!state) return; + const warningMessages: React.ReactNode[] = []; + Object.entries(state.layers).forEach(([layerId, layer]) => { + let dateHistogramInterval: null | moment.Duration = null; + const dateHistogramColumn = layer.columnOrder.find( + (colId) => layer.columns[colId].operationType === 'date_histogram' + ); + if (!dateHistogramColumn) { + return; + } + if (dateHistogramColumn && activeData && activeData[layerId]) { + const column = activeData[layerId].columns.find((col) => col.id === dateHistogramColumn); + if (column) { + dateHistogramInterval = search.aggs.parseInterval( + search.aggs.getDateHistogramMetaDataByDatatableColumn(column)?.interval || '' + ); + } + } + if (dateHistogramInterval === null) { + return; + } + const shiftInterval = dateHistogramInterval.asMilliseconds(); + let timeShifts: number[] = []; + const timeShiftMap: Record = {}; + Object.entries(layer.columns).forEach(([columnId, column]) => { + if (column.isBucketed) return; + let duration: number = 0; + if (column.timeShift) { + duration = (parseTimeShift(column.timeShift) as moment.Duration).asMilliseconds(); + } + timeShifts.push(duration); + if (!timeShiftMap[duration]) { + timeShiftMap[duration] = []; + } + timeShiftMap[duration].push(columnId); + }); + timeShifts = uniq(timeShifts); + + if (timeShifts.length < 2) { + return; + } + + timeShifts.forEach((timeShift) => { + if (timeShift === 0) return; + if (timeShift < shiftInterval) { + timeShiftMap[timeShift].forEach((columnId) => { + warningMessages.push( + {layer.columns[columnId].label}, + interval: dateHistogramInterval?.humanize(), + columnTimeShift: layer.columns[columnId].timeShift!, + }} + /> + ); + }); + } else if (!Number.isInteger(timeShift / shiftInterval)) { + timeShiftMap[timeShift].forEach((columnId) => { + warningMessages.push( + {layer.columns[columnId].label}, + interval: dateHistogramInterval?.humanize(), + columnTimeShift: layer.columns[columnId].timeShift!, + }} + /> + ); + }); + } + }); + }); + return warningMessages; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index 4ce82aa3a79c0e..f0155cc6d59fd7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -7,7 +7,7 @@ import { IStorageWrapper } from 'src/plugins/kibana_utils/public'; import { getIndexPatternDatasource, IndexPatternColumn } from './indexpattern'; -import { DatasourcePublicAPI, Operation, Datasource } from '../types'; +import { DatasourcePublicAPI, Operation, Datasource, FramePublicAPI } from '../types'; import { coreMock } from 'src/core/public/mocks'; import { IndexPatternPersistedState, IndexPatternPrivateState } from './types'; import { dataPluginMock } from '../../../../../src/plugins/data/public/mocks'; @@ -18,6 +18,7 @@ import { operationDefinitionMap, getErrorMessages } from './operations'; import { createMockedFullReference } from './operations/mocks'; import { indexPatternFieldEditorPluginMock } from 'src/plugins/index_pattern_field_editor/public/mocks'; import { uiActionsPluginMock } from '../../../../../src/plugins/ui_actions/public/mocks'; +import React from 'react'; jest.mock('./loader'); jest.mock('../id_generator'); @@ -1304,6 +1305,127 @@ describe('IndexPattern Data Source', () => { }); }); + describe('#getWarningMessages', () => { + it('should return mismatched time shifts', () => { + const state: IndexPatternPrivateState = { + indexPatternRefs: [], + existingFields: {}, + isFirstExistenceFetch: false, + indexPatterns: expectedIndexPatterns, + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1', 'col2', 'col3', 'col4', 'col5'], + columns: { + col1: { + operationType: 'date_histogram', + params: { + interval: '12h', + }, + label: '', + dataType: 'date', + isBucketed: true, + sourceField: 'timestamp', + }, + col2: { + operationType: 'count', + label: '', + dataType: 'number', + isBucketed: false, + sourceField: 'records', + }, + col3: { + operationType: 'count', + timeShift: '1h', + label: '', + dataType: 'number', + isBucketed: false, + sourceField: 'records', + }, + col4: { + operationType: 'count', + timeShift: '13h', + label: '', + dataType: 'number', + isBucketed: false, + sourceField: 'records', + }, + col5: { + operationType: 'count', + timeShift: '1w', + label: '', + dataType: 'number', + isBucketed: false, + sourceField: 'records', + }, + }, + }, + }, + currentIndexPatternId: '1', + }; + const warnings = indexPatternDatasource.getWarningMessages!(state, ({ + activeData: { + first: { + type: 'datatable', + rows: [], + columns: [ + { + id: 'col1', + name: 'col1', + meta: { + type: 'date', + source: 'esaggs', + sourceParams: { + type: 'date_histogram', + params: { + used_interval: '12h', + }, + }, + }, + }, + ], + }, + }, + } as unknown) as FramePublicAPI); + expect(warnings!.length).toBe(2); + expect((warnings![0] as React.ReactElement).props.id).toEqual( + 'xpack.lens.indexPattern.timeShiftSmallWarning' + ); + expect((warnings![1] as React.ReactElement).props.id).toEqual( + 'xpack.lens.indexPattern.timeShiftMultipleWarning' + ); + }); + + it('should prepend each error with its layer number on multi-layer chart', () => { + (getErrorMessages as jest.Mock).mockClear(); + (getErrorMessages as jest.Mock).mockReturnValueOnce(['error 1', 'error 2']); + const state: IndexPatternPrivateState = { + indexPatternRefs: [], + existingFields: {}, + isFirstExistenceFetch: false, + indexPatterns: expectedIndexPatterns, + layers: { + first: { + indexPatternId: '1', + columnOrder: [], + columns: {}, + }, + second: { + indexPatternId: '1', + columnOrder: [], + columns: {}, + }, + }, + currentIndexPatternId: '1', + }; + expect(indexPatternDatasource.getErrorMessages(state)).toEqual([ + { longMessage: 'Layer 1 error: error 1', shortMessage: '' }, + { longMessage: 'Layer 1 error: error 2', shortMessage: '' }, + ]); + expect(getErrorMessages).toHaveBeenCalledTimes(2); + }); + }); + describe('#updateStateOnCloseDimension', () => { it('should not update when there are no incomplete columns', () => { expect( diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index 0bf91b217c78aa..e66df35ea38f43 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -56,6 +56,7 @@ import { deleteColumn, isReferenced } from './operations'; import { UiActionsStart } from '../../../../../src/plugins/ui_actions/public'; import { GeoFieldWorkspacePanel } from '../editor_frame_service/editor_frame/workspace_panel/geo_field_workspace_panel'; import { DraggingIdentifier } from '../drag_drop'; +import { getTimeShiftWarningMessages } from './dimension_panel/time_shift'; export { OperationType, IndexPatternColumn, deleteColumn } from './operations'; @@ -457,6 +458,7 @@ export function getIndexPatternDatasource({ }); return messages.length ? messages : undefined; }, + getWarningMessages: getTimeShiftWarningMessages, checkIntegrity: (state) => { const ids = Object.values(state.layers || {}).map(({ indexPatternId }) => indexPatternId); return ids.filter((id) => !state.indexPatterns[id]); diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 6c799d3fd3d53e..cf082eab0f5420 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -259,6 +259,10 @@ export interface Datasource { * Check the internal state integrity and returns a list of missing references */ checkIntegrity: (state: T) => string[]; + /** + * The frame calls this function to display warnings about visualization + */ + getWarningMessages?: (state: T, frame: FramePublicAPI) => React.ReactNode[] | undefined; } /** From 047d99f518e4fa1d5db3f70d83e15e1e03e8d357 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 20 May 2021 14:30:52 +0200 Subject: [PATCH 22/31] improve warning --- .../dimension_panel/time_shift.tsx | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index 02eb3bd00e980d..823f41acf40bd3 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -294,7 +294,7 @@ export function getTimeShiftWarningMessages( if (!state) return; const warningMessages: React.ReactNode[] = []; Object.entries(state.layers).forEach(([layerId, layer]) => { - let dateHistogramInterval: null | moment.Duration = null; + let dateHistogramInterval: null | string = null; const dateHistogramColumn = layer.columnOrder.find( (colId) => layer.columns[colId].operationType === 'date_histogram' ); @@ -304,15 +304,14 @@ export function getTimeShiftWarningMessages( if (dateHistogramColumn && activeData && activeData[layerId]) { const column = activeData[layerId].columns.find((col) => col.id === dateHistogramColumn); if (column) { - dateHistogramInterval = search.aggs.parseInterval( - search.aggs.getDateHistogramMetaDataByDatatableColumn(column)?.interval || '' - ); + dateHistogramInterval = + search.aggs.getDateHistogramMetaDataByDatatableColumn(column)?.interval || null; } } if (dateHistogramInterval === null) { return; } - const shiftInterval = dateHistogramInterval.asMilliseconds(); + const shiftInterval = search.aggs.parseInterval(dateHistogramInterval)!.asMilliseconds(); let timeShifts: number[] = []; const timeShiftMap: Record = {}; Object.entries(layer.columns).forEach(([columnId, column]) => { @@ -344,8 +343,8 @@ export function getTimeShiftWarningMessages( defaultMessage="{label} uses a time shift of {columnTimeShift} which is smaller than the date histogram interval of {interval}. To prevent mismatched data, use a multiple of {interval} as time shift." values={{ label: {layer.columns[columnId].label}, - interval: dateHistogramInterval?.humanize(), - columnTimeShift: layer.columns[columnId].timeShift!, + interval: {dateHistogramInterval}, + columnTimeShift: {layer.columns[columnId].timeShift}, }} /> ); @@ -359,7 +358,7 @@ export function getTimeShiftWarningMessages( defaultMessage="{label} uses a time shift of {columnTimeShift} which is not a multiple of the date histogram interval of {interval}. To prevent mismatched data, use a multiple of {interval} as time shift." values={{ label: {layer.columns[columnId].label}, - interval: dateHistogramInterval?.humanize(), + interval: dateHistogramInterval, columnTimeShift: layer.columns[columnId].timeShift!, }} /> From f6620ff60ab0f19e7b6c4838943d3c5fca8ea90a Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 25 May 2021 11:23:03 +0200 Subject: [PATCH 23/31] review comments --- .../dimension_panel/dimension_editor.tsx | 2 +- .../dimension_panel/time_scaling.tsx | 8 +- .../dimension_panel/time_shift.tsx | 21 ++++- .../definitions/calculations/counter_rate.tsx | 6 +- .../definitions/calculations/differences.tsx | 4 +- .../calculations/moving_average.tsx | 4 +- .../definitions/calculations/utils.ts | 5 +- .../operations/definitions/cardinality.tsx | 28 ++++--- .../operations/definitions/count.tsx | 25 +++++- .../operations/definitions/last_value.tsx | 28 ++++--- .../operations/definitions/metrics.tsx | 11 ++- .../operations/definitions/percentile.tsx | 36 ++++++--- .../operations/time_scale_utils.test.ts | 77 ++++++++++++++++--- .../operations/time_scale_utils.ts | 30 ++++++-- 14 files changed, 218 insertions(+), 67 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 72d8c84b537680..70001175109309 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -489,7 +489,7 @@ export function DimensionEditor(props: DimensionEditorProps) { }, { title: i18n.translate('xpack.lens.indexPattern.timeShift.label', { - defaultMessage: 'Shift in time', + defaultMessage: 'Time shift', }), dataTestSubj: 'indexPattern-time-shift-enable', onClick: () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_scaling.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_scaling.tsx index bf5b64bf3d6152..61e5da5931e88e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_scaling.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_scaling.tsx @@ -27,7 +27,13 @@ export function setTimeScaling( const currentColumn = layer.columns[columnId]; const label = currentColumn.customLabel ? currentColumn.label - : adjustTimeScaleLabelSuffix(currentColumn.label, currentColumn.timeScale, timeScale); + : adjustTimeScaleLabelSuffix( + currentColumn.label, + currentColumn.timeScale, + timeScale, + currentColumn.timeShift, + currentColumn.timeShift + ); return { ...layer, columns: { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index 823f41acf40bd3..514588d534f1b4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -15,7 +15,11 @@ import React, { useEffect, useState } from 'react'; import { Query } from 'src/plugins/data/public'; import { search } from '../../../../../../src/plugins/data/public'; import { parseTimeShift } from '../../../../../../src/plugins/data/common'; -import { IndexPatternColumn, operationDefinitionMap } from '../operations'; +import { + adjustTimeScaleLabelSuffix, + IndexPatternColumn, + operationDefinitionMap, +} from '../operations'; import { IndexPattern, IndexPatternLayer, IndexPatternPrivateState } from '../types'; import { IndexPatternDimensionEditorProps } from './dimension_panel'; import { FramePublicAPI } from '../../types'; @@ -31,12 +35,23 @@ export function setTimeShift( layer: IndexPatternLayer, timeShift: string | undefined ) { + const currentColumn = layer.columns[columnId]; + const label = currentColumn.customLabel + ? currentColumn.label + : adjustTimeScaleLabelSuffix( + currentColumn.label, + currentColumn.timeScale, + currentColumn.timeScale, + currentColumn.timeShift, + timeShift + ); return { ...layer, columns: { ...layer.columns, [columnId]: { ...layer.columns[columnId], + label, timeShift, }, }, @@ -206,10 +221,10 @@ export function TimeShift({ display="columnCompressed" fullWidth label={i18n.translate('xpack.lens.indexPattern.timeShift.label', { - defaultMessage: 'Shift in time', + defaultMessage: 'Time shift', })} helpText={i18n.translate('xpack.lens.indexPattern.timeShift.help', { - defaultMessage: 'Time shift is specified by a number followed by a time unit', + defaultMessage: 'Enter the time shift number and unit', })} error={ (localValueTooSmall && diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx index 73d9136d55362f..823ec3eb58a924 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx @@ -70,7 +70,8 @@ export const counterRateOperation: OperationDefinition< ref && 'sourceField' in ref ? indexPattern.getFieldByName(ref.sourceField)?.displayName : undefined, - column.timeScale + column.timeScale, + column.timeShift ); }, toExpression: (layer, columnId) => { @@ -84,7 +85,8 @@ export const counterRateOperation: OperationDefinition< metric && 'sourceField' in metric ? indexPattern.getFieldByName(metric.sourceField)?.displayName : undefined, - timeScale + timeScale, + previousColumn?.timeShift ), dataType: 'number', operationType: 'counter_rate', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx index 220fcf6dc3c000..7c48b5742b8dbb 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/differences.tsx @@ -66,7 +66,7 @@ export const derivativeOperation: OperationDefinition< } }, getDefaultLabel: (column, indexPattern, columns) => { - return ofName(columns[column.references[0]]?.label, column.timeScale); + return ofName(columns[column.references[0]]?.label, column.timeScale, column.timeShift); }, toExpression: (layer, columnId) => { return dateBasedOperationToExpression(layer, columnId, 'derivative'); @@ -74,7 +74,7 @@ export const derivativeOperation: OperationDefinition< buildColumn: ({ referenceIds, previousColumn, layer }, columnParams) => { const ref = layer.columns[referenceIds[0]]; return { - label: ofName(ref?.label, previousColumn?.timeScale), + label: ofName(ref?.label, previousColumn?.timeScale, previousColumn?.timeShift), dataType: 'number', operationType: OPERATION_NAME, isBucketed: false, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx index 6351e076faecda..2ee4971fa67f4e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx @@ -80,7 +80,7 @@ export const movingAverageOperation: OperationDefinition< } }, getDefaultLabel: (column, indexPattern, columns) => { - return ofName(columns[column.references[0]]?.label, column.timeScale); + return ofName(columns[column.references[0]]?.label, column.timeScale, column.timeShift); }, toExpression: (layer, columnId) => { return dateBasedOperationToExpression(layer, columnId, 'moving_average', { @@ -94,7 +94,7 @@ export const movingAverageOperation: OperationDefinition< const metric = layer.columns[referenceIds[0]]; const { window = WINDOW_DEFAULT_VALUE } = columnParams; return { - label: ofName(metric?.label, previousColumn?.timeScale), + label: ofName(metric?.label, previousColumn?.timeScale, previousColumn?.timeShift), dataType: 'number', operationType: 'moving_average', isBucketed: false, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts index 59dbf74c11480c..1f4f097c6a7fb5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts @@ -16,10 +16,11 @@ import { operationDefinitionMap } from '..'; export const buildLabelFunction = (ofName: (name?: string) => string) => ( name?: string, - timeScale?: TimeScaleUnit + timeScale?: TimeScaleUnit, + timeShift?: string ) => { const rawLabel = ofName(name); - return adjustTimeScaleLabelSuffix(rawLabel, undefined, timeScale); + return adjustTimeScaleLabelSuffix(rawLabel, undefined, timeScale, undefined, timeShift); }; /** diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx index f53d6136ce2b6e..c5ea6d21ab4854 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx @@ -17,6 +17,7 @@ import { getSafeName, getFilter, } from './helpers'; +import { adjustTimeScaleLabelSuffix } from '../time_scale_utils'; const supportedTypes = new Set([ 'string', @@ -33,13 +34,19 @@ const SCALE = 'ratio'; const OPERATION_TYPE = 'unique_count'; const IS_BUCKETED = false; -function ofName(name: string) { - return i18n.translate('xpack.lens.indexPattern.cardinalityOf', { - defaultMessage: 'Unique count of {name}', - values: { - name, - }, - }); +function ofName(name: string, timeShift: string | undefined) { + return adjustTimeScaleLabelSuffix( + i18n.translate('xpack.lens.indexPattern.cardinalityOf', { + defaultMessage: 'Unique count of {name}', + values: { + name, + }, + }), + undefined, + undefined, + undefined, + timeShift + ); } export interface CardinalityIndexPatternColumn @@ -81,10 +88,11 @@ export const cardinalityOperation: OperationDefinition ofName(getSafeName(column.sourceField, indexPattern)), + getDefaultLabel: (column, indexPattern) => + ofName(getSafeName(column.sourceField, indexPattern), column.timeShift), buildColumn({ field, previousColumn }, columnParams) { return { - label: ofName(field.displayName), + label: ofName(field.displayName, previousColumn?.timeShift), dataType: 'number', operationType: OPERATION_TYPE, scale: SCALE, @@ -107,7 +115,7 @@ export const cardinalityOperation: OperationDefinition { return { ...oldColumn, - label: ofName(field.displayName), + label: ofName(field.displayName, oldColumn.timeShift), sourceField: field.name, }; }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx index 24d34940eccdd8..7233b88953eb29 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx @@ -38,7 +38,13 @@ export const countOperation: OperationDefinition { return { ...oldColumn, - label: adjustTimeScaleLabelSuffix(field.displayName, undefined, oldColumn.timeScale), + label: adjustTimeScaleLabelSuffix( + field.displayName, + undefined, + oldColumn.timeScale, + undefined, + oldColumn.timeShift + ), sourceField: field.name, }; }, @@ -51,10 +57,23 @@ export const countOperation: OperationDefinition adjustTimeScaleLabelSuffix(countLabel, undefined, column.timeScale), + getDefaultLabel: (column) => + adjustTimeScaleLabelSuffix( + countLabel, + undefined, + column.timeScale, + undefined, + column.timeShift + ), buildColumn({ field, previousColumn }, columnParams) { return { - label: adjustTimeScaleLabelSuffix(countLabel, undefined, previousColumn?.timeScale), + label: adjustTimeScaleLabelSuffix( + countLabel, + undefined, + previousColumn?.timeScale, + undefined, + previousColumn?.timeShift + ), dataType: 'number', operationType: 'count', isBucketed: false, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx index a2003204f534cc..dbfc5c70545122 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/last_value.tsx @@ -21,14 +21,21 @@ import { getSafeName, getFilter, } from './helpers'; +import { adjustTimeScaleLabelSuffix } from '../time_scale_utils'; -function ofName(name: string) { - return i18n.translate('xpack.lens.indexPattern.lastValueOf', { - defaultMessage: 'Last value of {name}', - values: { - name, - }, - }); +function ofName(name: string, timeShift: string | undefined) { + return adjustTimeScaleLabelSuffix( + i18n.translate('xpack.lens.indexPattern.lastValueOf', { + defaultMessage: 'Last value of {name}', + values: { + name, + }, + }), + undefined, + undefined, + undefined, + timeShift + ); } const supportedTypes = new Set(['string', 'boolean', 'number', 'ip']); @@ -96,7 +103,8 @@ export const lastValueOperation: OperationDefinition ofName(getSafeName(column.sourceField, indexPattern)), + getDefaultLabel: (column, indexPattern) => + ofName(getSafeName(column.sourceField, indexPattern), column.timeShift), input: 'field', onFieldChange: (oldColumn, field) => { const newParams = { ...oldColumn.params }; @@ -107,7 +115,7 @@ export const lastValueOperation: OperationDefinition>({ }) { const labelLookup = (name: string, column?: BaseIndexPatternColumn) => { const label = ofName(name); - if (!optionalTimeScaling) { - return label; - } - return adjustTimeScaleLabelSuffix(label, undefined, column?.timeScale); + return adjustTimeScaleLabelSuffix( + label, + undefined, + optionalTimeScaling ? column?.timeScale : undefined, + undefined, + column?.timeShift + ); }; return { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx index 100a3f8ca57b34..7624c1b9bf83b7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx @@ -20,6 +20,7 @@ import { getFilter, } from './helpers'; import { FieldBasedIndexPatternColumn } from './column_types'; +import { adjustTimeScaleLabelSuffix } from '../time_scale_utils'; export interface PercentileIndexPatternColumn extends FieldBasedIndexPatternColumn { operationType: 'percentile'; @@ -34,12 +35,18 @@ export interface PercentileIndexPatternColumn extends FieldBasedIndexPatternColu }; } -function ofName(name: string, percentile: number) { - return i18n.translate('xpack.lens.indexPattern.percentileOf', { - defaultMessage: - '{percentile, selectordinal, one {#st} two {#nd} few {#rd} other {#th}} percentile of {name}', - values: { name, percentile }, - }); +function ofName(name: string, percentile: number, timeShift: string | undefined) { + return adjustTimeScaleLabelSuffix( + i18n.translate('xpack.lens.indexPattern.percentileOf', { + defaultMessage: + '{percentile, selectordinal, one {#st} two {#nd} few {#rd} other {#th}} percentile of {name}', + values: { name, percentile }, + }), + undefined, + undefined, + undefined, + timeShift + ); } const DEFAULT_PERCENTILE_VALUE = 95; @@ -75,7 +82,11 @@ export const percentileOperation: OperationDefinition - ofName(getSafeName(column.sourceField, indexPattern), column.params.percentile), + ofName( + getSafeName(column.sourceField, indexPattern), + column.params.percentile, + column.timeShift + ), buildColumn: ({ field, previousColumn, indexPattern }, columnParams) => { const existingPercentileParam = previousColumn?.operationType === 'percentile' && @@ -85,7 +96,11 @@ export const percentileOperation: OperationDefinition { return { ...oldColumn, - label: ofName(field.displayName, oldColumn.params.percentile), + label: ofName(field.displayName, oldColumn.params.percentile, oldColumn.timeShift), sourceField: field.name, }; }, @@ -149,7 +164,8 @@ export const percentileOperation: OperationDefinition { describe('adjustTimeScaleLabelSuffix', () => { it('should should remove existing suffix', () => { - expect(adjustTimeScaleLabelSuffix('abc per second', 's', undefined)).toEqual('abc'); - expect(adjustTimeScaleLabelSuffix('abc per hour', 'h', undefined)).toEqual('abc'); + expect( + adjustTimeScaleLabelSuffix('abc per second', 's', undefined, undefined, undefined) + ).toEqual('abc'); + expect( + adjustTimeScaleLabelSuffix('abc per hour', 'h', undefined, undefined, undefined) + ).toEqual('abc'); + expect(adjustTimeScaleLabelSuffix('abc -3d', undefined, undefined, '3d', undefined)).toEqual( + 'abc' + ); + expect( + adjustTimeScaleLabelSuffix('abc per hour -3d', 'h', undefined, '3d', undefined) + ).toEqual('abc'); }); it('should add suffix', () => { - expect(adjustTimeScaleLabelSuffix('abc', undefined, 's')).toEqual('abc per second'); - expect(adjustTimeScaleLabelSuffix('abc', undefined, 'd')).toEqual('abc per day'); + expect(adjustTimeScaleLabelSuffix('abc', undefined, 's', undefined, undefined)).toEqual( + 'abc per second' + ); + expect(adjustTimeScaleLabelSuffix('abc', undefined, 'd', undefined, undefined)).toEqual( + 'abc per day' + ); + expect(adjustTimeScaleLabelSuffix('abc', undefined, undefined, undefined, '12h')).toEqual( + 'abc -12h' + ); + expect(adjustTimeScaleLabelSuffix('abc', undefined, 'h', undefined, '12h')).toEqual( + 'abc per hour -12h' + ); + }); + + it('should add and remove at the same time', () => { + expect(adjustTimeScaleLabelSuffix('abc per hour', 'h', undefined, undefined, '1d')).toEqual( + 'abc -1d' + ); + expect(adjustTimeScaleLabelSuffix('abc -1d', undefined, 'h', '1d', undefined)).toEqual( + 'abc per hour' + ); }); it('should change suffix', () => { - expect(adjustTimeScaleLabelSuffix('abc per second', 's', 'd')).toEqual('abc per day'); - expect(adjustTimeScaleLabelSuffix('abc per day', 'd', 's')).toEqual('abc per second'); + expect(adjustTimeScaleLabelSuffix('abc per second', 's', 'd', undefined, undefined)).toEqual( + 'abc per day' + ); + expect(adjustTimeScaleLabelSuffix('abc per day', 'd', 's', undefined, undefined)).toEqual( + 'abc per second' + ); + expect(adjustTimeScaleLabelSuffix('abc per day -3h', 'd', 's', '3h', '3h')).toEqual( + 'abc per second -3h' + ); + expect(adjustTimeScaleLabelSuffix('abc per day -3h', 'd', 'd', '3h', '4h')).toEqual( + 'abc per day -4h' + ); }); it('should keep current state', () => { - expect(adjustTimeScaleLabelSuffix('abc', undefined, undefined)).toEqual('abc'); - expect(adjustTimeScaleLabelSuffix('abc per day', 'd', 'd')).toEqual('abc per day'); + expect(adjustTimeScaleLabelSuffix('abc', undefined, undefined, undefined, undefined)).toEqual( + 'abc' + ); + expect(adjustTimeScaleLabelSuffix('abc per day', 'd', 'd', undefined, undefined)).toEqual( + 'abc per day' + ); + expect(adjustTimeScaleLabelSuffix('abc -1h', undefined, undefined, '1h', '1h')).toEqual( + 'abc -1h' + ); + expect(adjustTimeScaleLabelSuffix('abc per day -1h', 'd', 'd', '1h', '1h')).toEqual( + 'abc per day -1h' + ); }); it('should not fail on inconsistent input', () => { - expect(adjustTimeScaleLabelSuffix('abc', 's', undefined)).toEqual('abc'); - expect(adjustTimeScaleLabelSuffix('abc', 's', 'd')).toEqual('abc per day'); - expect(adjustTimeScaleLabelSuffix('abc per day', 's', undefined)).toEqual('abc per day'); + expect(adjustTimeScaleLabelSuffix('abc', 's', undefined, undefined, undefined)).toEqual( + 'abc' + ); + expect(adjustTimeScaleLabelSuffix('abc', 's', 'd', undefined, undefined)).toEqual( + 'abc per day' + ); + expect( + adjustTimeScaleLabelSuffix('abc per day', 's', undefined, undefined, undefined) + ).toEqual('abc per day'); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/time_scale_utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/time_scale_utils.ts index 07806a32665dda..a0b61060b9f3ad 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/time_scale_utils.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/time_scale_utils.ts @@ -12,24 +12,36 @@ import type { IndexPatternColumn } from './definitions'; export const DEFAULT_TIME_SCALE = 's' as TimeScaleUnit; +function getSuffix(scale: TimeScaleUnit | undefined, shift: string | undefined) { + return ( + (shift || scale ? ' ' : '') + + (scale ? unitSuffixesLong[scale] : '') + + (shift && scale ? ' ' : '') + + (shift ? `-${shift}` : '') + ); +} + export function adjustTimeScaleLabelSuffix( oldLabel: string, previousTimeScale: TimeScaleUnit | undefined, - newTimeScale: TimeScaleUnit | undefined + newTimeScale: TimeScaleUnit | undefined, + previousShift: string | undefined, + newShift: string | undefined ) { let cleanedLabel = oldLabel; // remove added suffix if column had a time scale previously - if (previousTimeScale) { - const suffixPosition = oldLabel.lastIndexOf(` ${unitSuffixesLong[previousTimeScale]}`); + if (previousTimeScale || previousShift) { + const suffix = getSuffix(previousTimeScale, previousShift); + const suffixPosition = oldLabel.lastIndexOf(suffix); if (suffixPosition !== -1) { cleanedLabel = oldLabel.substring(0, suffixPosition); } } - if (!newTimeScale) { + if (!newTimeScale && !newShift) { return cleanedLabel; } // add new suffix if column has a time scale now - return `${cleanedLabel} ${unitSuffixesLong[newTimeScale]}`; + return `${cleanedLabel}${getSuffix(newTimeScale, newShift)}`; } export function adjustTimeScaleOnOtherColumnChange( @@ -54,6 +66,12 @@ export function adjustTimeScaleOnOtherColumnChange return { ...column, timeScale: undefined, - label: adjustTimeScaleLabelSuffix(column.label, column.timeScale, undefined), + label: adjustTimeScaleLabelSuffix( + column.label, + column.timeScale, + undefined, + column.timeShift, + column.timeShift + ), }; } From eaa08195c306ed4dc0a31014866005aaebd9f3fc Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 25 May 2021 17:12:03 +0200 Subject: [PATCH 24/31] adjust wording --- .../operations/definitions/terms/index.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx index 651cf556b09d89..02d407fc7360e1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx @@ -75,11 +75,11 @@ function getDisallowedTermsMessage( return { message: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShifts', { defaultMessage: - "Can't use multiple time shifts in a single layer together with dynamic top values. Either use the same time shift for all metrics or use filters instead of top values.", + 'In a single layer, you are unable to combine multiple time shifts and dynamic top values. Use the same value for all time shifts, or use filters instead of top values.', }), fixAction: { label: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShiftsFixActionLabel', { - defaultMessage: 'Pin current top values', + defaultMessage: 'Use filters', }), newState: async (core: CoreStart, frame: FramePublicAPI, layerId: string) => { const currentColumn = layer.columns[columnId] as TermsIndexPatternColumn; From b066335608372fb0e4e507622d463bcf40f49e85 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 25 May 2021 19:03:02 +0200 Subject: [PATCH 25/31] fix test --- .../operations/definitions/terms/terms.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx index 3baf004cf01bda..f4b327c175ea6f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx @@ -1037,7 +1037,7 @@ describe('terms', () => { expect(termsOperation.getErrorMessage!(layer, 'col1', indexPattern)).toEqual([ expect.objectContaining({ message: - "Can't use multiple time shifts in a single layer together with dynamic top values. Either use the same time shift for all metrics or use filters instead of top values.", + 'In a single layer, you are unable to combine multiple time shifts and dynamic top values. Use the same value for all time shifts, or use filters instead of top values.', }), ]); }); From 9c045a7b3af2482ca9427a1dca52d98414b78549 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 26 May 2021 14:52:33 +0200 Subject: [PATCH 26/31] fix previous time shift error checks --- .../dimension_panel/time_shift.tsx | 6 +++++- .../indexpattern_datasource/indexpattern.test.ts | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index 514588d534f1b4..deaad50d70206c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -333,7 +333,11 @@ export function getTimeShiftWarningMessages( if (column.isBucketed) return; let duration: number = 0; if (column.timeShift) { - duration = (parseTimeShift(column.timeShift) as moment.Duration).asMilliseconds(); + const parsedTimeShift = parseTimeShift(column.timeShift); + if (parsedTimeShift === 'previous' || parsedTimeShift === 'invalid') { + return; + } + duration = parsedTimeShift.asMilliseconds(); } timeShifts.push(duration); if (!timeShiftMap[duration]) { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index f0155cc6d59fd7..fe212a878db840 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -1315,7 +1315,7 @@ describe('IndexPattern Data Source', () => { layers: { first: { indexPatternId: '1', - columnOrder: ['col1', 'col2', 'col3', 'col4', 'col5'], + columnOrder: ['col1', 'col2', 'col3', 'col4', 'col5', 'col6'], columns: { col1: { operationType: 'date_histogram', @@ -1358,6 +1358,14 @@ describe('IndexPattern Data Source', () => { isBucketed: false, sourceField: 'records', }, + col6: { + operationType: 'count', + timeShift: 'previous', + label: '', + dataType: 'number', + isBucketed: false, + sourceField: 'records', + }, }, }, }, From 026371fc9840fc76c01f085690392f56beac0d6a Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 27 May 2021 11:05:20 +0200 Subject: [PATCH 27/31] add label suffix for cumulative sum --- .../definitions/calculations/cumulative_sum.tsx | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx index 744d362707f142..c4f01e27be886f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx @@ -13,11 +13,12 @@ import { getErrorsForDateReference, dateBasedOperationToExpression, hasDateField, + buildLabelFunction, } from './utils'; import { OperationDefinition } from '..'; import { getFormatFromPreviousColumn, getFilter } from '../helpers'; -const ofName = (name?: string) => { +const ofName = buildLabelFunction((name?: string) => { return i18n.translate('xpack.lens.indexPattern.cumulativeSumOf', { defaultMessage: 'Cumulative sum of {name}', values: { @@ -28,7 +29,7 @@ const ofName = (name?: string) => { }), }, }); -}; +}); export type CumulativeSumIndexPatternColumn = FormattedIndexPatternColumn & ReferenceBasedIndexPatternColumn & { @@ -67,7 +68,9 @@ export const cumulativeSumOperation: OperationDefinition< return ofName( ref && 'sourceField' in ref ? indexPattern.getFieldByName(ref.sourceField)?.displayName - : undefined + : undefined, + undefined, + column.timeShift ); }, toExpression: (layer, columnId) => { @@ -79,7 +82,9 @@ export const cumulativeSumOperation: OperationDefinition< label: ofName( ref && 'sourceField' in ref ? indexPattern.getFieldByName(ref.sourceField)?.displayName - : undefined + : undefined, + undefined, + previousColumn?.timeShift ), dataType: 'number', operationType: 'cumulative_sum', From 6891e8b015c8578fa04da750ae29272aeeca804b Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 27 May 2021 18:55:46 +0200 Subject: [PATCH 28/31] review comments --- .../dimension_panel/time_shift.tsx | 6 ++++-- .../operations/definitions/terms/index.tsx | 9 ++++++--- .../operations/definitions/terms/terms.test.tsx | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index deaad50d70206c..538b395bf16241 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -11,7 +11,7 @@ import { EuiComboBox } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { uniq } from 'lodash'; import { i18n } from '@kbn/i18n'; -import React, { useEffect, useState } from 'react'; +import React, { useEffect, useRef, useState } from 'react'; import { Query } from 'src/plugins/data/public'; import { search } from '../../../../../../src/plugins/data/public'; import { parseTimeShift } from '../../../../../../src/plugins/data/common'; @@ -146,6 +146,7 @@ export function TimeShift({ activeData: IndexPatternDimensionEditorProps['activeData']; layerId: string; }) { + const focusSetRef = useRef(false); const [localValue, setLocalValue] = useState(selectedColumn.timeShift); useEffect(() => { setLocalValue(selectedColumn.timeShift); @@ -211,7 +212,8 @@ export function TimeShift({ ref={(r) => { if (r && isFocused) { const timeShiftInput = r.querySelector('[data-test-subj="comboBoxSearchInput"]'); - if (timeShiftInput instanceof HTMLInputElement) { + if (!focusSetRef.current && timeShiftInput instanceof HTMLInputElement) { + focusSetRef.current = true; timeShiftInput.focus(); } } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx index 02d407fc7360e1..b891a5061a5832 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx @@ -75,7 +75,7 @@ function getDisallowedTermsMessage( return { message: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShifts', { defaultMessage: - 'In a single layer, you are unable to combine multiple time shifts and dynamic top values. Use the same value for all time shifts, or use filters instead of top values.', + 'In a single layer, you are unable to combine metrics with different time shifts and dynamic top values. Use the same time shift value for all metrics, or use filters instead of top values.', }), fixAction: { label: i18n.translate('xpack.lens.indexPattern.termsWithMultipleShiftsFixActionLabel', { @@ -84,12 +84,15 @@ function getDisallowedTermsMessage( newState: async (core: CoreStart, frame: FramePublicAPI, layerId: string) => { const currentColumn = layer.columns[columnId] as TermsIndexPatternColumn; const fieldName = currentColumn.sourceField; + const activeDataFieldNameMatch = + frame.activeData?.[layerId].columns.find(({ id }) => id === columnId)?.meta.field === + fieldName; let currentTerms = uniq( frame.activeData?.[layerId].rows .map((row) => row[columnId] as string) .filter((term) => typeof term === 'string' && term !== '__other__') || [] ); - if (currentTerms.length === 0) { + if (!activeDataFieldNameMatch || currentTerms.length === 0) { const response: FieldStatsResponse = await core.http.post( `/api/lens/index_stats/${indexPattern.id}/field`, { @@ -115,7 +118,7 @@ function getDisallowedTermsMessage( ...layer.columns, [columnId]: { label: i18n.translate('xpack.lens.indexPattern.pinnedTopValuesLabel', { - defaultMessage: 'Pinned top values of {field}', + defaultMessage: 'Filters of {field}', values: { field: fieldName, }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx index f4b327c175ea6f..a0a46556a8459e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx @@ -1037,7 +1037,7 @@ describe('terms', () => { expect(termsOperation.getErrorMessage!(layer, 'col1', indexPattern)).toEqual([ expect.objectContaining({ message: - 'In a single layer, you are unable to combine multiple time shifts and dynamic top values. Use the same value for all time shifts, or use filters instead of top values.', + 'In a single layer, you are unable to combine metrics with different time shifts and dynamic top values. Use the same time shift value for all metrics, or use filters instead of top values.', }), ]); }); From 8e9f7248708c1e803a072d21540f049a1cf25e01 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 28 May 2021 09:26:55 +0200 Subject: [PATCH 29/31] fix functional test --- x-pack/test/functional/apps/lens/time_shift.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/functional/apps/lens/time_shift.ts b/x-pack/test/functional/apps/lens/time_shift.ts index 9ddd2c9daf58d3..57c2fc194d0c05 100644 --- a/x-pack/test/functional/apps/lens/time_shift.ts +++ b/x-pack/test/functional/apps/lens/time_shift.ts @@ -62,7 +62,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(await PageObjects.lens.getDatatableCellText(2, 2)).to.eql('5,541.5'); expect(await PageObjects.lens.getDatatableCellText(2, 3)).to.eql('3,628'); - expect(await PageObjects.lens.getDatatableHeaderText(0)).to.eql('Pinned top values of ip'); + expect(await PageObjects.lens.getDatatableHeaderText(0)).to.eql('Filters of ip'); }); }); } From 3849b044d3e9ee1c3a31a19d0023405c33faa82c Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 31 May 2021 09:57:26 +0200 Subject: [PATCH 30/31] review comments --- .../search/aggs/utils/parse_time_shift.ts | 5 +- .../workspace_panel/workspace_panel.tsx | 73 ++++++++++++------- .../dimension_panel/advanced_options.tsx | 11 ++- .../dimension_panel/time_shift.tsx | 6 +- .../operations/definitions/cardinality.tsx | 5 +- .../operations/definitions/count.tsx | 1 + .../operations/definitions/last_value.tsx | 1 + .../operations/definitions/metrics.tsx | 1 + .../operations/definitions/percentile.tsx | 1 + 9 files changed, 67 insertions(+), 37 deletions(-) diff --git a/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts b/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts index 256addac915209..4d8ee0f8891732 100644 --- a/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts +++ b/src/plugins/data/common/search/aggs/utils/parse_time_shift.ts @@ -16,10 +16,11 @@ type AllowedUnit = typeof allowedUnits[number]; * Allowed values are the string 'previous' and an integer followed by the units s,m,h,d,w,M,y * */ export const parseTimeShift = (val: string): moment.Duration | 'previous' | 'invalid' => { - if (val === 'previous') { + const trimmedVal = val.trim(); + if (trimmedVal === 'previous') { return 'previous'; } - const [, amount, unit] = val.match(/^(\d+)(\w)$/) || []; + const [, amount, unit] = trimmedVal.match(/^(\d+)(\w)$/) || []; const parsedAmount = Number(amount); if (Number.isNaN(parsedAmount) || !allowedUnits.includes(unit as AllowedUnit)) { return 'invalid'; diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx index 51e7e49a026c25..45abbf120042d0 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx @@ -19,6 +19,7 @@ import { EuiLink, EuiPageContentBody, EuiButton, + EuiSpacer, } from '@elastic/eui'; import { CoreStart, ApplicationStart } from 'kibana/public'; import { @@ -452,6 +453,41 @@ export const VisualizationWrapper = ({ [dispatchLens] ); + function renderFixAction( + validationError: + | { + shortMessage: string; + longMessage: string; + fixAction?: + | { label: string; newState: (framePublicAPI: FramePublicAPI) => Promise } + | undefined; + } + | undefined + ) { + return ( + validationError && + validationError.fixAction && + activeDatasourceId && ( + <> + { + const newState = await validationError.fixAction?.newState(framePublicAPI); + dispatch({ + type: 'UPDATE_DATASOURCE_STATE', + datasourceId: activeDatasourceId, + updater: newState, + }); + }} + > + {validationError.fixAction.label} + + + + ) + ); + } + if (localState.configurationValidationError?.length) { let showExtraErrors = null; let showExtraErrorsAction = null; @@ -460,14 +496,17 @@ export const VisualizationWrapper = ({ if (localState.expandError) { showExtraErrors = localState.configurationValidationError .slice(1) - .map(({ longMessage }) => ( -

- {longMessage} -

+ .map((validationError) => ( + <> +

+ {validationError.longMessage} +

+ {renderFixAction(validationError)} + )); } else { showExtraErrorsAction = ( @@ -499,23 +538,7 @@ export const VisualizationWrapper = ({

{localState.configurationValidationError[0].longMessage}

- {localState.configurationValidationError[0].fixAction && activeDatasourceId && ( - { - const newState = await localState.configurationValidationError?.[0].fixAction?.newState( - framePublicAPI - ); - dispatch({ - type: 'UPDATE_DATASOURCE_STATE', - datasourceId: activeDatasourceId, - updater: newState, - }); - }} - > - {localState.configurationValidationError[0].fixAction.label} - - )} + {renderFixAction(localState.configurationValidationError?.[0])} {showExtraErrors} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/advanced_options.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/advanced_options.tsx index ea5eb14d9c20eb..c8676faad0eea7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/advanced_options.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/advanced_options.tsx @@ -20,9 +20,7 @@ export function AdvancedOptions(props: { }) { const [popoverOpen, setPopoverOpen] = useState(false); const popoverOptions = props.options.filter((option) => option.showInPopover); - const inlineOptions = props.options - .filter((option) => option.inlineElement) - .map((option) => React.cloneElement(option.inlineElement!, { key: option.dataTestSubj })); + const inlineOptions = props.options.filter((option) => option.inlineElement); return ( <> @@ -74,7 +72,12 @@ export function AdvancedOptions(props: { {inlineOptions.length > 0 && ( <> - {inlineOptions} + {inlineOptions.map((option, index) => ( + <> + {React.cloneElement(option.inlineElement!, { key: option.dataTestSubj })} + {index !== inlineOptions.length - 1 && } + + ))} )} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx index 538b395bf16241..0ac02c15b34a50 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_shift.tsx @@ -35,6 +35,7 @@ export function setTimeShift( layer: IndexPatternLayer, timeShift: string | undefined ) { + const trimmedTimeShift = timeShift?.trim(); const currentColumn = layer.columns[columnId]; const label = currentColumn.customLabel ? currentColumn.label @@ -43,7 +44,7 @@ export function setTimeShift( currentColumn.timeScale, currentColumn.timeScale, currentColumn.timeShift, - timeShift + trimmedTimeShift ); return { ...layer, @@ -52,7 +53,7 @@ export function setTimeShift( [columnId]: { ...layer.columns[columnId], label, - timeShift, + timeShift: trimmedTimeShift, }, }, }; @@ -272,6 +273,7 @@ export function TimeShift({ onChange={(choices) => { if (choices.length === 0) { updateLayer(setTimeShift(columnId, layer, '')); + setLocalValue(''); return; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx index c5ea6d21ab4854..1911af0a6f679b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx @@ -84,10 +84,6 @@ export const cardinalityOperation: OperationDefinition ofName(getSafeName(column.sourceField, indexPattern), column.timeShift), buildColumn({ field, previousColumn }, columnParams) { @@ -109,6 +105,7 @@ export const cardinalityOperation: OperationDefinition>({ enabled: true, schema: 'metric', field: column.sourceField, + // time shift is added to wrapping aggFilteredMetric if filter is set timeShift: column.filter ? undefined : column.timeShift, }).toAst(); }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx index 169fefb4caeb54..aa8f951d46b4f2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx @@ -130,6 +130,7 @@ export const percentileOperation: OperationDefinition Date: Wed, 2 Jun 2021 13:47:17 +0200 Subject: [PATCH 31/31] fix tests --- .../dimension_panel/dimension_panel.test.tsx | 100 ++++++++---------- 1 file changed, 44 insertions(+), 56 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx index 2a07fb01930658..03db6141b917f9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx @@ -1418,23 +1418,20 @@ describe('IndexPatternDimensionEditorPanel', () => { .dive() .find('[data-test-subj="indexPattern-time-shift-enable"]') .prop('onClick')!({} as MouseEvent); - expect(props.setState).toHaveBeenCalledWith( - { - ...props.state, - layers: { - first: { - ...props.state.layers.first, - columns: { - ...props.state.layers.first.columns, - col2: expect.objectContaining({ - timeShift: '', - }), - }, + expect((props.setState as jest.Mock).mock.calls[0][0](props.state)).toEqual({ + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeShift: '', + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should carry over time shift to other operation if possible', () => { @@ -1448,23 +1445,20 @@ describe('IndexPatternDimensionEditorPanel', () => { wrapper .find('button[data-test-subj="lns-indexPatternDimension-count incompatible"]') .simulate('click'); - expect(props.setState).toHaveBeenCalledWith( - { - ...props.state, - layers: { - first: { - ...props.state.layers.first, - columns: { - ...props.state.layers.first.columns, - col2: expect.objectContaining({ - timeShift: '1d', - }), - }, + expect((props.setState as jest.Mock).mock.calls[0][0](props.state)).toEqual({ + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeShift: '1d', + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should allow to change time shift', () => { @@ -1473,23 +1467,20 @@ describe('IndexPatternDimensionEditorPanel', () => { }); wrapper = mount(); wrapper.find(TimeShift).find(EuiComboBox).prop('onCreateOption')!('1h', []); - expect(props.setState).toHaveBeenCalledWith( - { - ...props.state, - layers: { - first: { - ...props.state.layers.first, - columns: { - ...props.state.layers.first.columns, - col2: expect.objectContaining({ - timeShift: '1h', - }), - }, + expect((props.setState as jest.Mock).mock.calls[0][0](props.state)).toEqual({ + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeShift: '1h', + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); it('should allow to time shift', () => { @@ -1504,23 +1495,20 @@ describe('IndexPatternDimensionEditorPanel', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any {} as any ); - expect(props.setState).toHaveBeenCalledWith( - { - ...props.state, - layers: { - first: { - ...props.state.layers.first, - columns: { - ...props.state.layers.first.columns, - col2: expect.objectContaining({ - timeShift: undefined, - }), - }, + expect((props.setState as jest.Mock).mock.calls[0][0](props.state)).toEqual({ + ...props.state, + layers: { + first: { + ...props.state.layers.first, + columns: { + ...props.state.layers.first.columns, + col2: expect.objectContaining({ + timeShift: undefined, + }), }, }, }, - { shouldRemoveDimension: false, shouldReplaceDimension: true } - ); + }); }); });