From 2e57f002530499572a93628e518b025203f78c85 Mon Sep 17 00:00:00 2001 From: Anton Dosov Date: Wed, 8 Dec 2021 11:06:02 +0100 Subject: [PATCH] Add Sampler & Diversified Sampler aggs to AggConfigs (#120135) --- .../data/common/search/aggs/agg_types.ts | 4 + .../common/search/aggs/aggs_service.test.ts | 4 + .../search/aggs/buckets/bucket_agg_types.ts | 2 + .../aggs/buckets/diversified_sampler.ts | 62 +++++++++ .../buckets/diversified_sampler_fn.test.ts | 58 +++++++++ .../aggs/buckets/diversified_sampler_fn.ts | 90 +++++++++++++ .../data/common/search/aggs/buckets/index.ts | 4 + .../common/search/aggs/buckets/sampler.ts | 43 +++++++ .../search/aggs/buckets/sampler_fn.test.ts | 52 ++++++++ .../common/search/aggs/buckets/sampler_fn.ts | 77 +++++++++++ src/plugins/data/common/search/aggs/types.ts | 4 + .../public/search/aggs/aggs_service.test.ts | 4 +- .../heatmap/public/vis_type/heatmap.tsx | 27 +++- .../metric/public/metric_vis_type.ts | 9 +- .../pie/public/sample_vis.test.mocks.ts | 18 ++- .../vis_types/pie/public/vis_type/pie.ts | 18 ++- .../vis_types/table/public/table_vis_type.ts | 4 +- src/plugins/vis_types/vislib/public/gauge.ts | 9 +- src/plugins/vis_types/vislib/public/goal.ts | 9 +- .../point_series/point_series.mocks.ts | 81 ++++++++++-- .../xy/public/sample_vis.test.mocks.ts | 27 +++- .../vis_types/xy/public/vis_types/area.ts | 27 +++- .../xy/public/vis_types/histogram.ts | 27 +++- .../xy/public/vis_types/horizontal_bar.ts | 27 +++- .../vis_types/xy/public/vis_types/line.ts | 27 +++- .../run_pipeline/esaggs_sampler.ts | 121 ++++++++++++++++++ .../test_suites/run_pipeline/index.ts | 1 + 27 files changed, 798 insertions(+), 38 deletions(-) create mode 100644 src/plugins/data/common/search/aggs/buckets/diversified_sampler.ts create mode 100644 src/plugins/data/common/search/aggs/buckets/diversified_sampler_fn.test.ts create mode 100644 src/plugins/data/common/search/aggs/buckets/diversified_sampler_fn.ts create mode 100644 src/plugins/data/common/search/aggs/buckets/sampler.ts create mode 100644 src/plugins/data/common/search/aggs/buckets/sampler_fn.test.ts create mode 100644 src/plugins/data/common/search/aggs/buckets/sampler_fn.ts create mode 100644 test/interpreter_functional/test_suites/run_pipeline/esaggs_sampler.ts diff --git a/src/plugins/data/common/search/aggs/agg_types.ts b/src/plugins/data/common/search/aggs/agg_types.ts index dd930887f9d192..a84fddb19c5faf 100644 --- a/src/plugins/data/common/search/aggs/agg_types.ts +++ b/src/plugins/data/common/search/aggs/agg_types.ts @@ -62,6 +62,8 @@ export const getAggTypes = () => ({ { name: BUCKET_TYPES.SIGNIFICANT_TERMS, fn: buckets.getSignificantTermsBucketAgg }, { name: BUCKET_TYPES.GEOHASH_GRID, fn: buckets.getGeoHashBucketAgg }, { name: BUCKET_TYPES.GEOTILE_GRID, fn: buckets.getGeoTitleBucketAgg }, + { name: BUCKET_TYPES.SAMPLER, fn: buckets.getSamplerBucketAgg }, + { name: BUCKET_TYPES.DIVERSIFIED_SAMPLER, fn: buckets.getDiversifiedSamplerBucketAgg }, ], }); @@ -79,6 +81,8 @@ export const getAggTypesFunctions = () => [ buckets.aggDateHistogram, buckets.aggTerms, buckets.aggMultiTerms, + buckets.aggSampler, + buckets.aggDiversifiedSampler, metrics.aggAvg, metrics.aggBucketAvg, metrics.aggBucketMax, diff --git a/src/plugins/data/common/search/aggs/aggs_service.test.ts b/src/plugins/data/common/search/aggs/aggs_service.test.ts index be3fbae26174ab..571083c18156f7 100644 --- a/src/plugins/data/common/search/aggs/aggs_service.test.ts +++ b/src/plugins/data/common/search/aggs/aggs_service.test.ts @@ -73,6 +73,8 @@ describe('Aggs service', () => { "significant_terms", "geohash_grid", "geotile_grid", + "sampler", + "diversified_sampler", "foo", ] `); @@ -122,6 +124,8 @@ describe('Aggs service', () => { "significant_terms", "geohash_grid", "geotile_grid", + "sampler", + "diversified_sampler", ] `); expect(bStart.types.getAll().metrics.map((t) => t(aggTypesDependencies).name)) diff --git a/src/plugins/data/common/search/aggs/buckets/bucket_agg_types.ts b/src/plugins/data/common/search/aggs/buckets/bucket_agg_types.ts index 0c01bff90bfeef..671266ef159977 100644 --- a/src/plugins/data/common/search/aggs/buckets/bucket_agg_types.ts +++ b/src/plugins/data/common/search/aggs/buckets/bucket_agg_types.ts @@ -19,4 +19,6 @@ export enum BUCKET_TYPES { GEOHASH_GRID = 'geohash_grid', GEOTILE_GRID = 'geotile_grid', DATE_HISTOGRAM = 'date_histogram', + SAMPLER = 'sampler', + DIVERSIFIED_SAMPLER = 'diversified_sampler', } diff --git a/src/plugins/data/common/search/aggs/buckets/diversified_sampler.ts b/src/plugins/data/common/search/aggs/buckets/diversified_sampler.ts new file mode 100644 index 00000000000000..31ebaa094c368a --- /dev/null +++ b/src/plugins/data/common/search/aggs/buckets/diversified_sampler.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { i18n } from '@kbn/i18n'; +import { BucketAggType } from './bucket_agg_type'; +import { BaseAggParams } from '../types'; +import { aggDiversifiedSamplerFnName } from './diversified_sampler_fn'; + +export const DIVERSIFIED_SAMPLER_AGG_NAME = 'diversified_sampler'; + +const title = i18n.translate('data.search.aggs.buckets.diversifiedSamplerTitle', { + defaultMessage: 'Diversified sampler', + description: 'Diversified sampler aggregation title', +}); + +export interface AggParamsDiversifiedSampler extends BaseAggParams { + /** + * Is used to provide values used for de-duplication + */ + field: string; + + /** + * Limits how many top-scoring documents are collected in the sample processed on each shard. + */ + shard_size?: number; + + /** + * Limits how many documents are permitted per choice of de-duplicating value + */ + max_docs_per_value?: number; +} + +/** + * Like the sampler aggregation this is a filtering aggregation used to limit any sub aggregations' processing to a sample of the top-scoring documents. + * The diversified_sampler aggregation adds the ability to limit the number of matches that share a common value. + */ +export const getDiversifiedSamplerBucketAgg = () => + new BucketAggType({ + name: DIVERSIFIED_SAMPLER_AGG_NAME, + title, + customLabels: false, + expressionName: aggDiversifiedSamplerFnName, + params: [ + { + name: 'shard_size', + type: 'number', + }, + { + name: 'max_docs_per_value', + type: 'number', + }, + { + name: 'field', + type: 'field', + }, + ], + }); diff --git a/src/plugins/data/common/search/aggs/buckets/diversified_sampler_fn.test.ts b/src/plugins/data/common/search/aggs/buckets/diversified_sampler_fn.test.ts new file mode 100644 index 00000000000000..e874542289bb2f --- /dev/null +++ b/src/plugins/data/common/search/aggs/buckets/diversified_sampler_fn.test.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { functionWrapper } from '../test_helpers'; +import { aggDiversifiedSampler } from './diversified_sampler_fn'; + +describe('aggDiversifiedSampler', () => { + const fn = functionWrapper(aggDiversifiedSampler()); + + test('fills in defaults when only required args are provided', () => { + const actual = fn({ id: 'sampler', schema: 'bucket', field: 'author' }); + expect(actual).toMatchInlineSnapshot(` + Object { + "type": "agg_type", + "value": Object { + "enabled": true, + "id": "sampler", + "params": Object { + "field": "author", + "max_docs_per_value": undefined, + "shard_size": undefined, + }, + "schema": "bucket", + "type": "diversified_sampler", + }, + } + `); + }); + + test('includes optional params when they are provided', () => { + const actual = fn({ + id: 'sampler', + schema: 'bucket', + shard_size: 300, + field: 'author', + max_docs_per_value: 3, + }); + + expect(actual.value).toMatchInlineSnapshot(` + Object { + "enabled": true, + "id": "sampler", + "params": Object { + "field": "author", + "max_docs_per_value": 3, + "shard_size": 300, + }, + "schema": "bucket", + "type": "diversified_sampler", + } + `); + }); +}); diff --git a/src/plugins/data/common/search/aggs/buckets/diversified_sampler_fn.ts b/src/plugins/data/common/search/aggs/buckets/diversified_sampler_fn.ts new file mode 100644 index 00000000000000..0e1b235dd576dd --- /dev/null +++ b/src/plugins/data/common/search/aggs/buckets/diversified_sampler_fn.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { i18n } from '@kbn/i18n'; +import { ExpressionFunctionDefinition } from 'src/plugins/expressions/common'; +import { AggExpressionFunctionArgs, AggExpressionType, BUCKET_TYPES } from '../'; +import { DIVERSIFIED_SAMPLER_AGG_NAME } from './diversified_sampler'; + +export const aggDiversifiedSamplerFnName = 'aggDiversifiedSampler'; + +type Input = any; +type Arguments = AggExpressionFunctionArgs; + +type Output = AggExpressionType; +type FunctionDefinition = ExpressionFunctionDefinition< + typeof aggDiversifiedSamplerFnName, + Input, + Arguments, + Output +>; + +export const aggDiversifiedSampler = (): FunctionDefinition => ({ + name: aggDiversifiedSamplerFnName, + help: i18n.translate('data.search.aggs.function.buckets.diversifiedSampler.help', { + defaultMessage: 'Generates a serialized agg config for a Diversified sampler agg', + }), + type: 'agg_type', + args: { + id: { + types: ['string'], + help: i18n.translate('data.search.aggs.buckets.diversifiedSampler.id.help', { + defaultMessage: 'ID for this aggregation', + }), + }, + enabled: { + types: ['boolean'], + default: true, + help: i18n.translate('data.search.aggs.buckets.diversifiedSampler.enabled.help', { + defaultMessage: 'Specifies whether this aggregation should be enabled', + }), + }, + schema: { + types: ['string'], + help: i18n.translate('data.search.aggs.buckets.diversifiedSampler.schema.help', { + defaultMessage: 'Schema to use for this aggregation', + }), + }, + shard_size: { + types: ['number'], + help: i18n.translate('data.search.aggs.buckets.diversifiedSampler.shardSize.help', { + defaultMessage: + 'The shard_size parameter limits how many top-scoring documents are collected in the sample processed on each shard.', + }), + }, + max_docs_per_value: { + types: ['number'], + help: i18n.translate('data.search.aggs.buckets.diversifiedSampler.maxDocsPerValue.help', { + defaultMessage: + 'Limits how many documents are permitted per choice of de-duplicating value.', + }), + }, + field: { + types: ['string'], + help: i18n.translate('data.search.aggs.buckets.diversifiedSampler.field.help', { + defaultMessage: 'Used to provide values used for de-duplication.', + }), + }, + }, + fn: (input, args) => { + const { id, enabled, schema, ...rest } = args; + + return { + type: 'agg_type', + value: { + id, + enabled, + schema, + type: DIVERSIFIED_SAMPLER_AGG_NAME, + params: { + ...rest, + }, + }, + }; + }, +}); diff --git a/src/plugins/data/common/search/aggs/buckets/index.ts b/src/plugins/data/common/search/aggs/buckets/index.ts index 421fa0fcfdaf4e..bf96a9ef860c06 100644 --- a/src/plugins/data/common/search/aggs/buckets/index.ts +++ b/src/plugins/data/common/search/aggs/buckets/index.ts @@ -38,3 +38,7 @@ export * from './terms_fn'; export * from './terms'; export * from './multi_terms_fn'; export * from './multi_terms'; +export * from './sampler_fn'; +export * from './sampler'; +export * from './diversified_sampler_fn'; +export * from './diversified_sampler'; diff --git a/src/plugins/data/common/search/aggs/buckets/sampler.ts b/src/plugins/data/common/search/aggs/buckets/sampler.ts new file mode 100644 index 00000000000000..7eb4f741150956 --- /dev/null +++ b/src/plugins/data/common/search/aggs/buckets/sampler.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { i18n } from '@kbn/i18n'; +import { BucketAggType } from './bucket_agg_type'; +import { BaseAggParams } from '../types'; +import { aggSamplerFnName } from './sampler_fn'; + +export const SAMPLER_AGG_NAME = 'sampler'; + +const title = i18n.translate('data.search.aggs.buckets.samplerTitle', { + defaultMessage: 'Sampler', + description: 'Sampler aggregation title', +}); + +export interface AggParamsSampler extends BaseAggParams { + /** + * Limits how many top-scoring documents are collected in the sample processed on each shard. + */ + shard_size?: number; +} + +/** + * A filtering aggregation used to limit any sub aggregations' processing to a sample of the top-scoring documents. + */ +export const getSamplerBucketAgg = () => + new BucketAggType({ + name: SAMPLER_AGG_NAME, + title, + customLabels: false, + expressionName: aggSamplerFnName, + params: [ + { + name: 'shard_size', + type: 'number', + }, + ], + }); diff --git a/src/plugins/data/common/search/aggs/buckets/sampler_fn.test.ts b/src/plugins/data/common/search/aggs/buckets/sampler_fn.test.ts new file mode 100644 index 00000000000000..76ef901671e72b --- /dev/null +++ b/src/plugins/data/common/search/aggs/buckets/sampler_fn.test.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { functionWrapper } from '../test_helpers'; +import { aggSampler } from './sampler_fn'; + +describe('aggSampler', () => { + const fn = functionWrapper(aggSampler()); + + test('fills in defaults when only required args are provided', () => { + const actual = fn({ id: 'sampler', schema: 'bucket' }); + expect(actual).toMatchInlineSnapshot(` + Object { + "type": "agg_type", + "value": Object { + "enabled": true, + "id": "sampler", + "params": Object { + "shard_size": undefined, + }, + "schema": "bucket", + "type": "sampler", + }, + } + `); + }); + + test('includes optional params when they are provided', () => { + const actual = fn({ + id: 'sampler', + schema: 'bucket', + shard_size: 300, + }); + + expect(actual.value).toMatchInlineSnapshot(` + Object { + "enabled": true, + "id": "sampler", + "params": Object { + "shard_size": 300, + }, + "schema": "bucket", + "type": "sampler", + } + `); + }); +}); diff --git a/src/plugins/data/common/search/aggs/buckets/sampler_fn.ts b/src/plugins/data/common/search/aggs/buckets/sampler_fn.ts new file mode 100644 index 00000000000000..2cb30eb70a2300 --- /dev/null +++ b/src/plugins/data/common/search/aggs/buckets/sampler_fn.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { i18n } from '@kbn/i18n'; +import { ExpressionFunctionDefinition } from 'src/plugins/expressions/common'; +import { AggExpressionFunctionArgs, AggExpressionType, BUCKET_TYPES } from '../'; +import { SAMPLER_AGG_NAME } from './sampler'; + +export const aggSamplerFnName = 'aggSampler'; + +type Input = any; +type Arguments = AggExpressionFunctionArgs; + +type Output = AggExpressionType; +type FunctionDefinition = ExpressionFunctionDefinition< + typeof aggSamplerFnName, + Input, + Arguments, + Output +>; + +export const aggSampler = (): FunctionDefinition => ({ + name: aggSamplerFnName, + help: i18n.translate('data.search.aggs.function.buckets.sampler.help', { + defaultMessage: 'Generates a serialized agg config for a Sampler agg', + }), + type: 'agg_type', + args: { + id: { + types: ['string'], + help: i18n.translate('data.search.aggs.buckets.sampler.id.help', { + defaultMessage: 'ID for this aggregation', + }), + }, + enabled: { + types: ['boolean'], + default: true, + help: i18n.translate('data.search.aggs.buckets.sampler.enabled.help', { + defaultMessage: 'Specifies whether this aggregation should be enabled', + }), + }, + schema: { + types: ['string'], + help: i18n.translate('data.search.aggs.buckets.sampler.schema.help', { + defaultMessage: 'Schema to use for this aggregation', + }), + }, + shard_size: { + types: ['number'], + help: i18n.translate('data.search.aggs.buckets.sampler.shardSize.help', { + defaultMessage: + 'The shard_size parameter limits how many top-scoring documents are collected in the sample processed on each shard.', + }), + }, + }, + fn: (input, args) => { + const { id, enabled, schema, ...rest } = args; + + return { + type: 'agg_type', + value: { + id, + enabled, + schema, + type: SAMPLER_AGG_NAME, + params: { + ...rest, + }, + }, + }; + }, +}); diff --git a/src/plugins/data/common/search/aggs/types.ts b/src/plugins/data/common/search/aggs/types.ts index b9a977e0a8a097..9c4866c19714d6 100644 --- a/src/plugins/data/common/search/aggs/types.ts +++ b/src/plugins/data/common/search/aggs/types.ts @@ -90,6 +90,8 @@ import { aggFilteredMetric, aggSinglePercentile, } from './'; +import { AggParamsSampler } from './buckets/sampler'; +import { AggParamsDiversifiedSampler } from './buckets/diversified_sampler'; export type { IAggConfig, AggConfigSerialized } from './agg_config'; export type { CreateAggConfigParams, IAggConfigs } from './agg_configs'; @@ -166,6 +168,8 @@ export interface AggParamsMapping { [BUCKET_TYPES.DATE_HISTOGRAM]: AggParamsDateHistogram; [BUCKET_TYPES.TERMS]: AggParamsTerms; [BUCKET_TYPES.MULTI_TERMS]: AggParamsMultiTerms; + [BUCKET_TYPES.SAMPLER]: AggParamsSampler; + [BUCKET_TYPES.DIVERSIFIED_SAMPLER]: AggParamsDiversifiedSampler; [METRIC_TYPES.AVG]: AggParamsAvg; [METRIC_TYPES.CARDINALITY]: AggParamsCardinality; [METRIC_TYPES.COUNT]: BaseAggParams; diff --git a/src/plugins/data/public/search/aggs/aggs_service.test.ts b/src/plugins/data/public/search/aggs/aggs_service.test.ts index 20e07360a68e56..c7df4354cc76b5 100644 --- a/src/plugins/data/public/search/aggs/aggs_service.test.ts +++ b/src/plugins/data/public/search/aggs/aggs_service.test.ts @@ -53,7 +53,7 @@ describe('AggsService - public', () => { test('registers default agg types', () => { service.setup(setupDeps); const start = service.start(startDeps); - expect(start.types.getAll().buckets.length).toBe(12); + expect(start.types.getAll().buckets.length).toBe(14); expect(start.types.getAll().metrics.length).toBe(23); }); @@ -69,7 +69,7 @@ describe('AggsService - public', () => { ); const start = service.start(startDeps); - expect(start.types.getAll().buckets.length).toBe(13); + expect(start.types.getAll().buckets.length).toBe(15); expect(start.types.getAll().buckets.some(({ name }) => name === 'foo')).toBe(true); expect(start.types.getAll().metrics.length).toBe(24); expect(start.types.getAll().metrics.some(({ name }) => name === 'bar')).toBe(true); diff --git a/src/plugins/vis_types/heatmap/public/vis_type/heatmap.tsx b/src/plugins/vis_types/heatmap/public/vis_type/heatmap.tsx index 5d466c2f4b3c8f..e20c7eb0f8c21e 100644 --- a/src/plugins/vis_types/heatmap/public/vis_type/heatmap.tsx +++ b/src/plugins/vis_types/heatmap/public/vis_type/heatmap.tsx @@ -98,7 +98,14 @@ export const getHeatmapVisTypeDefinition = ({ }), min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], }, { group: AggGroupNames.Buckets, @@ -106,7 +113,14 @@ export const getHeatmapVisTypeDefinition = ({ title: i18n.translate('visTypeHeatmap.heatmap.groupTitle', { defaultMessage: 'Y-axis' }), min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], }, { group: AggGroupNames.Buckets, @@ -121,7 +135,14 @@ export const getHeatmapVisTypeDefinition = ({ }), min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], }, ], }, diff --git a/src/plugins/vis_types/metric/public/metric_vis_type.ts b/src/plugins/vis_types/metric/public/metric_vis_type.ts index d4db2ac9e46711..ffb34248aeccc6 100644 --- a/src/plugins/vis_types/metric/public/metric_vis_type.ts +++ b/src/plugins/vis_types/metric/public/metric_vis_type.ts @@ -86,7 +86,14 @@ export const createMetricVisTypeDefinition = (): VisTypeDefinition => }), min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], }, ], }, diff --git a/src/plugins/vis_types/pie/public/sample_vis.test.mocks.ts b/src/plugins/vis_types/pie/public/sample_vis.test.mocks.ts index 15a3675125f619..545456b6dcce08 100644 --- a/src/plugins/vis_types/pie/public/sample_vis.test.mocks.ts +++ b/src/plugins/vis_types/pie/public/sample_vis.test.mocks.ts @@ -87,7 +87,14 @@ export const samplePieVis = { title: 'Split slices', min: 0, max: null, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], editor: false, params: [], }, @@ -98,7 +105,14 @@ export const samplePieVis = { mustBeFirst: true, min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], params: [ { name: 'row', diff --git a/src/plugins/vis_types/pie/public/vis_type/pie.ts b/src/plugins/vis_types/pie/public/vis_type/pie.ts index 0d012ed95b5d93..f10af053bd1617 100644 --- a/src/plugins/vis_types/pie/public/vis_type/pie.ts +++ b/src/plugins/vis_types/pie/public/vis_type/pie.ts @@ -80,7 +80,14 @@ export const getPieVisTypeDefinition = ({ }), min: 0, max: Infinity, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], }, { group: AggGroupNames.Buckets, @@ -91,7 +98,14 @@ export const getPieVisTypeDefinition = ({ mustBeFirst: true, min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], }, ], }, diff --git a/src/plugins/vis_types/table/public/table_vis_type.ts b/src/plugins/vis_types/table/public/table_vis_type.ts index a641224e23f52d..2f1642e29107a9 100644 --- a/src/plugins/vis_types/table/public/table_vis_type.ts +++ b/src/plugins/vis_types/table/public/table_vis_type.ts @@ -62,7 +62,7 @@ export const tableVisTypeDefinition: VisTypeDefinition = { title: i18n.translate('visTypeTable.tableVisEditorConfig.schemas.bucketTitle', { defaultMessage: 'Split rows', }), - aggFilter: ['!filter'], + aggFilter: ['!filter', '!sampler', '!diversified_sampler', '!multi_terms'], }, { group: AggGroupNames.Buckets, @@ -72,7 +72,7 @@ export const tableVisTypeDefinition: VisTypeDefinition = { }), min: 0, max: 1, - aggFilter: ['!filter'], + aggFilter: ['!filter', '!sampler', '!diversified_sampler', '!multi_terms'], }, ], }, diff --git a/src/plugins/vis_types/vislib/public/gauge.ts b/src/plugins/vis_types/vislib/public/gauge.ts index 51cd7ea7622df2..31a44a5d1d73f4 100644 --- a/src/plugins/vis_types/vislib/public/gauge.ts +++ b/src/plugins/vis_types/vislib/public/gauge.ts @@ -132,7 +132,14 @@ export const gaugeVisTypeDefinition: VisTypeDefinition = { }), min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], }, ], }, diff --git a/src/plugins/vis_types/vislib/public/goal.ts b/src/plugins/vis_types/vislib/public/goal.ts index 05ad1f53904d74..26bc5987908392 100644 --- a/src/plugins/vis_types/vislib/public/goal.ts +++ b/src/plugins/vis_types/vislib/public/goal.ts @@ -96,7 +96,14 @@ export const goalVisTypeDefinition: VisTypeDefinition = { }), min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], }, ], }, diff --git a/src/plugins/vis_types/xy/public/editor/components/options/point_series/point_series.mocks.ts b/src/plugins/vis_types/xy/public/editor/components/options/point_series/point_series.mocks.ts index 41ab13d54f7c62..401afc5a7473a3 100644 --- a/src/plugins/vis_types/xy/public/editor/components/options/point_series/point_series.mocks.ts +++ b/src/plugins/vis_types/xy/public/editor/components/options/point_series/point_series.mocks.ts @@ -625,7 +625,14 @@ export const getVis = (bucketType: string) => { title: 'X-axis', min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], params: [], }, { @@ -634,7 +641,14 @@ export const getVis = (bucketType: string) => { title: 'Split series', min: 0, max: 3, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], params: [], }, { @@ -643,7 +657,14 @@ export const getVis = (bucketType: string) => { title: 'Split chart', min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], params: [ { name: 'row', @@ -688,7 +709,14 @@ export const getVis = (bucketType: string) => { title: 'X-axis', min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], params: [], }, { @@ -697,7 +725,14 @@ export const getVis = (bucketType: string) => { title: 'Split series', min: 0, max: 3, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], params: [], }, { @@ -706,7 +741,14 @@ export const getVis = (bucketType: string) => { title: 'Split chart', min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], params: [ { name: 'row', @@ -722,7 +764,14 @@ export const getVis = (bucketType: string) => { title: 'X-axis', min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], params: [], }, { @@ -731,7 +780,14 @@ export const getVis = (bucketType: string) => { title: 'Split series', min: 0, max: 3, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], params: [], }, { @@ -740,7 +796,14 @@ export const getVis = (bucketType: string) => { title: 'Split chart', min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], params: [ { name: 'row', diff --git a/src/plugins/vis_types/xy/public/sample_vis.test.mocks.ts b/src/plugins/vis_types/xy/public/sample_vis.test.mocks.ts index 6077732a9cc6b9..766929a2cd6549 100644 --- a/src/plugins/vis_types/xy/public/sample_vis.test.mocks.ts +++ b/src/plugins/vis_types/xy/public/sample_vis.test.mocks.ts @@ -149,7 +149,14 @@ export const sampleAreaVis = { title: 'X-axis', min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], editor: false, params: [], }, @@ -159,7 +166,14 @@ export const sampleAreaVis = { title: 'Split series', min: 0, max: 3, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], editor: false, params: [], }, @@ -169,7 +183,14 @@ export const sampleAreaVis = { title: 'Split chart', min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], params: [ { name: 'row', diff --git a/src/plugins/vis_types/xy/public/vis_types/area.ts b/src/plugins/vis_types/xy/public/vis_types/area.ts index 47fc190f130911..efeb4142ff0d7c 100644 --- a/src/plugins/vis_types/xy/public/vis_types/area.ts +++ b/src/plugins/vis_types/xy/public/vis_types/area.ts @@ -157,7 +157,14 @@ export const areaVisTypeDefinition = { }), min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], }, { group: AggGroupNames.Buckets, @@ -167,7 +174,14 @@ export const areaVisTypeDefinition = { }), min: 0, max: 3, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], }, { group: AggGroupNames.Buckets, @@ -177,7 +191,14 @@ export const areaVisTypeDefinition = { }), min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], }, ], }, diff --git a/src/plugins/vis_types/xy/public/vis_types/histogram.ts b/src/plugins/vis_types/xy/public/vis_types/histogram.ts index 5b312e8bc76a56..1cd346abec6e7a 100644 --- a/src/plugins/vis_types/xy/public/vis_types/histogram.ts +++ b/src/plugins/vis_types/xy/public/vis_types/histogram.ts @@ -160,7 +160,14 @@ export const histogramVisTypeDefinition = { }), min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], }, { group: AggGroupNames.Buckets, @@ -170,7 +177,14 @@ export const histogramVisTypeDefinition = { }), min: 0, max: 3, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], }, { group: AggGroupNames.Buckets, @@ -180,7 +194,14 @@ export const histogramVisTypeDefinition = { }), min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], }, ], }, diff --git a/src/plugins/vis_types/xy/public/vis_types/horizontal_bar.ts b/src/plugins/vis_types/xy/public/vis_types/horizontal_bar.ts index bf98fa20a93cf0..4e6056bbdae4f0 100644 --- a/src/plugins/vis_types/xy/public/vis_types/horizontal_bar.ts +++ b/src/plugins/vis_types/xy/public/vis_types/horizontal_bar.ts @@ -159,7 +159,14 @@ export const horizontalBarVisTypeDefinition = { }), min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], }, { group: AggGroupNames.Buckets, @@ -169,7 +176,14 @@ export const horizontalBarVisTypeDefinition = { }), min: 0, max: 3, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], }, { group: AggGroupNames.Buckets, @@ -179,7 +193,14 @@ export const horizontalBarVisTypeDefinition = { }), min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], }, ], }, diff --git a/src/plugins/vis_types/xy/public/vis_types/line.ts b/src/plugins/vis_types/xy/public/vis_types/line.ts index c76c197a118085..affcc64320df66 100644 --- a/src/plugins/vis_types/xy/public/vis_types/line.ts +++ b/src/plugins/vis_types/xy/public/vis_types/line.ts @@ -151,7 +151,14 @@ export const lineVisTypeDefinition = { title: i18n.translate('visTypeXy.line.segmentTitle', { defaultMessage: 'X-axis' }), min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], }, { group: AggGroupNames.Buckets, @@ -161,7 +168,14 @@ export const lineVisTypeDefinition = { }), min: 0, max: 3, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], }, { group: AggGroupNames.Buckets, @@ -171,7 +185,14 @@ export const lineVisTypeDefinition = { }), min: 0, max: 1, - aggFilter: ['!geohash_grid', '!geotile_grid', '!filter', '!multi_terms'], + aggFilter: [ + '!geohash_grid', + '!geotile_grid', + '!filter', + '!sampler', + '!diversified_sampler', + '!multi_terms', + ], }, ], }, diff --git a/test/interpreter_functional/test_suites/run_pipeline/esaggs_sampler.ts b/test/interpreter_functional/test_suites/run_pipeline/esaggs_sampler.ts new file mode 100644 index 00000000000000..d2411b24160678 --- /dev/null +++ b/test/interpreter_functional/test_suites/run_pipeline/esaggs_sampler.ts @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import expect from '@kbn/expect'; +import { ExpectExpression, expectExpressionProvider } from './helpers'; +import { FtrProviderContext } from '../../../functional/ftr_provider_context'; + +export default function ({ + getService, + updateBaselines, +}: FtrProviderContext & { updateBaselines: boolean }) { + let expectExpression: ExpectExpression; + + describe('esaggs_sampler', () => { + before(() => { + expectExpression = expectExpressionProvider({ getService, updateBaselines }); + }); + + const timeRange = { + from: '2015-09-21T00:00:00Z', + to: '2015-09-22T00:00:00Z', + }; + + describe('aggSampler', () => { + it('can execute aggSampler', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggSampler id="0" enabled=true schema="bucket"} + aggs={aggAvg id="1" enabled=true schema="metric" field="bytes"} + `; + const result = await expectExpression('sampler', expression).getResponse(); + + expect(result.columns.length).to.be(2); + const samplerColumn = result.columns[0]; + expect(samplerColumn.name).to.be('sampler'); + expect(samplerColumn.meta.sourceParams.params).to.eql({}); + + expect(result.rows.length).to.be(1); + expect(Object.keys(result.rows[0]).length).to.be(1); + const resultFromSample = result.rows[0]['col-1-1']; // check that sampler bucket doesn't produce columns + expect(typeof resultFromSample).to.be('number'); + expect(resultFromSample).to.greaterThan(0); // can't check exact metric using sample + }); + + it('can execute aggSampler with custom shard_size', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggSampler id="0" enabled=true schema="bucket" shard_size=20} + aggs={aggAvg id="1" enabled=true schema="metric" field="bytes"} + `; + const result = await expectExpression('sampler', expression).getResponse(); + + expect(result.columns.length).to.be(2); + const samplerColumn = result.columns[0]; + expect(samplerColumn.name).to.be('sampler'); + expect(samplerColumn.meta.sourceParams.params).to.eql({ shard_size: 20 }); + + expect(result.rows.length).to.be(1); + expect(Object.keys(result.rows[0]).length).to.be(1); // check that sampler bucket doesn't produce columns + const resultFromSample = result.rows[0]['col-1-1']; + expect(typeof resultFromSample).to.be('number'); + expect(resultFromSample).to.greaterThan(0); // can't check exact metric using sample + }); + }); + + describe('aggDiversifiedSampler', () => { + it('can execute aggDiversifiedSampler', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggDiversifiedSampler id="0" enabled=true schema="bucket" field="extension.raw"} + aggs={aggAvg id="1" enabled=true schema="metric" field="bytes"} + `; + const result = await expectExpression('sampler', expression).getResponse(); + + expect(result.columns.length).to.be(2); + const samplerColumn = result.columns[0]; + expect(samplerColumn.name).to.be('diversified_sampler'); + expect(samplerColumn.meta.sourceParams.params).to.eql({ field: 'extension.raw' }); + + expect(result.rows.length).to.be(1); + expect(Object.keys(result.rows[0]).length).to.be(1); + const resultFromSample = result.rows[0]['col-1-1']; // check that sampler bucket doesn't produce columns + expect(typeof resultFromSample).to.be('number'); + expect(resultFromSample).to.greaterThan(0); // can't check exact metric using sample + }); + + it('can execute aggSampler with custom shard_size and max_docs_per_value', async () => { + const expression = ` + kibana_context timeRange={timerange from='${timeRange.from}' to='${timeRange.to}'} + | esaggs index={indexPatternLoad id='logstash-*'} + aggs={aggDiversifiedSampler id="0" enabled=true schema="bucket" field="extension.raw" shard_size=20 max_docs_per_value=3} + aggs={aggAvg id="1" enabled=true schema="metric" field="bytes"} + `; + const result = await expectExpression('sampler', expression).getResponse(); + + expect(result.columns.length).to.be(2); + const samplerColumn = result.columns[0]; + expect(samplerColumn.name).to.be('diversified_sampler'); + expect(samplerColumn.meta.sourceParams.params).to.eql({ + field: 'extension.raw', + max_docs_per_value: 3, + shard_size: 20, + }); + + expect(result.rows.length).to.be(1); + expect(Object.keys(result.rows[0]).length).to.be(1); // check that sampler bucket doesn't produce columns + const resultFromSample = result.rows[0]['col-1-1']; + expect(typeof resultFromSample).to.be('number'); + expect(resultFromSample).to.greaterThan(0); // can't check exact metric using sample + }); + }); + }); +} diff --git a/test/interpreter_functional/test_suites/run_pipeline/index.ts b/test/interpreter_functional/test_suites/run_pipeline/index.ts index 32f59fcf3df9c8..fe2ccce23d94a8 100644 --- a/test/interpreter_functional/test_suites/run_pipeline/index.ts +++ b/test/interpreter_functional/test_suites/run_pipeline/index.ts @@ -44,5 +44,6 @@ export default function ({ getService, getPageObjects, loadTestFile }: FtrProvid loadTestFile(require.resolve('./esaggs')); loadTestFile(require.resolve('./esaggs_timeshift')); loadTestFile(require.resolve('./esaggs_multiterms')); + loadTestFile(require.resolve('./esaggs_sampler')); }); }